drivers/crypto/caam/caamalg_qi.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Freescale FSL CAAM support for crypto API over QI backend.
   4  * Based on caamalg.c
   5  *
   6  * Copyright 2013-2016 Freescale Semiconductor, Inc.
   7  * Copyright 2016-2018 NXP
   8  */
   9
  10 #include "compat.h"
  11 #include "ctrl.h"
  12 #include "regs.h"
  13 #include "intern.h"
  14 #include "desc_constr.h"
  15 #include "error.h"
  16 #include "sg_sw_qm.h"
  17 #include "key_gen.h"
  18 #include "qi.h"
  19 #include "jr.h"
  20 #include "caamalg_desc.h"
  21
  22 /*
  23  * crypto alg
  24  */
  25 #define CAAM_CRA_PRIORITY               2000
  26 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  27 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  28                                          SHA512_DIGEST_SIZE * 2)
  29
  30 #define DESC_MAX_USED_BYTES             (DESC_QI_AEAD_GIVENC_LEN + \
  31                                          CAAM_MAX_KEY_SIZE)
  32 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  33
  34 struct caam_alg_entry {
  35         int class1_alg_type;
  36         int class2_alg_type;
  37         bool rfc3686;
  38         bool geniv;
  39 };
  40
  41 struct caam_aead_alg {
  42         struct aead_alg aead;
  43         struct caam_alg_entry caam;
  44         bool registered;
  45 };
  46
  47 struct caam_skcipher_alg {
  48         struct skcipher_alg skcipher;
  49         struct caam_alg_entry caam;
  50         bool registered;
  51 };
  52
  53 /*
  54  * per-session context
  55  */
  56 struct caam_ctx {
  57         struct device *jrdev;
  58         u32 sh_desc_enc[DESC_MAX_USED_LEN];
  59         u32 sh_desc_dec[DESC_MAX_USED_LEN];
  60         u8 key[CAAM_MAX_KEY_SIZE];
  61         dma_addr_t key_dma;
  62         enum dma_data_direction dir;
  63         struct alginfo adata;
  64         struct alginfo cdata;
  65         unsigned int authsize;
  66         struct device *qidev;
  67         spinlock_t lock;        /* Protects multiple init of driver context */
  68         struct caam_drv_ctx *drv_ctx[NUM_OP];
  69 };
  70
  71 static int aead_set_sh_desc(struct crypto_aead *aead)
  72 {
  73         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
  74                                                  typeof(*alg), aead);
  75         struct caam_ctx *ctx = crypto_aead_ctx(aead);
  76         unsigned int ivsize = crypto_aead_ivsize(aead);
  77         u32 ctx1_iv_off = 0;
  78         u32 *nonce = NULL;
  79         unsigned int data_len[2];
  80         u32 inl_mask;
  81         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
  82                                OP_ALG_AAI_CTR_MOD128);
  83         const bool is_rfc3686 = alg->caam.rfc3686;
  84         struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
  85
  86         if (!ctx->cdata.keylen || !ctx->authsize)
  87                 return 0;
  88
  89         /*
  90          * AES-CTR needs to load IV in CONTEXT1 reg
  91          * at an offset of 128bits (16bytes)
  92          * CONTEXT1[255:128] = IV
  93          */
  94         if (ctr_mode)
  95                 ctx1_iv_off = 16;
  96
  97         /*
  98          * RFC3686 specific:
  99          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 100          */
 101         if (is_rfc3686) {
 102                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 103                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 104                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 105         }
 106
 107         data_len[0] = ctx->adata.keylen_pad;
 108         data_len[1] = ctx->cdata.keylen;
 109
 110         if (alg->caam.geniv)
 111                 goto skip_enc;
 112
 113         /* aead_encrypt shared descriptor */
 114         if (desc_inline_query(DESC_QI_AEAD_ENC_LEN +
 115                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 116                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 117                               ARRAY_SIZE(data_len)) < 0)
 118                 return -EINVAL;
 119
 120         if (inl_mask & 1)
 121                 ctx->adata.key_virt = ctx->key;
 122         else
 123                 ctx->adata.key_dma = ctx->key_dma;
 124
 125         if (inl_mask & 2)
 126                 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 127         else
 128                 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 129
 130         ctx->adata.key_inline = !!(inl_mask & 1);
 131         ctx->cdata.key_inline = !!(inl_mask & 2);
 132
 133         cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
 134                                ivsize, ctx->authsize, is_rfc3686, nonce,
 135                                ctx1_iv_off, true, ctrlpriv->era);
 136
 137 skip_enc:
 138         /* aead_decrypt shared descriptor */
 139         if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
 140                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 141                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 142                               ARRAY_SIZE(data_len)) < 0)
 143                 return -EINVAL;
 144
 145         if (inl_mask & 1)
 146                 ctx->adata.key_virt = ctx->key;
 147         else
 148                 ctx->adata.key_dma = ctx->key_dma;
 149
 150         if (inl_mask & 2)
 151                 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 152         else
 153                 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 154
 155         ctx->adata.key_inline = !!(inl_mask & 1);
 156         ctx->cdata.key_inline = !!(inl_mask & 2);
 157
 158         cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata,
 159                                ivsize, ctx->authsize, alg->caam.geniv,
 160                                is_rfc3686, nonce, ctx1_iv_off, true,
 161                                ctrlpriv->era);
 162
 163         if (!alg->caam.geniv)
 164                 goto skip_givenc;
 165
 166         /* aead_givencrypt shared descriptor */
 167         if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN +
 168                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 169                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 170                               ARRAY_SIZE(data_len)) < 0)
 171                 return -EINVAL;
 172
 173         if (inl_mask & 1)
 174                 ctx->adata.key_virt = ctx->key;
 175         else
 176                 ctx->adata.key_dma = ctx->key_dma;
 177
 178         if (inl_mask & 2)
 179                 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 180         else
 181                 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 182
 183         ctx->adata.key_inline = !!(inl_mask & 1);
 184         ctx->cdata.key_inline = !!(inl_mask & 2);
 185
 186         cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
 187                                   ivsize, ctx->authsize, is_rfc3686, nonce,
 188                                   ctx1_iv_off, true, ctrlpriv->era);
 189
 190 skip_givenc:
 191         return 0;
 192 }
 193
 194 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 195 {
 196         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 197
 198         ctx->authsize = authsize;
 199         aead_set_sh_desc(authenc);
 200
 201         return 0;
 202 }
 203
 204 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 205                        unsigned int keylen)
 206 {
 207         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 208         struct device *jrdev = ctx->jrdev;
 209         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 210         struct crypto_authenc_keys keys;
 211         int ret = 0;
 212
 213         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 214                 goto badkey;
 215
 216 #ifdef DEBUG
 217         dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
 218                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
 219                 keys.authkeylen);
 220         print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
 221                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 222 #endif
 223
 224         /*
 225          * If DKP is supported, use it in the shared descriptor to generate
 226          * the split key.
 227          */
 228         if (ctrlpriv->era >= 6) {
 229                 ctx->adata.keylen = keys.authkeylen;
 230                 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 231                                                       OP_ALG_ALGSEL_MASK);
 232
 233                 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 234                         goto badkey;
 235
 236                 memcpy(ctx->key, keys.authkey, keys.authkeylen);
 237                 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
 238                        keys.enckeylen);
 239                 dma_sync_single_for_device(jrdev, ctx->key_dma,
 240                                            ctx->adata.keylen_pad +
 241                                            keys.enckeylen, ctx->dir);
 242                 goto skip_split_key;
 243         }
 244
 245         ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey,
 246                             keys.authkeylen, CAAM_MAX_KEY_SIZE -
 247                             keys.enckeylen);
 248         if (ret)
 249                 goto badkey;
 250
 251         /* postpend encryption key to auth split key */
 252         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 253         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
 254                                    keys.enckeylen, ctx->dir);
 255 #ifdef DEBUG
 256         print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
 257                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 258                        ctx->adata.keylen_pad + keys.enckeylen, 1);
 259 #endif
 260
 261 skip_split_key:
 262         ctx->cdata.keylen = keys.enckeylen;
 263
 264         ret = aead_set_sh_desc(aead);
 265         if (ret)
 266                 goto badkey;
 267
 268         /* Now update the driver contexts with the new shared descriptor */
 269         if (ctx->drv_ctx[ENCRYPT]) {
 270                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 271                                           ctx->sh_desc_enc);
 272                 if (ret) {
 273                         dev_err(jrdev, "driver enc context update failed\n");
 274                         goto badkey;
 275                 }
 276         }
 277
 278         if (ctx->drv_ctx[DECRYPT]) {
 279                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 280                                           ctx->sh_desc_dec);
 281                 if (ret) {
 282                         dev_err(jrdev, "driver dec context update failed\n");
 283                         goto badkey;
 284                 }
 285         }
 286
 287         memzero_explicit(&keys, sizeof(keys));
 288         return ret;
 289 badkey:
 290         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 291         memzero_explicit(&keys, sizeof(keys));
 292         return -EINVAL;
 293 }
 294
 295 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 296                             unsigned int keylen)
 297 {
 298         struct crypto_authenc_keys keys;
 299         u32 flags;
 300         int err;
 301
 302         err = crypto_authenc_extractkeys(&keys, key, keylen);
 303         if (unlikely(err))
 304                 goto badkey;
 305
 306         err = -EINVAL;
 307         if (keys.enckeylen != DES3_EDE_KEY_SIZE)
 308                 goto badkey;
 309
 310         flags = crypto_aead_get_flags(aead);
 311         err = __des3_verify_key(&flags, keys.enckey);
 312         if (unlikely(err)) {
 313                 crypto_aead_set_flags(aead, flags);
 314                 goto out;
 315         }
 316
 317         err = aead_setkey(aead, key, keylen);
 318
 319 out:
 320         memzero_explicit(&keys, sizeof(keys));
 321         return err;
 322
 323 badkey:
 324         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 325         goto out;
 326 }
 327
 328 static int gcm_set_sh_desc(struct crypto_aead *aead)
 329 {
 330         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 331         unsigned int ivsize = crypto_aead_ivsize(aead);
 332         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 333                         ctx->cdata.keylen;
 334
 335         if (!ctx->cdata.keylen || !ctx->authsize)
 336                 return 0;
 337
 338         /*
 339          * Job Descriptor and Shared Descriptor
 340          * must fit into the 64-word Descriptor h/w Buffer
 341          */
 342         if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
 343                 ctx->cdata.key_inline = true;
 344                 ctx->cdata.key_virt = ctx->key;
 345         } else {
 346                 ctx->cdata.key_inline = false;
 347                 ctx->cdata.key_dma = ctx->key_dma;
 348         }
 349
 350         cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 351                               ctx->authsize, true);
 352
 353         /*
 354          * Job Descriptor and Shared Descriptor
 355          * must fit into the 64-word Descriptor h/w Buffer
 356          */
 357         if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
 358                 ctx->cdata.key_inline = true;
 359                 ctx->cdata.key_virt = ctx->key;
 360         } else {
 361                 ctx->cdata.key_inline = false;
 362                 ctx->cdata.key_dma = ctx->key_dma;
 363         }
 364
 365         cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 366                               ctx->authsize, true);
 367
 368         return 0;
 369 }
 370
 371 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 372 {
 373         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 374
 375         ctx->authsize = authsize;
 376         gcm_set_sh_desc(authenc);
 377
 378         return 0;
 379 }
 380
 381 static int gcm_setkey(struct crypto_aead *aead,
 382                       const u8 *key, unsigned int keylen)
 383 {
 384         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 385         struct device *jrdev = ctx->jrdev;
 386         int ret;
 387
 388 #ifdef DEBUG
 389         print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
 390                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 391 #endif
 392
 393         memcpy(ctx->key, key, keylen);
 394         dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
 395         ctx->cdata.keylen = keylen;
 396
 397         ret = gcm_set_sh_desc(aead);
 398         if (ret)
 399                 return ret;
 400
 401         /* Now update the driver contexts with the new shared descriptor */
 402         if (ctx->drv_ctx[ENCRYPT]) {
 403                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 404                                           ctx->sh_desc_enc);
 405                 if (ret) {
 406                         dev_err(jrdev, "driver enc context update failed\n");
 407                         return ret;
 408                 }
 409         }
 410
 411         if (ctx->drv_ctx[DECRYPT]) {
 412                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 413                                           ctx->sh_desc_dec);
 414                 if (ret) {
 415                         dev_err(jrdev, "driver dec context update failed\n");
 416                         return ret;
 417                 }
 418         }
 419
 420         return 0;
 421 }
 422
 423 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 424 {
 425         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 426         unsigned int ivsize = crypto_aead_ivsize(aead);
 427         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 428                         ctx->cdata.keylen;
 429
 430         if (!ctx->cdata.keylen || !ctx->authsize)
 431                 return 0;
 432
 433         ctx->cdata.key_virt = ctx->key;
 434
 435         /*
 436          * Job Descriptor and Shared Descriptor
 437          * must fit into the 64-word Descriptor h/w Buffer
 438          */
 439         if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
 440                 ctx->cdata.key_inline = true;
 441         } else {
 442                 ctx->cdata.key_inline = false;
 443                 ctx->cdata.key_dma = ctx->key_dma;
 444         }
 445
 446         cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 447                                   ctx->authsize, true);
 448
 449         /*
 450          * Job Descriptor and Shared Descriptor
 451          * must fit into the 64-word Descriptor h/w Buffer
 452          */
 453         if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
 454                 ctx->cdata.key_inline = true;
 455         } else {
 456                 ctx->cdata.key_inline = false;
 457                 ctx->cdata.key_dma = ctx->key_dma;
 458         }
 459
 460         cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 461                                   ctx->authsize, true);
 462
 463         return 0;
 464 }
 465
 466 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 467                                unsigned int authsize)
 468 {
 469         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 470
 471         ctx->authsize = authsize;
 472         rfc4106_set_sh_desc(authenc);
 473
 474         return 0;
 475 }
 476
 477 static int rfc4106_setkey(struct crypto_aead *aead,
 478                           const u8 *key, unsigned int keylen)
 479 {
 480         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 481         struct device *jrdev = ctx->jrdev;
 482         int ret;
 483
 484         if (keylen < 4)
 485                 return -EINVAL;
 486
 487 #ifdef DEBUG
 488         print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
 489                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 490 #endif
 491
 492         memcpy(ctx->key, key, keylen);
 493         /*
 494          * The last four bytes of the key material are used as the salt value
 495          * in the nonce. Update the AES key length.
 496          */
 497         ctx->cdata.keylen = keylen - 4;
 498         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
 499                                    ctx->dir);
 500
 501         ret = rfc4106_set_sh_desc(aead);
 502         if (ret)
 503                 return ret;
 504
 505         /* Now update the driver contexts with the new shared descriptor */
 506         if (ctx->drv_ctx[ENCRYPT]) {
 507                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 508                                           ctx->sh_desc_enc);
 509                 if (ret) {
 510                         dev_err(jrdev, "driver enc context update failed\n");
 511                         return ret;
 512                 }
 513         }
 514
 515         if (ctx->drv_ctx[DECRYPT]) {
 516                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 517                                           ctx->sh_desc_dec);
 518                 if (ret) {
 519                         dev_err(jrdev, "driver dec context update failed\n");
 520                         return ret;
 521                 }
 522         }
 523
 524         return 0;
 525 }
 526
 527 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 528 {
 529         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 530         unsigned int ivsize = crypto_aead_ivsize(aead);
 531         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 532                         ctx->cdata.keylen;
 533
 534         if (!ctx->cdata.keylen || !ctx->authsize)
 535                 return 0;
 536
 537         ctx->cdata.key_virt = ctx->key;
 538
 539         /*
 540          * Job Descriptor and Shared Descriptor
 541          * must fit into the 64-word Descriptor h/w Buffer
 542          */
 543         if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
 544                 ctx->cdata.key_inline = true;
 545         } else {
 546                 ctx->cdata.key_inline = false;
 547                 ctx->cdata.key_dma = ctx->key_dma;
 548         }
 549
 550         cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 551                                   ctx->authsize, true);
 552
 553         /*
 554          * Job Descriptor and Shared Descriptor
 555          * must fit into the 64-word Descriptor h/w Buffer
 556          */
 557         if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
 558                 ctx->cdata.key_inline = true;
 559         } else {
 560                 ctx->cdata.key_inline = false;
 561                 ctx->cdata.key_dma = ctx->key_dma;
 562         }
 563
 564         cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 565                                   ctx->authsize, true);
 566
 567         return 0;
 568 }
 569
 570 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 571                                unsigned int authsize)
 572 {
 573         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 574
 575         ctx->authsize = authsize;
 576         rfc4543_set_sh_desc(authenc);
 577
 578         return 0;
 579 }
 580
 581 static int rfc4543_setkey(struct crypto_aead *aead,
 582                           const u8 *key, unsigned int keylen)
 583 {
 584         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 585         struct device *jrdev = ctx->jrdev;
 586         int ret;
 587
 588         if (keylen < 4)
 589                 return -EINVAL;
 590
 591 #ifdef DEBUG
 592         print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
 593                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 594 #endif
 595
 596         memcpy(ctx->key, key, keylen);
 597         /*
 598          * The last four bytes of the key material are used as the salt value
 599          * in the nonce. Update the AES key length.
 600          */
 601         ctx->cdata.keylen = keylen - 4;
 602         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
 603                                    ctx->dir);
 604
 605         ret = rfc4543_set_sh_desc(aead);
 606         if (ret)
 607                 return ret;
 608
 609         /* Now update the driver contexts with the new shared descriptor */
 610         if (ctx->drv_ctx[ENCRYPT]) {
 611                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 612                                           ctx->sh_desc_enc);
 613                 if (ret) {
 614                         dev_err(jrdev, "driver enc context update failed\n");
 615                         return ret;
 616                 }
 617         }
 618
 619         if (ctx->drv_ctx[DECRYPT]) {
 620                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 621                                           ctx->sh_desc_dec);
 622                 if (ret) {
 623                         dev_err(jrdev, "driver dec context update failed\n");
 624                         return ret;
 625                 }
 626         }
 627
 628         return 0;
 629 }
 630
 631 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 632                            unsigned int keylen)
 633 {
 634         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 635         struct caam_skcipher_alg *alg =
 636                 container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
 637                              skcipher);
 638         struct device *jrdev = ctx->jrdev;
 639         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 640         u32 ctx1_iv_off = 0;
 641         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 642                                OP_ALG_AAI_CTR_MOD128);
 643         const bool is_rfc3686 = alg->caam.rfc3686;
 644         int ret = 0;
 645
 646 #ifdef DEBUG
 647         print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
 648                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 649 #endif
 650         /*
 651          * AES-CTR needs to load IV in CONTEXT1 reg
 652          * at an offset of 128bits (16bytes)
 653          * CONTEXT1[255:128] = IV
 654          */
 655         if (ctr_mode)
 656                 ctx1_iv_off = 16;
 657
 658         /*
 659          * RFC3686 specific:
 660          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 661          *      | *key = {KEY, NONCE}
 662          */
 663         if (is_rfc3686) {
 664                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 665                 keylen -= CTR_RFC3686_NONCE_SIZE;
 666         }
 667
 668         ctx->cdata.keylen = keylen;
 669         ctx->cdata.key_virt = key;
 670         ctx->cdata.key_inline = true;
 671
 672         /* skcipher encrypt, decrypt shared descriptors */
 673         cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 674                                    is_rfc3686, ctx1_iv_off);
 675         cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 676                                    is_rfc3686, ctx1_iv_off);
 677
 678         /* Now update the driver contexts with the new shared descriptor */
 679         if (ctx->drv_ctx[ENCRYPT]) {
 680                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 681                                           ctx->sh_desc_enc);
 682                 if (ret) {
 683                         dev_err(jrdev, "driver enc context update failed\n");
 684                         goto badkey;
 685                 }
 686         }
 687
 688         if (ctx->drv_ctx[DECRYPT]) {
 689                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 690                                           ctx->sh_desc_dec);
 691                 if (ret) {
 692                         dev_err(jrdev, "driver dec context update failed\n");
 693                         goto badkey;
 694                 }
 695         }
 696
 697         return ret;
 698 badkey:
 699         crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 700         return -EINVAL;
 701 }
 702
 703 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
 704                                 const u8 *key, unsigned int keylen)
 705 {
 706         return unlikely(des3_verify_key(skcipher, key)) ?:
 707                skcipher_setkey(skcipher, key, keylen);
 708 }
 709
 710 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 711                                unsigned int keylen)
 712 {
 713         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 714         struct device *jrdev = ctx->jrdev;
 715         int ret = 0;
 716
 717         if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
 718                 dev_err(jrdev, "key size mismatch\n");
 719                 goto badkey;
 720         }
 721
 722         ctx->cdata.keylen = keylen;
 723         ctx->cdata.key_virt = key;
 724         ctx->cdata.key_inline = true;
 725
 726         /* xts skcipher encrypt, decrypt shared descriptors */
 727         cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata);
 728         cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata);
 729
 730         /* Now update the driver contexts with the new shared descriptor */
 731         if (ctx->drv_ctx[ENCRYPT]) {
 732                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 733                                           ctx->sh_desc_enc);
 734                 if (ret) {
 735                         dev_err(jrdev, "driver enc context update failed\n");
 736                         goto badkey;
 737                 }
 738         }
 739
 740         if (ctx->drv_ctx[DECRYPT]) {
 741                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 742                                           ctx->sh_desc_dec);
 743                 if (ret) {
 744                         dev_err(jrdev, "driver dec context update failed\n");
 745                         goto badkey;
 746                 }
 747         }
 748
 749         return ret;
 750 badkey:
 751         crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 752         return -EINVAL;
 753 }
 754
 755 /*
 756  * aead_edesc - s/w-extended aead descriptor
 757  * @src_nents: number of segments in input scatterlist
 758  * @dst_nents: number of segments in output scatterlist
 759  * @iv_dma: dma address of iv for checking continuity and link table
 760  * @qm_sg_bytes: length of dma mapped h/w link table
 761  * @qm_sg_dma: bus physical mapped address of h/w link table
 762  * @assoclen: associated data length, in CAAM endianness
 763  * @assoclen_dma: bus physical mapped address of req->assoclen
 764  * @drv_req: driver-specific request structure
 765  * @sgt: the h/w link table, followed by IV
 766  */
 767 struct aead_edesc {
 768         int src_nents;
 769         int dst_nents;
 770         dma_addr_t iv_dma;
 771         int qm_sg_bytes;
 772         dma_addr_t qm_sg_dma;
 773         unsigned int assoclen;
 774         dma_addr_t assoclen_dma;
 775         struct caam_drv_req drv_req;
 776         struct qm_sg_entry sgt[0];
 777 };
 778
 779 /*
 780  * skcipher_edesc - s/w-extended skcipher descriptor
 781  * @src_nents: number of segments in input scatterlist
 782  * @dst_nents: number of segments in output scatterlist
 783  * @iv_dma: dma address of iv for checking continuity and link table
 784  * @qm_sg_bytes: length of dma mapped h/w link table
 785  * @qm_sg_dma: bus physical mapped address of h/w link table
 786  * @drv_req: driver-specific request structure
 787  * @sgt: the h/w link table, followed by IV
 788  */
 789 struct skcipher_edesc {
 790         int src_nents;
 791         int dst_nents;
 792         dma_addr_t iv_dma;
 793         int qm_sg_bytes;
 794         dma_addr_t qm_sg_dma;
 795         struct caam_drv_req drv_req;
 796         struct qm_sg_entry sgt[0];
 797 };
 798
 799 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
 800                                         enum optype type)
 801 {
 802         /*
 803          * This function is called on the fast path with values of 'type'
 804          * known at compile time. Invalid arguments are not expected and
 805          * thus no checks are made.
 806          */
 807         struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type];
 808         u32 *desc;
 809
 810         if (unlikely(!drv_ctx)) {
 811                 spin_lock(&ctx->lock);
 812
 813                 /* Read again to check if some other core init drv_ctx */
 814                 drv_ctx = ctx->drv_ctx[type];
 815                 if (!drv_ctx) {
 816                         int cpu;
 817
 818                         if (type == ENCRYPT)
 819                                 desc = ctx->sh_desc_enc;
 820                         else /* (type == DECRYPT) */
 821                                 desc = ctx->sh_desc_dec;
 822
 823                         cpu = smp_processor_id();
 824                         drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc);
 825                         if (!IS_ERR_OR_NULL(drv_ctx))
 826                                 drv_ctx->op_type = type;
 827
 828                         ctx->drv_ctx[type] = drv_ctx;
 829                 }
 830
 831                 spin_unlock(&ctx->lock);
 832         }
 833
 834         return drv_ctx;
 835 }
 836
 837 static void caam_unmap(struct device *dev, struct scatterlist *src,
 838                        struct scatterlist *dst, int src_nents,
 839                        int dst_nents, dma_addr_t iv_dma, int ivsize,
 840                        dma_addr_t qm_sg_dma, int qm_sg_bytes)
 841 {
 842         if (dst != src) {
 843                 if (src_nents)
 844                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 845                 if (dst_nents)
 846                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 847         } else {
 848                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 849         }
 850
 851         if (iv_dma)
 852                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
 853         if (qm_sg_bytes)
 854                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 855 }
 856
 857 static void aead_unmap(struct device *dev,
 858                        struct aead_edesc *edesc,
 859                        struct aead_request *req)
 860 {
 861         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 862         int ivsize = crypto_aead_ivsize(aead);
 863
 864         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
 865                    edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
 866         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 867 }
 868
 869 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
 870                            struct skcipher_request *req)
 871 {
 872         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
 873         int ivsize = crypto_skcipher_ivsize(skcipher);
 874
 875         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
 876                    edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
 877 }
 878
 879 static void aead_done(struct caam_drv_req *drv_req, u32 status)
 880 {
 881         struct device *qidev;
 882         struct aead_edesc *edesc;
 883         struct aead_request *aead_req = drv_req->app_ctx;
 884         struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
 885         struct caam_ctx *caam_ctx = crypto_aead_ctx(aead);
 886         int ecode = 0;
 887
 888         qidev = caam_ctx->qidev;
 889
 890         if (unlikely(status)) {
 891                 u32 ssrc = status & JRSTA_SSRC_MASK;
 892                 u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
 893
 894                 caam_jr_strstatus(qidev, status);
 895                 /*
 896                  * verify hw auth check passed else return -EBADMSG
 897                  */
 898                 if (ssrc == JRSTA_SSRC_CCB_ERROR &&
 899                     err_id == JRSTA_CCBERR_ERRID_ICVCHK)
 900                         ecode = -EBADMSG;
 901                 else
 902                         ecode = -EIO;
 903         }
 904
 905         edesc = container_of(drv_req, typeof(*edesc), drv_req);
 906         aead_unmap(qidev, edesc, aead_req);
 907
 908         aead_request_complete(aead_req, ecode);
 909         qi_cache_free(edesc);
 910 }
 911
 912 /*
 913  * allocate and map the aead extended descriptor
 914  */
 915 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 916                                            bool encrypt)
 917 {
 918         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 919         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 920         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 921                                                  typeof(*alg), aead);
 922         struct device *qidev = ctx->qidev;
 923         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 924                        GFP_KERNEL : GFP_ATOMIC;
 925         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 926         struct aead_edesc *edesc;
 927         dma_addr_t qm_sg_dma, iv_dma = 0;
 928         int ivsize = 0;
 929         unsigned int authsize = ctx->authsize;
 930         int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes;
 931         int in_len, out_len;
 932         struct qm_sg_entry *sg_table, *fd_sgt;
 933         struct caam_drv_ctx *drv_ctx;
 934
 935         drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
 936         if (IS_ERR_OR_NULL(drv_ctx))
 937                 return (struct aead_edesc *)drv_ctx;
 938
 939         /* allocate space for base edesc and hw desc commands, link tables */
 940         edesc = qi_cache_alloc(GFP_DMA | flags);
 941         if (unlikely(!edesc)) {
 942                 dev_err(qidev, "could not allocate extended descriptor\n");
 943                 return ERR_PTR(-ENOMEM);
 944         }
 945
 946         if (likely(req->src == req->dst)) {
 947                 src_nents = sg_nents_for_len(req->src, req->assoclen +
 948                                              req->cryptlen +
 949                                                 (encrypt ? authsize : 0));
 950                 if (unlikely(src_nents < 0)) {
 951                         dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
 952                                 req->assoclen + req->cryptlen +
 953                                 (encrypt ? authsize : 0));
 954                         qi_cache_free(edesc);
 955                         return ERR_PTR(src_nents);
 956                 }
 957
 958                 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
 959                                               DMA_BIDIRECTIONAL);
 960                 if (unlikely(!mapped_src_nents)) {
 961                         dev_err(qidev, "unable to map source\n");
 962                         qi_cache_free(edesc);
 963                         return ERR_PTR(-ENOMEM);
 964                 }
 965         } else {
 966                 src_nents = sg_nents_for_len(req->src, req->assoclen +
 967                                              req->cryptlen);
 968                 if (unlikely(src_nents < 0)) {
 969                         dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
 970                                 req->assoclen + req->cryptlen);
 971                         qi_cache_free(edesc);
 972                         return ERR_PTR(src_nents);
 973                 }
 974
 975                 dst_nents = sg_nents_for_len(req->dst, req->assoclen +
 976                                              req->cryptlen +
 977                                              (encrypt ? authsize :
 978                                                         (-authsize)));
 979                 if (unlikely(dst_nents < 0)) {
 980                         dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
 981                                 req->assoclen + req->cryptlen +
 982                                 (encrypt ? authsize : (-authsize)));
 983                         qi_cache_free(edesc);
 984                         return ERR_PTR(dst_nents);
 985                 }
 986
 987                 if (src_nents) {
 988                         mapped_src_nents = dma_map_sg(qidev, req->src,
 989                                                       src_nents, DMA_TO_DEVICE);
 990                         if (unlikely(!mapped_src_nents)) {
 991                                 dev_err(qidev, "unable to map source\n");
 992                                 qi_cache_free(edesc);
 993                                 return ERR_PTR(-ENOMEM);
 994                         }
 995                 } else {
 996                         mapped_src_nents = 0;
 997                 }
 998
 999                 if (dst_nents) {
1000                         mapped_dst_nents = dma_map_sg(qidev, req->dst,
1001                                                       dst_nents,
1002                                                       DMA_FROM_DEVICE);
1003                         if (unlikely(!mapped_dst_nents)) {
1004                                 dev_err(qidev, "unable to map destination\n");
1005                                 dma_unmap_sg(qidev, req->src, src_nents,
1006                                              DMA_TO_DEVICE);
1007                                 qi_cache_free(edesc);
1008                                 return ERR_PTR(-ENOMEM);
1009                         }
1010                 } else {
1011                         mapped_dst_nents = 0;
1012                 }
1013         }
1014
1015         if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
1016                 ivsize = crypto_aead_ivsize(aead);
1017
1018         /*
1019          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
1020          * Input is not contiguous.
1021          */
1022         qm_sg_ents = 1 + !!ivsize + mapped_src_nents +
1023                      (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
1024         sg_table = &edesc->sgt[0];
1025         qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
1026         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
1027                      CAAM_QI_MEMCACHE_SIZE)) {
1028                 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1029                         qm_sg_ents, ivsize);
1030                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1031                            0, 0, 0);
1032                 qi_cache_free(edesc);
1033                 return ERR_PTR(-ENOMEM);
1034         }
1035
1036         if (ivsize) {
1037                 u8 *iv = (u8 *)(sg_table + qm_sg_ents);
1038
1039                 /* Make sure IV is located in a DMAable area */
1040                 memcpy(iv, req->iv, ivsize);
1041
1042                 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
1043                 if (dma_mapping_error(qidev, iv_dma)) {
1044                         dev_err(qidev, "unable to map IV\n");
1045                         caam_unmap(qidev, req->src, req->dst, src_nents,
1046                                    dst_nents, 0, 0, 0, 0);
1047                         qi_cache_free(edesc);
1048                         return ERR_PTR(-ENOMEM);
1049                 }
1050         }
1051
1052         edesc->src_nents = src_nents;
1053         edesc->dst_nents = dst_nents;
1054         edesc->iv_dma = iv_dma;
1055         edesc->drv_req.app_ctx = req;
1056         edesc->drv_req.cbk = aead_done;
1057         edesc->drv_req.drv_ctx = drv_ctx;
1058
1059         edesc->assoclen = cpu_to_caam32(req->assoclen);
1060         edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4,
1061                                              DMA_TO_DEVICE);
1062         if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
1063                 dev_err(qidev, "unable to map assoclen\n");
1064                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1065                            iv_dma, ivsize, 0, 0);
1066                 qi_cache_free(edesc);
1067                 return ERR_PTR(-ENOMEM);
1068         }
1069
1070         dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
1071         qm_sg_index++;
1072         if (ivsize) {
1073                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
1074                 qm_sg_index++;
1075         }
1076         sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
1077         qm_sg_index += mapped_src_nents;
1078
1079         if (mapped_dst_nents > 1)
1080                 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
1081                                  qm_sg_index, 0);
1082
1083         qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
1084         if (dma_mapping_error(qidev, qm_sg_dma)) {
1085                 dev_err(qidev, "unable to map S/G table\n");
1086                 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1087                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1088                            iv_dma, ivsize, 0, 0);
1089                 qi_cache_free(edesc);
1090                 return ERR_PTR(-ENOMEM);
1091         }
1092
1093         edesc->qm_sg_dma = qm_sg_dma;
1094         edesc->qm_sg_bytes = qm_sg_bytes;
1095
1096         out_len = req->assoclen + req->cryptlen +
1097                   (encrypt ? ctx->authsize : (-ctx->authsize));
1098         in_len = 4 + ivsize + req->assoclen + req->cryptlen;
1099
1100         fd_sgt = &edesc->drv_req.fd_sgt[0];
1101         dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0);
1102
1103         if (req->dst == req->src) {
1104                 if (mapped_src_nents == 1)
1105                         dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src),
1106                                          out_len, 0);
1107                 else
1108                         dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +
1109                                              (1 + !!ivsize) * sizeof(*sg_table),
1110                                              out_len, 0);
1111         } else if (mapped_dst_nents == 1) {
1112                 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len,
1113                                  0);
1114         } else {
1115                 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) *
1116                                      qm_sg_index, out_len, 0);
1117         }
1118
1119         return edesc;
1120 }
1121
1122 static inline int aead_crypt(struct aead_request *req, bool encrypt)
1123 {
1124         struct aead_edesc *edesc;
1125         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1126         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1127         int ret;
1128
1129         if (unlikely(caam_congested))
1130                 return -EAGAIN;
1131
1132         /* allocate extended descriptor */
1133         edesc = aead_edesc_alloc(req, encrypt);
1134         if (IS_ERR_OR_NULL(edesc))
1135                 return PTR_ERR(edesc);
1136
1137         /* Create and submit job descriptor */
1138         ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1139         if (!ret) {
1140                 ret = -EINPROGRESS;
1141         } else {
1142                 aead_unmap(ctx->qidev, edesc, req);
1143                 qi_cache_free(edesc);
1144         }
1145
1146         return ret;
1147 }
1148
1149 static int aead_encrypt(struct aead_request *req)
1150 {
1151         return aead_crypt(req, true);
1152 }
1153
1154 static int aead_decrypt(struct aead_request *req)
1155 {
1156         return aead_crypt(req, false);
1157 }
1158
1159 static int ipsec_gcm_encrypt(struct aead_request *req)
1160 {
1161         if (req->assoclen < 8)
1162                 return -EINVAL;
1163
1164         return aead_crypt(req, true);
1165 }
1166
1167 static int ipsec_gcm_decrypt(struct aead_request *req)
1168 {
1169         if (req->assoclen < 8)
1170                 return -EINVAL;
1171
1172         return aead_crypt(req, false);
1173 }
1174
1175 static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
1176 {
1177         struct skcipher_edesc *edesc;
1178         struct skcipher_request *req = drv_req->app_ctx;
1179         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1180         struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher);
1181         struct device *qidev = caam_ctx->qidev;
1182         int ivsize = crypto_skcipher_ivsize(skcipher);
1183
1184 #ifdef DEBUG
1185         dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
1186 #endif
1187
1188         edesc = container_of(drv_req, typeof(*edesc), drv_req);
1189
1190         if (status)
1191                 caam_jr_strstatus(qidev, status);
1192
1193 #ifdef DEBUG
1194         print_hex_dump(KERN_ERR, "dstiv  @" __stringify(__LINE__)": ",
1195                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1196                        edesc->src_nents > 1 ? 100 : ivsize, 1);
1197         caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
1198                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1199                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1200 #endif
1201
1202         skcipher_unmap(qidev, edesc, req);
1203
1204         /*
1205          * The crypto API expects us to set the IV (req->iv) to the last
1206          * ciphertext block. This is used e.g. by the CTS mode.
1207          */
1208         if (edesc->drv_req.drv_ctx->op_type == ENCRYPT)
1209                 scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
1210                                          ivsize, ivsize, 0);
1211
1212         qi_cache_free(edesc);
1213         skcipher_request_complete(req, status);
1214 }
1215
1216 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1217                                                    bool encrypt)
1218 {
1219         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1220         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1221         struct device *qidev = ctx->qidev;
1222         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1223                        GFP_KERNEL : GFP_ATOMIC;
1224         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1225         struct skcipher_edesc *edesc;
1226         dma_addr_t iv_dma;
1227         u8 *iv;
1228         int ivsize = crypto_skcipher_ivsize(skcipher);
1229         int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1230         struct qm_sg_entry *sg_table, *fd_sgt;
1231         struct caam_drv_ctx *drv_ctx;
1232
1233         drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
1234         if (IS_ERR_OR_NULL(drv_ctx))
1235                 return (struct skcipher_edesc *)drv_ctx;
1236
1237         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1238         if (unlikely(src_nents < 0)) {
1239                 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
1240                         req->cryptlen);
1241                 return ERR_PTR(src_nents);
1242         }
1243
1244         if (unlikely(req->src != req->dst)) {
1245                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1246                 if (unlikely(dst_nents < 0)) {
1247                         dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
1248                                 req->cryptlen);
1249                         return ERR_PTR(dst_nents);
1250                 }
1251
1252                 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1253                                               DMA_TO_DEVICE);
1254                 if (unlikely(!mapped_src_nents)) {
1255                         dev_err(qidev, "unable to map source\n");
1256                         return ERR_PTR(-ENOMEM);
1257                 }
1258
1259                 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents,
1260                                               DMA_FROM_DEVICE);
1261                 if (unlikely(!mapped_dst_nents)) {
1262                         dev_err(qidev, "unable to map destination\n");
1263                         dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE);
1264                         return ERR_PTR(-ENOMEM);
1265                 }
1266         } else {
1267                 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1268                                               DMA_BIDIRECTIONAL);
1269                 if (unlikely(!mapped_src_nents)) {
1270                         dev_err(qidev, "unable to map source\n");
1271                         return ERR_PTR(-ENOMEM);
1272                 }
1273         }
1274
1275         qm_sg_ents = 1 + mapped_src_nents;
1276         dst_sg_idx = qm_sg_ents;
1277
1278         qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
1279         qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
1280         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1281                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1282                 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1283                         qm_sg_ents, ivsize);
1284                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1285                            0, 0, 0);
1286                 return ERR_PTR(-ENOMEM);
1287         }
1288
1289         /* allocate space for base edesc, link tables and IV */
1290         edesc = qi_cache_alloc(GFP_DMA | flags);
1291         if (unlikely(!edesc)) {
1292                 dev_err(qidev, "could not allocate extended descriptor\n");
1293                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1294                            0, 0, 0);
1295                 return ERR_PTR(-ENOMEM);
1296         }
1297
1298         /* Make sure IV is located in a DMAable area */
1299         sg_table = &edesc->sgt[0];
1300         iv = (u8 *)(sg_table + qm_sg_ents);
1301         memcpy(iv, req->iv, ivsize);
1302
1303         iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
1304         if (dma_mapping_error(qidev, iv_dma)) {
1305                 dev_err(qidev, "unable to map IV\n");
1306                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1307                            0, 0, 0);
1308                 qi_cache_free(edesc);
1309                 return ERR_PTR(-ENOMEM);
1310         }
1311
1312         edesc->src_nents = src_nents;
1313         edesc->dst_nents = dst_nents;
1314         edesc->iv_dma = iv_dma;
1315         edesc->qm_sg_bytes = qm_sg_bytes;
1316         edesc->drv_req.app_ctx = req;
1317         edesc->drv_req.cbk = skcipher_done;
1318         edesc->drv_req.drv_ctx = drv_ctx;
1319
1320         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1321         sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
1322
1323         if (mapped_dst_nents > 1)
1324                 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
1325                                  dst_sg_idx, 0);
1326
1327         edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
1328                                           DMA_TO_DEVICE);
1329         if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
1330                 dev_err(qidev, "unable to map S/G table\n");
1331                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1332                            iv_dma, ivsize, 0, 0);
1333                 qi_cache_free(edesc);
1334                 return ERR_PTR(-ENOMEM);
1335         }
1336
1337         fd_sgt = &edesc->drv_req.fd_sgt[0];
1338
1339         dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
1340                                   ivsize + req->cryptlen, 0);
1341
1342         if (req->src == req->dst) {
1343                 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
1344                                      sizeof(*sg_table), req->cryptlen, 0);
1345         } else if (mapped_dst_nents > 1) {
1346                 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
1347                                      sizeof(*sg_table), req->cryptlen, 0);
1348         } else {
1349                 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst),
1350                                  req->cryptlen, 0);
1351         }
1352
1353         return edesc;
1354 }
1355
1356 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
1357 {
1358         struct skcipher_edesc *edesc;
1359         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1360         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1361         int ivsize = crypto_skcipher_ivsize(skcipher);
1362         int ret;
1363
1364         if (unlikely(caam_congested))
1365                 return -EAGAIN;
1366
1367         /* allocate extended descriptor */
1368         edesc = skcipher_edesc_alloc(req, encrypt);
1369         if (IS_ERR(edesc))
1370                 return PTR_ERR(edesc);
1371
1372         /*
1373          * The crypto API expects us to set the IV (req->iv) to the last
1374          * ciphertext block.
1375          */
1376         if (!encrypt)
1377                 scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
1378                                          ivsize, ivsize, 0);
1379
1380         ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1381         if (!ret) {
1382                 ret = -EINPROGRESS;
1383         } else {
1384                 skcipher_unmap(ctx->qidev, edesc, req);
1385                 qi_cache_free(edesc);
1386         }
1387
1388         return ret;
1389 }
1390
1391 static int skcipher_encrypt(struct skcipher_request *req)
1392 {
1393         return skcipher_crypt(req, true);
1394 }
1395
1396 static int skcipher_decrypt(struct skcipher_request *req)
1397 {
1398         return skcipher_crypt(req, false);
1399 }
1400
1401 static struct caam_skcipher_alg driver_algs[] = {
1402         {
1403                 .skcipher = {
1404                         .base = {
1405                                 .cra_name = "cbc(aes)",
1406                                 .cra_driver_name = "cbc-aes-caam-qi",
1407                                 .cra_blocksize = AES_BLOCK_SIZE,
1408                         },
1409                         .setkey = skcipher_setkey,
1410                         .encrypt = skcipher_encrypt,
1411                         .decrypt = skcipher_decrypt,
1412                         .min_keysize = AES_MIN_KEY_SIZE,
1413                         .max_keysize = AES_MAX_KEY_SIZE,
1414                         .ivsize = AES_BLOCK_SIZE,
1415                 },
1416                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1417         },
1418         {
1419                 .skcipher = {
1420                         .base = {
1421                                 .cra_name = "cbc(des3_ede)",
1422                                 .cra_driver_name = "cbc-3des-caam-qi",
1423                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1424                         },
1425                         .setkey = des3_skcipher_setkey,
1426                         .encrypt = skcipher_encrypt,
1427                         .decrypt = skcipher_decrypt,
1428                         .min_keysize = DES3_EDE_KEY_SIZE,
1429                         .max_keysize = DES3_EDE_KEY_SIZE,
1430                         .ivsize = DES3_EDE_BLOCK_SIZE,
1431                 },
1432                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1433         },
1434         {
1435                 .skcipher = {
1436                         .base = {
1437                                 .cra_name = "cbc(des)",
1438                                 .cra_driver_name = "cbc-des-caam-qi",
1439                                 .cra_blocksize = DES_BLOCK_SIZE,
1440                         },
1441                         .setkey = skcipher_setkey,
1442                         .encrypt = skcipher_encrypt,
1443                         .decrypt = skcipher_decrypt,
1444                         .min_keysize = DES_KEY_SIZE,
1445                         .max_keysize = DES_KEY_SIZE,
1446                         .ivsize = DES_BLOCK_SIZE,
1447                 },
1448                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1449         },
1450         {
1451                 .skcipher = {
1452                         .base = {
1453                                 .cra_name = "ctr(aes)",
1454                                 .cra_driver_name = "ctr-aes-caam-qi",
1455                                 .cra_blocksize = 1,
1456                         },
1457                         .setkey = skcipher_setkey,
1458                         .encrypt = skcipher_encrypt,
1459                         .decrypt = skcipher_decrypt,
1460                         .min_keysize = AES_MIN_KEY_SIZE,
1461                         .max_keysize = AES_MAX_KEY_SIZE,
1462                         .ivsize = AES_BLOCK_SIZE,
1463                         .chunksize = AES_BLOCK_SIZE,
1464                 },
1465                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1466                                         OP_ALG_AAI_CTR_MOD128,
1467         },
1468         {
1469                 .skcipher = {
1470                         .base = {
1471                                 .cra_name = "rfc3686(ctr(aes))",
1472                                 .cra_driver_name = "rfc3686-ctr-aes-caam-qi",
1473                                 .cra_blocksize = 1,
1474                         },
1475                         .setkey = skcipher_setkey,
1476                         .encrypt = skcipher_encrypt,
1477                         .decrypt = skcipher_decrypt,
1478                         .min_keysize = AES_MIN_KEY_SIZE +
1479                                        CTR_RFC3686_NONCE_SIZE,
1480                         .max_keysize = AES_MAX_KEY_SIZE +
1481                                        CTR_RFC3686_NONCE_SIZE,
1482                         .ivsize = CTR_RFC3686_IV_SIZE,
1483                         .chunksize = AES_BLOCK_SIZE,
1484                 },
1485                 .caam = {
1486                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1487                                            OP_ALG_AAI_CTR_MOD128,
1488                         .rfc3686 = true,
1489                 },
1490         },
1491         {
1492                 .skcipher = {
1493                         .base = {
1494                                 .cra_name = "xts(aes)",
1495                                 .cra_driver_name = "xts-aes-caam-qi",
1496                                 .cra_blocksize = AES_BLOCK_SIZE,
1497                         },
1498                         .setkey = xts_skcipher_setkey,
1499                         .encrypt = skcipher_encrypt,
1500                         .decrypt = skcipher_decrypt,
1501                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1502                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1503                         .ivsize = AES_BLOCK_SIZE,
1504                 },
1505                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1506         },
1507 };
1508
1509 static struct caam_aead_alg driver_aeads[] = {
1510         {
1511                 .aead = {
1512                         .base = {
1513                                 .cra_name = "rfc4106(gcm(aes))",
1514                                 .cra_driver_name = "rfc4106-gcm-aes-caam-qi",
1515                                 .cra_blocksize = 1,
1516                         },
1517                         .setkey = rfc4106_setkey,
1518                         .setauthsize = rfc4106_setauthsize,
1519                         .encrypt = ipsec_gcm_encrypt,
1520                         .decrypt = ipsec_gcm_decrypt,
1521                         .ivsize = 8,
1522                         .maxauthsize = AES_BLOCK_SIZE,
1523                 },
1524                 .caam = {
1525                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1526                 },
1527         },
1528         {
1529                 .aead = {
1530                         .base = {
1531                                 .cra_name = "rfc4543(gcm(aes))",
1532                                 .cra_driver_name = "rfc4543-gcm-aes-caam-qi",
1533                                 .cra_blocksize = 1,
1534                         },
1535                         .setkey = rfc4543_setkey,
1536                         .setauthsize = rfc4543_setauthsize,
1537                         .encrypt = ipsec_gcm_encrypt,
1538                         .decrypt = ipsec_gcm_decrypt,
1539                         .ivsize = 8,
1540                         .maxauthsize = AES_BLOCK_SIZE,
1541                 },
1542                 .caam = {
1543                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1544                 },
1545         },
1546         /* Galois Counter Mode */
1547         {
1548                 .aead = {
1549                         .base = {
1550                                 .cra_name = "gcm(aes)",
1551                                 .cra_driver_name = "gcm-aes-caam-qi",
1552                                 .cra_blocksize = 1,
1553                         },
1554                         .setkey = gcm_setkey,
1555                         .setauthsize = gcm_setauthsize,
1556                         .encrypt = aead_encrypt,
1557                         .decrypt = aead_decrypt,
1558                         .ivsize = 12,
1559                         .maxauthsize = AES_BLOCK_SIZE,
1560                 },
1561                 .caam = {
1562                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1563                 }
1564         },
1565         /* single-pass ipsec_esp descriptor */
1566         {
1567                 .aead = {
1568                         .base = {
1569                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
1570                                 .cra_driver_name = "authenc-hmac-md5-"
1571                                                    "cbc-aes-caam-qi",
1572                                 .cra_blocksize = AES_BLOCK_SIZE,
1573                         },
1574                         .setkey = aead_setkey,
1575                         .setauthsize = aead_setauthsize,
1576                         .encrypt = aead_encrypt,
1577                         .decrypt = aead_decrypt,
1578                         .ivsize = AES_BLOCK_SIZE,
1579                         .maxauthsize = MD5_DIGEST_SIZE,
1580                 },
1581                 .caam = {
1582                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1583                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1584                                            OP_ALG_AAI_HMAC_PRECOMP,
1585                 }
1586         },
1587         {
1588                 .aead = {
1589                         .base = {
1590                                 .cra_name = "echainiv(authenc(hmac(md5),"
1591                                             "cbc(aes)))",
1592                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1593                                                    "cbc-aes-caam-qi",
1594                                 .cra_blocksize = AES_BLOCK_SIZE,
1595                         },
1596                         .setkey = aead_setkey,
1597                         .setauthsize = aead_setauthsize,
1598                         .encrypt = aead_encrypt,
1599                         .decrypt = aead_decrypt,
1600                         .ivsize = AES_BLOCK_SIZE,
1601                         .maxauthsize = MD5_DIGEST_SIZE,
1602                 },
1603                 .caam = {
1604                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1605                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1606                                            OP_ALG_AAI_HMAC_PRECOMP,
1607                         .geniv = true,
1608                 }
1609         },
1610         {
1611                 .aead = {
1612                         .base = {
1613                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1614                                 .cra_driver_name = "authenc-hmac-sha1-"
1615                                                    "cbc-aes-caam-qi",
1616                                 .cra_blocksize = AES_BLOCK_SIZE,
1617                         },
1618                         .setkey = aead_setkey,
1619                         .setauthsize = aead_setauthsize,
1620                         .encrypt = aead_encrypt,
1621                         .decrypt = aead_decrypt,
1622                         .ivsize = AES_BLOCK_SIZE,
1623                         .maxauthsize = SHA1_DIGEST_SIZE,
1624                 },
1625                 .caam = {
1626                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1627                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1628                                            OP_ALG_AAI_HMAC_PRECOMP,
1629                 }
1630         },
1631         {
1632                 .aead = {
1633                         .base = {
1634                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1635                                             "cbc(aes)))",
1636                                 .cra_driver_name = "echainiv-authenc-"
1637                                                    "hmac-sha1-cbc-aes-caam-qi",
1638                                 .cra_blocksize = AES_BLOCK_SIZE,
1639                         },
1640                         .setkey = aead_setkey,
1641                         .setauthsize = aead_setauthsize,
1642                         .encrypt = aead_encrypt,
1643                         .decrypt = aead_decrypt,
1644                         .ivsize = AES_BLOCK_SIZE,
1645                         .maxauthsize = SHA1_DIGEST_SIZE,
1646                 },
1647                 .caam = {
1648                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1649                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1650                                            OP_ALG_AAI_HMAC_PRECOMP,
1651                         .geniv = true,
1652                 },
1653         },
1654         {
1655                 .aead = {
1656                         .base = {
1657                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1658                                 .cra_driver_name = "authenc-hmac-sha224-"
1659                                                    "cbc-aes-caam-qi",
1660                                 .cra_blocksize = AES_BLOCK_SIZE,
1661                         },
1662                         .setkey = aead_setkey,
1663                         .setauthsize = aead_setauthsize,
1664                         .encrypt = aead_encrypt,
1665                         .decrypt = aead_decrypt,
1666                         .ivsize = AES_BLOCK_SIZE,
1667                         .maxauthsize = SHA224_DIGEST_SIZE,
1668                 },
1669                 .caam = {
1670                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1671                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1672                                            OP_ALG_AAI_HMAC_PRECOMP,
1673                 }
1674         },
1675         {
1676                 .aead = {
1677                         .base = {
1678                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1679                                             "cbc(aes)))",
1680                                 .cra_driver_name = "echainiv-authenc-"
1681                                                    "hmac-sha224-cbc-aes-caam-qi",
1682                                 .cra_blocksize = AES_BLOCK_SIZE,
1683                         },
1684                         .setkey = aead_setkey,
1685                         .setauthsize = aead_setauthsize,
1686                         .encrypt = aead_encrypt,
1687                         .decrypt = aead_decrypt,
1688                         .ivsize = AES_BLOCK_SIZE,
1689                         .maxauthsize = SHA224_DIGEST_SIZE,
1690                 },
1691                 .caam = {
1692                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1693                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1694                                            OP_ALG_AAI_HMAC_PRECOMP,
1695                         .geniv = true,
1696                 }
1697         },
1698         {
1699                 .aead = {
1700                         .base = {
1701                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1702                                 .cra_driver_name = "authenc-hmac-sha256-"
1703                                                    "cbc-aes-caam-qi",
1704                                 .cra_blocksize = AES_BLOCK_SIZE,
1705                         },
1706                         .setkey = aead_setkey,
1707                         .setauthsize = aead_setauthsize,
1708                         .encrypt = aead_encrypt,
1709                         .decrypt = aead_decrypt,
1710                         .ivsize = AES_BLOCK_SIZE,
1711                         .maxauthsize = SHA256_DIGEST_SIZE,
1712                 },
1713                 .caam = {
1714                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1715                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1716                                            OP_ALG_AAI_HMAC_PRECOMP,
1717                 }
1718         },
1719         {
1720                 .aead = {
1721                         .base = {
1722                                 .cra_name = "echainiv(authenc(hmac(sha256),"
1723                                             "cbc(aes)))",
1724                                 .cra_driver_name = "echainiv-authenc-"
1725                                                    "hmac-sha256-cbc-aes-"
1726                                                    "caam-qi",
1727                                 .cra_blocksize = AES_BLOCK_SIZE,
1728                         },
1729                         .setkey = aead_setkey,
1730                         .setauthsize = aead_setauthsize,
1731                         .encrypt = aead_encrypt,
1732                         .decrypt = aead_decrypt,
1733                         .ivsize = AES_BLOCK_SIZE,
1734                         .maxauthsize = SHA256_DIGEST_SIZE,
1735                 },
1736                 .caam = {
1737                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1738                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1739                                            OP_ALG_AAI_HMAC_PRECOMP,
1740                         .geniv = true,
1741                 }
1742         },
1743         {
1744                 .aead = {
1745                         .base = {
1746                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1747                                 .cra_driver_name = "authenc-hmac-sha384-"
1748                                                    "cbc-aes-caam-qi",
1749                                 .cra_blocksize = AES_BLOCK_SIZE,
1750                         },
1751                         .setkey = aead_setkey,
1752                         .setauthsize = aead_setauthsize,
1753                         .encrypt = aead_encrypt,
1754                         .decrypt = aead_decrypt,
1755                         .ivsize = AES_BLOCK_SIZE,
1756                         .maxauthsize = SHA384_DIGEST_SIZE,
1757                 },
1758                 .caam = {
1759                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1760                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1761                                            OP_ALG_AAI_HMAC_PRECOMP,
1762                 }
1763         },
1764         {
1765                 .aead = {
1766                         .base = {
1767                                 .cra_name = "echainiv(authenc(hmac(sha384),"
1768                                             "cbc(aes)))",
1769                                 .cra_driver_name = "echainiv-authenc-"
1770                                                    "hmac-sha384-cbc-aes-"
1771                                                    "caam-qi",
1772                                 .cra_blocksize = AES_BLOCK_SIZE,
1773                         },
1774                         .setkey = aead_setkey,
1775                         .setauthsize = aead_setauthsize,
1776                         .encrypt = aead_encrypt,
1777                         .decrypt = aead_decrypt,
1778                         .ivsize = AES_BLOCK_SIZE,
1779                         .maxauthsize = SHA384_DIGEST_SIZE,
1780                 },
1781                 .caam = {
1782                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1783                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1784                                            OP_ALG_AAI_HMAC_PRECOMP,
1785                         .geniv = true,
1786                 }
1787         },
1788         {
1789                 .aead = {
1790                         .base = {
1791                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1792                                 .cra_driver_name = "authenc-hmac-sha512-"
1793                                                    "cbc-aes-caam-qi",
1794                                 .cra_blocksize = AES_BLOCK_SIZE,
1795                         },
1796                         .setkey = aead_setkey,
1797                         .setauthsize = aead_setauthsize,
1798                         .encrypt = aead_encrypt,
1799                         .decrypt = aead_decrypt,
1800                         .ivsize = AES_BLOCK_SIZE,
1801                         .maxauthsize = SHA512_DIGEST_SIZE,
1802                 },
1803                 .caam = {
1804                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1805                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1806                                            OP_ALG_AAI_HMAC_PRECOMP,
1807                 }
1808         },
1809         {
1810                 .aead = {
1811                         .base = {
1812                                 .cra_name = "echainiv(authenc(hmac(sha512),"
1813                                             "cbc(aes)))",
1814                                 .cra_driver_name = "echainiv-authenc-"
1815                                                    "hmac-sha512-cbc-aes-"
1816                                                    "caam-qi",
1817                                 .cra_blocksize = AES_BLOCK_SIZE,
1818                         },
1819                         .setkey = aead_setkey,
1820                         .setauthsize = aead_setauthsize,
1821                         .encrypt = aead_encrypt,
1822                         .decrypt = aead_decrypt,
1823                         .ivsize = AES_BLOCK_SIZE,
1824                         .maxauthsize = SHA512_DIGEST_SIZE,
1825                 },
1826                 .caam = {
1827                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1828                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1829                                            OP_ALG_AAI_HMAC_PRECOMP,
1830                         .geniv = true,
1831                 }
1832         },
1833         {
1834                 .aead = {
1835                         .base = {
1836                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1837                                 .cra_driver_name = "authenc-hmac-md5-"
1838                                                    "cbc-des3_ede-caam-qi",
1839                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1840                         },
1841                         .setkey = des3_aead_setkey,
1842                         .setauthsize = aead_setauthsize,
1843                         .encrypt = aead_encrypt,
1844                         .decrypt = aead_decrypt,
1845                         .ivsize = DES3_EDE_BLOCK_SIZE,
1846                         .maxauthsize = MD5_DIGEST_SIZE,
1847                 },
1848                 .caam = {
1849                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1850                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1851                                            OP_ALG_AAI_HMAC_PRECOMP,
1852                 }
1853         },
1854         {
1855                 .aead = {
1856                         .base = {
1857                                 .cra_name = "echainiv(authenc(hmac(md5),"
1858                                             "cbc(des3_ede)))",
1859                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1860                                                    "cbc-des3_ede-caam-qi",
1861                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1862                         },
1863                         .setkey = des3_aead_setkey,
1864                         .setauthsize = aead_setauthsize,
1865                         .encrypt = aead_encrypt,
1866                         .decrypt = aead_decrypt,
1867                         .ivsize = DES3_EDE_BLOCK_SIZE,
1868                         .maxauthsize = MD5_DIGEST_SIZE,
1869                 },
1870                 .caam = {
1871                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1872                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1873                                            OP_ALG_AAI_HMAC_PRECOMP,
1874                         .geniv = true,
1875                 }
1876         },
1877         {
1878                 .aead = {
1879                         .base = {
1880                                 .cra_name = "authenc(hmac(sha1),"
1881                                             "cbc(des3_ede))",
1882                                 .cra_driver_name = "authenc-hmac-sha1-"
1883                                                    "cbc-des3_ede-caam-qi",
1884                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1885                         },
1886                         .setkey = des3_aead_setkey,
1887                         .setauthsize = aead_setauthsize,
1888                         .encrypt = aead_encrypt,
1889                         .decrypt = aead_decrypt,
1890                         .ivsize = DES3_EDE_BLOCK_SIZE,
1891                         .maxauthsize = SHA1_DIGEST_SIZE,
1892                 },
1893                 .caam = {
1894                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1895                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1896                                            OP_ALG_AAI_HMAC_PRECOMP,
1897                 },
1898         },
1899         {
1900                 .aead = {
1901                         .base = {
1902                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1903                                             "cbc(des3_ede)))",
1904                                 .cra_driver_name = "echainiv-authenc-"
1905                                                    "hmac-sha1-"
1906                                                    "cbc-des3_ede-caam-qi",
1907                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1908                         },
1909                         .setkey = des3_aead_setkey,
1910                         .setauthsize = aead_setauthsize,
1911                         .encrypt = aead_encrypt,
1912                         .decrypt = aead_decrypt,
1913                         .ivsize = DES3_EDE_BLOCK_SIZE,
1914                         .maxauthsize = SHA1_DIGEST_SIZE,
1915                 },
1916                 .caam = {
1917                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1918                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1919                                            OP_ALG_AAI_HMAC_PRECOMP,
1920                         .geniv = true,
1921                 }
1922         },
1923         {
1924                 .aead = {
1925                         .base = {
1926                                 .cra_name = "authenc(hmac(sha224),"
1927                                             "cbc(des3_ede))",
1928                                 .cra_driver_name = "authenc-hmac-sha224-"
1929                                                    "cbc-des3_ede-caam-qi",
1930                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1931                         },
1932                         .setkey = des3_aead_setkey,
1933                         .setauthsize = aead_setauthsize,
1934                         .encrypt = aead_encrypt,
1935                         .decrypt = aead_decrypt,
1936                         .ivsize = DES3_EDE_BLOCK_SIZE,
1937                         .maxauthsize = SHA224_DIGEST_SIZE,
1938                 },
1939                 .caam = {
1940                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1941                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1942                                            OP_ALG_AAI_HMAC_PRECOMP,
1943                 },
1944         },
1945         {
1946                 .aead = {
1947                         .base = {
1948                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1949                                             "cbc(des3_ede)))",
1950                                 .cra_driver_name = "echainiv-authenc-"
1951                                                    "hmac-sha224-"
1952                                                    "cbc-des3_ede-caam-qi",
1953                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1954                         },
1955                         .setkey = des3_aead_setkey,
1956                         .setauthsize = aead_setauthsize,
1957                         .encrypt = aead_encrypt,
1958                         .decrypt = aead_decrypt,
1959                         .ivsize = DES3_EDE_BLOCK_SIZE,
1960                         .maxauthsize = SHA224_DIGEST_SIZE,
1961                 },
1962                 .caam = {
1963                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1964                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1965                                            OP_ALG_AAI_HMAC_PRECOMP,
1966                         .geniv = true,
1967                 }
1968         },
1969         {
1970                 .aead = {
1971                         .base = {
1972                                 .cra_name = "authenc(hmac(sha256),"
1973                                             "cbc(des3_ede))",
1974                                 .cra_driver_name = "authenc-hmac-sha256-"
1975                                                    "cbc-des3_ede-caam-qi",
1976                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1977                         },
1978                         .setkey = des3_aead_setkey,
1979                         .setauthsize = aead_setauthsize,
1980                         .encrypt = aead_encrypt,
1981                         .decrypt = aead_decrypt,
1982                         .ivsize = DES3_EDE_BLOCK_SIZE,
1983                         .maxauthsize = SHA256_DIGEST_SIZE,
1984                 },
1985                 .caam = {
1986                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1987                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1988                                            OP_ALG_AAI_HMAC_PRECOMP,
1989                 },
1990         },
1991         {
1992                 .aead = {
1993                         .base = {
1994                                 .cra_name = "echainiv(authenc(hmac(sha256),"
1995                                             "cbc(des3_ede)))",
1996                                 .cra_driver_name = "echainiv-authenc-"
1997                                                    "hmac-sha256-"
1998                                                    "cbc-des3_ede-caam-qi",
1999                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2000                         },
2001                         .setkey = des3_aead_setkey,
2002                         .setauthsize = aead_setauthsize,
2003                         .encrypt = aead_encrypt,
2004                         .decrypt = aead_decrypt,
2005                         .ivsize = DES3_EDE_BLOCK_SIZE,
2006                         .maxauthsize = SHA256_DIGEST_SIZE,
2007                 },
2008                 .caam = {
2009                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2010                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2011                                            OP_ALG_AAI_HMAC_PRECOMP,
2012                         .geniv = true,
2013                 }
2014         },
2015         {
2016                 .aead = {
2017                         .base = {
2018                                 .cra_name = "authenc(hmac(sha384),"
2019                                             "cbc(des3_ede))",
2020                                 .cra_driver_name = "authenc-hmac-sha384-"
2021                                                    "cbc-des3_ede-caam-qi",
2022                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2023                         },
2024                         .setkey = des3_aead_setkey,
2025                         .setauthsize = aead_setauthsize,
2026                         .encrypt = aead_encrypt,
2027                         .decrypt = aead_decrypt,
2028                         .ivsize = DES3_EDE_BLOCK_SIZE,
2029                         .maxauthsize = SHA384_DIGEST_SIZE,
2030                 },
2031                 .caam = {
2032                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2033                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2034                                            OP_ALG_AAI_HMAC_PRECOMP,
2035                 },
2036         },
2037         {
2038                 .aead = {
2039                         .base = {
2040                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2041                                             "cbc(des3_ede)))",
2042                                 .cra_driver_name = "echainiv-authenc-"
2043                                                    "hmac-sha384-"
2044                                                    "cbc-des3_ede-caam-qi",
2045                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2046                         },
2047                         .setkey = des3_aead_setkey,
2048                         .setauthsize = aead_setauthsize,
2049                         .encrypt = aead_encrypt,
2050                         .decrypt = aead_decrypt,
2051                         .ivsize = DES3_EDE_BLOCK_SIZE,
2052                         .maxauthsize = SHA384_DIGEST_SIZE,
2053                 },
2054                 .caam = {
2055                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2056                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2057                                            OP_ALG_AAI_HMAC_PRECOMP,
2058                         .geniv = true,
2059                 }
2060         },
2061         {
2062                 .aead = {
2063                         .base = {
2064                                 .cra_name = "authenc(hmac(sha512),"
2065                                             "cbc(des3_ede))",
2066                                 .cra_driver_name = "authenc-hmac-sha512-"
2067                                                    "cbc-des3_ede-caam-qi",
2068                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2069                         },
2070                         .setkey = des3_aead_setkey,
2071                         .setauthsize = aead_setauthsize,
2072                         .encrypt = aead_encrypt,
2073                         .decrypt = aead_decrypt,
2074                         .ivsize = DES3_EDE_BLOCK_SIZE,
2075                         .maxauthsize = SHA512_DIGEST_SIZE,
2076                 },
2077                 .caam = {
2078                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2079                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2080                                            OP_ALG_AAI_HMAC_PRECOMP,
2081                 },
2082         },
2083         {
2084                 .aead = {
2085                         .base = {
2086                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2087                                             "cbc(des3_ede)))",
2088                                 .cra_driver_name = "echainiv-authenc-"
2089                                                    "hmac-sha512-"
2090                                                    "cbc-des3_ede-caam-qi",
2091                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2092                         },
2093                         .setkey = des3_aead_setkey,
2094                         .setauthsize = aead_setauthsize,
2095                         .encrypt = aead_encrypt,
2096                         .decrypt = aead_decrypt,
2097                         .ivsize = DES3_EDE_BLOCK_SIZE,
2098                         .maxauthsize = SHA512_DIGEST_SIZE,
2099                 },
2100                 .caam = {
2101                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2102                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2103                                            OP_ALG_AAI_HMAC_PRECOMP,
2104                         .geniv = true,
2105                 }
2106         },
2107         {
2108                 .aead = {
2109                         .base = {
2110                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2111                                 .cra_driver_name = "authenc-hmac-md5-"
2112                                                    "cbc-des-caam-qi",
2113                                 .cra_blocksize = DES_BLOCK_SIZE,
2114                         },
2115                         .setkey = aead_setkey,
2116                         .setauthsize = aead_setauthsize,
2117                         .encrypt = aead_encrypt,
2118                         .decrypt = aead_decrypt,
2119                         .ivsize = DES_BLOCK_SIZE,
2120                         .maxauthsize = MD5_DIGEST_SIZE,
2121                 },
2122                 .caam = {
2123                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2124                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2125                                            OP_ALG_AAI_HMAC_PRECOMP,
2126                 },
2127         },
2128         {
2129                 .aead = {
2130                         .base = {
2131                                 .cra_name = "echainiv(authenc(hmac(md5),"
2132                                             "cbc(des)))",
2133                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2134                                                    "cbc-des-caam-qi",
2135                                 .cra_blocksize = DES_BLOCK_SIZE,
2136                         },
2137                         .setkey = aead_setkey,
2138                         .setauthsize = aead_setauthsize,
2139                         .encrypt = aead_encrypt,
2140                         .decrypt = aead_decrypt,
2141                         .ivsize = DES_BLOCK_SIZE,
2142                         .maxauthsize = MD5_DIGEST_SIZE,
2143                 },
2144                 .caam = {
2145                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2146                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2147                                            OP_ALG_AAI_HMAC_PRECOMP,
2148                         .geniv = true,
2149                 }
2150         },
2151         {
2152                 .aead = {
2153                         .base = {
2154                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2155                                 .cra_driver_name = "authenc-hmac-sha1-"
2156                                                    "cbc-des-caam-qi",
2157                                 .cra_blocksize = DES_BLOCK_SIZE,
2158                         },
2159                         .setkey = aead_setkey,
2160                         .setauthsize = aead_setauthsize,
2161                         .encrypt = aead_encrypt,
2162                         .decrypt = aead_decrypt,
2163                         .ivsize = DES_BLOCK_SIZE,
2164                         .maxauthsize = SHA1_DIGEST_SIZE,
2165                 },
2166                 .caam = {
2167                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2168                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2169                                            OP_ALG_AAI_HMAC_PRECOMP,
2170                 },
2171         },
2172         {
2173                 .aead = {
2174                         .base = {
2175                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2176                                             "cbc(des)))",
2177                                 .cra_driver_name = "echainiv-authenc-"
2178                                                    "hmac-sha1-cbc-des-caam-qi",
2179                                 .cra_blocksize = DES_BLOCK_SIZE,
2180                         },
2181                         .setkey = aead_setkey,
2182                         .setauthsize = aead_setauthsize,
2183                         .encrypt = aead_encrypt,
2184                         .decrypt = aead_decrypt,
2185                         .ivsize = DES_BLOCK_SIZE,
2186                         .maxauthsize = SHA1_DIGEST_SIZE,
2187                 },
2188                 .caam = {
2189                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2190                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2191                                            OP_ALG_AAI_HMAC_PRECOMP,
2192                         .geniv = true,
2193                 }
2194         },
2195         {
2196                 .aead = {
2197                         .base = {
2198                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2199                                 .cra_driver_name = "authenc-hmac-sha224-"
2200                                                    "cbc-des-caam-qi",
2201                                 .cra_blocksize = DES_BLOCK_SIZE,
2202                         },
2203                         .setkey = aead_setkey,
2204                         .setauthsize = aead_setauthsize,
2205                         .encrypt = aead_encrypt,
2206                         .decrypt = aead_decrypt,
2207                         .ivsize = DES_BLOCK_SIZE,
2208                         .maxauthsize = SHA224_DIGEST_SIZE,
2209                 },
2210                 .caam = {
2211                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2212                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2213                                            OP_ALG_AAI_HMAC_PRECOMP,
2214                 },
2215         },
2216         {
2217                 .aead = {
2218                         .base = {
2219                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2220                                             "cbc(des)))",
2221                                 .cra_driver_name = "echainiv-authenc-"
2222                                                    "hmac-sha224-cbc-des-"
2223                                                    "caam-qi",
2224                                 .cra_blocksize = DES_BLOCK_SIZE,
2225                         },
2226                         .setkey = aead_setkey,
2227                         .setauthsize = aead_setauthsize,
2228                         .encrypt = aead_encrypt,
2229                         .decrypt = aead_decrypt,
2230                         .ivsize = DES_BLOCK_SIZE,
2231                         .maxauthsize = SHA224_DIGEST_SIZE,
2232                 },
2233                 .caam = {
2234                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2235                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2236                                            OP_ALG_AAI_HMAC_PRECOMP,
2237                         .geniv = true,
2238                 }
2239         },
2240         {
2241                 .aead = {
2242                         .base = {
2243                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2244                                 .cra_driver_name = "authenc-hmac-sha256-"
2245                                                    "cbc-des-caam-qi",
2246                                 .cra_blocksize = DES_BLOCK_SIZE,
2247                         },
2248                         .setkey = aead_setkey,
2249                         .setauthsize = aead_setauthsize,
2250                         .encrypt = aead_encrypt,
2251                         .decrypt = aead_decrypt,
2252                         .ivsize = DES_BLOCK_SIZE,
2253                         .maxauthsize = SHA256_DIGEST_SIZE,
2254                 },
2255                 .caam = {
2256                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2257                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2258                                            OP_ALG_AAI_HMAC_PRECOMP,
2259                 },
2260         },
2261         {
2262                 .aead = {
2263                         .base = {
2264                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2265                                             "cbc(des)))",
2266                                 .cra_driver_name = "echainiv-authenc-"
2267                                                    "hmac-sha256-cbc-des-"
2268                                                    "caam-qi",
2269                                 .cra_blocksize = DES_BLOCK_SIZE,
2270                         },
2271                         .setkey = aead_setkey,
2272                         .setauthsize = aead_setauthsize,
2273                         .encrypt = aead_encrypt,
2274                         .decrypt = aead_decrypt,
2275                         .ivsize = DES_BLOCK_SIZE,
2276                         .maxauthsize = SHA256_DIGEST_SIZE,
2277                 },
2278                 .caam = {
2279                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2280                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2281                                            OP_ALG_AAI_HMAC_PRECOMP,
2282                         .geniv = true,
2283                 },
2284         },
2285         {
2286                 .aead = {
2287                         .base = {
2288                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2289                                 .cra_driver_name = "authenc-hmac-sha384-"
2290                                                    "cbc-des-caam-qi",
2291                                 .cra_blocksize = DES_BLOCK_SIZE,
2292                         },
2293                         .setkey = aead_setkey,
2294                         .setauthsize = aead_setauthsize,
2295                         .encrypt = aead_encrypt,
2296                         .decrypt = aead_decrypt,
2297                         .ivsize = DES_BLOCK_SIZE,
2298                         .maxauthsize = SHA384_DIGEST_SIZE,
2299                 },
2300                 .caam = {
2301                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2302                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2303                                            OP_ALG_AAI_HMAC_PRECOMP,
2304                 },
2305         },
2306         {
2307                 .aead = {
2308                         .base = {
2309                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2310                                             "cbc(des)))",
2311                                 .cra_driver_name = "echainiv-authenc-"
2312                                                    "hmac-sha384-cbc-des-"
2313                                                    "caam-qi",
2314                                 .cra_blocksize = DES_BLOCK_SIZE,
2315                         },
2316                         .setkey = aead_setkey,
2317                         .setauthsize = aead_setauthsize,
2318                         .encrypt = aead_encrypt,
2319                         .decrypt = aead_decrypt,
2320                         .ivsize = DES_BLOCK_SIZE,
2321                         .maxauthsize = SHA384_DIGEST_SIZE,
2322                 },
2323                 .caam = {
2324                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2325                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2326                                            OP_ALG_AAI_HMAC_PRECOMP,
2327                         .geniv = true,
2328                 }
2329         },
2330         {
2331                 .aead = {
2332                         .base = {
2333                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2334                                 .cra_driver_name = "authenc-hmac-sha512-"
2335                                                    "cbc-des-caam-qi",
2336                                 .cra_blocksize = DES_BLOCK_SIZE,
2337                         },
2338                         .setkey = aead_setkey,
2339                         .setauthsize = aead_setauthsize,
2340                         .encrypt = aead_encrypt,
2341                         .decrypt = aead_decrypt,
2342                         .ivsize = DES_BLOCK_SIZE,
2343                         .maxauthsize = SHA512_DIGEST_SIZE,
2344                 },
2345                 .caam = {
2346                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2347                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2348                                            OP_ALG_AAI_HMAC_PRECOMP,
2349                 }
2350         },
2351         {
2352                 .aead = {
2353                         .base = {
2354                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2355                                             "cbc(des)))",
2356                                 .cra_driver_name = "echainiv-authenc-"
2357                                                    "hmac-sha512-cbc-des-"
2358                                                    "caam-qi",
2359                                 .cra_blocksize = DES_BLOCK_SIZE,
2360                         },
2361                         .setkey = aead_setkey,
2362                         .setauthsize = aead_setauthsize,
2363                         .encrypt = aead_encrypt,
2364                         .decrypt = aead_decrypt,
2365                         .ivsize = DES_BLOCK_SIZE,
2366                         .maxauthsize = SHA512_DIGEST_SIZE,
2367                 },
2368                 .caam = {
2369                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2370                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2371                                            OP_ALG_AAI_HMAC_PRECOMP,
2372                         .geniv = true,
2373                 }
2374         },
2375 };
2376
2377 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
2378                             bool uses_dkp)
2379 {
2380         struct caam_drv_private *priv;
2381
2382         /*
2383          * distribute tfms across job rings to ensure in-order
2384          * crypto request processing per tfm
2385          */
2386         ctx->jrdev = caam_jr_alloc();
2387         if (IS_ERR(ctx->jrdev)) {
2388                 pr_err("Job Ring Device allocation for transform failed\n");
2389                 return PTR_ERR(ctx->jrdev);
2390         }
2391
2392         priv = dev_get_drvdata(ctx->jrdev->parent);
2393         if (priv->era >= 6 && uses_dkp)
2394                 ctx->dir = DMA_BIDIRECTIONAL;
2395         else
2396                 ctx->dir = DMA_TO_DEVICE;
2397
2398         ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key),
2399                                       ctx->dir);
2400         if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) {
2401                 dev_err(ctx->jrdev, "unable to map key\n");
2402                 caam_jr_free(ctx->jrdev);
2403                 return -ENOMEM;
2404         }
2405
2406         /* copy descriptor header template value */
2407         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
2408         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
2409
2410         ctx->qidev = priv->qidev;
2411
2412         spin_lock_init(&ctx->lock);
2413         ctx->drv_ctx[ENCRYPT] = NULL;
2414         ctx->drv_ctx[DECRYPT] = NULL;
2415
2416         return 0;
2417 }
2418
2419 static int caam_cra_init(struct crypto_skcipher *tfm)
2420 {
2421         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
2422         struct caam_skcipher_alg *caam_alg =
2423                 container_of(alg, typeof(*caam_alg), skcipher);
2424
2425         return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam,
2426                                 false);
2427 }
2428
2429 static int caam_aead_init(struct crypto_aead *tfm)
2430 {
2431         struct aead_alg *alg = crypto_aead_alg(tfm);
2432         struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
2433                                                       aead);
2434         struct caam_ctx *ctx = crypto_aead_ctx(tfm);
2435
2436         return caam_init_common(ctx, &caam_alg->caam,
2437                                 alg->setkey == aead_setkey);
2438 }
2439
2440 static void caam_exit_common(struct caam_ctx *ctx)
2441 {
2442         caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
2443         caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
2444
2445         dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir);
2446
2447         caam_jr_free(ctx->jrdev);
2448 }
2449
2450 static void caam_cra_exit(struct crypto_skcipher *tfm)
2451 {
2452         caam_exit_common(crypto_skcipher_ctx(tfm));
2453 }
2454
2455 static void caam_aead_exit(struct crypto_aead *tfm)
2456 {
2457         caam_exit_common(crypto_aead_ctx(tfm));
2458 }
2459
2460 static void __exit caam_qi_algapi_exit(void)
2461 {
2462         int i;
2463
2464         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2465                 struct caam_aead_alg *t_alg = driver_aeads + i;
2466
2467                 if (t_alg->registered)
2468                         crypto_unregister_aead(&t_alg->aead);
2469         }
2470
2471         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2472                 struct caam_skcipher_alg *t_alg = driver_algs + i;
2473
2474                 if (t_alg->registered)
2475                         crypto_unregister_skcipher(&t_alg->skcipher);
2476         }
2477 }
2478
2479 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
2480 {
2481         struct skcipher_alg *alg = &t_alg->skcipher;
2482
2483         alg->base.cra_module = THIS_MODULE;
2484         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2485         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2486         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2487
2488         alg->init = caam_cra_init;
2489         alg->exit = caam_cra_exit;
2490 }
2491
2492 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
2493 {
2494         struct aead_alg *alg = &t_alg->aead;
2495
2496         alg->base.cra_module = THIS_MODULE;
2497         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2498         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2499         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2500
2501         alg->init = caam_aead_init;
2502         alg->exit = caam_aead_exit;
2503 }
2504
2505 static int __init caam_qi_algapi_init(void)
2506 {
2507         struct device_node *dev_node;
2508         struct platform_device *pdev;
2509         struct device *ctrldev;
2510         struct caam_drv_private *priv;
2511         int i = 0, err = 0;
2512         u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;
2513         unsigned int md_limit = SHA512_DIGEST_SIZE;
2514         bool registered = false;
2515
2516         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
2517         if (!dev_node) {
2518                 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
2519                 if (!dev_node)
2520                         return -ENODEV;
2521         }
2522
2523         pdev = of_find_device_by_node(dev_node);
2524         of_node_put(dev_node);
2525         if (!pdev)
2526                 return -ENODEV;
2527
2528         ctrldev = &pdev->dev;
2529         priv = dev_get_drvdata(ctrldev);
2530
2531         /*
2532          * If priv is NULL, it's probably because the caam driver wasn't
2533          * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
2534          */
2535         if (!priv || !priv->qi_present) {
2536                 err = -ENODEV;
2537                 goto out_put_dev;
2538         }
2539
2540         if (caam_dpaa2) {
2541                 dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n");
2542                 err = -ENODEV;
2543                 goto out_put_dev;
2544         }
2545
2546         /*
2547          * Register crypto algorithms the device supports.
2548          * First, detect presence and attributes of DES, AES, and MD blocks.
2549          */
2550         if (priv->era < 10) {
2551                 u32 cha_vid, cha_inst;
2552
2553                 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
2554                 aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
2555                 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2556
2557                 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
2558                 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
2559                            CHA_ID_LS_DES_SHIFT;
2560                 aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
2561                 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2562         } else {
2563                 u32 aesa, mdha;
2564
2565                 aesa = rd_reg32(&priv->ctrl->vreg.aesa);
2566                 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
2567
2568                 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2569                 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2570
2571                 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
2572                 aes_inst = aesa & CHA_VER_NUM_MASK;
2573                 md_inst = mdha & CHA_VER_NUM_MASK;
2574         }
2575
2576         /* If MD is present, limit digest size based on LP256 */
2577         if (md_inst && md_vid  == CHA_VER_VID_MD_LP256)
2578                 md_limit = SHA256_DIGEST_SIZE;
2579
2580         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2581                 struct caam_skcipher_alg *t_alg = driver_algs + i;
2582                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
2583
2584                 /* Skip DES algorithms if not supported by device */
2585                 if (!des_inst &&
2586                     ((alg_sel == OP_ALG_ALGSEL_3DES) ||
2587                      (alg_sel == OP_ALG_ALGSEL_DES)))
2588                         continue;
2589
2590                 /* Skip AES algorithms if not supported by device */
2591                 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
2592                         continue;
2593
2594                 caam_skcipher_alg_init(t_alg);
2595
2596                 err = crypto_register_skcipher(&t_alg->skcipher);
2597                 if (err) {
2598                         dev_warn(priv->qidev, "%s alg registration failed\n",
2599                                  t_alg->skcipher.base.cra_driver_name);
2600                         continue;
2601                 }
2602
2603                 t_alg->registered = true;
2604                 registered = true;
2605         }
2606
2607         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2608                 struct caam_aead_alg *t_alg = driver_aeads + i;
2609                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
2610                                  OP_ALG_ALGSEL_MASK;
2611                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
2612                                  OP_ALG_ALGSEL_MASK;
2613                 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
2614
2615                 /* Skip DES algorithms if not supported by device */
2616                 if (!des_inst &&
2617                     ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
2618                      (c1_alg_sel == OP_ALG_ALGSEL_DES)))
2619                         continue;
2620
2621                 /* Skip AES algorithms if not supported by device */
2622                 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
2623                         continue;
2624
2625                 /*
2626                  * Check support for AES algorithms not available
2627                  * on LP devices.
2628                  */
2629                 if (aes_vid  == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM)
2630                         continue;
2631
2632                 /*
2633                  * Skip algorithms requiring message digests
2634                  * if MD or MD size is not supported by device.
2635                  */
2636                 if (c2_alg_sel &&
2637                     (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
2638                         continue;
2639
2640                 caam_aead_alg_init(t_alg);
2641
2642                 err = crypto_register_aead(&t_alg->aead);
2643                 if (err) {
2644                         pr_warn("%s alg registration failed\n",
2645                                 t_alg->aead.base.cra_driver_name);
2646                         continue;
2647                 }
2648
2649                 t_alg->registered = true;
2650                 registered = true;
2651         }
2652
2653         if (registered)
2654                 dev_info(priv->qidev, "algorithms registered in /proc/crypto\n");
2655
2656 out_put_dev:
2657         put_device(ctrldev);
2658         return err;
2659 }
2660
2661 module_init(caam_qi_algapi_init);
2662 module_exit(caam_qi_algapi_exit);
2663
2664 MODULE_LICENSE("GPL");
2665 MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend");
2666 MODULE_AUTHOR("Freescale Semiconductor");