drivers/crypto/caam/caamalg_qi2.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /*
   3  * Copyright 2015-2016 Freescale Semiconductor Inc.
   4  * Copyright 2017-2019 NXP
   5  */
   6
   7 #include "compat.h"
   8 #include "regs.h"
   9 #include "caamalg_qi2.h"
  10 #include "dpseci_cmd.h"
  11 #include "desc_constr.h"
  12 #include "error.h"
  13 #include "sg_sw_sec4.h"
  14 #include "sg_sw_qm2.h"
  15 #include "key_gen.h"
  16 #include "caamalg_desc.h"
  17 #include "caamhash_desc.h"
  18 #include "dpseci-debugfs.h"
  19 #include <linux/fsl/mc.h>
  20 #include <soc/fsl/dpaa2-io.h>
  21 #include <soc/fsl/dpaa2-fd.h>
  22 #include <crypto/xts.h>
  23 #include <asm/unaligned.h>
  24
  25 #define CAAM_CRA_PRIORITY       2000
  26
  27 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  28 #define CAAM_MAX_KEY_SIZE       (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
  29                                  SHA512_DIGEST_SIZE * 2)
  30
  31 /*
  32  * This is a a cache of buffers, from which the users of CAAM QI driver
  33  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
  34  * NOTE: A more elegant solution would be to have some headroom in the frames
  35  *       being processed. This can be added by the dpaa2-eth driver. This would
  36  *       pose a problem for userspace application processing which cannot
  37  *       know of this limitation. So for now, this will work.
  38  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
  39  */
  40 static struct kmem_cache *qi_cache;
  41
  42 struct caam_alg_entry {
  43         struct device *dev;
  44         int class1_alg_type;
  45         int class2_alg_type;
  46         bool rfc3686;
  47         bool geniv;
  48         bool nodkp;
  49 };
  50
  51 struct caam_aead_alg {
  52         struct aead_alg aead;
  53         struct caam_alg_entry caam;
  54         bool registered;
  55 };
  56
  57 struct caam_skcipher_alg {
  58         struct skcipher_alg skcipher;
  59         struct caam_alg_entry caam;
  60         bool registered;
  61 };
  62
  63 /**
  64  * struct caam_ctx - per-session context
  65  * @flc: Flow Contexts array
  66  * @key:  [authentication key], encryption key
  67  * @flc_dma: I/O virtual addresses of the Flow Contexts
  68  * @key_dma: I/O virtual address of the key
  69  * @dir: DMA direction for mapping key and Flow Contexts
  70  * @dev: dpseci device
  71  * @adata: authentication algorithm details
  72  * @cdata: encryption algorithm details
  73  * @authsize: authentication tag (a.k.a. ICV / MAC) size
  74  * @xts_key_fallback: true if fallback tfm needs to be used due
  75  *                    to unsupported xts key lengths
  76  * @fallback: xts fallback tfm
  77  */
  78 struct caam_ctx {
  79         struct caam_flc flc[NUM_OP];
  80         u8 key[CAAM_MAX_KEY_SIZE];
  81         dma_addr_t flc_dma[NUM_OP];
  82         dma_addr_t key_dma;
  83         enum dma_data_direction dir;
  84         struct device *dev;
  85         struct alginfo adata;
  86         struct alginfo cdata;
  87         unsigned int authsize;
  88         bool xts_key_fallback;
  89         struct crypto_skcipher *fallback;
  90 };
  91
  92 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
  93                                      dma_addr_t iova_addr)
  94 {
  95         phys_addr_t phys_addr;
  96
  97         phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
  98                                    iova_addr;
  99
 100         return phys_to_virt(phys_addr);
 101 }
 102
 103 /*
 104  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
 105  *
 106  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
 107  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
 108  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
 109  * hosting 16 SG entries.
 110  *
 111  * @flags - flags that would be used for the equivalent kmalloc(..) call
 112  *
 113  * Returns a pointer to a retrieved buffer on success or NULL on failure.
 114  */
 115 static inline void *qi_cache_zalloc(gfp_t flags)
 116 {
 117         return kmem_cache_zalloc(qi_cache, flags);
 118 }
 119
 120 /*
 121  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
 122  *
 123  * @obj - buffer previously allocated by qi_cache_zalloc
 124  *
 125  * No checking is being done, the call is a passthrough call to
 126  * kmem_cache_free(...)
 127  */
 128 static inline void qi_cache_free(void *obj)
 129 {
 130         kmem_cache_free(qi_cache, obj);
 131 }
 132
 133 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
 134 {
 135         switch (crypto_tfm_alg_type(areq->tfm)) {
 136         case CRYPTO_ALG_TYPE_SKCIPHER:
 137                 return skcipher_request_ctx(skcipher_request_cast(areq));
 138         case CRYPTO_ALG_TYPE_AEAD:
 139                 return aead_request_ctx(container_of(areq, struct aead_request,
 140                                                      base));
 141         case CRYPTO_ALG_TYPE_AHASH:
 142                 return ahash_request_ctx(ahash_request_cast(areq));
 143         default:
 144                 return ERR_PTR(-EINVAL);
 145         }
 146 }
 147
 148 static void caam_unmap(struct device *dev, struct scatterlist *src,
 149                        struct scatterlist *dst, int src_nents,
 150                        int dst_nents, dma_addr_t iv_dma, int ivsize,
 151                        enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
 152                        int qm_sg_bytes)
 153 {
 154         if (dst != src) {
 155                 if (src_nents)
 156                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 157                 if (dst_nents)
 158                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 159         } else {
 160                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 161         }
 162
 163         if (iv_dma)
 164                 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 165
 166         if (qm_sg_bytes)
 167                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 168 }
 169
 170 static int aead_set_sh_desc(struct crypto_aead *aead)
 171 {
 172         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 173                                                  typeof(*alg), aead);
 174         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 175         unsigned int ivsize = crypto_aead_ivsize(aead);
 176         struct device *dev = ctx->dev;
 177         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
 178         struct caam_flc *flc;
 179         u32 *desc;
 180         u32 ctx1_iv_off = 0;
 181         u32 *nonce = NULL;
 182         unsigned int data_len[2];
 183         u32 inl_mask;
 184         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 185                                OP_ALG_AAI_CTR_MOD128);
 186         const bool is_rfc3686 = alg->caam.rfc3686;
 187
 188         if (!ctx->cdata.keylen || !ctx->authsize)
 189                 return 0;
 190
 191         /*
 192          * AES-CTR needs to load IV in CONTEXT1 reg
 193          * at an offset of 128bits (16bytes)
 194          * CONTEXT1[255:128] = IV
 195          */
 196         if (ctr_mode)
 197                 ctx1_iv_off = 16;
 198
 199         /*
 200          * RFC3686 specific:
 201          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 202          */
 203         if (is_rfc3686) {
 204                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 205                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 206                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 207         }
 208
 209         /*
 210          * In case |user key| > |derived key|, using DKP<imm,imm> would result
 211          * in invalid opcodes (last bytes of user key) in the resulting
 212          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
 213          * addresses are needed.
 214          */
 215         ctx->adata.key_virt = ctx->key;
 216         ctx->adata.key_dma = ctx->key_dma;
 217
 218         ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 219         ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 220
 221         data_len[0] = ctx->adata.keylen_pad;
 222         data_len[1] = ctx->cdata.keylen;
 223
 224         /* aead_encrypt shared descriptor */
 225         if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
 226                                                  DESC_QI_AEAD_ENC_LEN) +
 227                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 228                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 229                               ARRAY_SIZE(data_len)) < 0)
 230                 return -EINVAL;
 231
 232         ctx->adata.key_inline = !!(inl_mask & 1);
 233         ctx->cdata.key_inline = !!(inl_mask & 2);
 234
 235         flc = &ctx->flc[ENCRYPT];
 236         desc = flc->sh_desc;
 237
 238         if (alg->caam.geniv)
 239                 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
 240                                           ivsize, ctx->authsize, is_rfc3686,
 241                                           nonce, ctx1_iv_off, true,
 242                                           priv->sec_attr.era);
 243         else
 244                 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
 245                                        ivsize, ctx->authsize, is_rfc3686, nonce,
 246                                        ctx1_iv_off, true, priv->sec_attr.era);
 247
 248         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 249         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 250                                    sizeof(flc->flc) + desc_bytes(desc),
 251                                    ctx->dir);
 252
 253         /* aead_decrypt shared descriptor */
 254         if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
 255                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 256                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 257                               ARRAY_SIZE(data_len)) < 0)
 258                 return -EINVAL;
 259
 260         ctx->adata.key_inline = !!(inl_mask & 1);
 261         ctx->cdata.key_inline = !!(inl_mask & 2);
 262
 263         flc = &ctx->flc[DECRYPT];
 264         desc = flc->sh_desc;
 265         cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
 266                                ivsize, ctx->authsize, alg->caam.geniv,
 267                                is_rfc3686, nonce, ctx1_iv_off, true,
 268                                priv->sec_attr.era);
 269         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 270         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 271                                    sizeof(flc->flc) + desc_bytes(desc),
 272                                    ctx->dir);
 273
 274         return 0;
 275 }
 276
 277 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 278 {
 279         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 280
 281         ctx->authsize = authsize;
 282         aead_set_sh_desc(authenc);
 283
 284         return 0;
 285 }
 286
 287 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 288                        unsigned int keylen)
 289 {
 290         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 291         struct device *dev = ctx->dev;
 292         struct crypto_authenc_keys keys;
 293
 294         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 295                 goto badkey;
 296
 297         dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
 298                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
 299                 keys.authkeylen);
 300         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 301                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 302
 303         ctx->adata.keylen = keys.authkeylen;
 304         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 305                                               OP_ALG_ALGSEL_MASK);
 306
 307         if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 308                 goto badkey;
 309
 310         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 311         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 312         dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
 313                                    keys.enckeylen, ctx->dir);
 314         print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
 315                              DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 316                              ctx->adata.keylen_pad + keys.enckeylen, 1);
 317
 318         ctx->cdata.keylen = keys.enckeylen;
 319
 320         memzero_explicit(&keys, sizeof(keys));
 321         return aead_set_sh_desc(aead);
 322 badkey:
 323         memzero_explicit(&keys, sizeof(keys));
 324         return -EINVAL;
 325 }
 326
 327 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 328                             unsigned int keylen)
 329 {
 330         struct crypto_authenc_keys keys;
 331         int err;
 332
 333         err = crypto_authenc_extractkeys(&keys, key, keylen);
 334         if (unlikely(err))
 335                 goto out;
 336
 337         err = -EINVAL;
 338         if (keys.enckeylen != DES3_EDE_KEY_SIZE)
 339                 goto out;
 340
 341         err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
 342               aead_setkey(aead, key, keylen);
 343
 344 out:
 345         memzero_explicit(&keys, sizeof(keys));
 346         return err;
 347 }
 348
 349 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 350                                            bool encrypt)
 351 {
 352         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 353         struct caam_request *req_ctx = aead_request_ctx(req);
 354         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
 355         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
 356         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 357         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 358                                                  typeof(*alg), aead);
 359         struct device *dev = ctx->dev;
 360         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 361                       GFP_KERNEL : GFP_ATOMIC;
 362         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 363         int src_len, dst_len = 0;
 364         struct aead_edesc *edesc;
 365         dma_addr_t qm_sg_dma, iv_dma = 0;
 366         int ivsize = 0;
 367         unsigned int authsize = ctx->authsize;
 368         int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
 369         int in_len, out_len;
 370         struct dpaa2_sg_entry *sg_table;
 371
 372         /* allocate space for base edesc, link tables and IV */
 373         edesc = qi_cache_zalloc(GFP_DMA | flags);
 374         if (unlikely(!edesc)) {
 375                 dev_err(dev, "could not allocate extended descriptor\n");
 376                 return ERR_PTR(-ENOMEM);
 377         }
 378
 379         if (unlikely(req->dst != req->src)) {
 380                 src_len = req->assoclen + req->cryptlen;
 381                 dst_len = src_len + (encrypt ? authsize : (-authsize));
 382
 383                 src_nents = sg_nents_for_len(req->src, src_len);
 384                 if (unlikely(src_nents < 0)) {
 385                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 386                                 src_len);
 387                         qi_cache_free(edesc);
 388                         return ERR_PTR(src_nents);
 389                 }
 390
 391                 dst_nents = sg_nents_for_len(req->dst, dst_len);
 392                 if (unlikely(dst_nents < 0)) {
 393                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
 394                                 dst_len);
 395                         qi_cache_free(edesc);
 396                         return ERR_PTR(dst_nents);
 397                 }
 398
 399                 if (src_nents) {
 400                         mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 401                                                       DMA_TO_DEVICE);
 402                         if (unlikely(!mapped_src_nents)) {
 403                                 dev_err(dev, "unable to map source\n");
 404                                 qi_cache_free(edesc);
 405                                 return ERR_PTR(-ENOMEM);
 406                         }
 407                 } else {
 408                         mapped_src_nents = 0;
 409                 }
 410
 411                 if (dst_nents) {
 412                         mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
 413                                                       DMA_FROM_DEVICE);
 414                         if (unlikely(!mapped_dst_nents)) {
 415                                 dev_err(dev, "unable to map destination\n");
 416                                 dma_unmap_sg(dev, req->src, src_nents,
 417                                              DMA_TO_DEVICE);
 418                                 qi_cache_free(edesc);
 419                                 return ERR_PTR(-ENOMEM);
 420                         }
 421                 } else {
 422                         mapped_dst_nents = 0;
 423                 }
 424         } else {
 425                 src_len = req->assoclen + req->cryptlen +
 426                           (encrypt ? authsize : 0);
 427
 428                 src_nents = sg_nents_for_len(req->src, src_len);
 429                 if (unlikely(src_nents < 0)) {
 430                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 431                                 src_len);
 432                         qi_cache_free(edesc);
 433                         return ERR_PTR(src_nents);
 434                 }
 435
 436                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 437                                               DMA_BIDIRECTIONAL);
 438                 if (unlikely(!mapped_src_nents)) {
 439                         dev_err(dev, "unable to map source\n");
 440                         qi_cache_free(edesc);
 441                         return ERR_PTR(-ENOMEM);
 442                 }
 443         }
 444
 445         if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
 446                 ivsize = crypto_aead_ivsize(aead);
 447
 448         /*
 449          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
 450          * Input is not contiguous.
 451          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
 452          * the end of the table by allocating more S/G entries. Logic:
 453          * if (src != dst && output S/G)
 454          *      pad output S/G, if needed
 455          * else if (src == dst && S/G)
 456          *      overlapping S/Gs; pad one of them
 457          * else if (input S/G) ...
 458          *      pad input S/G, if needed
 459          */
 460         qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
 461         if (mapped_dst_nents > 1)
 462                 qm_sg_nents += pad_sg_nents(mapped_dst_nents);
 463         else if ((req->src == req->dst) && (mapped_src_nents > 1))
 464                 qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
 465                                   1 + !!ivsize +
 466                                   pad_sg_nents(mapped_src_nents));
 467         else
 468                 qm_sg_nents = pad_sg_nents(qm_sg_nents);
 469
 470         sg_table = &edesc->sgt[0];
 471         qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
 472         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
 473                      CAAM_QI_MEMCACHE_SIZE)) {
 474                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
 475                         qm_sg_nents, ivsize);
 476                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
 477                            0, DMA_NONE, 0, 0);
 478                 qi_cache_free(edesc);
 479                 return ERR_PTR(-ENOMEM);
 480         }
 481
 482         if (ivsize) {
 483                 u8 *iv = (u8 *)(sg_table + qm_sg_nents);
 484
 485                 /* Make sure IV is located in a DMAable area */
 486                 memcpy(iv, req->iv, ivsize);
 487
 488                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
 489                 if (dma_mapping_error(dev, iv_dma)) {
 490                         dev_err(dev, "unable to map IV\n");
 491                         caam_unmap(dev, req->src, req->dst, src_nents,
 492                                    dst_nents, 0, 0, DMA_NONE, 0, 0);
 493                         qi_cache_free(edesc);
 494                         return ERR_PTR(-ENOMEM);
 495                 }
 496         }
 497
 498         edesc->src_nents = src_nents;
 499         edesc->dst_nents = dst_nents;
 500         edesc->iv_dma = iv_dma;
 501
 502         if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
 503             OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
 504                 /*
 505                  * The associated data comes already with the IV but we need
 506                  * to skip it when we authenticate or encrypt...
 507                  */
 508                 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
 509         else
 510                 edesc->assoclen = cpu_to_caam32(req->assoclen);
 511         edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
 512                                              DMA_TO_DEVICE);
 513         if (dma_mapping_error(dev, edesc->assoclen_dma)) {
 514                 dev_err(dev, "unable to map assoclen\n");
 515                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 516                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 517                 qi_cache_free(edesc);
 518                 return ERR_PTR(-ENOMEM);
 519         }
 520
 521         dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
 522         qm_sg_index++;
 523         if (ivsize) {
 524                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
 525                 qm_sg_index++;
 526         }
 527         sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
 528         qm_sg_index += mapped_src_nents;
 529
 530         if (mapped_dst_nents > 1)
 531                 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
 532
 533         qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
 534         if (dma_mapping_error(dev, qm_sg_dma)) {
 535                 dev_err(dev, "unable to map S/G table\n");
 536                 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 537                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 538                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 539                 qi_cache_free(edesc);
 540                 return ERR_PTR(-ENOMEM);
 541         }
 542
 543         edesc->qm_sg_dma = qm_sg_dma;
 544         edesc->qm_sg_bytes = qm_sg_bytes;
 545
 546         out_len = req->assoclen + req->cryptlen +
 547                   (encrypt ? ctx->authsize : (-ctx->authsize));
 548         in_len = 4 + ivsize + req->assoclen + req->cryptlen;
 549
 550         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
 551         dpaa2_fl_set_final(in_fle, true);
 552         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
 553         dpaa2_fl_set_addr(in_fle, qm_sg_dma);
 554         dpaa2_fl_set_len(in_fle, in_len);
 555
 556         if (req->dst == req->src) {
 557                 if (mapped_src_nents == 1) {
 558                         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 559                         dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
 560                 } else {
 561                         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 562                         dpaa2_fl_set_addr(out_fle, qm_sg_dma +
 563                                           (1 + !!ivsize) * sizeof(*sg_table));
 564                 }
 565         } else if (!mapped_dst_nents) {
 566                 /*
 567                  * crypto engine requires the output entry to be present when
 568                  * "frame list" FD is used.
 569                  * Since engine does not support FMT=2'b11 (unused entry type),
 570                  * leaving out_fle zeroized is the best option.
 571                  */
 572                 goto skip_out_fle;
 573         } else if (mapped_dst_nents == 1) {
 574                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 575                 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
 576         } else {
 577                 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 578                 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
 579                                   sizeof(*sg_table));
 580         }
 581
 582         dpaa2_fl_set_len(out_fle, out_len);
 583
 584 skip_out_fle:
 585         return edesc;
 586 }
 587
 588 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
 589 {
 590         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 591         unsigned int ivsize = crypto_aead_ivsize(aead);
 592         struct device *dev = ctx->dev;
 593         struct caam_flc *flc;
 594         u32 *desc;
 595
 596         if (!ctx->cdata.keylen || !ctx->authsize)
 597                 return 0;
 598
 599         flc = &ctx->flc[ENCRYPT];
 600         desc = flc->sh_desc;
 601         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 602                                ctx->authsize, true, true);
 603         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 604         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 605                                    sizeof(flc->flc) + desc_bytes(desc),
 606                                    ctx->dir);
 607
 608         flc = &ctx->flc[DECRYPT];
 609         desc = flc->sh_desc;
 610         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 611                                ctx->authsize, false, true);
 612         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 613         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 614                                    sizeof(flc->flc) + desc_bytes(desc),
 615                                    ctx->dir);
 616
 617         return 0;
 618 }
 619
 620 static int chachapoly_setauthsize(struct crypto_aead *aead,
 621                                   unsigned int authsize)
 622 {
 623         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 624
 625         if (authsize != POLY1305_DIGEST_SIZE)
 626                 return -EINVAL;
 627
 628         ctx->authsize = authsize;
 629         return chachapoly_set_sh_desc(aead);
 630 }
 631
 632 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 633                              unsigned int keylen)
 634 {
 635         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 636         unsigned int ivsize = crypto_aead_ivsize(aead);
 637         unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 638
 639         if (keylen != CHACHA_KEY_SIZE + saltlen)
 640                 return -EINVAL;
 641
 642         ctx->cdata.key_virt = key;
 643         ctx->cdata.keylen = keylen - saltlen;
 644
 645         return chachapoly_set_sh_desc(aead);
 646 }
 647
 648 static int gcm_set_sh_desc(struct crypto_aead *aead)
 649 {
 650         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 651         struct device *dev = ctx->dev;
 652         unsigned int ivsize = crypto_aead_ivsize(aead);
 653         struct caam_flc *flc;
 654         u32 *desc;
 655         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 656                         ctx->cdata.keylen;
 657
 658         if (!ctx->cdata.keylen || !ctx->authsize)
 659                 return 0;
 660
 661         /*
 662          * AES GCM encrypt shared descriptor
 663          * Job Descriptor and Shared Descriptor
 664          * must fit into the 64-word Descriptor h/w Buffer
 665          */
 666         if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
 667                 ctx->cdata.key_inline = true;
 668                 ctx->cdata.key_virt = ctx->key;
 669         } else {
 670                 ctx->cdata.key_inline = false;
 671                 ctx->cdata.key_dma = ctx->key_dma;
 672         }
 673
 674         flc = &ctx->flc[ENCRYPT];
 675         desc = flc->sh_desc;
 676         cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 677         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 678         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 679                                    sizeof(flc->flc) + desc_bytes(desc),
 680                                    ctx->dir);
 681
 682         /*
 683          * Job Descriptor and Shared Descriptors
 684          * must all fit into the 64-word Descriptor h/w Buffer
 685          */
 686         if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
 687                 ctx->cdata.key_inline = true;
 688                 ctx->cdata.key_virt = ctx->key;
 689         } else {
 690                 ctx->cdata.key_inline = false;
 691                 ctx->cdata.key_dma = ctx->key_dma;
 692         }
 693
 694         flc = &ctx->flc[DECRYPT];
 695         desc = flc->sh_desc;
 696         cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 697         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 698         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 699                                    sizeof(flc->flc) + desc_bytes(desc),
 700                                    ctx->dir);
 701
 702         return 0;
 703 }
 704
 705 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 706 {
 707         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 708         int err;
 709
 710         err = crypto_gcm_check_authsize(authsize);
 711         if (err)
 712                 return err;
 713
 714         ctx->authsize = authsize;
 715         gcm_set_sh_desc(authenc);
 716
 717         return 0;
 718 }
 719
 720 static int gcm_setkey(struct crypto_aead *aead,
 721                       const u8 *key, unsigned int keylen)
 722 {
 723         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 724         struct device *dev = ctx->dev;
 725         int ret;
 726
 727         ret = aes_check_keylen(keylen);
 728         if (ret)
 729                 return ret;
 730         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 731                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 732
 733         memcpy(ctx->key, key, keylen);
 734         dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
 735         ctx->cdata.keylen = keylen;
 736
 737         return gcm_set_sh_desc(aead);
 738 }
 739
 740 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 741 {
 742         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 743         struct device *dev = ctx->dev;
 744         unsigned int ivsize = crypto_aead_ivsize(aead);
 745         struct caam_flc *flc;
 746         u32 *desc;
 747         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 748                         ctx->cdata.keylen;
 749
 750         if (!ctx->cdata.keylen || !ctx->authsize)
 751                 return 0;
 752
 753         ctx->cdata.key_virt = ctx->key;
 754
 755         /*
 756          * RFC4106 encrypt shared descriptor
 757          * Job Descriptor and Shared Descriptor
 758          * must fit into the 64-word Descriptor h/w Buffer
 759          */
 760         if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
 761                 ctx->cdata.key_inline = true;
 762         } else {
 763                 ctx->cdata.key_inline = false;
 764                 ctx->cdata.key_dma = ctx->key_dma;
 765         }
 766
 767         flc = &ctx->flc[ENCRYPT];
 768         desc = flc->sh_desc;
 769         cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 770                                   true);
 771         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 772         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 773                                    sizeof(flc->flc) + desc_bytes(desc),
 774                                    ctx->dir);
 775
 776         /*
 777          * Job Descriptor and Shared Descriptors
 778          * must all fit into the 64-word Descriptor h/w Buffer
 779          */
 780         if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
 781                 ctx->cdata.key_inline = true;
 782         } else {
 783                 ctx->cdata.key_inline = false;
 784                 ctx->cdata.key_dma = ctx->key_dma;
 785         }
 786
 787         flc = &ctx->flc[DECRYPT];
 788         desc = flc->sh_desc;
 789         cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 790                                   true);
 791         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 792         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 793                                    sizeof(flc->flc) + desc_bytes(desc),
 794                                    ctx->dir);
 795
 796         return 0;
 797 }
 798
 799 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 800                                unsigned int authsize)
 801 {
 802         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 803         int err;
 804
 805         err = crypto_rfc4106_check_authsize(authsize);
 806         if (err)
 807                 return err;
 808
 809         ctx->authsize = authsize;
 810         rfc4106_set_sh_desc(authenc);
 811
 812         return 0;
 813 }
 814
 815 static int rfc4106_setkey(struct crypto_aead *aead,
 816                           const u8 *key, unsigned int keylen)
 817 {
 818         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 819         struct device *dev = ctx->dev;
 820         int ret;
 821
 822         ret = aes_check_keylen(keylen - 4);
 823         if (ret)
 824                 return ret;
 825
 826         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 827                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 828
 829         memcpy(ctx->key, key, keylen);
 830         /*
 831          * The last four bytes of the key material are used as the salt value
 832          * in the nonce. Update the AES key length.
 833          */
 834         ctx->cdata.keylen = keylen - 4;
 835         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 836                                    ctx->dir);
 837
 838         return rfc4106_set_sh_desc(aead);
 839 }
 840
 841 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 842 {
 843         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 844         struct device *dev = ctx->dev;
 845         unsigned int ivsize = crypto_aead_ivsize(aead);
 846         struct caam_flc *flc;
 847         u32 *desc;
 848         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 849                         ctx->cdata.keylen;
 850
 851         if (!ctx->cdata.keylen || !ctx->authsize)
 852                 return 0;
 853
 854         ctx->cdata.key_virt = ctx->key;
 855
 856         /*
 857          * RFC4543 encrypt shared descriptor
 858          * Job Descriptor and Shared Descriptor
 859          * must fit into the 64-word Descriptor h/w Buffer
 860          */
 861         if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
 862                 ctx->cdata.key_inline = true;
 863         } else {
 864                 ctx->cdata.key_inline = false;
 865                 ctx->cdata.key_dma = ctx->key_dma;
 866         }
 867
 868         flc = &ctx->flc[ENCRYPT];
 869         desc = flc->sh_desc;
 870         cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 871                                   true);
 872         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 873         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 874                                    sizeof(flc->flc) + desc_bytes(desc),
 875                                    ctx->dir);
 876
 877         /*
 878          * Job Descriptor and Shared Descriptors
 879          * must all fit into the 64-word Descriptor h/w Buffer
 880          */
 881         if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
 882                 ctx->cdata.key_inline = true;
 883         } else {
 884                 ctx->cdata.key_inline = false;
 885                 ctx->cdata.key_dma = ctx->key_dma;
 886         }
 887
 888         flc = &ctx->flc[DECRYPT];
 889         desc = flc->sh_desc;
 890         cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 891                                   true);
 892         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 893         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 894                                    sizeof(flc->flc) + desc_bytes(desc),
 895                                    ctx->dir);
 896
 897         return 0;
 898 }
 899
 900 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 901                                unsigned int authsize)
 902 {
 903         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 904
 905         if (authsize != 16)
 906                 return -EINVAL;
 907
 908         ctx->authsize = authsize;
 909         rfc4543_set_sh_desc(authenc);
 910
 911         return 0;
 912 }
 913
 914 static int rfc4543_setkey(struct crypto_aead *aead,
 915                           const u8 *key, unsigned int keylen)
 916 {
 917         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 918         struct device *dev = ctx->dev;
 919         int ret;
 920
 921         ret = aes_check_keylen(keylen - 4);
 922         if (ret)
 923                 return ret;
 924
 925         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 926                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 927
 928         memcpy(ctx->key, key, keylen);
 929         /*
 930          * The last four bytes of the key material are used as the salt value
 931          * in the nonce. Update the AES key length.
 932          */
 933         ctx->cdata.keylen = keylen - 4;
 934         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 935                                    ctx->dir);
 936
 937         return rfc4543_set_sh_desc(aead);
 938 }
 939
 940 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 941                            unsigned int keylen, const u32 ctx1_iv_off)
 942 {
 943         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 944         struct caam_skcipher_alg *alg =
 945                 container_of(crypto_skcipher_alg(skcipher),
 946                              struct caam_skcipher_alg, skcipher);
 947         struct device *dev = ctx->dev;
 948         struct caam_flc *flc;
 949         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 950         u32 *desc;
 951         const bool is_rfc3686 = alg->caam.rfc3686;
 952
 953         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 954                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 955
 956         ctx->cdata.keylen = keylen;
 957         ctx->cdata.key_virt = key;
 958         ctx->cdata.key_inline = true;
 959
 960         /* skcipher_encrypt shared descriptor */
 961         flc = &ctx->flc[ENCRYPT];
 962         desc = flc->sh_desc;
 963         cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 964                                    ctx1_iv_off);
 965         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 966         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 967                                    sizeof(flc->flc) + desc_bytes(desc),
 968                                    ctx->dir);
 969
 970         /* skcipher_decrypt shared descriptor */
 971         flc = &ctx->flc[DECRYPT];
 972         desc = flc->sh_desc;
 973         cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 974                                    ctx1_iv_off);
 975         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 976         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 977                                    sizeof(flc->flc) + desc_bytes(desc),
 978                                    ctx->dir);
 979
 980         return 0;
 981 }
 982
 983 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 984                                const u8 *key, unsigned int keylen)
 985 {
 986         int err;
 987
 988         err = aes_check_keylen(keylen);
 989         if (err)
 990                 return err;
 991
 992         return skcipher_setkey(skcipher, key, keylen, 0);
 993 }
 994
 995 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
 996                                    const u8 *key, unsigned int keylen)
 997 {
 998         u32 ctx1_iv_off;
 999         int err;
1000
1001         /*
1002          * RFC3686 specific:
1003          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1004          *      | *key = {KEY, NONCE}
1005          */
1006         ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1007         keylen -= CTR_RFC3686_NONCE_SIZE;
1008
1009         err = aes_check_keylen(keylen);
1010         if (err)
1011                 return err;
1012
1013         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1014 }
1015
1016 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1017                                const u8 *key, unsigned int keylen)
1018 {
1019         u32 ctx1_iv_off;
1020         int err;
1021
1022         /*
1023          * AES-CTR needs to load IV in CONTEXT1 reg
1024          * at an offset of 128bits (16bytes)
1025          * CONTEXT1[255:128] = IV
1026          */
1027         ctx1_iv_off = 16;
1028
1029         err = aes_check_keylen(keylen);
1030         if (err)
1031                 return err;
1032
1033         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1034 }
1035
1036 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1037                                     const u8 *key, unsigned int keylen)
1038 {
1039         if (keylen != CHACHA_KEY_SIZE)
1040                 return -EINVAL;
1041
1042         return skcipher_setkey(skcipher, key, keylen, 0);
1043 }
1044
1045 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1046                                const u8 *key, unsigned int keylen)
1047 {
1048         return verify_skcipher_des_key(skcipher, key) ?:
1049                skcipher_setkey(skcipher, key, keylen, 0);
1050 }
1051
1052 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1053                                 const u8 *key, unsigned int keylen)
1054 {
1055         return verify_skcipher_des3_key(skcipher, key) ?:
1056                skcipher_setkey(skcipher, key, keylen, 0);
1057 }
1058
1059 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1060                                unsigned int keylen)
1061 {
1062         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1063         struct device *dev = ctx->dev;
1064         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
1065         struct caam_flc *flc;
1066         u32 *desc;
1067         int err;
1068
1069         err = xts_verify_key(skcipher, key, keylen);
1070         if (err) {
1071                 dev_dbg(dev, "key size mismatch\n");
1072                 return err;
1073         }
1074
1075         if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
1076                 ctx->xts_key_fallback = true;
1077
1078         if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) {
1079                 err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
1080                 if (err)
1081                         return err;
1082         }
1083
1084         ctx->cdata.keylen = keylen;
1085         ctx->cdata.key_virt = key;
1086         ctx->cdata.key_inline = true;
1087
1088         /* xts_skcipher_encrypt shared descriptor */
1089         flc = &ctx->flc[ENCRYPT];
1090         desc = flc->sh_desc;
1091         cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1092         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1093         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1094                                    sizeof(flc->flc) + desc_bytes(desc),
1095                                    ctx->dir);
1096
1097         /* xts_skcipher_decrypt shared descriptor */
1098         flc = &ctx->flc[DECRYPT];
1099         desc = flc->sh_desc;
1100         cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1101         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1102         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1103                                    sizeof(flc->flc) + desc_bytes(desc),
1104                                    ctx->dir);
1105
1106         return 0;
1107 }
1108
1109 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1110 {
1111         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1112         struct caam_request *req_ctx = skcipher_request_ctx(req);
1113         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1114         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1115         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1116         struct device *dev = ctx->dev;
1117         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1118                        GFP_KERNEL : GFP_ATOMIC;
1119         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1120         struct skcipher_edesc *edesc;
1121         dma_addr_t iv_dma;
1122         u8 *iv;
1123         int ivsize = crypto_skcipher_ivsize(skcipher);
1124         int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1125         struct dpaa2_sg_entry *sg_table;
1126
1127         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1128         if (unlikely(src_nents < 0)) {
1129                 dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1130                         req->cryptlen);
1131                 return ERR_PTR(src_nents);
1132         }
1133
1134         if (unlikely(req->dst != req->src)) {
1135                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1136                 if (unlikely(dst_nents < 0)) {
1137                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1138                                 req->cryptlen);
1139                         return ERR_PTR(dst_nents);
1140                 }
1141
1142                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1143                                               DMA_TO_DEVICE);
1144                 if (unlikely(!mapped_src_nents)) {
1145                         dev_err(dev, "unable to map source\n");
1146                         return ERR_PTR(-ENOMEM);
1147                 }
1148
1149                 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1150                                               DMA_FROM_DEVICE);
1151                 if (unlikely(!mapped_dst_nents)) {
1152                         dev_err(dev, "unable to map destination\n");
1153                         dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1154                         return ERR_PTR(-ENOMEM);
1155                 }
1156         } else {
1157                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1158                                               DMA_BIDIRECTIONAL);
1159                 if (unlikely(!mapped_src_nents)) {
1160                         dev_err(dev, "unable to map source\n");
1161                         return ERR_PTR(-ENOMEM);
1162                 }
1163         }
1164
1165         qm_sg_ents = 1 + mapped_src_nents;
1166         dst_sg_idx = qm_sg_ents;
1167
1168         /*
1169          * Input, output HW S/G tables: [IV, src][dst, IV]
1170          * IV entries point to the same buffer
1171          * If src == dst, S/G entries are reused (S/G tables overlap)
1172          *
1173          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1174          * the end of the table by allocating more S/G entries.
1175          */
1176         if (req->src != req->dst)
1177                 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1178         else
1179                 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1180
1181         qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1182         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1183                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1184                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1185                         qm_sg_ents, ivsize);
1186                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1187                            0, DMA_NONE, 0, 0);
1188                 return ERR_PTR(-ENOMEM);
1189         }
1190
1191         /* allocate space for base edesc, link tables and IV */
1192         edesc = qi_cache_zalloc(GFP_DMA | flags);
1193         if (unlikely(!edesc)) {
1194                 dev_err(dev, "could not allocate extended descriptor\n");
1195                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1196                            0, DMA_NONE, 0, 0);
1197                 return ERR_PTR(-ENOMEM);
1198         }
1199
1200         /* Make sure IV is located in a DMAable area */
1201         sg_table = &edesc->sgt[0];
1202         iv = (u8 *)(sg_table + qm_sg_ents);
1203         memcpy(iv, req->iv, ivsize);
1204
1205         iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1206         if (dma_mapping_error(dev, iv_dma)) {
1207                 dev_err(dev, "unable to map IV\n");
1208                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1209                            0, DMA_NONE, 0, 0);
1210                 qi_cache_free(edesc);
1211                 return ERR_PTR(-ENOMEM);
1212         }
1213
1214         edesc->src_nents = src_nents;
1215         edesc->dst_nents = dst_nents;
1216         edesc->iv_dma = iv_dma;
1217         edesc->qm_sg_bytes = qm_sg_bytes;
1218
1219         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1220         sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1221
1222         if (req->src != req->dst)
1223                 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1224
1225         dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1226                          ivsize, 0);
1227
1228         edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1229                                           DMA_TO_DEVICE);
1230         if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1231                 dev_err(dev, "unable to map S/G table\n");
1232                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1233                            iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1234                 qi_cache_free(edesc);
1235                 return ERR_PTR(-ENOMEM);
1236         }
1237
1238         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1239         dpaa2_fl_set_final(in_fle, true);
1240         dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1241         dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1242
1243         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1244         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1245
1246         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1247
1248         if (req->src == req->dst)
1249                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1250                                   sizeof(*sg_table));
1251         else
1252                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1253                                   sizeof(*sg_table));
1254
1255         return edesc;
1256 }
1257
1258 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1259                        struct aead_request *req)
1260 {
1261         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1262         int ivsize = crypto_aead_ivsize(aead);
1263
1264         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1265                    edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1266                    edesc->qm_sg_bytes);
1267         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1268 }
1269
1270 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1271                            struct skcipher_request *req)
1272 {
1273         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1274         int ivsize = crypto_skcipher_ivsize(skcipher);
1275
1276         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1277                    edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1278                    edesc->qm_sg_bytes);
1279 }
1280
1281 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1282 {
1283         struct crypto_async_request *areq = cbk_ctx;
1284         struct aead_request *req = container_of(areq, struct aead_request,
1285                                                 base);
1286         struct caam_request *req_ctx = to_caam_req(areq);
1287         struct aead_edesc *edesc = req_ctx->edesc;
1288         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1289         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1290         int ecode = 0;
1291
1292         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1293
1294         if (unlikely(status))
1295                 ecode = caam_qi2_strstatus(ctx->dev, status);
1296
1297         aead_unmap(ctx->dev, edesc, req);
1298         qi_cache_free(edesc);
1299         aead_request_complete(req, ecode);
1300 }
1301
1302 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1303 {
1304         struct crypto_async_request *areq = cbk_ctx;
1305         struct aead_request *req = container_of(areq, struct aead_request,
1306                                                 base);
1307         struct caam_request *req_ctx = to_caam_req(areq);
1308         struct aead_edesc *edesc = req_ctx->edesc;
1309         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1310         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1311         int ecode = 0;
1312
1313         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1314
1315         if (unlikely(status))
1316                 ecode = caam_qi2_strstatus(ctx->dev, status);
1317
1318         aead_unmap(ctx->dev, edesc, req);
1319         qi_cache_free(edesc);
1320         aead_request_complete(req, ecode);
1321 }
1322
1323 static int aead_encrypt(struct aead_request *req)
1324 {
1325         struct aead_edesc *edesc;
1326         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1327         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1328         struct caam_request *caam_req = aead_request_ctx(req);
1329         int ret;
1330
1331         /* allocate extended descriptor */
1332         edesc = aead_edesc_alloc(req, true);
1333         if (IS_ERR(edesc))
1334                 return PTR_ERR(edesc);
1335
1336         caam_req->flc = &ctx->flc[ENCRYPT];
1337         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1338         caam_req->cbk = aead_encrypt_done;
1339         caam_req->ctx = &req->base;
1340         caam_req->edesc = edesc;
1341         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1342         if (ret != -EINPROGRESS &&
1343             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1344                 aead_unmap(ctx->dev, edesc, req);
1345                 qi_cache_free(edesc);
1346         }
1347
1348         return ret;
1349 }
1350
1351 static int aead_decrypt(struct aead_request *req)
1352 {
1353         struct aead_edesc *edesc;
1354         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1355         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1356         struct caam_request *caam_req = aead_request_ctx(req);
1357         int ret;
1358
1359         /* allocate extended descriptor */
1360         edesc = aead_edesc_alloc(req, false);
1361         if (IS_ERR(edesc))
1362                 return PTR_ERR(edesc);
1363
1364         caam_req->flc = &ctx->flc[DECRYPT];
1365         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1366         caam_req->cbk = aead_decrypt_done;
1367         caam_req->ctx = &req->base;
1368         caam_req->edesc = edesc;
1369         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1370         if (ret != -EINPROGRESS &&
1371             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1372                 aead_unmap(ctx->dev, edesc, req);
1373                 qi_cache_free(edesc);
1374         }
1375
1376         return ret;
1377 }
1378
1379 static int ipsec_gcm_encrypt(struct aead_request *req)
1380 {
1381         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1382 }
1383
1384 static int ipsec_gcm_decrypt(struct aead_request *req)
1385 {
1386         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1387 }
1388
1389 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1390 {
1391         struct crypto_async_request *areq = cbk_ctx;
1392         struct skcipher_request *req = skcipher_request_cast(areq);
1393         struct caam_request *req_ctx = to_caam_req(areq);
1394         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1395         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1396         struct skcipher_edesc *edesc = req_ctx->edesc;
1397         int ecode = 0;
1398         int ivsize = crypto_skcipher_ivsize(skcipher);
1399
1400         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1401
1402         if (unlikely(status))
1403                 ecode = caam_qi2_strstatus(ctx->dev, status);
1404
1405         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1406                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1407                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1408         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1409                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1410                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1411
1412         skcipher_unmap(ctx->dev, edesc, req);
1413
1414         /*
1415          * The crypto API expects us to set the IV (req->iv) to the last
1416          * ciphertext block (CBC mode) or last counter (CTR mode).
1417          * This is used e.g. by the CTS mode.
1418          */
1419         if (!ecode)
1420                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1421                        ivsize);
1422
1423         qi_cache_free(edesc);
1424         skcipher_request_complete(req, ecode);
1425 }
1426
1427 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1428 {
1429         struct crypto_async_request *areq = cbk_ctx;
1430         struct skcipher_request *req = skcipher_request_cast(areq);
1431         struct caam_request *req_ctx = to_caam_req(areq);
1432         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1433         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1434         struct skcipher_edesc *edesc = req_ctx->edesc;
1435         int ecode = 0;
1436         int ivsize = crypto_skcipher_ivsize(skcipher);
1437
1438         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1439
1440         if (unlikely(status))
1441                 ecode = caam_qi2_strstatus(ctx->dev, status);
1442
1443         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1444                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1445                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1446         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1447                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1448                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1449
1450         skcipher_unmap(ctx->dev, edesc, req);
1451
1452         /*
1453          * The crypto API expects us to set the IV (req->iv) to the last
1454          * ciphertext block (CBC mode) or last counter (CTR mode).
1455          * This is used e.g. by the CTS mode.
1456          */
1457         if (!ecode)
1458                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1459                        ivsize);
1460
1461         qi_cache_free(edesc);
1462         skcipher_request_complete(req, ecode);
1463 }
1464
1465 static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1466 {
1467         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1468         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1469
1470         return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1471 }
1472
1473 static int skcipher_encrypt(struct skcipher_request *req)
1474 {
1475         struct skcipher_edesc *edesc;
1476         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1477         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1478         struct caam_request *caam_req = skcipher_request_ctx(req);
1479         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1480         int ret;
1481
1482         /*
1483          * XTS is expected to return an error even for input length = 0
1484          * Note that the case input length < block size will be caught during
1485          * HW offloading and return an error.
1486          */
1487         if (!req->cryptlen && !ctx->fallback)
1488                 return 0;
1489
1490         if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1491                               ctx->xts_key_fallback)) {
1492                 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1493                 skcipher_request_set_callback(&caam_req->fallback_req,
1494                                               req->base.flags,
1495                                               req->base.complete,
1496                                               req->base.data);
1497                 skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1498                                            req->dst, req->cryptlen, req->iv);
1499
1500                 return crypto_skcipher_encrypt(&caam_req->fallback_req);
1501         }
1502
1503         /* allocate extended descriptor */
1504         edesc = skcipher_edesc_alloc(req);
1505         if (IS_ERR(edesc))
1506                 return PTR_ERR(edesc);
1507
1508         caam_req->flc = &ctx->flc[ENCRYPT];
1509         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1510         caam_req->cbk = skcipher_encrypt_done;
1511         caam_req->ctx = &req->base;
1512         caam_req->edesc = edesc;
1513         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1514         if (ret != -EINPROGRESS &&
1515             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1516                 skcipher_unmap(ctx->dev, edesc, req);
1517                 qi_cache_free(edesc);
1518         }
1519
1520         return ret;
1521 }
1522
1523 static int skcipher_decrypt(struct skcipher_request *req)
1524 {
1525         struct skcipher_edesc *edesc;
1526         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1527         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1528         struct caam_request *caam_req = skcipher_request_ctx(req);
1529         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1530         int ret;
1531
1532         /*
1533          * XTS is expected to return an error even for input length = 0
1534          * Note that the case input length < block size will be caught during
1535          * HW offloading and return an error.
1536          */
1537         if (!req->cryptlen && !ctx->fallback)
1538                 return 0;
1539
1540         if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1541                               ctx->xts_key_fallback)) {
1542                 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1543                 skcipher_request_set_callback(&caam_req->fallback_req,
1544                                               req->base.flags,
1545                                               req->base.complete,
1546                                               req->base.data);
1547                 skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1548                                            req->dst, req->cryptlen, req->iv);
1549
1550                 return crypto_skcipher_decrypt(&caam_req->fallback_req);
1551         }
1552
1553         /* allocate extended descriptor */
1554         edesc = skcipher_edesc_alloc(req);
1555         if (IS_ERR(edesc))
1556                 return PTR_ERR(edesc);
1557
1558         caam_req->flc = &ctx->flc[DECRYPT];
1559         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1560         caam_req->cbk = skcipher_decrypt_done;
1561         caam_req->ctx = &req->base;
1562         caam_req->edesc = edesc;
1563         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1564         if (ret != -EINPROGRESS &&
1565             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1566                 skcipher_unmap(ctx->dev, edesc, req);
1567                 qi_cache_free(edesc);
1568         }
1569
1570         return ret;
1571 }
1572
1573 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1574                          bool uses_dkp)
1575 {
1576         dma_addr_t dma_addr;
1577         int i;
1578
1579         /* copy descriptor header template value */
1580         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1581         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1582
1583         ctx->dev = caam->dev;
1584         ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1585
1586         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1587                                         offsetof(struct caam_ctx, flc_dma),
1588                                         ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1589         if (dma_mapping_error(ctx->dev, dma_addr)) {
1590                 dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1591                 return -ENOMEM;
1592         }
1593
1594         for (i = 0; i < NUM_OP; i++)
1595                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1596         ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1597
1598         return 0;
1599 }
1600
1601 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1602 {
1603         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1604         struct caam_skcipher_alg *caam_alg =
1605                 container_of(alg, typeof(*caam_alg), skcipher);
1606         struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
1607         u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
1608         int ret = 0;
1609
1610         if (alg_aai == OP_ALG_AAI_XTS) {
1611                 const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
1612                 struct crypto_skcipher *fallback;
1613
1614                 fallback = crypto_alloc_skcipher(tfm_name, 0,
1615                                                  CRYPTO_ALG_NEED_FALLBACK);
1616                 if (IS_ERR(fallback)) {
1617                         dev_err(caam_alg->caam.dev,
1618                                 "Failed to allocate %s fallback: %ld\n",
1619                                 tfm_name, PTR_ERR(fallback));
1620                         return PTR_ERR(fallback);
1621                 }
1622
1623                 ctx->fallback = fallback;
1624                 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request) +
1625                                             crypto_skcipher_reqsize(fallback));
1626         } else {
1627                 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
1628         }
1629
1630         ret = caam_cra_init(ctx, &caam_alg->caam, false);
1631         if (ret && ctx->fallback)
1632                 crypto_free_skcipher(ctx->fallback);
1633
1634         return ret;
1635 }
1636
1637 static int caam_cra_init_aead(struct crypto_aead *tfm)
1638 {
1639         struct aead_alg *alg = crypto_aead_alg(tfm);
1640         struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1641                                                       aead);
1642
1643         crypto_aead_set_reqsize(tfm, sizeof(struct caam_request));
1644         return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam,
1645                              !caam_alg->caam.nodkp);
1646 }
1647
1648 static void caam_exit_common(struct caam_ctx *ctx)
1649 {
1650         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1651                                offsetof(struct caam_ctx, flc_dma), ctx->dir,
1652                                DMA_ATTR_SKIP_CPU_SYNC);
1653 }
1654
1655 static void caam_cra_exit(struct crypto_skcipher *tfm)
1656 {
1657         struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
1658
1659         if (ctx->fallback)
1660                 crypto_free_skcipher(ctx->fallback);
1661         caam_exit_common(ctx);
1662 }
1663
1664 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1665 {
1666         caam_exit_common(crypto_aead_ctx(tfm));
1667 }
1668
1669 static struct caam_skcipher_alg driver_algs[] = {
1670         {
1671                 .skcipher = {
1672                         .base = {
1673                                 .cra_name = "cbc(aes)",
1674                                 .cra_driver_name = "cbc-aes-caam-qi2",
1675                                 .cra_blocksize = AES_BLOCK_SIZE,
1676                         },
1677                         .setkey = aes_skcipher_setkey,
1678                         .encrypt = skcipher_encrypt,
1679                         .decrypt = skcipher_decrypt,
1680                         .min_keysize = AES_MIN_KEY_SIZE,
1681                         .max_keysize = AES_MAX_KEY_SIZE,
1682                         .ivsize = AES_BLOCK_SIZE,
1683                 },
1684                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1685         },
1686         {
1687                 .skcipher = {
1688                         .base = {
1689                                 .cra_name = "cbc(des3_ede)",
1690                                 .cra_driver_name = "cbc-3des-caam-qi2",
1691                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1692                         },
1693                         .setkey = des3_skcipher_setkey,
1694                         .encrypt = skcipher_encrypt,
1695                         .decrypt = skcipher_decrypt,
1696                         .min_keysize = DES3_EDE_KEY_SIZE,
1697                         .max_keysize = DES3_EDE_KEY_SIZE,
1698                         .ivsize = DES3_EDE_BLOCK_SIZE,
1699                 },
1700                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1701         },
1702         {
1703                 .skcipher = {
1704                         .base = {
1705                                 .cra_name = "cbc(des)",
1706                                 .cra_driver_name = "cbc-des-caam-qi2",
1707                                 .cra_blocksize = DES_BLOCK_SIZE,
1708                         },
1709                         .setkey = des_skcipher_setkey,
1710                         .encrypt = skcipher_encrypt,
1711                         .decrypt = skcipher_decrypt,
1712                         .min_keysize = DES_KEY_SIZE,
1713                         .max_keysize = DES_KEY_SIZE,
1714                         .ivsize = DES_BLOCK_SIZE,
1715                 },
1716                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1717         },
1718         {
1719                 .skcipher = {
1720                         .base = {
1721                                 .cra_name = "ctr(aes)",
1722                                 .cra_driver_name = "ctr-aes-caam-qi2",
1723                                 .cra_blocksize = 1,
1724                         },
1725                         .setkey = ctr_skcipher_setkey,
1726                         .encrypt = skcipher_encrypt,
1727                         .decrypt = skcipher_decrypt,
1728                         .min_keysize = AES_MIN_KEY_SIZE,
1729                         .max_keysize = AES_MAX_KEY_SIZE,
1730                         .ivsize = AES_BLOCK_SIZE,
1731                         .chunksize = AES_BLOCK_SIZE,
1732                 },
1733                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1734                                         OP_ALG_AAI_CTR_MOD128,
1735         },
1736         {
1737                 .skcipher = {
1738                         .base = {
1739                                 .cra_name = "rfc3686(ctr(aes))",
1740                                 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1741                                 .cra_blocksize = 1,
1742                         },
1743                         .setkey = rfc3686_skcipher_setkey,
1744                         .encrypt = skcipher_encrypt,
1745                         .decrypt = skcipher_decrypt,
1746                         .min_keysize = AES_MIN_KEY_SIZE +
1747                                        CTR_RFC3686_NONCE_SIZE,
1748                         .max_keysize = AES_MAX_KEY_SIZE +
1749                                        CTR_RFC3686_NONCE_SIZE,
1750                         .ivsize = CTR_RFC3686_IV_SIZE,
1751                         .chunksize = AES_BLOCK_SIZE,
1752                 },
1753                 .caam = {
1754                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1755                                            OP_ALG_AAI_CTR_MOD128,
1756                         .rfc3686 = true,
1757                 },
1758         },
1759         {
1760                 .skcipher = {
1761                         .base = {
1762                                 .cra_name = "xts(aes)",
1763                                 .cra_driver_name = "xts-aes-caam-qi2",
1764                                 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1765                                 .cra_blocksize = AES_BLOCK_SIZE,
1766                         },
1767                         .setkey = xts_skcipher_setkey,
1768                         .encrypt = skcipher_encrypt,
1769                         .decrypt = skcipher_decrypt,
1770                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1771                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1772                         .ivsize = AES_BLOCK_SIZE,
1773                 },
1774                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1775         },
1776         {
1777                 .skcipher = {
1778                         .base = {
1779                                 .cra_name = "chacha20",
1780                                 .cra_driver_name = "chacha20-caam-qi2",
1781                                 .cra_blocksize = 1,
1782                         },
1783                         .setkey = chacha20_skcipher_setkey,
1784                         .encrypt = skcipher_encrypt,
1785                         .decrypt = skcipher_decrypt,
1786                         .min_keysize = CHACHA_KEY_SIZE,
1787                         .max_keysize = CHACHA_KEY_SIZE,
1788                         .ivsize = CHACHA_IV_SIZE,
1789                 },
1790                 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1791         },
1792 };
1793
1794 static struct caam_aead_alg driver_aeads[] = {
1795         {
1796                 .aead = {
1797                         .base = {
1798                                 .cra_name = "rfc4106(gcm(aes))",
1799                                 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1800                                 .cra_blocksize = 1,
1801                         },
1802                         .setkey = rfc4106_setkey,
1803                         .setauthsize = rfc4106_setauthsize,
1804                         .encrypt = ipsec_gcm_encrypt,
1805                         .decrypt = ipsec_gcm_decrypt,
1806                         .ivsize = 8,
1807                         .maxauthsize = AES_BLOCK_SIZE,
1808                 },
1809                 .caam = {
1810                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1811                         .nodkp = true,
1812                 },
1813         },
1814         {
1815                 .aead = {
1816                         .base = {
1817                                 .cra_name = "rfc4543(gcm(aes))",
1818                                 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1819                                 .cra_blocksize = 1,
1820                         },
1821                         .setkey = rfc4543_setkey,
1822                         .setauthsize = rfc4543_setauthsize,
1823                         .encrypt = ipsec_gcm_encrypt,
1824                         .decrypt = ipsec_gcm_decrypt,
1825                         .ivsize = 8,
1826                         .maxauthsize = AES_BLOCK_SIZE,
1827                 },
1828                 .caam = {
1829                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1830                         .nodkp = true,
1831                 },
1832         },
1833         /* Galois Counter Mode */
1834         {
1835                 .aead = {
1836                         .base = {
1837                                 .cra_name = "gcm(aes)",
1838                                 .cra_driver_name = "gcm-aes-caam-qi2",
1839                                 .cra_blocksize = 1,
1840                         },
1841                         .setkey = gcm_setkey,
1842                         .setauthsize = gcm_setauthsize,
1843                         .encrypt = aead_encrypt,
1844                         .decrypt = aead_decrypt,
1845                         .ivsize = 12,
1846                         .maxauthsize = AES_BLOCK_SIZE,
1847                 },
1848                 .caam = {
1849                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1850                         .nodkp = true,
1851                 }
1852         },
1853         /* single-pass ipsec_esp descriptor */
1854         {
1855                 .aead = {
1856                         .base = {
1857                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
1858                                 .cra_driver_name = "authenc-hmac-md5-"
1859                                                    "cbc-aes-caam-qi2",
1860                                 .cra_blocksize = AES_BLOCK_SIZE,
1861                         },
1862                         .setkey = aead_setkey,
1863                         .setauthsize = aead_setauthsize,
1864                         .encrypt = aead_encrypt,
1865                         .decrypt = aead_decrypt,
1866                         .ivsize = AES_BLOCK_SIZE,
1867                         .maxauthsize = MD5_DIGEST_SIZE,
1868                 },
1869                 .caam = {
1870                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1871                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1872                                            OP_ALG_AAI_HMAC_PRECOMP,
1873                 }
1874         },
1875         {
1876                 .aead = {
1877                         .base = {
1878                                 .cra_name = "echainiv(authenc(hmac(md5),"
1879                                             "cbc(aes)))",
1880                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1881                                                    "cbc-aes-caam-qi2",
1882                                 .cra_blocksize = AES_BLOCK_SIZE,
1883                         },
1884                         .setkey = aead_setkey,
1885                         .setauthsize = aead_setauthsize,
1886                         .encrypt = aead_encrypt,
1887                         .decrypt = aead_decrypt,
1888                         .ivsize = AES_BLOCK_SIZE,
1889                         .maxauthsize = MD5_DIGEST_SIZE,
1890                 },
1891                 .caam = {
1892                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1893                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1894                                            OP_ALG_AAI_HMAC_PRECOMP,
1895                         .geniv = true,
1896                 }
1897         },
1898         {
1899                 .aead = {
1900                         .base = {
1901                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1902                                 .cra_driver_name = "authenc-hmac-sha1-"
1903                                                    "cbc-aes-caam-qi2",
1904                                 .cra_blocksize = AES_BLOCK_SIZE,
1905                         },
1906                         .setkey = aead_setkey,
1907                         .setauthsize = aead_setauthsize,
1908                         .encrypt = aead_encrypt,
1909                         .decrypt = aead_decrypt,
1910                         .ivsize = AES_BLOCK_SIZE,
1911                         .maxauthsize = SHA1_DIGEST_SIZE,
1912                 },
1913                 .caam = {
1914                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1915                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1916                                            OP_ALG_AAI_HMAC_PRECOMP,
1917                 }
1918         },
1919         {
1920                 .aead = {
1921                         .base = {
1922                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1923                                             "cbc(aes)))",
1924                                 .cra_driver_name = "echainiv-authenc-"
1925                                                    "hmac-sha1-cbc-aes-caam-qi2",
1926                                 .cra_blocksize = AES_BLOCK_SIZE,
1927                         },
1928                         .setkey = aead_setkey,
1929                         .setauthsize = aead_setauthsize,
1930                         .encrypt = aead_encrypt,
1931                         .decrypt = aead_decrypt,
1932                         .ivsize = AES_BLOCK_SIZE,
1933                         .maxauthsize = SHA1_DIGEST_SIZE,
1934                 },
1935                 .caam = {
1936                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1937                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1938                                            OP_ALG_AAI_HMAC_PRECOMP,
1939                         .geniv = true,
1940                 },
1941         },
1942         {
1943                 .aead = {
1944                         .base = {
1945                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1946                                 .cra_driver_name = "authenc-hmac-sha224-"
1947                                                    "cbc-aes-caam-qi2",
1948                                 .cra_blocksize = AES_BLOCK_SIZE,
1949                         },
1950                         .setkey = aead_setkey,
1951                         .setauthsize = aead_setauthsize,
1952                         .encrypt = aead_encrypt,
1953                         .decrypt = aead_decrypt,
1954                         .ivsize = AES_BLOCK_SIZE,
1955                         .maxauthsize = SHA224_DIGEST_SIZE,
1956                 },
1957                 .caam = {
1958                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1959                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1960                                            OP_ALG_AAI_HMAC_PRECOMP,
1961                 }
1962         },
1963         {
1964                 .aead = {
1965                         .base = {
1966                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1967                                             "cbc(aes)))",
1968                                 .cra_driver_name = "echainiv-authenc-"
1969                                                    "hmac-sha224-cbc-aes-caam-qi2",
1970                                 .cra_blocksize = AES_BLOCK_SIZE,
1971                         },
1972                         .setkey = aead_setkey,
1973                         .setauthsize = aead_setauthsize,
1974                         .encrypt = aead_encrypt,
1975                         .decrypt = aead_decrypt,
1976                         .ivsize = AES_BLOCK_SIZE,
1977                         .maxauthsize = SHA224_DIGEST_SIZE,
1978                 },
1979                 .caam = {
1980                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1981                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1982                                            OP_ALG_AAI_HMAC_PRECOMP,
1983                         .geniv = true,
1984                 }
1985         },
1986         {
1987                 .aead = {
1988                         .base = {
1989                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1990                                 .cra_driver_name = "authenc-hmac-sha256-"
1991                                                    "cbc-aes-caam-qi2",
1992                                 .cra_blocksize = AES_BLOCK_SIZE,
1993                         },
1994                         .setkey = aead_setkey,
1995                         .setauthsize = aead_setauthsize,
1996                         .encrypt = aead_encrypt,
1997                         .decrypt = aead_decrypt,
1998                         .ivsize = AES_BLOCK_SIZE,
1999                         .maxauthsize = SHA256_DIGEST_SIZE,
2000                 },
2001                 .caam = {
2002                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2003                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2004                                            OP_ALG_AAI_HMAC_PRECOMP,
2005                 }
2006         },
2007         {
2008                 .aead = {
2009                         .base = {
2010                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2011                                             "cbc(aes)))",
2012                                 .cra_driver_name = "echainiv-authenc-"
2013                                                    "hmac-sha256-cbc-aes-"
2014                                                    "caam-qi2",
2015                                 .cra_blocksize = AES_BLOCK_SIZE,
2016                         },
2017                         .setkey = aead_setkey,
2018                         .setauthsize = aead_setauthsize,
2019                         .encrypt = aead_encrypt,
2020                         .decrypt = aead_decrypt,
2021                         .ivsize = AES_BLOCK_SIZE,
2022                         .maxauthsize = SHA256_DIGEST_SIZE,
2023                 },
2024                 .caam = {
2025                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2026                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2027                                            OP_ALG_AAI_HMAC_PRECOMP,
2028                         .geniv = true,
2029                 }
2030         },
2031         {
2032                 .aead = {
2033                         .base = {
2034                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2035                                 .cra_driver_name = "authenc-hmac-sha384-"
2036                                                    "cbc-aes-caam-qi2",
2037                                 .cra_blocksize = AES_BLOCK_SIZE,
2038                         },
2039                         .setkey = aead_setkey,
2040                         .setauthsize = aead_setauthsize,
2041                         .encrypt = aead_encrypt,
2042                         .decrypt = aead_decrypt,
2043                         .ivsize = AES_BLOCK_SIZE,
2044                         .maxauthsize = SHA384_DIGEST_SIZE,
2045                 },
2046                 .caam = {
2047                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2048                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2049                                            OP_ALG_AAI_HMAC_PRECOMP,
2050                 }
2051         },
2052         {
2053                 .aead = {
2054                         .base = {
2055                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2056                                             "cbc(aes)))",
2057                                 .cra_driver_name = "echainiv-authenc-"
2058                                                    "hmac-sha384-cbc-aes-"
2059                                                    "caam-qi2",
2060                                 .cra_blocksize = AES_BLOCK_SIZE,
2061                         },
2062                         .setkey = aead_setkey,
2063                         .setauthsize = aead_setauthsize,
2064                         .encrypt = aead_encrypt,
2065                         .decrypt = aead_decrypt,
2066                         .ivsize = AES_BLOCK_SIZE,
2067                         .maxauthsize = SHA384_DIGEST_SIZE,
2068                 },
2069                 .caam = {
2070                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2071                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2072                                            OP_ALG_AAI_HMAC_PRECOMP,
2073                         .geniv = true,
2074                 }
2075         },
2076         {
2077                 .aead = {
2078                         .base = {
2079                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2080                                 .cra_driver_name = "authenc-hmac-sha512-"
2081                                                    "cbc-aes-caam-qi2",
2082                                 .cra_blocksize = AES_BLOCK_SIZE,
2083                         },
2084                         .setkey = aead_setkey,
2085                         .setauthsize = aead_setauthsize,
2086                         .encrypt = aead_encrypt,
2087                         .decrypt = aead_decrypt,
2088                         .ivsize = AES_BLOCK_SIZE,
2089                         .maxauthsize = SHA512_DIGEST_SIZE,
2090                 },
2091                 .caam = {
2092                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2093                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2094                                            OP_ALG_AAI_HMAC_PRECOMP,
2095                 }
2096         },
2097         {
2098                 .aead = {
2099                         .base = {
2100                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2101                                             "cbc(aes)))",
2102                                 .cra_driver_name = "echainiv-authenc-"
2103                                                    "hmac-sha512-cbc-aes-"
2104                                                    "caam-qi2",
2105                                 .cra_blocksize = AES_BLOCK_SIZE,
2106                         },
2107                         .setkey = aead_setkey,
2108                         .setauthsize = aead_setauthsize,
2109                         .encrypt = aead_encrypt,
2110                         .decrypt = aead_decrypt,
2111                         .ivsize = AES_BLOCK_SIZE,
2112                         .maxauthsize = SHA512_DIGEST_SIZE,
2113                 },
2114                 .caam = {
2115                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2116                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2117                                            OP_ALG_AAI_HMAC_PRECOMP,
2118                         .geniv = true,
2119                 }
2120         },
2121         {
2122                 .aead = {
2123                         .base = {
2124                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2125                                 .cra_driver_name = "authenc-hmac-md5-"
2126                                                    "cbc-des3_ede-caam-qi2",
2127                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2128                         },
2129                         .setkey = des3_aead_setkey,
2130                         .setauthsize = aead_setauthsize,
2131                         .encrypt = aead_encrypt,
2132                         .decrypt = aead_decrypt,
2133                         .ivsize = DES3_EDE_BLOCK_SIZE,
2134                         .maxauthsize = MD5_DIGEST_SIZE,
2135                 },
2136                 .caam = {
2137                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2138                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2139                                            OP_ALG_AAI_HMAC_PRECOMP,
2140                 }
2141         },
2142         {
2143                 .aead = {
2144                         .base = {
2145                                 .cra_name = "echainiv(authenc(hmac(md5),"
2146                                             "cbc(des3_ede)))",
2147                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2148                                                    "cbc-des3_ede-caam-qi2",
2149                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2150                         },
2151                         .setkey = des3_aead_setkey,
2152                         .setauthsize = aead_setauthsize,
2153                         .encrypt = aead_encrypt,
2154                         .decrypt = aead_decrypt,
2155                         .ivsize = DES3_EDE_BLOCK_SIZE,
2156                         .maxauthsize = MD5_DIGEST_SIZE,
2157                 },
2158                 .caam = {
2159                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2160                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2161                                            OP_ALG_AAI_HMAC_PRECOMP,
2162                         .geniv = true,
2163                 }
2164         },
2165         {
2166                 .aead = {
2167                         .base = {
2168                                 .cra_name = "authenc(hmac(sha1),"
2169                                             "cbc(des3_ede))",
2170                                 .cra_driver_name = "authenc-hmac-sha1-"
2171                                                    "cbc-des3_ede-caam-qi2",
2172                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2173                         },
2174                         .setkey = des3_aead_setkey,
2175                         .setauthsize = aead_setauthsize,
2176                         .encrypt = aead_encrypt,
2177                         .decrypt = aead_decrypt,
2178                         .ivsize = DES3_EDE_BLOCK_SIZE,
2179                         .maxauthsize = SHA1_DIGEST_SIZE,
2180                 },
2181                 .caam = {
2182                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2183                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2184                                            OP_ALG_AAI_HMAC_PRECOMP,
2185                 },
2186         },
2187         {
2188                 .aead = {
2189                         .base = {
2190                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2191                                             "cbc(des3_ede)))",
2192                                 .cra_driver_name = "echainiv-authenc-"
2193                                                    "hmac-sha1-"
2194                                                    "cbc-des3_ede-caam-qi2",
2195                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2196                         },
2197                         .setkey = des3_aead_setkey,
2198                         .setauthsize = aead_setauthsize,
2199                         .encrypt = aead_encrypt,
2200                         .decrypt = aead_decrypt,
2201                         .ivsize = DES3_EDE_BLOCK_SIZE,
2202                         .maxauthsize = SHA1_DIGEST_SIZE,
2203                 },
2204                 .caam = {
2205                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2206                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2207                                            OP_ALG_AAI_HMAC_PRECOMP,
2208                         .geniv = true,
2209                 }
2210         },
2211         {
2212                 .aead = {
2213                         .base = {
2214                                 .cra_name = "authenc(hmac(sha224),"
2215                                             "cbc(des3_ede))",
2216                                 .cra_driver_name = "authenc-hmac-sha224-"
2217                                                    "cbc-des3_ede-caam-qi2",
2218                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2219                         },
2220                         .setkey = des3_aead_setkey,
2221                         .setauthsize = aead_setauthsize,
2222                         .encrypt = aead_encrypt,
2223                         .decrypt = aead_decrypt,
2224                         .ivsize = DES3_EDE_BLOCK_SIZE,
2225                         .maxauthsize = SHA224_DIGEST_SIZE,
2226                 },
2227                 .caam = {
2228                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2229                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2230                                            OP_ALG_AAI_HMAC_PRECOMP,
2231                 },
2232         },
2233         {
2234                 .aead = {
2235                         .base = {
2236                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2237                                             "cbc(des3_ede)))",
2238                                 .cra_driver_name = "echainiv-authenc-"
2239                                                    "hmac-sha224-"
2240                                                    "cbc-des3_ede-caam-qi2",
2241                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2242                         },
2243                         .setkey = des3_aead_setkey,
2244                         .setauthsize = aead_setauthsize,
2245                         .encrypt = aead_encrypt,
2246                         .decrypt = aead_decrypt,
2247                         .ivsize = DES3_EDE_BLOCK_SIZE,
2248                         .maxauthsize = SHA224_DIGEST_SIZE,
2249                 },
2250                 .caam = {
2251                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2252                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2253                                            OP_ALG_AAI_HMAC_PRECOMP,
2254                         .geniv = true,
2255                 }
2256         },
2257         {
2258                 .aead = {
2259                         .base = {
2260                                 .cra_name = "authenc(hmac(sha256),"
2261                                             "cbc(des3_ede))",
2262                                 .cra_driver_name = "authenc-hmac-sha256-"
2263                                                    "cbc-des3_ede-caam-qi2",
2264                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2265                         },
2266                         .setkey = des3_aead_setkey,
2267                         .setauthsize = aead_setauthsize,
2268                         .encrypt = aead_encrypt,
2269                         .decrypt = aead_decrypt,
2270                         .ivsize = DES3_EDE_BLOCK_SIZE,
2271                         .maxauthsize = SHA256_DIGEST_SIZE,
2272                 },
2273                 .caam = {
2274                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2275                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2276                                            OP_ALG_AAI_HMAC_PRECOMP,
2277                 },
2278         },
2279         {
2280                 .aead = {
2281                         .base = {
2282                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2283                                             "cbc(des3_ede)))",
2284                                 .cra_driver_name = "echainiv-authenc-"
2285                                                    "hmac-sha256-"
2286                                                    "cbc-des3_ede-caam-qi2",
2287                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2288                         },
2289                         .setkey = des3_aead_setkey,
2290                         .setauthsize = aead_setauthsize,
2291                         .encrypt = aead_encrypt,
2292                         .decrypt = aead_decrypt,
2293                         .ivsize = DES3_EDE_BLOCK_SIZE,
2294                         .maxauthsize = SHA256_DIGEST_SIZE,
2295                 },
2296                 .caam = {
2297                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2298                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2299                                            OP_ALG_AAI_HMAC_PRECOMP,
2300                         .geniv = true,
2301                 }
2302         },
2303         {
2304                 .aead = {
2305                         .base = {
2306                                 .cra_name = "authenc(hmac(sha384),"
2307                                             "cbc(des3_ede))",
2308                                 .cra_driver_name = "authenc-hmac-sha384-"
2309                                                    "cbc-des3_ede-caam-qi2",
2310                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2311                         },
2312                         .setkey = des3_aead_setkey,
2313                         .setauthsize = aead_setauthsize,
2314                         .encrypt = aead_encrypt,
2315                         .decrypt = aead_decrypt,
2316                         .ivsize = DES3_EDE_BLOCK_SIZE,
2317                         .maxauthsize = SHA384_DIGEST_SIZE,
2318                 },
2319                 .caam = {
2320                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2321                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2322                                            OP_ALG_AAI_HMAC_PRECOMP,
2323                 },
2324         },
2325         {
2326                 .aead = {
2327                         .base = {
2328                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2329                                             "cbc(des3_ede)))",
2330                                 .cra_driver_name = "echainiv-authenc-"
2331                                                    "hmac-sha384-"
2332                                                    "cbc-des3_ede-caam-qi2",
2333                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2334                         },
2335                         .setkey = des3_aead_setkey,
2336                         .setauthsize = aead_setauthsize,
2337                         .encrypt = aead_encrypt,
2338                         .decrypt = aead_decrypt,
2339                         .ivsize = DES3_EDE_BLOCK_SIZE,
2340                         .maxauthsize = SHA384_DIGEST_SIZE,
2341                 },
2342                 .caam = {
2343                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2344                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2345                                            OP_ALG_AAI_HMAC_PRECOMP,
2346                         .geniv = true,
2347                 }
2348         },
2349         {
2350                 .aead = {
2351                         .base = {
2352                                 .cra_name = "authenc(hmac(sha512),"
2353                                             "cbc(des3_ede))",
2354                                 .cra_driver_name = "authenc-hmac-sha512-"
2355                                                    "cbc-des3_ede-caam-qi2",
2356                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2357                         },
2358                         .setkey = des3_aead_setkey,
2359                         .setauthsize = aead_setauthsize,
2360                         .encrypt = aead_encrypt,
2361                         .decrypt = aead_decrypt,
2362                         .ivsize = DES3_EDE_BLOCK_SIZE,
2363                         .maxauthsize = SHA512_DIGEST_SIZE,
2364                 },
2365                 .caam = {
2366                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2367                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2368                                            OP_ALG_AAI_HMAC_PRECOMP,
2369                 },
2370         },
2371         {
2372                 .aead = {
2373                         .base = {
2374                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2375                                             "cbc(des3_ede)))",
2376                                 .cra_driver_name = "echainiv-authenc-"
2377                                                    "hmac-sha512-"
2378                                                    "cbc-des3_ede-caam-qi2",
2379                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2380                         },
2381                         .setkey = des3_aead_setkey,
2382                         .setauthsize = aead_setauthsize,
2383                         .encrypt = aead_encrypt,
2384                         .decrypt = aead_decrypt,
2385                         .ivsize = DES3_EDE_BLOCK_SIZE,
2386                         .maxauthsize = SHA512_DIGEST_SIZE,
2387                 },
2388                 .caam = {
2389                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2390                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2391                                            OP_ALG_AAI_HMAC_PRECOMP,
2392                         .geniv = true,
2393                 }
2394         },
2395         {
2396                 .aead = {
2397                         .base = {
2398                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2399                                 .cra_driver_name = "authenc-hmac-md5-"
2400                                                    "cbc-des-caam-qi2",
2401                                 .cra_blocksize = DES_BLOCK_SIZE,
2402                         },
2403                         .setkey = aead_setkey,
2404                         .setauthsize = aead_setauthsize,
2405                         .encrypt = aead_encrypt,
2406                         .decrypt = aead_decrypt,
2407                         .ivsize = DES_BLOCK_SIZE,
2408                         .maxauthsize = MD5_DIGEST_SIZE,
2409                 },
2410                 .caam = {
2411                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2412                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2413                                            OP_ALG_AAI_HMAC_PRECOMP,
2414                 },
2415         },
2416         {
2417                 .aead = {
2418                         .base = {
2419                                 .cra_name = "echainiv(authenc(hmac(md5),"
2420                                             "cbc(des)))",
2421                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2422                                                    "cbc-des-caam-qi2",
2423                                 .cra_blocksize = DES_BLOCK_SIZE,
2424                         },
2425                         .setkey = aead_setkey,
2426                         .setauthsize = aead_setauthsize,
2427                         .encrypt = aead_encrypt,
2428                         .decrypt = aead_decrypt,
2429                         .ivsize = DES_BLOCK_SIZE,
2430                         .maxauthsize = MD5_DIGEST_SIZE,
2431                 },
2432                 .caam = {
2433                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2434                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2435                                            OP_ALG_AAI_HMAC_PRECOMP,
2436                         .geniv = true,
2437                 }
2438         },
2439         {
2440                 .aead = {
2441                         .base = {
2442                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2443                                 .cra_driver_name = "authenc-hmac-sha1-"
2444                                                    "cbc-des-caam-qi2",
2445                                 .cra_blocksize = DES_BLOCK_SIZE,
2446                         },
2447                         .setkey = aead_setkey,
2448                         .setauthsize = aead_setauthsize,
2449                         .encrypt = aead_encrypt,
2450                         .decrypt = aead_decrypt,
2451                         .ivsize = DES_BLOCK_SIZE,
2452                         .maxauthsize = SHA1_DIGEST_SIZE,
2453                 },
2454                 .caam = {
2455                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2456                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2457                                            OP_ALG_AAI_HMAC_PRECOMP,
2458                 },
2459         },
2460         {
2461                 .aead = {
2462                         .base = {
2463                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2464                                             "cbc(des)))",
2465                                 .cra_driver_name = "echainiv-authenc-"
2466                                                    "hmac-sha1-cbc-des-caam-qi2",
2467                                 .cra_blocksize = DES_BLOCK_SIZE,
2468                         },
2469                         .setkey = aead_setkey,
2470                         .setauthsize = aead_setauthsize,
2471                         .encrypt = aead_encrypt,
2472                         .decrypt = aead_decrypt,
2473                         .ivsize = DES_BLOCK_SIZE,
2474                         .maxauthsize = SHA1_DIGEST_SIZE,
2475                 },
2476                 .caam = {
2477                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2478                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2479                                            OP_ALG_AAI_HMAC_PRECOMP,
2480                         .geniv = true,
2481                 }
2482         },
2483         {
2484                 .aead = {
2485                         .base = {
2486                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2487                                 .cra_driver_name = "authenc-hmac-sha224-"
2488                                                    "cbc-des-caam-qi2",
2489                                 .cra_blocksize = DES_BLOCK_SIZE,
2490                         },
2491                         .setkey = aead_setkey,
2492                         .setauthsize = aead_setauthsize,
2493                         .encrypt = aead_encrypt,
2494                         .decrypt = aead_decrypt,
2495                         .ivsize = DES_BLOCK_SIZE,
2496                         .maxauthsize = SHA224_DIGEST_SIZE,
2497                 },
2498                 .caam = {
2499                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2500                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2501                                            OP_ALG_AAI_HMAC_PRECOMP,
2502                 },
2503         },
2504         {
2505                 .aead = {
2506                         .base = {
2507                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2508                                             "cbc(des)))",
2509                                 .cra_driver_name = "echainiv-authenc-"
2510                                                    "hmac-sha224-cbc-des-"
2511                                                    "caam-qi2",
2512                                 .cra_blocksize = DES_BLOCK_SIZE,
2513                         },
2514                         .setkey = aead_setkey,
2515                         .setauthsize = aead_setauthsize,
2516                         .encrypt = aead_encrypt,
2517                         .decrypt = aead_decrypt,
2518                         .ivsize = DES_BLOCK_SIZE,
2519                         .maxauthsize = SHA224_DIGEST_SIZE,
2520                 },
2521                 .caam = {
2522                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2523                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2524                                            OP_ALG_AAI_HMAC_PRECOMP,
2525                         .geniv = true,
2526                 }
2527         },
2528         {
2529                 .aead = {
2530                         .base = {
2531                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2532                                 .cra_driver_name = "authenc-hmac-sha256-"
2533                                                    "cbc-des-caam-qi2",
2534                                 .cra_blocksize = DES_BLOCK_SIZE,
2535                         },
2536                         .setkey = aead_setkey,
2537                         .setauthsize = aead_setauthsize,
2538                         .encrypt = aead_encrypt,
2539                         .decrypt = aead_decrypt,
2540                         .ivsize = DES_BLOCK_SIZE,
2541                         .maxauthsize = SHA256_DIGEST_SIZE,
2542                 },
2543                 .caam = {
2544                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2545                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2546                                            OP_ALG_AAI_HMAC_PRECOMP,
2547                 },
2548         },
2549         {
2550                 .aead = {
2551                         .base = {
2552                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2553                                             "cbc(des)))",
2554                                 .cra_driver_name = "echainiv-authenc-"
2555                                                    "hmac-sha256-cbc-des-"
2556                                                    "caam-qi2",
2557                                 .cra_blocksize = DES_BLOCK_SIZE,
2558                         },
2559                         .setkey = aead_setkey,
2560                         .setauthsize = aead_setauthsize,
2561                         .encrypt = aead_encrypt,
2562                         .decrypt = aead_decrypt,
2563                         .ivsize = DES_BLOCK_SIZE,
2564                         .maxauthsize = SHA256_DIGEST_SIZE,
2565                 },
2566                 .caam = {
2567                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2568                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2569                                            OP_ALG_AAI_HMAC_PRECOMP,
2570                         .geniv = true,
2571                 },
2572         },
2573         {
2574                 .aead = {
2575                         .base = {
2576                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2577                                 .cra_driver_name = "authenc-hmac-sha384-"
2578                                                    "cbc-des-caam-qi2",
2579                                 .cra_blocksize = DES_BLOCK_SIZE,
2580                         },
2581                         .setkey = aead_setkey,
2582                         .setauthsize = aead_setauthsize,
2583                         .encrypt = aead_encrypt,
2584                         .decrypt = aead_decrypt,
2585                         .ivsize = DES_BLOCK_SIZE,
2586                         .maxauthsize = SHA384_DIGEST_SIZE,
2587                 },
2588                 .caam = {
2589                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2590                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2591                                            OP_ALG_AAI_HMAC_PRECOMP,
2592                 },
2593         },
2594         {
2595                 .aead = {
2596                         .base = {
2597                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2598                                             "cbc(des)))",
2599                                 .cra_driver_name = "echainiv-authenc-"
2600                                                    "hmac-sha384-cbc-des-"
2601                                                    "caam-qi2",
2602                                 .cra_blocksize = DES_BLOCK_SIZE,
2603                         },
2604                         .setkey = aead_setkey,
2605                         .setauthsize = aead_setauthsize,
2606                         .encrypt = aead_encrypt,
2607                         .decrypt = aead_decrypt,
2608                         .ivsize = DES_BLOCK_SIZE,
2609                         .maxauthsize = SHA384_DIGEST_SIZE,
2610                 },
2611                 .caam = {
2612                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2613                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2614                                            OP_ALG_AAI_HMAC_PRECOMP,
2615                         .geniv = true,
2616                 }
2617         },
2618         {
2619                 .aead = {
2620                         .base = {
2621                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2622                                 .cra_driver_name = "authenc-hmac-sha512-"
2623                                                    "cbc-des-caam-qi2",
2624                                 .cra_blocksize = DES_BLOCK_SIZE,
2625                         },
2626                         .setkey = aead_setkey,
2627                         .setauthsize = aead_setauthsize,
2628                         .encrypt = aead_encrypt,
2629                         .decrypt = aead_decrypt,
2630                         .ivsize = DES_BLOCK_SIZE,
2631                         .maxauthsize = SHA512_DIGEST_SIZE,
2632                 },
2633                 .caam = {
2634                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2635                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2636                                            OP_ALG_AAI_HMAC_PRECOMP,
2637                 }
2638         },
2639         {
2640                 .aead = {
2641                         .base = {
2642                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2643                                             "cbc(des)))",
2644                                 .cra_driver_name = "echainiv-authenc-"
2645                                                    "hmac-sha512-cbc-des-"
2646                                                    "caam-qi2",
2647                                 .cra_blocksize = DES_BLOCK_SIZE,
2648                         },
2649                         .setkey = aead_setkey,
2650                         .setauthsize = aead_setauthsize,
2651                         .encrypt = aead_encrypt,
2652                         .decrypt = aead_decrypt,
2653                         .ivsize = DES_BLOCK_SIZE,
2654                         .maxauthsize = SHA512_DIGEST_SIZE,
2655                 },
2656                 .caam = {
2657                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2658                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2659                                            OP_ALG_AAI_HMAC_PRECOMP,
2660                         .geniv = true,
2661                 }
2662         },
2663         {
2664                 .aead = {
2665                         .base = {
2666                                 .cra_name = "authenc(hmac(md5),"
2667                                             "rfc3686(ctr(aes)))",
2668                                 .cra_driver_name = "authenc-hmac-md5-"
2669                                                    "rfc3686-ctr-aes-caam-qi2",
2670                                 .cra_blocksize = 1,
2671                         },
2672                         .setkey = aead_setkey,
2673                         .setauthsize = aead_setauthsize,
2674                         .encrypt = aead_encrypt,
2675                         .decrypt = aead_decrypt,
2676                         .ivsize = CTR_RFC3686_IV_SIZE,
2677                         .maxauthsize = MD5_DIGEST_SIZE,
2678                 },
2679                 .caam = {
2680                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2681                                            OP_ALG_AAI_CTR_MOD128,
2682                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2683                                            OP_ALG_AAI_HMAC_PRECOMP,
2684                         .rfc3686 = true,
2685                 },
2686         },
2687         {
2688                 .aead = {
2689                         .base = {
2690                                 .cra_name = "seqiv(authenc("
2691                                             "hmac(md5),rfc3686(ctr(aes))))",
2692                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
2693                                                    "rfc3686-ctr-aes-caam-qi2",
2694                                 .cra_blocksize = 1,
2695                         },
2696                         .setkey = aead_setkey,
2697                         .setauthsize = aead_setauthsize,
2698                         .encrypt = aead_encrypt,
2699                         .decrypt = aead_decrypt,
2700                         .ivsize = CTR_RFC3686_IV_SIZE,
2701                         .maxauthsize = MD5_DIGEST_SIZE,
2702                 },
2703                 .caam = {
2704                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2705                                            OP_ALG_AAI_CTR_MOD128,
2706                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2707                                            OP_ALG_AAI_HMAC_PRECOMP,
2708                         .rfc3686 = true,
2709                         .geniv = true,
2710                 },
2711         },
2712         {
2713                 .aead = {
2714                         .base = {
2715                                 .cra_name = "authenc(hmac(sha1),"
2716                                             "rfc3686(ctr(aes)))",
2717                                 .cra_driver_name = "authenc-hmac-sha1-"
2718                                                    "rfc3686-ctr-aes-caam-qi2",
2719                                 .cra_blocksize = 1,
2720                         },
2721                         .setkey = aead_setkey,
2722                         .setauthsize = aead_setauthsize,
2723                         .encrypt = aead_encrypt,
2724                         .decrypt = aead_decrypt,
2725                         .ivsize = CTR_RFC3686_IV_SIZE,
2726                         .maxauthsize = SHA1_DIGEST_SIZE,
2727                 },
2728                 .caam = {
2729                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2730                                            OP_ALG_AAI_CTR_MOD128,
2731                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2732                                            OP_ALG_AAI_HMAC_PRECOMP,
2733                         .rfc3686 = true,
2734                 },
2735         },
2736         {
2737                 .aead = {
2738                         .base = {
2739                                 .cra_name = "seqiv(authenc("
2740                                             "hmac(sha1),rfc3686(ctr(aes))))",
2741                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
2742                                                    "rfc3686-ctr-aes-caam-qi2",
2743                                 .cra_blocksize = 1,
2744                         },
2745                         .setkey = aead_setkey,
2746                         .setauthsize = aead_setauthsize,
2747                         .encrypt = aead_encrypt,
2748                         .decrypt = aead_decrypt,
2749                         .ivsize = CTR_RFC3686_IV_SIZE,
2750                         .maxauthsize = SHA1_DIGEST_SIZE,
2751                 },
2752                 .caam = {
2753                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2754                                            OP_ALG_AAI_CTR_MOD128,
2755                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2756                                            OP_ALG_AAI_HMAC_PRECOMP,
2757                         .rfc3686 = true,
2758                         .geniv = true,
2759                 },
2760         },
2761         {
2762                 .aead = {
2763                         .base = {
2764                                 .cra_name = "authenc(hmac(sha224),"
2765                                             "rfc3686(ctr(aes)))",
2766                                 .cra_driver_name = "authenc-hmac-sha224-"
2767                                                    "rfc3686-ctr-aes-caam-qi2",
2768                                 .cra_blocksize = 1,
2769                         },
2770                         .setkey = aead_setkey,
2771                         .setauthsize = aead_setauthsize,
2772                         .encrypt = aead_encrypt,
2773                         .decrypt = aead_decrypt,
2774                         .ivsize = CTR_RFC3686_IV_SIZE,
2775                         .maxauthsize = SHA224_DIGEST_SIZE,
2776                 },
2777                 .caam = {
2778                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2779                                            OP_ALG_AAI_CTR_MOD128,
2780                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2781                                            OP_ALG_AAI_HMAC_PRECOMP,
2782                         .rfc3686 = true,
2783                 },
2784         },
2785         {
2786                 .aead = {
2787                         .base = {
2788                                 .cra_name = "seqiv(authenc("
2789                                             "hmac(sha224),rfc3686(ctr(aes))))",
2790                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
2791                                                    "rfc3686-ctr-aes-caam-qi2",
2792                                 .cra_blocksize = 1,
2793                         },
2794                         .setkey = aead_setkey,
2795                         .setauthsize = aead_setauthsize,
2796                         .encrypt = aead_encrypt,
2797                         .decrypt = aead_decrypt,
2798                         .ivsize = CTR_RFC3686_IV_SIZE,
2799                         .maxauthsize = SHA224_DIGEST_SIZE,
2800                 },
2801                 .caam = {
2802                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2803                                            OP_ALG_AAI_CTR_MOD128,
2804                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2805                                            OP_ALG_AAI_HMAC_PRECOMP,
2806                         .rfc3686 = true,
2807                         .geniv = true,
2808                 },
2809         },
2810         {
2811                 .aead = {
2812                         .base = {
2813                                 .cra_name = "authenc(hmac(sha256),"
2814                                             "rfc3686(ctr(aes)))",
2815                                 .cra_driver_name = "authenc-hmac-sha256-"
2816                                                    "rfc3686-ctr-aes-caam-qi2",
2817                                 .cra_blocksize = 1,
2818                         },
2819                         .setkey = aead_setkey,
2820                         .setauthsize = aead_setauthsize,
2821                         .encrypt = aead_encrypt,
2822                         .decrypt = aead_decrypt,
2823                         .ivsize = CTR_RFC3686_IV_SIZE,
2824                         .maxauthsize = SHA256_DIGEST_SIZE,
2825                 },
2826                 .caam = {
2827                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2828                                            OP_ALG_AAI_CTR_MOD128,
2829                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2830                                            OP_ALG_AAI_HMAC_PRECOMP,
2831                         .rfc3686 = true,
2832                 },
2833         },
2834         {
2835                 .aead = {
2836                         .base = {
2837                                 .cra_name = "seqiv(authenc(hmac(sha256),"
2838                                             "rfc3686(ctr(aes))))",
2839                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
2840                                                    "rfc3686-ctr-aes-caam-qi2",
2841                                 .cra_blocksize = 1,
2842                         },
2843                         .setkey = aead_setkey,
2844                         .setauthsize = aead_setauthsize,
2845                         .encrypt = aead_encrypt,
2846                         .decrypt = aead_decrypt,
2847                         .ivsize = CTR_RFC3686_IV_SIZE,
2848                         .maxauthsize = SHA256_DIGEST_SIZE,
2849                 },
2850                 .caam = {
2851                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2852                                            OP_ALG_AAI_CTR_MOD128,
2853                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2854                                            OP_ALG_AAI_HMAC_PRECOMP,
2855                         .rfc3686 = true,
2856                         .geniv = true,
2857                 },
2858         },
2859         {
2860                 .aead = {
2861                         .base = {
2862                                 .cra_name = "authenc(hmac(sha384),"
2863                                             "rfc3686(ctr(aes)))",
2864                                 .cra_driver_name = "authenc-hmac-sha384-"
2865                                                    "rfc3686-ctr-aes-caam-qi2",
2866                                 .cra_blocksize = 1,
2867                         },
2868                         .setkey = aead_setkey,
2869                         .setauthsize = aead_setauthsize,
2870                         .encrypt = aead_encrypt,
2871                         .decrypt = aead_decrypt,
2872                         .ivsize = CTR_RFC3686_IV_SIZE,
2873                         .maxauthsize = SHA384_DIGEST_SIZE,
2874                 },
2875                 .caam = {
2876                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2877                                            OP_ALG_AAI_CTR_MOD128,
2878                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2879                                            OP_ALG_AAI_HMAC_PRECOMP,
2880                         .rfc3686 = true,
2881                 },
2882         },
2883         {
2884                 .aead = {
2885                         .base = {
2886                                 .cra_name = "seqiv(authenc(hmac(sha384),"
2887                                             "rfc3686(ctr(aes))))",
2888                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
2889                                                    "rfc3686-ctr-aes-caam-qi2",
2890                                 .cra_blocksize = 1,
2891                         },
2892                         .setkey = aead_setkey,
2893                         .setauthsize = aead_setauthsize,
2894                         .encrypt = aead_encrypt,
2895                         .decrypt = aead_decrypt,
2896                         .ivsize = CTR_RFC3686_IV_SIZE,
2897                         .maxauthsize = SHA384_DIGEST_SIZE,
2898                 },
2899                 .caam = {
2900                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2901                                            OP_ALG_AAI_CTR_MOD128,
2902                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2903                                            OP_ALG_AAI_HMAC_PRECOMP,
2904                         .rfc3686 = true,
2905                         .geniv = true,
2906                 },
2907         },
2908         {
2909                 .aead = {
2910                         .base = {
2911                                 .cra_name = "rfc7539(chacha20,poly1305)",
2912                                 .cra_driver_name = "rfc7539-chacha20-poly1305-"
2913                                                    "caam-qi2",
2914                                 .cra_blocksize = 1,
2915                         },
2916                         .setkey = chachapoly_setkey,
2917                         .setauthsize = chachapoly_setauthsize,
2918                         .encrypt = aead_encrypt,
2919                         .decrypt = aead_decrypt,
2920                         .ivsize = CHACHAPOLY_IV_SIZE,
2921                         .maxauthsize = POLY1305_DIGEST_SIZE,
2922                 },
2923                 .caam = {
2924                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2925                                            OP_ALG_AAI_AEAD,
2926                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2927                                            OP_ALG_AAI_AEAD,
2928                         .nodkp = true,
2929                 },
2930         },
2931         {
2932                 .aead = {
2933                         .base = {
2934                                 .cra_name = "rfc7539esp(chacha20,poly1305)",
2935                                 .cra_driver_name = "rfc7539esp-chacha20-"
2936                                                    "poly1305-caam-qi2",
2937                                 .cra_blocksize = 1,
2938                         },
2939                         .setkey = chachapoly_setkey,
2940                         .setauthsize = chachapoly_setauthsize,
2941                         .encrypt = aead_encrypt,
2942                         .decrypt = aead_decrypt,
2943                         .ivsize = 8,
2944                         .maxauthsize = POLY1305_DIGEST_SIZE,
2945                 },
2946                 .caam = {
2947                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2948                                            OP_ALG_AAI_AEAD,
2949                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2950                                            OP_ALG_AAI_AEAD,
2951                         .nodkp = true,
2952                 },
2953         },
2954         {
2955                 .aead = {
2956                         .base = {
2957                                 .cra_name = "authenc(hmac(sha512),"
2958                                             "rfc3686(ctr(aes)))",
2959                                 .cra_driver_name = "authenc-hmac-sha512-"
2960                                                    "rfc3686-ctr-aes-caam-qi2",
2961                                 .cra_blocksize = 1,
2962                         },
2963                         .setkey = aead_setkey,
2964                         .setauthsize = aead_setauthsize,
2965                         .encrypt = aead_encrypt,
2966                         .decrypt = aead_decrypt,
2967                         .ivsize = CTR_RFC3686_IV_SIZE,
2968                         .maxauthsize = SHA512_DIGEST_SIZE,
2969                 },
2970                 .caam = {
2971                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2972                                            OP_ALG_AAI_CTR_MOD128,
2973                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2974                                            OP_ALG_AAI_HMAC_PRECOMP,
2975                         .rfc3686 = true,
2976                 },
2977         },
2978         {
2979                 .aead = {
2980                         .base = {
2981                                 .cra_name = "seqiv(authenc(hmac(sha512),"
2982                                             "rfc3686(ctr(aes))))",
2983                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
2984                                                    "rfc3686-ctr-aes-caam-qi2",
2985                                 .cra_blocksize = 1,
2986                         },
2987                         .setkey = aead_setkey,
2988                         .setauthsize = aead_setauthsize,
2989                         .encrypt = aead_encrypt,
2990                         .decrypt = aead_decrypt,
2991                         .ivsize = CTR_RFC3686_IV_SIZE,
2992                         .maxauthsize = SHA512_DIGEST_SIZE,
2993                 },
2994                 .caam = {
2995                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2996                                            OP_ALG_AAI_CTR_MOD128,
2997                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2998                                            OP_ALG_AAI_HMAC_PRECOMP,
2999                         .rfc3686 = true,
3000                         .geniv = true,
3001                 },
3002         },
3003 };
3004
3005 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3006 {
3007         struct skcipher_alg *alg = &t_alg->skcipher;
3008
3009         alg->base.cra_module = THIS_MODULE;
3010         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3011         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3012         alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3013                               CRYPTO_ALG_KERN_DRIVER_ONLY);
3014
3015         alg->init = caam_cra_init_skcipher;
3016         alg->exit = caam_cra_exit;
3017 }
3018
3019 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3020 {
3021         struct aead_alg *alg = &t_alg->aead;
3022
3023         alg->base.cra_module = THIS_MODULE;
3024         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3025         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3026         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3027                               CRYPTO_ALG_KERN_DRIVER_ONLY;
3028
3029         alg->init = caam_cra_init_aead;
3030         alg->exit = caam_cra_exit_aead;
3031 }
3032
3033 /* max hash key is max split key size */
3034 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
3035
3036 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
3037
3038 /* caam context sizes for hashes: running digest + 8 */
3039 #define HASH_MSG_LEN                    8
3040 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
3041
3042 enum hash_optype {
3043         UPDATE = 0,
3044         UPDATE_FIRST,
3045         FINALIZE,
3046         DIGEST,
3047         HASH_NUM_OP
3048 };
3049
3050 /**
3051  * struct caam_hash_ctx - ahash per-session context
3052  * @flc: Flow Contexts array
3053  * @key: authentication key
3054  * @flc_dma: I/O virtual addresses of the Flow Contexts
3055  * @dev: dpseci device
3056  * @ctx_len: size of Context Register
3057  * @adata: hashing algorithm details
3058  */
3059 struct caam_hash_ctx {
3060         struct caam_flc flc[HASH_NUM_OP];
3061         u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3062         dma_addr_t flc_dma[HASH_NUM_OP];
3063         struct device *dev;
3064         int ctx_len;
3065         struct alginfo adata;
3066 };
3067
3068 /* ahash state */
3069 struct caam_hash_state {
3070         struct caam_request caam_req;
3071         dma_addr_t buf_dma;
3072         dma_addr_t ctx_dma;
3073         int ctx_dma_len;
3074         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3075         int buflen;
3076         int next_buflen;
3077         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
3078         int (*update)(struct ahash_request *req);
3079         int (*final)(struct ahash_request *req);
3080         int (*finup)(struct ahash_request *req);
3081 };
3082
3083 struct caam_export_state {
3084         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
3085         u8 caam_ctx[MAX_CTX_LEN];
3086         int buflen;
3087         int (*update)(struct ahash_request *req);
3088         int (*final)(struct ahash_request *req);
3089         int (*finup)(struct ahash_request *req);
3090 };
3091
3092 /* Map current buffer in state (if length > 0) and put it in link table */
3093 static inline int buf_map_to_qm_sg(struct device *dev,
3094                                    struct dpaa2_sg_entry *qm_sg,
3095                                    struct caam_hash_state *state)
3096 {
3097         int buflen = state->buflen;
3098
3099         if (!buflen)
3100                 return 0;
3101
3102         state->buf_dma = dma_map_single(dev, state->buf, buflen,
3103                                         DMA_TO_DEVICE);
3104         if (dma_mapping_error(dev, state->buf_dma)) {
3105                 dev_err(dev, "unable to map buf\n");
3106                 state->buf_dma = 0;
3107                 return -ENOMEM;
3108         }
3109
3110         dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3111
3112         return 0;
3113 }
3114
3115 /* Map state->caam_ctx, and add it to link table */
3116 static inline int ctx_map_to_qm_sg(struct device *dev,
3117                                    struct caam_hash_state *state, int ctx_len,
3118                                    struct dpaa2_sg_entry *qm_sg, u32 flag)
3119 {
3120         state->ctx_dma_len = ctx_len;
3121         state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3122         if (dma_mapping_error(dev, state->ctx_dma)) {
3123                 dev_err(dev, "unable to map ctx\n");
3124                 state->ctx_dma = 0;
3125                 return -ENOMEM;
3126         }
3127
3128         dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3129
3130         return 0;
3131 }
3132
3133 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3134 {
3135         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3136         int digestsize = crypto_ahash_digestsize(ahash);
3137         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3138         struct caam_flc *flc;
3139         u32 *desc;
3140
3141         /* ahash_update shared descriptor */
3142         flc = &ctx->flc[UPDATE];
3143         desc = flc->sh_desc;
3144         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3145                           ctx->ctx_len, true, priv->sec_attr.era);
3146         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3147         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3148                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3149         print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3150                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3151                              1);
3152
3153         /* ahash_update_first shared descriptor */
3154         flc = &ctx->flc[UPDATE_FIRST];
3155         desc = flc->sh_desc;
3156         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3157                           ctx->ctx_len, false, priv->sec_attr.era);
3158         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3159         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3160                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3161         print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3162                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3163                              1);
3164
3165         /* ahash_final shared descriptor */
3166         flc = &ctx->flc[FINALIZE];
3167         desc = flc->sh_desc;
3168         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3169                           ctx->ctx_len, true, priv->sec_attr.era);
3170         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3171         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3172                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3173         print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3174                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3175                              1);
3176
3177         /* ahash_digest shared descriptor */
3178         flc = &ctx->flc[DIGEST];
3179         desc = flc->sh_desc;
3180         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3181                           ctx->ctx_len, false, priv->sec_attr.era);
3182         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3183         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3184                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3185         print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3186                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3187                              1);
3188
3189         return 0;
3190 }
3191
3192 struct split_key_sh_result {
3193         struct completion completion;
3194         int err;
3195         struct device *dev;
3196 };
3197
3198 static void split_key_sh_done(void *cbk_ctx, u32 err)
3199 {
3200         struct split_key_sh_result *res = cbk_ctx;
3201
3202         dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3203
3204         res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3205         complete(&res->completion);
3206 }
3207
3208 /* Digest hash size if it is too large */
3209 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3210                            u32 digestsize)
3211 {
3212         struct caam_request *req_ctx;
3213         u32 *desc;
3214         struct split_key_sh_result result;
3215         dma_addr_t key_dma;
3216         struct caam_flc *flc;
3217         dma_addr_t flc_dma;
3218         int ret = -ENOMEM;
3219         struct dpaa2_fl_entry *in_fle, *out_fle;
3220
3221         req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
3222         if (!req_ctx)
3223                 return -ENOMEM;
3224
3225         in_fle = &req_ctx->fd_flt[1];
3226         out_fle = &req_ctx->fd_flt[0];
3227
3228         flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
3229         if (!flc)
3230                 goto err_flc;
3231
3232         key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3233         if (dma_mapping_error(ctx->dev, key_dma)) {
3234                 dev_err(ctx->dev, "unable to map key memory\n");
3235                 goto err_key_dma;
3236         }
3237
3238         desc = flc->sh_desc;
3239
3240         init_sh_desc(desc, 0);
3241
3242         /* descriptor to perform unkeyed hash on key_in */
3243         append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3244                          OP_ALG_AS_INITFINAL);
3245         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3246                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3247         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3248                          LDST_SRCDST_BYTE_CONTEXT);
3249
3250         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3251         flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3252                                  desc_bytes(desc), DMA_TO_DEVICE);
3253         if (dma_mapping_error(ctx->dev, flc_dma)) {
3254                 dev_err(ctx->dev, "unable to map shared descriptor\n");
3255                 goto err_flc_dma;
3256         }
3257
3258         dpaa2_fl_set_final(in_fle, true);
3259         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3260         dpaa2_fl_set_addr(in_fle, key_dma);
3261         dpaa2_fl_set_len(in_fle, *keylen);
3262         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3263         dpaa2_fl_set_addr(out_fle, key_dma);
3264         dpaa2_fl_set_len(out_fle, digestsize);
3265
3266         print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3267                              DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3268         print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3269                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3270                              1);
3271
3272         result.err = 0;
3273         init_completion(&result.completion);
3274         result.dev = ctx->dev;
3275
3276         req_ctx->flc = flc;
3277         req_ctx->flc_dma = flc_dma;
3278         req_ctx->cbk = split_key_sh_done;
3279         req_ctx->ctx = &result;
3280
3281         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3282         if (ret == -EINPROGRESS) {
3283                 /* in progress */
3284                 wait_for_completion(&result.completion);
3285                 ret = result.err;
3286                 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3287                                      DUMP_PREFIX_ADDRESS, 16, 4, key,
3288                                      digestsize, 1);
3289         }
3290
3291         dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3292                          DMA_TO_DEVICE);
3293 err_flc_dma:
3294         dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3295 err_key_dma:
3296         kfree(flc);
3297 err_flc:
3298         kfree(req_ctx);
3299
3300         *keylen = digestsize;
3301
3302         return ret;
3303 }
3304
3305 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3306                         unsigned int keylen)
3307 {
3308         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3309         unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3310         unsigned int digestsize = crypto_ahash_digestsize(ahash);
3311         int ret;
3312         u8 *hashed_key = NULL;
3313
3314         dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3315
3316         if (keylen > blocksize) {
3317                 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
3318                 if (!hashed_key)
3319                         return -ENOMEM;
3320                 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3321                 if (ret)
3322                         goto bad_free_key;
3323                 key = hashed_key;
3324         }
3325
3326         ctx->adata.keylen = keylen;
3327         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3328                                               OP_ALG_ALGSEL_MASK);
3329         if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3330                 goto bad_free_key;
3331
3332         ctx->adata.key_virt = key;
3333         ctx->adata.key_inline = true;
3334
3335         /*
3336          * In case |user key| > |derived key|, using DKP<imm,imm> would result
3337          * in invalid opcodes (last bytes of user key) in the resulting
3338          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3339          * addresses are needed.
3340          */
3341         if (keylen > ctx->adata.keylen_pad) {
3342                 memcpy(ctx->key, key, keylen);
3343                 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3344                                            ctx->adata.keylen_pad,
3345                                            DMA_TO_DEVICE);
3346         }
3347
3348         ret = ahash_set_sh_desc(ahash);
3349         kfree(hashed_key);
3350         return ret;
3351 bad_free_key:
3352         kfree(hashed_key);
3353         return -EINVAL;
3354 }
3355
3356 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3357                                struct ahash_request *req)
3358 {
3359         struct caam_hash_state *state = ahash_request_ctx(req);
3360
3361         if (edesc->src_nents)
3362                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3363
3364         if (edesc->qm_sg_bytes)
3365                 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3366                                  DMA_TO_DEVICE);
3367
3368         if (state->buf_dma) {
3369                 dma_unmap_single(dev, state->buf_dma, state->buflen,
3370                                  DMA_TO_DEVICE);
3371                 state->buf_dma = 0;
3372         }
3373 }
3374
3375 static inline void ahash_unmap_ctx(struct device *dev,
3376                                    struct ahash_edesc *edesc,
3377                                    struct ahash_request *req, u32 flag)
3378 {
3379         struct caam_hash_state *state = ahash_request_ctx(req);
3380
3381         if (state->ctx_dma) {
3382                 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3383                 state->ctx_dma = 0;
3384         }
3385         ahash_unmap(dev, edesc, req);
3386 }
3387
3388 static void ahash_done(void *cbk_ctx, u32 status)
3389 {
3390         struct crypto_async_request *areq = cbk_ctx;
3391         struct ahash_request *req = ahash_request_cast(areq);
3392         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3393         struct caam_hash_state *state = ahash_request_ctx(req);
3394         struct ahash_edesc *edesc = state->caam_req.edesc;
3395         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3396         int digestsize = crypto_ahash_digestsize(ahash);
3397         int ecode = 0;
3398
3399         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3400
3401         if (unlikely(status))
3402                 ecode = caam_qi2_strstatus(ctx->dev, status);
3403
3404         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3405         memcpy(req->result, state->caam_ctx, digestsize);
3406         qi_cache_free(edesc);
3407
3408         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3409                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3410                              ctx->ctx_len, 1);
3411
3412         req->base.complete(&req->base, ecode);
3413 }
3414
3415 static void ahash_done_bi(void *cbk_ctx, u32 status)
3416 {
3417         struct crypto_async_request *areq = cbk_ctx;
3418         struct ahash_request *req = ahash_request_cast(areq);
3419         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3420         struct caam_hash_state *state = ahash_request_ctx(req);
3421         struct ahash_edesc *edesc = state->caam_req.edesc;
3422         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3423         int ecode = 0;
3424
3425         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3426
3427         if (unlikely(status))
3428                 ecode = caam_qi2_strstatus(ctx->dev, status);
3429
3430         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3431         qi_cache_free(edesc);
3432
3433         scatterwalk_map_and_copy(state->buf, req->src,
3434                                  req->nbytes - state->next_buflen,
3435                                  state->next_buflen, 0);
3436         state->buflen = state->next_buflen;
3437
3438         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3439                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3440                              state->buflen, 1);
3441
3442         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3443                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3444                              ctx->ctx_len, 1);
3445         if (req->result)
3446                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3447                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3448                                      crypto_ahash_digestsize(ahash), 1);
3449
3450         req->base.complete(&req->base, ecode);
3451 }
3452
3453 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3454 {
3455         struct crypto_async_request *areq = cbk_ctx;
3456         struct ahash_request *req = ahash_request_cast(areq);
3457         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3458         struct caam_hash_state *state = ahash_request_ctx(req);
3459         struct ahash_edesc *edesc = state->caam_req.edesc;
3460         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3461         int digestsize = crypto_ahash_digestsize(ahash);
3462         int ecode = 0;
3463
3464         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3465
3466         if (unlikely(status))
3467                 ecode = caam_qi2_strstatus(ctx->dev, status);
3468
3469         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3470         memcpy(req->result, state->caam_ctx, digestsize);
3471         qi_cache_free(edesc);
3472
3473         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3474                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3475                              ctx->ctx_len, 1);
3476
3477         req->base.complete(&req->base, ecode);
3478 }
3479
3480 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3481 {
3482         struct crypto_async_request *areq = cbk_ctx;
3483         struct ahash_request *req = ahash_request_cast(areq);
3484         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3485         struct caam_hash_state *state = ahash_request_ctx(req);
3486         struct ahash_edesc *edesc = state->caam_req.edesc;
3487         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3488         int ecode = 0;
3489
3490         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3491
3492         if (unlikely(status))
3493                 ecode = caam_qi2_strstatus(ctx->dev, status);
3494
3495         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3496         qi_cache_free(edesc);
3497
3498         scatterwalk_map_and_copy(state->buf, req->src,
3499                                  req->nbytes - state->next_buflen,
3500                                  state->next_buflen, 0);
3501         state->buflen = state->next_buflen;
3502
3503         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3504                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3505                              state->buflen, 1);
3506
3507         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3508                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3509                              ctx->ctx_len, 1);
3510         if (req->result)
3511                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3512                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3513                                      crypto_ahash_digestsize(ahash), 1);
3514
3515         req->base.complete(&req->base, ecode);
3516 }
3517
3518 static int ahash_update_ctx(struct ahash_request *req)
3519 {
3520         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3521         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3522         struct caam_hash_state *state = ahash_request_ctx(req);
3523         struct caam_request *req_ctx = &state->caam_req;
3524         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3525         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3526         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3527                       GFP_KERNEL : GFP_ATOMIC;
3528         u8 *buf = state->buf;
3529         int *buflen = &state->buflen;
3530         int *next_buflen = &state->next_buflen;
3531         int in_len = *buflen + req->nbytes, to_hash;
3532         int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3533         struct ahash_edesc *edesc;
3534         int ret = 0;
3535
3536         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3537         to_hash = in_len - *next_buflen;
3538
3539         if (to_hash) {
3540                 struct dpaa2_sg_entry *sg_table;
3541                 int src_len = req->nbytes - *next_buflen;
3542
3543                 src_nents = sg_nents_for_len(req->src, src_len);
3544                 if (src_nents < 0) {
3545                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3546                         return src_nents;
3547                 }
3548
3549                 if (src_nents) {
3550                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3551                                                   DMA_TO_DEVICE);
3552                         if (!mapped_nents) {
3553                                 dev_err(ctx->dev, "unable to DMA map source\n");
3554                                 return -ENOMEM;
3555                         }
3556                 } else {
3557                         mapped_nents = 0;
3558                 }
3559
3560                 /* allocate space for base edesc and link tables */
3561                 edesc = qi_cache_zalloc(GFP_DMA | flags);
3562                 if (!edesc) {
3563                         dma_unmap_sg(ctx->dev, req->src, src_nents,
3564                                      DMA_TO_DEVICE);
3565                         return -ENOMEM;
3566                 }
3567
3568                 edesc->src_nents = src_nents;
3569                 qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3570                 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3571                               sizeof(*sg_table);
3572                 sg_table = &edesc->sgt[0];
3573
3574                 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3575                                        DMA_BIDIRECTIONAL);
3576                 if (ret)
3577                         goto unmap_ctx;
3578
3579                 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3580                 if (ret)
3581                         goto unmap_ctx;
3582
3583                 if (mapped_nents) {
3584                         sg_to_qm_sg_last(req->src, src_len,
3585                                          sg_table + qm_sg_src_index, 0);
3586                 } else {
3587                         dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3588                                            true);
3589                 }
3590
3591                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3592                                                   qm_sg_bytes, DMA_TO_DEVICE);
3593                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3594                         dev_err(ctx->dev, "unable to map S/G table\n");
3595                         ret = -ENOMEM;
3596                         goto unmap_ctx;
3597                 }
3598                 edesc->qm_sg_bytes = qm_sg_bytes;
3599
3600                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3601                 dpaa2_fl_set_final(in_fle, true);
3602                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3603                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3604                 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3605                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3606                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3607                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3608
3609                 req_ctx->flc = &ctx->flc[UPDATE];
3610                 req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3611                 req_ctx->cbk = ahash_done_bi;
3612                 req_ctx->ctx = &req->base;
3613                 req_ctx->edesc = edesc;
3614
3615                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3616                 if (ret != -EINPROGRESS &&
3617                     !(ret == -EBUSY &&
3618                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3619                         goto unmap_ctx;
3620         } else if (*next_buflen) {
3621                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3622                                          req->nbytes, 0);
3623                 *buflen = *next_buflen;
3624
3625                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3626                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
3627                                      *buflen, 1);
3628         }
3629
3630         return ret;
3631 unmap_ctx:
3632         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3633         qi_cache_free(edesc);
3634         return ret;
3635 }
3636
3637 static int ahash_final_ctx(struct ahash_request *req)
3638 {
3639         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3640         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3641         struct caam_hash_state *state = ahash_request_ctx(req);
3642         struct caam_request *req_ctx = &state->caam_req;
3643         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3644         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3645         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3646                       GFP_KERNEL : GFP_ATOMIC;
3647         int buflen = state->buflen;
3648         int qm_sg_bytes;
3649         int digestsize = crypto_ahash_digestsize(ahash);
3650         struct ahash_edesc *edesc;
3651         struct dpaa2_sg_entry *sg_table;
3652         int ret;
3653
3654         /* allocate space for base edesc and link tables */
3655         edesc = qi_cache_zalloc(GFP_DMA | flags);
3656         if (!edesc)
3657                 return -ENOMEM;
3658
3659         qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3660         sg_table = &edesc->sgt[0];
3661
3662         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3663                                DMA_BIDIRECTIONAL);
3664         if (ret)
3665                 goto unmap_ctx;
3666
3667         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3668         if (ret)
3669                 goto unmap_ctx;
3670
3671         dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3672
3673         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3674                                           DMA_TO_DEVICE);
3675         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3676                 dev_err(ctx->dev, "unable to map S/G table\n");
3677                 ret = -ENOMEM;
3678                 goto unmap_ctx;
3679         }
3680         edesc->qm_sg_bytes = qm_sg_bytes;
3681
3682         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3683         dpaa2_fl_set_final(in_fle, true);
3684         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3685         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3686         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3687         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3688         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3689         dpaa2_fl_set_len(out_fle, digestsize);
3690
3691         req_ctx->flc = &ctx->flc[FINALIZE];
3692         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3693         req_ctx->cbk = ahash_done_ctx_src;
3694         req_ctx->ctx = &req->base;
3695         req_ctx->edesc = edesc;
3696
3697         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3698         if (ret == -EINPROGRESS ||
3699             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3700                 return ret;
3701
3702 unmap_ctx:
3703         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3704         qi_cache_free(edesc);
3705         return ret;
3706 }
3707
3708 static int ahash_finup_ctx(struct ahash_request *req)
3709 {
3710         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3711         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3712         struct caam_hash_state *state = ahash_request_ctx(req);
3713         struct caam_request *req_ctx = &state->caam_req;
3714         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3715         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3716         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3717                       GFP_KERNEL : GFP_ATOMIC;
3718         int buflen = state->buflen;
3719         int qm_sg_bytes, qm_sg_src_index;
3720         int src_nents, mapped_nents;
3721         int digestsize = crypto_ahash_digestsize(ahash);
3722         struct ahash_edesc *edesc;
3723         struct dpaa2_sg_entry *sg_table;
3724         int ret;
3725
3726         src_nents = sg_nents_for_len(req->src, req->nbytes);
3727         if (src_nents < 0) {
3728                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3729                 return src_nents;
3730         }
3731
3732         if (src_nents) {
3733                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3734                                           DMA_TO_DEVICE);
3735                 if (!mapped_nents) {
3736                         dev_err(ctx->dev, "unable to DMA map source\n");
3737                         return -ENOMEM;
3738                 }
3739         } else {
3740                 mapped_nents = 0;
3741         }
3742
3743         /* allocate space for base edesc and link tables */
3744         edesc = qi_cache_zalloc(GFP_DMA | flags);
3745         if (!edesc) {
3746                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3747                 return -ENOMEM;
3748         }
3749
3750         edesc->src_nents = src_nents;
3751         qm_sg_src_index = 1 + (buflen ? 1 : 0);
3752         qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3753                       sizeof(*sg_table);
3754         sg_table = &edesc->sgt[0];
3755
3756         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3757                                DMA_BIDIRECTIONAL);
3758         if (ret)
3759                 goto unmap_ctx;
3760
3761         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3762         if (ret)
3763                 goto unmap_ctx;
3764
3765         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3766
3767         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3768                                           DMA_TO_DEVICE);
3769         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3770                 dev_err(ctx->dev, "unable to map S/G table\n");
3771                 ret = -ENOMEM;
3772                 goto unmap_ctx;
3773         }
3774         edesc->qm_sg_bytes = qm_sg_bytes;
3775
3776         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3777         dpaa2_fl_set_final(in_fle, true);
3778         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3779         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3780         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3781         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3782         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3783         dpaa2_fl_set_len(out_fle, digestsize);
3784
3785         req_ctx->flc = &ctx->flc[FINALIZE];
3786         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3787         req_ctx->cbk = ahash_done_ctx_src;
3788         req_ctx->ctx = &req->base;
3789         req_ctx->edesc = edesc;
3790
3791         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3792         if (ret == -EINPROGRESS ||
3793             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3794                 return ret;
3795
3796 unmap_ctx:
3797         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3798         qi_cache_free(edesc);
3799         return ret;
3800 }
3801
3802 static int ahash_digest(struct ahash_request *req)
3803 {
3804         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3805         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3806         struct caam_hash_state *state = ahash_request_ctx(req);
3807         struct caam_request *req_ctx = &state->caam_req;
3808         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3809         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3810         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3811                       GFP_KERNEL : GFP_ATOMIC;
3812         int digestsize = crypto_ahash_digestsize(ahash);
3813         int src_nents, mapped_nents;
3814         struct ahash_edesc *edesc;
3815         int ret = -ENOMEM;
3816
3817         state->buf_dma = 0;
3818
3819         src_nents = sg_nents_for_len(req->src, req->nbytes);
3820         if (src_nents < 0) {
3821                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3822                 return src_nents;
3823         }
3824
3825         if (src_nents) {
3826                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3827                                           DMA_TO_DEVICE);
3828                 if (!mapped_nents) {
3829                         dev_err(ctx->dev, "unable to map source for DMA\n");
3830                         return ret;
3831                 }
3832         } else {
3833                 mapped_nents = 0;
3834         }
3835
3836         /* allocate space for base edesc and link tables */
3837         edesc = qi_cache_zalloc(GFP_DMA | flags);
3838         if (!edesc) {
3839                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3840                 return ret;
3841         }
3842
3843         edesc->src_nents = src_nents;
3844         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3845
3846         if (mapped_nents > 1) {
3847                 int qm_sg_bytes;
3848                 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3849
3850                 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3851                 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3852                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3853                                                   qm_sg_bytes, DMA_TO_DEVICE);
3854                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3855                         dev_err(ctx->dev, "unable to map S/G table\n");
3856                         goto unmap;
3857                 }
3858                 edesc->qm_sg_bytes = qm_sg_bytes;
3859                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3860                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3861         } else {
3862                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3863                 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3864         }
3865
3866         state->ctx_dma_len = digestsize;
3867         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3868                                         DMA_FROM_DEVICE);
3869         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3870                 dev_err(ctx->dev, "unable to map ctx\n");
3871                 state->ctx_dma = 0;
3872                 goto unmap;
3873         }
3874
3875         dpaa2_fl_set_final(in_fle, true);
3876         dpaa2_fl_set_len(in_fle, req->nbytes);
3877         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3878         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3879         dpaa2_fl_set_len(out_fle, digestsize);
3880
3881         req_ctx->flc = &ctx->flc[DIGEST];
3882         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3883         req_ctx->cbk = ahash_done;
3884         req_ctx->ctx = &req->base;
3885         req_ctx->edesc = edesc;
3886         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3887         if (ret == -EINPROGRESS ||
3888             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3889                 return ret;
3890
3891 unmap:
3892         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3893         qi_cache_free(edesc);
3894         return ret;
3895 }
3896
3897 static int ahash_final_no_ctx(struct ahash_request *req)
3898 {
3899         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3900         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3901         struct caam_hash_state *state = ahash_request_ctx(req);
3902         struct caam_request *req_ctx = &state->caam_req;
3903         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3904         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3905         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3906                       GFP_KERNEL : GFP_ATOMIC;
3907         u8 *buf = state->buf;
3908         int buflen = state->buflen;
3909         int digestsize = crypto_ahash_digestsize(ahash);
3910         struct ahash_edesc *edesc;
3911         int ret = -ENOMEM;
3912
3913         /* allocate space for base edesc and link tables */
3914         edesc = qi_cache_zalloc(GFP_DMA | flags);
3915         if (!edesc)
3916                 return ret;
3917
3918         if (buflen) {
3919                 state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3920                                                 DMA_TO_DEVICE);
3921                 if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3922                         dev_err(ctx->dev, "unable to map src\n");
3923                         goto unmap;
3924                 }
3925         }
3926
3927         state->ctx_dma_len = digestsize;
3928         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3929                                         DMA_FROM_DEVICE);
3930         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3931                 dev_err(ctx->dev, "unable to map ctx\n");
3932                 state->ctx_dma = 0;
3933                 goto unmap;
3934         }
3935
3936         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3937         dpaa2_fl_set_final(in_fle, true);
3938         /*
3939          * crypto engine requires the input entry to be present when
3940          * "frame list" FD is used.
3941          * Since engine does not support FMT=2'b11 (unused entry type), leaving
3942          * in_fle zeroized (except for "Final" flag) is the best option.
3943          */
3944         if (buflen) {
3945                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3946                 dpaa2_fl_set_addr(in_fle, state->buf_dma);
3947                 dpaa2_fl_set_len(in_fle, buflen);
3948         }
3949         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3950         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3951         dpaa2_fl_set_len(out_fle, digestsize);
3952
3953         req_ctx->flc = &ctx->flc[DIGEST];
3954         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3955         req_ctx->cbk = ahash_done;
3956         req_ctx->ctx = &req->base;
3957         req_ctx->edesc = edesc;
3958
3959         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3960         if (ret == -EINPROGRESS ||
3961             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3962                 return ret;
3963
3964 unmap:
3965         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3966         qi_cache_free(edesc);
3967         return ret;
3968 }
3969
3970 static int ahash_update_no_ctx(struct ahash_request *req)
3971 {
3972         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3973         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3974         struct caam_hash_state *state = ahash_request_ctx(req);
3975         struct caam_request *req_ctx = &state->caam_req;
3976         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3977         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3978         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3979                       GFP_KERNEL : GFP_ATOMIC;
3980         u8 *buf = state->buf;
3981         int *buflen = &state->buflen;
3982         int *next_buflen = &state->next_buflen;
3983         int in_len = *buflen + req->nbytes, to_hash;
3984         int qm_sg_bytes, src_nents, mapped_nents;
3985         struct ahash_edesc *edesc;
3986         int ret = 0;
3987
3988         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3989         to_hash = in_len - *next_buflen;
3990
3991         if (to_hash) {
3992                 struct dpaa2_sg_entry *sg_table;
3993                 int src_len = req->nbytes - *next_buflen;
3994
3995                 src_nents = sg_nents_for_len(req->src, src_len);
3996                 if (src_nents < 0) {
3997                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3998                         return src_nents;
3999                 }
4000
4001                 if (src_nents) {
4002                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4003                                                   DMA_TO_DEVICE);
4004                         if (!mapped_nents) {
4005                                 dev_err(ctx->dev, "unable to DMA map source\n");
4006                                 return -ENOMEM;
4007                         }
4008                 } else {
4009                         mapped_nents = 0;
4010                 }
4011
4012                 /* allocate space for base edesc and link tables */
4013                 edesc = qi_cache_zalloc(GFP_DMA | flags);
4014                 if (!edesc) {
4015                         dma_unmap_sg(ctx->dev, req->src, src_nents,
4016                                      DMA_TO_DEVICE);
4017                         return -ENOMEM;
4018                 }
4019
4020                 edesc->src_nents = src_nents;
4021                 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
4022                               sizeof(*sg_table);
4023                 sg_table = &edesc->sgt[0];
4024
4025                 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4026                 if (ret)
4027                         goto unmap_ctx;
4028
4029                 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
4030
4031                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4032                                                   qm_sg_bytes, DMA_TO_DEVICE);
4033                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4034                         dev_err(ctx->dev, "unable to map S/G table\n");
4035                         ret = -ENOMEM;
4036                         goto unmap_ctx;
4037                 }
4038                 edesc->qm_sg_bytes = qm_sg_bytes;
4039
4040                 state->ctx_dma_len = ctx->ctx_len;
4041                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4042                                                 ctx->ctx_len, DMA_FROM_DEVICE);
4043                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4044                         dev_err(ctx->dev, "unable to map ctx\n");
4045                         state->ctx_dma = 0;
4046                         ret = -ENOMEM;
4047                         goto unmap_ctx;
4048                 }
4049
4050                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4051                 dpaa2_fl_set_final(in_fle, true);
4052                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4053                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4054                 dpaa2_fl_set_len(in_fle, to_hash);
4055                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4056                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4057                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4058
4059                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4060                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4061                 req_ctx->cbk = ahash_done_ctx_dst;
4062                 req_ctx->ctx = &req->base;
4063                 req_ctx->edesc = edesc;
4064
4065                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4066                 if (ret != -EINPROGRESS &&
4067                     !(ret == -EBUSY &&
4068                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4069                         goto unmap_ctx;
4070
4071                 state->update = ahash_update_ctx;
4072                 state->finup = ahash_finup_ctx;
4073                 state->final = ahash_final_ctx;
4074         } else if (*next_buflen) {
4075                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
4076                                          req->nbytes, 0);
4077                 *buflen = *next_buflen;
4078
4079                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4080                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
4081                                      *buflen, 1);
4082         }
4083
4084         return ret;
4085 unmap_ctx:
4086         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4087         qi_cache_free(edesc);
4088         return ret;
4089 }
4090
4091 static int ahash_finup_no_ctx(struct ahash_request *req)
4092 {
4093         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4094         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4095         struct caam_hash_state *state = ahash_request_ctx(req);
4096         struct caam_request *req_ctx = &state->caam_req;
4097         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4098         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4099         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4100                       GFP_KERNEL : GFP_ATOMIC;
4101         int buflen = state->buflen;
4102         int qm_sg_bytes, src_nents, mapped_nents;
4103         int digestsize = crypto_ahash_digestsize(ahash);
4104         struct ahash_edesc *edesc;
4105         struct dpaa2_sg_entry *sg_table;
4106         int ret = -ENOMEM;
4107
4108         src_nents = sg_nents_for_len(req->src, req->nbytes);
4109         if (src_nents < 0) {
4110                 dev_err(ctx->dev, "Invalid number of src SG.\n");
4111                 return src_nents;
4112         }
4113
4114         if (src_nents) {
4115                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4116                                           DMA_TO_DEVICE);
4117                 if (!mapped_nents) {
4118                         dev_err(ctx->dev, "unable to DMA map source\n");
4119                         return ret;
4120                 }
4121         } else {
4122                 mapped_nents = 0;
4123         }
4124
4125         /* allocate space for base edesc and link tables */
4126         edesc = qi_cache_zalloc(GFP_DMA | flags);
4127         if (!edesc) {
4128                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4129                 return ret;
4130         }
4131
4132         edesc->src_nents = src_nents;
4133         qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4134         sg_table = &edesc->sgt[0];
4135
4136         ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4137         if (ret)
4138                 goto unmap;
4139
4140         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4141
4142         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4143                                           DMA_TO_DEVICE);
4144         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4145                 dev_err(ctx->dev, "unable to map S/G table\n");
4146                 ret = -ENOMEM;
4147                 goto unmap;
4148         }
4149         edesc->qm_sg_bytes = qm_sg_bytes;
4150
4151         state->ctx_dma_len = digestsize;
4152         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4153                                         DMA_FROM_DEVICE);
4154         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4155                 dev_err(ctx->dev, "unable to map ctx\n");
4156                 state->ctx_dma = 0;
4157                 ret = -ENOMEM;
4158                 goto unmap;
4159         }
4160
4161         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4162         dpaa2_fl_set_final(in_fle, true);
4163         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4164         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4165         dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4166         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4167         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4168         dpaa2_fl_set_len(out_fle, digestsize);
4169
4170         req_ctx->flc = &ctx->flc[DIGEST];
4171         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4172         req_ctx->cbk = ahash_done;
4173         req_ctx->ctx = &req->base;
4174         req_ctx->edesc = edesc;
4175         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4176         if (ret != -EINPROGRESS &&
4177             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4178                 goto unmap;
4179
4180         return ret;
4181 unmap:
4182         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4183         qi_cache_free(edesc);
4184         return ret;
4185 }
4186
4187 static int ahash_update_first(struct ahash_request *req)
4188 {
4189         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4190         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4191         struct caam_hash_state *state = ahash_request_ctx(req);
4192         struct caam_request *req_ctx = &state->caam_req;
4193         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4194         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4195         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4196                       GFP_KERNEL : GFP_ATOMIC;
4197         u8 *buf = state->buf;
4198         int *buflen = &state->buflen;
4199         int *next_buflen = &state->next_buflen;
4200         int to_hash;
4201         int src_nents, mapped_nents;
4202         struct ahash_edesc *edesc;
4203         int ret = 0;
4204
4205         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4206                                       1);
4207         to_hash = req->nbytes - *next_buflen;
4208
4209         if (to_hash) {
4210                 struct dpaa2_sg_entry *sg_table;
4211                 int src_len = req->nbytes - *next_buflen;
4212
4213                 src_nents = sg_nents_for_len(req->src, src_len);
4214                 if (src_nents < 0) {
4215                         dev_err(ctx->dev, "Invalid number of src SG.\n");
4216                         return src_nents;
4217                 }
4218
4219                 if (src_nents) {
4220                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4221                                                   DMA_TO_DEVICE);
4222                         if (!mapped_nents) {
4223                                 dev_err(ctx->dev, "unable to map source for DMA\n");
4224                                 return -ENOMEM;
4225                         }
4226                 } else {
4227                         mapped_nents = 0;
4228                 }
4229
4230                 /* allocate space for base edesc and link tables */
4231                 edesc = qi_cache_zalloc(GFP_DMA | flags);
4232                 if (!edesc) {
4233                         dma_unmap_sg(ctx->dev, req->src, src_nents,
4234                                      DMA_TO_DEVICE);
4235                         return -ENOMEM;
4236                 }
4237
4238                 edesc->src_nents = src_nents;
4239                 sg_table = &edesc->sgt[0];
4240
4241                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4242                 dpaa2_fl_set_final(in_fle, true);
4243                 dpaa2_fl_set_len(in_fle, to_hash);
4244
4245                 if (mapped_nents > 1) {
4246                         int qm_sg_bytes;
4247
4248                         sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4249                         qm_sg_bytes = pad_sg_nents(mapped_nents) *
4250                                       sizeof(*sg_table);
4251                         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4252                                                           qm_sg_bytes,
4253                                                           DMA_TO_DEVICE);
4254                         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4255                                 dev_err(ctx->dev, "unable to map S/G table\n");
4256                                 ret = -ENOMEM;
4257                                 goto unmap_ctx;
4258                         }
4259                         edesc->qm_sg_bytes = qm_sg_bytes;
4260                         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4261                         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4262                 } else {
4263                         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4264                         dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4265                 }
4266
4267                 state->ctx_dma_len = ctx->ctx_len;
4268                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4269                                                 ctx->ctx_len, DMA_FROM_DEVICE);
4270                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4271                         dev_err(ctx->dev, "unable to map ctx\n");
4272                         state->ctx_dma = 0;
4273                         ret = -ENOMEM;
4274                         goto unmap_ctx;
4275                 }
4276
4277                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4278                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4279                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4280
4281                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4282                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4283                 req_ctx->cbk = ahash_done_ctx_dst;
4284                 req_ctx->ctx = &req->base;
4285                 req_ctx->edesc = edesc;
4286
4287                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4288                 if (ret != -EINPROGRESS &&
4289                     !(ret == -EBUSY && req->base.flags &
4290                       CRYPTO_TFM_REQ_MAY_BACKLOG))
4291                         goto unmap_ctx;
4292
4293                 state->update = ahash_update_ctx;
4294                 state->finup = ahash_finup_ctx;
4295                 state->final = ahash_final_ctx;
4296         } else if (*next_buflen) {
4297                 state->update = ahash_update_no_ctx;
4298                 state->finup = ahash_finup_no_ctx;
4299                 state->final = ahash_final_no_ctx;
4300                 scatterwalk_map_and_copy(buf, req->src, 0,
4301                                          req->nbytes, 0);
4302                 *buflen = *next_buflen;
4303
4304                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4305                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
4306                                      *buflen, 1);
4307         }
4308
4309         return ret;
4310 unmap_ctx:
4311         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4312         qi_cache_free(edesc);
4313         return ret;
4314 }
4315
4316 static int ahash_finup_first(struct ahash_request *req)
4317 {
4318         return ahash_digest(req);
4319 }
4320
4321 static int ahash_init(struct ahash_request *req)
4322 {
4323         struct caam_hash_state *state = ahash_request_ctx(req);
4324
4325         state->update = ahash_update_first;
4326         state->finup = ahash_finup_first;
4327         state->final = ahash_final_no_ctx;
4328
4329         state->ctx_dma = 0;
4330         state->ctx_dma_len = 0;
4331         state->buf_dma = 0;
4332         state->buflen = 0;
4333         state->next_buflen = 0;
4334
4335         return 0;
4336 }
4337
4338 static int ahash_update(struct ahash_request *req)
4339 {
4340         struct caam_hash_state *state = ahash_request_ctx(req);
4341
4342         return state->update(req);
4343 }
4344
4345 static int ahash_finup(struct ahash_request *req)
4346 {
4347         struct caam_hash_state *state = ahash_request_ctx(req);
4348
4349         return state->finup(req);
4350 }
4351
4352 static int ahash_final(struct ahash_request *req)
4353 {
4354         struct caam_hash_state *state = ahash_request_ctx(req);
4355
4356         return state->final(req);
4357 }
4358
4359 static int ahash_export(struct ahash_request *req, void *out)
4360 {
4361         struct caam_hash_state *state = ahash_request_ctx(req);
4362         struct caam_export_state *export = out;
4363         u8 *buf = state->buf;
4364         int len = state->buflen;
4365
4366         memcpy(export->buf, buf, len);
4367         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4368         export->buflen = len;
4369         export->update = state->update;
4370         export->final = state->final;
4371         export->finup = state->finup;
4372
4373         return 0;
4374 }
4375
4376 static int ahash_import(struct ahash_request *req, const void *in)
4377 {
4378         struct caam_hash_state *state = ahash_request_ctx(req);
4379         const struct caam_export_state *export = in;
4380
4381         memset(state, 0, sizeof(*state));
4382         memcpy(state->buf, export->buf, export->buflen);
4383         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4384         state->buflen = export->buflen;
4385         state->update = export->update;
4386         state->final = export->final;
4387         state->finup = export->finup;
4388
4389         return 0;
4390 }
4391
4392 struct caam_hash_template {
4393         char name[CRYPTO_MAX_ALG_NAME];
4394         char driver_name[CRYPTO_MAX_ALG_NAME];
4395         char hmac_name[CRYPTO_MAX_ALG_NAME];
4396         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4397         unsigned int blocksize;
4398         struct ahash_alg template_ahash;
4399         u32 alg_type;
4400 };
4401
4402 /* ahash descriptors */
4403 static struct caam_hash_template driver_hash[] = {
4404         {
4405                 .name = "sha1",
4406                 .driver_name = "sha1-caam-qi2",
4407                 .hmac_name = "hmac(sha1)",
4408                 .hmac_driver_name = "hmac-sha1-caam-qi2",
4409                 .blocksize = SHA1_BLOCK_SIZE,
4410                 .template_ahash = {
4411                         .init = ahash_init,
4412                         .update = ahash_update,
4413                         .final = ahash_final,
4414                         .finup = ahash_finup,
4415                         .digest = ahash_digest,
4416                         .export = ahash_export,
4417                         .import = ahash_import,
4418                         .setkey = ahash_setkey,
4419                         .halg = {
4420                                 .digestsize = SHA1_DIGEST_SIZE,
4421                                 .statesize = sizeof(struct caam_export_state),
4422                         },
4423                 },
4424                 .alg_type = OP_ALG_ALGSEL_SHA1,
4425         }, {
4426                 .name = "sha224",
4427                 .driver_name = "sha224-caam-qi2",
4428                 .hmac_name = "hmac(sha224)",
4429                 .hmac_driver_name = "hmac-sha224-caam-qi2",
4430                 .blocksize = SHA224_BLOCK_SIZE,
4431                 .template_ahash = {
4432                         .init = ahash_init,
4433                         .update = ahash_update,
4434                         .final = ahash_final,
4435                         .finup = ahash_finup,
4436                         .digest = ahash_digest,
4437                         .export = ahash_export,
4438                         .import = ahash_import,
4439                         .setkey = ahash_setkey,
4440                         .halg = {
4441                                 .digestsize = SHA224_DIGEST_SIZE,
4442                                 .statesize = sizeof(struct caam_export_state),
4443                         },
4444                 },
4445                 .alg_type = OP_ALG_ALGSEL_SHA224,
4446         }, {
4447                 .name = "sha256",
4448                 .driver_name = "sha256-caam-qi2",
4449                 .hmac_name = "hmac(sha256)",
4450                 .hmac_driver_name = "hmac-sha256-caam-qi2",
4451                 .blocksize = SHA256_BLOCK_SIZE,
4452                 .template_ahash = {
4453                         .init = ahash_init,
4454                         .update = ahash_update,
4455                         .final = ahash_final,
4456                         .finup = ahash_finup,
4457                         .digest = ahash_digest,
4458                         .export = ahash_export,
4459                         .import = ahash_import,
4460                         .setkey = ahash_setkey,
4461                         .halg = {
4462                                 .digestsize = SHA256_DIGEST_SIZE,
4463                                 .statesize = sizeof(struct caam_export_state),
4464                         },
4465                 },
4466                 .alg_type = OP_ALG_ALGSEL_SHA256,
4467         }, {
4468                 .name = "sha384",
4469                 .driver_name = "sha384-caam-qi2",
4470                 .hmac_name = "hmac(sha384)",
4471                 .hmac_driver_name = "hmac-sha384-caam-qi2",
4472                 .blocksize = SHA384_BLOCK_SIZE,
4473                 .template_ahash = {
4474                         .init = ahash_init,
4475                         .update = ahash_update,
4476                         .final = ahash_final,
4477                         .finup = ahash_finup,
4478                         .digest = ahash_digest,
4479                         .export = ahash_export,
4480                         .import = ahash_import,
4481                         .setkey = ahash_setkey,
4482                         .halg = {
4483                                 .digestsize = SHA384_DIGEST_SIZE,
4484                                 .statesize = sizeof(struct caam_export_state),
4485                         },
4486                 },
4487                 .alg_type = OP_ALG_ALGSEL_SHA384,
4488         }, {
4489                 .name = "sha512",
4490                 .driver_name = "sha512-caam-qi2",
4491                 .hmac_name = "hmac(sha512)",
4492                 .hmac_driver_name = "hmac-sha512-caam-qi2",
4493                 .blocksize = SHA512_BLOCK_SIZE,
4494                 .template_ahash = {
4495                         .init = ahash_init,
4496                         .update = ahash_update,
4497                         .final = ahash_final,
4498                         .finup = ahash_finup,
4499                         .digest = ahash_digest,
4500                         .export = ahash_export,
4501                         .import = ahash_import,
4502                         .setkey = ahash_setkey,
4503                         .halg = {
4504                                 .digestsize = SHA512_DIGEST_SIZE,
4505                                 .statesize = sizeof(struct caam_export_state),
4506                         },
4507                 },
4508                 .alg_type = OP_ALG_ALGSEL_SHA512,
4509         }, {
4510                 .name = "md5",
4511                 .driver_name = "md5-caam-qi2",
4512                 .hmac_name = "hmac(md5)",
4513                 .hmac_driver_name = "hmac-md5-caam-qi2",
4514                 .blocksize = MD5_BLOCK_WORDS * 4,
4515                 .template_ahash = {
4516                         .init = ahash_init,
4517                         .update = ahash_update,
4518                         .final = ahash_final,
4519                         .finup = ahash_finup,
4520                         .digest = ahash_digest,
4521                         .export = ahash_export,
4522                         .import = ahash_import,
4523                         .setkey = ahash_setkey,
4524                         .halg = {
4525                                 .digestsize = MD5_DIGEST_SIZE,
4526                                 .statesize = sizeof(struct caam_export_state),
4527                         },
4528                 },
4529                 .alg_type = OP_ALG_ALGSEL_MD5,
4530         }
4531 };
4532
4533 struct caam_hash_alg {
4534         struct list_head entry;
4535         struct device *dev;
4536         int alg_type;
4537         struct ahash_alg ahash_alg;
4538 };
4539
4540 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4541 {
4542         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4543         struct crypto_alg *base = tfm->__crt_alg;
4544         struct hash_alg_common *halg =
4545                  container_of(base, struct hash_alg_common, base);
4546         struct ahash_alg *alg =
4547                  container_of(halg, struct ahash_alg, halg);
4548         struct caam_hash_alg *caam_hash =
4549                  container_of(alg, struct caam_hash_alg, ahash_alg);
4550         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4551         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4552         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4553                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4554                                          HASH_MSG_LEN + 32,
4555                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4556                                          HASH_MSG_LEN + 64,
4557                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4558         dma_addr_t dma_addr;
4559         int i;
4560
4561         ctx->dev = caam_hash->dev;
4562
4563         if (alg->setkey) {
4564                 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4565                                                           ARRAY_SIZE(ctx->key),
4566                                                           DMA_TO_DEVICE,
4567                                                           DMA_ATTR_SKIP_CPU_SYNC);
4568                 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4569                         dev_err(ctx->dev, "unable to map key\n");
4570                         return -ENOMEM;
4571                 }
4572         }
4573
4574         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4575                                         DMA_BIDIRECTIONAL,
4576                                         DMA_ATTR_SKIP_CPU_SYNC);
4577         if (dma_mapping_error(ctx->dev, dma_addr)) {
4578                 dev_err(ctx->dev, "unable to map shared descriptors\n");
4579                 if (ctx->adata.key_dma)
4580                         dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4581                                                ARRAY_SIZE(ctx->key),
4582                                                DMA_TO_DEVICE,
4583                                                DMA_ATTR_SKIP_CPU_SYNC);
4584                 return -ENOMEM;
4585         }
4586
4587         for (i = 0; i < HASH_NUM_OP; i++)
4588                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4589
4590         /* copy descriptor header template value */
4591         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4592
4593         ctx->ctx_len = runninglen[(ctx->adata.algtype &
4594                                    OP_ALG_ALGSEL_SUBMASK) >>
4595                                   OP_ALG_ALGSEL_SHIFT];
4596
4597         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4598                                  sizeof(struct caam_hash_state));
4599
4600         /*
4601          * For keyed hash algorithms shared descriptors
4602          * will be created later in setkey() callback
4603          */
4604         return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
4605 }
4606
4607 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4608 {
4609         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4610
4611         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4612                                DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4613         if (ctx->adata.key_dma)
4614                 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4615                                        ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4616                                        DMA_ATTR_SKIP_CPU_SYNC);
4617 }
4618
4619 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4620         struct caam_hash_template *template, bool keyed)
4621 {
4622         struct caam_hash_alg *t_alg;
4623         struct ahash_alg *halg;
4624         struct crypto_alg *alg;
4625
4626         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4627         if (!t_alg)
4628                 return ERR_PTR(-ENOMEM);
4629
4630         t_alg->ahash_alg = template->template_ahash;
4631         halg = &t_alg->ahash_alg;
4632         alg = &halg->halg.base;
4633
4634         if (keyed) {
4635                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4636                          template->hmac_name);
4637                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4638                          template->hmac_driver_name);
4639         } else {
4640                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4641                          template->name);
4642                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4643                          template->driver_name);
4644                 t_alg->ahash_alg.setkey = NULL;
4645         }
4646         alg->cra_module = THIS_MODULE;
4647         alg->cra_init = caam_hash_cra_init;
4648         alg->cra_exit = caam_hash_cra_exit;
4649         alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
4650         alg->cra_priority = CAAM_CRA_PRIORITY;
4651         alg->cra_blocksize = template->blocksize;
4652         alg->cra_alignmask = 0;
4653         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4654
4655         t_alg->alg_type = template->alg_type;
4656         t_alg->dev = dev;
4657
4658         return t_alg;
4659 }
4660
4661 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4662 {
4663         struct dpaa2_caam_priv_per_cpu *ppriv;
4664
4665         ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4666         napi_schedule_irqoff(&ppriv->napi);
4667 }
4668
4669 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4670 {
4671         struct device *dev = priv->dev;
4672         struct dpaa2_io_notification_ctx *nctx;
4673         struct dpaa2_caam_priv_per_cpu *ppriv;
4674         int err, i = 0, cpu;
4675
4676         for_each_online_cpu(cpu) {
4677                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4678                 ppriv->priv = priv;
4679                 nctx = &ppriv->nctx;
4680                 nctx->is_cdan = 0;
4681                 nctx->id = ppriv->rsp_fqid;
4682                 nctx->desired_cpu = cpu;
4683                 nctx->cb = dpaa2_caam_fqdan_cb;
4684
4685                 /* Register notification callbacks */
4686                 ppriv->dpio = dpaa2_io_service_select(cpu);
4687                 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4688                 if (unlikely(err)) {
4689                         dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4690                         nctx->cb = NULL;
4691                         /*
4692                          * If no affine DPIO for this core, there's probably
4693                          * none available for next cores either. Signal we want
4694                          * to retry later, in case the DPIO devices weren't
4695                          * probed yet.
4696                          */
4697                         err = -EPROBE_DEFER;
4698                         goto err;
4699                 }
4700
4701                 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4702                                                      dev);
4703                 if (unlikely(!ppriv->store)) {
4704                         dev_err(dev, "dpaa2_io_store_create() failed\n");
4705                         err = -ENOMEM;
4706                         goto err;
4707                 }
4708
4709                 if (++i == priv->num_pairs)
4710                         break;
4711         }
4712
4713         return 0;
4714
4715 err:
4716         for_each_online_cpu(cpu) {
4717                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4718                 if (!ppriv->nctx.cb)
4719                         break;
4720                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4721         }
4722
4723         for_each_online_cpu(cpu) {
4724                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4725                 if (!ppriv->store)
4726                         break;
4727                 dpaa2_io_store_destroy(ppriv->store);
4728         }
4729
4730         return err;
4731 }
4732
4733 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4734 {
4735         struct dpaa2_caam_priv_per_cpu *ppriv;
4736         int i = 0, cpu;
4737
4738         for_each_online_cpu(cpu) {
4739                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4740                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4741                                             priv->dev);
4742                 dpaa2_io_store_destroy(ppriv->store);
4743
4744                 if (++i == priv->num_pairs)
4745                         return;
4746         }
4747 }
4748
4749 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4750 {
4751         struct dpseci_rx_queue_cfg rx_queue_cfg;
4752         struct device *dev = priv->dev;
4753         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4754         struct dpaa2_caam_priv_per_cpu *ppriv;
4755         int err = 0, i = 0, cpu;
4756
4757         /* Configure Rx queues */
4758         for_each_online_cpu(cpu) {
4759                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4760
4761                 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4762                                        DPSECI_QUEUE_OPT_USER_CTX;
4763                 rx_queue_cfg.order_preservation_en = 0;
4764                 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4765                 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4766                 /*
4767                  * Rx priority (WQ) doesn't really matter, since we use
4768                  * pull mode, i.e. volatile dequeues from specific FQs
4769                  */
4770                 rx_queue_cfg.dest_cfg.priority = 0;
4771                 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4772
4773                 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4774                                           &rx_queue_cfg);
4775                 if (err) {
4776                         dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4777                                 err);
4778                         return err;
4779                 }
4780
4781                 if (++i == priv->num_pairs)
4782                         break;
4783         }
4784
4785         return err;
4786 }
4787
4788 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4789 {
4790         struct device *dev = priv->dev;
4791
4792         if (!priv->cscn_mem)
4793                 return;
4794
4795         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4796         kfree(priv->cscn_mem);
4797 }
4798
4799 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4800 {
4801         struct device *dev = priv->dev;
4802         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4803         int err;
4804
4805         if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
4806                 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
4807                 if (err)
4808                         dev_err(dev, "dpseci_reset() failed\n");
4809         }
4810
4811         dpaa2_dpseci_congestion_free(priv);
4812         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4813 }
4814
4815 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4816                                   const struct dpaa2_fd *fd)
4817 {
4818         struct caam_request *req;
4819         u32 fd_err;
4820
4821         if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4822                 dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4823                 return;
4824         }
4825
4826         fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4827         if (unlikely(fd_err))
4828                 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4829
4830         /*
4831          * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4832          * in FD[ERR] or FD[FRC].
4833          */
4834         req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4835         dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4836                          DMA_BIDIRECTIONAL);
4837         req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4838 }
4839
4840 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4841 {
4842         int err;
4843
4844         /* Retry while portal is busy */
4845         do {
4846                 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4847                                                ppriv->store);
4848         } while (err == -EBUSY);
4849
4850         if (unlikely(err))
4851                 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4852
4853         return err;
4854 }
4855
4856 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4857 {
4858         struct dpaa2_dq *dq;
4859         int cleaned = 0, is_last;
4860
4861         do {
4862                 dq = dpaa2_io_store_next(ppriv->store, &is_last);
4863                 if (unlikely(!dq)) {
4864                         if (unlikely(!is_last)) {
4865                                 dev_dbg(ppriv->priv->dev,
4866                                         "FQ %d returned no valid frames\n",
4867                                         ppriv->rsp_fqid);
4868                                 /*
4869                                  * MUST retry until we get some sort of
4870                                  * valid response token (be it "empty dequeue"
4871                                  * or a valid frame).
4872                                  */
4873                                 continue;
4874                         }
4875                         break;
4876                 }
4877
4878                 /* Process FD */
4879                 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4880                 cleaned++;
4881         } while (!is_last);
4882
4883         return cleaned;
4884 }
4885
4886 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4887 {
4888         struct dpaa2_caam_priv_per_cpu *ppriv;
4889         struct dpaa2_caam_priv *priv;
4890         int err, cleaned = 0, store_cleaned;
4891
4892         ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4893         priv = ppriv->priv;
4894
4895         if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4896                 return 0;
4897
4898         do {
4899                 store_cleaned = dpaa2_caam_store_consume(ppriv);
4900                 cleaned += store_cleaned;
4901
4902                 if (store_cleaned == 0 ||
4903                     cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4904                         break;
4905
4906                 /* Try to dequeue some more */
4907                 err = dpaa2_caam_pull_fq(ppriv);
4908                 if (unlikely(err))
4909                         break;
4910         } while (1);
4911
4912         if (cleaned < budget) {
4913                 napi_complete_done(napi, cleaned);
4914                 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4915                 if (unlikely(err))
4916                         dev_err(priv->dev, "Notification rearm failed: %d\n",
4917                                 err);
4918         }
4919
4920         return cleaned;
4921 }
4922
4923 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4924                                          u16 token)
4925 {
4926         struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4927         struct device *dev = priv->dev;
4928         int err;
4929
4930         /*
4931          * Congestion group feature supported starting with DPSECI API v5.1
4932          * and only when object has been created with this capability.
4933          */
4934         if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4935             !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4936                 return 0;
4937
4938         priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
4939                                  GFP_KERNEL | GFP_DMA);
4940         if (!priv->cscn_mem)
4941                 return -ENOMEM;
4942
4943         priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
4944         priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
4945                                         DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4946         if (dma_mapping_error(dev, priv->cscn_dma)) {
4947                 dev_err(dev, "Error mapping CSCN memory area\n");
4948                 err = -ENOMEM;
4949                 goto err_dma_map;
4950         }
4951
4952         cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4953         cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4954         cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4955         cong_notif_cfg.message_ctx = (uintptr_t)priv;
4956         cong_notif_cfg.message_iova = priv->cscn_dma;
4957         cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4958                                         DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4959                                         DPSECI_CGN_MODE_COHERENT_WRITE;
4960
4961         err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4962                                                  &cong_notif_cfg);
4963         if (err) {
4964                 dev_err(dev, "dpseci_set_congestion_notification failed\n");
4965                 goto err_set_cong;
4966         }
4967
4968         return 0;
4969
4970 err_set_cong:
4971         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4972 err_dma_map:
4973         kfree(priv->cscn_mem);
4974
4975         return err;
4976 }
4977
4978 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4979 {
4980         struct device *dev = &ls_dev->dev;
4981         struct dpaa2_caam_priv *priv;
4982         struct dpaa2_caam_priv_per_cpu *ppriv;
4983         int err, cpu;
4984         u8 i;
4985
4986         priv = dev_get_drvdata(dev);
4987
4988         priv->dev = dev;
4989         priv->dpsec_id = ls_dev->obj_desc.id;
4990
4991         /* Get a handle for the DPSECI this interface is associate with */
4992         err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
4993         if (err) {
4994                 dev_err(dev, "dpseci_open() failed: %d\n", err);
4995                 goto err_open;
4996         }
4997
4998         err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
4999                                      &priv->minor_ver);
5000         if (err) {
5001                 dev_err(dev, "dpseci_get_api_version() failed\n");
5002                 goto err_get_vers;
5003         }
5004
5005         dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
5006
5007         if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
5008                 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
5009                 if (err) {
5010                         dev_err(dev, "dpseci_reset() failed\n");
5011                         goto err_get_vers;
5012                 }
5013         }
5014
5015         err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
5016                                     &priv->dpseci_attr);
5017         if (err) {
5018                 dev_err(dev, "dpseci_get_attributes() failed\n");
5019                 goto err_get_vers;
5020         }
5021
5022         err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
5023                                   &priv->sec_attr);
5024         if (err) {
5025                 dev_err(dev, "dpseci_get_sec_attr() failed\n");
5026                 goto err_get_vers;
5027         }
5028
5029         err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
5030         if (err) {
5031                 dev_err(dev, "setup_congestion() failed\n");
5032                 goto err_get_vers;
5033         }
5034
5035         priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
5036                               priv->dpseci_attr.num_tx_queues);
5037         if (priv->num_pairs > num_online_cpus()) {
5038                 dev_warn(dev, "%d queues won't be used\n",
5039                          priv->num_pairs - num_online_cpus());
5040                 priv->num_pairs = num_online_cpus();
5041         }
5042
5043         for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
5044                 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5045                                           &priv->rx_queue_attr[i]);
5046                 if (err) {
5047                         dev_err(dev, "dpseci_get_rx_queue() failed\n");
5048                         goto err_get_rx_queue;
5049                 }
5050         }
5051
5052         for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
5053                 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5054                                           &priv->tx_queue_attr[i]);
5055                 if (err) {
5056                         dev_err(dev, "dpseci_get_tx_queue() failed\n");
5057                         goto err_get_rx_queue;
5058                 }
5059         }
5060
5061         i = 0;
5062         for_each_online_cpu(cpu) {
5063                 u8 j;
5064
5065                 j = i % priv->num_pairs;
5066
5067                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
5068                 ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
5069
5070                 /*
5071                  * Allow all cores to enqueue, while only some of them
5072                  * will take part in dequeuing.
5073                  */
5074                 if (++i > priv->num_pairs)
5075                         continue;
5076
5077                 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
5078                 ppriv->prio = j;
5079
5080                 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
5081                         priv->rx_queue_attr[j].fqid,
5082                         priv->tx_queue_attr[j].fqid);
5083
5084                 ppriv->net_dev.dev = *dev;
5085                 INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
5086                 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll,
5087                                DPAA2_CAAM_NAPI_WEIGHT);
5088         }
5089
5090         return 0;
5091
5092 err_get_rx_queue:
5093         dpaa2_dpseci_congestion_free(priv);
5094 err_get_vers:
5095         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
5096 err_open:
5097         return err;
5098 }
5099
5100 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
5101 {
5102         struct device *dev = priv->dev;
5103         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5104         struct dpaa2_caam_priv_per_cpu *ppriv;
5105         int i;
5106
5107         for (i = 0; i < priv->num_pairs; i++) {
5108                 ppriv = per_cpu_ptr(priv->ppriv, i);
5109                 napi_enable(&ppriv->napi);
5110         }
5111
5112         return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
5113 }
5114
5115 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
5116 {
5117         struct device *dev = priv->dev;
5118         struct dpaa2_caam_priv_per_cpu *ppriv;
5119         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5120         int i, err = 0, enabled;
5121
5122         err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5123         if (err) {
5124                 dev_err(dev, "dpseci_disable() failed\n");
5125                 return err;
5126         }
5127
5128         err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5129         if (err) {
5130                 dev_err(dev, "dpseci_is_enabled() failed\n");
5131                 return err;
5132         }
5133
5134         dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5135
5136         for (i = 0; i < priv->num_pairs; i++) {
5137                 ppriv = per_cpu_ptr(priv->ppriv, i);
5138                 napi_disable(&ppriv->napi);
5139                 netif_napi_del(&ppriv->napi);
5140         }
5141
5142         return 0;
5143 }
5144
5145 static struct list_head hash_list;
5146
5147 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5148 {
5149         struct device *dev;
5150         struct dpaa2_caam_priv *priv;
5151         int i, err = 0;
5152         bool registered = false;
5153
5154         /*
5155          * There is no way to get CAAM endianness - there is no direct register
5156          * space access and MC f/w does not provide this attribute.
5157          * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5158          * property.
5159          */
5160         caam_little_end = true;
5161
5162         caam_imx = false;
5163
5164         dev = &dpseci_dev->dev;
5165
5166         priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5167         if (!priv)
5168                 return -ENOMEM;
5169
5170         dev_set_drvdata(dev, priv);
5171
5172         priv->domain = iommu_get_domain_for_dev(dev);
5173
5174         qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5175                                      0, SLAB_CACHE_DMA, NULL);
5176         if (!qi_cache) {
5177                 dev_err(dev, "Can't allocate SEC cache\n");
5178                 return -ENOMEM;
5179         }
5180
5181         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5182         if (err) {
5183                 dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5184                 goto err_dma_mask;
5185         }
5186
5187         /* Obtain a MC portal */
5188         err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5189         if (err) {
5190                 if (err == -ENXIO)
5191                         err = -EPROBE_DEFER;
5192                 else
5193                         dev_err(dev, "MC portal allocation failed\n");
5194
5195                 goto err_dma_mask;
5196         }
5197
5198         priv->ppriv = alloc_percpu(*priv->ppriv);
5199         if (!priv->ppriv) {
5200                 dev_err(dev, "alloc_percpu() failed\n");
5201                 err = -ENOMEM;
5202                 goto err_alloc_ppriv;
5203         }
5204
5205         /* DPSECI initialization */
5206         err = dpaa2_dpseci_setup(dpseci_dev);
5207         if (err) {
5208                 dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5209                 goto err_dpseci_setup;
5210         }
5211
5212         /* DPIO */
5213         err = dpaa2_dpseci_dpio_setup(priv);
5214         if (err) {
5215                 dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n");
5216                 goto err_dpio_setup;
5217         }
5218
5219         /* DPSECI binding to DPIO */
5220         err = dpaa2_dpseci_bind(priv);
5221         if (err) {
5222                 dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5223                 goto err_bind;
5224         }
5225
5226         /* DPSECI enable */
5227         err = dpaa2_dpseci_enable(priv);
5228         if (err) {
5229                 dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5230                 goto err_bind;
5231         }
5232
5233         dpaa2_dpseci_debugfs_init(priv);
5234
5235         /* register crypto algorithms the device supports */
5236         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5237                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5238                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5239
5240                 /* Skip DES algorithms if not supported by device */
5241                 if (!priv->sec_attr.des_acc_num &&
5242                     (alg_sel == OP_ALG_ALGSEL_3DES ||
5243                      alg_sel == OP_ALG_ALGSEL_DES))
5244                         continue;
5245
5246                 /* Skip AES algorithms if not supported by device */
5247                 if (!priv->sec_attr.aes_acc_num &&
5248                     alg_sel == OP_ALG_ALGSEL_AES)
5249                         continue;
5250
5251                 /* Skip CHACHA20 algorithms if not supported by device */
5252                 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5253                     !priv->sec_attr.ccha_acc_num)
5254                         continue;
5255
5256                 t_alg->caam.dev = dev;
5257                 caam_skcipher_alg_init(t_alg);
5258
5259                 err = crypto_register_skcipher(&t_alg->skcipher);
5260                 if (err) {
5261                         dev_warn(dev, "%s alg registration failed: %d\n",
5262                                  t_alg->skcipher.base.cra_driver_name, err);
5263                         continue;
5264                 }
5265
5266                 t_alg->registered = true;
5267                 registered = true;
5268         }
5269
5270         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5271                 struct caam_aead_alg *t_alg = driver_aeads + i;
5272                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5273                                  OP_ALG_ALGSEL_MASK;
5274                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5275                                  OP_ALG_ALGSEL_MASK;
5276
5277                 /* Skip DES algorithms if not supported by device */
5278                 if (!priv->sec_attr.des_acc_num &&
5279                     (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5280                      c1_alg_sel == OP_ALG_ALGSEL_DES))
5281                         continue;
5282
5283                 /* Skip AES algorithms if not supported by device */
5284                 if (!priv->sec_attr.aes_acc_num &&
5285                     c1_alg_sel == OP_ALG_ALGSEL_AES)
5286                         continue;
5287
5288                 /* Skip CHACHA20 algorithms if not supported by device */
5289                 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5290                     !priv->sec_attr.ccha_acc_num)
5291                         continue;
5292
5293                 /* Skip POLY1305 algorithms if not supported by device */
5294                 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5295                     !priv->sec_attr.ptha_acc_num)
5296                         continue;
5297
5298                 /*
5299                  * Skip algorithms requiring message digests
5300                  * if MD not supported by device.
5301                  */
5302                 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5303                     !priv->sec_attr.md_acc_num)
5304                         continue;
5305
5306                 t_alg->caam.dev = dev;
5307                 caam_aead_alg_init(t_alg);
5308
5309                 err = crypto_register_aead(&t_alg->aead);
5310                 if (err) {
5311                         dev_warn(dev, "%s alg registration failed: %d\n",
5312                                  t_alg->aead.base.cra_driver_name, err);
5313                         continue;
5314                 }
5315
5316                 t_alg->registered = true;
5317                 registered = true;
5318         }
5319         if (registered)
5320                 dev_info(dev, "algorithms registered in /proc/crypto\n");
5321
5322         /* register hash algorithms the device supports */
5323         INIT_LIST_HEAD(&hash_list);
5324
5325         /*
5326          * Skip registration of any hashing algorithms if MD block
5327          * is not present.
5328          */
5329         if (!priv->sec_attr.md_acc_num)
5330                 return 0;
5331
5332         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5333                 struct caam_hash_alg *t_alg;
5334                 struct caam_hash_template *alg = driver_hash + i;
5335
5336                 /* register hmac version */
5337                 t_alg = caam_hash_alloc(dev, alg, true);
5338                 if (IS_ERR(t_alg)) {
5339                         err = PTR_ERR(t_alg);
5340                         dev_warn(dev, "%s hash alg allocation failed: %d\n",
5341                                  alg->hmac_driver_name, err);
5342                         continue;
5343                 }
5344
5345                 err = crypto_register_ahash(&t_alg->ahash_alg);
5346                 if (err) {
5347                         dev_warn(dev, "%s alg registration failed: %d\n",
5348                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5349                                  err);
5350                         kfree(t_alg);
5351                 } else {
5352                         list_add_tail(&t_alg->entry, &hash_list);
5353                 }
5354
5355                 /* register unkeyed version */
5356                 t_alg = caam_hash_alloc(dev, alg, false);
5357                 if (IS_ERR(t_alg)) {
5358                         err = PTR_ERR(t_alg);
5359                         dev_warn(dev, "%s alg allocation failed: %d\n",
5360                                  alg->driver_name, err);
5361                         continue;
5362                 }
5363
5364                 err = crypto_register_ahash(&t_alg->ahash_alg);
5365                 if (err) {
5366                         dev_warn(dev, "%s alg registration failed: %d\n",
5367                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5368                                  err);
5369                         kfree(t_alg);
5370                 } else {
5371                         list_add_tail(&t_alg->entry, &hash_list);
5372                 }
5373         }
5374         if (!list_empty(&hash_list))
5375                 dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5376
5377         return err;
5378
5379 err_bind:
5380         dpaa2_dpseci_dpio_free(priv);
5381 err_dpio_setup:
5382         dpaa2_dpseci_free(priv);
5383 err_dpseci_setup:
5384         free_percpu(priv->ppriv);
5385 err_alloc_ppriv:
5386         fsl_mc_portal_free(priv->mc_io);
5387 err_dma_mask:
5388         kmem_cache_destroy(qi_cache);
5389
5390         return err;
5391 }
5392
5393 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5394 {
5395         struct device *dev;
5396         struct dpaa2_caam_priv *priv;
5397         int i;
5398
5399         dev = &ls_dev->dev;
5400         priv = dev_get_drvdata(dev);
5401
5402         dpaa2_dpseci_debugfs_exit(priv);
5403
5404         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5405                 struct caam_aead_alg *t_alg = driver_aeads + i;
5406
5407                 if (t_alg->registered)
5408                         crypto_unregister_aead(&t_alg->aead);
5409         }
5410
5411         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5412                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5413
5414                 if (t_alg->registered)
5415                         crypto_unregister_skcipher(&t_alg->skcipher);
5416         }
5417
5418         if (hash_list.next) {
5419                 struct caam_hash_alg *t_hash_alg, *p;
5420
5421                 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5422                         crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5423                         list_del(&t_hash_alg->entry);
5424                         kfree(t_hash_alg);
5425                 }
5426         }
5427
5428         dpaa2_dpseci_disable(priv);
5429         dpaa2_dpseci_dpio_free(priv);
5430         dpaa2_dpseci_free(priv);
5431         free_percpu(priv->ppriv);
5432         fsl_mc_portal_free(priv->mc_io);
5433         kmem_cache_destroy(qi_cache);
5434
5435         return 0;
5436 }
5437
5438 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5439 {
5440         struct dpaa2_fd fd;
5441         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5442         struct dpaa2_caam_priv_per_cpu *ppriv;
5443         int err = 0, i;
5444
5445         if (IS_ERR(req))
5446                 return PTR_ERR(req);
5447
5448         if (priv->cscn_mem) {
5449                 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5450                                         DPAA2_CSCN_SIZE,
5451                                         DMA_FROM_DEVICE);
5452                 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
5453                         dev_dbg_ratelimited(dev, "Dropping request\n");
5454                         return -EBUSY;
5455                 }
5456         }
5457
5458         dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5459
5460         req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5461                                          DMA_BIDIRECTIONAL);
5462         if (dma_mapping_error(dev, req->fd_flt_dma)) {
5463                 dev_err(dev, "DMA mapping error for QI enqueue request\n");
5464                 goto err_out;
5465         }
5466
5467         memset(&fd, 0, sizeof(fd));
5468         dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5469         dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5470         dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5471         dpaa2_fd_set_flc(&fd, req->flc_dma);
5472
5473         ppriv = raw_cpu_ptr(priv->ppriv);
5474         for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5475                 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5476                                                   &fd);
5477                 if (err != -EBUSY)
5478                         break;
5479
5480                 cpu_relax();
5481         }
5482
5483         if (unlikely(err)) {
5484                 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5485                 goto err_out;
5486         }
5487
5488         return -EINPROGRESS;
5489
5490 err_out:
5491         dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5492                          DMA_BIDIRECTIONAL);
5493         return -EIO;
5494 }
5495 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5496
5497 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5498         {
5499                 .vendor = FSL_MC_VENDOR_FREESCALE,
5500                 .obj_type = "dpseci",
5501         },
5502         { .vendor = 0x0 }
5503 };
5504 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table);
5505
5506 static struct fsl_mc_driver dpaa2_caam_driver = {
5507         .driver = {
5508                 .name           = KBUILD_MODNAME,
5509                 .owner          = THIS_MODULE,
5510         },
5511         .probe          = dpaa2_caam_probe,
5512         .remove         = dpaa2_caam_remove,
5513         .match_id_table = dpaa2_caam_match_id_table
5514 };
5515
5516 MODULE_LICENSE("Dual BSD/GPL");
5517 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5518 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5519
5520 module_fsl_mc_driver(dpaa2_caam_driver);