drivers/crypto/caam/caamhash.c

   1 /*
   2  * caam - Freescale FSL CAAM support for ahash functions of crypto API
   3  *
   4  * Copyright 2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on caamalg.c crypto API driver.
   7  *
   8  * relationship of digest job descriptor or first job descriptor after init to
   9  * shared descriptors:
  10  *
  11  * ---------------                     ---------------
  12  * | JobDesc #1  |-------------------->|  ShareDesc  |
  13  * | *(packet 1) |                     |  (hashKey)  |
  14  * ---------------                     | (operation) |
  15  *                                     ---------------
  16  *
  17  * relationship of subsequent job descriptors to shared descriptors:
  18  *
  19  * ---------------                     ---------------
  20  * | JobDesc #2  |-------------------->|  ShareDesc  |
  21  * | *(packet 2) |      |------------->|  (hashKey)  |
  22  * ---------------      |    |-------->| (operation) |
  23  *       .              |    |         | (load ctx2) |
  24  *       .              |    |         ---------------
  25  * ---------------      |    |
  26  * | JobDesc #3  |------|    |
  27  * | *(packet 3) |           |
  28  * ---------------           |
  29  *       .                   |
  30  *       .                   |
  31  * ---------------           |
  32  * | JobDesc #4  |------------
  33  * | *(packet 4) |
  34  * ---------------
  35  *
  36  * The SharedDesc never changes for a connection unless rekeyed, but
  37  * each packet will likely be in a different place. So all we need
  38  * to know to process the packet is where the input is, where the
  39  * output goes, and what context we want to process with. Context is
  40  * in the SharedDesc, packet references in the JobDesc.
  41  *
  42  * So, a job desc looks like:
  43  *
  44  * ---------------------
  45  * | Header            |
  46  * | ShareDesc Pointer |
  47  * | SEQ_OUT_PTR       |
  48  * | (output buffer)   |
  49  * | (output length)   |
  50  * | SEQ_IN_PTR        |
  51  * | (input buffer)    |
  52  * | (input length)    |
  53  * ---------------------
  54  */
  55
  56 #include "compat.h"
  57
  58 #include "regs.h"
  59 #include "intern.h"
  60 #include "desc_constr.h"
  61 #include "jr.h"
  62 #include "error.h"
  63 #include "sg_sw_sec4.h"
  64 #include "key_gen.h"
  65
  66 #define CAAM_CRA_PRIORITY               3000
  67
  68 /* max hash key is max split key size */
  69 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
  70
  71 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
  72 #define CAAM_MAX_HASH_DIGEST_SIZE       SHA512_DIGEST_SIZE
  73
  74 /* length of descriptors text */
  75 #define DESC_AHASH_BASE                 (3 * CAAM_CMD_SZ)
  76 #define DESC_AHASH_UPDATE_LEN           (6 * CAAM_CMD_SZ)
  77 #define DESC_AHASH_UPDATE_FIRST_LEN     (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
  78 #define DESC_AHASH_FINAL_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
  79 #define DESC_AHASH_FINUP_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
  80 #define DESC_AHASH_DIGEST_LEN           (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
  81
  82 #define DESC_HASH_MAX_USED_BYTES        (DESC_AHASH_FINAL_LEN + \
  83                                          CAAM_MAX_HASH_KEY_SIZE)
  84 #define DESC_HASH_MAX_USED_LEN          (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
  85
  86 /* caam context sizes for hashes: running digest + 8 */
  87 #define HASH_MSG_LEN                    8
  88 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
  89
  90 #ifdef DEBUG
  91 /* for print_hex_dumps with line references */
  92 #define debug(format, arg...) printk(format, arg)
  93 #else
  94 #define debug(format, arg...)
  95 #endif
  96
  97
  98 static struct list_head hash_list;
  99
 100 /* ahash per-session context */
 101 struct caam_hash_ctx {
 102         u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 103         u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 104         u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 105         u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 106         dma_addr_t sh_desc_update_dma ____cacheline_aligned;
 107         dma_addr_t sh_desc_update_first_dma;
 108         dma_addr_t sh_desc_fin_dma;
 109         dma_addr_t sh_desc_digest_dma;
 110         struct device *jrdev;
 111         u8 key[CAAM_MAX_HASH_KEY_SIZE];
 112         int ctx_len;
 113         struct alginfo adata;
 114 };
 115
 116 /* ahash state */
 117 struct caam_hash_state {
 118         dma_addr_t buf_dma;
 119         dma_addr_t ctx_dma;
 120         u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
 121         int buflen_0;
 122         u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
 123         int buflen_1;
 124         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
 125         int (*update)(struct ahash_request *req);
 126         int (*final)(struct ahash_request *req);
 127         int (*finup)(struct ahash_request *req);
 128         int current_buf;
 129 };
 130
 131 struct caam_export_state {
 132         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
 133         u8 caam_ctx[MAX_CTX_LEN];
 134         int buflen;
 135         int (*update)(struct ahash_request *req);
 136         int (*final)(struct ahash_request *req);
 137         int (*finup)(struct ahash_request *req);
 138 };
 139
 140 static inline void switch_buf(struct caam_hash_state *state)
 141 {
 142         state->current_buf ^= 1;
 143 }
 144
 145 static inline u8 *current_buf(struct caam_hash_state *state)
 146 {
 147         return state->current_buf ? state->buf_1 : state->buf_0;
 148 }
 149
 150 static inline u8 *alt_buf(struct caam_hash_state *state)
 151 {
 152         return state->current_buf ? state->buf_0 : state->buf_1;
 153 }
 154
 155 static inline int *current_buflen(struct caam_hash_state *state)
 156 {
 157         return state->current_buf ? &state->buflen_1 : &state->buflen_0;
 158 }
 159
 160 static inline int *alt_buflen(struct caam_hash_state *state)
 161 {
 162         return state->current_buf ? &state->buflen_0 : &state->buflen_1;
 163 }
 164
 165 /* Common job descriptor seq in/out ptr routines */
 166
 167 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
 168 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
 169                                       struct caam_hash_state *state,
 170                                       int ctx_len)
 171 {
 172         state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
 173                                         ctx_len, DMA_FROM_DEVICE);
 174         if (dma_mapping_error(jrdev, state->ctx_dma)) {
 175                 dev_err(jrdev, "unable to map ctx\n");
 176                 state->ctx_dma = 0;
 177                 return -ENOMEM;
 178         }
 179
 180         append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
 181
 182         return 0;
 183 }
 184
 185 /* Map req->result, and append seq_out_ptr command that points to it */
 186 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev,
 187                                                 u8 *result, int digestsize)
 188 {
 189         dma_addr_t dst_dma;
 190
 191         dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE);
 192         append_seq_out_ptr(desc, dst_dma, digestsize, 0);
 193
 194         return dst_dma;
 195 }
 196
 197 /* Map current buffer in state (if length > 0) and put it in link table */
 198 static inline int buf_map_to_sec4_sg(struct device *jrdev,
 199                                      struct sec4_sg_entry *sec4_sg,
 200                                      struct caam_hash_state *state)
 201 {
 202         int buflen = *current_buflen(state);
 203
 204         if (!buflen)
 205                 return 0;
 206
 207         state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen,
 208                                         DMA_TO_DEVICE);
 209         if (dma_mapping_error(jrdev, state->buf_dma)) {
 210                 dev_err(jrdev, "unable to map buf\n");
 211                 state->buf_dma = 0;
 212                 return -ENOMEM;
 213         }
 214
 215         dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0);
 216
 217         return 0;
 218 }
 219
 220 /* Map state->caam_ctx, and add it to link table */
 221 static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev,
 222                                      struct caam_hash_state *state, int ctx_len,
 223                                      struct sec4_sg_entry *sec4_sg, u32 flag)
 224 {
 225         state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
 226         if (dma_mapping_error(jrdev, state->ctx_dma)) {
 227                 dev_err(jrdev, "unable to map ctx\n");
 228                 state->ctx_dma = 0;
 229                 return -ENOMEM;
 230         }
 231
 232         dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
 233
 234         return 0;
 235 }
 236
 237 /*
 238  * For ahash update, final and finup (import_ctx = true)
 239  *     import context, read and write to seqout
 240  * For ahash firsts and digest (import_ctx = false)
 241  *     read and write to seqout
 242  */
 243 static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize,
 244                                      struct caam_hash_ctx *ctx, bool import_ctx)
 245 {
 246         u32 op = ctx->adata.algtype;
 247         u32 *skip_key_load;
 248
 249         init_sh_desc(desc, HDR_SHARE_SERIAL);
 250
 251         /* Append key if it has been set; ahash update excluded */
 252         if ((state != OP_ALG_AS_UPDATE) && (ctx->adata.keylen)) {
 253                 /* Skip key loading if already shared */
 254                 skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 255                                             JUMP_COND_SHRD);
 256
 257                 append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad,
 258                                   ctx->adata.keylen, CLASS_2 |
 259                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 260
 261                 set_jump_tgt_here(desc, skip_key_load);
 262
 263                 op |= OP_ALG_AAI_HMAC_PRECOMP;
 264         }
 265
 266         /* If needed, import context from software */
 267         if (import_ctx)
 268                 append_seq_load(desc, ctx->ctx_len, LDST_CLASS_2_CCB |
 269                                 LDST_SRCDST_BYTE_CONTEXT);
 270
 271         /* Class 2 operation */
 272         append_operation(desc, op | state | OP_ALG_ENCRYPT);
 273
 274         /*
 275          * Load from buf and/or src and write to req->result or state->context
 276          * Calculate remaining bytes to read
 277          */
 278         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 279         /* Read remaining bytes */
 280         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
 281                              FIFOLD_TYPE_MSG | KEY_VLF);
 282         /* Store class2 context bytes */
 283         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
 284                          LDST_SRCDST_BYTE_CONTEXT);
 285 }
 286
 287 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 288 {
 289         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 290         int digestsize = crypto_ahash_digestsize(ahash);
 291         struct device *jrdev = ctx->jrdev;
 292         u32 *desc;
 293
 294         /* ahash_update shared descriptor */
 295         desc = ctx->sh_desc_update;
 296         ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true);
 297         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
 298                                    desc_bytes(desc), DMA_TO_DEVICE);
 299 #ifdef DEBUG
 300         print_hex_dump(KERN_ERR,
 301                        "ahash update shdesc@"__stringify(__LINE__)": ",
 302                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 303 #endif
 304
 305         /* ahash_update_first shared descriptor */
 306         desc = ctx->sh_desc_update_first;
 307         ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false);
 308         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
 309                                    desc_bytes(desc), DMA_TO_DEVICE);
 310 #ifdef DEBUG
 311         print_hex_dump(KERN_ERR,
 312                        "ahash update first shdesc@"__stringify(__LINE__)": ",
 313                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 314 #endif
 315
 316         /* ahash_final shared descriptor */
 317         desc = ctx->sh_desc_fin;
 318         ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true);
 319         dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
 320                                    desc_bytes(desc), DMA_TO_DEVICE);
 321 #ifdef DEBUG
 322         print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
 323                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 324                        desc_bytes(desc), 1);
 325 #endif
 326
 327         /* ahash_digest shared descriptor */
 328         desc = ctx->sh_desc_digest;
 329         ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false);
 330         dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
 331                                    desc_bytes(desc), DMA_TO_DEVICE);
 332 #ifdef DEBUG
 333         print_hex_dump(KERN_ERR,
 334                        "ahash digest shdesc@"__stringify(__LINE__)": ",
 335                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 336                        desc_bytes(desc), 1);
 337 #endif
 338
 339         return 0;
 340 }
 341
 342 /* Digest hash size if it is too large */
 343 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 344                            u32 *keylen, u8 *key_out, u32 digestsize)
 345 {
 346         struct device *jrdev = ctx->jrdev;
 347         u32 *desc;
 348         struct split_key_result result;
 349         dma_addr_t src_dma, dst_dma;
 350         int ret;
 351
 352         desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
 353         if (!desc) {
 354                 dev_err(jrdev, "unable to allocate key input memory\n");
 355                 return -ENOMEM;
 356         }
 357
 358         init_job_desc(desc, 0);
 359
 360         src_dma = dma_map_single(jrdev, (void *)key_in, *keylen,
 361                                  DMA_TO_DEVICE);
 362         if (dma_mapping_error(jrdev, src_dma)) {
 363                 dev_err(jrdev, "unable to map key input memory\n");
 364                 kfree(desc);
 365                 return -ENOMEM;
 366         }
 367         dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize,
 368                                  DMA_FROM_DEVICE);
 369         if (dma_mapping_error(jrdev, dst_dma)) {
 370                 dev_err(jrdev, "unable to map key output memory\n");
 371                 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
 372                 kfree(desc);
 373                 return -ENOMEM;
 374         }
 375
 376         /* Job descriptor to perform unkeyed hash on key_in */
 377         append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
 378                          OP_ALG_AS_INITFINAL);
 379         append_seq_in_ptr(desc, src_dma, *keylen, 0);
 380         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
 381                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
 382         append_seq_out_ptr(desc, dst_dma, digestsize, 0);
 383         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
 384                          LDST_SRCDST_BYTE_CONTEXT);
 385
 386 #ifdef DEBUG
 387         print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
 388                        DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
 389         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 390                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 391 #endif
 392
 393         result.err = 0;
 394         init_completion(&result.completion);
 395
 396         ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
 397         if (!ret) {
 398                 /* in progress */
 399                 wait_for_completion_interruptible(&result.completion);
 400                 ret = result.err;
 401 #ifdef DEBUG
 402                 print_hex_dump(KERN_ERR,
 403                                "digested key@"__stringify(__LINE__)": ",
 404                                DUMP_PREFIX_ADDRESS, 16, 4, key_in,
 405                                digestsize, 1);
 406 #endif
 407         }
 408         dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
 409         dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE);
 410
 411         *keylen = digestsize;
 412
 413         kfree(desc);
 414
 415         return ret;
 416 }
 417
 418 static int ahash_setkey(struct crypto_ahash *ahash,
 419                         const u8 *key, unsigned int keylen)
 420 {
 421         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 422         int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
 423         int digestsize = crypto_ahash_digestsize(ahash);
 424         int ret;
 425         u8 *hashed_key = NULL;
 426
 427 #ifdef DEBUG
 428         printk(KERN_ERR "keylen %d\n", keylen);
 429 #endif
 430
 431         if (keylen > blocksize) {
 432                 hashed_key = kmalloc_array(digestsize,
 433                                            sizeof(*hashed_key),
 434                                            GFP_KERNEL | GFP_DMA);
 435                 if (!hashed_key)
 436                         return -ENOMEM;
 437                 ret = hash_digest_key(ctx, key, &keylen, hashed_key,
 438                                       digestsize);
 439                 if (ret)
 440                         goto bad_free_key;
 441                 key = hashed_key;
 442         }
 443
 444         ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen,
 445                             CAAM_MAX_HASH_KEY_SIZE);
 446         if (ret)
 447                 goto bad_free_key;
 448
 449 #ifdef DEBUG
 450         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 451                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 452                        ctx->adata.keylen_pad, 1);
 453 #endif
 454
 455         kfree(hashed_key);
 456         return ahash_set_sh_desc(ahash);
 457  bad_free_key:
 458         kfree(hashed_key);
 459         crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
 460         return -EINVAL;
 461 }
 462
 463 /*
 464  * ahash_edesc - s/w-extended ahash descriptor
 465  * @dst_dma: physical mapped address of req->result
 466  * @sec4_sg_dma: physical mapped address of h/w link table
 467  * @src_nents: number of segments in input scatterlist
 468  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 469  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 470  * @sec4_sg: h/w link table
 471  */
 472 struct ahash_edesc {
 473         dma_addr_t dst_dma;
 474         dma_addr_t sec4_sg_dma;
 475         int src_nents;
 476         int sec4_sg_bytes;
 477         u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
 478         struct sec4_sg_entry sec4_sg[0];
 479 };
 480
 481 static inline void ahash_unmap(struct device *dev,
 482                         struct ahash_edesc *edesc,
 483                         struct ahash_request *req, int dst_len)
 484 {
 485         struct caam_hash_state *state = ahash_request_ctx(req);
 486
 487         if (edesc->src_nents)
 488                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
 489         if (edesc->dst_dma)
 490                 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
 491
 492         if (edesc->sec4_sg_bytes)
 493                 dma_unmap_single(dev, edesc->sec4_sg_dma,
 494                                  edesc->sec4_sg_bytes, DMA_TO_DEVICE);
 495
 496         if (state->buf_dma) {
 497                 dma_unmap_single(dev, state->buf_dma, *current_buflen(state),
 498                                  DMA_TO_DEVICE);
 499                 state->buf_dma = 0;
 500         }
 501 }
 502
 503 static inline void ahash_unmap_ctx(struct device *dev,
 504                         struct ahash_edesc *edesc,
 505                         struct ahash_request *req, int dst_len, u32 flag)
 506 {
 507         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 508         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 509         struct caam_hash_state *state = ahash_request_ctx(req);
 510
 511         if (state->ctx_dma) {
 512                 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
 513                 state->ctx_dma = 0;
 514         }
 515         ahash_unmap(dev, edesc, req, dst_len);
 516 }
 517
 518 static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
 519                        void *context)
 520 {
 521         struct ahash_request *req = context;
 522         struct ahash_edesc *edesc;
 523         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 524         int digestsize = crypto_ahash_digestsize(ahash);
 525 #ifdef DEBUG
 526         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 527         struct caam_hash_state *state = ahash_request_ctx(req);
 528
 529         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 530 #endif
 531
 532         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 533         if (err)
 534                 caam_jr_strstatus(jrdev, err);
 535
 536         ahash_unmap(jrdev, edesc, req, digestsize);
 537         kfree(edesc);
 538
 539 #ifdef DEBUG
 540         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 541                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 542                        ctx->ctx_len, 1);
 543         if (req->result)
 544                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 545                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 546                                digestsize, 1);
 547 #endif
 548
 549         req->base.complete(&req->base, err);
 550 }
 551
 552 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
 553                             void *context)
 554 {
 555         struct ahash_request *req = context;
 556         struct ahash_edesc *edesc;
 557         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 558         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 559         struct caam_hash_state *state = ahash_request_ctx(req);
 560 #ifdef DEBUG
 561         int digestsize = crypto_ahash_digestsize(ahash);
 562
 563         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 564 #endif
 565
 566         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 567         if (err)
 568                 caam_jr_strstatus(jrdev, err);
 569
 570         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
 571         switch_buf(state);
 572         kfree(edesc);
 573
 574 #ifdef DEBUG
 575         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 576                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 577                        ctx->ctx_len, 1);
 578         if (req->result)
 579                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 580                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 581                                digestsize, 1);
 582 #endif
 583
 584         req->base.complete(&req->base, err);
 585 }
 586
 587 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
 588                                void *context)
 589 {
 590         struct ahash_request *req = context;
 591         struct ahash_edesc *edesc;
 592         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 593         int digestsize = crypto_ahash_digestsize(ahash);
 594 #ifdef DEBUG
 595         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 596         struct caam_hash_state *state = ahash_request_ctx(req);
 597
 598         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 599 #endif
 600
 601         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 602         if (err)
 603                 caam_jr_strstatus(jrdev, err);
 604
 605         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);
 606         kfree(edesc);
 607
 608 #ifdef DEBUG
 609         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 610                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 611                        ctx->ctx_len, 1);
 612         if (req->result)
 613                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 614                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 615                                digestsize, 1);
 616 #endif
 617
 618         req->base.complete(&req->base, err);
 619 }
 620
 621 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
 622                                void *context)
 623 {
 624         struct ahash_request *req = context;
 625         struct ahash_edesc *edesc;
 626         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 627         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 628         struct caam_hash_state *state = ahash_request_ctx(req);
 629 #ifdef DEBUG
 630         int digestsize = crypto_ahash_digestsize(ahash);
 631
 632         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 633 #endif
 634
 635         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 636         if (err)
 637                 caam_jr_strstatus(jrdev, err);
 638
 639         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
 640         switch_buf(state);
 641         kfree(edesc);
 642
 643 #ifdef DEBUG
 644         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 645                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 646                        ctx->ctx_len, 1);
 647         if (req->result)
 648                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 649                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 650                                digestsize, 1);
 651 #endif
 652
 653         req->base.complete(&req->base, err);
 654 }
 655
 656 /*
 657  * Allocate an enhanced descriptor, which contains the hardware descriptor
 658  * and space for hardware scatter table containing sg_num entries.
 659  */
 660 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx,
 661                                              int sg_num, u32 *sh_desc,
 662                                              dma_addr_t sh_desc_dma,
 663                                              gfp_t flags)
 664 {
 665         struct ahash_edesc *edesc;
 666         unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
 667
 668         edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
 669         if (!edesc) {
 670                 dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
 671                 return NULL;
 672         }
 673
 674         init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
 675                              HDR_SHARE_DEFER | HDR_REVERSE);
 676
 677         return edesc;
 678 }
 679
 680 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
 681                                struct ahash_edesc *edesc,
 682                                struct ahash_request *req, int nents,
 683                                unsigned int first_sg,
 684                                unsigned int first_bytes, size_t to_hash)
 685 {
 686         dma_addr_t src_dma;
 687         u32 options;
 688
 689         if (nents > 1 || first_sg) {
 690                 struct sec4_sg_entry *sg = edesc->sec4_sg;
 691                 unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
 692
 693                 sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
 694
 695                 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
 696                 if (dma_mapping_error(ctx->jrdev, src_dma)) {
 697                         dev_err(ctx->jrdev, "unable to map S/G table\n");
 698                         return -ENOMEM;
 699                 }
 700
 701                 edesc->sec4_sg_bytes = sgsize;
 702                 edesc->sec4_sg_dma = src_dma;
 703                 options = LDST_SGF;
 704         } else {
 705                 src_dma = sg_dma_address(req->src);
 706                 options = 0;
 707         }
 708
 709         append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
 710                           options);
 711
 712         return 0;
 713 }
 714
 715 /* submit update job descriptor */
 716 static int ahash_update_ctx(struct ahash_request *req)
 717 {
 718         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 719         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 720         struct caam_hash_state *state = ahash_request_ctx(req);
 721         struct device *jrdev = ctx->jrdev;
 722         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
 723                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
 724         u8 *buf = current_buf(state);
 725         int *buflen = current_buflen(state);
 726         u8 *next_buf = alt_buf(state);
 727         int *next_buflen = alt_buflen(state), last_buflen;
 728         int in_len = *buflen + req->nbytes, to_hash;
 729         u32 *desc;
 730         int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
 731         struct ahash_edesc *edesc;
 732         int ret = 0;
 733
 734         last_buflen = *next_buflen;
 735         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
 736         to_hash = in_len - *next_buflen;
 737
 738         if (to_hash) {
 739                 src_nents = sg_nents_for_len(req->src,
 740                                              req->nbytes - (*next_buflen));
 741                 if (src_nents < 0) {
 742                         dev_err(jrdev, "Invalid number of src SG.\n");
 743                         return src_nents;
 744                 }
 745
 746                 if (src_nents) {
 747                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
 748                                                   DMA_TO_DEVICE);
 749                         if (!mapped_nents) {
 750                                 dev_err(jrdev, "unable to DMA map source\n");
 751                                 return -ENOMEM;
 752                         }
 753                 } else {
 754                         mapped_nents = 0;
 755                 }
 756
 757                 sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
 758                 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
 759                                  sizeof(struct sec4_sg_entry);
 760
 761                 /*
 762                  * allocate space for base edesc and hw desc commands,
 763                  * link tables
 764                  */
 765                 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
 766                                           ctx->sh_desc_update,
 767                                           ctx->sh_desc_update_dma, flags);
 768                 if (!edesc) {
 769                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 770                         return -ENOMEM;
 771                 }
 772
 773                 edesc->src_nents = src_nents;
 774                 edesc->sec4_sg_bytes = sec4_sg_bytes;
 775
 776                 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 777                                          edesc->sec4_sg, DMA_BIDIRECTIONAL);
 778                 if (ret)
 779                         goto unmap_ctx;
 780
 781                 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
 782                 if (ret)
 783                         goto unmap_ctx;
 784
 785                 if (mapped_nents) {
 786                         sg_to_sec4_sg_last(req->src, mapped_nents,
 787                                            edesc->sec4_sg + sec4_sg_src_index,
 788                                            0);
 789                         if (*next_buflen)
 790                                 scatterwalk_map_and_copy(next_buf, req->src,
 791                                                          to_hash - *buflen,
 792                                                          *next_buflen, 0);
 793                 } else {
 794                         (edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
 795                                 cpu_to_caam32(SEC4_SG_LEN_FIN);
 796                 }
 797
 798                 desc = edesc->hw_desc;
 799
 800                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 801                                                      sec4_sg_bytes,
 802                                                      DMA_TO_DEVICE);
 803                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 804                         dev_err(jrdev, "unable to map S/G table\n");
 805                         ret = -ENOMEM;
 806                         goto unmap_ctx;
 807                 }
 808
 809                 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
 810                                        to_hash, LDST_SGF);
 811
 812                 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
 813
 814 #ifdef DEBUG
 815                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 816                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
 817                                desc_bytes(desc), 1);
 818 #endif
 819
 820                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
 821                 if (ret)
 822                         goto unmap_ctx;
 823
 824                 ret = -EINPROGRESS;
 825         } else if (*next_buflen) {
 826                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
 827                                          req->nbytes, 0);
 828                 *buflen = *next_buflen;
 829                 *next_buflen = last_buflen;
 830         }
 831 #ifdef DEBUG
 832         print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
 833                        DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
 834         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
 835                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
 836                        *next_buflen, 1);
 837 #endif
 838
 839         return ret;
 840  unmap_ctx:
 841         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
 842         kfree(edesc);
 843         return ret;
 844 }
 845
 846 static int ahash_final_ctx(struct ahash_request *req)
 847 {
 848         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 849         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 850         struct caam_hash_state *state = ahash_request_ctx(req);
 851         struct device *jrdev = ctx->jrdev;
 852         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
 853                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
 854         int buflen = *current_buflen(state);
 855         u32 *desc;
 856         int sec4_sg_bytes, sec4_sg_src_index;
 857         int digestsize = crypto_ahash_digestsize(ahash);
 858         struct ahash_edesc *edesc;
 859         int ret;
 860
 861         sec4_sg_src_index = 1 + (buflen ? 1 : 0);
 862         sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
 863
 864         /* allocate space for base edesc and hw desc commands, link tables */
 865         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
 866                                   ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
 867                                   flags);
 868         if (!edesc)
 869                 return -ENOMEM;
 870
 871         desc = edesc->hw_desc;
 872
 873         edesc->sec4_sg_bytes = sec4_sg_bytes;
 874         edesc->src_nents = 0;
 875
 876         ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 877                                  edesc->sec4_sg, DMA_TO_DEVICE);
 878         if (ret)
 879                 goto unmap_ctx;
 880
 881         ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
 882         if (ret)
 883                 goto unmap_ctx;
 884
 885         (edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
 886                 cpu_to_caam32(SEC4_SG_LEN_FIN);
 887
 888         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 889                                             sec4_sg_bytes, DMA_TO_DEVICE);
 890         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 891                 dev_err(jrdev, "unable to map S/G table\n");
 892                 ret = -ENOMEM;
 893                 goto unmap_ctx;
 894         }
 895
 896         append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
 897                           LDST_SGF);
 898
 899         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
 900                                                 digestsize);
 901         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 902                 dev_err(jrdev, "unable to map dst\n");
 903                 ret = -ENOMEM;
 904                 goto unmap_ctx;
 905         }
 906
 907 #ifdef DEBUG
 908         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 909                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 910 #endif
 911
 912         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
 913         if (ret)
 914                 goto unmap_ctx;
 915
 916         return -EINPROGRESS;
 917  unmap_ctx:
 918         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
 919         kfree(edesc);
 920         return ret;
 921 }
 922
 923 static int ahash_finup_ctx(struct ahash_request *req)
 924 {
 925         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 926         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 927         struct caam_hash_state *state = ahash_request_ctx(req);
 928         struct device *jrdev = ctx->jrdev;
 929         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
 930                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
 931         int buflen = *current_buflen(state);
 932         u32 *desc;
 933         int sec4_sg_src_index;
 934         int src_nents, mapped_nents;
 935         int digestsize = crypto_ahash_digestsize(ahash);
 936         struct ahash_edesc *edesc;
 937         int ret;
 938
 939         src_nents = sg_nents_for_len(req->src, req->nbytes);
 940         if (src_nents < 0) {
 941                 dev_err(jrdev, "Invalid number of src SG.\n");
 942                 return src_nents;
 943         }
 944
 945         if (src_nents) {
 946                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
 947                                           DMA_TO_DEVICE);
 948                 if (!mapped_nents) {
 949                         dev_err(jrdev, "unable to DMA map source\n");
 950                         return -ENOMEM;
 951                 }
 952         } else {
 953                 mapped_nents = 0;
 954         }
 955
 956         sec4_sg_src_index = 1 + (buflen ? 1 : 0);
 957
 958         /* allocate space for base edesc and hw desc commands, link tables */
 959         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
 960                                   ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
 961                                   flags);
 962         if (!edesc) {
 963                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 964                 return -ENOMEM;
 965         }
 966
 967         desc = edesc->hw_desc;
 968
 969         edesc->src_nents = src_nents;
 970
 971         ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 972                                  edesc->sec4_sg, DMA_TO_DEVICE);
 973         if (ret)
 974                 goto unmap_ctx;
 975
 976         ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
 977         if (ret)
 978                 goto unmap_ctx;
 979
 980         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
 981                                   sec4_sg_src_index, ctx->ctx_len + buflen,
 982                                   req->nbytes);
 983         if (ret)
 984                 goto unmap_ctx;
 985
 986         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
 987                                                 digestsize);
 988         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 989                 dev_err(jrdev, "unable to map dst\n");
 990                 ret = -ENOMEM;
 991                 goto unmap_ctx;
 992         }
 993
 994 #ifdef DEBUG
 995         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 996                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 997 #endif
 998
 999         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
1000         if (ret)
1001                 goto unmap_ctx;
1002
1003         return -EINPROGRESS;
1004  unmap_ctx:
1005         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1006         kfree(edesc);
1007         return ret;
1008 }
1009
1010 static int ahash_digest(struct ahash_request *req)
1011 {
1012         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1013         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1014         struct caam_hash_state *state = ahash_request_ctx(req);
1015         struct device *jrdev = ctx->jrdev;
1016         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1017                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1018         u32 *desc;
1019         int digestsize = crypto_ahash_digestsize(ahash);
1020         int src_nents, mapped_nents;
1021         struct ahash_edesc *edesc;
1022         int ret;
1023
1024         state->buf_dma = 0;
1025
1026         src_nents = sg_nents_for_len(req->src, req->nbytes);
1027         if (src_nents < 0) {
1028                 dev_err(jrdev, "Invalid number of src SG.\n");
1029                 return src_nents;
1030         }
1031
1032         if (src_nents) {
1033                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1034                                           DMA_TO_DEVICE);
1035                 if (!mapped_nents) {
1036                         dev_err(jrdev, "unable to map source for DMA\n");
1037                         return -ENOMEM;
1038                 }
1039         } else {
1040                 mapped_nents = 0;
1041         }
1042
1043         /* allocate space for base edesc and hw desc commands, link tables */
1044         edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0,
1045                                   ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1046                                   flags);
1047         if (!edesc) {
1048                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1049                 return -ENOMEM;
1050         }
1051
1052         edesc->src_nents = src_nents;
1053
1054         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1055                                   req->nbytes);
1056         if (ret) {
1057                 ahash_unmap(jrdev, edesc, req, digestsize);
1058                 kfree(edesc);
1059                 return ret;
1060         }
1061
1062         desc = edesc->hw_desc;
1063
1064         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1065                                                 digestsize);
1066         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1067                 dev_err(jrdev, "unable to map dst\n");
1068                 ahash_unmap(jrdev, edesc, req, digestsize);
1069                 kfree(edesc);
1070                 return -ENOMEM;
1071         }
1072
1073 #ifdef DEBUG
1074         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1075                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1076 #endif
1077
1078         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1079         if (!ret) {
1080                 ret = -EINPROGRESS;
1081         } else {
1082                 ahash_unmap(jrdev, edesc, req, digestsize);
1083                 kfree(edesc);
1084         }
1085
1086         return ret;
1087 }
1088
1089 /* submit ahash final if it the first job descriptor */
1090 static int ahash_final_no_ctx(struct ahash_request *req)
1091 {
1092         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1093         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1094         struct caam_hash_state *state = ahash_request_ctx(req);
1095         struct device *jrdev = ctx->jrdev;
1096         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1097                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1098         u8 *buf = current_buf(state);
1099         int buflen = *current_buflen(state);
1100         u32 *desc;
1101         int digestsize = crypto_ahash_digestsize(ahash);
1102         struct ahash_edesc *edesc;
1103         int ret;
1104
1105         /* allocate space for base edesc and hw desc commands, link tables */
1106         edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest,
1107                                   ctx->sh_desc_digest_dma, flags);
1108         if (!edesc)
1109                 return -ENOMEM;
1110
1111         desc = edesc->hw_desc;
1112
1113         state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
1114         if (dma_mapping_error(jrdev, state->buf_dma)) {
1115                 dev_err(jrdev, "unable to map src\n");
1116                 goto unmap;
1117         }
1118
1119         append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
1120
1121         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1122                                                 digestsize);
1123         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1124                 dev_err(jrdev, "unable to map dst\n");
1125                 goto unmap;
1126         }
1127         edesc->src_nents = 0;
1128
1129 #ifdef DEBUG
1130         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1131                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1132 #endif
1133
1134         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1135         if (!ret) {
1136                 ret = -EINPROGRESS;
1137         } else {
1138                 ahash_unmap(jrdev, edesc, req, digestsize);
1139                 kfree(edesc);
1140         }
1141
1142         return ret;
1143  unmap:
1144         ahash_unmap(jrdev, edesc, req, digestsize);
1145         kfree(edesc);
1146         return -ENOMEM;
1147
1148 }
1149
1150 /* submit ahash update if it the first job descriptor after update */
1151 static int ahash_update_no_ctx(struct ahash_request *req)
1152 {
1153         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1154         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1155         struct caam_hash_state *state = ahash_request_ctx(req);
1156         struct device *jrdev = ctx->jrdev;
1157         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1158                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1159         u8 *buf = current_buf(state);
1160         int *buflen = current_buflen(state);
1161         u8 *next_buf = alt_buf(state);
1162         int *next_buflen = alt_buflen(state);
1163         int in_len = *buflen + req->nbytes, to_hash;
1164         int sec4_sg_bytes, src_nents, mapped_nents;
1165         struct ahash_edesc *edesc;
1166         u32 *desc;
1167         int ret = 0;
1168
1169         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
1170         to_hash = in_len - *next_buflen;
1171
1172         if (to_hash) {
1173                 src_nents = sg_nents_for_len(req->src,
1174                                              req->nbytes - *next_buflen);
1175                 if (src_nents < 0) {
1176                         dev_err(jrdev, "Invalid number of src SG.\n");
1177                         return src_nents;
1178                 }
1179
1180                 if (src_nents) {
1181                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1182                                                   DMA_TO_DEVICE);
1183                         if (!mapped_nents) {
1184                                 dev_err(jrdev, "unable to DMA map source\n");
1185                                 return -ENOMEM;
1186                         }
1187                 } else {
1188                         mapped_nents = 0;
1189                 }
1190
1191                 sec4_sg_bytes = (1 + mapped_nents) *
1192                                 sizeof(struct sec4_sg_entry);
1193
1194                 /*
1195                  * allocate space for base edesc and hw desc commands,
1196                  * link tables
1197                  */
1198                 edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
1199                                           ctx->sh_desc_update_first,
1200                                           ctx->sh_desc_update_first_dma,
1201                                           flags);
1202                 if (!edesc) {
1203                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1204                         return -ENOMEM;
1205                 }
1206
1207                 edesc->src_nents = src_nents;
1208                 edesc->sec4_sg_bytes = sec4_sg_bytes;
1209                 edesc->dst_dma = 0;
1210
1211                 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1212                 if (ret)
1213                         goto unmap_ctx;
1214
1215                 sg_to_sec4_sg_last(req->src, mapped_nents,
1216                                    edesc->sec4_sg + 1, 0);
1217
1218                 if (*next_buflen) {
1219                         scatterwalk_map_and_copy(next_buf, req->src,
1220                                                  to_hash - *buflen,
1221                                                  *next_buflen, 0);
1222                 }
1223
1224                 desc = edesc->hw_desc;
1225
1226                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1227                                                     sec4_sg_bytes,
1228                                                     DMA_TO_DEVICE);
1229                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1230                         dev_err(jrdev, "unable to map S/G table\n");
1231                         ret = -ENOMEM;
1232                         goto unmap_ctx;
1233                 }
1234
1235                 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
1236
1237                 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1238                 if (ret)
1239                         goto unmap_ctx;
1240
1241 #ifdef DEBUG
1242                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1243                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
1244                                desc_bytes(desc), 1);
1245 #endif
1246
1247                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1248                 if (ret)
1249                         goto unmap_ctx;
1250
1251                 ret = -EINPROGRESS;
1252                 state->update = ahash_update_ctx;
1253                 state->finup = ahash_finup_ctx;
1254                 state->final = ahash_final_ctx;
1255         } else if (*next_buflen) {
1256                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
1257                                          req->nbytes, 0);
1258                 *buflen = *next_buflen;
1259                 *next_buflen = 0;
1260         }
1261 #ifdef DEBUG
1262         print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
1263                        DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
1264         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1265                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1266                        *next_buflen, 1);
1267 #endif
1268
1269         return ret;
1270  unmap_ctx:
1271         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1272         kfree(edesc);
1273         return ret;
1274 }
1275
1276 /* submit ahash finup if it the first job descriptor after update */
1277 static int ahash_finup_no_ctx(struct ahash_request *req)
1278 {
1279         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1280         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1281         struct caam_hash_state *state = ahash_request_ctx(req);
1282         struct device *jrdev = ctx->jrdev;
1283         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1284                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1285         int buflen = *current_buflen(state);
1286         u32 *desc;
1287         int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
1288         int digestsize = crypto_ahash_digestsize(ahash);
1289         struct ahash_edesc *edesc;
1290         int ret;
1291
1292         src_nents = sg_nents_for_len(req->src, req->nbytes);
1293         if (src_nents < 0) {
1294                 dev_err(jrdev, "Invalid number of src SG.\n");
1295                 return src_nents;
1296         }
1297
1298         if (src_nents) {
1299                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1300                                           DMA_TO_DEVICE);
1301                 if (!mapped_nents) {
1302                         dev_err(jrdev, "unable to DMA map source\n");
1303                         return -ENOMEM;
1304                 }
1305         } else {
1306                 mapped_nents = 0;
1307         }
1308
1309         sec4_sg_src_index = 2;
1310         sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
1311                          sizeof(struct sec4_sg_entry);
1312
1313         /* allocate space for base edesc and hw desc commands, link tables */
1314         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
1315                                   ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1316                                   flags);
1317         if (!edesc) {
1318                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1319                 return -ENOMEM;
1320         }
1321
1322         desc = edesc->hw_desc;
1323
1324         edesc->src_nents = src_nents;
1325         edesc->sec4_sg_bytes = sec4_sg_bytes;
1326
1327         ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1328         if (ret)
1329                 goto unmap;
1330
1331         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
1332                                   req->nbytes);
1333         if (ret) {
1334                 dev_err(jrdev, "unable to map S/G table\n");
1335                 goto unmap;
1336         }
1337
1338         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1339                                                 digestsize);
1340         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1341                 dev_err(jrdev, "unable to map dst\n");
1342                 goto unmap;
1343         }
1344
1345 #ifdef DEBUG
1346         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1347                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1348 #endif
1349
1350         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1351         if (!ret) {
1352                 ret = -EINPROGRESS;
1353         } else {
1354                 ahash_unmap(jrdev, edesc, req, digestsize);
1355                 kfree(edesc);
1356         }
1357
1358         return ret;
1359  unmap:
1360         ahash_unmap(jrdev, edesc, req, digestsize);
1361         kfree(edesc);
1362         return -ENOMEM;
1363
1364 }
1365
1366 /* submit first update job descriptor after init */
1367 static int ahash_update_first(struct ahash_request *req)
1368 {
1369         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1370         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1371         struct caam_hash_state *state = ahash_request_ctx(req);
1372         struct device *jrdev = ctx->jrdev;
1373         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1374                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1375         u8 *next_buf = alt_buf(state);
1376         int *next_buflen = alt_buflen(state);
1377         int to_hash;
1378         u32 *desc;
1379         int src_nents, mapped_nents;
1380         struct ahash_edesc *edesc;
1381         int ret = 0;
1382
1383         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
1384                                       1);
1385         to_hash = req->nbytes - *next_buflen;
1386
1387         if (to_hash) {
1388                 src_nents = sg_nents_for_len(req->src,
1389                                              req->nbytes - *next_buflen);
1390                 if (src_nents < 0) {
1391                         dev_err(jrdev, "Invalid number of src SG.\n");
1392                         return src_nents;
1393                 }
1394
1395                 if (src_nents) {
1396                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1397                                                   DMA_TO_DEVICE);
1398                         if (!mapped_nents) {
1399                                 dev_err(jrdev, "unable to map source for DMA\n");
1400                                 return -ENOMEM;
1401                         }
1402                 } else {
1403                         mapped_nents = 0;
1404                 }
1405
1406                 /*
1407                  * allocate space for base edesc and hw desc commands,
1408                  * link tables
1409                  */
1410                 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ?
1411                                           mapped_nents : 0,
1412                                           ctx->sh_desc_update_first,
1413                                           ctx->sh_desc_update_first_dma,
1414                                           flags);
1415                 if (!edesc) {
1416                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1417                         return -ENOMEM;
1418                 }
1419
1420                 edesc->src_nents = src_nents;
1421                 edesc->dst_dma = 0;
1422
1423                 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1424                                           to_hash);
1425                 if (ret)
1426                         goto unmap_ctx;
1427
1428                 if (*next_buflen)
1429                         scatterwalk_map_and_copy(next_buf, req->src, to_hash,
1430                                                  *next_buflen, 0);
1431
1432                 desc = edesc->hw_desc;
1433
1434                 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1435                 if (ret)
1436                         goto unmap_ctx;
1437
1438 #ifdef DEBUG
1439                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1440                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
1441                                desc_bytes(desc), 1);
1442 #endif
1443
1444                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1445                 if (ret)
1446                         goto unmap_ctx;
1447
1448                 ret = -EINPROGRESS;
1449                 state->update = ahash_update_ctx;
1450                 state->finup = ahash_finup_ctx;
1451                 state->final = ahash_final_ctx;
1452         } else if (*next_buflen) {
1453                 state->update = ahash_update_no_ctx;
1454                 state->finup = ahash_finup_no_ctx;
1455                 state->final = ahash_final_no_ctx;
1456                 scatterwalk_map_and_copy(next_buf, req->src, 0,
1457                                          req->nbytes, 0);
1458                 switch_buf(state);
1459         }
1460 #ifdef DEBUG
1461         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1462                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1463                        *next_buflen, 1);
1464 #endif
1465
1466         return ret;
1467  unmap_ctx:
1468         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1469         kfree(edesc);
1470         return ret;
1471 }
1472
1473 static int ahash_finup_first(struct ahash_request *req)
1474 {
1475         return ahash_digest(req);
1476 }
1477
1478 static int ahash_init(struct ahash_request *req)
1479 {
1480         struct caam_hash_state *state = ahash_request_ctx(req);
1481
1482         state->update = ahash_update_first;
1483         state->finup = ahash_finup_first;
1484         state->final = ahash_final_no_ctx;
1485
1486         state->ctx_dma = 0;
1487         state->current_buf = 0;
1488         state->buf_dma = 0;
1489         state->buflen_0 = 0;
1490         state->buflen_1 = 0;
1491
1492         return 0;
1493 }
1494
1495 static int ahash_update(struct ahash_request *req)
1496 {
1497         struct caam_hash_state *state = ahash_request_ctx(req);
1498
1499         return state->update(req);
1500 }
1501
1502 static int ahash_finup(struct ahash_request *req)
1503 {
1504         struct caam_hash_state *state = ahash_request_ctx(req);
1505
1506         return state->finup(req);
1507 }
1508
1509 static int ahash_final(struct ahash_request *req)
1510 {
1511         struct caam_hash_state *state = ahash_request_ctx(req);
1512
1513         return state->final(req);
1514 }
1515
1516 static int ahash_export(struct ahash_request *req, void *out)
1517 {
1518         struct caam_hash_state *state = ahash_request_ctx(req);
1519         struct caam_export_state *export = out;
1520         int len;
1521         u8 *buf;
1522
1523         if (state->current_buf) {
1524                 buf = state->buf_1;
1525                 len = state->buflen_1;
1526         } else {
1527                 buf = state->buf_0;
1528                 len = state->buflen_0;
1529         }
1530
1531         memcpy(export->buf, buf, len);
1532         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
1533         export->buflen = len;
1534         export->update = state->update;
1535         export->final = state->final;
1536         export->finup = state->finup;
1537
1538         return 0;
1539 }
1540
1541 static int ahash_import(struct ahash_request *req, const void *in)
1542 {
1543         struct caam_hash_state *state = ahash_request_ctx(req);
1544         const struct caam_export_state *export = in;
1545
1546         memset(state, 0, sizeof(*state));
1547         memcpy(state->buf_0, export->buf, export->buflen);
1548         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
1549         state->buflen_0 = export->buflen;
1550         state->update = export->update;
1551         state->final = export->final;
1552         state->finup = export->finup;
1553
1554         return 0;
1555 }
1556
1557 struct caam_hash_template {
1558         char name[CRYPTO_MAX_ALG_NAME];
1559         char driver_name[CRYPTO_MAX_ALG_NAME];
1560         char hmac_name[CRYPTO_MAX_ALG_NAME];
1561         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
1562         unsigned int blocksize;
1563         struct ahash_alg template_ahash;
1564         u32 alg_type;
1565 };
1566
1567 /* ahash descriptors */
1568 static struct caam_hash_template driver_hash[] = {
1569         {
1570                 .name = "sha1",
1571                 .driver_name = "sha1-caam",
1572                 .hmac_name = "hmac(sha1)",
1573                 .hmac_driver_name = "hmac-sha1-caam",
1574                 .blocksize = SHA1_BLOCK_SIZE,
1575                 .template_ahash = {
1576                         .init = ahash_init,
1577                         .update = ahash_update,
1578                         .final = ahash_final,
1579                         .finup = ahash_finup,
1580                         .digest = ahash_digest,
1581                         .export = ahash_export,
1582                         .import = ahash_import,
1583                         .setkey = ahash_setkey,
1584                         .halg = {
1585                                 .digestsize = SHA1_DIGEST_SIZE,
1586                                 .statesize = sizeof(struct caam_export_state),
1587                         },
1588                 },
1589                 .alg_type = OP_ALG_ALGSEL_SHA1,
1590         }, {
1591                 .name = "sha224",
1592                 .driver_name = "sha224-caam",
1593                 .hmac_name = "hmac(sha224)",
1594                 .hmac_driver_name = "hmac-sha224-caam",
1595                 .blocksize = SHA224_BLOCK_SIZE,
1596                 .template_ahash = {
1597                         .init = ahash_init,
1598                         .update = ahash_update,
1599                         .final = ahash_final,
1600                         .finup = ahash_finup,
1601                         .digest = ahash_digest,
1602                         .export = ahash_export,
1603                         .import = ahash_import,
1604                         .setkey = ahash_setkey,
1605                         .halg = {
1606                                 .digestsize = SHA224_DIGEST_SIZE,
1607                                 .statesize = sizeof(struct caam_export_state),
1608                         },
1609                 },
1610                 .alg_type = OP_ALG_ALGSEL_SHA224,
1611         }, {
1612                 .name = "sha256",
1613                 .driver_name = "sha256-caam",
1614                 .hmac_name = "hmac(sha256)",
1615                 .hmac_driver_name = "hmac-sha256-caam",
1616                 .blocksize = SHA256_BLOCK_SIZE,
1617                 .template_ahash = {
1618                         .init = ahash_init,
1619                         .update = ahash_update,
1620                         .final = ahash_final,
1621                         .finup = ahash_finup,
1622                         .digest = ahash_digest,
1623                         .export = ahash_export,
1624                         .import = ahash_import,
1625                         .setkey = ahash_setkey,
1626                         .halg = {
1627                                 .digestsize = SHA256_DIGEST_SIZE,
1628                                 .statesize = sizeof(struct caam_export_state),
1629                         },
1630                 },
1631                 .alg_type = OP_ALG_ALGSEL_SHA256,
1632         }, {
1633                 .name = "sha384",
1634                 .driver_name = "sha384-caam",
1635                 .hmac_name = "hmac(sha384)",
1636                 .hmac_driver_name = "hmac-sha384-caam",
1637                 .blocksize = SHA384_BLOCK_SIZE,
1638                 .template_ahash = {
1639                         .init = ahash_init,
1640                         .update = ahash_update,
1641                         .final = ahash_final,
1642                         .finup = ahash_finup,
1643                         .digest = ahash_digest,
1644                         .export = ahash_export,
1645                         .import = ahash_import,
1646                         .setkey = ahash_setkey,
1647                         .halg = {
1648                                 .digestsize = SHA384_DIGEST_SIZE,
1649                                 .statesize = sizeof(struct caam_export_state),
1650                         },
1651                 },
1652                 .alg_type = OP_ALG_ALGSEL_SHA384,
1653         }, {
1654                 .name = "sha512",
1655                 .driver_name = "sha512-caam",
1656                 .hmac_name = "hmac(sha512)",
1657                 .hmac_driver_name = "hmac-sha512-caam",
1658                 .blocksize = SHA512_BLOCK_SIZE,
1659                 .template_ahash = {
1660                         .init = ahash_init,
1661                         .update = ahash_update,
1662                         .final = ahash_final,
1663                         .finup = ahash_finup,
1664                         .digest = ahash_digest,
1665                         .export = ahash_export,
1666                         .import = ahash_import,
1667                         .setkey = ahash_setkey,
1668                         .halg = {
1669                                 .digestsize = SHA512_DIGEST_SIZE,
1670                                 .statesize = sizeof(struct caam_export_state),
1671                         },
1672                 },
1673                 .alg_type = OP_ALG_ALGSEL_SHA512,
1674         }, {
1675                 .name = "md5",
1676                 .driver_name = "md5-caam",
1677                 .hmac_name = "hmac(md5)",
1678                 .hmac_driver_name = "hmac-md5-caam",
1679                 .blocksize = MD5_BLOCK_WORDS * 4,
1680                 .template_ahash = {
1681                         .init = ahash_init,
1682                         .update = ahash_update,
1683                         .final = ahash_final,
1684                         .finup = ahash_finup,
1685                         .digest = ahash_digest,
1686                         .export = ahash_export,
1687                         .import = ahash_import,
1688                         .setkey = ahash_setkey,
1689                         .halg = {
1690                                 .digestsize = MD5_DIGEST_SIZE,
1691                                 .statesize = sizeof(struct caam_export_state),
1692                         },
1693                 },
1694                 .alg_type = OP_ALG_ALGSEL_MD5,
1695         },
1696 };
1697
1698 struct caam_hash_alg {
1699         struct list_head entry;
1700         int alg_type;
1701         struct ahash_alg ahash_alg;
1702 };
1703
1704 static int caam_hash_cra_init(struct crypto_tfm *tfm)
1705 {
1706         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
1707         struct crypto_alg *base = tfm->__crt_alg;
1708         struct hash_alg_common *halg =
1709                  container_of(base, struct hash_alg_common, base);
1710         struct ahash_alg *alg =
1711                  container_of(halg, struct ahash_alg, halg);
1712         struct caam_hash_alg *caam_hash =
1713                  container_of(alg, struct caam_hash_alg, ahash_alg);
1714         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1715         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
1716         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
1717                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
1718                                          HASH_MSG_LEN + 32,
1719                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
1720                                          HASH_MSG_LEN + 64,
1721                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
1722         dma_addr_t dma_addr;
1723
1724         /*
1725          * Get a Job ring from Job Ring driver to ensure in-order
1726          * crypto request processing per tfm
1727          */
1728         ctx->jrdev = caam_jr_alloc();
1729         if (IS_ERR(ctx->jrdev)) {
1730                 pr_err("Job Ring Device allocation for transform failed\n");
1731                 return PTR_ERR(ctx->jrdev);
1732         }
1733
1734         dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
1735                                         offsetof(struct caam_hash_ctx,
1736                                                  sh_desc_update_dma),
1737                                         DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
1738         if (dma_mapping_error(ctx->jrdev, dma_addr)) {
1739                 dev_err(ctx->jrdev, "unable to map shared descriptors\n");
1740                 caam_jr_free(ctx->jrdev);
1741                 return -ENOMEM;
1742         }
1743
1744         ctx->sh_desc_update_dma = dma_addr;
1745         ctx->sh_desc_update_first_dma = dma_addr +
1746                                         offsetof(struct caam_hash_ctx,
1747                                                  sh_desc_update_first);
1748         ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx,
1749                                                    sh_desc_fin);
1750         ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx,
1751                                                       sh_desc_digest);
1752
1753         /* copy descriptor header template value */
1754         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
1755
1756         ctx->ctx_len = runninglen[(ctx->adata.algtype &
1757                                    OP_ALG_ALGSEL_SUBMASK) >>
1758                                   OP_ALG_ALGSEL_SHIFT];
1759
1760         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1761                                  sizeof(struct caam_hash_state));
1762         return ahash_set_sh_desc(ahash);
1763 }
1764
1765 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
1766 {
1767         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1768
1769         dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
1770                                offsetof(struct caam_hash_ctx,
1771                                         sh_desc_update_dma),
1772                                DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
1773         caam_jr_free(ctx->jrdev);
1774 }
1775
1776 static void __exit caam_algapi_hash_exit(void)
1777 {
1778         struct caam_hash_alg *t_alg, *n;
1779
1780         if (!hash_list.next)
1781                 return;
1782
1783         list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
1784                 crypto_unregister_ahash(&t_alg->ahash_alg);
1785                 list_del(&t_alg->entry);
1786                 kfree(t_alg);
1787         }
1788 }
1789
1790 static struct caam_hash_alg *
1791 caam_hash_alloc(struct caam_hash_template *template,
1792                 bool keyed)
1793 {
1794         struct caam_hash_alg *t_alg;
1795         struct ahash_alg *halg;
1796         struct crypto_alg *alg;
1797
1798         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
1799         if (!t_alg) {
1800                 pr_err("failed to allocate t_alg\n");
1801                 return ERR_PTR(-ENOMEM);
1802         }
1803
1804         t_alg->ahash_alg = template->template_ahash;
1805         halg = &t_alg->ahash_alg;
1806         alg = &halg->halg.base;
1807
1808         if (keyed) {
1809                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1810                          template->hmac_name);
1811                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1812                          template->hmac_driver_name);
1813         } else {
1814                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1815                          template->name);
1816                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1817                          template->driver_name);
1818                 t_alg->ahash_alg.setkey = NULL;
1819         }
1820         alg->cra_module = THIS_MODULE;
1821         alg->cra_init = caam_hash_cra_init;
1822         alg->cra_exit = caam_hash_cra_exit;
1823         alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
1824         alg->cra_priority = CAAM_CRA_PRIORITY;
1825         alg->cra_blocksize = template->blocksize;
1826         alg->cra_alignmask = 0;
1827         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH;
1828         alg->cra_type = &crypto_ahash_type;
1829
1830         t_alg->alg_type = template->alg_type;
1831
1832         return t_alg;
1833 }
1834
1835 static int __init caam_algapi_hash_init(void)
1836 {
1837         struct device_node *dev_node;
1838         struct platform_device *pdev;
1839         struct device *ctrldev;
1840         int i = 0, err = 0;
1841         struct caam_drv_private *priv;
1842         unsigned int md_limit = SHA512_DIGEST_SIZE;
1843         u32 cha_inst, cha_vid;
1844
1845         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1846         if (!dev_node) {
1847                 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
1848                 if (!dev_node)
1849                         return -ENODEV;
1850         }
1851
1852         pdev = of_find_device_by_node(dev_node);
1853         if (!pdev) {
1854                 of_node_put(dev_node);
1855                 return -ENODEV;
1856         }
1857
1858         ctrldev = &pdev->dev;
1859         priv = dev_get_drvdata(ctrldev);
1860         of_node_put(dev_node);
1861
1862         /*
1863          * If priv is NULL, it's probably because the caam driver wasn't
1864          * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
1865          */
1866         if (!priv)
1867                 return -ENODEV;
1868
1869         /*
1870          * Register crypto algorithms the device supports.  First, identify
1871          * presence and attributes of MD block.
1872          */
1873         cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
1874         cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
1875
1876         /*
1877          * Skip registration of any hashing algorithms if MD block
1878          * is not present.
1879          */
1880         if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT))
1881                 return -ENODEV;
1882
1883         /* Limit digest size based on LP256 */
1884         if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)
1885                 md_limit = SHA256_DIGEST_SIZE;
1886
1887         INIT_LIST_HEAD(&hash_list);
1888
1889         /* register crypto algorithms the device supports */
1890         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
1891                 struct caam_hash_alg *t_alg;
1892                 struct caam_hash_template *alg = driver_hash + i;
1893
1894                 /* If MD size is not supported by device, skip registration */
1895                 if (alg->template_ahash.halg.digestsize > md_limit)
1896                         continue;
1897
1898                 /* register hmac version */
1899                 t_alg = caam_hash_alloc(alg, true);
1900                 if (IS_ERR(t_alg)) {
1901                         err = PTR_ERR(t_alg);
1902                         pr_warn("%s alg allocation failed\n", alg->driver_name);
1903                         continue;
1904                 }
1905
1906                 err = crypto_register_ahash(&t_alg->ahash_alg);
1907                 if (err) {
1908                         pr_warn("%s alg registration failed: %d\n",
1909                                 t_alg->ahash_alg.halg.base.cra_driver_name,
1910                                 err);
1911                         kfree(t_alg);
1912                 } else
1913                         list_add_tail(&t_alg->entry, &hash_list);
1914
1915                 /* register unkeyed version */
1916                 t_alg = caam_hash_alloc(alg, false);
1917                 if (IS_ERR(t_alg)) {
1918                         err = PTR_ERR(t_alg);
1919                         pr_warn("%s alg allocation failed\n", alg->driver_name);
1920                         continue;
1921                 }
1922
1923                 err = crypto_register_ahash(&t_alg->ahash_alg);
1924                 if (err) {
1925                         pr_warn("%s alg registration failed: %d\n",
1926                                 t_alg->ahash_alg.halg.base.cra_driver_name,
1927                                 err);
1928                         kfree(t_alg);
1929                 } else
1930                         list_add_tail(&t_alg->entry, &hash_list);
1931         }
1932
1933         return err;
1934 }
1935
1936 module_init(caam_algapi_hash_init);
1937 module_exit(caam_algapi_hash_exit);
1938
1939 MODULE_LICENSE("GPL");
1940 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
1941 MODULE_AUTHOR("Freescale Semiconductor - NMG");