drivers/crypto/ccp/ccp-ops.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * AMD Cryptographic Coprocessor (CCP) driver
   4  *
   5  * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
   6  *
   7  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   8  * Author: Gary R Hook <gary.hook@amd.com>
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/kernel.h>
  13 #include <linux/pci.h>
  14 #include <linux/interrupt.h>
  15 #include <crypto/scatterwalk.h>
  16 #include <crypto/des.h>
  17 #include <linux/ccp.h>
  18
  19 #include "ccp-dev.h"
  20
  21 /* SHA initial context values */
  22 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  23         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  24         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  25         cpu_to_be32(SHA1_H4),
  26 };
  27
  28 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  29         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  30         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  31         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  32         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  33 };
  34
  35 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  36         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  37         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  38         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  39         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  40 };
  41
  42 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  43         cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  44         cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  45         cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  46         cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  47 };
  48
  49 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  50         cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  51         cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  52         cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  53         cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  54 };
  55
  56 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  57                                         ccp_gen_jobid(ccp) : 0)
  58
  59 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  60 {
  61         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  62 }
  63
  64 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  65 {
  66         if (wa->dma_count)
  67                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
  68
  69         wa->dma_count = 0;
  70 }
  71
  72 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  73                                 struct scatterlist *sg, u64 len,
  74                                 enum dma_data_direction dma_dir)
  75 {
  76         memset(wa, 0, sizeof(*wa));
  77
  78         wa->sg = sg;
  79         if (!sg)
  80                 return 0;
  81
  82         wa->nents = sg_nents_for_len(sg, len);
  83         if (wa->nents < 0)
  84                 return wa->nents;
  85
  86         wa->bytes_left = len;
  87         wa->sg_used = 0;
  88
  89         if (len == 0)
  90                 return 0;
  91
  92         if (dma_dir == DMA_NONE)
  93                 return 0;
  94
  95         wa->dma_sg = sg;
  96         wa->dma_dev = dev;
  97         wa->dma_dir = dma_dir;
  98         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
  99         if (!wa->dma_count)
 100                 return -ENOMEM;
 101
 102         return 0;
 103 }
 104
 105 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 106 {
 107         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 108
 109         if (!wa->sg)
 110                 return;
 111
 112         wa->sg_used += nbytes;
 113         wa->bytes_left -= nbytes;
 114         if (wa->sg_used == wa->sg->length) {
 115                 wa->sg = sg_next(wa->sg);
 116                 wa->sg_used = 0;
 117         }
 118 }
 119
 120 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 121 {
 122         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 123                 if (wa->address)
 124                         dma_pool_free(wa->dma_pool, wa->address,
 125                                       wa->dma.address);
 126         } else {
 127                 if (wa->dma.address)
 128                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 129                                          wa->dma.dir);
 130                 kfree(wa->address);
 131         }
 132
 133         wa->address = NULL;
 134         wa->dma.address = 0;
 135 }
 136
 137 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 138                                 struct ccp_cmd_queue *cmd_q,
 139                                 unsigned int len,
 140                                 enum dma_data_direction dir)
 141 {
 142         memset(wa, 0, sizeof(*wa));
 143
 144         if (!len)
 145                 return 0;
 146
 147         wa->dev = cmd_q->ccp->dev;
 148         wa->length = len;
 149
 150         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 151                 wa->dma_pool = cmd_q->dma_pool;
 152
 153                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 154                                              &wa->dma.address);
 155                 if (!wa->address)
 156                         return -ENOMEM;
 157
 158                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 159
 160                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 161         } else {
 162                 wa->address = kzalloc(len, GFP_KERNEL);
 163                 if (!wa->address)
 164                         return -ENOMEM;
 165
 166                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 167                                                  dir);
 168                 if (dma_mapping_error(wa->dev, wa->dma.address))
 169                         return -ENOMEM;
 170
 171                 wa->dma.length = len;
 172         }
 173         wa->dma.dir = dir;
 174
 175         return 0;
 176 }
 177
 178 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 179                            struct scatterlist *sg, unsigned int sg_offset,
 180                            unsigned int len)
 181 {
 182         WARN_ON(!wa->address);
 183
 184         if (len > (wa->length - wa_offset))
 185                 return -EINVAL;
 186
 187         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 188                                  0);
 189         return 0;
 190 }
 191
 192 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 193                             struct scatterlist *sg, unsigned int sg_offset,
 194                             unsigned int len)
 195 {
 196         WARN_ON(!wa->address);
 197
 198         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 199                                  1);
 200 }
 201
 202 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 203                                    unsigned int wa_offset,
 204                                    struct scatterlist *sg,
 205                                    unsigned int sg_offset,
 206                                    unsigned int len)
 207 {
 208         u8 *p, *q;
 209         int     rc;
 210
 211         rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
 212         if (rc)
 213                 return rc;
 214
 215         p = wa->address + wa_offset;
 216         q = p + len - 1;
 217         while (p < q) {
 218                 *p = *p ^ *q;
 219                 *q = *p ^ *q;
 220                 *p = *p ^ *q;
 221                 p++;
 222                 q--;
 223         }
 224         return 0;
 225 }
 226
 227 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 228                                     unsigned int wa_offset,
 229                                     struct scatterlist *sg,
 230                                     unsigned int sg_offset,
 231                                     unsigned int len)
 232 {
 233         u8 *p, *q;
 234
 235         p = wa->address + wa_offset;
 236         q = p + len - 1;
 237         while (p < q) {
 238                 *p = *p ^ *q;
 239                 *q = *p ^ *q;
 240                 *p = *p ^ *q;
 241                 p++;
 242                 q--;
 243         }
 244
 245         ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
 246 }
 247
 248 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 249 {
 250         ccp_dm_free(&data->dm_wa);
 251         ccp_sg_free(&data->sg_wa);
 252 }
 253
 254 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 255                          struct scatterlist *sg, u64 sg_len,
 256                          unsigned int dm_len,
 257                          enum dma_data_direction dir)
 258 {
 259         int ret;
 260
 261         memset(data, 0, sizeof(*data));
 262
 263         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 264                                    dir);
 265         if (ret)
 266                 goto e_err;
 267
 268         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 269         if (ret)
 270                 goto e_err;
 271
 272         return 0;
 273
 274 e_err:
 275         ccp_free_data(data, cmd_q);
 276
 277         return ret;
 278 }
 279
 280 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 281 {
 282         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 283         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 284         unsigned int buf_count, nbytes;
 285
 286         /* Clear the buffer if setting it */
 287         if (!from)
 288                 memset(dm_wa->address, 0, dm_wa->length);
 289
 290         if (!sg_wa->sg)
 291                 return 0;
 292
 293         /* Perform the copy operation
 294          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 295          *   an unsigned int
 296          */
 297         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 298         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 299                                  nbytes, from);
 300
 301         /* Update the structures and generate the count */
 302         buf_count = 0;
 303         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 304                 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
 305                              dm_wa->length - buf_count);
 306                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 307
 308                 buf_count += nbytes;
 309                 ccp_update_sg_workarea(sg_wa, nbytes);
 310         }
 311
 312         return buf_count;
 313 }
 314
 315 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 316 {
 317         return ccp_queue_buf(data, 0);
 318 }
 319
 320 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 321 {
 322         return ccp_queue_buf(data, 1);
 323 }
 324
 325 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 326                              struct ccp_op *op, unsigned int block_size,
 327                              bool blocksize_op)
 328 {
 329         unsigned int sg_src_len, sg_dst_len, op_len;
 330
 331         /* The CCP can only DMA from/to one address each per operation. This
 332          * requires that we find the smallest DMA area between the source
 333          * and destination. The resulting len values will always be <= UINT_MAX
 334          * because the dma length is an unsigned int.
 335          */
 336         sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
 337         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 338
 339         if (dst) {
 340                 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
 341                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 342                 op_len = min(sg_src_len, sg_dst_len);
 343         } else {
 344                 op_len = sg_src_len;
 345         }
 346
 347         /* The data operation length will be at least block_size in length
 348          * or the smaller of available sg room remaining for the source or
 349          * the destination
 350          */
 351         op_len = max(op_len, block_size);
 352
 353         /* Unless we have to buffer data, there's no reason to wait */
 354         op->soc = 0;
 355
 356         if (sg_src_len < block_size) {
 357                 /* Not enough data in the sg element, so it
 358                  * needs to be buffered into a blocksize chunk
 359                  */
 360                 int cp_len = ccp_fill_queue_buf(src);
 361
 362                 op->soc = 1;
 363                 op->src.u.dma.address = src->dm_wa.dma.address;
 364                 op->src.u.dma.offset = 0;
 365                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 366         } else {
 367                 /* Enough data in the sg element, but we need to
 368                  * adjust for any previously copied data
 369                  */
 370                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 371                 op->src.u.dma.offset = src->sg_wa.sg_used;
 372                 op->src.u.dma.length = op_len & ~(block_size - 1);
 373
 374                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 375         }
 376
 377         if (dst) {
 378                 if (sg_dst_len < block_size) {
 379                         /* Not enough room in the sg element or we're on the
 380                          * last piece of data (when using padding), so the
 381                          * output needs to be buffered into a blocksize chunk
 382                          */
 383                         op->soc = 1;
 384                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 385                         op->dst.u.dma.offset = 0;
 386                         op->dst.u.dma.length = op->src.u.dma.length;
 387                 } else {
 388                         /* Enough room in the sg element, but we need to
 389                          * adjust for any previously used area
 390                          */
 391                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 392                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 393                         op->dst.u.dma.length = op->src.u.dma.length;
 394                 }
 395         }
 396 }
 397
 398 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 399                              struct ccp_op *op)
 400 {
 401         op->init = 0;
 402
 403         if (dst) {
 404                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 405                         ccp_empty_queue_buf(dst);
 406                 else
 407                         ccp_update_sg_workarea(&dst->sg_wa,
 408                                                op->dst.u.dma.length);
 409         }
 410 }
 411
 412 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 413                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 414                                u32 byte_swap, bool from)
 415 {
 416         struct ccp_op op;
 417
 418         memset(&op, 0, sizeof(op));
 419
 420         op.cmd_q = cmd_q;
 421         op.jobid = jobid;
 422         op.eom = 1;
 423
 424         if (from) {
 425                 op.soc = 1;
 426                 op.src.type = CCP_MEMTYPE_SB;
 427                 op.src.u.sb = sb;
 428                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 429                 op.dst.u.dma.address = wa->dma.address;
 430                 op.dst.u.dma.length = wa->length;
 431         } else {
 432                 op.src.type = CCP_MEMTYPE_SYSTEM;
 433                 op.src.u.dma.address = wa->dma.address;
 434                 op.src.u.dma.length = wa->length;
 435                 op.dst.type = CCP_MEMTYPE_SB;
 436                 op.dst.u.sb = sb;
 437         }
 438
 439         op.u.passthru.byte_swap = byte_swap;
 440
 441         return cmd_q->ccp->vdata->perform->passthru(&op);
 442 }
 443
 444 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 445                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 446                           u32 byte_swap)
 447 {
 448         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 449 }
 450
 451 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 452                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 453                             u32 byte_swap)
 454 {
 455         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 456 }
 457
 458 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 459                                 struct ccp_cmd *cmd)
 460 {
 461         struct ccp_aes_engine *aes = &cmd->u.aes;
 462         struct ccp_dm_workarea key, ctx;
 463         struct ccp_data src;
 464         struct ccp_op op;
 465         unsigned int dm_offset;
 466         int ret;
 467
 468         if (!((aes->key_len == AES_KEYSIZE_128) ||
 469               (aes->key_len == AES_KEYSIZE_192) ||
 470               (aes->key_len == AES_KEYSIZE_256)))
 471                 return -EINVAL;
 472
 473         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 474                 return -EINVAL;
 475
 476         if (aes->iv_len != AES_BLOCK_SIZE)
 477                 return -EINVAL;
 478
 479         if (!aes->key || !aes->iv || !aes->src)
 480                 return -EINVAL;
 481
 482         if (aes->cmac_final) {
 483                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 484                         return -EINVAL;
 485
 486                 if (!aes->cmac_key)
 487                         return -EINVAL;
 488         }
 489
 490         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 491         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 492
 493         ret = -EIO;
 494         memset(&op, 0, sizeof(op));
 495         op.cmd_q = cmd_q;
 496         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 497         op.sb_key = cmd_q->sb_key;
 498         op.sb_ctx = cmd_q->sb_ctx;
 499         op.init = 1;
 500         op.u.aes.type = aes->type;
 501         op.u.aes.mode = aes->mode;
 502         op.u.aes.action = aes->action;
 503
 504         /* All supported key sizes fit in a single (32-byte) SB entry
 505          * and must be in little endian format. Use the 256-bit byte
 506          * swap passthru option to convert from big endian to little
 507          * endian.
 508          */
 509         ret = ccp_init_dm_workarea(&key, cmd_q,
 510                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 511                                    DMA_TO_DEVICE);
 512         if (ret)
 513                 return ret;
 514
 515         dm_offset = CCP_SB_BYTES - aes->key_len;
 516         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 517         if (ret)
 518                 goto e_key;
 519         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 520                              CCP_PASSTHRU_BYTESWAP_256BIT);
 521         if (ret) {
 522                 cmd->engine_error = cmd_q->cmd_error;
 523                 goto e_key;
 524         }
 525
 526         /* The AES context fits in a single (32-byte) SB entry and
 527          * must be in little endian format. Use the 256-bit byte swap
 528          * passthru option to convert from big endian to little endian.
 529          */
 530         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 531                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 532                                    DMA_BIDIRECTIONAL);
 533         if (ret)
 534                 goto e_key;
 535
 536         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 537         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 538         if (ret)
 539                 goto e_ctx;
 540         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 541                              CCP_PASSTHRU_BYTESWAP_256BIT);
 542         if (ret) {
 543                 cmd->engine_error = cmd_q->cmd_error;
 544                 goto e_ctx;
 545         }
 546
 547         /* Send data to the CCP AES engine */
 548         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 549                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 550         if (ret)
 551                 goto e_ctx;
 552
 553         while (src.sg_wa.bytes_left) {
 554                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 555                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 556                         op.eom = 1;
 557
 558                         /* Push the K1/K2 key to the CCP now */
 559                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 560                                                op.sb_ctx,
 561                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 562                         if (ret) {
 563                                 cmd->engine_error = cmd_q->cmd_error;
 564                                 goto e_src;
 565                         }
 566
 567                         ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 568                                               aes->cmac_key_len);
 569                         if (ret)
 570                                 goto e_src;
 571                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 572                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 573                         if (ret) {
 574                                 cmd->engine_error = cmd_q->cmd_error;
 575                                 goto e_src;
 576                         }
 577                 }
 578
 579                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 580                 if (ret) {
 581                         cmd->engine_error = cmd_q->cmd_error;
 582                         goto e_src;
 583                 }
 584
 585                 ccp_process_data(&src, NULL, &op);
 586         }
 587
 588         /* Retrieve the AES context - convert from LE to BE using
 589          * 32-byte (256-bit) byteswapping
 590          */
 591         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 592                                CCP_PASSTHRU_BYTESWAP_256BIT);
 593         if (ret) {
 594                 cmd->engine_error = cmd_q->cmd_error;
 595                 goto e_src;
 596         }
 597
 598         /* ...but we only need AES_BLOCK_SIZE bytes */
 599         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 600         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 601
 602 e_src:
 603         ccp_free_data(&src, cmd_q);
 604
 605 e_ctx:
 606         ccp_dm_free(&ctx);
 607
 608 e_key:
 609         ccp_dm_free(&key);
 610
 611         return ret;
 612 }
 613
 614 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
 615                                struct ccp_cmd *cmd)
 616 {
 617         struct ccp_aes_engine *aes = &cmd->u.aes;
 618         struct ccp_dm_workarea key, ctx, final_wa, tag;
 619         struct ccp_data src, dst;
 620         struct ccp_data aad;
 621         struct ccp_op op;
 622
 623         unsigned long long *final;
 624         unsigned int dm_offset;
 625         unsigned int authsize;
 626         unsigned int jobid;
 627         unsigned int ilen;
 628         bool in_place = true; /* Default value */
 629         int ret;
 630
 631         struct scatterlist *p_inp, sg_inp[2];
 632         struct scatterlist *p_tag, sg_tag[2];
 633         struct scatterlist *p_outp, sg_outp[2];
 634         struct scatterlist *p_aad;
 635
 636         if (!aes->iv)
 637                 return -EINVAL;
 638
 639         if (!((aes->key_len == AES_KEYSIZE_128) ||
 640                 (aes->key_len == AES_KEYSIZE_192) ||
 641                 (aes->key_len == AES_KEYSIZE_256)))
 642                 return -EINVAL;
 643
 644         if (!aes->key) /* Gotta have a key SGL */
 645                 return -EINVAL;
 646
 647         /* Zero defaults to 16 bytes, the maximum size */
 648         authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
 649         switch (authsize) {
 650         case 16:
 651         case 15:
 652         case 14:
 653         case 13:
 654         case 12:
 655         case 8:
 656         case 4:
 657                 break;
 658         default:
 659                 return -EINVAL;
 660         }
 661
 662         /* First, decompose the source buffer into AAD & PT,
 663          * and the destination buffer into AAD, CT & tag, or
 664          * the input into CT & tag.
 665          * It is expected that the input and output SGs will
 666          * be valid, even if the AAD and input lengths are 0.
 667          */
 668         p_aad = aes->src;
 669         p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
 670         p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
 671         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 672                 ilen = aes->src_len;
 673                 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
 674         } else {
 675                 /* Input length for decryption includes tag */
 676                 ilen = aes->src_len - authsize;
 677                 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
 678         }
 679
 680         jobid = CCP_NEW_JOBID(cmd_q->ccp);
 681
 682         memset(&op, 0, sizeof(op));
 683         op.cmd_q = cmd_q;
 684         op.jobid = jobid;
 685         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 686         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 687         op.init = 1;
 688         op.u.aes.type = aes->type;
 689
 690         /* Copy the key to the LSB */
 691         ret = ccp_init_dm_workarea(&key, cmd_q,
 692                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 693                                    DMA_TO_DEVICE);
 694         if (ret)
 695                 return ret;
 696
 697         dm_offset = CCP_SB_BYTES - aes->key_len;
 698         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 699         if (ret)
 700                 goto e_key;
 701         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 702                              CCP_PASSTHRU_BYTESWAP_256BIT);
 703         if (ret) {
 704                 cmd->engine_error = cmd_q->cmd_error;
 705                 goto e_key;
 706         }
 707
 708         /* Copy the context (IV) to the LSB.
 709          * There is an assumption here that the IV is 96 bits in length, plus
 710          * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
 711          */
 712         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 713                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 714                                    DMA_BIDIRECTIONAL);
 715         if (ret)
 716                 goto e_key;
 717
 718         dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
 719         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 720         if (ret)
 721                 goto e_ctx;
 722
 723         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 724                              CCP_PASSTHRU_BYTESWAP_256BIT);
 725         if (ret) {
 726                 cmd->engine_error = cmd_q->cmd_error;
 727                 goto e_ctx;
 728         }
 729
 730         op.init = 1;
 731         if (aes->aad_len > 0) {
 732                 /* Step 1: Run a GHASH over the Additional Authenticated Data */
 733                 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
 734                                     AES_BLOCK_SIZE,
 735                                     DMA_TO_DEVICE);
 736                 if (ret)
 737                         goto e_ctx;
 738
 739                 op.u.aes.mode = CCP_AES_MODE_GHASH;
 740                 op.u.aes.action = CCP_AES_GHASHAAD;
 741
 742                 while (aad.sg_wa.bytes_left) {
 743                         ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
 744
 745                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 746                         if (ret) {
 747                                 cmd->engine_error = cmd_q->cmd_error;
 748                                 goto e_aad;
 749                         }
 750
 751                         ccp_process_data(&aad, NULL, &op);
 752                         op.init = 0;
 753                 }
 754         }
 755
 756         op.u.aes.mode = CCP_AES_MODE_GCTR;
 757         op.u.aes.action = aes->action;
 758
 759         if (ilen > 0) {
 760                 /* Step 2: Run a GCTR over the plaintext */
 761                 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
 762
 763                 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
 764                                     AES_BLOCK_SIZE,
 765                                     in_place ? DMA_BIDIRECTIONAL
 766                                              : DMA_TO_DEVICE);
 767                 if (ret)
 768                         goto e_ctx;
 769
 770                 if (in_place) {
 771                         dst = src;
 772                 } else {
 773                         ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
 774                                             AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 775                         if (ret)
 776                                 goto e_src;
 777                 }
 778
 779                 op.soc = 0;
 780                 op.eom = 0;
 781                 op.init = 1;
 782                 while (src.sg_wa.bytes_left) {
 783                         ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 784                         if (!src.sg_wa.bytes_left) {
 785                                 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
 786
 787                                 if (nbytes) {
 788                                         op.eom = 1;
 789                                         op.u.aes.size = (nbytes * 8) - 1;
 790                                 }
 791                         }
 792
 793                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 794                         if (ret) {
 795                                 cmd->engine_error = cmd_q->cmd_error;
 796                                 goto e_dst;
 797                         }
 798
 799                         ccp_process_data(&src, &dst, &op);
 800                         op.init = 0;
 801                 }
 802         }
 803
 804         /* Step 3: Update the IV portion of the context with the original IV */
 805         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 806                                CCP_PASSTHRU_BYTESWAP_256BIT);
 807         if (ret) {
 808                 cmd->engine_error = cmd_q->cmd_error;
 809                 goto e_dst;
 810         }
 811
 812         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 813         if (ret)
 814                 goto e_dst;
 815
 816         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 817                              CCP_PASSTHRU_BYTESWAP_256BIT);
 818         if (ret) {
 819                 cmd->engine_error = cmd_q->cmd_error;
 820                 goto e_dst;
 821         }
 822
 823         /* Step 4: Concatenate the lengths of the AAD and source, and
 824          * hash that 16 byte buffer.
 825          */
 826         ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
 827                                    DMA_BIDIRECTIONAL);
 828         if (ret)
 829                 goto e_dst;
 830         final = (unsigned long long *) final_wa.address;
 831         final[0] = cpu_to_be64(aes->aad_len * 8);
 832         final[1] = cpu_to_be64(ilen * 8);
 833
 834         memset(&op, 0, sizeof(op));
 835         op.cmd_q = cmd_q;
 836         op.jobid = jobid;
 837         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 838         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 839         op.init = 1;
 840         op.u.aes.type = aes->type;
 841         op.u.aes.mode = CCP_AES_MODE_GHASH;
 842         op.u.aes.action = CCP_AES_GHASHFINAL;
 843         op.src.type = CCP_MEMTYPE_SYSTEM;
 844         op.src.u.dma.address = final_wa.dma.address;
 845         op.src.u.dma.length = AES_BLOCK_SIZE;
 846         op.dst.type = CCP_MEMTYPE_SYSTEM;
 847         op.dst.u.dma.address = final_wa.dma.address;
 848         op.dst.u.dma.length = AES_BLOCK_SIZE;
 849         op.eom = 1;
 850         op.u.aes.size = 0;
 851         ret = cmd_q->ccp->vdata->perform->aes(&op);
 852         if (ret)
 853                 goto e_dst;
 854
 855         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 856                 /* Put the ciphered tag after the ciphertext. */
 857                 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
 858         } else {
 859                 /* Does this ciphered tag match the input? */
 860                 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
 861                                            DMA_BIDIRECTIONAL);
 862                 if (ret)
 863                         goto e_tag;
 864                 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
 865                 if (ret)
 866                         goto e_tag;
 867
 868                 ret = crypto_memneq(tag.address, final_wa.address,
 869                                     authsize) ? -EBADMSG : 0;
 870                 ccp_dm_free(&tag);
 871         }
 872
 873 e_tag:
 874         ccp_dm_free(&final_wa);
 875
 876 e_dst:
 877         if (ilen > 0 && !in_place)
 878                 ccp_free_data(&dst, cmd_q);
 879
 880 e_src:
 881         if (ilen > 0)
 882                 ccp_free_data(&src, cmd_q);
 883
 884 e_aad:
 885         if (aes->aad_len)
 886                 ccp_free_data(&aad, cmd_q);
 887
 888 e_ctx:
 889         ccp_dm_free(&ctx);
 890
 891 e_key:
 892         ccp_dm_free(&key);
 893
 894         return ret;
 895 }
 896
 897 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 898 {
 899         struct ccp_aes_engine *aes = &cmd->u.aes;
 900         struct ccp_dm_workarea key, ctx;
 901         struct ccp_data src, dst;
 902         struct ccp_op op;
 903         unsigned int dm_offset;
 904         bool in_place = false;
 905         int ret;
 906
 907         if (aes->mode == CCP_AES_MODE_CMAC)
 908                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
 909
 910         if (aes->mode == CCP_AES_MODE_GCM)
 911                 return ccp_run_aes_gcm_cmd(cmd_q, cmd);
 912
 913         if (!((aes->key_len == AES_KEYSIZE_128) ||
 914               (aes->key_len == AES_KEYSIZE_192) ||
 915               (aes->key_len == AES_KEYSIZE_256)))
 916                 return -EINVAL;
 917
 918         if (((aes->mode == CCP_AES_MODE_ECB) ||
 919              (aes->mode == CCP_AES_MODE_CBC)) &&
 920             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 921                 return -EINVAL;
 922
 923         if (!aes->key || !aes->src || !aes->dst)
 924                 return -EINVAL;
 925
 926         if (aes->mode != CCP_AES_MODE_ECB) {
 927                 if (aes->iv_len != AES_BLOCK_SIZE)
 928                         return -EINVAL;
 929
 930                 if (!aes->iv)
 931                         return -EINVAL;
 932         }
 933
 934         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 935         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 936
 937         ret = -EIO;
 938         memset(&op, 0, sizeof(op));
 939         op.cmd_q = cmd_q;
 940         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 941         op.sb_key = cmd_q->sb_key;
 942         op.sb_ctx = cmd_q->sb_ctx;
 943         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 944         op.u.aes.type = aes->type;
 945         op.u.aes.mode = aes->mode;
 946         op.u.aes.action = aes->action;
 947
 948         /* All supported key sizes fit in a single (32-byte) SB entry
 949          * and must be in little endian format. Use the 256-bit byte
 950          * swap passthru option to convert from big endian to little
 951          * endian.
 952          */
 953         ret = ccp_init_dm_workarea(&key, cmd_q,
 954                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 955                                    DMA_TO_DEVICE);
 956         if (ret)
 957                 return ret;
 958
 959         dm_offset = CCP_SB_BYTES - aes->key_len;
 960         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 961         if (ret)
 962                 goto e_key;
 963         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 964                              CCP_PASSTHRU_BYTESWAP_256BIT);
 965         if (ret) {
 966                 cmd->engine_error = cmd_q->cmd_error;
 967                 goto e_key;
 968         }
 969
 970         /* The AES context fits in a single (32-byte) SB entry and
 971          * must be in little endian format. Use the 256-bit byte swap
 972          * passthru option to convert from big endian to little endian.
 973          */
 974         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 975                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 976                                    DMA_BIDIRECTIONAL);
 977         if (ret)
 978                 goto e_key;
 979
 980         if (aes->mode != CCP_AES_MODE_ECB) {
 981                 /* Load the AES context - convert to LE */
 982                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 983                 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 984                 if (ret)
 985                         goto e_ctx;
 986                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 987                                      CCP_PASSTHRU_BYTESWAP_256BIT);
 988                 if (ret) {
 989                         cmd->engine_error = cmd_q->cmd_error;
 990                         goto e_ctx;
 991                 }
 992         }
 993         switch (aes->mode) {
 994         case CCP_AES_MODE_CFB: /* CFB128 only */
 995         case CCP_AES_MODE_CTR:
 996                 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
 997                 break;
 998         default:
 999                 op.u.aes.size = 0;
1000         }
1001
1002         /* Prepare the input and output data workareas. For in-place
1003          * operations we need to set the dma direction to BIDIRECTIONAL
1004          * and copy the src workarea to the dst workarea.
1005          */
1006         if (sg_virt(aes->src) == sg_virt(aes->dst))
1007                 in_place = true;
1008
1009         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1010                             AES_BLOCK_SIZE,
1011                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1012         if (ret)
1013                 goto e_ctx;
1014
1015         if (in_place) {
1016                 dst = src;
1017         } else {
1018                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1019                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1020                 if (ret)
1021                         goto e_src;
1022         }
1023
1024         /* Send data to the CCP AES engine */
1025         while (src.sg_wa.bytes_left) {
1026                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1027                 if (!src.sg_wa.bytes_left) {
1028                         op.eom = 1;
1029
1030                         /* Since we don't retrieve the AES context in ECB
1031                          * mode we have to wait for the operation to complete
1032                          * on the last piece of data
1033                          */
1034                         if (aes->mode == CCP_AES_MODE_ECB)
1035                                 op.soc = 1;
1036                 }
1037
1038                 ret = cmd_q->ccp->vdata->perform->aes(&op);
1039                 if (ret) {
1040                         cmd->engine_error = cmd_q->cmd_error;
1041                         goto e_dst;
1042                 }
1043
1044                 ccp_process_data(&src, &dst, &op);
1045         }
1046
1047         if (aes->mode != CCP_AES_MODE_ECB) {
1048                 /* Retrieve the AES context - convert from LE to BE using
1049                  * 32-byte (256-bit) byteswapping
1050                  */
1051                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1052                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1053                 if (ret) {
1054                         cmd->engine_error = cmd_q->cmd_error;
1055                         goto e_dst;
1056                 }
1057
1058                 /* ...but we only need AES_BLOCK_SIZE bytes */
1059                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1060                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1061         }
1062
1063 e_dst:
1064         if (!in_place)
1065                 ccp_free_data(&dst, cmd_q);
1066
1067 e_src:
1068         ccp_free_data(&src, cmd_q);
1069
1070 e_ctx:
1071         ccp_dm_free(&ctx);
1072
1073 e_key:
1074         ccp_dm_free(&key);
1075
1076         return ret;
1077 }
1078
1079 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1080                                struct ccp_cmd *cmd)
1081 {
1082         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1083         struct ccp_dm_workarea key, ctx;
1084         struct ccp_data src, dst;
1085         struct ccp_op op;
1086         unsigned int unit_size, dm_offset;
1087         bool in_place = false;
1088         unsigned int sb_count;
1089         enum ccp_aes_type aestype;
1090         int ret;
1091
1092         switch (xts->unit_size) {
1093         case CCP_XTS_AES_UNIT_SIZE_16:
1094                 unit_size = 16;
1095                 break;
1096         case CCP_XTS_AES_UNIT_SIZE_512:
1097                 unit_size = 512;
1098                 break;
1099         case CCP_XTS_AES_UNIT_SIZE_1024:
1100                 unit_size = 1024;
1101                 break;
1102         case CCP_XTS_AES_UNIT_SIZE_2048:
1103                 unit_size = 2048;
1104                 break;
1105         case CCP_XTS_AES_UNIT_SIZE_4096:
1106                 unit_size = 4096;
1107                 break;
1108
1109         default:
1110                 return -EINVAL;
1111         }
1112
1113         if (xts->key_len == AES_KEYSIZE_128)
1114                 aestype = CCP_AES_TYPE_128;
1115         else if (xts->key_len == AES_KEYSIZE_256)
1116                 aestype = CCP_AES_TYPE_256;
1117         else
1118                 return -EINVAL;
1119
1120         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1121                 return -EINVAL;
1122
1123         if (xts->iv_len != AES_BLOCK_SIZE)
1124                 return -EINVAL;
1125
1126         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1127                 return -EINVAL;
1128
1129         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1130         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1131
1132         ret = -EIO;
1133         memset(&op, 0, sizeof(op));
1134         op.cmd_q = cmd_q;
1135         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1136         op.sb_key = cmd_q->sb_key;
1137         op.sb_ctx = cmd_q->sb_ctx;
1138         op.init = 1;
1139         op.u.xts.type = aestype;
1140         op.u.xts.action = xts->action;
1141         op.u.xts.unit_size = xts->unit_size;
1142
1143         /* A version 3 device only supports 128-bit keys, which fits into a
1144          * single SB entry. A version 5 device uses a 512-bit vector, so two
1145          * SB entries.
1146          */
1147         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1148                 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1149         else
1150                 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1151         ret = ccp_init_dm_workarea(&key, cmd_q,
1152                                    sb_count * CCP_SB_BYTES,
1153                                    DMA_TO_DEVICE);
1154         if (ret)
1155                 return ret;
1156
1157         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1158                 /* All supported key sizes must be in little endian format.
1159                  * Use the 256-bit byte swap passthru option to convert from
1160                  * big endian to little endian.
1161                  */
1162                 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1163                 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1164                 if (ret)
1165                         goto e_key;
1166                 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1167                 if (ret)
1168                         goto e_key;
1169         } else {
1170                 /* Version 5 CCPs use a 512-bit space for the key: each portion
1171                  * occupies 256 bits, or one entire slot, and is zero-padded.
1172                  */
1173                 unsigned int pad;
1174
1175                 dm_offset = CCP_SB_BYTES;
1176                 pad = dm_offset - xts->key_len;
1177                 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1178                 if (ret)
1179                         goto e_key;
1180                 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1181                                       xts->key_len, xts->key_len);
1182                 if (ret)
1183                         goto e_key;
1184         }
1185         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1186                              CCP_PASSTHRU_BYTESWAP_256BIT);
1187         if (ret) {
1188                 cmd->engine_error = cmd_q->cmd_error;
1189                 goto e_key;
1190         }
1191
1192         /* The AES context fits in a single (32-byte) SB entry and
1193          * for XTS is already in little endian format so no byte swapping
1194          * is needed.
1195          */
1196         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1197                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1198                                    DMA_BIDIRECTIONAL);
1199         if (ret)
1200                 goto e_key;
1201
1202         ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1203         if (ret)
1204                 goto e_ctx;
1205         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1206                              CCP_PASSTHRU_BYTESWAP_NOOP);
1207         if (ret) {
1208                 cmd->engine_error = cmd_q->cmd_error;
1209                 goto e_ctx;
1210         }
1211
1212         /* Prepare the input and output data workareas. For in-place
1213          * operations we need to set the dma direction to BIDIRECTIONAL
1214          * and copy the src workarea to the dst workarea.
1215          */
1216         if (sg_virt(xts->src) == sg_virt(xts->dst))
1217                 in_place = true;
1218
1219         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1220                             unit_size,
1221                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1222         if (ret)
1223                 goto e_ctx;
1224
1225         if (in_place) {
1226                 dst = src;
1227         } else {
1228                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1229                                     unit_size, DMA_FROM_DEVICE);
1230                 if (ret)
1231                         goto e_src;
1232         }
1233
1234         /* Send data to the CCP AES engine */
1235         while (src.sg_wa.bytes_left) {
1236                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1237                 if (!src.sg_wa.bytes_left)
1238                         op.eom = 1;
1239
1240                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1241                 if (ret) {
1242                         cmd->engine_error = cmd_q->cmd_error;
1243                         goto e_dst;
1244                 }
1245
1246                 ccp_process_data(&src, &dst, &op);
1247         }
1248
1249         /* Retrieve the AES context - convert from LE to BE using
1250          * 32-byte (256-bit) byteswapping
1251          */
1252         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1253                                CCP_PASSTHRU_BYTESWAP_256BIT);
1254         if (ret) {
1255                 cmd->engine_error = cmd_q->cmd_error;
1256                 goto e_dst;
1257         }
1258
1259         /* ...but we only need AES_BLOCK_SIZE bytes */
1260         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1261         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1262
1263 e_dst:
1264         if (!in_place)
1265                 ccp_free_data(&dst, cmd_q);
1266
1267 e_src:
1268         ccp_free_data(&src, cmd_q);
1269
1270 e_ctx:
1271         ccp_dm_free(&ctx);
1272
1273 e_key:
1274         ccp_dm_free(&key);
1275
1276         return ret;
1277 }
1278
1279 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1280 {
1281         struct ccp_des3_engine *des3 = &cmd->u.des3;
1282
1283         struct ccp_dm_workarea key, ctx;
1284         struct ccp_data src, dst;
1285         struct ccp_op op;
1286         unsigned int dm_offset;
1287         unsigned int len_singlekey;
1288         bool in_place = false;
1289         int ret;
1290
1291         /* Error checks */
1292         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1293                 return -EINVAL;
1294
1295         if (!cmd_q->ccp->vdata->perform->des3)
1296                 return -EINVAL;
1297
1298         if (des3->key_len != DES3_EDE_KEY_SIZE)
1299                 return -EINVAL;
1300
1301         if (((des3->mode == CCP_DES3_MODE_ECB) ||
1302                 (des3->mode == CCP_DES3_MODE_CBC)) &&
1303                 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1304                 return -EINVAL;
1305
1306         if (!des3->key || !des3->src || !des3->dst)
1307                 return -EINVAL;
1308
1309         if (des3->mode != CCP_DES3_MODE_ECB) {
1310                 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1311                         return -EINVAL;
1312
1313                 if (!des3->iv)
1314                         return -EINVAL;
1315         }
1316
1317         ret = -EIO;
1318         /* Zero out all the fields of the command desc */
1319         memset(&op, 0, sizeof(op));
1320
1321         /* Set up the Function field */
1322         op.cmd_q = cmd_q;
1323         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1324         op.sb_key = cmd_q->sb_key;
1325
1326         op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1327         op.u.des3.type = des3->type;
1328         op.u.des3.mode = des3->mode;
1329         op.u.des3.action = des3->action;
1330
1331         /*
1332          * All supported key sizes fit in a single (32-byte) KSB entry and
1333          * (like AES) must be in little endian format. Use the 256-bit byte
1334          * swap passthru option to convert from big endian to little endian.
1335          */
1336         ret = ccp_init_dm_workarea(&key, cmd_q,
1337                                    CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1338                                    DMA_TO_DEVICE);
1339         if (ret)
1340                 return ret;
1341
1342         /*
1343          * The contents of the key triplet are in the reverse order of what
1344          * is required by the engine. Copy the 3 pieces individually to put
1345          * them where they belong.
1346          */
1347         dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1348
1349         len_singlekey = des3->key_len / 3;
1350         ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1351                               des3->key, 0, len_singlekey);
1352         if (ret)
1353                 goto e_key;
1354         ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1355                               des3->key, len_singlekey, len_singlekey);
1356         if (ret)
1357                 goto e_key;
1358         ret = ccp_set_dm_area(&key, dm_offset,
1359                               des3->key, 2 * len_singlekey, len_singlekey);
1360         if (ret)
1361                 goto e_key;
1362
1363         /* Copy the key to the SB */
1364         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1365                              CCP_PASSTHRU_BYTESWAP_256BIT);
1366         if (ret) {
1367                 cmd->engine_error = cmd_q->cmd_error;
1368                 goto e_key;
1369         }
1370
1371         /*
1372          * The DES3 context fits in a single (32-byte) KSB entry and
1373          * must be in little endian format. Use the 256-bit byte swap
1374          * passthru option to convert from big endian to little endian.
1375          */
1376         if (des3->mode != CCP_DES3_MODE_ECB) {
1377                 op.sb_ctx = cmd_q->sb_ctx;
1378
1379                 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1380                                            CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1381                                            DMA_BIDIRECTIONAL);
1382                 if (ret)
1383                         goto e_key;
1384
1385                 /* Load the context into the LSB */
1386                 dm_offset = CCP_SB_BYTES - des3->iv_len;
1387                 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1388                                       des3->iv_len);
1389                 if (ret)
1390                         goto e_ctx;
1391
1392                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1393                                      CCP_PASSTHRU_BYTESWAP_256BIT);
1394                 if (ret) {
1395                         cmd->engine_error = cmd_q->cmd_error;
1396                         goto e_ctx;
1397                 }
1398         }
1399
1400         /*
1401          * Prepare the input and output data workareas. For in-place
1402          * operations we need to set the dma direction to BIDIRECTIONAL
1403          * and copy the src workarea to the dst workarea.
1404          */
1405         if (sg_virt(des3->src) == sg_virt(des3->dst))
1406                 in_place = true;
1407
1408         ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1409                         DES3_EDE_BLOCK_SIZE,
1410                         in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1411         if (ret)
1412                 goto e_ctx;
1413
1414         if (in_place)
1415                 dst = src;
1416         else {
1417                 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1418                                 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1419                 if (ret)
1420                         goto e_src;
1421         }
1422
1423         /* Send data to the CCP DES3 engine */
1424         while (src.sg_wa.bytes_left) {
1425                 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1426                 if (!src.sg_wa.bytes_left) {
1427                         op.eom = 1;
1428
1429                         /* Since we don't retrieve the context in ECB mode
1430                          * we have to wait for the operation to complete
1431                          * on the last piece of data
1432                          */
1433                         op.soc = 0;
1434                 }
1435
1436                 ret = cmd_q->ccp->vdata->perform->des3(&op);
1437                 if (ret) {
1438                         cmd->engine_error = cmd_q->cmd_error;
1439                         goto e_dst;
1440                 }
1441
1442                 ccp_process_data(&src, &dst, &op);
1443         }
1444
1445         if (des3->mode != CCP_DES3_MODE_ECB) {
1446                 /* Retrieve the context and make BE */
1447                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1448                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1449                 if (ret) {
1450                         cmd->engine_error = cmd_q->cmd_error;
1451                         goto e_dst;
1452                 }
1453
1454                 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1455                 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1456                                 DES3_EDE_BLOCK_SIZE);
1457         }
1458 e_dst:
1459         if (!in_place)
1460                 ccp_free_data(&dst, cmd_q);
1461
1462 e_src:
1463         ccp_free_data(&src, cmd_q);
1464
1465 e_ctx:
1466         if (des3->mode != CCP_DES3_MODE_ECB)
1467                 ccp_dm_free(&ctx);
1468
1469 e_key:
1470         ccp_dm_free(&key);
1471
1472         return ret;
1473 }
1474
1475 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1476 {
1477         struct ccp_sha_engine *sha = &cmd->u.sha;
1478         struct ccp_dm_workarea ctx;
1479         struct ccp_data src;
1480         struct ccp_op op;
1481         unsigned int ioffset, ooffset;
1482         unsigned int digest_size;
1483         int sb_count;
1484         const void *init;
1485         u64 block_size;
1486         int ctx_size;
1487         int ret;
1488
1489         switch (sha->type) {
1490         case CCP_SHA_TYPE_1:
1491                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1492                         return -EINVAL;
1493                 block_size = SHA1_BLOCK_SIZE;
1494                 break;
1495         case CCP_SHA_TYPE_224:
1496                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1497                         return -EINVAL;
1498                 block_size = SHA224_BLOCK_SIZE;
1499                 break;
1500         case CCP_SHA_TYPE_256:
1501                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1502                         return -EINVAL;
1503                 block_size = SHA256_BLOCK_SIZE;
1504                 break;
1505         case CCP_SHA_TYPE_384:
1506                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1507                     || sha->ctx_len < SHA384_DIGEST_SIZE)
1508                         return -EINVAL;
1509                 block_size = SHA384_BLOCK_SIZE;
1510                 break;
1511         case CCP_SHA_TYPE_512:
1512                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1513                     || sha->ctx_len < SHA512_DIGEST_SIZE)
1514                         return -EINVAL;
1515                 block_size = SHA512_BLOCK_SIZE;
1516                 break;
1517         default:
1518                 return -EINVAL;
1519         }
1520
1521         if (!sha->ctx)
1522                 return -EINVAL;
1523
1524         if (!sha->final && (sha->src_len & (block_size - 1)))
1525                 return -EINVAL;
1526
1527         /* The version 3 device can't handle zero-length input */
1528         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1529
1530                 if (!sha->src_len) {
1531                         unsigned int digest_len;
1532                         const u8 *sha_zero;
1533
1534                         /* Not final, just return */
1535                         if (!sha->final)
1536                                 return 0;
1537
1538                         /* CCP can't do a zero length sha operation so the
1539                          * caller must buffer the data.
1540                          */
1541                         if (sha->msg_bits)
1542                                 return -EINVAL;
1543
1544                         /* The CCP cannot perform zero-length sha operations
1545                          * so the caller is required to buffer data for the
1546                          * final operation. However, a sha operation for a
1547                          * message with a total length of zero is valid so
1548                          * known values are required to supply the result.
1549                          */
1550                         switch (sha->type) {
1551                         case CCP_SHA_TYPE_1:
1552                                 sha_zero = sha1_zero_message_hash;
1553                                 digest_len = SHA1_DIGEST_SIZE;
1554                                 break;
1555                         case CCP_SHA_TYPE_224:
1556                                 sha_zero = sha224_zero_message_hash;
1557                                 digest_len = SHA224_DIGEST_SIZE;
1558                                 break;
1559                         case CCP_SHA_TYPE_256:
1560                                 sha_zero = sha256_zero_message_hash;
1561                                 digest_len = SHA256_DIGEST_SIZE;
1562                                 break;
1563                         default:
1564                                 return -EINVAL;
1565                         }
1566
1567                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1568                                                  digest_len, 1);
1569
1570                         return 0;
1571                 }
1572         }
1573
1574         /* Set variables used throughout */
1575         switch (sha->type) {
1576         case CCP_SHA_TYPE_1:
1577                 digest_size = SHA1_DIGEST_SIZE;
1578                 init = (void *) ccp_sha1_init;
1579                 ctx_size = SHA1_DIGEST_SIZE;
1580                 sb_count = 1;
1581                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1582                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1583                 else
1584                         ooffset = ioffset = 0;
1585                 break;
1586         case CCP_SHA_TYPE_224:
1587                 digest_size = SHA224_DIGEST_SIZE;
1588                 init = (void *) ccp_sha224_init;
1589                 ctx_size = SHA256_DIGEST_SIZE;
1590                 sb_count = 1;
1591                 ioffset = 0;
1592                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1593                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1594                 else
1595                         ooffset = 0;
1596                 break;
1597         case CCP_SHA_TYPE_256:
1598                 digest_size = SHA256_DIGEST_SIZE;
1599                 init = (void *) ccp_sha256_init;
1600                 ctx_size = SHA256_DIGEST_SIZE;
1601                 sb_count = 1;
1602                 ooffset = ioffset = 0;
1603                 break;
1604         case CCP_SHA_TYPE_384:
1605                 digest_size = SHA384_DIGEST_SIZE;
1606                 init = (void *) ccp_sha384_init;
1607                 ctx_size = SHA512_DIGEST_SIZE;
1608                 sb_count = 2;
1609                 ioffset = 0;
1610                 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1611                 break;
1612         case CCP_SHA_TYPE_512:
1613                 digest_size = SHA512_DIGEST_SIZE;
1614                 init = (void *) ccp_sha512_init;
1615                 ctx_size = SHA512_DIGEST_SIZE;
1616                 sb_count = 2;
1617                 ooffset = ioffset = 0;
1618                 break;
1619         default:
1620                 ret = -EINVAL;
1621                 goto e_data;
1622         }
1623
1624         /* For zero-length plaintext the src pointer is ignored;
1625          * otherwise both parts must be valid
1626          */
1627         if (sha->src_len && !sha->src)
1628                 return -EINVAL;
1629
1630         memset(&op, 0, sizeof(op));
1631         op.cmd_q = cmd_q;
1632         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1633         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1634         op.u.sha.type = sha->type;
1635         op.u.sha.msg_bits = sha->msg_bits;
1636
1637         /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1638          * SHA384/512 require 2 adjacent SB slots, with the right half in the
1639          * first slot, and the left half in the second. Each portion must then
1640          * be in little endian format: use the 256-bit byte swap option.
1641          */
1642         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1643                                    DMA_BIDIRECTIONAL);
1644         if (ret)
1645                 return ret;
1646         if (sha->first) {
1647                 switch (sha->type) {
1648                 case CCP_SHA_TYPE_1:
1649                 case CCP_SHA_TYPE_224:
1650                 case CCP_SHA_TYPE_256:
1651                         memcpy(ctx.address + ioffset, init, ctx_size);
1652                         break;
1653                 case CCP_SHA_TYPE_384:
1654                 case CCP_SHA_TYPE_512:
1655                         memcpy(ctx.address + ctx_size / 2, init,
1656                                ctx_size / 2);
1657                         memcpy(ctx.address, init + ctx_size / 2,
1658                                ctx_size / 2);
1659                         break;
1660                 default:
1661                         ret = -EINVAL;
1662                         goto e_ctx;
1663                 }
1664         } else {
1665                 /* Restore the context */
1666                 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1667                                       sb_count * CCP_SB_BYTES);
1668                 if (ret)
1669                         goto e_ctx;
1670         }
1671
1672         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1673                              CCP_PASSTHRU_BYTESWAP_256BIT);
1674         if (ret) {
1675                 cmd->engine_error = cmd_q->cmd_error;
1676                 goto e_ctx;
1677         }
1678
1679         if (sha->src) {
1680                 /* Send data to the CCP SHA engine; block_size is set above */
1681                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1682                                     block_size, DMA_TO_DEVICE);
1683                 if (ret)
1684                         goto e_ctx;
1685
1686                 while (src.sg_wa.bytes_left) {
1687                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1688                         if (sha->final && !src.sg_wa.bytes_left)
1689                                 op.eom = 1;
1690
1691                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1692                         if (ret) {
1693                                 cmd->engine_error = cmd_q->cmd_error;
1694                                 goto e_data;
1695                         }
1696
1697                         ccp_process_data(&src, NULL, &op);
1698                 }
1699         } else {
1700                 op.eom = 1;
1701                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1702                 if (ret) {
1703                         cmd->engine_error = cmd_q->cmd_error;
1704                         goto e_data;
1705                 }
1706         }
1707
1708         /* Retrieve the SHA context - convert from LE to BE using
1709          * 32-byte (256-bit) byteswapping to BE
1710          */
1711         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1712                                CCP_PASSTHRU_BYTESWAP_256BIT);
1713         if (ret) {
1714                 cmd->engine_error = cmd_q->cmd_error;
1715                 goto e_data;
1716         }
1717
1718         if (sha->final) {
1719                 /* Finishing up, so get the digest */
1720                 switch (sha->type) {
1721                 case CCP_SHA_TYPE_1:
1722                 case CCP_SHA_TYPE_224:
1723                 case CCP_SHA_TYPE_256:
1724                         ccp_get_dm_area(&ctx, ooffset,
1725                                         sha->ctx, 0,
1726                                         digest_size);
1727                         break;
1728                 case CCP_SHA_TYPE_384:
1729                 case CCP_SHA_TYPE_512:
1730                         ccp_get_dm_area(&ctx, 0,
1731                                         sha->ctx, LSB_ITEM_SIZE - ooffset,
1732                                         LSB_ITEM_SIZE);
1733                         ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1734                                         sha->ctx, 0,
1735                                         LSB_ITEM_SIZE - ooffset);
1736                         break;
1737                 default:
1738                         ret = -EINVAL;
1739                         goto e_ctx;
1740                 }
1741         } else {
1742                 /* Stash the context */
1743                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1744                                 sb_count * CCP_SB_BYTES);
1745         }
1746
1747         if (sha->final && sha->opad) {
1748                 /* HMAC operation, recursively perform final SHA */
1749                 struct ccp_cmd hmac_cmd;
1750                 struct scatterlist sg;
1751                 u8 *hmac_buf;
1752
1753                 if (sha->opad_len != block_size) {
1754                         ret = -EINVAL;
1755                         goto e_data;
1756                 }
1757
1758                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1759                 if (!hmac_buf) {
1760                         ret = -ENOMEM;
1761                         goto e_data;
1762                 }
1763                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1764
1765                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1766                 switch (sha->type) {
1767                 case CCP_SHA_TYPE_1:
1768                 case CCP_SHA_TYPE_224:
1769                 case CCP_SHA_TYPE_256:
1770                         memcpy(hmac_buf + block_size,
1771                                ctx.address + ooffset,
1772                                digest_size);
1773                         break;
1774                 case CCP_SHA_TYPE_384:
1775                 case CCP_SHA_TYPE_512:
1776                         memcpy(hmac_buf + block_size,
1777                                ctx.address + LSB_ITEM_SIZE + ooffset,
1778                                LSB_ITEM_SIZE);
1779                         memcpy(hmac_buf + block_size +
1780                                (LSB_ITEM_SIZE - ooffset),
1781                                ctx.address,
1782                                LSB_ITEM_SIZE);
1783                         break;
1784                 default:
1785                         ret = -EINVAL;
1786                         goto e_ctx;
1787                 }
1788
1789                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1790                 hmac_cmd.engine = CCP_ENGINE_SHA;
1791                 hmac_cmd.u.sha.type = sha->type;
1792                 hmac_cmd.u.sha.ctx = sha->ctx;
1793                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1794                 hmac_cmd.u.sha.src = &sg;
1795                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1796                 hmac_cmd.u.sha.opad = NULL;
1797                 hmac_cmd.u.sha.opad_len = 0;
1798                 hmac_cmd.u.sha.first = 1;
1799                 hmac_cmd.u.sha.final = 1;
1800                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1801
1802                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1803                 if (ret)
1804                         cmd->engine_error = hmac_cmd.engine_error;
1805
1806                 kfree(hmac_buf);
1807         }
1808
1809 e_data:
1810         if (sha->src)
1811                 ccp_free_data(&src, cmd_q);
1812
1813 e_ctx:
1814         ccp_dm_free(&ctx);
1815
1816         return ret;
1817 }
1818
1819 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1820 {
1821         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1822         struct ccp_dm_workarea exp, src, dst;
1823         struct ccp_op op;
1824         unsigned int sb_count, i_len, o_len;
1825         int ret;
1826
1827         /* Check against the maximum allowable size, in bits */
1828         if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1829                 return -EINVAL;
1830
1831         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1832                 return -EINVAL;
1833
1834         memset(&op, 0, sizeof(op));
1835         op.cmd_q = cmd_q;
1836         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1837
1838         /* The RSA modulus must precede the message being acted upon, so
1839          * it must be copied to a DMA area where the message and the
1840          * modulus can be concatenated.  Therefore the input buffer
1841          * length required is twice the output buffer length (which
1842          * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
1843          * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1844          * required.
1845          */
1846         o_len = 32 * ((rsa->key_size + 255) / 256);
1847         i_len = o_len * 2;
1848
1849         sb_count = 0;
1850         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1851                 /* sb_count is the number of storage block slots required
1852                  * for the modulus.
1853                  */
1854                 sb_count = o_len / CCP_SB_BYTES;
1855                 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1856                                                                 sb_count);
1857                 if (!op.sb_key)
1858                         return -EIO;
1859         } else {
1860                 /* A version 5 device allows a modulus size that will not fit
1861                  * in the LSB, so the command will transfer it from memory.
1862                  * Set the sb key to the default, even though it's not used.
1863                  */
1864                 op.sb_key = cmd_q->sb_key;
1865         }
1866
1867         /* The RSA exponent must be in little endian format. Reverse its
1868          * byte order.
1869          */
1870         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1871         if (ret)
1872                 goto e_sb;
1873
1874         ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1875         if (ret)
1876                 goto e_exp;
1877
1878         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1879                 /* Copy the exponent to the local storage block, using
1880                  * as many 32-byte blocks as were allocated above. It's
1881                  * already little endian, so no further change is required.
1882                  */
1883                 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1884                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1885                 if (ret) {
1886                         cmd->engine_error = cmd_q->cmd_error;
1887                         goto e_exp;
1888                 }
1889         } else {
1890                 /* The exponent can be retrieved from memory via DMA. */
1891                 op.exp.u.dma.address = exp.dma.address;
1892                 op.exp.u.dma.offset = 0;
1893         }
1894
1895         /* Concatenate the modulus and the message. Both the modulus and
1896          * the operands must be in little endian format.  Since the input
1897          * is in big endian format it must be converted.
1898          */
1899         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1900         if (ret)
1901                 goto e_exp;
1902
1903         ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1904         if (ret)
1905                 goto e_src;
1906         ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1907         if (ret)
1908                 goto e_src;
1909
1910         /* Prepare the output area for the operation */
1911         ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1912         if (ret)
1913                 goto e_src;
1914
1915         op.soc = 1;
1916         op.src.u.dma.address = src.dma.address;
1917         op.src.u.dma.offset = 0;
1918         op.src.u.dma.length = i_len;
1919         op.dst.u.dma.address = dst.dma.address;
1920         op.dst.u.dma.offset = 0;
1921         op.dst.u.dma.length = o_len;
1922
1923         op.u.rsa.mod_size = rsa->key_size;
1924         op.u.rsa.input_len = i_len;
1925
1926         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1927         if (ret) {
1928                 cmd->engine_error = cmd_q->cmd_error;
1929                 goto e_dst;
1930         }
1931
1932         ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1933
1934 e_dst:
1935         ccp_dm_free(&dst);
1936
1937 e_src:
1938         ccp_dm_free(&src);
1939
1940 e_exp:
1941         ccp_dm_free(&exp);
1942
1943 e_sb:
1944         if (sb_count)
1945                 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1946
1947         return ret;
1948 }
1949
1950 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1951                                 struct ccp_cmd *cmd)
1952 {
1953         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1954         struct ccp_dm_workarea mask;
1955         struct ccp_data src, dst;
1956         struct ccp_op op;
1957         bool in_place = false;
1958         unsigned int i;
1959         int ret = 0;
1960
1961         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1962                 return -EINVAL;
1963
1964         if (!pt->src || !pt->dst)
1965                 return -EINVAL;
1966
1967         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1968                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1969                         return -EINVAL;
1970                 if (!pt->mask)
1971                         return -EINVAL;
1972         }
1973
1974         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1975
1976         memset(&op, 0, sizeof(op));
1977         op.cmd_q = cmd_q;
1978         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1979
1980         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1981                 /* Load the mask */
1982                 op.sb_key = cmd_q->sb_key;
1983
1984                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1985                                            CCP_PASSTHRU_SB_COUNT *
1986                                            CCP_SB_BYTES,
1987                                            DMA_TO_DEVICE);
1988                 if (ret)
1989                         return ret;
1990
1991                 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1992                 if (ret)
1993                         goto e_mask;
1994                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1995                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1996                 if (ret) {
1997                         cmd->engine_error = cmd_q->cmd_error;
1998                         goto e_mask;
1999                 }
2000         }
2001
2002         /* Prepare the input and output data workareas. For in-place
2003          * operations we need to set the dma direction to BIDIRECTIONAL
2004          * and copy the src workarea to the dst workarea.
2005          */
2006         if (sg_virt(pt->src) == sg_virt(pt->dst))
2007                 in_place = true;
2008
2009         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2010                             CCP_PASSTHRU_MASKSIZE,
2011                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2012         if (ret)
2013                 goto e_mask;
2014
2015         if (in_place) {
2016                 dst = src;
2017         } else {
2018                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2019                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2020                 if (ret)
2021                         goto e_src;
2022         }
2023
2024         /* Send data to the CCP Passthru engine
2025          *   Because the CCP engine works on a single source and destination
2026          *   dma address at a time, each entry in the source scatterlist
2027          *   (after the dma_map_sg call) must be less than or equal to the
2028          *   (remaining) length in the destination scatterlist entry and the
2029          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2030          */
2031         dst.sg_wa.sg_used = 0;
2032         for (i = 1; i <= src.sg_wa.dma_count; i++) {
2033                 if (!dst.sg_wa.sg ||
2034                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
2035                         ret = -EINVAL;
2036                         goto e_dst;
2037                 }
2038
2039                 if (i == src.sg_wa.dma_count) {
2040                         op.eom = 1;
2041                         op.soc = 1;
2042                 }
2043
2044                 op.src.type = CCP_MEMTYPE_SYSTEM;
2045                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2046                 op.src.u.dma.offset = 0;
2047                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2048
2049                 op.dst.type = CCP_MEMTYPE_SYSTEM;
2050                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2051                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2052                 op.dst.u.dma.length = op.src.u.dma.length;
2053
2054                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2055                 if (ret) {
2056                         cmd->engine_error = cmd_q->cmd_error;
2057                         goto e_dst;
2058                 }
2059
2060                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
2061                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
2062                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2063                         dst.sg_wa.sg_used = 0;
2064                 }
2065                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2066         }
2067
2068 e_dst:
2069         if (!in_place)
2070                 ccp_free_data(&dst, cmd_q);
2071
2072 e_src:
2073         ccp_free_data(&src, cmd_q);
2074
2075 e_mask:
2076         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2077                 ccp_dm_free(&mask);
2078
2079         return ret;
2080 }
2081
2082 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2083                                       struct ccp_cmd *cmd)
2084 {
2085         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2086         struct ccp_dm_workarea mask;
2087         struct ccp_op op;
2088         int ret;
2089
2090         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2091                 return -EINVAL;
2092
2093         if (!pt->src_dma || !pt->dst_dma)
2094                 return -EINVAL;
2095
2096         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2097                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2098                         return -EINVAL;
2099                 if (!pt->mask)
2100                         return -EINVAL;
2101         }
2102
2103         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2104
2105         memset(&op, 0, sizeof(op));
2106         op.cmd_q = cmd_q;
2107         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2108
2109         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2110                 /* Load the mask */
2111                 op.sb_key = cmd_q->sb_key;
2112
2113                 mask.length = pt->mask_len;
2114                 mask.dma.address = pt->mask;
2115                 mask.dma.length = pt->mask_len;
2116
2117                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2118                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2119                 if (ret) {
2120                         cmd->engine_error = cmd_q->cmd_error;
2121                         return ret;
2122                 }
2123         }
2124
2125         /* Send data to the CCP Passthru engine */
2126         op.eom = 1;
2127         op.soc = 1;
2128
2129         op.src.type = CCP_MEMTYPE_SYSTEM;
2130         op.src.u.dma.address = pt->src_dma;
2131         op.src.u.dma.offset = 0;
2132         op.src.u.dma.length = pt->src_len;
2133
2134         op.dst.type = CCP_MEMTYPE_SYSTEM;
2135         op.dst.u.dma.address = pt->dst_dma;
2136         op.dst.u.dma.offset = 0;
2137         op.dst.u.dma.length = pt->src_len;
2138
2139         ret = cmd_q->ccp->vdata->perform->passthru(&op);
2140         if (ret)
2141                 cmd->engine_error = cmd_q->cmd_error;
2142
2143         return ret;
2144 }
2145
2146 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2147 {
2148         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2149         struct ccp_dm_workarea src, dst;
2150         struct ccp_op op;
2151         int ret;
2152         u8 *save;
2153
2154         if (!ecc->u.mm.operand_1 ||
2155             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2156                 return -EINVAL;
2157
2158         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2159                 if (!ecc->u.mm.operand_2 ||
2160                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2161                         return -EINVAL;
2162
2163         if (!ecc->u.mm.result ||
2164             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2165                 return -EINVAL;
2166
2167         memset(&op, 0, sizeof(op));
2168         op.cmd_q = cmd_q;
2169         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2170
2171         /* Concatenate the modulus and the operands. Both the modulus and
2172          * the operands must be in little endian format.  Since the input
2173          * is in big endian format it must be converted and placed in a
2174          * fixed length buffer.
2175          */
2176         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2177                                    DMA_TO_DEVICE);
2178         if (ret)
2179                 return ret;
2180
2181         /* Save the workarea address since it is updated in order to perform
2182          * the concatenation
2183          */
2184         save = src.address;
2185
2186         /* Copy the ECC modulus */
2187         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2188         if (ret)
2189                 goto e_src;
2190         src.address += CCP_ECC_OPERAND_SIZE;
2191
2192         /* Copy the first operand */
2193         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2194                                       ecc->u.mm.operand_1_len);
2195         if (ret)
2196                 goto e_src;
2197         src.address += CCP_ECC_OPERAND_SIZE;
2198
2199         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2200                 /* Copy the second operand */
2201                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2202                                               ecc->u.mm.operand_2_len);
2203                 if (ret)
2204                         goto e_src;
2205                 src.address += CCP_ECC_OPERAND_SIZE;
2206         }
2207
2208         /* Restore the workarea address */
2209         src.address = save;
2210
2211         /* Prepare the output area for the operation */
2212         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2213                                    DMA_FROM_DEVICE);
2214         if (ret)
2215                 goto e_src;
2216
2217         op.soc = 1;
2218         op.src.u.dma.address = src.dma.address;
2219         op.src.u.dma.offset = 0;
2220         op.src.u.dma.length = src.length;
2221         op.dst.u.dma.address = dst.dma.address;
2222         op.dst.u.dma.offset = 0;
2223         op.dst.u.dma.length = dst.length;
2224
2225         op.u.ecc.function = cmd->u.ecc.function;
2226
2227         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2228         if (ret) {
2229                 cmd->engine_error = cmd_q->cmd_error;
2230                 goto e_dst;
2231         }
2232
2233         ecc->ecc_result = le16_to_cpup(
2234                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2235         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2236                 ret = -EIO;
2237                 goto e_dst;
2238         }
2239
2240         /* Save the ECC result */
2241         ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2242                                 CCP_ECC_MODULUS_BYTES);
2243
2244 e_dst:
2245         ccp_dm_free(&dst);
2246
2247 e_src:
2248         ccp_dm_free(&src);
2249
2250         return ret;
2251 }
2252
2253 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2254 {
2255         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2256         struct ccp_dm_workarea src, dst;
2257         struct ccp_op op;
2258         int ret;
2259         u8 *save;
2260
2261         if (!ecc->u.pm.point_1.x ||
2262             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2263             !ecc->u.pm.point_1.y ||
2264             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2265                 return -EINVAL;
2266
2267         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2268                 if (!ecc->u.pm.point_2.x ||
2269                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2270                     !ecc->u.pm.point_2.y ||
2271                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2272                         return -EINVAL;
2273         } else {
2274                 if (!ecc->u.pm.domain_a ||
2275                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2276                         return -EINVAL;
2277
2278                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2279                         if (!ecc->u.pm.scalar ||
2280                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2281                                 return -EINVAL;
2282         }
2283
2284         if (!ecc->u.pm.result.x ||
2285             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2286             !ecc->u.pm.result.y ||
2287             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2288                 return -EINVAL;
2289
2290         memset(&op, 0, sizeof(op));
2291         op.cmd_q = cmd_q;
2292         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2293
2294         /* Concatenate the modulus and the operands. Both the modulus and
2295          * the operands must be in little endian format.  Since the input
2296          * is in big endian format it must be converted and placed in a
2297          * fixed length buffer.
2298          */
2299         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2300                                    DMA_TO_DEVICE);
2301         if (ret)
2302                 return ret;
2303
2304         /* Save the workarea address since it is updated in order to perform
2305          * the concatenation
2306          */
2307         save = src.address;
2308
2309         /* Copy the ECC modulus */
2310         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2311         if (ret)
2312                 goto e_src;
2313         src.address += CCP_ECC_OPERAND_SIZE;
2314
2315         /* Copy the first point X and Y coordinate */
2316         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2317                                       ecc->u.pm.point_1.x_len);
2318         if (ret)
2319                 goto e_src;
2320         src.address += CCP_ECC_OPERAND_SIZE;
2321         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2322                                       ecc->u.pm.point_1.y_len);
2323         if (ret)
2324                 goto e_src;
2325         src.address += CCP_ECC_OPERAND_SIZE;
2326
2327         /* Set the first point Z coordinate to 1 */
2328         *src.address = 0x01;
2329         src.address += CCP_ECC_OPERAND_SIZE;
2330
2331         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2332                 /* Copy the second point X and Y coordinate */
2333                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2334                                               ecc->u.pm.point_2.x_len);
2335                 if (ret)
2336                         goto e_src;
2337                 src.address += CCP_ECC_OPERAND_SIZE;
2338                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2339                                               ecc->u.pm.point_2.y_len);
2340                 if (ret)
2341                         goto e_src;
2342                 src.address += CCP_ECC_OPERAND_SIZE;
2343
2344                 /* Set the second point Z coordinate to 1 */
2345                 *src.address = 0x01;
2346                 src.address += CCP_ECC_OPERAND_SIZE;
2347         } else {
2348                 /* Copy the Domain "a" parameter */
2349                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2350                                               ecc->u.pm.domain_a_len);
2351                 if (ret)
2352                         goto e_src;
2353                 src.address += CCP_ECC_OPERAND_SIZE;
2354
2355                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2356                         /* Copy the scalar value */
2357                         ret = ccp_reverse_set_dm_area(&src, 0,
2358                                                       ecc->u.pm.scalar, 0,
2359                                                       ecc->u.pm.scalar_len);
2360                         if (ret)
2361                                 goto e_src;
2362                         src.address += CCP_ECC_OPERAND_SIZE;
2363                 }
2364         }
2365
2366         /* Restore the workarea address */
2367         src.address = save;
2368
2369         /* Prepare the output area for the operation */
2370         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2371                                    DMA_FROM_DEVICE);
2372         if (ret)
2373                 goto e_src;
2374
2375         op.soc = 1;
2376         op.src.u.dma.address = src.dma.address;
2377         op.src.u.dma.offset = 0;
2378         op.src.u.dma.length = src.length;
2379         op.dst.u.dma.address = dst.dma.address;
2380         op.dst.u.dma.offset = 0;
2381         op.dst.u.dma.length = dst.length;
2382
2383         op.u.ecc.function = cmd->u.ecc.function;
2384
2385         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2386         if (ret) {
2387                 cmd->engine_error = cmd_q->cmd_error;
2388                 goto e_dst;
2389         }
2390
2391         ecc->ecc_result = le16_to_cpup(
2392                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2393         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2394                 ret = -EIO;
2395                 goto e_dst;
2396         }
2397
2398         /* Save the workarea address since it is updated as we walk through
2399          * to copy the point math result
2400          */
2401         save = dst.address;
2402
2403         /* Save the ECC result X and Y coordinates */
2404         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2405                                 CCP_ECC_MODULUS_BYTES);
2406         dst.address += CCP_ECC_OUTPUT_SIZE;
2407         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2408                                 CCP_ECC_MODULUS_BYTES);
2409         dst.address += CCP_ECC_OUTPUT_SIZE;
2410
2411         /* Restore the workarea address */
2412         dst.address = save;
2413
2414 e_dst:
2415         ccp_dm_free(&dst);
2416
2417 e_src:
2418         ccp_dm_free(&src);
2419
2420         return ret;
2421 }
2422
2423 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2424 {
2425         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2426
2427         ecc->ecc_result = 0;
2428
2429         if (!ecc->mod ||
2430             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2431                 return -EINVAL;
2432
2433         switch (ecc->function) {
2434         case CCP_ECC_FUNCTION_MMUL_384BIT:
2435         case CCP_ECC_FUNCTION_MADD_384BIT:
2436         case CCP_ECC_FUNCTION_MINV_384BIT:
2437                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2438
2439         case CCP_ECC_FUNCTION_PADD_384BIT:
2440         case CCP_ECC_FUNCTION_PMUL_384BIT:
2441         case CCP_ECC_FUNCTION_PDBL_384BIT:
2442                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2443
2444         default:
2445                 return -EINVAL;
2446         }
2447 }
2448
2449 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2450 {
2451         int ret;
2452
2453         cmd->engine_error = 0;
2454         cmd_q->cmd_error = 0;
2455         cmd_q->int_rcvd = 0;
2456         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2457
2458         switch (cmd->engine) {
2459         case CCP_ENGINE_AES:
2460                 ret = ccp_run_aes_cmd(cmd_q, cmd);
2461                 break;
2462         case CCP_ENGINE_XTS_AES_128:
2463                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2464                 break;
2465         case CCP_ENGINE_DES3:
2466                 ret = ccp_run_des3_cmd(cmd_q, cmd);
2467                 break;
2468         case CCP_ENGINE_SHA:
2469                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2470                 break;
2471         case CCP_ENGINE_RSA:
2472                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2473                 break;
2474         case CCP_ENGINE_PASSTHRU:
2475                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2476                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2477                 else
2478                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
2479                 break;
2480         case CCP_ENGINE_ECC:
2481                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2482                 break;
2483         default:
2484                 ret = -EINVAL;
2485         }
2486
2487         return ret;
2488 }