drivers/crypto/ccree/cc_aead.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <crypto/algapi.h>
   7 #include <crypto/internal/aead.h>
   8 #include <crypto/authenc.h>
   9 #include <crypto/des.h>
  10 #include <linux/rtnetlink.h>
  11 #include "cc_driver.h"
  12 #include "cc_buffer_mgr.h"
  13 #include "cc_aead.h"
  14 #include "cc_request_mgr.h"
  15 #include "cc_hash.h"
  16 #include "cc_sram_mgr.h"
  17
  18 #define template_aead   template_u.aead
  19
  20 #define MAX_AEAD_SETKEY_SEQ 12
  21 #define MAX_AEAD_PROCESS_SEQ 23
  22
  23 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE)
  24 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE)
  25
  26 #define AES_CCM_RFC4309_NONCE_SIZE 3
  27 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE
  28
  29 /* Value of each ICV_CMP byte (of 8) in case of success */
  30 #define ICV_VERIF_OK 0x01
  31
  32 struct cc_aead_handle {
  33         cc_sram_addr_t sram_workspace_addr;
  34         struct list_head aead_list;
  35 };
  36
  37 struct cc_hmac_s {
  38         u8 *padded_authkey;
  39         u8 *ipad_opad; /* IPAD, OPAD*/
  40         dma_addr_t padded_authkey_dma_addr;
  41         dma_addr_t ipad_opad_dma_addr;
  42 };
  43
  44 struct cc_xcbc_s {
  45         u8 *xcbc_keys; /* K1,K2,K3 */
  46         dma_addr_t xcbc_keys_dma_addr;
  47 };
  48
  49 struct cc_aead_ctx {
  50         struct cc_drvdata *drvdata;
  51         u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */
  52         u8 *enckey;
  53         dma_addr_t enckey_dma_addr;
  54         union {
  55                 struct cc_hmac_s hmac;
  56                 struct cc_xcbc_s xcbc;
  57         } auth_state;
  58         unsigned int enc_keylen;
  59         unsigned int auth_keylen;
  60         unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */
  61         unsigned int hash_len;
  62         enum drv_cipher_mode cipher_mode;
  63         enum cc_flow_mode flow_mode;
  64         enum drv_hash_mode auth_mode;
  65 };
  66
  67 static inline bool valid_assoclen(struct aead_request *req)
  68 {
  69         return ((req->assoclen == 16) || (req->assoclen == 20));
  70 }
  71
  72 static void cc_aead_exit(struct crypto_aead *tfm)
  73 {
  74         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
  75         struct device *dev = drvdata_to_dev(ctx->drvdata);
  76
  77         dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm),
  78                 crypto_tfm_alg_name(&tfm->base));
  79
  80         /* Unmap enckey buffer */
  81         if (ctx->enckey) {
  82                 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey,
  83                                   ctx->enckey_dma_addr);
  84                 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n",
  85                         &ctx->enckey_dma_addr);
  86                 ctx->enckey_dma_addr = 0;
  87                 ctx->enckey = NULL;
  88         }
  89
  90         if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
  91                 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
  92
  93                 if (xcbc->xcbc_keys) {
  94                         dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3,
  95                                           xcbc->xcbc_keys,
  96                                           xcbc->xcbc_keys_dma_addr);
  97                 }
  98                 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n",
  99                         &xcbc->xcbc_keys_dma_addr);
 100                 xcbc->xcbc_keys_dma_addr = 0;
 101                 xcbc->xcbc_keys = NULL;
 102         } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */
 103                 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 104
 105                 if (hmac->ipad_opad) {
 106                         dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE,
 107                                           hmac->ipad_opad,
 108                                           hmac->ipad_opad_dma_addr);
 109                         dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n",
 110                                 &hmac->ipad_opad_dma_addr);
 111                         hmac->ipad_opad_dma_addr = 0;
 112                         hmac->ipad_opad = NULL;
 113                 }
 114                 if (hmac->padded_authkey) {
 115                         dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE,
 116                                           hmac->padded_authkey,
 117                                           hmac->padded_authkey_dma_addr);
 118                         dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n",
 119                                 &hmac->padded_authkey_dma_addr);
 120                         hmac->padded_authkey_dma_addr = 0;
 121                         hmac->padded_authkey = NULL;
 122                 }
 123         }
 124 }
 125
 126 static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm)
 127 {
 128         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 129
 130         return cc_get_default_hash_len(ctx->drvdata);
 131 }
 132
 133 static int cc_aead_init(struct crypto_aead *tfm)
 134 {
 135         struct aead_alg *alg = crypto_aead_alg(tfm);
 136         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 137         struct cc_crypto_alg *cc_alg =
 138                         container_of(alg, struct cc_crypto_alg, aead_alg);
 139         struct device *dev = drvdata_to_dev(cc_alg->drvdata);
 140
 141         dev_dbg(dev, "Initializing context @%p for %s\n", ctx,
 142                 crypto_tfm_alg_name(&tfm->base));
 143
 144         /* Initialize modes in instance */
 145         ctx->cipher_mode = cc_alg->cipher_mode;
 146         ctx->flow_mode = cc_alg->flow_mode;
 147         ctx->auth_mode = cc_alg->auth_mode;
 148         ctx->drvdata = cc_alg->drvdata;
 149         crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx));
 150
 151         /* Allocate key buffer, cache line aligned */
 152         ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
 153                                          &ctx->enckey_dma_addr, GFP_KERNEL);
 154         if (!ctx->enckey) {
 155                 dev_err(dev, "Failed allocating key buffer\n");
 156                 goto init_failed;
 157         }
 158         dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n",
 159                 ctx->enckey);
 160
 161         /* Set default authlen value */
 162
 163         if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
 164                 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
 165                 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3;
 166
 167                 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */
 168                 /* (and temporary for user key - up to 256b) */
 169                 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size,
 170                                                      &xcbc->xcbc_keys_dma_addr,
 171                                                      GFP_KERNEL);
 172                 if (!xcbc->xcbc_keys) {
 173                         dev_err(dev, "Failed allocating buffer for XCBC keys\n");
 174                         goto init_failed;
 175                 }
 176         } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */
 177                 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 178                 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE;
 179                 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr;
 180
 181                 /* Allocate dma-coherent buffer for IPAD + OPAD */
 182                 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size,
 183                                                      &hmac->ipad_opad_dma_addr,
 184                                                      GFP_KERNEL);
 185
 186                 if (!hmac->ipad_opad) {
 187                         dev_err(dev, "Failed allocating IPAD/OPAD buffer\n");
 188                         goto init_failed;
 189                 }
 190
 191                 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n",
 192                         hmac->ipad_opad);
 193
 194                 hmac->padded_authkey = dma_alloc_coherent(dev,
 195                                                           MAX_HMAC_BLOCK_SIZE,
 196                                                           pkey_dma,
 197                                                           GFP_KERNEL);
 198
 199                 if (!hmac->padded_authkey) {
 200                         dev_err(dev, "failed to allocate padded_authkey\n");
 201                         goto init_failed;
 202                 }
 203         } else {
 204                 ctx->auth_state.hmac.ipad_opad = NULL;
 205                 ctx->auth_state.hmac.padded_authkey = NULL;
 206         }
 207         ctx->hash_len = cc_get_aead_hash_len(tfm);
 208
 209         return 0;
 210
 211 init_failed:
 212         cc_aead_exit(tfm);
 213         return -ENOMEM;
 214 }
 215
 216 static void cc_aead_complete(struct device *dev, void *cc_req, int err)
 217 {
 218         struct aead_request *areq = (struct aead_request *)cc_req;
 219         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 220         struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req);
 221         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 222
 223         cc_unmap_aead_request(dev, areq);
 224
 225         /* Restore ordinary iv pointer */
 226         areq->iv = areq_ctx->backup_iv;
 227
 228         if (err)
 229                 goto done;
 230
 231         if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
 232                 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
 233                            ctx->authsize) != 0) {
 234                         dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n",
 235                                 ctx->authsize, ctx->cipher_mode);
 236                         /* In case of payload authentication failure, MUST NOT
 237                          * revealed the decrypted message --> zero its memory.
 238                          */
 239                         cc_zero_sgl(areq->dst, areq_ctx->cryptlen);
 240                         err = -EBADMSG;
 241                 }
 242         } else { /*ENCRYPT*/
 243                 if (areq_ctx->is_icv_fragmented) {
 244                         u32 skip = areq->cryptlen + areq_ctx->dst_offset;
 245
 246                         cc_copy_sg_portion(dev, areq_ctx->mac_buf,
 247                                            areq_ctx->dst_sgl, skip,
 248                                            (skip + ctx->authsize),
 249                                            CC_SG_FROM_BUF);
 250                 }
 251
 252                 /* If an IV was generated, copy it back to the user provided
 253                  * buffer.
 254                  */
 255                 if (areq_ctx->backup_giv) {
 256                         if (ctx->cipher_mode == DRV_CIPHER_CTR)
 257                                 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
 258                                        CTR_RFC3686_NONCE_SIZE,
 259                                        CTR_RFC3686_IV_SIZE);
 260                         else if (ctx->cipher_mode == DRV_CIPHER_CCM)
 261                                 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
 262                                        CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE);
 263                 }
 264         }
 265 done:
 266         aead_request_complete(areq, err);
 267 }
 268
 269 static unsigned int xcbc_setkey(struct cc_hw_desc *desc,
 270                                 struct cc_aead_ctx *ctx)
 271 {
 272         /* Load the AES key */
 273         hw_desc_init(&desc[0]);
 274         /* We are using for the source/user key the same buffer
 275          * as for the output keys, * because after this key loading it
 276          * is not needed anymore
 277          */
 278         set_din_type(&desc[0], DMA_DLLI,
 279                      ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen,
 280                      NS_BIT);
 281         set_cipher_mode(&desc[0], DRV_CIPHER_ECB);
 282         set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT);
 283         set_key_size_aes(&desc[0], ctx->auth_keylen);
 284         set_flow_mode(&desc[0], S_DIN_to_AES);
 285         set_setup_mode(&desc[0], SETUP_LOAD_KEY0);
 286
 287         hw_desc_init(&desc[1]);
 288         set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
 289         set_flow_mode(&desc[1], DIN_AES_DOUT);
 290         set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr,
 291                       AES_KEYSIZE_128, NS_BIT, 0);
 292
 293         hw_desc_init(&desc[2]);
 294         set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
 295         set_flow_mode(&desc[2], DIN_AES_DOUT);
 296         set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
 297                                          + AES_KEYSIZE_128),
 298                               AES_KEYSIZE_128, NS_BIT, 0);
 299
 300         hw_desc_init(&desc[3]);
 301         set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
 302         set_flow_mode(&desc[3], DIN_AES_DOUT);
 303         set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
 304                                           + 2 * AES_KEYSIZE_128),
 305                               AES_KEYSIZE_128, NS_BIT, 0);
 306
 307         return 4;
 308 }
 309
 310 static int hmac_setkey(struct cc_hw_desc *desc, struct cc_aead_ctx *ctx)
 311 {
 312         unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
 313         unsigned int digest_ofs = 0;
 314         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
 315                         DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
 316         unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
 317                         CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
 318         struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 319
 320         unsigned int idx = 0;
 321         int i;
 322
 323         /* calc derived HMAC key */
 324         for (i = 0; i < 2; i++) {
 325                 /* Load hash initial state */
 326                 hw_desc_init(&desc[idx]);
 327                 set_cipher_mode(&desc[idx], hash_mode);
 328                 set_din_sram(&desc[idx],
 329                              cc_larval_digest_addr(ctx->drvdata,
 330                                                    ctx->auth_mode),
 331                              digest_size);
 332                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 333                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 334                 idx++;
 335
 336                 /* Load the hash current length*/
 337                 hw_desc_init(&desc[idx]);
 338                 set_cipher_mode(&desc[idx], hash_mode);
 339                 set_din_const(&desc[idx], 0, ctx->hash_len);
 340                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 341                 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 342                 idx++;
 343
 344                 /* Prepare ipad key */
 345                 hw_desc_init(&desc[idx]);
 346                 set_xor_val(&desc[idx], hmac_pad_const[i]);
 347                 set_cipher_mode(&desc[idx], hash_mode);
 348                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 349                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
 350                 idx++;
 351
 352                 /* Perform HASH update */
 353                 hw_desc_init(&desc[idx]);
 354                 set_din_type(&desc[idx], DMA_DLLI,
 355                              hmac->padded_authkey_dma_addr,
 356                              SHA256_BLOCK_SIZE, NS_BIT);
 357                 set_cipher_mode(&desc[idx], hash_mode);
 358                 set_xor_active(&desc[idx]);
 359                 set_flow_mode(&desc[idx], DIN_HASH);
 360                 idx++;
 361
 362                 /* Get the digset */
 363                 hw_desc_init(&desc[idx]);
 364                 set_cipher_mode(&desc[idx], hash_mode);
 365                 set_dout_dlli(&desc[idx],
 366                               (hmac->ipad_opad_dma_addr + digest_ofs),
 367                               digest_size, NS_BIT, 0);
 368                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 369                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 370                 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 371                 idx++;
 372
 373                 digest_ofs += digest_size;
 374         }
 375
 376         return idx;
 377 }
 378
 379 static int validate_keys_sizes(struct cc_aead_ctx *ctx)
 380 {
 381         struct device *dev = drvdata_to_dev(ctx->drvdata);
 382
 383         dev_dbg(dev, "enc_keylen=%u  authkeylen=%u\n",
 384                 ctx->enc_keylen, ctx->auth_keylen);
 385
 386         switch (ctx->auth_mode) {
 387         case DRV_HASH_SHA1:
 388         case DRV_HASH_SHA256:
 389                 break;
 390         case DRV_HASH_XCBC_MAC:
 391                 if (ctx->auth_keylen != AES_KEYSIZE_128 &&
 392                     ctx->auth_keylen != AES_KEYSIZE_192 &&
 393                     ctx->auth_keylen != AES_KEYSIZE_256)
 394                         return -ENOTSUPP;
 395                 break;
 396         case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */
 397                 if (ctx->auth_keylen > 0)
 398                         return -EINVAL;
 399                 break;
 400         default:
 401                 dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode);
 402                 return -EINVAL;
 403         }
 404         /* Check cipher key size */
 405         if (ctx->flow_mode == S_DIN_to_DES) {
 406                 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) {
 407                         dev_err(dev, "Invalid cipher(3DES) key size: %u\n",
 408                                 ctx->enc_keylen);
 409                         return -EINVAL;
 410                 }
 411         } else { /* Default assumed to be AES ciphers */
 412                 if (ctx->enc_keylen != AES_KEYSIZE_128 &&
 413                     ctx->enc_keylen != AES_KEYSIZE_192 &&
 414                     ctx->enc_keylen != AES_KEYSIZE_256) {
 415                         dev_err(dev, "Invalid cipher(AES) key size: %u\n",
 416                                 ctx->enc_keylen);
 417                         return -EINVAL;
 418                 }
 419         }
 420
 421         return 0; /* All tests of keys sizes passed */
 422 }
 423
 424 /* This function prepers the user key so it can pass to the hmac processing
 425  * (copy to intenral buffer or hash in case of key longer than block
 426  */
 427 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *key,
 428                                  unsigned int keylen)
 429 {
 430         dma_addr_t key_dma_addr = 0;
 431         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 432         struct device *dev = drvdata_to_dev(ctx->drvdata);
 433         u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode);
 434         struct cc_crypto_req cc_req = {};
 435         unsigned int blocksize;
 436         unsigned int digestsize;
 437         unsigned int hashmode;
 438         unsigned int idx = 0;
 439         int rc = 0;
 440         struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
 441         dma_addr_t padded_authkey_dma_addr =
 442                 ctx->auth_state.hmac.padded_authkey_dma_addr;
 443
 444         switch (ctx->auth_mode) { /* auth_key required and >0 */
 445         case DRV_HASH_SHA1:
 446                 blocksize = SHA1_BLOCK_SIZE;
 447                 digestsize = SHA1_DIGEST_SIZE;
 448                 hashmode = DRV_HASH_HW_SHA1;
 449                 break;
 450         case DRV_HASH_SHA256:
 451         default:
 452                 blocksize = SHA256_BLOCK_SIZE;
 453                 digestsize = SHA256_DIGEST_SIZE;
 454                 hashmode = DRV_HASH_HW_SHA256;
 455         }
 456
 457         if (keylen != 0) {
 458                 key_dma_addr = dma_map_single(dev, (void *)key, keylen,
 459                                               DMA_TO_DEVICE);
 460                 if (dma_mapping_error(dev, key_dma_addr)) {
 461                         dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
 462                                 key, keylen);
 463                         return -ENOMEM;
 464                 }
 465                 if (keylen > blocksize) {
 466                         /* Load hash initial state */
 467                         hw_desc_init(&desc[idx]);
 468                         set_cipher_mode(&desc[idx], hashmode);
 469                         set_din_sram(&desc[idx], larval_addr, digestsize);
 470                         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 471                         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 472                         idx++;
 473
 474                         /* Load the hash current length*/
 475                         hw_desc_init(&desc[idx]);
 476                         set_cipher_mode(&desc[idx], hashmode);
 477                         set_din_const(&desc[idx], 0, ctx->hash_len);
 478                         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
 479                         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 480                         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 481                         idx++;
 482
 483                         hw_desc_init(&desc[idx]);
 484                         set_din_type(&desc[idx], DMA_DLLI,
 485                                      key_dma_addr, keylen, NS_BIT);
 486                         set_flow_mode(&desc[idx], DIN_HASH);
 487                         idx++;
 488
 489                         /* Get hashed key */
 490                         hw_desc_init(&desc[idx]);
 491                         set_cipher_mode(&desc[idx], hashmode);
 492                         set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
 493                                       digestsize, NS_BIT, 0);
 494                         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 495                         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 496                         set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 497                         set_cipher_config0(&desc[idx],
 498                                            HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 499                         idx++;
 500
 501                         hw_desc_init(&desc[idx]);
 502                         set_din_const(&desc[idx], 0, (blocksize - digestsize));
 503                         set_flow_mode(&desc[idx], BYPASS);
 504                         set_dout_dlli(&desc[idx], (padded_authkey_dma_addr +
 505                                       digestsize), (blocksize - digestsize),
 506                                       NS_BIT, 0);
 507                         idx++;
 508                 } else {
 509                         hw_desc_init(&desc[idx]);
 510                         set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
 511                                      keylen, NS_BIT);
 512                         set_flow_mode(&desc[idx], BYPASS);
 513                         set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
 514                                       keylen, NS_BIT, 0);
 515                         idx++;
 516
 517                         if ((blocksize - keylen) != 0) {
 518                                 hw_desc_init(&desc[idx]);
 519                                 set_din_const(&desc[idx], 0,
 520                                               (blocksize - keylen));
 521                                 set_flow_mode(&desc[idx], BYPASS);
 522                                 set_dout_dlli(&desc[idx],
 523                                               (padded_authkey_dma_addr +
 524                                                keylen),
 525                                               (blocksize - keylen), NS_BIT, 0);
 526                                 idx++;
 527                         }
 528                 }
 529         } else {
 530                 hw_desc_init(&desc[idx]);
 531                 set_din_const(&desc[idx], 0, (blocksize - keylen));
 532                 set_flow_mode(&desc[idx], BYPASS);
 533                 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
 534                               blocksize, NS_BIT, 0);
 535                 idx++;
 536         }
 537
 538         rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
 539         if (rc)
 540                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 541
 542         if (key_dma_addr)
 543                 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE);
 544
 545         return rc;
 546 }
 547
 548 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
 549                           unsigned int keylen)
 550 {
 551         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 552         struct cc_crypto_req cc_req = {};
 553         struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
 554         unsigned int seq_len = 0;
 555         struct device *dev = drvdata_to_dev(ctx->drvdata);
 556         const u8 *enckey, *authkey;
 557         int rc;
 558
 559         dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n",
 560                 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen);
 561
 562         /* STAT_PHASE_0: Init and sanity checks */
 563
 564         if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */
 565                 struct crypto_authenc_keys keys;
 566
 567                 rc = crypto_authenc_extractkeys(&keys, key, keylen);
 568                 if (rc)
 569                         goto badkey;
 570                 enckey = keys.enckey;
 571                 authkey = keys.authkey;
 572                 ctx->enc_keylen = keys.enckeylen;
 573                 ctx->auth_keylen = keys.authkeylen;
 574
 575                 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
 576                         /* the nonce is stored in bytes at end of key */
 577                         rc = -EINVAL;
 578                         if (ctx->enc_keylen <
 579                             (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE))
 580                                 goto badkey;
 581                         /* Copy nonce from last 4 bytes in CTR key to
 582                          *  first 4 bytes in CTR IV
 583                          */
 584                         memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen -
 585                                CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE);
 586                         /* Set CTR key size */
 587                         ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE;
 588                 }
 589         } else { /* non-authenc - has just one key */
 590                 enckey = key;
 591                 authkey = NULL;
 592                 ctx->enc_keylen = keylen;
 593                 ctx->auth_keylen = 0;
 594         }
 595
 596         rc = validate_keys_sizes(ctx);
 597         if (rc)
 598                 goto badkey;
 599
 600         /* STAT_PHASE_1: Copy key to ctx */
 601
 602         /* Get key material */
 603         memcpy(ctx->enckey, enckey, ctx->enc_keylen);
 604         if (ctx->enc_keylen == 24)
 605                 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24);
 606         if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
 607                 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey,
 608                        ctx->auth_keylen);
 609         } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */
 610                 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen);
 611                 if (rc)
 612                         goto badkey;
 613         }
 614
 615         /* STAT_PHASE_2: Create sequence */
 616
 617         switch (ctx->auth_mode) {
 618         case DRV_HASH_SHA1:
 619         case DRV_HASH_SHA256:
 620                 seq_len = hmac_setkey(desc, ctx);
 621                 break;
 622         case DRV_HASH_XCBC_MAC:
 623                 seq_len = xcbc_setkey(desc, ctx);
 624                 break;
 625         case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */
 626                 break; /* No auth. key setup */
 627         default:
 628                 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
 629                 rc = -ENOTSUPP;
 630                 goto badkey;
 631         }
 632
 633         /* STAT_PHASE_3: Submit sequence to HW */
 634
 635         if (seq_len > 0) { /* For CCM there is no sequence to setup the key */
 636                 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len);
 637                 if (rc) {
 638                         dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 639                         goto setkey_error;
 640                 }
 641         }
 642
 643         /* Update STAT_PHASE_3 */
 644         return rc;
 645
 646 badkey:
 647         crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 648
 649 setkey_error:
 650         return rc;
 651 }
 652
 653 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
 654                                  unsigned int keylen)
 655 {
 656         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 657
 658         if (keylen < 3)
 659                 return -EINVAL;
 660
 661         keylen -= 3;
 662         memcpy(ctx->ctr_nonce, key + keylen, 3);
 663
 664         return cc_aead_setkey(tfm, key, keylen);
 665 }
 666
 667 static int cc_aead_setauthsize(struct crypto_aead *authenc,
 668                                unsigned int authsize)
 669 {
 670         struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
 671         struct device *dev = drvdata_to_dev(ctx->drvdata);
 672
 673         /* Unsupported auth. sizes */
 674         if (authsize == 0 ||
 675             authsize > crypto_aead_maxauthsize(authenc)) {
 676                 return -ENOTSUPP;
 677         }
 678
 679         ctx->authsize = authsize;
 680         dev_dbg(dev, "authlen=%d\n", ctx->authsize);
 681
 682         return 0;
 683 }
 684
 685 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,
 686                                       unsigned int authsize)
 687 {
 688         switch (authsize) {
 689         case 8:
 690         case 12:
 691         case 16:
 692                 break;
 693         default:
 694                 return -EINVAL;
 695         }
 696
 697         return cc_aead_setauthsize(authenc, authsize);
 698 }
 699
 700 static int cc_ccm_setauthsize(struct crypto_aead *authenc,
 701                               unsigned int authsize)
 702 {
 703         switch (authsize) {
 704         case 4:
 705         case 6:
 706         case 8:
 707         case 10:
 708         case 12:
 709         case 14:
 710         case 16:
 711                 break;
 712         default:
 713                 return -EINVAL;
 714         }
 715
 716         return cc_aead_setauthsize(authenc, authsize);
 717 }
 718
 719 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode,
 720                               struct cc_hw_desc desc[], unsigned int *seq_size)
 721 {
 722         struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
 723         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 724         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 725         enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;
 726         unsigned int idx = *seq_size;
 727         struct device *dev = drvdata_to_dev(ctx->drvdata);
 728
 729         switch (assoc_dma_type) {
 730         case CC_DMA_BUF_DLLI:
 731                 dev_dbg(dev, "ASSOC buffer type DLLI\n");
 732                 hw_desc_init(&desc[idx]);
 733                 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src),
 734                              areq->assoclen, NS_BIT);
 735                 set_flow_mode(&desc[idx], flow_mode);
 736                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
 737                     areq_ctx->cryptlen > 0)
 738                         set_din_not_last_indication(&desc[idx]);
 739                 break;
 740         case CC_DMA_BUF_MLLI:
 741                 dev_dbg(dev, "ASSOC buffer type MLLI\n");
 742                 hw_desc_init(&desc[idx]);
 743                 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr,
 744                              areq_ctx->assoc.mlli_nents, NS_BIT);
 745                 set_flow_mode(&desc[idx], flow_mode);
 746                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
 747                     areq_ctx->cryptlen > 0)
 748                         set_din_not_last_indication(&desc[idx]);
 749                 break;
 750         case CC_DMA_BUF_NULL:
 751         default:
 752                 dev_err(dev, "Invalid ASSOC buffer type\n");
 753         }
 754
 755         *seq_size = (++idx);
 756 }
 757
 758 static void cc_proc_authen_desc(struct aead_request *areq,
 759                                 unsigned int flow_mode,
 760                                 struct cc_hw_desc desc[],
 761                                 unsigned int *seq_size, int direct)
 762 {
 763         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 764         enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
 765         unsigned int idx = *seq_size;
 766         struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
 767         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 768         struct device *dev = drvdata_to_dev(ctx->drvdata);
 769
 770         switch (data_dma_type) {
 771         case CC_DMA_BUF_DLLI:
 772         {
 773                 struct scatterlist *cipher =
 774                         (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
 775                         areq_ctx->dst_sgl : areq_ctx->src_sgl;
 776
 777                 unsigned int offset =
 778                         (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
 779                         areq_ctx->dst_offset : areq_ctx->src_offset;
 780                 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n");
 781                 hw_desc_init(&desc[idx]);
 782                 set_din_type(&desc[idx], DMA_DLLI,
 783                              (sg_dma_address(cipher) + offset),
 784                              areq_ctx->cryptlen, NS_BIT);
 785                 set_flow_mode(&desc[idx], flow_mode);
 786                 break;
 787         }
 788         case CC_DMA_BUF_MLLI:
 789         {
 790                 /* DOUBLE-PASS flow (as default)
 791                  * assoc. + iv + data -compact in one table
 792                  * if assoclen is ZERO only IV perform
 793                  */
 794                 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr;
 795                 u32 mlli_nents = areq_ctx->assoc.mlli_nents;
 796
 797                 if (areq_ctx->is_single_pass) {
 798                         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
 799                                 mlli_addr = areq_ctx->dst.sram_addr;
 800                                 mlli_nents = areq_ctx->dst.mlli_nents;
 801                         } else {
 802                                 mlli_addr = areq_ctx->src.sram_addr;
 803                                 mlli_nents = areq_ctx->src.mlli_nents;
 804                         }
 805                 }
 806
 807                 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n");
 808                 hw_desc_init(&desc[idx]);
 809                 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents,
 810                              NS_BIT);
 811                 set_flow_mode(&desc[idx], flow_mode);
 812                 break;
 813         }
 814         case CC_DMA_BUF_NULL:
 815         default:
 816                 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n");
 817         }
 818
 819         *seq_size = (++idx);
 820 }
 821
 822 static void cc_proc_cipher_desc(struct aead_request *areq,
 823                                 unsigned int flow_mode,
 824                                 struct cc_hw_desc desc[],
 825                                 unsigned int *seq_size)
 826 {
 827         unsigned int idx = *seq_size;
 828         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 829         enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
 830         struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
 831         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 832         struct device *dev = drvdata_to_dev(ctx->drvdata);
 833
 834         if (areq_ctx->cryptlen == 0)
 835                 return; /*null processing*/
 836
 837         switch (data_dma_type) {
 838         case CC_DMA_BUF_DLLI:
 839                 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n");
 840                 hw_desc_init(&desc[idx]);
 841                 set_din_type(&desc[idx], DMA_DLLI,
 842                              (sg_dma_address(areq_ctx->src_sgl) +
 843                               areq_ctx->src_offset), areq_ctx->cryptlen,
 844                               NS_BIT);
 845                 set_dout_dlli(&desc[idx],
 846                               (sg_dma_address(areq_ctx->dst_sgl) +
 847                                areq_ctx->dst_offset),
 848                               areq_ctx->cryptlen, NS_BIT, 0);
 849                 set_flow_mode(&desc[idx], flow_mode);
 850                 break;
 851         case CC_DMA_BUF_MLLI:
 852                 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n");
 853                 hw_desc_init(&desc[idx]);
 854                 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr,
 855                              areq_ctx->src.mlli_nents, NS_BIT);
 856                 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr,
 857                               areq_ctx->dst.mlli_nents, NS_BIT, 0);
 858                 set_flow_mode(&desc[idx], flow_mode);
 859                 break;
 860         case CC_DMA_BUF_NULL:
 861         default:
 862                 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n");
 863         }
 864
 865         *seq_size = (++idx);
 866 }
 867
 868 static void cc_proc_digest_desc(struct aead_request *req,
 869                                 struct cc_hw_desc desc[],
 870                                 unsigned int *seq_size)
 871 {
 872         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 873         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 874         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
 875         unsigned int idx = *seq_size;
 876         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
 877                                 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
 878         int direct = req_ctx->gen_ctx.op_type;
 879
 880         /* Get final ICV result */
 881         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
 882                 hw_desc_init(&desc[idx]);
 883                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 884                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 885                 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize,
 886                               NS_BIT, 1);
 887                 set_queue_last_ind(ctx->drvdata, &desc[idx]);
 888                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
 889                         set_aes_not_hash_mode(&desc[idx]);
 890                         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
 891                 } else {
 892                         set_cipher_config0(&desc[idx],
 893                                            HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 894                         set_cipher_mode(&desc[idx], hash_mode);
 895                 }
 896         } else { /*Decrypt*/
 897                 /* Get ICV out from hardware */
 898                 hw_desc_init(&desc[idx]);
 899                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 900                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 901                 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr,
 902                               ctx->authsize, NS_BIT, 1);
 903                 set_queue_last_ind(ctx->drvdata, &desc[idx]);
 904                 set_cipher_config0(&desc[idx],
 905                                    HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 906                 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 907                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
 908                         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
 909                         set_aes_not_hash_mode(&desc[idx]);
 910                 } else {
 911                         set_cipher_mode(&desc[idx], hash_mode);
 912                 }
 913         }
 914
 915         *seq_size = (++idx);
 916 }
 917
 918 static void cc_set_cipher_desc(struct aead_request *req,
 919                                struct cc_hw_desc desc[],
 920                                unsigned int *seq_size)
 921 {
 922         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 923         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 924         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
 925         unsigned int hw_iv_size = req_ctx->hw_iv_size;
 926         unsigned int idx = *seq_size;
 927         int direct = req_ctx->gen_ctx.op_type;
 928
 929         /* Setup cipher state */
 930         hw_desc_init(&desc[idx]);
 931         set_cipher_config0(&desc[idx], direct);
 932         set_flow_mode(&desc[idx], ctx->flow_mode);
 933         set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr,
 934                      hw_iv_size, NS_BIT);
 935         if (ctx->cipher_mode == DRV_CIPHER_CTR)
 936                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
 937         else
 938                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 939         set_cipher_mode(&desc[idx], ctx->cipher_mode);
 940         idx++;
 941
 942         /* Setup enc. key */
 943         hw_desc_init(&desc[idx]);
 944         set_cipher_config0(&desc[idx], direct);
 945         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 946         set_flow_mode(&desc[idx], ctx->flow_mode);
 947         if (ctx->flow_mode == S_DIN_to_AES) {
 948                 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
 949                              ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
 950                               ctx->enc_keylen), NS_BIT);
 951                 set_key_size_aes(&desc[idx], ctx->enc_keylen);
 952         } else {
 953                 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
 954                              ctx->enc_keylen, NS_BIT);
 955                 set_key_size_des(&desc[idx], ctx->enc_keylen);
 956         }
 957         set_cipher_mode(&desc[idx], ctx->cipher_mode);
 958         idx++;
 959
 960         *seq_size = idx;
 961 }
 962
 963 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[],
 964                            unsigned int *seq_size, unsigned int data_flow_mode)
 965 {
 966         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
 967         int direct = req_ctx->gen_ctx.op_type;
 968         unsigned int idx = *seq_size;
 969
 970         if (req_ctx->cryptlen == 0)
 971                 return; /*null processing*/
 972
 973         cc_set_cipher_desc(req, desc, &idx);
 974         cc_proc_cipher_desc(req, data_flow_mode, desc, &idx);
 975         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
 976                 /* We must wait for DMA to write all cipher */
 977                 hw_desc_init(&desc[idx]);
 978                 set_din_no_dma(&desc[idx], 0, 0xfffff0);
 979                 set_dout_no_dma(&desc[idx], 0, 0, 1);
 980                 idx++;
 981         }
 982
 983         *seq_size = idx;
 984 }
 985
 986 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[],
 987                              unsigned int *seq_size)
 988 {
 989         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 990         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 991         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
 992                                 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
 993         unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
 994                                 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
 995         unsigned int idx = *seq_size;
 996
 997         /* Loading hash ipad xor key state */
 998         hw_desc_init(&desc[idx]);
 999         set_cipher_mode(&desc[idx], hash_mode);
1000         set_din_type(&desc[idx], DMA_DLLI,
1001                      ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size,
1002                      NS_BIT);
1003         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1004         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1005         idx++;
1006
1007         /* Load init. digest len (64 bytes) */
1008         hw_desc_init(&desc[idx]);
1009         set_cipher_mode(&desc[idx], hash_mode);
1010         set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1011                      ctx->hash_len);
1012         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1013         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1014         idx++;
1015
1016         *seq_size = idx;
1017 }
1018
1019 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[],
1020                              unsigned int *seq_size)
1021 {
1022         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1023         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1024         unsigned int idx = *seq_size;
1025
1026         /* Loading MAC state */
1027         hw_desc_init(&desc[idx]);
1028         set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE);
1029         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1030         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1031         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1032         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1033         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1034         set_aes_not_hash_mode(&desc[idx]);
1035         idx++;
1036
1037         /* Setup XCBC MAC K1 */
1038         hw_desc_init(&desc[idx]);
1039         set_din_type(&desc[idx], DMA_DLLI,
1040                      ctx->auth_state.xcbc.xcbc_keys_dma_addr,
1041                      AES_KEYSIZE_128, NS_BIT);
1042         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1043         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1044         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1045         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1046         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1047         set_aes_not_hash_mode(&desc[idx]);
1048         idx++;
1049
1050         /* Setup XCBC MAC K2 */
1051         hw_desc_init(&desc[idx]);
1052         set_din_type(&desc[idx], DMA_DLLI,
1053                      (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1054                       AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1055         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1056         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1057         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1058         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1059         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1060         set_aes_not_hash_mode(&desc[idx]);
1061         idx++;
1062
1063         /* Setup XCBC MAC K3 */
1064         hw_desc_init(&desc[idx]);
1065         set_din_type(&desc[idx], DMA_DLLI,
1066                      (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1067                       2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1068         set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
1069         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1070         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1071         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1072         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1073         set_aes_not_hash_mode(&desc[idx]);
1074         idx++;
1075
1076         *seq_size = idx;
1077 }
1078
1079 static void cc_proc_header_desc(struct aead_request *req,
1080                                 struct cc_hw_desc desc[],
1081                                 unsigned int *seq_size)
1082 {
1083         unsigned int idx = *seq_size;
1084         /* Hash associated data */
1085         if (req->assoclen > 0)
1086                 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1087
1088         /* Hash IV */
1089         *seq_size = idx;
1090 }
1091
1092 static void cc_proc_scheme_desc(struct aead_request *req,
1093                                 struct cc_hw_desc desc[],
1094                                 unsigned int *seq_size)
1095 {
1096         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1097         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1098         struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle;
1099         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1100                                 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1101         unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1102                                 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1103         unsigned int idx = *seq_size;
1104
1105         hw_desc_init(&desc[idx]);
1106         set_cipher_mode(&desc[idx], hash_mode);
1107         set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1108                       ctx->hash_len);
1109         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1110         set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
1111         set_cipher_do(&desc[idx], DO_PAD);
1112         idx++;
1113
1114         /* Get final ICV result */
1115         hw_desc_init(&desc[idx]);
1116         set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1117                       digest_size);
1118         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1119         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1120         set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1121         set_cipher_mode(&desc[idx], hash_mode);
1122         idx++;
1123
1124         /* Loading hash opad xor key state */
1125         hw_desc_init(&desc[idx]);
1126         set_cipher_mode(&desc[idx], hash_mode);
1127         set_din_type(&desc[idx], DMA_DLLI,
1128                      (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size),
1129                      digest_size, NS_BIT);
1130         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1131         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1132         idx++;
1133
1134         /* Load init. digest len (64 bytes) */
1135         hw_desc_init(&desc[idx]);
1136         set_cipher_mode(&desc[idx], hash_mode);
1137         set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1138                      ctx->hash_len);
1139         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1140         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1141         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1142         idx++;
1143
1144         /* Perform HASH update */
1145         hw_desc_init(&desc[idx]);
1146         set_din_sram(&desc[idx], aead_handle->sram_workspace_addr,
1147                      digest_size);
1148         set_flow_mode(&desc[idx], DIN_HASH);
1149         idx++;
1150
1151         *seq_size = idx;
1152 }
1153
1154 static void cc_mlli_to_sram(struct aead_request *req,
1155                             struct cc_hw_desc desc[], unsigned int *seq_size)
1156 {
1157         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1158         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1159         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1160         struct device *dev = drvdata_to_dev(ctx->drvdata);
1161
1162         if (req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
1163             req_ctx->data_buff_type == CC_DMA_BUF_MLLI ||
1164             !req_ctx->is_single_pass) {
1165                 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n",
1166                         (unsigned int)ctx->drvdata->mlli_sram_addr,
1167                         req_ctx->mlli_params.mlli_len);
1168                 /* Copy MLLI table host-to-sram */
1169                 hw_desc_init(&desc[*seq_size]);
1170                 set_din_type(&desc[*seq_size], DMA_DLLI,
1171                              req_ctx->mlli_params.mlli_dma_addr,
1172                              req_ctx->mlli_params.mlli_len, NS_BIT);
1173                 set_dout_sram(&desc[*seq_size],
1174                               ctx->drvdata->mlli_sram_addr,
1175                               req_ctx->mlli_params.mlli_len);
1176                 set_flow_mode(&desc[*seq_size], BYPASS);
1177                 (*seq_size)++;
1178         }
1179 }
1180
1181 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct,
1182                                           enum cc_flow_mode setup_flow_mode,
1183                                           bool is_single_pass)
1184 {
1185         enum cc_flow_mode data_flow_mode;
1186
1187         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1188                 if (setup_flow_mode == S_DIN_to_AES)
1189                         data_flow_mode = is_single_pass ?
1190                                 AES_to_HASH_and_DOUT : DIN_AES_DOUT;
1191                 else
1192                         data_flow_mode = is_single_pass ?
1193                                 DES_to_HASH_and_DOUT : DIN_DES_DOUT;
1194         } else { /* Decrypt */
1195                 if (setup_flow_mode == S_DIN_to_AES)
1196                         data_flow_mode = is_single_pass ?
1197                                 AES_and_HASH : DIN_AES_DOUT;
1198                 else
1199                         data_flow_mode = is_single_pass ?
1200                                 DES_and_HASH : DIN_DES_DOUT;
1201         }
1202
1203         return data_flow_mode;
1204 }
1205
1206 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1207                             unsigned int *seq_size)
1208 {
1209         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1210         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1211         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1212         int direct = req_ctx->gen_ctx.op_type;
1213         unsigned int data_flow_mode =
1214                 cc_get_data_flow(direct, ctx->flow_mode,
1215                                  req_ctx->is_single_pass);
1216
1217         if (req_ctx->is_single_pass) {
1218                 /**
1219                  * Single-pass flow
1220                  */
1221                 cc_set_hmac_desc(req, desc, seq_size);
1222                 cc_set_cipher_desc(req, desc, seq_size);
1223                 cc_proc_header_desc(req, desc, seq_size);
1224                 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1225                 cc_proc_scheme_desc(req, desc, seq_size);
1226                 cc_proc_digest_desc(req, desc, seq_size);
1227                 return;
1228         }
1229
1230         /**
1231          * Double-pass flow
1232          * Fallback for unsupported single-pass modes,
1233          * i.e. using assoc. data of non-word-multiple
1234          */
1235         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1236                 /* encrypt first.. */
1237                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1238                 /* authenc after..*/
1239                 cc_set_hmac_desc(req, desc, seq_size);
1240                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1241                 cc_proc_scheme_desc(req, desc, seq_size);
1242                 cc_proc_digest_desc(req, desc, seq_size);
1243
1244         } else { /*DECRYPT*/
1245                 /* authenc first..*/
1246                 cc_set_hmac_desc(req, desc, seq_size);
1247                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1248                 cc_proc_scheme_desc(req, desc, seq_size);
1249                 /* decrypt after.. */
1250                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1251                 /* read the digest result with setting the completion bit
1252                  * must be after the cipher operation
1253                  */
1254                 cc_proc_digest_desc(req, desc, seq_size);
1255         }
1256 }
1257
1258 static void
1259 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1260                 unsigned int *seq_size)
1261 {
1262         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1263         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1264         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1265         int direct = req_ctx->gen_ctx.op_type;
1266         unsigned int data_flow_mode =
1267                 cc_get_data_flow(direct, ctx->flow_mode,
1268                                  req_ctx->is_single_pass);
1269
1270         if (req_ctx->is_single_pass) {
1271                 /**
1272                  * Single-pass flow
1273                  */
1274                 cc_set_xcbc_desc(req, desc, seq_size);
1275                 cc_set_cipher_desc(req, desc, seq_size);
1276                 cc_proc_header_desc(req, desc, seq_size);
1277                 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1278                 cc_proc_digest_desc(req, desc, seq_size);
1279                 return;
1280         }
1281
1282         /**
1283          * Double-pass flow
1284          * Fallback for unsupported single-pass modes,
1285          * i.e. using assoc. data of non-word-multiple
1286          */
1287         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1288                 /* encrypt first.. */
1289                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1290                 /* authenc after.. */
1291                 cc_set_xcbc_desc(req, desc, seq_size);
1292                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1293                 cc_proc_digest_desc(req, desc, seq_size);
1294         } else { /*DECRYPT*/
1295                 /* authenc first.. */
1296                 cc_set_xcbc_desc(req, desc, seq_size);
1297                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1298                 /* decrypt after..*/
1299                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1300                 /* read the digest result with setting the completion bit
1301                  * must be after the cipher operation
1302                  */
1303                 cc_proc_digest_desc(req, desc, seq_size);
1304         }
1305 }
1306
1307 static int validate_data_size(struct cc_aead_ctx *ctx,
1308                               enum drv_crypto_direction direct,
1309                               struct aead_request *req)
1310 {
1311         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1312         struct device *dev = drvdata_to_dev(ctx->drvdata);
1313         unsigned int assoclen = req->assoclen;
1314         unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?
1315                         (req->cryptlen - ctx->authsize) : req->cryptlen;
1316
1317         if (direct == DRV_CRYPTO_DIRECTION_DECRYPT &&
1318             req->cryptlen < ctx->authsize)
1319                 goto data_size_err;
1320
1321         areq_ctx->is_single_pass = true; /*defaulted to fast flow*/
1322
1323         switch (ctx->flow_mode) {
1324         case S_DIN_to_AES:
1325                 if (ctx->cipher_mode == DRV_CIPHER_CBC &&
1326                     !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE))
1327                         goto data_size_err;
1328                 if (ctx->cipher_mode == DRV_CIPHER_CCM)
1329                         break;
1330                 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1331                         if (areq_ctx->plaintext_authenticate_only)
1332                                 areq_ctx->is_single_pass = false;
1333                         break;
1334                 }
1335
1336                 if (!IS_ALIGNED(assoclen, sizeof(u32)))
1337                         areq_ctx->is_single_pass = false;
1338
1339                 if (ctx->cipher_mode == DRV_CIPHER_CTR &&
1340                     !IS_ALIGNED(cipherlen, sizeof(u32)))
1341                         areq_ctx->is_single_pass = false;
1342
1343                 break;
1344         case S_DIN_to_DES:
1345                 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE))
1346                         goto data_size_err;
1347                 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE))
1348                         areq_ctx->is_single_pass = false;
1349                 break;
1350         default:
1351                 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode);
1352                 goto data_size_err;
1353         }
1354
1355         return 0;
1356
1357 data_size_err:
1358         return -EINVAL;
1359 }
1360
1361 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size)
1362 {
1363         unsigned int len = 0;
1364
1365         if (header_size == 0)
1366                 return 0;
1367
1368         if (header_size < ((1UL << 16) - (1UL << 8))) {
1369                 len = 2;
1370
1371                 pa0_buff[0] = (header_size >> 8) & 0xFF;
1372                 pa0_buff[1] = header_size & 0xFF;
1373         } else {
1374                 len = 6;
1375
1376                 pa0_buff[0] = 0xFF;
1377                 pa0_buff[1] = 0xFE;
1378                 pa0_buff[2] = (header_size >> 24) & 0xFF;
1379                 pa0_buff[3] = (header_size >> 16) & 0xFF;
1380                 pa0_buff[4] = (header_size >> 8) & 0xFF;
1381                 pa0_buff[5] = header_size & 0xFF;
1382         }
1383
1384         return len;
1385 }
1386
1387 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize)
1388 {
1389         __be32 data;
1390
1391         memset(block, 0, csize);
1392         block += csize;
1393
1394         if (csize >= 4)
1395                 csize = 4;
1396         else if (msglen > (1 << (8 * csize)))
1397                 return -EOVERFLOW;
1398
1399         data = cpu_to_be32(msglen);
1400         memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
1401
1402         return 0;
1403 }
1404
1405 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[],
1406                   unsigned int *seq_size)
1407 {
1408         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1409         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1410         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1411         unsigned int idx = *seq_size;
1412         unsigned int cipher_flow_mode;
1413         dma_addr_t mac_result;
1414
1415         if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1416                 cipher_flow_mode = AES_to_HASH_and_DOUT;
1417                 mac_result = req_ctx->mac_buf_dma_addr;
1418         } else { /* Encrypt */
1419                 cipher_flow_mode = AES_and_HASH;
1420                 mac_result = req_ctx->icv_dma_addr;
1421         }
1422
1423         /* load key */
1424         hw_desc_init(&desc[idx]);
1425         set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1426         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1427                      ((ctx->enc_keylen == 24) ?  CC_AES_KEY_SIZE_MAX :
1428                       ctx->enc_keylen), NS_BIT);
1429         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1430         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1431         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1432         set_flow_mode(&desc[idx], S_DIN_to_AES);
1433         idx++;
1434
1435         /* load ctr state */
1436         hw_desc_init(&desc[idx]);
1437         set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1438         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1439         set_din_type(&desc[idx], DMA_DLLI,
1440                      req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT);
1441         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1442         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1443         set_flow_mode(&desc[idx], S_DIN_to_AES);
1444         idx++;
1445
1446         /* load MAC key */
1447         hw_desc_init(&desc[idx]);
1448         set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1449         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1450                      ((ctx->enc_keylen == 24) ?  CC_AES_KEY_SIZE_MAX :
1451                       ctx->enc_keylen), NS_BIT);
1452         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1453         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1454         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1455         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1456         set_aes_not_hash_mode(&desc[idx]);
1457         idx++;
1458
1459         /* load MAC state */
1460         hw_desc_init(&desc[idx]);
1461         set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1462         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1463         set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1464                      AES_BLOCK_SIZE, NS_BIT);
1465         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1466         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1467         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1468         set_aes_not_hash_mode(&desc[idx]);
1469         idx++;
1470
1471         /* process assoc data */
1472         if (req->assoclen > 0) {
1473                 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1474         } else {
1475                 hw_desc_init(&desc[idx]);
1476                 set_din_type(&desc[idx], DMA_DLLI,
1477                              sg_dma_address(&req_ctx->ccm_adata_sg),
1478                              AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT);
1479                 set_flow_mode(&desc[idx], DIN_HASH);
1480                 idx++;
1481         }
1482
1483         /* process the cipher */
1484         if (req_ctx->cryptlen)
1485                 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx);
1486
1487         /* Read temporal MAC */
1488         hw_desc_init(&desc[idx]);
1489         set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1490         set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize,
1491                       NS_BIT, 0);
1492         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1493         set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1494         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1495         set_aes_not_hash_mode(&desc[idx]);
1496         idx++;
1497
1498         /* load AES-CTR state (for last MAC calculation)*/
1499         hw_desc_init(&desc[idx]);
1500         set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1501         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1502         set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr,
1503                      AES_BLOCK_SIZE, NS_BIT);
1504         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1505         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1506         set_flow_mode(&desc[idx], S_DIN_to_AES);
1507         idx++;
1508
1509         hw_desc_init(&desc[idx]);
1510         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1511         set_dout_no_dma(&desc[idx], 0, 0, 1);
1512         idx++;
1513
1514         /* encrypt the "T" value and store MAC in mac_state */
1515         hw_desc_init(&desc[idx]);
1516         set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1517                      ctx->authsize, NS_BIT);
1518         set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1519         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1520         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1521         idx++;
1522
1523         *seq_size = idx;
1524         return 0;
1525 }
1526
1527 static int config_ccm_adata(struct aead_request *req)
1528 {
1529         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1530         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1531         struct device *dev = drvdata_to_dev(ctx->drvdata);
1532         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1533         //unsigned int size_of_a = 0, rem_a_size = 0;
1534         unsigned int lp = req->iv[0];
1535         /* Note: The code assume that req->iv[0] already contains the value
1536          * of L' of RFC3610
1537          */
1538         unsigned int l = lp + 1;  /* This is L' of RFC 3610. */
1539         unsigned int m = ctx->authsize;  /* This is M' of RFC 3610. */
1540         u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET;
1541         u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET;
1542         u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
1543         unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1544                                  DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1545                                 req->cryptlen :
1546                                 (req->cryptlen - ctx->authsize);
1547         int rc;
1548
1549         memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1550         memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3);
1551
1552         /* taken from crypto/ccm.c */
1553         /* 2 <= L <= 8, so 1 <= L' <= 7. */
1554         if (l < 2 || l > 8) {
1555                 dev_err(dev, "illegal iv value %X\n", req->iv[0]);
1556                 return -EINVAL;
1557         }
1558         memcpy(b0, req->iv, AES_BLOCK_SIZE);
1559
1560         /* format control info per RFC 3610 and
1561          * NIST Special Publication 800-38C
1562          */
1563         *b0 |= (8 * ((m - 2) / 2));
1564         if (req->assoclen > 0)
1565                 *b0 |= 64;  /* Enable bit 6 if Adata exists. */
1566
1567         rc = set_msg_len(b0 + 16 - l, cryptlen, l);  /* Write L'. */
1568         if (rc) {
1569                 dev_err(dev, "message len overflow detected");
1570                 return rc;
1571         }
1572          /* END of "taken from crypto/ccm.c" */
1573
1574         /* l(a) - size of associated data. */
1575         req_ctx->ccm_hdr_size = format_ccm_a0(a0, req->assoclen);
1576
1577         memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1);
1578         req->iv[15] = 1;
1579
1580         memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE);
1581         ctr_count_0[15] = 0;
1582
1583         return 0;
1584 }
1585
1586 static void cc_proc_rfc4309_ccm(struct aead_request *req)
1587 {
1588         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1589         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1590         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1591
1592         /* L' */
1593         memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE);
1594         /* For RFC 4309, always use 4 bytes for message length
1595          * (at most 2^32-1 bytes).
1596          */
1597         areq_ctx->ctr_iv[0] = 3;
1598
1599         /* In RFC 4309 there is an 11-bytes nonce+IV part,
1600          * that we build here.
1601          */
1602         memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce,
1603                CCM_BLOCK_NONCE_SIZE);
1604         memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv,
1605                CCM_BLOCK_IV_SIZE);
1606         req->iv = areq_ctx->ctr_iv;
1607         req->assoclen -= CCM_BLOCK_IV_SIZE;
1608 }
1609
1610 static void cc_set_ghash_desc(struct aead_request *req,
1611                               struct cc_hw_desc desc[], unsigned int *seq_size)
1612 {
1613         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1614         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1615         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1616         unsigned int idx = *seq_size;
1617
1618         /* load key to AES*/
1619         hw_desc_init(&desc[idx]);
1620         set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1621         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1622         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1623                      ctx->enc_keylen, NS_BIT);
1624         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1625         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1626         set_flow_mode(&desc[idx], S_DIN_to_AES);
1627         idx++;
1628
1629         /* process one zero block to generate hkey */
1630         hw_desc_init(&desc[idx]);
1631         set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1632         set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE,
1633                       NS_BIT, 0);
1634         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1635         idx++;
1636
1637         /* Memory Barrier */
1638         hw_desc_init(&desc[idx]);
1639         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1640         set_dout_no_dma(&desc[idx], 0, 0, 1);
1641         idx++;
1642
1643         /* Load GHASH subkey */
1644         hw_desc_init(&desc[idx]);
1645         set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr,
1646                      AES_BLOCK_SIZE, NS_BIT);
1647         set_dout_no_dma(&desc[idx], 0, 0, 1);
1648         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1649         set_aes_not_hash_mode(&desc[idx]);
1650         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1651         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1652         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1653         idx++;
1654
1655         /* Configure Hash Engine to work with GHASH.
1656          * Since it was not possible to extend HASH submodes to add GHASH,
1657          * The following command is necessary in order to
1658          * select GHASH (according to HW designers)
1659          */
1660         hw_desc_init(&desc[idx]);
1661         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1662         set_dout_no_dma(&desc[idx], 0, 0, 1);
1663         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1664         set_aes_not_hash_mode(&desc[idx]);
1665         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1666         set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK
1667         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1668         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1669         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1670         idx++;
1671
1672         /* Load GHASH initial STATE (which is 0). (for any hash there is an
1673          * initial state)
1674          */
1675         hw_desc_init(&desc[idx]);
1676         set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1677         set_dout_no_dma(&desc[idx], 0, 0, 1);
1678         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1679         set_aes_not_hash_mode(&desc[idx]);
1680         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1681         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1682         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1683         idx++;
1684
1685         *seq_size = idx;
1686 }
1687
1688 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[],
1689                              unsigned int *seq_size)
1690 {
1691         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1692         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1693         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1694         unsigned int idx = *seq_size;
1695
1696         /* load key to AES*/
1697         hw_desc_init(&desc[idx]);
1698         set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1699         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1700         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1701                      ctx->enc_keylen, NS_BIT);
1702         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1703         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1704         set_flow_mode(&desc[idx], S_DIN_to_AES);
1705         idx++;
1706
1707         if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) {
1708                 /* load AES/CTR initial CTR value inc by 2*/
1709                 hw_desc_init(&desc[idx]);
1710                 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1711                 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1712                 set_din_type(&desc[idx], DMA_DLLI,
1713                              req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE,
1714                              NS_BIT);
1715                 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1716                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1717                 set_flow_mode(&desc[idx], S_DIN_to_AES);
1718                 idx++;
1719         }
1720
1721         *seq_size = idx;
1722 }
1723
1724 static void cc_proc_gcm_result(struct aead_request *req,
1725                                struct cc_hw_desc desc[],
1726                                unsigned int *seq_size)
1727 {
1728         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1729         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1730         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1731         dma_addr_t mac_result;
1732         unsigned int idx = *seq_size;
1733
1734         if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1735                 mac_result = req_ctx->mac_buf_dma_addr;
1736         } else { /* Encrypt */
1737                 mac_result = req_ctx->icv_dma_addr;
1738         }
1739
1740         /* process(ghash) gcm_block_len */
1741         hw_desc_init(&desc[idx]);
1742         set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr,
1743                      AES_BLOCK_SIZE, NS_BIT);
1744         set_flow_mode(&desc[idx], DIN_HASH);
1745         idx++;
1746
1747         /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */
1748         hw_desc_init(&desc[idx]);
1749         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1750         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1751         set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE,
1752                       NS_BIT, 0);
1753         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1754         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1755         set_aes_not_hash_mode(&desc[idx]);
1756
1757         idx++;
1758
1759         /* load AES/CTR initial CTR value inc by 1*/
1760         hw_desc_init(&desc[idx]);
1761         set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1762         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1763         set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr,
1764                      AES_BLOCK_SIZE, NS_BIT);
1765         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1766         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1767         set_flow_mode(&desc[idx], S_DIN_to_AES);
1768         idx++;
1769
1770         /* Memory Barrier */
1771         hw_desc_init(&desc[idx]);
1772         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1773         set_dout_no_dma(&desc[idx], 0, 0, 1);
1774         idx++;
1775
1776         /* process GCTR on stored GHASH and store MAC in mac_state*/
1777         hw_desc_init(&desc[idx]);
1778         set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1779         set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1780                      AES_BLOCK_SIZE, NS_BIT);
1781         set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1782         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1783         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1784         idx++;
1785
1786         *seq_size = idx;
1787 }
1788
1789 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[],
1790                   unsigned int *seq_size)
1791 {
1792         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1793         unsigned int cipher_flow_mode;
1794
1795         if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1796                 cipher_flow_mode = AES_and_HASH;
1797         } else { /* Encrypt */
1798                 cipher_flow_mode = AES_to_HASH_and_DOUT;
1799         }
1800
1801         //in RFC4543 no data to encrypt. just copy data from src to dest.
1802         if (req_ctx->plaintext_authenticate_only) {
1803                 cc_proc_cipher_desc(req, BYPASS, desc, seq_size);
1804                 cc_set_ghash_desc(req, desc, seq_size);
1805                 /* process(ghash) assoc data */
1806                 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1807                 cc_set_gctr_desc(req, desc, seq_size);
1808                 cc_proc_gcm_result(req, desc, seq_size);
1809                 return 0;
1810         }
1811
1812         // for gcm and rfc4106.
1813         cc_set_ghash_desc(req, desc, seq_size);
1814         /* process(ghash) assoc data */
1815         if (req->assoclen > 0)
1816                 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1817         cc_set_gctr_desc(req, desc, seq_size);
1818         /* process(gctr+ghash) */
1819         if (req_ctx->cryptlen)
1820                 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size);
1821         cc_proc_gcm_result(req, desc, seq_size);
1822
1823         return 0;
1824 }
1825
1826 static int config_gcm_context(struct aead_request *req)
1827 {
1828         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1829         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1830         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1831         struct device *dev = drvdata_to_dev(ctx->drvdata);
1832
1833         unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1834                                  DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1835                                 req->cryptlen :
1836                                 (req->cryptlen - ctx->authsize);
1837         __be32 counter = cpu_to_be32(2);
1838
1839         dev_dbg(dev, "%s() cryptlen = %d, req->assoclen = %d ctx->authsize = %d\n",
1840                 __func__, cryptlen, req->assoclen, ctx->authsize);
1841
1842         memset(req_ctx->hkey, 0, AES_BLOCK_SIZE);
1843
1844         memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1845
1846         memcpy(req->iv + 12, &counter, 4);
1847         memcpy(req_ctx->gcm_iv_inc2, req->iv, 16);
1848
1849         counter = cpu_to_be32(1);
1850         memcpy(req->iv + 12, &counter, 4);
1851         memcpy(req_ctx->gcm_iv_inc1, req->iv, 16);
1852
1853         if (!req_ctx->plaintext_authenticate_only) {
1854                 __be64 temp64;
1855
1856                 temp64 = cpu_to_be64(req->assoclen * 8);
1857                 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1858                 temp64 = cpu_to_be64(cryptlen * 8);
1859                 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1860         } else {
1861                 /* rfc4543=>  all data(AAD,IV,Plain) are considered additional
1862                  * data that is nothing is encrypted.
1863                  */
1864                 __be64 temp64;
1865
1866                 temp64 = cpu_to_be64((req->assoclen + GCM_BLOCK_RFC4_IV_SIZE +
1867                                       cryptlen) * 8);
1868                 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1869                 temp64 = 0;
1870                 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1871         }
1872
1873         return 0;
1874 }
1875
1876 static void cc_proc_rfc4_gcm(struct aead_request *req)
1877 {
1878         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1879         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1880         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1881
1882         memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET,
1883                ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE);
1884         memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv,
1885                GCM_BLOCK_RFC4_IV_SIZE);
1886         req->iv = areq_ctx->ctr_iv;
1887         req->assoclen -= GCM_BLOCK_RFC4_IV_SIZE;
1888 }
1889
1890 static int cc_proc_aead(struct aead_request *req,
1891                         enum drv_crypto_direction direct)
1892 {
1893         int rc = 0;
1894         int seq_len = 0;
1895         struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ];
1896         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1897         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1898         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1899         struct device *dev = drvdata_to_dev(ctx->drvdata);
1900         struct cc_crypto_req cc_req = {};
1901
1902         dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n",
1903                 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"),
1904                 ctx, req, req->iv, sg_virt(req->src), req->src->offset,
1905                 sg_virt(req->dst), req->dst->offset, req->cryptlen);
1906
1907         /* STAT_PHASE_0: Init and sanity checks */
1908
1909         /* Check data length according to mode */
1910         if (validate_data_size(ctx, direct, req)) {
1911                 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n",
1912                         req->cryptlen, req->assoclen);
1913                 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN);
1914                 return -EINVAL;
1915         }
1916
1917         /* Setup request structure */
1918         cc_req.user_cb = (void *)cc_aead_complete;
1919         cc_req.user_arg = (void *)req;
1920
1921         /* Setup request context */
1922         areq_ctx->gen_ctx.op_type = direct;
1923         areq_ctx->req_authsize = ctx->authsize;
1924         areq_ctx->cipher_mode = ctx->cipher_mode;
1925
1926         /* STAT_PHASE_1: Map buffers */
1927
1928         if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1929                 /* Build CTR IV - Copy nonce from last 4 bytes in
1930                  * CTR key to first 4 bytes in CTR IV
1931                  */
1932                 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,
1933                        CTR_RFC3686_NONCE_SIZE);
1934                 if (!areq_ctx->backup_giv) /*User none-generated IV*/
1935                         memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE,
1936                                req->iv, CTR_RFC3686_IV_SIZE);
1937                 /* Initialize counter portion of counter block */
1938                 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
1939                             CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1940
1941                 /* Replace with counter iv */
1942                 req->iv = areq_ctx->ctr_iv;
1943                 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE;
1944         } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) ||
1945                    (ctx->cipher_mode == DRV_CIPHER_GCTR)) {
1946                 areq_ctx->hw_iv_size = AES_BLOCK_SIZE;
1947                 if (areq_ctx->ctr_iv != req->iv) {
1948                         memcpy(areq_ctx->ctr_iv, req->iv,
1949                                crypto_aead_ivsize(tfm));
1950                         req->iv = areq_ctx->ctr_iv;
1951                 }
1952         }  else {
1953                 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm);
1954         }
1955
1956         if (ctx->cipher_mode == DRV_CIPHER_CCM) {
1957                 rc = config_ccm_adata(req);
1958                 if (rc) {
1959                         dev_dbg(dev, "config_ccm_adata() returned with a failure %d!",
1960                                 rc);
1961                         goto exit;
1962                 }
1963         } else {
1964                 areq_ctx->ccm_hdr_size = ccm_header_size_null;
1965         }
1966
1967         if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1968                 rc = config_gcm_context(req);
1969                 if (rc) {
1970                         dev_dbg(dev, "config_gcm_context() returned with a failure %d!",
1971                                 rc);
1972                         goto exit;
1973                 }
1974         }
1975
1976         rc = cc_map_aead_request(ctx->drvdata, req);
1977         if (rc) {
1978                 dev_err(dev, "map_request() failed\n");
1979                 goto exit;
1980         }
1981
1982         /* do we need to generate IV? */
1983         if (areq_ctx->backup_giv) {
1984                 /* set the DMA mapped IV address*/
1985                 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1986                         cc_req.ivgen_dma_addr[0] =
1987                                 areq_ctx->gen_ctx.iv_dma_addr +
1988                                 CTR_RFC3686_NONCE_SIZE;
1989                         cc_req.ivgen_dma_addr_len = 1;
1990                 } else if (ctx->cipher_mode == DRV_CIPHER_CCM) {
1991                         /* In ccm, the IV needs to exist both inside B0 and
1992                          * inside the counter.It is also copied to iv_dma_addr
1993                          * for other reasons (like returning it to the user).
1994                          * So, using 3 (identical) IV outputs.
1995                          */
1996                         cc_req.ivgen_dma_addr[0] =
1997                                 areq_ctx->gen_ctx.iv_dma_addr +
1998                                 CCM_BLOCK_IV_OFFSET;
1999                         cc_req.ivgen_dma_addr[1] =
2000                                 sg_dma_address(&areq_ctx->ccm_adata_sg) +
2001                                 CCM_B0_OFFSET + CCM_BLOCK_IV_OFFSET;
2002                         cc_req.ivgen_dma_addr[2] =
2003                                 sg_dma_address(&areq_ctx->ccm_adata_sg) +
2004                                 CCM_CTR_COUNT_0_OFFSET + CCM_BLOCK_IV_OFFSET;
2005                         cc_req.ivgen_dma_addr_len = 3;
2006                 } else {
2007                         cc_req.ivgen_dma_addr[0] =
2008                                 areq_ctx->gen_ctx.iv_dma_addr;
2009                         cc_req.ivgen_dma_addr_len = 1;
2010                 }
2011
2012                 /* set the IV size (8/16 B long)*/
2013                 cc_req.ivgen_size = crypto_aead_ivsize(tfm);
2014         }
2015
2016         /* STAT_PHASE_2: Create sequence */
2017
2018         /* Load MLLI tables to SRAM if necessary */
2019         cc_mlli_to_sram(req, desc, &seq_len);
2020
2021         /*TODO: move seq len by reference */
2022         switch (ctx->auth_mode) {
2023         case DRV_HASH_SHA1:
2024         case DRV_HASH_SHA256:
2025                 cc_hmac_authenc(req, desc, &seq_len);
2026                 break;
2027         case DRV_HASH_XCBC_MAC:
2028                 cc_xcbc_authenc(req, desc, &seq_len);
2029                 break;
2030         case DRV_HASH_NULL:
2031                 if (ctx->cipher_mode == DRV_CIPHER_CCM)
2032                         cc_ccm(req, desc, &seq_len);
2033                 if (ctx->cipher_mode == DRV_CIPHER_GCTR)
2034                         cc_gcm(req, desc, &seq_len);
2035                 break;
2036         default:
2037                 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
2038                 cc_unmap_aead_request(dev, req);
2039                 rc = -ENOTSUPP;
2040                 goto exit;
2041         }
2042
2043         /* STAT_PHASE_3: Lock HW and push sequence */
2044
2045         rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base);
2046
2047         if (rc != -EINPROGRESS && rc != -EBUSY) {
2048                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
2049                 cc_unmap_aead_request(dev, req);
2050         }
2051
2052 exit:
2053         return rc;
2054 }
2055
2056 static int cc_aead_encrypt(struct aead_request *req)
2057 {
2058         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2059         int rc;
2060
2061         /* No generated IV required */
2062         areq_ctx->backup_iv = req->iv;
2063         areq_ctx->backup_giv = NULL;
2064         areq_ctx->is_gcm4543 = false;
2065
2066         areq_ctx->plaintext_authenticate_only = false;
2067
2068         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2069         if (rc != -EINPROGRESS && rc != -EBUSY)
2070                 req->iv = areq_ctx->backup_iv;
2071
2072         return rc;
2073 }
2074
2075 static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
2076 {
2077         /* Very similar to cc_aead_encrypt() above. */
2078
2079         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2080         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2081         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2082         struct device *dev = drvdata_to_dev(ctx->drvdata);
2083         int rc = -EINVAL;
2084
2085         if (!valid_assoclen(req)) {
2086                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2087                 goto out;
2088         }
2089
2090         /* No generated IV required */
2091         areq_ctx->backup_iv = req->iv;
2092         areq_ctx->backup_giv = NULL;
2093         areq_ctx->is_gcm4543 = true;
2094
2095         cc_proc_rfc4309_ccm(req);
2096
2097         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2098         if (rc != -EINPROGRESS && rc != -EBUSY)
2099                 req->iv = areq_ctx->backup_iv;
2100 out:
2101         return rc;
2102 }
2103
2104 static int cc_aead_decrypt(struct aead_request *req)
2105 {
2106         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2107         int rc;
2108
2109         /* No generated IV required */
2110         areq_ctx->backup_iv = req->iv;
2111         areq_ctx->backup_giv = NULL;
2112         areq_ctx->is_gcm4543 = false;
2113
2114         areq_ctx->plaintext_authenticate_only = false;
2115
2116         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2117         if (rc != -EINPROGRESS && rc != -EBUSY)
2118                 req->iv = areq_ctx->backup_iv;
2119
2120         return rc;
2121 }
2122
2123 static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
2124 {
2125         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2126         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2127         struct device *dev = drvdata_to_dev(ctx->drvdata);
2128         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2129         int rc = -EINVAL;
2130
2131         if (!valid_assoclen(req)) {
2132                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2133                 goto out;
2134         }
2135
2136         /* No generated IV required */
2137         areq_ctx->backup_iv = req->iv;
2138         areq_ctx->backup_giv = NULL;
2139
2140         areq_ctx->is_gcm4543 = true;
2141         cc_proc_rfc4309_ccm(req);
2142
2143         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2144         if (rc != -EINPROGRESS && rc != -EBUSY)
2145                 req->iv = areq_ctx->backup_iv;
2146
2147 out:
2148         return rc;
2149 }
2150
2151 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2152                                  unsigned int keylen)
2153 {
2154         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2155         struct device *dev = drvdata_to_dev(ctx->drvdata);
2156
2157         dev_dbg(dev, "%s()  keylen %d, key %p\n", __func__, keylen, key);
2158
2159         if (keylen < 4)
2160                 return -EINVAL;
2161
2162         keylen -= 4;
2163         memcpy(ctx->ctr_nonce, key + keylen, 4);
2164
2165         return cc_aead_setkey(tfm, key, keylen);
2166 }
2167
2168 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2169                                  unsigned int keylen)
2170 {
2171         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2172         struct device *dev = drvdata_to_dev(ctx->drvdata);
2173
2174         dev_dbg(dev, "%s()  keylen %d, key %p\n", __func__, keylen, key);
2175
2176         if (keylen < 4)
2177                 return -EINVAL;
2178
2179         keylen -= 4;
2180         memcpy(ctx->ctr_nonce, key + keylen, 4);
2181
2182         return cc_aead_setkey(tfm, key, keylen);
2183 }
2184
2185 static int cc_gcm_setauthsize(struct crypto_aead *authenc,
2186                               unsigned int authsize)
2187 {
2188         switch (authsize) {
2189         case 4:
2190         case 8:
2191         case 12:
2192         case 13:
2193         case 14:
2194         case 15:
2195         case 16:
2196                 break;
2197         default:
2198                 return -EINVAL;
2199         }
2200
2201         return cc_aead_setauthsize(authenc, authsize);
2202 }
2203
2204 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,
2205                                       unsigned int authsize)
2206 {
2207         struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2208         struct device *dev = drvdata_to_dev(ctx->drvdata);
2209
2210         dev_dbg(dev, "authsize %d\n", authsize);
2211
2212         switch (authsize) {
2213         case 8:
2214         case 12:
2215         case 16:
2216                 break;
2217         default:
2218                 return -EINVAL;
2219         }
2220
2221         return cc_aead_setauthsize(authenc, authsize);
2222 }
2223
2224 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,
2225                                       unsigned int authsize)
2226 {
2227         struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2228         struct device *dev = drvdata_to_dev(ctx->drvdata);
2229
2230         dev_dbg(dev, "authsize %d\n", authsize);
2231
2232         if (authsize != 16)
2233                 return -EINVAL;
2234
2235         return cc_aead_setauthsize(authenc, authsize);
2236 }
2237
2238 static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
2239 {
2240         /* Very similar to cc_aead_encrypt() above. */
2241
2242         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2243         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2244         struct device *dev = drvdata_to_dev(ctx->drvdata);
2245         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2246         int rc = -EINVAL;
2247
2248         if (!valid_assoclen(req)) {
2249                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2250                 goto out;
2251         }
2252
2253         /* No generated IV required */
2254         areq_ctx->backup_iv = req->iv;
2255         areq_ctx->backup_giv = NULL;
2256
2257         areq_ctx->plaintext_authenticate_only = false;
2258
2259         cc_proc_rfc4_gcm(req);
2260         areq_ctx->is_gcm4543 = true;
2261
2262         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2263         if (rc != -EINPROGRESS && rc != -EBUSY)
2264                 req->iv = areq_ctx->backup_iv;
2265 out:
2266         return rc;
2267 }
2268
2269 static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
2270 {
2271         /* Very similar to cc_aead_encrypt() above. */
2272
2273         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2274         int rc;
2275
2276         //plaintext is not encryped with rfc4543
2277         areq_ctx->plaintext_authenticate_only = true;
2278
2279         /* No generated IV required */
2280         areq_ctx->backup_iv = req->iv;
2281         areq_ctx->backup_giv = NULL;
2282
2283         cc_proc_rfc4_gcm(req);
2284         areq_ctx->is_gcm4543 = true;
2285
2286         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2287         if (rc != -EINPROGRESS && rc != -EBUSY)
2288                 req->iv = areq_ctx->backup_iv;
2289
2290         return rc;
2291 }
2292
2293 static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
2294 {
2295         /* Very similar to cc_aead_decrypt() above. */
2296
2297         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2298         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2299         struct device *dev = drvdata_to_dev(ctx->drvdata);
2300         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2301         int rc = -EINVAL;
2302
2303         if (!valid_assoclen(req)) {
2304                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2305                 goto out;
2306         }
2307
2308         /* No generated IV required */
2309         areq_ctx->backup_iv = req->iv;
2310         areq_ctx->backup_giv = NULL;
2311
2312         areq_ctx->plaintext_authenticate_only = false;
2313
2314         cc_proc_rfc4_gcm(req);
2315         areq_ctx->is_gcm4543 = true;
2316
2317         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2318         if (rc != -EINPROGRESS && rc != -EBUSY)
2319                 req->iv = areq_ctx->backup_iv;
2320 out:
2321         return rc;
2322 }
2323
2324 static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
2325 {
2326         /* Very similar to cc_aead_decrypt() above. */
2327
2328         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2329         int rc;
2330
2331         //plaintext is not decryped with rfc4543
2332         areq_ctx->plaintext_authenticate_only = true;
2333
2334         /* No generated IV required */
2335         areq_ctx->backup_iv = req->iv;
2336         areq_ctx->backup_giv = NULL;
2337
2338         cc_proc_rfc4_gcm(req);
2339         areq_ctx->is_gcm4543 = true;
2340
2341         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2342         if (rc != -EINPROGRESS && rc != -EBUSY)
2343                 req->iv = areq_ctx->backup_iv;
2344
2345         return rc;
2346 }
2347
2348 /* aead alg */
2349 static struct cc_alg_template aead_algs[] = {
2350         {
2351                 .name = "authenc(hmac(sha1),cbc(aes))",
2352                 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree",
2353                 .blocksize = AES_BLOCK_SIZE,
2354                 .template_aead = {
2355                         .setkey = cc_aead_setkey,
2356                         .setauthsize = cc_aead_setauthsize,
2357                         .encrypt = cc_aead_encrypt,
2358                         .decrypt = cc_aead_decrypt,
2359                         .init = cc_aead_init,
2360                         .exit = cc_aead_exit,
2361                         .ivsize = AES_BLOCK_SIZE,
2362                         .maxauthsize = SHA1_DIGEST_SIZE,
2363                 },
2364                 .cipher_mode = DRV_CIPHER_CBC,
2365                 .flow_mode = S_DIN_to_AES,
2366                 .auth_mode = DRV_HASH_SHA1,
2367                 .min_hw_rev = CC_HW_REV_630,
2368                 .std_body = CC_STD_NIST,
2369         },
2370         {
2371                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
2372                 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree",
2373                 .blocksize = DES3_EDE_BLOCK_SIZE,
2374                 .template_aead = {
2375                         .setkey = cc_aead_setkey,
2376                         .setauthsize = cc_aead_setauthsize,
2377                         .encrypt = cc_aead_encrypt,
2378                         .decrypt = cc_aead_decrypt,
2379                         .init = cc_aead_init,
2380                         .exit = cc_aead_exit,
2381                         .ivsize = DES3_EDE_BLOCK_SIZE,
2382                         .maxauthsize = SHA1_DIGEST_SIZE,
2383                 },
2384                 .cipher_mode = DRV_CIPHER_CBC,
2385                 .flow_mode = S_DIN_to_DES,
2386                 .auth_mode = DRV_HASH_SHA1,
2387                 .min_hw_rev = CC_HW_REV_630,
2388                 .std_body = CC_STD_NIST,
2389         },
2390         {
2391                 .name = "authenc(hmac(sha256),cbc(aes))",
2392                 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree",
2393                 .blocksize = AES_BLOCK_SIZE,
2394                 .template_aead = {
2395                         .setkey = cc_aead_setkey,
2396                         .setauthsize = cc_aead_setauthsize,
2397                         .encrypt = cc_aead_encrypt,
2398                         .decrypt = cc_aead_decrypt,
2399                         .init = cc_aead_init,
2400                         .exit = cc_aead_exit,
2401                         .ivsize = AES_BLOCK_SIZE,
2402                         .maxauthsize = SHA256_DIGEST_SIZE,
2403                 },
2404                 .cipher_mode = DRV_CIPHER_CBC,
2405                 .flow_mode = S_DIN_to_AES,
2406                 .auth_mode = DRV_HASH_SHA256,
2407                 .min_hw_rev = CC_HW_REV_630,
2408                 .std_body = CC_STD_NIST,
2409         },
2410         {
2411                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
2412                 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree",
2413                 .blocksize = DES3_EDE_BLOCK_SIZE,
2414                 .template_aead = {
2415                         .setkey = cc_aead_setkey,
2416                         .setauthsize = cc_aead_setauthsize,
2417                         .encrypt = cc_aead_encrypt,
2418                         .decrypt = cc_aead_decrypt,
2419                         .init = cc_aead_init,
2420                         .exit = cc_aead_exit,
2421                         .ivsize = DES3_EDE_BLOCK_SIZE,
2422                         .maxauthsize = SHA256_DIGEST_SIZE,
2423                 },
2424                 .cipher_mode = DRV_CIPHER_CBC,
2425                 .flow_mode = S_DIN_to_DES,
2426                 .auth_mode = DRV_HASH_SHA256,
2427                 .min_hw_rev = CC_HW_REV_630,
2428                 .std_body = CC_STD_NIST,
2429         },
2430         {
2431                 .name = "authenc(xcbc(aes),cbc(aes))",
2432                 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree",
2433                 .blocksize = AES_BLOCK_SIZE,
2434                 .template_aead = {
2435                         .setkey = cc_aead_setkey,
2436                         .setauthsize = cc_aead_setauthsize,
2437                         .encrypt = cc_aead_encrypt,
2438                         .decrypt = cc_aead_decrypt,
2439                         .init = cc_aead_init,
2440                         .exit = cc_aead_exit,
2441                         .ivsize = AES_BLOCK_SIZE,
2442                         .maxauthsize = AES_BLOCK_SIZE,
2443                 },
2444                 .cipher_mode = DRV_CIPHER_CBC,
2445                 .flow_mode = S_DIN_to_AES,
2446                 .auth_mode = DRV_HASH_XCBC_MAC,
2447                 .min_hw_rev = CC_HW_REV_630,
2448                 .std_body = CC_STD_NIST,
2449         },
2450         {
2451                 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2452                 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree",
2453                 .blocksize = 1,
2454                 .template_aead = {
2455                         .setkey = cc_aead_setkey,
2456                         .setauthsize = cc_aead_setauthsize,
2457                         .encrypt = cc_aead_encrypt,
2458                         .decrypt = cc_aead_decrypt,
2459                         .init = cc_aead_init,
2460                         .exit = cc_aead_exit,
2461                         .ivsize = CTR_RFC3686_IV_SIZE,
2462                         .maxauthsize = SHA1_DIGEST_SIZE,
2463                 },
2464                 .cipher_mode = DRV_CIPHER_CTR,
2465                 .flow_mode = S_DIN_to_AES,
2466                 .auth_mode = DRV_HASH_SHA1,
2467                 .min_hw_rev = CC_HW_REV_630,
2468                 .std_body = CC_STD_NIST,
2469         },
2470         {
2471                 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2472                 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree",
2473                 .blocksize = 1,
2474                 .template_aead = {
2475                         .setkey = cc_aead_setkey,
2476                         .setauthsize = cc_aead_setauthsize,
2477                         .encrypt = cc_aead_encrypt,
2478                         .decrypt = cc_aead_decrypt,
2479                         .init = cc_aead_init,
2480                         .exit = cc_aead_exit,
2481                         .ivsize = CTR_RFC3686_IV_SIZE,
2482                         .maxauthsize = SHA256_DIGEST_SIZE,
2483                 },
2484                 .cipher_mode = DRV_CIPHER_CTR,
2485                 .flow_mode = S_DIN_to_AES,
2486                 .auth_mode = DRV_HASH_SHA256,
2487                 .min_hw_rev = CC_HW_REV_630,
2488                 .std_body = CC_STD_NIST,
2489         },
2490         {
2491                 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))",
2492                 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree",
2493                 .blocksize = 1,
2494                 .template_aead = {
2495                         .setkey = cc_aead_setkey,
2496                         .setauthsize = cc_aead_setauthsize,
2497                         .encrypt = cc_aead_encrypt,
2498                         .decrypt = cc_aead_decrypt,
2499                         .init = cc_aead_init,
2500                         .exit = cc_aead_exit,
2501                         .ivsize = CTR_RFC3686_IV_SIZE,
2502                         .maxauthsize = AES_BLOCK_SIZE,
2503                 },
2504                 .cipher_mode = DRV_CIPHER_CTR,
2505                 .flow_mode = S_DIN_to_AES,
2506                 .auth_mode = DRV_HASH_XCBC_MAC,
2507                 .min_hw_rev = CC_HW_REV_630,
2508                 .std_body = CC_STD_NIST,
2509         },
2510         {
2511                 .name = "ccm(aes)",
2512                 .driver_name = "ccm-aes-ccree",
2513                 .blocksize = 1,
2514                 .template_aead = {
2515                         .setkey = cc_aead_setkey,
2516                         .setauthsize = cc_ccm_setauthsize,
2517                         .encrypt = cc_aead_encrypt,
2518                         .decrypt = cc_aead_decrypt,
2519                         .init = cc_aead_init,
2520                         .exit = cc_aead_exit,
2521                         .ivsize = AES_BLOCK_SIZE,
2522                         .maxauthsize = AES_BLOCK_SIZE,
2523                 },
2524                 .cipher_mode = DRV_CIPHER_CCM,
2525                 .flow_mode = S_DIN_to_AES,
2526                 .auth_mode = DRV_HASH_NULL,
2527                 .min_hw_rev = CC_HW_REV_630,
2528                 .std_body = CC_STD_NIST,
2529         },
2530         {
2531                 .name = "rfc4309(ccm(aes))",
2532                 .driver_name = "rfc4309-ccm-aes-ccree",
2533                 .blocksize = 1,
2534                 .template_aead = {
2535                         .setkey = cc_rfc4309_ccm_setkey,
2536                         .setauthsize = cc_rfc4309_ccm_setauthsize,
2537                         .encrypt = cc_rfc4309_ccm_encrypt,
2538                         .decrypt = cc_rfc4309_ccm_decrypt,
2539                         .init = cc_aead_init,
2540                         .exit = cc_aead_exit,
2541                         .ivsize = CCM_BLOCK_IV_SIZE,
2542                         .maxauthsize = AES_BLOCK_SIZE,
2543                 },
2544                 .cipher_mode = DRV_CIPHER_CCM,
2545                 .flow_mode = S_DIN_to_AES,
2546                 .auth_mode = DRV_HASH_NULL,
2547                 .min_hw_rev = CC_HW_REV_630,
2548                 .std_body = CC_STD_NIST,
2549         },
2550         {
2551                 .name = "gcm(aes)",
2552                 .driver_name = "gcm-aes-ccree",
2553                 .blocksize = 1,
2554                 .template_aead = {
2555                         .setkey = cc_aead_setkey,
2556                         .setauthsize = cc_gcm_setauthsize,
2557                         .encrypt = cc_aead_encrypt,
2558                         .decrypt = cc_aead_decrypt,
2559                         .init = cc_aead_init,
2560                         .exit = cc_aead_exit,
2561                         .ivsize = 12,
2562                         .maxauthsize = AES_BLOCK_SIZE,
2563                 },
2564                 .cipher_mode = DRV_CIPHER_GCTR,
2565                 .flow_mode = S_DIN_to_AES,
2566                 .auth_mode = DRV_HASH_NULL,
2567                 .min_hw_rev = CC_HW_REV_630,
2568                 .std_body = CC_STD_NIST,
2569         },
2570         {
2571                 .name = "rfc4106(gcm(aes))",
2572                 .driver_name = "rfc4106-gcm-aes-ccree",
2573                 .blocksize = 1,
2574                 .template_aead = {
2575                         .setkey = cc_rfc4106_gcm_setkey,
2576                         .setauthsize = cc_rfc4106_gcm_setauthsize,
2577                         .encrypt = cc_rfc4106_gcm_encrypt,
2578                         .decrypt = cc_rfc4106_gcm_decrypt,
2579                         .init = cc_aead_init,
2580                         .exit = cc_aead_exit,
2581                         .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2582                         .maxauthsize = AES_BLOCK_SIZE,
2583                 },
2584                 .cipher_mode = DRV_CIPHER_GCTR,
2585                 .flow_mode = S_DIN_to_AES,
2586                 .auth_mode = DRV_HASH_NULL,
2587                 .min_hw_rev = CC_HW_REV_630,
2588                 .std_body = CC_STD_NIST,
2589         },
2590         {
2591                 .name = "rfc4543(gcm(aes))",
2592                 .driver_name = "rfc4543-gcm-aes-ccree",
2593                 .blocksize = 1,
2594                 .template_aead = {
2595                         .setkey = cc_rfc4543_gcm_setkey,
2596                         .setauthsize = cc_rfc4543_gcm_setauthsize,
2597                         .encrypt = cc_rfc4543_gcm_encrypt,
2598                         .decrypt = cc_rfc4543_gcm_decrypt,
2599                         .init = cc_aead_init,
2600                         .exit = cc_aead_exit,
2601                         .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2602                         .maxauthsize = AES_BLOCK_SIZE,
2603                 },
2604                 .cipher_mode = DRV_CIPHER_GCTR,
2605                 .flow_mode = S_DIN_to_AES,
2606                 .auth_mode = DRV_HASH_NULL,
2607                 .min_hw_rev = CC_HW_REV_630,
2608                 .std_body = CC_STD_NIST,
2609         },
2610 };
2611
2612 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
2613                                                 struct device *dev)
2614 {
2615         struct cc_crypto_alg *t_alg;
2616         struct aead_alg *alg;
2617
2618         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
2619         if (!t_alg)
2620                 return ERR_PTR(-ENOMEM);
2621
2622         alg = &tmpl->template_aead;
2623
2624         snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
2625         snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2626                  tmpl->driver_name);
2627         alg->base.cra_module = THIS_MODULE;
2628         alg->base.cra_priority = CC_CRA_PRIO;
2629
2630         alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx);
2631         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2632         alg->init = cc_aead_init;
2633         alg->exit = cc_aead_exit;
2634
2635         t_alg->aead_alg = *alg;
2636
2637         t_alg->cipher_mode = tmpl->cipher_mode;
2638         t_alg->flow_mode = tmpl->flow_mode;
2639         t_alg->auth_mode = tmpl->auth_mode;
2640
2641         return t_alg;
2642 }
2643
2644 int cc_aead_free(struct cc_drvdata *drvdata)
2645 {
2646         struct cc_crypto_alg *t_alg, *n;
2647         struct cc_aead_handle *aead_handle =
2648                 (struct cc_aead_handle *)drvdata->aead_handle;
2649
2650         if (aead_handle) {
2651                 /* Remove registered algs */
2652                 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list,
2653                                          entry) {
2654                         crypto_unregister_aead(&t_alg->aead_alg);
2655                         list_del(&t_alg->entry);
2656                         kfree(t_alg);
2657                 }
2658                 kfree(aead_handle);
2659                 drvdata->aead_handle = NULL;
2660         }
2661
2662         return 0;
2663 }
2664
2665 int cc_aead_alloc(struct cc_drvdata *drvdata)
2666 {
2667         struct cc_aead_handle *aead_handle;
2668         struct cc_crypto_alg *t_alg;
2669         int rc = -ENOMEM;
2670         int alg;
2671         struct device *dev = drvdata_to_dev(drvdata);
2672
2673         aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL);
2674         if (!aead_handle) {
2675                 rc = -ENOMEM;
2676                 goto fail0;
2677         }
2678
2679         INIT_LIST_HEAD(&aead_handle->aead_list);
2680         drvdata->aead_handle = aead_handle;
2681
2682         aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata,
2683                                                          MAX_HMAC_DIGEST_SIZE);
2684
2685         if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) {
2686                 dev_err(dev, "SRAM pool exhausted\n");
2687                 rc = -ENOMEM;
2688                 goto fail1;
2689         }
2690
2691         /* Linux crypto */
2692         for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) {
2693                 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) ||
2694                     !(drvdata->std_bodies & aead_algs[alg].std_body))
2695                         continue;
2696
2697                 t_alg = cc_create_aead_alg(&aead_algs[alg], dev);
2698                 if (IS_ERR(t_alg)) {
2699                         rc = PTR_ERR(t_alg);
2700                         dev_err(dev, "%s alg allocation failed\n",
2701                                 aead_algs[alg].driver_name);
2702                         goto fail1;
2703                 }
2704                 t_alg->drvdata = drvdata;
2705                 rc = crypto_register_aead(&t_alg->aead_alg);
2706                 if (rc) {
2707                         dev_err(dev, "%s alg registration failed\n",
2708                                 t_alg->aead_alg.base.cra_driver_name);
2709                         goto fail2;
2710                 } else {
2711                         list_add_tail(&t_alg->entry, &aead_handle->aead_list);
2712                         dev_dbg(dev, "Registered %s\n",
2713                                 t_alg->aead_alg.base.cra_driver_name);
2714                 }
2715         }
2716
2717         return 0;
2718
2719 fail2:
2720         kfree(t_alg);
2721 fail1:
2722         cc_aead_free(drvdata);
2723 fail0:
2724         return rc;
2725 }