drivers/crypto/inside-secure/safexcel_hash.c

   1 /*
   2  * Copyright (C) 2017 Marvell
   3  *
   4  * Antoine Tenart <antoine.tenart@free-electrons.com>
   5  *
   6  * This file is licensed under the terms of the GNU General Public
   7  * License version 2. This program is licensed "as is" without any
   8  * warranty of any kind, whether express or implied.
   9  */
  10
  11 #include <crypto/hmac.h>
  12 #include <crypto/sha.h>
  13 #include <linux/device.h>
  14 #include <linux/dma-mapping.h>
  15 #include <linux/dmapool.h>
  16
  17
  18 #include "safexcel.h"
  19
  20 struct safexcel_ahash_ctx {
  21         struct safexcel_context base;
  22         struct safexcel_crypto_priv *priv;
  23
  24         u32 alg;
  25         u32 digest;
  26
  27         u32 ipad[SHA1_DIGEST_SIZE / sizeof(u32)];
  28         u32 opad[SHA1_DIGEST_SIZE / sizeof(u32)];
  29 };
  30
  31 struct safexcel_ahash_req {
  32         bool last_req;
  33         bool finish;
  34         bool hmac;
  35
  36         u8 state_sz;    /* expected sate size, only set once */
  37         u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
  38
  39         u64 len;
  40         u64 processed;
  41
  42         u8 cache[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
  43         u8 cache_next[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
  44 };
  45
  46 struct safexcel_ahash_export_state {
  47         u64 len;
  48         u64 processed;
  49
  50         u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
  51         u8 cache[SHA256_BLOCK_SIZE];
  52 };
  53
  54 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
  55                                 u32 input_length, u32 result_length)
  56 {
  57         struct safexcel_token *token =
  58                 (struct safexcel_token *)cdesc->control_data.token;
  59
  60         token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
  61         token[0].packet_length = input_length;
  62         token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
  63         token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
  64
  65         token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
  66         token[1].packet_length = result_length;
  67         token[1].stat = EIP197_TOKEN_STAT_LAST_HASH |
  68                         EIP197_TOKEN_STAT_LAST_PACKET;
  69         token[1].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
  70                                 EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
  71 }
  72
  73 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
  74                                      struct safexcel_ahash_req *req,
  75                                      struct safexcel_command_desc *cdesc,
  76                                      unsigned int digestsize,
  77                                      unsigned int blocksize)
  78 {
  79         int i;
  80
  81         cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_OUT;
  82         cdesc->control_data.control0 |= ctx->alg;
  83         cdesc->control_data.control0 |= ctx->digest;
  84
  85         if (ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) {
  86                 if (req->processed) {
  87                         if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
  88                                 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(6);
  89                         else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224 ||
  90                                  ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
  91                                 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(9);
  92
  93                         cdesc->control_data.control1 |= CONTEXT_CONTROL_DIGEST_CNT;
  94                 } else {
  95                         cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH;
  96                 }
  97
  98                 if (!req->finish)
  99                         cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH;
 100
 101                 /*
 102                  * Copy the input digest if needed, and setup the context
 103                  * fields. Do this now as we need it to setup the first command
 104                  * descriptor.
 105                  */
 106                 if (req->processed) {
 107                         for (i = 0; i < digestsize / sizeof(u32); i++)
 108                                 ctx->base.ctxr->data[i] = cpu_to_le32(req->state[i]);
 109
 110                         if (req->finish)
 111                                 ctx->base.ctxr->data[i] = cpu_to_le32(req->processed / blocksize);
 112                 }
 113         } else if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC) {
 114                 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(10);
 115
 116                 memcpy(ctx->base.ctxr->data, ctx->ipad, digestsize);
 117                 memcpy(ctx->base.ctxr->data + digestsize / sizeof(u32),
 118                        ctx->opad, digestsize);
 119         }
 120 }
 121
 122 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
 123                                   struct crypto_async_request *async,
 124                                   bool *should_complete, int *ret)
 125 {
 126         struct safexcel_result_desc *rdesc;
 127         struct ahash_request *areq = ahash_request_cast(async);
 128         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 129         struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
 130         int cache_len, result_sz = sreq->state_sz;
 131
 132         *ret = 0;
 133
 134         spin_lock_bh(&priv->ring[ring].egress_lock);
 135         rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 136         if (IS_ERR(rdesc)) {
 137                 dev_err(priv->dev,
 138                         "hash: result: could not retrieve the result descriptor\n");
 139                 *ret = PTR_ERR(rdesc);
 140         } else if (rdesc->result_data.error_code) {
 141                 dev_err(priv->dev,
 142                         "hash: result: result descriptor error (%d)\n",
 143                         rdesc->result_data.error_code);
 144                 *ret = -EINVAL;
 145         }
 146
 147         safexcel_complete(priv, ring);
 148         spin_unlock_bh(&priv->ring[ring].egress_lock);
 149
 150         if (sreq->finish)
 151                 result_sz = crypto_ahash_digestsize(ahash);
 152         memcpy(sreq->state, areq->result, result_sz);
 153
 154         dma_unmap_sg(priv->dev, areq->src,
 155                      sg_nents_for_len(areq->src, areq->nbytes), DMA_TO_DEVICE);
 156
 157         safexcel_free_context(priv, async, sreq->state_sz);
 158
 159         cache_len = sreq->len - sreq->processed;
 160         if (cache_len)
 161                 memcpy(sreq->cache, sreq->cache_next, cache_len);
 162
 163         *should_complete = true;
 164
 165         return 1;
 166 }
 167
 168 static int safexcel_ahash_send(struct crypto_async_request *async, int ring,
 169                                struct safexcel_request *request, int *commands,
 170                                int *results)
 171 {
 172         struct ahash_request *areq = ahash_request_cast(async);
 173         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 174         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 175         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 176         struct safexcel_crypto_priv *priv = ctx->priv;
 177         struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
 178         struct safexcel_result_desc *rdesc;
 179         struct scatterlist *sg;
 180         int i, nents, queued, len, cache_len, extra, n_cdesc = 0, ret = 0;
 181
 182         queued = len = req->len - req->processed;
 183         if (queued < crypto_ahash_blocksize(ahash))
 184                 cache_len = queued;
 185         else
 186                 cache_len = queued - areq->nbytes;
 187
 188         /*
 189          * If this is not the last request and the queued data does not fit
 190          * into full blocks, cache it for the next send() call.
 191          */
 192         extra = queued & (crypto_ahash_blocksize(ahash) - 1);
 193         if (!req->last_req && extra) {
 194                 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 195                                    req->cache_next, extra, areq->nbytes - extra);
 196
 197                 queued -= extra;
 198                 len -= extra;
 199         }
 200
 201         spin_lock_bh(&priv->ring[ring].egress_lock);
 202
 203         /* Add a command descriptor for the cached data, if any */
 204         if (cache_len) {
 205                 ctx->base.cache = kzalloc(cache_len, EIP197_GFP_FLAGS(*async));
 206                 if (!ctx->base.cache) {
 207                         ret = -ENOMEM;
 208                         goto unlock;
 209                 }
 210                 memcpy(ctx->base.cache, req->cache, cache_len);
 211                 ctx->base.cache_dma = dma_map_single(priv->dev, ctx->base.cache,
 212                                                      cache_len, DMA_TO_DEVICE);
 213                 if (dma_mapping_error(priv->dev, ctx->base.cache_dma)) {
 214                         ret = -EINVAL;
 215                         goto free_cache;
 216                 }
 217
 218                 ctx->base.cache_sz = cache_len;
 219                 first_cdesc = safexcel_add_cdesc(priv, ring, 1,
 220                                                  (cache_len == len),
 221                                                  ctx->base.cache_dma,
 222                                                  cache_len, len,
 223                                                  ctx->base.ctxr_dma);
 224                 if (IS_ERR(first_cdesc)) {
 225                         ret = PTR_ERR(first_cdesc);
 226                         goto unmap_cache;
 227                 }
 228                 n_cdesc++;
 229
 230                 queued -= cache_len;
 231                 if (!queued)
 232                         goto send_command;
 233         }
 234
 235         /* Now handle the current ahash request buffer(s) */
 236         nents = dma_map_sg(priv->dev, areq->src,
 237                        sg_nents_for_len(areq->src, areq->nbytes),
 238                        DMA_TO_DEVICE);
 239         if (!nents) {
 240                 ret = -ENOMEM;
 241                 goto cdesc_rollback;
 242         }
 243
 244         for_each_sg(areq->src, sg, nents, i) {
 245                 int sglen = sg_dma_len(sg);
 246
 247                 /* Do not overflow the request */
 248                 if (queued - sglen < 0)
 249                         sglen = queued;
 250
 251                 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
 252                                            !(queued - sglen), sg_dma_address(sg),
 253                                            sglen, len, ctx->base.ctxr_dma);
 254                 if (IS_ERR(cdesc)) {
 255                         ret = PTR_ERR(cdesc);
 256                         goto cdesc_rollback;
 257                 }
 258                 n_cdesc++;
 259
 260                 if (n_cdesc == 1)
 261                         first_cdesc = cdesc;
 262
 263                 queued -= sglen;
 264                 if (!queued)
 265                         break;
 266         }
 267
 268 send_command:
 269         /* Setup the context options */
 270         safexcel_context_control(ctx, req, first_cdesc, req->state_sz,
 271                                  crypto_ahash_blocksize(ahash));
 272
 273         /* Add the token */
 274         safexcel_hash_token(first_cdesc, len, req->state_sz);
 275
 276         ctx->base.result_dma = dma_map_single(priv->dev, areq->result,
 277                                               req->state_sz, DMA_FROM_DEVICE);
 278         if (dma_mapping_error(priv->dev, ctx->base.result_dma)) {
 279                 ret = -EINVAL;
 280                 goto cdesc_rollback;
 281         }
 282
 283         /* Add a result descriptor */
 284         rdesc = safexcel_add_rdesc(priv, ring, 1, 1, ctx->base.result_dma,
 285                                    req->state_sz);
 286         if (IS_ERR(rdesc)) {
 287                 ret = PTR_ERR(rdesc);
 288                 goto cdesc_rollback;
 289         }
 290
 291         spin_unlock_bh(&priv->ring[ring].egress_lock);
 292
 293         req->processed += len;
 294         request->req = &areq->base;
 295         ctx->base.handle_result = safexcel_handle_result;
 296
 297         *commands = n_cdesc;
 298         *results = 1;
 299         return 0;
 300
 301 cdesc_rollback:
 302         for (i = 0; i < n_cdesc; i++)
 303                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 304 unmap_cache:
 305         if (ctx->base.cache_dma) {
 306                 dma_unmap_single(priv->dev, ctx->base.cache_dma,
 307                                  ctx->base.cache_sz, DMA_TO_DEVICE);
 308                 ctx->base.cache_sz = 0;
 309         }
 310 free_cache:
 311         kfree(ctx->base.cache);
 312         ctx->base.cache = NULL;
 313
 314 unlock:
 315         spin_unlock_bh(&priv->ring[ring].egress_lock);
 316         return ret;
 317 }
 318
 319 static inline bool safexcel_ahash_needs_inv_get(struct ahash_request *areq)
 320 {
 321         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 322         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 323         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 324         unsigned int state_w_sz = req->state_sz / sizeof(u32);
 325         int i;
 326
 327         for (i = 0; i < state_w_sz; i++)
 328                 if (ctx->base.ctxr->data[i] != cpu_to_le32(req->state[i]))
 329                         return true;
 330
 331         if (ctx->base.ctxr->data[state_w_sz] !=
 332             cpu_to_le32(req->processed / crypto_ahash_blocksize(ahash)))
 333                 return true;
 334
 335         return false;
 336 }
 337
 338 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 339                                       int ring,
 340                                       struct crypto_async_request *async,
 341                                       bool *should_complete, int *ret)
 342 {
 343         struct safexcel_result_desc *rdesc;
 344         struct ahash_request *areq = ahash_request_cast(async);
 345         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 346         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
 347         int enq_ret;
 348
 349         *ret = 0;
 350
 351         spin_lock_bh(&priv->ring[ring].egress_lock);
 352         rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 353         if (IS_ERR(rdesc)) {
 354                 dev_err(priv->dev,
 355                         "hash: invalidate: could not retrieve the result descriptor\n");
 356                 *ret = PTR_ERR(rdesc);
 357         } else if (rdesc->result_data.error_code) {
 358                 dev_err(priv->dev,
 359                         "hash: invalidate: result descriptor error (%d)\n",
 360                         rdesc->result_data.error_code);
 361                 *ret = -EINVAL;
 362         }
 363
 364         safexcel_complete(priv, ring);
 365         spin_unlock_bh(&priv->ring[ring].egress_lock);
 366
 367         if (ctx->base.exit_inv) {
 368                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 369                               ctx->base.ctxr_dma);
 370
 371                 *should_complete = true;
 372                 return 1;
 373         }
 374
 375         ring = safexcel_select_ring(priv);
 376         ctx->base.ring = ring;
 377         ctx->base.needs_inv = false;
 378         ctx->base.send = safexcel_ahash_send;
 379
 380         spin_lock_bh(&priv->ring[ring].queue_lock);
 381         enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
 382         spin_unlock_bh(&priv->ring[ring].queue_lock);
 383
 384         if (enq_ret != -EINPROGRESS)
 385                 *ret = enq_ret;
 386
 387         if (!priv->ring[ring].need_dequeue)
 388                 safexcel_dequeue(priv, ring);
 389
 390         *should_complete = false;
 391
 392         return 1;
 393 }
 394
 395 static int safexcel_ahash_send_inv(struct crypto_async_request *async,
 396                                    int ring, struct safexcel_request *request,
 397                                    int *commands, int *results)
 398 {
 399         struct ahash_request *areq = ahash_request_cast(async);
 400         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 401         int ret;
 402
 403         ctx->base.handle_result = safexcel_handle_inv_result;
 404         ret = safexcel_invalidate_cache(async, &ctx->base, ctx->priv,
 405                                         ctx->base.ctxr_dma, ring, request);
 406         if (unlikely(ret))
 407                 return ret;
 408
 409         *commands = 1;
 410         *results = 1;
 411
 412         return 0;
 413 }
 414
 415 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
 416 {
 417         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 418         struct safexcel_crypto_priv *priv = ctx->priv;
 419         struct ahash_request req;
 420         struct safexcel_inv_result result = {};
 421         int ring = ctx->base.ring;
 422
 423         memset(&req, 0, sizeof(struct ahash_request));
 424
 425         /* create invalidation request */
 426         init_completion(&result.completion);
 427         ahash_request_set_callback(&req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 428                                    safexcel_inv_complete, &result);
 429
 430         ahash_request_set_tfm(&req, __crypto_ahash_cast(tfm));
 431         ctx = crypto_tfm_ctx(req.base.tfm);
 432         ctx->base.exit_inv = true;
 433         ctx->base.send = safexcel_ahash_send_inv;
 434
 435         spin_lock_bh(&priv->ring[ring].queue_lock);
 436         crypto_enqueue_request(&priv->ring[ring].queue, &req.base);
 437         spin_unlock_bh(&priv->ring[ring].queue_lock);
 438
 439         if (!priv->ring[ring].need_dequeue)
 440                 safexcel_dequeue(priv, ring);
 441
 442         wait_for_completion_interruptible(&result.completion);
 443
 444         if (result.error) {
 445                 dev_warn(priv->dev, "hash: completion error (%d)\n",
 446                          result.error);
 447                 return result.error;
 448         }
 449
 450         return 0;
 451 }
 452
 453 static int safexcel_ahash_cache(struct ahash_request *areq)
 454 {
 455         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 456         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 457         int queued, cache_len;
 458
 459         cache_len = req->len - areq->nbytes - req->processed;
 460         queued = req->len - req->processed;
 461
 462         /*
 463          * In case there isn't enough bytes to proceed (less than a
 464          * block size), cache the data until we have enough.
 465          */
 466         if (cache_len + areq->nbytes <= crypto_ahash_blocksize(ahash)) {
 467                 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 468                                    req->cache + cache_len,
 469                                    areq->nbytes, 0);
 470                 return areq->nbytes;
 471         }
 472
 473         /* We could'nt cache all the data */
 474         return -E2BIG;
 475 }
 476
 477 static int safexcel_ahash_enqueue(struct ahash_request *areq)
 478 {
 479         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 480         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 481         struct safexcel_crypto_priv *priv = ctx->priv;
 482         int ret, ring;
 483
 484         ctx->base.send = safexcel_ahash_send;
 485
 486         if (req->processed && ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)
 487                 ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq);
 488
 489         if (ctx->base.ctxr) {
 490                 if (ctx->base.needs_inv)
 491                         ctx->base.send = safexcel_ahash_send_inv;
 492         } else {
 493                 ctx->base.ring = safexcel_select_ring(priv);
 494                 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
 495                                                  EIP197_GFP_FLAGS(areq->base),
 496                                                  &ctx->base.ctxr_dma);
 497                 if (!ctx->base.ctxr)
 498                         return -ENOMEM;
 499         }
 500
 501         ring = ctx->base.ring;
 502
 503         spin_lock_bh(&priv->ring[ring].queue_lock);
 504         ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
 505         spin_unlock_bh(&priv->ring[ring].queue_lock);
 506
 507         if (!priv->ring[ring].need_dequeue)
 508                 safexcel_dequeue(priv, ring);
 509
 510         return ret;
 511 }
 512
 513 static int safexcel_ahash_update(struct ahash_request *areq)
 514 {
 515         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 516         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 517         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 518
 519         /* If the request is 0 length, do nothing */
 520         if (!areq->nbytes)
 521                 return 0;
 522
 523         req->len += areq->nbytes;
 524
 525         safexcel_ahash_cache(areq);
 526
 527         /*
 528          * We're not doing partial updates when performing an hmac request.
 529          * Everything will be handled by the final() call.
 530          */
 531         if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC)
 532                 return 0;
 533
 534         if (req->hmac)
 535                 return safexcel_ahash_enqueue(areq);
 536
 537         if (!req->last_req &&
 538             req->len - req->processed > crypto_ahash_blocksize(ahash))
 539                 return safexcel_ahash_enqueue(areq);
 540
 541         return 0;
 542 }
 543
 544 static int safexcel_ahash_final(struct ahash_request *areq)
 545 {
 546         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 547         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 548
 549         req->last_req = true;
 550         req->finish = true;
 551
 552         /* If we have an overall 0 length request */
 553         if (!(req->len + areq->nbytes)) {
 554                 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
 555                         memcpy(areq->result, sha1_zero_message_hash,
 556                                SHA1_DIGEST_SIZE);
 557                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
 558                         memcpy(areq->result, sha224_zero_message_hash,
 559                                SHA224_DIGEST_SIZE);
 560                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
 561                         memcpy(areq->result, sha256_zero_message_hash,
 562                                SHA256_DIGEST_SIZE);
 563
 564                 return 0;
 565         }
 566
 567         return safexcel_ahash_enqueue(areq);
 568 }
 569
 570 static int safexcel_ahash_finup(struct ahash_request *areq)
 571 {
 572         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 573
 574         req->last_req = true;
 575         req->finish = true;
 576
 577         safexcel_ahash_update(areq);
 578         return safexcel_ahash_final(areq);
 579 }
 580
 581 static int safexcel_ahash_export(struct ahash_request *areq, void *out)
 582 {
 583         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 584         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 585         struct safexcel_ahash_export_state *export = out;
 586
 587         export->len = req->len;
 588         export->processed = req->processed;
 589
 590         memcpy(export->state, req->state, req->state_sz);
 591         memset(export->cache, 0, crypto_ahash_blocksize(ahash));
 592         memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash));
 593
 594         return 0;
 595 }
 596
 597 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
 598 {
 599         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 600         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 601         const struct safexcel_ahash_export_state *export = in;
 602         int ret;
 603
 604         ret = crypto_ahash_init(areq);
 605         if (ret)
 606                 return ret;
 607
 608         req->len = export->len;
 609         req->processed = export->processed;
 610
 611         memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash));
 612         memcpy(req->state, export->state, req->state_sz);
 613
 614         return 0;
 615 }
 616
 617 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
 618 {
 619         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 620         struct safexcel_alg_template *tmpl =
 621                 container_of(__crypto_ahash_alg(tfm->__crt_alg),
 622                              struct safexcel_alg_template, alg.ahash);
 623
 624         ctx->priv = tmpl->priv;
 625
 626         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 627                                  sizeof(struct safexcel_ahash_req));
 628         return 0;
 629 }
 630
 631 static int safexcel_sha1_init(struct ahash_request *areq)
 632 {
 633         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 634         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 635
 636         memset(req, 0, sizeof(*req));
 637
 638         req->state[0] = SHA1_H0;
 639         req->state[1] = SHA1_H1;
 640         req->state[2] = SHA1_H2;
 641         req->state[3] = SHA1_H3;
 642         req->state[4] = SHA1_H4;
 643
 644         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
 645         ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 646         req->state_sz = SHA1_DIGEST_SIZE;
 647
 648         return 0;
 649 }
 650
 651 static int safexcel_sha1_digest(struct ahash_request *areq)
 652 {
 653         int ret = safexcel_sha1_init(areq);
 654
 655         if (ret)
 656                 return ret;
 657
 658         return safexcel_ahash_finup(areq);
 659 }
 660
 661 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
 662 {
 663         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 664         struct safexcel_crypto_priv *priv = ctx->priv;
 665         int ret;
 666
 667         /* context not allocated, skip invalidation */
 668         if (!ctx->base.ctxr)
 669                 return;
 670
 671         ret = safexcel_ahash_exit_inv(tfm);
 672         if (ret)
 673                 dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
 674 }
 675
 676 struct safexcel_alg_template safexcel_alg_sha1 = {
 677         .type = SAFEXCEL_ALG_TYPE_AHASH,
 678         .alg.ahash = {
 679                 .init = safexcel_sha1_init,
 680                 .update = safexcel_ahash_update,
 681                 .final = safexcel_ahash_final,
 682                 .finup = safexcel_ahash_finup,
 683                 .digest = safexcel_sha1_digest,
 684                 .export = safexcel_ahash_export,
 685                 .import = safexcel_ahash_import,
 686                 .halg = {
 687                         .digestsize = SHA1_DIGEST_SIZE,
 688                         .statesize = sizeof(struct safexcel_ahash_export_state),
 689                         .base = {
 690                                 .cra_name = "sha1",
 691                                 .cra_driver_name = "safexcel-sha1",
 692                                 .cra_priority = 300,
 693                                 .cra_flags = CRYPTO_ALG_ASYNC |
 694                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
 695                                 .cra_blocksize = SHA1_BLOCK_SIZE,
 696                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
 697                                 .cra_init = safexcel_ahash_cra_init,
 698                                 .cra_exit = safexcel_ahash_cra_exit,
 699                                 .cra_module = THIS_MODULE,
 700                         },
 701                 },
 702         },
 703 };
 704
 705 static int safexcel_hmac_sha1_init(struct ahash_request *areq)
 706 {
 707         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 708
 709         safexcel_sha1_init(areq);
 710         ctx->digest = CONTEXT_CONTROL_DIGEST_HMAC;
 711         return 0;
 712 }
 713
 714 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
 715 {
 716         int ret = safexcel_hmac_sha1_init(areq);
 717
 718         if (ret)
 719                 return ret;
 720
 721         return safexcel_ahash_finup(areq);
 722 }
 723
 724 struct safexcel_ahash_result {
 725         struct completion completion;
 726         int error;
 727 };
 728
 729 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)
 730 {
 731         struct safexcel_ahash_result *result = req->data;
 732
 733         if (error == -EINPROGRESS)
 734                 return;
 735
 736         result->error = error;
 737         complete(&result->completion);
 738 }
 739
 740 static int safexcel_hmac_init_pad(struct ahash_request *areq,
 741                                   unsigned int blocksize, const u8 *key,
 742                                   unsigned int keylen, u8 *ipad, u8 *opad)
 743 {
 744         struct safexcel_ahash_result result;
 745         struct scatterlist sg;
 746         int ret, i;
 747         u8 *keydup;
 748
 749         if (keylen <= blocksize) {
 750                 memcpy(ipad, key, keylen);
 751         } else {
 752                 keydup = kmemdup(key, keylen, GFP_KERNEL);
 753                 if (!keydup)
 754                         return -ENOMEM;
 755
 756                 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
 757                                            safexcel_ahash_complete, &result);
 758                 sg_init_one(&sg, keydup, keylen);
 759                 ahash_request_set_crypt(areq, &sg, ipad, keylen);
 760                 init_completion(&result.completion);
 761
 762                 ret = crypto_ahash_digest(areq);
 763                 if (ret == -EINPROGRESS) {
 764                         wait_for_completion_interruptible(&result.completion);
 765                         ret = result.error;
 766                 }
 767
 768                 /* Avoid leaking */
 769                 memzero_explicit(keydup, keylen);
 770                 kfree(keydup);
 771
 772                 if (ret)
 773                         return ret;
 774
 775                 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
 776         }
 777
 778         memset(ipad + keylen, 0, blocksize - keylen);
 779         memcpy(opad, ipad, blocksize);
 780
 781         for (i = 0; i < blocksize; i++) {
 782                 ipad[i] ^= HMAC_IPAD_VALUE;
 783                 opad[i] ^= HMAC_OPAD_VALUE;
 784         }
 785
 786         return 0;
 787 }
 788
 789 static int safexcel_hmac_init_iv(struct ahash_request *areq,
 790                                  unsigned int blocksize, u8 *pad, void *state)
 791 {
 792         struct safexcel_ahash_result result;
 793         struct safexcel_ahash_req *req;
 794         struct scatterlist sg;
 795         int ret;
 796
 797         ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
 798                                    safexcel_ahash_complete, &result);
 799         sg_init_one(&sg, pad, blocksize);
 800         ahash_request_set_crypt(areq, &sg, pad, blocksize);
 801         init_completion(&result.completion);
 802
 803         ret = crypto_ahash_init(areq);
 804         if (ret)
 805                 return ret;
 806
 807         req = ahash_request_ctx(areq);
 808         req->hmac = true;
 809         req->last_req = true;
 810
 811         ret = crypto_ahash_update(areq);
 812         if (ret && ret != -EINPROGRESS)
 813                 return ret;
 814
 815         wait_for_completion_interruptible(&result.completion);
 816         if (result.error)
 817                 return result.error;
 818
 819         return crypto_ahash_export(areq, state);
 820 }
 821
 822 static int safexcel_hmac_setkey(const char *alg, const u8 *key,
 823                                 unsigned int keylen, void *istate, void *ostate)
 824 {
 825         struct ahash_request *areq;
 826         struct crypto_ahash *tfm;
 827         unsigned int blocksize;
 828         u8 *ipad, *opad;
 829         int ret;
 830
 831         tfm = crypto_alloc_ahash(alg, CRYPTO_ALG_TYPE_AHASH,
 832                                  CRYPTO_ALG_TYPE_AHASH_MASK);
 833         if (IS_ERR(tfm))
 834                 return PTR_ERR(tfm);
 835
 836         areq = ahash_request_alloc(tfm, GFP_KERNEL);
 837         if (!areq) {
 838                 ret = -ENOMEM;
 839                 goto free_ahash;
 840         }
 841
 842         crypto_ahash_clear_flags(tfm, ~0);
 843         blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
 844
 845         ipad = kzalloc(2 * blocksize, GFP_KERNEL);
 846         if (!ipad) {
 847                 ret = -ENOMEM;
 848                 goto free_request;
 849         }
 850
 851         opad = ipad + blocksize;
 852
 853         ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
 854         if (ret)
 855                 goto free_ipad;
 856
 857         ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
 858         if (ret)
 859                 goto free_ipad;
 860
 861         ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
 862
 863 free_ipad:
 864         kfree(ipad);
 865 free_request:
 866         ahash_request_free(areq);
 867 free_ahash:
 868         crypto_free_ahash(tfm);
 869
 870         return ret;
 871 }
 872
 873 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
 874                                      unsigned int keylen)
 875 {
 876         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
 877         struct safexcel_ahash_export_state istate, ostate;
 878         int ret, i;
 879
 880         ret = safexcel_hmac_setkey("safexcel-sha1", key, keylen, &istate, &ostate);
 881         if (ret)
 882                 return ret;
 883
 884         for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) {
 885                 if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) ||
 886                     ctx->opad[i] != le32_to_cpu(ostate.state[i])) {
 887                         ctx->base.needs_inv = true;
 888                         break;
 889                 }
 890         }
 891
 892         memcpy(ctx->ipad, &istate.state, SHA1_DIGEST_SIZE);
 893         memcpy(ctx->opad, &ostate.state, SHA1_DIGEST_SIZE);
 894
 895         return 0;
 896 }
 897
 898 struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
 899         .type = SAFEXCEL_ALG_TYPE_AHASH,
 900         .alg.ahash = {
 901                 .init = safexcel_hmac_sha1_init,
 902                 .update = safexcel_ahash_update,
 903                 .final = safexcel_ahash_final,
 904                 .finup = safexcel_ahash_finup,
 905                 .digest = safexcel_hmac_sha1_digest,
 906                 .setkey = safexcel_hmac_sha1_setkey,
 907                 .export = safexcel_ahash_export,
 908                 .import = safexcel_ahash_import,
 909                 .halg = {
 910                         .digestsize = SHA1_DIGEST_SIZE,
 911                         .statesize = sizeof(struct safexcel_ahash_export_state),
 912                         .base = {
 913                                 .cra_name = "hmac(sha1)",
 914                                 .cra_driver_name = "safexcel-hmac-sha1",
 915                                 .cra_priority = 300,
 916                                 .cra_flags = CRYPTO_ALG_ASYNC |
 917                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
 918                                 .cra_blocksize = SHA1_BLOCK_SIZE,
 919                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
 920                                 .cra_init = safexcel_ahash_cra_init,
 921                                 .cra_exit = safexcel_ahash_cra_exit,
 922                                 .cra_module = THIS_MODULE,
 923                         },
 924                 },
 925         },
 926 };
 927
 928 static int safexcel_sha256_init(struct ahash_request *areq)
 929 {
 930         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 931         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 932
 933         memset(req, 0, sizeof(*req));
 934
 935         req->state[0] = SHA256_H0;
 936         req->state[1] = SHA256_H1;
 937         req->state[2] = SHA256_H2;
 938         req->state[3] = SHA256_H3;
 939         req->state[4] = SHA256_H4;
 940         req->state[5] = SHA256_H5;
 941         req->state[6] = SHA256_H6;
 942         req->state[7] = SHA256_H7;
 943
 944         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
 945         ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 946         req->state_sz = SHA256_DIGEST_SIZE;
 947
 948         return 0;
 949 }
 950
 951 static int safexcel_sha256_digest(struct ahash_request *areq)
 952 {
 953         int ret = safexcel_sha256_init(areq);
 954
 955         if (ret)
 956                 return ret;
 957
 958         return safexcel_ahash_finup(areq);
 959 }
 960
 961 struct safexcel_alg_template safexcel_alg_sha256 = {
 962         .type = SAFEXCEL_ALG_TYPE_AHASH,
 963         .alg.ahash = {
 964                 .init = safexcel_sha256_init,
 965                 .update = safexcel_ahash_update,
 966                 .final = safexcel_ahash_final,
 967                 .finup = safexcel_ahash_finup,
 968                 .digest = safexcel_sha256_digest,
 969                 .export = safexcel_ahash_export,
 970                 .import = safexcel_ahash_import,
 971                 .halg = {
 972                         .digestsize = SHA256_DIGEST_SIZE,
 973                         .statesize = sizeof(struct safexcel_ahash_export_state),
 974                         .base = {
 975                                 .cra_name = "sha256",
 976                                 .cra_driver_name = "safexcel-sha256",
 977                                 .cra_priority = 300,
 978                                 .cra_flags = CRYPTO_ALG_ASYNC |
 979                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
 980                                 .cra_blocksize = SHA256_BLOCK_SIZE,
 981                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
 982                                 .cra_init = safexcel_ahash_cra_init,
 983                                 .cra_exit = safexcel_ahash_cra_exit,
 984                                 .cra_module = THIS_MODULE,
 985                         },
 986                 },
 987         },
 988 };
 989
 990 static int safexcel_sha224_init(struct ahash_request *areq)
 991 {
 992         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 993         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 994
 995         memset(req, 0, sizeof(*req));
 996
 997         req->state[0] = SHA224_H0;
 998         req->state[1] = SHA224_H1;
 999         req->state[2] = SHA224_H2;
1000         req->state[3] = SHA224_H3;
1001         req->state[4] = SHA224_H4;
1002         req->state[5] = SHA224_H5;
1003         req->state[6] = SHA224_H6;
1004         req->state[7] = SHA224_H7;
1005
1006         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1007         ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1008         req->state_sz = SHA256_DIGEST_SIZE;
1009
1010         return 0;
1011 }
1012
1013 static int safexcel_sha224_digest(struct ahash_request *areq)
1014 {
1015         int ret = safexcel_sha224_init(areq);
1016
1017         if (ret)
1018                 return ret;
1019
1020         return safexcel_ahash_finup(areq);
1021 }
1022
1023 struct safexcel_alg_template safexcel_alg_sha224 = {
1024         .type = SAFEXCEL_ALG_TYPE_AHASH,
1025         .alg.ahash = {
1026                 .init = safexcel_sha224_init,
1027                 .update = safexcel_ahash_update,
1028                 .final = safexcel_ahash_final,
1029                 .finup = safexcel_ahash_finup,
1030                 .digest = safexcel_sha224_digest,
1031                 .export = safexcel_ahash_export,
1032                 .import = safexcel_ahash_import,
1033                 .halg = {
1034                         .digestsize = SHA224_DIGEST_SIZE,
1035                         .statesize = sizeof(struct safexcel_ahash_export_state),
1036                         .base = {
1037                                 .cra_name = "sha224",
1038                                 .cra_driver_name = "safexcel-sha224",
1039                                 .cra_priority = 300,
1040                                 .cra_flags = CRYPTO_ALG_ASYNC |
1041                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1042                                 .cra_blocksize = SHA224_BLOCK_SIZE,
1043                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1044                                 .cra_init = safexcel_ahash_cra_init,
1045                                 .cra_exit = safexcel_ahash_cra_exit,
1046                                 .cra_module = THIS_MODULE,
1047                         },
1048                 },
1049         },
1050 };