drivers/crypto/cavium/cpt/cptvf_reqmanager.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2016 Cavium, Inc.
   4  */
   5
   6 #include "cptvf.h"
   7 #include "cptvf_algs.h"
   8 #include "request_manager.h"
   9
  10 /**
  11  * get_free_pending_entry - get free entry from pending queue
  12  * @param pqinfo: pending_qinfo structure
  13  * @param qno: queue number
  14  */
  15 static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
  16                                                     int qlen)
  17 {
  18         struct pending_entry *ent = NULL;
  19
  20         ent = &q->head[q->rear];
  21         if (unlikely(ent->busy)) {
  22                 ent = NULL;
  23                 goto no_free_entry;
  24         }
  25
  26         q->rear++;
  27         if (unlikely(q->rear == qlen))
  28                 q->rear = 0;
  29
  30 no_free_entry:
  31         return ent;
  32 }
  33
  34 static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
  35                                            int qno)
  36 {
  37         struct pending_queue *queue = &pqinfo->queue[qno];
  38
  39         queue->front++;
  40         if (unlikely(queue->front == pqinfo->qlen))
  41                 queue->front = 0;
  42 }
  43
  44 static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
  45                                  int buf_count, u8 *buffer)
  46 {
  47         int ret = 0, i, j;
  48         int components;
  49         struct sglist_component *sg_ptr = NULL;
  50         struct pci_dev *pdev = cptvf->pdev;
  51
  52         if (unlikely(!list)) {
  53                 dev_err(&pdev->dev, "Input List pointer is NULL\n");
  54                 return -EFAULT;
  55         }
  56
  57         for (i = 0; i < buf_count; i++) {
  58                 if (likely(list[i].vptr)) {
  59                         list[i].dma_addr = dma_map_single(&pdev->dev,
  60                                                           list[i].vptr,
  61                                                           list[i].size,
  62                                                           DMA_BIDIRECTIONAL);
  63                         if (unlikely(dma_mapping_error(&pdev->dev,
  64                                                        list[i].dma_addr))) {
  65                                 dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
  66                                         i);
  67                                 ret = -EIO;
  68                                 goto sg_cleanup;
  69                         }
  70                 }
  71         }
  72
  73         components = buf_count / 4;
  74         sg_ptr = (struct sglist_component *)buffer;
  75         for (i = 0; i < components; i++) {
  76                 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
  77                 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
  78                 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
  79                 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
  80                 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
  81                 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
  82                 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
  83                 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
  84                 sg_ptr++;
  85         }
  86
  87         components = buf_count % 4;
  88
  89         switch (components) {
  90         case 3:
  91                 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
  92                 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
  93                 fallthrough;
  94         case 2:
  95                 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
  96                 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
  97                 fallthrough;
  98         case 1:
  99                 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
 100                 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
 101                 break;
 102         default:
 103                 break;
 104         }
 105
 106         return ret;
 107
 108 sg_cleanup:
 109         for (j = 0; j < i; j++) {
 110                 if (list[j].dma_addr) {
 111                         dma_unmap_single(&pdev->dev, list[i].dma_addr,
 112                                          list[i].size, DMA_BIDIRECTIONAL);
 113                 }
 114
 115                 list[j].dma_addr = 0;
 116         }
 117
 118         return ret;
 119 }
 120
 121 static inline int setup_sgio_list(struct cpt_vf *cptvf,
 122                                   struct cpt_info_buffer *info,
 123                                   struct cpt_request_info *req)
 124 {
 125         u16 g_sz_bytes = 0, s_sz_bytes = 0;
 126         int ret = 0;
 127         struct pci_dev *pdev = cptvf->pdev;
 128
 129         if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
 130                 dev_err(&pdev->dev, "Request SG components are higher than supported\n");
 131                 ret = -EINVAL;
 132                 goto  scatter_gather_clean;
 133         }
 134
 135         /* Setup gather (input) components */
 136         g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
 137         info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
 138         if (!info->gather_components) {
 139                 ret = -ENOMEM;
 140                 goto  scatter_gather_clean;
 141         }
 142
 143         ret = setup_sgio_components(cptvf, req->in,
 144                                     req->incnt,
 145                                     info->gather_components);
 146         if (ret) {
 147                 dev_err(&pdev->dev, "Failed to setup gather list\n");
 148                 ret = -EFAULT;
 149                 goto  scatter_gather_clean;
 150         }
 151
 152         /* Setup scatter (output) components */
 153         s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
 154         info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
 155         if (!info->scatter_components) {
 156                 ret = -ENOMEM;
 157                 goto  scatter_gather_clean;
 158         }
 159
 160         ret = setup_sgio_components(cptvf, req->out,
 161                                     req->outcnt,
 162                                     info->scatter_components);
 163         if (ret) {
 164                 dev_err(&pdev->dev, "Failed to setup gather list\n");
 165                 ret = -EFAULT;
 166                 goto  scatter_gather_clean;
 167         }
 168
 169         /* Create and initialize DPTR */
 170         info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
 171         info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
 172         if (!info->in_buffer) {
 173                 ret = -ENOMEM;
 174                 goto  scatter_gather_clean;
 175         }
 176
 177         ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);
 178         ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);
 179         ((__be16 *)info->in_buffer)[2] = 0;
 180         ((__be16 *)info->in_buffer)[3] = 0;
 181
 182         memcpy(&info->in_buffer[8], info->gather_components,
 183                g_sz_bytes);
 184         memcpy(&info->in_buffer[8 + g_sz_bytes],
 185                info->scatter_components, s_sz_bytes);
 186
 187         info->dptr_baddr = dma_map_single(&pdev->dev,
 188                                           (void *)info->in_buffer,
 189                                           info->dlen,
 190                                           DMA_BIDIRECTIONAL);
 191         if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
 192                 dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
 193                 ret = -EIO;
 194                 goto  scatter_gather_clean;
 195         }
 196
 197         /* Create and initialize RPTR */
 198         info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
 199         if (!info->out_buffer) {
 200                 ret = -ENOMEM;
 201                 goto scatter_gather_clean;
 202         }
 203
 204         *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
 205         info->alternate_caddr = (u64 *)info->out_buffer;
 206         info->rptr_baddr = dma_map_single(&pdev->dev,
 207                                           (void *)info->out_buffer,
 208                                           COMPLETION_CODE_SIZE,
 209                                           DMA_BIDIRECTIONAL);
 210         if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
 211                 dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
 212                         COMPLETION_CODE_SIZE);
 213                 ret = -EIO;
 214                 goto  scatter_gather_clean;
 215         }
 216
 217         return 0;
 218
 219 scatter_gather_clean:
 220         return ret;
 221 }
 222
 223 static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
 224                      u32 qno)
 225 {
 226         struct pci_dev *pdev = cptvf->pdev;
 227         struct command_qinfo *qinfo = NULL;
 228         struct command_queue *queue;
 229         struct command_chunk *chunk;
 230         u8 *ent;
 231         int ret = 0;
 232
 233         if (unlikely(qno >= cptvf->nr_queues)) {
 234                 dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
 235                         qno, cptvf->nr_queues);
 236                 return -EINVAL;
 237         }
 238
 239         qinfo = &cptvf->cqinfo;
 240         queue = &qinfo->queue[qno];
 241         /* lock commad queue */
 242         spin_lock(&queue->lock);
 243         ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
 244         memcpy(ent, (void *)cmd, qinfo->cmd_size);
 245
 246         if (++queue->idx >= queue->qhead->size / 64) {
 247                 struct hlist_node *node;
 248
 249                 hlist_for_each(node, &queue->chead) {
 250                         chunk = hlist_entry(node, struct command_chunk,
 251                                             nextchunk);
 252                         if (chunk == queue->qhead) {
 253                                 continue;
 254                         } else {
 255                                 queue->qhead = chunk;
 256                                 break;
 257                         }
 258                 }
 259                 queue->idx = 0;
 260         }
 261         /* make sure all memory stores are done before ringing doorbell */
 262         smp_wmb();
 263         cptvf_write_vq_doorbell(cptvf, 1);
 264         /* unlock command queue */
 265         spin_unlock(&queue->lock);
 266
 267         return ret;
 268 }
 269
 270 static void do_request_cleanup(struct cpt_vf *cptvf,
 271                         struct cpt_info_buffer *info)
 272 {
 273         int i;
 274         struct pci_dev *pdev = cptvf->pdev;
 275         struct cpt_request_info *req;
 276
 277         if (info->dptr_baddr)
 278                 dma_unmap_single(&pdev->dev, info->dptr_baddr,
 279                                  info->dlen, DMA_BIDIRECTIONAL);
 280
 281         if (info->rptr_baddr)
 282                 dma_unmap_single(&pdev->dev, info->rptr_baddr,
 283                                  COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
 284
 285         if (info->comp_baddr)
 286                 dma_unmap_single(&pdev->dev, info->comp_baddr,
 287                                  sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
 288
 289         if (info->req) {
 290                 req = info->req;
 291                 for (i = 0; i < req->outcnt; i++) {
 292                         if (req->out[i].dma_addr)
 293                                 dma_unmap_single(&pdev->dev,
 294                                                  req->out[i].dma_addr,
 295                                                  req->out[i].size,
 296                                                  DMA_BIDIRECTIONAL);
 297                 }
 298
 299                 for (i = 0; i < req->incnt; i++) {
 300                         if (req->in[i].dma_addr)
 301                                 dma_unmap_single(&pdev->dev,
 302                                                  req->in[i].dma_addr,
 303                                                  req->in[i].size,
 304                                                  DMA_BIDIRECTIONAL);
 305                 }
 306         }
 307
 308         kfree_sensitive(info->scatter_components);
 309         kfree_sensitive(info->gather_components);
 310         kfree_sensitive(info->out_buffer);
 311         kfree_sensitive(info->in_buffer);
 312         kfree_sensitive((void *)info->completion_addr);
 313         kfree_sensitive(info);
 314 }
 315
 316 static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
 317 {
 318         struct pci_dev *pdev = cptvf->pdev;
 319
 320         if (!info) {
 321                 dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");
 322                 return;
 323         }
 324
 325         do_request_cleanup(cptvf, info);
 326 }
 327
 328 static inline void process_pending_queue(struct cpt_vf *cptvf,
 329                                          struct pending_qinfo *pqinfo,
 330                                          int qno)
 331 {
 332         struct pci_dev *pdev = cptvf->pdev;
 333         struct pending_queue *pqueue = &pqinfo->queue[qno];
 334         struct pending_entry *pentry = NULL;
 335         struct cpt_info_buffer *info = NULL;
 336         union cpt_res_s *status = NULL;
 337         unsigned char ccode;
 338
 339         while (1) {
 340                 spin_lock_bh(&pqueue->lock);
 341                 pentry = &pqueue->head[pqueue->front];
 342                 if (unlikely(!pentry->busy)) {
 343                         spin_unlock_bh(&pqueue->lock);
 344                         break;
 345                 }
 346
 347                 info = (struct cpt_info_buffer *)pentry->post_arg;
 348                 if (unlikely(!info)) {
 349                         dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
 350                         pending_queue_inc_front(pqinfo, qno);
 351                         spin_unlock_bh(&pqueue->lock);
 352                         continue;
 353                 }
 354
 355                 status = (union cpt_res_s *)pentry->completion_addr;
 356                 ccode = status->s.compcode;
 357                 if ((status->s.compcode == CPT_COMP_E_FAULT) ||
 358                     (status->s.compcode == CPT_COMP_E_SWERR)) {
 359                         dev_err(&pdev->dev, "Request failed with %s\n",
 360                                 (status->s.compcode == CPT_COMP_E_FAULT) ?
 361                                 "DMA Fault" : "Software error");
 362                         pentry->completion_addr = NULL;
 363                         pentry->busy = false;
 364                         atomic64_dec((&pqueue->pending_count));
 365                         pentry->post_arg = NULL;
 366                         pending_queue_inc_front(pqinfo, qno);
 367                         do_request_cleanup(cptvf, info);
 368                         spin_unlock_bh(&pqueue->lock);
 369                         break;
 370                 } else if (status->s.compcode == COMPLETION_CODE_INIT) {
 371                         /* check for timeout */
 372                         if (time_after_eq(jiffies,
 373                                           (info->time_in +
 374                                           (CPT_COMMAND_TIMEOUT * HZ)))) {
 375                                 dev_err(&pdev->dev, "Request timed out");
 376                                 pentry->completion_addr = NULL;
 377                                 pentry->busy = false;
 378                                 atomic64_dec((&pqueue->pending_count));
 379                                 pentry->post_arg = NULL;
 380                                 pending_queue_inc_front(pqinfo, qno);
 381                                 do_request_cleanup(cptvf, info);
 382                                 spin_unlock_bh(&pqueue->lock);
 383                                 break;
 384                         } else if ((*info->alternate_caddr ==
 385                                 (~COMPLETION_CODE_INIT)) &&
 386                                 (info->extra_time < TIME_IN_RESET_COUNT)) {
 387                                 info->time_in = jiffies;
 388                                 info->extra_time++;
 389                                 spin_unlock_bh(&pqueue->lock);
 390                                 break;
 391                         }
 392                 }
 393
 394                 pentry->completion_addr = NULL;
 395                 pentry->busy = false;
 396                 pentry->post_arg = NULL;
 397                 atomic64_dec((&pqueue->pending_count));
 398                 pending_queue_inc_front(pqinfo, qno);
 399                 spin_unlock_bh(&pqueue->lock);
 400
 401                 do_post_process(info->cptvf, info);
 402                 /*
 403                  * Calling callback after we find
 404                  * that the request has been serviced
 405                  */
 406                 pentry->callback(ccode, pentry->callback_arg);
 407         }
 408 }
 409
 410 int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
 411 {
 412         int ret = 0, clear = 0, queue = 0;
 413         struct cpt_info_buffer *info = NULL;
 414         struct cptvf_request *cpt_req = NULL;
 415         union ctrl_info *ctrl = NULL;
 416         union cpt_res_s *result = NULL;
 417         struct pending_entry *pentry = NULL;
 418         struct pending_queue *pqueue = NULL;
 419         struct pci_dev *pdev = cptvf->pdev;
 420         u8 group = 0;
 421         struct cpt_vq_command vq_cmd;
 422         union cpt_inst_s cptinst;
 423
 424         info = kzalloc(sizeof(*info), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
 425         if (unlikely(!info)) {
 426                 dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
 427                 return -ENOMEM;
 428         }
 429
 430         cpt_req = (struct cptvf_request *)&req->req;
 431         ctrl = (union ctrl_info *)&req->ctrl;
 432
 433         info->cptvf = cptvf;
 434         group = ctrl->s.grp;
 435         ret = setup_sgio_list(cptvf, info, req);
 436         if (ret) {
 437                 dev_err(&pdev->dev, "Setting up SG list failed");
 438                 goto request_cleanup;
 439         }
 440
 441         cpt_req->dlen = info->dlen;
 442         /*
 443          * Get buffer for union cpt_res_s response
 444          * structure and its physical address
 445          */
 446         info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
 447         if (unlikely(!info->completion_addr)) {
 448                 dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
 449                 ret = -ENOMEM;
 450                 goto request_cleanup;
 451         }
 452
 453         result = (union cpt_res_s *)info->completion_addr;
 454         result->s.compcode = COMPLETION_CODE_INIT;
 455         info->comp_baddr = dma_map_single(&pdev->dev,
 456                                                (void *)info->completion_addr,
 457                                                sizeof(union cpt_res_s),
 458                                                DMA_BIDIRECTIONAL);
 459         if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
 460                 dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
 461                         sizeof(union cpt_res_s));
 462                 ret = -EFAULT;
 463                 goto  request_cleanup;
 464         }
 465
 466         /* Fill the VQ command */
 467         vq_cmd.cmd.u64 = 0;
 468         vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
 469         vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
 470         vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
 471         vq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);
 472
 473         vq_cmd.dptr = info->dptr_baddr;
 474         vq_cmd.rptr = info->rptr_baddr;
 475         vq_cmd.cptr.u64 = 0;
 476         vq_cmd.cptr.s.grp = group;
 477         /* Get Pending Entry to submit command */
 478         /* Always queue 0, because 1 queue per VF */
 479         queue = 0;
 480         pqueue = &cptvf->pqinfo.queue[queue];
 481
 482         if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
 483                 dev_err(&pdev->dev, "pending threshold reached\n");
 484                 process_pending_queue(cptvf, &cptvf->pqinfo, queue);
 485         }
 486
 487 get_pending_entry:
 488         spin_lock_bh(&pqueue->lock);
 489         pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
 490         if (unlikely(!pentry)) {
 491                 spin_unlock_bh(&pqueue->lock);
 492                 if (clear == 0) {
 493                         process_pending_queue(cptvf, &cptvf->pqinfo, queue);
 494                         clear = 1;
 495                         goto get_pending_entry;
 496                 }
 497                 dev_err(&pdev->dev, "Get free entry failed\n");
 498                 dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
 499                         queue, pqueue->rear, pqueue->front);
 500                 ret = -EFAULT;
 501                 goto request_cleanup;
 502         }
 503
 504         pentry->completion_addr = info->completion_addr;
 505         pentry->post_arg = (void *)info;
 506         pentry->callback = req->callback;
 507         pentry->callback_arg = req->callback_arg;
 508         info->pentry = pentry;
 509         pentry->busy = true;
 510         atomic64_inc(&pqueue->pending_count);
 511
 512         /* Send CPT command */
 513         info->pentry = pentry;
 514         info->time_in = jiffies;
 515         info->req = req;
 516
 517         /* Create the CPT_INST_S type command for HW intrepretation */
 518         cptinst.s.doneint = true;
 519         cptinst.s.res_addr = (u64)info->comp_baddr;
 520         cptinst.s.tag = 0;
 521         cptinst.s.grp = 0;
 522         cptinst.s.wq_ptr = 0;
 523         cptinst.s.ei0 = vq_cmd.cmd.u64;
 524         cptinst.s.ei1 = vq_cmd.dptr;
 525         cptinst.s.ei2 = vq_cmd.rptr;
 526         cptinst.s.ei3 = vq_cmd.cptr.u64;
 527
 528         ret = send_cpt_command(cptvf, &cptinst, queue);
 529         spin_unlock_bh(&pqueue->lock);
 530         if (unlikely(ret)) {
 531                 dev_err(&pdev->dev, "Send command failed for AE\n");
 532                 ret = -EFAULT;
 533                 goto request_cleanup;
 534         }
 535
 536         return 0;
 537
 538 request_cleanup:
 539         dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
 540         do_request_cleanup(cptvf, info);
 541
 542         return ret;
 543 }
 544
 545 void vq_post_process(struct cpt_vf *cptvf, u32 qno)
 546 {
 547         struct pci_dev *pdev = cptvf->pdev;
 548
 549         if (unlikely(qno > cptvf->nr_queues)) {
 550                 dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
 551                         qno);
 552                 return;
 553         }
 554
 555         process_pending_queue(cptvf, &cptvf->pqinfo, qno);
 556 }
 557
 558 int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
 559 {
 560         struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
 561         struct pci_dev *pdev = cptvf->pdev;
 562
 563         if (!cpt_device_ready(cptvf)) {
 564                 dev_err(&pdev->dev, "CPT Device is not ready");
 565                 return -ENODEV;
 566         }
 567
 568         if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
 569                 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
 570                         cptvf->vfid);
 571                 return -EINVAL;
 572         } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
 573                 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
 574                         cptvf->vfid);
 575                 return -EINVAL;
 576         }
 577
 578         return process_request(cptvf, req);
 579 }