Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
[sfrench/cifs-2.6.git] / drivers / crypto / cavium / nitrox / nitrox_reqmgr.c
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/gfp.h>
3 #include <linux/workqueue.h>
4 #include <crypto/internal/skcipher.h>
5
6 #include "nitrox_dev.h"
7 #include "nitrox_req.h"
8 #include "nitrox_csr.h"
9
10 /* SLC_STORE_INFO */
11 #define MIN_UDD_LEN 16
12 /* PKT_IN_HDR + SLC_STORE_INFO */
13 #define FDATA_SIZE 32
14 /* Base destination port for the solicited requests */
15 #define SOLICIT_BASE_DPORT 256
16 #define PENDING_SIG     0xFFFFFFFFFFFFFFFFUL
17
18 #define REQ_NOT_POSTED 1
19 #define REQ_BACKLOG    2
20 #define REQ_POSTED     3
21
22 /**
23  * Response codes from SE microcode
24  * 0x00 - Success
25  *   Completion with no error
26  * 0x43 - ERR_GC_DATA_LEN_INVALID
27  *   Invalid Data length if Encryption Data length is
28  *   less than 16 bytes for AES-XTS and AES-CTS.
29  * 0x45 - ERR_GC_CTX_LEN_INVALID
30  *   Invalid context length: CTXL != 23 words.
31  * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
32  *   DOCSIS support is enabled with other than
33  *   AES/DES-CBC mode encryption.
34  * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
35  *   Authentication offset is other than 0 with
36  *   Encryption IV source = 0.
37  *   Authentication offset is other than 8 (DES)/16 (AES)
38  *   with Encryption IV source = 1
39  * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
40  *   CRC32 is enabled for other than DOCSIS encryption.
41  * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
42  *   Invalid flag options in AES-CCM IV.
43  */
44
45 static inline int incr_index(int index, int count, int max)
46 {
47         if ((index + count) >= max)
48                 index = index + count - max;
49         else
50                 index += count;
51
52         return index;
53 }
54
55 /**
56  * dma_free_sglist - unmap and free the sg lists.
57  * @ndev: N5 device
58  * @sgtbl: SG table
59  */
60 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
61 {
62         struct nitrox_device *ndev = sr->ndev;
63         struct device *dev = DEV(ndev);
64         struct nitrox_sglist *sglist;
65
66         /* unmap in sgbuf */
67         sglist = sr->in.sglist;
68         if (!sglist)
69                 goto out_unmap;
70
71         /* unmap iv */
72         dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL);
73         /* unmpa src sglist */
74         dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir);
75         /* unamp gather component */
76         dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE);
77         kfree(sr->in.sglist);
78         kfree(sr->in.sgcomp);
79         sr->in.sglist = NULL;
80         sr->in.buf = NULL;
81         sr->in.map_bufs_cnt = 0;
82
83 out_unmap:
84         /* unmap out sgbuf */
85         sglist = sr->out.sglist;
86         if (!sglist)
87                 return;
88
89         /* unmap orh */
90         dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
91
92         /* unmap dst sglist */
93         if (!sr->inplace) {
94                 dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3),
95                              sr->out.dir);
96         }
97         /* unmap completion */
98         dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
99
100         /* unmap scatter component */
101         dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE);
102         kfree(sr->out.sglist);
103         kfree(sr->out.sgcomp);
104         sr->out.sglist = NULL;
105         sr->out.buf = NULL;
106         sr->out.map_bufs_cnt = 0;
107 }
108
109 static void softreq_destroy(struct nitrox_softreq *sr)
110 {
111         softreq_unmap_sgbufs(sr);
112         kfree(sr);
113 }
114
115 /**
116  * create_sg_component - create SG componets for N5 device.
117  * @sr: Request structure
118  * @sgtbl: SG table
119  * @nr_comp: total number of components required
120  *
121  * Component structure
122  *
123  *   63     48 47     32 31    16 15      0
124  *   --------------------------------------
125  *   |   LEN0  |  LEN1  |  LEN2  |  LEN3  |
126  *   |-------------------------------------
127  *   |               PTR0                 |
128  *   --------------------------------------
129  *   |               PTR1                 |
130  *   --------------------------------------
131  *   |               PTR2                 |
132  *   --------------------------------------
133  *   |               PTR3                 |
134  *   --------------------------------------
135  *
136  *   Returns 0 if success or a negative errno code on error.
137  */
138 static int create_sg_component(struct nitrox_softreq *sr,
139                                struct nitrox_sgtable *sgtbl, int map_nents)
140 {
141         struct nitrox_device *ndev = sr->ndev;
142         struct nitrox_sgcomp *sgcomp;
143         struct nitrox_sglist *sglist;
144         dma_addr_t dma;
145         size_t sz_comp;
146         int i, j, nr_sgcomp;
147
148         nr_sgcomp = roundup(map_nents, 4) / 4;
149
150         /* each component holds 4 dma pointers */
151         sz_comp = nr_sgcomp * sizeof(*sgcomp);
152         sgcomp = kzalloc(sz_comp, sr->gfp);
153         if (!sgcomp)
154                 return -ENOMEM;
155
156         sgtbl->sgcomp = sgcomp;
157         sgtbl->nr_sgcomp = nr_sgcomp;
158
159         sglist = sgtbl->sglist;
160         /* populate device sg component */
161         for (i = 0; i < nr_sgcomp; i++) {
162                 for (j = 0; j < 4; j++) {
163                         sgcomp->len[j] = cpu_to_be16(sglist->len);
164                         sgcomp->dma[j] = cpu_to_be64(sglist->dma);
165                         sglist++;
166                 }
167                 sgcomp++;
168         }
169         /* map the device sg component */
170         dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
171         if (dma_mapping_error(DEV(ndev), dma)) {
172                 kfree(sgtbl->sgcomp);
173                 sgtbl->sgcomp = NULL;
174                 return -ENOMEM;
175         }
176
177         sgtbl->dma = dma;
178         sgtbl->len = sz_comp;
179
180         return 0;
181 }
182
183 /**
184  * dma_map_inbufs - DMA map input sglist and creates sglist component
185  *                  for N5 device.
186  * @sr: Request structure
187  * @req: Crypto request structre
188  *
189  * Returns 0 if successful or a negative errno code on error.
190  */
191 static int dma_map_inbufs(struct nitrox_softreq *sr,
192                           struct se_crypto_request *req)
193 {
194         struct device *dev = DEV(sr->ndev);
195         struct scatterlist *sg = req->src;
196         struct nitrox_sglist *glist;
197         int i, nents, ret = 0;
198         dma_addr_t dma;
199         size_t sz;
200
201         nents = sg_nents(req->src);
202
203         /* creater gather list IV and src entries */
204         sz = roundup((1 + nents), 4) * sizeof(*glist);
205         glist = kzalloc(sz, sr->gfp);
206         if (!glist)
207                 return -ENOMEM;
208
209         sr->in.sglist = glist;
210         /* map IV */
211         dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL);
212         if (dma_mapping_error(dev, dma)) {
213                 ret = -EINVAL;
214                 goto iv_map_err;
215         }
216
217         sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
218         /* map src entries */
219         nents = dma_map_sg(dev, req->src, nents, sr->in.dir);
220         if (!nents) {
221                 ret = -EINVAL;
222                 goto src_map_err;
223         }
224         sr->in.buf = req->src;
225
226         /* store the mappings */
227         glist->len = req->ivsize;
228         glist->dma = dma;
229         glist++;
230         sr->in.total_bytes += req->ivsize;
231
232         for_each_sg(req->src, sg, nents, i) {
233                 glist->len = sg_dma_len(sg);
234                 glist->dma = sg_dma_address(sg);
235                 sr->in.total_bytes += glist->len;
236                 glist++;
237         }
238         /* roundup map count to align with entires in sg component */
239         sr->in.map_bufs_cnt = (1 + nents);
240
241         /* create NITROX gather component */
242         ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt);
243         if (ret)
244                 goto incomp_err;
245
246         return 0;
247
248 incomp_err:
249         dma_unmap_sg(dev, req->src, nents, sr->in.dir);
250         sr->in.map_bufs_cnt = 0;
251 src_map_err:
252         dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL);
253 iv_map_err:
254         kfree(sr->in.sglist);
255         sr->in.sglist = NULL;
256         return ret;
257 }
258
259 static int dma_map_outbufs(struct nitrox_softreq *sr,
260                            struct se_crypto_request *req)
261 {
262         struct device *dev = DEV(sr->ndev);
263         struct nitrox_sglist *glist = sr->in.sglist;
264         struct nitrox_sglist *slist;
265         struct scatterlist *sg;
266         int i, nents, map_bufs_cnt, ret = 0;
267         size_t sz;
268
269         nents = sg_nents(req->dst);
270
271         /* create scatter list ORH, IV, dst entries and Completion header */
272         sz = roundup((3 + nents), 4) * sizeof(*slist);
273         slist = kzalloc(sz, sr->gfp);
274         if (!slist)
275                 return -ENOMEM;
276
277         sr->out.sglist = slist;
278         sr->out.dir = DMA_BIDIRECTIONAL;
279         /* map ORH */
280         sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN,
281                                           sr->out.dir);
282         if (dma_mapping_error(dev, sr->resp.orh_dma)) {
283                 ret = -EINVAL;
284                 goto orh_map_err;
285         }
286
287         /* map completion */
288         sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion,
289                                                  COMP_HLEN, sr->out.dir);
290         if (dma_mapping_error(dev, sr->resp.completion_dma)) {
291                 ret = -EINVAL;
292                 goto compl_map_err;
293         }
294
295         sr->inplace = (req->src == req->dst) ? true : false;
296         /* out place */
297         if (!sr->inplace) {
298                 nents = dma_map_sg(dev, req->dst, nents, sr->out.dir);
299                 if (!nents) {
300                         ret = -EINVAL;
301                         goto dst_map_err;
302                 }
303         }
304         sr->out.buf = req->dst;
305
306         /* store the mappings */
307         /* orh */
308         slist->len = ORH_HLEN;
309         slist->dma = sr->resp.orh_dma;
310         slist++;
311
312         /* copy the glist mappings */
313         if (sr->inplace) {
314                 nents = sr->in.map_bufs_cnt - 1;
315                 map_bufs_cnt = sr->in.map_bufs_cnt;
316                 while (map_bufs_cnt--) {
317                         slist->len = glist->len;
318                         slist->dma = glist->dma;
319                         slist++;
320                         glist++;
321                 }
322         } else {
323                 /* copy iv mapping */
324                 slist->len = glist->len;
325                 slist->dma = glist->dma;
326                 slist++;
327                 /* copy remaining maps */
328                 for_each_sg(req->dst, sg, nents, i) {
329                         slist->len = sg_dma_len(sg);
330                         slist->dma = sg_dma_address(sg);
331                         slist++;
332                 }
333         }
334
335         /* completion */
336         slist->len = COMP_HLEN;
337         slist->dma = sr->resp.completion_dma;
338
339         sr->out.map_bufs_cnt = (3 + nents);
340
341         ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt);
342         if (ret)
343                 goto outcomp_map_err;
344
345         return 0;
346
347 outcomp_map_err:
348         if (!sr->inplace)
349                 dma_unmap_sg(dev, req->dst, nents, sr->out.dir);
350         sr->out.map_bufs_cnt = 0;
351         sr->out.buf = NULL;
352 dst_map_err:
353         dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
354         sr->resp.completion_dma = 0;
355 compl_map_err:
356         dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
357         sr->resp.orh_dma = 0;
358 orh_map_err:
359         kfree(sr->out.sglist);
360         sr->out.sglist = NULL;
361         return ret;
362 }
363
364 static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
365                                     struct se_crypto_request *creq)
366 {
367         int ret;
368
369         ret = dma_map_inbufs(sr, creq);
370         if (ret)
371                 return ret;
372
373         ret = dma_map_outbufs(sr, creq);
374         if (ret)
375                 softreq_unmap_sgbufs(sr);
376
377         return ret;
378 }
379
380 static inline void backlog_list_add(struct nitrox_softreq *sr,
381                                     struct nitrox_cmdq *cmdq)
382 {
383         INIT_LIST_HEAD(&sr->backlog);
384
385         spin_lock_bh(&cmdq->backlog_lock);
386         list_add_tail(&sr->backlog, &cmdq->backlog_head);
387         atomic_inc(&cmdq->backlog_count);
388         atomic_set(&sr->status, REQ_BACKLOG);
389         spin_unlock_bh(&cmdq->backlog_lock);
390 }
391
392 static inline void response_list_add(struct nitrox_softreq *sr,
393                                      struct nitrox_cmdq *cmdq)
394 {
395         INIT_LIST_HEAD(&sr->response);
396
397         spin_lock_bh(&cmdq->response_lock);
398         list_add_tail(&sr->response, &cmdq->response_head);
399         spin_unlock_bh(&cmdq->response_lock);
400 }
401
402 static inline void response_list_del(struct nitrox_softreq *sr,
403                                      struct nitrox_cmdq *cmdq)
404 {
405         spin_lock_bh(&cmdq->response_lock);
406         list_del(&sr->response);
407         spin_unlock_bh(&cmdq->response_lock);
408 }
409
410 static struct nitrox_softreq *
411 get_first_response_entry(struct nitrox_cmdq *cmdq)
412 {
413         return list_first_entry_or_null(&cmdq->response_head,
414                                         struct nitrox_softreq, response);
415 }
416
417 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
418 {
419         if (atomic_inc_return(&cmdq->pending_count) > qlen) {
420                 atomic_dec(&cmdq->pending_count);
421                 /* sync with other cpus */
422                 smp_mb__after_atomic();
423                 return true;
424         }
425         return false;
426 }
427
428 /**
429  * post_se_instr - Post SE instruction to Packet Input ring
430  * @sr: Request structure
431  *
432  * Returns 0 if successful or a negative error code,
433  * if no space in ring.
434  */
435 static void post_se_instr(struct nitrox_softreq *sr,
436                           struct nitrox_cmdq *cmdq)
437 {
438         struct nitrox_device *ndev = sr->ndev;
439         int idx;
440         u8 *ent;
441
442         spin_lock_bh(&cmdq->cmdq_lock);
443
444         idx = cmdq->write_idx;
445         /* copy the instruction */
446         ent = cmdq->head + (idx * cmdq->instr_size);
447         memcpy(ent, &sr->instr, cmdq->instr_size);
448
449         atomic_set(&sr->status, REQ_POSTED);
450         response_list_add(sr, cmdq);
451         sr->tstamp = jiffies;
452         /* flush the command queue updates */
453         dma_wmb();
454
455         /* Ring doorbell with count 1 */
456         writeq(1, cmdq->dbell_csr_addr);
457         /* orders the doorbell rings */
458         mmiowb();
459
460         cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
461
462         spin_unlock_bh(&cmdq->cmdq_lock);
463 }
464
465 static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
466 {
467         struct nitrox_device *ndev = cmdq->ndev;
468         struct nitrox_softreq *sr, *tmp;
469         int ret = 0;
470
471         if (!atomic_read(&cmdq->backlog_count))
472                 return 0;
473
474         spin_lock_bh(&cmdq->backlog_lock);
475
476         list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
477                 struct skcipher_request *skreq;
478
479                 /* submit until space available */
480                 if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
481                         ret = -ENOSPC;
482                         break;
483                 }
484                 /* delete from backlog list */
485                 list_del(&sr->backlog);
486                 atomic_dec(&cmdq->backlog_count);
487                 /* sync with other cpus */
488                 smp_mb__after_atomic();
489
490                 skreq = sr->skreq;
491                 /* post the command */
492                 post_se_instr(sr, cmdq);
493
494                 /* backlog requests are posted, wakeup with -EINPROGRESS */
495                 skcipher_request_complete(skreq, -EINPROGRESS);
496         }
497         spin_unlock_bh(&cmdq->backlog_lock);
498
499         return ret;
500 }
501
502 static int nitrox_enqueue_request(struct nitrox_softreq *sr)
503 {
504         struct nitrox_cmdq *cmdq = sr->cmdq;
505         struct nitrox_device *ndev = sr->ndev;
506
507         /* try to post backlog requests */
508         post_backlog_cmds(cmdq);
509
510         if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
511                 if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
512                         return -ENOSPC;
513                 /* add to backlog list */
514                 backlog_list_add(sr, cmdq);
515                 return -EBUSY;
516         }
517         post_se_instr(sr, cmdq);
518
519         return -EINPROGRESS;
520 }
521
522 /**
523  * nitrox_se_request - Send request to SE core
524  * @ndev: NITROX device
525  * @req: Crypto request
526  *
527  * Returns 0 on success, or a negative error code.
528  */
529 int nitrox_process_se_request(struct nitrox_device *ndev,
530                               struct se_crypto_request *req,
531                               completion_t callback,
532                               struct skcipher_request *skreq)
533 {
534         struct nitrox_softreq *sr;
535         dma_addr_t ctx_handle = 0;
536         int qno, ret = 0;
537
538         if (!nitrox_ready(ndev))
539                 return -ENODEV;
540
541         sr = kzalloc(sizeof(*sr), req->gfp);
542         if (!sr)
543                 return -ENOMEM;
544
545         sr->ndev = ndev;
546         sr->flags = req->flags;
547         sr->gfp = req->gfp;
548         sr->callback = callback;
549         sr->skreq = skreq;
550
551         atomic_set(&sr->status, REQ_NOT_POSTED);
552
553         WRITE_ONCE(sr->resp.orh, PENDING_SIG);
554         WRITE_ONCE(sr->resp.completion, PENDING_SIG);
555
556         ret = softreq_map_iobuf(sr, req);
557         if (ret) {
558                 kfree(sr);
559                 return ret;
560         }
561
562         /* get the context handle */
563         if (req->ctx_handle) {
564                 struct ctx_hdr *hdr;
565                 u8 *ctx_ptr;
566
567                 ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
568                 hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
569                 ctx_handle = hdr->ctx_dma;
570         }
571
572         /* select the queue */
573         qno = smp_processor_id() % ndev->nr_queues;
574
575         sr->cmdq = &ndev->pkt_cmdqs[qno];
576
577         /*
578          * 64-Byte Instruction Format
579          *
580          *  ----------------------
581          *  |      DPTR0         | 8 bytes
582          *  ----------------------
583          *  |  PKT_IN_INSTR_HDR  | 8 bytes
584          *  ----------------------
585          *  |    PKT_IN_HDR      | 16 bytes
586          *  ----------------------
587          *  |    SLC_INFO        | 16 bytes
588          *  ----------------------
589          *  |   Front data       | 16 bytes
590          *  ----------------------
591          */
592
593         /* fill the packet instruction */
594         /* word 0 */
595         sr->instr.dptr0 = cpu_to_be64(sr->in.dma);
596
597         /* word 1 */
598         sr->instr.ih.value = 0;
599         sr->instr.ih.s.g = 1;
600         sr->instr.ih.s.gsz = sr->in.map_bufs_cnt;
601         sr->instr.ih.s.ssz = sr->out.map_bufs_cnt;
602         sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
603         sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
604         sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
605
606         /* word 2 */
607         sr->instr.irh.value[0] = 0;
608         sr->instr.irh.s.uddl = MIN_UDD_LEN;
609         /* context length in 64-bit words */
610         sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
611         /* offset from solicit base port 256 */
612         sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
613         sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
614         sr->instr.irh.s.arg = req->ctrl.s.arg;
615         sr->instr.irh.s.opcode = req->opcode;
616         sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]);
617
618         /* word 3 */
619         sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
620
621         /* word 4 */
622         sr->instr.slc.value[0] = 0;
623         sr->instr.slc.s.ssz = sr->out.map_bufs_cnt;
624         sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
625
626         /* word 5 */
627         sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma);
628
629         /*
630          * No conversion for front data,
631          * It goes into payload
632          * put GP Header in front data
633          */
634         sr->instr.fdata[0] = *((u64 *)&req->gph);
635         sr->instr.fdata[1] = 0;
636
637         ret = nitrox_enqueue_request(sr);
638         if (ret == -ENOSPC)
639                 goto send_fail;
640
641         return ret;
642
643 send_fail:
644         softreq_destroy(sr);
645         return ret;
646 }
647
648 static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
649 {
650         return time_after_eq(jiffies, (tstamp + timeout));
651 }
652
653 void backlog_qflush_work(struct work_struct *work)
654 {
655         struct nitrox_cmdq *cmdq;
656
657         cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
658         post_backlog_cmds(cmdq);
659 }
660
661 /**
662  * process_request_list - process completed requests
663  * @ndev: N5 device
664  * @qno: queue to operate
665  *
666  * Returns the number of responses processed.
667  */
668 static void process_response_list(struct nitrox_cmdq *cmdq)
669 {
670         struct nitrox_device *ndev = cmdq->ndev;
671         struct nitrox_softreq *sr;
672         struct skcipher_request *skreq;
673         completion_t callback;
674         int req_completed = 0, err = 0, budget;
675
676         /* check all pending requests */
677         budget = atomic_read(&cmdq->pending_count);
678
679         while (req_completed < budget) {
680                 sr = get_first_response_entry(cmdq);
681                 if (!sr)
682                         break;
683
684                 if (atomic_read(&sr->status) != REQ_POSTED)
685                         break;
686
687                 /* check orh and completion bytes updates */
688                 if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) {
689                         /* request not completed, check for timeout */
690                         if (!cmd_timeout(sr->tstamp, ndev->timeout))
691                                 break;
692                         dev_err_ratelimited(DEV(ndev),
693                                             "Request timeout, orh 0x%016llx\n",
694                                             READ_ONCE(sr->resp.orh));
695                 }
696                 atomic_dec(&cmdq->pending_count);
697                 /* sync with other cpus */
698                 smp_mb__after_atomic();
699                 /* remove from response list */
700                 response_list_del(sr, cmdq);
701
702                 callback = sr->callback;
703                 skreq = sr->skreq;
704
705                 /* ORH error code */
706                 err = READ_ONCE(sr->resp.orh) & 0xff;
707                 softreq_destroy(sr);
708
709                 if (callback)
710                         callback(skreq, err);
711
712                 req_completed++;
713         }
714 }
715
716 /**
717  * pkt_slc_resp_handler - post processing of SE responses
718  */
719 void pkt_slc_resp_handler(unsigned long data)
720 {
721         struct bh_data *bh = (void *)(uintptr_t)(data);
722         struct nitrox_cmdq *cmdq = bh->cmdq;
723         union nps_pkt_slc_cnts pkt_slc_cnts;
724
725         /* read completion count */
726         pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr);
727         /* resend the interrupt if more work to do */
728         pkt_slc_cnts.s.resend = 1;
729
730         process_response_list(cmdq);
731
732         /*
733          * clear the interrupt with resend bit enabled,
734          * MSI-X interrupt generates if Completion count > Threshold
735          */
736         writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr);
737         /* order the writes */
738         mmiowb();
739
740         if (atomic_read(&cmdq->backlog_count))
741                 schedule_work(&cmdq->backlog_qflush);
742 }