Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma
[sfrench/cifs-2.6.git] / drivers / infiniband / hw / hns / hns_roce_hw_v1.c
1 /*
2  * Copyright (c) 2016 Hisilicon Limited.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32
33 #include <linux/platform_device.h>
34 #include <linux/acpi.h>
35 #include <linux/etherdevice.h>
36 #include <linux/interrupt.h>
37 #include <linux/of.h>
38 #include <linux/of_platform.h>
39 #include <rdma/ib_umem.h>
40 #include "hns_roce_common.h"
41 #include "hns_roce_device.h"
42 #include "hns_roce_cmd.h"
43 #include "hns_roce_hem.h"
44 #include "hns_roce_hw_v1.h"
45
46 static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg)
47 {
48         dseg->lkey = cpu_to_le32(sg->lkey);
49         dseg->addr = cpu_to_le64(sg->addr);
50         dseg->len  = cpu_to_le32(sg->length);
51 }
52
53 static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr,
54                           u32 rkey)
55 {
56         rseg->raddr = cpu_to_le64(remote_addr);
57         rseg->rkey  = cpu_to_le32(rkey);
58         rseg->len   = 0;
59 }
60
61 static int hns_roce_v1_post_send(struct ib_qp *ibqp,
62                                  const struct ib_send_wr *wr,
63                                  const struct ib_send_wr **bad_wr)
64 {
65         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
66         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
67         struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL;
68         struct hns_roce_wqe_ctrl_seg *ctrl = NULL;
69         struct hns_roce_wqe_data_seg *dseg = NULL;
70         struct hns_roce_qp *qp = to_hr_qp(ibqp);
71         struct device *dev = &hr_dev->pdev->dev;
72         struct hns_roce_sq_db sq_db;
73         int ps_opcode = 0, i = 0;
74         unsigned long flags = 0;
75         void *wqe = NULL;
76         u32 doorbell[2];
77         int nreq = 0;
78         u32 ind = 0;
79         int ret = 0;
80         u8 *smac;
81         int loopback;
82
83         if (unlikely(ibqp->qp_type != IB_QPT_GSI &&
84                 ibqp->qp_type != IB_QPT_RC)) {
85                 dev_err(dev, "un-supported QP type\n");
86                 *bad_wr = NULL;
87                 return -EOPNOTSUPP;
88         }
89
90         spin_lock_irqsave(&qp->sq.lock, flags);
91         ind = qp->sq_next_wqe;
92         for (nreq = 0; wr; ++nreq, wr = wr->next) {
93                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
94                         ret = -ENOMEM;
95                         *bad_wr = wr;
96                         goto out;
97                 }
98
99                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
100                         dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
101                                 wr->num_sge, qp->sq.max_gs);
102                         ret = -EINVAL;
103                         *bad_wr = wr;
104                         goto out;
105                 }
106
107                 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
108                 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
109                                                                       wr->wr_id;
110
111                 /* Corresponding to the RC and RD type wqe process separately */
112                 if (ibqp->qp_type == IB_QPT_GSI) {
113                         ud_sq_wqe = wqe;
114                         roce_set_field(ud_sq_wqe->dmac_h,
115                                        UD_SEND_WQE_U32_4_DMAC_0_M,
116                                        UD_SEND_WQE_U32_4_DMAC_0_S,
117                                        ah->av.mac[0]);
118                         roce_set_field(ud_sq_wqe->dmac_h,
119                                        UD_SEND_WQE_U32_4_DMAC_1_M,
120                                        UD_SEND_WQE_U32_4_DMAC_1_S,
121                                        ah->av.mac[1]);
122                         roce_set_field(ud_sq_wqe->dmac_h,
123                                        UD_SEND_WQE_U32_4_DMAC_2_M,
124                                        UD_SEND_WQE_U32_4_DMAC_2_S,
125                                        ah->av.mac[2]);
126                         roce_set_field(ud_sq_wqe->dmac_h,
127                                        UD_SEND_WQE_U32_4_DMAC_3_M,
128                                        UD_SEND_WQE_U32_4_DMAC_3_S,
129                                        ah->av.mac[3]);
130
131                         roce_set_field(ud_sq_wqe->u32_8,
132                                        UD_SEND_WQE_U32_8_DMAC_4_M,
133                                        UD_SEND_WQE_U32_8_DMAC_4_S,
134                                        ah->av.mac[4]);
135                         roce_set_field(ud_sq_wqe->u32_8,
136                                        UD_SEND_WQE_U32_8_DMAC_5_M,
137                                        UD_SEND_WQE_U32_8_DMAC_5_S,
138                                        ah->av.mac[5]);
139
140                         smac = (u8 *)hr_dev->dev_addr[qp->port];
141                         loopback = ether_addr_equal_unaligned(ah->av.mac,
142                                                               smac) ? 1 : 0;
143                         roce_set_bit(ud_sq_wqe->u32_8,
144                                      UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S,
145                                      loopback);
146
147                         roce_set_field(ud_sq_wqe->u32_8,
148                                        UD_SEND_WQE_U32_8_OPERATION_TYPE_M,
149                                        UD_SEND_WQE_U32_8_OPERATION_TYPE_S,
150                                        HNS_ROCE_WQE_OPCODE_SEND);
151                         roce_set_field(ud_sq_wqe->u32_8,
152                                        UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M,
153                                        UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S,
154                                        2);
155                         roce_set_bit(ud_sq_wqe->u32_8,
156                                 UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S,
157                                 1);
158
159                         ud_sq_wqe->u32_8 |= (wr->send_flags & IB_SEND_SIGNALED ?
160                                 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
161                                 (wr->send_flags & IB_SEND_SOLICITED ?
162                                 cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
163                                 ((wr->opcode == IB_WR_SEND_WITH_IMM) ?
164                                 cpu_to_le32(HNS_ROCE_WQE_IMM) : 0);
165
166                         roce_set_field(ud_sq_wqe->u32_16,
167                                        UD_SEND_WQE_U32_16_DEST_QP_M,
168                                        UD_SEND_WQE_U32_16_DEST_QP_S,
169                                        ud_wr(wr)->remote_qpn);
170                         roce_set_field(ud_sq_wqe->u32_16,
171                                        UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M,
172                                        UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S,
173                                        ah->av.stat_rate);
174
175                         roce_set_field(ud_sq_wqe->u32_36,
176                                        UD_SEND_WQE_U32_36_FLOW_LABEL_M,
177                                        UD_SEND_WQE_U32_36_FLOW_LABEL_S,
178                                        ah->av.sl_tclass_flowlabel &
179                                        HNS_ROCE_FLOW_LABEL_MASK);
180                         roce_set_field(ud_sq_wqe->u32_36,
181                                       UD_SEND_WQE_U32_36_PRIORITY_M,
182                                       UD_SEND_WQE_U32_36_PRIORITY_S,
183                                       le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
184                                       HNS_ROCE_SL_SHIFT);
185                         roce_set_field(ud_sq_wqe->u32_36,
186                                        UD_SEND_WQE_U32_36_SGID_INDEX_M,
187                                        UD_SEND_WQE_U32_36_SGID_INDEX_S,
188                                        hns_get_gid_index(hr_dev, qp->phy_port,
189                                                          ah->av.gid_index));
190
191                         roce_set_field(ud_sq_wqe->u32_40,
192                                        UD_SEND_WQE_U32_40_HOP_LIMIT_M,
193                                        UD_SEND_WQE_U32_40_HOP_LIMIT_S,
194                                        ah->av.hop_limit);
195                         roce_set_field(ud_sq_wqe->u32_40,
196                                        UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M,
197                                        UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S,
198                                        ah->av.sl_tclass_flowlabel >>
199                                        HNS_ROCE_TCLASS_SHIFT);
200
201                         memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN);
202
203                         ud_sq_wqe->va0_l =
204                                        cpu_to_le32((u32)wr->sg_list[0].addr);
205                         ud_sq_wqe->va0_h =
206                                        cpu_to_le32((wr->sg_list[0].addr) >> 32);
207                         ud_sq_wqe->l_key0 =
208                                        cpu_to_le32(wr->sg_list[0].lkey);
209
210                         ud_sq_wqe->va1_l =
211                                        cpu_to_le32((u32)wr->sg_list[1].addr);
212                         ud_sq_wqe->va1_h =
213                                        cpu_to_le32((wr->sg_list[1].addr) >> 32);
214                         ud_sq_wqe->l_key1 =
215                                        cpu_to_le32(wr->sg_list[1].lkey);
216                         ind++;
217                 } else if (ibqp->qp_type == IB_QPT_RC) {
218                         u32 tmp_len = 0;
219
220                         ctrl = wqe;
221                         memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
222                         for (i = 0; i < wr->num_sge; i++)
223                                 tmp_len += wr->sg_list[i].length;
224
225                         ctrl->msg_length =
226                           cpu_to_le32(le32_to_cpu(ctrl->msg_length) + tmp_len);
227
228                         ctrl->sgl_pa_h = 0;
229                         ctrl->flag = 0;
230
231                         switch (wr->opcode) {
232                         case IB_WR_SEND_WITH_IMM:
233                         case IB_WR_RDMA_WRITE_WITH_IMM:
234                                 ctrl->imm_data = wr->ex.imm_data;
235                                 break;
236                         case IB_WR_SEND_WITH_INV:
237                                 ctrl->inv_key =
238                                         cpu_to_le32(wr->ex.invalidate_rkey);
239                                 break;
240                         default:
241                                 ctrl->imm_data = 0;
242                                 break;
243                         }
244
245                         /*Ctrl field, ctrl set type: sig, solic, imm, fence */
246                         /* SO wait for conforming application scenarios */
247                         ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ?
248                                       cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
249                                       (wr->send_flags & IB_SEND_SOLICITED ?
250                                       cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
251                                       ((wr->opcode == IB_WR_SEND_WITH_IMM ||
252                                       wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) ?
253                                       cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) |
254                                       (wr->send_flags & IB_SEND_FENCE ?
255                                       (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0);
256
257                         wqe += sizeof(struct hns_roce_wqe_ctrl_seg);
258
259                         switch (wr->opcode) {
260                         case IB_WR_RDMA_READ:
261                                 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;
262                                 set_raddr_seg(wqe,  rdma_wr(wr)->remote_addr,
263                                                rdma_wr(wr)->rkey);
264                                 break;
265                         case IB_WR_RDMA_WRITE:
266                         case IB_WR_RDMA_WRITE_WITH_IMM:
267                                 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;
268                                 set_raddr_seg(wqe,  rdma_wr(wr)->remote_addr,
269                                               rdma_wr(wr)->rkey);
270                                 break;
271                         case IB_WR_SEND:
272                         case IB_WR_SEND_WITH_INV:
273                         case IB_WR_SEND_WITH_IMM:
274                                 ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;
275                                 break;
276                         case IB_WR_LOCAL_INV:
277                                 break;
278                         case IB_WR_ATOMIC_CMP_AND_SWP:
279                         case IB_WR_ATOMIC_FETCH_AND_ADD:
280                         case IB_WR_LSO:
281                         default:
282                                 ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;
283                                 break;
284                         }
285                         ctrl->flag |= cpu_to_le32(ps_opcode);
286                         wqe += sizeof(struct hns_roce_wqe_raddr_seg);
287
288                         dseg = wqe;
289                         if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
290                                 if (le32_to_cpu(ctrl->msg_length) >
291                                     hr_dev->caps.max_sq_inline) {
292                                         ret = -EINVAL;
293                                         *bad_wr = wr;
294                                         dev_err(dev, "inline len(1-%d)=%d, illegal",
295                                                 ctrl->msg_length,
296                                                 hr_dev->caps.max_sq_inline);
297                                         goto out;
298                                 }
299                                 for (i = 0; i < wr->num_sge; i++) {
300                                         memcpy(wqe, ((void *) (uintptr_t)
301                                                wr->sg_list[i].addr),
302                                                wr->sg_list[i].length);
303                                         wqe += wr->sg_list[i].length;
304                                 }
305                                 ctrl->flag |= cpu_to_le32(HNS_ROCE_WQE_INLINE);
306                         } else {
307                                 /*sqe num is two */
308                                 for (i = 0; i < wr->num_sge; i++)
309                                         set_data_seg(dseg + i, wr->sg_list + i);
310
311                                 ctrl->flag |= cpu_to_le32(wr->num_sge <<
312                                               HNS_ROCE_WQE_SGE_NUM_BIT);
313                         }
314                         ind++;
315                 }
316         }
317
318 out:
319         /* Set DB return */
320         if (likely(nreq)) {
321                 qp->sq.head += nreq;
322                 /* Memory barrier */
323                 wmb();
324
325                 sq_db.u32_4 = 0;
326                 sq_db.u32_8 = 0;
327                 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,
328                                SQ_DOORBELL_U32_4_SQ_HEAD_S,
329                               (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)));
330                 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SL_M,
331                                SQ_DOORBELL_U32_4_SL_S, qp->sl);
332                 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M,
333                                SQ_DOORBELL_U32_4_PORT_S, qp->phy_port);
334                 roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M,
335                                SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn);
336                 roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1);
337
338                 doorbell[0] = le32_to_cpu(sq_db.u32_4);
339                 doorbell[1] = le32_to_cpu(sq_db.u32_8);
340
341                 hns_roce_write64_k((__le32 *)doorbell, qp->sq.db_reg_l);
342                 qp->sq_next_wqe = ind;
343         }
344
345         spin_unlock_irqrestore(&qp->sq.lock, flags);
346
347         return ret;
348 }
349
350 static int hns_roce_v1_post_recv(struct ib_qp *ibqp,
351                                  const struct ib_recv_wr *wr,
352                                  const struct ib_recv_wr **bad_wr)
353 {
354         int ret = 0;
355         int nreq = 0;
356         int ind = 0;
357         int i = 0;
358         u32 reg_val;
359         unsigned long flags = 0;
360         struct hns_roce_rq_wqe_ctrl *ctrl = NULL;
361         struct hns_roce_wqe_data_seg *scat = NULL;
362         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
363         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
364         struct device *dev = &hr_dev->pdev->dev;
365         struct hns_roce_rq_db rq_db;
366         uint32_t doorbell[2] = {0};
367
368         spin_lock_irqsave(&hr_qp->rq.lock, flags);
369         ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
370
371         for (nreq = 0; wr; ++nreq, wr = wr->next) {
372                 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
373                         hr_qp->ibqp.recv_cq)) {
374                         ret = -ENOMEM;
375                         *bad_wr = wr;
376                         goto out;
377                 }
378
379                 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
380                         dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
381                                 wr->num_sge, hr_qp->rq.max_gs);
382                         ret = -EINVAL;
383                         *bad_wr = wr;
384                         goto out;
385                 }
386
387                 ctrl = get_recv_wqe(hr_qp, ind);
388
389                 roce_set_field(ctrl->rwqe_byte_12,
390                                RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M,
391                                RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S,
392                                wr->num_sge);
393
394                 scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1);
395
396                 for (i = 0; i < wr->num_sge; i++)
397                         set_data_seg(scat + i, wr->sg_list + i);
398
399                 hr_qp->rq.wrid[ind] = wr->wr_id;
400
401                 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
402         }
403
404 out:
405         if (likely(nreq)) {
406                 hr_qp->rq.head += nreq;
407                 /* Memory barrier */
408                 wmb();
409
410                 if (ibqp->qp_type == IB_QPT_GSI) {
411                         __le32 tmp;
412
413                         /* SW update GSI rq header */
414                         reg_val = roce_read(to_hr_dev(ibqp->device),
415                                             ROCEE_QP1C_CFG3_0_REG +
416                                             QP1C_CFGN_OFFSET * hr_qp->phy_port);
417                         tmp = cpu_to_le32(reg_val);
418                         roce_set_field(tmp,
419                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,
420                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,
421                                        hr_qp->rq.head);
422                         reg_val = le32_to_cpu(tmp);
423                         roce_write(to_hr_dev(ibqp->device),
424                                    ROCEE_QP1C_CFG3_0_REG +
425                                    QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val);
426                 } else {
427                         rq_db.u32_4 = 0;
428                         rq_db.u32_8 = 0;
429
430                         roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
431                                        RQ_DOORBELL_U32_4_RQ_HEAD_S,
432                                        hr_qp->rq.head);
433                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,
434                                        RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
435                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,
436                                        RQ_DOORBELL_U32_8_CMD_S, 1);
437                         roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
438                                      1);
439
440                         doorbell[0] = le32_to_cpu(rq_db.u32_4);
441                         doorbell[1] = le32_to_cpu(rq_db.u32_8);
442
443                         hns_roce_write64_k((__le32 *)doorbell,
444                                            hr_qp->rq.db_reg_l);
445                 }
446         }
447         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
448
449         return ret;
450 }
451
452 static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev,
453                                        int sdb_mode, int odb_mode)
454 {
455         __le32 tmp;
456         u32 val;
457
458         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
459         tmp = cpu_to_le32(val);
460         roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode);
461         roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode);
462         val = le32_to_cpu(tmp);
463         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
464 }
465
466 static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode,
467                                      u32 odb_mode)
468 {
469         __le32 tmp;
470         u32 val;
471
472         /* Configure SDB/ODB extend mode */
473         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
474         tmp = cpu_to_le32(val);
475         roce_set_bit(tmp, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode);
476         roce_set_bit(tmp, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode);
477         val = le32_to_cpu(tmp);
478         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
479 }
480
481 static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept,
482                              u32 sdb_alful)
483 {
484         __le32 tmp;
485         u32 val;
486
487         /* Configure SDB */
488         val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG);
489         tmp = cpu_to_le32(val);
490         roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M,
491                        ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful);
492         roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M,
493                        ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept);
494         val = le32_to_cpu(tmp);
495         roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val);
496 }
497
498 static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept,
499                              u32 odb_alful)
500 {
501         __le32 tmp;
502         u32 val;
503
504         /* Configure ODB */
505         val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG);
506         tmp = cpu_to_le32(val);
507         roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M,
508                        ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful);
509         roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M,
510                        ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept);
511         val = le32_to_cpu(tmp);
512         roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val);
513 }
514
515 static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
516                                  u32 ext_sdb_alful)
517 {
518         struct device *dev = &hr_dev->pdev->dev;
519         struct hns_roce_v1_priv *priv;
520         struct hns_roce_db_table *db;
521         dma_addr_t sdb_dma_addr;
522         __le32 tmp;
523         u32 val;
524
525         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
526         db = &priv->db_table;
527
528         /* Configure extend SDB threshold */
529         roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept);
530         roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful);
531
532         /* Configure extend SDB base addr */
533         sdb_dma_addr = db->ext_db->sdb_buf_list->map;
534         roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12));
535
536         /* Configure extend SDB depth */
537         val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG);
538         tmp = cpu_to_le32(val);
539         roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M,
540                        ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S,
541                        db->ext_db->esdb_dep);
542         /*
543          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
544          * using 4K page, and shift more 32 because of
545          * caculating the high 32 bit value evaluated to hardware.
546          */
547         roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M,
548                        ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44);
549         val = le32_to_cpu(tmp);
550         roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val);
551
552         dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep);
553         dev_dbg(dev, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
554                 ext_sdb_alept, ext_sdb_alful);
555 }
556
557 static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
558                                  u32 ext_odb_alful)
559 {
560         struct device *dev = &hr_dev->pdev->dev;
561         struct hns_roce_v1_priv *priv;
562         struct hns_roce_db_table *db;
563         dma_addr_t odb_dma_addr;
564         __le32 tmp;
565         u32 val;
566
567         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
568         db = &priv->db_table;
569
570         /* Configure extend ODB threshold */
571         roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept);
572         roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful);
573
574         /* Configure extend ODB base addr */
575         odb_dma_addr = db->ext_db->odb_buf_list->map;
576         roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12));
577
578         /* Configure extend ODB depth */
579         val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG);
580         tmp = cpu_to_le32(val);
581         roce_set_field(tmp, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M,
582                        ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S,
583                        db->ext_db->eodb_dep);
584         roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M,
585                        ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S,
586                        db->ext_db->eodb_dep);
587         val = le32_to_cpu(tmp);
588         roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val);
589
590         dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep);
591         dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
592                 ext_odb_alept, ext_odb_alful);
593 }
594
595 static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,
596                                 u32 odb_ext_mod)
597 {
598         struct device *dev = &hr_dev->pdev->dev;
599         struct hns_roce_v1_priv *priv;
600         struct hns_roce_db_table *db;
601         dma_addr_t sdb_dma_addr;
602         dma_addr_t odb_dma_addr;
603         int ret = 0;
604
605         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
606         db = &priv->db_table;
607
608         db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL);
609         if (!db->ext_db)
610                 return -ENOMEM;
611
612         if (sdb_ext_mod) {
613                 db->ext_db->sdb_buf_list = kmalloc(
614                                 sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL);
615                 if (!db->ext_db->sdb_buf_list) {
616                         ret = -ENOMEM;
617                         goto ext_sdb_buf_fail_out;
618                 }
619
620                 db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev,
621                                                      HNS_ROCE_V1_EXT_SDB_SIZE,
622                                                      &sdb_dma_addr, GFP_KERNEL);
623                 if (!db->ext_db->sdb_buf_list->buf) {
624                         ret = -ENOMEM;
625                         goto alloc_sq_db_buf_fail;
626                 }
627                 db->ext_db->sdb_buf_list->map = sdb_dma_addr;
628
629                 db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH);
630                 hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT,
631                                      HNS_ROCE_V1_EXT_SDB_ALFUL);
632         } else
633                 hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT,
634                                  HNS_ROCE_V1_SDB_ALFUL);
635
636         if (odb_ext_mod) {
637                 db->ext_db->odb_buf_list = kmalloc(
638                                 sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL);
639                 if (!db->ext_db->odb_buf_list) {
640                         ret = -ENOMEM;
641                         goto ext_odb_buf_fail_out;
642                 }
643
644                 db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev,
645                                                      HNS_ROCE_V1_EXT_ODB_SIZE,
646                                                      &odb_dma_addr, GFP_KERNEL);
647                 if (!db->ext_db->odb_buf_list->buf) {
648                         ret = -ENOMEM;
649                         goto alloc_otr_db_buf_fail;
650                 }
651                 db->ext_db->odb_buf_list->map = odb_dma_addr;
652
653                 db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH);
654                 hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT,
655                                      HNS_ROCE_V1_EXT_ODB_ALFUL);
656         } else
657                 hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT,
658                                  HNS_ROCE_V1_ODB_ALFUL);
659
660         hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod);
661
662         return 0;
663
664 alloc_otr_db_buf_fail:
665         kfree(db->ext_db->odb_buf_list);
666
667 ext_odb_buf_fail_out:
668         if (sdb_ext_mod) {
669                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
670                                   db->ext_db->sdb_buf_list->buf,
671                                   db->ext_db->sdb_buf_list->map);
672         }
673
674 alloc_sq_db_buf_fail:
675         if (sdb_ext_mod)
676                 kfree(db->ext_db->sdb_buf_list);
677
678 ext_sdb_buf_fail_out:
679         kfree(db->ext_db);
680         return ret;
681 }
682
683 static struct hns_roce_qp *hns_roce_v1_create_lp_qp(struct hns_roce_dev *hr_dev,
684                                                     struct ib_pd *pd)
685 {
686         struct device *dev = &hr_dev->pdev->dev;
687         struct ib_qp_init_attr init_attr;
688         struct ib_qp *qp;
689
690         memset(&init_attr, 0, sizeof(struct ib_qp_init_attr));
691         init_attr.qp_type               = IB_QPT_RC;
692         init_attr.sq_sig_type           = IB_SIGNAL_ALL_WR;
693         init_attr.cap.max_recv_wr       = HNS_ROCE_MIN_WQE_NUM;
694         init_attr.cap.max_send_wr       = HNS_ROCE_MIN_WQE_NUM;
695
696         qp = hns_roce_create_qp(pd, &init_attr, NULL);
697         if (IS_ERR(qp)) {
698                 dev_err(dev, "Create loop qp for mr free failed!");
699                 return NULL;
700         }
701
702         return to_hr_qp(qp);
703 }
704
705 static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
706 {
707         struct hns_roce_caps *caps = &hr_dev->caps;
708         struct device *dev = &hr_dev->pdev->dev;
709         struct ib_cq_init_attr cq_init_attr;
710         struct hns_roce_free_mr *free_mr;
711         struct ib_qp_attr attr = { 0 };
712         struct hns_roce_v1_priv *priv;
713         struct hns_roce_qp *hr_qp;
714         struct ib_device *ibdev;
715         struct ib_cq *cq;
716         struct ib_pd *pd;
717         union ib_gid dgid;
718         u64 subnet_prefix;
719         int attr_mask = 0;
720         int ret;
721         int i, j;
722         u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 };
723         u8 phy_port;
724         u8 port = 0;
725         u8 sl;
726
727         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
728         free_mr = &priv->free_mr;
729
730         /* Reserved cq for loop qp */
731         cq_init_attr.cqe                = HNS_ROCE_MIN_WQE_NUM * 2;
732         cq_init_attr.comp_vector        = 0;
733
734         ibdev = &hr_dev->ib_dev;
735         cq = rdma_zalloc_drv_obj(ibdev, ib_cq);
736         if (!cq)
737                 return -ENOMEM;
738
739         ret = hns_roce_ib_create_cq(cq, &cq_init_attr, NULL);
740         if (ret) {
741                 dev_err(dev, "Create cq for reserved loop qp failed!");
742                 goto alloc_cq_failed;
743         }
744         free_mr->mr_free_cq = to_hr_cq(cq);
745         free_mr->mr_free_cq->ib_cq.device               = &hr_dev->ib_dev;
746         free_mr->mr_free_cq->ib_cq.uobject              = NULL;
747         free_mr->mr_free_cq->ib_cq.comp_handler         = NULL;
748         free_mr->mr_free_cq->ib_cq.event_handler        = NULL;
749         free_mr->mr_free_cq->ib_cq.cq_context           = NULL;
750         atomic_set(&free_mr->mr_free_cq->ib_cq.usecnt, 0);
751
752         pd = rdma_zalloc_drv_obj(ibdev, ib_pd);
753         if (!pd)
754                 goto alloc_mem_failed;
755
756         pd->device  = ibdev;
757         ret = hns_roce_alloc_pd(pd, NULL);
758         if (ret)
759                 goto alloc_pd_failed;
760
761         free_mr->mr_free_pd = to_hr_pd(pd);
762         free_mr->mr_free_pd->ibpd.device  = &hr_dev->ib_dev;
763         free_mr->mr_free_pd->ibpd.uobject = NULL;
764         free_mr->mr_free_pd->ibpd.__internal_mr = NULL;
765         atomic_set(&free_mr->mr_free_pd->ibpd.usecnt, 0);
766
767         attr.qp_access_flags    = IB_ACCESS_REMOTE_WRITE;
768         attr.pkey_index         = 0;
769         attr.min_rnr_timer      = 0;
770         /* Disable read ability */
771         attr.max_dest_rd_atomic = 0;
772         attr.max_rd_atomic      = 0;
773         /* Use arbitrary values as rq_psn and sq_psn */
774         attr.rq_psn             = 0x0808;
775         attr.sq_psn             = 0x0808;
776         attr.retry_cnt          = 7;
777         attr.rnr_retry          = 7;
778         attr.timeout            = 0x12;
779         attr.path_mtu           = IB_MTU_256;
780         attr.ah_attr.type       = RDMA_AH_ATTR_TYPE_ROCE;
781         rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0);
782         rdma_ah_set_static_rate(&attr.ah_attr, 3);
783
784         subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
785         for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
786                 phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) :
787                                 (i % HNS_ROCE_MAX_PORTS);
788                 sl = i / HNS_ROCE_MAX_PORTS;
789
790                 for (j = 0; j < caps->num_ports; j++) {
791                         if (hr_dev->iboe.phy_port[j] == phy_port) {
792                                 queue_en[i] = 1;
793                                 port = j;
794                                 break;
795                         }
796                 }
797
798                 if (!queue_en[i])
799                         continue;
800
801                 free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd);
802                 if (!free_mr->mr_free_qp[i]) {
803                         dev_err(dev, "Create loop qp failed!\n");
804                         ret = -ENOMEM;
805                         goto create_lp_qp_failed;
806                 }
807                 hr_qp = free_mr->mr_free_qp[i];
808
809                 hr_qp->port             = port;
810                 hr_qp->phy_port         = phy_port;
811                 hr_qp->ibqp.qp_type     = IB_QPT_RC;
812                 hr_qp->ibqp.device      = &hr_dev->ib_dev;
813                 hr_qp->ibqp.uobject     = NULL;
814                 atomic_set(&hr_qp->ibqp.usecnt, 0);
815                 hr_qp->ibqp.pd          = pd;
816                 hr_qp->ibqp.recv_cq     = cq;
817                 hr_qp->ibqp.send_cq     = cq;
818
819                 rdma_ah_set_port_num(&attr.ah_attr, port + 1);
820                 rdma_ah_set_sl(&attr.ah_attr, sl);
821                 attr.port_num           = port + 1;
822
823                 attr.dest_qp_num        = hr_qp->qpn;
824                 memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr),
825                        hr_dev->dev_addr[port],
826                        ETH_ALEN);
827
828                 memcpy(&dgid.raw, &subnet_prefix, sizeof(u64));
829                 memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3);
830                 memcpy(&dgid.raw[13], hr_dev->dev_addr[port] + 3, 3);
831                 dgid.raw[11] = 0xff;
832                 dgid.raw[12] = 0xfe;
833                 dgid.raw[8] ^= 2;
834                 rdma_ah_set_dgid_raw(&attr.ah_attr, dgid.raw);
835
836                 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
837                                             IB_QPS_RESET, IB_QPS_INIT);
838                 if (ret) {
839                         dev_err(dev, "modify qp failed(%d)!\n", ret);
840                         goto create_lp_qp_failed;
841                 }
842
843                 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, IB_QP_DEST_QPN,
844                                             IB_QPS_INIT, IB_QPS_RTR);
845                 if (ret) {
846                         dev_err(dev, "modify qp failed(%d)!\n", ret);
847                         goto create_lp_qp_failed;
848                 }
849
850                 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
851                                             IB_QPS_RTR, IB_QPS_RTS);
852                 if (ret) {
853                         dev_err(dev, "modify qp failed(%d)!\n", ret);
854                         goto create_lp_qp_failed;
855                 }
856         }
857
858         return 0;
859
860 create_lp_qp_failed:
861         for (i -= 1; i >= 0; i--) {
862                 hr_qp = free_mr->mr_free_qp[i];
863                 if (hns_roce_v1_destroy_qp(&hr_qp->ibqp, NULL))
864                         dev_err(dev, "Destroy qp %d for mr free failed!\n", i);
865         }
866
867         hns_roce_dealloc_pd(pd, NULL);
868
869 alloc_pd_failed:
870         kfree(pd);
871
872 alloc_mem_failed:
873         hns_roce_ib_destroy_cq(cq, NULL);
874 alloc_cq_failed:
875         kfree(cq);
876         return ret;
877 }
878
879 static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
880 {
881         struct device *dev = &hr_dev->pdev->dev;
882         struct hns_roce_free_mr *free_mr;
883         struct hns_roce_v1_priv *priv;
884         struct hns_roce_qp *hr_qp;
885         int ret;
886         int i;
887
888         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
889         free_mr = &priv->free_mr;
890
891         for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
892                 hr_qp = free_mr->mr_free_qp[i];
893                 if (!hr_qp)
894                         continue;
895
896                 ret = hns_roce_v1_destroy_qp(&hr_qp->ibqp, NULL);
897                 if (ret)
898                         dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n",
899                                 i, ret);
900         }
901
902         hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL);
903         kfree(&free_mr->mr_free_cq->ib_cq);
904         hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd, NULL);
905         kfree(&free_mr->mr_free_pd->ibpd);
906 }
907
908 static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
909 {
910         struct device *dev = &hr_dev->pdev->dev;
911         struct hns_roce_v1_priv *priv;
912         struct hns_roce_db_table *db;
913         u32 sdb_ext_mod;
914         u32 odb_ext_mod;
915         u32 sdb_evt_mod;
916         u32 odb_evt_mod;
917         int ret = 0;
918
919         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
920         db = &priv->db_table;
921
922         memset(db, 0, sizeof(*db));
923
924         /* Default DB mode */
925         sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE;
926         odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE;
927         sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE;
928         odb_evt_mod = HNS_ROCE_ODB_POLL_MODE;
929
930         db->sdb_ext_mod = sdb_ext_mod;
931         db->odb_ext_mod = odb_ext_mod;
932
933         /* Init extend DB */
934         ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod);
935         if (ret) {
936                 dev_err(dev, "Failed in extend DB configuration.\n");
937                 return ret;
938         }
939
940         hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod);
941
942         return 0;
943 }
944
945 static void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work)
946 {
947         struct hns_roce_recreate_lp_qp_work *lp_qp_work;
948         struct hns_roce_dev *hr_dev;
949
950         lp_qp_work = container_of(work, struct hns_roce_recreate_lp_qp_work,
951                                   work);
952         hr_dev = to_hr_dev(lp_qp_work->ib_dev);
953
954         hns_roce_v1_release_lp_qp(hr_dev);
955
956         if (hns_roce_v1_rsv_lp_qp(hr_dev))
957                 dev_err(&hr_dev->pdev->dev, "create reserver qp failed\n");
958
959         if (lp_qp_work->comp_flag)
960                 complete(lp_qp_work->comp);
961
962         kfree(lp_qp_work);
963 }
964
965 static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev)
966 {
967         struct device *dev = &hr_dev->pdev->dev;
968         struct hns_roce_recreate_lp_qp_work *lp_qp_work;
969         struct hns_roce_free_mr *free_mr;
970         struct hns_roce_v1_priv *priv;
971         struct completion comp;
972         unsigned long end = HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS;
973
974         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
975         free_mr = &priv->free_mr;
976
977         lp_qp_work = kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work),
978                              GFP_KERNEL);
979         if (!lp_qp_work)
980                 return -ENOMEM;
981
982         INIT_WORK(&(lp_qp_work->work), hns_roce_v1_recreate_lp_qp_work_fn);
983
984         lp_qp_work->ib_dev = &(hr_dev->ib_dev);
985         lp_qp_work->comp = &comp;
986         lp_qp_work->comp_flag = 1;
987
988         init_completion(lp_qp_work->comp);
989
990         queue_work(free_mr->free_mr_wq, &(lp_qp_work->work));
991
992         while (end) {
993                 if (try_wait_for_completion(&comp))
994                         return 0;
995                 msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE);
996                 end -= HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE;
997         }
998
999         lp_qp_work->comp_flag = 0;
1000         if (try_wait_for_completion(&comp))
1001                 return 0;
1002
1003         dev_warn(dev, "recreate lp qp failed 20s timeout and return failed!\n");
1004         return -ETIMEDOUT;
1005 }
1006
1007 static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp *hr_qp)
1008 {
1009         struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
1010         struct device *dev = &hr_dev->pdev->dev;
1011         struct ib_send_wr send_wr;
1012         const struct ib_send_wr *bad_wr;
1013         int ret;
1014
1015         memset(&send_wr, 0, sizeof(send_wr));
1016         send_wr.next    = NULL;
1017         send_wr.num_sge = 0;
1018         send_wr.send_flags = 0;
1019         send_wr.sg_list = NULL;
1020         send_wr.wr_id   = (unsigned long long)&send_wr;
1021         send_wr.opcode  = IB_WR_RDMA_WRITE;
1022
1023         ret = hns_roce_v1_post_send(&hr_qp->ibqp, &send_wr, &bad_wr);
1024         if (ret) {
1025                 dev_err(dev, "Post write wqe for mr free failed(%d)!", ret);
1026                 return ret;
1027         }
1028
1029         return 0;
1030 }
1031
1032 static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
1033 {
1034         struct hns_roce_mr_free_work *mr_work;
1035         struct ib_wc wc[HNS_ROCE_V1_RESV_QP];
1036         struct hns_roce_free_mr *free_mr;
1037         struct hns_roce_cq *mr_free_cq;
1038         struct hns_roce_v1_priv *priv;
1039         struct hns_roce_dev *hr_dev;
1040         struct hns_roce_mr *hr_mr;
1041         struct hns_roce_qp *hr_qp;
1042         struct device *dev;
1043         unsigned long end =
1044                 msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
1045         int i;
1046         int ret;
1047         int ne = 0;
1048
1049         mr_work = container_of(work, struct hns_roce_mr_free_work, work);
1050         hr_mr = (struct hns_roce_mr *)mr_work->mr;
1051         hr_dev = to_hr_dev(mr_work->ib_dev);
1052         dev = &hr_dev->pdev->dev;
1053
1054         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1055         free_mr = &priv->free_mr;
1056         mr_free_cq = free_mr->mr_free_cq;
1057
1058         for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
1059                 hr_qp = free_mr->mr_free_qp[i];
1060                 if (!hr_qp)
1061                         continue;
1062                 ne++;
1063
1064                 ret = hns_roce_v1_send_lp_wqe(hr_qp);
1065                 if (ret) {
1066                         dev_err(dev,
1067                              "Send wqe (qp:0x%lx) for mr free failed(%d)!\n",
1068                              hr_qp->qpn, ret);
1069                         goto free_work;
1070                 }
1071         }
1072
1073         if (!ne) {
1074                 dev_err(dev, "Reserved loop qp is absent!\n");
1075                 goto free_work;
1076         }
1077
1078         do {
1079                 ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc);
1080                 if (ret < 0 && hr_qp) {
1081                         dev_err(dev,
1082                            "(qp:0x%lx) starts, Poll cqe failed(%d) for mr 0x%x free! Remain %d cqe\n",
1083                            hr_qp->qpn, ret, hr_mr->key, ne);
1084                         goto free_work;
1085                 }
1086                 ne -= ret;
1087                 usleep_range(HNS_ROCE_V1_FREE_MR_WAIT_VALUE * 1000,
1088                              (1 + HNS_ROCE_V1_FREE_MR_WAIT_VALUE) * 1000);
1089         } while (ne && time_before_eq(jiffies, end));
1090
1091         if (ne != 0)
1092                 dev_err(dev,
1093                         "Poll cqe for mr 0x%x free timeout! Remain %d cqe\n",
1094                         hr_mr->key, ne);
1095
1096 free_work:
1097         if (mr_work->comp_flag)
1098                 complete(mr_work->comp);
1099         kfree(mr_work);
1100 }
1101
1102 static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev,
1103                                 struct hns_roce_mr *mr, struct ib_udata *udata)
1104 {
1105         struct device *dev = &hr_dev->pdev->dev;
1106         struct hns_roce_mr_free_work *mr_work;
1107         struct hns_roce_free_mr *free_mr;
1108         struct hns_roce_v1_priv *priv;
1109         struct completion comp;
1110         unsigned long end = HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS;
1111         unsigned long start = jiffies;
1112         int npages;
1113         int ret = 0;
1114
1115         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1116         free_mr = &priv->free_mr;
1117
1118         if (mr->enabled) {
1119                 if (hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key)
1120                                        & (hr_dev->caps.num_mtpts - 1)))
1121                         dev_warn(dev, "HW2SW_MPT failed!\n");
1122         }
1123
1124         mr_work = kzalloc(sizeof(*mr_work), GFP_KERNEL);
1125         if (!mr_work) {
1126                 ret = -ENOMEM;
1127                 goto free_mr;
1128         }
1129
1130         INIT_WORK(&(mr_work->work), hns_roce_v1_mr_free_work_fn);
1131
1132         mr_work->ib_dev = &(hr_dev->ib_dev);
1133         mr_work->comp = &comp;
1134         mr_work->comp_flag = 1;
1135         mr_work->mr = (void *)mr;
1136         init_completion(mr_work->comp);
1137
1138         queue_work(free_mr->free_mr_wq, &(mr_work->work));
1139
1140         while (end) {
1141                 if (try_wait_for_completion(&comp))
1142                         goto free_mr;
1143                 msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE);
1144                 end -= HNS_ROCE_V1_FREE_MR_WAIT_VALUE;
1145         }
1146
1147         mr_work->comp_flag = 0;
1148         if (try_wait_for_completion(&comp))
1149                 goto free_mr;
1150
1151         dev_warn(dev, "Free mr work 0x%x over 50s and failed!\n", mr->key);
1152         ret = -ETIMEDOUT;
1153
1154 free_mr:
1155         dev_dbg(dev, "Free mr 0x%x use 0x%x us.\n",
1156                 mr->key, jiffies_to_usecs(jiffies) - jiffies_to_usecs(start));
1157
1158         if (mr->size != ~0ULL) {
1159                 npages = ib_umem_page_count(mr->umem);
1160                 dma_free_coherent(dev, npages * 8, mr->pbl_buf,
1161                                   mr->pbl_dma_addr);
1162         }
1163
1164         hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
1165                              key_to_hw_index(mr->key), 0);
1166
1167         ib_umem_release(mr->umem);
1168
1169         kfree(mr);
1170
1171         return ret;
1172 }
1173
1174 static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
1175 {
1176         struct device *dev = &hr_dev->pdev->dev;
1177         struct hns_roce_v1_priv *priv;
1178         struct hns_roce_db_table *db;
1179
1180         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1181         db = &priv->db_table;
1182
1183         if (db->sdb_ext_mod) {
1184                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
1185                                   db->ext_db->sdb_buf_list->buf,
1186                                   db->ext_db->sdb_buf_list->map);
1187                 kfree(db->ext_db->sdb_buf_list);
1188         }
1189
1190         if (db->odb_ext_mod) {
1191                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE,
1192                                   db->ext_db->odb_buf_list->buf,
1193                                   db->ext_db->odb_buf_list->map);
1194                 kfree(db->ext_db->odb_buf_list);
1195         }
1196
1197         kfree(db->ext_db);
1198 }
1199
1200 static int hns_roce_raq_init(struct hns_roce_dev *hr_dev)
1201 {
1202         int ret;
1203         u32 val;
1204         __le32 tmp;
1205         int raq_shift = 0;
1206         dma_addr_t addr;
1207         struct hns_roce_v1_priv *priv;
1208         struct hns_roce_raq_table *raq;
1209         struct device *dev = &hr_dev->pdev->dev;
1210
1211         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1212         raq = &priv->raq_table;
1213
1214         raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL);
1215         if (!raq->e_raq_buf)
1216                 return -ENOMEM;
1217
1218         raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE,
1219                                                  &addr, GFP_KERNEL);
1220         if (!raq->e_raq_buf->buf) {
1221                 ret = -ENOMEM;
1222                 goto err_dma_alloc_raq;
1223         }
1224         raq->e_raq_buf->map = addr;
1225
1226         /* Configure raq extended address. 48bit 4K align*/
1227         roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12);
1228
1229         /* Configure raq_shift */
1230         raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY);
1231         val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG);
1232         tmp = cpu_to_le32(val);
1233         roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M,
1234                        ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift);
1235         /*
1236          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1237          * using 4K page, and shift more 32 because of
1238          * caculating the high 32 bit value evaluated to hardware.
1239          */
1240         roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M,
1241                        ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S,
1242                        raq->e_raq_buf->map >> 44);
1243         val = le32_to_cpu(tmp);
1244         roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val);
1245         dev_dbg(dev, "Configure raq_shift 0x%x.\n", val);
1246
1247         /* Configure raq threshold */
1248         val = roce_read(hr_dev, ROCEE_RAQ_WL_REG);
1249         tmp = cpu_to_le32(val);
1250         roce_set_field(tmp, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M,
1251                        ROCEE_RAQ_WL_ROCEE_RAQ_WL_S,
1252                        HNS_ROCE_V1_EXT_RAQ_WF);
1253         val = le32_to_cpu(tmp);
1254         roce_write(hr_dev, ROCEE_RAQ_WL_REG, val);
1255         dev_dbg(dev, "Configure raq_wl 0x%x.\n", val);
1256
1257         /* Enable extend raq */
1258         val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG);
1259         tmp = cpu_to_le32(val);
1260         roce_set_field(tmp,
1261                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M,
1262                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S,
1263                        POL_TIME_INTERVAL_VAL);
1264         roce_set_bit(tmp, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1);
1265         roce_set_field(tmp,
1266                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M,
1267                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S,
1268                        2);
1269         roce_set_bit(tmp,
1270                      ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1);
1271         val = le32_to_cpu(tmp);
1272         roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val);
1273         dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val);
1274
1275         /* Enable raq drop */
1276         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
1277         tmp = cpu_to_le32(val);
1278         roce_set_bit(tmp, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1);
1279         val = le32_to_cpu(tmp);
1280         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
1281         dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val);
1282
1283         return 0;
1284
1285 err_dma_alloc_raq:
1286         kfree(raq->e_raq_buf);
1287         return ret;
1288 }
1289
1290 static void hns_roce_raq_free(struct hns_roce_dev *hr_dev)
1291 {
1292         struct device *dev = &hr_dev->pdev->dev;
1293         struct hns_roce_v1_priv *priv;
1294         struct hns_roce_raq_table *raq;
1295
1296         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1297         raq = &priv->raq_table;
1298
1299         dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf,
1300                           raq->e_raq_buf->map);
1301         kfree(raq->e_raq_buf);
1302 }
1303
1304 static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)
1305 {
1306         __le32 tmp;
1307         u32 val;
1308
1309         if (enable_flag) {
1310                 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
1311                  /* Open all ports */
1312                 tmp = cpu_to_le32(val);
1313                 roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
1314                                ROCEE_GLB_CFG_ROCEE_PORT_ST_S,
1315                                ALL_PORT_VAL_OPEN);
1316                 val = le32_to_cpu(tmp);
1317                 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
1318         } else {
1319                 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
1320                 /* Close all ports */
1321                 tmp = cpu_to_le32(val);
1322                 roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
1323                                ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0);
1324                 val = le32_to_cpu(tmp);
1325                 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
1326         }
1327 }
1328
1329 static int hns_roce_bt_init(struct hns_roce_dev *hr_dev)
1330 {
1331         struct device *dev = &hr_dev->pdev->dev;
1332         struct hns_roce_v1_priv *priv;
1333         int ret;
1334
1335         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1336
1337         priv->bt_table.qpc_buf.buf = dma_alloc_coherent(dev,
1338                 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.qpc_buf.map,
1339                 GFP_KERNEL);
1340         if (!priv->bt_table.qpc_buf.buf)
1341                 return -ENOMEM;
1342
1343         priv->bt_table.mtpt_buf.buf = dma_alloc_coherent(dev,
1344                 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.mtpt_buf.map,
1345                 GFP_KERNEL);
1346         if (!priv->bt_table.mtpt_buf.buf) {
1347                 ret = -ENOMEM;
1348                 goto err_failed_alloc_mtpt_buf;
1349         }
1350
1351         priv->bt_table.cqc_buf.buf = dma_alloc_coherent(dev,
1352                 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.cqc_buf.map,
1353                 GFP_KERNEL);
1354         if (!priv->bt_table.cqc_buf.buf) {
1355                 ret = -ENOMEM;
1356                 goto err_failed_alloc_cqc_buf;
1357         }
1358
1359         return 0;
1360
1361 err_failed_alloc_cqc_buf:
1362         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1363                 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
1364
1365 err_failed_alloc_mtpt_buf:
1366         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1367                 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
1368
1369         return ret;
1370 }
1371
1372 static void hns_roce_bt_free(struct hns_roce_dev *hr_dev)
1373 {
1374         struct device *dev = &hr_dev->pdev->dev;
1375         struct hns_roce_v1_priv *priv;
1376
1377         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1378
1379         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1380                 priv->bt_table.cqc_buf.buf, priv->bt_table.cqc_buf.map);
1381
1382         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1383                 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
1384
1385         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
1386                 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
1387 }
1388
1389 static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev)
1390 {
1391         struct device *dev = &hr_dev->pdev->dev;
1392         struct hns_roce_buf_list *tptr_buf;
1393         struct hns_roce_v1_priv *priv;
1394
1395         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1396         tptr_buf = &priv->tptr_table.tptr_buf;
1397
1398         /*
1399          * This buffer will be used for CQ's tptr(tail pointer), also
1400          * named ci(customer index). Every CQ will use 2 bytes to save
1401          * cqe ci in hip06. Hardware will read this area to get new ci
1402          * when the queue is almost full.
1403          */
1404         tptr_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,
1405                                            &tptr_buf->map, GFP_KERNEL);
1406         if (!tptr_buf->buf)
1407                 return -ENOMEM;
1408
1409         hr_dev->tptr_dma_addr = tptr_buf->map;
1410         hr_dev->tptr_size = HNS_ROCE_V1_TPTR_BUF_SIZE;
1411
1412         return 0;
1413 }
1414
1415 static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev)
1416 {
1417         struct device *dev = &hr_dev->pdev->dev;
1418         struct hns_roce_buf_list *tptr_buf;
1419         struct hns_roce_v1_priv *priv;
1420
1421         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1422         tptr_buf = &priv->tptr_table.tptr_buf;
1423
1424         dma_free_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,
1425                           tptr_buf->buf, tptr_buf->map);
1426 }
1427
1428 static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev)
1429 {
1430         struct device *dev = &hr_dev->pdev->dev;
1431         struct hns_roce_free_mr *free_mr;
1432         struct hns_roce_v1_priv *priv;
1433         int ret = 0;
1434
1435         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1436         free_mr = &priv->free_mr;
1437
1438         free_mr->free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr");
1439         if (!free_mr->free_mr_wq) {
1440                 dev_err(dev, "Create free mr workqueue failed!\n");
1441                 return -ENOMEM;
1442         }
1443
1444         ret = hns_roce_v1_rsv_lp_qp(hr_dev);
1445         if (ret) {
1446                 dev_err(dev, "Reserved loop qp failed(%d)!\n", ret);
1447                 flush_workqueue(free_mr->free_mr_wq);
1448                 destroy_workqueue(free_mr->free_mr_wq);
1449         }
1450
1451         return ret;
1452 }
1453
1454 static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev)
1455 {
1456         struct hns_roce_free_mr *free_mr;
1457         struct hns_roce_v1_priv *priv;
1458
1459         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
1460         free_mr = &priv->free_mr;
1461
1462         flush_workqueue(free_mr->free_mr_wq);
1463         destroy_workqueue(free_mr->free_mr_wq);
1464
1465         hns_roce_v1_release_lp_qp(hr_dev);
1466 }
1467
1468 /**
1469  * hns_roce_v1_reset - reset RoCE
1470  * @hr_dev: RoCE device struct pointer
1471  * @enable: true -- drop reset, false -- reset
1472  * return 0 - success , negative --fail
1473  */
1474 static int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)
1475 {
1476         struct device_node *dsaf_node;
1477         struct device *dev = &hr_dev->pdev->dev;
1478         struct device_node *np = dev->of_node;
1479         struct fwnode_handle *fwnode;
1480         int ret;
1481
1482         /* check if this is DT/ACPI case */
1483         if (dev_of_node(dev)) {
1484                 dsaf_node = of_parse_phandle(np, "dsaf-handle", 0);
1485                 if (!dsaf_node) {
1486                         dev_err(dev, "could not find dsaf-handle\n");
1487                         return -EINVAL;
1488                 }
1489                 fwnode = &dsaf_node->fwnode;
1490         } else if (is_acpi_device_node(dev->fwnode)) {
1491                 struct fwnode_reference_args args;
1492
1493                 ret = acpi_node_get_property_reference(dev->fwnode,
1494                                                        "dsaf-handle", 0, &args);
1495                 if (ret) {
1496                         dev_err(dev, "could not find dsaf-handle\n");
1497                         return ret;
1498                 }
1499                 fwnode = args.fwnode;
1500         } else {
1501                 dev_err(dev, "cannot read data from DT or ACPI\n");
1502                 return -ENXIO;
1503         }
1504
1505         ret = hns_dsaf_roce_reset(fwnode, false);
1506         if (ret)
1507                 return ret;
1508
1509         if (dereset) {
1510                 msleep(SLEEP_TIME_INTERVAL);
1511                 ret = hns_dsaf_roce_reset(fwnode, true);
1512         }
1513
1514         return ret;
1515 }
1516
1517 static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
1518 {
1519         int i = 0;
1520         struct hns_roce_caps *caps = &hr_dev->caps;
1521
1522         hr_dev->vendor_id = roce_read(hr_dev, ROCEE_VENDOR_ID_REG);
1523         hr_dev->vendor_part_id = roce_read(hr_dev, ROCEE_VENDOR_PART_ID_REG);
1524         hr_dev->sys_image_guid = roce_read(hr_dev, ROCEE_SYS_IMAGE_GUID_L_REG) |
1525                                 ((u64)roce_read(hr_dev,
1526                                             ROCEE_SYS_IMAGE_GUID_H_REG) << 32);
1527         hr_dev->hw_rev          = HNS_ROCE_HW_VER1;
1528
1529         caps->num_qps           = HNS_ROCE_V1_MAX_QP_NUM;
1530         caps->max_wqes          = HNS_ROCE_V1_MAX_WQE_NUM;
1531         caps->min_wqes          = HNS_ROCE_MIN_WQE_NUM;
1532         caps->num_cqs           = HNS_ROCE_V1_MAX_CQ_NUM;
1533         caps->min_cqes          = HNS_ROCE_MIN_CQE_NUM;
1534         caps->max_cqes          = HNS_ROCE_V1_MAX_CQE_NUM;
1535         caps->max_sq_sg         = HNS_ROCE_V1_SG_NUM;
1536         caps->max_rq_sg         = HNS_ROCE_V1_SG_NUM;
1537         caps->max_sq_inline     = HNS_ROCE_V1_INLINE_SIZE;
1538         caps->num_uars          = HNS_ROCE_V1_UAR_NUM;
1539         caps->phy_num_uars      = HNS_ROCE_V1_PHY_UAR_NUM;
1540         caps->num_aeq_vectors   = HNS_ROCE_V1_AEQE_VEC_NUM;
1541         caps->num_comp_vectors  = HNS_ROCE_V1_COMP_VEC_NUM;
1542         caps->num_other_vectors = HNS_ROCE_V1_ABNORMAL_VEC_NUM;
1543         caps->num_mtpts         = HNS_ROCE_V1_MAX_MTPT_NUM;
1544         caps->num_mtt_segs      = HNS_ROCE_V1_MAX_MTT_SEGS;
1545         caps->num_pds           = HNS_ROCE_V1_MAX_PD_NUM;
1546         caps->max_qp_init_rdma  = HNS_ROCE_V1_MAX_QP_INIT_RDMA;
1547         caps->max_qp_dest_rdma  = HNS_ROCE_V1_MAX_QP_DEST_RDMA;
1548         caps->max_sq_desc_sz    = HNS_ROCE_V1_MAX_SQ_DESC_SZ;
1549         caps->max_rq_desc_sz    = HNS_ROCE_V1_MAX_RQ_DESC_SZ;
1550         caps->qpc_entry_sz      = HNS_ROCE_V1_QPC_ENTRY_SIZE;
1551         caps->irrl_entry_sz     = HNS_ROCE_V1_IRRL_ENTRY_SIZE;
1552         caps->cqc_entry_sz      = HNS_ROCE_V1_CQC_ENTRY_SIZE;
1553         caps->mtpt_entry_sz     = HNS_ROCE_V1_MTPT_ENTRY_SIZE;
1554         caps->mtt_entry_sz      = HNS_ROCE_V1_MTT_ENTRY_SIZE;
1555         caps->cq_entry_sz       = HNS_ROCE_V1_CQE_ENTRY_SIZE;
1556         caps->page_size_cap     = HNS_ROCE_V1_PAGE_SIZE_SUPPORT;
1557         caps->reserved_lkey     = 0;
1558         caps->reserved_pds      = 0;
1559         caps->reserved_mrws     = 1;
1560         caps->reserved_uars     = 0;
1561         caps->reserved_cqs      = 0;
1562         caps->reserved_qps      = 12; /* 2 SQP per port, six ports total 12 */
1563         caps->chunk_sz          = HNS_ROCE_V1_TABLE_CHUNK_SIZE;
1564
1565         for (i = 0; i < caps->num_ports; i++)
1566                 caps->pkey_table_len[i] = 1;
1567
1568         for (i = 0; i < caps->num_ports; i++) {
1569                 /* Six ports shared 16 GID in v1 engine */
1570                 if (i >= (HNS_ROCE_V1_GID_NUM % caps->num_ports))
1571                         caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
1572                                                  caps->num_ports;
1573                 else
1574                         caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
1575                                                  caps->num_ports + 1;
1576         }
1577
1578         caps->ceqe_depth = HNS_ROCE_V1_COMP_EQE_NUM;
1579         caps->aeqe_depth = HNS_ROCE_V1_ASYNC_EQE_NUM;
1580         caps->local_ca_ack_delay = roce_read(hr_dev, ROCEE_ACK_DELAY_REG);
1581         caps->max_mtu = IB_MTU_2048;
1582
1583         return 0;
1584 }
1585
1586 static int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
1587 {
1588         int ret;
1589         u32 val;
1590         __le32 tmp;
1591         struct device *dev = &hr_dev->pdev->dev;
1592
1593         /* DMAE user config */
1594         val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG);
1595         tmp = cpu_to_le32(val);
1596         roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M,
1597                        ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf);
1598         roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M,
1599                        ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S,
1600                        1 << PAGES_SHIFT_16);
1601         val = le32_to_cpu(tmp);
1602         roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val);
1603
1604         val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG);
1605         tmp = cpu_to_le32(val);
1606         roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M,
1607                        ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf);
1608         roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M,
1609                        ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S,
1610                        1 << PAGES_SHIFT_16);
1611
1612         ret = hns_roce_db_init(hr_dev);
1613         if (ret) {
1614                 dev_err(dev, "doorbell init failed!\n");
1615                 return ret;
1616         }
1617
1618         ret = hns_roce_raq_init(hr_dev);
1619         if (ret) {
1620                 dev_err(dev, "raq init failed!\n");
1621                 goto error_failed_raq_init;
1622         }
1623
1624         ret = hns_roce_bt_init(hr_dev);
1625         if (ret) {
1626                 dev_err(dev, "bt init failed!\n");
1627                 goto error_failed_bt_init;
1628         }
1629
1630         ret = hns_roce_tptr_init(hr_dev);
1631         if (ret) {
1632                 dev_err(dev, "tptr init failed!\n");
1633                 goto error_failed_tptr_init;
1634         }
1635
1636         ret = hns_roce_free_mr_init(hr_dev);
1637         if (ret) {
1638                 dev_err(dev, "free mr init failed!\n");
1639                 goto error_failed_free_mr_init;
1640         }
1641
1642         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
1643
1644         return 0;
1645
1646 error_failed_free_mr_init:
1647         hns_roce_tptr_free(hr_dev);
1648
1649 error_failed_tptr_init:
1650         hns_roce_bt_free(hr_dev);
1651
1652 error_failed_bt_init:
1653         hns_roce_raq_free(hr_dev);
1654
1655 error_failed_raq_init:
1656         hns_roce_db_free(hr_dev);
1657         return ret;
1658 }
1659
1660 static void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)
1661 {
1662         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
1663         hns_roce_free_mr_free(hr_dev);
1664         hns_roce_tptr_free(hr_dev);
1665         hns_roce_bt_free(hr_dev);
1666         hns_roce_raq_free(hr_dev);
1667         hns_roce_db_free(hr_dev);
1668 }
1669
1670 static int hns_roce_v1_cmd_pending(struct hns_roce_dev *hr_dev)
1671 {
1672         u32 status = readl(hr_dev->reg_base + ROCEE_MB6_REG);
1673
1674         return (!!(status & (1 << HCR_GO_BIT)));
1675 }
1676
1677 static int hns_roce_v1_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
1678                                  u64 out_param, u32 in_modifier, u8 op_modifier,
1679                                  u16 op, u16 token, int event)
1680 {
1681         u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base + ROCEE_MB1_REG);
1682         unsigned long end;
1683         u32 val = 0;
1684         __le32 tmp;
1685
1686         end = msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS) + jiffies;
1687         while (hns_roce_v1_cmd_pending(hr_dev)) {
1688                 if (time_after(jiffies, end)) {
1689                         dev_err(hr_dev->dev, "jiffies=%d end=%d\n",
1690                                 (int)jiffies, (int)end);
1691                         return -EAGAIN;
1692                 }
1693                 cond_resched();
1694         }
1695
1696         tmp = cpu_to_le32(val);
1697         roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_M, ROCEE_MB6_ROCEE_MB_CMD_S,
1698                        op);
1699         roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_MDF_M,
1700                        ROCEE_MB6_ROCEE_MB_CMD_MDF_S, op_modifier);
1701         roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_EVENT_S, event);
1702         roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_HW_RUN_S, 1);
1703         roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_TOKEN_M,
1704                        ROCEE_MB6_ROCEE_MB_TOKEN_S, token);
1705
1706         val = le32_to_cpu(tmp);
1707         writeq(in_param, hcr + 0);
1708         writeq(out_param, hcr + 2);
1709         writel(in_modifier, hcr + 4);
1710         /* Memory barrier */
1711         wmb();
1712
1713         writel(val, hcr + 5);
1714
1715         return 0;
1716 }
1717
1718 static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev,
1719                                 unsigned long timeout)
1720 {
1721         u8 __iomem *hcr = hr_dev->reg_base + ROCEE_MB1_REG;
1722         unsigned long end = 0;
1723         u32 status = 0;
1724
1725         end = msecs_to_jiffies(timeout) + jiffies;
1726         while (hns_roce_v1_cmd_pending(hr_dev) && time_before(jiffies, end))
1727                 cond_resched();
1728
1729         if (hns_roce_v1_cmd_pending(hr_dev)) {
1730                 dev_err(hr_dev->dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
1731                 return -ETIMEDOUT;
1732         }
1733
1734         status = le32_to_cpu((__force __le32)
1735                               __raw_readl(hcr + HCR_STATUS_OFFSET));
1736         if ((status & STATUS_MASK) != 0x1) {
1737                 dev_err(hr_dev->dev, "mailbox status 0x%x!\n", status);
1738                 return -EBUSY;
1739         }
1740
1741         return 0;
1742 }
1743
1744 static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port,
1745                                int gid_index, const union ib_gid *gid,
1746                                const struct ib_gid_attr *attr)
1747 {
1748         unsigned long flags;
1749         u32 *p = NULL;
1750         u8 gid_idx = 0;
1751
1752         gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
1753
1754         spin_lock_irqsave(&hr_dev->iboe.lock, flags);
1755
1756         p = (u32 *)&gid->raw[0];
1757         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG +
1758                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1759
1760         p = (u32 *)&gid->raw[4];
1761         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG +
1762                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1763
1764         p = (u32 *)&gid->raw[8];
1765         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG +
1766                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1767
1768         p = (u32 *)&gid->raw[0xc];
1769         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG +
1770                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1771
1772         spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
1773
1774         return 0;
1775 }
1776
1777 static int hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
1778                                u8 *addr)
1779 {
1780         u32 reg_smac_l;
1781         u16 reg_smac_h;
1782         __le32 tmp;
1783         u16 *p_h;
1784         u32 *p;
1785         u32 val;
1786
1787         /*
1788          * When mac changed, loopback may fail
1789          * because of smac not equal to dmac.
1790          * We Need to release and create reserved qp again.
1791          */
1792         if (hr_dev->hw->dereg_mr) {
1793                 int ret;
1794
1795                 ret = hns_roce_v1_recreate_lp_qp(hr_dev);
1796                 if (ret && ret != -ETIMEDOUT)
1797                         return ret;
1798         }
1799
1800         p = (u32 *)(&addr[0]);
1801         reg_smac_l = *p;
1802         roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG +
1803                        PHY_PORT_OFFSET * phy_port);
1804
1805         val = roce_read(hr_dev,
1806                         ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1807         tmp = cpu_to_le32(val);
1808         p_h = (u16 *)(&addr[4]);
1809         reg_smac_h  = *p_h;
1810         roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_SMAC_H_M,
1811                        ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h);
1812         val = le32_to_cpu(tmp);
1813         roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1814                    val);
1815
1816         return 0;
1817 }
1818
1819 static void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port,
1820                                 enum ib_mtu mtu)
1821 {
1822         __le32 tmp;
1823         u32 val;
1824
1825         val = roce_read(hr_dev,
1826                         ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1827         tmp = cpu_to_le32(val);
1828         roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_PORT_MTU_M,
1829                        ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu);
1830         val = le32_to_cpu(tmp);
1831         roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1832                    val);
1833 }
1834
1835 static int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
1836                                   unsigned long mtpt_idx)
1837 {
1838         struct hns_roce_v1_mpt_entry *mpt_entry;
1839         struct sg_dma_page_iter sg_iter;
1840         u64 *pages;
1841         int i;
1842
1843         /* MPT filled into mailbox buf */
1844         mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf;
1845         memset(mpt_entry, 0, sizeof(*mpt_entry));
1846
1847         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_STATE_M,
1848                        MPT_BYTE_4_KEY_STATE_S, KEY_VALID);
1849         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_M,
1850                        MPT_BYTE_4_KEY_S, mr->key);
1851         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M,
1852                        MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K);
1853         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0);
1854         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S,
1855                      (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1856         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_OWN_S, 0);
1857         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M,
1858                        MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr->type);
1859         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0);
1860         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S,
1861                      (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1862         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S,
1863                      (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1864         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S,
1865                      (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1866         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S,
1867                      0);
1868         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0);
1869
1870         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1871                        MPT_BYTE_12_PBL_ADDR_H_S, 0);
1872         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M,
1873                        MPT_BYTE_12_MW_BIND_COUNTER_S, 0);
1874
1875         mpt_entry->virt_addr_l = cpu_to_le32((u32)mr->iova);
1876         mpt_entry->virt_addr_h = cpu_to_le32((u32)(mr->iova >> 32));
1877         mpt_entry->length = cpu_to_le32((u32)mr->size);
1878
1879         roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_PD_M,
1880                        MPT_BYTE_28_PD_S, mr->pd);
1881         roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M,
1882                        MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx);
1883         roce_set_field(mpt_entry->mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M,
1884                        MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx >> MTPT_IDX_SHIFT);
1885
1886         /* DMA memory register */
1887         if (mr->type == MR_TYPE_DMA)
1888                 return 0;
1889
1890         pages = (u64 *) __get_free_page(GFP_KERNEL);
1891         if (!pages)
1892                 return -ENOMEM;
1893
1894         i = 0;
1895         for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) {
1896                 pages[i] = ((u64)sg_page_iter_dma_address(&sg_iter)) >> 12;
1897
1898                 /* Directly record to MTPT table firstly 7 entry */
1899                 if (i >= HNS_ROCE_MAX_INNER_MTPT_NUM)
1900                         break;
1901                 i++;
1902         }
1903
1904         /* Register user mr */
1905         for (i = 0; i < HNS_ROCE_MAX_INNER_MTPT_NUM; i++) {
1906                 switch (i) {
1907                 case 0:
1908                         mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i]));
1909                         roce_set_field(mpt_entry->mpt_byte_36,
1910                                 MPT_BYTE_36_PA0_H_M,
1911                                 MPT_BYTE_36_PA0_H_S,
1912                                 (u32)(pages[i] >> PAGES_SHIFT_32));
1913                         break;
1914                 case 1:
1915                         roce_set_field(mpt_entry->mpt_byte_36,
1916                                        MPT_BYTE_36_PA1_L_M,
1917                                        MPT_BYTE_36_PA1_L_S, (u32)(pages[i]));
1918                         roce_set_field(mpt_entry->mpt_byte_40,
1919                                 MPT_BYTE_40_PA1_H_M,
1920                                 MPT_BYTE_40_PA1_H_S,
1921                                 (u32)(pages[i] >> PAGES_SHIFT_24));
1922                         break;
1923                 case 2:
1924                         roce_set_field(mpt_entry->mpt_byte_40,
1925                                        MPT_BYTE_40_PA2_L_M,
1926                                        MPT_BYTE_40_PA2_L_S, (u32)(pages[i]));
1927                         roce_set_field(mpt_entry->mpt_byte_44,
1928                                 MPT_BYTE_44_PA2_H_M,
1929                                 MPT_BYTE_44_PA2_H_S,
1930                                 (u32)(pages[i] >> PAGES_SHIFT_16));
1931                         break;
1932                 case 3:
1933                         roce_set_field(mpt_entry->mpt_byte_44,
1934                                        MPT_BYTE_44_PA3_L_M,
1935                                        MPT_BYTE_44_PA3_L_S, (u32)(pages[i]));
1936                         roce_set_field(mpt_entry->mpt_byte_48,
1937                                 MPT_BYTE_48_PA3_H_M,
1938                                 MPT_BYTE_48_PA3_H_S,
1939                                 (u32)(pages[i] >> PAGES_SHIFT_8));
1940                         break;
1941                 case 4:
1942                         mpt_entry->pa4_l = cpu_to_le32((u32)(pages[i]));
1943                         roce_set_field(mpt_entry->mpt_byte_56,
1944                                 MPT_BYTE_56_PA4_H_M,
1945                                 MPT_BYTE_56_PA4_H_S,
1946                                 (u32)(pages[i] >> PAGES_SHIFT_32));
1947                         break;
1948                 case 5:
1949                         roce_set_field(mpt_entry->mpt_byte_56,
1950                                        MPT_BYTE_56_PA5_L_M,
1951                                        MPT_BYTE_56_PA5_L_S, (u32)(pages[i]));
1952                         roce_set_field(mpt_entry->mpt_byte_60,
1953                                 MPT_BYTE_60_PA5_H_M,
1954                                 MPT_BYTE_60_PA5_H_S,
1955                                 (u32)(pages[i] >> PAGES_SHIFT_24));
1956                         break;
1957                 case 6:
1958                         roce_set_field(mpt_entry->mpt_byte_60,
1959                                        MPT_BYTE_60_PA6_L_M,
1960                                        MPT_BYTE_60_PA6_L_S, (u32)(pages[i]));
1961                         roce_set_field(mpt_entry->mpt_byte_64,
1962                                 MPT_BYTE_64_PA6_H_M,
1963                                 MPT_BYTE_64_PA6_H_S,
1964                                 (u32)(pages[i] >> PAGES_SHIFT_16));
1965                         break;
1966                 default:
1967                         break;
1968                 }
1969         }
1970
1971         free_page((unsigned long) pages);
1972
1973         mpt_entry->pbl_addr_l = cpu_to_le32((u32)(mr->pbl_dma_addr));
1974
1975         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1976                        MPT_BYTE_12_PBL_ADDR_H_S,
1977                        ((u32)(mr->pbl_dma_addr >> 32)));
1978
1979         return 0;
1980 }
1981
1982 static void *get_cqe(struct hns_roce_cq *hr_cq, int n)
1983 {
1984         return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1985                                    n * HNS_ROCE_V1_CQE_ENTRY_SIZE);
1986 }
1987
1988 static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n)
1989 {
1990         struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n & hr_cq->ib_cq.cqe);
1991
1992         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1993         return (roce_get_bit(hr_cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S) ^
1994                 !!(n & (hr_cq->ib_cq.cqe + 1))) ? hr_cqe : NULL;
1995 }
1996
1997 static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq)
1998 {
1999         return get_sw_cqe(hr_cq, hr_cq->cons_index);
2000 }
2001
2002 static void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2003 {
2004         __le32 doorbell[2];
2005
2006         doorbell[0] = cpu_to_le32(cons_index & ((hr_cq->cq_depth << 1) - 1));
2007         doorbell[1] = 0;
2008         roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
2009         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
2010                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
2011         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
2012                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0);
2013         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
2014                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn);
2015
2016         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
2017 }
2018
2019 static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2020                                    struct hns_roce_srq *srq)
2021 {
2022         struct hns_roce_cqe *cqe, *dest;
2023         u32 prod_index;
2024         int nfreed = 0;
2025         u8 owner_bit;
2026
2027         for (prod_index = hr_cq->cons_index; get_sw_cqe(hr_cq, prod_index);
2028              ++prod_index) {
2029                 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
2030                         break;
2031         }
2032
2033         /*
2034          * Now backwards through the CQ, removing CQ entries
2035          * that match our QP by overwriting them with next entries.
2036          */
2037         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
2038                 cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe);
2039                 if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2040                                      CQE_BYTE_16_LOCAL_QPN_S) &
2041                                      HNS_ROCE_CQE_QPN_MASK) == qpn) {
2042                         /* In v1 engine, not support SRQ */
2043                         ++nfreed;
2044                 } else if (nfreed) {
2045                         dest = get_cqe(hr_cq, (prod_index + nfreed) &
2046                                        hr_cq->ib_cq.cqe);
2047                         owner_bit = roce_get_bit(dest->cqe_byte_4,
2048                                                  CQE_BYTE_4_OWNER_S);
2049                         memcpy(dest, cqe, sizeof(*cqe));
2050                         roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S,
2051                                      owner_bit);
2052                 }
2053         }
2054
2055         if (nfreed) {
2056                 hr_cq->cons_index += nfreed;
2057                 /*
2058                  * Make sure update of buffer contents is done before
2059                  * updating consumer index.
2060                  */
2061                 wmb();
2062
2063                 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
2064         }
2065 }
2066
2067 static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2068                                  struct hns_roce_srq *srq)
2069 {
2070         spin_lock_irq(&hr_cq->lock);
2071         __hns_roce_v1_cq_clean(hr_cq, qpn, srq);
2072         spin_unlock_irq(&hr_cq->lock);
2073 }
2074
2075 static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
2076                                   struct hns_roce_cq *hr_cq, void *mb_buf,
2077                                   u64 *mtts, dma_addr_t dma_handle, int nent,
2078                                   u32 vector)
2079 {
2080         struct hns_roce_cq_context *cq_context = NULL;
2081         struct hns_roce_buf_list *tptr_buf;
2082         struct hns_roce_v1_priv *priv;
2083         dma_addr_t tptr_dma_addr;
2084         int offset;
2085
2086         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
2087         tptr_buf = &priv->tptr_table.tptr_buf;
2088
2089         cq_context = mb_buf;
2090         memset(cq_context, 0, sizeof(*cq_context));
2091
2092         /* Get the tptr for this CQ. */
2093         offset = hr_cq->cqn * HNS_ROCE_V1_TPTR_ENTRY_SIZE;
2094         tptr_dma_addr = tptr_buf->map + offset;
2095         hr_cq->tptr_addr = (u16 *)(tptr_buf->buf + offset);
2096
2097         /* Register cq_context members */
2098         roce_set_field(cq_context->cqc_byte_4,
2099                        CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M,
2100                        CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID);
2101         roce_set_field(cq_context->cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M,
2102                        CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq->cqn);
2103
2104         cq_context->cq_bt_l = cpu_to_le32((u32)dma_handle);
2105
2106         roce_set_field(cq_context->cqc_byte_12,
2107                        CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M,
2108                        CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S,
2109                        ((u64)dma_handle >> 32));
2110         roce_set_field(cq_context->cqc_byte_12,
2111                        CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M,
2112                        CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S,
2113                        ilog2((unsigned int)nent));
2114         roce_set_field(cq_context->cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M,
2115                        CQ_CONTEXT_CQC_BYTE_12_CEQN_S, vector);
2116
2117         cq_context->cur_cqe_ba0_l = cpu_to_le32((u32)(mtts[0]));
2118
2119         roce_set_field(cq_context->cqc_byte_20,
2120                        CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M,
2121                        CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S, (mtts[0]) >> 32);
2122         /* Dedicated hardware, directly set 0 */
2123         roce_set_field(cq_context->cqc_byte_20,
2124                        CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M,
2125                        CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0);
2126         /**
2127          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
2128          * using 4K page, and shift more 32 because of
2129          * caculating the high 32 bit value evaluated to hardware.
2130          */
2131         roce_set_field(cq_context->cqc_byte_20,
2132                        CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M,
2133                        CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S,
2134                        tptr_dma_addr >> 44);
2135
2136         cq_context->cqe_tptr_addr_l = cpu_to_le32((u32)(tptr_dma_addr >> 12));
2137
2138         roce_set_field(cq_context->cqc_byte_32,
2139                        CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M,
2140                        CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0);
2141         roce_set_bit(cq_context->cqc_byte_32,
2142                      CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0);
2143         roce_set_bit(cq_context->cqc_byte_32,
2144                      CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0);
2145         roce_set_bit(cq_context->cqc_byte_32,
2146                      CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0);
2147         roce_set_bit(cq_context->cqc_byte_32,
2148                      CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S,
2149                      0);
2150         /* The initial value of cq's ci is 0 */
2151         roce_set_field(cq_context->cqc_byte_32,
2152                        CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M,
2153                        CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);
2154 }
2155
2156 static int hns_roce_v1_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
2157 {
2158         return -EOPNOTSUPP;
2159 }
2160
2161 static int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq,
2162                                      enum ib_cq_notify_flags flags)
2163 {
2164         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2165         u32 notification_flag;
2166         __le32 doorbell[2];
2167
2168         notification_flag = (flags & IB_CQ_SOLICITED_MASK) ==
2169                             IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;
2170         /*
2171          * flags = 0; Notification Flag = 1, next
2172          * flags = 1; Notification Flag = 0, solocited
2173          */
2174         doorbell[0] =
2175                 cpu_to_le32(hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
2176         roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
2177         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
2178                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
2179         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
2180                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1);
2181         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
2182                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S,
2183                        hr_cq->cqn | notification_flag);
2184
2185         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
2186
2187         return 0;
2188 }
2189
2190 static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
2191                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
2192 {
2193         int qpn;
2194         int is_send;
2195         u16 wqe_ctr;
2196         u32 status;
2197         u32 opcode;
2198         struct hns_roce_cqe *cqe;
2199         struct hns_roce_qp *hr_qp;
2200         struct hns_roce_wq *wq;
2201         struct hns_roce_wqe_ctrl_seg *sq_wqe;
2202         struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2203         struct device *dev = &hr_dev->pdev->dev;
2204
2205         /* Find cqe according consumer index */
2206         cqe = next_cqe_sw(hr_cq);
2207         if (!cqe)
2208                 return -EAGAIN;
2209
2210         ++hr_cq->cons_index;
2211         /* Memory barrier */
2212         rmb();
2213         /* 0->SQ, 1->RQ */
2214         is_send  = !(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S));
2215
2216         /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
2217         if (roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2218                            CQE_BYTE_16_LOCAL_QPN_S) <= 1) {
2219                 qpn = roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_PORT_NUM_M,
2220                                      CQE_BYTE_20_PORT_NUM_S) +
2221                       roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2222                                      CQE_BYTE_16_LOCAL_QPN_S) *
2223                                      HNS_ROCE_MAX_PORTS;
2224         } else {
2225                 qpn = roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
2226                                      CQE_BYTE_16_LOCAL_QPN_S);
2227         }
2228
2229         if (!*cur_qp || (qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->qpn) {
2230                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2231                 if (unlikely(!hr_qp)) {
2232                         dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n",
2233                                 hr_cq->cqn, (qpn & HNS_ROCE_CQE_QPN_MASK));
2234                         return -EINVAL;
2235                 }
2236
2237                 *cur_qp = hr_qp;
2238         }
2239
2240         wc->qp = &(*cur_qp)->ibqp;
2241         wc->vendor_err = 0;
2242
2243         status = roce_get_field(cqe->cqe_byte_4,
2244                                 CQE_BYTE_4_STATUS_OF_THE_OPERATION_M,
2245                                 CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) &
2246                                 HNS_ROCE_CQE_STATUS_MASK;
2247         switch (status) {
2248         case HNS_ROCE_CQE_SUCCESS:
2249                 wc->status = IB_WC_SUCCESS;
2250                 break;
2251         case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR:
2252                 wc->status = IB_WC_LOC_LEN_ERR;
2253                 break;
2254         case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR:
2255                 wc->status = IB_WC_LOC_QP_OP_ERR;
2256                 break;
2257         case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR:
2258                 wc->status = IB_WC_LOC_PROT_ERR;
2259                 break;
2260         case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR:
2261                 wc->status = IB_WC_WR_FLUSH_ERR;
2262                 break;
2263         case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR:
2264                 wc->status = IB_WC_MW_BIND_ERR;
2265                 break;
2266         case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR:
2267                 wc->status = IB_WC_BAD_RESP_ERR;
2268                 break;
2269         case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR:
2270                 wc->status = IB_WC_LOC_ACCESS_ERR;
2271                 break;
2272         case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
2273                 wc->status = IB_WC_REM_INV_REQ_ERR;
2274                 break;
2275         case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR:
2276                 wc->status = IB_WC_REM_ACCESS_ERR;
2277                 break;
2278         case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR:
2279                 wc->status = IB_WC_REM_OP_ERR;
2280                 break;
2281         case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
2282                 wc->status = IB_WC_RETRY_EXC_ERR;
2283                 break;
2284         case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
2285                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2286                 break;
2287         default:
2288                 wc->status = IB_WC_GENERAL_ERR;
2289                 break;
2290         }
2291
2292         /* CQE status error, directly return */
2293         if (wc->status != IB_WC_SUCCESS)
2294                 return 0;
2295
2296         if (is_send) {
2297                 /* SQ conrespond to CQE */
2298                 sq_wqe = get_send_wqe(*cur_qp, roce_get_field(cqe->cqe_byte_4,
2299                                                 CQE_BYTE_4_WQE_INDEX_M,
2300                                                 CQE_BYTE_4_WQE_INDEX_S)&
2301                                                 ((*cur_qp)->sq.wqe_cnt-1));
2302                 switch (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_OPCODE_MASK) {
2303                 case HNS_ROCE_WQE_OPCODE_SEND:
2304                         wc->opcode = IB_WC_SEND;
2305                         break;
2306                 case HNS_ROCE_WQE_OPCODE_RDMA_READ:
2307                         wc->opcode = IB_WC_RDMA_READ;
2308                         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2309                         break;
2310                 case HNS_ROCE_WQE_OPCODE_RDMA_WRITE:
2311                         wc->opcode = IB_WC_RDMA_WRITE;
2312                         break;
2313                 case HNS_ROCE_WQE_OPCODE_LOCAL_INV:
2314                         wc->opcode = IB_WC_LOCAL_INV;
2315                         break;
2316                 case HNS_ROCE_WQE_OPCODE_UD_SEND:
2317                         wc->opcode = IB_WC_SEND;
2318                         break;
2319                 default:
2320                         wc->status = IB_WC_GENERAL_ERR;
2321                         break;
2322                 }
2323                 wc->wc_flags = (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_IMM ?
2324                                 IB_WC_WITH_IMM : 0);
2325
2326                 wq = &(*cur_qp)->sq;
2327                 if ((*cur_qp)->sq_signal_bits) {
2328                         /*
2329                          * If sg_signal_bit is 1,
2330                          * firstly tail pointer updated to wqe
2331                          * which current cqe correspond to
2332                          */
2333                         wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4,
2334                                                       CQE_BYTE_4_WQE_INDEX_M,
2335                                                       CQE_BYTE_4_WQE_INDEX_S);
2336                         wq->tail += (wqe_ctr - (u16)wq->tail) &
2337                                     (wq->wqe_cnt - 1);
2338                 }
2339                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2340                 ++wq->tail;
2341         } else {
2342                 /* RQ conrespond to CQE */
2343                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2344                 opcode = roce_get_field(cqe->cqe_byte_4,
2345                                         CQE_BYTE_4_OPERATION_TYPE_M,
2346                                         CQE_BYTE_4_OPERATION_TYPE_S) &
2347                                         HNS_ROCE_CQE_OPCODE_MASK;
2348                 switch (opcode) {
2349                 case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE:
2350                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2351                         wc->wc_flags = IB_WC_WITH_IMM;
2352                         wc->ex.imm_data =
2353                                 cpu_to_be32(le32_to_cpu(cqe->immediate_data));
2354                         break;
2355                 case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE:
2356                         if (roce_get_bit(cqe->cqe_byte_4,
2357                                          CQE_BYTE_4_IMM_INDICATOR_S)) {
2358                                 wc->opcode = IB_WC_RECV;
2359                                 wc->wc_flags = IB_WC_WITH_IMM;
2360                                 wc->ex.imm_data = cpu_to_be32(
2361                                         le32_to_cpu(cqe->immediate_data));
2362                         } else {
2363                                 wc->opcode = IB_WC_RECV;
2364                                 wc->wc_flags = 0;
2365                         }
2366                         break;
2367                 default:
2368                         wc->status = IB_WC_GENERAL_ERR;
2369                         break;
2370                 }
2371
2372                 /* Update tail pointer, record wr_id */
2373                 wq = &(*cur_qp)->rq;
2374                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2375                 ++wq->tail;
2376                 wc->sl = (u8)roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_SL_M,
2377                                             CQE_BYTE_20_SL_S);
2378                 wc->src_qp = (u8)roce_get_field(cqe->cqe_byte_20,
2379                                                 CQE_BYTE_20_REMOTE_QPN_M,
2380                                                 CQE_BYTE_20_REMOTE_QPN_S);
2381                 wc->wc_flags |= (roce_get_bit(cqe->cqe_byte_20,
2382                                               CQE_BYTE_20_GRH_PRESENT_S) ?
2383                                               IB_WC_GRH : 0);
2384                 wc->pkey_index = (u16)roce_get_field(cqe->cqe_byte_28,
2385                                                      CQE_BYTE_28_P_KEY_IDX_M,
2386                                                      CQE_BYTE_28_P_KEY_IDX_S);
2387         }
2388
2389         return 0;
2390 }
2391
2392 int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2393 {
2394         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2395         struct hns_roce_qp *cur_qp = NULL;
2396         unsigned long flags;
2397         int npolled;
2398         int ret = 0;
2399
2400         spin_lock_irqsave(&hr_cq->lock, flags);
2401
2402         for (npolled = 0; npolled < num_entries; ++npolled) {
2403                 ret = hns_roce_v1_poll_one(hr_cq, &cur_qp, wc + npolled);
2404                 if (ret)
2405                         break;
2406         }
2407
2408         if (npolled) {
2409                 *hr_cq->tptr_addr = hr_cq->cons_index &
2410                         ((hr_cq->cq_depth << 1) - 1);
2411
2412                 /* Memroy barrier */
2413                 wmb();
2414                 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
2415         }
2416
2417         spin_unlock_irqrestore(&hr_cq->lock, flags);
2418
2419         if (ret == 0 || ret == -EAGAIN)
2420                 return npolled;
2421         else
2422                 return ret;
2423 }
2424
2425 static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,
2426                                  struct hns_roce_hem_table *table, int obj,
2427                                  int step_idx)
2428 {
2429         struct device *dev = &hr_dev->pdev->dev;
2430         struct hns_roce_v1_priv *priv;
2431         unsigned long end = 0, flags = 0;
2432         __le32 bt_cmd_val[2] = {0};
2433         void __iomem *bt_cmd;
2434         u64 bt_ba = 0;
2435
2436         priv = (struct hns_roce_v1_priv *)hr_dev->priv;
2437
2438         switch (table->type) {
2439         case HEM_TYPE_QPC:
2440                 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
2441                         ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
2442                 bt_ba = priv->bt_table.qpc_buf.map >> 12;
2443                 break;
2444         case HEM_TYPE_MTPT:
2445                 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
2446                         ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_MTPT);
2447                 bt_ba = priv->bt_table.mtpt_buf.map >> 12;
2448                 break;
2449         case HEM_TYPE_CQC:
2450                 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
2451                         ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
2452                 bt_ba = priv->bt_table.cqc_buf.map >> 12;
2453                 break;
2454         case HEM_TYPE_SRQC:
2455                 dev_dbg(dev, "HEM_TYPE_SRQC not support.\n");
2456                 return -EINVAL;
2457         default:
2458                 return 0;
2459         }
2460         roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
2461                 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
2462         roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
2463         roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
2464
2465         spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags);
2466
2467         bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
2468
2469         end = HW_SYNC_TIMEOUT_MSECS;
2470         while (1) {
2471                 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
2472                         if (!end) {
2473                                 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
2474                                 spin_unlock_irqrestore(&hr_dev->bt_cmd_lock,
2475                                         flags);
2476                                 return -EBUSY;
2477                         }
2478                 } else {
2479                         break;
2480                 }
2481                 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
2482                 end -= HW_SYNC_SLEEP_TIME_INTERVAL;
2483         }
2484
2485         bt_cmd_val[0] = (__le32)bt_ba;
2486         roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
2487                 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, bt_ba >> 32);
2488         hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
2489
2490         spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags);
2491
2492         return 0;
2493 }
2494
2495 static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
2496                                  struct hns_roce_mtt *mtt,
2497                                  enum hns_roce_qp_state cur_state,
2498                                  enum hns_roce_qp_state new_state,
2499                                  struct hns_roce_qp_context *context,
2500                                  struct hns_roce_qp *hr_qp)
2501 {
2502         static const u16
2503         op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = {
2504                 [HNS_ROCE_QP_STATE_RST] = {
2505                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2506                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2507                 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
2508                 },
2509                 [HNS_ROCE_QP_STATE_INIT] = {
2510                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2511                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2512                 /* Note: In v1 engine, HW doesn't support RST2INIT.
2513                  * We use RST2INIT cmd instead of INIT2INIT.
2514                  */
2515                 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
2516                 [HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP,
2517                 },
2518                 [HNS_ROCE_QP_STATE_RTR] = {
2519                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2520                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2521                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP,
2522                 },
2523                 [HNS_ROCE_QP_STATE_RTS] = {
2524                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2525                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2526                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP,
2527                 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP,
2528                 },
2529                 [HNS_ROCE_QP_STATE_SQD] = {
2530                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2531                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2532                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP,
2533                 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP,
2534                 },
2535                 [HNS_ROCE_QP_STATE_ERR] = {
2536                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
2537                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
2538                 }
2539         };
2540
2541         struct hns_roce_cmd_mailbox *mailbox;
2542         struct device *dev = &hr_dev->pdev->dev;
2543         int ret = 0;
2544
2545         if (cur_state >= HNS_ROCE_QP_NUM_STATE ||
2546             new_state >= HNS_ROCE_QP_NUM_STATE ||
2547             !op[cur_state][new_state]) {
2548                 dev_err(dev, "[modify_qp]not support state %d to %d\n",
2549                         cur_state, new_state);
2550                 return -EINVAL;
2551         }
2552
2553         if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP)
2554                 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
2555                                          HNS_ROCE_CMD_2RST_QP,
2556                                          HNS_ROCE_CMD_TIMEOUT_MSECS);
2557
2558         if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP)
2559                 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
2560                                          HNS_ROCE_CMD_2ERR_QP,
2561                                          HNS_ROCE_CMD_TIMEOUT_MSECS);
2562
2563         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2564         if (IS_ERR(mailbox))
2565                 return PTR_ERR(mailbox);
2566
2567         memcpy(mailbox->buf, context, sizeof(*context));
2568
2569         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
2570                                 op[cur_state][new_state],
2571                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2572
2573         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2574         return ret;
2575 }
2576
2577 static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
2578                              int attr_mask, enum ib_qp_state cur_state,
2579                              enum ib_qp_state new_state)
2580 {
2581         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2582         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2583         struct hns_roce_sqp_context *context;
2584         struct device *dev = &hr_dev->pdev->dev;
2585         dma_addr_t dma_handle = 0;
2586         u32 __iomem *addr;
2587         int rq_pa_start;
2588         __le32 tmp;
2589         u32 reg_val;
2590         u64 *mtts;
2591
2592         context = kzalloc(sizeof(*context), GFP_KERNEL);
2593         if (!context)
2594                 return -ENOMEM;
2595
2596         /* Search QP buf's MTTs */
2597         mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
2598                                    hr_qp->mtt.first_seg, &dma_handle);
2599         if (!mtts) {
2600                 dev_err(dev, "qp buf pa find failed\n");
2601                 goto out;
2602         }
2603
2604         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2605                 roce_set_field(context->qp1c_bytes_4,
2606                                QP1C_BYTES_4_SQ_WQE_SHIFT_M,
2607                                QP1C_BYTES_4_SQ_WQE_SHIFT_S,
2608                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2609                 roce_set_field(context->qp1c_bytes_4,
2610                                QP1C_BYTES_4_RQ_WQE_SHIFT_M,
2611                                QP1C_BYTES_4_RQ_WQE_SHIFT_S,
2612                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2613                 roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M,
2614                                QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn);
2615
2616                 context->sq_rq_bt_l = cpu_to_le32((u32)(dma_handle));
2617                 roce_set_field(context->qp1c_bytes_12,
2618                                QP1C_BYTES_12_SQ_RQ_BT_H_M,
2619                                QP1C_BYTES_12_SQ_RQ_BT_H_S,
2620                                ((u32)(dma_handle >> 32)));
2621
2622                 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M,
2623                                QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head);
2624                 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M,
2625                                QP1C_BYTES_16_PORT_NUM_S, hr_qp->phy_port);
2626                 roce_set_bit(context->qp1c_bytes_16,
2627                              QP1C_BYTES_16_SIGNALING_TYPE_S,
2628                              le32_to_cpu(hr_qp->sq_signal_bits));
2629                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S,
2630                              1);
2631                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S,
2632                              1);
2633                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S,
2634                              0);
2635
2636                 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M,
2637                                QP1C_BYTES_20_SQ_HEAD_S, hr_qp->sq.head);
2638                 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M,
2639                                QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index);
2640
2641                 rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
2642                 context->cur_rq_wqe_ba_l =
2643                                 cpu_to_le32((u32)(mtts[rq_pa_start]));
2644
2645                 roce_set_field(context->qp1c_bytes_28,
2646                                QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M,
2647                                QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S,
2648                                (mtts[rq_pa_start]) >> 32);
2649                 roce_set_field(context->qp1c_bytes_28,
2650                                QP1C_BYTES_28_RQ_CUR_IDX_M,
2651                                QP1C_BYTES_28_RQ_CUR_IDX_S, 0);
2652
2653                 roce_set_field(context->qp1c_bytes_32,
2654                                QP1C_BYTES_32_RX_CQ_NUM_M,
2655                                QP1C_BYTES_32_RX_CQ_NUM_S,
2656                                to_hr_cq(ibqp->recv_cq)->cqn);
2657                 roce_set_field(context->qp1c_bytes_32,
2658                                QP1C_BYTES_32_TX_CQ_NUM_M,
2659                                QP1C_BYTES_32_TX_CQ_NUM_S,
2660                                to_hr_cq(ibqp->send_cq)->cqn);
2661
2662                 context->cur_sq_wqe_ba_l  = cpu_to_le32((u32)mtts[0]);
2663
2664                 roce_set_field(context->qp1c_bytes_40,
2665                                QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M,
2666                                QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S,
2667                                (mtts[0]) >> 32);
2668                 roce_set_field(context->qp1c_bytes_40,
2669                                QP1C_BYTES_40_SQ_CUR_IDX_M,
2670                                QP1C_BYTES_40_SQ_CUR_IDX_S, 0);
2671
2672                 /* Copy context to QP1C register */
2673                 addr = (u32 __iomem *)(hr_dev->reg_base +
2674                                        ROCEE_QP1C_CFG0_0_REG +
2675                                        hr_qp->phy_port * sizeof(*context));
2676
2677                 writel(le32_to_cpu(context->qp1c_bytes_4), addr);
2678                 writel(le32_to_cpu(context->sq_rq_bt_l), addr + 1);
2679                 writel(le32_to_cpu(context->qp1c_bytes_12), addr + 2);
2680                 writel(le32_to_cpu(context->qp1c_bytes_16), addr + 3);
2681                 writel(le32_to_cpu(context->qp1c_bytes_20), addr + 4);
2682                 writel(le32_to_cpu(context->cur_rq_wqe_ba_l), addr + 5);
2683                 writel(le32_to_cpu(context->qp1c_bytes_28), addr + 6);
2684                 writel(le32_to_cpu(context->qp1c_bytes_32), addr + 7);
2685                 writel(le32_to_cpu(context->cur_sq_wqe_ba_l), addr + 8);
2686                 writel(le32_to_cpu(context->qp1c_bytes_40), addr + 9);
2687         }
2688
2689         /* Modify QP1C status */
2690         reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG +
2691                             hr_qp->phy_port * sizeof(*context));
2692         tmp = cpu_to_le32(reg_val);
2693         roce_set_field(tmp, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M,
2694                        ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state);
2695         reg_val = le32_to_cpu(tmp);
2696         roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG +
2697                     hr_qp->phy_port * sizeof(*context), reg_val);
2698
2699         hr_qp->state = new_state;
2700         if (new_state == IB_QPS_RESET) {
2701                 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
2702                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
2703                 if (ibqp->send_cq != ibqp->recv_cq)
2704                         hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
2705                                              hr_qp->qpn, NULL);
2706
2707                 hr_qp->rq.head = 0;
2708                 hr_qp->rq.tail = 0;
2709                 hr_qp->sq.head = 0;
2710                 hr_qp->sq.tail = 0;
2711                 hr_qp->sq_next_wqe = 0;
2712         }
2713
2714         kfree(context);
2715         return 0;
2716
2717 out:
2718         kfree(context);
2719         return -EINVAL;
2720 }
2721
2722 static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
2723                             int attr_mask, enum ib_qp_state cur_state,
2724                             enum ib_qp_state new_state)
2725 {
2726         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2727         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2728         struct device *dev = &hr_dev->pdev->dev;
2729         struct hns_roce_qp_context *context;
2730         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2731         dma_addr_t dma_handle_2 = 0;
2732         dma_addr_t dma_handle = 0;
2733         __le32 doorbell[2] = {0};
2734         int rq_pa_start = 0;
2735         u64 *mtts_2 = NULL;
2736         int ret = -EINVAL;
2737         u64 *mtts = NULL;
2738         int port;
2739         u8 port_num;
2740         u8 *dmac;
2741         u8 *smac;
2742
2743         context = kzalloc(sizeof(*context), GFP_KERNEL);
2744         if (!context)
2745                 return -ENOMEM;
2746
2747         /* Search qp buf's mtts */
2748         mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
2749                                    hr_qp->mtt.first_seg, &dma_handle);
2750         if (mtts == NULL) {
2751                 dev_err(dev, "qp buf pa find failed\n");
2752                 goto out;
2753         }
2754
2755         /* Search IRRL's mtts */
2756         mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
2757                                      hr_qp->qpn, &dma_handle_2);
2758         if (mtts_2 == NULL) {
2759                 dev_err(dev, "qp irrl_table find failed\n");
2760                 goto out;
2761         }
2762
2763         /*
2764          * Reset to init
2765          *      Mandatory param:
2766          *      IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
2767          *      Optional param: NA
2768          */
2769         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2770                 roce_set_field(context->qpc_bytes_4,
2771                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2772                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2773                                to_hr_qp_type(hr_qp->ibqp.qp_type));
2774
2775                 roce_set_bit(context->qpc_bytes_4,
2776                              QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2777                 roce_set_bit(context->qpc_bytes_4,
2778                              QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2779                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
2780                 roce_set_bit(context->qpc_bytes_4,
2781                              QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2782                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
2783                              );
2784                 roce_set_bit(context->qpc_bytes_4,
2785                              QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S,
2786                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)
2787                              );
2788                 roce_set_bit(context->qpc_bytes_4,
2789                              QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2790                 roce_set_field(context->qpc_bytes_4,
2791                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2792                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2793                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2794                 roce_set_field(context->qpc_bytes_4,
2795                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
2796                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
2797                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2798                 roce_set_field(context->qpc_bytes_4,
2799                                QP_CONTEXT_QPC_BYTES_4_PD_M,
2800                                QP_CONTEXT_QPC_BYTES_4_PD_S,
2801                                to_hr_pd(ibqp->pd)->pdn);
2802                 hr_qp->access_flags = attr->qp_access_flags;
2803                 roce_set_field(context->qpc_bytes_8,
2804                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
2805                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
2806                                to_hr_cq(ibqp->send_cq)->cqn);
2807                 roce_set_field(context->qpc_bytes_8,
2808                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
2809                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
2810                                to_hr_cq(ibqp->recv_cq)->cqn);
2811
2812                 if (ibqp->srq)
2813                         roce_set_field(context->qpc_bytes_12,
2814                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
2815                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
2816                                        to_hr_srq(ibqp->srq)->srqn);
2817
2818                 roce_set_field(context->qpc_bytes_12,
2819                                QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2820                                QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2821                                attr->pkey_index);
2822                 hr_qp->pkey_index = attr->pkey_index;
2823                 roce_set_field(context->qpc_bytes_16,
2824                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2825                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2826
2827         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
2828                 roce_set_field(context->qpc_bytes_4,
2829                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2830                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2831                                to_hr_qp_type(hr_qp->ibqp.qp_type));
2832                 roce_set_bit(context->qpc_bytes_4,
2833                              QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2834                 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2835                         roce_set_bit(context->qpc_bytes_4,
2836                                      QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2837                                      !!(attr->qp_access_flags &
2838                                      IB_ACCESS_REMOTE_READ));
2839                         roce_set_bit(context->qpc_bytes_4,
2840                                      QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2841                                      !!(attr->qp_access_flags &
2842                                      IB_ACCESS_REMOTE_WRITE));
2843                 } else {
2844                         roce_set_bit(context->qpc_bytes_4,
2845                                      QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2846                                      !!(hr_qp->access_flags &
2847                                      IB_ACCESS_REMOTE_READ));
2848                         roce_set_bit(context->qpc_bytes_4,
2849                                      QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2850                                      !!(hr_qp->access_flags &
2851                                      IB_ACCESS_REMOTE_WRITE));
2852                 }
2853
2854                 roce_set_bit(context->qpc_bytes_4,
2855                              QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2856                 roce_set_field(context->qpc_bytes_4,
2857                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2858                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2859               &n