11a8d638bcd0ba1b8c8f4e0c7b9961fb604eb39f
[sfrench/cifs-2.6.git] / drivers / net / ethernet / mellanox / mlx5 / core / eq.c
1 /*
2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32
33 #include <linux/interrupt.h>
34 #include <linux/module.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/cmd.h>
37 #include "mlx5_core.h"
38 #ifdef CONFIG_MLX5_CORE_EN
39 #include "eswitch.h"
40 #endif
41
42 enum {
43         MLX5_EQE_SIZE           = sizeof(struct mlx5_eqe),
44         MLX5_EQE_OWNER_INIT_VAL = 0x1,
45 };
46
47 enum {
48         MLX5_EQ_STATE_ARMED             = 0x9,
49         MLX5_EQ_STATE_FIRED             = 0xa,
50         MLX5_EQ_STATE_ALWAYS_ARMED      = 0xb,
51 };
52
53 enum {
54         MLX5_NUM_SPARE_EQE      = 0x80,
55         MLX5_NUM_ASYNC_EQE      = 0x100,
56         MLX5_NUM_CMD_EQE        = 32,
57         MLX5_NUM_PF_DRAIN       = 64,
58 };
59
60 enum {
61         MLX5_EQ_DOORBEL_OFFSET  = 0x40,
62 };
63
64 #define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG)           | \
65                                (1ull << MLX5_EVENT_TYPE_COMM_EST)           | \
66                                (1ull << MLX5_EVENT_TYPE_SQ_DRAINED)         | \
67                                (1ull << MLX5_EVENT_TYPE_CQ_ERROR)           | \
68                                (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR)     | \
69                                (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED)    | \
70                                (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
71                                (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR)    | \
72                                (1ull << MLX5_EVENT_TYPE_PORT_CHANGE)        | \
73                                (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR)    | \
74                                (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE)       | \
75                                (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))
76
77 struct map_eq_in {
78         u64     mask;
79         u32     reserved;
80         u32     unmap_eqn;
81 };
82
83 struct cre_des_eq {
84         u8      reserved[15];
85         u8      eqn;
86 };
87
88 static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
89 {
90         u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {0};
91         u32 in[MLX5_ST_SZ_DW(destroy_eq_in)]   = {0};
92
93         MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
94         MLX5_SET(destroy_eq_in, in, eq_number, eqn);
95         return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
96 }
97
98 static struct mlx5_eqe *get_eqe(struct mlx5_eq *eq, u32 entry)
99 {
100         return mlx5_buf_offset(&eq->buf, entry * MLX5_EQE_SIZE);
101 }
102
103 static struct mlx5_eqe *next_eqe_sw(struct mlx5_eq *eq)
104 {
105         struct mlx5_eqe *eqe = get_eqe(eq, eq->cons_index & (eq->nent - 1));
106
107         return ((eqe->owner & 1) ^ !!(eq->cons_index & eq->nent)) ? NULL : eqe;
108 }
109
110 static const char *eqe_type_str(u8 type)
111 {
112         switch (type) {
113         case MLX5_EVENT_TYPE_COMP:
114                 return "MLX5_EVENT_TYPE_COMP";
115         case MLX5_EVENT_TYPE_PATH_MIG:
116                 return "MLX5_EVENT_TYPE_PATH_MIG";
117         case MLX5_EVENT_TYPE_COMM_EST:
118                 return "MLX5_EVENT_TYPE_COMM_EST";
119         case MLX5_EVENT_TYPE_SQ_DRAINED:
120                 return "MLX5_EVENT_TYPE_SQ_DRAINED";
121         case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
122                 return "MLX5_EVENT_TYPE_SRQ_LAST_WQE";
123         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
124                 return "MLX5_EVENT_TYPE_SRQ_RQ_LIMIT";
125         case MLX5_EVENT_TYPE_CQ_ERROR:
126                 return "MLX5_EVENT_TYPE_CQ_ERROR";
127         case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
128                 return "MLX5_EVENT_TYPE_WQ_CATAS_ERROR";
129         case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
130                 return "MLX5_EVENT_TYPE_PATH_MIG_FAILED";
131         case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
132                 return "MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR";
133         case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
134                 return "MLX5_EVENT_TYPE_WQ_ACCESS_ERROR";
135         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
136                 return "MLX5_EVENT_TYPE_SRQ_CATAS_ERROR";
137         case MLX5_EVENT_TYPE_INTERNAL_ERROR:
138                 return "MLX5_EVENT_TYPE_INTERNAL_ERROR";
139         case MLX5_EVENT_TYPE_PORT_CHANGE:
140                 return "MLX5_EVENT_TYPE_PORT_CHANGE";
141         case MLX5_EVENT_TYPE_GPIO_EVENT:
142                 return "MLX5_EVENT_TYPE_GPIO_EVENT";
143         case MLX5_EVENT_TYPE_PORT_MODULE_EVENT:
144                 return "MLX5_EVENT_TYPE_PORT_MODULE_EVENT";
145         case MLX5_EVENT_TYPE_REMOTE_CONFIG:
146                 return "MLX5_EVENT_TYPE_REMOTE_CONFIG";
147         case MLX5_EVENT_TYPE_DB_BF_CONGESTION:
148                 return "MLX5_EVENT_TYPE_DB_BF_CONGESTION";
149         case MLX5_EVENT_TYPE_STALL_EVENT:
150                 return "MLX5_EVENT_TYPE_STALL_EVENT";
151         case MLX5_EVENT_TYPE_CMD:
152                 return "MLX5_EVENT_TYPE_CMD";
153         case MLX5_EVENT_TYPE_PAGE_REQUEST:
154                 return "MLX5_EVENT_TYPE_PAGE_REQUEST";
155         case MLX5_EVENT_TYPE_PAGE_FAULT:
156                 return "MLX5_EVENT_TYPE_PAGE_FAULT";
157         default:
158                 return "Unrecognized event";
159         }
160 }
161
162 static enum mlx5_dev_event port_subtype_event(u8 subtype)
163 {
164         switch (subtype) {
165         case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
166                 return MLX5_DEV_EVENT_PORT_DOWN;
167         case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
168                 return MLX5_DEV_EVENT_PORT_UP;
169         case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
170                 return MLX5_DEV_EVENT_PORT_INITIALIZED;
171         case MLX5_PORT_CHANGE_SUBTYPE_LID:
172                 return MLX5_DEV_EVENT_LID_CHANGE;
173         case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
174                 return MLX5_DEV_EVENT_PKEY_CHANGE;
175         case MLX5_PORT_CHANGE_SUBTYPE_GUID:
176                 return MLX5_DEV_EVENT_GUID_CHANGE;
177         case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
178                 return MLX5_DEV_EVENT_CLIENT_REREG;
179         }
180         return -1;
181 }
182
183 static void eq_update_ci(struct mlx5_eq *eq, int arm)
184 {
185         __be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2);
186         u32 val = (eq->cons_index & 0xffffff) | (eq->eqn << 24);
187         __raw_writel((__force u32) cpu_to_be32(val), addr);
188         /* We still want ordering, just not swabbing, so add a barrier */
189         mb();
190 }
191
192 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
193 static void eqe_pf_action(struct work_struct *work)
194 {
195         struct mlx5_pagefault *pfault = container_of(work,
196                                                      struct mlx5_pagefault,
197                                                      work);
198         struct mlx5_eq *eq = pfault->eq;
199
200         mlx5_core_page_fault(eq->dev, pfault);
201         mempool_free(pfault, eq->pf_ctx.pool);
202 }
203
204 static void eq_pf_process(struct mlx5_eq *eq)
205 {
206         struct mlx5_core_dev *dev = eq->dev;
207         struct mlx5_eqe_page_fault *pf_eqe;
208         struct mlx5_pagefault *pfault;
209         struct mlx5_eqe *eqe;
210         int set_ci = 0;
211
212         while ((eqe = next_eqe_sw(eq))) {
213                 pfault = mempool_alloc(eq->pf_ctx.pool, GFP_ATOMIC);
214                 if (!pfault) {
215                         schedule_work(&eq->pf_ctx.work);
216                         break;
217                 }
218
219                 dma_rmb();
220                 pf_eqe = &eqe->data.page_fault;
221                 pfault->event_subtype = eqe->sub_type;
222                 pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed);
223
224                 mlx5_core_dbg(dev,
225                               "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n",
226                               eqe->sub_type, pfault->bytes_committed);
227
228                 switch (eqe->sub_type) {
229                 case MLX5_PFAULT_SUBTYPE_RDMA:
230                         /* RDMA based event */
231                         pfault->type =
232                                 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24;
233                         pfault->token =
234                                 be32_to_cpu(pf_eqe->rdma.pftype_token) &
235                                 MLX5_24BIT_MASK;
236                         pfault->rdma.r_key =
237                                 be32_to_cpu(pf_eqe->rdma.r_key);
238                         pfault->rdma.packet_size =
239                                 be16_to_cpu(pf_eqe->rdma.packet_length);
240                         pfault->rdma.rdma_op_len =
241                                 be32_to_cpu(pf_eqe->rdma.rdma_op_len);
242                         pfault->rdma.rdma_va =
243                                 be64_to_cpu(pf_eqe->rdma.rdma_va);
244                         mlx5_core_dbg(dev,
245                                       "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n",
246                                       pfault->type, pfault->token,
247                                       pfault->rdma.r_key);
248                         mlx5_core_dbg(dev,
249                                       "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n",
250                                       pfault->rdma.rdma_op_len,
251                                       pfault->rdma.rdma_va);
252                         break;
253
254                 case MLX5_PFAULT_SUBTYPE_WQE:
255                         /* WQE based event */
256                         pfault->type =
257                                 be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24;
258                         pfault->token =
259                                 be32_to_cpu(pf_eqe->wqe.token);
260                         pfault->wqe.wq_num =
261                                 be32_to_cpu(pf_eqe->wqe.pftype_wq) &
262                                 MLX5_24BIT_MASK;
263                         pfault->wqe.wqe_index =
264                                 be16_to_cpu(pf_eqe->wqe.wqe_index);
265                         pfault->wqe.packet_size =
266                                 be16_to_cpu(pf_eqe->wqe.packet_length);
267                         mlx5_core_dbg(dev,
268                                       "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n",
269                                       pfault->type, pfault->token,
270                                       pfault->wqe.wq_num,
271                                       pfault->wqe.wqe_index);
272                         break;
273
274                 default:
275                         mlx5_core_warn(dev,
276                                        "Unsupported page fault event sub-type: 0x%02hhx\n",
277                                        eqe->sub_type);
278                         /* Unsupported page faults should still be
279                          * resolved by the page fault handler
280                          */
281                 }
282
283                 pfault->eq = eq;
284                 INIT_WORK(&pfault->work, eqe_pf_action);
285                 queue_work(eq->pf_ctx.wq, &pfault->work);
286
287                 ++eq->cons_index;
288                 ++set_ci;
289
290                 if (unlikely(set_ci >= MLX5_NUM_SPARE_EQE)) {
291                         eq_update_ci(eq, 0);
292                         set_ci = 0;
293                 }
294         }
295
296         eq_update_ci(eq, 1);
297 }
298
299 static irqreturn_t mlx5_eq_pf_int(int irq, void *eq_ptr)
300 {
301         struct mlx5_eq *eq = eq_ptr;
302         unsigned long flags;
303
304         if (spin_trylock_irqsave(&eq->pf_ctx.lock, flags)) {
305                 eq_pf_process(eq);
306                 spin_unlock_irqrestore(&eq->pf_ctx.lock, flags);
307         } else {
308                 schedule_work(&eq->pf_ctx.work);
309         }
310
311         return IRQ_HANDLED;
312 }
313
314 /* mempool_refill() was proposed but unfortunately wasn't accepted
315  * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html
316  * Chip workaround.
317  */
318 static void mempool_refill(mempool_t *pool)
319 {
320         while (pool->curr_nr < pool->min_nr)
321                 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool);
322 }
323
324 static void eq_pf_action(struct work_struct *work)
325 {
326         struct mlx5_eq *eq = container_of(work, struct mlx5_eq, pf_ctx.work);
327
328         mempool_refill(eq->pf_ctx.pool);
329
330         spin_lock_irq(&eq->pf_ctx.lock);
331         eq_pf_process(eq);
332         spin_unlock_irq(&eq->pf_ctx.lock);
333 }
334
335 static int init_pf_ctx(struct mlx5_eq_pagefault *pf_ctx, const char *name)
336 {
337         spin_lock_init(&pf_ctx->lock);
338         INIT_WORK(&pf_ctx->work, eq_pf_action);
339
340         pf_ctx->wq = alloc_ordered_workqueue(name,
341                                              WQ_MEM_RECLAIM);
342         if (!pf_ctx->wq)
343                 return -ENOMEM;
344
345         pf_ctx->pool = mempool_create_kmalloc_pool
346                 (MLX5_NUM_PF_DRAIN, sizeof(struct mlx5_pagefault));
347         if (!pf_ctx->pool)
348                 goto err_wq;
349
350         return 0;
351 err_wq:
352         destroy_workqueue(pf_ctx->wq);
353         return -ENOMEM;
354 }
355
356 int mlx5_core_page_fault_resume(struct mlx5_core_dev *dev, u32 token,
357                                 u32 wq_num, u8 type, int error)
358 {
359         u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = {0};
360         u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)]   = {0};
361
362         MLX5_SET(page_fault_resume_in, in, opcode,
363                  MLX5_CMD_OP_PAGE_FAULT_RESUME);
364         MLX5_SET(page_fault_resume_in, in, error, !!error);
365         MLX5_SET(page_fault_resume_in, in, page_fault_type, type);
366         MLX5_SET(page_fault_resume_in, in, wq_number, wq_num);
367         MLX5_SET(page_fault_resume_in, in, token, token);
368
369         return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
370 }
371 EXPORT_SYMBOL_GPL(mlx5_core_page_fault_resume);
372 #endif
373
374 static irqreturn_t mlx5_eq_int(int irq, void *eq_ptr)
375 {
376         struct mlx5_eq *eq = eq_ptr;
377         struct mlx5_core_dev *dev = eq->dev;
378         struct mlx5_eqe *eqe;
379         int set_ci = 0;
380         u32 cqn = -1;
381         u32 rsn;
382         u8 port;
383
384         while ((eqe = next_eqe_sw(eq))) {
385                 /*
386                  * Make sure we read EQ entry contents after we've
387                  * checked the ownership bit.
388                  */
389                 dma_rmb();
390
391                 mlx5_core_dbg(eq->dev, "eqn %d, eqe type %s\n",
392                               eq->eqn, eqe_type_str(eqe->type));
393                 switch (eqe->type) {
394                 case MLX5_EVENT_TYPE_COMP:
395                         cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff;
396                         mlx5_cq_completion(dev, cqn);
397                         break;
398
399                 case MLX5_EVENT_TYPE_PATH_MIG:
400                 case MLX5_EVENT_TYPE_COMM_EST:
401                 case MLX5_EVENT_TYPE_SQ_DRAINED:
402                 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
403                 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
404                 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
405                 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
406                 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
407                         rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
408                         rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
409                         mlx5_core_dbg(dev, "event %s(%d) arrived on resource 0x%x\n",
410                                       eqe_type_str(eqe->type), eqe->type, rsn);
411                         mlx5_rsc_event(dev, rsn, eqe->type);
412                         break;
413
414                 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
415                 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
416                         rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
417                         mlx5_core_dbg(dev, "SRQ event %s(%d): srqn 0x%x\n",
418                                       eqe_type_str(eqe->type), eqe->type, rsn);
419                         mlx5_srq_event(dev, rsn, eqe->type);
420                         break;
421
422                 case MLX5_EVENT_TYPE_CMD:
423                         mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector));
424                         break;
425
426                 case MLX5_EVENT_TYPE_PORT_CHANGE:
427                         port = (eqe->data.port.port >> 4) & 0xf;
428                         switch (eqe->sub_type) {
429                         case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
430                         case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
431                         case MLX5_PORT_CHANGE_SUBTYPE_LID:
432                         case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
433                         case MLX5_PORT_CHANGE_SUBTYPE_GUID:
434                         case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
435                         case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
436                                 if (dev->event)
437                                         dev->event(dev, port_subtype_event(eqe->sub_type),
438                                                    (unsigned long)port);
439                                 break;
440                         default:
441                                 mlx5_core_warn(dev, "Port event with unrecognized subtype: port %d, sub_type %d\n",
442                                                port, eqe->sub_type);
443                         }
444                         break;
445                 case MLX5_EVENT_TYPE_CQ_ERROR:
446                         cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
447                         mlx5_core_warn(dev, "CQ error on CQN 0x%x, syndrom 0x%x\n",
448                                        cqn, eqe->data.cq_err.syndrome);
449                         mlx5_cq_event(dev, cqn, eqe->type);
450                         break;
451
452                 case MLX5_EVENT_TYPE_PAGE_REQUEST:
453                         {
454                                 u16 func_id = be16_to_cpu(eqe->data.req_pages.func_id);
455                                 s32 npages = be32_to_cpu(eqe->data.req_pages.num_pages);
456
457                                 mlx5_core_dbg(dev, "page request for func 0x%x, npages %d\n",
458                                               func_id, npages);
459                                 mlx5_core_req_pages_handler(dev, func_id, npages);
460                         }
461                         break;
462
463 #ifdef CONFIG_MLX5_CORE_EN
464                 case MLX5_EVENT_TYPE_NIC_VPORT_CHANGE:
465                         mlx5_eswitch_vport_event(dev->priv.eswitch, eqe);
466                         break;
467 #endif
468
469                 case MLX5_EVENT_TYPE_PORT_MODULE_EVENT:
470                         mlx5_port_module_event(dev, eqe);
471                         break;
472
473                 default:
474                         mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n",
475                                        eqe->type, eq->eqn);
476                         break;
477                 }
478
479                 ++eq->cons_index;
480                 ++set_ci;
481
482                 /* The HCA will think the queue has overflowed if we
483                  * don't tell it we've been processing events.  We
484                  * create our EQs with MLX5_NUM_SPARE_EQE extra
485                  * entries, so we must update our consumer index at
486                  * least that often.
487                  */
488                 if (unlikely(set_ci >= MLX5_NUM_SPARE_EQE)) {
489                         eq_update_ci(eq, 0);
490                         set_ci = 0;
491                 }
492         }
493
494         eq_update_ci(eq, 1);
495
496         if (cqn != -1)
497                 tasklet_schedule(&eq->tasklet_ctx.task);
498
499         return IRQ_HANDLED;
500 }
501
502 static void init_eq_buf(struct mlx5_eq *eq)
503 {
504         struct mlx5_eqe *eqe;
505         int i;
506
507         for (i = 0; i < eq->nent; i++) {
508                 eqe = get_eqe(eq, i);
509                 eqe->owner = MLX5_EQE_OWNER_INIT_VAL;
510         }
511 }
512
513 int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
514                        int nent, u64 mask, const char *name,
515                        struct mlx5_uar *uar, enum mlx5_eq_type type)
516 {
517         u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
518         struct mlx5_priv *priv = &dev->priv;
519         irq_handler_t handler;
520         __be64 *pas;
521         void *eqc;
522         int inlen;
523         u32 *in;
524         int err;
525
526         eq->type = type;
527         eq->nent = roundup_pow_of_two(nent + MLX5_NUM_SPARE_EQE);
528         eq->cons_index = 0;
529         err = mlx5_buf_alloc(dev, eq->nent * MLX5_EQE_SIZE, &eq->buf);
530         if (err)
531                 return err;
532
533 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
534         if (type == MLX5_EQ_TYPE_PF)
535                 handler = mlx5_eq_pf_int;
536         else
537 #endif
538                 handler = mlx5_eq_int;
539
540         init_eq_buf(eq);
541
542         inlen = MLX5_ST_SZ_BYTES(create_eq_in) +
543                 MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->buf.npages;
544
545         in = mlx5_vzalloc(inlen);
546         if (!in) {
547                 err = -ENOMEM;
548                 goto err_buf;
549         }
550
551         pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas);
552         mlx5_fill_page_array(&eq->buf, pas);
553
554         MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
555         MLX5_SET64(create_eq_in, in, event_bitmask, mask);
556
557         eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
558         MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent));
559         MLX5_SET(eqc, eqc, uar_page, uar->index);
560         MLX5_SET(eqc, eqc, intr, vecidx);
561         MLX5_SET(eqc, eqc, log_page_size,
562                  eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
563
564         err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
565         if (err)
566                 goto err_in;
567
568         snprintf(priv->irq_info[vecidx].name, MLX5_MAX_IRQ_NAME, "%s@pci:%s",
569                  name, pci_name(dev->pdev));
570
571         eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
572         eq->irqn = priv->msix_arr[vecidx].vector;
573         eq->dev = dev;
574         eq->doorbell = uar->map + MLX5_EQ_DOORBEL_OFFSET;
575         err = request_irq(eq->irqn, handler, 0,
576                           priv->irq_info[vecidx].name, eq);
577         if (err)
578                 goto err_eq;
579
580         err = mlx5_debug_eq_add(dev, eq);
581         if (err)
582                 goto err_irq;
583
584 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
585         if (type == MLX5_EQ_TYPE_PF) {
586                 err = init_pf_ctx(&eq->pf_ctx, name);
587                 if (err)
588                         goto err_irq;
589         } else
590 #endif
591         {
592                 INIT_LIST_HEAD(&eq->tasklet_ctx.list);
593                 INIT_LIST_HEAD(&eq->tasklet_ctx.process_list);
594                 spin_lock_init(&eq->tasklet_ctx.lock);
595                 tasklet_init(&eq->tasklet_ctx.task, mlx5_cq_tasklet_cb,
596                              (unsigned long)&eq->tasklet_ctx);
597         }
598
599         /* EQs are created in ARMED state
600          */
601         eq_update_ci(eq, 1);
602
603         kvfree(in);
604         return 0;
605
606 err_irq:
607         free_irq(priv->msix_arr[vecidx].vector, eq);
608
609 err_eq:
610         mlx5_cmd_destroy_eq(dev, eq->eqn);
611
612 err_in:
613         kvfree(in);
614
615 err_buf:
616         mlx5_buf_free(dev, &eq->buf);
617         return err;
618 }
619 EXPORT_SYMBOL_GPL(mlx5_create_map_eq);
620
621 int mlx5_destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
622 {
623         int err;
624
625         mlx5_debug_eq_remove(dev, eq);
626         free_irq(eq->irqn, eq);
627         err = mlx5_cmd_destroy_eq(dev, eq->eqn);
628         if (err)
629                 mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
630                                eq->eqn);
631         synchronize_irq(eq->irqn);
632
633         if (eq->type == MLX5_EQ_TYPE_COMP) {
634                 tasklet_disable(&eq->tasklet_ctx.task);
635 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
636         } else if (eq->type == MLX5_EQ_TYPE_PF) {
637                 cancel_work_sync(&eq->pf_ctx.work);
638                 destroy_workqueue(eq->pf_ctx.wq);
639                 mempool_destroy(eq->pf_ctx.pool);
640 #endif
641         }
642         mlx5_buf_free(dev, &eq->buf);
643
644         return err;
645 }
646 EXPORT_SYMBOL_GPL(mlx5_destroy_unmap_eq);
647
648 u32 mlx5_get_msix_vec(struct mlx5_core_dev *dev, int vecidx)
649 {
650         return dev->priv.msix_arr[MLX5_EQ_VEC_ASYNC].vector;
651 }
652
653 int mlx5_eq_init(struct mlx5_core_dev *dev)
654 {
655         int err;
656
657         spin_lock_init(&dev->priv.eq_table.lock);
658
659         err = mlx5_eq_debugfs_init(dev);
660
661         return err;
662 }
663
664
665 void mlx5_eq_cleanup(struct mlx5_core_dev *dev)
666 {
667         mlx5_eq_debugfs_cleanup(dev);
668 }
669
670 int mlx5_start_eqs(struct mlx5_core_dev *dev)
671 {
672         struct mlx5_eq_table *table = &dev->priv.eq_table;
673         u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
674         int err;
675
676
677         if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH &&
678             MLX5_CAP_GEN(dev, vport_group_manager) &&
679             mlx5_core_is_pf(dev))
680                 async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE);
681
682         if (MLX5_CAP_GEN(dev, port_module_event))
683                 async_event_mask |= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT);
684         else
685                 mlx5_core_dbg(dev, "port_module_event is not set\n");
686
687         err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
688                                  MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD,
689                                  "mlx5_cmd_eq", &dev->priv.bfregi.uars[0],
690                                  MLX5_EQ_TYPE_ASYNC);
691         if (err) {
692                 mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err);
693                 return err;
694         }
695
696         mlx5_cmd_use_events(dev);
697
698         err = mlx5_create_map_eq(dev, &table->async_eq, MLX5_EQ_VEC_ASYNC,
699                                  MLX5_NUM_ASYNC_EQE, async_event_mask,
700                                  "mlx5_async_eq", &dev->priv.bfregi.uars[0],
701                                  MLX5_EQ_TYPE_ASYNC);
702         if (err) {
703                 mlx5_core_warn(dev, "failed to create async EQ %d\n", err);
704                 goto err1;
705         }
706
707         err = mlx5_create_map_eq(dev, &table->pages_eq,
708                                  MLX5_EQ_VEC_PAGES,
709                                  /* TODO: sriov max_vf + */ 1,
710                                  1 << MLX5_EVENT_TYPE_PAGE_REQUEST, "mlx5_pages_eq",
711                                  &dev->priv.bfregi.uars[0],
712                                  MLX5_EQ_TYPE_ASYNC);
713         if (err) {
714                 mlx5_core_warn(dev, "failed to create pages EQ %d\n", err);
715                 goto err2;
716         }
717
718 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
719         if (MLX5_CAP_GEN(dev, pg)) {
720                 err = mlx5_create_map_eq(dev, &table->pfault_eq,
721                                          MLX5_EQ_VEC_PFAULT,
722                                          MLX5_NUM_ASYNC_EQE,
723                                          1 << MLX5_EVENT_TYPE_PAGE_FAULT,
724                                          "mlx5_page_fault_eq",
725                                          &dev->priv.bfregi.uars[0],
726                                          MLX5_EQ_TYPE_PF);
727                 if (err) {
728                         mlx5_core_warn(dev, "failed to create page fault EQ %d\n",
729                                        err);
730                         goto err3;
731                 }
732         }
733
734         return err;
735 err3:
736         mlx5_destroy_unmap_eq(dev, &table->pages_eq);
737 #else
738         return err;
739 #endif
740
741 err2:
742         mlx5_destroy_unmap_eq(dev, &table->async_eq);
743
744 err1:
745         mlx5_cmd_use_polling(dev);
746         mlx5_destroy_unmap_eq(dev, &table->cmd_eq);
747         return err;
748 }
749
750 int mlx5_stop_eqs(struct mlx5_core_dev *dev)
751 {
752         struct mlx5_eq_table *table = &dev->priv.eq_table;
753         int err;
754
755 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
756         if (MLX5_CAP_GEN(dev, pg)) {
757                 err = mlx5_destroy_unmap_eq(dev, &table->pfault_eq);
758                 if (err)
759                         return err;
760         }
761 #endif
762
763         err = mlx5_destroy_unmap_eq(dev, &table->pages_eq);
764         if (err)
765                 return err;
766
767         mlx5_destroy_unmap_eq(dev, &table->async_eq);
768         mlx5_cmd_use_polling(dev);
769
770         err = mlx5_destroy_unmap_eq(dev, &table->cmd_eq);
771         if (err)
772                 mlx5_cmd_use_events(dev);
773
774         return err;
775 }
776
777 int mlx5_core_eq_query(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
778                        u32 *out, int outlen)
779 {
780         u32 in[MLX5_ST_SZ_DW(query_eq_in)] = {0};
781
782         MLX5_SET(query_eq_in, in, opcode, MLX5_CMD_OP_QUERY_EQ);
783         MLX5_SET(query_eq_in, in, eq_number, eq->eqn);
784         return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
785 }
786 EXPORT_SYMBOL_GPL(mlx5_core_eq_query);