net/smc: use queue pair number when matching link group
[sfrench/cifs-2.6.git] / net / smc / smc_core.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
4  *
5  *  Basic Transport Functions exploiting Infiniband API
6  *
7  *  Copyright IBM Corp. 2016
8  *
9  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
10  */
11
12 #include <linux/socket.h>
13 #include <linux/if_vlan.h>
14 #include <linux/random.h>
15 #include <linux/workqueue.h>
16 #include <net/tcp.h>
17 #include <net/sock.h>
18 #include <rdma/ib_verbs.h>
19 #include <rdma/ib_cache.h>
20
21 #include "smc.h"
22 #include "smc_clc.h"
23 #include "smc_core.h"
24 #include "smc_ib.h"
25 #include "smc_wr.h"
26 #include "smc_llc.h"
27 #include "smc_cdc.h"
28 #include "smc_close.h"
29 #include "smc_ism.h"
30
31 #define SMC_LGR_NUM_INCR                256
32 #define SMC_LGR_FREE_DELAY_SERV         (600 * HZ)
33 #define SMC_LGR_FREE_DELAY_CLNT         (SMC_LGR_FREE_DELAY_SERV + 10 * HZ)
34 #define SMC_LGR_FREE_DELAY_FAST         (8 * HZ)
35
36 static struct smc_lgr_list smc_lgr_list = {     /* established link groups */
37         .lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock),
38         .list = LIST_HEAD_INIT(smc_lgr_list.list),
39         .num = 0,
40 };
41
42 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
43                          struct smc_buf_desc *buf_desc);
44
45 static void smc_lgr_schedule_free_work(struct smc_link_group *lgr)
46 {
47         /* client link group creation always follows the server link group
48          * creation. For client use a somewhat higher removal delay time,
49          * otherwise there is a risk of out-of-sync link groups.
50          */
51         mod_delayed_work(system_wq, &lgr->free_work,
52                          (!lgr->is_smcd && lgr->role == SMC_CLNT) ?
53                          SMC_LGR_FREE_DELAY_CLNT : SMC_LGR_FREE_DELAY_SERV);
54 }
55
56 void smc_lgr_schedule_free_work_fast(struct smc_link_group *lgr)
57 {
58         mod_delayed_work(system_wq, &lgr->free_work, SMC_LGR_FREE_DELAY_FAST);
59 }
60
61 /* Register connection's alert token in our lookup structure.
62  * To use rbtrees we have to implement our own insert core.
63  * Requires @conns_lock
64  * @smc         connection to register
65  * Returns 0 on success, != otherwise.
66  */
67 static void smc_lgr_add_alert_token(struct smc_connection *conn)
68 {
69         struct rb_node **link, *parent = NULL;
70         u32 token = conn->alert_token_local;
71
72         link = &conn->lgr->conns_all.rb_node;
73         while (*link) {
74                 struct smc_connection *cur = rb_entry(*link,
75                                         struct smc_connection, alert_node);
76
77                 parent = *link;
78                 if (cur->alert_token_local > token)
79                         link = &parent->rb_left;
80                 else
81                         link = &parent->rb_right;
82         }
83         /* Put the new node there */
84         rb_link_node(&conn->alert_node, parent, link);
85         rb_insert_color(&conn->alert_node, &conn->lgr->conns_all);
86 }
87
88 /* Register connection in link group by assigning an alert token
89  * registered in a search tree.
90  * Requires @conns_lock
91  * Note that '0' is a reserved value and not assigned.
92  */
93 static void smc_lgr_register_conn(struct smc_connection *conn)
94 {
95         struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
96         static atomic_t nexttoken = ATOMIC_INIT(0);
97
98         /* find a new alert_token_local value not yet used by some connection
99          * in this link group
100          */
101         sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */
102         while (!conn->alert_token_local) {
103                 conn->alert_token_local = atomic_inc_return(&nexttoken);
104                 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr))
105                         conn->alert_token_local = 0;
106         }
107         smc_lgr_add_alert_token(conn);
108         conn->lgr->conns_num++;
109 }
110
111 /* Unregister connection and reset the alert token of the given connection<
112  */
113 static void __smc_lgr_unregister_conn(struct smc_connection *conn)
114 {
115         struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
116         struct smc_link_group *lgr = conn->lgr;
117
118         rb_erase(&conn->alert_node, &lgr->conns_all);
119         lgr->conns_num--;
120         conn->alert_token_local = 0;
121         conn->lgr = NULL;
122         sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */
123 }
124
125 /* Unregister connection from lgr
126  */
127 static void smc_lgr_unregister_conn(struct smc_connection *conn)
128 {
129         struct smc_link_group *lgr = conn->lgr;
130
131         write_lock_bh(&lgr->conns_lock);
132         if (conn->alert_token_local) {
133                 __smc_lgr_unregister_conn(conn);
134         }
135         write_unlock_bh(&lgr->conns_lock);
136 }
137
138 /* Send delete link, either as client to request the initiation
139  * of the DELETE LINK sequence from server; or as server to
140  * initiate the delete processing. See smc_llc_rx_delete_link().
141  */
142 static int smc_link_send_delete(struct smc_link *lnk)
143 {
144         if (lnk->state == SMC_LNK_ACTIVE &&
145             !smc_llc_send_delete_link(lnk, SMC_LLC_REQ, true)) {
146                 smc_llc_link_deleting(lnk);
147                 return 0;
148         }
149         return -ENOTCONN;
150 }
151
152 static void smc_lgr_free_work(struct work_struct *work)
153 {
154         struct smc_link_group *lgr = container_of(to_delayed_work(work),
155                                                   struct smc_link_group,
156                                                   free_work);
157         bool conns;
158
159         spin_lock_bh(&smc_lgr_list.lock);
160         if (list_empty(&lgr->list))
161                 goto free;
162         read_lock_bh(&lgr->conns_lock);
163         conns = RB_EMPTY_ROOT(&lgr->conns_all);
164         read_unlock_bh(&lgr->conns_lock);
165         if (!conns) { /* number of lgr connections is no longer zero */
166                 spin_unlock_bh(&smc_lgr_list.lock);
167                 return;
168         }
169         list_del_init(&lgr->list); /* remove from smc_lgr_list */
170 free:
171         spin_unlock_bh(&smc_lgr_list.lock);
172
173         if (!lgr->is_smcd && !lgr->terminating) {
174                 /* try to send del link msg, on error free lgr immediately */
175                 if (!smc_link_send_delete(&lgr->lnk[SMC_SINGLE_LINK])) {
176                         /* reschedule in case we never receive a response */
177                         smc_lgr_schedule_free_work(lgr);
178                         return;
179                 }
180         }
181
182         if (!delayed_work_pending(&lgr->free_work)) {
183                 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
184
185                 if (!lgr->is_smcd && lnk->state != SMC_LNK_INACTIVE)
186                         smc_llc_link_inactive(lnk);
187                 smc_lgr_free(lgr);
188         }
189 }
190
191 /* create a new SMC link group */
192 static int smc_lgr_create(struct smc_sock *smc, bool is_smcd,
193                           struct smc_ib_device *smcibdev, u8 ibport,
194                           char *peer_systemid, unsigned short vlan_id,
195                           struct smcd_dev *smcismdev, u64 peer_gid)
196 {
197         struct smc_link_group *lgr;
198         struct smc_link *lnk;
199         u8 rndvec[3];
200         int rc = 0;
201         int i;
202
203         if (is_smcd && vlan_id) {
204                 rc = smc_ism_get_vlan(smcismdev, vlan_id);
205                 if (rc)
206                         goto out;
207         }
208
209         lgr = kzalloc(sizeof(*lgr), GFP_KERNEL);
210         if (!lgr) {
211                 rc = -ENOMEM;
212                 goto out;
213         }
214         lgr->is_smcd = is_smcd;
215         lgr->sync_err = 0;
216         lgr->vlan_id = vlan_id;
217         rwlock_init(&lgr->sndbufs_lock);
218         rwlock_init(&lgr->rmbs_lock);
219         rwlock_init(&lgr->conns_lock);
220         for (i = 0; i < SMC_RMBE_SIZES; i++) {
221                 INIT_LIST_HEAD(&lgr->sndbufs[i]);
222                 INIT_LIST_HEAD(&lgr->rmbs[i]);
223         }
224         smc_lgr_list.num += SMC_LGR_NUM_INCR;
225         memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE);
226         INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work);
227         lgr->conns_all = RB_ROOT;
228         if (is_smcd) {
229                 /* SMC-D specific settings */
230                 lgr->peer_gid = peer_gid;
231                 lgr->smcd = smcismdev;
232         } else {
233                 /* SMC-R specific settings */
234                 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
235                 memcpy(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN);
236
237                 lnk = &lgr->lnk[SMC_SINGLE_LINK];
238                 /* initialize link */
239                 lnk->state = SMC_LNK_ACTIVATING;
240                 lnk->link_id = SMC_SINGLE_LINK;
241                 lnk->smcibdev = smcibdev;
242                 lnk->ibport = ibport;
243                 lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu;
244                 if (!smcibdev->initialized)
245                         smc_ib_setup_per_ibdev(smcibdev);
246                 get_random_bytes(rndvec, sizeof(rndvec));
247                 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) +
248                         (rndvec[2] << 16);
249                 rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport,
250                                           vlan_id, lnk->gid, &lnk->sgid_index);
251                 if (rc)
252                         goto free_lgr;
253                 rc = smc_llc_link_init(lnk);
254                 if (rc)
255                         goto free_lgr;
256                 rc = smc_wr_alloc_link_mem(lnk);
257                 if (rc)
258                         goto clear_llc_lnk;
259                 rc = smc_ib_create_protection_domain(lnk);
260                 if (rc)
261                         goto free_link_mem;
262                 rc = smc_ib_create_queue_pair(lnk);
263                 if (rc)
264                         goto dealloc_pd;
265                 rc = smc_wr_create_link(lnk);
266                 if (rc)
267                         goto destroy_qp;
268         }
269         smc->conn.lgr = lgr;
270         spin_lock_bh(&smc_lgr_list.lock);
271         list_add(&lgr->list, &smc_lgr_list.list);
272         spin_unlock_bh(&smc_lgr_list.lock);
273         return 0;
274
275 destroy_qp:
276         smc_ib_destroy_queue_pair(lnk);
277 dealloc_pd:
278         smc_ib_dealloc_protection_domain(lnk);
279 free_link_mem:
280         smc_wr_free_link_mem(lnk);
281 clear_llc_lnk:
282         smc_llc_link_clear(lnk);
283 free_lgr:
284         kfree(lgr);
285 out:
286         return rc;
287 }
288
289 static void smc_buf_unuse(struct smc_connection *conn,
290                           struct smc_link_group *lgr)
291 {
292         if (conn->sndbuf_desc)
293                 conn->sndbuf_desc->used = 0;
294         if (conn->rmb_desc) {
295                 if (!conn->rmb_desc->regerr) {
296                         conn->rmb_desc->reused = 1;
297                         conn->rmb_desc->used = 0;
298                 } else {
299                         /* buf registration failed, reuse not possible */
300                         write_lock_bh(&lgr->rmbs_lock);
301                         list_del(&conn->rmb_desc->list);
302                         write_unlock_bh(&lgr->rmbs_lock);
303
304                         smc_buf_free(lgr, true, conn->rmb_desc);
305                 }
306         }
307 }
308
309 /* remove a finished connection from its link group */
310 void smc_conn_free(struct smc_connection *conn)
311 {
312         struct smc_link_group *lgr = conn->lgr;
313
314         if (!lgr)
315                 return;
316         if (lgr->is_smcd) {
317                 smc_ism_unset_conn(conn);
318                 tasklet_kill(&conn->rx_tsklet);
319         } else {
320                 smc_cdc_tx_dismiss_slots(conn);
321         }
322         smc_lgr_unregister_conn(conn);          /* unsets conn->lgr */
323         smc_buf_unuse(conn, lgr);               /* allow buffer reuse */
324
325         if (!lgr->conns_num)
326                 smc_lgr_schedule_free_work(lgr);
327 }
328
329 static void smc_link_clear(struct smc_link *lnk)
330 {
331         lnk->peer_qpn = 0;
332         smc_llc_link_clear(lnk);
333         smc_ib_modify_qp_reset(lnk);
334         smc_wr_free_link(lnk);
335         smc_ib_destroy_queue_pair(lnk);
336         smc_ib_dealloc_protection_domain(lnk);
337         smc_wr_free_link_mem(lnk);
338 }
339
340 static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb,
341                           struct smc_buf_desc *buf_desc)
342 {
343         struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
344
345         if (is_rmb) {
346                 if (buf_desc->mr_rx[SMC_SINGLE_LINK])
347                         smc_ib_put_memory_region(
348                                         buf_desc->mr_rx[SMC_SINGLE_LINK]);
349                 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
350                                     DMA_FROM_DEVICE);
351         } else {
352                 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
353                                     DMA_TO_DEVICE);
354         }
355         sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]);
356         if (buf_desc->pages)
357                 __free_pages(buf_desc->pages, buf_desc->order);
358         kfree(buf_desc);
359 }
360
361 static void smcd_buf_free(struct smc_link_group *lgr, bool is_dmb,
362                           struct smc_buf_desc *buf_desc)
363 {
364         if (is_dmb) {
365                 /* restore original buf len */
366                 buf_desc->len += sizeof(struct smcd_cdc_msg);
367                 smc_ism_unregister_dmb(lgr->smcd, buf_desc);
368         } else {
369                 kfree(buf_desc->cpu_addr);
370         }
371         kfree(buf_desc);
372 }
373
374 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
375                          struct smc_buf_desc *buf_desc)
376 {
377         if (lgr->is_smcd)
378                 smcd_buf_free(lgr, is_rmb, buf_desc);
379         else
380                 smcr_buf_free(lgr, is_rmb, buf_desc);
381 }
382
383 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb)
384 {
385         struct smc_buf_desc *buf_desc, *bf_desc;
386         struct list_head *buf_list;
387         int i;
388
389         for (i = 0; i < SMC_RMBE_SIZES; i++) {
390                 if (is_rmb)
391                         buf_list = &lgr->rmbs[i];
392                 else
393                         buf_list = &lgr->sndbufs[i];
394                 list_for_each_entry_safe(buf_desc, bf_desc, buf_list,
395                                          list) {
396                         list_del(&buf_desc->list);
397                         smc_buf_free(lgr, is_rmb, buf_desc);
398                 }
399         }
400 }
401
402 static void smc_lgr_free_bufs(struct smc_link_group *lgr)
403 {
404         /* free send buffers */
405         __smc_lgr_free_bufs(lgr, false);
406         /* free rmbs */
407         __smc_lgr_free_bufs(lgr, true);
408 }
409
410 /* remove a link group */
411 void smc_lgr_free(struct smc_link_group *lgr)
412 {
413         smc_lgr_free_bufs(lgr);
414         if (lgr->is_smcd)
415                 smc_ism_put_vlan(lgr->smcd, lgr->vlan_id);
416         else
417                 smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]);
418         kfree(lgr);
419 }
420
421 void smc_lgr_forget(struct smc_link_group *lgr)
422 {
423         spin_lock_bh(&smc_lgr_list.lock);
424         /* do not use this link group for new connections */
425         if (!list_empty(&lgr->list))
426                 list_del_init(&lgr->list);
427         spin_unlock_bh(&smc_lgr_list.lock);
428 }
429
430 /* terminate linkgroup abnormally */
431 static void __smc_lgr_terminate(struct smc_link_group *lgr)
432 {
433         struct smc_connection *conn;
434         struct smc_sock *smc;
435         struct rb_node *node;
436
437         if (lgr->terminating)
438                 return; /* lgr already terminating */
439         lgr->terminating = 1;
440         if (!list_empty(&lgr->list)) /* forget lgr */
441                 list_del_init(&lgr->list);
442         if (!lgr->is_smcd)
443                 smc_llc_link_inactive(&lgr->lnk[SMC_SINGLE_LINK]);
444
445         write_lock_bh(&lgr->conns_lock);
446         node = rb_first(&lgr->conns_all);
447         while (node) {
448                 conn = rb_entry(node, struct smc_connection, alert_node);
449                 smc = container_of(conn, struct smc_sock, conn);
450                 sock_hold(&smc->sk); /* sock_put in close work */
451                 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;
452                 __smc_lgr_unregister_conn(conn);
453                 write_unlock_bh(&lgr->conns_lock);
454                 if (!schedule_work(&conn->close_work))
455                         sock_put(&smc->sk);
456                 write_lock_bh(&lgr->conns_lock);
457                 node = rb_first(&lgr->conns_all);
458         }
459         write_unlock_bh(&lgr->conns_lock);
460         if (!lgr->is_smcd)
461                 wake_up(&lgr->lnk[SMC_SINGLE_LINK].wr_reg_wait);
462         smc_lgr_schedule_free_work(lgr);
463 }
464
465 void smc_lgr_terminate(struct smc_link_group *lgr)
466 {
467         spin_lock_bh(&smc_lgr_list.lock);
468         __smc_lgr_terminate(lgr);
469         spin_unlock_bh(&smc_lgr_list.lock);
470 }
471
472 /* Called when IB port is terminated */
473 void smc_port_terminate(struct smc_ib_device *smcibdev, u8 ibport)
474 {
475         struct smc_link_group *lgr, *l;
476
477         spin_lock_bh(&smc_lgr_list.lock);
478         list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) {
479                 if (!lgr->is_smcd &&
480                     lgr->lnk[SMC_SINGLE_LINK].smcibdev == smcibdev &&
481                     lgr->lnk[SMC_SINGLE_LINK].ibport == ibport)
482                         __smc_lgr_terminate(lgr);
483         }
484         spin_unlock_bh(&smc_lgr_list.lock);
485 }
486
487 /* Called when SMC-D device is terminated or peer is lost */
488 void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid)
489 {
490         struct smc_link_group *lgr, *l;
491         LIST_HEAD(lgr_free_list);
492
493         /* run common cleanup function and build free list */
494         spin_lock_bh(&smc_lgr_list.lock);
495         list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) {
496                 if (lgr->is_smcd && lgr->smcd == dev &&
497                     (!peer_gid || lgr->peer_gid == peer_gid) &&
498                     !list_empty(&lgr->list)) {
499                         __smc_lgr_terminate(lgr);
500                         list_move(&lgr->list, &lgr_free_list);
501                 }
502         }
503         spin_unlock_bh(&smc_lgr_list.lock);
504
505         /* cancel the regular free workers and actually free lgrs */
506         list_for_each_entry_safe(lgr, l, &lgr_free_list, list) {
507                 list_del_init(&lgr->list);
508                 cancel_delayed_work_sync(&lgr->free_work);
509                 smc_lgr_free(lgr);
510         }
511 }
512
513 /* Determine vlan of internal TCP socket.
514  * @vlan_id: address to store the determined vlan id into
515  */
516 int smc_vlan_by_tcpsk(struct socket *clcsock, unsigned short *vlan_id)
517 {
518         struct dst_entry *dst = sk_dst_get(clcsock->sk);
519         struct net_device *ndev;
520         int i, nest_lvl, rc = 0;
521
522         *vlan_id = 0;
523         if (!dst) {
524                 rc = -ENOTCONN;
525                 goto out;
526         }
527         if (!dst->dev) {
528                 rc = -ENODEV;
529                 goto out_rel;
530         }
531
532         ndev = dst->dev;
533         if (is_vlan_dev(ndev)) {
534                 *vlan_id = vlan_dev_vlan_id(ndev);
535                 goto out_rel;
536         }
537
538         rtnl_lock();
539         nest_lvl = dev_get_nest_level(ndev);
540         for (i = 0; i < nest_lvl; i++) {
541                 struct list_head *lower = &ndev->adj_list.lower;
542
543                 if (list_empty(lower))
544                         break;
545                 lower = lower->next;
546                 ndev = (struct net_device *)netdev_lower_get_next(ndev, &lower);
547                 if (is_vlan_dev(ndev)) {
548                         *vlan_id = vlan_dev_vlan_id(ndev);
549                         break;
550                 }
551         }
552         rtnl_unlock();
553
554 out_rel:
555         dst_release(dst);
556 out:
557         return rc;
558 }
559
560 static bool smcr_lgr_match(struct smc_link_group *lgr,
561                            struct smc_clc_msg_local *lcl,
562                            enum smc_lgr_role role, u32 clcqpn)
563 {
564         return !memcmp(lgr->peer_systemid, lcl->id_for_peer,
565                        SMC_SYSTEMID_LEN) &&
566                 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid,
567                         SMC_GID_SIZE) &&
568                 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac,
569                         sizeof(lcl->mac)) &&
570                 lgr->role == role &&
571                 (lgr->role == SMC_SERV ||
572                  lgr->lnk[SMC_SINGLE_LINK].peer_qpn == clcqpn);
573 }
574
575 static bool smcd_lgr_match(struct smc_link_group *lgr,
576                            struct smcd_dev *smcismdev, u64 peer_gid)
577 {
578         return lgr->peer_gid == peer_gid && lgr->smcd == smcismdev;
579 }
580
581 /* create a new SMC connection (and a new link group if necessary) */
582 int smc_conn_create(struct smc_sock *smc, bool is_smcd, int srv_first_contact,
583                     struct smc_ib_device *smcibdev, u8 ibport, u32 clcqpn,
584                     struct smc_clc_msg_local *lcl, struct smcd_dev *smcd,
585                     u64 peer_gid)
586 {
587         struct smc_connection *conn = &smc->conn;
588         int local_contact = SMC_FIRST_CONTACT;
589         struct smc_link_group *lgr;
590         unsigned short vlan_id;
591         enum smc_lgr_role role;
592         int rc = 0;
593
594         role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
595         rc = smc_vlan_by_tcpsk(smc->clcsock, &vlan_id);
596         if (rc)
597                 return rc;
598
599         if ((role == SMC_CLNT) && srv_first_contact)
600                 /* create new link group as well */
601                 goto create;
602
603         /* determine if an existing link group can be reused */
604         spin_lock_bh(&smc_lgr_list.lock);
605         list_for_each_entry(lgr, &smc_lgr_list.list, list) {
606                 write_lock_bh(&lgr->conns_lock);
607                 if ((is_smcd ? smcd_lgr_match(lgr, smcd, peer_gid) :
608                      smcr_lgr_match(lgr, lcl, role, clcqpn)) &&
609                     !lgr->sync_err &&
610                     lgr->vlan_id == vlan_id &&
611                     (role == SMC_CLNT ||
612                      lgr->conns_num < SMC_RMBS_PER_LGR_MAX)) {
613                         /* link group found */
614                         local_contact = SMC_REUSE_CONTACT;
615                         conn->lgr = lgr;
616                         smc_lgr_register_conn(conn); /* add smc conn to lgr */
617                         write_unlock_bh(&lgr->conns_lock);
618                         break;
619                 }
620                 write_unlock_bh(&lgr->conns_lock);
621         }
622         spin_unlock_bh(&smc_lgr_list.lock);
623
624         if (role == SMC_CLNT && !srv_first_contact &&
625             (local_contact == SMC_FIRST_CONTACT)) {
626                 /* Server reuses a link group, but Client wants to start
627                  * a new one
628                  * send out_of_sync decline, reason synchr. error
629                  */
630                 return -ENOLINK;
631         }
632
633 create:
634         if (local_contact == SMC_FIRST_CONTACT) {
635                 rc = smc_lgr_create(smc, is_smcd, smcibdev, ibport,
636                                     lcl->id_for_peer, vlan_id, smcd, peer_gid);
637                 if (rc)
638                         goto out;
639                 smc_lgr_register_conn(conn); /* add smc conn to lgr */
640         }
641         conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE;
642         conn->local_tx_ctrl.len = SMC_WR_TX_SIZE;
643         conn->urg_state = SMC_URG_READ;
644         if (is_smcd) {
645                 conn->rx_off = sizeof(struct smcd_cdc_msg);
646                 smcd_cdc_rx_init(conn); /* init tasklet for this conn */
647         }
648 #ifndef KERNEL_HAS_ATOMIC64
649         spin_lock_init(&conn->acurs_lock);
650 #endif
651
652 out:
653         return rc ? rc : local_contact;
654 }
655
656 /* convert the RMB size into the compressed notation - minimum 16K.
657  * In contrast to plain ilog2, this rounds towards the next power of 2,
658  * so the socket application gets at least its desired sndbuf / rcvbuf size.
659  */
660 static u8 smc_compress_bufsize(int size)
661 {
662         u8 compressed;
663
664         if (size <= SMC_BUF_MIN_SIZE)
665                 return 0;
666
667         size = (size - 1) >> 14;
668         compressed = ilog2(size) + 1;
669         if (compressed >= SMC_RMBE_SIZES)
670                 compressed = SMC_RMBE_SIZES - 1;
671         return compressed;
672 }
673
674 /* convert the RMB size from compressed notation into integer */
675 int smc_uncompress_bufsize(u8 compressed)
676 {
677         u32 size;
678
679         size = 0x00000001 << (((int)compressed) + 14);
680         return (int)size;
681 }
682
683 /* try to reuse a sndbuf or rmb description slot for a certain
684  * buffer size; if not available, return NULL
685  */
686 static struct smc_buf_desc *smc_buf_get_slot(int compressed_bufsize,
687                                              rwlock_t *lock,
688                                              struct list_head *buf_list)
689 {
690         struct smc_buf_desc *buf_slot;
691
692         read_lock_bh(lock);
693         list_for_each_entry(buf_slot, buf_list, list) {
694                 if (cmpxchg(&buf_slot->used, 0, 1) == 0) {
695                         read_unlock_bh(lock);
696                         return buf_slot;
697                 }
698         }
699         read_unlock_bh(lock);
700         return NULL;
701 }
702
703 /* one of the conditions for announcing a receiver's current window size is
704  * that it "results in a minimum increase in the window size of 10% of the
705  * receive buffer space" [RFC7609]
706  */
707 static inline int smc_rmb_wnd_update_limit(int rmbe_size)
708 {
709         return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2);
710 }
711
712 static struct smc_buf_desc *smcr_new_buf_create(struct smc_link_group *lgr,
713                                                 bool is_rmb, int bufsize)
714 {
715         struct smc_buf_desc *buf_desc;
716         struct smc_link *lnk;
717         int rc;
718
719         /* try to alloc a new buffer */
720         buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
721         if (!buf_desc)
722                 return ERR_PTR(-ENOMEM);
723
724         buf_desc->order = get_order(bufsize);
725         buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN |
726                                       __GFP_NOMEMALLOC | __GFP_COMP |
727                                       __GFP_NORETRY | __GFP_ZERO,
728                                       buf_desc->order);
729         if (!buf_desc->pages) {
730                 kfree(buf_desc);
731                 return ERR_PTR(-EAGAIN);
732         }
733         buf_desc->cpu_addr = (void *)page_address(buf_desc->pages);
734
735         /* build the sg table from the pages */
736         lnk = &lgr->lnk[SMC_SINGLE_LINK];
737         rc = sg_alloc_table(&buf_desc->sgt[SMC_SINGLE_LINK], 1,
738                             GFP_KERNEL);
739         if (rc) {
740                 smc_buf_free(lgr, is_rmb, buf_desc);
741                 return ERR_PTR(rc);
742         }
743         sg_set_buf(buf_desc->sgt[SMC_SINGLE_LINK].sgl,
744                    buf_desc->cpu_addr, bufsize);
745
746         /* map sg table to DMA address */
747         rc = smc_ib_buf_map_sg(lnk->smcibdev, buf_desc,
748                                is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
749         /* SMC protocol depends on mapping to one DMA address only */
750         if (rc != 1)  {
751                 smc_buf_free(lgr, is_rmb, buf_desc);
752                 return ERR_PTR(-EAGAIN);
753         }
754
755         /* create a new memory region for the RMB */
756         if (is_rmb) {
757                 rc = smc_ib_get_memory_region(lnk->roce_pd,
758                                               IB_ACCESS_REMOTE_WRITE |
759                                               IB_ACCESS_LOCAL_WRITE,
760                                               buf_desc);
761                 if (rc) {
762                         smc_buf_free(lgr, is_rmb, buf_desc);
763                         return ERR_PTR(rc);
764                 }
765         }
766
767         buf_desc->len = bufsize;
768         return buf_desc;
769 }
770
771 #define SMCD_DMBE_SIZES         7 /* 0 -> 16KB, 1 -> 32KB, .. 6 -> 1MB */
772
773 static struct smc_buf_desc *smcd_new_buf_create(struct smc_link_group *lgr,
774                                                 bool is_dmb, int bufsize)
775 {
776         struct smc_buf_desc *buf_desc;
777         int rc;
778
779         if (smc_compress_bufsize(bufsize) > SMCD_DMBE_SIZES)
780                 return ERR_PTR(-EAGAIN);
781
782         /* try to alloc a new DMB */
783         buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
784         if (!buf_desc)
785                 return ERR_PTR(-ENOMEM);
786         if (is_dmb) {
787                 rc = smc_ism_register_dmb(lgr, bufsize, buf_desc);
788                 if (rc) {
789                         kfree(buf_desc);
790                         return ERR_PTR(-EAGAIN);
791                 }
792                 buf_desc->pages = virt_to_page(buf_desc->cpu_addr);
793                 /* CDC header stored in buf. So, pretend it was smaller */
794                 buf_desc->len = bufsize - sizeof(struct smcd_cdc_msg);
795         } else {
796                 buf_desc->cpu_addr = kzalloc(bufsize, GFP_KERNEL |
797                                              __GFP_NOWARN | __GFP_NORETRY |
798                                              __GFP_NOMEMALLOC);
799                 if (!buf_desc->cpu_addr) {
800                         kfree(buf_desc);
801                         return ERR_PTR(-EAGAIN);
802                 }
803                 buf_desc->len = bufsize;
804         }
805         return buf_desc;
806 }
807
808 static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb)
809 {
810         struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM);
811         struct smc_connection *conn = &smc->conn;
812         struct smc_link_group *lgr = conn->lgr;
813         struct list_head *buf_list;
814         int bufsize, bufsize_short;
815         int sk_buf_size;
816         rwlock_t *lock;
817
818         if (is_rmb)
819                 /* use socket recv buffer size (w/o overhead) as start value */
820                 sk_buf_size = smc->sk.sk_rcvbuf / 2;
821         else
822                 /* use socket send buffer size (w/o overhead) as start value */
823                 sk_buf_size = smc->sk.sk_sndbuf / 2;
824
825         for (bufsize_short = smc_compress_bufsize(sk_buf_size);
826              bufsize_short >= 0; bufsize_short--) {
827
828                 if (is_rmb) {
829                         lock = &lgr->rmbs_lock;
830                         buf_list = &lgr->rmbs[bufsize_short];
831                 } else {
832                         lock = &lgr->sndbufs_lock;
833                         buf_list = &lgr->sndbufs[bufsize_short];
834                 }
835                 bufsize = smc_uncompress_bufsize(bufsize_short);
836                 if ((1 << get_order(bufsize)) > SG_MAX_SINGLE_ALLOC)
837                         continue;
838
839                 /* check for reusable slot in the link group */
840                 buf_desc = smc_buf_get_slot(bufsize_short, lock, buf_list);
841                 if (buf_desc) {
842                         memset(buf_desc->cpu_addr, 0, bufsize);
843                         break; /* found reusable slot */
844                 }
845
846                 if (is_smcd)
847                         buf_desc = smcd_new_buf_create(lgr, is_rmb, bufsize);
848                 else
849                         buf_desc = smcr_new_buf_create(lgr, is_rmb, bufsize);
850
851                 if (PTR_ERR(buf_desc) == -ENOMEM)
852                         break;
853                 if (IS_ERR(buf_desc))
854                         continue;
855
856                 buf_desc->used = 1;
857                 write_lock_bh(lock);
858                 list_add(&buf_desc->list, buf_list);
859                 write_unlock_bh(lock);
860                 break; /* found */
861         }
862
863         if (IS_ERR(buf_desc))
864                 return -ENOMEM;
865
866         if (is_rmb) {
867                 conn->rmb_desc = buf_desc;
868                 conn->rmbe_size_short = bufsize_short;
869                 smc->sk.sk_rcvbuf = bufsize * 2;
870                 atomic_set(&conn->bytes_to_rcv, 0);
871                 conn->rmbe_update_limit =
872                         smc_rmb_wnd_update_limit(buf_desc->len);
873                 if (is_smcd)
874                         smc_ism_set_conn(conn); /* map RMB/smcd_dev to conn */
875         } else {
876                 conn->sndbuf_desc = buf_desc;
877                 smc->sk.sk_sndbuf = bufsize * 2;
878                 atomic_set(&conn->sndbuf_space, bufsize);
879         }
880         return 0;
881 }
882
883 void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn)
884 {
885         struct smc_link_group *lgr = conn->lgr;
886
887         if (!conn->lgr || conn->lgr->is_smcd)
888                 return;
889         smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
890                                conn->sndbuf_desc, DMA_TO_DEVICE);
891 }
892
893 void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn)
894 {
895         struct smc_link_group *lgr = conn->lgr;
896
897         if (!conn->lgr || conn->lgr->is_smcd)
898                 return;
899         smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
900                                   conn->sndbuf_desc, DMA_TO_DEVICE);
901 }
902
903 void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn)
904 {
905         struct smc_link_group *lgr = conn->lgr;
906
907         if (!conn->lgr || conn->lgr->is_smcd)
908                 return;
909         smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
910                                conn->rmb_desc, DMA_FROM_DEVICE);
911 }
912
913 void smc_rmb_sync_sg_for_device(struct smc_connection *conn)
914 {
915         struct smc_link_group *lgr = conn->lgr;
916
917         if (!conn->lgr || conn->lgr->is_smcd)
918                 return;
919         smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
920                                   conn->rmb_desc, DMA_FROM_DEVICE);
921 }
922
923 /* create the send and receive buffer for an SMC socket;
924  * receive buffers are called RMBs;
925  * (even though the SMC protocol allows more than one RMB-element per RMB,
926  * the Linux implementation uses just one RMB-element per RMB, i.e. uses an
927  * extra RMB for every connection in a link group
928  */
929 int smc_buf_create(struct smc_sock *smc, bool is_smcd)
930 {
931         int rc;
932
933         /* create send buffer */
934         rc = __smc_buf_create(smc, is_smcd, false);
935         if (rc)
936                 return rc;
937         /* create rmb */
938         rc = __smc_buf_create(smc, is_smcd, true);
939         if (rc)
940                 smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc);
941         return rc;
942 }
943
944 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)
945 {
946         int i;
947
948         for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) {
949                 if (!test_and_set_bit(i, lgr->rtokens_used_mask))
950                         return i;
951         }
952         return -ENOSPC;
953 }
954
955 /* add a new rtoken from peer */
956 int smc_rtoken_add(struct smc_link_group *lgr, __be64 nw_vaddr, __be32 nw_rkey)
957 {
958         u64 dma_addr = be64_to_cpu(nw_vaddr);
959         u32 rkey = ntohl(nw_rkey);
960         int i;
961
962         for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
963                 if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) &&
964                     (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) &&
965                     test_bit(i, lgr->rtokens_used_mask)) {
966                         /* already in list */
967                         return i;
968                 }
969         }
970         i = smc_rmb_reserve_rtoken_idx(lgr);
971         if (i < 0)
972                 return i;
973         lgr->rtokens[i][SMC_SINGLE_LINK].rkey = rkey;
974         lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = dma_addr;
975         return i;
976 }
977
978 /* delete an rtoken */
979 int smc_rtoken_delete(struct smc_link_group *lgr, __be32 nw_rkey)
980 {
981         u32 rkey = ntohl(nw_rkey);
982         int i;
983
984         for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
985                 if (lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey &&
986                     test_bit(i, lgr->rtokens_used_mask)) {
987                         lgr->rtokens[i][SMC_SINGLE_LINK].rkey = 0;
988                         lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = 0;
989
990                         clear_bit(i, lgr->rtokens_used_mask);
991                         return 0;
992                 }
993         }
994         return -ENOENT;
995 }
996
997 /* save rkey and dma_addr received from peer during clc handshake */
998 int smc_rmb_rtoken_handling(struct smc_connection *conn,
999                             struct smc_clc_msg_accept_confirm *clc)
1000 {
1001         conn->rtoken_idx = smc_rtoken_add(conn->lgr, clc->rmb_dma_addr,
1002                                           clc->rmb_rkey);
1003         if (conn->rtoken_idx < 0)
1004                 return conn->rtoken_idx;
1005         return 0;
1006 }
1007
1008 /* Called (from smc_exit) when module is removed */
1009 void smc_core_exit(void)
1010 {
1011         struct smc_link_group *lgr, *lg;
1012         LIST_HEAD(lgr_freeing_list);
1013
1014         spin_lock_bh(&smc_lgr_list.lock);
1015         if (!list_empty(&smc_lgr_list.list))
1016                 list_splice_init(&smc_lgr_list.list, &lgr_freeing_list);
1017         spin_unlock_bh(&smc_lgr_list.lock);
1018         list_for_each_entry_safe(lgr, lg, &lgr_freeing_list, list) {
1019                 list_del_init(&lgr->list);
1020                 if (!lgr->is_smcd) {
1021                         struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
1022
1023                         if (lnk->state == SMC_LNK_ACTIVE)
1024                                 smc_llc_send_delete_link(lnk, SMC_LLC_REQ,
1025                                                          false);
1026                         smc_llc_link_inactive(lnk);
1027                 }
1028                 cancel_delayed_work_sync(&lgr->free_work);
1029                 smc_lgr_free(lgr); /* free link group */
1030         }
1031 }