drivers/target/iscsi/cxgbit/cxgbit_cm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2016 Chelsio Communications, Inc.
   4  */
   5
   6 #include <linux/module.h>
   7 #include <linux/list.h>
   8 #include <linux/workqueue.h>
   9 #include <linux/skbuff.h>
  10 #include <linux/timer.h>
  11 #include <linux/notifier.h>
  12 #include <linux/inetdevice.h>
  13 #include <linux/ip.h>
  14 #include <linux/tcp.h>
  15 #include <linux/if_vlan.h>
  16
  17 #include <net/neighbour.h>
  18 #include <net/netevent.h>
  19 #include <net/route.h>
  20 #include <net/tcp.h>
  21 #include <net/ip6_route.h>
  22 #include <net/addrconf.h>
  23
  24 #include <libcxgb_cm.h>
  25 #include "cxgbit.h"
  26 #include "clip_tbl.h"
  27
  28 static void cxgbit_init_wr_wait(struct cxgbit_wr_wait *wr_waitp)
  29 {
  30         wr_waitp->ret = 0;
  31         reinit_completion(&wr_waitp->completion);
  32 }
  33
  34 static void
  35 cxgbit_wake_up(struct cxgbit_wr_wait *wr_waitp, const char *func, u8 ret)
  36 {
  37         if (ret == CPL_ERR_NONE)
  38                 wr_waitp->ret = 0;
  39         else
  40                 wr_waitp->ret = -EIO;
  41
  42         if (wr_waitp->ret)
  43                 pr_err("%s: err:%u", func, ret);
  44
  45         complete(&wr_waitp->completion);
  46 }
  47
  48 static int
  49 cxgbit_wait_for_reply(struct cxgbit_device *cdev,
  50                       struct cxgbit_wr_wait *wr_waitp, u32 tid, u32 timeout,
  51                       const char *func)
  52 {
  53         int ret;
  54
  55         if (!test_bit(CDEV_STATE_UP, &cdev->flags)) {
  56                 wr_waitp->ret = -EIO;
  57                 goto out;
  58         }
  59
  60         ret = wait_for_completion_timeout(&wr_waitp->completion, timeout * HZ);
  61         if (!ret) {
  62                 pr_info("%s - Device %s not responding tid %u\n",
  63                         func, pci_name(cdev->lldi.pdev), tid);
  64                 wr_waitp->ret = -ETIMEDOUT;
  65         }
  66 out:
  67         if (wr_waitp->ret)
  68                 pr_info("%s: FW reply %d tid %u\n",
  69                         pci_name(cdev->lldi.pdev), wr_waitp->ret, tid);
  70         return wr_waitp->ret;
  71 }
  72
  73 static int cxgbit_np_hashfn(const struct cxgbit_np *cnp)
  74 {
  75         return ((unsigned long)cnp >> 10) & (NP_INFO_HASH_SIZE - 1);
  76 }
  77
  78 static struct np_info *
  79 cxgbit_np_hash_add(struct cxgbit_device *cdev, struct cxgbit_np *cnp,
  80                    unsigned int stid)
  81 {
  82         struct np_info *p = kzalloc(sizeof(*p), GFP_KERNEL);
  83
  84         if (p) {
  85                 int bucket = cxgbit_np_hashfn(cnp);
  86
  87                 p->cnp = cnp;
  88                 p->stid = stid;
  89                 spin_lock(&cdev->np_lock);
  90                 p->next = cdev->np_hash_tab[bucket];
  91                 cdev->np_hash_tab[bucket] = p;
  92                 spin_unlock(&cdev->np_lock);
  93         }
  94
  95         return p;
  96 }
  97
  98 static int
  99 cxgbit_np_hash_find(struct cxgbit_device *cdev, struct cxgbit_np *cnp)
 100 {
 101         int stid = -1, bucket = cxgbit_np_hashfn(cnp);
 102         struct np_info *p;
 103
 104         spin_lock(&cdev->np_lock);
 105         for (p = cdev->np_hash_tab[bucket]; p; p = p->next) {
 106                 if (p->cnp == cnp) {
 107                         stid = p->stid;
 108                         break;
 109                 }
 110         }
 111         spin_unlock(&cdev->np_lock);
 112
 113         return stid;
 114 }
 115
 116 static int cxgbit_np_hash_del(struct cxgbit_device *cdev, struct cxgbit_np *cnp)
 117 {
 118         int stid = -1, bucket = cxgbit_np_hashfn(cnp);
 119         struct np_info *p, **prev = &cdev->np_hash_tab[bucket];
 120
 121         spin_lock(&cdev->np_lock);
 122         for (p = *prev; p; prev = &p->next, p = p->next) {
 123                 if (p->cnp == cnp) {
 124                         stid = p->stid;
 125                         *prev = p->next;
 126                         kfree(p);
 127                         break;
 128                 }
 129         }
 130         spin_unlock(&cdev->np_lock);
 131
 132         return stid;
 133 }
 134
 135 void _cxgbit_free_cnp(struct kref *kref)
 136 {
 137         struct cxgbit_np *cnp;
 138
 139         cnp = container_of(kref, struct cxgbit_np, kref);
 140         kfree(cnp);
 141 }
 142
 143 static int
 144 cxgbit_create_server6(struct cxgbit_device *cdev, unsigned int stid,
 145                       struct cxgbit_np *cnp)
 146 {
 147         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)
 148                                      &cnp->com.local_addr;
 149         int addr_type;
 150         int ret;
 151
 152         pr_debug("%s: dev = %s; stid = %u; sin6_port = %u\n",
 153                  __func__, cdev->lldi.ports[0]->name, stid, sin6->sin6_port);
 154
 155         addr_type = ipv6_addr_type((const struct in6_addr *)
 156                                    &sin6->sin6_addr);
 157         if (addr_type != IPV6_ADDR_ANY) {
 158                 ret = cxgb4_clip_get(cdev->lldi.ports[0],
 159                                      (const u32 *)&sin6->sin6_addr.s6_addr, 1);
 160                 if (ret) {
 161                         pr_err("Unable to find clip table entry. laddr %pI6. Error:%d.\n",
 162                                sin6->sin6_addr.s6_addr, ret);
 163                         return -ENOMEM;
 164                 }
 165         }
 166
 167         cxgbit_get_cnp(cnp);
 168         cxgbit_init_wr_wait(&cnp->com.wr_wait);
 169
 170         ret = cxgb4_create_server6(cdev->lldi.ports[0],
 171                                    stid, &sin6->sin6_addr,
 172                                    sin6->sin6_port,
 173                                    cdev->lldi.rxq_ids[0]);
 174         if (!ret)
 175                 ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait,
 176                                             0, 10, __func__);
 177         else if (ret > 0)
 178                 ret = net_xmit_errno(ret);
 179         else
 180                 cxgbit_put_cnp(cnp);
 181
 182         if (ret) {
 183                 if (ret != -ETIMEDOUT)
 184                         cxgb4_clip_release(cdev->lldi.ports[0],
 185                                    (const u32 *)&sin6->sin6_addr.s6_addr, 1);
 186
 187                 pr_err("create server6 err %d stid %d laddr %pI6 lport %d\n",
 188                        ret, stid, sin6->sin6_addr.s6_addr,
 189                        ntohs(sin6->sin6_port));
 190         }
 191
 192         return ret;
 193 }
 194
 195 static int
 196 cxgbit_create_server4(struct cxgbit_device *cdev, unsigned int stid,
 197                       struct cxgbit_np *cnp)
 198 {
 199         struct sockaddr_in *sin = (struct sockaddr_in *)
 200                                    &cnp->com.local_addr;
 201         int ret;
 202
 203         pr_debug("%s: dev = %s; stid = %u; sin_port = %u\n",
 204                  __func__, cdev->lldi.ports[0]->name, stid, sin->sin_port);
 205
 206         cxgbit_get_cnp(cnp);
 207         cxgbit_init_wr_wait(&cnp->com.wr_wait);
 208
 209         ret = cxgb4_create_server(cdev->lldi.ports[0],
 210                                   stid, sin->sin_addr.s_addr,
 211                                   sin->sin_port, 0,
 212                                   cdev->lldi.rxq_ids[0]);
 213         if (!ret)
 214                 ret = cxgbit_wait_for_reply(cdev,
 215                                             &cnp->com.wr_wait,
 216                                             0, 10, __func__);
 217         else if (ret > 0)
 218                 ret = net_xmit_errno(ret);
 219         else
 220                 cxgbit_put_cnp(cnp);
 221
 222         if (ret)
 223                 pr_err("create server failed err %d stid %d laddr %pI4 lport %d\n",
 224                        ret, stid, &sin->sin_addr, ntohs(sin->sin_port));
 225         return ret;
 226 }
 227
 228 struct cxgbit_device *cxgbit_find_device(struct net_device *ndev, u8 *port_id)
 229 {
 230         struct cxgbit_device *cdev;
 231         u8 i;
 232
 233         list_for_each_entry(cdev, &cdev_list_head, list) {
 234                 struct cxgb4_lld_info *lldi = &cdev->lldi;
 235
 236                 for (i = 0; i < lldi->nports; i++) {
 237                         if (lldi->ports[i] == ndev) {
 238                                 if (port_id)
 239                                         *port_id = i;
 240                                 return cdev;
 241                         }
 242                 }
 243         }
 244
 245         return NULL;
 246 }
 247
 248 static struct net_device *cxgbit_get_real_dev(struct net_device *ndev)
 249 {
 250         if (ndev->priv_flags & IFF_BONDING) {
 251                 pr_err("Bond devices are not supported. Interface:%s\n",
 252                        ndev->name);
 253                 return NULL;
 254         }
 255
 256         if (is_vlan_dev(ndev))
 257                 return vlan_dev_real_dev(ndev);
 258
 259         return ndev;
 260 }
 261
 262 static struct net_device *cxgbit_ipv4_netdev(__be32 saddr)
 263 {
 264         struct net_device *ndev;
 265
 266         ndev = __ip_dev_find(&init_net, saddr, false);
 267         if (!ndev)
 268                 return NULL;
 269
 270         return cxgbit_get_real_dev(ndev);
 271 }
 272
 273 static struct net_device *cxgbit_ipv6_netdev(struct in6_addr *addr6)
 274 {
 275         struct net_device *ndev = NULL;
 276         bool found = false;
 277
 278         if (IS_ENABLED(CONFIG_IPV6)) {
 279                 for_each_netdev_rcu(&init_net, ndev)
 280                         if (ipv6_chk_addr(&init_net, addr6, ndev, 1)) {
 281                                 found = true;
 282                                 break;
 283                         }
 284         }
 285         if (!found)
 286                 return NULL;
 287         return cxgbit_get_real_dev(ndev);
 288 }
 289
 290 static struct cxgbit_device *cxgbit_find_np_cdev(struct cxgbit_np *cnp)
 291 {
 292         struct sockaddr_storage *sockaddr = &cnp->com.local_addr;
 293         int ss_family = sockaddr->ss_family;
 294         struct net_device *ndev = NULL;
 295         struct cxgbit_device *cdev = NULL;
 296
 297         rcu_read_lock();
 298         if (ss_family == AF_INET) {
 299                 struct sockaddr_in *sin;
 300
 301                 sin = (struct sockaddr_in *)sockaddr;
 302                 ndev = cxgbit_ipv4_netdev(sin->sin_addr.s_addr);
 303         } else if (ss_family == AF_INET6) {
 304                 struct sockaddr_in6 *sin6;
 305
 306                 sin6 = (struct sockaddr_in6 *)sockaddr;
 307                 ndev = cxgbit_ipv6_netdev(&sin6->sin6_addr);
 308         }
 309         if (!ndev)
 310                 goto out;
 311
 312         cdev = cxgbit_find_device(ndev, NULL);
 313 out:
 314         rcu_read_unlock();
 315         return cdev;
 316 }
 317
 318 static bool cxgbit_inaddr_any(struct cxgbit_np *cnp)
 319 {
 320         struct sockaddr_storage *sockaddr = &cnp->com.local_addr;
 321         int ss_family = sockaddr->ss_family;
 322         int addr_type;
 323
 324         if (ss_family == AF_INET) {
 325                 struct sockaddr_in *sin;
 326
 327                 sin = (struct sockaddr_in *)sockaddr;
 328                 if (sin->sin_addr.s_addr == htonl(INADDR_ANY))
 329                         return true;
 330         } else if (ss_family == AF_INET6) {
 331                 struct sockaddr_in6 *sin6;
 332
 333                 sin6 = (struct sockaddr_in6 *)sockaddr;
 334                 addr_type = ipv6_addr_type((const struct in6_addr *)
 335                                 &sin6->sin6_addr);
 336                 if (addr_type == IPV6_ADDR_ANY)
 337                         return true;
 338         }
 339         return false;
 340 }
 341
 342 static int
 343 __cxgbit_setup_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp)
 344 {
 345         int stid, ret;
 346         int ss_family = cnp->com.local_addr.ss_family;
 347
 348         if (!test_bit(CDEV_STATE_UP, &cdev->flags))
 349                 return -EINVAL;
 350
 351         stid = cxgb4_alloc_stid(cdev->lldi.tids, ss_family, cnp);
 352         if (stid < 0)
 353                 return -EINVAL;
 354
 355         if (!cxgbit_np_hash_add(cdev, cnp, stid)) {
 356                 cxgb4_free_stid(cdev->lldi.tids, stid, ss_family);
 357                 return -EINVAL;
 358         }
 359
 360         if (ss_family == AF_INET)
 361                 ret = cxgbit_create_server4(cdev, stid, cnp);
 362         else
 363                 ret = cxgbit_create_server6(cdev, stid, cnp);
 364
 365         if (ret) {
 366                 if (ret != -ETIMEDOUT)
 367                         cxgb4_free_stid(cdev->lldi.tids, stid,
 368                                         ss_family);
 369                 cxgbit_np_hash_del(cdev, cnp);
 370                 return ret;
 371         }
 372         return ret;
 373 }
 374
 375 static int cxgbit_setup_cdev_np(struct cxgbit_np *cnp)
 376 {
 377         struct cxgbit_device *cdev;
 378         int ret = -1;
 379
 380         mutex_lock(&cdev_list_lock);
 381         cdev = cxgbit_find_np_cdev(cnp);
 382         if (!cdev)
 383                 goto out;
 384
 385         if (cxgbit_np_hash_find(cdev, cnp) >= 0)
 386                 goto out;
 387
 388         if (__cxgbit_setup_cdev_np(cdev, cnp))
 389                 goto out;
 390
 391         cnp->com.cdev = cdev;
 392         ret = 0;
 393 out:
 394         mutex_unlock(&cdev_list_lock);
 395         return ret;
 396 }
 397
 398 static int cxgbit_setup_all_np(struct cxgbit_np *cnp)
 399 {
 400         struct cxgbit_device *cdev;
 401         int ret;
 402         u32 count = 0;
 403
 404         mutex_lock(&cdev_list_lock);
 405         list_for_each_entry(cdev, &cdev_list_head, list) {
 406                 if (cxgbit_np_hash_find(cdev, cnp) >= 0) {
 407                         mutex_unlock(&cdev_list_lock);
 408                         return -1;
 409                 }
 410         }
 411
 412         list_for_each_entry(cdev, &cdev_list_head, list) {
 413                 ret = __cxgbit_setup_cdev_np(cdev, cnp);
 414                 if (ret == -ETIMEDOUT)
 415                         break;
 416                 if (ret != 0)
 417                         continue;
 418                 count++;
 419         }
 420         mutex_unlock(&cdev_list_lock);
 421
 422         return count ? 0 : -1;
 423 }
 424
 425 int cxgbit_setup_np(struct iscsi_np *np, struct sockaddr_storage *ksockaddr)
 426 {
 427         struct cxgbit_np *cnp;
 428         int ret;
 429
 430         if ((ksockaddr->ss_family != AF_INET) &&
 431             (ksockaddr->ss_family != AF_INET6))
 432                 return -EINVAL;
 433
 434         cnp = kzalloc(sizeof(*cnp), GFP_KERNEL);
 435         if (!cnp)
 436                 return -ENOMEM;
 437
 438         init_waitqueue_head(&cnp->accept_wait);
 439         init_completion(&cnp->com.wr_wait.completion);
 440         init_completion(&cnp->accept_comp);
 441         INIT_LIST_HEAD(&cnp->np_accept_list);
 442         spin_lock_init(&cnp->np_accept_lock);
 443         kref_init(&cnp->kref);
 444         memcpy(&np->np_sockaddr, ksockaddr,
 445                sizeof(struct sockaddr_storage));
 446         memcpy(&cnp->com.local_addr, &np->np_sockaddr,
 447                sizeof(cnp->com.local_addr));
 448
 449         cnp->np = np;
 450         cnp->com.cdev = NULL;
 451
 452         if (cxgbit_inaddr_any(cnp))
 453                 ret = cxgbit_setup_all_np(cnp);
 454         else
 455                 ret = cxgbit_setup_cdev_np(cnp);
 456
 457         if (ret) {
 458                 cxgbit_put_cnp(cnp);
 459                 return -EINVAL;
 460         }
 461
 462         np->np_context = cnp;
 463         cnp->com.state = CSK_STATE_LISTEN;
 464         return 0;
 465 }
 466
 467 static void
 468 cxgbit_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
 469                      struct cxgbit_sock *csk)
 470 {
 471         conn->login_family = np->np_sockaddr.ss_family;
 472         conn->login_sockaddr = csk->com.remote_addr;
 473         conn->local_sockaddr = csk->com.local_addr;
 474 }
 475
 476 int cxgbit_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
 477 {
 478         struct cxgbit_np *cnp = np->np_context;
 479         struct cxgbit_sock *csk;
 480         int ret = 0;
 481
 482 accept_wait:
 483         ret = wait_for_completion_interruptible(&cnp->accept_comp);
 484         if (ret)
 485                 return -ENODEV;
 486
 487         spin_lock_bh(&np->np_thread_lock);
 488         if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
 489                 spin_unlock_bh(&np->np_thread_lock);
 490                 /**
 491                  * No point in stalling here when np_thread
 492                  * is in state RESET/SHUTDOWN/EXIT - bail
 493                  **/
 494                 return -ENODEV;
 495         }
 496         spin_unlock_bh(&np->np_thread_lock);
 497
 498         spin_lock_bh(&cnp->np_accept_lock);
 499         if (list_empty(&cnp->np_accept_list)) {
 500                 spin_unlock_bh(&cnp->np_accept_lock);
 501                 goto accept_wait;
 502         }
 503
 504         csk = list_first_entry(&cnp->np_accept_list,
 505                                struct cxgbit_sock,
 506                                accept_node);
 507
 508         list_del_init(&csk->accept_node);
 509         spin_unlock_bh(&cnp->np_accept_lock);
 510         conn->context = csk;
 511         csk->conn = conn;
 512
 513         cxgbit_set_conn_info(np, conn, csk);
 514         return 0;
 515 }
 516
 517 static int
 518 __cxgbit_free_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp)
 519 {
 520         int stid, ret;
 521         bool ipv6 = false;
 522
 523         stid = cxgbit_np_hash_del(cdev, cnp);
 524         if (stid < 0)
 525                 return -EINVAL;
 526         if (!test_bit(CDEV_STATE_UP, &cdev->flags))
 527                 return -EINVAL;
 528
 529         if (cnp->np->np_sockaddr.ss_family == AF_INET6)
 530                 ipv6 = true;
 531
 532         cxgbit_get_cnp(cnp);
 533         cxgbit_init_wr_wait(&cnp->com.wr_wait);
 534         ret = cxgb4_remove_server(cdev->lldi.ports[0], stid,
 535                                   cdev->lldi.rxq_ids[0], ipv6);
 536
 537         if (ret > 0)
 538                 ret = net_xmit_errno(ret);
 539
 540         if (ret) {
 541                 cxgbit_put_cnp(cnp);
 542                 return ret;
 543         }
 544
 545         ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait,
 546                                     0, 10, __func__);
 547         if (ret == -ETIMEDOUT)
 548                 return ret;
 549
 550         if (ipv6 && cnp->com.cdev) {
 551                 struct sockaddr_in6 *sin6;
 552
 553                 sin6 = (struct sockaddr_in6 *)&cnp->com.local_addr;
 554                 cxgb4_clip_release(cdev->lldi.ports[0],
 555                                    (const u32 *)&sin6->sin6_addr.s6_addr,
 556                                    1);
 557         }
 558
 559         cxgb4_free_stid(cdev->lldi.tids, stid,
 560                         cnp->com.local_addr.ss_family);
 561         return 0;
 562 }
 563
 564 static void cxgbit_free_all_np(struct cxgbit_np *cnp)
 565 {
 566         struct cxgbit_device *cdev;
 567         int ret;
 568
 569         mutex_lock(&cdev_list_lock);
 570         list_for_each_entry(cdev, &cdev_list_head, list) {
 571                 ret = __cxgbit_free_cdev_np(cdev, cnp);
 572                 if (ret == -ETIMEDOUT)
 573                         break;
 574         }
 575         mutex_unlock(&cdev_list_lock);
 576 }
 577
 578 static void cxgbit_free_cdev_np(struct cxgbit_np *cnp)
 579 {
 580         struct cxgbit_device *cdev;
 581         bool found = false;
 582
 583         mutex_lock(&cdev_list_lock);
 584         list_for_each_entry(cdev, &cdev_list_head, list) {
 585                 if (cdev == cnp->com.cdev) {
 586                         found = true;
 587                         break;
 588                 }
 589         }
 590         if (!found)
 591                 goto out;
 592
 593         __cxgbit_free_cdev_np(cdev, cnp);
 594 out:
 595         mutex_unlock(&cdev_list_lock);
 596 }
 597
 598 static void __cxgbit_free_conn(struct cxgbit_sock *csk);
 599
 600 void cxgbit_free_np(struct iscsi_np *np)
 601 {
 602         struct cxgbit_np *cnp = np->np_context;
 603         struct cxgbit_sock *csk, *tmp;
 604
 605         cnp->com.state = CSK_STATE_DEAD;
 606         if (cnp->com.cdev)
 607                 cxgbit_free_cdev_np(cnp);
 608         else
 609                 cxgbit_free_all_np(cnp);
 610
 611         spin_lock_bh(&cnp->np_accept_lock);
 612         list_for_each_entry_safe(csk, tmp, &cnp->np_accept_list, accept_node) {
 613                 list_del_init(&csk->accept_node);
 614                 __cxgbit_free_conn(csk);
 615         }
 616         spin_unlock_bh(&cnp->np_accept_lock);
 617
 618         np->np_context = NULL;
 619         cxgbit_put_cnp(cnp);
 620 }
 621
 622 static void cxgbit_send_halfclose(struct cxgbit_sock *csk)
 623 {
 624         struct sk_buff *skb;
 625         u32 len = roundup(sizeof(struct cpl_close_con_req), 16);
 626
 627         skb = alloc_skb(len, GFP_ATOMIC);
 628         if (!skb)
 629                 return;
 630
 631         cxgb_mk_close_con_req(skb, len, csk->tid, csk->txq_idx,
 632                               NULL, NULL);
 633
 634         cxgbit_skcb_flags(skb) |= SKCBF_TX_FLAG_COMPL;
 635         __skb_queue_tail(&csk->txq, skb);
 636         cxgbit_push_tx_frames(csk);
 637 }
 638
 639 static void cxgbit_arp_failure_discard(void *handle, struct sk_buff *skb)
 640 {
 641         struct cxgbit_sock *csk = handle;
 642
 643         pr_debug("%s cxgbit_device %p\n", __func__, handle);
 644         kfree_skb(skb);
 645         cxgbit_put_csk(csk);
 646 }
 647
 648 static void cxgbit_abort_arp_failure(void *handle, struct sk_buff *skb)
 649 {
 650         struct cxgbit_device *cdev = handle;
 651         struct cpl_abort_req *req = cplhdr(skb);
 652
 653         pr_debug("%s cdev %p\n", __func__, cdev);
 654         req->cmd = CPL_ABORT_NO_RST;
 655         cxgbit_ofld_send(cdev, skb);
 656 }
 657
 658 static int cxgbit_send_abort_req(struct cxgbit_sock *csk)
 659 {
 660         struct sk_buff *skb;
 661         u32 len = roundup(sizeof(struct cpl_abort_req), 16);
 662
 663         pr_debug("%s: csk %p tid %u; state %d\n",
 664                  __func__, csk, csk->tid, csk->com.state);
 665
 666         __skb_queue_purge(&csk->txq);
 667
 668         if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags))
 669                 cxgbit_send_tx_flowc_wr(csk);
 670
 671         skb = __skb_dequeue(&csk->skbq);
 672         cxgb_mk_abort_req(skb, len, csk->tid, csk->txq_idx,
 673                           csk->com.cdev, cxgbit_abort_arp_failure);
 674
 675         return cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
 676 }
 677
 678 static void
 679 __cxgbit_abort_conn(struct cxgbit_sock *csk, struct sk_buff *skb)
 680 {
 681         __kfree_skb(skb);
 682
 683         if (csk->com.state != CSK_STATE_ESTABLISHED)
 684                 goto no_abort;
 685
 686         set_bit(CSK_ABORT_RPL_WAIT, &csk->com.flags);
 687         csk->com.state = CSK_STATE_ABORTING;
 688
 689         cxgbit_send_abort_req(csk);
 690
 691         return;
 692
 693 no_abort:
 694         cxgbit_wake_up(&csk->com.wr_wait, __func__, CPL_ERR_NONE);
 695         cxgbit_put_csk(csk);
 696 }
 697
 698 void cxgbit_abort_conn(struct cxgbit_sock *csk)
 699 {
 700         struct sk_buff *skb = alloc_skb(0, GFP_KERNEL | __GFP_NOFAIL);
 701
 702         cxgbit_get_csk(csk);
 703         cxgbit_init_wr_wait(&csk->com.wr_wait);
 704
 705         spin_lock_bh(&csk->lock);
 706         if (csk->lock_owner) {
 707                 cxgbit_skcb_rx_backlog_fn(skb) = __cxgbit_abort_conn;
 708                 __skb_queue_tail(&csk->backlogq, skb);
 709         } else {
 710                 __cxgbit_abort_conn(csk, skb);
 711         }
 712         spin_unlock_bh(&csk->lock);
 713
 714         cxgbit_wait_for_reply(csk->com.cdev, &csk->com.wr_wait,
 715                               csk->tid, 600, __func__);
 716 }
 717
 718 static void __cxgbit_free_conn(struct cxgbit_sock *csk)
 719 {
 720         struct iscsi_conn *conn = csk->conn;
 721         bool release = false;
 722
 723         pr_debug("%s: state %d\n",
 724                  __func__, csk->com.state);
 725
 726         spin_lock_bh(&csk->lock);
 727         switch (csk->com.state) {
 728         case CSK_STATE_ESTABLISHED:
 729                 if (conn && (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)) {
 730                         csk->com.state = CSK_STATE_CLOSING;
 731                         cxgbit_send_halfclose(csk);
 732                 } else {
 733                         csk->com.state = CSK_STATE_ABORTING;
 734                         cxgbit_send_abort_req(csk);
 735                 }
 736                 break;
 737         case CSK_STATE_CLOSING:
 738                 csk->com.state = CSK_STATE_MORIBUND;
 739                 cxgbit_send_halfclose(csk);
 740                 break;
 741         case CSK_STATE_DEAD:
 742                 release = true;
 743                 break;
 744         default:
 745                 pr_err("%s: csk %p; state %d\n",
 746                        __func__, csk, csk->com.state);
 747         }
 748         spin_unlock_bh(&csk->lock);
 749
 750         if (release)
 751                 cxgbit_put_csk(csk);
 752 }
 753
 754 void cxgbit_free_conn(struct iscsi_conn *conn)
 755 {
 756         __cxgbit_free_conn(conn->context);
 757 }
 758
 759 static void cxgbit_set_emss(struct cxgbit_sock *csk, u16 opt)
 760 {
 761         csk->emss = csk->com.cdev->lldi.mtus[TCPOPT_MSS_G(opt)] -
 762                         ((csk->com.remote_addr.ss_family == AF_INET) ?
 763                         sizeof(struct iphdr) : sizeof(struct ipv6hdr)) -
 764                         sizeof(struct tcphdr);
 765         csk->mss = csk->emss;
 766         if (TCPOPT_TSTAMP_G(opt))
 767                 csk->emss -= round_up(TCPOLEN_TIMESTAMP, 4);
 768         if (csk->emss < 128)
 769                 csk->emss = 128;
 770         if (csk->emss & 7)
 771                 pr_info("Warning: misaligned mtu idx %u mss %u emss=%u\n",
 772                         TCPOPT_MSS_G(opt), csk->mss, csk->emss);
 773         pr_debug("%s mss_idx %u mss %u emss=%u\n", __func__, TCPOPT_MSS_G(opt),
 774                  csk->mss, csk->emss);
 775 }
 776
 777 static void cxgbit_free_skb(struct cxgbit_sock *csk)
 778 {
 779         struct sk_buff *skb;
 780
 781         __skb_queue_purge(&csk->txq);
 782         __skb_queue_purge(&csk->rxq);
 783         __skb_queue_purge(&csk->backlogq);
 784         __skb_queue_purge(&csk->ppodq);
 785         __skb_queue_purge(&csk->skbq);
 786
 787         while ((skb = cxgbit_sock_dequeue_wr(csk)))
 788                 kfree_skb(skb);
 789
 790         __kfree_skb(csk->lro_hskb);
 791 }
 792
 793 void _cxgbit_free_csk(struct kref *kref)
 794 {
 795         struct cxgbit_sock *csk;
 796         struct cxgbit_device *cdev;
 797
 798         csk = container_of(kref, struct cxgbit_sock, kref);
 799
 800         pr_debug("%s csk %p state %d\n", __func__, csk, csk->com.state);
 801
 802         if (csk->com.local_addr.ss_family == AF_INET6) {
 803                 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)
 804                                              &csk->com.local_addr;
 805                 cxgb4_clip_release(csk->com.cdev->lldi.ports[0],
 806                                    (const u32 *)
 807                                    &sin6->sin6_addr.s6_addr, 1);
 808         }
 809
 810         cxgb4_remove_tid(csk->com.cdev->lldi.tids, 0, csk->tid,
 811                          csk->com.local_addr.ss_family);
 812         dst_release(csk->dst);
 813         cxgb4_l2t_release(csk->l2t);
 814
 815         cdev = csk->com.cdev;
 816         spin_lock_bh(&cdev->cskq.lock);
 817         list_del(&csk->list);
 818         spin_unlock_bh(&cdev->cskq.lock);
 819
 820         cxgbit_free_skb(csk);
 821         cxgbit_put_cnp(csk->cnp);
 822         cxgbit_put_cdev(cdev);
 823
 824         kfree(csk);
 825 }
 826
 827 static void cxgbit_set_tcp_window(struct cxgbit_sock *csk, struct port_info *pi)
 828 {
 829         unsigned int linkspeed;
 830         u8 scale;
 831
 832         linkspeed = pi->link_cfg.speed;
 833         scale = linkspeed / SPEED_10000;
 834
 835 #define CXGBIT_10G_RCV_WIN (256 * 1024)
 836         csk->rcv_win = CXGBIT_10G_RCV_WIN;
 837         if (scale)
 838                 csk->rcv_win *= scale;
 839
 840 #define CXGBIT_10G_SND_WIN (256 * 1024)
 841         csk->snd_win = CXGBIT_10G_SND_WIN;
 842         if (scale)
 843                 csk->snd_win *= scale;
 844
 845         pr_debug("%s snd_win %d rcv_win %d\n",
 846                  __func__, csk->snd_win, csk->rcv_win);
 847 }
 848
 849 #ifdef CONFIG_CHELSIO_T4_DCB
 850 static u8 cxgbit_get_iscsi_dcb_state(struct net_device *ndev)
 851 {
 852         return ndev->dcbnl_ops->getstate(ndev);
 853 }
 854
 855 static int cxgbit_select_priority(int pri_mask)
 856 {
 857         if (!pri_mask)
 858                 return 0;
 859
 860         return (ffs(pri_mask) - 1);
 861 }
 862
 863 static u8 cxgbit_get_iscsi_dcb_priority(struct net_device *ndev, u16 local_port)
 864 {
 865         int ret;
 866         u8 caps;
 867
 868         struct dcb_app iscsi_dcb_app = {
 869                 .protocol = local_port
 870         };
 871
 872         ret = (int)ndev->dcbnl_ops->getcap(ndev, DCB_CAP_ATTR_DCBX, &caps);
 873
 874         if (ret)
 875                 return 0;
 876
 877         if (caps & DCB_CAP_DCBX_VER_IEEE) {
 878                 iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_STREAM;
 879                 ret = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app);
 880                 if (!ret) {
 881                         iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_ANY;
 882                         ret = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app);
 883                 }
 884         } else if (caps & DCB_CAP_DCBX_VER_CEE) {
 885                 iscsi_dcb_app.selector = DCB_APP_IDTYPE_PORTNUM;
 886
 887                 ret = dcb_getapp(ndev, &iscsi_dcb_app);
 888         }
 889
 890         pr_info("iSCSI priority is set to %u\n", cxgbit_select_priority(ret));
 891
 892         return cxgbit_select_priority(ret);
 893 }
 894 #endif
 895
 896 static int
 897 cxgbit_offload_init(struct cxgbit_sock *csk, int iptype, __u8 *peer_ip,
 898                     u16 local_port, struct dst_entry *dst,
 899                     struct cxgbit_device *cdev)
 900 {
 901         struct neighbour *n;
 902         int ret, step;
 903         struct net_device *ndev;
 904         u16 rxq_idx, port_id;
 905 #ifdef CONFIG_CHELSIO_T4_DCB
 906         u8 priority = 0;
 907 #endif
 908
 909         n = dst_neigh_lookup(dst, peer_ip);
 910         if (!n)
 911                 return -ENODEV;
 912
 913         rcu_read_lock();
 914         if (!(n->nud_state & NUD_VALID))
 915                 neigh_event_send(n, NULL);
 916
 917         ret = -ENOMEM;
 918         if (n->dev->flags & IFF_LOOPBACK) {
 919                 if (iptype == 4)
 920                         ndev = cxgbit_ipv4_netdev(*(__be32 *)peer_ip);
 921                 else if (IS_ENABLED(CONFIG_IPV6))
 922                         ndev = cxgbit_ipv6_netdev((struct in6_addr *)peer_ip);
 923                 else
 924                         ndev = NULL;
 925
 926                 if (!ndev) {
 927                         ret = -ENODEV;
 928                         goto out;
 929                 }
 930
 931                 csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t,
 932                                          n, ndev, 0);
 933                 if (!csk->l2t)
 934                         goto out;
 935                 csk->mtu = ndev->mtu;
 936                 csk->tx_chan = cxgb4_port_chan(ndev);
 937                 csk->smac_idx =
 938                                ((struct port_info *)netdev_priv(ndev))->smt_idx;
 939                 step = cdev->lldi.ntxq /
 940                         cdev->lldi.nchan;
 941                 csk->txq_idx = cxgb4_port_idx(ndev) * step;
 942                 step = cdev->lldi.nrxq /
 943                         cdev->lldi.nchan;
 944                 csk->ctrlq_idx = cxgb4_port_idx(ndev);
 945                 csk->rss_qid = cdev->lldi.rxq_ids[
 946                                 cxgb4_port_idx(ndev) * step];
 947                 csk->port_id = cxgb4_port_idx(ndev);
 948                 cxgbit_set_tcp_window(csk,
 949                                       (struct port_info *)netdev_priv(ndev));
 950         } else {
 951                 ndev = cxgbit_get_real_dev(n->dev);
 952                 if (!ndev) {
 953                         ret = -ENODEV;
 954                         goto out;
 955                 }
 956
 957 #ifdef CONFIG_CHELSIO_T4_DCB
 958                 if (cxgbit_get_iscsi_dcb_state(ndev))
 959                         priority = cxgbit_get_iscsi_dcb_priority(ndev,
 960                                                                  local_port);
 961
 962                 csk->dcb_priority = priority;
 963
 964                 csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, priority);
 965 #else
 966                 csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, 0);
 967 #endif
 968                 if (!csk->l2t)
 969                         goto out;
 970                 port_id = cxgb4_port_idx(ndev);
 971                 csk->mtu = dst_mtu(dst);
 972                 csk->tx_chan = cxgb4_port_chan(ndev);
 973                 csk->smac_idx =
 974                                ((struct port_info *)netdev_priv(ndev))->smt_idx;
 975                 step = cdev->lldi.ntxq /
 976                         cdev->lldi.nports;
 977                 csk->txq_idx = (port_id * step) +
 978                                 (cdev->selectq[port_id][0]++ % step);
 979                 csk->ctrlq_idx = cxgb4_port_idx(ndev);
 980                 step = cdev->lldi.nrxq /
 981                         cdev->lldi.nports;
 982                 rxq_idx = (port_id * step) +
 983                                 (cdev->selectq[port_id][1]++ % step);
 984                 csk->rss_qid = cdev->lldi.rxq_ids[rxq_idx];
 985                 csk->port_id = port_id;
 986                 cxgbit_set_tcp_window(csk,
 987                                       (struct port_info *)netdev_priv(ndev));
 988         }
 989         ret = 0;
 990 out:
 991         rcu_read_unlock();
 992         neigh_release(n);
 993         return ret;
 994 }
 995
 996 int cxgbit_ofld_send(struct cxgbit_device *cdev, struct sk_buff *skb)
 997 {
 998         int ret = 0;
 999
1000         if (!test_bit(CDEV_STATE_UP, &cdev->flags)) {
1001                 kfree_skb(skb);
1002                 pr_err("%s - device not up - dropping\n", __func__);
1003                 return -EIO;
1004         }
1005
1006         ret = cxgb4_ofld_send(cdev->lldi.ports[0], skb);
1007         if (ret < 0)
1008                 kfree_skb(skb);
1009         return ret < 0 ? ret : 0;
1010 }
1011
1012 static void cxgbit_release_tid(struct cxgbit_device *cdev, u32 tid)
1013 {
1014         u32 len = roundup(sizeof(struct cpl_tid_release), 16);
1015         struct sk_buff *skb;
1016
1017         skb = alloc_skb(len, GFP_ATOMIC);
1018         if (!skb)
1019                 return;
1020
1021         cxgb_mk_tid_release(skb, len, tid, 0);
1022         cxgbit_ofld_send(cdev, skb);
1023 }
1024
1025 int
1026 cxgbit_l2t_send(struct cxgbit_device *cdev, struct sk_buff *skb,
1027                 struct l2t_entry *l2e)
1028 {
1029         int ret = 0;
1030
1031         if (!test_bit(CDEV_STATE_UP, &cdev->flags)) {
1032                 kfree_skb(skb);
1033                 pr_err("%s - device not up - dropping\n", __func__);
1034                 return -EIO;
1035         }
1036
1037         ret = cxgb4_l2t_send(cdev->lldi.ports[0], skb, l2e);
1038         if (ret < 0)
1039                 kfree_skb(skb);
1040         return ret < 0 ? ret : 0;
1041 }
1042
1043 static void cxgbit_send_rx_credits(struct cxgbit_sock *csk, struct sk_buff *skb)
1044 {
1045         if (csk->com.state != CSK_STATE_ESTABLISHED) {
1046                 __kfree_skb(skb);
1047                 return;
1048         }
1049
1050         cxgbit_ofld_send(csk->com.cdev, skb);
1051 }
1052
1053 /*
1054  * CPL connection rx data ack: host ->
1055  * Send RX credits through an RX_DATA_ACK CPL message.
1056  * Returns the number of credits sent.
1057  */
1058 int cxgbit_rx_data_ack(struct cxgbit_sock *csk)
1059 {
1060         struct sk_buff *skb;
1061         u32 len = roundup(sizeof(struct cpl_rx_data_ack), 16);
1062         u32 credit_dack;
1063
1064         skb = alloc_skb(len, GFP_KERNEL);
1065         if (!skb)
1066                 return -1;
1067
1068         credit_dack = RX_DACK_CHANGE_F | RX_DACK_MODE_V(1) |
1069                       RX_CREDITS_V(csk->rx_credits);
1070
1071         cxgb_mk_rx_data_ack(skb, len, csk->tid, csk->ctrlq_idx,
1072                             credit_dack);
1073
1074         csk->rx_credits = 0;
1075
1076         spin_lock_bh(&csk->lock);
1077         if (csk->lock_owner) {
1078                 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_send_rx_credits;
1079                 __skb_queue_tail(&csk->backlogq, skb);
1080                 spin_unlock_bh(&csk->lock);
1081                 return 0;
1082         }
1083
1084         cxgbit_send_rx_credits(csk, skb);
1085         spin_unlock_bh(&csk->lock);
1086
1087         return 0;
1088 }
1089
1090 #define FLOWC_WR_NPARAMS_MIN    9
1091 #define FLOWC_WR_NPARAMS_MAX    11
1092 static int cxgbit_alloc_csk_skb(struct cxgbit_sock *csk)
1093 {
1094         struct sk_buff *skb;
1095         u32 len, flowclen;
1096         u8 i;
1097
1098         flowclen = offsetof(struct fw_flowc_wr,
1099                             mnemval[FLOWC_WR_NPARAMS_MAX]);
1100
1101         len = max_t(u32, sizeof(struct cpl_abort_req),
1102                     sizeof(struct cpl_abort_rpl));
1103
1104         len = max(len, flowclen);
1105         len = roundup(len, 16);
1106
1107         for (i = 0; i < 3; i++) {
1108                 skb = alloc_skb(len, GFP_ATOMIC);
1109                 if (!skb)
1110                         goto out;
1111                 __skb_queue_tail(&csk->skbq, skb);
1112         }
1113
1114         skb = alloc_skb(LRO_SKB_MIN_HEADROOM, GFP_ATOMIC);
1115         if (!skb)
1116                 goto out;
1117
1118         memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1119         csk->lro_hskb = skb;
1120
1121         return 0;
1122 out:
1123         __skb_queue_purge(&csk->skbq);
1124         return -ENOMEM;
1125 }
1126
1127 static void
1128 cxgbit_pass_accept_rpl(struct cxgbit_sock *csk, struct cpl_pass_accept_req *req)
1129 {
1130         struct sk_buff *skb;
1131         const struct tcphdr *tcph;
1132         struct cpl_t5_pass_accept_rpl *rpl5;
1133         struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1134         unsigned int len = roundup(sizeof(*rpl5), 16);
1135         unsigned int mtu_idx;
1136         u64 opt0;
1137         u32 opt2, hlen;
1138         u32 wscale;
1139         u32 win;
1140
1141         pr_debug("%s csk %p tid %u\n", __func__, csk, csk->tid);
1142
1143         skb = alloc_skb(len, GFP_ATOMIC);
1144         if (!skb) {
1145                 cxgbit_put_csk(csk);
1146                 return;
1147         }
1148
1149         rpl5 = __skb_put_zero(skb, len);
1150
1151         INIT_TP_WR(rpl5, csk->tid);
1152         OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
1153                                                      csk->tid));
1154         cxgb_best_mtu(csk->com.cdev->lldi.mtus, csk->mtu, &mtu_idx,
1155                       req->tcpopt.tstamp,
1156                       (csk->com.remote_addr.ss_family == AF_INET) ? 0 : 1);
1157         wscale = cxgb_compute_wscale(csk->rcv_win);
1158         /*
1159          * Specify the largest window that will fit in opt0. The
1160          * remainder will be specified in the rx_data_ack.
1161          */
1162         win = csk->rcv_win >> 10;
1163         if (win > RCV_BUFSIZ_M)
1164                 win = RCV_BUFSIZ_M;
1165         opt0 =  TCAM_BYPASS_F |
1166                 WND_SCALE_V(wscale) |
1167                 MSS_IDX_V(mtu_idx) |
1168                 L2T_IDX_V(csk->l2t->idx) |
1169                 TX_CHAN_V(csk->tx_chan) |
1170                 SMAC_SEL_V(csk->smac_idx) |
1171                 DSCP_V(csk->tos >> 2) |
1172                 ULP_MODE_V(ULP_MODE_ISCSI) |
1173                 RCV_BUFSIZ_V(win);
1174
1175         opt2 = RX_CHANNEL_V(0) |
1176                 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
1177
1178         if (!is_t5(lldi->adapter_type))
1179                 opt2 |= RX_FC_DISABLE_F;
1180
1181         if (req->tcpopt.tstamp)
1182                 opt2 |= TSTAMPS_EN_F;
1183         if (req->tcpopt.sack)
1184                 opt2 |= SACK_EN_F;
1185         if (wscale)
1186                 opt2 |= WND_SCALE_EN_F;
1187
1188         hlen = ntohl(req->hdr_len);
1189
1190         if (is_t5(lldi->adapter_type))
1191                 tcph = (struct tcphdr *)((u8 *)(req + 1) +
1192                        ETH_HDR_LEN_G(hlen) + IP_HDR_LEN_G(hlen));
1193         else
1194                 tcph = (struct tcphdr *)((u8 *)(req + 1) +
1195                        T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
1196
1197         if (tcph->ece && tcph->cwr)
1198                 opt2 |= CCTRL_ECN_V(1);
1199
1200         opt2 |= RX_COALESCE_V(3);
1201         opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
1202
1203         opt2 |= T5_ISS_F;
1204         rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1);
1205
1206         opt2 |= T5_OPT_2_VALID_F;
1207
1208         rpl5->opt0 = cpu_to_be64(opt0);
1209         rpl5->opt2 = cpu_to_be32(opt2);
1210         set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->ctrlq_idx);
1211         t4_set_arp_err_handler(skb, csk, cxgbit_arp_failure_discard);
1212         cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
1213 }
1214
1215 static void
1216 cxgbit_pass_accept_req(struct cxgbit_device *cdev, struct sk_buff *skb)
1217 {
1218         struct cxgbit_sock *csk = NULL;
1219         struct cxgbit_np *cnp;
1220         struct cpl_pass_accept_req *req = cplhdr(skb);
1221         unsigned int stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1222         struct tid_info *t = cdev->lldi.tids;
1223         unsigned int tid = GET_TID(req);
1224         u16 peer_mss = ntohs(req->tcpopt.mss);
1225         unsigned short hdrs;
1226
1227         struct dst_entry *dst;
1228         __u8 local_ip[16], peer_ip[16];
1229         __be16 local_port, peer_port;
1230         int ret;
1231         int iptype;
1232
1233         pr_debug("%s: cdev = %p; stid = %u; tid = %u\n",
1234                  __func__, cdev, stid, tid);
1235
1236         cnp = lookup_stid(t, stid);
1237         if (!cnp) {
1238                 pr_err("%s connect request on invalid stid %d\n",
1239                        __func__, stid);
1240                 goto rel_skb;
1241         }
1242
1243         if (cnp->com.state != CSK_STATE_LISTEN) {
1244                 pr_err("%s - listening parent not in CSK_STATE_LISTEN\n",
1245                        __func__);
1246                 goto reject;
1247         }
1248
1249         csk = lookup_tid(t, tid);
1250         if (csk) {
1251                 pr_err("%s csk not null tid %u\n",
1252                        __func__, tid);
1253                 goto rel_skb;
1254         }
1255
1256         cxgb_get_4tuple(req, cdev->lldi.adapter_type, &iptype, local_ip,
1257                         peer_ip, &local_port, &peer_port);
1258
1259         /* Find output route */
1260         if (iptype == 4)  {
1261                 pr_debug("%s parent sock %p tid %u laddr %pI4 raddr %pI4 "
1262                          "lport %d rport %d peer_mss %d\n"
1263                          , __func__, cnp, tid,
1264                          local_ip, peer_ip, ntohs(local_port),
1265                          ntohs(peer_port), peer_mss);
1266                 dst = cxgb_find_route(&cdev->lldi, cxgbit_get_real_dev,
1267                                       *(__be32 *)local_ip,
1268                                       *(__be32 *)peer_ip,
1269                                       local_port, peer_port,
1270                                       PASS_OPEN_TOS_G(ntohl(req->tos_stid)));
1271         } else {
1272                 pr_debug("%s parent sock %p tid %u laddr %pI6 raddr %pI6 "
1273                          "lport %d rport %d peer_mss %d\n"
1274                          , __func__, cnp, tid,
1275                          local_ip, peer_ip, ntohs(local_port),
1276                          ntohs(peer_port), peer_mss);
1277                 dst = cxgb_find_route6(&cdev->lldi, cxgbit_get_real_dev,
1278                                        local_ip, peer_ip,
1279                                        local_port, peer_port,
1280                                        PASS_OPEN_TOS_G(ntohl(req->tos_stid)),
1281                                        ((struct sockaddr_in6 *)
1282                                         &cnp->com.local_addr)->sin6_scope_id);
1283         }
1284         if (!dst) {
1285                 pr_err("%s - failed to find dst entry!\n",
1286                        __func__);
1287                 goto reject;
1288         }
1289
1290         csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
1291         if (!csk) {
1292                 dst_release(dst);
1293                 goto rel_skb;
1294         }
1295
1296         ret = cxgbit_offload_init(csk, iptype, peer_ip, ntohs(local_port),
1297                                   dst, cdev);
1298         if (ret) {
1299                 pr_err("%s - failed to allocate l2t entry!\n",
1300                        __func__);
1301                 dst_release(dst);
1302                 kfree(csk);
1303                 goto reject;
1304         }
1305
1306         kref_init(&csk->kref);
1307         init_completion(&csk->com.wr_wait.completion);
1308
1309         INIT_LIST_HEAD(&csk->accept_node);
1310
1311         hdrs = (iptype == 4 ? sizeof(struct iphdr) : sizeof(struct ipv6hdr)) +
1312                 sizeof(struct tcphdr) + (req->tcpopt.tstamp ? 12 : 0);
1313         if (peer_mss && csk->mtu > (peer_mss + hdrs))
1314                 csk->mtu = peer_mss + hdrs;
1315
1316         csk->com.state = CSK_STATE_CONNECTING;
1317         csk->com.cdev = cdev;
1318         csk->cnp = cnp;
1319         csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
1320         csk->dst = dst;
1321         csk->tid = tid;
1322         csk->wr_cred = cdev->lldi.wr_cred -
1323                         DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16);
1324         csk->wr_max_cred = csk->wr_cred;
1325         csk->wr_una_cred = 0;
1326
1327         if (iptype == 4) {
1328                 struct sockaddr_in *sin = (struct sockaddr_in *)
1329                                           &csk->com.local_addr;
1330                 sin->sin_family = AF_INET;
1331                 sin->sin_port = local_port;
1332                 sin->sin_addr.s_addr = *(__be32 *)local_ip;
1333
1334                 sin = (struct sockaddr_in *)&csk->com.remote_addr;
1335                 sin->sin_family = AF_INET;
1336                 sin->sin_port = peer_port;
1337                 sin->sin_addr.s_addr = *(__be32 *)peer_ip;
1338         } else {
1339                 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)
1340                                             &csk->com.local_addr;
1341
1342                 sin6->sin6_family = PF_INET6;
1343                 sin6->sin6_port = local_port;
1344                 memcpy(sin6->sin6_addr.s6_addr, local_ip, 16);
1345                 cxgb4_clip_get(cdev->lldi.ports[0],
1346                                (const u32 *)&sin6->sin6_addr.s6_addr,
1347                                1);
1348
1349                 sin6 = (struct sockaddr_in6 *)&csk->com.remote_addr;
1350                 sin6->sin6_family = PF_INET6;
1351                 sin6->sin6_port = peer_port;
1352                 memcpy(sin6->sin6_addr.s6_addr, peer_ip, 16);
1353         }
1354
1355         skb_queue_head_init(&csk->rxq);
1356         skb_queue_head_init(&csk->txq);
1357         skb_queue_head_init(&csk->ppodq);
1358         skb_queue_head_init(&csk->backlogq);
1359         skb_queue_head_init(&csk->skbq);
1360         cxgbit_sock_reset_wr_list(csk);
1361         spin_lock_init(&csk->lock);
1362         init_waitqueue_head(&csk->waitq);
1363         init_waitqueue_head(&csk->ack_waitq);
1364         csk->lock_owner = false;
1365
1366         if (cxgbit_alloc_csk_skb(csk)) {
1367                 dst_release(dst);
1368                 kfree(csk);
1369                 goto rel_skb;
1370         }
1371
1372         cxgbit_get_cnp(cnp);
1373         cxgbit_get_cdev(cdev);
1374
1375         spin_lock(&cdev->cskq.lock);
1376         list_add_tail(&csk->list, &cdev->cskq.list);
1377         spin_unlock(&cdev->cskq.lock);
1378         cxgb4_insert_tid(t, csk, tid, csk->com.local_addr.ss_family);
1379         cxgbit_pass_accept_rpl(csk, req);
1380         goto rel_skb;
1381
1382 reject:
1383         cxgbit_release_tid(cdev, tid);
1384 rel_skb:
1385         __kfree_skb(skb);
1386 }
1387
1388 static u32
1389 cxgbit_tx_flowc_wr_credits(struct cxgbit_sock *csk, u32 *nparamsp,
1390                            u32 *flowclenp)
1391 {
1392         u32 nparams, flowclen16, flowclen;
1393
1394         nparams = FLOWC_WR_NPARAMS_MIN;
1395
1396         if (csk->snd_wscale)
1397                 nparams++;
1398
1399 #ifdef CONFIG_CHELSIO_T4_DCB
1400         nparams++;
1401 #endif
1402         flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
1403         flowclen16 = DIV_ROUND_UP(flowclen, 16);
1404         flowclen = flowclen16 * 16;
1405         /*
1406          * Return the number of 16-byte credits used by the flowc request.
1407          * Pass back the nparams and actual flowc length if requested.
1408          */
1409         if (nparamsp)
1410                 *nparamsp = nparams;
1411         if (flowclenp)
1412                 *flowclenp = flowclen;
1413         return flowclen16;
1414 }
1415
1416 u32 cxgbit_send_tx_flowc_wr(struct cxgbit_sock *csk)
1417 {
1418         struct cxgbit_device *cdev = csk->com.cdev;
1419         struct fw_flowc_wr *flowc;
1420         u32 nparams, flowclen16, flowclen;
1421         struct sk_buff *skb;
1422         u8 index;
1423
1424 #ifdef CONFIG_CHELSIO_T4_DCB
1425         u16 vlan = ((struct l2t_entry *)csk->l2t)->vlan;
1426 #endif
1427
1428         flowclen16 = cxgbit_tx_flowc_wr_credits(csk, &nparams, &flowclen);
1429
1430         skb = __skb_dequeue(&csk->skbq);
1431         flowc = __skb_put_zero(skb, flowclen);
1432
1433         flowc->op_to_nparams = cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) |
1434                                            FW_FLOWC_WR_NPARAMS_V(nparams));
1435         flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) |
1436                                           FW_WR_FLOWID_V(csk->tid));
1437         flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
1438         flowc->mnemval[0].val = cpu_to_be32(FW_PFVF_CMD_PFN_V
1439                                             (csk->com.cdev->lldi.pf));
1440         flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
1441         flowc->mnemval[1].val = cpu_to_be32(csk->tx_chan);
1442         flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
1443         flowc->mnemval[2].val = cpu_to_be32(csk->tx_chan);
1444         flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
1445         flowc->mnemval[3].val = cpu_to_be32(csk->rss_qid);
1446         flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
1447         flowc->mnemval[4].val = cpu_to_be32(csk->snd_nxt);
1448         flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
1449         flowc->mnemval[5].val = cpu_to_be32(csk->rcv_nxt);
1450         flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
1451         flowc->mnemval[6].val = cpu_to_be32(csk->snd_win);
1452         flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
1453         flowc->mnemval[7].val = cpu_to_be32(csk->emss);
1454
1455         flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;
1456         if (test_bit(CDEV_ISO_ENABLE, &cdev->flags))
1457                 flowc->mnemval[8].val = cpu_to_be32(CXGBIT_MAX_ISO_PAYLOAD);
1458         else
1459                 flowc->mnemval[8].val = cpu_to_be32(16384);
1460
1461         index = 9;
1462
1463         if (csk->snd_wscale) {
1464                 flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_RCV_SCALE;
1465                 flowc->mnemval[index].val = cpu_to_be32(csk->snd_wscale);
1466                 index++;
1467         }
1468
1469 #ifdef CONFIG_CHELSIO_T4_DCB
1470         flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_DCBPRIO;
1471         if (vlan == VLAN_NONE) {
1472                 pr_warn("csk %u without VLAN Tag on DCB Link\n", csk->tid);
1473                 flowc->mnemval[index].val = cpu_to_be32(0);
1474         } else
1475                 flowc->mnemval[index].val = cpu_to_be32(
1476                                 (vlan & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT);
1477 #endif
1478
1479         pr_debug("%s: csk %p; tx_chan = %u; rss_qid = %u; snd_seq = %u;"
1480                  " rcv_seq = %u; snd_win = %u; emss = %u\n",
1481                  __func__, csk, csk->tx_chan, csk->rss_qid, csk->snd_nxt,
1482                  csk->rcv_nxt, csk->snd_win, csk->emss);
1483         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
1484         cxgbit_ofld_send(csk->com.cdev, skb);
1485         return flowclen16;
1486 }
1487
1488 int cxgbit_setup_conn_digest(struct cxgbit_sock *csk)
1489 {
1490         struct sk_buff *skb;
1491         struct cpl_set_tcb_field *req;
1492         u8 hcrc = csk->submode & CXGBIT_SUBMODE_HCRC;
1493         u8 dcrc = csk->submode & CXGBIT_SUBMODE_DCRC;
1494         unsigned int len = roundup(sizeof(*req), 16);
1495         int ret;
1496
1497         skb = alloc_skb(len, GFP_KERNEL);
1498         if (!skb)
1499                 return -ENOMEM;
1500
1501         /*  set up ulp submode */
1502         req = __skb_put_zero(skb, len);
1503
1504         INIT_TP_WR(req, csk->tid);
1505         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));
1506         req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid));
1507         req->word_cookie = htons(0);
1508         req->mask = cpu_to_be64(0x3 << 4);
1509         req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) |
1510                                 (dcrc ? ULP_CRC_DATA : 0)) << 4);
1511         set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx);
1512
1513         cxgbit_get_csk(csk);
1514         cxgbit_init_wr_wait(&csk->com.wr_wait);
1515
1516         cxgbit_ofld_send(csk->com.cdev, skb);
1517
1518         ret = cxgbit_wait_for_reply(csk->com.cdev,
1519                                     &csk->com.wr_wait,
1520                                     csk->tid, 5, __func__);
1521         if (ret)
1522                 return -1;
1523
1524         return 0;
1525 }
1526
1527 int cxgbit_setup_conn_pgidx(struct cxgbit_sock *csk, u32 pg_idx)
1528 {
1529         struct sk_buff *skb;
1530         struct cpl_set_tcb_field *req;
1531         unsigned int len = roundup(sizeof(*req), 16);
1532         int ret;
1533
1534         skb = alloc_skb(len, GFP_KERNEL);
1535         if (!skb)
1536                 return -ENOMEM;
1537
1538         req = __skb_put_zero(skb, len);
1539
1540         INIT_TP_WR(req, csk->tid);
1541         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));
1542         req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid));
1543         req->word_cookie = htons(0);
1544         req->mask = cpu_to_be64(0x3 << 8);
1545         req->val = cpu_to_be64(pg_idx << 8);
1546         set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx);
1547
1548         cxgbit_get_csk(csk);
1549         cxgbit_init_wr_wait(&csk->com.wr_wait);
1550
1551         cxgbit_ofld_send(csk->com.cdev, skb);
1552
1553         ret = cxgbit_wait_for_reply(csk->com.cdev,
1554                                     &csk->com.wr_wait,
1555                                     csk->tid, 5, __func__);
1556         if (ret)
1557                 return -1;
1558
1559         return 0;
1560 }
1561
1562 static void
1563 cxgbit_pass_open_rpl(struct cxgbit_device *cdev, struct sk_buff *skb)
1564 {
1565         struct cpl_pass_open_rpl *rpl = cplhdr(skb);
1566         struct tid_info *t = cdev->lldi.tids;
1567         unsigned int stid = GET_TID(rpl);
1568         struct cxgbit_np *cnp = lookup_stid(t, stid);
1569
1570         pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1571                  __func__, cnp, stid, rpl->status);
1572
1573         if (!cnp) {
1574                 pr_info("%s stid %d lookup failure\n", __func__, stid);
1575                 goto rel_skb;
1576         }
1577
1578         cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status);
1579         cxgbit_put_cnp(cnp);
1580 rel_skb:
1581         __kfree_skb(skb);
1582 }
1583
1584 static void
1585 cxgbit_close_listsrv_rpl(struct cxgbit_device *cdev, struct sk_buff *skb)
1586 {
1587         struct cpl_close_listsvr_rpl *rpl = cplhdr(skb);
1588         struct tid_info *t = cdev->lldi.tids;
1589         unsigned int stid = GET_TID(rpl);
1590         struct cxgbit_np *cnp = lookup_stid(t, stid);
1591
1592         pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1593                  __func__, cnp, stid, rpl->status);
1594
1595         if (!cnp) {
1596                 pr_info("%s stid %d lookup failure\n", __func__, stid);
1597                 goto rel_skb;
1598         }
1599
1600         cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status);
1601         cxgbit_put_cnp(cnp);
1602 rel_skb:
1603         __kfree_skb(skb);
1604 }
1605
1606 static void
1607 cxgbit_pass_establish(struct cxgbit_device *cdev, struct sk_buff *skb)
1608 {
1609         struct cpl_pass_establish *req = cplhdr(skb);
1610         struct tid_info *t = cdev->lldi.tids;
1611         unsigned int tid = GET_TID(req);
1612         struct cxgbit_sock *csk;
1613         struct cxgbit_np *cnp;
1614         u16 tcp_opt = be16_to_cpu(req->tcp_opt);
1615         u32 snd_isn = be32_to_cpu(req->snd_isn);
1616         u32 rcv_isn = be32_to_cpu(req->rcv_isn);
1617
1618         csk = lookup_tid(t, tid);
1619         if (unlikely(!csk)) {
1620                 pr_err("can't find connection for tid %u.\n", tid);
1621                 goto rel_skb;
1622         }
1623         cnp = csk->cnp;
1624
1625         pr_debug("%s: csk %p; tid %u; cnp %p\n",
1626                  __func__, csk, tid, cnp);
1627
1628         csk->write_seq = snd_isn;
1629         csk->snd_una = snd_isn;
1630         csk->snd_nxt = snd_isn;
1631
1632         csk->rcv_nxt = rcv_isn;
1633
1634         if (csk->rcv_win > (RCV_BUFSIZ_M << 10))
1635                 csk->rx_credits = (csk->rcv_win - (RCV_BUFSIZ_M << 10));
1636
1637         csk->snd_wscale = TCPOPT_SND_WSCALE_G(tcp_opt);
1638         cxgbit_set_emss(csk, tcp_opt);
1639         dst_confirm(csk->dst);
1640         csk->com.state = CSK_STATE_ESTABLISHED;
1641         spin_lock_bh(&cnp->np_accept_lock);
1642         list_add_tail(&csk->accept_node, &cnp->np_accept_list);
1643         spin_unlock_bh(&cnp->np_accept_lock);
1644         complete(&cnp->accept_comp);
1645 rel_skb:
1646         __kfree_skb(skb);
1647 }
1648
1649 static void cxgbit_queue_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1650 {
1651         cxgbit_skcb_flags(skb) = 0;
1652         spin_lock_bh(&csk->rxq.lock);
1653         __skb_queue_tail(&csk->rxq, skb);
1654         spin_unlock_bh(&csk->rxq.lock);
1655         wake_up(&csk->waitq);
1656 }
1657
1658 static void cxgbit_peer_close(struct cxgbit_sock *csk, struct sk_buff *skb)
1659 {
1660         pr_debug("%s: csk %p; tid %u; state %d\n",
1661                  __func__, csk, csk->tid, csk->com.state);
1662
1663         switch (csk->com.state) {
1664         case CSK_STATE_ESTABLISHED:
1665                 csk->com.state = CSK_STATE_CLOSING;
1666                 cxgbit_queue_rx_skb(csk, skb);
1667                 return;
1668         case CSK_STATE_CLOSING:
1669                 /* simultaneous close */
1670                 csk->com.state = CSK_STATE_MORIBUND;
1671                 break;
1672         case CSK_STATE_MORIBUND:
1673                 csk->com.state = CSK_STATE_DEAD;
1674                 cxgbit_put_csk(csk);
1675                 break;
1676         case CSK_STATE_ABORTING:
1677                 break;
1678         default:
1679                 pr_info("%s: cpl_peer_close in bad state %d\n",
1680                         __func__, csk->com.state);
1681         }
1682
1683         __kfree_skb(skb);
1684 }
1685
1686 static void cxgbit_close_con_rpl(struct cxgbit_sock *csk, struct sk_buff *skb)
1687 {
1688         pr_debug("%s: csk %p; tid %u; state %d\n",
1689                  __func__, csk, csk->tid, csk->com.state);
1690
1691         switch (csk->com.state) {
1692         case CSK_STATE_CLOSING:
1693                 csk->com.state = CSK_STATE_MORIBUND;
1694                 break;
1695         case CSK_STATE_MORIBUND:
1696                 csk->com.state = CSK_STATE_DEAD;
1697                 cxgbit_put_csk(csk);
1698                 break;
1699         case CSK_STATE_ABORTING:
1700         case CSK_STATE_DEAD:
1701                 break;
1702         default:
1703                 pr_info("%s: cpl_close_con_rpl in bad state %d\n",
1704                         __func__, csk->com.state);
1705         }
1706
1707         __kfree_skb(skb);
1708 }
1709
1710 static void cxgbit_abort_req_rss(struct cxgbit_sock *csk, struct sk_buff *skb)
1711 {
1712         struct cpl_abort_req_rss *hdr = cplhdr(skb);
1713         unsigned int tid = GET_TID(hdr);
1714         struct sk_buff *rpl_skb;
1715         bool release = false;
1716         bool wakeup_thread = false;
1717         u32 len = roundup(sizeof(struct cpl_abort_rpl), 16);
1718
1719         pr_debug("%s: csk %p; tid %u; state %d\n",
1720                  __func__, csk, tid, csk->com.state);
1721
1722         if (cxgb_is_neg_adv(hdr->status)) {
1723                 pr_err("%s: got neg advise %d on tid %u\n",
1724                        __func__, hdr->status, tid);
1725                 goto rel_skb;
1726         }
1727
1728         switch (csk->com.state) {
1729         case CSK_STATE_CONNECTING:
1730         case CSK_STATE_MORIBUND:
1731                 csk->com.state = CSK_STATE_DEAD;
1732                 release = true;
1733                 break;
1734         case CSK_STATE_ESTABLISHED:
1735                 csk->com.state = CSK_STATE_DEAD;
1736                 wakeup_thread = true;
1737                 break;
1738         case CSK_STATE_CLOSING:
1739                 csk->com.state = CSK_STATE_DEAD;
1740                 if (!csk->conn)
1741                         release = true;
1742                 break;
1743         case CSK_STATE_ABORTING:
1744                 break;
1745         default:
1746                 pr_info("%s: cpl_abort_req_rss in bad state %d\n",
1747                         __func__, csk->com.state);
1748                 csk->com.state = CSK_STATE_DEAD;
1749         }
1750
1751         __skb_queue_purge(&csk->txq);
1752
1753         if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags))
1754                 cxgbit_send_tx_flowc_wr(csk);
1755
1756         rpl_skb = __skb_dequeue(&csk->skbq);
1757
1758         cxgb_mk_abort_rpl(rpl_skb, len, csk->tid, csk->txq_idx);
1759         cxgbit_ofld_send(csk->com.cdev, rpl_skb);
1760
1761         if (wakeup_thread) {
1762                 cxgbit_queue_rx_skb(csk, skb);
1763                 return;
1764         }
1765
1766         if (release)
1767                 cxgbit_put_csk(csk);
1768 rel_skb:
1769         __kfree_skb(skb);
1770 }
1771
1772 static void cxgbit_abort_rpl_rss(struct cxgbit_sock *csk, struct sk_buff *skb)
1773 {
1774         struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
1775
1776         pr_debug("%s: csk %p; tid %u; state %d\n",
1777                  __func__, csk, csk->tid, csk->com.state);
1778
1779         switch (csk->com.state) {
1780         case CSK_STATE_ABORTING:
1781                 csk->com.state = CSK_STATE_DEAD;
1782                 if (test_bit(CSK_ABORT_RPL_WAIT, &csk->com.flags))
1783                         cxgbit_wake_up(&csk->com.wr_wait, __func__,
1784                                        rpl->status);
1785                 cxgbit_put_csk(csk);
1786                 break;
1787         default:
1788                 pr_info("%s: cpl_abort_rpl_rss in state %d\n",
1789                         __func__, csk->com.state);
1790         }
1791
1792         __kfree_skb(skb);
1793 }
1794
1795 static bool cxgbit_credit_err(const struct cxgbit_sock *csk)
1796 {
1797         const struct sk_buff *skb = csk->wr_pending_head;
1798         u32 credit = 0;
1799
1800         if (unlikely(csk->wr_cred > csk->wr_max_cred)) {
1801                 pr_err("csk 0x%p, tid %u, credit %u > %u\n",
1802                        csk, csk->tid, csk->wr_cred, csk->wr_max_cred);
1803                 return true;
1804         }
1805
1806         while (skb) {
1807                 credit += (__force u32)skb->csum;
1808                 skb = cxgbit_skcb_tx_wr_next(skb);
1809         }
1810
1811         if (unlikely((csk->wr_cred + credit) != csk->wr_max_cred)) {
1812                 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
1813                        csk, csk->tid, csk->wr_cred,
1814                        credit, csk->wr_max_cred);
1815
1816                 return true;
1817         }
1818
1819         return false;
1820 }
1821
1822 static void cxgbit_fw4_ack(struct cxgbit_sock *csk, struct sk_buff *skb)
1823 {
1824         struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)cplhdr(skb);
1825         u32 credits = rpl->credits;
1826         u32 snd_una = ntohl(rpl->snd_una);
1827
1828         csk->wr_cred += credits;
1829         if (csk->wr_una_cred > (csk->wr_max_cred - csk->wr_cred))
1830                 csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred;
1831
1832         while (credits) {
1833                 struct sk_buff *p = cxgbit_sock_peek_wr(csk);
1834                 u32 csum;
1835
1836                 if (unlikely(!p)) {
1837                         pr_err("csk 0x%p,%u, cr %u,%u+%u, empty.\n",
1838                                csk, csk->tid, credits,
1839                                csk->wr_cred, csk->wr_una_cred);
1840                         break;
1841                 }
1842
1843                 csum = (__force u32)p->csum;
1844                 if (unlikely(credits < csum)) {
1845                         pr_warn("csk 0x%p,%u, cr %u,%u+%u, < %u.\n",
1846                                 csk,  csk->tid,
1847                                 credits, csk->wr_cred, csk->wr_una_cred,
1848                                 csum);
1849                         p->csum = (__force __wsum)(csum - credits);
1850                         break;
1851                 }
1852
1853                 cxgbit_sock_dequeue_wr(csk);
1854                 credits -= csum;
1855                 kfree_skb(p);
1856         }
1857
1858         if (unlikely(cxgbit_credit_err(csk))) {
1859                 cxgbit_queue_rx_skb(csk, skb);
1860                 return;
1861         }
1862
1863         if (rpl->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
1864                 if (unlikely(before(snd_una, csk->snd_una))) {
1865                         pr_warn("csk 0x%p,%u, snd_una %u/%u.",
1866                                 csk, csk->tid, snd_una,
1867                                 csk->snd_una);
1868                         goto rel_skb;
1869                 }
1870
1871                 if (csk->snd_una != snd_una) {
1872                         csk->snd_una = snd_una;
1873                         dst_confirm(csk->dst);
1874                         wake_up(&csk->ack_waitq);
1875                 }
1876         }
1877
1878         if (skb_queue_len(&csk->txq))
1879                 cxgbit_push_tx_frames(csk);
1880
1881 rel_skb:
1882         __kfree_skb(skb);
1883 }
1884
1885 static void cxgbit_set_tcb_rpl(struct cxgbit_device *cdev, struct sk_buff *skb)
1886 {
1887         struct cxgbit_sock *csk;
1888         struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data;
1889         unsigned int tid = GET_TID(rpl);
1890         struct cxgb4_lld_info *lldi = &cdev->lldi;
1891         struct tid_info *t = lldi->tids;
1892
1893         csk = lookup_tid(t, tid);
1894         if (unlikely(!csk)) {
1895                 pr_err("can't find connection for tid %u.\n", tid);
1896                 goto rel_skb;
1897         } else {
1898                 cxgbit_wake_up(&csk->com.wr_wait, __func__, rpl->status);
1899         }
1900
1901         cxgbit_put_csk(csk);
1902 rel_skb:
1903         __kfree_skb(skb);
1904 }
1905
1906 static void cxgbit_rx_data(struct cxgbit_device *cdev, struct sk_buff *skb)
1907 {
1908         struct cxgbit_sock *csk;
1909         struct cpl_rx_data *cpl = cplhdr(skb);
1910         unsigned int tid = GET_TID(cpl);
1911         struct cxgb4_lld_info *lldi = &cdev->lldi;
1912         struct tid_info *t = lldi->tids;
1913
1914         csk = lookup_tid(t, tid);
1915         if (unlikely(!csk)) {
1916                 pr_err("can't find conn. for tid %u.\n", tid);
1917                 goto rel_skb;
1918         }
1919
1920         cxgbit_queue_rx_skb(csk, skb);
1921         return;
1922 rel_skb:
1923         __kfree_skb(skb);
1924 }
1925
1926 static void
1927 __cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb)
1928 {
1929         spin_lock(&csk->lock);
1930         if (csk->lock_owner) {
1931                 __skb_queue_tail(&csk->backlogq, skb);
1932                 spin_unlock(&csk->lock);
1933                 return;
1934         }
1935
1936         cxgbit_skcb_rx_backlog_fn(skb)(csk, skb);
1937         spin_unlock(&csk->lock);
1938 }
1939
1940 static void cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb)
1941 {
1942         cxgbit_get_csk(csk);
1943         __cxgbit_process_rx_cpl(csk, skb);
1944         cxgbit_put_csk(csk);
1945 }
1946
1947 static void cxgbit_rx_cpl(struct cxgbit_device *cdev, struct sk_buff *skb)
1948 {
1949         struct cxgbit_sock *csk;
1950         struct cpl_tx_data *cpl = cplhdr(skb);
1951         struct cxgb4_lld_info *lldi = &cdev->lldi;
1952         struct tid_info *t = lldi->tids;
1953         unsigned int tid = GET_TID(cpl);
1954         u8 opcode = cxgbit_skcb_rx_opcode(skb);
1955         bool ref = true;
1956
1957         switch (opcode) {
1958         case CPL_FW4_ACK:
1959                         cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_fw4_ack;
1960                         ref = false;
1961                         break;
1962         case CPL_PEER_CLOSE:
1963                         cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_peer_close;
1964                         break;
1965         case CPL_CLOSE_CON_RPL:
1966                         cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_close_con_rpl;
1967                         break;
1968         case CPL_ABORT_REQ_RSS:
1969                         cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_req_rss;
1970                         break;
1971         case CPL_ABORT_RPL_RSS:
1972                         cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_rpl_rss;
1973                         break;
1974         default:
1975                 goto rel_skb;
1976         }
1977
1978         csk = lookup_tid(t, tid);
1979         if (unlikely(!csk)) {
1980                 pr_err("can't find conn. for tid %u.\n", tid);
1981                 goto rel_skb;
1982         }
1983
1984         if (ref)
1985                 cxgbit_process_rx_cpl(csk, skb);
1986         else
1987                 __cxgbit_process_rx_cpl(csk, skb);
1988
1989         return;
1990 rel_skb:
1991         __kfree_skb(skb);
1992 }
1993
1994 cxgbit_cplhandler_func cxgbit_cplhandlers[NUM_CPL_CMDS] = {
1995         [CPL_PASS_OPEN_RPL]     = cxgbit_pass_open_rpl,
1996         [CPL_CLOSE_LISTSRV_RPL] = cxgbit_close_listsrv_rpl,
1997         [CPL_PASS_ACCEPT_REQ]   = cxgbit_pass_accept_req,
1998         [CPL_PASS_ESTABLISH]    = cxgbit_pass_establish,
1999         [CPL_SET_TCB_RPL]       = cxgbit_set_tcb_rpl,
2000         [CPL_RX_DATA]           = cxgbit_rx_data,
2001         [CPL_FW4_ACK]           = cxgbit_rx_cpl,
2002         [CPL_PEER_CLOSE]        = cxgbit_rx_cpl,
2003         [CPL_CLOSE_CON_RPL]     = cxgbit_rx_cpl,
2004         [CPL_ABORT_REQ_RSS]     = cxgbit_rx_cpl,
2005         [CPL_ABORT_RPL_RSS]     = cxgbit_rx_cpl,
2006 };