drivers/net/wireless/ath/wil6210/txrx.c

   1 /*
   2  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
   3  * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
   4  *
   5  * Permission to use, copy, modify, and/or distribute this software for any
   6  * purpose with or without fee is hereby granted, provided that the above
   7  * copyright notice and this permission notice appear in all copies.
   8  *
   9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  16  */
  17
  18 #include <linux/etherdevice.h>
  19 #include <net/ieee80211_radiotap.h>
  20 #include <linux/if_arp.h>
  21 #include <linux/moduleparam.h>
  22 #include <linux/ip.h>
  23 #include <linux/ipv6.h>
  24 #include <net/ipv6.h>
  25 #include <linux/prefetch.h>
  26
  27 #include "wil6210.h"
  28 #include "wmi.h"
  29 #include "txrx.h"
  30 #include "trace.h"
  31 #include "txrx_edma.h"
  32
  33 bool rx_align_2;
  34 module_param(rx_align_2, bool, 0444);
  35 MODULE_PARM_DESC(rx_align_2, " align Rx buffers on 4*n+2, default - no");
  36
  37 bool rx_large_buf;
  38 module_param(rx_large_buf, bool, 0444);
  39 MODULE_PARM_DESC(rx_large_buf, " allocate 8KB RX buffers, default - no");
  40
  41 /* Drop Tx packets in case Tx ring is full */
  42 bool drop_if_ring_full;
  43
  44 static inline uint wil_rx_snaplen(void)
  45 {
  46         return rx_align_2 ? 6 : 0;
  47 }
  48
  49 /* wil_ring_wmark_low - low watermark for available descriptor space */
  50 static inline int wil_ring_wmark_low(struct wil_ring *ring)
  51 {
  52         return ring->size / 8;
  53 }
  54
  55 /* wil_ring_wmark_high - high watermark for available descriptor space */
  56 static inline int wil_ring_wmark_high(struct wil_ring *ring)
  57 {
  58         return ring->size / 4;
  59 }
  60
  61 /* returns true if num avail descriptors is lower than wmark_low */
  62 static inline int wil_ring_avail_low(struct wil_ring *ring)
  63 {
  64         return wil_ring_avail_tx(ring) < wil_ring_wmark_low(ring);
  65 }
  66
  67 /* returns true if num avail descriptors is higher than wmark_high */
  68 static inline int wil_ring_avail_high(struct wil_ring *ring)
  69 {
  70         return wil_ring_avail_tx(ring) > wil_ring_wmark_high(ring);
  71 }
  72
  73 /* returns true when all tx vrings are empty */
  74 bool wil_is_tx_idle(struct wil6210_priv *wil)
  75 {
  76         int i;
  77         unsigned long data_comp_to;
  78         int min_ring_id = wil_get_min_tx_ring_id(wil);
  79
  80         for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
  81                 struct wil_ring *vring = &wil->ring_tx[i];
  82                 int vring_index = vring - wil->ring_tx;
  83                 struct wil_ring_tx_data *txdata =
  84                         &wil->ring_tx_data[vring_index];
  85
  86                 spin_lock(&txdata->lock);
  87
  88                 if (!vring->va || !txdata->enabled) {
  89                         spin_unlock(&txdata->lock);
  90                         continue;
  91                 }
  92
  93                 data_comp_to = jiffies + msecs_to_jiffies(
  94                                         WIL_DATA_COMPLETION_TO_MS);
  95                 if (test_bit(wil_status_napi_en, wil->status)) {
  96                         while (!wil_ring_is_empty(vring)) {
  97                                 if (time_after(jiffies, data_comp_to)) {
  98                                         wil_dbg_pm(wil,
  99                                                    "TO waiting for idle tx\n");
 100                                         spin_unlock(&txdata->lock);
 101                                         return false;
 102                                 }
 103                                 wil_dbg_ratelimited(wil,
 104                                                     "tx vring is not empty -> NAPI\n");
 105                                 spin_unlock(&txdata->lock);
 106                                 napi_synchronize(&wil->napi_tx);
 107                                 msleep(20);
 108                                 spin_lock(&txdata->lock);
 109                                 if (!vring->va || !txdata->enabled)
 110                                         break;
 111                         }
 112                 }
 113
 114                 spin_unlock(&txdata->lock);
 115         }
 116
 117         return true;
 118 }
 119
 120 static int wil_vring_alloc(struct wil6210_priv *wil, struct wil_ring *vring)
 121 {
 122         struct device *dev = wil_to_dev(wil);
 123         size_t sz = vring->size * sizeof(vring->va[0]);
 124         uint i;
 125
 126         wil_dbg_misc(wil, "vring_alloc:\n");
 127
 128         BUILD_BUG_ON(sizeof(vring->va[0]) != 32);
 129
 130         vring->swhead = 0;
 131         vring->swtail = 0;
 132         vring->ctx = kcalloc(vring->size, sizeof(vring->ctx[0]), GFP_KERNEL);
 133         if (!vring->ctx) {
 134                 vring->va = NULL;
 135                 return -ENOMEM;
 136         }
 137
 138         /* vring->va should be aligned on its size rounded up to power of 2
 139          * This is granted by the dma_alloc_coherent.
 140          *
 141          * HW has limitation that all vrings addresses must share the same
 142          * upper 16 msb bits part of 48 bits address. To workaround that,
 143          * if we are using more than 32 bit addresses switch to 32 bit
 144          * allocation before allocating vring memory.
 145          *
 146          * There's no check for the return value of dma_set_mask_and_coherent,
 147          * since we assume if we were able to set the mask during
 148          * initialization in this system it will not fail if we set it again
 149          */
 150         if (wil->dma_addr_size > 32)
 151                 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 152
 153         vring->va = dma_alloc_coherent(dev, sz, &vring->pa, GFP_KERNEL);
 154         if (!vring->va) {
 155                 kfree(vring->ctx);
 156                 vring->ctx = NULL;
 157                 return -ENOMEM;
 158         }
 159
 160         if (wil->dma_addr_size > 32)
 161                 dma_set_mask_and_coherent(dev,
 162                                           DMA_BIT_MASK(wil->dma_addr_size));
 163
 164         /* initially, all descriptors are SW owned
 165          * For Tx and Rx, ownership bit is at the same location, thus
 166          * we can use any
 167          */
 168         for (i = 0; i < vring->size; i++) {
 169                 volatile struct vring_tx_desc *_d =
 170                         &vring->va[i].tx.legacy;
 171
 172                 _d->dma.status = TX_DMA_STATUS_DU;
 173         }
 174
 175         wil_dbg_misc(wil, "vring[%d] 0x%p:%pad 0x%p\n", vring->size,
 176                      vring->va, &vring->pa, vring->ctx);
 177
 178         return 0;
 179 }
 180
 181 static void wil_txdesc_unmap(struct device *dev, union wil_tx_desc *desc,
 182                              struct wil_ctx *ctx)
 183 {
 184         struct vring_tx_desc *d = &desc->legacy;
 185         dma_addr_t pa = wil_desc_addr(&d->dma.addr);
 186         u16 dmalen = le16_to_cpu(d->dma.length);
 187
 188         switch (ctx->mapped_as) {
 189         case wil_mapped_as_single:
 190                 dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE);
 191                 break;
 192         case wil_mapped_as_page:
 193                 dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE);
 194                 break;
 195         default:
 196                 break;
 197         }
 198 }
 199
 200 static void wil_vring_free(struct wil6210_priv *wil, struct wil_ring *vring)
 201 {
 202         struct device *dev = wil_to_dev(wil);
 203         size_t sz = vring->size * sizeof(vring->va[0]);
 204
 205         lockdep_assert_held(&wil->mutex);
 206         if (!vring->is_rx) {
 207                 int vring_index = vring - wil->ring_tx;
 208
 209                 wil_dbg_misc(wil, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n",
 210                              vring_index, vring->size, vring->va,
 211                              &vring->pa, vring->ctx);
 212         } else {
 213                 wil_dbg_misc(wil, "free Rx vring [%d] 0x%p:%pad 0x%p\n",
 214                              vring->size, vring->va,
 215                              &vring->pa, vring->ctx);
 216         }
 217
 218         while (!wil_ring_is_empty(vring)) {
 219                 dma_addr_t pa;
 220                 u16 dmalen;
 221                 struct wil_ctx *ctx;
 222
 223                 if (!vring->is_rx) {
 224                         struct vring_tx_desc dd, *d = &dd;
 225                         volatile struct vring_tx_desc *_d =
 226                                         &vring->va[vring->swtail].tx.legacy;
 227
 228                         ctx = &vring->ctx[vring->swtail];
 229                         if (!ctx) {
 230                                 wil_dbg_txrx(wil,
 231                                              "ctx(%d) was already completed\n",
 232                                              vring->swtail);
 233                                 vring->swtail = wil_ring_next_tail(vring);
 234                                 continue;
 235                         }
 236                         *d = *_d;
 237                         wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx);
 238                         if (ctx->skb)
 239                                 dev_kfree_skb_any(ctx->skb);
 240                         vring->swtail = wil_ring_next_tail(vring);
 241                 } else { /* rx */
 242                         struct vring_rx_desc dd, *d = &dd;
 243                         volatile struct vring_rx_desc *_d =
 244                                 &vring->va[vring->swhead].rx.legacy;
 245
 246                         ctx = &vring->ctx[vring->swhead];
 247                         *d = *_d;
 248                         pa = wil_desc_addr(&d->dma.addr);
 249                         dmalen = le16_to_cpu(d->dma.length);
 250                         dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE);
 251                         kfree_skb(ctx->skb);
 252                         wil_ring_advance_head(vring, 1);
 253                 }
 254         }
 255         dma_free_coherent(dev, sz, (void *)vring->va, vring->pa);
 256         kfree(vring->ctx);
 257         vring->pa = 0;
 258         vring->va = NULL;
 259         vring->ctx = NULL;
 260 }
 261
 262 /**
 263  * Allocate one skb for Rx VRING
 264  *
 265  * Safe to call from IRQ
 266  */
 267 static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct wil_ring *vring,
 268                                u32 i, int headroom)
 269 {
 270         struct device *dev = wil_to_dev(wil);
 271         unsigned int sz = wil->rx_buf_len + ETH_HLEN + wil_rx_snaplen();
 272         struct vring_rx_desc dd, *d = &dd;
 273         volatile struct vring_rx_desc *_d = &vring->va[i].rx.legacy;
 274         dma_addr_t pa;
 275         struct sk_buff *skb = dev_alloc_skb(sz + headroom);
 276
 277         if (unlikely(!skb))
 278                 return -ENOMEM;
 279
 280         skb_reserve(skb, headroom);
 281         skb_put(skb, sz);
 282
 283         /**
 284          * Make sure that the network stack calculates checksum for packets
 285          * which failed the HW checksum calculation
 286          */
 287         skb->ip_summed = CHECKSUM_NONE;
 288
 289         pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE);
 290         if (unlikely(dma_mapping_error(dev, pa))) {
 291                 kfree_skb(skb);
 292                 return -ENOMEM;
 293         }
 294
 295         d->dma.d0 = RX_DMA_D0_CMD_DMA_RT | RX_DMA_D0_CMD_DMA_IT;
 296         wil_desc_addr_set(&d->dma.addr, pa);
 297         /* ip_length don't care */
 298         /* b11 don't care */
 299         /* error don't care */
 300         d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */
 301         d->dma.length = cpu_to_le16(sz);
 302         *_d = *d;
 303         vring->ctx[i].skb = skb;
 304
 305         return 0;
 306 }
 307
 308 /**
 309  * Adds radiotap header
 310  *
 311  * Any error indicated as "Bad FCS"
 312  *
 313  * Vendor data for 04:ce:14-1 (Wilocity-1) consists of:
 314  *  - Rx descriptor: 32 bytes
 315  *  - Phy info
 316  */
 317 static void wil_rx_add_radiotap_header(struct wil6210_priv *wil,
 318                                        struct sk_buff *skb)
 319 {
 320         struct wil6210_rtap {
 321                 struct ieee80211_radiotap_header rthdr;
 322                 /* fields should be in the order of bits in rthdr.it_present */
 323                 /* flags */
 324                 u8 flags;
 325                 /* channel */
 326                 __le16 chnl_freq __aligned(2);
 327                 __le16 chnl_flags;
 328                 /* MCS */
 329                 u8 mcs_present;
 330                 u8 mcs_flags;
 331                 u8 mcs_index;
 332         } __packed;
 333         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 334         struct wil6210_rtap *rtap;
 335         int rtap_len = sizeof(struct wil6210_rtap);
 336         struct ieee80211_channel *ch = wil->monitor_chandef.chan;
 337
 338         if (skb_headroom(skb) < rtap_len &&
 339             pskb_expand_head(skb, rtap_len, 0, GFP_ATOMIC)) {
 340                 wil_err(wil, "Unable to expand headroom to %d\n", rtap_len);
 341                 return;
 342         }
 343
 344         rtap = skb_push(skb, rtap_len);
 345         memset(rtap, 0, rtap_len);
 346
 347         rtap->rthdr.it_version = PKTHDR_RADIOTAP_VERSION;
 348         rtap->rthdr.it_len = cpu_to_le16(rtap_len);
 349         rtap->rthdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) |
 350                         (1 << IEEE80211_RADIOTAP_CHANNEL) |
 351                         (1 << IEEE80211_RADIOTAP_MCS));
 352         if (d->dma.status & RX_DMA_STATUS_ERROR)
 353                 rtap->flags |= IEEE80211_RADIOTAP_F_BADFCS;
 354
 355         rtap->chnl_freq = cpu_to_le16(ch ? ch->center_freq : 58320);
 356         rtap->chnl_flags = cpu_to_le16(0);
 357
 358         rtap->mcs_present = IEEE80211_RADIOTAP_MCS_HAVE_MCS;
 359         rtap->mcs_flags = 0;
 360         rtap->mcs_index = wil_rxdesc_mcs(d);
 361 }
 362
 363 static bool wil_is_rx_idle(struct wil6210_priv *wil)
 364 {
 365         struct vring_rx_desc *_d;
 366         struct wil_ring *ring = &wil->ring_rx;
 367
 368         _d = (struct vring_rx_desc *)&ring->va[ring->swhead].rx.legacy;
 369         if (_d->dma.status & RX_DMA_STATUS_DU)
 370                 return false;
 371
 372         return true;
 373 }
 374
 375 static int wil_rx_get_cid_by_skb(struct wil6210_priv *wil, struct sk_buff *skb)
 376 {
 377         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 378         int mid = wil_rxdesc_mid(d);
 379         struct wil6210_vif *vif = wil->vifs[mid];
 380         /* cid from DMA descriptor is limited to 3 bits.
 381          * In case of cid>=8, the value would be cid modulo 8 and we need to
 382          * find real cid by locating the transmitter (ta) inside sta array
 383          */
 384         int cid = wil_rxdesc_cid(d);
 385         unsigned int snaplen = wil_rx_snaplen();
 386         struct ieee80211_hdr_3addr *hdr;
 387         int i;
 388         unsigned char *ta;
 389         u8 ftype;
 390
 391         /* in monitor mode there are no connections */
 392         if (vif->wdev.iftype == NL80211_IFTYPE_MONITOR)
 393                 return cid;
 394
 395         ftype = wil_rxdesc_ftype(d) << 2;
 396         if (likely(ftype == IEEE80211_FTYPE_DATA)) {
 397                 if (unlikely(skb->len < ETH_HLEN + snaplen)) {
 398                         wil_err_ratelimited(wil,
 399                                             "Short data frame, len = %d\n",
 400                                             skb->len);
 401                         return -ENOENT;
 402                 }
 403                 ta = wil_skb_get_sa(skb);
 404         } else {
 405                 if (unlikely(skb->len < sizeof(struct ieee80211_hdr_3addr))) {
 406                         wil_err_ratelimited(wil, "Short frame, len = %d\n",
 407                                             skb->len);
 408                         return -ENOENT;
 409                 }
 410                 hdr = (void *)skb->data;
 411                 ta = hdr->addr2;
 412         }
 413
 414         if (wil->max_assoc_sta <= WIL6210_RX_DESC_MAX_CID)
 415                 return cid;
 416
 417         /* assuming no concurrency between AP interfaces and STA interfaces.
 418          * multista is used only in P2P_GO or AP mode. In other modes return
 419          * cid from the rx descriptor
 420          */
 421         if (vif->wdev.iftype != NL80211_IFTYPE_P2P_GO &&
 422             vif->wdev.iftype != NL80211_IFTYPE_AP)
 423                 return cid;
 424
 425         /* For Rx packets cid from rx descriptor is limited to 3 bits (0..7),
 426          * to find the real cid, compare transmitter address with the stored
 427          * stations mac address in the driver sta array
 428          */
 429         for (i = cid; i < wil->max_assoc_sta; i += WIL6210_RX_DESC_MAX_CID) {
 430                 if (wil->sta[i].status != wil_sta_unused &&
 431                     ether_addr_equal(wil->sta[i].addr, ta)) {
 432                         cid = i;
 433                         break;
 434                 }
 435         }
 436         if (i >= wil->max_assoc_sta) {
 437                 wil_err_ratelimited(wil, "Could not find cid for frame with transmit addr = %pM, iftype = %d, frametype = %d, len = %d\n",
 438                                     ta, vif->wdev.iftype, ftype, skb->len);
 439                 cid = -ENOENT;
 440         }
 441
 442         return cid;
 443 }
 444
 445 /**
 446  * reap 1 frame from @swhead
 447  *
 448  * Rx descriptor copied to skb->cb
 449  *
 450  * Safe to call from IRQ
 451  */
 452 static struct sk_buff *wil_vring_reap_rx(struct wil6210_priv *wil,
 453                                          struct wil_ring *vring)
 454 {
 455         struct device *dev = wil_to_dev(wil);
 456         struct wil6210_vif *vif;
 457         struct net_device *ndev;
 458         volatile struct vring_rx_desc *_d;
 459         struct vring_rx_desc *d;
 460         struct sk_buff *skb;
 461         dma_addr_t pa;
 462         unsigned int snaplen = wil_rx_snaplen();
 463         unsigned int sz = wil->rx_buf_len + ETH_HLEN + snaplen;
 464         u16 dmalen;
 465         u8 ftype;
 466         int cid, mid;
 467         int i;
 468         struct wil_net_stats *stats;
 469
 470         BUILD_BUG_ON(sizeof(struct skb_rx_info) > sizeof(skb->cb));
 471
 472 again:
 473         if (unlikely(wil_ring_is_empty(vring)))
 474                 return NULL;
 475
 476         i = (int)vring->swhead;
 477         _d = &vring->va[i].rx.legacy;
 478         if (unlikely(!(_d->dma.status & RX_DMA_STATUS_DU))) {
 479                 /* it is not error, we just reached end of Rx done area */
 480                 return NULL;
 481         }
 482
 483         skb = vring->ctx[i].skb;
 484         vring->ctx[i].skb = NULL;
 485         wil_ring_advance_head(vring, 1);
 486         if (!skb) {
 487                 wil_err(wil, "No Rx skb at [%d]\n", i);
 488                 goto again;
 489         }
 490         d = wil_skb_rxdesc(skb);
 491         *d = *_d;
 492         pa = wil_desc_addr(&d->dma.addr);
 493
 494         dma_unmap_single(dev, pa, sz, DMA_FROM_DEVICE);
 495         dmalen = le16_to_cpu(d->dma.length);
 496
 497         trace_wil6210_rx(i, d);
 498         wil_dbg_txrx(wil, "Rx[%3d] : %d bytes\n", i, dmalen);
 499         wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4,
 500                           (const void *)d, sizeof(*d), false);
 501
 502         mid = wil_rxdesc_mid(d);
 503         vif = wil->vifs[mid];
 504
 505         if (unlikely(!vif)) {
 506                 wil_dbg_txrx(wil, "skipped RX descriptor with invalid mid %d",
 507                              mid);
 508                 kfree_skb(skb);
 509                 goto again;
 510         }
 511         ndev = vif_to_ndev(vif);
 512         if (unlikely(dmalen > sz)) {
 513                 wil_err_ratelimited(wil, "Rx size too large: %d bytes!\n",
 514                                     dmalen);
 515                 kfree_skb(skb);
 516                 goto again;
 517         }
 518         skb_trim(skb, dmalen);
 519
 520         prefetch(skb->data);
 521
 522         wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1,
 523                           skb->data, skb_headlen(skb), false);
 524
 525         cid = wil_rx_get_cid_by_skb(wil, skb);
 526         if (cid == -ENOENT) {
 527                 kfree_skb(skb);
 528                 goto again;
 529         }
 530         wil_skb_set_cid(skb, (u8)cid);
 531         stats = &wil->sta[cid].stats;
 532
 533         stats->last_mcs_rx = wil_rxdesc_mcs(d);
 534         if (stats->last_mcs_rx < ARRAY_SIZE(stats->rx_per_mcs))
 535                 stats->rx_per_mcs[stats->last_mcs_rx]++;
 536
 537         /* use radiotap header only if required */
 538         if (ndev->type == ARPHRD_IEEE80211_RADIOTAP)
 539                 wil_rx_add_radiotap_header(wil, skb);
 540
 541         /* no extra checks if in sniffer mode */
 542         if (ndev->type != ARPHRD_ETHER)
 543                 return skb;
 544         /* Non-data frames may be delivered through Rx DMA channel (ex: BAR)
 545          * Driver should recognize it by frame type, that is found
 546          * in Rx descriptor. If type is not data, it is 802.11 frame as is
 547          */
 548         ftype = wil_rxdesc_ftype(d) << 2;
 549         if (unlikely(ftype != IEEE80211_FTYPE_DATA)) {
 550                 u8 fc1 = wil_rxdesc_fc1(d);
 551                 int tid = wil_rxdesc_tid(d);
 552                 u16 seq = wil_rxdesc_seq(d);
 553
 554                 wil_dbg_txrx(wil,
 555                              "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
 556                              fc1, mid, cid, tid, seq);
 557                 stats->rx_non_data_frame++;
 558                 if (wil_is_back_req(fc1)) {
 559                         wil_dbg_txrx(wil,
 560                                      "BAR: MID %d CID %d TID %d Seq 0x%03x\n",
 561                                      mid, cid, tid, seq);
 562                         wil_rx_bar(wil, vif, cid, tid, seq);
 563                 } else {
 564                         /* print again all info. One can enable only this
 565                          * without overhead for printing every Rx frame
 566                          */
 567                         wil_dbg_txrx(wil,
 568                                      "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
 569                                      fc1, mid, cid, tid, seq);
 570                         wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4,
 571                                           (const void *)d, sizeof(*d), false);
 572                         wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1,
 573                                           skb->data, skb_headlen(skb), false);
 574                 }
 575                 kfree_skb(skb);
 576                 goto again;
 577         }
 578
 579         /* L4 IDENT is on when HW calculated checksum, check status
 580          * and in case of error drop the packet
 581          * higher stack layers will handle retransmission (if required)
 582          */
 583         if (likely(d->dma.status & RX_DMA_STATUS_L4I)) {
 584                 /* L4 protocol identified, csum calculated */
 585                 if (likely((d->dma.error & RX_DMA_ERROR_L4_ERR) == 0))
 586                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 587                 /* If HW reports bad checksum, let IP stack re-check it
 588                  * For example, HW don't understand Microsoft IP stack that
 589                  * mis-calculates TCP checksum - if it should be 0x0,
 590                  * it writes 0xffff in violation of RFC 1624
 591                  */
 592                 else
 593                         stats->rx_csum_err++;
 594         }
 595
 596         if (snaplen) {
 597                 /* Packet layout
 598                  * +-------+-------+---------+------------+------+
 599                  * | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA |
 600                  * +-------+-------+---------+------------+------+
 601                  * Need to remove SNAP, shifting SA and DA forward
 602                  */
 603                 memmove(skb->data + snaplen, skb->data, 2 * ETH_ALEN);
 604                 skb_pull(skb, snaplen);
 605         }
 606
 607         return skb;
 608 }
 609
 610 /**
 611  * allocate and fill up to @count buffers in rx ring
 612  * buffers posted at @swtail
 613  * Note: we have a single RX queue for servicing all VIFs, but we
 614  * allocate skbs with headroom according to main interface only. This
 615  * means it will not work with monitor interface together with other VIFs.
 616  * Currently we only support monitor interface on its own without other VIFs,
 617  * and we will need to fix this code once we add support.
 618  */
 619 static int wil_rx_refill(struct wil6210_priv *wil, int count)
 620 {
 621         struct net_device *ndev = wil->main_ndev;
 622         struct wil_ring *v = &wil->ring_rx;
 623         u32 next_tail;
 624         int rc = 0;
 625         int headroom = ndev->type == ARPHRD_IEEE80211_RADIOTAP ?
 626                         WIL6210_RTAP_SIZE : 0;
 627
 628         for (; next_tail = wil_ring_next_tail(v),
 629              (next_tail != v->swhead) && (count-- > 0);
 630              v->swtail = next_tail) {
 631                 rc = wil_vring_alloc_skb(wil, v, v->swtail, headroom);
 632                 if (unlikely(rc)) {
 633                         wil_err_ratelimited(wil, "Error %d in rx refill[%d]\n",
 634                                             rc, v->swtail);
 635                         break;
 636                 }
 637         }
 638
 639         /* make sure all writes to descriptors (shared memory) are done before
 640          * committing them to HW
 641          */
 642         wmb();
 643
 644         wil_w(wil, v->hwtail, v->swtail);
 645
 646         return rc;
 647 }
 648
 649 /**
 650  * reverse_memcmp - Compare two areas of memory, in reverse order
 651  * @cs: One area of memory
 652  * @ct: Another area of memory
 653  * @count: The size of the area.
 654  *
 655  * Cut'n'paste from original memcmp (see lib/string.c)
 656  * with minimal modifications
 657  */
 658 int reverse_memcmp(const void *cs, const void *ct, size_t count)
 659 {
 660         const unsigned char *su1, *su2;
 661         int res = 0;
 662
 663         for (su1 = cs + count - 1, su2 = ct + count - 1; count > 0;
 664              --su1, --su2, count--) {
 665                 res = *su1 - *su2;
 666                 if (res)
 667                         break;
 668         }
 669         return res;
 670 }
 671
 672 static int wil_rx_crypto_check(struct wil6210_priv *wil, struct sk_buff *skb)
 673 {
 674         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 675         int cid = wil_skb_get_cid(skb);
 676         int tid = wil_rxdesc_tid(d);
 677         int key_id = wil_rxdesc_key_id(d);
 678         int mc = wil_rxdesc_mcast(d);
 679         struct wil_sta_info *s = &wil->sta[cid];
 680         struct wil_tid_crypto_rx *c = mc ? &s->group_crypto_rx :
 681                                       &s->tid_crypto_rx[tid];
 682         struct wil_tid_crypto_rx_single *cc = &c->key_id[key_id];
 683         const u8 *pn = (u8 *)&d->mac.pn_15_0;
 684
 685         if (!cc->key_set) {
 686                 wil_err_ratelimited(wil,
 687                                     "Key missing. CID %d TID %d MCast %d KEY_ID %d\n",
 688                                     cid, tid, mc, key_id);
 689                 return -EINVAL;
 690         }
 691
 692         if (reverse_memcmp(pn, cc->pn, IEEE80211_GCMP_PN_LEN) <= 0) {
 693                 wil_err_ratelimited(wil,
 694                                     "Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n",
 695                                     cid, tid, mc, key_id, pn, cc->pn);
 696                 return -EINVAL;
 697         }
 698         memcpy(cc->pn, pn, IEEE80211_GCMP_PN_LEN);
 699
 700         return 0;
 701 }
 702
 703 static int wil_rx_error_check(struct wil6210_priv *wil, struct sk_buff *skb,
 704                               struct wil_net_stats *stats)
 705 {
 706         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 707
 708         if ((d->dma.status & RX_DMA_STATUS_ERROR) &&
 709             (d->dma.error & RX_DMA_ERROR_MIC)) {
 710                 stats->rx_mic_error++;
 711                 wil_dbg_txrx(wil, "MIC error, dropping packet\n");
 712                 return -EFAULT;
 713         }
 714
 715         return 0;
 716 }
 717
 718 static void wil_get_netif_rx_params(struct sk_buff *skb, int *cid,
 719                                     int *security)
 720 {
 721         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
 722
 723         *cid = wil_skb_get_cid(skb);
 724         *security = wil_rxdesc_security(d);
 725 }
 726
 727 /*
 728  * Check if skb is ptk eapol key message
 729  *
 730  * returns a pointer to the start of the eapol key structure, NULL
 731  * if frame is not PTK eapol key
 732  */
 733 static struct wil_eapol_key *wil_is_ptk_eapol_key(struct wil6210_priv *wil,
 734                                                   struct sk_buff *skb)
 735 {
 736         u8 *buf;
 737         const struct wil_1x_hdr *hdr;
 738         struct wil_eapol_key *key;
 739         u16 key_info;
 740         int len = skb->len;
 741
 742         if (!skb_mac_header_was_set(skb)) {
 743                 wil_err(wil, "mac header was not set\n");
 744                 return NULL;
 745         }
 746
 747         len -= skb_mac_offset(skb);
 748
 749         if (len < sizeof(struct ethhdr) + sizeof(struct wil_1x_hdr) +
 750             sizeof(struct wil_eapol_key))
 751                 return NULL;
 752
 753         buf = skb_mac_header(skb) + sizeof(struct ethhdr);
 754
 755         hdr = (const struct wil_1x_hdr *)buf;
 756         if (hdr->type != WIL_1X_TYPE_EAPOL_KEY)
 757                 return NULL;
 758
 759         key = (struct wil_eapol_key *)(buf + sizeof(struct wil_1x_hdr));
 760         if (key->type != WIL_EAPOL_KEY_TYPE_WPA &&
 761             key->type != WIL_EAPOL_KEY_TYPE_RSN)
 762                 return NULL;
 763
 764         key_info = be16_to_cpu(key->key_info);
 765         if (!(key_info & WIL_KEY_INFO_KEY_TYPE)) /* check if pairwise */
 766                 return NULL;
 767
 768         return key;
 769 }
 770
 771 static bool wil_skb_is_eap_3(struct wil6210_priv *wil, struct sk_buff *skb)
 772 {
 773         struct wil_eapol_key *key;
 774         u16 key_info;
 775
 776         key = wil_is_ptk_eapol_key(wil, skb);
 777         if (!key)
 778                 return false;
 779
 780         key_info = be16_to_cpu(key->key_info);
 781         if (key_info & (WIL_KEY_INFO_MIC |
 782                         WIL_KEY_INFO_ENCR_KEY_DATA)) {
 783                 /* 3/4 of 4-Way Handshake */
 784                 wil_dbg_misc(wil, "EAPOL key message 3\n");
 785                 return true;
 786         }
 787         /* 1/4 of 4-Way Handshake */
 788         wil_dbg_misc(wil, "EAPOL key message 1\n");
 789
 790         return false;
 791 }
 792
 793 static bool wil_skb_is_eap_4(struct wil6210_priv *wil, struct sk_buff *skb)
 794 {
 795         struct wil_eapol_key *key;
 796         u32 *nonce, i;
 797
 798         key = wil_is_ptk_eapol_key(wil, skb);
 799         if (!key)
 800                 return false;
 801
 802         nonce = (u32 *)key->key_nonce;
 803         for (i = 0; i < WIL_EAP_NONCE_LEN / sizeof(u32); i++, nonce++) {
 804                 if (*nonce != 0) {
 805                         /* message 2/4 */
 806                         wil_dbg_misc(wil, "EAPOL key message 2\n");
 807                         return false;
 808                 }
 809         }
 810         wil_dbg_misc(wil, "EAPOL key message 4\n");
 811
 812         return true;
 813 }
 814
 815 void wil_enable_tx_key_worker(struct work_struct *work)
 816 {
 817         struct wil6210_vif *vif = container_of(work,
 818                         struct wil6210_vif, enable_tx_key_worker);
 819         struct wil6210_priv *wil = vif_to_wil(vif);
 820         int rc, cid;
 821
 822         rtnl_lock();
 823         if (vif->ptk_rekey_state != WIL_REKEY_WAIT_M4_SENT) {
 824                 wil_dbg_misc(wil, "Invalid rekey state = %d\n",
 825                              vif->ptk_rekey_state);
 826                 rtnl_unlock();
 827                 return;
 828         }
 829
 830         cid =  wil_find_cid_by_idx(wil, vif->mid, 0);
 831         if (!wil_cid_valid(wil, cid)) {
 832                 wil_err(wil, "Invalid cid = %d\n", cid);
 833                 rtnl_unlock();
 834                 return;
 835         }
 836
 837         wil_dbg_misc(wil, "Apply PTK key after eapol was sent out\n");
 838         rc = wmi_add_cipher_key(vif, 0, wil->sta[cid].addr, 0, NULL,
 839                                 WMI_KEY_USE_APPLY_PTK);
 840
 841         vif->ptk_rekey_state = WIL_REKEY_IDLE;
 842         rtnl_unlock();
 843
 844         if (rc)
 845                 wil_err(wil, "Apply PTK key failed %d\n", rc);
 846 }
 847
 848 void wil_tx_complete_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb)
 849 {
 850         struct wil6210_priv *wil = vif_to_wil(vif);
 851         struct wireless_dev *wdev = vif_to_wdev(vif);
 852         bool q = false;
 853
 854         if (wdev->iftype != NL80211_IFTYPE_STATION ||
 855             !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities))
 856                 return;
 857
 858         /* check if skb is an EAP message 4/4 */
 859         if (!wil_skb_is_eap_4(wil, skb))
 860                 return;
 861
 862         spin_lock_bh(&wil->eap_lock);
 863         switch (vif->ptk_rekey_state) {
 864         case WIL_REKEY_IDLE:
 865                 /* ignore idle state, can happen due to M4 retransmission */
 866                 break;
 867         case WIL_REKEY_M3_RECEIVED:
 868                 vif->ptk_rekey_state = WIL_REKEY_IDLE;
 869                 break;
 870         case WIL_REKEY_WAIT_M4_SENT:
 871                 q = true;
 872                 break;
 873         default:
 874                 wil_err(wil, "Unknown rekey state = %d",
 875                         vif->ptk_rekey_state);
 876         }
 877         spin_unlock_bh(&wil->eap_lock);
 878
 879         if (q) {
 880                 q = queue_work(wil->wmi_wq, &vif->enable_tx_key_worker);
 881                 wil_dbg_misc(wil, "queue_work of enable_tx_key_worker -> %d\n",
 882                              q);
 883         }
 884 }
 885
 886 static void wil_rx_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb)
 887 {
 888         struct wil6210_priv *wil = vif_to_wil(vif);
 889         struct wireless_dev *wdev = vif_to_wdev(vif);
 890
 891         if (wdev->iftype != NL80211_IFTYPE_STATION ||
 892             !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities))
 893                 return;
 894
 895         /* check if skb is a EAP message 3/4 */
 896         if (!wil_skb_is_eap_3(wil, skb))
 897                 return;
 898
 899         if (vif->ptk_rekey_state == WIL_REKEY_IDLE)
 900                 vif->ptk_rekey_state = WIL_REKEY_M3_RECEIVED;
 901 }
 902
 903 /*
 904  * Pass Rx packet to the netif. Update statistics.
 905  * Called in softirq context (NAPI poll).
 906  */
 907 void wil_netif_rx(struct sk_buff *skb, struct net_device *ndev, int cid,
 908                   struct wil_net_stats *stats, bool gro)
 909 {
 910         gro_result_t rc = GRO_NORMAL;
 911         struct wil6210_vif *vif = ndev_to_vif(ndev);
 912         struct wil6210_priv *wil = ndev_to_wil(ndev);
 913         struct wireless_dev *wdev = vif_to_wdev(vif);
 914         unsigned int len = skb->len;
 915         u8 *sa, *da = wil_skb_get_da(skb);
 916         /* here looking for DA, not A1, thus Rxdesc's 'mcast' indication
 917          * is not suitable, need to look at data
 918          */
 919         int mcast = is_multicast_ether_addr(da);
 920         struct sk_buff *xmit_skb = NULL;
 921         static const char * const gro_res_str[] = {
 922                 [GRO_MERGED]            = "GRO_MERGED",
 923                 [GRO_MERGED_FREE]       = "GRO_MERGED_FREE",
 924                 [GRO_HELD]              = "GRO_HELD",
 925                 [GRO_NORMAL]            = "GRO_NORMAL",
 926                 [GRO_DROP]              = "GRO_DROP",
 927                 [GRO_CONSUMED]          = "GRO_CONSUMED",
 928         };
 929
 930         if (wdev->iftype == NL80211_IFTYPE_STATION) {
 931                 sa = wil_skb_get_sa(skb);
 932                 if (mcast && ether_addr_equal(sa, ndev->dev_addr)) {
 933                         /* mcast packet looped back to us */
 934                         rc = GRO_DROP;
 935                         dev_kfree_skb(skb);
 936                         goto stats;
 937                 }
 938         } else if (wdev->iftype == NL80211_IFTYPE_AP && !vif->ap_isolate) {
 939                 if (mcast) {
 940                         /* send multicast frames both to higher layers in
 941                          * local net stack and back to the wireless medium
 942                          */
 943                         xmit_skb = skb_copy(skb, GFP_ATOMIC);
 944                 } else {
 945                         int xmit_cid = wil_find_cid(wil, vif->mid, da);
 946
 947                         if (xmit_cid >= 0) {
 948                                 /* The destination station is associated to
 949                                  * this AP (in this VLAN), so send the frame
 950                                  * directly to it and do not pass it to local
 951                                  * net stack.
 952                                  */
 953                                 xmit_skb = skb;
 954                                 skb = NULL;
 955                         }
 956                 }
 957         }
 958         if (xmit_skb) {
 959                 /* Send to wireless media and increase priority by 256 to
 960                  * keep the received priority instead of reclassifying
 961                  * the frame (see cfg80211_classify8021d).
 962                  */
 963                 xmit_skb->dev = ndev;
 964                 xmit_skb->priority += 256;
 965                 xmit_skb->protocol = htons(ETH_P_802_3);
 966                 skb_reset_network_header(xmit_skb);
 967                 skb_reset_mac_header(xmit_skb);
 968                 wil_dbg_txrx(wil, "Rx -> Tx %d bytes\n", len);
 969                 dev_queue_xmit(xmit_skb);
 970         }
 971
 972         if (skb) { /* deliver to local stack */
 973                 skb->protocol = eth_type_trans(skb, ndev);
 974                 skb->dev = ndev;
 975
 976                 if (skb->protocol == cpu_to_be16(ETH_P_PAE))
 977                         wil_rx_handle_eapol(vif, skb);
 978
 979                 if (gro)
 980                         rc = napi_gro_receive(&wil->napi_rx, skb);
 981                 else
 982                         netif_rx_ni(skb);
 983                 wil_dbg_txrx(wil, "Rx complete %d bytes => %s\n",
 984                              len, gro_res_str[rc]);
 985         }
 986 stats:
 987         /* statistics. rc set to GRO_NORMAL for AP bridging */
 988         if (unlikely(rc == GRO_DROP)) {
 989                 ndev->stats.rx_dropped++;
 990                 stats->rx_dropped++;
 991                 wil_dbg_txrx(wil, "Rx drop %d bytes\n", len);
 992         } else {
 993                 ndev->stats.rx_packets++;
 994                 stats->rx_packets++;
 995                 ndev->stats.rx_bytes += len;
 996                 stats->rx_bytes += len;
 997                 if (mcast)
 998                         ndev->stats.multicast++;
 999         }
1000 }
1001
1002 void wil_netif_rx_any(struct sk_buff *skb, struct net_device *ndev)
1003 {
1004         int cid, security;
1005         struct wil6210_priv *wil = ndev_to_wil(ndev);
1006         struct wil_net_stats *stats;
1007
1008         wil->txrx_ops.get_netif_rx_params(skb, &cid, &security);
1009
1010         stats = &wil->sta[cid].stats;
1011
1012         skb_orphan(skb);
1013
1014         if (security && (wil->txrx_ops.rx_crypto_check(wil, skb) != 0)) {
1015                 wil_dbg_txrx(wil, "Rx drop %d bytes\n", skb->len);
1016                 dev_kfree_skb(skb);
1017                 ndev->stats.rx_dropped++;
1018                 stats->rx_replay++;
1019                 stats->rx_dropped++;
1020                 return;
1021         }
1022
1023         /* check errors reported by HW and update statistics */
1024         if (unlikely(wil->txrx_ops.rx_error_check(wil, skb, stats))) {
1025                 dev_kfree_skb(skb);
1026                 return;
1027         }
1028
1029         wil_netif_rx(skb, ndev, cid, stats, true);
1030 }
1031
1032 /**
1033  * Proceed all completed skb's from Rx VRING
1034  *
1035  * Safe to call from NAPI poll, i.e. softirq with interrupts enabled
1036  */
1037 void wil_rx_handle(struct wil6210_priv *wil, int *quota)
1038 {
1039         struct net_device *ndev = wil->main_ndev;
1040         struct wireless_dev *wdev = ndev->ieee80211_ptr;
1041         struct wil_ring *v = &wil->ring_rx;
1042         struct sk_buff *skb;
1043
1044         if (unlikely(!v->va)) {
1045                 wil_err(wil, "Rx IRQ while Rx not yet initialized\n");
1046                 return;
1047         }
1048         wil_dbg_txrx(wil, "rx_handle\n");
1049         while ((*quota > 0) && (NULL != (skb = wil_vring_reap_rx(wil, v)))) {
1050                 (*quota)--;
1051
1052                 /* monitor is currently supported on main interface only */
1053                 if (wdev->iftype == NL80211_IFTYPE_MONITOR) {
1054                         skb->dev = ndev;
1055                         skb_reset_mac_header(skb);
1056                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1057                         skb->pkt_type = PACKET_OTHERHOST;
1058                         skb->protocol = htons(ETH_P_802_2);
1059                         wil_netif_rx_any(skb, ndev);
1060                 } else {
1061                         wil_rx_reorder(wil, skb);
1062                 }
1063         }
1064         wil_rx_refill(wil, v->size);
1065 }
1066
1067 static void wil_rx_buf_len_init(struct wil6210_priv *wil)
1068 {
1069         wil->rx_buf_len = rx_large_buf ?
1070                 WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD;
1071         if (mtu_max > wil->rx_buf_len) {
1072                 /* do not allow RX buffers to be smaller than mtu_max, for
1073                  * backward compatibility (mtu_max parameter was also used
1074                  * to support receiving large packets)
1075                  */
1076                 wil_info(wil, "Override RX buffer to mtu_max(%d)\n", mtu_max);
1077                 wil->rx_buf_len = mtu_max;
1078         }
1079 }
1080
1081 static int wil_rx_init(struct wil6210_priv *wil, uint order)
1082 {
1083         struct wil_ring *vring = &wil->ring_rx;
1084         int rc;
1085
1086         wil_dbg_misc(wil, "rx_init\n");
1087
1088         if (vring->va) {
1089                 wil_err(wil, "Rx ring already allocated\n");
1090                 return -EINVAL;
1091         }
1092
1093         wil_rx_buf_len_init(wil);
1094
1095         vring->size = 1 << order;
1096         vring->is_rx = true;
1097         rc = wil_vring_alloc(wil, vring);
1098         if (rc)
1099                 return rc;
1100
1101         rc = wmi_rx_chain_add(wil, vring);
1102         if (rc)
1103                 goto err_free;
1104
1105         rc = wil_rx_refill(wil, vring->size);
1106         if (rc)
1107                 goto err_free;
1108
1109         return 0;
1110  err_free:
1111         wil_vring_free(wil, vring);
1112
1113         return rc;
1114 }
1115
1116 static void wil_rx_fini(struct wil6210_priv *wil)
1117 {
1118         struct wil_ring *vring = &wil->ring_rx;
1119
1120         wil_dbg_misc(wil, "rx_fini\n");
1121
1122         if (vring->va)
1123                 wil_vring_free(wil, vring);
1124 }
1125
1126 static int wil_tx_desc_map(union wil_tx_desc *desc, dma_addr_t pa,
1127                            u32 len, int vring_index)
1128 {
1129         struct vring_tx_desc *d = &desc->legacy;
1130
1131         wil_desc_addr_set(&d->dma.addr, pa);
1132         d->dma.ip_length = 0;
1133         /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/
1134         d->dma.b11 = 0/*14 | BIT(7)*/;
1135         d->dma.error = 0;
1136         d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */
1137         d->dma.length = cpu_to_le16((u16)len);
1138         d->dma.d0 = (vring_index << DMA_CFG_DESC_TX_0_QID_POS);
1139         d->mac.d[0] = 0;
1140         d->mac.d[1] = 0;
1141         d->mac.d[2] = 0;
1142         d->mac.ucode_cmd = 0;
1143         /* translation type:  0 - bypass; 1 - 802.3; 2 - native wifi */
1144         d->mac.d[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS) |
1145                       (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS);
1146
1147         return 0;
1148 }
1149
1150 void wil_tx_data_init(struct wil_ring_tx_data *txdata)
1151 {
1152         spin_lock_bh(&txdata->lock);
1153         txdata->dot1x_open = 0;
1154         txdata->enabled = 0;
1155         txdata->idle = 0;
1156         txdata->last_idle = 0;
1157         txdata->begin = 0;
1158         txdata->agg_wsize = 0;
1159         txdata->agg_timeout = 0;
1160         txdata->agg_amsdu = 0;
1161         txdata->addba_in_progress = false;
1162         txdata->mid = U8_MAX;
1163         spin_unlock_bh(&txdata->lock);
1164 }
1165
1166 static int wil_vring_init_tx(struct wil6210_vif *vif, int id, int size,
1167                              int cid, int tid)
1168 {
1169         struct wil6210_priv *wil = vif_to_wil(vif);
1170         int rc;
1171         struct wmi_vring_cfg_cmd cmd = {
1172                 .action = cpu_to_le32(WMI_VRING_CMD_ADD),
1173                 .vring_cfg = {
1174                         .tx_sw_ring = {
1175                                 .max_mpdu_size =
1176                                         cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1177                                 .ring_size = cpu_to_le16(size),
1178                         },
1179                         .ringid = id,
1180                         .encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1181                         .mac_ctrl = 0,
1182                         .to_resolution = 0,
1183                         .agg_max_wsize = 0,
1184                         .schd_params = {
1185                                 .priority = cpu_to_le16(0),
1186                                 .timeslot_us = cpu_to_le16(0xfff),
1187                         },
1188                 },
1189         };
1190         struct {
1191                 struct wmi_cmd_hdr wmi;
1192                 struct wmi_vring_cfg_done_event cmd;
1193         } __packed reply = {
1194                 .cmd = {.status = WMI_FW_STATUS_FAILURE},
1195         };
1196         struct wil_ring *vring = &wil->ring_tx[id];
1197         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];
1198
1199         if (cid >= WIL6210_RX_DESC_MAX_CID) {
1200                 cmd.vring_cfg.cidxtid = CIDXTID_EXTENDED_CID_TID;
1201                 cmd.vring_cfg.cid = cid;
1202                 cmd.vring_cfg.tid = tid;
1203         } else {
1204                 cmd.vring_cfg.cidxtid = mk_cidxtid(cid, tid);
1205         }
1206
1207         wil_dbg_misc(wil, "vring_init_tx: max_mpdu_size %d\n",
1208                      cmd.vring_cfg.tx_sw_ring.max_mpdu_size);
1209         lockdep_assert_held(&wil->mutex);
1210
1211         if (vring->va) {
1212                 wil_err(wil, "Tx ring [%d] already allocated\n", id);
1213                 rc = -EINVAL;
1214                 goto out;
1215         }
1216
1217         wil_tx_data_init(txdata);
1218         vring->is_rx = false;
1219         vring->size = size;
1220         rc = wil_vring_alloc(wil, vring);
1221         if (rc)
1222                 goto out;
1223
1224         wil->ring2cid_tid[id][0] = cid;
1225         wil->ring2cid_tid[id][1] = tid;
1226
1227         cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1228
1229         if (!vif->privacy)
1230                 txdata->dot1x_open = true;
1231         rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd),
1232                       WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply),
1233                       WIL_WMI_CALL_GENERAL_TO_MS);
1234         if (rc)
1235                 goto out_free;
1236
1237         if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1238                 wil_err(wil, "Tx config failed, status 0x%02x\n",
1239                         reply.cmd.status);
1240                 rc = -EINVAL;
1241                 goto out_free;
1242         }
1243
1244         spin_lock_bh(&txdata->lock);
1245         vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
1246         txdata->mid = vif->mid;
1247         txdata->enabled = 1;
1248         spin_unlock_bh(&txdata->lock);
1249
1250         if (txdata->dot1x_open && (agg_wsize >= 0))
1251                 wil_addba_tx_request(wil, id, agg_wsize);
1252
1253         return 0;
1254  out_free:
1255         spin_lock_bh(&txdata->lock);
1256         txdata->dot1x_open = false;
1257         txdata->enabled = 0;
1258         spin_unlock_bh(&txdata->lock);
1259         wil_vring_free(wil, vring);
1260         wil->ring2cid_tid[id][0] = wil->max_assoc_sta;
1261         wil->ring2cid_tid[id][1] = 0;
1262
1263  out:
1264
1265         return rc;
1266 }
1267
1268 static int wil_tx_vring_modify(struct wil6210_vif *vif, int ring_id, int cid,
1269                                int tid)
1270 {
1271         struct wil6210_priv *wil = vif_to_wil(vif);
1272         int rc;
1273         struct wmi_vring_cfg_cmd cmd = {
1274                 .action = cpu_to_le32(WMI_VRING_CMD_MODIFY),
1275                 .vring_cfg = {
1276                         .tx_sw_ring = {
1277                                 .max_mpdu_size =
1278                                         cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1279                                 .ring_size = 0,
1280                         },
1281                         .ringid = ring_id,
1282                         .cidxtid = mk_cidxtid(cid, tid),
1283                         .encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1284                         .mac_ctrl = 0,
1285                         .to_resolution = 0,
1286                         .agg_max_wsize = 0,
1287                         .schd_params = {
1288                                 .priority = cpu_to_le16(0),
1289                                 .timeslot_us = cpu_to_le16(0xfff),
1290                         },
1291                 },
1292         };
1293         struct {
1294                 struct wmi_cmd_hdr wmi;
1295                 struct wmi_vring_cfg_done_event cmd;
1296         } __packed reply = {
1297                 .cmd = {.status = WMI_FW_STATUS_FAILURE},
1298         };
1299         struct wil_ring *vring = &wil->ring_tx[ring_id];
1300         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id];
1301
1302         wil_dbg_misc(wil, "vring_modify: ring %d cid %d tid %d\n", ring_id,
1303                      cid, tid);
1304         lockdep_assert_held(&wil->mutex);
1305
1306         if (!vring->va) {
1307                 wil_err(wil, "Tx ring [%d] not allocated\n", ring_id);
1308                 return -EINVAL;
1309         }
1310
1311         if (wil->ring2cid_tid[ring_id][0] != cid ||
1312             wil->ring2cid_tid[ring_id][1] != tid) {
1313                 wil_err(wil, "ring info does not match cid=%u tid=%u\n",
1314                         wil->ring2cid_tid[ring_id][0],
1315                         wil->ring2cid_tid[ring_id][1]);
1316         }
1317
1318         cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1319
1320         rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd),
1321                       WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply),
1322                       WIL_WMI_CALL_GENERAL_TO_MS);
1323         if (rc)
1324                 goto fail;
1325
1326         if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1327                 wil_err(wil, "Tx modify failed, status 0x%02x\n",
1328                         reply.cmd.status);
1329                 rc = -EINVAL;
1330                 goto fail;
1331         }
1332
1333         /* set BA aggregation window size to 0 to force a new BA with the
1334          * new AP
1335          */
1336         txdata->agg_wsize = 0;
1337         if (txdata->dot1x_open && agg_wsize >= 0)
1338                 wil_addba_tx_request(wil, ring_id, agg_wsize);
1339
1340         return 0;
1341 fail:
1342         spin_lock_bh(&txdata->lock);
1343         txdata->dot1x_open = false;
1344         txdata->enabled = 0;
1345         spin_unlock_bh(&txdata->lock);
1346         wil->ring2cid_tid[ring_id][0] = wil->max_assoc_sta;
1347         wil->ring2cid_tid[ring_id][1] = 0;
1348         return rc;
1349 }
1350
1351 int wil_vring_init_bcast(struct wil6210_vif *vif, int id, int size)
1352 {
1353         struct wil6210_priv *wil = vif_to_wil(vif);
1354         int rc;
1355         struct wmi_bcast_vring_cfg_cmd cmd = {
1356                 .action = cpu_to_le32(WMI_VRING_CMD_ADD),
1357                 .vring_cfg = {
1358                         .tx_sw_ring = {
1359                                 .max_mpdu_size =
1360                                         cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1361                                 .ring_size = cpu_to_le16(size),
1362                         },
1363                         .ringid = id,
1364                         .encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1365                 },
1366         };
1367         struct {
1368                 struct wmi_cmd_hdr wmi;
1369                 struct wmi_vring_cfg_done_event cmd;
1370         } __packed reply = {
1371                 .cmd = {.status = WMI_FW_STATUS_FAILURE},
1372         };
1373         struct wil_ring *vring = &wil->ring_tx[id];
1374         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];
1375
1376         wil_dbg_misc(wil, "vring_init_bcast: max_mpdu_size %d\n",
1377                      cmd.vring_cfg.tx_sw_ring.max_mpdu_size);
1378         lockdep_assert_held(&wil->mutex);
1379
1380         if (vring->va) {
1381                 wil_err(wil, "Tx ring [%d] already allocated\n", id);
1382                 rc = -EINVAL;
1383                 goto out;
1384         }
1385
1386         wil_tx_data_init(txdata);
1387         vring->is_rx = false;
1388         vring->size = size;
1389         rc = wil_vring_alloc(wil, vring);
1390         if (rc)
1391                 goto out;
1392
1393         wil->ring2cid_tid[id][0] = wil->max_assoc_sta; /* CID */
1394         wil->ring2cid_tid[id][1] = 0; /* TID */
1395
1396         cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1397
1398         if (!vif->privacy)
1399                 txdata->dot1x_open = true;
1400         rc = wmi_call(wil, WMI_BCAST_VRING_CFG_CMDID, vif->mid,
1401                       &cmd, sizeof(cmd),
1402                       WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply),
1403                       WIL_WMI_CALL_GENERAL_TO_MS);
1404         if (rc)
1405                 goto out_free;
1406
1407         if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1408                 wil_err(wil, "Tx config failed, status 0x%02x\n",
1409                         reply.cmd.status);
1410                 rc = -EINVAL;
1411                 goto out_free;
1412         }
1413
1414         spin_lock_bh(&txdata->lock);
1415         vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
1416         txdata->mid = vif->mid;
1417         txdata->enabled = 1;
1418         spin_unlock_bh(&txdata->lock);
1419
1420         return 0;
1421  out_free:
1422         spin_lock_bh(&txdata->lock);
1423         txdata->enabled = 0;
1424         txdata->dot1x_open = false;
1425         spin_unlock_bh(&txdata->lock);
1426         wil_vring_free(wil, vring);
1427  out:
1428
1429         return rc;
1430 }
1431
1432 static struct wil_ring *wil_find_tx_ucast(struct wil6210_priv *wil,
1433                                           struct wil6210_vif *vif,
1434                                           struct sk_buff *skb)
1435 {
1436         int i, cid;
1437         const u8 *da = wil_skb_get_da(skb);
1438         int min_ring_id = wil_get_min_tx_ring_id(wil);
1439
1440         cid = wil_find_cid(wil, vif->mid, da);
1441
1442         if (cid < 0 || cid >= wil->max_assoc_sta)
1443                 return NULL;
1444
1445         /* TODO: fix for multiple TID */
1446         for (i = min_ring_id; i < ARRAY_SIZE(wil->ring2cid_tid); i++) {
1447                 if (!wil->ring_tx_data[i].dot1x_open &&
1448                     skb->protocol != cpu_to_be16(ETH_P_PAE))
1449                         continue;
1450                 if (wil->ring2cid_tid[i][0] == cid) {
1451                         struct wil_ring *v = &wil->ring_tx[i];
1452                         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i];
1453
1454                         wil_dbg_txrx(wil, "find_tx_ucast: (%pM) -> [%d]\n",
1455                                      da, i);
1456                         if (v->va && txdata->enabled) {
1457                                 return v;
1458                         } else {
1459                                 wil_dbg_txrx(wil,
1460                                              "find_tx_ucast: vring[%d] not valid\n",
1461                                              i);
1462                                 return NULL;
1463                         }
1464                 }
1465         }
1466
1467         return NULL;
1468 }
1469
1470 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
1471                        struct wil_ring *ring, struct sk_buff *skb);
1472
1473 static struct wil_ring *wil_find_tx_ring_sta(struct wil6210_priv *wil,
1474                                              struct wil6210_vif *vif,
1475                                              struct sk_buff *skb)
1476 {
1477         struct wil_ring *ring;
1478         int i;
1479         u8 cid;
1480         struct wil_ring_tx_data  *txdata;
1481         int min_ring_id = wil_get_min_tx_ring_id(wil);
1482
1483         /* In the STA mode, it is expected to have only 1 VRING
1484          * for the AP we connected to.
1485          * find 1-st vring eligible for this skb and use it.
1486          */
1487         for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
1488                 ring = &wil->ring_tx[i];
1489                 txdata = &wil->ring_tx_data[i];
1490                 if (!ring->va || !txdata->enabled || txdata->mid != vif->mid)
1491                         continue;
1492
1493                 cid = wil->ring2cid_tid[i][0];
1494                 if (cid >= wil->max_assoc_sta) /* skip BCAST */
1495                         continue;
1496
1497                 if (!wil->ring_tx_data[i].dot1x_open &&
1498                     skb->protocol != cpu_to_be16(ETH_P_PAE))
1499                         continue;
1500
1501                 wil_dbg_txrx(wil, "Tx -> ring %d\n", i);
1502
1503                 return ring;
1504         }
1505
1506         wil_dbg_txrx(wil, "Tx while no rings active?\n");
1507
1508         return NULL;
1509 }
1510
1511 /* Use one of 2 strategies:
1512  *
1513  * 1. New (real broadcast):
1514  *    use dedicated broadcast vring
1515  * 2. Old (pseudo-DMS):
1516  *    Find 1-st vring and return it;
1517  *    duplicate skb and send it to other active vrings;
1518  *    in all cases override dest address to unicast peer's address
1519  * Use old strategy when new is not supported yet:
1520  *  - for PBSS
1521  */
1522 static struct wil_ring *wil_find_tx_bcast_1(struct wil6210_priv *wil,
1523                                             struct wil6210_vif *vif,
1524                                             struct sk_buff *skb)
1525 {
1526         struct wil_ring *v;
1527         struct wil_ring_tx_data *txdata;
1528         int i = vif->bcast_ring;
1529
1530         if (i < 0)
1531                 return NULL;
1532         v = &wil->ring_tx[i];
1533         txdata = &wil->ring_tx_data[i];
1534         if (!v->va || !txdata->enabled)
1535                 return NULL;
1536         if (!wil->ring_tx_data[i].dot1x_open &&
1537             skb->protocol != cpu_to_be16(ETH_P_PAE))
1538                 return NULL;
1539
1540         return v;
1541 }
1542
1543 static void wil_set_da_for_vring(struct wil6210_priv *wil,
1544                                  struct sk_buff *skb, int vring_index)
1545 {
1546         u8 *da = wil_skb_get_da(skb);
1547         int cid = wil->ring2cid_tid[vring_index][0];
1548
1549         ether_addr_copy(da, wil->sta[cid].addr);
1550 }
1551
1552 static struct wil_ring *wil_find_tx_bcast_2(struct wil6210_priv *wil,
1553                                             struct wil6210_vif *vif,
1554                                             struct sk_buff *skb)
1555 {
1556         struct wil_ring *v, *v2;
1557         struct sk_buff *skb2;
1558         int i;
1559         u8 cid;
1560         const u8 *src = wil_skb_get_sa(skb);
1561         struct wil_ring_tx_data *txdata, *txdata2;
1562         int min_ring_id = wil_get_min_tx_ring_id(wil);
1563
1564         /* find 1-st vring eligible for data */
1565         for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
1566                 v = &wil->ring_tx[i];
1567                 txdata = &wil->ring_tx_data[i];
1568                 if (!v->va || !txdata->enabled || txdata->mid != vif->mid)
1569                         continue;
1570
1571                 cid = wil->ring2cid_tid[i][0];
1572                 if (cid >= wil->max_assoc_sta) /* skip BCAST */
1573                         continue;
1574                 if (!wil->ring_tx_data[i].dot1x_open &&
1575                     skb->protocol != cpu_to_be16(ETH_P_PAE))
1576                         continue;
1577
1578                 /* don't Tx back to source when re-routing Rx->Tx at the AP */
1579                 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN))
1580                         continue;
1581
1582                 goto found;
1583         }
1584
1585         wil_dbg_txrx(wil, "Tx while no vrings active?\n");
1586
1587         return NULL;
1588
1589 found:
1590         wil_dbg_txrx(wil, "BCAST -> ring %d\n", i);
1591         wil_set_da_for_vring(wil, skb, i);
1592
1593         /* find other active vrings and duplicate skb for each */
1594         for (i++; i < WIL6210_MAX_TX_RINGS; i++) {
1595                 v2 = &wil->ring_tx[i];
1596                 txdata2 = &wil->ring_tx_data[i];
1597                 if (!v2->va || txdata2->mid != vif->mid)
1598                         continue;
1599                 cid = wil->ring2cid_tid[i][0];
1600                 if (cid >= wil->max_assoc_sta) /* skip BCAST */
1601                         continue;
1602                 if (!wil->ring_tx_data[i].dot1x_open &&
1603                     skb->protocol != cpu_to_be16(ETH_P_PAE))
1604                         continue;
1605
1606                 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN))
1607                         continue;
1608
1609                 skb2 = skb_copy(skb, GFP_ATOMIC);
1610                 if (skb2) {
1611                         wil_dbg_txrx(wil, "BCAST DUP -> ring %d\n", i);
1612                         wil_set_da_for_vring(wil, skb2, i);
1613                         wil_tx_ring(wil, vif, v2, skb2);
1614                         /* successful call to wil_tx_ring takes skb2 ref */
1615                         dev_kfree_skb_any(skb2);
1616                 } else {
1617                         wil_err(wil, "skb_copy failed\n");
1618                 }
1619         }
1620
1621         return v;
1622 }
1623
1624 static inline
1625 void wil_tx_desc_set_nr_frags(struct vring_tx_desc *d, int nr_frags)
1626 {
1627         d->mac.d[2] |= (nr_frags << MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS);
1628 }
1629
1630 /**
1631  * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding
1632  * @skb is used to obtain the protocol and headers length.
1633  * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data,
1634  * 2 - middle, 3 - last descriptor.
1635  */
1636
1637 static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc *d,
1638                                           struct sk_buff *skb,
1639                                           int tso_desc_type, bool is_ipv4,
1640                                           int tcp_hdr_len, int skb_net_hdr_len)
1641 {
1642         d->dma.b11 = ETH_HLEN; /* MAC header length */
1643         d->dma.b11 |= is_ipv4 << DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS;
1644
1645         d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS);
1646         /* L4 header len: TCP header length */
1647         d->dma.d0 |= (tcp_hdr_len & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1648
1649         /* Setup TSO: bit and desc type */
1650         d->dma.d0 |= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS)) |
1651                 (tso_desc_type << DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS);
1652         d->dma.d0 |= (is_ipv4 << DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS);
1653
1654         d->dma.ip_length = skb_net_hdr_len;
1655         /* Enable TCP/UDP checksum */
1656         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS);
1657         /* Calculate pseudo-header */
1658         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS);
1659 }
1660
1661 /**
1662  * Sets the descriptor @d up for csum. The corresponding
1663  * @skb is used to obtain the protocol and headers length.
1664  * Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6.
1665  * Note, if d==NULL, the function only returns the protocol result.
1666  *
1667  * It is very similar to previous wil_tx_desc_offload_setup_tso. This
1668  * is "if unrolling" to optimize the critical path.
1669  */
1670
1671 static int wil_tx_desc_offload_setup(struct vring_tx_desc *d,
1672                                      struct sk_buff *skb){
1673         int protocol;
1674
1675         if (skb->ip_summed != CHECKSUM_PARTIAL)
1676                 return 0;
1677
1678         d->dma.b11 = ETH_HLEN; /* MAC header length */
1679
1680         switch (skb->protocol) {
1681         case cpu_to_be16(ETH_P_IP):
1682                 protocol = ip_hdr(skb)->protocol;
1683                 d->dma.b11 |= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS);
1684                 break;
1685         case cpu_to_be16(ETH_P_IPV6):
1686                 protocol = ipv6_hdr(skb)->nexthdr;
1687                 break;
1688         default:
1689                 return -EINVAL;
1690         }
1691
1692         switch (protocol) {
1693         case IPPROTO_TCP:
1694                 d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS);
1695                 /* L4 header len: TCP header length */
1696                 d->dma.d0 |=
1697                 (tcp_hdrlen(skb) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1698                 break;
1699         case IPPROTO_UDP:
1700                 /* L4 header len: UDP header length */
1701                 d->dma.d0 |=
1702                 (sizeof(struct udphdr) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1703                 break;
1704         default:
1705                 return -EINVAL;
1706         }
1707
1708         d->dma.ip_length = skb_network_header_len(skb);
1709         /* Enable TCP/UDP checksum */
1710         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS);
1711         /* Calculate pseudo-header */
1712         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS);
1713
1714         return 0;
1715 }
1716
1717 static inline void wil_tx_last_desc(struct vring_tx_desc *d)
1718 {
1719         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS) |
1720               BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS) |
1721               BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS);
1722 }
1723
1724 static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc *d)
1725 {
1726         d->dma.d0 |= wil_tso_type_lst <<
1727                   DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS;
1728 }
1729
1730 static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct wil6210_vif *vif,
1731                               struct wil_ring *vring, struct sk_buff *skb)
1732 {
1733         struct device *dev = wil_to_dev(wil);
1734
1735         /* point to descriptors in shared memory */
1736         volatile struct vring_tx_desc *_desc = NULL, *_hdr_desc,
1737                                       *_first_desc = NULL;
1738
1739         /* pointers to shadow descriptors */
1740         struct vring_tx_desc desc_mem, hdr_desc_mem, first_desc_mem,
1741                              *d = &hdr_desc_mem, *hdr_desc = &hdr_desc_mem,
1742                              *first_desc = &first_desc_mem;
1743
1744         /* pointer to shadow descriptors' context */
1745         struct wil_ctx *hdr_ctx, *first_ctx = NULL;
1746
1747         int descs_used = 0; /* total number of used descriptors */
1748         int sg_desc_cnt = 0; /* number of descriptors for current mss*/
1749
1750         u32 swhead = vring->swhead;
1751         int used, avail = wil_ring_avail_tx(vring);
1752         int nr_frags = skb_shinfo(skb)->nr_frags;
1753         int min_desc_required = nr_frags + 1;
1754         int mss = skb_shinfo(skb)->gso_size;    /* payload size w/o headers */
1755         int f, len, hdrlen, headlen;
1756         int vring_index = vring - wil->ring_tx;
1757         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[vring_index];
1758         uint i = swhead;
1759         dma_addr_t pa;
1760         const skb_frag_t *frag = NULL;
1761         int rem_data = mss;
1762         int lenmss;
1763         int hdr_compensation_need = true;
1764         int desc_tso_type = wil_tso_type_first;
1765         bool is_ipv4;
1766         int tcp_hdr_len;
1767         int skb_net_hdr_len;
1768         int gso_type;
1769         int rc = -EINVAL;
1770
1771         wil_dbg_txrx(wil, "tx_vring_tso: %d bytes to vring %d\n", skb->len,
1772                      vring_index);
1773
1774         if (unlikely(!txdata->enabled))
1775                 return -EINVAL;
1776
1777         /* A typical page 4K is 3-4 payloads, we assume each fragment
1778          * is a full payload, that's how min_desc_required has been
1779          * calculated. In real we might need more or less descriptors,
1780          * this is the initial check only.
1781          */
1782         if (unlikely(avail < min_desc_required)) {
1783                 wil_err_ratelimited(wil,
1784                                     "TSO: Tx ring[%2d] full. No space for %d fragments\n",
1785                                     vring_index, min_desc_required);
1786                 return -ENOMEM;
1787         }
1788
1789         /* Header Length = MAC header len + IP header len + TCP header len*/
1790         hdrlen = ETH_HLEN +
1791                 (int)skb_network_header_len(skb) +
1792                 tcp_hdrlen(skb);
1793
1794         gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4);
1795         switch (gso_type) {
1796         case SKB_GSO_TCPV4:
1797                 /* TCP v4, zero out the IP length and IPv4 checksum fields
1798                  * as required by the offloading doc
1799                  */
1800                 ip_hdr(skb)->tot_len = 0;
1801                 ip_hdr(skb)->check = 0;
1802                 is_ipv4 = true;
1803                 break;
1804         case SKB_GSO_TCPV6:
1805                 /* TCP v6, zero out the payload length */
1806                 ipv6_hdr(skb)->payload_len = 0;
1807                 is_ipv4 = false;
1808                 break;
1809         default:
1810                 /* other than TCPv4 or TCPv6 types are not supported for TSO.
1811                  * It is also illegal for both to be set simultaneously
1812                  */
1813                 return -EINVAL;
1814         }
1815
1816         if (skb->ip_summed != CHECKSUM_PARTIAL)
1817                 return -EINVAL;
1818
1819         /* tcp header length and skb network header length are fixed for all
1820          * packet's descriptors - read then once here
1821          */
1822         tcp_hdr_len = tcp_hdrlen(skb);
1823         skb_net_hdr_len = skb_network_header_len(skb);
1824
1825         _hdr_desc = &vring->va[i].tx.legacy;
1826
1827         pa = dma_map_single(dev, skb->data, hdrlen, DMA_TO_DEVICE);
1828         if (unlikely(dma_mapping_error(dev, pa))) {
1829                 wil_err(wil, "TSO: Skb head DMA map error\n");
1830                 goto err_exit;
1831         }
1832
1833         wil->txrx_ops.tx_desc_map((union wil_tx_desc *)hdr_desc, pa,
1834                                   hdrlen, vring_index);
1835         wil_tx_desc_offload_setup_tso(hdr_desc, skb, wil_tso_type_hdr, is_ipv4,
1836                                       tcp_hdr_len, skb_net_hdr_len);
1837         wil_tx_last_desc(hdr_desc);
1838
1839         vring->ctx[i].mapped_as = wil_mapped_as_single;
1840         hdr_ctx = &vring->ctx[i];
1841
1842         descs_used++;
1843         headlen = skb_headlen(skb) - hdrlen;
1844
1845         for (f = headlen ? -1 : 0; f < nr_frags; f++)  {
1846                 if (headlen) {
1847                         len = headlen;
1848                         wil_dbg_txrx(wil, "TSO: process skb head, len %u\n",
1849                                      len);
1850                 } else {
1851                         frag = &skb_shinfo(skb)->frags[f];
1852                         len = skb_frag_size(frag);
1853                         wil_dbg_txrx(wil, "TSO: frag[%d]: len %u\n", f, len);
1854                 }
1855
1856                 while (len) {
1857                         wil_dbg_txrx(wil,
1858                                      "TSO: len %d, rem_data %d, descs_used %d\n",
1859                                      len, rem_data, descs_used);
1860
1861                         if (descs_used == avail)  {
1862                                 wil_err_ratelimited(wil, "TSO: ring overflow\n");
1863                                 rc = -ENOMEM;
1864                                 goto mem_error;
1865                         }
1866
1867                         lenmss = min_t(int, rem_data, len);
1868                         i = (swhead + descs_used) % vring->size;
1869                         wil_dbg_txrx(wil, "TSO: lenmss %d, i %d\n", lenmss, i);
1870
1871                         if (!headlen) {
1872                                 pa = skb_frag_dma_map(dev, frag,
1873                                                       skb_frag_size(frag) - len,
1874                                                       lenmss, DMA_TO_DEVICE);
1875                                 vring->ctx[i].mapped_as = wil_mapped_as_page;
1876                         } else {
1877                                 pa = dma_map_single(dev,
1878                                                     skb->data +
1879                                                     skb_headlen(skb) - headlen,
1880                                                     lenmss,
1881                                                     DMA_TO_DEVICE);
1882                                 vring->ctx[i].mapped_as = wil_mapped_as_single;
1883                                 headlen -= lenmss;
1884                         }
1885
1886                         if (unlikely(dma_mapping_error(dev, pa))) {
1887                                 wil_err(wil, "TSO: DMA map page error\n");
1888                                 goto mem_error;
1889                         }
1890
1891                         _desc = &vring->va[i].tx.legacy;
1892
1893                         if (!_first_desc) {
1894                                 _first_desc = _desc;
1895                                 first_ctx = &vring->ctx[i];
1896                                 d = first_desc;
1897                         } else {
1898                                 d = &desc_mem;
1899                         }
1900
1901                         wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,
1902                                                   pa, lenmss, vring_index);
1903                         wil_tx_desc_offload_setup_tso(d, skb, desc_tso_type,
1904                                                       is_ipv4, tcp_hdr_len,
1905                                                       skb_net_hdr_len);
1906
1907                         /* use tso_type_first only once */
1908                         desc_tso_type = wil_tso_type_mid;
1909
1910                         descs_used++;  /* desc used so far */
1911                         sg_desc_cnt++; /* desc used for this segment */
1912                         len -= lenmss;
1913                         rem_data -= lenmss;
1914
1915                         wil_dbg_txrx(wil,
1916                                      "TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n",
1917                                      len, rem_data, descs_used, sg_desc_cnt);
1918
1919                         /* Close the segment if reached mss size or last frag*/
1920                         if (rem_data == 0 || (f == nr_frags - 1 && len == 0)) {
1921                                 if (hdr_compensation_need) {
1922                                         /* first segment include hdr desc for
1923                                          * release
1924                                          */
1925                                         hdr_ctx->nr_frags = sg_desc_cnt;
1926                                         wil_tx_desc_set_nr_frags(first_desc,
1927                                                                  sg_desc_cnt +
1928                                                                  1);
1929                                         hdr_compensation_need = false;
1930                                 } else {
1931                                         wil_tx_desc_set_nr_frags(first_desc,
1932                                                                  sg_desc_cnt);
1933                                 }
1934                                 first_ctx->nr_frags = sg_desc_cnt - 1;
1935
1936                                 wil_tx_last_desc(d);
1937
1938                                 /* first descriptor may also be the last
1939                                  * for this mss - make sure not to copy
1940                                  * it twice
1941                                  */
1942                                 if (first_desc != d)
1943                                         *_first_desc = *first_desc;
1944
1945                                 /*last descriptor will be copied at the end
1946                                  * of this TS processing
1947                                  */
1948                                 if (f < nr_frags - 1 || len > 0)
1949                                         *_desc = *d;
1950
1951                                 rem_data = mss;
1952                                 _first_desc = NULL;
1953                                 sg_desc_cnt = 0;
1954                         } else if (first_desc != d) /* update mid descriptor */
1955                                         *_desc = *d;
1956                 }
1957         }
1958
1959         if (!_desc)
1960                 goto mem_error;
1961
1962         /* first descriptor may also be the last.
1963          * in this case d pointer is invalid
1964          */
1965         if (_first_desc == _desc)
1966                 d = first_desc;
1967
1968         /* Last data descriptor */
1969         wil_set_tx_desc_last_tso(d);
1970         *_desc = *d;
1971
1972         /* Fill the total number of descriptors in first desc (hdr)*/
1973         wil_tx_desc_set_nr_frags(hdr_desc, descs_used);
1974         *_hdr_desc = *hdr_desc;
1975
1976         /* hold reference to skb
1977          * to prevent skb release before accounting
1978          * in case of immediate "tx done"
1979          */
1980         vring->ctx[i].skb = skb_get(skb);
1981
1982         /* performance monitoring */
1983         used = wil_ring_used_tx(vring);
1984         if (wil_val_in_range(wil->ring_idle_trsh,
1985                              used, used + descs_used)) {
1986                 txdata->idle += get_cycles() - txdata->last_idle;
1987                 wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",
1988                              vring_index, used, used + descs_used);
1989         }
1990
1991         /* Make sure to advance the head only after descriptor update is done.
1992          * This will prevent a race condition where the completion thread
1993          * will see the DU bit set from previous run and will handle the
1994          * skb before it was completed.
1995          */
1996         wmb();
1997
1998         /* advance swhead */
1999         wil_ring_advance_head(vring, descs_used);
2000         wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, vring->swhead);
2001
2002         /* make sure all writes to descriptors (shared memory) are done before
2003          * committing them to HW
2004          */
2005         wmb();
2006
2007         if (wil->tx_latency)
2008                 *(ktime_t *)&skb->cb = ktime_get();
2009         else
2010                 memset(skb->cb, 0, sizeof(ktime_t));
2011
2012         wil_w(wil, vring->hwtail, vring->swhead);
2013         return 0;
2014
2015 mem_error:
2016         while (descs_used > 0) {
2017                 struct wil_ctx *ctx;
2018
2019                 i = (swhead + descs_used - 1) % vring->size;
2020                 d = (struct vring_tx_desc *)&vring->va[i].tx.legacy;
2021                 _desc = &vring->va[i].tx.legacy;
2022                 *d = *_desc;
2023                 _desc->dma.status = TX_DMA_STATUS_DU;
2024                 ctx = &vring->ctx[i];
2025                 wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx);
2026                 memset(ctx, 0, sizeof(*ctx));
2027                 descs_used--;
2028         }
2029 err_exit:
2030         return rc;
2031 }
2032
2033 static int __wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
2034                          struct wil_ring *ring, struct sk_buff *skb)
2035 {
2036         struct device *dev = wil_to_dev(wil);
2037         struct vring_tx_desc dd, *d = &dd;
2038         volatile struct vring_tx_desc *_d;
2039         u32 swhead = ring->swhead;
2040         int avail = wil_ring_avail_tx(ring);
2041         int nr_frags = skb_shinfo(skb)->nr_frags;
2042         uint f = 0;
2043         int ring_index = ring - wil->ring_tx;
2044         struct wil_ring_tx_data  *txdata = &wil->ring_tx_data[ring_index];
2045         uint i = swhead;
2046         dma_addr_t pa;
2047         int used;
2048         bool mcast = (ring_index == vif->bcast_ring);
2049         uint len = skb_headlen(skb);
2050
2051         wil_dbg_txrx(wil, "tx_ring: %d bytes to ring %d, nr_frags %d\n",
2052                      skb->len, ring_index, nr_frags);
2053
2054         if (unlikely(!txdata->enabled))
2055                 return -EINVAL;
2056
2057         if (unlikely(avail < 1 + nr_frags)) {
2058                 wil_err_ratelimited(wil,
2059                                     "Tx ring[%2d] full. No space for %d fragments\n",
2060                                     ring_index, 1 + nr_frags);
2061                 return -ENOMEM;
2062         }
2063         _d = &ring->va[i].tx.legacy;
2064
2065         pa = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
2066
2067         wil_dbg_txrx(wil, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", ring_index,
2068                      skb_headlen(skb), skb->data, &pa);
2069         wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET, 16, 1,
2070                           skb->data, skb_headlen(skb), false);
2071
2072         if (unlikely(dma_mapping_error(dev, pa)))
2073                 return -EINVAL;
2074         ring->ctx[i].mapped_as = wil_mapped_as_single;
2075         /* 1-st segment */
2076         wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, pa, len,
2077                                    ring_index);
2078         if (unlikely(mcast)) {
2079                 d->mac.d[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS); /* MCS 0 */
2080                 if (unlikely(len > WIL_BCAST_MCS0_LIMIT)) /* set MCS 1 */
2081                         d->mac.d[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS);
2082         }
2083         /* Process TCP/UDP checksum offloading */
2084         if (unlikely(wil_tx_desc_offload_setup(d, skb))) {
2085                 wil_err(wil, "Tx[%2d] Failed to set cksum, drop packet\n",
2086                         ring_index);
2087                 goto dma_error;
2088         }
2089
2090         ring->ctx[i].nr_frags = nr_frags;
2091         wil_tx_desc_set_nr_frags(d, nr_frags + 1);
2092
2093         /* middle segments */
2094         for (; f < nr_frags; f++) {
2095                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2096                 int len = skb_frag_size(frag);
2097
2098                 *_d = *d;
2099                 wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);
2100                 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4,
2101                                   (const void *)d, sizeof(*d), false);
2102                 i = (swhead + f + 1) % ring->size;
2103                 _d = &ring->va[i].tx.legacy;
2104                 pa = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag),
2105                                       DMA_TO_DEVICE);
2106                 if (unlikely(dma_mapping_error(dev, pa))) {
2107                         wil_err(wil, "Tx[%2d] failed to map fragment\n",
2108                                 ring_index);
2109                         goto dma_error;
2110                 }
2111                 ring->ctx[i].mapped_as = wil_mapped_as_page;
2112                 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,
2113                                            pa, len, ring_index);
2114                 /* no need to check return code -
2115                  * if it succeeded for 1-st descriptor,
2116                  * it will succeed here too
2117                  */
2118                 wil_tx_desc_offload_setup(d, skb);
2119         }
2120         /* for the last seg only */
2121         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS);
2122         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS);
2123         d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS);
2124         *_d = *d;
2125         wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);
2126         wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4,
2127                           (const void *)d, sizeof(*d), false);
2128
2129         /* hold reference to skb
2130          * to prevent skb release before accounting
2131          * in case of immediate "tx done"
2132          */
2133         ring->ctx[i].skb = skb_get(skb);
2134
2135         /* performance monitoring */
2136         used = wil_ring_used_tx(ring);
2137         if (wil_val_in_range(wil->ring_idle_trsh,
2138                              used, used + nr_frags + 1)) {
2139                 txdata->idle += get_cycles() - txdata->last_idle;
2140                 wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",
2141                              ring_index, used, used + nr_frags + 1);
2142         }
2143
2144         /* Make sure to advance the head only after descriptor update is done.
2145          * This will prevent a race condition where the completion thread
2146          * will see the DU bit set from previous run and will handle the
2147          * skb before it was completed.
2148          */
2149         wmb();
2150
2151         /* advance swhead */
2152         wil_ring_advance_head(ring, nr_frags + 1);
2153         wil_dbg_txrx(wil, "Tx[%2d] swhead %d -> %d\n", ring_index, swhead,
2154                      ring->swhead);
2155         trace_wil6210_tx(ring_index, swhead, skb->len, nr_frags);
2156
2157         /* make sure all writes to descriptors (shared memory) are done before
2158          * committing them to HW
2159          */
2160         wmb();
2161
2162         if (wil->tx_latency)
2163                 *(ktime_t *)&skb->cb = ktime_get();
2164         else
2165                 memset(skb->cb, 0, sizeof(ktime_t));
2166
2167         wil_w(wil, ring->hwtail, ring->swhead);
2168
2169         return 0;
2170  dma_error:
2171         /* unmap what we have mapped */
2172         nr_frags = f + 1; /* frags mapped + one for skb head */
2173         for (f = 0; f < nr_frags; f++) {
2174                 struct wil_ctx *ctx;
2175
2176                 i = (swhead + f) % ring->size;
2177                 ctx = &ring->ctx[i];
2178                 _d = &ring->va[i].tx.legacy;
2179                 *d = *_d;
2180                 _d->dma.status = TX_DMA_STATUS_DU;
2181                 wil->txrx_ops.tx_desc_unmap(dev,
2182                                             (union wil_tx_desc *)d,
2183                                             ctx);
2184
2185                 memset(ctx, 0, sizeof(*ctx));
2186         }
2187
2188         return -EINVAL;
2189 }
2190
2191 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
2192                        struct wil_ring *ring, struct sk_buff *skb)
2193 {
2194         int ring_index = ring - wil->ring_tx;
2195         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index];
2196         int rc;
2197
2198         spin_lock(&txdata->lock);
2199
2200         if (test_bit(wil_status_suspending, wil->status) ||
2201             test_bit(wil_status_suspended, wil->status) ||
2202             test_bit(wil_status_resuming, wil->status)) {
2203                 wil_dbg_txrx(wil,
2204                              "suspend/resume in progress. drop packet\n");
2205                 spin_unlock(&txdata->lock);
2206                 return -EINVAL;
2207         }
2208
2209         rc = (skb_is_gso(skb) ? wil->txrx_ops.tx_ring_tso : __wil_tx_ring)
2210              (wil, vif, ring, skb);
2211
2212         spin_unlock(&txdata->lock);
2213
2214         return rc;
2215 }
2216
2217 /**
2218  * Check status of tx vrings and stop/wake net queues if needed
2219  * It will start/stop net queues of a specific VIF net_device.
2220  *
2221  * This function does one of two checks:
2222  * In case check_stop is true, will check if net queues need to be stopped. If
2223  * the conditions for stopping are met, netif_tx_stop_all_queues() is called.
2224  * In case check_stop is false, will check if net queues need to be waked. If
2225  * the conditions for waking are met, netif_tx_wake_all_queues() is called.
2226  * vring is the vring which is currently being modified by either adding
2227  * descriptors (tx) into it or removing descriptors (tx complete) from it. Can
2228  * be null when irrelevant (e.g. connect/disconnect events).
2229  *
2230  * The implementation is to stop net queues if modified vring has low
2231  * descriptor availability. Wake if all vrings are not in low descriptor
2232  * availability and modified vring has high descriptor availability.
2233  */
2234 static inline void __wil_update_net_queues(struct wil6210_priv *wil,
2235                                            struct wil6210_vif *vif,
2236                                            struct wil_ring *ring,
2237                                            bool check_stop)
2238 {
2239         int i;
2240         int min_ring_id = wil_get_min_tx_ring_id(wil);
2241
2242         if (unlikely(!vif))
2243                 return;
2244
2245         if (ring)
2246                 wil_dbg_txrx(wil, "vring %d, mid %d, check_stop=%d, stopped=%d",
2247                              (int)(ring - wil->ring_tx), vif->mid, check_stop,
2248                              vif->net_queue_stopped);
2249         else
2250                 wil_dbg_txrx(wil, "check_stop=%d, mid=%d, stopped=%d",
2251                              check_stop, vif->mid, vif->net_queue_stopped);
2252
2253         if (ring && drop_if_ring_full)
2254                 /* no need to stop/wake net queues */
2255                 return;
2256
2257         if (check_stop == vif->net_queue_stopped)
2258                 /* net queues already in desired state */
2259                 return;
2260
2261         if (check_stop) {
2262                 if (!ring || unlikely(wil_ring_avail_low(ring))) {
2263                         /* not enough room in the vring */
2264                         netif_tx_stop_all_queues(vif_to_ndev(vif));
2265                         vif->net_queue_stopped = true;
2266                         wil_dbg_txrx(wil, "netif_tx_stop called\n");
2267                 }
2268                 return;
2269         }
2270
2271         /* Do not wake the queues in suspend flow */
2272         if (test_bit(wil_status_suspending, wil->status) ||
2273             test_bit(wil_status_suspended, wil->status))
2274                 return;
2275
2276         /* check wake */
2277         for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
2278                 struct wil_ring *cur_ring = &wil->ring_tx[i];
2279                 struct wil_ring_tx_data  *txdata = &wil->ring_tx_data[i];
2280
2281                 if (txdata->mid != vif->mid || !cur_ring->va ||
2282                     !txdata->enabled || cur_ring == ring)
2283                         continue;
2284
2285                 if (wil_ring_avail_low(cur_ring)) {
2286                         wil_dbg_txrx(wil, "ring %d full, can't wake\n",
2287                                      (int)(cur_ring - wil->ring_tx));
2288                         return;
2289                 }
2290         }
2291
2292         if (!ring || wil_ring_avail_high(ring)) {
2293                 /* enough room in the ring */
2294                 wil_dbg_txrx(wil, "calling netif_tx_wake\n");
2295                 netif_tx_wake_all_queues(vif_to_ndev(vif));
2296                 vif->net_queue_stopped = false;
2297         }
2298 }
2299
2300 void wil_update_net_queues(struct wil6210_priv *wil, struct wil6210_vif *vif,
2301                            struct wil_ring *ring, bool check_stop)
2302 {
2303         spin_lock(&wil->net_queue_lock);
2304         __wil_update_net_queues(wil, vif, ring, check_stop);
2305         spin_unlock(&wil->net_queue_lock);
2306 }
2307
2308 void wil_update_net_queues_bh(struct wil6210_priv *wil, struct wil6210_vif *vif,
2309                               struct wil_ring *ring, bool check_stop)
2310 {
2311         spin_lock_bh(&wil->net_queue_lock);
2312         __wil_update_net_queues(wil, vif, ring, check_stop);
2313         spin_unlock_bh(&wil->net_queue_lock);
2314 }
2315
2316 netdev_tx_t wil_start_xmit(struct sk_buff *skb, struct net_device *ndev)
2317 {
2318         struct wil6210_vif *vif = ndev_to_vif(ndev);
2319         struct wil6210_priv *wil = vif_to_wil(vif);
2320         const u8 *da = wil_skb_get_da(skb);
2321         bool bcast = is_multicast_ether_addr(da);
2322         struct wil_ring *ring;
2323         static bool pr_once_fw;
2324         int rc;
2325
2326         wil_dbg_txrx(wil, "start_xmit\n");
2327         if (unlikely(!test_bit(wil_status_fwready, wil->status))) {
2328                 if (!pr_once_fw) {
2329                         wil_err(wil, "FW not ready\n");
2330                         pr_once_fw = true;
2331                 }
2332                 goto drop;
2333         }
2334         if (unlikely(!test_bit(wil_vif_fwconnected, vif->status))) {
2335                 wil_dbg_ratelimited(wil,
2336                                     "VIF not connected, packet dropped\n");
2337                 goto drop;
2338         }
2339         if (unlikely(vif->wdev.iftype == NL80211_IFTYPE_MONITOR)) {
2340                 wil_err(wil, "Xmit in monitor mode not supported\n");
2341                 goto drop;
2342         }
2343         pr_once_fw = false;
2344
2345         /* find vring */
2346         if (vif->wdev.iftype == NL80211_IFTYPE_STATION && !vif->pbss) {
2347                 /* in STA mode (ESS), all to same VRING (to AP) */
2348                 ring = wil_find_tx_ring_sta(wil, vif, skb);
2349         } else if (bcast) {
2350                 if (vif->pbss)
2351                         /* in pbss, no bcast VRING - duplicate skb in
2352                          * all stations VRINGs
2353                          */
2354                         ring = wil_find_tx_bcast_2(wil, vif, skb);
2355                 else if (vif->wdev.iftype == NL80211_IFTYPE_AP)
2356                         /* AP has a dedicated bcast VRING */
2357                         ring = wil_find_tx_bcast_1(wil, vif, skb);
2358                 else
2359                         /* unexpected combination, fallback to duplicating
2360                          * the skb in all stations VRINGs
2361                          */
2362                         ring = wil_find_tx_bcast_2(wil, vif, skb);
2363         } else {
2364                 /* unicast, find specific VRING by dest. address */
2365                 ring = wil_find_tx_ucast(wil, vif, skb);
2366         }
2367         if (unlikely(!ring)) {
2368                 wil_dbg_txrx(wil, "No Tx RING found for %pM\n", da);
2369                 goto drop;
2370         }
2371         /* set up vring entry */
2372         rc = wil_tx_ring(wil, vif, ring, skb);
2373
2374         switch (rc) {
2375         case 0:
2376                 /* shall we stop net queues? */
2377                 wil_update_net_queues_bh(wil, vif, ring, true);
2378                 /* statistics will be updated on the tx_complete */
2379                 dev_kfree_skb_any(skb);
2380                 return NETDEV_TX_OK;
2381         case -ENOMEM:
2382                 if (drop_if_ring_full)
2383                         goto drop;
2384                 return NETDEV_TX_BUSY;
2385         default:
2386                 break; /* goto drop; */
2387         }
2388  drop:
2389         ndev->stats.tx_dropped++;
2390         dev_kfree_skb_any(skb);
2391
2392         return NET_XMIT_DROP;
2393 }
2394
2395 void wil_tx_latency_calc(struct wil6210_priv *wil, struct sk_buff *skb,
2396                          struct wil_sta_info *sta)
2397 {
2398         int skb_time_us;
2399         int bin;
2400
2401         if (!wil->tx_latency)
2402                 return;
2403
2404         if (ktime_to_ms(*(ktime_t *)&skb->cb) == 0)
2405                 return;
2406
2407         skb_time_us = ktime_us_delta(ktime_get(), *(ktime_t *)&skb->cb);
2408         bin = skb_time_us / wil->tx_latency_res;
2409         bin = min_t(int, bin, WIL_NUM_LATENCY_BINS - 1);
2410
2411         wil_dbg_txrx(wil, "skb time %dus => bin %d\n", skb_time_us, bin);
2412         sta->tx_latency_bins[bin]++;
2413         sta->stats.tx_latency_total_us += skb_time_us;
2414         if (skb_time_us < sta->stats.tx_latency_min_us)
2415                 sta->stats.tx_latency_min_us = skb_time_us;
2416         if (skb_time_us > sta->stats.tx_latency_max_us)
2417                 sta->stats.tx_latency_max_us = skb_time_us;
2418 }
2419
2420 /**
2421  * Clean up transmitted skb's from the Tx VRING
2422  *
2423  * Return number of descriptors cleared
2424  *
2425  * Safe to call from IRQ
2426  */
2427 int wil_tx_complete(struct wil6210_vif *vif, int ringid)
2428 {
2429         struct wil6210_priv *wil = vif_to_wil(vif);
2430         struct net_device *ndev = vif_to_ndev(vif);
2431         struct device *dev = wil_to_dev(wil);
2432         struct wil_ring *vring = &wil->ring_tx[ringid];
2433         struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ringid];
2434         int done = 0;
2435         int cid = wil->ring2cid_tid[ringid][0];
2436         struct wil_net_stats *stats = NULL;
2437         volatile struct vring_tx_desc *_d;
2438         int used_before_complete;
2439         int used_new;
2440
2441         if (unlikely(!vring->va)) {
2442                 wil_err(wil, "Tx irq[%d]: vring not initialized\n", ringid);
2443                 return 0;
2444         }
2445
2446         if (unlikely(!txdata->enabled)) {
2447                 wil_info(wil, "Tx irq[%d]: vring disabled\n", ringid);
2448                 return 0;
2449         }
2450
2451         wil_dbg_txrx(wil, "tx_complete: (%d)\n", ringid);
2452
2453         used_before_complete = wil_ring_used_tx(vring);
2454
2455         if (cid < wil->max_assoc_sta)
2456                 stats = &wil->sta[cid].stats;
2457
2458         while (!wil_ring_is_empty(vring)) {
2459                 int new_swtail;
2460                 struct wil_ctx *ctx = &vring->ctx[vring->swtail];
2461                 /**
2462                  * For the fragmented skb, HW will set DU bit only for the
2463                  * last fragment. look for it.
2464                  * In TSO the first DU will include hdr desc
2465                  */
2466                 int lf = (vring->swtail + ctx->nr_frags) % vring->size;
2467                 /* TODO: check we are not past head */
2468
2469                 _d = &vring->va[lf].tx.legacy;
2470                 if (unlikely(!(_d->dma.status & TX_DMA_STATUS_DU)))
2471                         break;
2472
2473                 new_swtail = (lf + 1) % vring->size;
2474                 while (vring->swtail != new_swtail) {
2475                         struct vring_tx_desc dd, *d = &dd;
2476                         u16 dmalen;
2477                         struct sk_buff *skb;
2478
2479                         ctx = &vring->ctx[vring->swtail];
2480                         skb = ctx->skb;
2481                         _d = &vring->va[vring->swtail].tx.legacy;
2482
2483                         *d = *_d;
2484
2485                         dmalen = le16_to_cpu(d->dma.length);
2486                         trace_wil6210_tx_done(ringid, vring->swtail, dmalen,
2487                                               d->dma.error);
2488                         wil_dbg_txrx(wil,
2489                                      "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n",
2490                                      ringid, vring->swtail, dmalen,
2491                                      d->dma.status, d->dma.error);
2492                         wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE, 32, 4,
2493                                           (const void *)d, sizeof(*d), false);
2494
2495                         wil->txrx_ops.tx_desc_unmap(dev,
2496                                                     (union wil_tx_desc *)d,
2497                                                     ctx);
2498
2499                         if (skb) {
2500                                 if (likely(d->dma.error == 0)) {
2501                                         ndev->stats.tx_packets++;
2502                                         ndev->stats.tx_bytes += skb->len;
2503                                         if (stats) {
2504                                                 stats->tx_packets++;
2505                                                 stats->tx_bytes += skb->len;
2506
2507                                                 wil_tx_latency_calc(wil, skb,
2508                                                         &wil->sta[cid]);
2509                                         }
2510                                 } else {
2511                                         ndev->stats.tx_errors++;
2512                                         if (stats)
2513                                                 stats->tx_errors++;
2514                                 }
2515
2516                                 if (skb->protocol == cpu_to_be16(ETH_P_PAE))
2517                                         wil_tx_complete_handle_eapol(vif, skb);
2518
2519                                 wil_consume_skb(skb, d->dma.error == 0);
2520                         }
2521                         memset(ctx, 0, sizeof(*ctx));
2522                         /* Make sure the ctx is zeroed before updating the tail
2523                          * to prevent a case where wil_tx_ring will see
2524                          * this descriptor as used and handle it before ctx zero
2525                          * is completed.
2526                          */
2527                         wmb();
2528                         /* There is no need to touch HW descriptor:
2529                          * - ststus bit TX_DMA_STATUS_DU is set by design,
2530                          *   so hardware will not try to process this desc.,
2531                          * - rest of descriptor will be initialized on Tx.
2532                          */
2533                         vring->swtail = wil_ring_next_tail(vring);
2534                         done++;
2535                 }
2536         }
2537
2538         /* performance monitoring */
2539         used_new = wil_ring_used_tx(vring);
2540         if (wil_val_in_range(wil->ring_idle_trsh,
2541                              used_new, used_before_complete)) {
2542                 wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n",
2543                              ringid, used_before_complete, used_new);
2544                 txdata->last_idle = get_cycles();
2545         }
2546
2547         /* shall we wake net queues? */
2548         if (done)
2549                 wil_update_net_queues(wil, vif, vring, false);
2550
2551         return done;
2552 }
2553
2554 static inline int wil_tx_init(struct wil6210_priv *wil)
2555 {
2556         return 0;
2557 }
2558
2559 static inline void wil_tx_fini(struct wil6210_priv *wil) {}
2560
2561 static void wil_get_reorder_params(struct wil6210_priv *wil,
2562                                    struct sk_buff *skb, int *tid, int *cid,
2563                                    int *mid, u16 *seq, int *mcast, int *retry)
2564 {
2565         struct vring_rx_desc *d = wil_skb_rxdesc(skb);
2566
2567         *tid = wil_rxdesc_tid(d);
2568         *cid = wil_skb_get_cid(skb);
2569         *mid = wil_rxdesc_mid(d);
2570         *seq = wil_rxdesc_seq(d);
2571         *mcast = wil_rxdesc_mcast(d);
2572         *retry = wil_rxdesc_retry(d);
2573 }
2574
2575 void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil)
2576 {
2577         wil->txrx_ops.configure_interrupt_moderation =
2578                 wil_configure_interrupt_moderation;
2579         /* TX ops */
2580         wil->txrx_ops.tx_desc_map = wil_tx_desc_map;
2581         wil->txrx_ops.tx_desc_unmap = wil_txdesc_unmap;
2582         wil->txrx_ops.tx_ring_tso =  __wil_tx_vring_tso;
2583         wil->txrx_ops.ring_init_tx = wil_vring_init_tx;
2584         wil->txrx_ops.ring_fini_tx = wil_vring_free;
2585         wil->txrx_ops.ring_init_bcast = wil_vring_init_bcast;
2586         wil->txrx_ops.tx_init = wil_tx_init;
2587         wil->txrx_ops.tx_fini = wil_tx_fini;
2588         wil->txrx_ops.tx_ring_modify = wil_tx_vring_modify;
2589         /* RX ops */
2590         wil->txrx_ops.rx_init = wil_rx_init;
2591         wil->txrx_ops.wmi_addba_rx_resp = wmi_addba_rx_resp;
2592         wil->txrx_ops.get_reorder_params = wil_get_reorder_params;
2593         wil->txrx_ops.get_netif_rx_params =
2594                 wil_get_netif_rx_params;
2595         wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check;
2596         wil->txrx_ops.rx_error_check = wil_rx_error_check;
2597         wil->txrx_ops.is_rx_idle = wil_is_rx_idle;
2598         wil->txrx_ops.rx_fini = wil_rx_fini;
2599 }