2 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
4 * Permission to use, copy, modify, and/or distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 #include <linux/dma-mapping.h>
21 #define DMA_DUMMY_TXWI ((void *) ~0)
24 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q)
29 spin_lock_init(&q->lock);
30 INIT_LIST_HEAD(&q->swq);
32 size = q->ndesc * sizeof(struct mt76_desc);
33 q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
37 size = q->ndesc * sizeof(*q->entry);
38 q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
42 /* clear descriptors */
43 for (i = 0; i < q->ndesc; i++)
44 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
46 iowrite32(q->desc_dma, &q->regs->desc_base);
47 iowrite32(0, &q->regs->cpu_idx);
48 iowrite32(0, &q->regs->dma_idx);
49 iowrite32(q->ndesc, &q->regs->ring_size);
55 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
56 struct mt76_queue_buf *buf, int nbufs, u32 info,
57 struct sk_buff *skb, void *txwi)
59 struct mt76_desc *desc;
64 q->entry[q->head].txwi = DMA_DUMMY_TXWI;
66 for (i = 0; i < nbufs; i += 2, buf += 2) {
67 u32 buf0 = buf[0].addr, buf1 = 0;
69 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
72 ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
76 ctrl |= MT_DMA_CTL_LAST_SEC0;
77 else if (i == nbufs - 2)
78 ctrl |= MT_DMA_CTL_LAST_SEC1;
81 q->head = (q->head + 1) % q->ndesc;
85 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
86 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
87 WRITE_ONCE(desc->info, cpu_to_le32(info));
88 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
93 q->entry[idx].txwi = txwi;
94 q->entry[idx].skb = skb;
100 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
101 struct mt76_queue_entry *prev_e)
103 struct mt76_queue_entry *e = &q->entry[idx];
104 __le32 __ctrl = READ_ONCE(q->desc[idx].ctrl);
105 u32 ctrl = le32_to_cpu(__ctrl);
107 if (!e->txwi || !e->skb) {
108 __le32 addr = READ_ONCE(q->desc[idx].buf0);
109 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);
111 dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
115 if (!(ctrl & MT_DMA_CTL_LAST_SEC0)) {
116 __le32 addr = READ_ONCE(q->desc[idx].buf1);
117 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN1, ctrl);
119 dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
123 if (e->txwi == DMA_DUMMY_TXWI)
127 memset(e, 0, sizeof(*e));
131 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
133 iowrite32(q->desc_dma, &q->regs->desc_base);
134 iowrite32(q->ndesc, &q->regs->ring_size);
135 q->head = ioread32(&q->regs->dma_idx);
137 iowrite32(q->head, &q->regs->cpu_idx);
141 mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
143 struct mt76_queue *q = &dev->q_tx[qid];
144 struct mt76_queue_entry entry;
151 spin_lock_bh(&q->lock);
155 last = ioread32(&q->regs->dma_idx);
157 while (q->queued && q->tail != last) {
158 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
162 q->tail = (q->tail + 1) % q->ndesc;
166 spin_unlock_bh(&q->lock);
167 dev->drv->tx_complete_skb(dev, q, &entry, flush);
168 spin_lock_bh(&q->lock);
172 mt76_put_txwi(dev, entry.txwi);
176 if (!flush && q->tail == last)
177 last = ioread32(&q->regs->dma_idx);
181 mt76_txq_schedule(dev, q);
183 mt76_dma_sync_idx(dev, q);
185 wake = wake && q->stopped &&
186 qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
191 wake_up(&dev->tx_wait);
193 spin_unlock_bh(&q->lock);
196 ieee80211_wake_queue(dev->hw, qid);
200 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
201 int *len, u32 *info, bool *more)
203 struct mt76_queue_entry *e = &q->entry[idx];
204 struct mt76_desc *desc = &q->desc[idx];
207 int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
209 buf_addr = le32_to_cpu(READ_ONCE(desc->buf0));
211 u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
212 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
213 *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
217 *info = le32_to_cpu(desc->info);
219 dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
226 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
227 int *len, u32 *info, bool *more)
235 if (!flush && !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
238 q->tail = (q->tail + 1) % q->ndesc;
241 return mt76_dma_get_buf(dev, q, idx, len, info, more);
245 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
247 iowrite32(q->head, &q->regs->cpu_idx);
251 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, enum mt76_txq_id qid,
252 struct sk_buff *skb, u32 tx_info)
254 struct mt76_queue *q = &dev->q_tx[qid];
255 struct mt76_queue_buf buf;
258 addr = dma_map_single(dev->dev, skb->data, skb->len,
260 if (dma_mapping_error(dev->dev, addr))
266 spin_lock_bh(&q->lock);
267 mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
268 mt76_dma_kick_queue(dev, q);
269 spin_unlock_bh(&q->lock);
274 int mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
275 struct sk_buff *skb, struct mt76_wcid *wcid,
276 struct ieee80211_sta *sta)
278 struct mt76_queue_entry e;
279 struct mt76_txwi_cache *t;
280 struct mt76_queue_buf buf[32];
281 struct sk_buff *iter;
287 t = mt76_get_txwi(dev);
289 ieee80211_free_txskb(dev->hw, skb);
293 skb->prev = skb->next = NULL;
294 dma_sync_single_for_cpu(dev->dev, t->dma_addr, sizeof(t->txwi),
296 ret = dev->drv->tx_prepare_skb(dev, &t->txwi, skb, q, wcid, sta,
298 dma_sync_single_for_device(dev->dev, t->dma_addr, sizeof(t->txwi),
303 len = skb->len - skb->data_len;
304 addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
305 if (dma_mapping_error(dev->dev, addr)) {
311 buf[n].addr = t->dma_addr;
312 buf[n++].len = dev->drv->txwi_size;
316 skb_walk_frags(skb, iter) {
317 if (n == ARRAY_SIZE(buf))
320 addr = dma_map_single(dev->dev, iter->data, iter->len,
322 if (dma_mapping_error(dev->dev, addr))
326 buf[n++].len = iter->len;
329 if (q->queued + (n + 1) / 2 >= q->ndesc - 1)
332 return mt76_dma_add_buf(dev, q, buf, n, tx_info, skb, t);
336 for (n--; n > 0; n--)
337 dma_unmap_single(dev->dev, buf[n].addr, buf[n].len,
343 dev->drv->tx_complete_skb(dev, q, &e, true);
344 mt76_put_txwi(dev, t);
347 EXPORT_SYMBOL_GPL(mt76_dma_tx_queue_skb);
350 mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
355 int len = SKB_WITH_OVERHEAD(q->buf_size);
356 int offset = q->buf_offset;
359 spin_lock_bh(&q->lock);
361 while (q->queued < q->ndesc - 1) {
362 struct mt76_queue_buf qbuf;
364 buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
368 addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
369 if (dma_mapping_error(dev->dev, addr)) {
374 qbuf.addr = addr + offset;
375 qbuf.len = len - offset;
376 idx = mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
381 mt76_dma_kick_queue(dev, q);
383 spin_unlock_bh(&q->lock);
389 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
395 spin_lock_bh(&q->lock);
397 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
403 spin_unlock_bh(&q->lock);
408 page = virt_to_page(q->rx_page.va);
409 __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
410 memset(&q->rx_page, 0, sizeof(q->rx_page));
414 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
416 struct mt76_queue *q = &dev->q_rx[qid];
419 for (i = 0; i < q->ndesc; i++)
420 q->desc[i].ctrl &= ~cpu_to_le32(MT_DMA_CTL_DMA_DONE);
422 mt76_dma_rx_cleanup(dev, q);
423 mt76_dma_sync_idx(dev, q);
424 mt76_dma_rx_fill(dev, q);
428 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
431 struct page *page = virt_to_head_page(data);
432 int offset = data - page_address(page);
433 struct sk_buff *skb = q->rx_head;
435 offset += q->buf_offset;
436 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, len,
443 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
447 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
449 int len, data_len, done = 0;
454 while (done < budget) {
457 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
462 data_len = q->buf_size;
464 data_len = SKB_WITH_OVERHEAD(q->buf_size);
466 if (data_len < len + q->buf_offset) {
467 dev_kfree_skb(q->rx_head);
475 mt76_add_fragment(dev, q, data, len, more);
479 skb = build_skb(data, q->buf_size);
484 skb_reserve(skb, q->buf_offset);
486 if (q == &dev->q_rx[MT_RXQ_MCU]) {
487 u32 *rxfce = (u32 *) skb->cb;
499 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
502 mt76_dma_rx_fill(dev, q);
507 mt76_dma_rx_poll(struct napi_struct *napi, int budget)
509 struct mt76_dev *dev;
510 int qid, done = 0, cur;
512 dev = container_of(napi->dev, struct mt76_dev, napi_dev);
513 qid = napi - dev->napi;
518 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
519 mt76_rx_poll_complete(dev, qid, napi);
521 } while (cur && done < budget);
527 dev->drv->rx_poll_complete(dev, qid);
534 mt76_dma_init(struct mt76_dev *dev)
538 init_dummy_netdev(&dev->napi_dev);
540 for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
541 netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
543 mt76_dma_rx_fill(dev, &dev->q_rx[i]);
544 skb_queue_head_init(&dev->rx_skb[i]);
545 napi_enable(&dev->napi[i]);
551 static const struct mt76_queue_ops mt76_dma_ops = {
552 .init = mt76_dma_init,
553 .alloc = mt76_dma_alloc_queue,
554 .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
555 .tx_queue_skb = mt76_dma_tx_queue_skb,
556 .tx_cleanup = mt76_dma_tx_cleanup,
557 .rx_reset = mt76_dma_rx_reset,
558 .kick = mt76_dma_kick_queue,
561 void mt76_dma_attach(struct mt76_dev *dev)
563 dev->queue_ops = &mt76_dma_ops;
565 EXPORT_SYMBOL_GPL(mt76_dma_attach);
567 void mt76_dma_cleanup(struct mt76_dev *dev)
571 for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
572 mt76_dma_tx_cleanup(dev, i, true);
574 for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
575 netif_napi_del(&dev->napi[i]);
576 mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
579 EXPORT_SYMBOL_GPL(mt76_dma_cleanup);