1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
4 * Copyright (C) 2015-2019 Google, Inc.
8 #include "gve_adminq.h"
10 #include <linux/tcp.h>
11 #include <linux/vmalloc.h>
12 #include <linux/skbuff.h>
14 static inline void gve_tx_put_doorbell(struct gve_priv *priv,
15 struct gve_queue_resources *q_resources,
18 iowrite32be(val, &priv->db_bar2[be32_to_cpu(q_resources->db_index)]);
21 /* gvnic can only transmit from a Registered Segment.
22 * We copy skb payloads into the registered segment before writing Tx
23 * descriptors and ringing the Tx doorbell.
25 * gve_tx_fifo_* manages the Registered Segment as a FIFO - clients must
26 * free allocations in the order they were allocated.
29 static int gve_tx_fifo_init(struct gve_priv *priv, struct gve_tx_fifo *fifo)
31 fifo->base = vmap(fifo->qpl->pages, fifo->qpl->num_entries, VM_MAP,
33 if (unlikely(!fifo->base)) {
34 netif_err(priv, drv, priv->dev, "Failed to vmap fifo, qpl_id = %d\n",
39 fifo->size = fifo->qpl->num_entries * PAGE_SIZE;
40 atomic_set(&fifo->available, fifo->size);
45 static void gve_tx_fifo_release(struct gve_priv *priv, struct gve_tx_fifo *fifo)
47 WARN(atomic_read(&fifo->available) != fifo->size,
48 "Releasing non-empty fifo");
53 static int gve_tx_fifo_pad_alloc_one_frag(struct gve_tx_fifo *fifo,
56 return (fifo->head + bytes < fifo->size) ? 0 : fifo->size - fifo->head;
59 static bool gve_tx_fifo_can_alloc(struct gve_tx_fifo *fifo, size_t bytes)
61 return (atomic_read(&fifo->available) <= bytes) ? false : true;
64 /* gve_tx_alloc_fifo - Allocate fragment(s) from Tx FIFO
65 * @fifo: FIFO to allocate from
66 * @bytes: Allocation size
67 * @iov: Scatter-gather elements to fill with allocation fragment base/len
69 * Returns number of valid elements in iov[] or negative on error.
71 * Allocations from a given FIFO must be externally synchronized but concurrent
72 * allocation and frees are allowed.
74 static int gve_tx_alloc_fifo(struct gve_tx_fifo *fifo, size_t bytes,
75 struct gve_tx_iovec iov[2])
77 size_t overflow, padding;
84 /* This check happens before we know how much padding is needed to
85 * align to a cacheline boundary for the payload, but that is fine,
86 * because the FIFO head always start aligned, and the FIFO's boundaries
87 * are aligned, so if there is space for the data, there is space for
88 * the padding to the next alignment.
90 WARN(!gve_tx_fifo_can_alloc(fifo, bytes),
91 "Reached %s when there's not enough space in the fifo", __func__);
95 iov[0].iov_offset = fifo->head;
96 iov[0].iov_len = bytes;
99 if (fifo->head > fifo->size) {
100 /* If the allocation did not fit in the tail fragment of the
101 * FIFO, also use the head fragment.
104 overflow = fifo->head - fifo->size;
105 iov[0].iov_len -= overflow;
106 iov[1].iov_offset = 0; /* Start of fifo*/
107 iov[1].iov_len = overflow;
109 fifo->head = overflow;
112 /* Re-align to a cacheline boundary */
113 aligned_head = L1_CACHE_ALIGN(fifo->head);
114 padding = aligned_head - fifo->head;
115 iov[nfrags - 1].iov_padding = padding;
116 atomic_sub(bytes + padding, &fifo->available);
117 fifo->head = aligned_head;
119 if (fifo->head == fifo->size)
125 /* gve_tx_free_fifo - Return space to Tx FIFO
126 * @fifo: FIFO to return fragments to
127 * @bytes: Bytes to free
129 static void gve_tx_free_fifo(struct gve_tx_fifo *fifo, size_t bytes)
131 atomic_add(bytes, &fifo->available);
134 static void gve_tx_remove_from_block(struct gve_priv *priv, int queue_idx)
136 struct gve_notify_block *block =
137 &priv->ntfy_blocks[gve_tx_idx_to_ntfy(priv, queue_idx)];
142 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
143 u32 to_do, bool try_to_wake);
145 static void gve_tx_free_ring(struct gve_priv *priv, int idx)
147 struct gve_tx_ring *tx = &priv->tx[idx];
148 struct device *hdev = &priv->pdev->dev;
152 gve_tx_remove_from_block(priv, idx);
153 slots = tx->mask + 1;
154 gve_clean_tx_done(priv, tx, tx->req, false);
155 netdev_tx_reset_queue(tx->netdev_txq);
157 dma_free_coherent(hdev, sizeof(*tx->q_resources),
158 tx->q_resources, tx->q_resources_bus);
159 tx->q_resources = NULL;
161 if (!tx->raw_addressing) {
162 gve_tx_fifo_release(priv, &tx->tx_fifo);
163 gve_unassign_qpl(priv, tx->tx_fifo.qpl->id);
164 tx->tx_fifo.qpl = NULL;
167 bytes = sizeof(*tx->desc) * slots;
168 dma_free_coherent(hdev, bytes, tx->desc, tx->bus);
174 netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx);
177 static void gve_tx_add_to_block(struct gve_priv *priv, int queue_idx)
179 int ntfy_idx = gve_tx_idx_to_ntfy(priv, queue_idx);
180 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
181 struct gve_tx_ring *tx = &priv->tx[queue_idx];
184 tx->ntfy_id = ntfy_idx;
187 static int gve_tx_alloc_ring(struct gve_priv *priv, int idx)
189 struct gve_tx_ring *tx = &priv->tx[idx];
190 struct device *hdev = &priv->pdev->dev;
191 u32 slots = priv->tx_desc_cnt;
194 /* Make sure everything is zeroed to start */
195 memset(tx, 0, sizeof(*tx));
198 tx->mask = slots - 1;
201 tx->info = vzalloc(sizeof(*tx->info) * slots);
206 bytes = sizeof(*tx->desc) * slots;
207 tx->desc = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL);
209 goto abort_with_info;
211 tx->raw_addressing = priv->raw_addressing;
212 tx->dev = &priv->pdev->dev;
213 if (!tx->raw_addressing) {
214 tx->tx_fifo.qpl = gve_assign_tx_qpl(priv);
215 if (!tx->tx_fifo.qpl)
216 goto abort_with_desc;
218 if (gve_tx_fifo_init(priv, &tx->tx_fifo))
223 dma_alloc_coherent(hdev,
224 sizeof(*tx->q_resources),
225 &tx->q_resources_bus,
227 if (!tx->q_resources)
228 goto abort_with_fifo;
230 netif_dbg(priv, drv, priv->dev, "tx[%d]->bus=%lx\n", idx,
231 (unsigned long)tx->bus);
232 tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx);
233 gve_tx_add_to_block(priv, idx);
238 if (!tx->raw_addressing)
239 gve_tx_fifo_release(priv, &tx->tx_fifo);
241 if (!tx->raw_addressing)
242 gve_unassign_qpl(priv, tx->tx_fifo.qpl->id);
244 dma_free_coherent(hdev, bytes, tx->desc, tx->bus);
252 int gve_tx_alloc_rings(struct gve_priv *priv)
257 for (i = 0; i < priv->tx_cfg.num_queues; i++) {
258 err = gve_tx_alloc_ring(priv, i);
260 netif_err(priv, drv, priv->dev,
261 "Failed to alloc tx ring=%d: err=%d\n",
266 /* Unallocate if there was an error */
270 for (j = 0; j < i; j++)
271 gve_tx_free_ring(priv, j);
276 void gve_tx_free_rings(struct gve_priv *priv)
280 for (i = 0; i < priv->tx_cfg.num_queues; i++)
281 gve_tx_free_ring(priv, i);
284 /* gve_tx_avail - Calculates the number of slots available in the ring
285 * @tx: tx ring to check
287 * Returns the number of slots available
289 * The capacity of the queue is mask + 1. We don't need to reserve an entry.
291 static inline u32 gve_tx_avail(struct gve_tx_ring *tx)
293 return tx->mask + 1 - (tx->req - tx->done);
296 static inline int gve_skb_fifo_bytes_required(struct gve_tx_ring *tx,
299 int pad_bytes, align_hdr_pad;
303 hlen = skb_is_gso(skb) ? skb_checksum_start_offset(skb) +
304 tcp_hdrlen(skb) : skb_headlen(skb);
306 pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo,
308 /* We need to take into account the header alignment padding. */
309 align_hdr_pad = L1_CACHE_ALIGN(hlen) - hlen;
310 bytes = align_hdr_pad + pad_bytes + skb->len;
315 /* The most descriptors we could need is MAX_SKB_FRAGS + 3 : 1 for each skb frag,
316 * +1 for the skb linear portion, +1 for when tcp hdr needs to be in separate descriptor,
317 * and +1 if the payload wraps to the beginning of the FIFO.
319 #define MAX_TX_DESC_NEEDED (MAX_SKB_FRAGS + 3)
320 static void gve_tx_unmap_buf(struct device *dev, struct gve_tx_buffer_state *info)
323 dma_unmap_single(dev, dma_unmap_addr(&info->buf, dma),
324 dma_unmap_len(&info->buf, len),
326 dma_unmap_len_set(&info->buf, len, 0);
328 dma_unmap_page(dev, dma_unmap_addr(&info->buf, dma),
329 dma_unmap_len(&info->buf, len),
331 dma_unmap_len_set(&info->buf, len, 0);
335 /* Check if sufficient resources (descriptor ring space, FIFO space) are
336 * available to transmit the given number of bytes.
338 static inline bool gve_can_tx(struct gve_tx_ring *tx, int bytes_required)
340 bool can_alloc = true;
342 if (!tx->raw_addressing)
343 can_alloc = gve_tx_fifo_can_alloc(&tx->tx_fifo, bytes_required);
345 return (gve_tx_avail(tx) >= MAX_TX_DESC_NEEDED && can_alloc);
348 /* Stops the queue if the skb cannot be transmitted. */
349 static int gve_maybe_stop_tx(struct gve_tx_ring *tx, struct sk_buff *skb)
351 int bytes_required = 0;
353 if (!tx->raw_addressing)
354 bytes_required = gve_skb_fifo_bytes_required(tx, skb);
356 if (likely(gve_can_tx(tx, bytes_required)))
359 /* No space, so stop the queue */
361 netif_tx_stop_queue(tx->netdev_txq);
362 smp_mb(); /* sync with restarting queue in gve_clean_tx_done() */
364 /* Now check for resources again, in case gve_clean_tx_done() freed
365 * resources after we checked and we stopped the queue after
366 * gve_clean_tx_done() checked.
368 * gve_maybe_stop_tx() gve_clean_tx_done()
369 * nsegs/can_alloc test failed
371 * if (tx queue stopped)
372 * netif_tx_queue_wake()
373 * netif_tx_stop_queue()
374 * Need to check again for space here!
376 if (likely(!gve_can_tx(tx, bytes_required)))
379 netif_tx_start_queue(tx->netdev_txq);
384 static void gve_tx_fill_pkt_desc(union gve_tx_desc *pkt_desc,
385 struct sk_buff *skb, bool is_gso,
386 int l4_hdr_offset, u32 desc_cnt,
389 /* l4_hdr_offset and csum_offset are in units of 16-bit words */
391 pkt_desc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM;
392 pkt_desc->pkt.l4_csum_offset = skb->csum_offset >> 1;
393 pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
394 } else if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
395 pkt_desc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM;
396 pkt_desc->pkt.l4_csum_offset = skb->csum_offset >> 1;
397 pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
399 pkt_desc->pkt.type_flags = GVE_TXD_STD;
400 pkt_desc->pkt.l4_csum_offset = 0;
401 pkt_desc->pkt.l4_hdr_offset = 0;
403 pkt_desc->pkt.desc_cnt = desc_cnt;
404 pkt_desc->pkt.len = cpu_to_be16(skb->len);
405 pkt_desc->pkt.seg_len = cpu_to_be16(hlen);
406 pkt_desc->pkt.seg_addr = cpu_to_be64(addr);
409 static void gve_tx_fill_seg_desc(union gve_tx_desc *seg_desc,
410 struct sk_buff *skb, bool is_gso,
413 seg_desc->seg.type_flags = GVE_TXD_SEG;
415 if (skb_is_gso_v6(skb))
416 seg_desc->seg.type_flags |= GVE_TXSF_IPV6;
417 seg_desc->seg.l3_offset = skb_network_offset(skb) >> 1;
418 seg_desc->seg.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
420 seg_desc->seg.seg_len = cpu_to_be16(len);
421 seg_desc->seg.seg_addr = cpu_to_be64(addr);
424 static void gve_dma_sync_for_device(struct device *dev, dma_addr_t *page_buses,
425 u64 iov_offset, u64 iov_len)
427 u64 last_page = (iov_offset + iov_len - 1) / PAGE_SIZE;
428 u64 first_page = iov_offset / PAGE_SIZE;
431 for (page = first_page; page <= last_page; page++)
432 dma_sync_single_for_device(dev, page_buses[page], PAGE_SIZE, DMA_TO_DEVICE);
435 static int gve_tx_add_skb_copy(struct gve_priv *priv, struct gve_tx_ring *tx, struct sk_buff *skb)
437 int pad_bytes, hlen, hdr_nfrags, payload_nfrags, l4_hdr_offset;
438 union gve_tx_desc *pkt_desc, *seg_desc;
439 struct gve_tx_buffer_state *info;
440 bool is_gso = skb_is_gso(skb);
441 u32 idx = tx->req & tx->mask;
447 info = &tx->info[idx];
448 pkt_desc = &tx->desc[idx];
450 l4_hdr_offset = skb_checksum_start_offset(skb);
451 /* If the skb is gso, then we want the tcp header in the first segment
452 * otherwise we want the linear portion of the skb (which will contain
453 * the checksum because skb->csum_start and skb->csum_offset are given
454 * relative to skb->head) in the first segment.
456 hlen = is_gso ? l4_hdr_offset + tcp_hdrlen(skb) :
460 /* We don't want to split the header, so if necessary, pad to the end
461 * of the fifo and then put the header at the beginning of the fifo.
463 pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, hlen);
464 hdr_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, hlen + pad_bytes,
466 WARN(!hdr_nfrags, "hdr_nfrags should never be 0!");
467 payload_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, skb->len - hlen,
468 &info->iov[payload_iov]);
470 gve_tx_fill_pkt_desc(pkt_desc, skb, is_gso, l4_hdr_offset,
471 1 + payload_nfrags, hlen,
472 info->iov[hdr_nfrags - 1].iov_offset);
474 skb_copy_bits(skb, 0,
475 tx->tx_fifo.base + info->iov[hdr_nfrags - 1].iov_offset,
477 gve_dma_sync_for_device(&priv->pdev->dev, tx->tx_fifo.qpl->page_buses,
478 info->iov[hdr_nfrags - 1].iov_offset,
479 info->iov[hdr_nfrags - 1].iov_len);
482 for (i = payload_iov; i < payload_nfrags + payload_iov; i++) {
483 next_idx = (tx->req + 1 + i - payload_iov) & tx->mask;
484 seg_desc = &tx->desc[next_idx];
486 gve_tx_fill_seg_desc(seg_desc, skb, is_gso,
487 info->iov[i].iov_len,
488 info->iov[i].iov_offset);
490 skb_copy_bits(skb, copy_offset,
491 tx->tx_fifo.base + info->iov[i].iov_offset,
492 info->iov[i].iov_len);
493 gve_dma_sync_for_device(&priv->pdev->dev, tx->tx_fifo.qpl->page_buses,
494 info->iov[i].iov_offset,
495 info->iov[i].iov_len);
496 copy_offset += info->iov[i].iov_len;
499 return 1 + payload_nfrags;
502 static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,
505 const struct skb_shared_info *shinfo = skb_shinfo(skb);
506 int hlen, payload_nfrags, l4_hdr_offset;
507 union gve_tx_desc *pkt_desc, *seg_desc;
508 struct gve_tx_buffer_state *info;
509 bool is_gso = skb_is_gso(skb);
510 u32 idx = tx->req & tx->mask;
511 struct gve_tx_dma_buf *buf;
516 info = &tx->info[idx];
517 pkt_desc = &tx->desc[idx];
519 l4_hdr_offset = skb_checksum_start_offset(skb);
520 /* If the skb is gso, then we want only up to the tcp header in the first segment
521 * to efficiently replicate on each segment otherwise we want the linear portion
522 * of the skb (which will contain the checksum because skb->csum_start and
523 * skb->csum_offset are given relative to skb->head) in the first segment.
525 hlen = is_gso ? l4_hdr_offset + tcp_hdrlen(skb) : skb_headlen(skb);
526 len = skb_headlen(skb);
530 addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE);
531 if (unlikely(dma_mapping_error(tx->dev, addr))) {
532 tx->dma_mapping_error++;
536 dma_unmap_len_set(buf, len, len);
537 dma_unmap_addr_set(buf, dma, addr);
539 payload_nfrags = shinfo->nr_frags;
541 /* For gso the rest of the linear portion of the skb needs to
542 * be in its own descriptor.
545 gve_tx_fill_pkt_desc(pkt_desc, skb, is_gso, l4_hdr_offset,
546 1 + payload_nfrags, hlen, addr);
550 idx = (tx->req + 1) & tx->mask;
551 seg_desc = &tx->desc[idx];
552 gve_tx_fill_seg_desc(seg_desc, skb, is_gso, len, addr);
554 gve_tx_fill_pkt_desc(pkt_desc, skb, is_gso, l4_hdr_offset,
555 1 + payload_nfrags, hlen, addr);
558 for (i = 0; i < shinfo->nr_frags; i++) {
559 const skb_frag_t *frag = &shinfo->frags[i];
561 idx = (idx + 1) & tx->mask;
562 seg_desc = &tx->desc[idx];
563 len = skb_frag_size(frag);
564 addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE);
565 if (unlikely(dma_mapping_error(tx->dev, addr))) {
566 tx->dma_mapping_error++;
569 buf = &tx->info[idx].buf;
570 tx->info[idx].skb = NULL;
571 dma_unmap_len_set(buf, len, len);
572 dma_unmap_addr_set(buf, dma, addr);
574 gve_tx_fill_seg_desc(seg_desc, skb, is_gso, len, addr);
577 return 1 + payload_nfrags;
580 i += (payload_nfrags == shinfo->nr_frags ? 1 : 2);
583 gve_tx_unmap_buf(tx->dev, &tx->info[idx & tx->mask]);
590 netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev)
592 struct gve_priv *priv = netdev_priv(dev);
593 struct gve_tx_ring *tx;
596 WARN(skb_get_queue_mapping(skb) >= priv->tx_cfg.num_queues,
597 "skb queue index out of range");
598 tx = &priv->tx[skb_get_queue_mapping(skb)];
599 if (unlikely(gve_maybe_stop_tx(tx, skb))) {
600 /* We need to ring the txq doorbell -- we have stopped the Tx
601 * queue for want of resources, but prior calls to gve_tx()
602 * may have added descriptors without ringing the doorbell.
605 gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
606 return NETDEV_TX_BUSY;
608 if (tx->raw_addressing)
609 nsegs = gve_tx_add_skb_no_copy(priv, tx, skb);
611 nsegs = gve_tx_add_skb_copy(priv, tx, skb);
613 /* If the packet is getting sent, we need to update the skb */
615 netdev_tx_sent_queue(tx->netdev_txq, skb->len);
616 skb_tx_timestamp(skb);
619 dev_kfree_skb_any(skb);
622 if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more())
625 /* Give packets to NIC. Even if this packet failed to send the doorbell
626 * might need to be rung because of xmit_more.
628 gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
632 #define GVE_TX_START_THRESH PAGE_SIZE
634 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
635 u32 to_do, bool try_to_wake)
637 struct gve_tx_buffer_state *info;
638 u64 pkts = 0, bytes = 0;
639 size_t space_freed = 0;
644 for (j = 0; j < to_do; j++) {
645 idx = tx->done & tx->mask;
646 netif_info(priv, tx_done, priv->dev,
647 "[%d] %s: idx=%d (req=%u done=%u)\n",
648 tx->q_num, __func__, idx, tx->req, tx->done);
649 info = &tx->info[idx];
652 /* Unmap the buffer */
653 if (tx->raw_addressing)
654 gve_tx_unmap_buf(tx->dev, info);
661 dev_consume_skb_any(skb);
662 if (tx->raw_addressing)
665 for (i = 0; i < ARRAY_SIZE(info->iov); i++) {
666 space_freed += info->iov[i].iov_len + info->iov[i].iov_padding;
667 info->iov[i].iov_len = 0;
668 info->iov[i].iov_padding = 0;
673 if (!tx->raw_addressing)
674 gve_tx_free_fifo(&tx->tx_fifo, space_freed);
675 u64_stats_update_begin(&tx->statss);
676 tx->bytes_done += bytes;
677 tx->pkt_done += pkts;
678 u64_stats_update_end(&tx->statss);
679 netdev_tx_completed_queue(tx->netdev_txq, pkts, bytes);
681 /* start the queue if we've stopped it */
683 /* Make sure that the doorbells are synced */
686 if (try_to_wake && netif_tx_queue_stopped(tx->netdev_txq) &&
687 likely(gve_can_tx(tx, GVE_TX_START_THRESH))) {
689 netif_tx_wake_queue(tx->netdev_txq);
695 __be32 gve_tx_load_event_counter(struct gve_priv *priv,
696 struct gve_tx_ring *tx)
698 u32 counter_index = be32_to_cpu((tx->q_resources->counter_index));
700 return READ_ONCE(priv->counter_array[counter_index]);
703 bool gve_tx_poll(struct gve_notify_block *block, int budget)
705 struct gve_priv *priv = block->priv;
706 struct gve_tx_ring *tx = block->tx;
711 /* If budget is 0, do all the work */
715 /* Find out how much work there is to be done */
716 tx->last_nic_done = gve_tx_load_event_counter(priv, tx);
717 nic_done = be32_to_cpu(tx->last_nic_done);
719 /* Do as much work as we have that the budget will
722 to_do = min_t(u32, (nic_done - tx->done), budget);
723 gve_clean_tx_done(priv, tx, to_do, true);
725 /* If we still have work we want to repoll */
726 repoll |= (nic_done != tx->done);