1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
4 * Copyright (C) 2015-2019 Google, Inc.
8 #include "gve_adminq.h"
9 #include <linux/etherdevice.h>
11 static void gve_rx_remove_from_block(struct gve_priv *priv, int queue_idx)
13 struct gve_notify_block *block =
14 &priv->ntfy_blocks[gve_rx_idx_to_ntfy(priv, queue_idx)];
19 static void gve_rx_free_ring(struct gve_priv *priv, int idx)
21 struct gve_rx_ring *rx = &priv->rx[idx];
22 struct device *dev = &priv->pdev->dev;
26 gve_rx_remove_from_block(priv, idx);
28 bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
29 dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
30 rx->desc.desc_ring = NULL;
32 dma_free_coherent(dev, sizeof(*rx->q_resources),
33 rx->q_resources, rx->q_resources_bus);
34 rx->q_resources = NULL;
36 gve_unassign_qpl(priv, rx->data.qpl->id);
38 kvfree(rx->data.page_info);
40 slots = rx->data.mask + 1;
41 bytes = sizeof(*rx->data.data_ring) * slots;
42 dma_free_coherent(dev, bytes, rx->data.data_ring,
44 rx->data.data_ring = NULL;
45 netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
48 static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
49 struct gve_rx_data_slot *slot,
50 dma_addr_t addr, struct page *page)
52 page_info->page = page;
53 page_info->page_offset = 0;
54 page_info->page_address = page_address(page);
55 slot->qpl_offset = cpu_to_be64(addr);
58 static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
60 struct gve_priv *priv = rx->gve;
64 /* Allocate one page per Rx queue slot. Each page is split into two
65 * packet buffers, when possible we "page flip" between the two.
67 slots = rx->data.mask + 1;
69 rx->data.page_info = kvzalloc(slots *
70 sizeof(*rx->data.page_info), GFP_KERNEL);
71 if (!rx->data.page_info)
74 rx->data.qpl = gve_assign_rx_qpl(priv);
76 for (i = 0; i < slots; i++) {
77 struct page *page = rx->data.qpl->pages[i];
78 dma_addr_t addr = i * PAGE_SIZE;
80 gve_setup_rx_buffer(&rx->data.page_info[i],
81 &rx->data.data_ring[i], addr, page);
87 static void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx)
89 u32 ntfy_idx = gve_rx_idx_to_ntfy(priv, queue_idx);
90 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
91 struct gve_rx_ring *rx = &priv->rx[queue_idx];
94 rx->ntfy_id = ntfy_idx;
97 static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
99 struct gve_rx_ring *rx = &priv->rx[idx];
100 struct device *hdev = &priv->pdev->dev;
106 netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
107 /* Make sure everything is zeroed to start with */
108 memset(rx, 0, sizeof(*rx));
113 slots = priv->rx_pages_per_qpl;
114 rx->data.mask = slots - 1;
116 /* alloc rx data ring */
117 bytes = sizeof(*rx->data.data_ring) * slots;
118 rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
121 if (!rx->data.data_ring)
123 filled_pages = gve_prefill_rx_pages(rx);
124 if (filled_pages < 0) {
126 goto abort_with_slots;
128 rx->desc.fill_cnt = filled_pages;
129 /* Ensure data ring slots (packet buffers) are visible. */
132 /* Alloc gve_queue_resources */
134 dma_alloc_coherent(hdev,
135 sizeof(*rx->q_resources),
136 &rx->q_resources_bus,
138 if (!rx->q_resources) {
142 netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
143 (unsigned long)rx->data.data_bus);
145 /* alloc rx desc ring */
146 bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
147 npages = bytes / PAGE_SIZE;
148 if (npages * PAGE_SIZE != bytes) {
150 goto abort_with_q_resources;
153 rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
155 if (!rx->desc.desc_ring) {
157 goto abort_with_q_resources;
159 rx->desc.mask = slots - 1;
162 gve_rx_add_to_block(priv, idx);
166 abort_with_q_resources:
167 dma_free_coherent(hdev, sizeof(*rx->q_resources),
168 rx->q_resources, rx->q_resources_bus);
169 rx->q_resources = NULL;
171 kvfree(rx->data.page_info);
173 bytes = sizeof(*rx->data.data_ring) * slots;
174 dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
175 rx->data.data_ring = NULL;
180 int gve_rx_alloc_rings(struct gve_priv *priv)
185 for (i = 0; i < priv->rx_cfg.num_queues; i++) {
186 err = gve_rx_alloc_ring(priv, i);
188 netif_err(priv, drv, priv->dev,
189 "Failed to alloc rx ring=%d: err=%d\n",
194 /* Unallocate if there was an error */
198 for (j = 0; j < i; j++)
199 gve_rx_free_ring(priv, j);
204 void gve_rx_free_rings(struct gve_priv *priv)
208 for (i = 0; i < priv->rx_cfg.num_queues; i++)
209 gve_rx_free_ring(priv, i);
212 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx)
214 u32 db_idx = be32_to_cpu(rx->q_resources->db_index);
216 iowrite32be(rx->desc.fill_cnt, &priv->db_bar2[db_idx]);
219 static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
221 if (likely(pkt_flags & (GVE_RXF_TCP | GVE_RXF_UDP)))
222 return PKT_HASH_TYPE_L4;
223 if (pkt_flags & (GVE_RXF_IPV4 | GVE_RXF_IPV6))
224 return PKT_HASH_TYPE_L3;
225 return PKT_HASH_TYPE_L2;
228 static struct sk_buff *gve_rx_copy(struct net_device *dev,
229 struct napi_struct *napi,
230 struct gve_rx_slot_page_info *page_info,
233 struct sk_buff *skb = napi_alloc_skb(napi, len);
234 void *va = page_info->page_address + GVE_RX_PAD +
235 page_info->page_offset;
242 skb_copy_to_linear_data(skb, va, len);
244 skb->protocol = eth_type_trans(skb, dev);
248 static struct sk_buff *gve_rx_add_frags(struct net_device *dev,
249 struct napi_struct *napi,
250 struct gve_rx_slot_page_info *page_info,
253 struct sk_buff *skb = napi_get_frags(napi);
258 skb_add_rx_frag(skb, 0, page_info->page,
259 page_info->page_offset +
260 GVE_RX_PAD, len, PAGE_SIZE / 2);
265 static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info,
266 struct gve_rx_data_slot *data_ring)
268 u64 addr = be64_to_cpu(data_ring->qpl_offset);
270 page_info->page_offset ^= PAGE_SIZE / 2;
271 addr ^= PAGE_SIZE / 2;
272 data_ring->qpl_offset = cpu_to_be64(addr);
275 static bool gve_rx(struct gve_rx_ring *rx, struct gve_rx_desc *rx_desc,
276 netdev_features_t feat)
278 struct gve_rx_slot_page_info *page_info;
279 struct gve_priv *priv = rx->gve;
280 struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
281 struct net_device *dev = priv->dev;
287 /* drop this packet */
288 if (unlikely(rx_desc->flags_seq & GVE_RXF_ERR))
291 len = be16_to_cpu(rx_desc->len) - GVE_RX_PAD;
292 idx = rx->data.cnt & rx->data.mask;
293 page_info = &rx->data.page_info[idx];
295 /* gvnic can only receive into registered segments. If the buffer
296 * can't be recycled, our only choice is to copy the data out of
297 * it so that we can return it to the device.
300 if (PAGE_SIZE == 4096) {
301 if (len <= priv->rx_copybreak) {
302 /* Just copy small packets */
303 skb = gve_rx_copy(dev, napi, page_info, len);
306 if (unlikely(!gve_can_recycle_pages(dev))) {
307 skb = gve_rx_copy(dev, napi, page_info, len);
310 pagecount = page_count(page_info->page);
311 if (pagecount == 1) {
312 /* No part of this page is used by any SKBs; we attach
313 * the page fragment to a new SKB and pass it up the
316 skb = gve_rx_add_frags(dev, napi, page_info, len);
319 /* Make sure the kernel stack can't release the page */
320 get_page(page_info->page);
321 /* "flip" to other packet buffer on this page */
322 gve_rx_flip_buff(page_info, &rx->data.data_ring[idx]);
323 } else if (pagecount >= 2) {
324 /* We have previously passed the other half of this
325 * page up the stack, but it has not yet been freed.
327 skb = gve_rx_copy(dev, napi, page_info, len);
329 WARN(pagecount < 1, "Pagecount should never be < 1");
333 skb = gve_rx_copy(dev, napi, page_info, len);
337 /* We didn't manage to allocate an skb but we haven't had any
338 * reset worthy failures.
345 if (likely(feat & NETIF_F_RXCSUM)) {
346 /* NIC passes up the partial sum */
348 skb->ip_summed = CHECKSUM_COMPLETE;
350 skb->ip_summed = CHECKSUM_NONE;
351 skb->csum = csum_unfold(rx_desc->csum);
354 /* parse flags & pass relevant info up */
355 if (likely(feat & NETIF_F_RXHASH) &&
356 gve_needs_rss(rx_desc->flags_seq))
357 skb_set_hash(skb, be32_to_cpu(rx_desc->rss_hash),
358 gve_rss_type(rx_desc->flags_seq));
360 if (skb_is_nonlinear(skb))
361 napi_gro_frags(napi);
363 napi_gro_receive(napi, skb);
367 static bool gve_rx_work_pending(struct gve_rx_ring *rx)
369 struct gve_rx_desc *desc;
373 next_idx = rx->desc.cnt & rx->desc.mask;
374 desc = rx->desc.desc_ring + next_idx;
376 flags_seq = desc->flags_seq;
377 /* Make sure we have synchronized the seq no with the device */
380 return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
383 bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
384 netdev_features_t feat)
386 struct gve_priv *priv = rx->gve;
387 struct gve_rx_desc *desc;
388 u32 cnt = rx->desc.cnt;
389 u32 idx = cnt & rx->desc.mask;
393 desc = rx->desc.desc_ring + idx;
394 while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
395 work_done < budget) {
396 netif_info(priv, rx_status, priv->dev,
397 "[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
398 rx->q_num, idx, desc, desc->flags_seq);
399 netif_info(priv, rx_status, priv->dev,
400 "[%d] seqno=%d rx->desc.seqno=%d\n",
401 rx->q_num, GVE_SEQNO(desc->flags_seq),
403 bytes += be16_to_cpu(desc->len) - GVE_RX_PAD;
404 if (!gve_rx(rx, desc, feat))
405 gve_schedule_reset(priv);
407 idx = cnt & rx->desc.mask;
408 desc = rx->desc.desc_ring + idx;
409 rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
416 u64_stats_update_begin(&rx->statss);
417 rx->rpackets += work_done;
419 u64_stats_update_end(&rx->statss);
421 rx->desc.fill_cnt += work_done;
423 /* restock desc ring slots */
424 dma_wmb(); /* Ensure descs are visible before ringing doorbell */
425 gve_rx_write_doorbell(priv, rx);
426 return gve_rx_work_pending(rx);
429 bool gve_rx_poll(struct gve_notify_block *block, int budget)
431 struct gve_rx_ring *rx = block->rx;
432 netdev_features_t feat;
435 feat = block->napi.dev->features;
437 /* If budget is 0, do all the work */
442 repoll |= gve_clean_rx_done(rx, budget, feat);
444 repoll |= gve_rx_work_pending(rx);