drivers/net/ethernet/amazon/ena/ena_xdp.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
   4  */
   5
   6 #include "ena_xdp.h"
   7
   8 static int validate_xdp_req_id(struct ena_ring *tx_ring, u16 req_id)
   9 {
  10         struct ena_tx_buffer *tx_info;
  11
  12         tx_info = &tx_ring->tx_buffer_info[req_id];
  13         if (likely(tx_info->xdpf))
  14                 return 0;
  15
  16         return handle_invalid_req_id(tx_ring, req_id, tx_info, true);
  17 }
  18
  19 static int ena_xdp_tx_map_frame(struct ena_ring *tx_ring,
  20                                 struct ena_tx_buffer *tx_info,
  21                                 struct xdp_frame *xdpf,
  22                                 struct ena_com_tx_ctx *ena_tx_ctx)
  23 {
  24         struct ena_adapter *adapter = tx_ring->adapter;
  25         struct ena_com_buf *ena_buf;
  26         int push_len = 0;
  27         dma_addr_t dma;
  28         void *data;
  29         u32 size;
  30
  31         tx_info->xdpf = xdpf;
  32         data = tx_info->xdpf->data;
  33         size = tx_info->xdpf->len;
  34
  35         if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
  36                 /* Designate part of the packet for LLQ */
  37                 push_len = min_t(u32, size, tx_ring->tx_max_header_size);
  38
  39                 ena_tx_ctx->push_header = data;
  40
  41                 size -= push_len;
  42                 data += push_len;
  43         }
  44
  45         ena_tx_ctx->header_len = push_len;
  46
  47         if (size > 0) {
  48                 dma = dma_map_single(tx_ring->dev,
  49                                      data,
  50                                      size,
  51                                      DMA_TO_DEVICE);
  52                 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
  53                         goto error_report_dma_error;
  54
  55                 tx_info->map_linear_data = 0;
  56
  57                 ena_buf = tx_info->bufs;
  58                 ena_buf->paddr = dma;
  59                 ena_buf->len = size;
  60
  61                 ena_tx_ctx->ena_bufs = ena_buf;
  62                 ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1;
  63         }
  64
  65         return 0;
  66
  67 error_report_dma_error:
  68         ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,
  69                           &tx_ring->syncp);
  70         netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n");
  71
  72         return -EINVAL;
  73 }
  74
  75 int ena_xdp_xmit_frame(struct ena_ring *tx_ring,
  76                        struct ena_adapter *adapter,
  77                        struct xdp_frame *xdpf,
  78                        int flags)
  79 {
  80         struct ena_com_tx_ctx ena_tx_ctx = {};
  81         struct ena_tx_buffer *tx_info;
  82         u16 next_to_use, req_id;
  83         int rc;
  84
  85         next_to_use = tx_ring->next_to_use;
  86         req_id = tx_ring->free_ids[next_to_use];
  87         tx_info = &tx_ring->tx_buffer_info[req_id];
  88         tx_info->num_of_bufs = 0;
  89
  90         rc = ena_xdp_tx_map_frame(tx_ring, tx_info, xdpf, &ena_tx_ctx);
  91         if (unlikely(rc))
  92                 goto err;
  93
  94         ena_tx_ctx.req_id = req_id;
  95
  96         rc = ena_xmit_common(adapter,
  97                              tx_ring,
  98                              tx_info,
  99                              &ena_tx_ctx,
 100                              next_to_use,
 101                              xdpf->len);
 102         if (rc)
 103                 goto error_unmap_dma;
 104
 105         /* trigger the dma engine. ena_ring_tx_doorbell()
 106          * calls a memory barrier inside it.
 107          */
 108         if (flags & XDP_XMIT_FLUSH)
 109                 ena_ring_tx_doorbell(tx_ring);
 110
 111         return rc;
 112
 113 error_unmap_dma:
 114         ena_unmap_tx_buff(tx_ring, tx_info);
 115 err:
 116         tx_info->xdpf = NULL;
 117
 118         return rc;
 119 }
 120
 121 int ena_xdp_xmit(struct net_device *dev, int n,
 122                  struct xdp_frame **frames, u32 flags)
 123 {
 124         struct ena_adapter *adapter = netdev_priv(dev);
 125         struct ena_ring *tx_ring;
 126         int qid, i, nxmit = 0;
 127
 128         if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
 129                 return -EINVAL;
 130
 131         if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
 132                 return -ENETDOWN;
 133
 134         /* We assume that all rings have the same XDP program */
 135         if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog))
 136                 return -ENXIO;
 137
 138         qid = smp_processor_id() % adapter->xdp_num_queues;
 139         qid += adapter->xdp_first_ring;
 140         tx_ring = &adapter->tx_ring[qid];
 141
 142         /* Other CPU ids might try to send thorugh this queue */
 143         spin_lock(&tx_ring->xdp_tx_lock);
 144
 145         for (i = 0; i < n; i++) {
 146                 if (ena_xdp_xmit_frame(tx_ring, adapter, frames[i], 0))
 147                         break;
 148                 nxmit++;
 149         }
 150
 151         /* Ring doorbell to make device aware of the packets */
 152         if (flags & XDP_XMIT_FLUSH)
 153                 ena_ring_tx_doorbell(tx_ring);
 154
 155         spin_unlock(&tx_ring->xdp_tx_lock);
 156
 157         /* Return number of packets sent */
 158         return nxmit;
 159 }
 160
 161 static void ena_init_all_xdp_queues(struct ena_adapter *adapter)
 162 {
 163         adapter->xdp_first_ring = adapter->num_io_queues;
 164         adapter->xdp_num_queues = adapter->num_io_queues;
 165
 166         ena_init_io_rings(adapter,
 167                           adapter->xdp_first_ring,
 168                           adapter->xdp_num_queues);
 169 }
 170
 171 int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter)
 172 {
 173         u32 xdp_first_ring = adapter->xdp_first_ring;
 174         u32 xdp_num_queues = adapter->xdp_num_queues;
 175         int rc = 0;
 176
 177         rc = ena_setup_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues);
 178         if (rc)
 179                 goto setup_err;
 180
 181         rc = ena_create_io_tx_queues_in_range(adapter, xdp_first_ring, xdp_num_queues);
 182         if (rc)
 183                 goto create_err;
 184
 185         return 0;
 186
 187 create_err:
 188         ena_free_all_io_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues);
 189 setup_err:
 190         return rc;
 191 }
 192
 193 /* Provides a way for both kernel and bpf-prog to know
 194  * more about the RX-queue a given XDP frame arrived on.
 195  */
 196 int ena_xdp_register_rxq_info(struct ena_ring *rx_ring)
 197 {
 198         int rc;
 199
 200         rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0);
 201
 202         netif_dbg(rx_ring->adapter, ifup, rx_ring->netdev, "Registering RX info for queue %d",
 203                   rx_ring->qid);
 204         if (rc) {
 205                 netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
 206                           "Failed to register xdp rx queue info. RX queue num %d rc: %d\n",
 207                           rx_ring->qid, rc);
 208                 goto err;
 209         }
 210
 211         rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL);
 212
 213         if (rc) {
 214                 netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
 215                           "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n",
 216                           rx_ring->qid, rc);
 217                 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 218         }
 219
 220 err:
 221         return rc;
 222 }
 223
 224 void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring)
 225 {
 226         netif_dbg(rx_ring->adapter, ifdown, rx_ring->netdev,
 227                   "Unregistering RX info for queue %d",
 228                   rx_ring->qid);
 229         xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq);
 230         xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 231 }
 232
 233 void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,
 234                                           struct bpf_prog *prog,
 235                                           int first, int count)
 236 {
 237         struct bpf_prog *old_bpf_prog;
 238         struct ena_ring *rx_ring;
 239         int i = 0;
 240
 241         for (i = first; i < count; i++) {
 242                 rx_ring = &adapter->rx_ring[i];
 243                 old_bpf_prog = xchg(&rx_ring->xdp_bpf_prog, prog);
 244
 245                 if (!old_bpf_prog && prog) {
 246                         rx_ring->rx_headroom = XDP_PACKET_HEADROOM;
 247                 } else if (old_bpf_prog && !prog) {
 248                         rx_ring->rx_headroom = NET_SKB_PAD;
 249                 }
 250         }
 251 }
 252
 253 static void ena_xdp_exchange_program(struct ena_adapter *adapter,
 254                                      struct bpf_prog *prog)
 255 {
 256         struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog);
 257
 258         ena_xdp_exchange_program_rx_in_range(adapter,
 259                                              prog,
 260                                              0,
 261                                              adapter->num_io_queues);
 262
 263         if (old_bpf_prog)
 264                 bpf_prog_put(old_bpf_prog);
 265 }
 266
 267 static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter)
 268 {
 269         bool was_up;
 270         int rc;
 271
 272         was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
 273
 274         if (was_up)
 275                 ena_down(adapter);
 276
 277         adapter->xdp_first_ring = 0;
 278         adapter->xdp_num_queues = 0;
 279         ena_xdp_exchange_program(adapter, NULL);
 280         if (was_up) {
 281                 rc = ena_up(adapter);
 282                 if (rc)
 283                         return rc;
 284         }
 285         return 0;
 286 }
 287
 288 static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf)
 289 {
 290         struct ena_adapter *adapter = netdev_priv(netdev);
 291         struct bpf_prog *prog = bpf->prog;
 292         struct bpf_prog *old_bpf_prog;
 293         int rc, prev_mtu;
 294         bool is_up;
 295
 296         is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
 297         rc = ena_xdp_allowed(adapter);
 298         if (rc == ENA_XDP_ALLOWED) {
 299                 old_bpf_prog = adapter->xdp_bpf_prog;
 300                 if (prog) {
 301                         if (!is_up) {
 302                                 ena_init_all_xdp_queues(adapter);
 303                         } else if (!old_bpf_prog) {
 304                                 ena_down(adapter);
 305                                 ena_init_all_xdp_queues(adapter);
 306                         }
 307                         ena_xdp_exchange_program(adapter, prog);
 308
 309                         netif_dbg(adapter, drv, adapter->netdev, "Set a new XDP program\n");
 310
 311                         if (is_up && !old_bpf_prog) {
 312                                 rc = ena_up(adapter);
 313                                 if (rc)
 314                                         return rc;
 315                         }
 316                         xdp_features_set_redirect_target(netdev, false);
 317                 } else if (old_bpf_prog) {
 318                         xdp_features_clear_redirect_target(netdev);
 319                         netif_dbg(adapter, drv, adapter->netdev, "Removing XDP program\n");
 320
 321                         rc = ena_destroy_and_free_all_xdp_queues(adapter);
 322                         if (rc)
 323                                 return rc;
 324                 }
 325
 326                 prev_mtu = netdev->max_mtu;
 327                 netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu;
 328
 329                 if (!old_bpf_prog)
 330                         netif_info(adapter, drv, adapter->netdev,
 331                                    "XDP program is set, changing the max_mtu from %d to %d",
 332                                    prev_mtu, netdev->max_mtu);
 333
 334         } else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) {
 335                 netif_err(adapter, drv, adapter->netdev,
 336                           "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on",
 337                           netdev->mtu, ENA_XDP_MAX_MTU);
 338                 NL_SET_ERR_MSG_MOD(bpf->extack,
 339                                    "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info");
 340                 return -EINVAL;
 341         } else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) {
 342                 netif_err(adapter, drv, adapter->netdev,
 343                           "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n",
 344                           adapter->num_io_queues, adapter->max_num_io_queues);
 345                 NL_SET_ERR_MSG_MOD(bpf->extack,
 346                                    "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info");
 347                 return -EINVAL;
 348         }
 349
 350         return 0;
 351 }
 352
 353 /* This is the main xdp callback, it's used by the kernel to set/unset the xdp
 354  * program as well as to query the current xdp program id.
 355  */
 356 int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
 357 {
 358         switch (bpf->command) {
 359         case XDP_SETUP_PROG:
 360                 return ena_xdp_set(netdev, bpf);
 361         default:
 362                 return -EINVAL;
 363         }
 364         return 0;
 365 }
 366
 367 static int ena_clean_xdp_irq(struct ena_ring *tx_ring, u32 budget)
 368 {
 369         u32 total_done = 0;
 370         u16 next_to_clean;
 371         int tx_pkts = 0;
 372         u16 req_id;
 373         int rc;
 374
 375         if (unlikely(!tx_ring))
 376                 return 0;
 377         next_to_clean = tx_ring->next_to_clean;
 378
 379         while (tx_pkts < budget) {
 380                 struct ena_tx_buffer *tx_info;
 381                 struct xdp_frame *xdpf;
 382
 383                 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
 384                                                 &req_id);
 385                 if (rc) {
 386                         if (unlikely(rc == -EINVAL))
 387                                 handle_invalid_req_id(tx_ring, req_id, NULL, true);
 388                         break;
 389                 }
 390
 391                 /* validate that the request id points to a valid xdp_frame */
 392                 rc = validate_xdp_req_id(tx_ring, req_id);
 393                 if (rc)
 394                         break;
 395
 396                 tx_info = &tx_ring->tx_buffer_info[req_id];
 397
 398                 tx_info->last_jiffies = 0;
 399
 400                 xdpf = tx_info->xdpf;
 401                 tx_info->xdpf = NULL;
 402                 ena_unmap_tx_buff(tx_ring, tx_info);
 403                 xdp_return_frame(xdpf);
 404
 405                 tx_pkts++;
 406                 total_done += tx_info->tx_descs;
 407                 tx_ring->free_ids[next_to_clean] = req_id;
 408                 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
 409                                                      tx_ring->ring_size);
 410
 411                 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
 412                           "tx_poll: q %d pkt #%d req_id %d\n", tx_ring->qid, tx_pkts, req_id);
 413         }
 414
 415         tx_ring->next_to_clean = next_to_clean;
 416         ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
 417
 418         netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
 419                   "tx_poll: q %d done. total pkts: %d\n",
 420                   tx_ring->qid, tx_pkts);
 421
 422         return tx_pkts;
 423 }
 424
 425 /* This is the XDP napi callback. XDP queues use a separate napi callback
 426  * than Rx/Tx queues.
 427  */
 428 int ena_xdp_io_poll(struct napi_struct *napi, int budget)
 429 {
 430         struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
 431         struct ena_ring *tx_ring;
 432         u32 work_done;
 433         int ret;
 434
 435         tx_ring = ena_napi->tx_ring;
 436
 437         if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
 438             test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
 439                 napi_complete_done(napi, 0);
 440                 return 0;
 441         }
 442
 443         work_done = ena_clean_xdp_irq(tx_ring, budget);
 444
 445         /* If the device is about to reset or down, avoid unmask
 446          * the interrupt and return 0 so NAPI won't reschedule
 447          */
 448         if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags))) {
 449                 napi_complete_done(napi, 0);
 450                 ret = 0;
 451         } else if (budget > work_done) {
 452                 ena_increase_stat(&tx_ring->tx_stats.napi_comp, 1,
 453                                   &tx_ring->syncp);
 454                 if (napi_complete_done(napi, work_done))
 455                         ena_unmask_interrupt(tx_ring, NULL);
 456
 457                 ena_update_ring_numa_node(tx_ring, NULL);
 458                 ret = work_done;
 459         } else {
 460                 ret = budget;
 461         }
 462
 463         u64_stats_update_begin(&tx_ring->syncp);
 464         tx_ring->tx_stats.tx_poll++;
 465         u64_stats_update_end(&tx_ring->syncp);
 466         tx_ring->tx_stats.last_napi_jiffies = jiffies;
 467
 468         return ret;
 469 }