net/mlx5e: XDP Tx, no inline copy on ConnectX-5

[sfrench/cifs-2.6.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_rx.c

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 20f116f8c4575fab8b6c25148e105dc74877a200..b039b87742a64422454ea105a8073b6451977253 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -33,6 +33,7 @@
 #include <linux/ip.h>
 #include <linux/ipv6.h>
 #include <linux/tcp.h>
+#include <linux/bpf_trace.h>
 #include <net/busy_poll.h>
 #include "en.h"
 #include "en_tc.h"
@@ -155,17 +156,15 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
        return mlx5e_decompress_cqes_cont(rq, cq, 1, budget_rem) - 1;
 }
 
-void mlx5e_modify_rx_cqe_compression(struct mlx5e_priv *priv, bool val)
+void mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool val)
 {
        bool was_opened;
 
        if (!MLX5_CAP_GEN(priv->mdev, cqe_compression))
                return;
 
-       mutex_lock(&priv->state_lock);
-
        if (MLX5E_GET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS) == val)
-               goto unlock;
+               return;
 
        was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
        if (was_opened)
@@ -176,8 +175,6 @@ void mlx5e_modify_rx_cqe_compression(struct mlx5e_priv *priv, bool val)
        if (was_opened)
                mlx5e_open_locked(priv->netdev);
 
-unlock:
-       mutex_unlock(&priv->state_lock);
 }
 
 #define RQ_PAGE_SIZE(rq) ((1 << rq->buff.page_order) << PAGE_SHIFT)
@@ -193,6 +190,9 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
                return false;
        }
 
+       if (unlikely(page_is_pfmemalloc(dma_info->page)))
+               return false;
+
        cache->page_cache[cache->tail] = *dma_info;
        cache->tail = tail_next;
        return true;
@@ -644,7 +644,7 @@ static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_sq *sq)
        mlx5e_tx_notify_hw(sq, &wqe->ctrl, 0);
 }
 
-static inline void mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
+static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
                                        struct mlx5e_dma_info *di,
                                        const struct xdp_buff *xdp)
 {
@@ -657,16 +657,17 @@ static inline void mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
        struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
        struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
        struct mlx5_wqe_data_seg *dseg;
+       u8 ds_cnt = MLX5E_XDP_TX_DS_COUNT;
 
        ptrdiff_t data_offset = xdp->data - xdp->data_hard_start;
-       dma_addr_t dma_addr  = di->addr + data_offset + MLX5E_XDP_MIN_INLINE;
+       dma_addr_t dma_addr  = di->addr + data_offset;
        unsigned int dma_len = xdp->data_end - xdp->data;
 
        if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE ||
                     MLX5E_SW2HW_MTU(rq->netdev->mtu) < dma_len)) {
                rq->stats.xdp_drop++;
                mlx5e_page_release(rq, di, true);
-               return;
+               return false;
        }
 
        if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_XDP_TX_WQEBBS))) {
@@ -677,20 +678,25 @@ static inline void mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
                }
                rq->stats.xdp_tx_full++;
                mlx5e_page_release(rq, di, true);
-               return;
+               return false;
        }
 
-       dma_len -= MLX5E_XDP_MIN_INLINE;
        dma_sync_single_for_device(sq->pdev, dma_addr, dma_len,
                                   PCI_DMA_TODEVICE);
 
        memset(wqe, 0, sizeof(*wqe));
 
-       /* copy the inline part */
-       memcpy(eseg->inline_hdr_start, xdp->data, MLX5E_XDP_MIN_INLINE);
-       eseg->inline_hdr_sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
+       dseg = (struct mlx5_wqe_data_seg *)eseg + 1;
+       /* copy the inline part if required */
+       if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
+               memcpy(eseg->inline_hdr.start, xdp->data, MLX5E_XDP_MIN_INLINE);
+               eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
+               dma_len  -= MLX5E_XDP_MIN_INLINE;
+               dma_addr += MLX5E_XDP_MIN_INLINE;
 
-       dseg = (struct mlx5_wqe_data_seg *)cseg + (MLX5E_XDP_TX_DS_COUNT - 1);
+               ds_cnt   += MLX5E_XDP_IHS_DS_COUNT;
+               dseg++;
+       }
 
        /* write the dma part */
        dseg->addr       = cpu_to_be64(dma_addr);
@@ -698,7 +704,7 @@ static inline void mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
        dseg->lkey       = sq->mkey_be;
 
        cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND);
-       cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | MLX5E_XDP_TX_DS_COUNT);
+       cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
 
        sq->db.xdp.di[pi] = *di;
        wi->opcode     = MLX5_OPCODE_SEND;
@@ -707,6 +713,7 @@ static inline void mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
 
        sq->db.xdp.doorbell = true;
        rq->stats.xdp_tx++;
+       return true;
 }
 
 /* returns true if packet was consumed by xdp */
@@ -732,11 +739,13 @@ static inline int mlx5e_xdp_handle(struct mlx5e_rq *rq,
                *len = xdp.data_end - xdp.data;
                return false;
        case XDP_TX:
-               mlx5e_xmit_xdp_frame(rq, di, &xdp);
+               if (unlikely(!mlx5e_xmit_xdp_frame(rq, di, &xdp)))
+                       trace_xdp_exception(rq->netdev, prog, act);
                return true;
        default:
                bpf_warn_invalid_xdp_action(act);
        case XDP_ABORTED:
+               trace_xdp_exception(rq->netdev, prog, act);
        case XDP_DROP:
                rq->stats.xdp_drop++;
                mlx5e_page_release(rq, di, true);