net/mlx5e: Do not recycle RX pages in interface down flow

[sfrench/cifs-2.6.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_rx.c

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index d3a1dd20e41d4c8b3d68669d1fc40a9b4e1e63e4..97db5eeca9f3d6145e764a6c2e33371443322a16 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -44,6 +44,7 @@
 #include "en_rep.h"
 #include "ipoib/ipoib.h"
 #include "en_accel/ipsec_rxtx.h"
+#include "en_accel/tls_rxtx.h"
 #include "lib/clock.h"
 
 static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
@@ -276,10 +277,11 @@ static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
 }
 
 static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
-                                    struct mlx5e_wqe_frag_info *frag)
+                                    struct mlx5e_wqe_frag_info *frag,
+                                    bool recycle)
 {
        if (frag->last_in_page)
-               mlx5e_page_release(rq, frag->di, true);
+               mlx5e_page_release(rq, frag->di, recycle);
 }
 
 static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
@@ -307,25 +309,26 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
 
 free_frags:
        while (--i >= 0)
-               mlx5e_put_rx_frag(rq, --frag);
+               mlx5e_put_rx_frag(rq, --frag, true);
 
        return err;
 }
 
 static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
-                                    struct mlx5e_wqe_frag_info *wi)
+                                    struct mlx5e_wqe_frag_info *wi,
+                                    bool recycle)
 {
        int i;
 
        for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
-               mlx5e_put_rx_frag(rq, wi);
+               mlx5e_put_rx_frag(rq, wi, recycle);
 }
 
 void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
 {
        struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
 
-       mlx5e_free_rx_wqe(rq, wi);
+       mlx5e_free_rx_wqe(rq, wi, false);
 }
 
 static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
@@ -395,7 +398,8 @@ mlx5e_copy_skb_header_mpwqe(struct device *pdev,
        }
 }
 
-void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
+static void
+mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
 {
        const bool no_xdp_xmit =
                bitmap_empty(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
@@ -404,7 +408,7 @@ void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
 
        for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
                if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
-                       mlx5e_page_release(rq, &dma_info[i], true);
+                       mlx5e_page_release(rq, &dma_info[i], recycle);
 }
 
 static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
@@ -487,7 +491,7 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 
        sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
        sq->pc += MLX5E_UMR_WQEBBS;
-       mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &umr_wqe->ctrl);
+       mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &umr_wqe->ctrl);
 
        return 0;
 
@@ -504,8 +508,8 @@ err_unmap:
 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 {
        struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
-
-       mlx5e_free_rx_mpwqe(rq, wi);
+       /* Don't recycle, this function is called on rq/netdev close */
+       mlx5e_free_rx_mpwqe(rq, wi, false);
 }
 
 bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
@@ -601,6 +605,8 @@ bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
 
        if (!rq->mpwqe.umr_in_progress)
                mlx5e_alloc_rx_mpwqe(rq, wq->head);
+       else
+               rq->stats->congst_umr += mlx5_wq_ll_missing(wq) > 2;
 
        return false;
 }
@@ -795,6 +801,11 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
        struct net_device *netdev = rq->netdev;
 
        skb->mac_len = ETH_HLEN;
+
+#ifdef CONFIG_MLX5_EN_TLS
+       mlx5e_tls_handle_rx_skb(netdev, skb, &cqe_bcnt);
+#endif
+
        if (lro_num_seg > 1) {
                mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);
                skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg);
@@ -1105,7 +1116,7 @@ void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
        napi_gro_receive(rq->cq.napi, skb);
 
 free_wqe:
-       mlx5e_free_rx_wqe(rq, wi);
+       mlx5e_free_rx_wqe(rq, wi, true);
 wq_cyc_pop:
        mlx5_wq_cyc_pop(wq);
 }
@@ -1147,7 +1158,7 @@ void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
        napi_gro_receive(rq->cq.napi, skb);
 
 free_wqe:
-       mlx5e_free_rx_wqe(rq, wi);
+       mlx5e_free_rx_wqe(rq, wi, true);
 wq_cyc_pop:
        mlx5_wq_cyc_pop(wq);
 }
@@ -1261,7 +1272,10 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
        }
 
        if (unlikely(mpwrq_is_filler_cqe(cqe))) {
-               rq->stats->mpwqe_filler++;
+               struct mlx5e_rq_stats *stats = rq->stats;
+
+               stats->mpwqe_filler_cqes++;
+               stats->mpwqe_filler_strides += cstrides;
                goto mpwrq_cqe_out;
        }
 
@@ -1281,7 +1295,7 @@ mpwrq_cqe_out:
 
        wq  = &rq->mpwqe.wq;
        wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
-       mlx5e_free_rx_mpwqe(rq, wi);
+       mlx5e_free_rx_mpwqe(rq, wi, true);
        mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
 }
 
@@ -1383,6 +1397,8 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
                } while (!last_wqe);
        } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
 
+       rq->stats->xdp_tx_cqe += i;
+
        mlx5_cqwq_update_db_record(&cq->wq);
 
        /* ensure cq space is freed before enabling more cqes */
@@ -1508,7 +1524,7 @@ void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
        napi_gro_receive(rq->cq.napi, skb);
 
 wq_free_wqe:
-       mlx5e_free_rx_wqe(rq, wi);
+       mlx5e_free_rx_wqe(rq, wi, true);
        mlx5_wq_cyc_pop(wq);
 }
 
@@ -1531,19 +1547,19 @@ void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
        skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt);
        if (unlikely(!skb)) {
                /* a DROP, save the page-reuse checks */
-               mlx5e_free_rx_wqe(rq, wi);
+               mlx5e_free_rx_wqe(rq, wi, true);
                goto wq_cyc_pop;
        }
-       skb = mlx5e_ipsec_handle_rx_skb(rq->netdev, skb);
+       skb = mlx5e_ipsec_handle_rx_skb(rq->netdev, skb, &cqe_bcnt);
        if (unlikely(!skb)) {
-               mlx5e_free_rx_wqe(rq, wi);
+               mlx5e_free_rx_wqe(rq, wi, true);
                goto wq_cyc_pop;
        }
 
        mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
        napi_gro_receive(rq->cq.napi, skb);
 
-       mlx5e_free_rx_wqe(rq, wi);
+       mlx5e_free_rx_wqe(rq, wi, true);
 wq_cyc_pop:
        mlx5_wq_cyc_pop(wq);
 }