net/mlx5e: Add RSS support for hairpin

[sfrench/cifs-2.6.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_main.c

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index d9d8227f195f0e151ba948e0622ea90a411817c4..8e380a89ec63abf28998e8d7868f85c8ed52e137 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -582,6 +582,10 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
                goto err_rq_wq_destroy;
        }
 
+       err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix);
+       if (err < 0)
+               goto err_rq_wq_destroy;
+
        rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
        rq->buff.headroom = params->rq_headroom;
 
@@ -674,8 +678,17 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
                wqe->data.lkey = rq->mkey_be;
        }
 
-       INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
-       rq->am.mode = params->rx_cq_moderation.cq_period_mode;
+       INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work);
+
+       switch (params->rx_cq_moderation.cq_period_mode) {
+       case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
+               rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
+               break;
+       case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
+       default:
+               rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+       }
+
        rq->page_cache.head = 0;
        rq->page_cache.tail = 0;
 
@@ -687,6 +700,7 @@ err_destroy_umr_mkey:
 err_rq_wq_destroy:
        if (rq->xdp_prog)
                bpf_prog_put(rq->xdp_prog);
+       xdp_rxq_info_unreg(&rq->xdp_rxq);
        mlx5_wq_destroy(&rq->wq_ctrl);
 
        return err;
@@ -699,6 +713,8 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
        if (rq->xdp_prog)
                bpf_prog_put(rq->xdp_prog);
 
+       xdp_rxq_info_unreg(&rq->xdp_rxq);
+
        switch (rq->wq_type) {
        case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
                mlx5e_rq_free_mpwqe_info(rq);
@@ -919,7 +935,7 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
        if (err)
                goto err_destroy_rq;
 
-       if (params->rx_am_enabled)
+       if (params->rx_dim_enabled)
                c->rq.state |= BIT(MLX5E_RQ_STATE_AM);
 
        return 0;
@@ -952,7 +968,7 @@ static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
 
 static void mlx5e_close_rq(struct mlx5e_rq *rq)
 {
-       cancel_work_sync(&rq->am.work);
+       cancel_work_sync(&rq->dim.work);
        mlx5e_destroy_rq(rq);
        mlx5e_free_rx_descs(rq);
        mlx5e_free_rq(rq);
@@ -1565,7 +1581,7 @@ static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
 }
 
 static int mlx5e_open_cq(struct mlx5e_channel *c,
-                        struct mlx5e_cq_moder moder,
+                        struct net_dim_cq_moder moder,
                         struct mlx5e_cq_param *param,
                         struct mlx5e_cq *cq)
 {
@@ -1747,7 +1763,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
                              struct mlx5e_channel_param *cparam,
                              struct mlx5e_channel **cp)
 {
-       struct mlx5e_cq_moder icocq_moder = {0, 0};
+       struct net_dim_cq_moder icocq_moder = {0, 0};
        struct net_device *netdev = priv->netdev;
        int cpu = mlx5e_get_cpu(priv, ix);
        struct mlx5e_channel *c;
@@ -1999,7 +2015,7 @@ static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
 
        mlx5e_build_common_cq_param(priv, param);
 
-       param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
+       param->cq_period_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 }
 
 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
@@ -2203,7 +2219,7 @@ static int mlx5e_rx_hash_fn(int hfunc)
               MLX5_RX_HASH_FN_INVERTED_XOR8;
 }
 
-static int mlx5e_bits_invert(unsigned long a, int size)
+int mlx5e_bits_invert(unsigned long a, int size)
 {
        int inv = 0;
        int i;
@@ -2669,7 +2685,7 @@ void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
                netif_carrier_on(netdev);
 }
 
-void mlx5e_timestamp_set(struct mlx5e_priv *priv)
+void mlx5e_timestamp_init(struct mlx5e_priv *priv)
 {
        priv->tstamp.tx_type   = HWTSTAMP_TX_OFF;
        priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE;
@@ -2690,7 +2706,6 @@ int mlx5e_open_locked(struct net_device *netdev)
        mlx5e_activate_priv_channels(priv);
        if (priv->profile->update_carrier)
                priv->profile->update_carrier(priv);
-       mlx5e_timestamp_set(priv);
 
        if (priv->profile->update_stats)
                queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
@@ -2766,6 +2781,9 @@ static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
        if (err)
                return err;
 
+       /* Mark as unused given "Drop-RQ" packets never reach XDP */
+       xdp_rxq_info_unused(&rq->xdp_rxq);
+
        rq->mdev = mdev;
 
        return 0;
@@ -3219,12 +3237,12 @@ static int mlx5e_set_mac(struct net_device *netdev, void *addr)
        return 0;
 }
 
-#define MLX5E_SET_FEATURE(netdev, feature, enable)     \
+#define MLX5E_SET_FEATURE(features, feature, enable)   \
        do {                                            \
                if (enable)                             \
-                       netdev->features |= feature;    \
+                       *features |= feature;           \
                else                                    \
-                       netdev->features &= ~feature;   \
+                       *features &= ~feature;          \
        } while (0)
 
 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
@@ -3347,6 +3365,7 @@ static int set_feature_arfs(struct net_device *netdev, bool enable)
 #endif
 
 static int mlx5e_handle_feature(struct net_device *netdev,
+                               netdev_features_t *features,
                                netdev_features_t wanted_features,
                                netdev_features_t feature,
                                mlx5e_feature_handler feature_handler)
@@ -3365,34 +3384,40 @@ static int mlx5e_handle_feature(struct net_device *netdev,
                return err;
        }
 
-       MLX5E_SET_FEATURE(netdev, feature, enable);
+       MLX5E_SET_FEATURE(features, feature, enable);
        return 0;
 }
 
 static int mlx5e_set_features(struct net_device *netdev,
                              netdev_features_t features)
 {
+       netdev_features_t oper_features = netdev->features;
        int err;
 
-       err  = mlx5e_handle_feature(netdev, features, NETIF_F_LRO,
-                                   set_feature_lro);
-       err |= mlx5e_handle_feature(netdev, features,
+       err  = mlx5e_handle_feature(netdev, &oper_features, features,
+                                   NETIF_F_LRO, set_feature_lro);
+       err |= mlx5e_handle_feature(netdev, &oper_features, features,
                                    NETIF_F_HW_VLAN_CTAG_FILTER,
                                    set_feature_cvlan_filter);
-       err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_TC,
-                                   set_feature_tc_num_filters);
-       err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXALL,
-                                   set_feature_rx_all);
-       err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXFCS,
-                                   set_feature_rx_fcs);
-       err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_VLAN_CTAG_RX,
-                                   set_feature_rx_vlan);
+       err |= mlx5e_handle_feature(netdev, &oper_features, features,
+                                   NETIF_F_HW_TC, set_feature_tc_num_filters);
+       err |= mlx5e_handle_feature(netdev, &oper_features, features,
+                                   NETIF_F_RXALL, set_feature_rx_all);
+       err |= mlx5e_handle_feature(netdev, &oper_features, features,
+                                   NETIF_F_RXFCS, set_feature_rx_fcs);
+       err |= mlx5e_handle_feature(netdev, &oper_features, features,
+                                   NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan);
 #ifdef CONFIG_RFS_ACCEL
-       err |= mlx5e_handle_feature(netdev, features, NETIF_F_NTUPLE,
-                                   set_feature_arfs);
+       err |= mlx5e_handle_feature(netdev, &oper_features, features,
+                                   NETIF_F_NTUPLE, set_feature_arfs);
 #endif
 
-       return err ? -EINVAL : 0;
+       if (err) {
+               netdev->features = oper_features;
+               return -EINVAL;
+       }
+
+       return 0;
 }
 
 static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
@@ -4038,9 +4063,18 @@ void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
                params->rx_cq_moderation.usec =
                        MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
 
-       if (params->rx_am_enabled)
-               params->rx_cq_moderation =
-                       mlx5e_am_get_def_profile(cq_period_mode);
+       if (params->rx_dim_enabled) {
+               switch (cq_period_mode) {
+               case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
+                       params->rx_cq_moderation =
+                               net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE);
+                       break;
+               case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
+               default:
+                       params->rx_cq_moderation =
+                               net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE);
+               }
+       }
 
        MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
                        params->rx_cq_moderation.cq_period_mode ==
@@ -4102,7 +4136,7 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
        cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
                        MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
                        MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
-       params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
+       params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
        mlx5e_set_rx_cq_mode_params(params, cq_period_mode);
        mlx5e_set_tx_cq_mode_params(params, cq_period_mode);
 
@@ -4139,6 +4173,8 @@ static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
        INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
        INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
        INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
+
+       mlx5e_timestamp_init(priv);
 }
 
 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
@@ -4307,9 +4343,6 @@ static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
 {
        mlx5e_ipsec_cleanup(priv);
        mlx5e_vxlan_cleanup(priv);
-
-       if (priv->channels.params.xdp_prog)
-               bpf_prog_put(priv->channels.params.xdp_prog);
 }
 
 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)