drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c

   1 /*
   2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include "en.h"
  34
  35 /* Adaptive moderation profiles */
  36 #define MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
  37 #define MLX5E_RX_AM_DEF_PROFILE_CQE 1
  38 #define MLX5E_RX_AM_DEF_PROFILE_EQE 1
  39 #define MLX5E_PARAMS_AM_NUM_PROFILES 5
  40
  41 /* All profiles sizes must be MLX5E_PARAMS_AM_NUM_PROFILES */
  42 #define MLX5_AM_EQE_PROFILES { \
  43         {1,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  44         {8,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  45         {64,  MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  46         {128, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  47         {256, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  48 }
  49
  50 #define MLX5_AM_CQE_PROFILES { \
  51         {2,  256},             \
  52         {8,  128},             \
  53         {16, 64},              \
  54         {32, 64},              \
  55         {64, 64}               \
  56 }
  57
  58 static const struct mlx5e_cq_moder
  59 profile[MLX5_CQ_PERIOD_NUM_MODES][MLX5E_PARAMS_AM_NUM_PROFILES] = {
  60         MLX5_AM_EQE_PROFILES,
  61         MLX5_AM_CQE_PROFILES,
  62 };
  63
  64 static inline struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix)
  65 {
  66         return profile[cq_period_mode][ix];
  67 }
  68
  69 struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode)
  70 {
  71         int default_profile_ix;
  72
  73         if (rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
  74                 default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_CQE;
  75         else /* MLX5_CQ_PERIOD_MODE_START_FROM_EQE */
  76                 default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_EQE;
  77
  78         return profile[rx_cq_period_mode][default_profile_ix];
  79 }
  80
  81 /* Adaptive moderation logic */
  82 enum {
  83         MLX5E_AM_START_MEASURE,
  84         MLX5E_AM_MEASURE_IN_PROGRESS,
  85         MLX5E_AM_APPLY_NEW_PROFILE,
  86 };
  87
  88 enum {
  89         MLX5E_AM_PARKING_ON_TOP,
  90         MLX5E_AM_PARKING_TIRED,
  91         MLX5E_AM_GOING_RIGHT,
  92         MLX5E_AM_GOING_LEFT,
  93 };
  94
  95 enum {
  96         MLX5E_AM_STATS_WORSE,
  97         MLX5E_AM_STATS_SAME,
  98         MLX5E_AM_STATS_BETTER,
  99 };
 100
 101 enum {
 102         MLX5E_AM_STEPPED,
 103         MLX5E_AM_TOO_TIRED,
 104         MLX5E_AM_ON_EDGE,
 105 };
 106
 107 static bool mlx5e_am_on_top(struct mlx5e_rx_am *am)
 108 {
 109         switch (am->tune_state) {
 110         case MLX5E_AM_PARKING_ON_TOP:
 111         case MLX5E_AM_PARKING_TIRED:
 112                 return true;
 113         case MLX5E_AM_GOING_RIGHT:
 114                 return (am->steps_left > 1) && (am->steps_right == 1);
 115         default: /* MLX5E_AM_GOING_LEFT */
 116                 return (am->steps_right > 1) && (am->steps_left == 1);
 117         }
 118 }
 119
 120 static void mlx5e_am_turn(struct mlx5e_rx_am *am)
 121 {
 122         switch (am->tune_state) {
 123         case MLX5E_AM_PARKING_ON_TOP:
 124         case MLX5E_AM_PARKING_TIRED:
 125                 break;
 126         case MLX5E_AM_GOING_RIGHT:
 127                 am->tune_state = MLX5E_AM_GOING_LEFT;
 128                 am->steps_left = 0;
 129                 break;
 130         case MLX5E_AM_GOING_LEFT:
 131                 am->tune_state = MLX5E_AM_GOING_RIGHT;
 132                 am->steps_right = 0;
 133                 break;
 134         }
 135 }
 136
 137 static int mlx5e_am_step(struct mlx5e_rx_am *am)
 138 {
 139         if (am->tired == (MLX5E_PARAMS_AM_NUM_PROFILES * 2))
 140                 return MLX5E_AM_TOO_TIRED;
 141
 142         switch (am->tune_state) {
 143         case MLX5E_AM_PARKING_ON_TOP:
 144         case MLX5E_AM_PARKING_TIRED:
 145                 break;
 146         case MLX5E_AM_GOING_RIGHT:
 147                 if (am->profile_ix == (MLX5E_PARAMS_AM_NUM_PROFILES - 1))
 148                         return MLX5E_AM_ON_EDGE;
 149                 am->profile_ix++;
 150                 am->steps_right++;
 151                 break;
 152         case MLX5E_AM_GOING_LEFT:
 153                 if (am->profile_ix == 0)
 154                         return MLX5E_AM_ON_EDGE;
 155                 am->profile_ix--;
 156                 am->steps_left++;
 157                 break;
 158         }
 159
 160         am->tired++;
 161         return MLX5E_AM_STEPPED;
 162 }
 163
 164 static void mlx5e_am_park_on_top(struct mlx5e_rx_am *am)
 165 {
 166         am->steps_right  = 0;
 167         am->steps_left   = 0;
 168         am->tired        = 0;
 169         am->tune_state   = MLX5E_AM_PARKING_ON_TOP;
 170 }
 171
 172 static void mlx5e_am_park_tired(struct mlx5e_rx_am *am)
 173 {
 174         am->steps_right  = 0;
 175         am->steps_left   = 0;
 176         am->tune_state   = MLX5E_AM_PARKING_TIRED;
 177 }
 178
 179 static void mlx5e_am_exit_parking(struct mlx5e_rx_am *am)
 180 {
 181         am->tune_state = am->profile_ix ? MLX5E_AM_GOING_LEFT :
 182                                           MLX5E_AM_GOING_RIGHT;
 183         mlx5e_am_step(am);
 184 }
 185
 186 #define IS_SIGNIFICANT_DIFF(val, ref) \
 187         (((100 * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */
 188
 189 static int mlx5e_am_stats_compare(struct mlx5e_rx_am_stats *curr,
 190                                   struct mlx5e_rx_am_stats *prev)
 191 {
 192         if (!prev->bpms)
 193                 return curr->bpms ? MLX5E_AM_STATS_BETTER :
 194                                     MLX5E_AM_STATS_SAME;
 195
 196         if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
 197                 return (curr->bpms > prev->bpms) ? MLX5E_AM_STATS_BETTER :
 198                                                    MLX5E_AM_STATS_WORSE;
 199
 200         if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
 201                 return (curr->ppms > prev->ppms) ? MLX5E_AM_STATS_BETTER :
 202                                                    MLX5E_AM_STATS_WORSE;
 203
 204         if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
 205                 return (curr->epms < prev->epms) ? MLX5E_AM_STATS_BETTER :
 206                                                    MLX5E_AM_STATS_WORSE;
 207
 208         return MLX5E_AM_STATS_SAME;
 209 }
 210
 211 static bool mlx5e_am_decision(struct mlx5e_rx_am_stats *curr_stats,
 212                               struct mlx5e_rx_am *am)
 213 {
 214         int prev_state = am->tune_state;
 215         int prev_ix = am->profile_ix;
 216         int stats_res;
 217         int step_res;
 218
 219         switch (am->tune_state) {
 220         case MLX5E_AM_PARKING_ON_TOP:
 221                 stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
 222                 if (stats_res != MLX5E_AM_STATS_SAME)
 223                         mlx5e_am_exit_parking(am);
 224                 break;
 225
 226         case MLX5E_AM_PARKING_TIRED:
 227                 am->tired--;
 228                 if (!am->tired)
 229                         mlx5e_am_exit_parking(am);
 230                 break;
 231
 232         case MLX5E_AM_GOING_RIGHT:
 233         case MLX5E_AM_GOING_LEFT:
 234                 stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
 235                 if (stats_res != MLX5E_AM_STATS_BETTER)
 236                         mlx5e_am_turn(am);
 237
 238                 if (mlx5e_am_on_top(am)) {
 239                         mlx5e_am_park_on_top(am);
 240                         break;
 241                 }
 242
 243                 step_res = mlx5e_am_step(am);
 244                 switch (step_res) {
 245                 case MLX5E_AM_ON_EDGE:
 246                         mlx5e_am_park_on_top(am);
 247                         break;
 248                 case MLX5E_AM_TOO_TIRED:
 249                         mlx5e_am_park_tired(am);
 250                         break;
 251                 }
 252
 253                 break;
 254         }
 255
 256         if ((prev_state     != MLX5E_AM_PARKING_ON_TOP) ||
 257             (am->tune_state != MLX5E_AM_PARKING_ON_TOP))
 258                 am->prev_stats = *curr_stats;
 259
 260         return am->profile_ix != prev_ix;
 261 }
 262
 263 static void mlx5e_am_sample(struct mlx5e_rq *rq,
 264                             struct mlx5e_rx_am_sample *s)
 265 {
 266         s->time      = ktime_get();
 267         s->pkt_ctr   = rq->stats.packets;
 268         s->byte_ctr  = rq->stats.bytes;
 269         s->event_ctr = rq->cq.event_ctr;
 270 }
 271
 272 #define MLX5E_AM_NEVENTS 64
 273 #define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE)
 274 #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
 275
 276 static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start,
 277                                 struct mlx5e_rx_am_sample *end,
 278                                 struct mlx5e_rx_am_stats *curr_stats)
 279 {
 280         /* u32 holds up to 71 minutes, should be enough */
 281         u32 delta_us = ktime_us_delta(end->time, start->time);
 282         u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr);
 283         u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr,
 284                              start->byte_ctr);
 285
 286         if (!delta_us)
 287                 return;
 288
 289         curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us);
 290         curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us);
 291         curr_stats->epms = DIV_ROUND_UP(MLX5E_AM_NEVENTS * USEC_PER_MSEC,
 292                                         delta_us);
 293 }
 294
 295 void mlx5e_rx_am_work(struct work_struct *work)
 296 {
 297         struct mlx5e_rx_am *am = container_of(work, struct mlx5e_rx_am,
 298                                               work);
 299         struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am);
 300         struct mlx5e_cq_moder cur_profile = profile[am->mode][am->profile_ix];
 301
 302         mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq,
 303                                        cur_profile.usec, cur_profile.pkts);
 304
 305         am->state = MLX5E_AM_START_MEASURE;
 306 }
 307
 308 void mlx5e_rx_am(struct mlx5e_rq *rq)
 309 {
 310         struct mlx5e_rx_am *am = &rq->am;
 311         struct mlx5e_rx_am_sample end_sample;
 312         struct mlx5e_rx_am_stats curr_stats;
 313         u16 nevents;
 314
 315         switch (am->state) {
 316         case MLX5E_AM_MEASURE_IN_PROGRESS:
 317                 nevents = BIT_GAP(BITS_PER_TYPE(u16), rq->cq.event_ctr,
 318                                   am->start_sample.event_ctr);
 319                 if (nevents < MLX5E_AM_NEVENTS)
 320                         break;
 321                 mlx5e_am_sample(rq, &end_sample);
 322                 mlx5e_am_calc_stats(&am->start_sample, &end_sample,
 323                                     &curr_stats);
 324                 if (mlx5e_am_decision(&curr_stats, am)) {
 325                         am->state = MLX5E_AM_APPLY_NEW_PROFILE;
 326                         schedule_work(&am->work);
 327                         break;
 328                 }
 329                 /* fall through */
 330         case MLX5E_AM_START_MEASURE:
 331                 mlx5e_am_sample(rq, &am->start_sample);
 332                 am->state = MLX5E_AM_MEASURE_IN_PROGRESS;
 333                 break;
 334         case MLX5E_AM_APPLY_NEW_PROFILE:
 335                 break;
 336         }
 337 }