include/linux/net_dim.h

   1 /*
   2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
   3  * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #ifndef NET_DIM_H
  35 #define NET_DIM_H
  36
  37 #include <linux/module.h>
  38
  39 struct net_dim_cq_moder {
  40         u16 usec;
  41         u16 pkts;
  42         u8 cq_period_mode;
  43 };
  44
  45 struct net_dim_sample {
  46         ktime_t time;
  47         u32     pkt_ctr;
  48         u32     byte_ctr;
  49         u16     event_ctr;
  50 };
  51
  52 struct net_dim_stats {
  53         int ppms; /* packets per msec */
  54         int bpms; /* bytes per msec */
  55         int epms; /* events per msec */
  56 };
  57
  58 struct net_dim { /* Adaptive Moderation */
  59         u8                                      state;
  60         struct net_dim_stats                    prev_stats;
  61         struct net_dim_sample                   start_sample;
  62         struct work_struct                      work;
  63         u8                                      profile_ix;
  64         u8                                      mode;
  65         u8                                      tune_state;
  66         u8                                      steps_right;
  67         u8                                      steps_left;
  68         u8                                      tired;
  69 };
  70
  71 enum {
  72         NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0,
  73         NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1,
  74         NET_DIM_CQ_PERIOD_NUM_MODES
  75 };
  76
  77 /* Adaptive moderation logic */
  78 enum {
  79         NET_DIM_START_MEASURE,
  80         NET_DIM_MEASURE_IN_PROGRESS,
  81         NET_DIM_APPLY_NEW_PROFILE,
  82 };
  83
  84 enum {
  85         NET_DIM_PARKING_ON_TOP,
  86         NET_DIM_PARKING_TIRED,
  87         NET_DIM_GOING_RIGHT,
  88         NET_DIM_GOING_LEFT,
  89 };
  90
  91 enum {
  92         NET_DIM_STATS_WORSE,
  93         NET_DIM_STATS_SAME,
  94         NET_DIM_STATS_BETTER,
  95 };
  96
  97 enum {
  98         NET_DIM_STEPPED,
  99         NET_DIM_TOO_TIRED,
 100         NET_DIM_ON_EDGE,
 101 };
 102
 103 #define NET_DIM_PARAMS_NUM_PROFILES 5
 104 /* Adaptive moderation profiles */
 105 #define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
 106 #define NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE 128
 107 #define NET_DIM_DEF_PROFILE_CQE 1
 108 #define NET_DIM_DEF_PROFILE_EQE 1
 109
 110 /* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */
 111 #define NET_DIM_RX_EQE_PROFILES { \
 112         {1,   NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 113         {8,   NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 114         {64,  NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 115         {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 116         {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
 117 }
 118
 119 #define NET_DIM_RX_CQE_PROFILES { \
 120         {2,  256},             \
 121         {8,  128},             \
 122         {16, 64},              \
 123         {32, 64},              \
 124         {64, 64}               \
 125 }
 126
 127 #define NET_DIM_TX_EQE_PROFILES { \
 128         {1,   NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE},  \
 129         {8,   NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE},  \
 130         {32,  NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE},  \
 131         {64,  NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE},  \
 132         {128, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}   \
 133 }
 134
 135 #define NET_DIM_TX_CQE_PROFILES { \
 136         {5,  128},  \
 137         {8,  64},  \
 138         {16, 32},  \
 139         {32, 32},  \
 140         {64, 32}   \
 141 }
 142
 143 static const struct net_dim_cq_moder
 144 rx_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
 145         NET_DIM_RX_EQE_PROFILES,
 146         NET_DIM_RX_CQE_PROFILES,
 147 };
 148
 149 static const struct net_dim_cq_moder
 150 tx_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
 151         NET_DIM_TX_EQE_PROFILES,
 152         NET_DIM_TX_CQE_PROFILES,
 153 };
 154
 155 static inline struct net_dim_cq_moder
 156 net_dim_get_rx_moderation(u8 cq_period_mode, int ix)
 157 {
 158         struct net_dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix];
 159
 160         cq_moder.cq_period_mode = cq_period_mode;
 161         return cq_moder;
 162 }
 163
 164 static inline struct net_dim_cq_moder
 165 net_dim_get_def_rx_moderation(u8 cq_period_mode)
 166 {
 167         u8 profile_ix = cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
 168                         NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
 169
 170         return net_dim_get_rx_moderation(cq_period_mode, profile_ix);
 171 }
 172
 173 static inline struct net_dim_cq_moder
 174 net_dim_get_tx_moderation(u8 cq_period_mode, int ix)
 175 {
 176         struct net_dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix];
 177
 178         cq_moder.cq_period_mode = cq_period_mode;
 179         return cq_moder;
 180 }
 181
 182 static inline struct net_dim_cq_moder
 183 net_dim_get_def_tx_moderation(u8 cq_period_mode)
 184 {
 185         u8 profile_ix = cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
 186                         NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
 187
 188         return net_dim_get_tx_moderation(cq_period_mode, profile_ix);
 189 }
 190
 191 static inline bool net_dim_on_top(struct net_dim *dim)
 192 {
 193         switch (dim->tune_state) {
 194         case NET_DIM_PARKING_ON_TOP:
 195         case NET_DIM_PARKING_TIRED:
 196                 return true;
 197         case NET_DIM_GOING_RIGHT:
 198                 return (dim->steps_left > 1) && (dim->steps_right == 1);
 199         default: /* NET_DIM_GOING_LEFT */
 200                 return (dim->steps_right > 1) && (dim->steps_left == 1);
 201         }
 202 }
 203
 204 static inline void net_dim_turn(struct net_dim *dim)
 205 {
 206         switch (dim->tune_state) {
 207         case NET_DIM_PARKING_ON_TOP:
 208         case NET_DIM_PARKING_TIRED:
 209                 break;
 210         case NET_DIM_GOING_RIGHT:
 211                 dim->tune_state = NET_DIM_GOING_LEFT;
 212                 dim->steps_left = 0;
 213                 break;
 214         case NET_DIM_GOING_LEFT:
 215                 dim->tune_state = NET_DIM_GOING_RIGHT;
 216                 dim->steps_right = 0;
 217                 break;
 218         }
 219 }
 220
 221 static inline int net_dim_step(struct net_dim *dim)
 222 {
 223         if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
 224                 return NET_DIM_TOO_TIRED;
 225
 226         switch (dim->tune_state) {
 227         case NET_DIM_PARKING_ON_TOP:
 228         case NET_DIM_PARKING_TIRED:
 229                 break;
 230         case NET_DIM_GOING_RIGHT:
 231                 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
 232                         return NET_DIM_ON_EDGE;
 233                 dim->profile_ix++;
 234                 dim->steps_right++;
 235                 break;
 236         case NET_DIM_GOING_LEFT:
 237                 if (dim->profile_ix == 0)
 238                         return NET_DIM_ON_EDGE;
 239                 dim->profile_ix--;
 240                 dim->steps_left++;
 241                 break;
 242         }
 243
 244         dim->tired++;
 245         return NET_DIM_STEPPED;
 246 }
 247
 248 static inline void net_dim_park_on_top(struct net_dim *dim)
 249 {
 250         dim->steps_right  = 0;
 251         dim->steps_left   = 0;
 252         dim->tired        = 0;
 253         dim->tune_state   = NET_DIM_PARKING_ON_TOP;
 254 }
 255
 256 static inline void net_dim_park_tired(struct net_dim *dim)
 257 {
 258         dim->steps_right  = 0;
 259         dim->steps_left   = 0;
 260         dim->tune_state   = NET_DIM_PARKING_TIRED;
 261 }
 262
 263 static inline void net_dim_exit_parking(struct net_dim *dim)
 264 {
 265         dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT :
 266                                           NET_DIM_GOING_RIGHT;
 267         net_dim_step(dim);
 268 }
 269
 270 #define IS_SIGNIFICANT_DIFF(val, ref) \
 271         (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */
 272
 273 static inline int net_dim_stats_compare(struct net_dim_stats *curr,
 274                                         struct net_dim_stats *prev)
 275 {
 276         if (!prev->bpms)
 277                 return curr->bpms ? NET_DIM_STATS_BETTER :
 278                                     NET_DIM_STATS_SAME;
 279
 280         if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
 281                 return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER :
 282                                                    NET_DIM_STATS_WORSE;
 283
 284         if (!prev->ppms)
 285                 return curr->ppms ? NET_DIM_STATS_BETTER :
 286                                     NET_DIM_STATS_SAME;
 287
 288         if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
 289                 return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER :
 290                                                    NET_DIM_STATS_WORSE;
 291
 292         if (!prev->epms)
 293                 return NET_DIM_STATS_SAME;
 294
 295         if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
 296                 return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER :
 297                                                    NET_DIM_STATS_WORSE;
 298
 299         return NET_DIM_STATS_SAME;
 300 }
 301
 302 static inline bool net_dim_decision(struct net_dim_stats *curr_stats,
 303                                     struct net_dim *dim)
 304 {
 305         int prev_state = dim->tune_state;
 306         int prev_ix = dim->profile_ix;
 307         int stats_res;
 308         int step_res;
 309
 310         switch (dim->tune_state) {
 311         case NET_DIM_PARKING_ON_TOP:
 312                 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
 313                 if (stats_res != NET_DIM_STATS_SAME)
 314                         net_dim_exit_parking(dim);
 315                 break;
 316
 317         case NET_DIM_PARKING_TIRED:
 318                 dim->tired--;
 319                 if (!dim->tired)
 320                         net_dim_exit_parking(dim);
 321                 break;
 322
 323         case NET_DIM_GOING_RIGHT:
 324         case NET_DIM_GOING_LEFT:
 325                 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
 326                 if (stats_res != NET_DIM_STATS_BETTER)
 327                         net_dim_turn(dim);
 328
 329                 if (net_dim_on_top(dim)) {
 330                         net_dim_park_on_top(dim);
 331                         break;
 332                 }
 333
 334                 step_res = net_dim_step(dim);
 335                 switch (step_res) {
 336                 case NET_DIM_ON_EDGE:
 337                         net_dim_park_on_top(dim);
 338                         break;
 339                 case NET_DIM_TOO_TIRED:
 340                         net_dim_park_tired(dim);
 341                         break;
 342                 }
 343
 344                 break;
 345         }
 346
 347         if ((prev_state      != NET_DIM_PARKING_ON_TOP) ||
 348             (dim->tune_state != NET_DIM_PARKING_ON_TOP))
 349                 dim->prev_stats = *curr_stats;
 350
 351         return dim->profile_ix != prev_ix;
 352 }
 353
 354 static inline void net_dim_sample(u16 event_ctr,
 355                                   u64 packets,
 356                                   u64 bytes,
 357                                   struct net_dim_sample *s)
 358 {
 359         s->time      = ktime_get();
 360         s->pkt_ctr   = packets;
 361         s->byte_ctr  = bytes;
 362         s->event_ctr = event_ctr;
 363 }
 364
 365 #define NET_DIM_NEVENTS 64
 366 #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
 367
 368 static inline void net_dim_calc_stats(struct net_dim_sample *start,
 369                                       struct net_dim_sample *end,
 370                                       struct net_dim_stats *curr_stats)
 371 {
 372         /* u32 holds up to 71 minutes, should be enough */
 373         u32 delta_us = ktime_us_delta(end->time, start->time);
 374         u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr);
 375         u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr,
 376                              start->byte_ctr);
 377
 378         if (!delta_us)
 379                 return;
 380
 381         curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us);
 382         curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us);
 383         curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC,
 384                                         delta_us);
 385 }
 386
 387 static inline void net_dim(struct net_dim *dim,
 388                            struct net_dim_sample end_sample)
 389 {
 390         struct net_dim_stats curr_stats;
 391         u16 nevents;
 392
 393         switch (dim->state) {
 394         case NET_DIM_MEASURE_IN_PROGRESS:
 395                 nevents = BIT_GAP(BITS_PER_TYPE(u16),
 396                                   end_sample.event_ctr,
 397                                   dim->start_sample.event_ctr);
 398                 if (nevents < NET_DIM_NEVENTS)
 399                         break;
 400                 net_dim_calc_stats(&dim->start_sample, &end_sample,
 401                                    &curr_stats);
 402                 if (net_dim_decision(&curr_stats, dim)) {
 403                         dim->state = NET_DIM_APPLY_NEW_PROFILE;
 404                         schedule_work(&dim->work);
 405                         break;
 406                 }
 407                 /* fall through */
 408         case NET_DIM_START_MEASURE:
 409                 dim->state = NET_DIM_MEASURE_IN_PROGRESS;
 410                 break;
 411         case NET_DIM_APPLY_NEW_PROFILE:
 412                 break;
 413         }
 414 }
 415
 416 #endif /* NET_DIM_H */