1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0 3 * 4 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 5 * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 /* This file implements Dynamic Interrupt Moderation, DIM */ 37 38 #ifndef _LINUXKPI_LINUX_NET_DIM_H 39 #define _LINUXKPI_LINUX_NET_DIM_H 40 41 #include <asm/types.h> 42 43 #include <linux/workqueue.h> 44 #include <linux/ktime.h> 45 46 struct net_dim_cq_moder { 47 u16 usec; 48 u16 pkts; 49 u8 cq_period_mode; 50 }; 51 52 struct net_dim_sample { 53 ktime_t time; 54 u32 pkt_ctr; 55 u32 byte_ctr; 56 u16 event_ctr; 57 }; 58 59 struct net_dim_stats { 60 int ppms; /* packets per msec */ 61 int bpms; /* bytes per msec */ 62 int epms; /* events per msec */ 63 }; 64 65 struct net_dim { /* Adaptive Moderation */ 66 u8 state; 67 struct net_dim_stats prev_stats; 68 struct net_dim_sample start_sample; 69 struct work_struct work; 70 u16 event_ctr; 71 u8 profile_ix; 72 u8 mode; 73 u8 tune_state; 74 u8 steps_right; 75 u8 steps_left; 76 u8 tired; 77 }; 78 79 enum { 80 NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, 81 NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, 82 NET_DIM_CQ_PERIOD_NUM_MODES = 0x2, 83 NET_DIM_CQ_PERIOD_MODE_DISABLED = 0xFF, 84 }; 85 86 /* Adaptive moderation logic */ 87 enum { 88 NET_DIM_START_MEASURE, 89 NET_DIM_MEASURE_IN_PROGRESS, 90 NET_DIM_APPLY_NEW_PROFILE, 91 }; 92 93 enum { 94 NET_DIM_PARKING_ON_TOP, 95 NET_DIM_PARKING_TIRED, 96 NET_DIM_GOING_RIGHT, 97 NET_DIM_GOING_LEFT, 98 }; 99 100 enum { 101 NET_DIM_STATS_WORSE, 102 NET_DIM_STATS_SAME, 103 NET_DIM_STATS_BETTER, 104 }; 105 106 enum { 107 NET_DIM_STEPPED, 108 NET_DIM_TOO_TIRED, 109 NET_DIM_ON_EDGE, 110 }; 111 112 #define NET_DIM_PARAMS_NUM_PROFILES 5 113 /* Adaptive moderation profiles */ 114 #define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 115 #define NET_DIM_DEF_PROFILE_CQE 1 116 #define NET_DIM_DEF_PROFILE_EQE 1 117 118 /* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */ 119 #define NET_DIM_EQE_PROFILES { \ 120 {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 121 {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 122 {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 123 {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 124 {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 125 } 126 127 #define NET_DIM_CQE_PROFILES { \ 128 {2, 256}, \ 129 {8, 128}, \ 130 {16, 64}, \ 131 {32, 64}, \ 132 {64, 64} \ 133 } 134 135 static const struct net_dim_cq_moder 136 net_dim_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { 137 NET_DIM_EQE_PROFILES, 138 NET_DIM_CQE_PROFILES, 139 }; 140 141 static inline struct net_dim_cq_moder 142 net_dim_get_profile(u8 cq_period_mode, 143 int ix) 144 { 145 struct net_dim_cq_moder cq_moder; 146 147 cq_moder = net_dim_profile[cq_period_mode][ix]; 148 cq_moder.cq_period_mode = cq_period_mode; 149 return cq_moder; 150 } 151 152 static inline struct net_dim_cq_moder 153 net_dim_get_def_profile(u8 rx_cq_period_mode) 154 { 155 int default_profile_ix; 156 157 if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE) 158 default_profile_ix = NET_DIM_DEF_PROFILE_CQE; 159 else /* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */ 160 default_profile_ix = NET_DIM_DEF_PROFILE_EQE; 161 162 return net_dim_get_profile(rx_cq_period_mode, default_profile_ix); 163 } 164 165 static inline bool 166 net_dim_on_top(struct net_dim *dim) 167 { 168 switch (dim->tune_state) { 169 case NET_DIM_PARKING_ON_TOP: 170 case NET_DIM_PARKING_TIRED: 171 return true; 172 case NET_DIM_GOING_RIGHT: 173 return (dim->steps_left > 1) && (dim->steps_right == 1); 174 default: /* NET_DIM_GOING_LEFT */ 175 return (dim->steps_right > 1) && (dim->steps_left == 1); 176 } 177 } 178 179 static inline void 180 net_dim_turn(struct net_dim *dim) 181 { 182 switch (dim->tune_state) { 183 case NET_DIM_PARKING_ON_TOP: 184 case NET_DIM_PARKING_TIRED: 185 break; 186 case NET_DIM_GOING_RIGHT: 187 dim->tune_state = NET_DIM_GOING_LEFT; 188 dim->steps_left = 0; 189 break; 190 case NET_DIM_GOING_LEFT: 191 dim->tune_state = NET_DIM_GOING_RIGHT; 192 dim->steps_right = 0; 193 break; 194 } 195 } 196 197 static inline int 198 net_dim_step(struct net_dim *dim) 199 { 200 if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) 201 return NET_DIM_TOO_TIRED; 202 203 switch (dim->tune_state) { 204 case NET_DIM_PARKING_ON_TOP: 205 case NET_DIM_PARKING_TIRED: 206 break; 207 case NET_DIM_GOING_RIGHT: 208 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) 209 return NET_DIM_ON_EDGE; 210 dim->profile_ix++; 211 dim->steps_right++; 212 break; 213 case NET_DIM_GOING_LEFT: 214 if (dim->profile_ix == 0) 215 return NET_DIM_ON_EDGE; 216 dim->profile_ix--; 217 dim->steps_left++; 218 break; 219 } 220 221 dim->tired++; 222 return NET_DIM_STEPPED; 223 } 224 225 static inline void 226 net_dim_park_on_top(struct net_dim *dim) 227 { 228 dim->steps_right = 0; 229 dim->steps_left = 0; 230 dim->tired = 0; 231 dim->tune_state = NET_DIM_PARKING_ON_TOP; 232 } 233 234 static inline void 235 net_dim_park_tired(struct net_dim *dim) 236 { 237 dim->steps_right = 0; 238 dim->steps_left = 0; 239 dim->tune_state = NET_DIM_PARKING_TIRED; 240 } 241 242 static inline void 243 net_dim_exit_parking(struct net_dim *dim) 244 { 245 dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT : 246 NET_DIM_GOING_RIGHT; 247 net_dim_step(dim); 248 } 249 250 #define IS_SIGNIFICANT_DIFF(val, ref) \ 251 (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10% 252 * difference */ 253 254 static inline int 255 net_dim_stats_compare(struct net_dim_stats *curr, 256 struct net_dim_stats *prev) 257 { 258 if (!prev->bpms) 259 return curr->bpms ? NET_DIM_STATS_BETTER : 260 NET_DIM_STATS_SAME; 261 262 if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) 263 return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER : 264 NET_DIM_STATS_WORSE; 265 266 if (!prev->ppms) 267 return curr->ppms ? NET_DIM_STATS_BETTER : 268 NET_DIM_STATS_SAME; 269 270 if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) 271 return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER : 272 NET_DIM_STATS_WORSE; 273 274 if (!prev->epms) 275 return NET_DIM_STATS_SAME; 276 277 if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) 278 return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER : 279 NET_DIM_STATS_WORSE; 280 281 return NET_DIM_STATS_SAME; 282 } 283 284 static inline bool 285 net_dim_decision(struct net_dim_stats *curr_stats, 286 struct net_dim *dim) 287 { 288 int prev_state = dim->tune_state; 289 int prev_ix = dim->profile_ix; 290 int stats_res; 291 int step_res; 292 293 switch (dim->tune_state) { 294 case NET_DIM_PARKING_ON_TOP: 295 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); 296 if (stats_res != NET_DIM_STATS_SAME) 297 net_dim_exit_parking(dim); 298 break; 299 300 case NET_DIM_PARKING_TIRED: 301 dim->tired--; 302 if (!dim->tired) 303 net_dim_exit_parking(dim); 304 break; 305 306 case NET_DIM_GOING_RIGHT: 307 case NET_DIM_GOING_LEFT: 308 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); 309 if (stats_res != NET_DIM_STATS_BETTER) 310 net_dim_turn(dim); 311 312 if (net_dim_on_top(dim)) { 313 net_dim_park_on_top(dim); 314 break; 315 } 316 step_res = net_dim_step(dim); 317 switch (step_res) { 318 case NET_DIM_ON_EDGE: 319 net_dim_park_on_top(dim); 320 break; 321 case NET_DIM_TOO_TIRED: 322 net_dim_park_tired(dim); 323 break; 324 } 325 326 break; 327 } 328 329 if ((prev_state != NET_DIM_PARKING_ON_TOP) || 330 (dim->tune_state != NET_DIM_PARKING_ON_TOP)) 331 dim->prev_stats = *curr_stats; 332 333 return dim->profile_ix != prev_ix; 334 } 335 336 static inline void 337 net_dim_sample(u16 event_ctr, 338 u64 packets, 339 u64 bytes, 340 struct net_dim_sample *s) 341 { 342 s->time = ktime_get(); 343 s->pkt_ctr = packets; 344 s->byte_ctr = bytes; 345 s->event_ctr = event_ctr; 346 } 347 348 #define NET_DIM_NEVENTS 64 349 #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1)) 350 351 static inline void 352 net_dim_calc_stats(struct net_dim_sample *start, 353 struct net_dim_sample *end, 354 struct net_dim_stats *curr_stats) 355 { 356 /* u32 holds up to 71 minutes, should be enough */ 357 u32 delta_us = ktime_us_delta(end->time, start->time); 358 u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr); 359 u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr, 360 start->byte_ctr); 361 362 if (!delta_us) 363 return; 364 365 curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); 366 curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); 367 curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC, 368 delta_us); 369 } 370 371 static inline void 372 net_dim(struct net_dim *dim, 373 u64 packets, u64 bytes) 374 { 375 struct net_dim_stats curr_stats; 376 struct net_dim_sample end_sample; 377 u16 nevents; 378 379 dim->event_ctr++; 380 381 switch (dim->state) { 382 case NET_DIM_MEASURE_IN_PROGRESS: 383 nevents = BIT_GAP(BITS_PER_TYPE(u16), 384 dim->event_ctr, 385 dim->start_sample.event_ctr); 386 if (nevents < NET_DIM_NEVENTS) 387 break; 388 net_dim_sample(dim->event_ctr, packets, bytes, &end_sample); 389 net_dim_calc_stats(&dim->start_sample, &end_sample, 390 &curr_stats); 391 if (net_dim_decision(&curr_stats, dim)) { 392 dim->state = NET_DIM_APPLY_NEW_PROFILE; 393 schedule_work(&dim->work); 394 break; 395 } 396 /* FALLTHROUGH */ 397 case NET_DIM_START_MEASURE: 398 net_dim_sample(dim->event_ctr, packets, bytes, &dim->start_sample); 399 dim->state = NET_DIM_MEASURE_IN_PROGRESS; 400 break; 401 case NET_DIM_APPLY_NEW_PROFILE: 402 break; 403 default: 404 break; 405 } 406 } 407 408 #endif /* _LINUXKPI_LINUX_NET_DIM_H */ 409