1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0 3 * 4 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 5 * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 * 35 * $FreeBSD$ 36 */ 37 38 /* This file implements Dynamic Interrupt Moderation, DIM */ 39 40 #ifndef NET_DIM_H 41 #define NET_DIM_H 42 43 #include <asm/types.h> 44 45 #include <linux/workqueue.h> 46 #include <linux/ktime.h> 47 48 struct net_dim_cq_moder { 49 u16 usec; 50 u16 pkts; 51 u8 cq_period_mode; 52 }; 53 54 struct net_dim_sample { 55 ktime_t time; 56 u32 pkt_ctr; 57 u32 byte_ctr; 58 u16 event_ctr; 59 }; 60 61 struct net_dim_stats { 62 int ppms; /* packets per msec */ 63 int bpms; /* bytes per msec */ 64 int epms; /* events per msec */ 65 }; 66 67 struct net_dim { /* Adaptive Moderation */ 68 u8 state; 69 struct net_dim_stats prev_stats; 70 struct net_dim_sample start_sample; 71 struct work_struct work; 72 u16 event_ctr; 73 u8 profile_ix; 74 u8 mode; 75 u8 tune_state; 76 u8 steps_right; 77 u8 steps_left; 78 u8 tired; 79 }; 80 81 enum { 82 NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, 83 NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, 84 NET_DIM_CQ_PERIOD_NUM_MODES = 0x2, 85 NET_DIM_CQ_PERIOD_MODE_DISABLED = 0xFF, 86 }; 87 88 /* Adaptive moderation logic */ 89 enum { 90 NET_DIM_START_MEASURE, 91 NET_DIM_MEASURE_IN_PROGRESS, 92 NET_DIM_APPLY_NEW_PROFILE, 93 }; 94 95 enum { 96 NET_DIM_PARKING_ON_TOP, 97 NET_DIM_PARKING_TIRED, 98 NET_DIM_GOING_RIGHT, 99 NET_DIM_GOING_LEFT, 100 }; 101 102 enum { 103 NET_DIM_STATS_WORSE, 104 NET_DIM_STATS_SAME, 105 NET_DIM_STATS_BETTER, 106 }; 107 108 enum { 109 NET_DIM_STEPPED, 110 NET_DIM_TOO_TIRED, 111 NET_DIM_ON_EDGE, 112 }; 113 114 #define NET_DIM_PARAMS_NUM_PROFILES 5 115 /* Adaptive moderation profiles */ 116 #define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 117 #define NET_DIM_DEF_PROFILE_CQE 1 118 #define NET_DIM_DEF_PROFILE_EQE 1 119 120 /* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */ 121 #define NET_DIM_EQE_PROFILES { \ 122 {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 123 {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 124 {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 125 {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 126 {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 127 } 128 129 #define NET_DIM_CQE_PROFILES { \ 130 {2, 256}, \ 131 {8, 128}, \ 132 {16, 64}, \ 133 {32, 64}, \ 134 {64, 64} \ 135 } 136 137 static const struct net_dim_cq_moder 138 net_dim_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { 139 NET_DIM_EQE_PROFILES, 140 NET_DIM_CQE_PROFILES, 141 }; 142 143 static inline struct net_dim_cq_moder 144 net_dim_get_profile(u8 cq_period_mode, 145 int ix) 146 { 147 struct net_dim_cq_moder cq_moder; 148 149 cq_moder = net_dim_profile[cq_period_mode][ix]; 150 cq_moder.cq_period_mode = cq_period_mode; 151 return cq_moder; 152 } 153 154 static inline struct net_dim_cq_moder 155 net_dim_get_def_profile(u8 rx_cq_period_mode) 156 { 157 int default_profile_ix; 158 159 if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE) 160 default_profile_ix = NET_DIM_DEF_PROFILE_CQE; 161 else /* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */ 162 default_profile_ix = NET_DIM_DEF_PROFILE_EQE; 163 164 return net_dim_get_profile(rx_cq_period_mode, default_profile_ix); 165 } 166 167 static inline bool 168 net_dim_on_top(struct net_dim *dim) 169 { 170 switch (dim->tune_state) { 171 case NET_DIM_PARKING_ON_TOP: 172 case NET_DIM_PARKING_TIRED: 173 return true; 174 case NET_DIM_GOING_RIGHT: 175 return (dim->steps_left > 1) && (dim->steps_right == 1); 176 default: /* NET_DIM_GOING_LEFT */ 177 return (dim->steps_right > 1) && (dim->steps_left == 1); 178 } 179 } 180 181 static inline void 182 net_dim_turn(struct net_dim *dim) 183 { 184 switch (dim->tune_state) { 185 case NET_DIM_PARKING_ON_TOP: 186 case NET_DIM_PARKING_TIRED: 187 break; 188 case NET_DIM_GOING_RIGHT: 189 dim->tune_state = NET_DIM_GOING_LEFT; 190 dim->steps_left = 0; 191 break; 192 case NET_DIM_GOING_LEFT: 193 dim->tune_state = NET_DIM_GOING_RIGHT; 194 dim->steps_right = 0; 195 break; 196 } 197 } 198 199 static inline int 200 net_dim_step(struct net_dim *dim) 201 { 202 if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) 203 return NET_DIM_TOO_TIRED; 204 205 switch (dim->tune_state) { 206 case NET_DIM_PARKING_ON_TOP: 207 case NET_DIM_PARKING_TIRED: 208 break; 209 case NET_DIM_GOING_RIGHT: 210 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) 211 return NET_DIM_ON_EDGE; 212 dim->profile_ix++; 213 dim->steps_right++; 214 break; 215 case NET_DIM_GOING_LEFT: 216 if (dim->profile_ix == 0) 217 return NET_DIM_ON_EDGE; 218 dim->profile_ix--; 219 dim->steps_left++; 220 break; 221 } 222 223 dim->tired++; 224 return NET_DIM_STEPPED; 225 } 226 227 static inline void 228 net_dim_park_on_top(struct net_dim *dim) 229 { 230 dim->steps_right = 0; 231 dim->steps_left = 0; 232 dim->tired = 0; 233 dim->tune_state = NET_DIM_PARKING_ON_TOP; 234 } 235 236 static inline void 237 net_dim_park_tired(struct net_dim *dim) 238 { 239 dim->steps_right = 0; 240 dim->steps_left = 0; 241 dim->tune_state = NET_DIM_PARKING_TIRED; 242 } 243 244 static inline void 245 net_dim_exit_parking(struct net_dim *dim) 246 { 247 dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT : 248 NET_DIM_GOING_RIGHT; 249 net_dim_step(dim); 250 } 251 252 #define IS_SIGNIFICANT_DIFF(val, ref) \ 253 (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10% 254 * difference */ 255 256 static inline int 257 net_dim_stats_compare(struct net_dim_stats *curr, 258 struct net_dim_stats *prev) 259 { 260 if (!prev->bpms) 261 return curr->bpms ? NET_DIM_STATS_BETTER : 262 NET_DIM_STATS_SAME; 263 264 if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) 265 return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER : 266 NET_DIM_STATS_WORSE; 267 268 if (!prev->ppms) 269 return curr->ppms ? NET_DIM_STATS_BETTER : 270 NET_DIM_STATS_SAME; 271 272 if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) 273 return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER : 274 NET_DIM_STATS_WORSE; 275 276 if (!prev->epms) 277 return NET_DIM_STATS_SAME; 278 279 if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) 280 return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER : 281 NET_DIM_STATS_WORSE; 282 283 return NET_DIM_STATS_SAME; 284 } 285 286 static inline bool 287 net_dim_decision(struct net_dim_stats *curr_stats, 288 struct net_dim *dim) 289 { 290 int prev_state = dim->tune_state; 291 int prev_ix = dim->profile_ix; 292 int stats_res; 293 int step_res; 294 295 switch (dim->tune_state) { 296 case NET_DIM_PARKING_ON_TOP: 297 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); 298 if (stats_res != NET_DIM_STATS_SAME) 299 net_dim_exit_parking(dim); 300 break; 301 302 case NET_DIM_PARKING_TIRED: 303 dim->tired--; 304 if (!dim->tired) 305 net_dim_exit_parking(dim); 306 break; 307 308 case NET_DIM_GOING_RIGHT: 309 case NET_DIM_GOING_LEFT: 310 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); 311 if (stats_res != NET_DIM_STATS_BETTER) 312 net_dim_turn(dim); 313 314 if (net_dim_on_top(dim)) { 315 net_dim_park_on_top(dim); 316 break; 317 } 318 step_res = net_dim_step(dim); 319 switch (step_res) { 320 case NET_DIM_ON_EDGE: 321 net_dim_park_on_top(dim); 322 break; 323 case NET_DIM_TOO_TIRED: 324 net_dim_park_tired(dim); 325 break; 326 } 327 328 break; 329 } 330 331 if ((prev_state != NET_DIM_PARKING_ON_TOP) || 332 (dim->tune_state != NET_DIM_PARKING_ON_TOP)) 333 dim->prev_stats = *curr_stats; 334 335 return dim->profile_ix != prev_ix; 336 } 337 338 static inline void 339 net_dim_sample(u16 event_ctr, 340 u64 packets, 341 u64 bytes, 342 struct net_dim_sample *s) 343 { 344 s->time = ktime_get(); 345 s->pkt_ctr = packets; 346 s->byte_ctr = bytes; 347 s->event_ctr = event_ctr; 348 } 349 350 #define NET_DIM_NEVENTS 64 351 #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1)) 352 353 static inline void 354 net_dim_calc_stats(struct net_dim_sample *start, 355 struct net_dim_sample *end, 356 struct net_dim_stats *curr_stats) 357 { 358 /* u32 holds up to 71 minutes, should be enough */ 359 u32 delta_us = ktime_us_delta(end->time, start->time); 360 u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr); 361 u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr, 362 start->byte_ctr); 363 364 if (!delta_us) 365 return; 366 367 curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); 368 curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); 369 curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC, 370 delta_us); 371 } 372 373 static inline void 374 net_dim(struct net_dim *dim, 375 u64 packets, u64 bytes) 376 { 377 struct net_dim_stats curr_stats; 378 struct net_dim_sample end_sample; 379 u16 nevents; 380 381 dim->event_ctr++; 382 383 switch (dim->state) { 384 case NET_DIM_MEASURE_IN_PROGRESS: 385 nevents = BIT_GAP(BITS_PER_TYPE(u16), 386 dim->event_ctr, 387 dim->start_sample.event_ctr); 388 if (nevents < NET_DIM_NEVENTS) 389 break; 390 net_dim_sample(dim->event_ctr, packets, bytes, &end_sample); 391 net_dim_calc_stats(&dim->start_sample, &end_sample, 392 &curr_stats); 393 if (net_dim_decision(&curr_stats, dim)) { 394 dim->state = NET_DIM_APPLY_NEW_PROFILE; 395 schedule_work(&dim->work); 396 break; 397 } 398 /* FALLTHROUGH */ 399 case NET_DIM_START_MEASURE: 400 net_dim_sample(dim->event_ctr, packets, bytes, &dim->start_sample); 401 dim->state = NET_DIM_MEASURE_IN_PROGRESS; 402 break; 403 case NET_DIM_APPLY_NEW_PROFILE: 404 break; 405 default: 406 break; 407 } 408 } 409 410 #endif /* NET_DIM_H */ 411