1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
3 *
4 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
5 * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35
36 /* This file implements Dynamic Interrupt Moderation, DIM */
37
38 #ifndef _LINUXKPI_LINUX_NET_DIM_H
39 #define _LINUXKPI_LINUX_NET_DIM_H
40
41 #include <asm/types.h>
42
43 #include <linux/workqueue.h>
44 #include <linux/ktime.h>
45
46 struct net_dim_cq_moder {
47 u16 usec;
48 u16 pkts;
49 u8 cq_period_mode;
50 };
51
52 struct net_dim_sample {
53 ktime_t time;
54 u32 pkt_ctr;
55 u32 byte_ctr;
56 u16 event_ctr;
57 };
58
59 struct net_dim_stats {
60 int ppms; /* packets per msec */
61 int bpms; /* bytes per msec */
62 int epms; /* events per msec */
63 };
64
65 struct net_dim { /* Adaptive Moderation */
66 u8 state;
67 struct net_dim_stats prev_stats;
68 struct net_dim_sample start_sample;
69 struct work_struct work;
70 u16 event_ctr;
71 u8 profile_ix;
72 u8 mode;
73 u8 tune_state;
74 u8 steps_right;
75 u8 steps_left;
76 u8 tired;
77 };
78
79 enum {
80 NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0,
81 NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1,
82 NET_DIM_CQ_PERIOD_NUM_MODES = 0x2,
83 NET_DIM_CQ_PERIOD_MODE_DISABLED = 0xFF,
84 };
85
86 /* Adaptive moderation logic */
87 enum {
88 NET_DIM_START_MEASURE,
89 NET_DIM_MEASURE_IN_PROGRESS,
90 NET_DIM_APPLY_NEW_PROFILE,
91 };
92
93 enum {
94 NET_DIM_PARKING_ON_TOP,
95 NET_DIM_PARKING_TIRED,
96 NET_DIM_GOING_RIGHT,
97 NET_DIM_GOING_LEFT,
98 };
99
100 enum {
101 NET_DIM_STATS_WORSE,
102 NET_DIM_STATS_SAME,
103 NET_DIM_STATS_BETTER,
104 };
105
106 enum {
107 NET_DIM_STEPPED,
108 NET_DIM_TOO_TIRED,
109 NET_DIM_ON_EDGE,
110 };
111
112 #define NET_DIM_PARAMS_NUM_PROFILES 5
113 /* Adaptive moderation profiles */
114 #define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
115 #define NET_DIM_DEF_PROFILE_CQE 1
116 #define NET_DIM_DEF_PROFILE_EQE 1
117
118 /* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */
119 #define NET_DIM_EQE_PROFILES { \
120 {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
121 {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
122 {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
123 {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
124 {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
125 }
126
127 #define NET_DIM_CQE_PROFILES { \
128 {2, 256}, \
129 {8, 128}, \
130 {16, 64}, \
131 {32, 64}, \
132 {64, 64} \
133 }
134
135 static const struct net_dim_cq_moder
136 net_dim_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
137 NET_DIM_EQE_PROFILES,
138 NET_DIM_CQE_PROFILES,
139 };
140
141 static inline struct net_dim_cq_moder
net_dim_get_profile(u8 cq_period_mode,int ix)142 net_dim_get_profile(u8 cq_period_mode,
143 int ix)
144 {
145 struct net_dim_cq_moder cq_moder;
146
147 cq_moder = net_dim_profile[cq_period_mode][ix];
148 cq_moder.cq_period_mode = cq_period_mode;
149 return cq_moder;
150 }
151
152 static inline struct net_dim_cq_moder
net_dim_get_def_profile(u8 rx_cq_period_mode)153 net_dim_get_def_profile(u8 rx_cq_period_mode)
154 {
155 int default_profile_ix;
156
157 if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE)
158 default_profile_ix = NET_DIM_DEF_PROFILE_CQE;
159 else /* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */
160 default_profile_ix = NET_DIM_DEF_PROFILE_EQE;
161
162 return net_dim_get_profile(rx_cq_period_mode, default_profile_ix);
163 }
164
165 static inline bool
net_dim_on_top(struct net_dim * dim)166 net_dim_on_top(struct net_dim *dim)
167 {
168 switch (dim->tune_state) {
169 case NET_DIM_PARKING_ON_TOP:
170 case NET_DIM_PARKING_TIRED:
171 return true;
172 case NET_DIM_GOING_RIGHT:
173 return (dim->steps_left > 1) && (dim->steps_right == 1);
174 default: /* NET_DIM_GOING_LEFT */
175 return (dim->steps_right > 1) && (dim->steps_left == 1);
176 }
177 }
178
179 static inline void
net_dim_turn(struct net_dim * dim)180 net_dim_turn(struct net_dim *dim)
181 {
182 switch (dim->tune_state) {
183 case NET_DIM_PARKING_ON_TOP:
184 case NET_DIM_PARKING_TIRED:
185 break;
186 case NET_DIM_GOING_RIGHT:
187 dim->tune_state = NET_DIM_GOING_LEFT;
188 dim->steps_left = 0;
189 break;
190 case NET_DIM_GOING_LEFT:
191 dim->tune_state = NET_DIM_GOING_RIGHT;
192 dim->steps_right = 0;
193 break;
194 }
195 }
196
197 static inline int
net_dim_step(struct net_dim * dim)198 net_dim_step(struct net_dim *dim)
199 {
200 if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
201 return NET_DIM_TOO_TIRED;
202
203 switch (dim->tune_state) {
204 case NET_DIM_PARKING_ON_TOP:
205 case NET_DIM_PARKING_TIRED:
206 break;
207 case NET_DIM_GOING_RIGHT:
208 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
209 return NET_DIM_ON_EDGE;
210 dim->profile_ix++;
211 dim->steps_right++;
212 break;
213 case NET_DIM_GOING_LEFT:
214 if (dim->profile_ix == 0)
215 return NET_DIM_ON_EDGE;
216 dim->profile_ix--;
217 dim->steps_left++;
218 break;
219 }
220
221 dim->tired++;
222 return NET_DIM_STEPPED;
223 }
224
225 static inline void
net_dim_park_on_top(struct net_dim * dim)226 net_dim_park_on_top(struct net_dim *dim)
227 {
228 dim->steps_right = 0;
229 dim->steps_left = 0;
230 dim->tired = 0;
231 dim->tune_state = NET_DIM_PARKING_ON_TOP;
232 }
233
234 static inline void
net_dim_park_tired(struct net_dim * dim)235 net_dim_park_tired(struct net_dim *dim)
236 {
237 dim->steps_right = 0;
238 dim->steps_left = 0;
239 dim->tune_state = NET_DIM_PARKING_TIRED;
240 }
241
242 static inline void
net_dim_exit_parking(struct net_dim * dim)243 net_dim_exit_parking(struct net_dim *dim)
244 {
245 dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT :
246 NET_DIM_GOING_RIGHT;
247 net_dim_step(dim);
248 }
249
250 #define IS_SIGNIFICANT_DIFF(val, ref) \
251 (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10%
252 * difference */
253
254 static inline int
net_dim_stats_compare(struct net_dim_stats * curr,struct net_dim_stats * prev)255 net_dim_stats_compare(struct net_dim_stats *curr,
256 struct net_dim_stats *prev)
257 {
258 if (!prev->bpms)
259 return curr->bpms ? NET_DIM_STATS_BETTER :
260 NET_DIM_STATS_SAME;
261
262 if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
263 return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER :
264 NET_DIM_STATS_WORSE;
265
266 if (!prev->ppms)
267 return curr->ppms ? NET_DIM_STATS_BETTER :
268 NET_DIM_STATS_SAME;
269
270 if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
271 return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER :
272 NET_DIM_STATS_WORSE;
273
274 if (!prev->epms)
275 return NET_DIM_STATS_SAME;
276
277 if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
278 return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER :
279 NET_DIM_STATS_WORSE;
280
281 return NET_DIM_STATS_SAME;
282 }
283
284 static inline bool
net_dim_decision(struct net_dim_stats * curr_stats,struct net_dim * dim)285 net_dim_decision(struct net_dim_stats *curr_stats,
286 struct net_dim *dim)
287 {
288 int prev_state = dim->tune_state;
289 int prev_ix = dim->profile_ix;
290 int stats_res;
291 int step_res;
292
293 switch (dim->tune_state) {
294 case NET_DIM_PARKING_ON_TOP:
295 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
296 if (stats_res != NET_DIM_STATS_SAME)
297 net_dim_exit_parking(dim);
298 break;
299
300 case NET_DIM_PARKING_TIRED:
301 dim->tired--;
302 if (!dim->tired)
303 net_dim_exit_parking(dim);
304 break;
305
306 case NET_DIM_GOING_RIGHT:
307 case NET_DIM_GOING_LEFT:
308 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
309 if (stats_res != NET_DIM_STATS_BETTER)
310 net_dim_turn(dim);
311
312 if (net_dim_on_top(dim)) {
313 net_dim_park_on_top(dim);
314 break;
315 }
316 step_res = net_dim_step(dim);
317 switch (step_res) {
318 case NET_DIM_ON_EDGE:
319 net_dim_park_on_top(dim);
320 break;
321 case NET_DIM_TOO_TIRED:
322 net_dim_park_tired(dim);
323 break;
324 }
325
326 break;
327 }
328
329 if ((prev_state != NET_DIM_PARKING_ON_TOP) ||
330 (dim->tune_state != NET_DIM_PARKING_ON_TOP))
331 dim->prev_stats = *curr_stats;
332
333 return dim->profile_ix != prev_ix;
334 }
335
336 static inline void
net_dim_sample(u16 event_ctr,u64 packets,u64 bytes,struct net_dim_sample * s)337 net_dim_sample(u16 event_ctr,
338 u64 packets,
339 u64 bytes,
340 struct net_dim_sample *s)
341 {
342 s->time = ktime_get();
343 s->pkt_ctr = packets;
344 s->byte_ctr = bytes;
345 s->event_ctr = event_ctr;
346 }
347
348 #define NET_DIM_NEVENTS 64
349 #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
350
351 static inline void
net_dim_calc_stats(struct net_dim_sample * start,struct net_dim_sample * end,struct net_dim_stats * curr_stats)352 net_dim_calc_stats(struct net_dim_sample *start,
353 struct net_dim_sample *end,
354 struct net_dim_stats *curr_stats)
355 {
356 /* u32 holds up to 71 minutes, should be enough */
357 u32 delta_us = ktime_us_delta(end->time, start->time);
358 u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr);
359 u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr,
360 start->byte_ctr);
361
362 if (!delta_us)
363 return;
364
365 curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us);
366 curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us);
367 curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC,
368 delta_us);
369 }
370
371 static inline void
net_dim(struct net_dim * dim,u64 packets,u64 bytes)372 net_dim(struct net_dim *dim,
373 u64 packets, u64 bytes)
374 {
375 struct net_dim_stats curr_stats;
376 struct net_dim_sample end_sample;
377 u16 nevents;
378
379 dim->event_ctr++;
380
381 switch (dim->state) {
382 case NET_DIM_MEASURE_IN_PROGRESS:
383 nevents = BIT_GAP(BITS_PER_TYPE(u16),
384 dim->event_ctr,
385 dim->start_sample.event_ctr);
386 if (nevents < NET_DIM_NEVENTS)
387 break;
388 net_dim_sample(dim->event_ctr, packets, bytes, &end_sample);
389 net_dim_calc_stats(&dim->start_sample, &end_sample,
390 &curr_stats);
391 if (net_dim_decision(&curr_stats, dim)) {
392 dim->state = NET_DIM_APPLY_NEW_PROFILE;
393 schedule_work(&dim->work);
394 break;
395 }
396 /* FALLTHROUGH */
397 case NET_DIM_START_MEASURE:
398 net_dim_sample(dim->event_ctr, packets, bytes, &dim->start_sample);
399 dim->state = NET_DIM_MEASURE_IN_PROGRESS;
400 break;
401 case NET_DIM_APPLY_NEW_PROFILE:
402 break;
403 default:
404 break;
405 }
406 }
407
408 #endif /* _LINUXKPI_LINUX_NET_DIM_H */
409