1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
4 * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
5 * Copyright (c) 2022 David Vernet <dvernet@meta.com>
6 */
7 #ifndef _GNU_SOURCE
8 #define _GNU_SOURCE
9 #endif
10 #include <stdio.h>
11 #include <stdlib.h>
12 #include <unistd.h>
13 #include <inttypes.h>
14 #include <signal.h>
15 #include <libgen.h>
16 #include <sys/mman.h>
17 #include <sys/stat.h>
18 #include <fcntl.h>
19 #include <limits.h>
20 #include <string.h>
21 #include <time.h>
22 #include <bpf/bpf.h>
23 #include <scx/common.h>
24 #include "scx_qmap.h"
25 #include "scx_qmap.bpf.skel.h"
26
27 const char help_fmt[] =
28 "A simple five-level FIFO queue sched_ext scheduler.\n"
29 "\n"
30 "It also demonstrates hierarchical sub-scheduling: a scheduler can hand some\n"
31 "of its cpus to a child cgroup that runs its own scheduler. Run one qmap as\n"
32 "the parent, then run another qmap on a child cgroup with -c to attach it\n"
33 "beneath the parent.\n"
34 "\n"
35 "The policy below is deliberately simplistic and the resulting behavior can\n"
36 "look odd. qmap is a demo: it exists to exercise every sub-scheduling\n"
37 "primitive the kernel offers with as little code as possible, not to schedule\n"
38 "well.\n"
39 "\n"
40 "A parent divides the full cpus it holds among itself and its children in\n"
41 "proportion to cpu.weight. The cpus left over by rounding are time-shared,\n"
42 "handed to each participant in turn every -R ms. A cpu a scheduler only\n"
43 "holds a time-share of is never handed further down, and a parent left with\n"
44 "no full cpu of its own shuts its children down.\n"
45 "\n"
46 "See the top-of-file comment in .bpf.c for the design.\n"
47 "\n"
48 "Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n"
49 " [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I]\n"
50 " [-F COUNT] [-i SEC] [-R MS] [-J MODE] [-v]\n"
51 "\n"
52 " -s SLICE_US Override slice duration\n"
53 " -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n"
54 " -t COUNT Stall every COUNT'th user thread\n"
55 " -T COUNT Stall every COUNT'th kernel thread\n"
56 " -N COUNT Size of the task_ctx arena slab (default 16384)\n"
57 " -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n"
58 " -b COUNT Dispatch upto COUNT tasks together\n"
59 " -P Print out DSQ content and event counters to trace_pipe every second\n"
60 " -M Print out debug messages to trace_pipe\n"
61 " -H Boost nice -20 tasks in SHARED_DSQ, use with -b\n"
62 " -c CG_PATH Cgroup path to attach as sub-scheduler, must run parent scheduler first\n"
63 " -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n"
64 " -D LEN Set scx_exit_info.dump buffer length\n"
65 " -S Suppress qmap-specific debug dump\n"
66 " -p Switch only tasks on SCHED_EXT policy instead of all\n"
67 " -I Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n"
68 " -F COUNT IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n"
69 " -C MODE cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n"
70 " -i SEC Stats interval, seconds (default 5)\n"
71 " -R MS Round-robin period for time-shared cpus, ms (default 200)\n"
72 " -J MODE Fault injection (wrong-cid: dispatch to a cid not held,\n"
73 " init-fail/cgrp-init-fail: fail init_task/cpuctl_init for\n"
74 " \"qmfail*\" comms/cgroups)\n"
75 " -B PPT Rescue bandwidth in parts per thousand, 0 disables (root only, default 20)\n"
76 " -q US Rescue batch quantum in microseconds (root only, default 5000)\n"
77 " -v Print libbpf debug messages\n"
78 " -h Display this help and exit\n";
79
80 static bool verbose;
81 static volatile int exit_req;
82
libbpf_print_fn(enum libbpf_print_level level,const char * format,va_list args)83 static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
84 {
85 if (level == LIBBPF_DEBUG && !verbose)
86 return 0;
87 return vfprintf(stderr, format, args);
88 }
89
sigint_handler(int dummy)90 static void sigint_handler(int dummy)
91 {
92 exit_req = 1;
93 }
94
invoke_flush_alloc(struct scx_qmap * skel)95 static void invoke_flush_alloc(struct scx_qmap *skel)
96 {
97 LIBBPF_OPTS(bpf_test_run_opts, opts);
98
99 bpf_prog_test_run_opts(bpf_program__fd(skel->progs.flush_alloc), &opts);
100 }
101
102 /* previous counter snapshots for the per-interval hier stats */
103 struct hier_prev {
104 u64 alloc_ns[MAX_SUB_SCHEDS];
105 u64 self_alloc_ns;
106 u64 alloc_window_ns;
107 u64 nr_dsps[MAX_SUB_SCHEDS];
108 u64 nr_reenq_cap;
109 u64 nr_reenq_immed;
110 u64 nr_inject_attempts;
111 u64 nr_rescue_dsp;
112 };
113
114 /* current wall-clock time as "HH:MM:SS" for the startup and interval headers */
tstamp(char * buf,size_t sz)115 static const char *tstamp(char *buf, size_t sz)
116 {
117 time_t now = time(NULL);
118
119 strftime(buf, sz, "%H:%M:%S", localtime(&now));
120 return buf;
121 }
122
123 /* format the cids whose cid_owner[] matches @owner as "0-3,8", "-" if none */
format_cid_ranges(struct qmap_arena * qa,s32 owner,char * buf,size_t sz)124 static void format_cid_ranges(struct qmap_arena *qa, s32 owner, char *buf, size_t sz)
125 {
126 u32 nr = qa->nr_cids, cid;
127 size_t off = 0;
128 s32 start = -1;
129
130 buf[0] = '\0';
131 for (cid = 0; cid <= nr; cid++) {
132 bool match = cid < nr && qa->part.cid_owner[cid] == owner;
133 int n;
134
135 if (match) {
136 if (start < 0)
137 start = cid;
138 continue;
139 }
140 if (start < 0)
141 continue;
142
143 if (start == (s32)cid - 1)
144 n = snprintf(buf + off, sz - off, "%s%d",
145 off ? "," : "", start);
146 else
147 n = snprintf(buf + off, sz - off, "%s%d-%d",
148 off ? "," : "", start, cid - 1);
149 if (n < 0 || (size_t)n >= sz - off) {
150 strcpy(&buf[sz - 4], "...");
151 return;
152 }
153 off += n;
154 start = -1;
155 }
156 if (!off)
157 strcpy(buf, "-");
158 }
159
160 /* partition summary + one row per sched: weight, cpus, dispatch rate, cids */
print_hier(struct qmap_arena * qa,struct hier_prev * prev,u64 own_cgid)161 static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cgid)
162 {
163 char ranges[128], who[16];
164 const char *rr = "-";
165 double secs;
166 u32 i;
167
168 /*
169 * account_alloc() bumps alloc_window_ns together with the per-owner
170 * counters, so dividing by the same window yields exact cid counts.
171 */
172 secs = (qa->alloc_window_ns - prev->alloc_window_ns) / 1e9;
173 prev->alloc_window_ns = qa->alloc_window_ns;
174
175 /* resolve the live shared-pool holder */
176 if (qa->part.nr_shared && qa->part.nr_rr) {
177 u64 cgid = qa->part.rr_slots[qa->part.rr_pos];
178
179 rr = "self";
180 if (cgid) {
181 rr = "?";
182 for (i = 0; i < MAX_SUB_SCHEDS; i++) {
183 if (qa->sub_sched_ctxs[i].cgroup_id == cgid) {
184 snprintf(who, sizeof(who), "sub%u", i);
185 rr = who;
186 break;
187 }
188 }
189 }
190 }
191
192 format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges));
193 printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu rescue=+%llu\n",
194 (unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr,
195 (unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap),
196 (unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed),
197 (unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts),
198 (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp));
199 prev->nr_reenq_cap = qa->nr_reenq_cap;
200 prev->nr_reenq_immed = qa->nr_reenq_immed;
201 prev->nr_inject_attempts = qa->nr_inject_attempts;
202 prev->nr_rescue_dsp = qa->nr_rescue_dsp;
203
204 printf("hier : %-4s %10s %4s %6s %8s %s\n",
205 "", "cgroup", "w", "alloc", "disp/s", "cids");
206
207 format_cid_ranges(qa, CID_SELF, ranges, sizeof(ranges));
208 printf("hier : %-4s %10llu %4u %6.2f %8s %s\n", "self",
209 (unsigned long long)own_cgid, 100,
210 secs > 0 ? (qa->self_alloc_ns - prev->self_alloc_ns) / (secs * 1e9) : 0.0,
211 "-", ranges);
212 prev->self_alloc_ns = qa->self_alloc_ns;
213
214 for (i = 0; i < MAX_SUB_SCHEDS; i++) {
215 struct sub_sched_ctx *sc = &qa->sub_sched_ctxs[i];
216
217 if (!sc->cgroup_id)
218 continue;
219
220 snprintf(who, sizeof(who), "sub%u", i);
221 format_cid_ranges(qa, i, ranges, sizeof(ranges));
222 printf("hier : %-4s %10llu %4u %6.2f %8.1f %s\n", who,
223 (unsigned long long)sc->cgroup_id, sc->weight,
224 secs > 0 ? (qa->alloc_ns[i] - prev->alloc_ns[i]) / (secs * 1e9) : 0.0,
225 secs > 0 ? (sc->nr_dsps - prev->nr_dsps[i]) / secs : 0.0,
226 ranges);
227 prev->alloc_ns[i] = qa->alloc_ns[i];
228 prev->nr_dsps[i] = sc->nr_dsps;
229 }
230 }
231
main(int argc,char ** argv)232 int main(int argc, char **argv)
233 {
234 struct scx_qmap *skel;
235 struct bpf_link *link;
236 struct qmap_arena *qa;
237 u32 test_error_cnt = 0;
238 u64 ecode;
239 int opt, stats_intv = 5, i, round_robin_ms = 200;
240 struct hier_prev hprev = {};
241 const char *sub_cg_path = NULL;
242 char tbuf[32];
243 u32 inject_mode = 0;
244 u64 own_cgid = 0;
245 s32 cid_override_shard_sz = 4;
246
247 libbpf_set_print(libbpf_print_fn);
248 signal(SIGINT, sigint_handler);
249 signal(SIGTERM, sigint_handler);
250
251 if (libbpf_num_possible_cpus() > SCX_QMAP_MAX_CPUS) {
252 fprintf(stderr,
253 "scx_qmap: %d possible CPUs exceeds compile-time cap %d; "
254 "rebuild with larger SCX_QMAP_MAX_CPUS\n",
255 libbpf_num_possible_cpus(), SCX_QMAP_MAX_CPUS);
256 return 1;
257 }
258 restart:
259 optind = 1;
260 skel = SCX_OPS_CID_OPEN(qmap_ops, scx_qmap);
261
262 skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
263 skel->rodata->max_tasks = 16384;
264
265 while ((opt = getopt(argc, argv,
266 "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:B:q:vh")) != -1) {
267 switch (opt) {
268 case 's':
269 skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
270 break;
271 case 'e':
272 test_error_cnt = strtoul(optarg, NULL, 0);
273 break;
274 case 't':
275 skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0);
276 break;
277 case 'T':
278 skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0);
279 break;
280 case 'l':
281 skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0);
282 break;
283 case 'b':
284 skel->rodata->dsp_batch = strtoul(optarg, NULL, 0);
285 break;
286 case 'N':
287 skel->rodata->max_tasks = strtoul(optarg, NULL, 0);
288 break;
289 case 'P':
290 skel->rodata->print_dsqs_and_events = true;
291 break;
292 case 'M':
293 skel->rodata->print_msgs = true;
294 break;
295 case 'H':
296 skel->rodata->highpri_boosting = true;
297 break;
298 case 'c': {
299 struct stat st;
300 if (stat(optarg, &st) < 0) {
301 perror("stat");
302 return 1;
303 }
304 skel->struct_ops.qmap_ops->sub_cgroup_id = st.st_ino;
305 skel->rodata->sub_cgroup_id = st.st_ino;
306 own_cgid = st.st_ino;
307 sub_cg_path = optarg;
308 break;
309 }
310 case 'd':
311 skel->rodata->disallow_tgid = strtol(optarg, NULL, 0);
312 if (skel->rodata->disallow_tgid < 0)
313 skel->rodata->disallow_tgid = getpid();
314 break;
315 case 'D':
316 skel->struct_ops.qmap_ops->exit_dump_len = strtoul(optarg, NULL, 0);
317 break;
318 case 'S':
319 skel->rodata->suppress_dump = true;
320 break;
321 case 'p':
322 skel->struct_ops.qmap_ops->flags |= SCX_OPS_SWITCH_PARTIAL;
323 break;
324 case 'I':
325 skel->struct_ops.qmap_ops->flags |= SCX_OPS_ALWAYS_ENQ_IMMED;
326 break;
327 case 'F':
328 skel->rodata->immed_stress_nth = strtoul(optarg, NULL, 0);
329 break;
330 case 'C': {
331 u32 nr_cpus = libbpf_num_possible_cpus();
332 u32 mode;
333
334 if (!strcmp(optarg, "shuffle"))
335 mode = QMAP_CID_OVR_SHUFFLE;
336 else if (!strcmp(optarg, "bad-dup"))
337 mode = QMAP_CID_OVR_BAD_DUP;
338 else if (!strcmp(optarg, "bad-range"))
339 mode = QMAP_CID_OVR_BAD_RANGE;
340 else if (!strcmp(optarg, "bad-mono"))
341 mode = QMAP_CID_OVR_BAD_MONO;
342 else {
343 fprintf(stderr, "unknown cid-override mode '%s'\n", optarg);
344 return 1;
345 }
346 skel->rodata->cid_override_mode = mode;
347 cid_override_shard_sz = 4;
348
349 /*
350 * bad-mono needs >= 3 shards to build a 0-based but
351 * non-monotonic shard_start. Shrink the shard size so
352 * the test runs on any machine with >= 3 cpus.
353 */
354 if (mode == QMAP_CID_OVR_BAD_MONO) {
355 if (nr_cpus < 3) {
356 fprintf(stderr, "bad-mono needs >= 3 cpus (have %u)\n",
357 nr_cpus);
358 return 1;
359 }
360 cid_override_shard_sz = nr_cpus / 3;
361 }
362
363 /* shards of shard_sz each */
364 skel->rodata->cid_override_nr_shards =
365 (nr_cpus + cid_override_shard_sz - 1) / cid_override_shard_sz;
366 break;
367 }
368 case 'i':
369 stats_intv = atoi(optarg);
370 if (stats_intv < 1)
371 stats_intv = 1;
372 break;
373 case 'R':
374 round_robin_ms = atoi(optarg);
375 if (round_robin_ms < 10)
376 round_robin_ms = 10;
377 break;
378 case 'J':
379 if (!strcmp(optarg, "wrong-cid"))
380 inject_mode = QMAP_INJ_WRONG_CID;
381 else if (!strcmp(optarg, "init-fail"))
382 inject_mode = QMAP_INJ_INIT_FAIL;
383 else if (!strcmp(optarg, "cgrp-init-fail"))
384 inject_mode = QMAP_INJ_CGRP_INIT_FAIL;
385 else
386 inject_mode = strtoul(optarg, NULL, 0);
387 break;
388 case 'B': {
389 u32 ppt = strtoul(optarg, NULL, 0);
390
391 if (!ppt)
392 ppt = __COMPAT_ENUM_OR_ZERO("scx_consts", "SCX_RESCUE_DISABLE");
393 skel->struct_ops.qmap_ops->rescue_bandwidth_ppt = ppt;
394 break;
395 }
396 case 'q':
397 skel->struct_ops.qmap_ops->rescue_quantum_us = strtoul(optarg, NULL, 0);
398 break;
399 case 'v':
400 verbose = true;
401 break;
402 default:
403 fprintf(stderr, help_fmt, basename(argv[0]));
404 return opt != 'h';
405 }
406 }
407
408 skel->rodata->round_robin_ns = (u64)round_robin_ms * 1000000;
409
410 SCX_OPS_LOAD(skel, qmap_ops, scx_qmap, uei);
411
412 qa = &skel->arena->qa;
413
414 /*
415 * The cid-override arrays live in the arena, which is mmapped at load.
416 * Populate them before qmap_init_cids() consumes them at attach.
417 */
418 if (skel->rodata->cid_override_mode) {
419 u32 mode = skel->rodata->cid_override_mode;
420 u32 nr_cpus = libbpf_num_possible_cpus();
421 u32 i;
422
423 /* shuffle: reversed cpu_to_cid; others: identity */
424 for (i = 0; i < nr_cpus; i++) {
425 if (mode == QMAP_CID_OVR_SHUFFLE)
426 qa->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i;
427 else
428 qa->cid_override_cpu_to_cid[i] = i;
429 }
430 if (mode == QMAP_CID_OVR_BAD_DUP && nr_cpus >= 2)
431 qa->cid_override_cpu_to_cid[1] = 0;
432 if (mode == QMAP_CID_OVR_BAD_RANGE)
433 qa->cid_override_cpu_to_cid[0] = (s32)nr_cpus;
434
435 for (i = 0; i < skel->rodata->cid_override_nr_shards; i++)
436 qa->cid_override_shard_start[i] = i * cid_override_shard_sz;
437
438 if (mode == QMAP_CID_OVR_BAD_MONO) {
439 /* swap [1] and [2] to break monotonicity */
440 s32 tmp = qa->cid_override_shard_start[1];
441 qa->cid_override_shard_start[1] = qa->cid_override_shard_start[2];
442 qa->cid_override_shard_start[2] = tmp;
443 }
444 }
445
446 link = SCX_OPS_ATTACH(skel, qmap_ops, scx_qmap);
447
448 qa->test_error_cnt = test_error_cnt;
449 qa->inject_mode = inject_mode;
450
451 if (sub_cg_path)
452 printf("%s scx_qmap started: sub-scheduler on %s, stats every %ds\n",
453 tstamp(tbuf, sizeof(tbuf)), sub_cg_path, stats_intv);
454 else
455 printf("%s scx_qmap started: root scheduler, stats every %ds\n",
456 tstamp(tbuf, sizeof(tbuf)), stats_intv);
457 fflush(stdout);
458
459 while (!exit_req && !UEI_EXITED(skel, uei)) {
460 long nr_enqueued = qa->nr_enqueued;
461 long nr_dispatched = qa->nr_dispatched;
462
463 printf("---- %s ----\n",
464 tstamp(tbuf, sizeof(tbuf)));
465 printf("stats : enq=%lu dsp=%lu delta=%ld reenq/cid0=%llu/%llu deq=%llu core=%llu enq_ddsp=%llu\n",
466 nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched,
467 (unsigned long long)qa->nr_reenqueued,
468 (unsigned long long)qa->nr_reenqueued_cid0,
469 (unsigned long long)qa->nr_dequeued,
470 (unsigned long long)qa->nr_core_sched_execed,
471 (unsigned long long)qa->nr_ddsp_from_enq);
472 printf(" exp_local=%llu exp_remote=%llu exp_timer=%llu exp_lost=%llu\n",
473 (unsigned long long)qa->nr_expedited_local,
474 (unsigned long long)qa->nr_expedited_remote,
475 (unsigned long long)qa->nr_expedited_from_timer,
476 (unsigned long long)qa->nr_expedited_lost);
477 if (__COMPAT_has_ksym("scx_bpf_cidperf_cur"))
478 printf("cpuperf: cur min/avg/max=%u/%u/%u target min/avg/max=%u/%u/%u\n",
479 qa->cpuperf_min,
480 qa->cpuperf_avg,
481 qa->cpuperf_max,
482 qa->cpuperf_target_min,
483 qa->cpuperf_target_avg,
484 qa->cpuperf_target_max);
485
486 invoke_flush_alloc(skel);
487 print_hier(qa, &hprev, own_cgid);
488 fflush(stdout);
489
490 for (i = 0; i < stats_intv && !exit_req && !UEI_EXITED(skel, uei); i++)
491 sleep(1);
492 }
493
494 bpf_link__destroy(link);
495 ecode = UEI_REPORT(skel, uei);
496 scx_qmap__destroy(skel);
497
498 if (!exit_req && UEI_ECODE_RESTART(ecode))
499 goto restart;
500 return 0;
501 }
502