1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Verify that ops.dequeue() is called for tasks leaving BPF custody through 4 * an SCX-internal cross-CPU migration (move_remote_task_to_local_dsq()). 5 * 6 * Copyright (c) 2026 Google LLC. 7 */ 8 #define _GNU_SOURCE 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <unistd.h> 13 #include <signal.h> 14 #include <time.h> 15 #include <sched.h> 16 #include <bpf/bpf.h> 17 #include <scx/common.h> 18 #include <sys/wait.h> 19 #include "scx_test.h" 20 #include "dequeue_remote.bpf.skel.h" 21 22 #define MAX_WORKERS 64 23 #define RUN_MS 2000 24 25 static int nr_cpus; 26 27 static long long now_ms(void) 28 { 29 struct timespec ts; 30 31 clock_gettime(CLOCK_MONOTONIC, &ts); 32 return ts.tv_sec * 1000LL + ts.tv_nsec / 1000000; 33 } 34 35 /* mix of short bursts and sleeps to generate lots of enqueues and wakeups */ 36 static void worker_fn(int id) 37 { 38 long long end = now_ms() + RUN_MS; 39 volatile unsigned long sum = 0; 40 41 while (now_ms() < end) { 42 unsigned long j; 43 44 for (j = 0; j < 20000 + id * 1000; j++) 45 sum += j; 46 if (id & 1) 47 usleep(100); 48 else 49 sched_yield(); 50 } 51 52 exit(0); 53 } 54 55 static enum scx_test_status run_scenario(struct dequeue_remote *skel, 56 bool use_move_to_local, 57 const char *name) 58 { 59 enum scx_test_status ret = SCX_TEST_PASS; 60 struct bpf_link *link; 61 pid_t pids[MAX_WORKERS]; 62 int nr_workers, nr_forked, i; 63 64 nr_workers = 2 * nr_cpus; 65 if (nr_workers < 4) 66 nr_workers = 4; 67 if (nr_workers > MAX_WORKERS) 68 nr_workers = MAX_WORKERS; 69 70 skel->bss->test_use_move_to_local = use_move_to_local; 71 skel->bss->enqueue_cnt = 0; 72 skel->bss->dequeue_cnt = 0; 73 skel->bss->dispatch_dequeue_cnt = 0; 74 skel->bss->change_dequeue_cnt = 0; 75 skel->bss->remote_dispatch_cnt = 0; 76 skel->bss->remote_running_cnt = 0; 77 skel->bss->missed_dequeue_cnt = 0; 78 skel->bss->core_sched_exec_dequeue_cnt = 0; 79 memset(&skel->data->uei, 0, sizeof(skel->data->uei)); 80 81 link = bpf_map__attach_struct_ops(skel->maps.dequeue_remote_ops); 82 SCX_FAIL_IF(!link, "Failed to attach struct_ops for %s", name); 83 84 fflush(stdout); 85 fflush(stderr); 86 87 for (nr_forked = 0; nr_forked < nr_workers; nr_forked++) { 88 pid_t pid = fork(); 89 90 if (pid < 0) { 91 SCX_ERR("Failed to fork worker %d", nr_forked); 92 ret = SCX_TEST_FAIL; 93 break; 94 } 95 if (pid == 0) 96 worker_fn(nr_forked); 97 pids[nr_forked] = pid; 98 } 99 100 /* on failure, kill the remaining workers but still reap them */ 101 for (i = 0; i < nr_forked; i++) { 102 int status; 103 104 if (ret != SCX_TEST_PASS) 105 kill(pids[i], SIGKILL); 106 107 if (waitpid(pids[i], &status, 0) != pids[i]) { 108 SCX_ERR("Failed to wait for worker %d", i); 109 ret = SCX_TEST_FAIL; 110 } else if (ret == SCX_TEST_PASS && status != 0) { 111 SCX_ERR("Worker %d exited with status %d", i, status); 112 ret = SCX_TEST_FAIL; 113 } 114 } 115 116 bpf_link__destroy(link); 117 118 if (ret != SCX_TEST_PASS) 119 return ret; 120 121 printf("%s:\n", name); 122 printf(" workers: %d\n", nr_workers); 123 printf(" enqueues: %lu\n", (unsigned long)skel->bss->enqueue_cnt); 124 printf(" dequeues: %lu (dispatch: %lu, property_change: %lu)\n", 125 (unsigned long)skel->bss->dequeue_cnt, 126 (unsigned long)skel->bss->dispatch_dequeue_cnt, 127 (unsigned long)skel->bss->change_dequeue_cnt); 128 if (!use_move_to_local) 129 printf(" remote SCX_DSQ_LOCAL_ON dispatches: %lu\n", 130 (unsigned long)skel->bss->remote_dispatch_cnt); 131 printf(" ran on a CPU other than the enqueue CPU: %lu\n", 132 (unsigned long)skel->bss->remote_running_cnt); 133 printf(" ran without ops.dequeue(): %lu\n", 134 (unsigned long)skel->bss->missed_dequeue_cnt); 135 printf(" late SCX_DEQ_CORE_SCHED_EXEC dequeues: %lu\n", 136 (unsigned long)skel->bss->core_sched_exec_dequeue_cnt); 137 138 if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG)) 139 SCX_ERR("Scheduler exited with kind=%lld: %s", 140 (long long)skel->data->uei.kind, skel->data->uei.msg); 141 SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG)); 142 143 /* the test is meaningless if no task was moved across CPUs */ 144 SCX_GT(skel->bss->remote_running_cnt, 0); 145 SCX_EQ(skel->bss->enqueue_cnt, skel->bss->dequeue_cnt); 146 147 return SCX_TEST_PASS; 148 } 149 150 static enum scx_test_status setup(void **ctx) 151 { 152 struct dequeue_remote *skel; 153 cpu_set_t cpus; 154 155 SCX_FAIL_IF(sched_getaffinity(0, sizeof(cpus), &cpus), 156 "Failed to get CPU affinity"); 157 nr_cpus = CPU_COUNT(&cpus); 158 if (nr_cpus < 2) { 159 fprintf(stderr, "Skipping: requires at least 2 usable CPUs\n"); 160 return SCX_TEST_SKIP; 161 } 162 163 skel = SCX_OPS_OPEN(dequeue_remote_ops, dequeue_remote); 164 SCX_OPS_LOAD(skel, dequeue_remote_ops, dequeue_remote, uei); 165 166 *ctx = skel; 167 168 return SCX_TEST_PASS; 169 } 170 171 static enum scx_test_status run(void *ctx) 172 { 173 struct dequeue_remote *skel = ctx; 174 enum scx_test_status status; 175 176 status = run_scenario(skel, false, 177 "BPF queue -> SCX_DSQ_LOCAL_ON | cpu"); 178 if (status != SCX_TEST_PASS) 179 return status; 180 181 status = run_scenario(skel, true, 182 "user DSQ -> scx_bpf_dsq_move_to_local()"); 183 if (status != SCX_TEST_PASS) 184 return status; 185 186 return SCX_TEST_PASS; 187 } 188 189 static void cleanup(void *ctx) 190 { 191 struct dequeue_remote *skel = ctx; 192 193 dequeue_remote__destroy(skel); 194 } 195 196 struct scx_test dequeue_remote_test = { 197 .name = "dequeue_remote", 198 .description = "Verify ops.dequeue() on SCX-internal cross-CPU migrations", 199 .setup = setup, 200 .run = run, 201 .cleanup = cleanup, 202 }; 203 204 REGISTER_SCX_TEST(&dequeue_remote_test) 205