1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2026 KylinSoft Corporation. */ 3 #include <vmlinux.h> 4 #include <stdbool.h> 5 #include <errno.h> 6 #include <bpf/bpf_helpers.h> 7 #include <bpf/bpf_tracing.h> 8 #include <bpf/bpf_core_read.h> 9 10 char _license[] SEC("license") = "GPL"; 11 12 extern bool CONFIG_PREEMPT_RT __kconfig __weak; 13 14 /* 15 * Bounds for the hardirq busy-wait that holds the scheduling handler 16 * inside the real window: task_work_add() has published the callback, 17 * but the handler's tail state transition has not run yet. 18 * irq_work_single() clears BUSY only after the handler returns, so 19 * while this wait runs, the context's scheduling irq_work is BUSY, 20 * even though the first callback may already have completed and 21 * returned the context to STANDBY. 22 * 23 * The wait is bounded both by a bpf_ktime_get_ns() check and by an 24 * iteration budget. The time bound is checked against a monotonic 25 * clock: host vCPU preemption can stretch the wall-clock duration of 26 * the window, which only makes the window longer, never shorter. 27 */ 28 #define BUSY_WAIT_NS 4000000ULL 29 #define BUSY_WAIT_ITERS 1000000ULL 30 31 int trigger_tid; 32 int target_tid; 33 34 /* 35 * High 32 bits of the result slots hold the sequence, low 32 bits the 36 * signed error. BUSY_WINDOW low bits: 1 open, 2 the in-window attempt 37 * was seen before the handler returned, 3 the wait bound was reached 38 * without it. Must match the test. 39 */ 40 enum task_work_busy_status { 41 BUSY_ARM_SEQ, 42 BUSY_WINDOW, 43 BUSY_ACK, 44 BUSY_GEN1_RESULT, 45 BUSY_GEN2_RESULT, 46 BUSY_CB1_SEQ, 47 BUSY_CB2_COUNT, 48 BUSY_CB2_SEQ, 49 BUSY_RECOVER_SEQ, 50 BUSY_LAST_WAIT_NS, 51 BUSY_STATUS_MAX, 52 }; 53 54 struct { 55 __uint(type, BPF_MAP_TYPE_ARRAY); 56 __uint(max_entries, BUSY_STATUS_MAX); 57 __type(key, int); 58 __type(value, __s64); 59 } status SEC(".maps"); 60 61 struct task_work_busy_value { 62 __u32 seq; 63 char data[60]; 64 struct bpf_task_work tw; 65 }; 66 67 /* 68 * max_entries must cover TASK_WORK_BUSY_MAX_ATTEMPTS in the test: each 69 * attempt schedules into its own fresh value. 70 */ 71 struct { 72 __uint(type, BPF_MAP_TYPE_HASH); 73 __uint(map_flags, BPF_F_NO_PREALLOC); 74 __uint(max_entries, 128); 75 __type(key, int); 76 __type(value, struct task_work_busy_value); 77 } hmap SEC(".maps"); 78 79 static __always_inline void set_status(int key, __s64 value) 80 { 81 __s64 *slot; 82 83 slot = bpf_map_lookup_elem(&status, &key); 84 if (slot) 85 *slot = value; 86 } 87 88 static __always_inline __s64 get_status(int key) 89 { 90 __s64 *slot = bpf_map_lookup_elem(&status, &key); 91 92 return slot ? *slot : 0; 93 } 94 95 struct busy_wait_ctx { 96 __u64 seq; 97 __u64 start; 98 bool acked; 99 }; 100 101 /* bpf_loop() callback: return 0 continues the loop, return 1 stops it. */ 102 static long busy_wait_cb(__u32 i, void *pctx) 103 { 104 struct busy_wait_ctx *w = pctx; 105 106 if (((__u64)get_status(BUSY_ACK) >> 32) == w->seq) { 107 w->acked = true; 108 return 1; 109 } 110 if (bpf_ktime_get_ns() - w->start > BUSY_WAIT_NS) 111 return 1; 112 return 0; 113 } 114 115 static int busy_cb_gen1(struct bpf_map *map, void *key, void *value) 116 { 117 struct task_work_busy_value *work = value; 118 __u64 we = (__u64)get_status(BUSY_WINDOW) >> 32; 119 120 /* 121 * Backstop for deliveries from abandoned attempts: accept only 122 * while no window for a different sequence is published. The 123 * test waits for the old workers to disappear from PID lookup, 124 * after their exit task_work, before resetting the shared state. 125 * This callback may run before fexit publishes the window state. 126 */ 127 if (we && we != (__u64)work->seq) 128 return 0; 129 set_status(BUSY_CB1_SEQ, work->seq); 130 return 0; 131 } 132 133 static int busy_cb_gen2(struct bpf_map *map, void *key, void *value) 134 { 135 struct task_work_busy_value *work = value; 136 __s64 *count; 137 int key2 = BUSY_CB2_COUNT; 138 139 /* 140 * Count only deliveries belonging to the sequence whose window is 141 * (or was) published: a late delivery from an abandoned attempt 142 * must not satisfy a later exactly-once check. 143 */ 144 if ((__u64)work->seq != ((__u64)get_status(BUSY_WINDOW) >> 32)) 145 return 0; 146 set_status(BUSY_CB2_SEQ, work->seq); 147 count = bpf_map_lookup_elem(&status, &key2); 148 if (count) 149 __sync_fetch_and_add(count, 1); 150 return 0; 151 } 152 153 SEC("fexit/task_work_add") 154 int BPF_PROG(busy_fexit_task_work_add, struct task_struct *task, 155 struct callback_head *work, int notify, int retval) 156 { 157 struct busy_wait_ctx w = {}; 158 __u64 seq, elapsed; 159 160 seq = (__u64)get_status(BUSY_ARM_SEQ); 161 if (!seq) 162 return 0; 163 164 /* 165 * One-shot arm. Only this test's own first-round add may consume 166 * it and hold the window: a successful TWA_SIGNAL add of the 167 * bpf_task_work callback, issued from the trigger thread's CPU by 168 * the self-IPI'd scheduling handler and targeting the target 169 * thread. Everything else leaves the arm untouched. 170 */ 171 if (retval != 0 || notify != TWA_SIGNAL || 172 BPF_CORE_READ(task, pid) != target_tid) 173 return 0; 174 if ((__u32)bpf_get_current_pid_tgid() != (__u32)trigger_tid) 175 return 0; 176 177 set_status(BUSY_ARM_SEQ, 0); 178 set_status(BUSY_WINDOW, (seq << 32) | 1); 179 180 w.seq = seq; 181 w.start = bpf_ktime_get_ns(); 182 bpf_loop(BUSY_WAIT_ITERS, busy_wait_cb, &w, 0); 183 elapsed = bpf_ktime_get_ns() - w.start; 184 185 /* 186 * Publish elapsed time for logging after worker cleanup. Window 187 * validity depends on seeing ACK here, not on this diagnostic. 188 */ 189 set_status(BUSY_LAST_WAIT_NS, (__s64)elapsed); 190 if (w.acked) 191 set_status(BUSY_WINDOW, (seq << 32) | 2); 192 else 193 set_status(BUSY_WINDOW, (seq << 32) | 3); 194 return 0; 195 } 196 197 /* 198 * Attempt the second scheduling round on the same context, targeting the 199 * trigger thread. The attempt uses the value keyed by its own sequence, 200 * so it can never touch another attempt's context. Returns true if a 201 * terminal result was published into res_key; a transient -EBUSY is left 202 * unpublished for the caller's retry policy, mirroring the race test's 203 * convention. 204 */ 205 static __always_inline bool busy_try_gen2(__u64 seq, int res_key) 206 { 207 struct task_work_busy_value *work; 208 struct task_struct *task; 209 int key = (int)seq, err; 210 211 task = bpf_task_from_pid(trigger_tid); 212 if (!task) { 213 set_status(res_key, ((__s64)seq << 32) | (__u32)-ESRCH); 214 return true; 215 } 216 work = bpf_map_lookup_elem(&hmap, &key); 217 if (!work) { 218 bpf_task_release(task); 219 set_status(res_key, ((__s64)seq << 32) | (__u32)-ENOENT); 220 return true; 221 } 222 work->seq = seq; 223 err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, 224 busy_cb_gen2); 225 bpf_task_release(task); 226 if (err == -EBUSY) 227 return false; 228 set_status(res_key, ((__s64)seq << 32) | (__u32)err); 229 return true; 230 } 231 232 SEC("tracepoint/syscalls/sys_enter_getppid") 233 int busy_sched_work(void *ctx) 234 { 235 __u32 tid = (__u32)bpf_get_current_pid_tgid(); 236 struct task_work_busy_value *work; 237 struct task_struct *task; 238 __u64 window, seq; 239 int key, err; 240 241 /* Also keep the kconfig extern in object BTF for the userspace skip. */ 242 if (CONFIG_PREEMPT_RT) 243 return 0; 244 245 if (tid == (__u32)trigger_tid) { 246 /* 247 * Trigger thread: schedule the first round once per armed 248 * sequence, targeting the target thread. The kfunc queues 249 * the scheduling irq_work and returns; the handler then 250 * parks inside the fexit window above. A fresh value has no 251 * expected BUSY source, so publish every scheduling result. 252 */ 253 seq = (__u64)get_status(BUSY_ARM_SEQ); 254 if (!seq || 255 ((__u64)get_status(BUSY_GEN1_RESULT) >> 32) == seq) 256 return 0; 257 key = (int)seq; 258 task = bpf_task_from_pid(target_tid); 259 if (!task) { 260 set_status(BUSY_GEN1_RESULT, 261 ((__s64)seq << 32) | (__u32)-ESRCH); 262 set_status(BUSY_ARM_SEQ, 0); 263 return 0; 264 } 265 work = bpf_map_lookup_elem(&hmap, &key); 266 if (!work) { 267 bpf_task_release(task); 268 set_status(BUSY_GEN1_RESULT, 269 ((__s64)seq << 32) | (__u32)-ENOENT); 270 set_status(BUSY_ARM_SEQ, 0); 271 return 0; 272 } 273 work->seq = seq; 274 err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, 275 busy_cb_gen1); 276 bpf_task_release(task); 277 set_status(BUSY_GEN1_RESULT, ((__s64)seq << 32) | (__u32)err); 278 return 0; 279 } 280 281 if (tid != (__u32)target_tid) 282 return 0; 283 284 window = (__u64)get_status(BUSY_WINDOW); 285 seq = window >> 32; 286 if (!seq || (__u32)window == 0) 287 return 0; 288 /* 289 * The first callback must have completed for this sequence. Being 290 * inside the target thread's own subsequent syscall proves the 291 * callback wrapper has returned and the context is STANDBY, which 292 * is stronger than observing the completion marker from another 293 * thread. 294 */ 295 if ((__u64)get_status(BUSY_CB1_SEQ) != seq) 296 return 0; 297 298 if ((__u64)get_status(BUSY_ACK) >> 32 != seq) { 299 /* 300 * The in-window attempt, exactly once per sequence. On a 301 * fixed kernel it must be rejected with -EBUSY: the context 302 * is STANDBY, the map and program references are alive and 303 * no other scheduler exists, so only the BUSY gate can 304 * reject here. 305 */ 306 if (!busy_try_gen2(seq, BUSY_ACK)) 307 set_status(BUSY_ACK, 308 ((__s64)seq << 32) | (__u32)-EBUSY); 309 return 0; 310 } 311 312 /* 313 * Recovery, gated by the test: the fexit program releasing the 314 * window does not mean the whole old handler has returned and 315 * BUSY has cleared yet, so retry until the schedule succeeds. 316 */ 317 if ((__u64)get_status(BUSY_RECOVER_SEQ) != seq) 318 return 0; 319 if ((__s32)get_status(BUSY_ACK) == 0) 320 return 0; /* scheduled in-window; nothing to recover */ 321 if (((__u64)get_status(BUSY_GEN2_RESULT) >> 32) == seq) 322 return 0; /* terminal recovery result published */ 323 busy_try_gen2(seq, BUSY_GEN2_RESULT); 324 return 0; 325 } 326