1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 * 5 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 6 * Copyright (c) 2022 Tejun Heo <tj@kernel.org> 7 * Copyright (c) 2022 David Vernet <dvernet@meta.com> 8 */ 9 #include <linux/btf_ids.h> 10 #include "ext_idle.h" 11 12 /* 13 * NOTE: sched_ext is in the process of growing multiple scheduler support and 14 * scx_root usage is in a transitional state. Naked dereferences are safe if the 15 * caller is one of the tasks attached to SCX and explicit RCU dereference is 16 * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but 17 * are used as temporary markers to indicate that the dereferences need to be 18 * updated to point to the associated scheduler instances rather than scx_root. 19 */ 20 static struct scx_sched __rcu *scx_root; 21 22 /* 23 * During exit, a task may schedule after losing its PIDs. When disabling the 24 * BPF scheduler, we need to be able to iterate tasks in every state to 25 * guarantee system safety. Maintain a dedicated task list which contains every 26 * task between its fork and eventual free. 27 */ 28 static DEFINE_RAW_SPINLOCK(scx_tasks_lock); 29 static LIST_HEAD(scx_tasks); 30 31 /* ops enable/disable */ 32 static DEFINE_MUTEX(scx_enable_mutex); 33 DEFINE_STATIC_KEY_FALSE(__scx_enabled); 34 DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); 35 static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); 36 static int scx_bypass_depth; 37 static cpumask_var_t scx_bypass_lb_donee_cpumask; 38 static cpumask_var_t scx_bypass_lb_resched_cpumask; 39 static bool scx_aborting; 40 static bool scx_init_task_enabled; 41 static bool scx_switching_all; 42 DEFINE_STATIC_KEY_FALSE(__scx_switched_all); 43 44 /* 45 * Tracks whether scx_enable() called scx_bypass(true). Used to balance bypass 46 * depth on enable failure. Will be removed when bypass depth is moved into the 47 * sched instance. 48 */ 49 static bool scx_bypassed_for_enable; 50 51 static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); 52 static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); 53 54 /* 55 * A monotically increasing sequence number that is incremented every time a 56 * scheduler is enabled. This can be used by to check if any custom sched_ext 57 * scheduler has ever been used in the system. 58 */ 59 static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); 60 61 /* 62 * The maximum amount of time in jiffies that a task may be runnable without 63 * being scheduled on a CPU. If this timeout is exceeded, it will trigger 64 * scx_error(). 65 */ 66 static unsigned long scx_watchdog_timeout; 67 68 /* 69 * The last time the delayed work was run. This delayed work relies on 70 * ksoftirqd being able to run to service timer interrupts, so it's possible 71 * that this work itself could get wedged. To account for this, we check that 72 * it's not stalled in the timer tick, and trigger an error if it is. 73 */ 74 static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; 75 76 static struct delayed_work scx_watchdog_work; 77 78 /* 79 * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence 80 * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu 81 * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated 82 * lazily when enabling and freed when disabling to avoid waste when sched_ext 83 * isn't active. 84 */ 85 struct scx_kick_syncs { 86 struct rcu_head rcu; 87 unsigned long syncs[]; 88 }; 89 90 static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); 91 92 /* 93 * Direct dispatch marker. 94 * 95 * Non-NULL values are used for direct dispatch from enqueue path. A valid 96 * pointer points to the task currently being enqueued. An ERR_PTR value is used 97 * to indicate that direct dispatch has already happened. 98 */ 99 static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); 100 101 static const struct rhashtable_params dsq_hash_params = { 102 .key_len = sizeof_field(struct scx_dispatch_q, id), 103 .key_offset = offsetof(struct scx_dispatch_q, id), 104 .head_offset = offsetof(struct scx_dispatch_q, hash_node), 105 }; 106 107 static LLIST_HEAD(dsqs_to_free); 108 109 /* dispatch buf */ 110 struct scx_dsp_buf_ent { 111 struct task_struct *task; 112 unsigned long qseq; 113 u64 dsq_id; 114 u64 enq_flags; 115 }; 116 117 static u32 scx_dsp_max_batch; 118 119 struct scx_dsp_ctx { 120 struct rq *rq; 121 u32 cursor; 122 u32 nr_tasks; 123 struct scx_dsp_buf_ent buf[]; 124 }; 125 126 static struct scx_dsp_ctx __percpu *scx_dsp_ctx; 127 128 /* string formatting from BPF */ 129 struct scx_bstr_buf { 130 u64 data[MAX_BPRINTF_VARARGS]; 131 char line[SCX_EXIT_MSG_LEN]; 132 }; 133 134 static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); 135 static struct scx_bstr_buf scx_exit_bstr_buf; 136 137 /* ops debug dump */ 138 struct scx_dump_data { 139 s32 cpu; 140 bool first; 141 s32 cursor; 142 struct seq_buf *s; 143 const char *prefix; 144 struct scx_bstr_buf buf; 145 }; 146 147 static struct scx_dump_data scx_dump_data = { 148 .cpu = -1, 149 }; 150 151 /* /sys/kernel/sched_ext interface */ 152 static struct kset *scx_kset; 153 154 /* 155 * Parameters that can be adjusted through /sys/module/sched_ext/parameters. 156 * There usually is no reason to modify these as normal scheduler operation 157 * shouldn't be affected by them. The knobs are primarily for debugging. 158 */ 159 static u64 scx_slice_dfl = SCX_SLICE_DFL; 160 static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC; 161 static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US; 162 163 static int set_slice_us(const char *val, const struct kernel_param *kp) 164 { 165 return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC); 166 } 167 168 static const struct kernel_param_ops slice_us_param_ops = { 169 .set = set_slice_us, 170 .get = param_get_uint, 171 }; 172 173 static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp) 174 { 175 return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC); 176 } 177 178 static const struct kernel_param_ops bypass_lb_intv_us_param_ops = { 179 .set = set_bypass_lb_intv_us, 180 .get = param_get_uint, 181 }; 182 183 #undef MODULE_PARAM_PREFIX 184 #define MODULE_PARAM_PREFIX "sched_ext." 185 186 module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600); 187 MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)"); 188 module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600); 189 MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)"); 190 191 #undef MODULE_PARAM_PREFIX 192 193 #define CREATE_TRACE_POINTS 194 #include <trace/events/sched_ext.h> 195 196 static void process_ddsp_deferred_locals(struct rq *rq); 197 static bool task_dead_and_done(struct task_struct *p); 198 static u32 reenq_local(struct rq *rq); 199 static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); 200 static bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, 201 s64 exit_code, const char *fmt, va_list args); 202 203 static __printf(4, 5) bool scx_exit(struct scx_sched *sch, 204 enum scx_exit_kind kind, s64 exit_code, 205 const char *fmt, ...) 206 { 207 va_list args; 208 bool ret; 209 210 va_start(args, fmt); 211 ret = scx_vexit(sch, kind, exit_code, fmt, args); 212 va_end(args); 213 214 return ret; 215 } 216 217 #define scx_error(sch, fmt, args...) scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) 218 #define scx_verror(sch, fmt, args) scx_vexit((sch), SCX_EXIT_ERROR, 0, fmt, args) 219 220 #define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) 221 222 static long jiffies_delta_msecs(unsigned long at, unsigned long now) 223 { 224 if (time_after(at, now)) 225 return jiffies_to_msecs(at - now); 226 else 227 return -(long)jiffies_to_msecs(now - at); 228 } 229 230 /* if the highest set bit is N, return a mask with bits [N+1, 31] set */ 231 static u32 higher_bits(u32 flags) 232 { 233 return ~((1 << fls(flags)) - 1); 234 } 235 236 /* return the mask with only the highest bit set */ 237 static u32 highest_bit(u32 flags) 238 { 239 int bit = fls(flags); 240 return ((u64)1 << bit) >> 1; 241 } 242 243 static bool u32_before(u32 a, u32 b) 244 { 245 return (s32)(a - b) < 0; 246 } 247 248 static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, 249 struct task_struct *p) 250 { 251 return sch->global_dsqs[cpu_to_node(task_cpu(p))]; 252 } 253 254 static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id) 255 { 256 return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params); 257 } 258 259 static const struct sched_class *scx_setscheduler_class(struct task_struct *p) 260 { 261 if (p->sched_class == &stop_sched_class) 262 return &stop_sched_class; 263 264 return __setscheduler_class(p->policy, p->prio); 265 } 266 267 /* 268 * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX 269 * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate 270 * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check 271 * whether it's running from an allowed context. 272 * 273 * @mask is constant, always inline to cull the mask calculations. 274 */ 275 static __always_inline void scx_kf_allow(u32 mask) 276 { 277 /* nesting is allowed only in increasing scx_kf_mask order */ 278 WARN_ONCE((mask | higher_bits(mask)) & current->scx.kf_mask, 279 "invalid nesting current->scx.kf_mask=0x%x mask=0x%x\n", 280 current->scx.kf_mask, mask); 281 current->scx.kf_mask |= mask; 282 barrier(); 283 } 284 285 static void scx_kf_disallow(u32 mask) 286 { 287 barrier(); 288 current->scx.kf_mask &= ~mask; 289 } 290 291 /* 292 * Track the rq currently locked. 293 * 294 * This allows kfuncs to safely operate on rq from any scx ops callback, 295 * knowing which rq is already locked. 296 */ 297 DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); 298 299 static inline void update_locked_rq(struct rq *rq) 300 { 301 /* 302 * Check whether @rq is actually locked. This can help expose bugs 303 * or incorrect assumptions about the context in which a kfunc or 304 * callback is executed. 305 */ 306 if (rq) 307 lockdep_assert_rq_held(rq); 308 __this_cpu_write(scx_locked_rq_state, rq); 309 } 310 311 #define SCX_CALL_OP(sch, mask, op, rq, args...) \ 312 do { \ 313 if (rq) \ 314 update_locked_rq(rq); \ 315 if (mask) { \ 316 scx_kf_allow(mask); \ 317 (sch)->ops.op(args); \ 318 scx_kf_disallow(mask); \ 319 } else { \ 320 (sch)->ops.op(args); \ 321 } \ 322 if (rq) \ 323 update_locked_rq(NULL); \ 324 } while (0) 325 326 #define SCX_CALL_OP_RET(sch, mask, op, rq, args...) \ 327 ({ \ 328 __typeof__((sch)->ops.op(args)) __ret; \ 329 \ 330 if (rq) \ 331 update_locked_rq(rq); \ 332 if (mask) { \ 333 scx_kf_allow(mask); \ 334 __ret = (sch)->ops.op(args); \ 335 scx_kf_disallow(mask); \ 336 } else { \ 337 __ret = (sch)->ops.op(args); \ 338 } \ 339 if (rq) \ 340 update_locked_rq(NULL); \ 341 __ret; \ 342 }) 343 344 /* 345 * Some kfuncs are allowed only on the tasks that are subjects of the 346 * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such 347 * restrictions, the following SCX_CALL_OP_*() variants should be used when 348 * invoking scx_ops operations that take task arguments. These can only be used 349 * for non-nesting operations due to the way the tasks are tracked. 350 * 351 * kfuncs which can only operate on such tasks can in turn use 352 * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on 353 * the specific task. 354 */ 355 #define SCX_CALL_OP_TASK(sch, mask, op, rq, task, args...) \ 356 do { \ 357 BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ 358 current->scx.kf_tasks[0] = task; \ 359 SCX_CALL_OP((sch), mask, op, rq, task, ##args); \ 360 current->scx.kf_tasks[0] = NULL; \ 361 } while (0) 362 363 #define SCX_CALL_OP_TASK_RET(sch, mask, op, rq, task, args...) \ 364 ({ \ 365 __typeof__((sch)->ops.op(task, ##args)) __ret; \ 366 BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ 367 current->scx.kf_tasks[0] = task; \ 368 __ret = SCX_CALL_OP_RET((sch), mask, op, rq, task, ##args); \ 369 current->scx.kf_tasks[0] = NULL; \ 370 __ret; \ 371 }) 372 373 #define SCX_CALL_OP_2TASKS_RET(sch, mask, op, rq, task0, task1, args...) \ 374 ({ \ 375 __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ 376 BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ 377 current->scx.kf_tasks[0] = task0; \ 378 current->scx.kf_tasks[1] = task1; \ 379 __ret = SCX_CALL_OP_RET((sch), mask, op, rq, task0, task1, ##args); \ 380 current->scx.kf_tasks[0] = NULL; \ 381 current->scx.kf_tasks[1] = NULL; \ 382 __ret; \ 383 }) 384 385 /* @mask is constant, always inline to cull unnecessary branches */ 386 static __always_inline bool scx_kf_allowed(struct scx_sched *sch, u32 mask) 387 { 388 if (unlikely(!(current->scx.kf_mask & mask))) { 389 scx_error(sch, "kfunc with mask 0x%x called from an operation only allowing 0x%x", 390 mask, current->scx.kf_mask); 391 return false; 392 } 393 394 /* 395 * Enforce nesting boundaries. e.g. A kfunc which can be called from 396 * DISPATCH must not be called if we're running DEQUEUE which is nested 397 * inside ops.dispatch(). We don't need to check boundaries for any 398 * blocking kfuncs as the verifier ensures they're only called from 399 * sleepable progs. 400 */ 401 if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && 402 (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { 403 scx_error(sch, "cpu_release kfunc called from a nested operation"); 404 return false; 405 } 406 407 if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && 408 (current->scx.kf_mask & higher_bits(SCX_KF_DISPATCH)))) { 409 scx_error(sch, "dispatch kfunc called from a nested operation"); 410 return false; 411 } 412 413 return true; 414 } 415 416 /* see SCX_CALL_OP_TASK() */ 417 static __always_inline bool scx_kf_allowed_on_arg_tasks(struct scx_sched *sch, 418 u32 mask, 419 struct task_struct *p) 420 { 421 if (!scx_kf_allowed(sch, mask)) 422 return false; 423 424 if (unlikely((p != current->scx.kf_tasks[0] && 425 p != current->scx.kf_tasks[1]))) { 426 scx_error(sch, "called on a task not being operated on"); 427 return false; 428 } 429 430 return true; 431 } 432 433 /** 434 * nldsq_next_task - Iterate to the next task in a non-local DSQ 435 * @dsq: user dsq being iterated 436 * @cur: current position, %NULL to start iteration 437 * @rev: walk backwards 438 * 439 * Returns %NULL when iteration is finished. 440 */ 441 static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, 442 struct task_struct *cur, bool rev) 443 { 444 struct list_head *list_node; 445 struct scx_dsq_list_node *dsq_lnode; 446 447 lockdep_assert_held(&dsq->lock); 448 449 if (cur) 450 list_node = &cur->scx.dsq_list.node; 451 else 452 list_node = &dsq->list; 453 454 /* find the next task, need to skip BPF iteration cursors */ 455 do { 456 if (rev) 457 list_node = list_node->prev; 458 else 459 list_node = list_node->next; 460 461 if (list_node == &dsq->list) 462 return NULL; 463 464 dsq_lnode = container_of(list_node, struct scx_dsq_list_node, 465 node); 466 } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); 467 468 return container_of(dsq_lnode, struct task_struct, scx.dsq_list); 469 } 470 471 #define nldsq_for_each_task(p, dsq) \ 472 for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ 473 (p) = nldsq_next_task((dsq), (p), false)) 474 475 476 /* 477 * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] 478 * dispatch order. BPF-visible iterator is opaque and larger to allow future 479 * changes without breaking backward compatibility. Can be used with 480 * bpf_for_each(). See bpf_iter_scx_dsq_*(). 481 */ 482 enum scx_dsq_iter_flags { 483 /* iterate in the reverse dispatch order */ 484 SCX_DSQ_ITER_REV = 1U << 16, 485 486 __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, 487 __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, 488 489 __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, 490 __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | 491 __SCX_DSQ_ITER_HAS_SLICE | 492 __SCX_DSQ_ITER_HAS_VTIME, 493 }; 494 495 struct bpf_iter_scx_dsq_kern { 496 struct scx_dsq_list_node cursor; 497 struct scx_dispatch_q *dsq; 498 u64 slice; 499 u64 vtime; 500 } __attribute__((aligned(8))); 501 502 struct bpf_iter_scx_dsq { 503 u64 __opaque[6]; 504 } __attribute__((aligned(8))); 505 506 507 /* 508 * SCX task iterator. 509 */ 510 struct scx_task_iter { 511 struct sched_ext_entity cursor; 512 struct task_struct *locked_task; 513 struct rq *rq; 514 struct rq_flags rf; 515 u32 cnt; 516 bool list_locked; 517 }; 518 519 /** 520 * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration 521 * @iter: iterator to init 522 * 523 * Initialize @iter and return with scx_tasks_lock held. Once initialized, @iter 524 * must eventually be stopped with scx_task_iter_stop(). 525 * 526 * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock() 527 * between this and the first next() call or between any two next() calls. If 528 * the locks are released between two next() calls, the caller is responsible 529 * for ensuring that the task being iterated remains accessible either through 530 * RCU read lock or obtaining a reference count. 531 * 532 * All tasks which existed when the iteration started are guaranteed to be 533 * visited as long as they are not dead. 534 */ 535 static void scx_task_iter_start(struct scx_task_iter *iter) 536 { 537 memset(iter, 0, sizeof(*iter)); 538 539 raw_spin_lock_irq(&scx_tasks_lock); 540 541 iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; 542 list_add(&iter->cursor.tasks_node, &scx_tasks); 543 iter->list_locked = true; 544 } 545 546 static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) 547 { 548 if (iter->locked_task) { 549 __balance_callbacks(iter->rq, &iter->rf); 550 task_rq_unlock(iter->rq, iter->locked_task, &iter->rf); 551 iter->locked_task = NULL; 552 } 553 } 554 555 /** 556 * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator 557 * @iter: iterator to unlock 558 * 559 * If @iter is in the middle of a locked iteration, it may be locking the rq of 560 * the task currently being visited in addition to scx_tasks_lock. Unlock both. 561 * This function can be safely called anytime during an iteration. The next 562 * iterator operation will automatically restore the necessary locking. 563 */ 564 static void scx_task_iter_unlock(struct scx_task_iter *iter) 565 { 566 __scx_task_iter_rq_unlock(iter); 567 if (iter->list_locked) { 568 iter->list_locked = false; 569 raw_spin_unlock_irq(&scx_tasks_lock); 570 } 571 } 572 573 static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter) 574 { 575 if (!iter->list_locked) { 576 raw_spin_lock_irq(&scx_tasks_lock); 577 iter->list_locked = true; 578 } 579 } 580 581 /** 582 * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock 583 * @iter: iterator to exit 584 * 585 * Exit a previously initialized @iter. Must be called with scx_tasks_lock held 586 * which is released on return. If the iterator holds a task's rq lock, that rq 587 * lock is also released. See scx_task_iter_start() for details. 588 */ 589 static void scx_task_iter_stop(struct scx_task_iter *iter) 590 { 591 __scx_task_iter_maybe_relock(iter); 592 list_del_init(&iter->cursor.tasks_node); 593 scx_task_iter_unlock(iter); 594 } 595 596 /** 597 * scx_task_iter_next - Next task 598 * @iter: iterator to walk 599 * 600 * Visit the next task. See scx_task_iter_start() for details. Locks are dropped 601 * and re-acquired every %SCX_TASK_ITER_BATCH iterations to avoid causing stalls 602 * by holding scx_tasks_lock for too long. 603 */ 604 static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) 605 { 606 struct list_head *cursor = &iter->cursor.tasks_node; 607 struct sched_ext_entity *pos; 608 609 if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) { 610 scx_task_iter_unlock(iter); 611 cond_resched(); 612 } 613 614 __scx_task_iter_maybe_relock(iter); 615 616 list_for_each_entry(pos, cursor, tasks_node) { 617 if (&pos->tasks_node == &scx_tasks) 618 return NULL; 619 if (!(pos->flags & SCX_TASK_CURSOR)) { 620 list_move(cursor, &pos->tasks_node); 621 return container_of(pos, struct task_struct, scx); 622 } 623 } 624 625 /* can't happen, should always terminate at scx_tasks above */ 626 BUG(); 627 } 628 629 /** 630 * scx_task_iter_next_locked - Next non-idle task with its rq locked 631 * @iter: iterator to walk 632 * 633 * Visit the non-idle task with its rq lock held. Allows callers to specify 634 * whether they would like to filter out dead tasks. See scx_task_iter_start() 635 * for details. 636 */ 637 static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) 638 { 639 struct task_struct *p; 640 641 __scx_task_iter_rq_unlock(iter); 642 643 while ((p = scx_task_iter_next(iter))) { 644 /* 645 * scx_task_iter is used to prepare and move tasks into SCX 646 * while loading the BPF scheduler and vice-versa while 647 * unloading. The init_tasks ("swappers") should be excluded 648 * from the iteration because: 649 * 650 * - It's unsafe to use __setschduler_prio() on an init_task to 651 * determine the sched_class to use as it won't preserve its 652 * idle_sched_class. 653 * 654 * - ops.init/exit_task() can easily be confused if called with 655 * init_tasks as they, e.g., share PID 0. 656 * 657 * As init_tasks are never scheduled through SCX, they can be 658 * skipped safely. Note that is_idle_task() which tests %PF_IDLE 659 * doesn't work here: 660 * 661 * - %PF_IDLE may not be set for an init_task whose CPU hasn't 662 * yet been onlined. 663 * 664 * - %PF_IDLE can be set on tasks that are not init_tasks. See 665 * play_idle_precise() used by CONFIG_IDLE_INJECT. 666 * 667 * Test for idle_sched_class as only init_tasks are on it. 668 */ 669 if (p->sched_class != &idle_sched_class) 670 break; 671 } 672 if (!p) 673 return NULL; 674 675 iter->rq = task_rq_lock(p, &iter->rf); 676 iter->locked_task = p; 677 678 return p; 679 } 680 681 /** 682 * scx_add_event - Increase an event counter for 'name' by 'cnt' 683 * @sch: scx_sched to account events for 684 * @name: an event name defined in struct scx_event_stats 685 * @cnt: the number of the event occurred 686 * 687 * This can be used when preemption is not disabled. 688 */ 689 #define scx_add_event(sch, name, cnt) do { \ 690 this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ 691 trace_sched_ext_event(#name, (cnt)); \ 692 } while(0) 693 694 /** 695 * __scx_add_event - Increase an event counter for 'name' by 'cnt' 696 * @sch: scx_sched to account events for 697 * @name: an event name defined in struct scx_event_stats 698 * @cnt: the number of the event occurred 699 * 700 * This should be used only when preemption is disabled. 701 */ 702 #define __scx_add_event(sch, name, cnt) do { \ 703 __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ 704 trace_sched_ext_event(#name, cnt); \ 705 } while(0) 706 707 /** 708 * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e' 709 * @dst_e: destination event stats 710 * @src_e: source event stats 711 * @kind: a kind of event to be aggregated 712 */ 713 #define scx_agg_event(dst_e, src_e, kind) do { \ 714 (dst_e)->kind += READ_ONCE((src_e)->kind); \ 715 } while(0) 716 717 /** 718 * scx_dump_event - Dump an event 'kind' in 'events' to 's' 719 * @s: output seq_buf 720 * @events: event stats 721 * @kind: a kind of event to dump 722 */ 723 #define scx_dump_event(s, events, kind) do { \ 724 dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ 725 } while (0) 726 727 728 static void scx_read_events(struct scx_sched *sch, 729 struct scx_event_stats *events); 730 731 static enum scx_enable_state scx_enable_state(void) 732 { 733 return atomic_read(&scx_enable_state_var); 734 } 735 736 static enum scx_enable_state scx_set_enable_state(enum scx_enable_state to) 737 { 738 return atomic_xchg(&scx_enable_state_var, to); 739 } 740 741 static bool scx_tryset_enable_state(enum scx_enable_state to, 742 enum scx_enable_state from) 743 { 744 int from_v = from; 745 746 return atomic_try_cmpxchg(&scx_enable_state_var, &from_v, to); 747 } 748 749 /** 750 * wait_ops_state - Busy-wait the specified ops state to end 751 * @p: target task 752 * @opss: state to wait the end of 753 * 754 * Busy-wait for @p to transition out of @opss. This can only be used when the 755 * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also 756 * has load_acquire semantics to ensure that the caller can see the updates made 757 * in the enqueueing and dispatching paths. 758 */ 759 static void wait_ops_state(struct task_struct *p, unsigned long opss) 760 { 761 do { 762 cpu_relax(); 763 } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); 764 } 765 766 static inline bool __cpu_valid(s32 cpu) 767 { 768 return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); 769 } 770 771 /** 772 * ops_cpu_valid - Verify a cpu number, to be used on ops input args 773 * @sch: scx_sched to abort on error 774 * @cpu: cpu number which came from a BPF ops 775 * @where: extra information reported on error 776 * 777 * @cpu is a cpu number which came from the BPF scheduler and can be any value. 778 * Verify that it is in range and one of the possible cpus. If invalid, trigger 779 * an ops error. 780 */ 781 static bool ops_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) 782 { 783 if (__cpu_valid(cpu)) { 784 return true; 785 } else { 786 scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: ""); 787 return false; 788 } 789 } 790 791 /** 792 * ops_sanitize_err - Sanitize a -errno value 793 * @sch: scx_sched to error out on error 794 * @ops_name: operation to blame on failure 795 * @err: -errno value to sanitize 796 * 797 * Verify @err is a valid -errno. If not, trigger scx_error() and return 798 * -%EPROTO. This is necessary because returning a rogue -errno up the chain can 799 * cause misbehaviors. For an example, a large negative return from 800 * ops.init_task() triggers an oops when passed up the call chain because the 801 * value fails IS_ERR() test after being encoded with ERR_PTR() and then is 802 * handled as a pointer. 803 */ 804 static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) 805 { 806 if (err < 0 && err >= -MAX_ERRNO) 807 return err; 808 809 scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); 810 return -EPROTO; 811 } 812 813 static void run_deferred(struct rq *rq) 814 { 815 process_ddsp_deferred_locals(rq); 816 817 if (local_read(&rq->scx.reenq_local_deferred)) { 818 local_set(&rq->scx.reenq_local_deferred, 0); 819 reenq_local(rq); 820 } 821 } 822 823 static void deferred_bal_cb_workfn(struct rq *rq) 824 { 825 run_deferred(rq); 826 } 827 828 static void deferred_irq_workfn(struct irq_work *irq_work) 829 { 830 struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); 831 832 raw_spin_rq_lock(rq); 833 run_deferred(rq); 834 raw_spin_rq_unlock(rq); 835 } 836 837 /** 838 * schedule_deferred - Schedule execution of deferred actions on an rq 839 * @rq: target rq 840 * 841 * Schedule execution of deferred actions on @rq. Deferred actions are executed 842 * with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks 843 * to other rqs. 844 */ 845 static void schedule_deferred(struct rq *rq) 846 { 847 /* 848 * Queue an irq work. They are executed on IRQ re-enable which may take 849 * a bit longer than the scheduler hook in schedule_deferred_locked(). 850 */ 851 irq_work_queue(&rq->scx.deferred_irq_work); 852 } 853 854 /** 855 * schedule_deferred_locked - Schedule execution of deferred actions on an rq 856 * @rq: target rq 857 * 858 * Schedule execution of deferred actions on @rq. Equivalent to 859 * schedule_deferred() but requires @rq to be locked and can be more efficient. 860 */ 861 static void schedule_deferred_locked(struct rq *rq) 862 { 863 lockdep_assert_rq_held(rq); 864 865 /* 866 * If in the middle of waking up a task, task_woken_scx() will be called 867 * afterwards which will then run the deferred actions, no need to 868 * schedule anything. 869 */ 870 if (rq->scx.flags & SCX_RQ_IN_WAKEUP) 871 return; 872 873 /* Don't do anything if there already is a deferred operation. */ 874 if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) 875 return; 876 877 /* 878 * If in balance, the balance callbacks will be called before rq lock is 879 * released. Schedule one. 880 * 881 * 882 * We can't directly insert the callback into the 883 * rq's list: The call can drop its lock and make the pending balance 884 * callback visible to unrelated code paths that call rq_pin_lock(). 885 * 886 * Just let balance_one() know that it must do it itself. 887 */ 888 if (rq->scx.flags & SCX_RQ_IN_BALANCE) { 889 rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; 890 return; 891 } 892 893 /* 894 * No scheduler hooks available. Use the generic irq_work path. The 895 * above WAKEUP and BALANCE paths should cover most of the cases and the 896 * time to IRQ re-enable shouldn't be long. 897 */ 898 schedule_deferred(rq); 899 } 900 901 /** 902 * touch_core_sched - Update timestamp used for core-sched task ordering 903 * @rq: rq to read clock from, must be locked 904 * @p: task to update the timestamp for 905 * 906 * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to 907 * implement global or local-DSQ FIFO ordering for core-sched. Should be called 908 * when a task becomes runnable and its turn on the CPU ends (e.g. slice 909 * exhaustion). 910 */ 911 static void touch_core_sched(struct rq *rq, struct task_struct *p) 912 { 913 lockdep_assert_rq_held(rq); 914 915 #ifdef CONFIG_SCHED_CORE 916 /* 917 * It's okay to update the timestamp spuriously. Use 918 * sched_core_disabled() which is cheaper than enabled(). 919 * 920 * As this is used to determine ordering between tasks of sibling CPUs, 921 * it may be better to use per-core dispatch sequence instead. 922 */ 923 if (!sched_core_disabled()) 924 p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); 925 #endif 926 } 927 928 /** 929 * touch_core_sched_dispatch - Update core-sched timestamp on dispatch 930 * @rq: rq to read clock from, must be locked 931 * @p: task being dispatched 932 * 933 * If the BPF scheduler implements custom core-sched ordering via 934 * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO 935 * ordering within each local DSQ. This function is called from dispatch paths 936 * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. 937 */ 938 static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) 939 { 940 lockdep_assert_rq_held(rq); 941 942 #ifdef CONFIG_SCHED_CORE 943 if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) 944 touch_core_sched(rq, p); 945 #endif 946 } 947 948 static void update_curr_scx(struct rq *rq) 949 { 950 struct task_struct *curr = rq->curr; 951 s64 delta_exec; 952 953 delta_exec = update_curr_common(rq); 954 if (unlikely(delta_exec <= 0)) 955 return; 956 957 if (curr->scx.slice != SCX_SLICE_INF) { 958 curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); 959 if (!curr->scx.slice) 960 touch_core_sched(rq, curr); 961 } 962 963 dl_server_update(&rq->ext_server, delta_exec); 964 } 965 966 static bool scx_dsq_priq_less(struct rb_node *node_a, 967 const struct rb_node *node_b) 968 { 969 const struct task_struct *a = 970 container_of(node_a, struct task_struct, scx.dsq_priq); 971 const struct task_struct *b = 972 container_of(node_b, struct task_struct, scx.dsq_priq); 973 974 return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); 975 } 976 977 static void dsq_mod_nr(struct scx_dispatch_q *dsq, s32 delta) 978 { 979 /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */ 980 WRITE_ONCE(dsq->nr, dsq->nr + delta); 981 } 982 983 static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) 984 { 985 p->scx.slice = READ_ONCE(scx_slice_dfl); 986 __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); 987 } 988 989 static void local_dsq_post_enq(struct scx_dispatch_q *dsq, struct task_struct *p, 990 u64 enq_flags) 991 { 992 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); 993 bool preempt = false; 994 995 /* 996 * If @rq is in balance, the CPU is already vacant and looking for the 997 * next task to run. No need to preempt or trigger resched after moving 998 * @p into its local DSQ. 999 */ 1000 if (rq->scx.flags & SCX_RQ_IN_BALANCE) 1001 return; 1002 1003 if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && 1004 rq->curr->sched_class == &ext_sched_class) { 1005 rq->curr->scx.slice = 0; 1006 preempt = true; 1007 } 1008 1009 if (preempt || sched_class_above(&ext_sched_class, rq->curr->sched_class)) 1010 resched_curr(rq); 1011 } 1012 1013 static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq, 1014 struct task_struct *p, u64 enq_flags) 1015 { 1016 bool is_local = dsq->id == SCX_DSQ_LOCAL; 1017 1018 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); 1019 WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || 1020 !RB_EMPTY_NODE(&p->scx.dsq_priq)); 1021 1022 if (!is_local) { 1023 raw_spin_lock_nested(&dsq->lock, 1024 (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0); 1025 1026 if (unlikely(dsq->id == SCX_DSQ_INVALID)) { 1027 scx_error(sch, "attempting to dispatch to a destroyed dsq"); 1028 /* fall back to the global dsq */ 1029 raw_spin_unlock(&dsq->lock); 1030 dsq = find_global_dsq(sch, p); 1031 raw_spin_lock(&dsq->lock); 1032 } 1033 } 1034 1035 if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && 1036 (enq_flags & SCX_ENQ_DSQ_PRIQ))) { 1037 /* 1038 * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from 1039 * their FIFO queues. To avoid confusion and accidentally 1040 * starving vtime-dispatched tasks by FIFO-dispatched tasks, we 1041 * disallow any internal DSQ from doing vtime ordering of 1042 * tasks. 1043 */ 1044 scx_error(sch, "cannot use vtime ordering for built-in DSQs"); 1045 enq_flags &= ~SCX_ENQ_DSQ_PRIQ; 1046 } 1047 1048 if (enq_flags & SCX_ENQ_DSQ_PRIQ) { 1049 struct rb_node *rbp; 1050 1051 /* 1052 * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are 1053 * linked to both the rbtree and list on PRIQs, this can only be 1054 * tested easily when adding the first task. 1055 */ 1056 if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && 1057 nldsq_next_task(dsq, NULL, false))) 1058 scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks", 1059 dsq->id); 1060 1061 p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; 1062 rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); 1063 1064 /* 1065 * Find the previous task and insert after it on the list so 1066 * that @dsq->list is vtime ordered. 1067 */ 1068 rbp = rb_prev(&p->scx.dsq_priq); 1069 if (rbp) { 1070 struct task_struct *prev = 1071 container_of(rbp, struct task_struct, 1072 scx.dsq_priq); 1073 list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); 1074 /* first task unchanged - no update needed */ 1075 } else { 1076 list_add(&p->scx.dsq_list.node, &dsq->list); 1077 /* not builtin and new task is at head - use fastpath */ 1078 rcu_assign_pointer(dsq->first_task, p); 1079 } 1080 } else { 1081 /* a FIFO DSQ shouldn't be using PRIQ enqueuing */ 1082 if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) 1083 scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", 1084 dsq->id); 1085 1086 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { 1087 list_add(&p->scx.dsq_list.node, &dsq->list); 1088 /* new task inserted at head - use fastpath */ 1089 if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) 1090 rcu_assign_pointer(dsq->first_task, p); 1091 } else { 1092 bool was_empty; 1093 1094 was_empty = list_empty(&dsq->list); 1095 list_add_tail(&p->scx.dsq_list.node, &dsq->list); 1096 if (was_empty && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) 1097 rcu_assign_pointer(dsq->first_task, p); 1098 } 1099 } 1100 1101 /* seq records the order tasks are queued, used by BPF DSQ iterator */ 1102 dsq->seq++; 1103 p->scx.dsq_seq = dsq->seq; 1104 1105 dsq_mod_nr(dsq, 1); 1106 p->scx.dsq = dsq; 1107 1108 /* 1109 * scx.ddsp_dsq_id and scx.ddsp_enq_flags are only relevant on the 1110 * direct dispatch path, but we clear them here because the direct 1111 * dispatch verdict may be overridden on the enqueue path during e.g. 1112 * bypass. 1113 */ 1114 p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; 1115 p->scx.ddsp_enq_flags = 0; 1116 1117 /* 1118 * We're transitioning out of QUEUEING or DISPATCHING. store_release to 1119 * match waiters' load_acquire. 1120 */ 1121 if (enq_flags & SCX_ENQ_CLEAR_OPSS) 1122 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 1123 1124 if (is_local) 1125 local_dsq_post_enq(dsq, p, enq_flags); 1126 else 1127 raw_spin_unlock(&dsq->lock); 1128 } 1129 1130 static void task_unlink_from_dsq(struct task_struct *p, 1131 struct scx_dispatch_q *dsq) 1132 { 1133 WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); 1134 1135 if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { 1136 rb_erase(&p->scx.dsq_priq, &dsq->priq); 1137 RB_CLEAR_NODE(&p->scx.dsq_priq); 1138 p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; 1139 } 1140 1141 list_del_init(&p->scx.dsq_list.node); 1142 dsq_mod_nr(dsq, -1); 1143 1144 if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && dsq->first_task == p) { 1145 struct task_struct *first_task; 1146 1147 first_task = nldsq_next_task(dsq, NULL, false); 1148 rcu_assign_pointer(dsq->first_task, first_task); 1149 } 1150 } 1151 1152 static void dispatch_dequeue(struct rq *rq, struct task_struct *p) 1153 { 1154 struct scx_dispatch_q *dsq = p->scx.dsq; 1155 bool is_local = dsq == &rq->scx.local_dsq; 1156 1157 lockdep_assert_rq_held(rq); 1158 1159 if (!dsq) { 1160 /* 1161 * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals. 1162 * Unlinking is all that's needed to cancel. 1163 */ 1164 if (unlikely(!list_empty(&p->scx.dsq_list.node))) 1165 list_del_init(&p->scx.dsq_list.node); 1166 1167 /* 1168 * When dispatching directly from the BPF scheduler to a local 1169 * DSQ, the task isn't associated with any DSQ but 1170 * @p->scx.holding_cpu may be set under the protection of 1171 * %SCX_OPSS_DISPATCHING. 1172 */ 1173 if (p->scx.holding_cpu >= 0) 1174 p->scx.holding_cpu = -1; 1175 1176 return; 1177 } 1178 1179 if (!is_local) 1180 raw_spin_lock(&dsq->lock); 1181 1182 /* 1183 * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't 1184 * change underneath us. 1185 */ 1186 if (p->scx.holding_cpu < 0) { 1187 /* @p must still be on @dsq, dequeue */ 1188 task_unlink_from_dsq(p, dsq); 1189 } else { 1190 /* 1191 * We're racing against dispatch_to_local_dsq() which already 1192 * removed @p from @dsq and set @p->scx.holding_cpu. Clear the 1193 * holding_cpu which tells dispatch_to_local_dsq() that it lost 1194 * the race. 1195 */ 1196 WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); 1197 p->scx.holding_cpu = -1; 1198 } 1199 p->scx.dsq = NULL; 1200 1201 if (!is_local) 1202 raw_spin_unlock(&dsq->lock); 1203 } 1204 1205 /* 1206 * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq 1207 * and dsq are locked. 1208 */ 1209 static void dispatch_dequeue_locked(struct task_struct *p, 1210 struct scx_dispatch_q *dsq) 1211 { 1212 lockdep_assert_rq_held(task_rq(p)); 1213 lockdep_assert_held(&dsq->lock); 1214 1215 task_unlink_from_dsq(p, dsq); 1216 p->scx.dsq = NULL; 1217 } 1218 1219 static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, 1220 struct rq *rq, u64 dsq_id, 1221 struct task_struct *p) 1222 { 1223 struct scx_dispatch_q *dsq; 1224 1225 if (dsq_id == SCX_DSQ_LOCAL) 1226 return &rq->scx.local_dsq; 1227 1228 if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { 1229 s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; 1230 1231 if (!ops_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) 1232 return find_global_dsq(sch, p); 1233 1234 return &cpu_rq(cpu)->scx.local_dsq; 1235 } 1236 1237 if (dsq_id == SCX_DSQ_GLOBAL) 1238 dsq = find_global_dsq(sch, p); 1239 else 1240 dsq = find_user_dsq(sch, dsq_id); 1241 1242 if (unlikely(!dsq)) { 1243 scx_error(sch, "non-existent DSQ 0x%llx for %s[%d]", 1244 dsq_id, p->comm, p->pid); 1245 return find_global_dsq(sch, p); 1246 } 1247 1248 return dsq; 1249 } 1250 1251 static void mark_direct_dispatch(struct scx_sched *sch, 1252 struct task_struct *ddsp_task, 1253 struct task_struct *p, u64 dsq_id, 1254 u64 enq_flags) 1255 { 1256 /* 1257 * Mark that dispatch already happened from ops.select_cpu() or 1258 * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value 1259 * which can never match a valid task pointer. 1260 */ 1261 __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); 1262 1263 /* @p must match the task on the enqueue path */ 1264 if (unlikely(p != ddsp_task)) { 1265 if (IS_ERR(ddsp_task)) 1266 scx_error(sch, "%s[%d] already direct-dispatched", 1267 p->comm, p->pid); 1268 else 1269 scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]", 1270 ddsp_task->comm, ddsp_task->pid, 1271 p->comm, p->pid); 1272 return; 1273 } 1274 1275 WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); 1276 WARN_ON_ONCE(p->scx.ddsp_enq_flags); 1277 1278 p->scx.ddsp_dsq_id = dsq_id; 1279 p->scx.ddsp_enq_flags = enq_flags; 1280 } 1281 1282 static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, 1283 u64 enq_flags) 1284 { 1285 struct rq *rq = task_rq(p); 1286 struct scx_dispatch_q *dsq = 1287 find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, p); 1288 1289 touch_core_sched_dispatch(rq, p); 1290 1291 p->scx.ddsp_enq_flags |= enq_flags; 1292 1293 /* 1294 * We are in the enqueue path with @rq locked and pinned, and thus can't 1295 * double lock a remote rq and enqueue to its local DSQ. For 1296 * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer 1297 * the enqueue so that it's executed when @rq can be unlocked. 1298 */ 1299 if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { 1300 unsigned long opss; 1301 1302 opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; 1303 1304 switch (opss & SCX_OPSS_STATE_MASK) { 1305 case SCX_OPSS_NONE: 1306 break; 1307 case SCX_OPSS_QUEUEING: 1308 /* 1309 * As @p was never passed to the BPF side, _release is 1310 * not strictly necessary. Still do it for consistency. 1311 */ 1312 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 1313 break; 1314 default: 1315 WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()", 1316 p->comm, p->pid, opss); 1317 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 1318 break; 1319 } 1320 1321 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); 1322 list_add_tail(&p->scx.dsq_list.node, 1323 &rq->scx.ddsp_deferred_locals); 1324 schedule_deferred_locked(rq); 1325 return; 1326 } 1327 1328 dispatch_enqueue(sch, dsq, p, 1329 p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); 1330 } 1331 1332 static bool scx_rq_online(struct rq *rq) 1333 { 1334 /* 1335 * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates 1336 * the online state as seen from the BPF scheduler. cpu_active() test 1337 * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will 1338 * stay set until the current scheduling operation is complete even if 1339 * we aren't locking @rq. 1340 */ 1341 return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); 1342 } 1343 1344 static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, 1345 int sticky_cpu) 1346 { 1347 struct scx_sched *sch = scx_root; 1348 struct task_struct **ddsp_taskp; 1349 struct scx_dispatch_q *dsq; 1350 unsigned long qseq; 1351 1352 WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); 1353 1354 /* rq migration */ 1355 if (sticky_cpu == cpu_of(rq)) 1356 goto local_norefill; 1357 1358 /* 1359 * If !scx_rq_online(), we already told the BPF scheduler that the CPU 1360 * is offline and are just running the hotplug path. Don't bother the 1361 * BPF scheduler. 1362 */ 1363 if (!scx_rq_online(rq)) 1364 goto local; 1365 1366 if (scx_rq_bypassing(rq)) { 1367 __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); 1368 goto bypass; 1369 } 1370 1371 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) 1372 goto direct; 1373 1374 /* see %SCX_OPS_ENQ_EXITING */ 1375 if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && 1376 unlikely(p->flags & PF_EXITING)) { 1377 __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); 1378 goto local; 1379 } 1380 1381 /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ 1382 if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && 1383 is_migration_disabled(p)) { 1384 __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); 1385 goto local; 1386 } 1387 1388 if (unlikely(!SCX_HAS_OP(sch, enqueue))) 1389 goto global; 1390 1391 /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ 1392 qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; 1393 1394 WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); 1395 atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); 1396 1397 ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); 1398 WARN_ON_ONCE(*ddsp_taskp); 1399 *ddsp_taskp = p; 1400 1401 SCX_CALL_OP_TASK(sch, SCX_KF_ENQUEUE, enqueue, rq, p, enq_flags); 1402 1403 *ddsp_taskp = NULL; 1404 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) 1405 goto direct; 1406 1407 /* 1408 * If not directly dispatched, QUEUEING isn't clear yet and dispatch or 1409 * dequeue may be waiting. The store_release matches their load_acquire. 1410 */ 1411 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); 1412 return; 1413 1414 direct: 1415 direct_dispatch(sch, p, enq_flags); 1416 return; 1417 local_norefill: 1418 dispatch_enqueue(sch, &rq->scx.local_dsq, p, enq_flags); 1419 return; 1420 local: 1421 dsq = &rq->scx.local_dsq; 1422 goto enqueue; 1423 global: 1424 dsq = find_global_dsq(sch, p); 1425 goto enqueue; 1426 bypass: 1427 dsq = &task_rq(p)->scx.bypass_dsq; 1428 goto enqueue; 1429 1430 enqueue: 1431 /* 1432 * For task-ordering, slice refill must be treated as implying the end 1433 * of the current slice. Otherwise, the longer @p stays on the CPU, the 1434 * higher priority it becomes from scx_prio_less()'s POV. 1435 */ 1436 touch_core_sched(rq, p); 1437 refill_task_slice_dfl(sch, p); 1438 dispatch_enqueue(sch, dsq, p, enq_flags); 1439 } 1440 1441 static bool task_runnable(const struct task_struct *p) 1442 { 1443 return !list_empty(&p->scx.runnable_node); 1444 } 1445 1446 static void set_task_runnable(struct rq *rq, struct task_struct *p) 1447 { 1448 lockdep_assert_rq_held(rq); 1449 1450 if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { 1451 p->scx.runnable_at = jiffies; 1452 p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; 1453 } 1454 1455 /* 1456 * list_add_tail() must be used. scx_bypass() depends on tasks being 1457 * appended to the runnable_list. 1458 */ 1459 list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); 1460 } 1461 1462 static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) 1463 { 1464 list_del_init(&p->scx.runnable_node); 1465 if (reset_runnable_at) 1466 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; 1467 } 1468 1469 static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) 1470 { 1471 struct scx_sched *sch = scx_root; 1472 int sticky_cpu = p->scx.sticky_cpu; 1473 1474 if (enq_flags & ENQUEUE_WAKEUP) 1475 rq->scx.flags |= SCX_RQ_IN_WAKEUP; 1476 1477 enq_flags |= rq->scx.extra_enq_flags; 1478 1479 if (sticky_cpu >= 0) 1480 p->scx.sticky_cpu = -1; 1481 1482 /* 1483 * Restoring a running task will be immediately followed by 1484 * set_next_task_scx() which expects the task to not be on the BPF 1485 * scheduler as tasks can only start running through local DSQs. Force 1486 * direct-dispatch into the local DSQ by setting the sticky_cpu. 1487 */ 1488 if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) 1489 sticky_cpu = cpu_of(rq); 1490 1491 if (p->scx.flags & SCX_TASK_QUEUED) { 1492 WARN_ON_ONCE(!task_runnable(p)); 1493 goto out; 1494 } 1495 1496 set_task_runnable(rq, p); 1497 p->scx.flags |= SCX_TASK_QUEUED; 1498 rq->scx.nr_running++; 1499 add_nr_running(rq, 1); 1500 1501 if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) 1502 SCX_CALL_OP_TASK(sch, SCX_KF_REST, runnable, rq, p, enq_flags); 1503 1504 if (enq_flags & SCX_ENQ_WAKEUP) 1505 touch_core_sched(rq, p); 1506 1507 /* Start dl_server if this is the first task being enqueued */ 1508 if (rq->scx.nr_running == 1) 1509 dl_server_start(&rq->ext_server); 1510 1511 do_enqueue_task(rq, p, enq_flags, sticky_cpu); 1512 out: 1513 rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; 1514 1515 if ((enq_flags & SCX_ENQ_CPU_SELECTED) && 1516 unlikely(cpu_of(rq) != p->scx.selected_cpu)) 1517 __scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1); 1518 } 1519 1520 static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) 1521 { 1522 struct scx_sched *sch = scx_root; 1523 unsigned long opss; 1524 1525 /* dequeue is always temporary, don't reset runnable_at */ 1526 clr_task_runnable(p, false); 1527 1528 /* acquire ensures that we see the preceding updates on QUEUED */ 1529 opss = atomic_long_read_acquire(&p->scx.ops_state); 1530 1531 switch (opss & SCX_OPSS_STATE_MASK) { 1532 case SCX_OPSS_NONE: 1533 break; 1534 case SCX_OPSS_QUEUEING: 1535 /* 1536 * QUEUEING is started and finished while holding @p's rq lock. 1537 * As we're holding the rq lock now, we shouldn't see QUEUEING. 1538 */ 1539 BUG(); 1540 case SCX_OPSS_QUEUED: 1541 if (SCX_HAS_OP(sch, dequeue)) 1542 SCX_CALL_OP_TASK(sch, SCX_KF_REST, dequeue, rq, 1543 p, deq_flags); 1544 1545 if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, 1546 SCX_OPSS_NONE)) 1547 break; 1548 fallthrough; 1549 case SCX_OPSS_DISPATCHING: 1550 /* 1551 * If @p is being dispatched from the BPF scheduler to a DSQ, 1552 * wait for the transfer to complete so that @p doesn't get 1553 * added to its DSQ after dequeueing is complete. 1554 * 1555 * As we're waiting on DISPATCHING with the rq locked, the 1556 * dispatching side shouldn't try to lock the rq while 1557 * DISPATCHING is set. See dispatch_to_local_dsq(). 1558 * 1559 * DISPATCHING shouldn't have qseq set and control can reach 1560 * here with NONE @opss from the above QUEUED case block. 1561 * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. 1562 */ 1563 wait_ops_state(p, SCX_OPSS_DISPATCHING); 1564 BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); 1565 break; 1566 } 1567 } 1568 1569 static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) 1570 { 1571 struct scx_sched *sch = scx_root; 1572 1573 if (!(p->scx.flags & SCX_TASK_QUEUED)) { 1574 WARN_ON_ONCE(task_runnable(p)); 1575 return true; 1576 } 1577 1578 ops_dequeue(rq, p, deq_flags); 1579 1580 /* 1581 * A currently running task which is going off @rq first gets dequeued 1582 * and then stops running. As we want running <-> stopping transitions 1583 * to be contained within runnable <-> quiescent transitions, trigger 1584 * ->stopping() early here instead of in put_prev_task_scx(). 1585 * 1586 * @p may go through multiple stopping <-> running transitions between 1587 * here and put_prev_task_scx() if task attribute changes occur while 1588 * balance_one() leaves @rq unlocked. However, they don't contain any 1589 * information meaningful to the BPF scheduler and can be suppressed by 1590 * skipping the callbacks if the task is !QUEUED. 1591 */ 1592 if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { 1593 update_curr_scx(rq); 1594 SCX_CALL_OP_TASK(sch, SCX_KF_REST, stopping, rq, p, false); 1595 } 1596 1597 if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) 1598 SCX_CALL_OP_TASK(sch, SCX_KF_REST, quiescent, rq, p, deq_flags); 1599 1600 if (deq_flags & SCX_DEQ_SLEEP) 1601 p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; 1602 else 1603 p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; 1604 1605 p->scx.flags &= ~SCX_TASK_QUEUED; 1606 rq->scx.nr_running--; 1607 sub_nr_running(rq, 1); 1608 1609 dispatch_dequeue(rq, p); 1610 return true; 1611 } 1612 1613 static void yield_task_scx(struct rq *rq) 1614 { 1615 struct scx_sched *sch = scx_root; 1616 struct task_struct *p = rq->donor; 1617 1618 if (SCX_HAS_OP(sch, yield)) 1619 SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, yield, rq, p, NULL); 1620 else 1621 p->scx.slice = 0; 1622 } 1623 1624 static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) 1625 { 1626 struct scx_sched *sch = scx_root; 1627 struct task_struct *from = rq->donor; 1628 1629 if (SCX_HAS_OP(sch, yield)) 1630 return SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, yield, rq, 1631 from, to); 1632 else 1633 return false; 1634 } 1635 1636 static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags, 1637 struct scx_dispatch_q *src_dsq, 1638 struct rq *dst_rq) 1639 { 1640 struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; 1641 1642 /* @dsq is locked and @p is on @dst_rq */ 1643 lockdep_assert_held(&src_dsq->lock); 1644 lockdep_assert_rq_held(dst_rq); 1645 1646 WARN_ON_ONCE(p->scx.holding_cpu >= 0); 1647 1648 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) 1649 list_add(&p->scx.dsq_list.node, &dst_dsq->list); 1650 else 1651 list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); 1652 1653 dsq_mod_nr(dst_dsq, 1); 1654 p->scx.dsq = dst_dsq; 1655 1656 local_dsq_post_enq(dst_dsq, p, enq_flags); 1657 } 1658 1659 /** 1660 * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ 1661 * @p: task to move 1662 * @enq_flags: %SCX_ENQ_* 1663 * @src_rq: rq to move the task from, locked on entry, released on return 1664 * @dst_rq: rq to move the task into, locked on return 1665 * 1666 * Move @p which is currently on @src_rq to @dst_rq's local DSQ. 1667 */ 1668 static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, 1669 struct rq *src_rq, struct rq *dst_rq) 1670 { 1671 lockdep_assert_rq_held(src_rq); 1672 1673 /* the following marks @p MIGRATING which excludes dequeue */ 1674 deactivate_task(src_rq, p, 0); 1675 set_task_cpu(p, cpu_of(dst_rq)); 1676 p->scx.sticky_cpu = cpu_of(dst_rq); 1677 1678 raw_spin_rq_unlock(src_rq); 1679 raw_spin_rq_lock(dst_rq); 1680 1681 /* 1682 * We want to pass scx-specific enq_flags but activate_task() will 1683 * truncate the upper 32 bit. As we own @rq, we can pass them through 1684 * @rq->scx.extra_enq_flags instead. 1685 */ 1686 WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); 1687 WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); 1688 dst_rq->scx.extra_enq_flags = enq_flags; 1689 activate_task(dst_rq, p, 0); 1690 dst_rq->scx.extra_enq_flags = 0; 1691 } 1692 1693 /* 1694 * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two 1695 * differences: 1696 * 1697 * - is_cpu_allowed() asks "Can this task run on this CPU?" while 1698 * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to 1699 * this CPU?". 1700 * 1701 * While migration is disabled, is_cpu_allowed() has to say "yes" as the task 1702 * must be allowed to finish on the CPU that it's currently on regardless of 1703 * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the 1704 * BPF scheduler shouldn't attempt to migrate a task which has migration 1705 * disabled. 1706 * 1707 * - The BPF scheduler is bypassed while the rq is offline and we can always say 1708 * no to the BPF scheduler initiated migrations while offline. 1709 * 1710 * The caller must ensure that @p and @rq are on different CPUs. 1711 */ 1712 static bool task_can_run_on_remote_rq(struct scx_sched *sch, 1713 struct task_struct *p, struct rq *rq, 1714 bool enforce) 1715 { 1716 int cpu = cpu_of(rq); 1717 1718 WARN_ON_ONCE(task_cpu(p) == cpu); 1719 1720 /* 1721 * If @p has migration disabled, @p->cpus_ptr is updated to contain only 1722 * the pinned CPU in migrate_disable_switch() while @p is being switched 1723 * out. However, put_prev_task_scx() is called before @p->cpus_ptr is 1724 * updated and thus another CPU may see @p on a DSQ inbetween leading to 1725 * @p passing the below task_allowed_on_cpu() check while migration is 1726 * disabled. 1727 * 1728 * Test the migration disabled state first as the race window is narrow 1729 * and the BPF scheduler failing to check migration disabled state can 1730 * easily be masked if task_allowed_on_cpu() is done first. 1731 */ 1732 if (unlikely(is_migration_disabled(p))) { 1733 if (enforce) 1734 scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d", 1735 p->comm, p->pid, task_cpu(p), cpu); 1736 return false; 1737 } 1738 1739 /* 1740 * We don't require the BPF scheduler to avoid dispatching to offline 1741 * CPUs mostly for convenience but also because CPUs can go offline 1742 * between scx_bpf_dsq_insert() calls and here. Trigger error iff the 1743 * picked CPU is outside the allowed mask. 1744 */ 1745 if (!task_allowed_on_cpu(p, cpu)) { 1746 if (enforce) 1747 scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]", 1748 cpu, p->comm, p->pid); 1749 return false; 1750 } 1751 1752 if (!scx_rq_online(rq)) { 1753 if (enforce) 1754 __scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1); 1755 return false; 1756 } 1757 1758 return true; 1759 } 1760 1761 /** 1762 * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq 1763 * @p: target task 1764 * @dsq: locked DSQ @p is currently on 1765 * @src_rq: rq @p is currently on, stable with @dsq locked 1766 * 1767 * Called with @dsq locked but no rq's locked. We want to move @p to a different 1768 * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is 1769 * required when transferring into a local DSQ. Even when transferring into a 1770 * non-local DSQ, it's better to use the same mechanism to protect against 1771 * dequeues and maintain the invariant that @p->scx.dsq can only change while 1772 * @src_rq is locked, which e.g. scx_dump_task() depends on. 1773 * 1774 * We want to grab @src_rq but that can deadlock if we try while locking @dsq, 1775 * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As 1776 * this may race with dequeue, which can't drop the rq lock or fail, do a little 1777 * dancing from our side. 1778 * 1779 * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets 1780 * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu 1781 * would be cleared to -1. While other cpus may have updated it to different 1782 * values afterwards, as this operation can't be preempted or recurse, the 1783 * holding_cpu can never become this CPU again before we're done. Thus, we can 1784 * tell whether we lost to dequeue by testing whether the holding_cpu still 1785 * points to this CPU. See dispatch_dequeue() for the counterpart. 1786 * 1787 * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is 1788 * still valid. %false if lost to dequeue. 1789 */ 1790 static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, 1791 struct scx_dispatch_q *dsq, 1792 struct rq *src_rq) 1793 { 1794 s32 cpu = raw_smp_processor_id(); 1795 1796 lockdep_assert_held(&dsq->lock); 1797 1798 WARN_ON_ONCE(p->scx.holding_cpu >= 0); 1799 task_unlink_from_dsq(p, dsq); 1800 p->scx.holding_cpu = cpu; 1801 1802 raw_spin_unlock(&dsq->lock); 1803 raw_spin_rq_lock(src_rq); 1804 1805 /* task_rq couldn't have changed if we're still the holding cpu */ 1806 return likely(p->scx.holding_cpu == cpu) && 1807 !WARN_ON_ONCE(src_rq != task_rq(p)); 1808 } 1809 1810 static bool consume_remote_task(struct rq *this_rq, struct task_struct *p, 1811 struct scx_dispatch_q *dsq, struct rq *src_rq) 1812 { 1813 raw_spin_rq_unlock(this_rq); 1814 1815 if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { 1816 move_remote_task_to_local_dsq(p, 0, src_rq, this_rq); 1817 return true; 1818 } else { 1819 raw_spin_rq_unlock(src_rq); 1820 raw_spin_rq_lock(this_rq); 1821 return false; 1822 } 1823 } 1824 1825 /** 1826 * move_task_between_dsqs() - Move a task from one DSQ to another 1827 * @sch: scx_sched being operated on 1828 * @p: target task 1829 * @enq_flags: %SCX_ENQ_* 1830 * @src_dsq: DSQ @p is currently on, must not be a local DSQ 1831 * @dst_dsq: DSQ @p is being moved to, can be any DSQ 1832 * 1833 * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local 1834 * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq 1835 * will change. As @p's task_rq is locked, this function doesn't need to use the 1836 * holding_cpu mechanism. 1837 * 1838 * On return, @src_dsq is unlocked and only @p's new task_rq, which is the 1839 * return value, is locked. 1840 */ 1841 static struct rq *move_task_between_dsqs(struct scx_sched *sch, 1842 struct task_struct *p, u64 enq_flags, 1843 struct scx_dispatch_q *src_dsq, 1844 struct scx_dispatch_q *dst_dsq) 1845 { 1846 struct rq *src_rq = task_rq(p), *dst_rq; 1847 1848 BUG_ON(src_dsq->id == SCX_DSQ_LOCAL); 1849 lockdep_assert_held(&src_dsq->lock); 1850 lockdep_assert_rq_held(src_rq); 1851 1852 if (dst_dsq->id == SCX_DSQ_LOCAL) { 1853 dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); 1854 if (src_rq != dst_rq && 1855 unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { 1856 dst_dsq = find_global_dsq(sch, p); 1857 dst_rq = src_rq; 1858 } 1859 } else { 1860 /* no need to migrate if destination is a non-local DSQ */ 1861 dst_rq = src_rq; 1862 } 1863 1864 /* 1865 * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different 1866 * CPU, @p will be migrated. 1867 */ 1868 if (dst_dsq->id == SCX_DSQ_LOCAL) { 1869 /* @p is going from a non-local DSQ to a local DSQ */ 1870 if (src_rq == dst_rq) { 1871 task_unlink_from_dsq(p, src_dsq); 1872 move_local_task_to_local_dsq(p, enq_flags, 1873 src_dsq, dst_rq); 1874 raw_spin_unlock(&src_dsq->lock); 1875 } else { 1876 raw_spin_unlock(&src_dsq->lock); 1877 move_remote_task_to_local_dsq(p, enq_flags, 1878 src_rq, dst_rq); 1879 } 1880 } else { 1881 /* 1882 * @p is going from a non-local DSQ to a non-local DSQ. As 1883 * $src_dsq is already locked, do an abbreviated dequeue. 1884 */ 1885 dispatch_dequeue_locked(p, src_dsq); 1886 raw_spin_unlock(&src_dsq->lock); 1887 1888 dispatch_enqueue(sch, dst_dsq, p, enq_flags); 1889 } 1890 1891 return dst_rq; 1892 } 1893 1894 static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, 1895 struct scx_dispatch_q *dsq) 1896 { 1897 struct task_struct *p; 1898 retry: 1899 /* 1900 * The caller can't expect to successfully consume a task if the task's 1901 * addition to @dsq isn't guaranteed to be visible somehow. Test 1902 * @dsq->list without locking and skip if it seems empty. 1903 */ 1904 if (list_empty(&dsq->list)) 1905 return false; 1906 1907 raw_spin_lock(&dsq->lock); 1908 1909 nldsq_for_each_task(p, dsq) { 1910 struct rq *task_rq = task_rq(p); 1911 1912 /* 1913 * This loop can lead to multiple lockup scenarios, e.g. the BPF 1914 * scheduler can put an enormous number of affinitized tasks into 1915 * a contended DSQ, or the outer retry loop can repeatedly race 1916 * against scx_bypass() dequeueing tasks from @dsq trying to put 1917 * the system into the bypass mode. This can easily live-lock the 1918 * machine. If aborting, exit from all non-bypass DSQs. 1919 */ 1920 if (unlikely(READ_ONCE(scx_aborting)) && dsq->id != SCX_DSQ_BYPASS) 1921 break; 1922 1923 if (rq == task_rq) { 1924 task_unlink_from_dsq(p, dsq); 1925 move_local_task_to_local_dsq(p, 0, dsq, rq); 1926 raw_spin_unlock(&dsq->lock); 1927 return true; 1928 } 1929 1930 if (task_can_run_on_remote_rq(sch, p, rq, false)) { 1931 if (likely(consume_remote_task(rq, p, dsq, task_rq))) 1932 return true; 1933 goto retry; 1934 } 1935 } 1936 1937 raw_spin_unlock(&dsq->lock); 1938 return false; 1939 } 1940 1941 static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) 1942 { 1943 int node = cpu_to_node(cpu_of(rq)); 1944 1945 return consume_dispatch_q(sch, rq, sch->global_dsqs[node]); 1946 } 1947 1948 /** 1949 * dispatch_to_local_dsq - Dispatch a task to a local dsq 1950 * @sch: scx_sched being operated on 1951 * @rq: current rq which is locked 1952 * @dst_dsq: destination DSQ 1953 * @p: task to dispatch 1954 * @enq_flags: %SCX_ENQ_* 1955 * 1956 * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local 1957 * DSQ. This function performs all the synchronization dancing needed because 1958 * local DSQs are protected with rq locks. 1959 * 1960 * The caller must have exclusive ownership of @p (e.g. through 1961 * %SCX_OPSS_DISPATCHING). 1962 */ 1963 static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, 1964 struct scx_dispatch_q *dst_dsq, 1965 struct task_struct *p, u64 enq_flags) 1966 { 1967 struct rq *src_rq = task_rq(p); 1968 struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); 1969 struct rq *locked_rq = rq; 1970 1971 /* 1972 * We're synchronized against dequeue through DISPATCHING. As @p can't 1973 * be dequeued, its task_rq and cpus_allowed are stable too. 1974 * 1975 * If dispatching to @rq that @p is already on, no lock dancing needed. 1976 */ 1977 if (rq == src_rq && rq == dst_rq) { 1978 dispatch_enqueue(sch, dst_dsq, p, 1979 enq_flags | SCX_ENQ_CLEAR_OPSS); 1980 return; 1981 } 1982 1983 if (src_rq != dst_rq && 1984 unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { 1985 dispatch_enqueue(sch, find_global_dsq(sch, p), p, 1986 enq_flags | SCX_ENQ_CLEAR_OPSS); 1987 return; 1988 } 1989 1990 /* 1991 * @p is on a possibly remote @src_rq which we need to lock to move the 1992 * task. If dequeue is in progress, it'd be locking @src_rq and waiting 1993 * on DISPATCHING, so we can't grab @src_rq lock while holding 1994 * DISPATCHING. 1995 * 1996 * As DISPATCHING guarantees that @p is wholly ours, we can pretend that 1997 * we're moving from a DSQ and use the same mechanism - mark the task 1998 * under transfer with holding_cpu, release DISPATCHING and then follow 1999 * the same protocol. See unlink_dsq_and_lock_src_rq(). 2000 */ 2001 p->scx.holding_cpu = raw_smp_processor_id(); 2002 2003 /* store_release ensures that dequeue sees the above */ 2004 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 2005 2006 /* switch to @src_rq lock */ 2007 if (locked_rq != src_rq) { 2008 raw_spin_rq_unlock(locked_rq); 2009 locked_rq = src_rq; 2010 raw_spin_rq_lock(src_rq); 2011 } 2012 2013 /* task_rq couldn't have changed if we're still the holding cpu */ 2014 if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && 2015 !WARN_ON_ONCE(src_rq != task_rq(p))) { 2016 /* 2017 * If @p is staying on the same rq, there's no need to go 2018 * through the full deactivate/activate cycle. Optimize by 2019 * abbreviating move_remote_task_to_local_dsq(). 2020 */ 2021 if (src_rq == dst_rq) { 2022 p->scx.holding_cpu = -1; 2023 dispatch_enqueue(sch, &dst_rq->scx.local_dsq, p, 2024 enq_flags); 2025 } else { 2026 move_remote_task_to_local_dsq(p, enq_flags, 2027 src_rq, dst_rq); 2028 /* task has been moved to dst_rq, which is now locked */ 2029 locked_rq = dst_rq; 2030 } 2031 2032 /* if the destination CPU is idle, wake it up */ 2033 if (sched_class_above(p->sched_class, dst_rq->curr->sched_class)) 2034 resched_curr(dst_rq); 2035 } 2036 2037 /* switch back to @rq lock */ 2038 if (locked_rq != rq) { 2039 raw_spin_rq_unlock(locked_rq); 2040 raw_spin_rq_lock(rq); 2041 } 2042 } 2043 2044 /** 2045 * finish_dispatch - Asynchronously finish dispatching a task 2046 * @rq: current rq which is locked 2047 * @p: task to finish dispatching 2048 * @qseq_at_dispatch: qseq when @p started getting dispatched 2049 * @dsq_id: destination DSQ ID 2050 * @enq_flags: %SCX_ENQ_* 2051 * 2052 * Dispatching to local DSQs may need to wait for queueing to complete or 2053 * require rq lock dancing. As we don't wanna do either while inside 2054 * ops.dispatch() to avoid locking order inversion, we split dispatching into 2055 * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the 2056 * task and its qseq. Once ops.dispatch() returns, this function is called to 2057 * finish up. 2058 * 2059 * There is no guarantee that @p is still valid for dispatching or even that it 2060 * was valid in the first place. Make sure that the task is still owned by the 2061 * BPF scheduler and claim the ownership before dispatching. 2062 */ 2063 static void finish_dispatch(struct scx_sched *sch, struct rq *rq, 2064 struct task_struct *p, 2065 unsigned long qseq_at_dispatch, 2066 u64 dsq_id, u64 enq_flags) 2067 { 2068 struct scx_dispatch_q *dsq; 2069 unsigned long opss; 2070 2071 touch_core_sched_dispatch(rq, p); 2072 retry: 2073 /* 2074 * No need for _acquire here. @p is accessed only after a successful 2075 * try_cmpxchg to DISPATCHING. 2076 */ 2077 opss = atomic_long_read(&p->scx.ops_state); 2078 2079 switch (opss & SCX_OPSS_STATE_MASK) { 2080 case SCX_OPSS_DISPATCHING: 2081 case SCX_OPSS_NONE: 2082 /* someone else already got to it */ 2083 return; 2084 case SCX_OPSS_QUEUED: 2085 /* 2086 * If qseq doesn't match, @p has gone through at least one 2087 * dispatch/dequeue and re-enqueue cycle between 2088 * scx_bpf_dsq_insert() and here and we have no claim on it. 2089 */ 2090 if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) 2091 return; 2092 2093 /* 2094 * While we know @p is accessible, we don't yet have a claim on 2095 * it - the BPF scheduler is allowed to dispatch tasks 2096 * spuriously and there can be a racing dequeue attempt. Let's 2097 * claim @p by atomically transitioning it from QUEUED to 2098 * DISPATCHING. 2099 */ 2100 if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, 2101 SCX_OPSS_DISPATCHING))) 2102 break; 2103 goto retry; 2104 case SCX_OPSS_QUEUEING: 2105 /* 2106 * do_enqueue_task() is in the process of transferring the task 2107 * to the BPF scheduler while holding @p's rq lock. As we aren't 2108 * holding any kernel or BPF resource that the enqueue path may 2109 * depend upon, it's safe to wait. 2110 */ 2111 wait_ops_state(p, opss); 2112 goto retry; 2113 } 2114 2115 BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); 2116 2117 dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, p); 2118 2119 if (dsq->id == SCX_DSQ_LOCAL) 2120 dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); 2121 else 2122 dispatch_enqueue(sch, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); 2123 } 2124 2125 static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) 2126 { 2127 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 2128 u32 u; 2129 2130 for (u = 0; u < dspc->cursor; u++) { 2131 struct scx_dsp_buf_ent *ent = &dspc->buf[u]; 2132 2133 finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, 2134 ent->enq_flags); 2135 } 2136 2137 dspc->nr_tasks += dspc->cursor; 2138 dspc->cursor = 0; 2139 } 2140 2141 static inline void maybe_queue_balance_callback(struct rq *rq) 2142 { 2143 lockdep_assert_rq_held(rq); 2144 2145 if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) 2146 return; 2147 2148 queue_balance_callback(rq, &rq->scx.deferred_bal_cb, 2149 deferred_bal_cb_workfn); 2150 2151 rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; 2152 } 2153 2154 static int balance_one(struct rq *rq, struct task_struct *prev) 2155 { 2156 struct scx_sched *sch = scx_root; 2157 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 2158 bool prev_on_scx = prev->sched_class == &ext_sched_class; 2159 bool prev_on_rq = prev->scx.flags & SCX_TASK_QUEUED; 2160 int nr_loops = SCX_DSP_MAX_LOOPS; 2161 2162 lockdep_assert_rq_held(rq); 2163 rq->scx.flags |= SCX_RQ_IN_BALANCE; 2164 rq->scx.flags &= ~SCX_RQ_BAL_KEEP; 2165 2166 if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && 2167 unlikely(rq->scx.cpu_released)) { 2168 /* 2169 * If the previous sched_class for the current CPU was not SCX, 2170 * notify the BPF scheduler that it again has control of the 2171 * core. This callback complements ->cpu_release(), which is 2172 * emitted in switch_class(). 2173 */ 2174 if (SCX_HAS_OP(sch, cpu_acquire)) 2175 SCX_CALL_OP(sch, SCX_KF_REST, cpu_acquire, rq, 2176 cpu_of(rq), NULL); 2177 rq->scx.cpu_released = false; 2178 } 2179 2180 if (prev_on_scx) { 2181 update_curr_scx(rq); 2182 2183 /* 2184 * If @prev is runnable & has slice left, it has priority and 2185 * fetching more just increases latency for the fetched tasks. 2186 * Tell pick_task_scx() to keep running @prev. If the BPF 2187 * scheduler wants to handle this explicitly, it should 2188 * implement ->cpu_release(). 2189 * 2190 * See scx_disable_workfn() for the explanation on the bypassing 2191 * test. 2192 */ 2193 if (prev_on_rq && prev->scx.slice && !scx_rq_bypassing(rq)) { 2194 rq->scx.flags |= SCX_RQ_BAL_KEEP; 2195 goto has_tasks; 2196 } 2197 } 2198 2199 /* if there already are tasks to run, nothing to do */ 2200 if (rq->scx.local_dsq.nr) 2201 goto has_tasks; 2202 2203 if (consume_global_dsq(sch, rq)) 2204 goto has_tasks; 2205 2206 if (scx_rq_bypassing(rq)) { 2207 if (consume_dispatch_q(sch, rq, &rq->scx.bypass_dsq)) 2208 goto has_tasks; 2209 else 2210 goto no_tasks; 2211 } 2212 2213 if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) 2214 goto no_tasks; 2215 2216 dspc->rq = rq; 2217 2218 /* 2219 * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, 2220 * the local DSQ might still end up empty after a successful 2221 * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() 2222 * produced some tasks, retry. The BPF scheduler may depend on this 2223 * looping behavior to simplify its implementation. 2224 */ 2225 do { 2226 dspc->nr_tasks = 0; 2227 2228 SCX_CALL_OP(sch, SCX_KF_DISPATCH, dispatch, rq, 2229 cpu_of(rq), prev_on_scx ? prev : NULL); 2230 2231 flush_dispatch_buf(sch, rq); 2232 2233 if (prev_on_rq && prev->scx.slice) { 2234 rq->scx.flags |= SCX_RQ_BAL_KEEP; 2235 goto has_tasks; 2236 } 2237 if (rq->scx.local_dsq.nr) 2238 goto has_tasks; 2239 if (consume_global_dsq(sch, rq)) 2240 goto has_tasks; 2241 2242 /* 2243 * ops.dispatch() can trap us in this loop by repeatedly 2244 * dispatching ineligible tasks. Break out once in a while to 2245 * allow the watchdog to run. As IRQ can't be enabled in 2246 * balance(), we want to complete this scheduling cycle and then 2247 * start a new one. IOW, we want to call resched_curr() on the 2248 * next, most likely idle, task, not the current one. Use 2249 * scx_kick_cpu() for deferred kicking. 2250 */ 2251 if (unlikely(!--nr_loops)) { 2252 scx_kick_cpu(sch, cpu_of(rq), 0); 2253 break; 2254 } 2255 } while (dspc->nr_tasks); 2256 2257 no_tasks: 2258 /* 2259 * Didn't find another task to run. Keep running @prev unless 2260 * %SCX_OPS_ENQ_LAST is in effect. 2261 */ 2262 if (prev_on_rq && 2263 (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_rq_bypassing(rq))) { 2264 rq->scx.flags |= SCX_RQ_BAL_KEEP; 2265 __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); 2266 goto has_tasks; 2267 } 2268 rq->scx.flags &= ~SCX_RQ_IN_BALANCE; 2269 return false; 2270 2271 has_tasks: 2272 rq->scx.flags &= ~SCX_RQ_IN_BALANCE; 2273 return true; 2274 } 2275 2276 static void process_ddsp_deferred_locals(struct rq *rq) 2277 { 2278 struct task_struct *p; 2279 2280 lockdep_assert_rq_held(rq); 2281 2282 /* 2283 * Now that @rq can be unlocked, execute the deferred enqueueing of 2284 * tasks directly dispatched to the local DSQs of other CPUs. See 2285 * direct_dispatch(). Keep popping from the head instead of using 2286 * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq 2287 * temporarily. 2288 */ 2289 while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, 2290 struct task_struct, scx.dsq_list.node))) { 2291 struct scx_sched *sch = scx_root; 2292 struct scx_dispatch_q *dsq; 2293 2294 list_del_init(&p->scx.dsq_list.node); 2295 2296 dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, p); 2297 if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) 2298 dispatch_to_local_dsq(sch, rq, dsq, p, 2299 p->scx.ddsp_enq_flags); 2300 } 2301 } 2302 2303 static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) 2304 { 2305 struct scx_sched *sch = scx_root; 2306 2307 if (p->scx.flags & SCX_TASK_QUEUED) { 2308 /* 2309 * Core-sched might decide to execute @p before it is 2310 * dispatched. Call ops_dequeue() to notify the BPF scheduler. 2311 */ 2312 ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); 2313 dispatch_dequeue(rq, p); 2314 } 2315 2316 p->se.exec_start = rq_clock_task(rq); 2317 2318 /* see dequeue_task_scx() on why we skip when !QUEUED */ 2319 if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) 2320 SCX_CALL_OP_TASK(sch, SCX_KF_REST, running, rq, p); 2321 2322 clr_task_runnable(p, true); 2323 2324 /* 2325 * @p is getting newly scheduled or got kicked after someone updated its 2326 * slice. Refresh whether tick can be stopped. See scx_can_stop_tick(). 2327 */ 2328 if ((p->scx.slice == SCX_SLICE_INF) != 2329 (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { 2330 if (p->scx.slice == SCX_SLICE_INF) 2331 rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; 2332 else 2333 rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; 2334 2335 sched_update_tick_dependency(rq); 2336 2337 /* 2338 * For now, let's refresh the load_avgs just when transitioning 2339 * in and out of nohz. In the future, we might want to add a 2340 * mechanism which calls the following periodically on 2341 * tick-stopped CPUs. 2342 */ 2343 update_other_load_avgs(rq); 2344 } 2345 } 2346 2347 static enum scx_cpu_preempt_reason 2348 preempt_reason_from_class(const struct sched_class *class) 2349 { 2350 if (class == &stop_sched_class) 2351 return SCX_CPU_PREEMPT_STOP; 2352 if (class == &dl_sched_class) 2353 return SCX_CPU_PREEMPT_DL; 2354 if (class == &rt_sched_class) 2355 return SCX_CPU_PREEMPT_RT; 2356 return SCX_CPU_PREEMPT_UNKNOWN; 2357 } 2358 2359 static void switch_class(struct rq *rq, struct task_struct *next) 2360 { 2361 struct scx_sched *sch = scx_root; 2362 const struct sched_class *next_class = next->sched_class; 2363 2364 if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) 2365 return; 2366 2367 /* 2368 * The callback is conceptually meant to convey that the CPU is no 2369 * longer under the control of SCX. Therefore, don't invoke the callback 2370 * if the next class is below SCX (in which case the BPF scheduler has 2371 * actively decided not to schedule any tasks on the CPU). 2372 */ 2373 if (sched_class_above(&ext_sched_class, next_class)) 2374 return; 2375 2376 /* 2377 * At this point we know that SCX was preempted by a higher priority 2378 * sched_class, so invoke the ->cpu_release() callback if we have not 2379 * done so already. We only send the callback once between SCX being 2380 * preempted, and it regaining control of the CPU. 2381 * 2382 * ->cpu_release() complements ->cpu_acquire(), which is emitted the 2383 * next time that balance_one() is invoked. 2384 */ 2385 if (!rq->scx.cpu_released) { 2386 if (SCX_HAS_OP(sch, cpu_release)) { 2387 struct scx_cpu_release_args args = { 2388 .reason = preempt_reason_from_class(next_class), 2389 .task = next, 2390 }; 2391 2392 SCX_CALL_OP(sch, SCX_KF_CPU_RELEASE, cpu_release, rq, 2393 cpu_of(rq), &args); 2394 } 2395 rq->scx.cpu_released = true; 2396 } 2397 } 2398 2399 static void put_prev_task_scx(struct rq *rq, struct task_struct *p, 2400 struct task_struct *next) 2401 { 2402 struct scx_sched *sch = scx_root; 2403 2404 /* see kick_cpus_irq_workfn() */ 2405 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); 2406 2407 update_curr_scx(rq); 2408 2409 /* see dequeue_task_scx() on why we skip when !QUEUED */ 2410 if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) 2411 SCX_CALL_OP_TASK(sch, SCX_KF_REST, stopping, rq, p, true); 2412 2413 if (p->scx.flags & SCX_TASK_QUEUED) { 2414 set_task_runnable(rq, p); 2415 2416 /* 2417 * If @p has slice left and is being put, @p is getting 2418 * preempted by a higher priority scheduler class or core-sched 2419 * forcing a different task. Leave it at the head of the local 2420 * DSQ. 2421 */ 2422 if (p->scx.slice && !scx_rq_bypassing(rq)) { 2423 dispatch_enqueue(sch, &rq->scx.local_dsq, p, 2424 SCX_ENQ_HEAD); 2425 goto switch_class; 2426 } 2427 2428 /* 2429 * If @p is runnable but we're about to enter a lower 2430 * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell 2431 * ops.enqueue() that @p is the only one available for this cpu, 2432 * which should trigger an explicit follow-up scheduling event. 2433 */ 2434 if (next && sched_class_above(&ext_sched_class, next->sched_class)) { 2435 WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST)); 2436 do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); 2437 } else { 2438 do_enqueue_task(rq, p, 0, -1); 2439 } 2440 } 2441 2442 switch_class: 2443 if (next && next->sched_class != &ext_sched_class) 2444 switch_class(rq, next); 2445 } 2446 2447 static struct task_struct *first_local_task(struct rq *rq) 2448 { 2449 return list_first_entry_or_null(&rq->scx.local_dsq.list, 2450 struct task_struct, scx.dsq_list.node); 2451 } 2452 2453 static struct task_struct * 2454 do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) 2455 { 2456 struct task_struct *prev = rq->curr; 2457 bool keep_prev; 2458 struct task_struct *p; 2459 2460 /* see kick_cpus_irq_workfn() */ 2461 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); 2462 2463 rq->next_class = &ext_sched_class; 2464 2465 rq_unpin_lock(rq, rf); 2466 balance_one(rq, prev); 2467 rq_repin_lock(rq, rf); 2468 maybe_queue_balance_callback(rq); 2469 2470 /* 2471 * If any higher-priority sched class enqueued a runnable task on 2472 * this rq during balance_one(), abort and return RETRY_TASK, so 2473 * that the scheduler loop can restart. 2474 * 2475 * If @force_scx is true, always try to pick a SCHED_EXT task, 2476 * regardless of any higher-priority sched classes activity. 2477 */ 2478 if (!force_scx && sched_class_above(rq->next_class, &ext_sched_class)) 2479 return RETRY_TASK; 2480 2481 keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; 2482 if (unlikely(keep_prev && 2483 prev->sched_class != &ext_sched_class)) { 2484 WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); 2485 keep_prev = false; 2486 } 2487 2488 /* 2489 * If balance_one() is telling us to keep running @prev, replenish slice 2490 * if necessary and keep running @prev. Otherwise, pop the first one 2491 * from the local DSQ. 2492 */ 2493 if (keep_prev) { 2494 p = prev; 2495 if (!p->scx.slice) 2496 refill_task_slice_dfl(rcu_dereference_sched(scx_root), p); 2497 } else { 2498 p = first_local_task(rq); 2499 if (!p) 2500 return NULL; 2501 2502 if (unlikely(!p->scx.slice)) { 2503 struct scx_sched *sch = rcu_dereference_sched(scx_root); 2504 2505 if (!scx_rq_bypassing(rq) && !sch->warned_zero_slice) { 2506 printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n", 2507 p->comm, p->pid, __func__); 2508 sch->warned_zero_slice = true; 2509 } 2510 refill_task_slice_dfl(sch, p); 2511 } 2512 } 2513 2514 return p; 2515 } 2516 2517 static struct task_struct *pick_task_scx(struct rq *rq, struct rq_flags *rf) 2518 { 2519 return do_pick_task_scx(rq, rf, false); 2520 } 2521 2522 /* 2523 * Select the next task to run from the ext scheduling class. 2524 * 2525 * Use do_pick_task_scx() directly with @force_scx enabled, since the 2526 * dl_server must always select a sched_ext task. 2527 */ 2528 static struct task_struct * 2529 ext_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) 2530 { 2531 if (!scx_enabled()) 2532 return NULL; 2533 2534 return do_pick_task_scx(dl_se->rq, rf, true); 2535 } 2536 2537 /* 2538 * Initialize the ext server deadline entity. 2539 */ 2540 void ext_server_init(struct rq *rq) 2541 { 2542 struct sched_dl_entity *dl_se = &rq->ext_server; 2543 2544 init_dl_entity(dl_se); 2545 2546 dl_server_init(dl_se, rq, ext_server_pick_task); 2547 } 2548 2549 #ifdef CONFIG_SCHED_CORE 2550 /** 2551 * scx_prio_less - Task ordering for core-sched 2552 * @a: task A 2553 * @b: task B 2554 * @in_fi: in forced idle state 2555 * 2556 * Core-sched is implemented as an additional scheduling layer on top of the 2557 * usual sched_class'es and needs to find out the expected task ordering. For 2558 * SCX, core-sched calls this function to interrogate the task ordering. 2559 * 2560 * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used 2561 * to implement the default task ordering. The older the timestamp, the higher 2562 * priority the task - the global FIFO ordering matching the default scheduling 2563 * behavior. 2564 * 2565 * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to 2566 * implement FIFO ordering within each local DSQ. See pick_task_scx(). 2567 */ 2568 bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, 2569 bool in_fi) 2570 { 2571 struct scx_sched *sch = scx_root; 2572 2573 /* 2574 * The const qualifiers are dropped from task_struct pointers when 2575 * calling ops.core_sched_before(). Accesses are controlled by the 2576 * verifier. 2577 */ 2578 if (SCX_HAS_OP(sch, core_sched_before) && 2579 !scx_rq_bypassing(task_rq(a))) 2580 return SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, core_sched_before, 2581 NULL, 2582 (struct task_struct *)a, 2583 (struct task_struct *)b); 2584 else 2585 return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); 2586 } 2587 #endif /* CONFIG_SCHED_CORE */ 2588 2589 static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) 2590 { 2591 struct scx_sched *sch = scx_root; 2592 bool rq_bypass; 2593 2594 /* 2595 * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it 2596 * can be a good migration opportunity with low cache and memory 2597 * footprint. Returning a CPU different than @prev_cpu triggers 2598 * immediate rq migration. However, for SCX, as the current rq 2599 * association doesn't dictate where the task is going to run, this 2600 * doesn't fit well. If necessary, we can later add a dedicated method 2601 * which can decide to preempt self to force it through the regular 2602 * scheduling path. 2603 */ 2604 if (unlikely(wake_flags & WF_EXEC)) 2605 return prev_cpu; 2606 2607 rq_bypass = scx_rq_bypassing(task_rq(p)); 2608 if (likely(SCX_HAS_OP(sch, select_cpu)) && !rq_bypass) { 2609 s32 cpu; 2610 struct task_struct **ddsp_taskp; 2611 2612 ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); 2613 WARN_ON_ONCE(*ddsp_taskp); 2614 *ddsp_taskp = p; 2615 2616 cpu = SCX_CALL_OP_TASK_RET(sch, 2617 SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU, 2618 select_cpu, NULL, p, prev_cpu, 2619 wake_flags); 2620 p->scx.selected_cpu = cpu; 2621 *ddsp_taskp = NULL; 2622 if (ops_cpu_valid(sch, cpu, "from ops.select_cpu()")) 2623 return cpu; 2624 else 2625 return prev_cpu; 2626 } else { 2627 s32 cpu; 2628 2629 cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); 2630 if (cpu >= 0) { 2631 refill_task_slice_dfl(sch, p); 2632 p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; 2633 } else { 2634 cpu = prev_cpu; 2635 } 2636 p->scx.selected_cpu = cpu; 2637 2638 if (rq_bypass) 2639 __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); 2640 return cpu; 2641 } 2642 } 2643 2644 static void task_woken_scx(struct rq *rq, struct task_struct *p) 2645 { 2646 run_deferred(rq); 2647 } 2648 2649 static void set_cpus_allowed_scx(struct task_struct *p, 2650 struct affinity_context *ac) 2651 { 2652 struct scx_sched *sch = scx_root; 2653 2654 set_cpus_allowed_common(p, ac); 2655 2656 if (task_dead_and_done(p)) 2657 return; 2658 2659 /* 2660 * The effective cpumask is stored in @p->cpus_ptr which may temporarily 2661 * differ from the configured one in @p->cpus_mask. Always tell the bpf 2662 * scheduler the effective one. 2663 * 2664 * Fine-grained memory write control is enforced by BPF making the const 2665 * designation pointless. Cast it away when calling the operation. 2666 */ 2667 if (SCX_HAS_OP(sch, set_cpumask)) 2668 SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_cpumask, NULL, 2669 p, (struct cpumask *)p->cpus_ptr); 2670 } 2671 2672 static void handle_hotplug(struct rq *rq, bool online) 2673 { 2674 struct scx_sched *sch = scx_root; 2675 int cpu = cpu_of(rq); 2676 2677 atomic_long_inc(&scx_hotplug_seq); 2678 2679 /* 2680 * scx_root updates are protected by cpus_read_lock() and will stay 2681 * stable here. Note that we can't depend on scx_enabled() test as the 2682 * hotplug ops need to be enabled before __scx_enabled is set. 2683 */ 2684 if (unlikely(!sch)) 2685 return; 2686 2687 if (scx_enabled()) 2688 scx_idle_update_selcpu_topology(&sch->ops); 2689 2690 if (online && SCX_HAS_OP(sch, cpu_online)) 2691 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cpu_online, NULL, cpu); 2692 else if (!online && SCX_HAS_OP(sch, cpu_offline)) 2693 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cpu_offline, NULL, cpu); 2694 else 2695 scx_exit(sch, SCX_EXIT_UNREG_KERN, 2696 SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, 2697 "cpu %d going %s, exiting scheduler", cpu, 2698 online ? "online" : "offline"); 2699 } 2700 2701 void scx_rq_activate(struct rq *rq) 2702 { 2703 handle_hotplug(rq, true); 2704 } 2705 2706 void scx_rq_deactivate(struct rq *rq) 2707 { 2708 handle_hotplug(rq, false); 2709 } 2710 2711 static void rq_online_scx(struct rq *rq) 2712 { 2713 rq->scx.flags |= SCX_RQ_ONLINE; 2714 } 2715 2716 static void rq_offline_scx(struct rq *rq) 2717 { 2718 rq->scx.flags &= ~SCX_RQ_ONLINE; 2719 } 2720 2721 2722 static bool check_rq_for_timeouts(struct rq *rq) 2723 { 2724 struct scx_sched *sch; 2725 struct task_struct *p; 2726 struct rq_flags rf; 2727 bool timed_out = false; 2728 2729 rq_lock_irqsave(rq, &rf); 2730 sch = rcu_dereference_bh(scx_root); 2731 if (unlikely(!sch)) 2732 goto out_unlock; 2733 2734 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { 2735 unsigned long last_runnable = p->scx.runnable_at; 2736 2737 if (unlikely(time_after(jiffies, 2738 last_runnable + scx_watchdog_timeout))) { 2739 u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); 2740 2741 scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, 2742 "%s[%d] failed to run for %u.%03us", 2743 p->comm, p->pid, dur_ms / 1000, dur_ms % 1000); 2744 timed_out = true; 2745 break; 2746 } 2747 } 2748 out_unlock: 2749 rq_unlock_irqrestore(rq, &rf); 2750 return timed_out; 2751 } 2752 2753 static void scx_watchdog_workfn(struct work_struct *work) 2754 { 2755 int cpu; 2756 2757 WRITE_ONCE(scx_watchdog_timestamp, jiffies); 2758 2759 for_each_online_cpu(cpu) { 2760 if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) 2761 break; 2762 2763 cond_resched(); 2764 } 2765 queue_delayed_work(system_unbound_wq, to_delayed_work(work), 2766 scx_watchdog_timeout / 2); 2767 } 2768 2769 void scx_tick(struct rq *rq) 2770 { 2771 struct scx_sched *sch; 2772 unsigned long last_check; 2773 2774 if (!scx_enabled()) 2775 return; 2776 2777 sch = rcu_dereference_bh(scx_root); 2778 if (unlikely(!sch)) 2779 return; 2780 2781 last_check = READ_ONCE(scx_watchdog_timestamp); 2782 if (unlikely(time_after(jiffies, 2783 last_check + READ_ONCE(scx_watchdog_timeout)))) { 2784 u32 dur_ms = jiffies_to_msecs(jiffies - last_check); 2785 2786 scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, 2787 "watchdog failed to check in for %u.%03us", 2788 dur_ms / 1000, dur_ms % 1000); 2789 } 2790 2791 update_other_load_avgs(rq); 2792 } 2793 2794 static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) 2795 { 2796 struct scx_sched *sch = scx_root; 2797 2798 update_curr_scx(rq); 2799 2800 /* 2801 * While disabling, always resched and refresh core-sched timestamp as 2802 * we can't trust the slice management or ops.core_sched_before(). 2803 */ 2804 if (scx_rq_bypassing(rq)) { 2805 curr->scx.slice = 0; 2806 touch_core_sched(rq, curr); 2807 } else if (SCX_HAS_OP(sch, tick)) { 2808 SCX_CALL_OP_TASK(sch, SCX_KF_REST, tick, rq, curr); 2809 } 2810 2811 if (!curr->scx.slice) 2812 resched_curr(rq); 2813 } 2814 2815 #ifdef CONFIG_EXT_GROUP_SCHED 2816 static struct cgroup *tg_cgrp(struct task_group *tg) 2817 { 2818 /* 2819 * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup, 2820 * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the 2821 * root cgroup. 2822 */ 2823 if (tg && tg->css.cgroup) 2824 return tg->css.cgroup; 2825 else 2826 return &cgrp_dfl_root.cgrp; 2827 } 2828 2829 #define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg), 2830 2831 #else /* CONFIG_EXT_GROUP_SCHED */ 2832 2833 #define SCX_INIT_TASK_ARGS_CGROUP(tg) 2834 2835 #endif /* CONFIG_EXT_GROUP_SCHED */ 2836 2837 static enum scx_task_state scx_get_task_state(const struct task_struct *p) 2838 { 2839 return (p->scx.flags & SCX_TASK_STATE_MASK) >> SCX_TASK_STATE_SHIFT; 2840 } 2841 2842 static void scx_set_task_state(struct task_struct *p, enum scx_task_state state) 2843 { 2844 enum scx_task_state prev_state = scx_get_task_state(p); 2845 bool warn = false; 2846 2847 BUILD_BUG_ON(SCX_TASK_NR_STATES > (1 << SCX_TASK_STATE_BITS)); 2848 2849 switch (state) { 2850 case SCX_TASK_NONE: 2851 break; 2852 case SCX_TASK_INIT: 2853 warn = prev_state != SCX_TASK_NONE; 2854 break; 2855 case SCX_TASK_READY: 2856 warn = prev_state == SCX_TASK_NONE; 2857 break; 2858 case SCX_TASK_ENABLED: 2859 warn = prev_state != SCX_TASK_READY; 2860 break; 2861 default: 2862 warn = true; 2863 return; 2864 } 2865 2866 WARN_ONCE(warn, "sched_ext: Invalid task state transition %d -> %d for %s[%d]", 2867 prev_state, state, p->comm, p->pid); 2868 2869 p->scx.flags &= ~SCX_TASK_STATE_MASK; 2870 p->scx.flags |= state << SCX_TASK_STATE_SHIFT; 2871 } 2872 2873 static int scx_init_task(struct task_struct *p, struct task_group *tg, bool fork) 2874 { 2875 struct scx_sched *sch = scx_root; 2876 int ret; 2877 2878 p->scx.disallow = false; 2879 2880 if (SCX_HAS_OP(sch, init_task)) { 2881 struct scx_init_task_args args = { 2882 SCX_INIT_TASK_ARGS_CGROUP(tg) 2883 .fork = fork, 2884 }; 2885 2886 ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, init_task, NULL, 2887 p, &args); 2888 if (unlikely(ret)) { 2889 ret = ops_sanitize_err(sch, "init_task", ret); 2890 return ret; 2891 } 2892 } 2893 2894 scx_set_task_state(p, SCX_TASK_INIT); 2895 2896 if (p->scx.disallow) { 2897 if (!fork) { 2898 struct rq *rq; 2899 struct rq_flags rf; 2900 2901 rq = task_rq_lock(p, &rf); 2902 2903 /* 2904 * We're in the load path and @p->policy will be applied 2905 * right after. Reverting @p->policy here and rejecting 2906 * %SCHED_EXT transitions from scx_check_setscheduler() 2907 * guarantees that if ops.init_task() sets @p->disallow, 2908 * @p can never be in SCX. 2909 */ 2910 if (p->policy == SCHED_EXT) { 2911 p->policy = SCHED_NORMAL; 2912 atomic_long_inc(&scx_nr_rejected); 2913 } 2914 2915 task_rq_unlock(rq, p, &rf); 2916 } else if (p->policy == SCHED_EXT) { 2917 scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", 2918 p->comm, p->pid); 2919 } 2920 } 2921 2922 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; 2923 return 0; 2924 } 2925 2926 static void scx_enable_task(struct task_struct *p) 2927 { 2928 struct scx_sched *sch = scx_root; 2929 struct rq *rq = task_rq(p); 2930 u32 weight; 2931 2932 lockdep_assert_rq_held(rq); 2933 2934 /* 2935 * Set the weight before calling ops.enable() so that the scheduler 2936 * doesn't see a stale value if they inspect the task struct. 2937 */ 2938 if (task_has_idle_policy(p)) 2939 weight = WEIGHT_IDLEPRIO; 2940 else 2941 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; 2942 2943 p->scx.weight = sched_weight_to_cgroup(weight); 2944 2945 if (SCX_HAS_OP(sch, enable)) 2946 SCX_CALL_OP_TASK(sch, SCX_KF_REST, enable, rq, p); 2947 scx_set_task_state(p, SCX_TASK_ENABLED); 2948 2949 if (SCX_HAS_OP(sch, set_weight)) 2950 SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_weight, rq, 2951 p, p->scx.weight); 2952 } 2953 2954 static void scx_disable_task(struct task_struct *p) 2955 { 2956 struct scx_sched *sch = scx_root; 2957 struct rq *rq = task_rq(p); 2958 2959 lockdep_assert_rq_held(rq); 2960 WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); 2961 2962 if (SCX_HAS_OP(sch, disable)) 2963 SCX_CALL_OP_TASK(sch, SCX_KF_REST, disable, rq, p); 2964 scx_set_task_state(p, SCX_TASK_READY); 2965 } 2966 2967 static void scx_exit_task(struct task_struct *p) 2968 { 2969 struct scx_sched *sch = scx_root; 2970 struct scx_exit_task_args args = { 2971 .cancelled = false, 2972 }; 2973 2974 lockdep_assert_rq_held(task_rq(p)); 2975 2976 switch (scx_get_task_state(p)) { 2977 case SCX_TASK_NONE: 2978 return; 2979 case SCX_TASK_INIT: 2980 args.cancelled = true; 2981 break; 2982 case SCX_TASK_READY: 2983 break; 2984 case SCX_TASK_ENABLED: 2985 scx_disable_task(p); 2986 break; 2987 default: 2988 WARN_ON_ONCE(true); 2989 return; 2990 } 2991 2992 if (SCX_HAS_OP(sch, exit_task)) 2993 SCX_CALL_OP_TASK(sch, SCX_KF_REST, exit_task, task_rq(p), 2994 p, &args); 2995 scx_set_task_state(p, SCX_TASK_NONE); 2996 } 2997 2998 void init_scx_entity(struct sched_ext_entity *scx) 2999 { 3000 memset(scx, 0, sizeof(*scx)); 3001 INIT_LIST_HEAD(&scx->dsq_list.node); 3002 RB_CLEAR_NODE(&scx->dsq_priq); 3003 scx->sticky_cpu = -1; 3004 scx->holding_cpu = -1; 3005 INIT_LIST_HEAD(&scx->runnable_node); 3006 scx->runnable_at = jiffies; 3007 scx->ddsp_dsq_id = SCX_DSQ_INVALID; 3008 scx->slice = READ_ONCE(scx_slice_dfl); 3009 } 3010 3011 void scx_pre_fork(struct task_struct *p) 3012 { 3013 /* 3014 * BPF scheduler enable/disable paths want to be able to iterate and 3015 * update all tasks which can become complex when racing forks. As 3016 * enable/disable are very cold paths, let's use a percpu_rwsem to 3017 * exclude forks. 3018 */ 3019 percpu_down_read(&scx_fork_rwsem); 3020 } 3021 3022 int scx_fork(struct task_struct *p) 3023 { 3024 percpu_rwsem_assert_held(&scx_fork_rwsem); 3025 3026 if (scx_init_task_enabled) 3027 return scx_init_task(p, task_group(p), true); 3028 else 3029 return 0; 3030 } 3031 3032 void scx_post_fork(struct task_struct *p) 3033 { 3034 if (scx_init_task_enabled) { 3035 scx_set_task_state(p, SCX_TASK_READY); 3036 3037 /* 3038 * Enable the task immediately if it's running on sched_ext. 3039 * Otherwise, it'll be enabled in switching_to_scx() if and 3040 * when it's ever configured to run with a SCHED_EXT policy. 3041 */ 3042 if (p->sched_class == &ext_sched_class) { 3043 struct rq_flags rf; 3044 struct rq *rq; 3045 3046 rq = task_rq_lock(p, &rf); 3047 scx_enable_task(p); 3048 task_rq_unlock(rq, p, &rf); 3049 } 3050 } 3051 3052 raw_spin_lock_irq(&scx_tasks_lock); 3053 list_add_tail(&p->scx.tasks_node, &scx_tasks); 3054 raw_spin_unlock_irq(&scx_tasks_lock); 3055 3056 percpu_up_read(&scx_fork_rwsem); 3057 } 3058 3059 void scx_cancel_fork(struct task_struct *p) 3060 { 3061 if (scx_enabled()) { 3062 struct rq *rq; 3063 struct rq_flags rf; 3064 3065 rq = task_rq_lock(p, &rf); 3066 WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY); 3067 scx_exit_task(p); 3068 task_rq_unlock(rq, p, &rf); 3069 } 3070 3071 percpu_up_read(&scx_fork_rwsem); 3072 } 3073 3074 /** 3075 * task_dead_and_done - Is a task dead and done running? 3076 * @p: target task 3077 * 3078 * Once sched_ext_dead() removes the dead task from scx_tasks and exits it, the 3079 * task no longer exists from SCX's POV. However, certain sched_class ops may be 3080 * invoked on these dead tasks leading to failures - e.g. sched_setscheduler() 3081 * may try to switch a task which finished sched_ext_dead() back into SCX 3082 * triggering invalid SCX task state transitions and worse. 3083 * 3084 * Once a task has finished the final switch, sched_ext_dead() is the only thing 3085 * that needs to happen on the task. Use this test to short-circuit sched_class 3086 * operations which may be called on dead tasks. 3087 */ 3088 static bool task_dead_and_done(struct task_struct *p) 3089 { 3090 struct rq *rq = task_rq(p); 3091 3092 lockdep_assert_rq_held(rq); 3093 3094 /* 3095 * In do_task_dead(), a dying task sets %TASK_DEAD with preemption 3096 * disabled and __schedule(). If @p has %TASK_DEAD set and off CPU, @p 3097 * won't ever run again. 3098 */ 3099 return unlikely(READ_ONCE(p->__state) == TASK_DEAD) && 3100 !task_on_cpu(rq, p); 3101 } 3102 3103 void sched_ext_dead(struct task_struct *p) 3104 { 3105 unsigned long flags; 3106 3107 /* 3108 * By the time control reaches here, @p has %TASK_DEAD set, switched out 3109 * for the last time and then dropped the rq lock - task_dead_and_done() 3110 * should be returning %true nullifying the straggling sched_class ops. 3111 * Remove from scx_tasks and exit @p. 3112 */ 3113 raw_spin_lock_irqsave(&scx_tasks_lock, flags); 3114 list_del_init(&p->scx.tasks_node); 3115 raw_spin_unlock_irqrestore(&scx_tasks_lock, flags); 3116 3117 /* 3118 * @p is off scx_tasks and wholly ours. scx_enable()'s READY -> ENABLED 3119 * transitions can't race us. Disable ops for @p. 3120 */ 3121 if (scx_get_task_state(p) != SCX_TASK_NONE) { 3122 struct rq_flags rf; 3123 struct rq *rq; 3124 3125 rq = task_rq_lock(p, &rf); 3126 scx_exit_task(p); 3127 task_rq_unlock(rq, p, &rf); 3128 } 3129 } 3130 3131 static void reweight_task_scx(struct rq *rq, struct task_struct *p, 3132 const struct load_weight *lw) 3133 { 3134 struct scx_sched *sch = scx_root; 3135 3136 lockdep_assert_rq_held(task_rq(p)); 3137 3138 if (task_dead_and_done(p)) 3139 return; 3140 3141 p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); 3142 if (SCX_HAS_OP(sch, set_weight)) 3143 SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_weight, rq, 3144 p, p->scx.weight); 3145 } 3146 3147 static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio) 3148 { 3149 } 3150 3151 static void switching_to_scx(struct rq *rq, struct task_struct *p) 3152 { 3153 struct scx_sched *sch = scx_root; 3154 3155 if (task_dead_and_done(p)) 3156 return; 3157 3158 scx_enable_task(p); 3159 3160 /* 3161 * set_cpus_allowed_scx() is not called while @p is associated with a 3162 * different scheduler class. Keep the BPF scheduler up-to-date. 3163 */ 3164 if (SCX_HAS_OP(sch, set_cpumask)) 3165 SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_cpumask, rq, 3166 p, (struct cpumask *)p->cpus_ptr); 3167 } 3168 3169 static void switched_from_scx(struct rq *rq, struct task_struct *p) 3170 { 3171 if (task_dead_and_done(p)) 3172 return; 3173 3174 scx_disable_task(p); 3175 } 3176 3177 static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags) {} 3178 3179 static void switched_to_scx(struct rq *rq, struct task_struct *p) {} 3180 3181 int scx_check_setscheduler(struct task_struct *p, int policy) 3182 { 3183 lockdep_assert_rq_held(task_rq(p)); 3184 3185 /* if disallow, reject transitioning into SCX */ 3186 if (scx_enabled() && READ_ONCE(p->scx.disallow) && 3187 p->policy != policy && policy == SCHED_EXT) 3188 return -EACCES; 3189 3190 return 0; 3191 } 3192 3193 #ifdef CONFIG_NO_HZ_FULL 3194 bool scx_can_stop_tick(struct rq *rq) 3195 { 3196 struct task_struct *p = rq->curr; 3197 3198 if (scx_rq_bypassing(rq)) 3199 return false; 3200 3201 if (p->sched_class != &ext_sched_class) 3202 return true; 3203 3204 /* 3205 * @rq can dispatch from different DSQs, so we can't tell whether it 3206 * needs the tick or not by looking at nr_running. Allow stopping ticks 3207 * iff the BPF scheduler indicated so. See set_next_task_scx(). 3208 */ 3209 return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; 3210 } 3211 #endif 3212 3213 #ifdef CONFIG_EXT_GROUP_SCHED 3214 3215 DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem); 3216 static bool scx_cgroup_enabled; 3217 3218 void scx_tg_init(struct task_group *tg) 3219 { 3220 tg->scx.weight = CGROUP_WEIGHT_DFL; 3221 tg->scx.bw_period_us = default_bw_period_us(); 3222 tg->scx.bw_quota_us = RUNTIME_INF; 3223 tg->scx.idle = false; 3224 } 3225 3226 int scx_tg_online(struct task_group *tg) 3227 { 3228 struct scx_sched *sch = scx_root; 3229 int ret = 0; 3230 3231 WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); 3232 3233 if (scx_cgroup_enabled) { 3234 if (SCX_HAS_OP(sch, cgroup_init)) { 3235 struct scx_cgroup_init_args args = 3236 { .weight = tg->scx.weight, 3237 .bw_period_us = tg->scx.bw_period_us, 3238 .bw_quota_us = tg->scx.bw_quota_us, 3239 .bw_burst_us = tg->scx.bw_burst_us }; 3240 3241 ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, cgroup_init, 3242 NULL, tg->css.cgroup, &args); 3243 if (ret) 3244 ret = ops_sanitize_err(sch, "cgroup_init", ret); 3245 } 3246 if (ret == 0) 3247 tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; 3248 } else { 3249 tg->scx.flags |= SCX_TG_ONLINE; 3250 } 3251 3252 return ret; 3253 } 3254 3255 void scx_tg_offline(struct task_group *tg) 3256 { 3257 struct scx_sched *sch = scx_root; 3258 3259 WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); 3260 3261 if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && 3262 (tg->scx.flags & SCX_TG_INITED)) 3263 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_exit, NULL, 3264 tg->css.cgroup); 3265 tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); 3266 } 3267 3268 int scx_cgroup_can_attach(struct cgroup_taskset *tset) 3269 { 3270 struct scx_sched *sch = scx_root; 3271 struct cgroup_subsys_state *css; 3272 struct task_struct *p; 3273 int ret; 3274 3275 if (!scx_cgroup_enabled) 3276 return 0; 3277 3278 cgroup_taskset_for_each(p, css, tset) { 3279 struct cgroup *from = tg_cgrp(task_group(p)); 3280 struct cgroup *to = tg_cgrp(css_tg(css)); 3281 3282 WARN_ON_ONCE(p->scx.cgrp_moving_from); 3283 3284 /* 3285 * sched_move_task() omits identity migrations. Let's match the 3286 * behavior so that ops.cgroup_prep_move() and ops.cgroup_move() 3287 * always match one-to-one. 3288 */ 3289 if (from == to) 3290 continue; 3291 3292 if (SCX_HAS_OP(sch, cgroup_prep_move)) { 3293 ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, 3294 cgroup_prep_move, NULL, 3295 p, from, css->cgroup); 3296 if (ret) 3297 goto err; 3298 } 3299 3300 p->scx.cgrp_moving_from = from; 3301 } 3302 3303 return 0; 3304 3305 err: 3306 cgroup_taskset_for_each(p, css, tset) { 3307 if (SCX_HAS_OP(sch, cgroup_cancel_move) && 3308 p->scx.cgrp_moving_from) 3309 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_cancel_move, NULL, 3310 p, p->scx.cgrp_moving_from, css->cgroup); 3311 p->scx.cgrp_moving_from = NULL; 3312 } 3313 3314 return ops_sanitize_err(sch, "cgroup_prep_move", ret); 3315 } 3316 3317 void scx_cgroup_move_task(struct task_struct *p) 3318 { 3319 struct scx_sched *sch = scx_root; 3320 3321 if (!scx_cgroup_enabled) 3322 return; 3323 3324 /* 3325 * @p must have ops.cgroup_prep_move() called on it and thus 3326 * cgrp_moving_from set. 3327 */ 3328 if (SCX_HAS_OP(sch, cgroup_move) && 3329 !WARN_ON_ONCE(!p->scx.cgrp_moving_from)) 3330 SCX_CALL_OP_TASK(sch, SCX_KF_UNLOCKED, cgroup_move, NULL, 3331 p, p->scx.cgrp_moving_from, 3332 tg_cgrp(task_group(p))); 3333 p->scx.cgrp_moving_from = NULL; 3334 } 3335 3336 void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) 3337 { 3338 struct scx_sched *sch = scx_root; 3339 struct cgroup_subsys_state *css; 3340 struct task_struct *p; 3341 3342 if (!scx_cgroup_enabled) 3343 return; 3344 3345 cgroup_taskset_for_each(p, css, tset) { 3346 if (SCX_HAS_OP(sch, cgroup_cancel_move) && 3347 p->scx.cgrp_moving_from) 3348 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_cancel_move, NULL, 3349 p, p->scx.cgrp_moving_from, css->cgroup); 3350 p->scx.cgrp_moving_from = NULL; 3351 } 3352 } 3353 3354 void scx_group_set_weight(struct task_group *tg, unsigned long weight) 3355 { 3356 struct scx_sched *sch = scx_root; 3357 3358 percpu_down_read(&scx_cgroup_ops_rwsem); 3359 3360 if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && 3361 tg->scx.weight != weight) 3362 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_set_weight, NULL, 3363 tg_cgrp(tg), weight); 3364 3365 tg->scx.weight = weight; 3366 3367 percpu_up_read(&scx_cgroup_ops_rwsem); 3368 } 3369 3370 void scx_group_set_idle(struct task_group *tg, bool idle) 3371 { 3372 struct scx_sched *sch = scx_root; 3373 3374 percpu_down_read(&scx_cgroup_ops_rwsem); 3375 3376 if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) 3377 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_set_idle, NULL, 3378 tg_cgrp(tg), idle); 3379 3380 /* Update the task group's idle state */ 3381 tg->scx.idle = idle; 3382 3383 percpu_up_read(&scx_cgroup_ops_rwsem); 3384 } 3385 3386 void scx_group_set_bandwidth(struct task_group *tg, 3387 u64 period_us, u64 quota_us, u64 burst_us) 3388 { 3389 struct scx_sched *sch = scx_root; 3390 3391 percpu_down_read(&scx_cgroup_ops_rwsem); 3392 3393 if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && 3394 (tg->scx.bw_period_us != period_us || 3395 tg->scx.bw_quota_us != quota_us || 3396 tg->scx.bw_burst_us != burst_us)) 3397 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_set_bandwidth, NULL, 3398 tg_cgrp(tg), period_us, quota_us, burst_us); 3399 3400 tg->scx.bw_period_us = period_us; 3401 tg->scx.bw_quota_us = quota_us; 3402 tg->scx.bw_burst_us = burst_us; 3403 3404 percpu_up_read(&scx_cgroup_ops_rwsem); 3405 } 3406 3407 static void scx_cgroup_lock(void) 3408 { 3409 percpu_down_write(&scx_cgroup_ops_rwsem); 3410 cgroup_lock(); 3411 } 3412 3413 static void scx_cgroup_unlock(void) 3414 { 3415 cgroup_unlock(); 3416 percpu_up_write(&scx_cgroup_ops_rwsem); 3417 } 3418 3419 #else /* CONFIG_EXT_GROUP_SCHED */ 3420 3421 static void scx_cgroup_lock(void) {} 3422 static void scx_cgroup_unlock(void) {} 3423 3424 #endif /* CONFIG_EXT_GROUP_SCHED */ 3425 3426 /* 3427 * Omitted operations: 3428 * 3429 * - wakeup_preempt: NOOP as it isn't useful in the wakeup path because the task 3430 * isn't tied to the CPU at that point. Preemption is implemented by resetting 3431 * the victim task's slice to 0 and triggering reschedule on the target CPU. 3432 * 3433 * - migrate_task_rq: Unnecessary as task to cpu mapping is transient. 3434 * 3435 * - task_fork/dead: We need fork/dead notifications for all tasks regardless of 3436 * their current sched_class. Call them directly from sched core instead. 3437 */ 3438 DEFINE_SCHED_CLASS(ext) = { 3439 .enqueue_task = enqueue_task_scx, 3440 .dequeue_task = dequeue_task_scx, 3441 .yield_task = yield_task_scx, 3442 .yield_to_task = yield_to_task_scx, 3443 3444 .wakeup_preempt = wakeup_preempt_scx, 3445 3446 .pick_task = pick_task_scx, 3447 3448 .put_prev_task = put_prev_task_scx, 3449 .set_next_task = set_next_task_scx, 3450 3451 .select_task_rq = select_task_rq_scx, 3452 .task_woken = task_woken_scx, 3453 .set_cpus_allowed = set_cpus_allowed_scx, 3454 3455 .rq_online = rq_online_scx, 3456 .rq_offline = rq_offline_scx, 3457 3458 .task_tick = task_tick_scx, 3459 3460 .switching_to = switching_to_scx, 3461 .switched_from = switched_from_scx, 3462 .switched_to = switched_to_scx, 3463 .reweight_task = reweight_task_scx, 3464 .prio_changed = prio_changed_scx, 3465 3466 .update_curr = update_curr_scx, 3467 3468 #ifdef CONFIG_UCLAMP_TASK 3469 .uclamp_enabled = 1, 3470 #endif 3471 }; 3472 3473 static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) 3474 { 3475 memset(dsq, 0, sizeof(*dsq)); 3476 3477 raw_spin_lock_init(&dsq->lock); 3478 INIT_LIST_HEAD(&dsq->list); 3479 dsq->id = dsq_id; 3480 } 3481 3482 static void free_dsq_irq_workfn(struct irq_work *irq_work) 3483 { 3484 struct llist_node *to_free = llist_del_all(&dsqs_to_free); 3485 struct scx_dispatch_q *dsq, *tmp_dsq; 3486 3487 llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) 3488 kfree_rcu(dsq, rcu); 3489 } 3490 3491 static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); 3492 3493 static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) 3494 { 3495 struct scx_dispatch_q *dsq; 3496 unsigned long flags; 3497 3498 rcu_read_lock(); 3499 3500 dsq = find_user_dsq(sch, dsq_id); 3501 if (!dsq) 3502 goto out_unlock_rcu; 3503 3504 raw_spin_lock_irqsave(&dsq->lock, flags); 3505 3506 if (dsq->nr) { 3507 scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)", 3508 dsq->id, dsq->nr); 3509 goto out_unlock_dsq; 3510 } 3511 3512 if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node, 3513 dsq_hash_params)) 3514 goto out_unlock_dsq; 3515 3516 /* 3517 * Mark dead by invalidating ->id to prevent dispatch_enqueue() from 3518 * queueing more tasks. As this function can be called from anywhere, 3519 * freeing is bounced through an irq work to avoid nesting RCU 3520 * operations inside scheduler locks. 3521 */ 3522 dsq->id = SCX_DSQ_INVALID; 3523 llist_add(&dsq->free_node, &dsqs_to_free); 3524 irq_work_queue(&free_dsq_irq_work); 3525 3526 out_unlock_dsq: 3527 raw_spin_unlock_irqrestore(&dsq->lock, flags); 3528 out_unlock_rcu: 3529 rcu_read_unlock(); 3530 } 3531 3532 #ifdef CONFIG_EXT_GROUP_SCHED 3533 static void scx_cgroup_exit(struct scx_sched *sch) 3534 { 3535 struct cgroup_subsys_state *css; 3536 3537 scx_cgroup_enabled = false; 3538 3539 /* 3540 * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk 3541 * cgroups and exit all the inited ones, all online cgroups are exited. 3542 */ 3543 css_for_each_descendant_post(css, &root_task_group.css) { 3544 struct task_group *tg = css_tg(css); 3545 3546 if (!(tg->scx.flags & SCX_TG_INITED)) 3547 continue; 3548 tg->scx.flags &= ~SCX_TG_INITED; 3549 3550 if (!sch->ops.cgroup_exit) 3551 continue; 3552 3553 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_exit, NULL, 3554 css->cgroup); 3555 } 3556 } 3557 3558 static int scx_cgroup_init(struct scx_sched *sch) 3559 { 3560 struct cgroup_subsys_state *css; 3561 int ret; 3562 3563 /* 3564 * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk 3565 * cgroups and init, all online cgroups are initialized. 3566 */ 3567 css_for_each_descendant_pre(css, &root_task_group.css) { 3568 struct task_group *tg = css_tg(css); 3569 struct scx_cgroup_init_args args = { 3570 .weight = tg->scx.weight, 3571 .bw_period_us = tg->scx.bw_period_us, 3572 .bw_quota_us = tg->scx.bw_quota_us, 3573 .bw_burst_us = tg->scx.bw_burst_us, 3574 }; 3575 3576 if ((tg->scx.flags & 3577 (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) 3578 continue; 3579 3580 if (!sch->ops.cgroup_init) { 3581 tg->scx.flags |= SCX_TG_INITED; 3582 continue; 3583 } 3584 3585 ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, cgroup_init, NULL, 3586 css->cgroup, &args); 3587 if (ret) { 3588 css_put(css); 3589 scx_error(sch, "ops.cgroup_init() failed (%d)", ret); 3590 return ret; 3591 } 3592 tg->scx.flags |= SCX_TG_INITED; 3593 } 3594 3595 WARN_ON_ONCE(scx_cgroup_enabled); 3596 scx_cgroup_enabled = true; 3597 3598 return 0; 3599 } 3600 3601 #else 3602 static void scx_cgroup_exit(struct scx_sched *sch) {} 3603 static int scx_cgroup_init(struct scx_sched *sch) { return 0; } 3604 #endif 3605 3606 3607 /******************************************************************************** 3608 * Sysfs interface and ops enable/disable. 3609 */ 3610 3611 #define SCX_ATTR(_name) \ 3612 static struct kobj_attribute scx_attr_##_name = { \ 3613 .attr = { .name = __stringify(_name), .mode = 0444 }, \ 3614 .show = scx_attr_##_name##_show, \ 3615 } 3616 3617 static ssize_t scx_attr_state_show(struct kobject *kobj, 3618 struct kobj_attribute *ka, char *buf) 3619 { 3620 return sysfs_emit(buf, "%s\n", scx_enable_state_str[scx_enable_state()]); 3621 } 3622 SCX_ATTR(state); 3623 3624 static ssize_t scx_attr_switch_all_show(struct kobject *kobj, 3625 struct kobj_attribute *ka, char *buf) 3626 { 3627 return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all)); 3628 } 3629 SCX_ATTR(switch_all); 3630 3631 static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj, 3632 struct kobj_attribute *ka, char *buf) 3633 { 3634 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected)); 3635 } 3636 SCX_ATTR(nr_rejected); 3637 3638 static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj, 3639 struct kobj_attribute *ka, char *buf) 3640 { 3641 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq)); 3642 } 3643 SCX_ATTR(hotplug_seq); 3644 3645 static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, 3646 struct kobj_attribute *ka, char *buf) 3647 { 3648 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq)); 3649 } 3650 SCX_ATTR(enable_seq); 3651 3652 static struct attribute *scx_global_attrs[] = { 3653 &scx_attr_state.attr, 3654 &scx_attr_switch_all.attr, 3655 &scx_attr_nr_rejected.attr, 3656 &scx_attr_hotplug_seq.attr, 3657 &scx_attr_enable_seq.attr, 3658 NULL, 3659 }; 3660 3661 static const struct attribute_group scx_global_attr_group = { 3662 .attrs = scx_global_attrs, 3663 }; 3664 3665 static void free_exit_info(struct scx_exit_info *ei); 3666 3667 static void scx_sched_free_rcu_work(struct work_struct *work) 3668 { 3669 struct rcu_work *rcu_work = to_rcu_work(work); 3670 struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); 3671 struct rhashtable_iter rht_iter; 3672 struct scx_dispatch_q *dsq; 3673 int node; 3674 3675 irq_work_sync(&sch->error_irq_work); 3676 kthread_destroy_worker(sch->helper); 3677 3678 free_percpu(sch->pcpu); 3679 3680 for_each_node_state(node, N_POSSIBLE) 3681 kfree(sch->global_dsqs[node]); 3682 kfree(sch->global_dsqs); 3683 3684 rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); 3685 do { 3686 rhashtable_walk_start(&rht_iter); 3687 3688 while ((dsq = rhashtable_walk_next(&rht_iter)) && !IS_ERR(dsq)) 3689 destroy_dsq(sch, dsq->id); 3690 3691 rhashtable_walk_stop(&rht_iter); 3692 } while (dsq == ERR_PTR(-EAGAIN)); 3693 rhashtable_walk_exit(&rht_iter); 3694 3695 rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); 3696 free_exit_info(sch->exit_info); 3697 kfree(sch); 3698 } 3699 3700 static void scx_kobj_release(struct kobject *kobj) 3701 { 3702 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); 3703 3704 INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); 3705 queue_rcu_work(system_unbound_wq, &sch->rcu_work); 3706 } 3707 3708 static ssize_t scx_attr_ops_show(struct kobject *kobj, 3709 struct kobj_attribute *ka, char *buf) 3710 { 3711 return sysfs_emit(buf, "%s\n", scx_root->ops.name); 3712 } 3713 SCX_ATTR(ops); 3714 3715 #define scx_attr_event_show(buf, at, events, kind) ({ \ 3716 sysfs_emit_at(buf, at, "%s %llu\n", #kind, (events)->kind); \ 3717 }) 3718 3719 static ssize_t scx_attr_events_show(struct kobject *kobj, 3720 struct kobj_attribute *ka, char *buf) 3721 { 3722 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); 3723 struct scx_event_stats events; 3724 int at = 0; 3725 3726 scx_read_events(sch, &events); 3727 at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK); 3728 at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); 3729 at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST); 3730 at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING); 3731 at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); 3732 at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL); 3733 at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION); 3734 at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH); 3735 at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE); 3736 return at; 3737 } 3738 SCX_ATTR(events); 3739 3740 static struct attribute *scx_sched_attrs[] = { 3741 &scx_attr_ops.attr, 3742 &scx_attr_events.attr, 3743 NULL, 3744 }; 3745 ATTRIBUTE_GROUPS(scx_sched); 3746 3747 static const struct kobj_type scx_ktype = { 3748 .release = scx_kobj_release, 3749 .sysfs_ops = &kobj_sysfs_ops, 3750 .default_groups = scx_sched_groups, 3751 }; 3752 3753 static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) 3754 { 3755 return add_uevent_var(env, "SCXOPS=%s", scx_root->ops.name); 3756 } 3757 3758 static const struct kset_uevent_ops scx_uevent_ops = { 3759 .uevent = scx_uevent, 3760 }; 3761 3762 /* 3763 * Used by sched_fork() and __setscheduler_prio() to pick the matching 3764 * sched_class. dl/rt are already handled. 3765 */ 3766 bool task_should_scx(int policy) 3767 { 3768 if (!scx_enabled() || unlikely(scx_enable_state() == SCX_DISABLING)) 3769 return false; 3770 if (READ_ONCE(scx_switching_all)) 3771 return true; 3772 return policy == SCHED_EXT; 3773 } 3774 3775 bool scx_allow_ttwu_queue(const struct task_struct *p) 3776 { 3777 struct scx_sched *sch; 3778 3779 if (!scx_enabled()) 3780 return true; 3781 3782 sch = rcu_dereference_sched(scx_root); 3783 if (unlikely(!sch)) 3784 return true; 3785 3786 if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP) 3787 return true; 3788 3789 if (unlikely(p->sched_class != &ext_sched_class)) 3790 return true; 3791 3792 return false; 3793 } 3794 3795 /** 3796 * handle_lockup - sched_ext common lockup handler 3797 * @fmt: format string 3798 * 3799 * Called on system stall or lockup condition and initiates abort of sched_ext 3800 * if enabled, which may resolve the reported lockup. 3801 * 3802 * Returns %true if sched_ext is enabled and abort was initiated, which may 3803 * resolve the lockup. %false if sched_ext is not enabled or abort was already 3804 * initiated by someone else. 3805 */ 3806 static __printf(1, 2) bool handle_lockup(const char *fmt, ...) 3807 { 3808 struct scx_sched *sch; 3809 va_list args; 3810 bool ret; 3811 3812 guard(rcu)(); 3813 3814 sch = rcu_dereference(scx_root); 3815 if (unlikely(!sch)) 3816 return false; 3817 3818 switch (scx_enable_state()) { 3819 case SCX_ENABLING: 3820 case SCX_ENABLED: 3821 va_start(args, fmt); 3822 ret = scx_verror(sch, fmt, args); 3823 va_end(args); 3824 return ret; 3825 default: 3826 return false; 3827 } 3828 } 3829 3830 /** 3831 * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler 3832 * 3833 * While there are various reasons why RCU CPU stalls can occur on a system 3834 * that may not be caused by the current BPF scheduler, try kicking out the 3835 * current scheduler in an attempt to recover the system to a good state before 3836 * issuing panics. 3837 * 3838 * Returns %true if sched_ext is enabled and abort was initiated, which may 3839 * resolve the reported RCU stall. %false if sched_ext is not enabled or someone 3840 * else already initiated abort. 3841 */ 3842 bool scx_rcu_cpu_stall(void) 3843 { 3844 return handle_lockup("RCU CPU stall detected!"); 3845 } 3846 3847 /** 3848 * scx_softlockup - sched_ext softlockup handler 3849 * @dur_s: number of seconds of CPU stuck due to soft lockup 3850 * 3851 * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can 3852 * live-lock the system by making many CPUs target the same DSQ to the point 3853 * where soft-lockup detection triggers. This function is called from 3854 * soft-lockup watchdog when the triggering point is close and tries to unjam 3855 * the system and aborting the BPF scheduler. 3856 */ 3857 void scx_softlockup(u32 dur_s) 3858 { 3859 if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) 3860 return; 3861 3862 printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", 3863 smp_processor_id(), dur_s); 3864 } 3865 3866 /** 3867 * scx_hardlockup - sched_ext hardlockup handler 3868 * 3869 * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting 3870 * numerous affinitized tasks in a single queue and directing all CPUs at it. 3871 * Try kicking out the current scheduler in an attempt to recover the system to 3872 * a good state before taking more drastic actions. 3873 * 3874 * Returns %true if sched_ext is enabled and abort was initiated, which may 3875 * resolve the reported hardlockdup. %false if sched_ext is not enabled or 3876 * someone else already initiated abort. 3877 */ 3878 bool scx_hardlockup(int cpu) 3879 { 3880 if (!handle_lockup("hard lockup - CPU %d", cpu)) 3881 return false; 3882 3883 printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", 3884 cpu); 3885 return true; 3886 } 3887 3888 static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq, 3889 struct cpumask *donee_mask, struct cpumask *resched_mask, 3890 u32 nr_donor_target, u32 nr_donee_target) 3891 { 3892 struct scx_dispatch_q *donor_dsq = &rq->scx.bypass_dsq; 3893 struct task_struct *p, *n; 3894 struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, 0, 0); 3895 s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; 3896 u32 nr_balanced = 0, min_delta_us; 3897 3898 /* 3899 * All we want to guarantee is reasonable forward progress. No reason to 3900 * fine tune. Assuming every task on @donor_dsq runs their full slice, 3901 * consider offloading iff the total queued duration is over the 3902 * threshold. 3903 */ 3904 min_delta_us = scx_bypass_lb_intv_us / SCX_BYPASS_LB_MIN_DELTA_DIV; 3905 if (delta < DIV_ROUND_UP(min_delta_us, scx_slice_bypass_us)) 3906 return 0; 3907 3908 raw_spin_rq_lock_irq(rq); 3909 raw_spin_lock(&donor_dsq->lock); 3910 list_add(&cursor.node, &donor_dsq->list); 3911 resume: 3912 n = container_of(&cursor, struct task_struct, scx.dsq_list); 3913 n = nldsq_next_task(donor_dsq, n, false); 3914 3915 while ((p = n)) { 3916 struct rq *donee_rq; 3917 struct scx_dispatch_q *donee_dsq; 3918 int donee; 3919 3920 n = nldsq_next_task(donor_dsq, n, false); 3921 3922 if (donor_dsq->nr <= nr_donor_target) 3923 break; 3924 3925 if (cpumask_empty(donee_mask)) 3926 break; 3927 3928 donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); 3929 if (donee >= nr_cpu_ids) 3930 continue; 3931 3932 donee_rq = cpu_rq(donee); 3933 donee_dsq = &donee_rq->scx.bypass_dsq; 3934 3935 /* 3936 * $p's rq is not locked but $p's DSQ lock protects its 3937 * scheduling properties making this test safe. 3938 */ 3939 if (!task_can_run_on_remote_rq(sch, p, donee_rq, false)) 3940 continue; 3941 3942 /* 3943 * Moving $p from one non-local DSQ to another. The source rq 3944 * and DSQ are already locked. Do an abbreviated dequeue and 3945 * then perform enqueue without unlocking $donor_dsq. 3946 * 3947 * We don't want to drop and reacquire the lock on each 3948 * iteration as @donor_dsq can be very long and potentially 3949 * highly contended. Donee DSQs are less likely to be contended. 3950 * The nested locking is safe as only this LB moves tasks 3951 * between bypass DSQs. 3952 */ 3953 dispatch_dequeue_locked(p, donor_dsq); 3954 dispatch_enqueue(sch, donee_dsq, p, SCX_ENQ_NESTED); 3955 3956 /* 3957 * $donee might have been idle and need to be woken up. No need 3958 * to be clever. Kick every CPU that receives tasks. 3959 */ 3960 cpumask_set_cpu(donee, resched_mask); 3961 3962 if (READ_ONCE(donee_dsq->nr) >= nr_donee_target) 3963 cpumask_clear_cpu(donee, donee_mask); 3964 3965 nr_balanced++; 3966 if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { 3967 list_move_tail(&cursor.node, &n->scx.dsq_list.node); 3968 raw_spin_unlock(&donor_dsq->lock); 3969 raw_spin_rq_unlock_irq(rq); 3970 cpu_relax(); 3971 raw_spin_rq_lock_irq(rq); 3972 raw_spin_lock(&donor_dsq->lock); 3973 goto resume; 3974 } 3975 } 3976 3977 list_del_init(&cursor.node); 3978 raw_spin_unlock(&donor_dsq->lock); 3979 raw_spin_rq_unlock_irq(rq); 3980 3981 return nr_balanced; 3982 } 3983 3984 static void bypass_lb_node(struct scx_sched *sch, int node) 3985 { 3986 const struct cpumask *node_mask = cpumask_of_node(node); 3987 struct cpumask *donee_mask = scx_bypass_lb_donee_cpumask; 3988 struct cpumask *resched_mask = scx_bypass_lb_resched_cpumask; 3989 u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0; 3990 u32 nr_target, nr_donor_target; 3991 u32 before_min = U32_MAX, before_max = 0; 3992 u32 after_min = U32_MAX, after_max = 0; 3993 int cpu; 3994 3995 /* count the target tasks and CPUs */ 3996 for_each_cpu_and(cpu, cpu_online_mask, node_mask) { 3997 u32 nr = READ_ONCE(cpu_rq(cpu)->scx.bypass_dsq.nr); 3998 3999 nr_tasks += nr; 4000 nr_cpus++; 4001 4002 before_min = min(nr, before_min); 4003 before_max = max(nr, before_max); 4004 } 4005 4006 if (!nr_cpus) 4007 return; 4008 4009 /* 4010 * We don't want CPUs to have more than $nr_donor_target tasks and 4011 * balancing to fill donee CPUs upto $nr_target. Once targets are 4012 * calculated, find the donee CPUs. 4013 */ 4014 nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); 4015 nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100); 4016 4017 cpumask_clear(donee_mask); 4018 for_each_cpu_and(cpu, cpu_online_mask, node_mask) { 4019 if (READ_ONCE(cpu_rq(cpu)->scx.bypass_dsq.nr) < nr_target) 4020 cpumask_set_cpu(cpu, donee_mask); 4021 } 4022 4023 /* iterate !donee CPUs and see if they should be offloaded */ 4024 cpumask_clear(resched_mask); 4025 for_each_cpu_and(cpu, cpu_online_mask, node_mask) { 4026 struct rq *rq = cpu_rq(cpu); 4027 struct scx_dispatch_q *donor_dsq = &rq->scx.bypass_dsq; 4028 4029 if (cpumask_empty(donee_mask)) 4030 break; 4031 if (cpumask_test_cpu(cpu, donee_mask)) 4032 continue; 4033 if (READ_ONCE(donor_dsq->nr) <= nr_donor_target) 4034 continue; 4035 4036 nr_balanced += bypass_lb_cpu(sch, rq, donee_mask, resched_mask, 4037 nr_donor_target, nr_target); 4038 } 4039 4040 for_each_cpu(cpu, resched_mask) 4041 resched_cpu(cpu); 4042 4043 for_each_cpu_and(cpu, cpu_online_mask, node_mask) { 4044 u32 nr = READ_ONCE(cpu_rq(cpu)->scx.bypass_dsq.nr); 4045 4046 after_min = min(nr, after_min); 4047 after_max = max(nr, after_max); 4048 4049 } 4050 4051 trace_sched_ext_bypass_lb(node, nr_cpus, nr_tasks, nr_balanced, 4052 before_min, before_max, after_min, after_max); 4053 } 4054 4055 /* 4056 * In bypass mode, all tasks are put on the per-CPU bypass DSQs. If the machine 4057 * is over-saturated and the BPF scheduler skewed tasks into few CPUs, some 4058 * bypass DSQs can be overloaded. If there are enough tasks to saturate other 4059 * lightly loaded CPUs, such imbalance can lead to very high execution latency 4060 * on the overloaded CPUs and thus to hung tasks and RCU stalls. To avoid such 4061 * outcomes, a simple load balancing mechanism is implemented by the following 4062 * timer which runs periodically while bypass mode is in effect. 4063 */ 4064 static void scx_bypass_lb_timerfn(struct timer_list *timer) 4065 { 4066 struct scx_sched *sch; 4067 int node; 4068 u32 intv_us; 4069 4070 sch = rcu_dereference_all(scx_root); 4071 if (unlikely(!sch) || !READ_ONCE(scx_bypass_depth)) 4072 return; 4073 4074 for_each_node_with_cpus(node) 4075 bypass_lb_node(sch, node); 4076 4077 intv_us = READ_ONCE(scx_bypass_lb_intv_us); 4078 if (intv_us) 4079 mod_timer(timer, jiffies + usecs_to_jiffies(intv_us)); 4080 } 4081 4082 static DEFINE_TIMER(scx_bypass_lb_timer, scx_bypass_lb_timerfn); 4083 4084 /** 4085 * scx_bypass - [Un]bypass scx_ops and guarantee forward progress 4086 * @bypass: true for bypass, false for unbypass 4087 * 4088 * Bypassing guarantees that all runnable tasks make forward progress without 4089 * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might 4090 * be held by tasks that the BPF scheduler is forgetting to run, which 4091 * unfortunately also excludes toggling the static branches. 4092 * 4093 * Let's work around by overriding a couple ops and modifying behaviors based on 4094 * the DISABLING state and then cycling the queued tasks through dequeue/enqueue 4095 * to force global FIFO scheduling. 4096 * 4097 * - ops.select_cpu() is ignored and the default select_cpu() is used. 4098 * 4099 * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order. 4100 * %SCX_OPS_ENQ_LAST is also ignored. 4101 * 4102 * - ops.dispatch() is ignored. 4103 * 4104 * - balance_one() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice 4105 * can't be trusted. Whenever a tick triggers, the running task is rotated to 4106 * the tail of the queue with core_sched_at touched. 4107 * 4108 * - pick_next_task() suppresses zero slice warning. 4109 * 4110 * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM 4111 * operations. 4112 * 4113 * - scx_prio_less() reverts to the default core_sched_at order. 4114 */ 4115 static void scx_bypass(bool bypass) 4116 { 4117 static DEFINE_RAW_SPINLOCK(bypass_lock); 4118 static unsigned long bypass_timestamp; 4119 struct scx_sched *sch; 4120 unsigned long flags; 4121 int cpu; 4122 4123 raw_spin_lock_irqsave(&bypass_lock, flags); 4124 sch = rcu_dereference_bh(scx_root); 4125 4126 if (bypass) { 4127 u32 intv_us; 4128 4129 WRITE_ONCE(scx_bypass_depth, scx_bypass_depth + 1); 4130 WARN_ON_ONCE(scx_bypass_depth <= 0); 4131 if (scx_bypass_depth != 1) 4132 goto unlock; 4133 WRITE_ONCE(scx_slice_dfl, scx_slice_bypass_us * NSEC_PER_USEC); 4134 bypass_timestamp = ktime_get_ns(); 4135 if (sch) 4136 scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1); 4137 4138 intv_us = READ_ONCE(scx_bypass_lb_intv_us); 4139 if (intv_us && !timer_pending(&scx_bypass_lb_timer)) { 4140 scx_bypass_lb_timer.expires = 4141 jiffies + usecs_to_jiffies(intv_us); 4142 add_timer_global(&scx_bypass_lb_timer); 4143 } 4144 } else { 4145 WRITE_ONCE(scx_bypass_depth, scx_bypass_depth - 1); 4146 WARN_ON_ONCE(scx_bypass_depth < 0); 4147 if (scx_bypass_depth != 0) 4148 goto unlock; 4149 WRITE_ONCE(scx_slice_dfl, SCX_SLICE_DFL); 4150 if (sch) 4151 scx_add_event(sch, SCX_EV_BYPASS_DURATION, 4152 ktime_get_ns() - bypass_timestamp); 4153 } 4154 4155 /* 4156 * No task property is changing. We just need to make sure all currently 4157 * queued tasks are re-queued according to the new scx_rq_bypassing() 4158 * state. As an optimization, walk each rq's runnable_list instead of 4159 * the scx_tasks list. 4160 * 4161 * This function can't trust the scheduler and thus can't use 4162 * cpus_read_lock(). Walk all possible CPUs instead of online. 4163 */ 4164 for_each_possible_cpu(cpu) { 4165 struct rq *rq = cpu_rq(cpu); 4166 struct task_struct *p, *n; 4167 4168 raw_spin_rq_lock(rq); 4169 4170 if (bypass) { 4171 WARN_ON_ONCE(rq->scx.flags & SCX_RQ_BYPASSING); 4172 rq->scx.flags |= SCX_RQ_BYPASSING; 4173 } else { 4174 WARN_ON_ONCE(!(rq->scx.flags & SCX_RQ_BYPASSING)); 4175 rq->scx.flags &= ~SCX_RQ_BYPASSING; 4176 } 4177 4178 /* 4179 * We need to guarantee that no tasks are on the BPF scheduler 4180 * while bypassing. Either we see enabled or the enable path 4181 * sees scx_rq_bypassing() before moving tasks to SCX. 4182 */ 4183 if (!scx_enabled()) { 4184 raw_spin_rq_unlock(rq); 4185 continue; 4186 } 4187 4188 /* 4189 * The use of list_for_each_entry_safe_reverse() is required 4190 * because each task is going to be removed from and added back 4191 * to the runnable_list during iteration. Because they're added 4192 * to the tail of the list, safe reverse iteration can still 4193 * visit all nodes. 4194 */ 4195 list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, 4196 scx.runnable_node) { 4197 /* cycling deq/enq is enough, see the function comment */ 4198 scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { 4199 /* nothing */ ; 4200 } 4201 } 4202 4203 /* resched to restore ticks and idle state */ 4204 if (cpu_online(cpu) || cpu == smp_processor_id()) 4205 resched_curr(rq); 4206 4207 raw_spin_rq_unlock(rq); 4208 } 4209 4210 unlock: 4211 raw_spin_unlock_irqrestore(&bypass_lock, flags); 4212 } 4213 4214 static void free_exit_info(struct scx_exit_info *ei) 4215 { 4216 kvfree(ei->dump); 4217 kfree(ei->msg); 4218 kfree(ei->bt); 4219 kfree(ei); 4220 } 4221 4222 static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len) 4223 { 4224 struct scx_exit_info *ei; 4225 4226 ei = kzalloc(sizeof(*ei), GFP_KERNEL); 4227 if (!ei) 4228 return NULL; 4229 4230 ei->bt = kcalloc(SCX_EXIT_BT_LEN, sizeof(ei->bt[0]), GFP_KERNEL); 4231 ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); 4232 ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL); 4233 4234 if (!ei->bt || !ei->msg || !ei->dump) { 4235 free_exit_info(ei); 4236 return NULL; 4237 } 4238 4239 return ei; 4240 } 4241 4242 static const char *scx_exit_reason(enum scx_exit_kind kind) 4243 { 4244 switch (kind) { 4245 case SCX_EXIT_UNREG: 4246 return "unregistered from user space"; 4247 case SCX_EXIT_UNREG_BPF: 4248 return "unregistered from BPF"; 4249 case SCX_EXIT_UNREG_KERN: 4250 return "unregistered from the main kernel"; 4251 case SCX_EXIT_SYSRQ: 4252 return "disabled by sysrq-S"; 4253 case SCX_EXIT_ERROR: 4254 return "runtime error"; 4255 case SCX_EXIT_ERROR_BPF: 4256 return "scx_bpf_error"; 4257 case SCX_EXIT_ERROR_STALL: 4258 return "runnable task stall"; 4259 default: 4260 return "<UNKNOWN>"; 4261 } 4262 } 4263 4264 static void free_kick_syncs(void) 4265 { 4266 int cpu; 4267 4268 for_each_possible_cpu(cpu) { 4269 struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); 4270 struct scx_kick_syncs *to_free; 4271 4272 to_free = rcu_replace_pointer(*ksyncs, NULL, true); 4273 if (to_free) 4274 kvfree_rcu(to_free, rcu); 4275 } 4276 } 4277 4278 static void scx_disable_workfn(struct kthread_work *work) 4279 { 4280 struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); 4281 struct scx_exit_info *ei = sch->exit_info; 4282 struct scx_task_iter sti; 4283 struct task_struct *p; 4284 int kind, cpu; 4285 4286 kind = atomic_read(&sch->exit_kind); 4287 while (true) { 4288 if (kind == SCX_EXIT_DONE) /* already disabled? */ 4289 return; 4290 WARN_ON_ONCE(kind == SCX_EXIT_NONE); 4291 if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) 4292 break; 4293 } 4294 ei->kind = kind; 4295 ei->reason = scx_exit_reason(ei->kind); 4296 4297 /* guarantee forward progress by bypassing scx_ops */ 4298 scx_bypass(true); 4299 WRITE_ONCE(scx_aborting, false); 4300 4301 switch (scx_set_enable_state(SCX_DISABLING)) { 4302 case SCX_DISABLING: 4303 WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); 4304 break; 4305 case SCX_DISABLED: 4306 pr_warn("sched_ext: ops error detected without ops (%s)\n", 4307 sch->exit_info->msg); 4308 WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); 4309 goto done; 4310 default: 4311 break; 4312 } 4313 4314 /* 4315 * Here, every runnable task is guaranteed to make forward progress and 4316 * we can safely use blocking synchronization constructs. Actually 4317 * disable ops. 4318 */ 4319 mutex_lock(&scx_enable_mutex); 4320 4321 static_branch_disable(&__scx_switched_all); 4322 WRITE_ONCE(scx_switching_all, false); 4323 4324 /* 4325 * Shut down cgroup support before tasks so that the cgroup attach path 4326 * doesn't race against scx_exit_task(). 4327 */ 4328 scx_cgroup_lock(); 4329 scx_cgroup_exit(sch); 4330 scx_cgroup_unlock(); 4331 4332 /* 4333 * The BPF scheduler is going away. All tasks including %TASK_DEAD ones 4334 * must be switched out and exited synchronously. 4335 */ 4336 percpu_down_write(&scx_fork_rwsem); 4337 4338 scx_init_task_enabled = false; 4339 4340 scx_task_iter_start(&sti); 4341 while ((p = scx_task_iter_next_locked(&sti))) { 4342 unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 4343 const struct sched_class *old_class = p->sched_class; 4344 const struct sched_class *new_class = scx_setscheduler_class(p); 4345 4346 update_rq_clock(task_rq(p)); 4347 4348 if (old_class != new_class) 4349 queue_flags |= DEQUEUE_CLASS; 4350 4351 scoped_guard (sched_change, p, queue_flags) { 4352 p->sched_class = new_class; 4353 } 4354 4355 scx_exit_task(p); 4356 } 4357 scx_task_iter_stop(&sti); 4358 percpu_up_write(&scx_fork_rwsem); 4359 4360 /* 4361 * Invalidate all the rq clocks to prevent getting outdated 4362 * rq clocks from a previous scx scheduler. 4363 */ 4364 for_each_possible_cpu(cpu) { 4365 struct rq *rq = cpu_rq(cpu); 4366 scx_rq_clock_invalidate(rq); 4367 } 4368 4369 /* no task is on scx, turn off all the switches and flush in-progress calls */ 4370 static_branch_disable(&__scx_enabled); 4371 bitmap_zero(sch->has_op, SCX_OPI_END); 4372 scx_idle_disable(); 4373 synchronize_rcu(); 4374 4375 if (ei->kind >= SCX_EXIT_ERROR) { 4376 pr_err("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", 4377 sch->ops.name, ei->reason); 4378 4379 if (ei->msg[0] != '\0') 4380 pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg); 4381 #ifdef CONFIG_STACKTRACE 4382 stack_trace_print(ei->bt, ei->bt_len, 2); 4383 #endif 4384 } else { 4385 pr_info("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", 4386 sch->ops.name, ei->reason); 4387 } 4388 4389 if (sch->ops.exit) 4390 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, exit, NULL, ei); 4391 4392 cancel_delayed_work_sync(&scx_watchdog_work); 4393 4394 /* 4395 * scx_root clearing must be inside cpus_read_lock(). See 4396 * handle_hotplug(). 4397 */ 4398 cpus_read_lock(); 4399 RCU_INIT_POINTER(scx_root, NULL); 4400 cpus_read_unlock(); 4401 4402 /* 4403 * Delete the kobject from the hierarchy synchronously. Otherwise, sysfs 4404 * could observe an object of the same name still in the hierarchy when 4405 * the next scheduler is loaded. 4406 */ 4407 kobject_del(&sch->kobj); 4408 4409 free_percpu(scx_dsp_ctx); 4410 scx_dsp_ctx = NULL; 4411 scx_dsp_max_batch = 0; 4412 free_kick_syncs(); 4413 4414 if (scx_bypassed_for_enable) { 4415 scx_bypassed_for_enable = false; 4416 scx_bypass(false); 4417 } 4418 4419 mutex_unlock(&scx_enable_mutex); 4420 4421 WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); 4422 done: 4423 scx_bypass(false); 4424 } 4425 4426 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) 4427 { 4428 int none = SCX_EXIT_NONE; 4429 4430 if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) 4431 return false; 4432 4433 /* 4434 * Some CPUs may be trapped in the dispatch paths. Set the aborting 4435 * flag to break potential live-lock scenarios, ensuring we can 4436 * successfully reach scx_bypass(). 4437 */ 4438 WRITE_ONCE(scx_aborting, true); 4439 return true; 4440 } 4441 4442 static void scx_disable(enum scx_exit_kind kind) 4443 { 4444 struct scx_sched *sch; 4445 4446 if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) 4447 kind = SCX_EXIT_ERROR; 4448 4449 rcu_read_lock(); 4450 sch = rcu_dereference(scx_root); 4451 if (sch) { 4452 scx_claim_exit(sch, kind); 4453 kthread_queue_work(sch->helper, &sch->disable_work); 4454 } 4455 rcu_read_unlock(); 4456 } 4457 4458 static void dump_newline(struct seq_buf *s) 4459 { 4460 trace_sched_ext_dump(""); 4461 4462 /* @s may be zero sized and seq_buf triggers WARN if so */ 4463 if (s->size) 4464 seq_buf_putc(s, '\n'); 4465 } 4466 4467 static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...) 4468 { 4469 va_list args; 4470 4471 #ifdef CONFIG_TRACEPOINTS 4472 if (trace_sched_ext_dump_enabled()) { 4473 /* protected by scx_dump_state()::dump_lock */ 4474 static char line_buf[SCX_EXIT_MSG_LEN]; 4475 4476 va_start(args, fmt); 4477 vscnprintf(line_buf, sizeof(line_buf), fmt, args); 4478 va_end(args); 4479 4480 trace_sched_ext_dump(line_buf); 4481 } 4482 #endif 4483 /* @s may be zero sized and seq_buf triggers WARN if so */ 4484 if (s->size) { 4485 va_start(args, fmt); 4486 seq_buf_vprintf(s, fmt, args); 4487 va_end(args); 4488 4489 seq_buf_putc(s, '\n'); 4490 } 4491 } 4492 4493 static void dump_stack_trace(struct seq_buf *s, const char *prefix, 4494 const unsigned long *bt, unsigned int len) 4495 { 4496 unsigned int i; 4497 4498 for (i = 0; i < len; i++) 4499 dump_line(s, "%s%pS", prefix, (void *)bt[i]); 4500 } 4501 4502 static void ops_dump_init(struct seq_buf *s, const char *prefix) 4503 { 4504 struct scx_dump_data *dd = &scx_dump_data; 4505 4506 lockdep_assert_irqs_disabled(); 4507 4508 dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */ 4509 dd->first = true; 4510 dd->cursor = 0; 4511 dd->s = s; 4512 dd->prefix = prefix; 4513 } 4514 4515 static void ops_dump_flush(void) 4516 { 4517 struct scx_dump_data *dd = &scx_dump_data; 4518 char *line = dd->buf.line; 4519 4520 if (!dd->cursor) 4521 return; 4522 4523 /* 4524 * There's something to flush and this is the first line. Insert a blank 4525 * line to distinguish ops dump. 4526 */ 4527 if (dd->first) { 4528 dump_newline(dd->s); 4529 dd->first = false; 4530 } 4531 4532 /* 4533 * There may be multiple lines in $line. Scan and emit each line 4534 * separately. 4535 */ 4536 while (true) { 4537 char *end = line; 4538 char c; 4539 4540 while (*end != '\n' && *end != '\0') 4541 end++; 4542 4543 /* 4544 * If $line overflowed, it may not have newline at the end. 4545 * Always emit with a newline. 4546 */ 4547 c = *end; 4548 *end = '\0'; 4549 dump_line(dd->s, "%s%s", dd->prefix, line); 4550 if (c == '\0') 4551 break; 4552 4553 /* move to the next line */ 4554 end++; 4555 if (*end == '\0') 4556 break; 4557 line = end; 4558 } 4559 4560 dd->cursor = 0; 4561 } 4562 4563 static void ops_dump_exit(void) 4564 { 4565 ops_dump_flush(); 4566 scx_dump_data.cpu = -1; 4567 } 4568 4569 static void scx_dump_task(struct seq_buf *s, struct scx_dump_ctx *dctx, 4570 struct task_struct *p, char marker) 4571 { 4572 static unsigned long bt[SCX_EXIT_BT_LEN]; 4573 struct scx_sched *sch = scx_root; 4574 char dsq_id_buf[19] = "(n/a)"; 4575 unsigned long ops_state = atomic_long_read(&p->scx.ops_state); 4576 unsigned int bt_len = 0; 4577 4578 if (p->scx.dsq) 4579 scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx", 4580 (unsigned long long)p->scx.dsq->id); 4581 4582 dump_newline(s); 4583 dump_line(s, " %c%c %s[%d] %+ldms", 4584 marker, task_state_to_char(p), p->comm, p->pid, 4585 jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); 4586 dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", 4587 scx_get_task_state(p), p->scx.flags & ~SCX_TASK_STATE_MASK, 4588 p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, 4589 ops_state >> SCX_OPSS_QSEQ_SHIFT); 4590 dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", 4591 p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); 4592 dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", 4593 p->scx.dsq_vtime, p->scx.slice, p->scx.weight); 4594 dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), 4595 p->migration_disabled); 4596 4597 if (SCX_HAS_OP(sch, dump_task)) { 4598 ops_dump_init(s, " "); 4599 SCX_CALL_OP(sch, SCX_KF_REST, dump_task, NULL, dctx, p); 4600 ops_dump_exit(); 4601 } 4602 4603 #ifdef CONFIG_STACKTRACE 4604 bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1); 4605 #endif 4606 if (bt_len) { 4607 dump_newline(s); 4608 dump_stack_trace(s, " ", bt, bt_len); 4609 } 4610 } 4611 4612 static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len) 4613 { 4614 static DEFINE_SPINLOCK(dump_lock); 4615 static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n"; 4616 struct scx_sched *sch = scx_root; 4617 struct scx_dump_ctx dctx = { 4618 .kind = ei->kind, 4619 .exit_code = ei->exit_code, 4620 .reason = ei->reason, 4621 .at_ns = ktime_get_ns(), 4622 .at_jiffies = jiffies, 4623 }; 4624 struct seq_buf s; 4625 struct scx_event_stats events; 4626 unsigned long flags; 4627 char *buf; 4628 int cpu; 4629 4630 spin_lock_irqsave(&dump_lock, flags); 4631 4632 seq_buf_init(&s, ei->dump, dump_len); 4633 4634 if (ei->kind == SCX_EXIT_NONE) { 4635 dump_line(&s, "Debug dump triggered by %s", ei->reason); 4636 } else { 4637 dump_line(&s, "%s[%d] triggered exit kind %d:", 4638 current->comm, current->pid, ei->kind); 4639 dump_line(&s, " %s (%s)", ei->reason, ei->msg); 4640 dump_newline(&s); 4641 dump_line(&s, "Backtrace:"); 4642 dump_stack_trace(&s, " ", ei->bt, ei->bt_len); 4643 } 4644 4645 if (SCX_HAS_OP(sch, dump)) { 4646 ops_dump_init(&s, ""); 4647 SCX_CALL_OP(sch, SCX_KF_UNLOCKED, dump, NULL, &dctx); 4648 ops_dump_exit(); 4649 } 4650 4651 dump_newline(&s); 4652 dump_line(&s, "CPU states"); 4653 dump_line(&s, "----------"); 4654 4655 for_each_possible_cpu(cpu) { 4656 struct rq *rq = cpu_rq(cpu); 4657 struct rq_flags rf; 4658 struct task_struct *p; 4659 struct seq_buf ns; 4660 size_t avail, used; 4661 bool idle; 4662 4663 rq_lock_irqsave(rq, &rf); 4664 4665 idle = list_empty(&rq->scx.runnable_list) && 4666 rq->curr->sched_class == &idle_sched_class; 4667 4668 if (idle && !SCX_HAS_OP(sch, dump_cpu)) 4669 goto next; 4670 4671 /* 4672 * We don't yet know whether ops.dump_cpu() will produce output 4673 * and we may want to skip the default CPU dump if it doesn't. 4674 * Use a nested seq_buf to generate the standard dump so that we 4675 * can decide whether to commit later. 4676 */ 4677 avail = seq_buf_get_buf(&s, &buf); 4678 seq_buf_init(&ns, buf, avail); 4679 4680 dump_newline(&ns); 4681 dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", 4682 cpu, rq->scx.nr_running, rq->scx.flags, 4683 rq->scx.cpu_released, rq->scx.ops_qseq, 4684 rq->scx.kick_sync); 4685 dump_line(&ns, " curr=%s[%d] class=%ps", 4686 rq->curr->comm, rq->curr->pid, 4687 rq->curr->sched_class); 4688 if (!cpumask_empty(rq->scx.cpus_to_kick)) 4689 dump_line(&ns, " cpus_to_kick : %*pb", 4690 cpumask_pr_args(rq->scx.cpus_to_kick)); 4691 if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) 4692 dump_line(&ns, " idle_to_kick : %*pb", 4693 cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); 4694 if (!cpumask_empty(rq->scx.cpus_to_preempt)) 4695 dump_line(&ns, " cpus_to_preempt: %*pb", 4696 cpumask_pr_args(rq->scx.cpus_to_preempt)); 4697 if (!cpumask_empty(rq->scx.cpus_to_wait)) 4698 dump_line(&ns, " cpus_to_wait : %*pb", 4699 cpumask_pr_args(rq->scx.cpus_to_wait)); 4700 4701 used = seq_buf_used(&ns); 4702 if (SCX_HAS_OP(sch, dump_cpu)) { 4703 ops_dump_init(&ns, " "); 4704 SCX_CALL_OP(sch, SCX_KF_REST, dump_cpu, NULL, 4705 &dctx, cpu, idle); 4706 ops_dump_exit(); 4707 } 4708 4709 /* 4710 * If idle && nothing generated by ops.dump_cpu(), there's 4711 * nothing interesting. Skip. 4712 */ 4713 if (idle && used == seq_buf_used(&ns)) 4714 goto next; 4715 4716 /* 4717 * $s may already have overflowed when $ns was created. If so, 4718 * calling commit on it will trigger BUG. 4719 */ 4720 if (avail) { 4721 seq_buf_commit(&s, seq_buf_used(&ns)); 4722 if (seq_buf_has_overflowed(&ns)) 4723 seq_buf_set_overflow(&s); 4724 } 4725 4726 if (rq->curr->sched_class == &ext_sched_class) 4727 scx_dump_task(&s, &dctx, rq->curr, '*'); 4728 4729 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) 4730 scx_dump_task(&s, &dctx, p, ' '); 4731 next: 4732 rq_unlock_irqrestore(rq, &rf); 4733 } 4734 4735 dump_newline(&s); 4736 dump_line(&s, "Event counters"); 4737 dump_line(&s, "--------------"); 4738 4739 scx_read_events(sch, &events); 4740 scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK); 4741 scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); 4742 scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST); 4743 scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING); 4744 scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); 4745 scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL); 4746 scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION); 4747 scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH); 4748 scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE); 4749 4750 if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) 4751 memcpy(ei->dump + dump_len - sizeof(trunc_marker), 4752 trunc_marker, sizeof(trunc_marker)); 4753 4754 spin_unlock_irqrestore(&dump_lock, flags); 4755 } 4756 4757 static void scx_error_irq_workfn(struct irq_work *irq_work) 4758 { 4759 struct scx_sched *sch = container_of(irq_work, struct scx_sched, error_irq_work); 4760 struct scx_exit_info *ei = sch->exit_info; 4761 4762 if (ei->kind >= SCX_EXIT_ERROR) 4763 scx_dump_state(ei, sch->ops.exit_dump_len); 4764 4765 kthread_queue_work(sch->helper, &sch->disable_work); 4766 } 4767 4768 static bool scx_vexit(struct scx_sched *sch, 4769 enum scx_exit_kind kind, s64 exit_code, 4770 const char *fmt, va_list args) 4771 { 4772 struct scx_exit_info *ei = sch->exit_info; 4773 4774 if (!scx_claim_exit(sch, kind)) 4775 return false; 4776 4777 ei->exit_code = exit_code; 4778 #ifdef CONFIG_STACKTRACE 4779 if (kind >= SCX_EXIT_ERROR) 4780 ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); 4781 #endif 4782 vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); 4783 4784 /* 4785 * Set ei->kind and ->reason for scx_dump_state(). They'll be set again 4786 * in scx_disable_workfn(). 4787 */ 4788 ei->kind = kind; 4789 ei->reason = scx_exit_reason(ei->kind); 4790 4791 irq_work_queue(&sch->error_irq_work); 4792 return true; 4793 } 4794 4795 static int alloc_kick_syncs(void) 4796 { 4797 int cpu; 4798 4799 /* 4800 * Allocate per-CPU arrays sized by nr_cpu_ids. Use kvzalloc as size 4801 * can exceed percpu allocator limits on large machines. 4802 */ 4803 for_each_possible_cpu(cpu) { 4804 struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); 4805 struct scx_kick_syncs *new_ksyncs; 4806 4807 WARN_ON_ONCE(rcu_access_pointer(*ksyncs)); 4808 4809 new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids), 4810 GFP_KERNEL, cpu_to_node(cpu)); 4811 if (!new_ksyncs) { 4812 free_kick_syncs(); 4813 return -ENOMEM; 4814 } 4815 4816 rcu_assign_pointer(*ksyncs, new_ksyncs); 4817 } 4818 4819 return 0; 4820 } 4821 4822 static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops) 4823 { 4824 struct scx_sched *sch; 4825 int node, ret; 4826 4827 sch = kzalloc(sizeof(*sch), GFP_KERNEL); 4828 if (!sch) 4829 return ERR_PTR(-ENOMEM); 4830 4831 sch->exit_info = alloc_exit_info(ops->exit_dump_len); 4832 if (!sch->exit_info) { 4833 ret = -ENOMEM; 4834 goto err_free_sch; 4835 } 4836 4837 ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params); 4838 if (ret < 0) 4839 goto err_free_ei; 4840 4841 sch->global_dsqs = kcalloc(nr_node_ids, sizeof(sch->global_dsqs[0]), 4842 GFP_KERNEL); 4843 if (!sch->global_dsqs) { 4844 ret = -ENOMEM; 4845 goto err_free_hash; 4846 } 4847 4848 for_each_node_state(node, N_POSSIBLE) { 4849 struct scx_dispatch_q *dsq; 4850 4851 dsq = kzalloc_node(sizeof(*dsq), GFP_KERNEL, node); 4852 if (!dsq) { 4853 ret = -ENOMEM; 4854 goto err_free_gdsqs; 4855 } 4856 4857 init_dsq(dsq, SCX_DSQ_GLOBAL); 4858 sch->global_dsqs[node] = dsq; 4859 } 4860 4861 sch->pcpu = alloc_percpu(struct scx_sched_pcpu); 4862 if (!sch->pcpu) { 4863 ret = -ENOMEM; 4864 goto err_free_gdsqs; 4865 } 4866 4867 sch->helper = kthread_run_worker(0, "sched_ext_helper"); 4868 if (IS_ERR(sch->helper)) { 4869 ret = PTR_ERR(sch->helper); 4870 goto err_free_pcpu; 4871 } 4872 4873 sched_set_fifo(sch->helper->task); 4874 4875 atomic_set(&sch->exit_kind, SCX_EXIT_NONE); 4876 init_irq_work(&sch->error_irq_work, scx_error_irq_workfn); 4877 kthread_init_work(&sch->disable_work, scx_disable_workfn); 4878 sch->ops = *ops; 4879 ops->priv = sch; 4880 4881 sch->kobj.kset = scx_kset; 4882 ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); 4883 if (ret < 0) 4884 goto err_stop_helper; 4885 4886 return sch; 4887 4888 err_stop_helper: 4889 kthread_destroy_worker(sch->helper); 4890 err_free_pcpu: 4891 free_percpu(sch->pcpu); 4892 err_free_gdsqs: 4893 for_each_node_state(node, N_POSSIBLE) 4894 kfree(sch->global_dsqs[node]); 4895 kfree(sch->global_dsqs); 4896 err_free_hash: 4897 rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); 4898 err_free_ei: 4899 free_exit_info(sch->exit_info); 4900 err_free_sch: 4901 kfree(sch); 4902 return ERR_PTR(ret); 4903 } 4904 4905 static int check_hotplug_seq(struct scx_sched *sch, 4906 const struct sched_ext_ops *ops) 4907 { 4908 unsigned long long global_hotplug_seq; 4909 4910 /* 4911 * If a hotplug event has occurred between when a scheduler was 4912 * initialized, and when we were able to attach, exit and notify user 4913 * space about it. 4914 */ 4915 if (ops->hotplug_seq) { 4916 global_hotplug_seq = atomic_long_read(&scx_hotplug_seq); 4917 if (ops->hotplug_seq != global_hotplug_seq) { 4918 scx_exit(sch, SCX_EXIT_UNREG_KERN, 4919 SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, 4920 "expected hotplug seq %llu did not match actual %llu", 4921 ops->hotplug_seq, global_hotplug_seq); 4922 return -EBUSY; 4923 } 4924 } 4925 4926 return 0; 4927 } 4928 4929 static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) 4930 { 4931 /* 4932 * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the 4933 * ops.enqueue() callback isn't implemented. 4934 */ 4935 if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) { 4936 scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); 4937 return -EINVAL; 4938 } 4939 4940 /* 4941 * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle 4942 * selection policy to be enabled. 4943 */ 4944 if ((ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) && 4945 (ops->update_idle && !(ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE))) { 4946 scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled"); 4947 return -EINVAL; 4948 } 4949 4950 if (ops->flags & SCX_OPS_HAS_CGROUP_WEIGHT) 4951 pr_warn("SCX_OPS_HAS_CGROUP_WEIGHT is deprecated and a noop\n"); 4952 4953 if (ops->cpu_acquire || ops->cpu_release) 4954 pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); 4955 4956 return 0; 4957 } 4958 4959 static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) 4960 { 4961 struct scx_sched *sch; 4962 struct scx_task_iter sti; 4963 struct task_struct *p; 4964 unsigned long timeout; 4965 int i, cpu, ret; 4966 4967 if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), 4968 cpu_possible_mask)) { 4969 pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); 4970 return -EINVAL; 4971 } 4972 4973 mutex_lock(&scx_enable_mutex); 4974 4975 if (scx_enable_state() != SCX_DISABLED) { 4976 ret = -EBUSY; 4977 goto err_unlock; 4978 } 4979 4980 ret = alloc_kick_syncs(); 4981 if (ret) 4982 goto err_unlock; 4983 4984 sch = scx_alloc_and_add_sched(ops); 4985 if (IS_ERR(sch)) { 4986 ret = PTR_ERR(sch); 4987 goto err_free_ksyncs; 4988 } 4989 4990 /* 4991 * Transition to ENABLING and clear exit info to arm the disable path. 4992 * Failure triggers full disabling from here on. 4993 */ 4994 WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); 4995 WARN_ON_ONCE(scx_root); 4996 if (WARN_ON_ONCE(READ_ONCE(scx_aborting))) 4997 WRITE_ONCE(scx_aborting, false); 4998 4999 atomic_long_set(&scx_nr_rejected, 0); 5000 5001 for_each_possible_cpu(cpu) 5002 cpu_rq(cpu)->scx.cpuperf_target = SCX_CPUPERF_ONE; 5003 5004 /* 5005 * Keep CPUs stable during enable so that the BPF scheduler can track 5006 * online CPUs by watching ->on/offline_cpu() after ->init(). 5007 */ 5008 cpus_read_lock(); 5009 5010 /* 5011 * Make the scheduler instance visible. Must be inside cpus_read_lock(). 5012 * See handle_hotplug(). 5013 */ 5014 rcu_assign_pointer(scx_root, sch); 5015 5016 scx_idle_enable(ops); 5017 5018 if (sch->ops.init) { 5019 ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, init, NULL); 5020 if (ret) { 5021 ret = ops_sanitize_err(sch, "init", ret); 5022 cpus_read_unlock(); 5023 scx_error(sch, "ops.init() failed (%d)", ret); 5024 goto err_disable; 5025 } 5026 sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; 5027 } 5028 5029 for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) 5030 if (((void (**)(void))ops)[i]) 5031 set_bit(i, sch->has_op); 5032 5033 ret = check_hotplug_seq(sch, ops); 5034 if (ret) { 5035 cpus_read_unlock(); 5036 goto err_disable; 5037 } 5038 scx_idle_update_selcpu_topology(ops); 5039 5040 cpus_read_unlock(); 5041 5042 ret = validate_ops(sch, ops); 5043 if (ret) 5044 goto err_disable; 5045 5046 WARN_ON_ONCE(scx_dsp_ctx); 5047 scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; 5048 scx_dsp_ctx = __alloc_percpu(struct_size_t(struct scx_dsp_ctx, buf, 5049 scx_dsp_max_batch), 5050 __alignof__(struct scx_dsp_ctx)); 5051 if (!scx_dsp_ctx) { 5052 ret = -ENOMEM; 5053 goto err_disable; 5054 } 5055 5056 if (ops->timeout_ms) 5057 timeout = msecs_to_jiffies(ops->timeout_ms); 5058 else 5059 timeout = SCX_WATCHDOG_MAX_TIMEOUT; 5060 5061 WRITE_ONCE(scx_watchdog_timeout, timeout); 5062 WRITE_ONCE(scx_watchdog_timestamp, jiffies); 5063 queue_delayed_work(system_unbound_wq, &scx_watchdog_work, 5064 scx_watchdog_timeout / 2); 5065 5066 /* 5067 * Once __scx_enabled is set, %current can be switched to SCX anytime. 5068 * This can lead to stalls as some BPF schedulers (e.g. userspace 5069 * scheduling) may not function correctly before all tasks are switched. 5070 * Init in bypass mode to guarantee forward progress. 5071 */ 5072 scx_bypass(true); 5073 scx_bypassed_for_enable = true; 5074 5075 for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) 5076 if (((void (**)(void))ops)[i]) 5077 set_bit(i, sch->has_op); 5078 5079 if (sch->ops.cpu_acquire || sch->ops.cpu_release) 5080 sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; 5081 5082 /* 5083 * Lock out forks, cgroup on/offlining and moves before opening the 5084 * floodgate so that they don't wander into the operations prematurely. 5085 */ 5086 percpu_down_write(&scx_fork_rwsem); 5087 5088 WARN_ON_ONCE(scx_init_task_enabled); 5089 scx_init_task_enabled = true; 5090 5091 /* 5092 * Enable ops for every task. Fork is excluded by scx_fork_rwsem 5093 * preventing new tasks from being added. No need to exclude tasks 5094 * leaving as sched_ext_free() can handle both prepped and enabled 5095 * tasks. Prep all tasks first and then enable them with preemption 5096 * disabled. 5097 * 5098 * All cgroups should be initialized before scx_init_task() so that the 5099 * BPF scheduler can reliably track each task's cgroup membership from 5100 * scx_init_task(). Lock out cgroup on/offlining and task migrations 5101 * while tasks are being initialized so that scx_cgroup_can_attach() 5102 * never sees uninitialized tasks. 5103 */ 5104 scx_cgroup_lock(); 5105 ret = scx_cgroup_init(sch); 5106 if (ret) 5107 goto err_disable_unlock_all; 5108 5109 scx_task_iter_start(&sti); 5110 while ((p = scx_task_iter_next_locked(&sti))) { 5111 /* 5112 * @p may already be dead, have lost all its usages counts and 5113 * be waiting for RCU grace period before being freed. @p can't 5114 * be initialized for SCX in such cases and should be ignored. 5115 */ 5116 if (!tryget_task_struct(p)) 5117 continue; 5118 5119 scx_task_iter_unlock(&sti); 5120 5121 ret = scx_init_task(p, task_group(p), false); 5122 if (ret) { 5123 put_task_struct(p); 5124 scx_task_iter_stop(&sti); 5125 scx_error(sch, "ops.init_task() failed (%d) for %s[%d]", 5126 ret, p->comm, p->pid); 5127 goto err_disable_unlock_all; 5128 } 5129 5130 scx_set_task_state(p, SCX_TASK_READY); 5131 5132 put_task_struct(p); 5133 } 5134 scx_task_iter_stop(&sti); 5135 scx_cgroup_unlock(); 5136 percpu_up_write(&scx_fork_rwsem); 5137 5138 /* 5139 * All tasks are READY. It's safe to turn on scx_enabled() and switch 5140 * all eligible tasks. 5141 */ 5142 WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); 5143 static_branch_enable(&__scx_enabled); 5144 5145 /* 5146 * We're fully committed and can't fail. The task READY -> ENABLED 5147 * transitions here are synchronized against sched_ext_free() through 5148 * scx_tasks_lock. 5149 */ 5150 percpu_down_write(&scx_fork_rwsem); 5151 scx_task_iter_start(&sti); 5152 while ((p = scx_task_iter_next_locked(&sti))) { 5153 unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; 5154 const struct sched_class *old_class = p->sched_class; 5155 const struct sched_class *new_class = scx_setscheduler_class(p); 5156 5157 if (scx_get_task_state(p) != SCX_TASK_READY) 5158 continue; 5159 5160 if (old_class != new_class) 5161 queue_flags |= DEQUEUE_CLASS; 5162 5163 scoped_guard (sched_change, p, queue_flags) { 5164 p->scx.slice = READ_ONCE(scx_slice_dfl); 5165 p->sched_class = new_class; 5166 } 5167 } 5168 scx_task_iter_stop(&sti); 5169 percpu_up_write(&scx_fork_rwsem); 5170 5171 scx_bypassed_for_enable = false; 5172 scx_bypass(false); 5173 5174 if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { 5175 WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); 5176 goto err_disable; 5177 } 5178 5179 if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) 5180 static_branch_enable(&__scx_switched_all); 5181 5182 pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", 5183 sch->ops.name, scx_switched_all() ? "" : " (partial)"); 5184 kobject_uevent(&sch->kobj, KOBJ_ADD); 5185 mutex_unlock(&scx_enable_mutex); 5186 5187 atomic_long_inc(&scx_enable_seq); 5188 5189 return 0; 5190 5191 err_free_ksyncs: 5192 free_kick_syncs(); 5193 err_unlock: 5194 mutex_unlock(&scx_enable_mutex); 5195 return ret; 5196 5197 err_disable_unlock_all: 5198 scx_cgroup_unlock(); 5199 percpu_up_write(&scx_fork_rwsem); 5200 /* we'll soon enter disable path, keep bypass on */ 5201 err_disable: 5202 mutex_unlock(&scx_enable_mutex); 5203 /* 5204 * Returning an error code here would not pass all the error information 5205 * to userspace. Record errno using scx_error() for cases scx_error() 5206 * wasn't already invoked and exit indicating success so that the error 5207 * is notified through ops.exit() with all the details. 5208 * 5209 * Flush scx_disable_work to ensure that error is reported before init 5210 * completion. sch's base reference will be put by bpf_scx_unreg(). 5211 */ 5212 scx_error(sch, "scx_enable() failed (%d)", ret); 5213 kthread_flush_work(&sch->disable_work); 5214 return 0; 5215 } 5216 5217 5218 /******************************************************************************** 5219 * bpf_struct_ops plumbing. 5220 */ 5221 #include <linux/bpf_verifier.h> 5222 #include <linux/bpf.h> 5223 #include <linux/btf.h> 5224 5225 static const struct btf_type *task_struct_type; 5226 5227 static bool bpf_scx_is_valid_access(int off, int size, 5228 enum bpf_access_type type, 5229 const struct bpf_prog *prog, 5230 struct bpf_insn_access_aux *info) 5231 { 5232 if (type != BPF_READ) 5233 return false; 5234 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 5235 return false; 5236 if (off % size != 0) 5237 return false; 5238 5239 return btf_ctx_access(off, size, type, prog, info); 5240 } 5241 5242 static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, 5243 const struct bpf_reg_state *reg, int off, 5244 int size) 5245 { 5246 const struct btf_type *t; 5247 5248 t = btf_type_by_id(reg->btf, reg->btf_id); 5249 if (t == task_struct_type) { 5250 if (off >= offsetof(struct task_struct, scx.slice) && 5251 off + size <= offsetofend(struct task_struct, scx.slice)) 5252 return SCALAR_VALUE; 5253 if (off >= offsetof(struct task_struct, scx.dsq_vtime) && 5254 off + size <= offsetofend(struct task_struct, scx.dsq_vtime)) 5255 return SCALAR_VALUE; 5256 if (off >= offsetof(struct task_struct, scx.disallow) && 5257 off + size <= offsetofend(struct task_struct, scx.disallow)) 5258 return SCALAR_VALUE; 5259 } 5260 5261 return -EACCES; 5262 } 5263 5264 static const struct bpf_verifier_ops bpf_scx_verifier_ops = { 5265 .get_func_proto = bpf_base_func_proto, 5266 .is_valid_access = bpf_scx_is_valid_access, 5267 .btf_struct_access = bpf_scx_btf_struct_access, 5268 }; 5269 5270 static int bpf_scx_init_member(const struct btf_type *t, 5271 const struct btf_member *member, 5272 void *kdata, const void *udata) 5273 { 5274 const struct sched_ext_ops *uops = udata; 5275 struct sched_ext_ops *ops = kdata; 5276 u32 moff = __btf_member_bit_offset(t, member) / 8; 5277 int ret; 5278 5279 switch (moff) { 5280 case offsetof(struct sched_ext_ops, dispatch_max_batch): 5281 if (*(u32 *)(udata + moff) > INT_MAX) 5282 return -E2BIG; 5283 ops->dispatch_max_batch = *(u32 *)(udata + moff); 5284 return 1; 5285 case offsetof(struct sched_ext_ops, flags): 5286 if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) 5287 return -EINVAL; 5288 ops->flags = *(u64 *)(udata + moff); 5289 return 1; 5290 case offsetof(struct sched_ext_ops, name): 5291 ret = bpf_obj_name_cpy(ops->name, uops->name, 5292 sizeof(ops->name)); 5293 if (ret < 0) 5294 return ret; 5295 if (ret == 0) 5296 return -EINVAL; 5297 return 1; 5298 case offsetof(struct sched_ext_ops, timeout_ms): 5299 if (msecs_to_jiffies(*(u32 *)(udata + moff)) > 5300 SCX_WATCHDOG_MAX_TIMEOUT) 5301 return -E2BIG; 5302 ops->timeout_ms = *(u32 *)(udata + moff); 5303 return 1; 5304 case offsetof(struct sched_ext_ops, exit_dump_len): 5305 ops->exit_dump_len = 5306 *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN; 5307 return 1; 5308 case offsetof(struct sched_ext_ops, hotplug_seq): 5309 ops->hotplug_seq = *(u64 *)(udata + moff); 5310 return 1; 5311 } 5312 5313 return 0; 5314 } 5315 5316 static int bpf_scx_check_member(const struct btf_type *t, 5317 const struct btf_member *member, 5318 const struct bpf_prog *prog) 5319 { 5320 u32 moff = __btf_member_bit_offset(t, member) / 8; 5321 5322 switch (moff) { 5323 case offsetof(struct sched_ext_ops, init_task): 5324 #ifdef CONFIG_EXT_GROUP_SCHED 5325 case offsetof(struct sched_ext_ops, cgroup_init): 5326 case offsetof(struct sched_ext_ops, cgroup_exit): 5327 case offsetof(struct sched_ext_ops, cgroup_prep_move): 5328 #endif 5329 case offsetof(struct sched_ext_ops, cpu_online): 5330 case offsetof(struct sched_ext_ops, cpu_offline): 5331 case offsetof(struct sched_ext_ops, init): 5332 case offsetof(struct sched_ext_ops, exit): 5333 break; 5334 default: 5335 if (prog->sleepable) 5336 return -EINVAL; 5337 } 5338 5339 return 0; 5340 } 5341 5342 static int bpf_scx_reg(void *kdata, struct bpf_link *link) 5343 { 5344 return scx_enable(kdata, link); 5345 } 5346 5347 static void bpf_scx_unreg(void *kdata, struct bpf_link *link) 5348 { 5349 struct sched_ext_ops *ops = kdata; 5350 struct scx_sched *sch = ops->priv; 5351 5352 scx_disable(SCX_EXIT_UNREG); 5353 kthread_flush_work(&sch->disable_work); 5354 kobject_put(&sch->kobj); 5355 } 5356 5357 static int bpf_scx_init(struct btf *btf) 5358 { 5359 task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]); 5360 5361 return 0; 5362 } 5363 5364 static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link) 5365 { 5366 /* 5367 * sched_ext does not support updating the actively-loaded BPF 5368 * scheduler, as registering a BPF scheduler can always fail if the 5369 * scheduler returns an error code for e.g. ops.init(), ops.init_task(), 5370 * etc. Similarly, we can always race with unregistration happening 5371 * elsewhere, such as with sysrq. 5372 */ 5373 return -EOPNOTSUPP; 5374 } 5375 5376 static int bpf_scx_validate(void *kdata) 5377 { 5378 return 0; 5379 } 5380 5381 static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; } 5382 static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {} 5383 static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {} 5384 static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {} 5385 static void sched_ext_ops__tick(struct task_struct *p) {} 5386 static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {} 5387 static void sched_ext_ops__running(struct task_struct *p) {} 5388 static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {} 5389 static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {} 5390 static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; } 5391 static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; } 5392 static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {} 5393 static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {} 5394 static void sched_ext_ops__update_idle(s32 cpu, bool idle) {} 5395 static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {} 5396 static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {} 5397 static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; } 5398 static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {} 5399 static void sched_ext_ops__enable(struct task_struct *p) {} 5400 static void sched_ext_ops__disable(struct task_struct *p) {} 5401 #ifdef CONFIG_EXT_GROUP_SCHED 5402 static s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; } 5403 static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {} 5404 static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; } 5405 static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} 5406 static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} 5407 static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {} 5408 static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {} 5409 static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {} 5410 #endif 5411 static void sched_ext_ops__cpu_online(s32 cpu) {} 5412 static void sched_ext_ops__cpu_offline(s32 cpu) {} 5413 static s32 sched_ext_ops__init(void) { return -EINVAL; } 5414 static void sched_ext_ops__exit(struct scx_exit_info *info) {} 5415 static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {} 5416 static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {} 5417 static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {} 5418 5419 static struct sched_ext_ops __bpf_ops_sched_ext_ops = { 5420 .select_cpu = sched_ext_ops__select_cpu, 5421 .enqueue = sched_ext_ops__enqueue, 5422 .dequeue = sched_ext_ops__dequeue, 5423 .dispatch = sched_ext_ops__dispatch, 5424 .tick = sched_ext_ops__tick, 5425 .runnable = sched_ext_ops__runnable, 5426 .running = sched_ext_ops__running, 5427 .stopping = sched_ext_ops__stopping, 5428 .quiescent = sched_ext_ops__quiescent, 5429 .yield = sched_ext_ops__yield, 5430 .core_sched_before = sched_ext_ops__core_sched_before, 5431 .set_weight = sched_ext_ops__set_weight, 5432 .set_cpumask = sched_ext_ops__set_cpumask, 5433 .update_idle = sched_ext_ops__update_idle, 5434 .cpu_acquire = sched_ext_ops__cpu_acquire, 5435 .cpu_release = sched_ext_ops__cpu_release, 5436 .init_task = sched_ext_ops__init_task, 5437 .exit_task = sched_ext_ops__exit_task, 5438 .enable = sched_ext_ops__enable, 5439 .disable = sched_ext_ops__disable, 5440 #ifdef CONFIG_EXT_GROUP_SCHED 5441 .cgroup_init = sched_ext_ops__cgroup_init, 5442 .cgroup_exit = sched_ext_ops__cgroup_exit, 5443 .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, 5444 .cgroup_move = sched_ext_ops__cgroup_move, 5445 .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, 5446 .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, 5447 .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, 5448 .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, 5449 #endif 5450 .cpu_online = sched_ext_ops__cpu_online, 5451 .cpu_offline = sched_ext_ops__cpu_offline, 5452 .init = sched_ext_ops__init, 5453 .exit = sched_ext_ops__exit, 5454 .dump = sched_ext_ops__dump, 5455 .dump_cpu = sched_ext_ops__dump_cpu, 5456 .dump_task = sched_ext_ops__dump_task, 5457 }; 5458 5459 static struct bpf_struct_ops bpf_sched_ext_ops = { 5460 .verifier_ops = &bpf_scx_verifier_ops, 5461 .reg = bpf_scx_reg, 5462 .unreg = bpf_scx_unreg, 5463 .check_member = bpf_scx_check_member, 5464 .init_member = bpf_scx_init_member, 5465 .init = bpf_scx_init, 5466 .update = bpf_scx_update, 5467 .validate = bpf_scx_validate, 5468 .name = "sched_ext_ops", 5469 .owner = THIS_MODULE, 5470 .cfi_stubs = &__bpf_ops_sched_ext_ops 5471 }; 5472 5473 5474 /******************************************************************************** 5475 * System integration and init. 5476 */ 5477 5478 static void sysrq_handle_sched_ext_reset(u8 key) 5479 { 5480 scx_disable(SCX_EXIT_SYSRQ); 5481 } 5482 5483 static const struct sysrq_key_op sysrq_sched_ext_reset_op = { 5484 .handler = sysrq_handle_sched_ext_reset, 5485 .help_msg = "reset-sched-ext(S)", 5486 .action_msg = "Disable sched_ext and revert all tasks to CFS", 5487 .enable_mask = SYSRQ_ENABLE_RTNICE, 5488 }; 5489 5490 static void sysrq_handle_sched_ext_dump(u8 key) 5491 { 5492 struct scx_exit_info ei = { .kind = SCX_EXIT_NONE, .reason = "SysRq-D" }; 5493 5494 if (scx_enabled()) 5495 scx_dump_state(&ei, 0); 5496 } 5497 5498 static const struct sysrq_key_op sysrq_sched_ext_dump_op = { 5499 .handler = sysrq_handle_sched_ext_dump, 5500 .help_msg = "dump-sched-ext(D)", 5501 .action_msg = "Trigger sched_ext debug dump", 5502 .enable_mask = SYSRQ_ENABLE_RTNICE, 5503 }; 5504 5505 static bool can_skip_idle_kick(struct rq *rq) 5506 { 5507 lockdep_assert_rq_held(rq); 5508 5509 /* 5510 * We can skip idle kicking if @rq is going to go through at least one 5511 * full SCX scheduling cycle before going idle. Just checking whether 5512 * curr is not idle is insufficient because we could be racing 5513 * balance_one() trying to pull the next task from a remote rq, which 5514 * may fail, and @rq may become idle afterwards. 5515 * 5516 * The race window is small and we don't and can't guarantee that @rq is 5517 * only kicked while idle anyway. Skip only when sure. 5518 */ 5519 return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); 5520 } 5521 5522 static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) 5523 { 5524 struct rq *rq = cpu_rq(cpu); 5525 struct scx_rq *this_scx = &this_rq->scx; 5526 const struct sched_class *cur_class; 5527 bool should_wait = false; 5528 unsigned long flags; 5529 5530 raw_spin_rq_lock_irqsave(rq, flags); 5531 cur_class = rq->curr->sched_class; 5532 5533 /* 5534 * During CPU hotplug, a CPU may depend on kicking itself to make 5535 * forward progress. Allow kicking self regardless of online state. If 5536 * @cpu is running a higher class task, we have no control over @cpu. 5537 * Skip kicking. 5538 */ 5539 if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && 5540 !sched_class_above(cur_class, &ext_sched_class)) { 5541 if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { 5542 if (cur_class == &ext_sched_class) 5543 rq->curr->scx.slice = 0; 5544 cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); 5545 } 5546 5547 if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { 5548 if (cur_class == &ext_sched_class) { 5549 ksyncs[cpu] = rq->scx.kick_sync; 5550 should_wait = true; 5551 } else { 5552 cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); 5553 } 5554 } 5555 5556 resched_curr(rq); 5557 } else { 5558 cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); 5559 cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); 5560 } 5561 5562 raw_spin_rq_unlock_irqrestore(rq, flags); 5563 5564 return should_wait; 5565 } 5566 5567 static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) 5568 { 5569 struct rq *rq = cpu_rq(cpu); 5570 unsigned long flags; 5571 5572 raw_spin_rq_lock_irqsave(rq, flags); 5573 5574 if (!can_skip_idle_kick(rq) && 5575 (cpu_online(cpu) || cpu == cpu_of(this_rq))) 5576 resched_curr(rq); 5577 5578 raw_spin_rq_unlock_irqrestore(rq, flags); 5579 } 5580 5581 static void kick_cpus_irq_workfn(struct irq_work *irq_work) 5582 { 5583 struct rq *this_rq = this_rq(); 5584 struct scx_rq *this_scx = &this_rq->scx; 5585 struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); 5586 bool should_wait = false; 5587 unsigned long *ksyncs; 5588 s32 cpu; 5589 5590 if (unlikely(!ksyncs_pcpu)) { 5591 pr_warn_once("kick_cpus_irq_workfn() called with NULL scx_kick_syncs"); 5592 return; 5593 } 5594 5595 ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs; 5596 5597 for_each_cpu(cpu, this_scx->cpus_to_kick) { 5598 should_wait |= kick_one_cpu(cpu, this_rq, ksyncs); 5599 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); 5600 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); 5601 } 5602 5603 for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { 5604 kick_one_cpu_if_idle(cpu, this_rq); 5605 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); 5606 } 5607 5608 if (!should_wait) 5609 return; 5610 5611 for_each_cpu(cpu, this_scx->cpus_to_wait) { 5612 unsigned long *wait_kick_sync = &cpu_rq(cpu)->scx.kick_sync; 5613 5614 /* 5615 * Busy-wait until the task running at the time of kicking is no 5616 * longer running. This can be used to implement e.g. core 5617 * scheduling. 5618 * 5619 * smp_cond_load_acquire() pairs with store_releases in 5620 * pick_task_scx() and put_prev_task_scx(). The former breaks 5621 * the wait if SCX's scheduling path is entered even if the same 5622 * task is picked subsequently. The latter is necessary to break 5623 * the wait when $cpu is taken by a higher sched class. 5624 */ 5625 if (cpu != cpu_of(this_rq)) 5626 smp_cond_load_acquire(wait_kick_sync, VAL != ksyncs[cpu]); 5627 5628 cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); 5629 } 5630 } 5631 5632 /** 5633 * print_scx_info - print out sched_ext scheduler state 5634 * @log_lvl: the log level to use when printing 5635 * @p: target task 5636 * 5637 * If a sched_ext scheduler is enabled, print the name and state of the 5638 * scheduler. If @p is on sched_ext, print further information about the task. 5639 * 5640 * This function can be safely called on any task as long as the task_struct 5641 * itself is accessible. While safe, this function isn't synchronized and may 5642 * print out mixups or garbages of limited length. 5643 */ 5644 void print_scx_info(const char *log_lvl, struct task_struct *p) 5645 { 5646 struct scx_sched *sch = scx_root; 5647 enum scx_enable_state state = scx_enable_state(); 5648 const char *all = READ_ONCE(scx_switching_all) ? "+all" : ""; 5649 char runnable_at_buf[22] = "?"; 5650 struct sched_class *class; 5651 unsigned long runnable_at; 5652 5653 if (state == SCX_DISABLED) 5654 return; 5655 5656 /* 5657 * Carefully check if the task was running on sched_ext, and then 5658 * carefully copy the time it's been runnable, and its state. 5659 */ 5660 if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) || 5661 class != &ext_sched_class) { 5662 printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name, 5663 scx_enable_state_str[state], all); 5664 return; 5665 } 5666 5667 if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, 5668 sizeof(runnable_at))) 5669 scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms", 5670 jiffies_delta_msecs(runnable_at, jiffies)); 5671 5672 /* print everything onto one line to conserve console space */ 5673 printk("%sSched_ext: %s (%s%s), task: runnable_at=%s", 5674 log_lvl, sch->ops.name, scx_enable_state_str[state], all, 5675 runnable_at_buf); 5676 } 5677 5678 static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr) 5679 { 5680 /* 5681 * SCX schedulers often have userspace components which are sometimes 5682 * involved in critial scheduling paths. PM operations involve freezing 5683 * userspace which can lead to scheduling misbehaviors including stalls. 5684 * Let's bypass while PM operations are in progress. 5685 */ 5686 switch (event) { 5687 case PM_HIBERNATION_PREPARE: 5688 case PM_SUSPEND_PREPARE: 5689 case PM_RESTORE_PREPARE: 5690 scx_bypass(true); 5691 break; 5692 case PM_POST_HIBERNATION: 5693 case PM_POST_SUSPEND: 5694 case PM_POST_RESTORE: 5695 scx_bypass(false); 5696 break; 5697 } 5698 5699 return NOTIFY_OK; 5700 } 5701 5702 static struct notifier_block scx_pm_notifier = { 5703 .notifier_call = scx_pm_handler, 5704 }; 5705 5706 void __init init_sched_ext_class(void) 5707 { 5708 s32 cpu, v; 5709 5710 /* 5711 * The following is to prevent the compiler from optimizing out the enum 5712 * definitions so that BPF scheduler implementations can use them 5713 * through the generated vmlinux.h. 5714 */ 5715 WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT | 5716 SCX_TG_ONLINE); 5717 5718 scx_idle_init_masks(); 5719 5720 for_each_possible_cpu(cpu) { 5721 struct rq *rq = cpu_rq(cpu); 5722 int n = cpu_to_node(cpu); 5723 5724 init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL); 5725 init_dsq(&rq->scx.bypass_dsq, SCX_DSQ_BYPASS); 5726 INIT_LIST_HEAD(&rq->scx.runnable_list); 5727 INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); 5728 5729 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n)); 5730 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); 5731 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); 5732 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); 5733 rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); 5734 rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); 5735 5736 if (cpu_online(cpu)) 5737 cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; 5738 } 5739 5740 register_sysrq_key('S', &sysrq_sched_ext_reset_op); 5741 register_sysrq_key('D', &sysrq_sched_ext_dump_op); 5742 INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); 5743 } 5744 5745 5746 /******************************************************************************** 5747 * Helpers that can be called from the BPF scheduler. 5748 */ 5749 static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p, 5750 u64 enq_flags) 5751 { 5752 if (!scx_kf_allowed(sch, SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) 5753 return false; 5754 5755 lockdep_assert_irqs_disabled(); 5756 5757 if (unlikely(!p)) { 5758 scx_error(sch, "called with NULL task"); 5759 return false; 5760 } 5761 5762 if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { 5763 scx_error(sch, "invalid enq_flags 0x%llx", enq_flags); 5764 return false; 5765 } 5766 5767 return true; 5768 } 5769 5770 static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, 5771 u64 dsq_id, u64 enq_flags) 5772 { 5773 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 5774 struct task_struct *ddsp_task; 5775 5776 ddsp_task = __this_cpu_read(direct_dispatch_task); 5777 if (ddsp_task) { 5778 mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags); 5779 return; 5780 } 5781 5782 if (unlikely(dspc->cursor >= scx_dsp_max_batch)) { 5783 scx_error(sch, "dispatch buffer overflow"); 5784 return; 5785 } 5786 5787 dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){ 5788 .task = p, 5789 .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, 5790 .dsq_id = dsq_id, 5791 .enq_flags = enq_flags, 5792 }; 5793 } 5794 5795 __bpf_kfunc_start_defs(); 5796 5797 /** 5798 * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ 5799 * @p: task_struct to insert 5800 * @dsq_id: DSQ to insert into 5801 * @slice: duration @p can run for in nsecs, 0 to keep the current value 5802 * @enq_flags: SCX_ENQ_* 5803 * 5804 * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to 5805 * call this function spuriously. Can be called from ops.enqueue(), 5806 * ops.select_cpu(), and ops.dispatch(). 5807 * 5808 * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch 5809 * and @p must match the task being enqueued. 5810 * 5811 * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p 5812 * will be directly inserted into the corresponding dispatch queue after 5813 * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be 5814 * inserted into the local DSQ of the CPU returned by ops.select_cpu(). 5815 * @enq_flags are OR'd with the enqueue flags on the enqueue path before the 5816 * task is inserted. 5817 * 5818 * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id 5819 * and this function can be called upto ops.dispatch_max_batch times to insert 5820 * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the 5821 * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the 5822 * counter. 5823 * 5824 * This function doesn't have any locking restrictions and may be called under 5825 * BPF locks (in the future when BPF introduces more flexible locking). 5826 * 5827 * @p is allowed to run for @slice. The scheduling path is triggered on slice 5828 * exhaustion. If zero, the current residual slice is maintained. If 5829 * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with 5830 * scx_bpf_kick_cpu() to trigger scheduling. 5831 * 5832 * Returns %true on successful insertion, %false on failure. On the root 5833 * scheduler, %false return triggers scheduler abort and the caller doesn't need 5834 * to check the return value. 5835 */ 5836 __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, 5837 u64 slice, u64 enq_flags) 5838 { 5839 struct scx_sched *sch; 5840 5841 guard(rcu)(); 5842 sch = rcu_dereference(scx_root); 5843 if (unlikely(!sch)) 5844 return false; 5845 5846 if (!scx_dsq_insert_preamble(sch, p, enq_flags)) 5847 return false; 5848 5849 if (slice) 5850 p->scx.slice = slice; 5851 else 5852 p->scx.slice = p->scx.slice ?: 1; 5853 5854 scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); 5855 5856 return true; 5857 } 5858 5859 /* 5860 * COMPAT: Will be removed in v6.23 along with the ___v2 suffix. 5861 */ 5862 __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, 5863 u64 slice, u64 enq_flags) 5864 { 5865 scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags); 5866 } 5867 5868 static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, 5869 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) 5870 { 5871 if (!scx_dsq_insert_preamble(sch, p, enq_flags)) 5872 return false; 5873 5874 if (slice) 5875 p->scx.slice = slice; 5876 else 5877 p->scx.slice = p->scx.slice ?: 1; 5878 5879 p->scx.dsq_vtime = vtime; 5880 5881 scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); 5882 5883 return true; 5884 } 5885 5886 struct scx_bpf_dsq_insert_vtime_args { 5887 /* @p can't be packed together as KF_RCU is not transitive */ 5888 u64 dsq_id; 5889 u64 slice; 5890 u64 vtime; 5891 u64 enq_flags; 5892 }; 5893 5894 /** 5895 * __scx_bpf_dsq_insert_vtime - Arg-wrapped vtime DSQ insertion 5896 * @p: task_struct to insert 5897 * @args: struct containing the rest of the arguments 5898 * @args->dsq_id: DSQ to insert into 5899 * @args->slice: duration @p can run for in nsecs, 0 to keep the current value 5900 * @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ 5901 * @args->enq_flags: SCX_ENQ_* 5902 * 5903 * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument 5904 * limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided 5905 * as an inline wrapper in common.bpf.h. 5906 * 5907 * Insert @p into the vtime priority queue of the DSQ identified by 5908 * @args->dsq_id. Tasks queued into the priority queue are ordered by 5909 * @args->vtime. All other aspects are identical to scx_bpf_dsq_insert(). 5910 * 5911 * @args->vtime ordering is according to time_before64() which considers 5912 * wrapping. A numerically larger vtime may indicate an earlier position in the 5913 * ordering and vice-versa. 5914 * 5915 * A DSQ can only be used as a FIFO or priority queue at any given time and this 5916 * function must not be called on a DSQ which already has one or more FIFO tasks 5917 * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and 5918 * SCX_DSQ_GLOBAL) cannot be used as priority queues. 5919 * 5920 * Returns %true on successful insertion, %false on failure. On the root 5921 * scheduler, %false return triggers scheduler abort and the caller doesn't need 5922 * to check the return value. 5923 */ 5924 __bpf_kfunc bool 5925 __scx_bpf_dsq_insert_vtime(struct task_struct *p, 5926 struct scx_bpf_dsq_insert_vtime_args *args) 5927 { 5928 struct scx_sched *sch; 5929 5930 guard(rcu)(); 5931 5932 sch = rcu_dereference(scx_root); 5933 if (unlikely(!sch)) 5934 return false; 5935 5936 return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice, 5937 args->vtime, args->enq_flags); 5938 } 5939 5940 /* 5941 * COMPAT: Will be removed in v6.23. 5942 */ 5943 __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, 5944 u64 slice, u64 vtime, u64 enq_flags) 5945 { 5946 struct scx_sched *sch; 5947 5948 guard(rcu)(); 5949 5950 sch = rcu_dereference(scx_root); 5951 if (unlikely(!sch)) 5952 return; 5953 5954 scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags); 5955 } 5956 5957 __bpf_kfunc_end_defs(); 5958 5959 BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch) 5960 BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_RCU) 5961 BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_RCU) 5962 BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_RCU) 5963 BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU) 5964 BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch) 5965 5966 static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { 5967 .owner = THIS_MODULE, 5968 .set = &scx_kfunc_ids_enqueue_dispatch, 5969 }; 5970 5971 static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, 5972 struct task_struct *p, u64 dsq_id, u64 enq_flags) 5973 { 5974 struct scx_sched *sch = scx_root; 5975 struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; 5976 struct rq *this_rq, *src_rq, *locked_rq; 5977 bool dispatched = false; 5978 bool in_balance; 5979 unsigned long flags; 5980 5981 if (!scx_kf_allowed_if_unlocked() && 5982 !scx_kf_allowed(sch, SCX_KF_DISPATCH)) 5983 return false; 5984 5985 /* 5986 * If the BPF scheduler keeps calling this function repeatedly, it can 5987 * cause similar live-lock conditions as consume_dispatch_q(). 5988 */ 5989 if (unlikely(READ_ONCE(scx_aborting))) 5990 return false; 5991 5992 /* 5993 * Can be called from either ops.dispatch() locking this_rq() or any 5994 * context where no rq lock is held. If latter, lock @p's task_rq which 5995 * we'll likely need anyway. 5996 */ 5997 src_rq = task_rq(p); 5998 5999 local_irq_save(flags); 6000 this_rq = this_rq(); 6001 in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; 6002 6003 if (in_balance) { 6004 if (this_rq != src_rq) { 6005 raw_spin_rq_unlock(this_rq); 6006 raw_spin_rq_lock(src_rq); 6007 } 6008 } else { 6009 raw_spin_rq_lock(src_rq); 6010 } 6011 6012 locked_rq = src_rq; 6013 raw_spin_lock(&src_dsq->lock); 6014 6015 /* 6016 * Did someone else get to it? @p could have already left $src_dsq, got 6017 * re-enqueud, or be in the process of being consumed by someone else. 6018 */ 6019 if (unlikely(p->scx.dsq != src_dsq || 6020 u32_before(kit->cursor.priv, p->scx.dsq_seq) || 6021 p->scx.holding_cpu >= 0) || 6022 WARN_ON_ONCE(src_rq != task_rq(p))) { 6023 raw_spin_unlock(&src_dsq->lock); 6024 goto out; 6025 } 6026 6027 /* @p is still on $src_dsq and stable, determine the destination */ 6028 dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, p); 6029 6030 /* 6031 * Apply vtime and slice updates before moving so that the new time is 6032 * visible before inserting into $dst_dsq. @p is still on $src_dsq but 6033 * this is safe as we're locking it. 6034 */ 6035 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) 6036 p->scx.dsq_vtime = kit->vtime; 6037 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) 6038 p->scx.slice = kit->slice; 6039 6040 /* execute move */ 6041 locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); 6042 dispatched = true; 6043 out: 6044 if (in_balance) { 6045 if (this_rq != locked_rq) { 6046 raw_spin_rq_unlock(locked_rq); 6047 raw_spin_rq_lock(this_rq); 6048 } 6049 } else { 6050 raw_spin_rq_unlock_irqrestore(locked_rq, flags); 6051 } 6052 6053 kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | 6054 __SCX_DSQ_ITER_HAS_VTIME); 6055 return dispatched; 6056 } 6057 6058 __bpf_kfunc_start_defs(); 6059 6060 /** 6061 * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots 6062 * 6063 * Can only be called from ops.dispatch(). 6064 */ 6065 __bpf_kfunc u32 scx_bpf_dispatch_nr_slots(void) 6066 { 6067 struct scx_sched *sch; 6068 6069 guard(rcu)(); 6070 6071 sch = rcu_dereference(scx_root); 6072 if (unlikely(!sch)) 6073 return 0; 6074 6075 if (!scx_kf_allowed(sch, SCX_KF_DISPATCH)) 6076 return 0; 6077 6078 return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx->cursor); 6079 } 6080 6081 /** 6082 * scx_bpf_dispatch_cancel - Cancel the latest dispatch 6083 * 6084 * Cancel the latest dispatch. Can be called multiple times to cancel further 6085 * dispatches. Can only be called from ops.dispatch(). 6086 */ 6087 __bpf_kfunc void scx_bpf_dispatch_cancel(void) 6088 { 6089 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 6090 struct scx_sched *sch; 6091 6092 guard(rcu)(); 6093 6094 sch = rcu_dereference(scx_root); 6095 if (unlikely(!sch)) 6096 return; 6097 6098 if (!scx_kf_allowed(sch, SCX_KF_DISPATCH)) 6099 return; 6100 6101 if (dspc->cursor > 0) 6102 dspc->cursor--; 6103 else 6104 scx_error(sch, "dispatch buffer underflow"); 6105 } 6106 6107 /** 6108 * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ 6109 * @dsq_id: DSQ to move task from 6110 * 6111 * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's 6112 * local DSQ for execution. Can only be called from ops.dispatch(). 6113 * 6114 * This function flushes the in-flight dispatches from scx_bpf_dsq_insert() 6115 * before trying to move from the specified DSQ. It may also grab rq locks and 6116 * thus can't be called under any BPF locks. 6117 * 6118 * Returns %true if a task has been moved, %false if there isn't any task to 6119 * move. 6120 */ 6121 __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id) 6122 { 6123 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 6124 struct scx_dispatch_q *dsq; 6125 struct scx_sched *sch; 6126 6127 guard(rcu)(); 6128 6129 sch = rcu_dereference(scx_root); 6130 if (unlikely(!sch)) 6131 return false; 6132 6133 if (!scx_kf_allowed(sch, SCX_KF_DISPATCH)) 6134 return false; 6135 6136 flush_dispatch_buf(sch, dspc->rq); 6137 6138 dsq = find_user_dsq(sch, dsq_id); 6139 if (unlikely(!dsq)) { 6140 scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id); 6141 return false; 6142 } 6143 6144 if (consume_dispatch_q(sch, dspc->rq, dsq)) { 6145 /* 6146 * A successfully consumed task can be dequeued before it starts 6147 * running while the CPU is trying to migrate other dispatched 6148 * tasks. Bump nr_tasks to tell balance_one() to retry on empty 6149 * local DSQ. 6150 */ 6151 dspc->nr_tasks++; 6152 return true; 6153 } else { 6154 return false; 6155 } 6156 } 6157 6158 /** 6159 * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs 6160 * @it__iter: DSQ iterator in progress 6161 * @slice: duration the moved task can run for in nsecs 6162 * 6163 * Override the slice of the next task that will be moved from @it__iter using 6164 * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous 6165 * slice duration is kept. 6166 */ 6167 __bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, 6168 u64 slice) 6169 { 6170 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; 6171 6172 kit->slice = slice; 6173 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE; 6174 } 6175 6176 /** 6177 * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs 6178 * @it__iter: DSQ iterator in progress 6179 * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ 6180 * 6181 * Override the vtime of the next task that will be moved from @it__iter using 6182 * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice 6183 * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the 6184 * override is ignored and cleared. 6185 */ 6186 __bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, 6187 u64 vtime) 6188 { 6189 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; 6190 6191 kit->vtime = vtime; 6192 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME; 6193 } 6194 6195 /** 6196 * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ 6197 * @it__iter: DSQ iterator in progress 6198 * @p: task to transfer 6199 * @dsq_id: DSQ to move @p to 6200 * @enq_flags: SCX_ENQ_* 6201 * 6202 * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ 6203 * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can 6204 * be the destination. 6205 * 6206 * For the transfer to be successful, @p must still be on the DSQ and have been 6207 * queued before the DSQ iteration started. This function doesn't care whether 6208 * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have 6209 * been queued before the iteration started. 6210 * 6211 * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update. 6212 * 6213 * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq 6214 * lock (e.g. BPF timers or SYSCALL programs). 6215 * 6216 * Returns %true if @p has been consumed, %false if @p had already been 6217 * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local 6218 * DSQ. 6219 */ 6220 __bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, 6221 struct task_struct *p, u64 dsq_id, 6222 u64 enq_flags) 6223 { 6224 return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, 6225 p, dsq_id, enq_flags); 6226 } 6227 6228 /** 6229 * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ 6230 * @it__iter: DSQ iterator in progress 6231 * @p: task to transfer 6232 * @dsq_id: DSQ to move @p to 6233 * @enq_flags: SCX_ENQ_* 6234 * 6235 * Transfer @p which is on the DSQ currently iterated by @it__iter to the 6236 * priority queue of the DSQ specified by @dsq_id. The destination must be a 6237 * user DSQ as only user DSQs support priority queue. 6238 * 6239 * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice() 6240 * and scx_bpf_dsq_move_set_vtime() to update. 6241 * 6242 * All other aspects are identical to scx_bpf_dsq_move(). See 6243 * scx_bpf_dsq_insert_vtime() for more information on @vtime. 6244 */ 6245 __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, 6246 struct task_struct *p, u64 dsq_id, 6247 u64 enq_flags) 6248 { 6249 return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, 6250 p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); 6251 } 6252 6253 __bpf_kfunc_end_defs(); 6254 6255 BTF_KFUNCS_START(scx_kfunc_ids_dispatch) 6256 BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) 6257 BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel) 6258 BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local) 6259 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) 6260 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) 6261 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) 6262 BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) 6263 BTF_KFUNCS_END(scx_kfunc_ids_dispatch) 6264 6265 static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { 6266 .owner = THIS_MODULE, 6267 .set = &scx_kfunc_ids_dispatch, 6268 }; 6269 6270 static u32 reenq_local(struct rq *rq) 6271 { 6272 LIST_HEAD(tasks); 6273 u32 nr_enqueued = 0; 6274 struct task_struct *p, *n; 6275 6276 lockdep_assert_rq_held(rq); 6277 6278 /* 6279 * The BPF scheduler may choose to dispatch tasks back to 6280 * @rq->scx.local_dsq. Move all candidate tasks off to a private list 6281 * first to avoid processing the same tasks repeatedly. 6282 */ 6283 list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, 6284 scx.dsq_list.node) { 6285 /* 6286 * If @p is being migrated, @p's current CPU may not agree with 6287 * its allowed CPUs and the migration_cpu_stop is about to 6288 * deactivate and re-activate @p anyway. Skip re-enqueueing. 6289 * 6290 * While racing sched property changes may also dequeue and 6291 * re-enqueue a migrating task while its current CPU and allowed 6292 * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to 6293 * the current local DSQ for running tasks and thus are not 6294 * visible to the BPF scheduler. 6295 */ 6296 if (p->migration_pending) 6297 continue; 6298 6299 dispatch_dequeue(rq, p); 6300 list_add_tail(&p->scx.dsq_list.node, &tasks); 6301 } 6302 6303 list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { 6304 list_del_init(&p->scx.dsq_list.node); 6305 do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); 6306 nr_enqueued++; 6307 } 6308 6309 return nr_enqueued; 6310 } 6311 6312 __bpf_kfunc_start_defs(); 6313 6314 /** 6315 * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ 6316 * 6317 * Iterate over all of the tasks currently enqueued on the local DSQ of the 6318 * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of 6319 * processed tasks. Can only be called from ops.cpu_release(). 6320 * 6321 * COMPAT: Will be removed in v6.23 along with the ___v2 suffix on the void 6322 * returning variant that can be called from anywhere. 6323 */ 6324 __bpf_kfunc u32 scx_bpf_reenqueue_local(void) 6325 { 6326 struct scx_sched *sch; 6327 struct rq *rq; 6328 6329 guard(rcu)(); 6330 sch = rcu_dereference(scx_root); 6331 if (unlikely(!sch)) 6332 return 0; 6333 6334 if (!scx_kf_allowed(sch, SCX_KF_CPU_RELEASE)) 6335 return 0; 6336 6337 rq = cpu_rq(smp_processor_id()); 6338 lockdep_assert_rq_held(rq); 6339 6340 return reenq_local(rq); 6341 } 6342 6343 __bpf_kfunc_end_defs(); 6344 6345 BTF_KFUNCS_START(scx_kfunc_ids_cpu_release) 6346 BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) 6347 BTF_KFUNCS_END(scx_kfunc_ids_cpu_release) 6348 6349 static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { 6350 .owner = THIS_MODULE, 6351 .set = &scx_kfunc_ids_cpu_release, 6352 }; 6353 6354 __bpf_kfunc_start_defs(); 6355 6356 /** 6357 * scx_bpf_create_dsq - Create a custom DSQ 6358 * @dsq_id: DSQ to create 6359 * @node: NUMA node to allocate from 6360 * 6361 * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable 6362 * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. 6363 */ 6364 __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) 6365 { 6366 struct scx_dispatch_q *dsq; 6367 struct scx_sched *sch; 6368 s32 ret; 6369 6370 if (unlikely(node >= (int)nr_node_ids || 6371 (node < 0 && node != NUMA_NO_NODE))) 6372 return -EINVAL; 6373 6374 if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) 6375 return -EINVAL; 6376 6377 dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); 6378 if (!dsq) 6379 return -ENOMEM; 6380 6381 init_dsq(dsq, dsq_id); 6382 6383 rcu_read_lock(); 6384 6385 sch = rcu_dereference(scx_root); 6386 if (sch) 6387 ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node, 6388 dsq_hash_params); 6389 else 6390 ret = -ENODEV; 6391 6392 rcu_read_unlock(); 6393 if (ret) 6394 kfree(dsq); 6395 return ret; 6396 } 6397 6398 __bpf_kfunc_end_defs(); 6399 6400 BTF_KFUNCS_START(scx_kfunc_ids_unlocked) 6401 BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_SLEEPABLE) 6402 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) 6403 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) 6404 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) 6405 BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) 6406 BTF_KFUNCS_END(scx_kfunc_ids_unlocked) 6407 6408 static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = { 6409 .owner = THIS_MODULE, 6410 .set = &scx_kfunc_ids_unlocked, 6411 }; 6412 6413 __bpf_kfunc_start_defs(); 6414 6415 /** 6416 * scx_bpf_task_set_slice - Set task's time slice 6417 * @p: task of interest 6418 * @slice: time slice to set in nsecs 6419 * 6420 * Set @p's time slice to @slice. Returns %true on success, %false if the 6421 * calling scheduler doesn't have authority over @p. 6422 */ 6423 __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice) 6424 { 6425 p->scx.slice = slice; 6426 return true; 6427 } 6428 6429 /** 6430 * scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering 6431 * @p: task of interest 6432 * @vtime: virtual time to set 6433 * 6434 * Set @p's virtual time to @vtime. Returns %true on success, %false if the 6435 * calling scheduler doesn't have authority over @p. 6436 */ 6437 __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime) 6438 { 6439 p->scx.dsq_vtime = vtime; 6440 return true; 6441 } 6442 6443 static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) 6444 { 6445 struct rq *this_rq; 6446 unsigned long irq_flags; 6447 6448 if (!ops_cpu_valid(sch, cpu, NULL)) 6449 return; 6450 6451 local_irq_save(irq_flags); 6452 6453 this_rq = this_rq(); 6454 6455 /* 6456 * While bypassing for PM ops, IRQ handling may not be online which can 6457 * lead to irq_work_queue() malfunction such as infinite busy wait for 6458 * IRQ status update. Suppress kicking. 6459 */ 6460 if (scx_rq_bypassing(this_rq)) 6461 goto out; 6462 6463 /* 6464 * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting 6465 * rq locks. We can probably be smarter and avoid bouncing if called 6466 * from ops which don't hold a rq lock. 6467 */ 6468 if (flags & SCX_KICK_IDLE) { 6469 struct rq *target_rq = cpu_rq(cpu); 6470 6471 if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) 6472 scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); 6473 6474 if (raw_spin_rq_trylock(target_rq)) { 6475 if (can_skip_idle_kick(target_rq)) { 6476 raw_spin_rq_unlock(target_rq); 6477 goto out; 6478 } 6479 raw_spin_rq_unlock(target_rq); 6480 } 6481 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); 6482 } else { 6483 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); 6484 6485 if (flags & SCX_KICK_PREEMPT) 6486 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); 6487 if (flags & SCX_KICK_WAIT) 6488 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); 6489 } 6490 6491 irq_work_queue(&this_rq->scx.kick_cpus_irq_work); 6492 out: 6493 local_irq_restore(irq_flags); 6494 } 6495 6496 /** 6497 * scx_bpf_kick_cpu - Trigger reschedule on a CPU 6498 * @cpu: cpu to kick 6499 * @flags: %SCX_KICK_* flags 6500 * 6501 * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or 6502 * trigger rescheduling on a busy CPU. This can be called from any online 6503 * scx_ops operation and the actual kicking is performed asynchronously through 6504 * an irq work. 6505 */ 6506 __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags) 6507 { 6508 struct scx_sched *sch; 6509 6510 guard(rcu)(); 6511 sch = rcu_dereference(scx_root); 6512 if (likely(sch)) 6513 scx_kick_cpu(sch, cpu, flags); 6514 } 6515 6516 /** 6517 * scx_bpf_dsq_nr_queued - Return the number of queued tasks 6518 * @dsq_id: id of the DSQ 6519 * 6520 * Return the number of tasks in the DSQ matching @dsq_id. If not found, 6521 * -%ENOENT is returned. 6522 */ 6523 __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id) 6524 { 6525 struct scx_sched *sch; 6526 struct scx_dispatch_q *dsq; 6527 s32 ret; 6528 6529 preempt_disable(); 6530 6531 sch = rcu_dereference_sched(scx_root); 6532 if (unlikely(!sch)) { 6533 ret = -ENODEV; 6534 goto out; 6535 } 6536 6537 if (dsq_id == SCX_DSQ_LOCAL) { 6538 ret = READ_ONCE(this_rq()->scx.local_dsq.nr); 6539 goto out; 6540 } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { 6541 s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; 6542 6543 if (ops_cpu_valid(sch, cpu, NULL)) { 6544 ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); 6545 goto out; 6546 } 6547 } else { 6548 dsq = find_user_dsq(sch, dsq_id); 6549 if (dsq) { 6550 ret = READ_ONCE(dsq->nr); 6551 goto out; 6552 } 6553 } 6554 ret = -ENOENT; 6555 out: 6556 preempt_enable(); 6557 return ret; 6558 } 6559 6560 /** 6561 * scx_bpf_destroy_dsq - Destroy a custom DSQ 6562 * @dsq_id: DSQ to destroy 6563 * 6564 * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with 6565 * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is 6566 * empty and no further tasks are dispatched to it. Ignored if called on a DSQ 6567 * which doesn't exist. Can be called from any online scx_ops operations. 6568 */ 6569 __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id) 6570 { 6571 struct scx_sched *sch; 6572 6573 rcu_read_lock(); 6574 sch = rcu_dereference(scx_root); 6575 if (sch) 6576 destroy_dsq(sch, dsq_id); 6577 rcu_read_unlock(); 6578 } 6579 6580 /** 6581 * bpf_iter_scx_dsq_new - Create a DSQ iterator 6582 * @it: iterator to initialize 6583 * @dsq_id: DSQ to iterate 6584 * @flags: %SCX_DSQ_ITER_* 6585 * 6586 * Initialize BPF iterator @it which can be used with bpf_for_each() to walk 6587 * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes 6588 * tasks which are already queued when this function is invoked. 6589 */ 6590 __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, 6591 u64 flags) 6592 { 6593 struct bpf_iter_scx_dsq_kern *kit = (void *)it; 6594 struct scx_sched *sch; 6595 6596 BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) > 6597 sizeof(struct bpf_iter_scx_dsq)); 6598 BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != 6599 __alignof__(struct bpf_iter_scx_dsq)); 6600 BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & 6601 ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1)); 6602 6603 /* 6604 * next() and destroy() will be called regardless of the return value. 6605 * Always clear $kit->dsq. 6606 */ 6607 kit->dsq = NULL; 6608 6609 sch = rcu_dereference_check(scx_root, rcu_read_lock_bh_held()); 6610 if (unlikely(!sch)) 6611 return -ENODEV; 6612 6613 if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) 6614 return -EINVAL; 6615 6616 kit->dsq = find_user_dsq(sch, dsq_id); 6617 if (!kit->dsq) 6618 return -ENOENT; 6619 6620 kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, flags, 6621 READ_ONCE(kit->dsq->seq)); 6622 6623 return 0; 6624 } 6625 6626 /** 6627 * bpf_iter_scx_dsq_next - Progress a DSQ iterator 6628 * @it: iterator to progress 6629 * 6630 * Return the next task. See bpf_iter_scx_dsq_new(). 6631 */ 6632 __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) 6633 { 6634 struct bpf_iter_scx_dsq_kern *kit = (void *)it; 6635 bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV; 6636 struct task_struct *p; 6637 unsigned long flags; 6638 6639 if (!kit->dsq) 6640 return NULL; 6641 6642 raw_spin_lock_irqsave(&kit->dsq->lock, flags); 6643 6644 if (list_empty(&kit->cursor.node)) 6645 p = NULL; 6646 else 6647 p = container_of(&kit->cursor, struct task_struct, scx.dsq_list); 6648 6649 /* 6650 * Only tasks which were queued before the iteration started are 6651 * visible. This bounds BPF iterations and guarantees that vtime never 6652 * jumps in the other direction while iterating. 6653 */ 6654 do { 6655 p = nldsq_next_task(kit->dsq, p, rev); 6656 } while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq))); 6657 6658 if (p) { 6659 if (rev) 6660 list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node); 6661 else 6662 list_move(&kit->cursor.node, &p->scx.dsq_list.node); 6663 } else { 6664 list_del_init(&kit->cursor.node); 6665 } 6666 6667 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); 6668 6669 return p; 6670 } 6671 6672 /** 6673 * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator 6674 * @it: iterator to destroy 6675 * 6676 * Undo scx_iter_scx_dsq_new(). 6677 */ 6678 __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) 6679 { 6680 struct bpf_iter_scx_dsq_kern *kit = (void *)it; 6681 6682 if (!kit->dsq) 6683 return; 6684 6685 if (!list_empty(&kit->cursor.node)) { 6686 unsigned long flags; 6687 6688 raw_spin_lock_irqsave(&kit->dsq->lock, flags); 6689 list_del_init(&kit->cursor.node); 6690 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); 6691 } 6692 kit->dsq = NULL; 6693 } 6694 6695 /** 6696 * scx_bpf_dsq_peek - Lockless peek at the first element. 6697 * @dsq_id: DSQ to examine. 6698 * 6699 * Read the first element in the DSQ. This is semantically equivalent to using 6700 * the DSQ iterator, but is lockfree. Of course, like any lockless operation, 6701 * this provides only a point-in-time snapshot, and the contents may change 6702 * by the time any subsequent locking operation reads the queue. 6703 * 6704 * Returns the pointer, or NULL indicates an empty queue OR internal error. 6705 */ 6706 __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id) 6707 { 6708 struct scx_sched *sch; 6709 struct scx_dispatch_q *dsq; 6710 6711 sch = rcu_dereference(scx_root); 6712 if (unlikely(!sch)) 6713 return NULL; 6714 6715 if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) { 6716 scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id); 6717 return NULL; 6718 } 6719 6720 dsq = find_user_dsq(sch, dsq_id); 6721 if (unlikely(!dsq)) { 6722 scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id); 6723 return NULL; 6724 } 6725 6726 return rcu_dereference(dsq->first_task); 6727 } 6728 6729 __bpf_kfunc_end_defs(); 6730 6731 static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, 6732 size_t line_size, char *fmt, unsigned long long *data, 6733 u32 data__sz) 6734 { 6735 struct bpf_bprintf_data bprintf_data = { .get_bin_args = true }; 6736 s32 ret; 6737 6738 if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || 6739 (data__sz && !data)) { 6740 scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz); 6741 return -EINVAL; 6742 } 6743 6744 ret = copy_from_kernel_nofault(data_buf, data, data__sz); 6745 if (ret < 0) { 6746 scx_error(sch, "failed to read data fields (%d)", ret); 6747 return ret; 6748 } 6749 6750 ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8, 6751 &bprintf_data); 6752 if (ret < 0) { 6753 scx_error(sch, "format preparation failed (%d)", ret); 6754 return ret; 6755 } 6756 6757 ret = bstr_printf(line_buf, line_size, fmt, 6758 bprintf_data.bin_args); 6759 bpf_bprintf_cleanup(&bprintf_data); 6760 if (ret < 0) { 6761 scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz); 6762 return ret; 6763 } 6764 6765 return ret; 6766 } 6767 6768 static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, 6769 char *fmt, unsigned long long *data, u32 data__sz) 6770 { 6771 return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line), 6772 fmt, data, data__sz); 6773 } 6774 6775 __bpf_kfunc_start_defs(); 6776 6777 /** 6778 * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler. 6779 * @exit_code: Exit value to pass to user space via struct scx_exit_info. 6780 * @fmt: error message format string 6781 * @data: format string parameters packaged using ___bpf_fill() macro 6782 * @data__sz: @data len, must end in '__sz' for the verifier 6783 * 6784 * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops 6785 * disabling. 6786 */ 6787 __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, 6788 unsigned long long *data, u32 data__sz) 6789 { 6790 struct scx_sched *sch; 6791 unsigned long flags; 6792 6793 raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); 6794 sch = rcu_dereference_bh(scx_root); 6795 if (likely(sch) && 6796 bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) 6797 scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); 6798 raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); 6799 } 6800 6801 /** 6802 * scx_bpf_error_bstr - Indicate fatal error 6803 * @fmt: error message format string 6804 * @data: format string parameters packaged using ___bpf_fill() macro 6805 * @data__sz: @data len, must end in '__sz' for the verifier 6806 * 6807 * Indicate that the BPF scheduler encountered a fatal error and initiate ops 6808 * disabling. 6809 */ 6810 __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, 6811 u32 data__sz) 6812 { 6813 struct scx_sched *sch; 6814 unsigned long flags; 6815 6816 raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); 6817 sch = rcu_dereference_bh(scx_root); 6818 if (likely(sch) && 6819 bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) 6820 scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); 6821 raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); 6822 } 6823 6824 /** 6825 * scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler 6826 * @fmt: format string 6827 * @data: format string parameters packaged using ___bpf_fill() macro 6828 * @data__sz: @data len, must end in '__sz' for the verifier 6829 * 6830 * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and 6831 * dump_task() to generate extra debug dump specific to the BPF scheduler. 6832 * 6833 * The extra dump may be multiple lines. A single line may be split over 6834 * multiple calls. The last line is automatically terminated. 6835 */ 6836 __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, 6837 u32 data__sz) 6838 { 6839 struct scx_sched *sch; 6840 struct scx_dump_data *dd = &scx_dump_data; 6841 struct scx_bstr_buf *buf = &dd->buf; 6842 s32 ret; 6843 6844 guard(rcu)(); 6845 6846 sch = rcu_dereference(scx_root); 6847 if (unlikely(!sch)) 6848 return; 6849 6850 if (raw_smp_processor_id() != dd->cpu) { 6851 scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends"); 6852 return; 6853 } 6854 6855 /* append the formatted string to the line buf */ 6856 ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, 6857 sizeof(buf->line) - dd->cursor, fmt, data, data__sz); 6858 if (ret < 0) { 6859 dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", 6860 dd->prefix, fmt, data, data__sz, ret); 6861 return; 6862 } 6863 6864 dd->cursor += ret; 6865 dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line)); 6866 6867 if (!dd->cursor) 6868 return; 6869 6870 /* 6871 * If the line buf overflowed or ends in a newline, flush it into the 6872 * dump. This is to allow the caller to generate a single line over 6873 * multiple calls. As ops_dump_flush() can also handle multiple lines in 6874 * the line buf, the only case which can lead to an unexpected 6875 * truncation is when the caller keeps generating newlines in the middle 6876 * instead of the end consecutively. Don't do that. 6877 */ 6878 if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n') 6879 ops_dump_flush(); 6880 } 6881 6882 /** 6883 * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ 6884 * 6885 * Iterate over all of the tasks currently enqueued on the local DSQ of the 6886 * caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from 6887 * anywhere. 6888 */ 6889 __bpf_kfunc void scx_bpf_reenqueue_local___v2(void) 6890 { 6891 struct rq *rq; 6892 6893 guard(preempt)(); 6894 6895 rq = this_rq(); 6896 local_set(&rq->scx.reenq_local_deferred, 1); 6897 schedule_deferred(rq); 6898 } 6899 6900 /** 6901 * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU 6902 * @cpu: CPU of interest 6903 * 6904 * Return the maximum relative capacity of @cpu in relation to the most 6905 * performant CPU in the system. The return value is in the range [1, 6906 * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur(). 6907 */ 6908 __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu) 6909 { 6910 struct scx_sched *sch; 6911 6912 guard(rcu)(); 6913 6914 sch = rcu_dereference(scx_root); 6915 if (likely(sch) && ops_cpu_valid(sch, cpu, NULL)) 6916 return arch_scale_cpu_capacity(cpu); 6917 else 6918 return SCX_CPUPERF_ONE; 6919 } 6920 6921 /** 6922 * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU 6923 * @cpu: CPU of interest 6924 * 6925 * Return the current relative performance of @cpu in relation to its maximum. 6926 * The return value is in the range [1, %SCX_CPUPERF_ONE]. 6927 * 6928 * The current performance level of a CPU in relation to the maximum performance 6929 * available in the system can be calculated as follows: 6930 * 6931 * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE 6932 * 6933 * The result is in the range [1, %SCX_CPUPERF_ONE]. 6934 */ 6935 __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu) 6936 { 6937 struct scx_sched *sch; 6938 6939 guard(rcu)(); 6940 6941 sch = rcu_dereference(scx_root); 6942 if (likely(sch) && ops_cpu_valid(sch, cpu, NULL)) 6943 return arch_scale_freq_capacity(cpu); 6944 else 6945 return SCX_CPUPERF_ONE; 6946 } 6947 6948 /** 6949 * scx_bpf_cpuperf_set - Set the relative performance target of a CPU 6950 * @cpu: CPU of interest 6951 * @perf: target performance level [0, %SCX_CPUPERF_ONE] 6952 * 6953 * Set the target performance level of @cpu to @perf. @perf is in linear 6954 * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the 6955 * schedutil cpufreq governor chooses the target frequency. 6956 * 6957 * The actual performance level chosen, CPU grouping, and the overhead and 6958 * latency of the operations are dependent on the hardware and cpufreq driver in 6959 * use. Consult hardware and cpufreq documentation for more information. The 6960 * current performance level can be monitored using scx_bpf_cpuperf_cur(). 6961 */ 6962 __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf) 6963 { 6964 struct scx_sched *sch; 6965 6966 guard(rcu)(); 6967 6968 sch = rcu_dereference(scx_root); 6969 if (unlikely(!sch)) 6970 return; 6971 6972 if (unlikely(perf > SCX_CPUPERF_ONE)) { 6973 scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); 6974 return; 6975 } 6976 6977 if (ops_cpu_valid(sch, cpu, NULL)) { 6978 struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq(); 6979 struct rq_flags rf; 6980 6981 /* 6982 * When called with an rq lock held, restrict the operation 6983 * to the corresponding CPU to prevent ABBA deadlocks. 6984 */ 6985 if (locked_rq && rq != locked_rq) { 6986 scx_error(sch, "Invalid target CPU %d", cpu); 6987 return; 6988 } 6989 6990 /* 6991 * If no rq lock is held, allow to operate on any CPU by 6992 * acquiring the corresponding rq lock. 6993 */ 6994 if (!locked_rq) { 6995 rq_lock_irqsave(rq, &rf); 6996 update_rq_clock(rq); 6997 } 6998 6999 rq->scx.cpuperf_target = perf; 7000 cpufreq_update_util(rq, 0); 7001 7002 if (!locked_rq) 7003 rq_unlock_irqrestore(rq, &rf); 7004 } 7005 } 7006 7007 /** 7008 * scx_bpf_nr_node_ids - Return the number of possible node IDs 7009 * 7010 * All valid node IDs in the system are smaller than the returned value. 7011 */ 7012 __bpf_kfunc u32 scx_bpf_nr_node_ids(void) 7013 { 7014 return nr_node_ids; 7015 } 7016 7017 /** 7018 * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs 7019 * 7020 * All valid CPU IDs in the system are smaller than the returned value. 7021 */ 7022 __bpf_kfunc u32 scx_bpf_nr_cpu_ids(void) 7023 { 7024 return nr_cpu_ids; 7025 } 7026 7027 /** 7028 * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask 7029 */ 7030 __bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void) 7031 { 7032 return cpu_possible_mask; 7033 } 7034 7035 /** 7036 * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask 7037 */ 7038 __bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void) 7039 { 7040 return cpu_online_mask; 7041 } 7042 7043 /** 7044 * scx_bpf_put_cpumask - Release a possible/online cpumask 7045 * @cpumask: cpumask to release 7046 */ 7047 __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) 7048 { 7049 /* 7050 * Empty function body because we aren't actually acquiring or releasing 7051 * a reference to a global cpumask, which is read-only in the caller and 7052 * is never released. The acquire / release semantics here are just used 7053 * to make the cpumask is a trusted pointer in the caller. 7054 */ 7055 } 7056 7057 /** 7058 * scx_bpf_task_running - Is task currently running? 7059 * @p: task of interest 7060 */ 7061 __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) 7062 { 7063 return task_rq(p)->curr == p; 7064 } 7065 7066 /** 7067 * scx_bpf_task_cpu - CPU a task is currently associated with 7068 * @p: task of interest 7069 */ 7070 __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) 7071 { 7072 return task_cpu(p); 7073 } 7074 7075 /** 7076 * scx_bpf_cpu_rq - Fetch the rq of a CPU 7077 * @cpu: CPU of the rq 7078 */ 7079 __bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu) 7080 { 7081 struct scx_sched *sch; 7082 7083 guard(rcu)(); 7084 7085 sch = rcu_dereference(scx_root); 7086 if (unlikely(!sch)) 7087 return NULL; 7088 7089 if (!ops_cpu_valid(sch, cpu, NULL)) 7090 return NULL; 7091 7092 if (!sch->warned_deprecated_rq) { 7093 printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; " 7094 "use scx_bpf_locked_rq() when holding rq lock " 7095 "or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__); 7096 sch->warned_deprecated_rq = true; 7097 } 7098 7099 return cpu_rq(cpu); 7100 } 7101 7102 /** 7103 * scx_bpf_locked_rq - Return the rq currently locked by SCX 7104 * 7105 * Returns the rq if a rq lock is currently held by SCX. 7106 * Otherwise emits an error and returns NULL. 7107 */ 7108 __bpf_kfunc struct rq *scx_bpf_locked_rq(void) 7109 { 7110 struct scx_sched *sch; 7111 struct rq *rq; 7112 7113 guard(preempt)(); 7114 7115 sch = rcu_dereference_sched(scx_root); 7116 if (unlikely(!sch)) 7117 return NULL; 7118 7119 rq = scx_locked_rq(); 7120 if (!rq) { 7121 scx_error(sch, "accessing rq without holding rq lock"); 7122 return NULL; 7123 } 7124 7125 return rq; 7126 } 7127 7128 /** 7129 * scx_bpf_cpu_curr - Return remote CPU's curr task 7130 * @cpu: CPU of interest 7131 * 7132 * Callers must hold RCU read lock (KF_RCU). 7133 */ 7134 __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu) 7135 { 7136 struct scx_sched *sch; 7137 7138 guard(rcu)(); 7139 7140 sch = rcu_dereference(scx_root); 7141 if (unlikely(!sch)) 7142 return NULL; 7143 7144 if (!ops_cpu_valid(sch, cpu, NULL)) 7145 return NULL; 7146 7147 return rcu_dereference(cpu_rq(cpu)->curr); 7148 } 7149 7150 /** 7151 * scx_bpf_task_cgroup - Return the sched cgroup of a task 7152 * @p: task of interest 7153 * 7154 * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with 7155 * from the scheduler's POV. SCX operations should use this function to 7156 * determine @p's current cgroup as, unlike following @p->cgroups, 7157 * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all 7158 * rq-locked operations. Can be called on the parameter tasks of rq-locked 7159 * operations. The restriction guarantees that @p's rq is locked by the caller. 7160 */ 7161 #ifdef CONFIG_CGROUP_SCHED 7162 __bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) 7163 { 7164 struct task_group *tg = p->sched_task_group; 7165 struct cgroup *cgrp = &cgrp_dfl_root.cgrp; 7166 struct scx_sched *sch; 7167 7168 guard(rcu)(); 7169 7170 sch = rcu_dereference(scx_root); 7171 if (unlikely(!sch)) 7172 goto out; 7173 7174 if (!scx_kf_allowed_on_arg_tasks(sch, __SCX_KF_RQ_LOCKED, p)) 7175 goto out; 7176 7177 cgrp = tg_cgrp(tg); 7178 7179 out: 7180 cgroup_get(cgrp); 7181 return cgrp; 7182 } 7183 #endif 7184 7185 /** 7186 * scx_bpf_now - Returns a high-performance monotonically non-decreasing 7187 * clock for the current CPU. The clock returned is in nanoseconds. 7188 * 7189 * It provides the following properties: 7190 * 7191 * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently 7192 * to account for execution time and track tasks' runtime properties. 7193 * Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which 7194 * eventually reads a hardware timestamp counter -- is neither performant nor 7195 * scalable. scx_bpf_now() aims to provide a high-performance clock by 7196 * using the rq clock in the scheduler core whenever possible. 7197 * 7198 * 2) High enough resolution for the BPF scheduler use cases: In most BPF 7199 * scheduler use cases, the required clock resolution is lower than the most 7200 * accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically 7201 * uses the rq clock in the scheduler core whenever it is valid. It considers 7202 * that the rq clock is valid from the time the rq clock is updated 7203 * (update_rq_clock) until the rq is unlocked (rq_unpin_lock). 7204 * 7205 * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now() 7206 * guarantees the clock never goes backward when comparing them in the same 7207 * CPU. On the other hand, when comparing clocks in different CPUs, there 7208 * is no such guarantee -- the clock can go backward. It provides a 7209 * monotonically *non-decreasing* clock so that it would provide the same 7210 * clock values in two different scx_bpf_now() calls in the same CPU 7211 * during the same period of when the rq clock is valid. 7212 */ 7213 __bpf_kfunc u64 scx_bpf_now(void) 7214 { 7215 struct rq *rq; 7216 u64 clock; 7217 7218 preempt_disable(); 7219 7220 rq = this_rq(); 7221 if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { 7222 /* 7223 * If the rq clock is valid, use the cached rq clock. 7224 * 7225 * Note that scx_bpf_now() is re-entrant between a process 7226 * context and an interrupt context (e.g., timer interrupt). 7227 * However, we don't need to consider the race between them 7228 * because such race is not observable from a caller. 7229 */ 7230 clock = READ_ONCE(rq->scx.clock); 7231 } else { 7232 /* 7233 * Otherwise, return a fresh rq clock. 7234 * 7235 * The rq clock is updated outside of the rq lock. 7236 * In this case, keep the updated rq clock invalid so the next 7237 * kfunc call outside the rq lock gets a fresh rq clock. 7238 */ 7239 clock = sched_clock_cpu(cpu_of(rq)); 7240 } 7241 7242 preempt_enable(); 7243 7244 return clock; 7245 } 7246 7247 static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) 7248 { 7249 struct scx_event_stats *e_cpu; 7250 int cpu; 7251 7252 /* Aggregate per-CPU event counters into @events. */ 7253 memset(events, 0, sizeof(*events)); 7254 for_each_possible_cpu(cpu) { 7255 e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; 7256 scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK); 7257 scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); 7258 scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST); 7259 scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING); 7260 scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); 7261 scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL); 7262 scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION); 7263 scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH); 7264 scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE); 7265 } 7266 } 7267 7268 /* 7269 * scx_bpf_events - Get a system-wide event counter to 7270 * @events: output buffer from a BPF program 7271 * @events__sz: @events len, must end in '__sz'' for the verifier 7272 */ 7273 __bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, 7274 size_t events__sz) 7275 { 7276 struct scx_sched *sch; 7277 struct scx_event_stats e_sys; 7278 7279 rcu_read_lock(); 7280 sch = rcu_dereference(scx_root); 7281 if (sch) 7282 scx_read_events(sch, &e_sys); 7283 else 7284 memset(&e_sys, 0, sizeof(e_sys)); 7285 rcu_read_unlock(); 7286 7287 /* 7288 * We cannot entirely trust a BPF-provided size since a BPF program 7289 * might be compiled against a different vmlinux.h, of which 7290 * scx_event_stats would be larger (a newer vmlinux.h) or smaller 7291 * (an older vmlinux.h). Hence, we use the smaller size to avoid 7292 * memory corruption. 7293 */ 7294 events__sz = min(events__sz, sizeof(*events)); 7295 memcpy(events, &e_sys, events__sz); 7296 } 7297 7298 __bpf_kfunc_end_defs(); 7299 7300 BTF_KFUNCS_START(scx_kfunc_ids_any) 7301 BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_RCU); 7302 BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_RCU); 7303 BTF_ID_FLAGS(func, scx_bpf_kick_cpu) 7304 BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) 7305 BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) 7306 BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_RCU_PROTECTED | KF_RET_NULL) 7307 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_ITER_NEW | KF_RCU_PROTECTED) 7308 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL) 7309 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY) 7310 BTF_ID_FLAGS(func, scx_bpf_exit_bstr) 7311 BTF_ID_FLAGS(func, scx_bpf_error_bstr) 7312 BTF_ID_FLAGS(func, scx_bpf_dump_bstr) 7313 BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2) 7314 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap) 7315 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur) 7316 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set) 7317 BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) 7318 BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) 7319 BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) 7320 BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) 7321 BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) 7322 BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) 7323 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) 7324 BTF_ID_FLAGS(func, scx_bpf_cpu_rq) 7325 BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_RET_NULL) 7326 BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_RET_NULL | KF_RCU_PROTECTED) 7327 #ifdef CONFIG_CGROUP_SCHED 7328 BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_RCU | KF_ACQUIRE) 7329 #endif 7330 BTF_ID_FLAGS(func, scx_bpf_now) 7331 BTF_ID_FLAGS(func, scx_bpf_events) 7332 BTF_KFUNCS_END(scx_kfunc_ids_any) 7333 7334 static const struct btf_kfunc_id_set scx_kfunc_set_any = { 7335 .owner = THIS_MODULE, 7336 .set = &scx_kfunc_ids_any, 7337 }; 7338 7339 static int __init scx_init(void) 7340 { 7341 int ret; 7342 7343 /* 7344 * kfunc registration can't be done from init_sched_ext_class() as 7345 * register_btf_kfunc_id_set() needs most of the system to be up. 7346 * 7347 * Some kfuncs are context-sensitive and can only be called from 7348 * specific SCX ops. They are grouped into BTF sets accordingly. 7349 * Unfortunately, BPF currently doesn't have a way of enforcing such 7350 * restrictions. Eventually, the verifier should be able to enforce 7351 * them. For now, register them the same and make each kfunc explicitly 7352 * check using scx_kf_allowed(). 7353 */ 7354 if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7355 &scx_kfunc_set_enqueue_dispatch)) || 7356 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7357 &scx_kfunc_set_dispatch)) || 7358 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7359 &scx_kfunc_set_cpu_release)) || 7360 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7361 &scx_kfunc_set_unlocked)) || 7362 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, 7363 &scx_kfunc_set_unlocked)) || 7364 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7365 &scx_kfunc_set_any)) || 7366 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, 7367 &scx_kfunc_set_any)) || 7368 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, 7369 &scx_kfunc_set_any))) { 7370 pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret); 7371 return ret; 7372 } 7373 7374 ret = scx_idle_init(); 7375 if (ret) { 7376 pr_err("sched_ext: Failed to initialize idle tracking (%d)\n", ret); 7377 return ret; 7378 } 7379 7380 ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); 7381 if (ret) { 7382 pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); 7383 return ret; 7384 } 7385 7386 ret = register_pm_notifier(&scx_pm_notifier); 7387 if (ret) { 7388 pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); 7389 return ret; 7390 } 7391 7392 scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj); 7393 if (!scx_kset) { 7394 pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n"); 7395 return -ENOMEM; 7396 } 7397 7398 ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group); 7399 if (ret < 0) { 7400 pr_err("sched_ext: Failed to add global attributes\n"); 7401 return ret; 7402 } 7403 7404 if (!alloc_cpumask_var(&scx_bypass_lb_donee_cpumask, GFP_KERNEL) || 7405 !alloc_cpumask_var(&scx_bypass_lb_resched_cpumask, GFP_KERNEL)) { 7406 pr_err("sched_ext: Failed to allocate cpumasks\n"); 7407 return -ENOMEM; 7408 } 7409 7410 return 0; 7411 } 7412 __initcall(scx_init); 7413