1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 * 5 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 6 * Copyright (c) 2022 Tejun Heo <tj@kernel.org> 7 * Copyright (c) 2022 David Vernet <dvernet@meta.com> 8 */ 9 #include <linux/bitmap.h> 10 #include <linux/btf_ids.h> 11 #include <linux/rhashtable.h> 12 #include <linux/sched/clock.h> 13 #include <linux/sched/isolation.h> 14 #include <linux/suspend.h> 15 #include <linux/sysrq.h> 16 17 #include "../pelt.h" 18 #include "internal.h" 19 #include "cid.h" 20 #include "arena.h" 21 #include "idle.h" 22 #include "sub.h" 23 #include "inlines.h" 24 25 DEFINE_RAW_SPINLOCK(scx_sched_lock); 26 27 /* 28 * Block a retained proxy donor, telling put_prev_task_scx() that the donor is 29 * about to be dequeued and must not be reenqueued. 30 */ 31 static void scx_block_proxy_donor(struct rq *rq, struct task_struct *p) 32 { 33 rq->scx.flags |= SCX_RQ_PROXY_BLOCKING; 34 sched_proxy_block_task(rq, p); 35 rq->scx.flags &= ~SCX_RQ_PROXY_BLOCKING; 36 } 37 38 bool __scx_allow_proxy_exec(const struct task_struct *p) 39 { 40 struct scx_sched *sch; 41 42 if (p->sched_class != &ext_sched_class) 43 return true; 44 45 sch = scx_task_sched(p); 46 return !sch || (sch->ops.flags & SCX_OPS_ENQ_BLOCKED); 47 } 48 49 /* 50 * End retained proxy execution before changing @p's BPF scheduler ownership. 51 * Called with @p's pi and rq locks held immediately before 52 * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock 53 * is updated only once. 54 */ 55 void scx_prepare_task_sched_change(struct task_struct *p) 56 { 57 lockdep_assert_held(&p->pi_lock); 58 lockdep_assert_rq_held(task_rq(p)); 59 60 update_rq_clock(task_rq(p)); 61 scx_block_proxy_donor(task_rq(p), p); 62 } 63 64 /* 65 * NOTE: sched_ext is in the process of growing multiple scheduler support and 66 * scx_root usage is in a transitional state. Naked dereferences are safe if the 67 * caller is one of the tasks attached to SCX and explicit RCU dereference is 68 * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but 69 * are used as temporary markers to indicate that the dereferences need to be 70 * updated to point to the associated scheduler instances rather than scx_root. 71 */ 72 struct scx_sched __rcu *scx_root; 73 74 /* 75 * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock. 76 * Readers can hold either or rcu_read_lock(). 77 */ 78 LIST_HEAD(scx_sched_all); 79 80 #ifdef CONFIG_EXT_SUB_SCHED 81 static __always_inline int scx_sched_cmpfn(struct rhashtable_compare_arg *arg, 82 const void *ptr) 83 { 84 const struct scx_sched *sch = ptr; 85 86 BUILD_BUG_ON(sizeof_field(struct scx_sched, ops.sub_cgroup_id) != sizeof(u64)); 87 88 return sch->ops.sub_cgroup_id != *(const u64 *)arg->key; 89 } 90 91 const struct rhashtable_params scx_sched_hash_params = { 92 .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), 93 .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), 94 .head_offset = offsetof(struct scx_sched, hash_node), 95 .obj_cmpfn = scx_sched_cmpfn, 96 .insecure_elasticity = true, /* inserted under scx_sched_lock */ 97 }; 98 99 struct rhashtable scx_sched_hash; 100 #endif 101 102 /* see SCX_OPS_TID_TO_TASK */ 103 static __always_inline int scx_tid_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) 104 { 105 const struct sched_ext_entity *scx = ptr; 106 107 BUILD_BUG_ON(sizeof_field(struct sched_ext_entity, tid) != sizeof(u64)); 108 109 return scx->tid != *(const u64 *)arg->key; 110 } 111 112 static const struct rhashtable_params scx_tid_hash_params = { 113 .key_len = sizeof_field(struct sched_ext_entity, tid), 114 .key_offset = offsetof(struct sched_ext_entity, tid), 115 .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), 116 .obj_cmpfn = scx_tid_cmpfn, 117 .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ 118 }; 119 static struct rhashtable scx_tid_hash; 120 121 /* 122 * During exit, a task may schedule after losing its PIDs. When disabling the 123 * BPF scheduler, we need to be able to iterate tasks in every state to 124 * guarantee system safety. Maintain a dedicated task list which contains every 125 * task between its fork and eventual free. 126 */ 127 static DEFINE_RAW_SPINLOCK(scx_tasks_lock); 128 static LIST_HEAD(scx_tasks); 129 130 /* ops enable/disable */ 131 DEFINE_MUTEX(scx_enable_mutex); 132 DEFINE_STATIC_KEY_FALSE(__scx_enabled); 133 DEFINE_PERCPU_RWSEM(scx_fork_rwsem); 134 static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); 135 static DEFINE_RAW_SPINLOCK(scx_bypass_lock); 136 static bool scx_init_task_enabled; 137 static bool scx_switching_all; 138 DEFINE_STATIC_KEY_FALSE(__scx_switched_all); 139 static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled); 140 141 /* 142 * Gates cgroup ops delivery. Set at the end of the cgroup init phase of root 143 * enable and cleared before root disable starts tearing down tasks, both under 144 * scx_cgroup_lock(). Holding cgroup_lock() and seeing %true guarantees no race 145 * against root tearing down tasks. 146 */ 147 bool scx_cgroup_enabled; 148 149 /* 150 * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler 151 * and the tid->task table is live. Wraps the static key so callers don't 152 * take the address, and hints "likely enabled" for the common case where 153 * the feature is in use. 154 */ 155 static inline bool scx_tid_to_task_enabled(void) 156 { 157 return static_branch_likely(&__scx_tid_to_task_enabled); 158 } 159 160 static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); 161 static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); 162 163 /* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */ 164 static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); 165 166 /* is @dsq synchronized by the containing rq lock instead of dsq->lock? */ 167 static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq) 168 { 169 switch (dsq->id) { 170 case SCX_DSQ_LOCAL: 171 case SCX_DSQ_REJECT: 172 case SCX_DSQ_RESCUE: 173 return true; 174 default: 175 return false; 176 } 177 } 178 179 /* Cursor for unique scx_sched instance ids. id 0 is reserved. */ 180 static atomic64_t scx_sched_id_cursor = ATOMIC64_INIT(0); 181 182 #ifdef CONFIG_EXT_SUB_SCHED 183 /* 184 * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit 185 * tasks for the sub-sched being enabled. Use a global variable instead of a 186 * per-task field as all enables are serialized. 187 */ 188 struct scx_sched *scx_enabling_sub_sched; 189 #else 190 #define scx_enabling_sub_sched (struct scx_sched *)NULL 191 #endif /* CONFIG_EXT_SUB_SCHED */ 192 193 /* 194 * A monotonically increasing sequence number that is incremented every time a 195 * scheduler is enabled. This can be used to check if any custom sched_ext 196 * scheduler has ever been used in the system. 197 */ 198 static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); 199 200 /* 201 * Watchdog interval. All scx_sched's share a single watchdog timer and the 202 * interval is half of the shortest sch->watchdog_timeout. 203 */ 204 static unsigned long scx_watchdog_interval; 205 206 /* 207 * The last time the delayed work was run. This delayed work relies on 208 * ksoftirqd being able to run to service timer interrupts, so it's possible 209 * that this work itself could get wedged. To account for this, we check that 210 * it's not stalled in the timer tick, and trigger an error if it is. 211 */ 212 static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; 213 214 static struct delayed_work scx_watchdog_work; 215 216 /* 217 * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence 218 * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu 219 * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated 220 * lazily when enabling and freed when disabling to avoid waste when sched_ext 221 * isn't active. 222 */ 223 struct scx_kick_syncs { 224 struct rcu_head rcu; 225 unsigned long syncs[]; 226 }; 227 228 static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); 229 230 /* 231 * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of 232 * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without 233 * further synchronization. See scx_alloc_tid(). 234 */ 235 struct scx_tid_alloc { 236 u64 next; 237 u64 end; 238 }; 239 static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); 240 241 /* 242 * Direct dispatch marker. 243 * 244 * Non-NULL values are used for direct dispatch from enqueue path. A valid 245 * pointer points to the task currently being enqueued. An ERR_PTR value is used 246 * to indicate that direct dispatch has already happened. 247 */ 248 static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); 249 250 static __always_inline int dsq_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) 251 { 252 const struct scx_dispatch_q *dsq = ptr; 253 254 BUILD_BUG_ON(sizeof_field(struct scx_dispatch_q, id) != sizeof(u64)); 255 256 return dsq->id != *(const u64 *)arg->key; 257 } 258 259 static const struct rhashtable_params dsq_hash_params = { 260 .key_len = sizeof_field(struct scx_dispatch_q, id), 261 .key_offset = offsetof(struct scx_dispatch_q, id), 262 .head_offset = offsetof(struct scx_dispatch_q, hash_node), 263 .obj_cmpfn = dsq_cmpfn, 264 }; 265 266 static LLIST_HEAD(dsqs_to_free); 267 268 /* ops debug dump */ 269 static DEFINE_RAW_SPINLOCK(scx_dump_lock); 270 271 struct scx_dump_data { 272 s32 cpu; 273 bool first; 274 s32 cursor; 275 struct seq_buf *s; 276 const char *prefix; 277 struct scx_bstr_buf buf; 278 }; 279 280 static struct scx_dump_data scx_dump_data = { 281 .cpu = -1, 282 }; 283 284 /* /sys/kernel/sched_ext interface */ 285 static struct kset *scx_kset; 286 287 /* 288 * Parameters that can be adjusted through /sys/module/sched_ext/parameters. 289 * There usually is no reason to modify these as normal scheduler operation 290 * shouldn't be affected by them. The knobs are primarily for debugging. 291 */ 292 static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC; 293 static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US; 294 295 static int set_slice_us(const char *val, const struct kernel_param *kp) 296 { 297 return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC); 298 } 299 300 static const struct kernel_param_ops slice_us_param_ops = { 301 .set = set_slice_us, 302 .get = param_get_uint, 303 }; 304 305 static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp) 306 { 307 return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC); 308 } 309 310 static const struct kernel_param_ops bypass_lb_intv_us_param_ops = { 311 .set = set_bypass_lb_intv_us, 312 .get = param_get_uint, 313 }; 314 315 #undef MODULE_PARAM_PREFIX 316 #define MODULE_PARAM_PREFIX "sched_ext." 317 318 module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600); 319 MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)"); 320 module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600); 321 MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)"); 322 323 #undef MODULE_PARAM_PREFIX 324 325 #define CREATE_TRACE_POINTS 326 #include <trace/events/sched_ext.h> 327 328 static void run_deferred(struct rq *rq); 329 static bool task_dead_and_done(struct task_struct *p); 330 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); 331 332 __printf(5, 6) bool __scx_exit(struct scx_sched *sch, 333 enum scx_exit_kind kind, s64 exit_code, 334 s32 exit_cpu, const char *fmt, ...) 335 { 336 va_list args; 337 bool ret; 338 339 va_start(args, fmt); 340 ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args); 341 va_end(args); 342 343 return ret; 344 } 345 346 static long jiffies_delta_msecs(unsigned long at, unsigned long now) 347 { 348 if (time_after(at, now)) 349 return jiffies_to_msecs(at - now); 350 else 351 return -(long)jiffies_to_msecs(now - at); 352 } 353 354 static bool u32_before(u32 a, u32 b) 355 { 356 return (s32)(a - b) < 0; 357 } 358 359 /** 360 * scx_is_descendant - Test whether sched is a descendant 361 * @sch: sched to test 362 * @ancestor: ancestor sched to test against 363 * 364 * Test whether @sch is a descendant of @ancestor. 365 */ 366 bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) 367 { 368 if (sch->level < ancestor->level) 369 return false; 370 return sch->ancestors[ancestor->level] == ancestor; 371 } 372 373 static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu) 374 { 375 return &sch->pnode[cpu_to_node(cpu)]->global_dsq; 376 } 377 378 static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id) 379 { 380 return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params); 381 } 382 383 static const struct sched_class *scx_setscheduler_class(struct task_struct *p) 384 { 385 if (p->sched_class == &stop_sched_class) 386 return &stop_sched_class; 387 388 return __setscheduler_class(p->policy, p->prio); 389 } 390 391 static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, struct rq *rq, 392 struct task_struct *p, u64 *enq_flags) 393 { 394 struct scx_sched *host = scx_exec_sched(sch, cpu_of(rq)); 395 struct scx_dispatch_q *dsq; 396 397 /* 398 * If @sch is a sub-sched which is bypassing, its tasks should go into 399 * the bypass DSQs of the nearest ancestor which is not bypassing. The 400 * not-bypassing ancestor is responsible for scheduling all tasks from 401 * bypassing sub-trees. If all ancestors including root are bypassing, 402 * all tasks should go to the root's bypass DSQs. 403 * 404 * Whenever a sched starts bypassing, all runnable tasks in its subtree 405 * are re-enqueued after scx_bypassing() is turned on, guaranteeing that 406 * all tasks are transferred to the right DSQs. 407 * 408 * A cpu's dispatch only descends into a host holding %SCX_CAP_BASE on 409 * it, so the bypass DSQ of a host without BASE here is never consumed. 410 * Send @p to rescue on this cpu instead, resolving against @host. 411 * scx_dispatch_enqueue() resolves a local DSQ again, against the 412 * bypassing @sch. %SCX_ENQ_IGNORE_CAPS makes that second resolution 413 * admit @p. 414 * 415 * A migration-disabled task can't leave this cpu, and a task with a 416 * migration pending must run here until the migration moves it. Either 417 * would wait on the unconsumed bypass DSQ until the host regains BASE. 418 * Resolve them too when rescue is disabled. The resolution admits both 419 * to the local DSQ, as it does any insert of such a task. 420 */ 421 if (unlikely(scx_missing_caps(host, cpu_of(rq), SCX_CAP_BASE)) && 422 (scx_rescue_enabled() || is_migration_disabled(p) || p->migration_pending)) { 423 *enq_flags |= SCX_ENQ_RESCUE; 424 dsq = scx_resolve_local_dsq(host, rq, p, enq_flags); 425 *enq_flags |= SCX_ENQ_IGNORE_CAPS; 426 } else { 427 dsq = scx_bypass_dsq(host, cpu_of(rq)); 428 } 429 return dsq; 430 } 431 432 /** 433 * rq_is_open - Is the rq available for immediate execution of an SCX task? 434 * @rq: rq to test 435 * @enq_flags: optional %SCX_ENQ_* of the task being enqueued 436 * 437 * Returns %true if @rq is currently open for executing an SCX task. After a 438 * %false return, @rq is guaranteed to invoke SCX dispatch path at least once 439 * before going to idle and not inserting a task into @rq's local DSQ after a 440 * %false return doesn't cause @rq to stall. 441 */ 442 static bool rq_is_open(struct rq *rq, u64 enq_flags) 443 { 444 lockdep_assert_rq_held(rq); 445 446 /* 447 * A higher-priority class task is either running or in the process of 448 * waking up on @rq. 449 */ 450 if (sched_class_above(rq->next_class, &ext_sched_class)) 451 return false; 452 453 /* 454 * @rq is either in transition to or in idle and there is no 455 * higher-priority class task waking up on it. 456 */ 457 if (sched_class_above(&ext_sched_class, rq->next_class)) 458 return true; 459 460 /* 461 * @rq is either picking, in transition to, or running an SCX task. 462 */ 463 464 /* 465 * A local insertion can raise next_class to ext while the donor is 466 * still idle. Keep the rq open so that the local DSQ head is not 467 * unnecessarily reenqueued before the CPU can pick it. 468 */ 469 if (rq->donor == rq->idle) 470 return true; 471 472 /* 473 * If we're in the dispatch path holding rq lock, $donor may or may not 474 * be ready depending on whether the on-going dispatch decides to extend 475 * $donor's slice. We say yes here and resolve it at the end of dispatch. 476 * See dispatch_one(). 477 */ 478 if (rq->scx.flags & SCX_RQ_IN_DISPATCH) 479 return true; 480 481 /* 482 * The preemption flags clear $donor's slice if on SCX and kick dispatch, 483 * so allow it to avoid spuriously triggering reenq on a combined 484 * PREEMPT|IMMED insertion. 485 */ 486 if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { 487 struct task_struct *donor = rq->donor; 488 489 /* 490 * A protected slice refuses the preemption and the cpu stays 491 * occupied. See rq_owned_post_enq(). 492 */ 493 return donor->sched_class != &ext_sched_class || 494 likely(!(donor->scx.flags & SCX_TASK_PROTECTED)); 495 } 496 497 /* 498 * @rq is either in transition to or running an SCX task and can't go 499 * idle without another SCX dispatch cycle. 500 */ 501 return false; 502 } 503 504 /* 505 * Track the rq currently locked. 506 * 507 * This allows kfuncs to safely operate on rq from any scx ops callback, 508 * knowing which rq is already locked. 509 */ 510 DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); 511 512 /* 513 * Under core scheduling, a pick that releases the rq lock invalidates the 514 * core-wide selection it is part of. Count the releases so that the core-sched 515 * pick can tell whether one happened across dispatch. 516 */ 517 static void scx_rq_lock_drop(struct rq *rq) 518 { 519 lockdep_assert_rq_held(rq); 520 #ifdef CONFIG_SCHED_CORE 521 if (sched_core_enabled(rq)) 522 rq->scx.lock_drop_seq++; 523 #endif 524 } 525 526 static void switch_rq_lock(struct rq *from, struct rq *to) 527 { 528 bool tracked = scx_locked_rq() == from; 529 530 if (tracked) 531 update_locked_rq(NULL); 532 scx_rq_lock_drop(from); 533 raw_spin_rq_unlock(from); 534 raw_spin_rq_lock(to); 535 if (tracked) 536 update_locked_rq(to); 537 } 538 539 /* 540 * Flipped on enable per sch->is_cid_type. Declared in internal.h so 541 * subsystem inlines can read it. 542 */ 543 DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); 544 545 /** 546 * scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask 547 * @sch: scx_sched whose scratch to fill 548 * @cpumask: cpus to translate into cids 549 * 550 * The scratch lives in BPF-writable arena memory and its header can't be 551 * trusted, so it is rewritten from kernel geometry rather than read. Caller 552 * must hold an rq lock so this cpu is the sole kernel writer for as long as the 553 * returned address is in use. 554 */ 555 static struct scx_cmask *scx_fill_cmask_scratch(struct scx_sched *sch, 556 const struct cpumask *cpumask) 557 { 558 struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); 559 struct scx_cmask_ref ref; 560 561 scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); 562 scx_cmask_ref_from_cpumask(&ref, cpumask); 563 return kern_va; 564 } 565 566 /** 567 * scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task 568 * @sch: scx_sched being invoked 569 * @rq: rq to update as the currently-locked rq, or NULL 570 * @task: task whose affinity is changing 571 * @cpumask: new cpumask 572 * 573 * For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch 574 * and dispatch through the ops_cid union view. Caller must hold @rq's rq lock. 575 */ 576 static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, 577 struct task_struct *task, 578 const struct cpumask *cpumask) 579 { 580 if (scx_is_cid_type()) 581 SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, 582 scx_fill_cmask_scratch(sch, cpumask)); 583 else 584 SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); 585 } 586 587 enum scx_dsq_iter_flags { 588 /* iterate in the reverse dispatch order */ 589 SCX_DSQ_ITER_REV = 1U << 16, 590 591 __SCX_DSQ_ITER_DRAINED = 1U << 28, 592 __SCX_DSQ_ITER_HAS_REENQ = 1U << 29, 593 __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, 594 __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, 595 596 __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, 597 __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | 598 __SCX_DSQ_ITER_DRAINED | 599 __SCX_DSQ_ITER_HAS_REENQ | 600 __SCX_DSQ_ITER_HAS_SLICE | 601 __SCX_DSQ_ITER_HAS_VTIME, 602 }; 603 604 /** 605 * nldsq_next_task - Iterate to the next task in a non-local DSQ 606 * @dsq: non-local dsq being iterated 607 * @cur: current position, %NULL to start iteration 608 * @rev: walk backwards 609 * 610 * Returns %NULL when iteration is finished. 611 */ 612 static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, 613 struct task_struct *cur, bool rev) 614 { 615 struct list_head *list_node; 616 struct scx_dsq_list_node *dsq_lnode; 617 618 lockdep_assert_held(&dsq->lock); 619 620 if (cur) 621 list_node = &cur->scx.dsq_list.node; 622 else 623 list_node = &dsq->list; 624 625 /* find the next task, need to skip BPF iteration cursors */ 626 do { 627 if (rev) 628 list_node = list_node->prev; 629 else 630 list_node = list_node->next; 631 632 if (list_node == &dsq->list) 633 return NULL; 634 635 dsq_lnode = container_of(list_node, struct scx_dsq_list_node, 636 node); 637 } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); 638 639 return container_of(dsq_lnode, struct task_struct, scx.dsq_list); 640 } 641 642 #define nldsq_for_each_task(p, dsq) \ 643 for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ 644 (p) = nldsq_next_task((dsq), (p), false)) 645 646 /** 647 * nldsq_cursor_next_task - Iterate to the next task given a cursor in a non-local DSQ 648 * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() 649 * @dsq: non-local dsq being iterated 650 * 651 * Find the next task in a cursor based iteration. The caller must have 652 * initialized @cursor using INIT_DSQ_LIST_CURSOR() and can release the DSQ lock 653 * between the iteration steps. 654 * 655 * Only tasks which were queued before @cursor was initialized are visible. This 656 * bounds the iteration and guarantees that vtime never jumps in the other 657 * direction while iterating. 658 */ 659 static struct task_struct *nldsq_cursor_next_task(struct scx_dsq_list_node *cursor, 660 struct scx_dispatch_q *dsq) 661 { 662 bool rev = cursor->flags & SCX_DSQ_ITER_REV; 663 struct task_struct *p; 664 665 lockdep_assert_held(&dsq->lock); 666 BUG_ON(!(cursor->flags & SCX_DSQ_LNODE_ITER_CURSOR)); 667 668 if (list_empty(&cursor->node)) 669 p = NULL; 670 else 671 p = container_of(cursor, struct task_struct, scx.dsq_list); 672 673 /* skip cursors and tasks that were queued after @cursor init */ 674 do { 675 p = nldsq_next_task(dsq, p, rev); 676 } while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq))); 677 678 if (p) { 679 if (rev) 680 list_move_tail(&cursor->node, &p->scx.dsq_list.node); 681 else 682 list_move(&cursor->node, &p->scx.dsq_list.node); 683 } else { 684 list_del_init(&cursor->node); 685 } 686 687 return p; 688 } 689 690 /** 691 * nldsq_cursor_lost_task - Test whether someone else took the task since iteration 692 * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() 693 * @rq: rq @p was on 694 * @dsq: dsq @p was on 695 * @p: target task 696 * 697 * @p is a task returned by nldsq_cursor_next_task(). The locks may have been 698 * dropped and re-acquired inbetween. Verify that no one else took or is in the 699 * process of taking @p from @dsq. 700 * 701 * On %false return, the caller can assume full ownership of @p. 702 */ 703 static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor, 704 struct rq *rq, struct scx_dispatch_q *dsq, 705 struct task_struct *p) 706 { 707 lockdep_assert_rq_held(rq); 708 lockdep_assert_held(&dsq->lock); 709 710 /* 711 * @p could have already left $src_dsq, got re-enqueud, or be in the 712 * process of being consumed by someone else. 713 */ 714 if (unlikely(p->scx.dsq != dsq || 715 u32_before(cursor->priv, p->scx.dsq_seq) || 716 p->scx.holding_cpu >= 0)) 717 return true; 718 719 /* if @p has stayed on @dsq, its rq couldn't have changed */ 720 if (WARN_ON_ONCE(rq != task_rq(p))) 721 return true; 722 723 return false; 724 } 725 726 /* 727 * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] 728 * dispatch order. BPF-visible iterator is opaque and larger to allow future 729 * changes without breaking backward compatibility. Can be used with 730 * bpf_for_each(). See bpf_iter_scx_dsq_*(). 731 */ 732 struct bpf_iter_scx_dsq_kern { 733 struct scx_dsq_list_node cursor; 734 struct scx_dispatch_q *dsq; 735 u64 slice; 736 u64 vtime; 737 } __attribute__((aligned(8))); 738 739 struct bpf_iter_scx_dsq { 740 u64 __opaque[6]; 741 } __attribute__((aligned(8))); 742 743 744 u32 scx_get_task_state(const struct task_struct *p) 745 { 746 return p->scx.flags & SCX_TASK_STATE_MASK; 747 } 748 749 void scx_set_task_state(struct task_struct *p, u32 state) 750 { 751 u32 prev_state = scx_get_task_state(p); 752 bool warn = false; 753 754 switch (state) { 755 case SCX_TASK_NONE: 756 warn = prev_state == SCX_TASK_DEAD; 757 break; 758 case SCX_TASK_INIT_BEGIN: 759 warn = prev_state != SCX_TASK_NONE; 760 break; 761 case SCX_TASK_INIT: 762 warn = prev_state != SCX_TASK_INIT_BEGIN; 763 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; 764 break; 765 case SCX_TASK_READY: 766 warn = !(prev_state == SCX_TASK_INIT || 767 prev_state == SCX_TASK_ENABLED); 768 break; 769 case SCX_TASK_ENABLED: 770 warn = prev_state != SCX_TASK_READY; 771 break; 772 case SCX_TASK_DEAD: 773 warn = !(prev_state == SCX_TASK_NONE || 774 prev_state == SCX_TASK_INIT_BEGIN); 775 break; 776 default: 777 WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]", 778 prev_state, state, p->comm, p->pid); 779 return; 780 } 781 782 WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]", 783 prev_state, state, p->comm, p->pid); 784 785 p->scx.flags &= ~SCX_TASK_STATE_MASK; 786 p->scx.flags |= state; 787 } 788 789 /** 790 * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration 791 * @iter: iterator to init 792 * @cgrp: Optional root of cgroup subhierarchy to iterate 793 * 794 * Initialize @iter. Once initialized, @iter must eventually be stopped with 795 * scx_task_iter_stop(). 796 * 797 * If @cgrp is %NULL, scx_tasks is used for iteration and this function returns 798 * with scx_tasks_lock held and @iter->cursor inserted into scx_tasks. 799 * 800 * If @cgrp is not %NULL, @cgrp and its descendants' tasks are walked using 801 * @iter->css_iter. The caller must be holding cgroup_lock() to prevent cgroup 802 * task migrations. 803 * 804 * The two modes of iterations are largely independent and it's likely that 805 * scx_tasks can be removed in favor of always using cgroup iteration if 806 * CONFIG_SCHED_CLASS_EXT depends on CONFIG_CGROUPS. 807 * 808 * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock() 809 * between this and the first next() call or between any two next() calls. If 810 * the locks are released between two next() calls, the caller is responsible 811 * for ensuring that the task being iterated remains accessible either through 812 * RCU read lock or obtaining a reference count. 813 * 814 * All tasks which existed when the iteration started are guaranteed to be 815 * visited as long as they are not dead. 816 */ 817 void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) 818 { 819 memset(iter, 0, sizeof(*iter)); 820 821 #ifdef CONFIG_EXT_SUB_SCHED 822 if (cgrp) { 823 lockdep_assert_held(&cgroup_mutex); 824 iter->cgrp = cgrp; 825 iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self); 826 css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, 827 &iter->css_iter); 828 return; 829 } 830 #endif 831 raw_spin_lock_irq(&scx_tasks_lock); 832 833 iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; 834 list_add(&iter->cursor.tasks_node, &scx_tasks); 835 iter->list_locked = true; 836 } 837 838 static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) 839 { 840 if (iter->locked_task) { 841 __balance_callbacks(iter->rq, &iter->rf); 842 task_rq_unlock(iter->rq, iter->locked_task, &iter->rf); 843 iter->locked_task = NULL; 844 } 845 } 846 847 /** 848 * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator 849 * @iter: iterator to unlock 850 * 851 * If @iter is in the middle of a locked iteration, it may be locking the rq of 852 * the task currently being visited in addition to scx_tasks_lock. Unlock both. 853 * This function can be safely called anytime during an iteration. The next 854 * iterator operation will automatically restore the necessary locking. 855 */ 856 void scx_task_iter_unlock(struct scx_task_iter *iter) 857 { 858 __scx_task_iter_rq_unlock(iter); 859 if (iter->list_locked) { 860 iter->list_locked = false; 861 raw_spin_unlock_irq(&scx_tasks_lock); 862 } 863 } 864 865 static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter) 866 { 867 if (!iter->list_locked) { 868 raw_spin_lock_irq(&scx_tasks_lock); 869 iter->list_locked = true; 870 } 871 } 872 873 /** 874 * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq 875 * @iter: iterator to relock 876 * @p: task whose rq to lock, or %NULL for scx_tasks_lock only 877 * 878 * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once 879 * re-acquired, both locks are managed by the iterator from here on. 880 */ 881 static void scx_task_iter_relock(struct scx_task_iter *iter, 882 struct task_struct *p) 883 { 884 __scx_task_iter_maybe_relock(iter); 885 if (p) { 886 iter->rq = task_rq_lock(p, &iter->rf); 887 iter->locked_task = p; 888 } 889 } 890 891 /** 892 * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock 893 * @iter: iterator to exit 894 * 895 * Exit a previously initialized @iter. Must be called with scx_tasks_lock held 896 * which is released on return. If the iterator holds a task's rq lock, that rq 897 * lock is also released. See scx_task_iter_start() for details. 898 */ 899 void scx_task_iter_stop(struct scx_task_iter *iter) 900 { 901 #ifdef CONFIG_EXT_SUB_SCHED 902 if (iter->cgrp) { 903 if (iter->css_pos) 904 css_task_iter_end(&iter->css_iter); 905 __scx_task_iter_rq_unlock(iter); 906 return; 907 } 908 #endif 909 __scx_task_iter_maybe_relock(iter); 910 list_del_init(&iter->cursor.tasks_node); 911 scx_task_iter_unlock(iter); 912 } 913 914 /** 915 * scx_task_iter_next - Next task 916 * @iter: iterator to walk 917 * 918 * Visit the next task. See scx_task_iter_start() for details. Locks are dropped 919 * and re-acquired every %SCX_TASK_ITER_BATCH iterations to avoid causing stalls 920 * by holding scx_tasks_lock for too long. 921 */ 922 static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) 923 { 924 struct list_head *cursor = &iter->cursor.tasks_node; 925 struct sched_ext_entity *pos; 926 927 if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) { 928 scx_task_iter_unlock(iter); 929 cond_resched(); 930 } 931 932 #ifdef CONFIG_EXT_SUB_SCHED 933 if (iter->cgrp) { 934 while (iter->css_pos) { 935 struct task_struct *p; 936 937 p = css_task_iter_next(&iter->css_iter); 938 if (p) 939 return p; 940 941 css_task_iter_end(&iter->css_iter); 942 iter->css_pos = css_next_descendant_pre(iter->css_pos, 943 &iter->cgrp->self); 944 if (iter->css_pos) 945 css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, 946 &iter->css_iter); 947 } 948 return NULL; 949 } 950 #endif 951 __scx_task_iter_maybe_relock(iter); 952 953 list_for_each_entry(pos, cursor, tasks_node) { 954 if (&pos->tasks_node == &scx_tasks) 955 return NULL; 956 if (!(pos->flags & SCX_TASK_CURSOR)) { 957 list_move(cursor, &pos->tasks_node); 958 return container_of(pos, struct task_struct, scx); 959 } 960 } 961 962 /* can't happen, should always terminate at scx_tasks above */ 963 BUG(); 964 } 965 966 /** 967 * scx_task_iter_next_locked - Next non-idle task with its rq locked 968 * @iter: iterator to walk 969 * 970 * Visit the non-idle task with its rq lock held. Allows callers to specify 971 * whether they would like to filter out dead tasks. See scx_task_iter_start() 972 * for details. 973 */ 974 struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) 975 { 976 struct task_struct *p; 977 978 __scx_task_iter_rq_unlock(iter); 979 980 while ((p = scx_task_iter_next(iter))) { 981 /* 982 * scx_task_iter is used to prepare and move tasks into SCX 983 * while loading the BPF scheduler and vice-versa while 984 * unloading. The init_tasks ("swappers") should be excluded 985 * from the iteration because: 986 * 987 * - It's unsafe to use __setscheduler_class() on an init_task 988 * to determine the sched_class to use as it won't preserve 989 * its idle_sched_class. 990 * 991 * - ops.init/exit_task() can easily be confused if called with 992 * init_tasks as they, e.g., share PID 0. 993 * 994 * As init_tasks are never scheduled through SCX, they can be 995 * skipped safely. Note that is_idle_task() which tests %PF_IDLE 996 * doesn't work here: 997 * 998 * - %PF_IDLE may not be set for an init_task whose CPU hasn't 999 * yet been onlined. 1000 * 1001 * - %PF_IDLE can be set on tasks that are not init_tasks. See 1002 * play_idle_precise() used by CONFIG_IDLE_INJECT. 1003 * 1004 * Test for idle_sched_class as only init_tasks are on it. 1005 */ 1006 if (p->sched_class == &idle_sched_class) 1007 continue; 1008 1009 iter->rq = task_rq_lock(p, &iter->rf); 1010 iter->locked_task = p; 1011 1012 /* 1013 * cgroup_task_dead() removes the dead tasks from cset->tasks 1014 * after sched_ext_dead() and cgroup iteration may see tasks 1015 * which already finished sched_ext_dead(). %SCX_TASK_DEAD is 1016 * set by sched_ext_dead() under @p's rq lock. Test it to 1017 * avoid visiting tasks which are already dead from SCX POV. 1018 */ 1019 if (scx_get_task_state(p) == SCX_TASK_DEAD) { 1020 __scx_task_iter_rq_unlock(iter); 1021 continue; 1022 } 1023 1024 return p; 1025 } 1026 return NULL; 1027 } 1028 1029 /** 1030 * scx_dump_event - Dump an event 'kind' in 'events' to 's' 1031 * @s: output seq_buf 1032 * @events: event stats 1033 * @kind: a kind of event to dump 1034 */ 1035 #define scx_dump_event(s, events, kind) do { \ 1036 scx_dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ 1037 } while (0) 1038 1039 1040 static void scx_read_events(struct scx_sched *sch, 1041 struct scx_event_stats *events); 1042 1043 static enum scx_enable_state scx_enable_state(void) 1044 { 1045 return atomic_read(&scx_enable_state_var); 1046 } 1047 1048 static enum scx_enable_state scx_set_enable_state(enum scx_enable_state to) 1049 { 1050 return atomic_xchg(&scx_enable_state_var, to); 1051 } 1052 1053 static bool scx_tryset_enable_state(enum scx_enable_state to, 1054 enum scx_enable_state from) 1055 { 1056 int from_v = from; 1057 1058 return atomic_try_cmpxchg(&scx_enable_state_var, &from_v, to); 1059 } 1060 1061 /** 1062 * wait_ops_state - Busy-wait the specified ops state to end 1063 * @p: target task 1064 * @opss: state to wait the end of 1065 * 1066 * Busy-wait for @p to transition out of @opss. This can only be used when the 1067 * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also 1068 * has load_acquire semantics to ensure that the caller can see the updates made 1069 * in the enqueueing and dispatching paths. 1070 */ 1071 static void wait_ops_state(struct task_struct *p, unsigned long opss) 1072 { 1073 do { 1074 cpu_relax(); 1075 } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); 1076 } 1077 1078 static inline bool __cpu_valid(s32 cpu) 1079 { 1080 return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); 1081 } 1082 1083 /** 1084 * scx_cpu_valid - Verify a cpu number, to be used on ops input args 1085 * @sch: scx_sched to abort on error 1086 * @cpu: cpu number which came from a BPF ops 1087 * @where: extra information reported on error 1088 * 1089 * @cpu is a cpu number which came from the BPF scheduler and can be any value. 1090 * Verify that it is in range and one of the possible cpus. If invalid, trigger 1091 * an ops error. 1092 */ 1093 bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) 1094 { 1095 if (__cpu_valid(cpu)) { 1096 return true; 1097 } else { 1098 scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: ""); 1099 return false; 1100 } 1101 } 1102 1103 static void deferred_bal_cb_workfn(struct rq *rq) 1104 { 1105 run_deferred(rq); 1106 } 1107 1108 static void deferred_irq_workfn(struct irq_work *irq_work) 1109 { 1110 struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); 1111 1112 raw_spin_rq_lock(rq); 1113 run_deferred(rq); 1114 scx_rq_lock_drop(rq); 1115 raw_spin_rq_unlock(rq); 1116 } 1117 1118 /** 1119 * schedule_deferred - Schedule execution of deferred actions on an rq 1120 * @rq: target rq 1121 * 1122 * Schedule execution of deferred actions on @rq. Deferred actions are executed 1123 * with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks 1124 * to other rqs. 1125 */ 1126 static void schedule_deferred(struct rq *rq) 1127 { 1128 /* 1129 * This is the fallback when schedule_deferred_locked() can't use 1130 * the cheaper balance callback or wakeup hook paths (the target 1131 * CPU is not in dispatch or wakeup). Currently, this is primarily 1132 * hit by reenqueue operations targeting a remote CPU. 1133 * 1134 * Queue on the target CPU. The deferred work can run from any CPU 1135 * correctly - the _locked() path already processes remote rqs from 1136 * the calling CPU - but targeting the owning CPU allows IPI delivery 1137 * without waiting for the calling CPU to re-enable IRQs and is 1138 * cheaper as the reenqueue runs locally. 1139 */ 1140 irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq)); 1141 } 1142 1143 /** 1144 * schedule_deferred_locked - Schedule execution of deferred actions on an rq 1145 * @rq: target rq 1146 * 1147 * Schedule execution of deferred actions on @rq. Equivalent to 1148 * schedule_deferred() but requires @rq to be locked and can be more efficient. 1149 */ 1150 static void schedule_deferred_locked(struct rq *rq) 1151 { 1152 lockdep_assert_rq_held(rq); 1153 1154 /* 1155 * If in the middle of waking up a task, task_woken_scx() will be called 1156 * afterwards which will then run the deferred actions, no need to 1157 * schedule anything. 1158 */ 1159 if (rq->scx.flags & SCX_RQ_IN_WAKEUP) 1160 return; 1161 1162 /* Don't do anything if there already is a deferred operation. */ 1163 if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) 1164 return; 1165 1166 /* 1167 * If in dispatch without proxy execution, the balance callbacks will 1168 * be called before rq lock is released. Schedule one. With proxy 1169 * execution, proxy resolution can zap the callbacks without another 1170 * SCX dispatch to requeue them, so use irq_work instead. 1171 * 1172 * We can't directly insert the callback into the 1173 * rq's list: The call can drop its lock and make the pending balance 1174 * callback visible to unrelated code paths that call rq_pin_lock(). 1175 * 1176 * Just let dispatch_one() know that it must do it itself. 1177 */ 1178 if ((rq->scx.flags & SCX_RQ_IN_DISPATCH) && !sched_proxy_exec()) { 1179 rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; 1180 return; 1181 } 1182 1183 /* 1184 * No scheduler hooks available. Use the generic irq_work path. The 1185 * above WAKEUP and DISPATCH paths should cover most of the cases and 1186 * the time to IRQ re-enable shouldn't be long. 1187 */ 1188 schedule_deferred(rq); 1189 } 1190 1191 static void resched_irq_workfn(struct irq_work *irq_work) 1192 { 1193 struct rq *rq = container_of(irq_work, struct rq, scx.resched_irq_work); 1194 1195 guard(rq_lock)(rq); 1196 /* a higher class task would only be picked again, see kick_one_cpu() */ 1197 if (!sched_class_above(rq->donor->sched_class, &ext_sched_class)) 1198 resched_curr(rq); 1199 scx_rq_lock_drop(rq); 1200 } 1201 1202 /** 1203 * scx_resched_curr_deferred - Reschedule @rq's cpu from hard irq work on it 1204 * @rq: rq to reschedule 1205 * 1206 * A kick, like a direct resched, only guarantees that a scheduling event on the 1207 * cpu completes after it, and that can be the one in progress. Work that a pick 1208 * processes only at the start of its dispatch, such as the ecaps syncs, can be 1209 * queued after that point and miss the pick. A pick on @rq's cpu runs with IRQs 1210 * off, so the irq work runs after it, and the scheduling event its resched 1211 * triggers starts after this call. A sibling's core scheduling pick for @rq 1212 * isn't covered. A cpu running a task of a higher class isn't rescheduled, as 1213 * its pick would return that task again. @rq's cpu must stay online until the 1214 * irq work is queued. 1215 */ 1216 void scx_resched_curr_deferred(struct rq *rq) 1217 { 1218 irq_work_queue_on(&rq->scx.resched_irq_work, cpu_of(rq)); 1219 } 1220 1221 #ifdef CONFIG_NO_HZ_FULL 1222 static void scx_proxy_tick_bal_cb(struct rq *rq) 1223 { 1224 sched_update_tick_dependency(rq); 1225 } 1226 1227 static void scx_proxy_update_tick(struct rq *rq, struct task_struct *next) 1228 { 1229 bool proxy = next != rq->donor; 1230 bool was_proxy = rq->scx.flags & SCX_RQ_PROXY_TICK; 1231 1232 if (likely(proxy == was_proxy)) 1233 return; 1234 1235 if (proxy) { 1236 /* Keep proxy execution tick-driven for now. */ 1237 rq->scx.flags |= SCX_RQ_PROXY_TICK; 1238 tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); 1239 } else { 1240 rq->scx.flags &= ~SCX_RQ_PROXY_TICK; 1241 /* 1242 * The selected donor is already visible, but rq->curr still 1243 * identifies the outgoing execution context. Reevaluate after 1244 * context_switch() updates rq->curr so sched_can_stop_tick() sees 1245 * the complete selection. 1246 */ 1247 queue_balance_callback(rq, &rq->scx.proxy_tick_bal_cb, 1248 scx_proxy_tick_bal_cb); 1249 } 1250 } 1251 #endif 1252 1253 /* 1254 * Complete sched_ext bookkeeping after proxy resolution and retry tasks which 1255 * couldn't be reenqueued by an earlier reject DSQ drain. 1256 */ 1257 void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) 1258 { 1259 lockdep_assert_rq_held(rq); 1260 1261 #ifdef CONFIG_NO_HZ_FULL 1262 if (scx_enabled() && tick_nohz_full_cpu(cpu_of(rq))) 1263 scx_proxy_update_tick(rq, next); 1264 #endif 1265 1266 /* 1267 * A higher priority class may wake up while @rq is transiently idle 1268 * for proxy resolution. wakeup_preempt_scx() isn't invoked in that 1269 * case, so reenqueue IMMED donors parked on the local DSQ here. 1270 */ 1271 if (rq->scx.nr_immed && sched_class_above(rq->next_class, &ext_sched_class)) 1272 scx_schedule_reenq_local(rq, 0); 1273 1274 if (rq->scx.flags & SCX_RQ_PROXY_RETRY) { 1275 rq->scx.flags &= ~SCX_RQ_PROXY_RETRY; 1276 schedule_deferred_locked(rq); 1277 } 1278 } 1279 1280 void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, 1281 u64 reenq_flags, struct rq *locked_rq) 1282 { 1283 struct rq *rq; 1284 1285 /* 1286 * Allowing reenqueues doesn't make sense while bypassing. This also 1287 * blocks from new reenqueues to be scheduled on dead scheds. 1288 */ 1289 if (unlikely(READ_ONCE(sch->bypass_depth))) 1290 return; 1291 1292 if (dsq->id == SCX_DSQ_LOCAL) { 1293 rq = container_of(dsq, struct rq, scx.local_dsq); 1294 1295 /* 1296 * An inactive cpu's local DSQ holds only tasks that are bound 1297 * to it or are being moved off, so a reenq of it is pointless. 1298 * Once balance push is installed, queue_balance_callback() also 1299 * drops the callback that would run a reenq requested from 1300 * dispatch, and irq_work_queue_on() can't reach an offline cpu. 1301 * The reenq's node would then stay on this rq's list after 1302 * @sch's pcpu is freed. 1303 */ 1304 if (unlikely(!cpu_active(cpu_of(rq)))) 1305 return; 1306 1307 /* 1308 * A sub-sched lacking baseline access on the target cid has no 1309 * business triggering IPIs. The lockless test is fine: slipping 1310 * through right after a revoke is harmless and a wrong denial 1311 * can't happen - if the caller has seen its ownership, so does 1312 * this test. 1313 */ 1314 if (unlikely(scx_missing_caps(sch, cpu_of(rq), SCX_CAP_BASE))) { 1315 __scx_add_event(sch, SCX_EV_SUB_REENQ_DENIED, 1); 1316 return; 1317 } 1318 1319 struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); 1320 struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local; 1321 1322 /* 1323 * Pairs with smp_mb() in process_deferred_reenq_locals() and 1324 * guarantees that there is a reenq_local() afterwards. 1325 */ 1326 smp_mb(); 1327 1328 if (list_empty(&drl->node) || 1329 (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) { 1330 1331 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); 1332 1333 if (list_empty(&drl->node)) 1334 list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals); 1335 WRITE_ONCE(drl->flags, drl->flags | reenq_flags); 1336 } 1337 } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { 1338 rq = this_rq(); 1339 1340 struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu_user, cpu_of(rq)); 1341 struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; 1342 1343 /* 1344 * Pairs with smp_mb() in process_deferred_reenq_users() and 1345 * guarantees that there is a reenq_user() afterwards. 1346 */ 1347 smp_mb(); 1348 1349 if (list_empty(&dru->node) || 1350 (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) { 1351 1352 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); 1353 1354 /* @dsq is being destroyed, see free_dsq_irq_workfn() */ 1355 if (unlikely(READ_ONCE(dsq->id) == SCX_DSQ_INVALID)) 1356 return; 1357 1358 if (list_empty(&dru->node)) 1359 list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); 1360 WRITE_ONCE(dru->flags, dru->flags | reenq_flags); 1361 } 1362 } else { 1363 /* destroyed since the caller resolved it, nothing to reenq */ 1364 if (READ_ONCE(dsq->id) != SCX_DSQ_INVALID) 1365 scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); 1366 return; 1367 } 1368 1369 if (rq == locked_rq) 1370 schedule_deferred_locked(rq); 1371 else 1372 schedule_deferred(rq); 1373 } 1374 1375 /* 1376 * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero 1377 * word means no request. Otherwise the fields are: 1378 * 1379 * 63 SCX_SLICE_OOB_PENDING, set on every request 1380 * 62-43 lower bits of issuing scheduler's id 1381 * 42-0 requested slice duration in nsecs 1382 * 1383 * A duration of SCX_SLICE_OOB_DUR_MASK means SCX_SLICE_INF. A finite dur 1384 * saturates at SCX_SLICE_OOB_DUR_MASK - 1. The id is used to detect and ignore 1385 * a request that outlived a task ownership change. 1386 * 1387 * Only the low 20 bits of sch->id are packed, which is enough to make 1388 * collisions practically impossible. A theoretical collision just lets a stale 1389 * request through once. 1390 */ 1391 enum scx_slice_oob_consts { 1392 SCX_SLICE_OOB_DUR_BITS = 43, 1393 SCX_SLICE_OOB_ID_BITS = 64 - SCX_SLICE_OOB_DUR_BITS - 1, 1394 1395 SCX_SLICE_OOB_DUR_MASK = (1LLU << SCX_SLICE_OOB_DUR_BITS) - 1, 1396 SCX_SLICE_OOB_ID_SHIFT = SCX_SLICE_OOB_DUR_BITS, 1397 SCX_SLICE_OOB_ID_MASK = (1LLU << SCX_SLICE_OOB_ID_BITS) - 1, 1398 SCX_SLICE_OOB_PENDING = 1LLU << 63, 1399 }; 1400 1401 /* 1402 * Slice and dsq_vtime write rules 1403 * 1404 * While @p is running, sleeping or queued on an rq-owned DSQ, both fields are 1405 * protected by the rq lock. While running, the rq lock is required because 1406 * update_curr_scx() RMWs the slice and the cap check for slice extension is 1407 * only reliable under the rq lock. 1408 * 1409 * While @p is queued on a user DSQ or on the BPF side, the kernel neither 1410 * consumes nor decides on the fields. Synchronizing the writers is the BPF 1411 * scheduler's responsibility. An rq-locked scx_bpf_task_set_slice() write and a 1412 * concurrent DSQ insertion commit can race each other and whichever lands last 1413 * wins. 1414 * 1415 * A DSQ insert kfunc doesn't update the fields directly. The verdict carries 1416 * the values and apply_slice_vtime() commits them at the insertion. 1417 * 1418 * scx_bpf_task_set_slice() may be called from any context and writes directly 1419 * only if @p's rq lock is already held, otherwise it bounces through 1420 * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration. 1421 * 1422 * While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is 1423 * refused. See set_task_slice_keep_oob(). A kernel slice, see 1424 * %SCX_TASK_KERNEL_SLICE, can be shortened but not extended. 1425 * 1426 * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so 1427 * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct 1428 * BPF stores from non-cid-form schedulers are outside these rules. 1429 */ 1430 1431 /* clear a pending slice request */ 1432 static void clear_task_slice_oob(struct task_struct *p) 1433 { 1434 if (unlikely(atomic64_read(&p->scx.slice_oob))) 1435 atomic64_set(&p->scx.slice_oob, 0); 1436 } 1437 1438 /** 1439 * dsq_insert_head - FIFO head insertion honoring %SCX_TASK_PROTECTED 1440 * @dsq: DSQ to insert into 1441 * @p: task being inserted 1442 * 1443 * A HEAD insert should land behind any leading protected tasks. Return %true 1444 * indicates whether @p became the first entry. 1445 */ 1446 static bool dsq_insert_head(struct scx_dispatch_q *dsq, struct task_struct *p) 1447 { 1448 struct list_head *pos = &dsq->list; 1449 struct scx_dsq_list_node *node; 1450 1451 /* 1452 * Only rq-owned DSQs can hold protected tasks and the associated rq 1453 * lock keeps their flags stable. 1454 */ 1455 if (!dsq_is_rq_owned(dsq)) { 1456 list_add(&p->scx.dsq_list.node, &dsq->list); 1457 return true; 1458 } 1459 1460 list_for_each_entry(node, &dsq->list, node) { 1461 struct task_struct *q; 1462 1463 if (WARN_ON_ONCE(node->flags & SCX_DSQ_LNODE_ITER_CURSOR)) 1464 continue; 1465 1466 q = container_of(node, struct task_struct, scx.dsq_list); 1467 if (!(q->scx.flags & SCX_TASK_PROTECTED)) 1468 break; 1469 1470 pos = &node->node; 1471 } 1472 1473 list_add(&p->scx.dsq_list.node, pos); 1474 1475 return pos == &dsq->list; 1476 } 1477 1478 /** 1479 * set_task_slice_keep_oob - Set @p's slice, leaving any pending oob request 1480 * @p: task of interest 1481 * @slice: slice to set 1482 * 1483 * While %SCX_TASK_PROTECTED is set, BPF schedulers may not modify the slice. 1484 * Refuse and return %false. 1485 */ 1486 static bool set_task_slice_keep_oob(struct task_struct *p, u64 slice) 1487 { 1488 lockdep_assert_rq_held(task_rq(p)); 1489 1490 if (unlikely(p->scx.flags & SCX_TASK_PROTECTED)) 1491 return false; 1492 1493 p->scx.slice = slice; 1494 return true; 1495 } 1496 1497 /* set @p's slice, superseding any pending out-of-band request */ 1498 bool scx_set_task_slice(struct task_struct *p, u64 slice) 1499 { 1500 if (!set_task_slice_keep_oob(p, slice)) 1501 return false; 1502 clear_task_slice_oob(p); 1503 return true; 1504 } 1505 1506 /** 1507 * scx_task_slice_ended - @p's slice is consumed or given up 1508 * @rq: rq @p is on 1509 * @p: task of interest 1510 * 1511 * End what rides on the slice - the protection, the kernel's ownership of the 1512 * slice, and the rescue if @p is being rescued. 1513 * 1514 * A dequeue normally ends the slice too. The exception is a save/restore pair 1515 * on the running task. Attribute changes like renice cycle the task through 1516 * dequeue and enqueue while it keeps executing, so the slice continues. A 1517 * queued task instead loses its DSQ position on any dequeue and the slice ends 1518 * with it. 1519 */ 1520 void scx_task_slice_ended(struct rq *rq, struct task_struct *p) 1521 { 1522 lockdep_assert_rq_held(rq); 1523 1524 p->scx.flags &= ~(SCX_TASK_PROTECTED | SCX_TASK_KERNEL_SLICE); 1525 if (unlikely(p == scx_rescuee(rq))) 1526 scx_rescue_end(rq); 1527 } 1528 1529 /* request @p's slice to be set to @slice, see the write rules above */ 1530 static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice) 1531 { 1532 u64 dur; 1533 1534 if (slice == SCX_SLICE_INF) { 1535 dur = SCX_SLICE_OOB_DUR_MASK; 1536 } else if (unlikely(slice >= SCX_SLICE_OOB_DUR_MASK)) { 1537 dur = SCX_SLICE_OOB_DUR_MASK - 1; 1538 scx_add_event(sch, SCX_EV_SLICE_CLAMPED, 1); 1539 } else { 1540 dur = slice; 1541 } 1542 1543 atomic64_set(&p->scx.slice_oob, SCX_SLICE_OOB_PENDING | 1544 ((sch->id & SCX_SLICE_OOB_ID_MASK) << SCX_SLICE_OOB_ID_SHIFT) | dur); 1545 } 1546 1547 /* 1548 * Apply a pending out-of-band slice request under @rq's lock. A request whose 1549 * packed id no longer matches @p's current owner is dropped. An extension needs 1550 * baseline cpu access on @p's cid and is refused on a kernel slice. Shortening 1551 * is always allowed. A protected slice refuses both. %SCX_EV_SLICE_DENIED 1552 * counts the denials. See the write rules above. 1553 */ 1554 static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) 1555 { 1556 u64 oob, dur, slice; 1557 1558 lockdep_assert_rq_held(rq); 1559 1560 if (likely(!atomic64_read(&p->scx.slice_oob))) 1561 return; 1562 1563 oob = atomic64_xchg(&p->scx.slice_oob, 0); 1564 if (unlikely(!oob)) 1565 return; 1566 1567 /* the issuing scheduler no longer owns @p, drop the request */ 1568 if (unlikely(((oob >> SCX_SLICE_OOB_ID_SHIFT) & SCX_SLICE_OOB_ID_MASK) != 1569 (scx_task_sched(p)->id & SCX_SLICE_OOB_ID_MASK))) 1570 return; 1571 1572 dur = oob & SCX_SLICE_OOB_DUR_MASK; 1573 slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur; 1574 1575 if (slice > p->scx.slice && 1576 unlikely(scx_task_has_kernel_slice(p) || 1577 scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) { 1578 __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); 1579 return; 1580 } 1581 1582 if (unlikely(!set_task_slice_keep_oob(p, slice))) 1583 __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); 1584 } 1585 1586 /* 1587 * A dsq insert kfunc doesn't write slice or dsq_vtime. The verdict carries them 1588 * and they are committed here, at the insertion. A zero @slice keeps the 1589 * current value, floored at 1 so the task isn't treated as expired. 1590 */ 1591 static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags) 1592 { 1593 if (slice) { 1594 p->scx.slice = slice; 1595 /* 1596 * An explicit slice supersedes a pending oob request. A carried 1597 * default refill is not an explicit request and must keep it. 1598 */ 1599 if (!(enq_flags & SCX_ENQ_SLICE_DFL)) 1600 clear_task_slice_oob(p); 1601 } else if (!p->scx.slice) { 1602 p->scx.slice = 1; 1603 } 1604 1605 if (enq_flags & SCX_ENQ_DSQ_PRIQ) 1606 p->scx.dsq_vtime = vtime; 1607 } 1608 1609 /* 1610 * refill_task_slice_dfl() supplies a slice when the scheduler left none and 1611 * expire_task_slice() takes it away to force the task off the cpu. Neither 1612 * touches a kernel slice, see %SCX_TASK_KERNEL_SLICE. 1613 */ 1614 static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) 1615 { 1616 if (scx_task_has_kernel_slice(p)) 1617 return; 1618 1619 /* 1620 * A default refill is not an explicit request, so it must not drop a 1621 * pending out-of-band one, which is applied when @p next runs. 1622 */ 1623 set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl)); 1624 __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); 1625 } 1626 1627 static void expire_task_slice(struct task_struct *p) 1628 { 1629 /* 1630 * Unlike the refill above, an expiry supersedes a pending out-of-band 1631 * request. A kept request would be applied at the next 1632 * update_curr_scx() and undo the expiry. 1633 */ 1634 if (!scx_task_has_kernel_slice(p)) 1635 scx_set_task_slice(p, 0); 1636 } 1637 1638 static void update_curr_scx(struct rq *rq) 1639 { 1640 struct task_struct *donor; 1641 s64 delta_exec; 1642 1643 /* 1644 * update_curr_scx() is selected through rq->donor->sched_class, not 1645 * rq->curr->sched_class, so @donor is always an EXT task here. If an EXT 1646 * owner executes for a FAIR donor, FAIR's update_curr() runs instead. 1647 */ 1648 donor = rq->donor; 1649 1650 /* apply even on 0 delta_exec, callers may still act on the slice */ 1651 apply_task_slice_oob(rq, donor); 1652 1653 delta_exec = update_curr_common(rq); 1654 if (unlikely(delta_exec <= 0)) 1655 return; 1656 1657 if (donor->scx.slice != SCX_SLICE_INF) 1658 donor->scx.slice -= min_t(u64, donor->scx.slice, delta_exec); 1659 1660 if (unlikely(donor == scx_rescuee(rq))) 1661 scx_rescue_charge(rq, delta_exec); 1662 1663 dl_server_update(&rq->ext_server, delta_exec); 1664 } 1665 1666 static bool scx_dsq_priq_less(struct rb_node *node_a, 1667 const struct rb_node *node_b) 1668 { 1669 const struct task_struct *a = 1670 container_of(node_a, struct task_struct, scx.dsq_priq); 1671 const struct task_struct *b = 1672 container_of(node_b, struct task_struct, scx.dsq_priq); 1673 1674 return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); 1675 } 1676 1677 static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) 1678 { 1679 /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */ 1680 WRITE_ONCE(dsq->nr, dsq->nr + 1); 1681 1682 /* 1683 * Once @p reaches a local DSQ, it can only leave it by being dispatched 1684 * to the CPU or dequeued. In both cases, the only way @p can go back to 1685 * the BPF sched is through enqueueing. If being inserted into a local 1686 * DSQ with IMMED, persist the state until the next enqueueing event in 1687 * scx_do_enqueue_task() so that we can maintain IMMED protection 1688 * through e.g. SAVE/RESTORE cycles and slice extensions. 1689 */ 1690 if (enq_flags & SCX_ENQ_IMMED) { 1691 if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { 1692 WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK)); 1693 return; 1694 } 1695 p->scx.flags |= SCX_TASK_IMMED; 1696 } 1697 1698 if (p->scx.flags & SCX_TASK_IMMED) { 1699 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); 1700 1701 if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) 1702 return; 1703 1704 rq->scx.nr_immed++; 1705 1706 /* 1707 * If @rq already had other tasks or the current task is not 1708 * done yet, @p can't go on the CPU immediately. Re-enqueue. 1709 */ 1710 if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) 1711 scx_schedule_reenq_local(rq, 0); 1712 } 1713 } 1714 1715 static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) 1716 { 1717 /* see dsq_inc_nr() */ 1718 WRITE_ONCE(dsq->nr, dsq->nr - 1); 1719 1720 if (p->scx.flags & SCX_TASK_IMMED) { 1721 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); 1722 1723 if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) || 1724 WARN_ON_ONCE(rq->scx.nr_immed <= 0)) 1725 return; 1726 1727 rq->scx.nr_immed--; 1728 } 1729 } 1730 1731 /* 1732 * Return true if @p is moving due to an internal SCX migration, false 1733 * otherwise. 1734 */ 1735 static inline bool task_scx_migrating(struct task_struct *p) 1736 { 1737 /* 1738 * We only need to check sticky_cpu: it is set to the destination 1739 * CPU in move_remote_task_to_local_dsq() before deactivate_task() 1740 * and cleared when the task is enqueued on the destination, so it 1741 * is only non-negative during an internal SCX migration. 1742 */ 1743 return p->scx.sticky_cpu >= 0; 1744 } 1745 1746 /* Must be called under the lock serializing @p's custody transfers. */ 1747 static bool task_leave_custody(struct task_struct *p) 1748 { 1749 if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) 1750 return false; 1751 1752 p->scx.flags &= ~SCX_TASK_IN_CUSTODY; 1753 return true; 1754 } 1755 1756 /* 1757 * A task with an infinite slice may be running with its tick stopped. Lazy 1758 * rescheduling doesn't send an IPI, so restore the tick dependency to guarantee 1759 * that the lazy request is promoted by a real scheduler tick. Set the lazy 1760 * request first so that the dependency update's IPI can also serve it on return 1761 * to user space. 1762 */ 1763 static void scx_resched_curr_lazy(struct rq *rq) 1764 { 1765 resched_curr_lazy(rq); 1766 1767 if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) { 1768 rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; 1769 if (rq->clock_update_flags < RQCF_UPDATED) 1770 update_rq_clock(rq); 1771 update_other_load_avgs(rq); 1772 sched_update_tick_dependency(rq); 1773 } 1774 } 1775 1776 static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, 1777 struct scx_dispatch_q *dsq, struct task_struct *p, 1778 u64 enq_flags) 1779 { 1780 if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) 1781 SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); 1782 1783 /* 1784 * Only local inserts get the wakeup treatment below. Rejects kick a 1785 * deferred reenq and rescue parks are paced by the rescue timer. Proxy 1786 * rejects which aren't ready when drained request a later retry from 1787 * scx_proxy_reenqueue_retry(). 1788 */ 1789 if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { 1790 if (dsq->id == SCX_DSQ_REJECT) 1791 schedule_deferred_locked(rq); 1792 return; 1793 } 1794 1795 /* 1796 * Note that @rq's lock may be dropped between this enqueue and @p 1797 * actually getting on CPU. This gives higher-class tasks (e.g. RT) 1798 * an opportunity to wake up on @rq and prevent @p from running. 1799 * Here are some concrete examples: 1800 * 1801 * Example 1: 1802 * 1803 * We dispatch two tasks from a single ops.dispatch(): 1804 * - First, a local task to this CPU's local DSQ; 1805 * - Second, a local/remote task to a remote CPU's local DSQ. 1806 * We must drop the local rq lock in order to finish the second 1807 * dispatch. In that time, an RT task can wake up on the local rq. 1808 * 1809 * Example 2: 1810 * 1811 * We dispatch a local/remote task to a remote CPU's local DSQ. 1812 * We must drop the remote rq lock before the dispatched task can run, 1813 * which gives an RT task an opportunity to wake up on the remote rq. 1814 * 1815 * Both examples work the same if we replace dispatching with moving 1816 * the tasks from a user-created DSQ. 1817 * 1818 * We must detect these wakeups so that we can re-enqueue IMMED tasks 1819 * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this 1820 * purpose, but for it to be invoked, we must ensure that we bump 1821 * @rq->next_class to &ext_sched_class if it's currently idle. 1822 * 1823 * wakeup_preempt() does the bumping, and since we only invoke it if 1824 * @rq->next_class is below &ext_sched_class, it will also 1825 * resched_curr(rq). 1826 */ 1827 if (sched_class_above(p->sched_class, rq->next_class)) 1828 wakeup_preempt(rq, p, 0); 1829 1830 /* 1831 * If @rq is in dispatch, the CPU is already vacant and looking for the 1832 * next task to run. No need to preempt or trigger resched after moving 1833 * @p into its local DSQ. 1834 * Note that the wakeup_preempt() above may have already triggered 1835 * a resched if @rq->next_class was idle. It's harmless, since 1836 * need_resched is cleared immediately after task pick. 1837 */ 1838 if (rq->scx.flags & SCX_RQ_IN_DISPATCH) 1839 return; 1840 1841 if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) && 1842 p != rq->donor && rq->donor->sched_class == &ext_sched_class) { 1843 if (likely(scx_set_task_slice(rq->donor, 0))) { 1844 if (enq_flags & SCX_ENQ_PREEMPT) 1845 resched_curr(rq); 1846 else 1847 scx_resched_curr_lazy(rq); 1848 } else { 1849 __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); 1850 } 1851 } 1852 } 1853 1854 static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, 1855 struct scx_dispatch_q *dsq, struct task_struct *p, 1856 u64 slice, u64 vtime, u64 enq_flags) 1857 { 1858 bool is_rq_owned = dsq_is_rq_owned(dsq); 1859 1860 if (dsq->id == SCX_DSQ_LOCAL) 1861 dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); 1862 1863 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); 1864 WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || 1865 !RB_EMPTY_NODE(&p->scx.dsq_priq)); 1866 1867 if (!is_rq_owned) { 1868 raw_spin_lock_nested(&dsq->lock, 1869 (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0); 1870 1871 if (unlikely(dsq->id == SCX_DSQ_INVALID)) { 1872 scx_error(sch, "attempting to dispatch to a destroyed dsq"); 1873 /* fall back to the global dsq */ 1874 raw_spin_unlock(&dsq->lock); 1875 dsq = find_global_dsq(sch, task_cpu(p)); 1876 raw_spin_lock(&dsq->lock); 1877 } 1878 } 1879 1880 if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && 1881 (enq_flags & SCX_ENQ_DSQ_PRIQ))) { 1882 /* 1883 * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from 1884 * their FIFO queues. To avoid confusion and accidentally 1885 * starving vtime-dispatched tasks by FIFO-dispatched tasks, we 1886 * disallow any internal DSQ from doing vtime ordering of 1887 * tasks. 1888 */ 1889 scx_error(sch, "cannot use vtime ordering for built-in DSQs"); 1890 enq_flags &= ~SCX_ENQ_DSQ_PRIQ; 1891 } 1892 1893 /* 1894 * @dsq is locked and @enq_flags is sanitized. Commit the carried slice 1895 * and vtime before the PRIQ insertion below reads the new dsq_vtime. 1896 */ 1897 if (enq_flags & SCX_ENQ_APPLY_SLICE) 1898 apply_slice_vtime(p, slice, vtime, enq_flags); 1899 1900 if (enq_flags & SCX_ENQ_DSQ_PRIQ) { 1901 struct rb_node *rbp; 1902 1903 /* 1904 * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are 1905 * linked to both the rbtree and list on PRIQs, this can only be 1906 * tested easily when adding the first task. 1907 */ 1908 if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && 1909 nldsq_next_task(dsq, NULL, false))) 1910 scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks", 1911 dsq->id); 1912 1913 p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; 1914 rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); 1915 1916 /* 1917 * Find the previous task and insert after it on the list so 1918 * that @dsq->list is vtime ordered. 1919 */ 1920 rbp = rb_prev(&p->scx.dsq_priq); 1921 if (rbp) { 1922 struct task_struct *prev = 1923 container_of(rbp, struct task_struct, 1924 scx.dsq_priq); 1925 list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); 1926 /* first task unchanged - no update needed */ 1927 } else { 1928 list_add(&p->scx.dsq_list.node, &dsq->list); 1929 /* not builtin and new task is at head - use fastpath */ 1930 rcu_assign_pointer(dsq->first_task, p); 1931 } 1932 } else { 1933 /* a FIFO DSQ shouldn't be using PRIQ enqueuing */ 1934 if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) 1935 scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", 1936 dsq->id); 1937 1938 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { 1939 /* new task inserted at head - use fastpath */ 1940 if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) 1941 rcu_assign_pointer(dsq->first_task, p); 1942 } else { 1943 /* 1944 * dsq->list can contain parked BPF iterator cursors, so 1945 * list_empty() here isn't a reliable proxy for "no real 1946 * task in the DSQ". Test dsq->first_task directly. 1947 */ 1948 list_add_tail(&p->scx.dsq_list.node, &dsq->list); 1949 if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) 1950 rcu_assign_pointer(dsq->first_task, p); 1951 } 1952 } 1953 1954 /* seq records the order tasks are queued, used by BPF DSQ iterator */ 1955 WRITE_ONCE(dsq->seq, dsq->seq + 1); 1956 p->scx.dsq_seq = dsq->seq; 1957 1958 dsq_inc_nr(dsq, p, enq_flags); 1959 p->scx.dsq = dsq; 1960 1961 /* 1962 * Update custody and call ops.dequeue() before clearing ops_state: 1963 * once ops_state is cleared, waiters in ops_dequeue() can proceed 1964 * and dequeue_task_scx() will RMW p->scx.flags. If we clear 1965 * ops_state first, both sides would modify p->scx.flags 1966 * concurrently in a non-atomic way. 1967 */ 1968 if (is_rq_owned) { 1969 rq_owned_post_enq(sch, rq, dsq, p, enq_flags); 1970 } else { 1971 bool call_dequeue = false; 1972 1973 /* 1974 * Global and bypass DSQs are terminal - the task leaves the 1975 * scheduler's custody, so ops.dequeue() fires. It can run 1976 * without @p's rq lock (finish_dispatch() passes the dispatch 1977 * rq); that's safe because dequeue_task_scx() waits on 1978 * SCX_OPSS_DISPATCHING (see the ops_state note above) and so 1979 * can't race it. A non-terminal DSQ keeps the task in custody. 1980 * The custody transfer happens under @dsq->lock so that later 1981 * consumers see the flag clear; the callback runs after 1982 * @dsq->lock is dropped because it may lock a DSQ itself. 1983 */ 1984 if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) 1985 call_dequeue = task_leave_custody(p); 1986 else 1987 p->scx.flags |= SCX_TASK_IN_CUSTODY; 1988 1989 raw_spin_unlock(&dsq->lock); 1990 1991 if (call_dequeue && SCX_HAS_OP(sch, dequeue)) 1992 SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); 1993 } 1994 1995 /* 1996 * We're transitioning out of QUEUEING or DISPATCHING. store_release to 1997 * match waiters' load_acquire. 1998 */ 1999 if (enq_flags & SCX_ENQ_CLEAR_OPSS) 2000 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 2001 } 2002 2003 void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq) 2004 { 2005 WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); 2006 2007 if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { 2008 rb_erase(&p->scx.dsq_priq, &dsq->priq); 2009 RB_CLEAR_NODE(&p->scx.dsq_priq); 2010 p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; 2011 } 2012 p->scx.dsq_flags &= ~SCX_TASK_DSQ_REENQ; 2013 2014 list_del_init(&p->scx.dsq_list.node); 2015 dsq_dec_nr(dsq, p); 2016 2017 if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && rcu_access_pointer(dsq->first_task) == p) { 2018 struct task_struct *first_task; 2019 2020 first_task = nldsq_next_task(dsq, NULL, false); 2021 rcu_assign_pointer(dsq->first_task, first_task); 2022 } 2023 } 2024 2025 void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) 2026 { 2027 struct scx_dispatch_q *dsq = p->scx.dsq; 2028 bool is_rq_owned = dsq && dsq_is_rq_owned(dsq); 2029 2030 lockdep_assert_rq_held(rq); 2031 2032 if (!dsq) { 2033 /* 2034 * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals. 2035 * Unlinking is all that's needed to cancel. 2036 */ 2037 if (unlikely(!list_empty(&p->scx.dsq_list.node))) 2038 list_del_init(&p->scx.dsq_list.node); 2039 2040 /* 2041 * When dispatch_to_local_dsq() or remote consumption moves a 2042 * task to a local DSQ, the task isn't associated with any DSQ 2043 * but @p->scx.holding_cpu may be set. Clearing holding_cpu 2044 * tells dispatch_to_local_dsq() that it lost to a dequeue. 2045 */ 2046 if (p->scx.holding_cpu >= 0) 2047 p->scx.holding_cpu = -1; 2048 2049 return; 2050 } 2051 2052 if (!is_rq_owned) 2053 raw_spin_lock(&dsq->lock); 2054 2055 /* 2056 * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't 2057 * change underneath us. 2058 */ 2059 if (p->scx.holding_cpu < 0) { 2060 /* @p must still be on @dsq, dequeue */ 2061 scx_task_unlink_from_dsq(p, dsq); 2062 } else { 2063 /* 2064 * We're racing against unlink_dsq_and_switch_rq_lock(), 2065 * which already removed @p from @dsq and set 2066 * @p->scx.holding_cpu. Clear holding_cpu to tell 2067 * unlink_dsq_and_switch_rq_lock() that it lost the race. 2068 */ 2069 WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); 2070 p->scx.holding_cpu = -1; 2071 } 2072 p->scx.dsq = NULL; 2073 2074 if (!is_rq_owned) 2075 raw_spin_unlock(&dsq->lock); 2076 } 2077 2078 /* 2079 * Abbreviated version of scx_dispatch_dequeue() that can be used when both 2080 * @p's rq and dsq are locked. 2081 */ 2082 static void dispatch_dequeue_locked(struct task_struct *p, 2083 struct scx_dispatch_q *dsq) 2084 { 2085 lockdep_assert_rq_held(task_rq(p)); 2086 lockdep_assert_held(&dsq->lock); 2087 2088 scx_task_unlink_from_dsq(p, dsq); 2089 p->scx.dsq = NULL; 2090 } 2091 2092 static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, 2093 struct rq *rq, u64 dsq_id, 2094 s32 tcpu) 2095 { 2096 struct scx_dispatch_q *dsq; 2097 2098 if (dsq_id == SCX_DSQ_LOCAL) 2099 return &rq->scx.local_dsq; 2100 2101 if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { 2102 s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); 2103 2104 if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) 2105 return find_global_dsq(sch, tcpu); 2106 2107 return &cpu_rq(cpu)->scx.local_dsq; 2108 } 2109 2110 if (dsq_id == SCX_DSQ_GLOBAL) 2111 dsq = find_global_dsq(sch, tcpu); 2112 else 2113 dsq = find_user_dsq(sch, dsq_id); 2114 2115 /* 2116 * Built-in DSQs are never inserted into dsq_hash, so REJECT and RESCUE 2117 * hit the error below. They cannot be reached with an ID. 2118 */ 2119 if (unlikely(!dsq)) { 2120 scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); 2121 return find_global_dsq(sch, tcpu); 2122 } 2123 2124 return dsq; 2125 } 2126 2127 static void mark_direct_dispatch(struct scx_sched *sch, 2128 struct task_struct *ddsp_task, 2129 struct task_struct *p, u64 dsq_id, 2130 u64 slice, u64 vtime, u64 enq_flags) 2131 { 2132 /* 2133 * Mark that dispatch already happened from ops.select_cpu() or 2134 * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value 2135 * which can never match a valid task pointer. 2136 */ 2137 __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); 2138 2139 /* @p must match the task on the enqueue path */ 2140 if (unlikely(p != ddsp_task)) { 2141 if (IS_ERR(ddsp_task)) 2142 scx_error(sch, "%s[%d] already direct-dispatched", 2143 p->comm, p->pid); 2144 else 2145 scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]", 2146 ddsp_task->comm, ddsp_task->pid, 2147 p->comm, p->pid); 2148 return; 2149 } 2150 2151 WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); 2152 WARN_ON_ONCE(p->scx.ddsp_enq_flags); 2153 2154 p->scx.ddsp_slice = slice; 2155 p->scx.ddsp_vtime = vtime; 2156 p->scx.ddsp_dsq_id = dsq_id; 2157 p->scx.ddsp_enq_flags = enq_flags; 2158 } 2159 2160 /* 2161 * Clear @p direct dispatch state when leaving the scheduler. 2162 * 2163 * Direct dispatch state must be cleared in the following cases: 2164 * - direct_dispatch(): cleared on the synchronous enqueue path, deferred 2165 * dispatch keeps the state until consumed 2166 * - process_ddsp_deferred_locals(): cleared after consuming deferred state, 2167 * - scx_do_enqueue_task(): cleared on enqueue fallbacks where the dispatch 2168 * verdict is ignored (local/global/bypass) 2169 * - dequeue_task_scx(): cleared after scx_dispatch_dequeue(), covering 2170 * deferred cancellation and holding_cpu races 2171 * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by 2172 * the scx_bypass() loop, so that stale state is not reused by a subsequent 2173 * scheduler instance 2174 */ 2175 static inline void clear_direct_dispatch(struct task_struct *p) 2176 { 2177 p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; 2178 p->scx.ddsp_enq_flags = 0; 2179 } 2180 2181 static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, 2182 u64 enq_flags) 2183 { 2184 struct rq *rq = task_rq(p); 2185 struct scx_dispatch_q *dsq = 2186 find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); 2187 u64 ddsp_enq_flags, slice, vtime; 2188 2189 p->scx.ddsp_enq_flags |= enq_flags; 2190 2191 /* 2192 * We are in the enqueue path with @rq locked and pinned, and thus can't 2193 * double lock a remote rq and enqueue to its local DSQ. For 2194 * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer 2195 * the enqueue so that it's executed when @rq can be unlocked. 2196 */ 2197 if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { 2198 unsigned long opss; 2199 2200 opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; 2201 2202 switch (opss & SCX_OPSS_STATE_MASK) { 2203 case SCX_OPSS_NONE: 2204 break; 2205 case SCX_OPSS_QUEUEING: 2206 /* 2207 * As @p was never passed to the BPF side, _release is 2208 * not strictly necessary. Still do it for consistency. 2209 */ 2210 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 2211 break; 2212 default: 2213 WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()", 2214 p->comm, p->pid, opss); 2215 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 2216 break; 2217 } 2218 2219 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); 2220 list_add_tail(&p->scx.dsq_list.node, 2221 &rq->scx.ddsp_deferred_locals); 2222 schedule_deferred_locked(rq); 2223 return; 2224 } 2225 2226 ddsp_enq_flags = p->scx.ddsp_enq_flags; 2227 slice = p->scx.ddsp_slice; 2228 vtime = p->scx.ddsp_vtime; 2229 clear_direct_dispatch(p); 2230 2231 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, 2232 ddsp_enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); 2233 } 2234 2235 bool scx_rq_online(struct rq *rq) 2236 { 2237 /* 2238 * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE is set 2239 * after the root's ops.cpu_online() and cleared when the cpu goes down. 2240 * cpu_active() test guarantees that, if this function returns %true, 2241 * %SCX_RQ_ONLINE will stay set until the current scheduling operation 2242 * is complete even if we aren't locking @rq. 2243 */ 2244 return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); 2245 } 2246 2247 void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, 2248 int sticky_cpu) 2249 { 2250 enum { 2251 ENQ_ACTION_NONE, 2252 ENQ_ACTION_DIRECT, 2253 ENQ_ACTION_LOCAL_NOREFILL, 2254 ENQ_ACTION_ENQUEUE, 2255 } action = ENQ_ACTION_NONE; 2256 struct scx_sched *sch = scx_task_sched(p); 2257 struct task_struct **ddsp_taskp; 2258 struct scx_dispatch_q *dsq; 2259 unsigned long qseq; 2260 2261 WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); 2262 2263 /* internal movements - rq migration / RESTORE */ 2264 if (sticky_cpu == cpu_of(rq)) 2265 goto local_norefill; 2266 2267 /* 2268 * Clear persistent TASK_IMMED for fresh enqueues, see dsq_inc_nr(). 2269 * Note that exiting and migration-disabled tasks that skip 2270 * ops.enqueue() below will lose IMMED protection unless 2271 * %SCX_OPS_ENQ_EXITING / %SCX_OPS_ENQ_MIGRATION_DISABLED are set. 2272 */ 2273 p->scx.flags &= ~SCX_TASK_IMMED; 2274 2275 /* 2276 * If !scx_rq_online(), we already told the BPF scheduler that the CPU 2277 * is offline and are just running the hotplug path. Don't bother the 2278 * BPF scheduler. 2279 */ 2280 if (!scx_rq_online(rq)) 2281 goto local; 2282 2283 if (scx_bypassing(sch, cpu_of(rq))) { 2284 __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); 2285 goto bypass; 2286 } 2287 2288 /* 2289 * A task reenqueued too many times without running means the scheduler 2290 * keeps re-deciding a placement it can't honor, e.g. re-inserting to a 2291 * cid it lacks caps on. Eject the owning scheduler and strand the task 2292 * to be picked up during sched exit. 2293 * 2294 * Bypass guarantees that every runnable task keeps running, and the 2295 * disable relies on that to take sleeping locks. The kernel places the 2296 * tasks of a bypassing scheduler and those on an offline rq, and 2297 * counting their reenqueues could strand a task under bypass and 2298 * deadlock the disable on a lock the task holds, so both tests stay 2299 * above the count. 2300 */ 2301 if (enq_flags & SCX_ENQ_REENQ) { 2302 if (++p->scx.reenq_cnt > 1) 2303 __scx_add_event(sch, SCX_EV_REENQ_REPEAT, 1); 2304 2305 if (unlikely(p->scx.reenq_cnt > SCX_REENQ_MAX_REPEAT)) { 2306 __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), 2307 "%s[%d] reenqueued %u times without running", 2308 p->comm, p->pid, p->scx.reenq_cnt); 2309 goto out; 2310 } 2311 } 2312 2313 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) 2314 goto direct; 2315 2316 /* 2317 * A full wakeup enqueue happens before is_blocked is cleared. Don't 2318 * report it as another blocked-donor admission. 2319 */ 2320 if ((sch->ops.flags & SCX_OPS_ENQ_BLOCKED) && p->is_blocked && 2321 !(enq_flags & SCX_ENQ_WAKEUP)) { 2322 enq_flags |= SCX_ENQ_BLOCKED; 2323 } else { 2324 /* see %SCX_OPS_ENQ_EXITING */ 2325 if (unlikely(p->flags & PF_EXITING) && 2326 !(sch->ops.flags & SCX_OPS_ENQ_EXITING)) { 2327 __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); 2328 enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ 2329 goto local; 2330 } 2331 2332 /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ 2333 if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && 2334 is_migration_disabled(p)) { 2335 __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); 2336 goto local; 2337 } 2338 } 2339 2340 if (unlikely(!SCX_HAS_OP(sch, enqueue))) 2341 goto global; 2342 2343 /* 2344 * DSQ bypass didn't trigger, enqueue on the BPF scheduler. Wrap the 2345 * per-task qseq counter where the QSEQ field wraps and skip 0, which is 2346 * what scx_bpf_dsq_insert() records for a task in NONE or DISPATCHING. 2347 */ 2348 p->scx.ops_qseq = ((p->scx.ops_qseq + 1) & 2349 (SCX_OPSS_QSEQ_MASK >> SCX_OPSS_QSEQ_SHIFT)) ?: 1; 2350 qseq = (unsigned long)p->scx.ops_qseq << SCX_OPSS_QSEQ_SHIFT; 2351 2352 WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); 2353 atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); 2354 2355 ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); 2356 WARN_ON_ONCE(*ddsp_taskp); 2357 *ddsp_taskp = p; 2358 2359 SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags); 2360 2361 *ddsp_taskp = NULL; 2362 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) 2363 goto direct; 2364 2365 /* 2366 * Task is now in BPF scheduler's custody. Set %SCX_TASK_IN_CUSTODY 2367 * so ops.dequeue() is called when it leaves custody. 2368 */ 2369 p->scx.flags |= SCX_TASK_IN_CUSTODY; 2370 2371 /* 2372 * If not directly dispatched, QUEUEING isn't clear yet and dispatch or 2373 * dequeue may be waiting. The store_release matches their load_acquire. 2374 */ 2375 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); 2376 goto out; 2377 2378 direct: 2379 action = ENQ_ACTION_DIRECT; 2380 goto out; 2381 local_norefill: 2382 action = ENQ_ACTION_LOCAL_NOREFILL; 2383 goto out; 2384 local: 2385 dsq = &rq->scx.local_dsq; 2386 goto enqueue; 2387 global: 2388 dsq = find_global_dsq(sch, task_cpu(p)); 2389 goto enqueue; 2390 bypass: 2391 dsq = bypass_enq_target_dsq(sch, rq, p, &enq_flags); 2392 goto enqueue; 2393 2394 enqueue: 2395 action = ENQ_ACTION_ENQUEUE; 2396 out: 2397 /* The reason is input to ops.enqueue(), not to the resulting placement. */ 2398 p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; 2399 2400 switch (action) { 2401 case ENQ_ACTION_DIRECT: 2402 direct_dispatch(sch, p, enq_flags); 2403 break; 2404 case ENQ_ACTION_LOCAL_NOREFILL: 2405 scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, 2406 enq_flags); 2407 break; 2408 case ENQ_ACTION_ENQUEUE: 2409 refill_task_slice_dfl(sch, p); 2410 clear_direct_dispatch(p); 2411 scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); 2412 break; 2413 case ENQ_ACTION_NONE: 2414 break; 2415 } 2416 } 2417 2418 static bool task_runnable(const struct task_struct *p) 2419 { 2420 return !list_empty(&p->scx.runnable_node); 2421 } 2422 2423 static void set_task_runnable(struct rq *rq, struct task_struct *p) 2424 { 2425 lockdep_assert_rq_held(rq); 2426 2427 if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { 2428 p->scx.runnable_at = jiffies; 2429 p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; 2430 } 2431 2432 /* 2433 * list_add_tail() must be used. scx_bypass() and drain_offline_rq() 2434 * depend on tasks being appended to the runnable_list. 2435 */ 2436 list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); 2437 2438 /* 2439 * Record the rq @p is runnable on, maintained under the rq lock so it 2440 * stays valid unlike task_cpu(), which a remote wakeup can move under 2441 * pi_lock alone. 2442 */ 2443 WRITE_ONCE(p->scx.runnable_cpu, cpu_of(rq)); 2444 } 2445 2446 static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) 2447 { 2448 list_del_init(&p->scx.runnable_node); 2449 WRITE_ONCE(p->scx.runnable_cpu, -1); 2450 if (reset_runnable_at) { 2451 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; 2452 p->scx.reenq_cnt = 0; 2453 } 2454 } 2455 2456 static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) 2457 { 2458 struct scx_sched *sch = scx_task_sched(p); 2459 int sticky_cpu = p->scx.sticky_cpu; 2460 u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; 2461 2462 /* 2463 * An SCX-internal migration ends on arrival. Clear sticky_cpu so @p can 2464 * leave custody when inserted into the destination DSQ. 2465 */ 2466 if (sticky_cpu >= 0) 2467 p->scx.sticky_cpu = -1; 2468 2469 /* 2470 * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue 2471 * returns. Only the core's wakeup path delivers one. The flags stashed 2472 * for a remote activation may carry the wakeup bit without it. 2473 */ 2474 if (core_enq_flags & ENQUEUE_WAKEUP) 2475 rq->scx.flags |= SCX_RQ_IN_WAKEUP; 2476 2477 /* 2478 * Restoring the current scheduling context will be immediately followed 2479 * by set_next_task_scx() which expects the task to not be on the BPF 2480 * scheduler as tasks can only start running through local DSQs. Force 2481 * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark 2482 * IGNORE_CAPS to force entry into the local DSQ. 2483 */ 2484 if (unlikely(enq_flags & ENQUEUE_RESTORE) && 2485 task_current_donor(rq, p)) { 2486 sticky_cpu = cpu_of(rq); 2487 enq_flags |= SCX_ENQ_IGNORE_CAPS; 2488 } 2489 2490 if (p->scx.flags & SCX_TASK_QUEUED) { 2491 WARN_ON_ONCE(!task_runnable(p)); 2492 goto out; 2493 } 2494 2495 set_task_runnable(rq, p); 2496 p->scx.flags |= SCX_TASK_QUEUED; 2497 rq->scx.nr_running++; 2498 add_nr_running(rq, 1); 2499 2500 if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) 2501 SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); 2502 2503 /* Start dl_server if this is the first task being enqueued */ 2504 if (rq->scx.nr_running == 1) 2505 dl_server_start(&rq->ext_server); 2506 2507 scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu); 2508 out: 2509 rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; 2510 2511 if ((enq_flags & SCX_ENQ_CPU_SELECTED) && 2512 unlikely(cpu_of(rq) != p->scx.selected_cpu)) 2513 __scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1); 2514 } 2515 2516 static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) 2517 { 2518 struct scx_sched *sch = scx_task_sched(p); 2519 unsigned long opss; 2520 2521 /* 2522 * Don't reset runnable_at. If @p is waiting, the wait continues across 2523 * temporary dequeues, e.g. for a migration or an attribute change. 2524 * scx_reset_task_fields() ends it if @p leaves its sched's control. 2525 * scx_bypass() restarts it when its requeue can change the sched 2526 * running @p. 2527 */ 2528 clr_task_runnable(p, false); 2529 2530 retry: 2531 /* acquire ensures that we see the preceding updates on QUEUED */ 2532 opss = atomic_long_read_acquire(&p->scx.ops_state); 2533 2534 switch (opss & SCX_OPSS_STATE_MASK) { 2535 case SCX_OPSS_NONE: 2536 break; 2537 case SCX_OPSS_QUEUEING: 2538 /* 2539 * QUEUEING is started and finished while holding @p's rq lock. 2540 * As we're holding the rq lock now, we shouldn't see QUEUEING. 2541 */ 2542 BUG(); 2543 case SCX_OPSS_QUEUED: 2544 /* 2545 * A queued task must always be in BPF scheduler's custody. If 2546 * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another 2547 * CPU has already passed task_leave_custody() (which clears the 2548 * flag), but has not yet written SCX_OPSS_NONE. That final 2549 * store does not require this rq's lock, so retrying with 2550 * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, 2551 * handled by the fallthrough) on a subsequent iteration. 2552 */ 2553 if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) { 2554 cpu_relax(); 2555 goto retry; 2556 } 2557 2558 if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, 2559 SCX_OPSS_NONE)) 2560 break; 2561 fallthrough; 2562 case SCX_OPSS_DISPATCHING: 2563 /* 2564 * If @p is being dispatched from the BPF scheduler to a DSQ, 2565 * wait for the transfer to complete so that @p doesn't get 2566 * added to its DSQ after dequeueing is complete. 2567 * 2568 * As we're waiting on DISPATCHING with the rq locked, the 2569 * dispatching side shouldn't try to lock the rq while 2570 * DISPATCHING is set. See dispatch_to_local_dsq(). 2571 * 2572 * DISPATCHING shouldn't have qseq set and control can reach 2573 * here with NONE @opss from the above QUEUED case block. 2574 * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. 2575 */ 2576 wait_ops_state(p, SCX_OPSS_DISPATCHING); 2577 BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); 2578 break; 2579 } 2580 2581 /* 2582 * Call ops.dequeue() if the task is still in BPF custody. 2583 * 2584 * The code that clears ops_state to %SCX_OPSS_NONE does not always 2585 * clear %SCX_TASK_IN_CUSTODY: in dispatch_to_local_dsq(), when 2586 * we're moving a task that was in %SCX_OPSS_DISPATCHING to a 2587 * remote CPU's local DSQ, we only set ops_state to %SCX_OPSS_NONE 2588 * so that a concurrent dequeue can proceed, but we clear 2589 * %SCX_TASK_IN_CUSTODY only when we later enqueue or move the 2590 * task. So we can see NONE + IN_CUSTODY here and we must handle 2591 * it. Similarly, after waiting on %SCX_OPSS_DISPATCHING we see 2592 * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until 2593 * it is enqueued on the destination. 2594 */ 2595 if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) 2596 SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); 2597 } 2598 2599 static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) 2600 { 2601 struct scx_sched *sch = scx_task_sched(p); 2602 u64 deq_flags = core_deq_flags; 2603 2604 /* 2605 * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property 2606 * change (not sleep). 2607 */ 2608 if (!(deq_flags & DEQUEUE_SLEEP)) 2609 deq_flags |= SCX_DEQ_SCHED_CHANGE; 2610 2611 if (!(p->scx.flags & SCX_TASK_QUEUED)) { 2612 WARN_ON_ONCE(task_runnable(p)); 2613 return true; 2614 } 2615 2616 ops_dequeue(rq, p, deq_flags); 2617 2618 /* 2619 * A current scheduling context which is going off @rq first gets 2620 * dequeued and then stops running. As we want running <-> stopping 2621 * transitions to be contained within runnable <-> quiescent transitions, 2622 * trigger ->stopping() early here instead of in put_prev_task_scx(). 2623 * 2624 * @p may go through multiple stopping <-> running transitions between 2625 * here and put_prev_task_scx() if task attribute changes occur while 2626 * dispatch_one() leaves @rq unlocked. However, they don't contain any 2627 * information meaningful to the BPF scheduler and can be suppressed by 2628 * skipping the callbacks if the task is !QUEUED. 2629 */ 2630 if (task_current_donor(rq, p) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) { 2631 if (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq))) { 2632 update_curr_scx(rq); 2633 if (SCX_HAS_OP(sch, stopping)) 2634 SCX_CALL_OP_TASK(sch, stopping, rq, p, false); 2635 } 2636 p->scx.flags &= ~SCX_TASK_RUN_TRACKED; 2637 } 2638 2639 if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) 2640 SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags); 2641 2642 if (deq_flags & SCX_DEQ_SLEEP) 2643 p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; 2644 else 2645 p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; 2646 2647 p->scx.flags &= ~SCX_TASK_QUEUED; 2648 rq->scx.nr_running--; 2649 sub_nr_running(rq, 1); 2650 2651 scx_dispatch_dequeue(rq, p); 2652 p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; 2653 2654 /* see scx_task_slice_ended() for the save/restore exception */ 2655 if (!((deq_flags & DEQUEUE_SAVE) && task_current_donor(rq, p))) 2656 scx_task_slice_ended(rq, p); 2657 2658 clear_direct_dispatch(p); 2659 return true; 2660 } 2661 2662 static void yield_task_scx(struct rq *rq) 2663 { 2664 struct task_struct *p = rq->donor; 2665 struct scx_sched *sch = scx_task_sched(p); 2666 2667 /* a yield gives the slice up */ 2668 scx_task_slice_ended(rq, p); 2669 2670 if (SCX_HAS_OP(sch, yield)) 2671 SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); 2672 else 2673 scx_set_task_slice(p, 0); 2674 } 2675 2676 static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) 2677 { 2678 struct task_struct *from = rq->donor; 2679 struct scx_sched *sch = scx_task_sched(from); 2680 2681 /* like a plain yield, giving the slice up ends the protection */ 2682 scx_task_slice_ended(rq, from); 2683 2684 if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) 2685 return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); 2686 else 2687 return false; 2688 } 2689 2690 static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags) 2691 { 2692 /* 2693 * Preemption between SCX tasks is implemented by resetting the victim 2694 * task's slice to 0 and triggering reschedule on the target CPU. 2695 * 2696 * A retained proxy donor can wake through ttwu_runnable() without another 2697 * ops.enqueue(). WF_TTWU_RQ identifies this path. Request rescheduling so 2698 * that ops.dispatch() can reconsider the task after ttwu_runnable() clears 2699 * is_blocked. A full wakeup activation has already enqueued the task and 2700 * doesn't need the additional reschedule. 2701 */ 2702 if (p->sched_class == &ext_sched_class) { 2703 if (sched_proxy_exec() && (wake_flags & WF_TTWU_RQ) && 2704 p->is_blocked) { 2705 struct scx_sched *sch = scx_task_sched(p); 2706 2707 if (sch && (sch->ops.flags & SCX_OPS_ENQ_BLOCKED)) 2708 resched_curr(rq); 2709 } 2710 return; 2711 } 2712 2713 /* 2714 * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. 2715 * This captures all preemption cases including: 2716 * 2717 * - A SCX task is currently running. 2718 * 2719 * - @rq is waking from idle due to a SCX task waking to it. 2720 * 2721 * - A higher-priority wakes up while SCX dispatch is in progress. 2722 */ 2723 if (rq->scx.nr_immed) 2724 scx_schedule_reenq_local(rq, 0); 2725 } 2726 2727 void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, 2728 u64 enq_flags, struct rq *dst_rq) 2729 { 2730 struct scx_dispatch_q *dst_dsq; 2731 2732 lockdep_assert_rq_held(dst_rq); 2733 2734 WARN_ON_ONCE(p->scx.holding_cpu >= 0); 2735 scx_wait_dispatching(p); 2736 2737 dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); 2738 2739 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) 2740 dsq_insert_head(dst_dsq, p); 2741 else 2742 list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); 2743 2744 dsq_inc_nr(dst_dsq, p, enq_flags); 2745 p->scx.dsq = dst_dsq; 2746 2747 rq_owned_post_enq(sch, dst_rq, dst_dsq, p, enq_flags); 2748 } 2749 2750 /** 2751 * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ 2752 * @sch: scheduler placing @p 2753 * @p: task to move 2754 * @enq_flags: %SCX_ENQ_* 2755 * @src_rq: rq to move the task from, locked on entry, released on return 2756 * @dst_rq: rq to move the task into, locked on return 2757 * 2758 * Move @p which is currently on @src_rq to @dst_rq's local DSQ. 2759 */ 2760 static void move_remote_task_to_local_dsq(struct scx_sched *sch, 2761 struct task_struct *p, u64 enq_flags, 2762 struct rq *src_rq, struct rq *dst_rq) 2763 { 2764 lockdep_assert_rq_held(src_rq); 2765 2766 /* 2767 * Set sticky_cpu before deactivate_task() to properly mark the 2768 * beginning of an SCX-internal migration. 2769 */ 2770 p->scx.sticky_cpu = cpu_of(dst_rq); 2771 deactivate_task(src_rq, p, 0); 2772 set_task_cpu(p, cpu_of(dst_rq)); 2773 2774 switch_rq_lock(src_rq, dst_rq); 2775 2776 /* 2777 * activate_task() below truncates enq_flags to 32 bits and re-derives 2778 * @p's owner, dropping our scx flags and the placing @sch. We own @rq, 2779 * so stash both across the call. The enqueue reads them back, keeping 2780 * the scx flags and checking caps against the placer, not the owner. 2781 */ 2782 WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); 2783 WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags || 2784 dst_rq->scx.remote_activate_sch); 2785 dst_rq->scx.remote_activate_enq_flags = enq_flags; 2786 dst_rq->scx.remote_activate_sch = sch; 2787 activate_task(dst_rq, p, 0); 2788 dst_rq->scx.remote_activate_enq_flags = 0; 2789 dst_rq->scx.remote_activate_sch = NULL; 2790 } 2791 2792 /* 2793 * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two 2794 * differences: 2795 * 2796 * - is_cpu_allowed() asks "Can this task run on this CPU?" while 2797 * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to 2798 * this CPU?". 2799 * 2800 * While migration is disabled, is_cpu_allowed() has to say "yes" as the task 2801 * must be allowed to finish on the CPU that it's currently on regardless of 2802 * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the 2803 * BPF scheduler shouldn't attempt to migrate a task which has migration 2804 * disabled. 2805 * 2806 * - The BPF scheduler is bypassed while the rq is offline and we can always say 2807 * no to the BPF scheduler initiated migrations while offline. 2808 * 2809 * The caller must ensure that @p and @rq are on different CPUs. If @enforce is 2810 * true, report violations attributable to BPF-directed migrations. The caller 2811 * must hold @p's rq lock to avoid reporting a transient race as a scheduler 2812 * error. 2813 */ 2814 static bool task_can_run_on_remote_rq(struct scx_sched *sch, 2815 struct task_struct *p, struct rq *rq, 2816 bool enforce) 2817 { 2818 s32 cpu = cpu_of(rq); 2819 2820 if (enforce) 2821 lockdep_assert_rq_held(task_rq(p)); 2822 2823 WARN_ON_ONCE(task_cpu(p) == cpu); 2824 2825 /* 2826 * proxy_set_task_cpu() preserves wake_cpu when moving a donor's scheduling 2827 * context to its lock owner's rq. For example, if BPF places donor D on 2828 * CPU0 while its lock owner runs on CPU1, proxy execution moves D to CPU1 2829 * but leaves D->wake_cpu pointing to CPU0. If D is later put on a shared 2830 * DSQ, CPU0 could otherwise consume it and move it back, only for proxy 2831 * execution to return it to CPU1 again. 2832 * 2833 * When task_cpu() == wake_cpu, the donor has not been proxy-migrated and 2834 * BPF may still choose its placement. An actively donating task is 2835 * rejected separately by task_proxy_running_or_donating(). 2836 */ 2837 if (sched_proxy_exec() && p->is_blocked && task_cpu(p) != p->wake_cpu) 2838 return false; 2839 2840 /* 2841 * If @p has migration disabled, @p->cpus_ptr is updated to contain only 2842 * the pinned CPU in migrate_disable_switch() while @p is being switched 2843 * out. However, put_prev_task_scx() is called before @p->cpus_ptr is 2844 * updated and thus another CPU may see @p on a DSQ inbetween leading to 2845 * @p passing the below task_allowed_on_cpu() check while migration is 2846 * disabled. 2847 * 2848 * Test the migration disabled state first as the race window is narrow 2849 * and the BPF scheduler failing to check migration disabled state can 2850 * easily be masked if task_allowed_on_cpu() is done first. 2851 */ 2852 if (unlikely(is_migration_disabled(p))) { 2853 if (enforce) 2854 scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d", 2855 p->comm, p->pid, task_cpu(p), cpu); 2856 return false; 2857 } 2858 2859 /* 2860 * We don't require the BPF scheduler to avoid dispatching to offline 2861 * CPUs mostly for convenience but also because CPUs can go offline 2862 * between scx_bpf_dsq_insert() calls and here. Trigger error iff the 2863 * picked CPU is outside the allowed mask. 2864 */ 2865 if (!task_allowed_on_cpu(p, cpu)) { 2866 if (enforce) 2867 scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]", 2868 cpu, p->comm, p->pid); 2869 return false; 2870 } 2871 2872 if (!scx_rq_online(rq)) { 2873 if (enforce) 2874 __scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1); 2875 return false; 2876 } 2877 2878 return true; 2879 } 2880 2881 /* 2882 * Proxy execution can change @p's execution and migration-disabled state 2883 * without touching its DSQ entry or clearing holding_cpu. Check those states 2884 * with @p's rq locked. Without proxy execution, the holding_cpu handshake is 2885 * sufficient and this must not affect the existing migration path. 2886 * 2887 * A BPF-directed transfer to a remote local DSQ performs a normal task 2888 * migration and thus cannot move a migration-disabled task. In contrast, 2889 * proxy_migrate_task() moves only a blocked donor's scheduling context towards 2890 * the mutex owner and preserves its execution home in wake_cpu. The latter is 2891 * therefore allowed even when the donor is migration-disabled. 2892 */ 2893 static bool task_proxy_running_or_donating(struct task_struct *p) 2894 { 2895 struct rq *src_rq = task_rq(p); 2896 2897 lockdep_assert_rq_held(src_rq); 2898 2899 if (!sched_proxy_exec()) 2900 return false; 2901 2902 /* @p may be rq->curr under another task's scheduling context. */ 2903 if (task_on_cpu(src_rq, p)) 2904 return true; 2905 2906 /* Don't move an active scheduling context off its source rq. */ 2907 if (task_current_donor(src_rq, p)) 2908 return true; 2909 2910 return false; 2911 } 2912 2913 static bool task_proxy_unsafe_to_move(struct task_struct *p) 2914 { 2915 if (!sched_proxy_exec()) 2916 return false; 2917 2918 return task_proxy_running_or_donating(p) || is_migration_disabled(p); 2919 } 2920 2921 /* 2922 * Park a task whose remote transfer raced with proxy execution. Reenqueueing 2923 * from the source rq makes the task's owning scheduler choose its placement 2924 * again and preserves sub-scheduler containment. 2925 */ 2926 static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq, 2927 struct task_struct *p, u64 enq_flags) 2928 { 2929 lockdep_assert_rq_held(rq); 2930 WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); 2931 2932 p->scx.flags |= SCX_TASK_REENQ_PROXY; 2933 scx_divert_strip_flags(p, &enq_flags); 2934 2935 scx_dispatch_enqueue(sch, rq, &rq->scx.reject_dsq, p, 0, 0, enq_flags); 2936 } 2937 2938 /** 2939 * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock 2940 * @p: target task 2941 * @dsq: locked DSQ @p is currently on 2942 * @locked_rq: currently locked rq 2943 * @src_rq: rq @p is currently on, stable with @dsq locked 2944 * 2945 * Called with @dsq and @locked_rq locked. We want to move @p to a different DSQ, 2946 * including any local DSQ, but are not locking @src_rq. Locking @src_rq is 2947 * required when transferring into a local DSQ. Even when transferring into a 2948 * non-local DSQ, it's better to use the same mechanism to protect against 2949 * dequeues and maintain the invariant that @p->scx.dsq can only change while 2950 * @src_rq is locked, which e.g. scx_dump_task() depends on. 2951 * 2952 * We want to grab @src_rq but that can deadlock if we try while locking @dsq, 2953 * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As 2954 * this may race with dequeue, which can't drop the rq lock or fail, do a little 2955 * dancing from our side. 2956 * 2957 * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets 2958 * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu 2959 * would be cleared to -1. While other cpus may have updated it to different 2960 * values afterwards, as this operation can't be preempted or recurse, the 2961 * holding_cpu can never become this CPU again before we're done. Thus, we can 2962 * tell whether we lost to dequeue by testing whether the holding_cpu still 2963 * points to this CPU. See scx_dispatch_dequeue() for the counterpart. 2964 * 2965 * On return, @dsq is unlocked and @src_rq is locked. Return %true if @p is 2966 * still valid and detached from @dsq, with holding_cpu cleared. Return %false 2967 * if @p was lost to a dequeue. 2968 */ 2969 static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, 2970 struct scx_dispatch_q *dsq, 2971 struct rq *locked_rq, 2972 struct rq *src_rq) 2973 { 2974 s32 cpu = raw_smp_processor_id(); 2975 2976 lockdep_assert_held(&dsq->lock); 2977 lockdep_assert_rq_held(locked_rq); 2978 2979 WARN_ON_ONCE(p->scx.holding_cpu >= 0); 2980 scx_task_unlink_from_dsq(p, dsq); 2981 p->scx.holding_cpu = cpu; 2982 2983 raw_spin_unlock(&dsq->lock); 2984 switch_rq_lock(locked_rq, src_rq); 2985 2986 /* task_rq couldn't have changed if we're still the holding cpu */ 2987 if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) { 2988 /* 2989 * Keep ->dsq set until we own @src_rq so that a racing dequeue 2990 * can find @dsq and clear holding_cpu. Once we own @src_rq, the 2991 * handshake is over. Clear ->dsq so that deactivate_task() 2992 * takes the !dsq path instead of retaking @dsq->lock. Clear 2993 * holding_cpu with it so that no caller has to. 2994 */ 2995 p->scx.dsq = NULL; 2996 p->scx.holding_cpu = -1; 2997 scx_wait_dispatching(p); 2998 return true; 2999 } 3000 3001 return false; 3002 } 3003 3004 static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, 3005 struct task_struct *p, u64 enq_flags, 3006 struct scx_dispatch_q *dsq, struct rq *src_rq) 3007 { 3008 if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { 3009 /* 3010 * Proxy execution may have changed @p's running or 3011 * migration-disabled state while switching rq locks without 3012 * clearing holding_cpu. Park it on the source rq and let its 3013 * owning scheduler choose its placement again. 3014 */ 3015 if (unlikely(task_proxy_unsafe_to_move(p))) { 3016 p->scx.dsq = NULL; 3017 scx_proxy_reject_task(sch, src_rq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); 3018 switch_rq_lock(src_rq, this_rq); 3019 return false; 3020 } 3021 3022 move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); 3023 return true; 3024 } else { 3025 switch_rq_lock(src_rq, this_rq); 3026 return false; 3027 } 3028 } 3029 3030 /** 3031 * move_task_between_dsqs() - Move a task from one DSQ to another 3032 * @sch: scx_sched being operated on 3033 * @p: target task 3034 * @enq_flags: %SCX_ENQ_* 3035 * @src_dsq: DSQ @p is currently on, must not be a local DSQ 3036 * @dst_dsq: DSQ @p is being moved to, can be any DSQ 3037 * 3038 * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local 3039 * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq 3040 * will change. As @p's task_rq is locked, this function doesn't need to use the 3041 * holding_cpu mechanism. 3042 * 3043 * On return, @src_dsq is unlocked and only @p's new task_rq, which is the 3044 * return value, is locked. 3045 */ 3046 static struct rq *move_task_between_dsqs(struct scx_sched *sch, 3047 struct task_struct *p, u64 enq_flags, 3048 struct scx_dispatch_q *src_dsq, 3049 struct scx_dispatch_q *dst_dsq) 3050 { 3051 struct rq *src_rq = task_rq(p), *dst_rq; 3052 3053 BUG_ON(src_dsq->id == SCX_DSQ_LOCAL); 3054 lockdep_assert_held(&src_dsq->lock); 3055 lockdep_assert_rq_held(src_rq); 3056 3057 scx_wait_dispatching(p); 3058 3059 if (dst_dsq->id == SCX_DSQ_LOCAL) { 3060 dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); 3061 /* 3062 * Unlike the rq-lock handoff paths, @src_rq has been locked 3063 * throughout this operation. Only active proxy state can race the 3064 * move here; let the enforcing check below diagnose an ordinary 3065 * migration-disabled task. 3066 */ 3067 if (src_rq != dst_rq && 3068 unlikely(task_proxy_running_or_donating(p))) { 3069 dispatch_dequeue_locked(p, src_dsq); 3070 raw_spin_unlock(&src_dsq->lock); 3071 scx_proxy_reject_task(sch, src_rq, p, enq_flags); 3072 return src_rq; 3073 } 3074 if (src_rq != dst_rq && 3075 unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { 3076 dst_dsq = find_global_dsq(sch, task_cpu(p)); 3077 dst_rq = src_rq; 3078 enq_flags |= SCX_ENQ_GDSQ_FALLBACK; 3079 } 3080 } else { 3081 /* no need to migrate if destination is a non-local DSQ */ 3082 dst_rq = src_rq; 3083 } 3084 3085 /* 3086 * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different 3087 * CPU, @p will be migrated. 3088 */ 3089 if (dst_dsq->id == SCX_DSQ_LOCAL) { 3090 /* @p is going from a non-local DSQ to a local DSQ */ 3091 if (src_rq == dst_rq) { 3092 scx_task_unlink_from_dsq(p, src_dsq); 3093 raw_spin_unlock(&src_dsq->lock); 3094 scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq); 3095 } else { 3096 dispatch_dequeue_locked(p, src_dsq); 3097 raw_spin_unlock(&src_dsq->lock); 3098 move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); 3099 } 3100 } else { 3101 /* 3102 * @p is going from a non-local DSQ to a non-local DSQ. As 3103 * $src_dsq is already locked, do an abbreviated dequeue. 3104 */ 3105 dispatch_dequeue_locked(p, src_dsq); 3106 raw_spin_unlock(&src_dsq->lock); 3107 3108 scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, 0, 0, enq_flags); 3109 } 3110 3111 return dst_rq; 3112 } 3113 3114 bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, 3115 struct scx_dispatch_q *dsq, u64 enq_flags) 3116 { 3117 struct task_struct *p; 3118 retry: 3119 /* 3120 * The caller can't expect to successfully consume a task if the task's 3121 * addition to @dsq isn't guaranteed to be visible somehow. Test 3122 * @dsq->list without locking and skip if it seems empty. 3123 */ 3124 if (list_empty(&dsq->list)) 3125 return false; 3126 3127 raw_spin_lock(&dsq->lock); 3128 3129 nldsq_for_each_task(p, dsq) { 3130 struct rq *task_rq = task_rq(p); 3131 3132 /* 3133 * This loop can lead to multiple lockup scenarios, e.g. the BPF 3134 * scheduler can put an enormous number of affinitized tasks into 3135 * a contended DSQ, or the outer retry loop can repeatedly race 3136 * against scx_bypass() dequeueing tasks from @dsq trying to put 3137 * the system into the bypass mode. This can easily live-lock the 3138 * machine. If aborting, exit from all non-bypass DSQs. 3139 */ 3140 if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS) 3141 break; 3142 3143 if (rq == task_rq) { 3144 scx_task_unlink_from_dsq(p, dsq); 3145 raw_spin_unlock(&dsq->lock); 3146 scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq); 3147 return true; 3148 } 3149 3150 if (task_can_run_on_remote_rq(sch, p, rq, false)) { 3151 if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq))) 3152 return true; 3153 goto retry; 3154 } 3155 } 3156 3157 raw_spin_unlock(&dsq->lock); 3158 return false; 3159 } 3160 3161 bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) 3162 { 3163 int node = cpu_to_node(cpu_of(rq)); 3164 3165 return scx_consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); 3166 } 3167 3168 /** 3169 * dispatch_to_local_dsq - Dispatch a task to a local dsq 3170 * @sch: scx_sched being operated on 3171 * @rq: current rq which is locked 3172 * @dst_dsq: destination DSQ 3173 * @p: task to dispatch 3174 * @slice: slice carried by the insert verdict, 0 keeps the current value 3175 * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts 3176 * @enq_flags: %SCX_ENQ_* 3177 * 3178 * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local 3179 * DSQ. This function performs all the synchronization dancing needed because 3180 * local DSQs are protected with rq locks. 3181 * 3182 * The caller must have exclusive ownership of @p (e.g. through 3183 * %SCX_OPSS_DISPATCHING). 3184 */ 3185 static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, 3186 struct scx_dispatch_q *dst_dsq, struct task_struct *p, 3187 u64 slice, u64 vtime, u64 enq_flags) 3188 { 3189 struct rq *src_rq = task_rq(p); 3190 struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); 3191 struct rq *locked_rq = rq; 3192 3193 /* 3194 * We're synchronized against dequeue through DISPATCHING. As @p can't 3195 * be dequeued, its task_rq and cpus_allowed are stable too. 3196 * 3197 * If dispatching to @rq that @p is already on, no lock dancing needed. 3198 */ 3199 if (rq == src_rq && rq == dst_rq) { 3200 scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime, 3201 enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); 3202 return; 3203 } 3204 3205 /* 3206 * @p is on a possibly remote @src_rq which we need to lock to move the 3207 * task. If dequeue is in progress, it'd be locking @src_rq and waiting 3208 * on DISPATCHING, so we can't grab @src_rq lock while holding 3209 * DISPATCHING. 3210 * 3211 * As DISPATCHING guarantees that @p is wholly ours, we can pretend that 3212 * we're moving from a DSQ and use the same mechanism - mark the task 3213 * under transfer with holding_cpu, release DISPATCHING and then follow 3214 * the same protocol. See unlink_dsq_and_switch_rq_lock(). 3215 */ 3216 p->scx.holding_cpu = raw_smp_processor_id(); 3217 3218 /* store_release ensures that dequeue sees the above */ 3219 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 3220 3221 /* switch to @src_rq lock */ 3222 if (locked_rq != src_rq) { 3223 switch_rq_lock(locked_rq, src_rq); 3224 locked_rq = src_rq; 3225 } 3226 3227 /* task_rq couldn't have changed if we're still the holding cpu */ 3228 if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && 3229 !WARN_ON_ONCE(src_rq != task_rq(p))) { 3230 bool fallback = false; 3231 3232 /* the handshake is over, see unlink_dsq_and_switch_rq_lock() */ 3233 p->scx.holding_cpu = -1; 3234 3235 /* 3236 * If @p is staying on the same rq, there's no need to go 3237 * through the full deactivate/activate cycle. Optimize by 3238 * abbreviating move_remote_task_to_local_dsq(). 3239 */ 3240 if (src_rq == dst_rq) { 3241 scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, 3242 slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); 3243 } else if (unlikely(task_proxy_unsafe_to_move(p))) { 3244 fallback = true; 3245 scx_proxy_reject_task(sch, src_rq, p, enq_flags); 3246 } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { 3247 fallback = true; 3248 scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), 3249 p, slice, vtime, 3250 enq_flags | SCX_ENQ_APPLY_SLICE | 3251 SCX_ENQ_GDSQ_FALLBACK); 3252 } else { 3253 apply_slice_vtime(p, slice, vtime, enq_flags); 3254 move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); 3255 /* task has been moved to dst_rq, which is now locked */ 3256 locked_rq = dst_rq; 3257 } 3258 3259 /* if the destination CPU is idle, wake it up */ 3260 if (!fallback && sched_class_above(p->sched_class, 3261 dst_rq->donor->sched_class)) 3262 resched_curr(dst_rq); 3263 } 3264 3265 /* switch back to @rq lock */ 3266 if (locked_rq != rq) 3267 switch_rq_lock(locked_rq, rq); 3268 } 3269 3270 /** 3271 * finish_dispatch - Asynchronously finish dispatching a task 3272 * @sch: the scheduler 3273 * @rq: current rq which is locked 3274 * @p: task to finish dispatching 3275 * @qseq_at_dispatch: qseq when @p started getting dispatched 3276 * @dsq_id: destination DSQ ID 3277 * @slice: slice carried by the insert verdict, 0 keeps the current value 3278 * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts 3279 * @enq_flags: %SCX_ENQ_* 3280 * 3281 * Dispatching to local DSQs may need to wait for queueing to complete or 3282 * require rq lock dancing. As we don't wanna do either while inside 3283 * ops.dispatch() to avoid locking order inversion, we split dispatching into 3284 * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the 3285 * task and its qseq. Once ops.dispatch() returns, this function is called to 3286 * finish up. 3287 * 3288 * There is no guarantee that @p is still valid for dispatching or even that it 3289 * was valid in the first place. Make sure that the task is still owned by the 3290 * BPF scheduler and claim the ownership before dispatching. 3291 */ 3292 static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p, 3293 unsigned long qseq_at_dispatch, u64 dsq_id, 3294 u64 slice, u64 vtime, u64 enq_flags) 3295 { 3296 struct scx_dispatch_q *dsq; 3297 unsigned long opss; 3298 3299 retry: 3300 /* 3301 * No need for _acquire here. @p is accessed only after a successful 3302 * try_cmpxchg to DISPATCHING. 3303 */ 3304 opss = atomic_long_read(&p->scx.ops_state); 3305 3306 switch (opss & SCX_OPSS_STATE_MASK) { 3307 case SCX_OPSS_DISPATCHING: 3308 case SCX_OPSS_NONE: 3309 /* someone else already got to it */ 3310 return; 3311 case SCX_OPSS_QUEUED: 3312 /* 3313 * If qseq doesn't match, @p has gone through at least one 3314 * dispatch/dequeue and re-enqueue cycle between 3315 * scx_bpf_dsq_insert() and here and we have no claim on it. 3316 */ 3317 if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) 3318 return; 3319 3320 /* see SCX_EV_INSERT_NOT_OWNED definition */ 3321 if (unlikely(!scx_task_on_sched(sch, p))) { 3322 __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); 3323 return; 3324 } 3325 3326 /* 3327 * While we know @p is accessible, we don't yet have a claim on 3328 * it - the BPF scheduler is allowed to dispatch tasks 3329 * spuriously and there can be a racing dequeue attempt. Let's 3330 * claim @p by atomically transitioning it from QUEUED to 3331 * DISPATCHING. 3332 */ 3333 if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, 3334 SCX_OPSS_DISPATCHING))) 3335 break; 3336 goto retry; 3337 case SCX_OPSS_QUEUEING: 3338 /* 3339 * scx_do_enqueue_task() is in the process of transferring the 3340 * task to the BPF scheduler while holding @p's rq lock. As we 3341 * aren't holding any kernel or BPF resource that the enqueue 3342 * path may depend upon, it's safe to wait. 3343 */ 3344 wait_ops_state(p, opss); 3345 goto retry; 3346 } 3347 3348 BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); 3349 3350 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); 3351 3352 if (dsq->id == SCX_DSQ_LOCAL) 3353 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); 3354 else 3355 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, 3356 enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); 3357 } 3358 3359 void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) 3360 { 3361 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; 3362 u32 u; 3363 3364 for (u = 0; u < dspc->cursor; u++) { 3365 struct scx_dsp_buf_ent *ent = &dspc->buf[u]; 3366 3367 finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, 3368 ent->slice, ent->vtime, ent->enq_flags); 3369 } 3370 3371 dspc->nr_tasks += dspc->cursor; 3372 dspc->cursor = 0; 3373 } 3374 3375 static inline void maybe_queue_balance_callback(struct rq *rq) 3376 { 3377 lockdep_assert_rq_held(rq); 3378 3379 if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) 3380 return; 3381 3382 queue_balance_callback(rq, &rq->scx.deferred_bal_cb, 3383 deferred_bal_cb_workfn); 3384 3385 rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; 3386 } 3387 3388 static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev) 3389 { 3390 struct scx_sched *root_sch = scx_root_protected_live(); 3391 enum scx_dsp_verdict verdict; 3392 s32 cpu = cpu_of(rq); 3393 3394 lockdep_assert_rq_held(rq); 3395 rq->scx.flags |= SCX_RQ_IN_DISPATCH; 3396 3397 scx_process_sync_ecaps(rq, prev); 3398 3399 if ((root_sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && 3400 unlikely(rq->scx.cpu_released)) { 3401 /* 3402 * If the previous sched_class for the current CPU was not SCX, 3403 * notify the BPF scheduler that it again has control of the 3404 * core. This callback complements ->cpu_release(), which is 3405 * emitted in switch_class(). 3406 */ 3407 if (root_sch->ops.cpu_acquire) 3408 SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL); 3409 rq->scx.cpu_released = false; 3410 } 3411 3412 /* the ecaps sync may have dropped the rq lock */ 3413 if (scx_dsp_prev_stale(rq, prev)) { 3414 verdict = SCX_DSP_RETRY; 3415 goto has_tasks; 3416 } 3417 3418 if (prev->sched_class == &ext_sched_class) { 3419 update_curr_scx(rq); 3420 3421 /* 3422 * Under bypass, @prev's slice can't be trusted as a cpu-form 3423 * scheduler can write it directly from BPF. Skipping the keep 3424 * below is not enough as the post-dispatch test in 3425 * scx_dispatch_sched() would keep @prev on the same slice. 3426 * 3427 * Expire it as the bypass tick would, so that neither test 3428 * keeps @prev on it and @prev rotates through the bypass 3429 * enqueue. A kernel slice is left alone and the keep below 3430 * keeps @prev on it, see %SCX_TASK_KERNEL_SLICE. 3431 * 3432 * TODO: Once the cpu form is deprecated and the kfuncs refuse 3433 * slice raises from a bypassing scheduler, slices can be 3434 * trusted under bypass. This expiry, the bypass tick's zeroing 3435 * and the related slice handling can then be cleaned up. 3436 */ 3437 if (scx_bypassing(scx_task_sched(prev), cpu)) 3438 expire_task_slice(prev); 3439 3440 /* 3441 * If @prev is runnable & has slice left, it has priority and 3442 * fetching more just increases latency for the fetched tasks. 3443 * Tell pick_task_scx() to keep running @prev. If the BPF 3444 * scheduler wants to handle this explicitly, it should 3445 * implement ->cpu_release(). 3446 */ 3447 if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { 3448 verdict = SCX_DSP_PREV; 3449 goto has_tasks; 3450 } 3451 } 3452 3453 /* if there already are tasks to run, nothing to do */ 3454 if (rq->scx.local_dsq.nr) { 3455 verdict = SCX_DSP_LOCAL; 3456 goto has_tasks; 3457 } 3458 3459 verdict = scx_dispatch_sched(root_sch, rq, prev, false); 3460 3461 /* scx_dispatch_sched() may have dropped the rq lock */ 3462 if (scx_dsp_prev_stale(rq, prev)) 3463 verdict = SCX_DSP_RETRY; 3464 if (verdict != SCX_DSP_NONE) 3465 goto has_tasks; 3466 3467 /* 3468 * Didn't find another task to run. Keep running @prev unless its own 3469 * scheduler set %SCX_OPS_ENQ_LAST and takes the enqueue instead, see 3470 * put_prev_task_scx(). Read the scheduler here as the dispatch above 3471 * may have dropped the rq lock while @prev changed class or scheduler. 3472 */ 3473 if (prev->scx.flags & SCX_TASK_QUEUED) { 3474 struct scx_sched *prev_sch = scx_task_sched(prev); 3475 3476 if ((!(prev_sch->ops.flags & SCX_OPS_ENQ_LAST) || 3477 scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) { 3478 __scx_add_event(prev_sch, SCX_EV_DISPATCH_KEEP_LAST, 1); 3479 verdict = SCX_DSP_PREV; 3480 goto has_tasks; 3481 } 3482 3483 /* 3484 * The scheduler preempts a rescuee like any other task by 3485 * zeroing its slice. The grant survives the preemption. The 3486 * rescuee gets the unserved remainder back and runs again 3487 * behind whatever the scheduler wanted to run first. 3488 * 3489 * Two sites restore the remainder. When another task won the 3490 * pick, put_prev_task_scx() restores it and queues the rescuee 3491 * at the tail of the local DSQ. 3492 * 3493 * When nothing else is runnable, the cpu must not go idle with 3494 * the rescuee queued. The keep above takes a rescuee whose 3495 * scheduler holds %SCX_CAP_BASE here and doesn't take the last 3496 * task through %SCX_OPS_ENQ_LAST. 3497 * 3498 * The pick then ends its zeroed slice, and the rescue with it, 3499 * and refills a default slice. The task continues as an 3500 * ordinary one, which is fine as its scheduler holds BASE here. 3501 * 3502 * For any other rescuee, a reinsert from put_prev_task_scx() 3503 * would come after the pick settled on idle. Keep it running on 3504 * the remainder here. 3505 */ 3506 if (unlikely(prev == scx_rescuee(rq)) && scx_rescue_keep(rq, prev)) { 3507 verdict = SCX_DSP_PREV; 3508 goto has_tasks; 3509 } 3510 } 3511 rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; 3512 return SCX_DSP_NONE; 3513 3514 has_tasks: 3515 /* 3516 * @rq may have extra IMMED tasks without reenq scheduled: 3517 * 3518 * - rq_is_open() can't reliably tell when and how slice is going to be 3519 * modified for $curr and allows IMMED tasks to be queued while 3520 * dispatch is in progress. 3521 * 3522 * - A non-IMMED HEAD task can get queued in front of an IMMED task 3523 * between the IMMED queueing and the subsequent scheduling event. 3524 * 3525 * - When dispatch keeps @prev, even a lone IMMED task queued during the 3526 * dispatch can't run right away. 3527 */ 3528 if (unlikely((rq->scx.local_dsq.nr > 1 || verdict == SCX_DSP_PREV) && 3529 rq->scx.nr_immed)) 3530 scx_schedule_reenq_local(rq, 0); 3531 3532 rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; 3533 return verdict; 3534 } 3535 3536 static void scx_start_task_running(struct rq *rq, struct task_struct *p) 3537 { 3538 struct scx_sched *sch = scx_task_sched(p); 3539 3540 if (p->scx.flags & SCX_TASK_RUN_TRACKED) 3541 return; 3542 3543 scx_cid_sched_update(rq, sch); 3544 3545 if (SCX_HAS_OP(sch, running)) 3546 SCX_CALL_OP_TASK(sch, running, rq, p); 3547 3548 p->scx.flags |= SCX_TASK_RUN_TRACKED; 3549 } 3550 3551 static void kick_sync_wait_bal_cb(struct rq *rq); 3552 3553 static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) 3554 { 3555 bool first = type == SNT_PICK; 3556 bool can_stop_tick; 3557 3558 /* 3559 * A blocked pick is provisional until proxy resolution completes, see 3560 * put_prev_task_scx(). A retry can repick the same donor, so 3561 * SNT_REPICK must set it again. 3562 */ 3563 rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; 3564 if (sched_proxy_exec() && p->is_blocked && 3565 (type == SNT_PICK || type == SNT_REPICK)) 3566 rq->scx.flags |= SCX_RQ_PROXY_PENDING; 3567 3568 /* 3569 * Defer the kick_sync wait to a balance callback which can drop rq lock 3570 * and enable IRQs. Waiting directly in the pick path would deadlock 3571 * against CPUs sending us IPIs (e.g., TLB flushes) while we wait for 3572 * them. 3573 * 3574 * Queue it only once the pick is final, so that the rq lock can't be 3575 * released with the callback queued (e.g., by another dispatch during 3576 * a core-wide selection). If no SCX task gets picked, there's nothing 3577 * to wait for and kick_sync_pending stays set for the next SCX pick. 3578 */ 3579 if (unlikely(rq->scx.kick_sync_pending) && (type == SNT_PICK || type == SNT_REPICK)) 3580 queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, kick_sync_wait_bal_cb); 3581 3582 if (type == SNT_REPICK) 3583 return; 3584 3585 if (p->scx.flags & SCX_TASK_QUEUED) { 3586 /* 3587 * Core-sched might decide to execute @p before it is 3588 * dispatched. Call ops_dequeue() to notify the BPF scheduler. 3589 */ 3590 ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); 3591 scx_dispatch_dequeue(rq, p); 3592 p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; 3593 } 3594 3595 p->se.exec_start = rq_clock_task(rq); 3596 3597 /* 3598 * See dequeue_task_scx() for why we skip when !QUEUED. 3599 * 3600 * During a normal switch (@first), a blocked task is only a provisional 3601 * donor. Proxy resolution may fail or migrate the donor to another CPU, 3602 * so defer ops.running() until scx_proxy_donor_start() confirms that 3603 * resolution succeeded. 3604 * 3605 * !@first denotes restoration after a SAVE/RESTORE cycle. The matching 3606 * dequeue already issued ops.stopping(), so restart the session here 3607 * regardless of the donor state. 3608 */ 3609 if ((p->scx.flags & SCX_TASK_QUEUED) && !(p->is_blocked && first)) 3610 scx_start_task_running(rq, p); 3611 3612 clr_task_runnable(p, true); 3613 3614 /* apply any pending out-of-band slice request before the tick decision */ 3615 apply_task_slice_oob(rq, p); 3616 can_stop_tick = p->scx.slice == SCX_SLICE_INF && !p->is_blocked; 3617 3618 /* 3619 * @p is getting newly scheduled or got kicked after someone updated its 3620 * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be 3621 * stopped. See scx_can_stop_tick(). 3622 * 3623 * Moreover, refresh the load_avgs just when transitioning in and out of 3624 * nohz. In the future, we might want to add a mechanism to update 3625 * load_avgs periodically on tick-stopped CPUs. 3626 */ 3627 if (can_stop_tick) { 3628 if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { 3629 /* 3630 * Bypass mode always assigns finite slices, so @p 3631 * can't have an infinite slice while bypassing. 3632 * Therefore, sched_update_tick_dependency() can safely 3633 * evaluate the outgoing task. 3634 */ 3635 rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; 3636 sched_update_tick_dependency(rq); 3637 3638 update_other_load_avgs(rq); 3639 } 3640 } else { 3641 if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) { 3642 rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; 3643 update_other_load_avgs(rq); 3644 } 3645 3646 /* 3647 * @rq still references the outgoing scheduling context. A finite 3648 * slice or a blocked proxy donor is sufficient by itself to require 3649 * the tick. 3650 */ 3651 if (tick_nohz_full_cpu(cpu_of(rq))) 3652 tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); 3653 } 3654 } 3655 3656 void scx_proxy_donor_start(struct rq *rq) 3657 { 3658 struct task_struct *donor = rq->donor; 3659 3660 lockdep_assert_rq_held(rq); 3661 rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; 3662 3663 if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED)) 3664 scx_start_task_running(rq, donor); 3665 } 3666 3667 static enum scx_cpu_preempt_reason 3668 preempt_reason_from_class(const struct sched_class *class) 3669 { 3670 if (class == &stop_sched_class) 3671 return SCX_CPU_PREEMPT_STOP; 3672 if (class == &dl_sched_class) 3673 return SCX_CPU_PREEMPT_DL; 3674 if (class == &rt_sched_class) 3675 return SCX_CPU_PREEMPT_RT; 3676 return SCX_CPU_PREEMPT_UNKNOWN; 3677 } 3678 3679 static void switch_class(struct rq *rq, struct task_struct *next) 3680 { 3681 struct scx_sched *sch = scx_root_protected_live(); 3682 const struct sched_class *next_class = next->sched_class; 3683 3684 if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) 3685 return; 3686 3687 /* 3688 * The callback is conceptually meant to convey that the CPU is no 3689 * longer under the control of SCX. Therefore, don't invoke the callback 3690 * if the next class is below SCX (in which case the BPF scheduler has 3691 * actively decided not to schedule any tasks on the CPU). 3692 */ 3693 if (sched_class_above(&ext_sched_class, next_class)) 3694 return; 3695 3696 /* 3697 * At this point we know that SCX was preempted by a higher priority 3698 * sched_class, so invoke the ->cpu_release() callback if we have not 3699 * done so already. We only send the callback once between SCX being 3700 * preempted, and it regaining control of the CPU. 3701 * 3702 * ->cpu_release() complements ->cpu_acquire(), which is emitted the 3703 * next time that dispatch_one() is invoked. 3704 */ 3705 if (!rq->scx.cpu_released) { 3706 if (sch->ops.cpu_release) { 3707 struct scx_cpu_release_args args = { 3708 .reason = preempt_reason_from_class(next_class), 3709 .task = next, 3710 }; 3711 3712 SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args); 3713 } 3714 rq->scx.cpu_released = true; 3715 } 3716 } 3717 3718 static void put_prev_task_scx(struct rq *rq, struct task_struct *p, 3719 struct task_struct *next) 3720 { 3721 struct scx_sched *sch = scx_task_sched(p); 3722 bool proxy_put = p->is_blocked && next == rq->idle && 3723 (rq->scx.flags & SCX_RQ_PROXY_PENDING); 3724 bool proxy_block = rq->scx.flags & SCX_RQ_PROXY_BLOCKING; 3725 bool rescue_keep = false; 3726 bool going_idle = !proxy_put && next && 3727 sched_class_above(&ext_sched_class, next->sched_class); 3728 3729 rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; 3730 3731 /* see kick_sync_wait_bal_cb() */ 3732 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); 3733 3734 update_curr_scx(rq); 3735 3736 /* 3737 * Preserve the running session when proxy execution refreshes the same 3738 * donor around an execution-context switch on this rq. 3739 */ 3740 if (next != p && (p->scx.flags & SCX_TASK_QUEUED) && 3741 (p->scx.flags & SCX_TASK_RUN_TRACKED)) { 3742 if (SCX_HAS_OP(sch, stopping)) 3743 SCX_CALL_OP_TASK(sch, stopping, rq, p, !proxy_block); 3744 3745 p->scx.flags &= ~SCX_TASK_RUN_TRACKED; 3746 } 3747 3748 /* 3749 * If the slice is consumed, protection ends with it. 3750 * 3751 * A rescuee keeps going when a preemption or ops.stopping() zeroed its 3752 * slice. Here another task won the pick. scx_rescue_keep() restores the 3753 * unserved remainder and the reinsert below queues the rescuee at the 3754 * tail of the local DSQ. When nothing else is runnable, dispatch_one() 3755 * keeps the rescuee running on the remainder instead. 3756 */ 3757 if (!p->scx.slice) { 3758 if (unlikely(p == scx_rescuee(rq))) 3759 rescue_keep = scx_rescue_keep(rq, p); 3760 if (!rescue_keep) 3761 scx_task_slice_ended(rq, p); 3762 } 3763 3764 if (p->scx.flags & SCX_TASK_QUEUED) { 3765 if (proxy_block) 3766 goto switch_class; 3767 3768 set_task_runnable(rq, p); 3769 3770 /* 3771 * If the donor is the same and only the mutex owner changes, 3772 * avoid triggering another ops.enqueue(): the BPF scheduler has 3773 * already admitted the donor, so it can continue running. 3774 */ 3775 if (p->is_blocked && next == p) 3776 goto switch_class; 3777 3778 /* 3779 * If @p has slice left and is being put, @p is getting 3780 * preempted by a higher priority scheduler class or core-sched 3781 * forcing a different task. Leave it at the head of the local 3782 * DSQ unless it was an IMMED task. IMMED tasks should not 3783 * linger on a busy CPU, reenqueue them to the BPF scheduler. 3784 * 3785 * The exception is an IMMED donor put by proxy_resched_idle(): 3786 * the CPU isn't busy, proxy resolution is only retrying through 3787 * idle, so keep the donor where it can be picked again. 3788 * 3789 * A task with a kernel slice stays on the local DSQ even while 3790 * @sch bypasses. Under bypass, rescue runs @p when the host 3791 * lacks %SCX_CAP_BASE, see bypass_enq_target_dsq(). 3792 */ 3793 if (p->scx.slice && (!scx_bypassing(sch, cpu_of(rq)) || 3794 scx_task_has_kernel_slice(p))) { 3795 if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) { 3796 p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; 3797 scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); 3798 } else { 3799 u64 enq_flags = 0; 3800 3801 /* 3802 * The enqueue caps gate the scheduler's 3803 * placement. The kernel placed a kernel-slice 3804 * task here itself, so reinsert it without a 3805 * cap check, at the head like any preempted 3806 * task. When scx_rescue_keep() restored a slice 3807 * the scheduler zeroed, the scheduler wanted 3808 * something else to run first, so append. 3809 */ 3810 if (unlikely(scx_task_has_kernel_slice(p))) { 3811 enq_flags |= SCX_ENQ_IGNORE_CAPS; 3812 if (!rescue_keep) 3813 enq_flags |= SCX_ENQ_HEAD; 3814 } else { 3815 enq_flags |= SCX_ENQ_HEAD; 3816 /* only require SCX_CAP_ENQ_IMMED, see scx_caps_for_enq() */ 3817 if (proxy_put && (p->scx.flags & SCX_TASK_IMMED)) 3818 enq_flags |= SCX_ENQ_IMMED; 3819 } 3820 3821 scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, 3822 enq_flags); 3823 } 3824 goto switch_class; 3825 } 3826 3827 /* 3828 * If @p is runnable but we're about to enter a lower 3829 * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell 3830 * ops.enqueue() that @p is the only one available for this cpu. 3831 * This doesn't apply if the baseline access on the CPU is lost or 3832 * proxy resolution temporarily switches to idle. 3833 * 3834 * Under core scheduling, a pick dispatches only when nothing is 3835 * locally runnable and can legitimately go idle with @p still 3836 * runnable (see do_pick_task_scx()). 3837 */ 3838 if (going_idle && scx_task_can_stay_on_cpu(rq, p)) { 3839 WARN_ON_ONCE(!sched_core_enabled(rq) && 3840 !(sch->ops.flags & SCX_OPS_ENQ_LAST)); 3841 scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); 3842 } else { 3843 scx_do_enqueue_task(rq, p, 0, -1); 3844 } 3845 3846 /* 3847 * @p may be on the local DSQ now. With %SCX_ENQ_LAST the 3848 * scheduler can put the last runnable task there. The plain 3849 * insert runs when @p cannot stay on this cpu, and rescue can 3850 * still admit it here. 3851 * 3852 * The cpu must not go idle with @p queued on its local DSQ. The 3853 * usual resched doesn't work: the pick has settled on @next and 3854 * @p is still curr, so resched_curr() would set need_resched on 3855 * @p, which __schedule() clears right after the pick. Set it on 3856 * @next instead. 3857 */ 3858 if (going_idle && p->scx.dsq == &rq->scx.local_dsq) 3859 set_tsk_need_resched(next); 3860 } 3861 3862 switch_class: 3863 if (next && next->sched_class != &ext_sched_class) { 3864 scx_cid_sched_update(rq, NULL); 3865 switch_class(rq, next); 3866 } 3867 } 3868 3869 static void kick_sync_wait_bal_cb(struct rq *rq) 3870 { 3871 struct scx_kick_syncs __rcu *ks; 3872 unsigned long *ksyncs; 3873 bool waited; 3874 s32 cpu; 3875 3876 /* 3877 * This callback is queued once the pick is final and flushed within 3878 * @rq's own scheduling pass. If a lock taker on a foreign CPU ever 3879 * flushes it, the snapshots are unrelated to that CPU. Don't get in the 3880 * way and leave the wait pending for @rq's next pick. 3881 */ 3882 if (unlikely(cpu_of(rq) != smp_processor_id())) 3883 return; 3884 3885 /* 3886 * Clear only when the wait actually runs. The callback can be dropped 3887 * without running (e.g., proxy exec zaps the balance callbacks and 3888 * picks again when find_proxy_task() fails), in which case the next 3889 * pick must queue it again. 3890 */ 3891 rq->scx.kick_sync_pending = false; 3892 3893 ks = __this_cpu_read(scx_kick_syncs); 3894 ksyncs = rcu_dereference_sched(ks)->syncs; 3895 3896 /* 3897 * Drop rq lock and enable IRQs while waiting. IRQs must be enabled 3898 * — a target CPU may be waiting for us to process an IPI (e.g. TLB 3899 * flush) while we wait for its kick_sync to advance. 3900 * 3901 * Also, keep advancing our own kick_sync so that new kick_sync waits 3902 * targeting us, which can start after we drop the lock, cannot form 3903 * cyclic dependencies. 3904 */ 3905 retry: 3906 waited = false; 3907 for_each_cpu(cpu, rq->scx.cpus_to_sync) { 3908 /* 3909 * smp_load_acquire() pairs with smp_store_release() on 3910 * kick_sync updates on the target CPUs. 3911 */ 3912 if (cpu == cpu_of(rq) || 3913 smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) { 3914 cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync); 3915 continue; 3916 } 3917 3918 scx_rq_lock_drop(rq); 3919 raw_spin_rq_unlock_irq(rq); 3920 while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { 3921 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); 3922 cpu_relax(); 3923 } 3924 raw_spin_rq_lock_irq(rq); 3925 waited = true; 3926 } 3927 3928 if (waited) 3929 goto retry; 3930 } 3931 3932 static struct task_struct *first_local_task(struct rq *rq) 3933 { 3934 return list_first_entry_or_null(&rq->scx.local_dsq.list, 3935 struct task_struct, scx.dsq_list.node); 3936 } 3937 3938 /* 3939 * Run dispatch for a pick. The follow-up work is queued by do_pick_task_scx() 3940 * once the pick is final. 3941 */ 3942 static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, 3943 struct task_struct *prev) 3944 { 3945 enum scx_dsp_verdict verdict; 3946 3947 rq_unpin_lock(rq, rf); 3948 verdict = dispatch_one(rq, prev); 3949 rq_repin_lock(rq, rf); 3950 3951 return verdict; 3952 } 3953 3954 #ifdef CONFIG_SCHED_CORE 3955 /* 3956 * Dispatch for a pick when core scheduling is enabled. The selection picks for 3957 * all SMT siblings and the rq_i->core_pick state it builds must stay atomic 3958 * throughout. If the dispatch released the rq lock, anything can have happened 3959 * in between - return %SCX_DSP_RETRY to restart the selection against current 3960 * state. 3961 */ 3962 static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf, 3963 struct task_struct *prev) 3964 { 3965 enum scx_dsp_verdict verdict; 3966 u32 seq = rq->scx.lock_drop_seq; 3967 3968 /* another dispatch is in flight on @rq, let that handle it */ 3969 if (rq->scx.flags & SCX_RQ_IN_DISPATCH) 3970 return SCX_DSP_NONE; 3971 3972 rq_unpin_lock(rq, rf); 3973 3974 verdict = dispatch_one(rq, prev); 3975 3976 /* 3977 * A pick that returns a task isn't final under core scheduling: the 3978 * selection goes on to pick the siblings, and their dispatch can 3979 * release the rq lock, which can't happen with balance callbacks 3980 * queued. Don't queue the deferred work as a balance callback, for this 3981 * CPU or a sibling, and run it from irq_work instead. 3982 * 3983 * do_pick_task_scx() leaves %SCX_RQ_BAL_CB_PENDING alone under core 3984 * scheduling, so that the early return above leaves it to the dispatch 3985 * in flight, which handles it here. 3986 */ 3987 if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { 3988 rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; 3989 schedule_deferred(rq); 3990 } 3991 3992 rq_repin_lock(rq, rf); 3993 3994 /* if dispatch_one() released the rq lock, restart the selection */ 3995 if (rq->scx.lock_drop_seq != seq) 3996 return SCX_DSP_RETRY; 3997 3998 return verdict; 3999 } 4000 #else /* CONFIG_SCHED_CORE */ 4001 static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf, 4002 struct task_struct *prev) 4003 { 4004 return SCX_DSP_NONE; 4005 } 4006 #endif /* CONFIG_SCHED_CORE */ 4007 4008 static struct task_struct * 4009 do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) 4010 { 4011 struct task_struct *prev = rq->donor; 4012 bool core_sched = sched_core_enabled(rq); 4013 enum scx_dsp_verdict verdict; 4014 struct task_struct *p; 4015 4016 /* A retry can abandon a provisional blocked-donor pick. */ 4017 rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; 4018 4019 /* see kick_sync_wait_bal_cb() */ 4020 smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); 4021 4022 rq_modified_begin(rq, &ext_sched_class); 4023 4024 /* 4025 * Core scheduling can get enabled while dispatch_pick() has the rq lock 4026 * released. Stick to @core_sched, so that the follow-up work below 4027 * matches the dispatch that ran. 4028 */ 4029 if (core_sched) 4030 verdict = dispatch_core_pick(rq, rf, prev); 4031 else 4032 verdict = dispatch_pick(rq, rf, prev); 4033 4034 if (verdict == SCX_DSP_RETRY) { 4035 p = RETRY_TASK; 4036 goto out; 4037 } 4038 4039 /* 4040 * If any higher-priority sched class enqueued a runnable task on this 4041 * rq during dispatch_one(), abort and return RETRY_TASK, so that the 4042 * scheduler loop can restart. 4043 * 4044 * If @force_scx is true, always try to pick a SCHED_EXT task, 4045 * regardless of any higher-priority sched classes activity. 4046 */ 4047 if (!force_scx && rq_modified_above(rq, &ext_sched_class)) { 4048 p = RETRY_TASK; 4049 goto out; 4050 } 4051 4052 /* 4053 * If we're keeping @prev, replenish slice if necessary and keep running 4054 * @prev. Otherwise, pop the first one from the local DSQ. 4055 */ 4056 if (verdict == SCX_DSP_PREV) { 4057 p = prev; 4058 if (!p->scx.slice) { 4059 /* the slice is consumed, protection ends */ 4060 scx_task_slice_ended(rq, p); 4061 refill_task_slice_dfl(scx_task_sched(p), p); 4062 } 4063 } else { 4064 p = first_local_task(rq); 4065 if (!p) 4066 goto out; 4067 4068 if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) { 4069 struct scx_sched *sch = scx_task_sched(p); 4070 4071 if (!scx_bypassing(sch, cpu_of(rq)) && 4072 !sch->warned_zero_slice) { 4073 printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n", 4074 p->comm, p->pid, __func__); 4075 sch->warned_zero_slice = true; 4076 } 4077 refill_task_slice_dfl(sch, p); 4078 } 4079 } 4080 4081 out: 4082 if (cpu_of(rq) != smp_processor_id()) 4083 return p; 4084 4085 /* 4086 * Balance callbacks must be consumed before the rq lock is released, so 4087 * queue them only when the pick is final. On RETRY_TASK, or after a 4088 * NULL pick from the dl_server, the core continues picking and can 4089 * release the rq lock (e.g., prev_balance() or another dispatch). In 4090 * that case, run the deferred work from irq_work, which doesn't depend 4091 * on the current rq lock section. 4092 */ 4093 if (p != RETRY_TASK && (p || !force_scx)) { 4094 /* under core scheduling, see dispatch_core_pick() */ 4095 if (!core_sched) 4096 maybe_queue_balance_callback(rq); 4097 } else if (!core_sched && (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { 4098 rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; 4099 schedule_deferred(rq); 4100 } 4101 4102 return p; 4103 } 4104 4105 static struct task_struct *pick_task_scx(struct rq *rq, struct rq_flags *rf) 4106 { 4107 return do_pick_task_scx(rq, rf, false); 4108 } 4109 4110 /* 4111 * Select the next task to run from the ext scheduling class. 4112 * 4113 * Use do_pick_task_scx() directly with @force_scx enabled, since the 4114 * dl_server must always select a sched_ext task. 4115 */ 4116 static struct task_struct * 4117 ext_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) 4118 { 4119 if (!scx_enabled()) 4120 return NULL; 4121 4122 return do_pick_task_scx(dl_se->rq, rf, true); 4123 } 4124 4125 /* 4126 * Initialize the ext server deadline entity. 4127 */ 4128 void ext_server_init(struct rq *rq) 4129 { 4130 struct sched_dl_entity *dl_se = &rq->ext_server; 4131 4132 init_dl_entity(dl_se); 4133 4134 dl_server_init(dl_se, rq, ext_server_pick_task); 4135 } 4136 4137 #ifdef CONFIG_SCHED_CORE 4138 /** 4139 * scx_prio_less - Task ordering for core-sched 4140 * @a: task A 4141 * @b: task B 4142 * @in_fi: in forced idle state 4143 * 4144 * Core-sched is implemented as an additional scheduling layer on top of the 4145 * usual sched_class'es and needs to find out the expected task ordering. For 4146 * SCX, core-sched calls this function to interrogate the task ordering. 4147 * 4148 * A pair of tasks owned by one scheduler is ordered by the owner's 4149 * ops.core_sched_before(). A pair spanning two schedulers is ordered by their 4150 * nearest common ancestor which implements the op - the one case where the op 4151 * is called on tasks that the scheduler delegated to its sub-schedulers and may 4152 * not be scheduling anymore. 4153 * 4154 * When neither applies, or the deciding scheduler is bypassing on either task's 4155 * CPU, the default ordering runs the task which has been waiting longer first. 4156 * A running task counts as the most recently serviced and orders after every 4157 * waiting task. Waiting tasks are compared by @p->scx.runnable_at. 4158 * 4159 * Return: %true if @a should run after @b. 4160 */ 4161 bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, 4162 bool in_fi) 4163 { 4164 struct scx_sched *sch_a = scx_task_sched(a); 4165 struct scx_sched *sch_b = scx_task_sched(b); 4166 struct scx_sched *sch = NULL; 4167 bool a_running, b_running; 4168 4169 if (sch_a == sch_b) { 4170 if (SCX_HAS_OP(sch_a, core_sched_before)) 4171 sch = sch_a; 4172 } else { 4173 s32 level; 4174 4175 for (level = min(sch_a->level, sch_b->level); level >= 0; level--) { 4176 struct scx_sched *anc = sch_a->ancestors[level]; 4177 4178 if (anc == sch_b->ancestors[level] && 4179 SCX_HAS_OP(anc, core_sched_before)) { 4180 sch = anc; 4181 break; 4182 } 4183 } 4184 } 4185 4186 /* 4187 * scx_prio_less() returns whether @a should run after @b while 4188 * ops.core_sched_before() returns whether its first argument should run 4189 * before the second. Swap the arguments. 4190 * 4191 * The const qualifiers are dropped from task_struct pointers when 4192 * calling ops.core_sched_before(). Accesses are controlled by the 4193 * verifier. 4194 */ 4195 if (sch && !scx_bypassing(sch, task_cpu(a)) && !scx_bypassing(sch, task_cpu(b))) 4196 return SCX_CALL_OP_2TASKS_RET(sch, core_sched_before, task_rq(a), 4197 (struct task_struct *)b, 4198 (struct task_struct *)a); 4199 4200 /* 4201 * runnable_at is refreshed only on enqueue, so a task which keeps 4202 * occupying its CPU carries a stale stamp. A running task is the most 4203 * recently serviced whatever its stamp says. Order it after every 4204 * waiting task. 4205 */ 4206 a_running = a->on_cpu; 4207 b_running = b->on_cpu; 4208 if (a_running != b_running) 4209 return a_running; 4210 4211 return time_after(a->scx.runnable_at, b->scx.runnable_at); 4212 } 4213 #endif /* CONFIG_SCHED_CORE */ 4214 4215 static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) 4216 { 4217 struct scx_sched *sch = scx_task_sched(p); 4218 bool bypassing; 4219 4220 /* 4221 * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it 4222 * can be a good migration opportunity with low cache and memory 4223 * footprint. Returning a CPU different than @prev_cpu triggers 4224 * immediate rq migration. However, for SCX, as the current rq 4225 * association doesn't dictate where the task is going to run, this 4226 * doesn't fit well. If necessary, we can later add a dedicated method 4227 * which can decide to preempt self to force it through the regular 4228 * scheduling path. 4229 */ 4230 if (unlikely(wake_flags & WF_EXEC)) 4231 return prev_cpu; 4232 4233 bypassing = scx_bypassing(sch, task_cpu(p)); 4234 if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) { 4235 s32 cpu; 4236 struct task_struct **ddsp_taskp; 4237 4238 ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); 4239 WARN_ON_ONCE(*ddsp_taskp); 4240 *ddsp_taskp = p; 4241 4242 this_rq()->scx.in_select_cpu = true; 4243 cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, 4244 scx_cpu_arg(prev_cpu), wake_flags); 4245 cpu = scx_cpu_ret(sch, cpu); 4246 this_rq()->scx.in_select_cpu = false; 4247 p->scx.selected_cpu = cpu; 4248 *ddsp_taskp = NULL; 4249 if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()")) 4250 return cpu; 4251 else 4252 return prev_cpu; 4253 } else { 4254 s32 cpu; 4255 4256 /* 4257 * While bypassing, the enqueue path routes @p to a bypass DSQ 4258 * without consulting the direct-dispatch target, making the 4259 * default selection pointless. It doesn't work anyway when the 4260 * scheduler does its own idle tracking and the built-in idle 4261 * cpumasks are not updated. Leave @p on @prev_cpu. 4262 */ 4263 if (bypassing) { 4264 __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); 4265 p->scx.selected_cpu = prev_cpu; 4266 return prev_cpu; 4267 } 4268 4269 cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); 4270 if (cpu >= 0) { 4271 /* 4272 * Carry the slice refill and let the insertion commit 4273 * it under rq lock. See the write rules. 4274 */ 4275 __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); 4276 p->scx.ddsp_slice = READ_ONCE(sch->slice_dfl); 4277 p->scx.ddsp_enq_flags = SCX_ENQ_SLICE_DFL; 4278 p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; 4279 } else { 4280 cpu = prev_cpu; 4281 } 4282 p->scx.selected_cpu = cpu; 4283 4284 return cpu; 4285 } 4286 } 4287 4288 static void task_woken_scx(struct rq *rq, struct task_struct *p) 4289 { 4290 run_deferred(rq); 4291 } 4292 4293 static void set_cpus_allowed_scx(struct task_struct *p, 4294 struct affinity_context *ac) 4295 { 4296 struct scx_sched *sch = scx_task_sched(p); 4297 4298 set_cpus_allowed_common(p, ac); 4299 4300 if (task_dead_and_done(p)) 4301 return; 4302 4303 /* 4304 * The effective cpumask is stored in @p->cpus_ptr which may temporarily 4305 * differ from the configured one in @p->cpus_mask. Always tell the bpf 4306 * scheduler the effective one. 4307 * 4308 * Fine-grained memory write control is enforced by BPF making the const 4309 * designation pointless. Cast it away when calling the operation. 4310 * 4311 * The cid form receives the initial mask when the task is enabled and 4312 * hears about changes only afterwards, see struct scx_enable_args. 4313 */ 4314 if (SCX_HAS_OP(sch, set_cpumask) && 4315 (!scx_is_cid_type() || scx_get_task_state(p) == SCX_TASK_ENABLED)) 4316 scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); 4317 } 4318 4319 static void drain_offline_rq(struct rq *rq) 4320 { 4321 struct task_struct *p, *n; 4322 4323 scx_rescue_flush(rq); 4324 4325 /* 4326 * An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks 4327 * onto the local DSQ so that they run here and balance_push() moves 4328 * them off. 4329 */ 4330 list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) { 4331 if (p->scx.dsq == &rq->scx.local_dsq) 4332 continue; 4333 guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK); 4334 } 4335 } 4336 4337 static void handle_hotplug(struct rq *rq, bool online) 4338 { 4339 struct scx_sched *sch = scx_root_protected(); 4340 s32 cpu = cpu_of(rq); 4341 s32 cpu_or_cid = cpu; 4342 4343 if (!online) { 4344 guard(rq_lock_irqsave)(rq); 4345 update_rq_clock(rq); 4346 rq->scx.flags &= ~SCX_RQ_ONLINE; 4347 drain_offline_rq(rq); 4348 } 4349 4350 atomic_long_inc(&scx_hotplug_seq); 4351 scx_update_frozen_cpus(cpu, online); 4352 4353 /* 4354 * scx_root updates are protected by cpus_read_lock() and will stay 4355 * stable here. Note that we can't depend on scx_enabled() test as the 4356 * hotplug ops need to be enabled before __scx_enabled is set. 4357 */ 4358 if (unlikely(!sch)) { 4359 /* no root to notify, set %SCX_RQ_ONLINE right away */ 4360 if (online) { 4361 guard(rq_lock_irqsave)(rq); 4362 rq->scx.flags |= SCX_RQ_ONLINE; 4363 } 4364 return; 4365 } 4366 4367 if (scx_enabled()) 4368 scx_idle_update_selcpu_topology(&sch->ops); 4369 4370 /* 4371 * The tables can't be retired while this function is running as the 4372 * retirement is inside cpus_read_lock. However, scx_cpu_arg() is 4373 * awkward here as the tables can be NULL after root enable failure and 4374 * lockdep would trigger without surrounding rcu_read_lock(). Open code 4375 * the translation. If the table is NULL, the ops are also cleared and 4376 * @cpu_or_cid goes unused. 4377 */ 4378 if (scx_is_cid_type()) { 4379 s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl, 4380 lockdep_is_cpus_held()); 4381 4382 if (tbl) { 4383 struct scx_sched *pos; 4384 4385 cpu_or_cid = tbl[cpu]; 4386 4387 guard(raw_spinlock_irqsave)(&scx_sched_lock); 4388 list_for_each_entry(pos, &scx_sched_all, all) { 4389 struct scx_cmask *mask = pos->online_cmask; 4390 4391 if (mask) 4392 __assign_bit(cpu_or_cid, (unsigned long *)mask->bits, 4393 online); 4394 } 4395 } 4396 } 4397 4398 /* 4399 * ecaps are delivered for @cpu only after ops.cpu_online() and before 4400 * ops.cpu_offline(). The ecaps ops may read online_cmask, so 4401 * scx_offline_ecaps() and scx_online_ecaps() run after its update. 4402 */ 4403 if (!online) 4404 scx_offline_ecaps(rq); 4405 4406 if (online && SCX_HAS_OP(sch, cpu_online)) 4407 SCX_CALL_OP(sch, cpu_online, NULL, cpu_or_cid); 4408 else if (!online && SCX_HAS_OP(sch, cpu_offline)) 4409 SCX_CALL_OP(sch, cpu_offline, NULL, cpu_or_cid); 4410 else 4411 scx_exit(sch, SCX_EXIT_UNREG_KERN, 4412 SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, 4413 "cpu %d going %s, exiting scheduler", cpu, 4414 online ? "online" : "offline"); 4415 4416 if (online) { 4417 /* 4418 * The cpu must not call ops.enqueue() or ops.dispatch() before 4419 * the root's ops.cpu_online(). The core's rq online state can't 4420 * gate them, as the sched domain rebuild in 4421 * sched_cpu_activate() can put @rq online before this function 4422 * runs. Set %SCX_RQ_ONLINE here. 4423 * 4424 * scx_online_ecaps() queues a reseed whose ecaps ops run in 4425 * dispatch context on the cpu and can insert into its local 4426 * DSQ. Set the flag first. 4427 */ 4428 scoped_guard (rq_lock_irqsave, rq) 4429 rq->scx.flags |= SCX_RQ_ONLINE; 4430 scx_online_ecaps(rq); 4431 } 4432 } 4433 4434 void scx_rq_activate(struct rq *rq) 4435 { 4436 handle_hotplug(rq, true); 4437 } 4438 4439 void scx_rq_deactivate(struct rq *rq) 4440 { 4441 handle_hotplug(rq, false); 4442 } 4443 4444 static bool check_rq_for_timeouts(struct rq *rq) 4445 { 4446 struct scx_sched *sch; 4447 struct task_struct *p; 4448 struct rq_flags rf; 4449 bool timed_out = false; 4450 4451 rq_lock_irqsave(rq, &rf); 4452 sch = rcu_dereference_bh(scx_root); 4453 if (unlikely(!sch)) 4454 goto out_unlock; 4455 4456 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { 4457 /* 4458 * @p's owner schedules @p, or its host while the owner is 4459 * bypassing. Test @p against that sched's timeout and blame it. 4460 */ 4461 struct scx_sched *sch = scx_exec_sched(scx_task_sched(p), cpu_of(rq)); 4462 unsigned long last_runnable = p->scx.runnable_at; 4463 4464 if (unlikely(time_after(jiffies, 4465 last_runnable + READ_ONCE(sch->watchdog_timeout)))) { 4466 u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); 4467 4468 __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), 4469 "%s[%d] failed to run for %u.%03us", 4470 p->comm, p->pid, dur_ms / 1000, 4471 dur_ms % 1000); 4472 timed_out = true; 4473 break; 4474 } 4475 } 4476 out_unlock: 4477 rq_unlock_irqrestore(rq, &rf); 4478 return timed_out; 4479 } 4480 4481 static void scx_watchdog_workfn(struct work_struct *work) 4482 { 4483 unsigned long intv; 4484 int cpu; 4485 4486 WRITE_ONCE(scx_watchdog_timestamp, jiffies); 4487 4488 for_each_online_cpu(cpu) { 4489 if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) 4490 break; 4491 4492 cond_resched(); 4493 } 4494 4495 intv = READ_ONCE(scx_watchdog_interval); 4496 if (intv < ULONG_MAX) 4497 queue_delayed_work(system_dfl_wq, to_delayed_work(work), intv); 4498 } 4499 4500 void scx_tick(struct rq *rq) 4501 { 4502 struct scx_sched *root; 4503 unsigned long last_check; 4504 4505 if (!scx_enabled()) 4506 return; 4507 4508 root = rcu_dereference_bh(scx_root); 4509 if (unlikely(!root)) 4510 return; 4511 4512 last_check = READ_ONCE(scx_watchdog_timestamp); 4513 if (unlikely(time_after(jiffies, 4514 last_check + READ_ONCE(root->watchdog_timeout)))) { 4515 u32 dur_ms = jiffies_to_msecs(jiffies - last_check); 4516 4517 scx_exit(root, SCX_EXIT_ERROR_STALL, 0, 4518 "watchdog failed to check in for %u.%03us", 4519 dur_ms / 1000, dur_ms % 1000); 4520 } 4521 4522 update_other_load_avgs(rq); 4523 } 4524 4525 static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued) 4526 { 4527 struct scx_sched *sch = scx_task_sched(donor); 4528 4529 update_curr_scx(rq); 4530 4531 /* 4532 * While disabling, always resched as we can't trust the slice 4533 * management. 4534 */ 4535 if (scx_bypassing(sch, cpu_of(rq))) 4536 expire_task_slice(donor); 4537 else if (SCX_HAS_OP(sch, tick)) 4538 SCX_CALL_OP_TASK(sch, tick, rq, donor); 4539 4540 if (!donor->scx.slice) { 4541 /* the slice can't be trusted while bypassing */ 4542 if (READ_ONCE(donor->scx.lazy_resched) && 4543 !scx_bypassing(sch, cpu_of(rq))) 4544 scx_resched_curr_lazy(rq); 4545 else 4546 resched_curr(rq); 4547 } 4548 } 4549 4550 #ifdef CONFIG_EXT_GROUP_SCHED 4551 static struct cgroup *tg_cgrp(struct task_group *tg) 4552 { 4553 /* 4554 * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup, 4555 * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the 4556 * root cgroup. 4557 */ 4558 if (tg && tg->css.cgroup) 4559 return tg->css.cgroup; 4560 else 4561 return &cgrp_dfl_root.cgrp; 4562 } 4563 4564 #define SCX_INIT_TASK_ARGS_CGROUP(cgrp) .cgroup = (cgrp), 4565 4566 #else /* CONFIG_EXT_GROUP_SCHED */ 4567 4568 #define SCX_INIT_TASK_ARGS_CGROUP(cgrp) 4569 4570 #endif /* CONFIG_EXT_GROUP_SCHED */ 4571 4572 /** 4573 * __scx_init_task - Initialize a task for a sched 4574 * @sch: sched to initialize @p for 4575 * @p: task of interest 4576 * @cgrp: cgroup @p is joining, %NULL for @p's current task_group's cgroup 4577 * @fork: %true if @p is being forked 4578 * 4579 * Pre-commit cgroup migration passes @cgrp explicitly as @p's task_group 4580 * still reflects the source. 4581 * 4582 * Return 0 on success, -errno on failure. 4583 */ 4584 int __scx_init_task(struct scx_sched *sch, struct task_struct *p, 4585 struct cgroup *cgrp, bool fork) 4586 { 4587 int ret; 4588 4589 p->scx.disallow = false; 4590 4591 if (SCX_HAS_OP(sch, init_task)) { 4592 struct scx_init_task_args args = { 4593 SCX_INIT_TASK_ARGS_CGROUP(cgrp ?: tg_cgrp(task_group(p))) 4594 .fork = fork, 4595 }; 4596 4597 ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); 4598 if (unlikely(ret)) { 4599 ret = scx_ops_sanitize_err(sch, "init_task", ret); 4600 return ret; 4601 } 4602 } 4603 4604 if (p->scx.disallow) { 4605 if (unlikely(scx_parent(sch))) { 4606 scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]", 4607 p->comm, p->pid); 4608 } else if (unlikely(fork)) { 4609 scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", 4610 p->comm, p->pid); 4611 } else if (unlikely(scx_enable_state() != SCX_ENABLING)) { 4612 scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] outside the enable path", 4613 p->comm, p->pid); 4614 } else { 4615 struct rq *rq; 4616 struct rq_flags rf; 4617 4618 rq = task_rq_lock(p, &rf); 4619 4620 /* 4621 * We're in the load path and @p->policy will be applied 4622 * right after. Reverting @p->policy here and rejecting 4623 * %SCHED_EXT transitions from scx_check_setscheduler() 4624 * guarantees that if ops.init_task() sets @p->disallow, 4625 * @p can never be in SCX. 4626 */ 4627 if (p->policy == SCHED_EXT) { 4628 p->policy = SCHED_NORMAL; 4629 atomic_long_inc(&scx_nr_rejected); 4630 } 4631 4632 task_rq_unlock(rq, p, &rf); 4633 } 4634 } 4635 4636 return 0; 4637 } 4638 4639 static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) 4640 { 4641 struct rq *rq = task_rq(p); 4642 u32 weight; 4643 4644 lockdep_assert_rq_held(rq); 4645 4646 /* 4647 * Verify the task is not in BPF scheduler's custody. If flag 4648 * transitions are consistent, the flag should always be clear 4649 * here. 4650 */ 4651 WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); 4652 4653 /* 4654 * Set the weight before calling ops.enable() so that the scheduler 4655 * doesn't see a stale value if they inspect the task struct. 4656 */ 4657 if (task_has_idle_policy(p)) 4658 weight = WEIGHT_IDLEPRIO; 4659 else 4660 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; 4661 4662 p->scx.weight = sched_weight_to_cgroup(weight); 4663 p->scx.lazy_resched = sch->ops.flags & SCX_OPS_LAZY_RESCHED; 4664 4665 if (SCX_HAS_OP(sch, enable)) { 4666 if (scx_is_cid_type()) { 4667 struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr); 4668 struct scx_enable_args args = { 4669 .cmask_arena_addr = scx_kaddr_to_arena(sch, cmask), 4670 }; 4671 4672 SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args); 4673 } else { 4674 SCX_CALL_OP_TASK(sch, enable, rq, p); 4675 } 4676 } 4677 4678 /* 4679 * The initial mask also goes out through set_cmask() so a scheduler can 4680 * track affinity there alone, and before set_weight() so that the mask 4681 * is in place when weight-dependent state is derived, see struct 4682 * scx_enable_args. 4683 */ 4684 if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) 4685 scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr); 4686 4687 if (SCX_HAS_OP(sch, set_weight)) 4688 SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); 4689 } 4690 4691 void scx_enable_task(struct scx_sched *sch, struct task_struct *p) 4692 { 4693 __scx_enable_task(sch, p); 4694 scx_set_task_state(p, SCX_TASK_ENABLED); 4695 } 4696 4697 /** 4698 * scx_reset_task_fields - Reset what @p carries from its sched's control 4699 * @rq: @p's rq, locked 4700 * @p: task leaving its sched's control 4701 * 4702 * The vtime, the slice and what rides on it, the reenqueue count and the 4703 * watchdog wait belong to the sched while it controls @p. The control ends when 4704 * @p is taken off the sched or leaves the ext class while staying on the sched. 4705 * Reset them so that @p starts clean wherever it goes. 4706 */ 4707 static void scx_reset_task_fields(struct rq *rq, struct task_struct *p) 4708 { 4709 lockdep_assert_rq_held(rq); 4710 4711 p->scx.dsq_vtime = 0; 4712 scx_task_slice_ended(rq, p); 4713 scx_set_task_slice(p, 0); 4714 p->scx.reenq_cnt = 0; 4715 4716 /* 4717 * If @p is runnable but not running, its wait started at runnable_at, 4718 * which ops_dequeue() didn't reset. Leaving the sched's control ends 4719 * the wait. Reset runnable_at the next time @p is enqueued on SCX, so 4720 * that the watchdog doesn't count the time @p spends elsewhere. 4721 */ 4722 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; 4723 } 4724 4725 static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) 4726 { 4727 struct rq *rq = task_rq(p); 4728 4729 lockdep_assert_rq_held(rq); 4730 WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); 4731 4732 clear_direct_dispatch(p); 4733 4734 if (SCX_HAS_OP(sch, disable)) 4735 SCX_CALL_OP_TASK(sch, disable, rq, p); 4736 scx_set_task_state(p, SCX_TASK_READY); 4737 4738 /* 4739 * Verify the task is not in BPF scheduler's custody. If flag 4740 * transitions are consistent, the flag should always be clear 4741 * here. 4742 */ 4743 WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); 4744 } 4745 4746 void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) 4747 { 4748 struct scx_exit_task_args args = { 4749 .cancelled = false, 4750 }; 4751 4752 lockdep_assert_held(&p->pi_lock); 4753 lockdep_assert_rq_held(task_rq(p)); 4754 4755 switch (scx_get_task_state(p)) { 4756 case SCX_TASK_NONE: 4757 /* 4758 * No op to call, but @p may still carry @sch's state, e.g. a 4759 * task punted by scx_fail_parent() and being rescued. 4760 */ 4761 scx_reset_task_fields(task_rq(p), p); 4762 return; 4763 case SCX_TASK_INIT: 4764 args.cancelled = true; 4765 break; 4766 case SCX_TASK_READY: 4767 scx_reset_task_fields(task_rq(p), p); 4768 break; 4769 case SCX_TASK_ENABLED: 4770 scx_disable_task(sch, p); 4771 /* after ops.disable() has seen the final values */ 4772 scx_reset_task_fields(task_rq(p), p); 4773 break; 4774 default: 4775 WARN_ON_ONCE(true); 4776 return; 4777 } 4778 4779 if (SCX_HAS_OP(sch, exit_task)) 4780 SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); 4781 } 4782 4783 /* 4784 * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never 4785 * ran. The task state has not been transitioned, so this mirrors the 4786 * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). 4787 */ 4788 void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) 4789 { 4790 struct scx_exit_task_args args = { .cancelled = true }; 4791 4792 lockdep_assert_held(&p->pi_lock); 4793 lockdep_assert_rq_held(task_rq(p)); 4794 4795 /* @p was never associated with @sch, dispatch on the explicit @sch */ 4796 if (SCX_HAS_OP(sch, exit_task)) 4797 __SCX_CALL_OP_TASK(sch, ops, exit_task, task_rq(p), p, &args); 4798 } 4799 4800 void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) 4801 { 4802 __scx_disable_and_exit_task(sch, p); 4803 4804 /* 4805 * If set, @p exited between __scx_init_task() and scx_enable_task() in 4806 * scx_sub_enable() and is initialized for both the associated sched and 4807 * its parent. Exit for the child too - scx_enable_task() never ran for 4808 * it, so undo only init_task. The flag is only set on the sub-enable 4809 * path, so it's always clear when @p arrives here in %SCX_TASK_NONE. 4810 */ 4811 if (p->scx.flags & SCX_TASK_SUB_INIT) { 4812 if (!WARN_ON_ONCE(!scx_enabling_sub_sched)) 4813 scx_sub_init_cancel_task(scx_enabling_sub_sched, p); 4814 p->scx.flags &= ~SCX_TASK_SUB_INIT; 4815 } 4816 4817 scx_set_task_sched(p, NULL); 4818 scx_set_task_state(p, SCX_TASK_NONE); 4819 } 4820 4821 void init_scx_entity(struct sched_ext_entity *scx) 4822 { 4823 memset(scx, 0, sizeof(*scx)); 4824 INIT_LIST_HEAD(&scx->dsq_list.node); 4825 RB_CLEAR_NODE(&scx->dsq_priq); 4826 scx->sticky_cpu = -1; 4827 scx->holding_cpu = -1; 4828 scx->runnable_cpu = -1; 4829 INIT_LIST_HEAD(&scx->runnable_node); 4830 scx->runnable_at = jiffies; 4831 scx->ddsp_dsq_id = SCX_DSQ_INVALID; 4832 scx->slice = SCX_SLICE_DFL; 4833 } 4834 4835 /* See scx_tid_alloc / scx_tid_cursor. */ 4836 static u64 scx_alloc_tid(void) 4837 { 4838 struct scx_tid_alloc *ta; 4839 4840 guard(preempt)(); 4841 ta = this_cpu_ptr(&scx_tid_alloc); 4842 4843 if (unlikely(ta->next >= ta->end)) { 4844 ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor); 4845 ta->end = ta->next + SCX_TID_CHUNK; 4846 } 4847 return ta->next++; 4848 } 4849 4850 static void scx_tid_hash_insert(struct task_struct *p) 4851 { 4852 int ret; 4853 4854 lockdep_assert_held(&scx_tasks_lock); 4855 4856 ret = rhashtable_lookup_insert_fast(&scx_tid_hash, 4857 &p->scx.tid_hash_node, 4858 scx_tid_hash_params); 4859 WARN_ON_ONCE(ret); 4860 } 4861 4862 void scx_pre_fork(struct task_struct *p) 4863 { 4864 /* 4865 * BPF scheduler enable/disable paths want to be able to iterate and 4866 * update all tasks which can become complex when racing forks. As 4867 * enable/disable are very cold paths, let's use a percpu_rwsem to 4868 * exclude forks. 4869 */ 4870 percpu_down_read(&scx_fork_rwsem); 4871 } 4872 4873 int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) 4874 { 4875 s32 ret; 4876 4877 percpu_rwsem_assert_held(&scx_fork_rwsem); 4878 4879 p->scx.tid = scx_alloc_tid(); 4880 4881 if (scx_init_task_enabled) { 4882 #ifdef CONFIG_EXT_SUB_SCHED 4883 struct scx_sched *sch = scx_cgroup_sched(kargs->cset->dfl_cgrp); 4884 #else 4885 struct scx_sched *sch = scx_root_protected_live(); 4886 #endif 4887 scx_set_task_state(p, SCX_TASK_INIT_BEGIN); 4888 ret = __scx_init_task(sch, p, NULL, true); 4889 if (unlikely(ret)) { 4890 scx_set_task_state(p, SCX_TASK_NONE); 4891 return ret; 4892 } 4893 scx_set_task_state(p, SCX_TASK_INIT); 4894 scx_set_task_sched(p, sch); 4895 } 4896 4897 return 0; 4898 } 4899 4900 void scx_post_fork(struct task_struct *p) 4901 { 4902 if (scx_init_task_enabled) { 4903 scx_set_task_state(p, SCX_TASK_READY); 4904 4905 /* 4906 * Enable the task immediately if it's running on sched_ext. 4907 * Otherwise, it'll be enabled in switching_to_scx() if and 4908 * when it's ever configured to run with a SCHED_EXT policy. 4909 */ 4910 if (p->sched_class == &ext_sched_class) { 4911 struct rq_flags rf; 4912 struct rq *rq; 4913 4914 rq = task_rq_lock(p, &rf); 4915 scx_enable_task(scx_task_sched(p), p); 4916 task_rq_unlock(rq, p, &rf); 4917 } 4918 } 4919 4920 scoped_guard(raw_spinlock_irq, &scx_tasks_lock) { 4921 list_add_tail(&p->scx.tasks_node, &scx_tasks); 4922 if (scx_tid_to_task_enabled()) 4923 scx_tid_hash_insert(p); 4924 } 4925 4926 percpu_up_read(&scx_fork_rwsem); 4927 } 4928 4929 void scx_cancel_fork(struct task_struct *p) 4930 { 4931 if (scx_init_task_enabled) { 4932 struct rq *rq; 4933 struct rq_flags rf; 4934 4935 rq = task_rq_lock(p, &rf); 4936 WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY); 4937 scx_disable_and_exit_task(scx_task_sched(p), p); 4938 task_rq_unlock(rq, p, &rf); 4939 } 4940 4941 percpu_up_read(&scx_fork_rwsem); 4942 } 4943 4944 /** 4945 * task_dead_and_done - Is a task dead and done running? 4946 * @p: target task 4947 * 4948 * Once sched_ext_dead() removes the dead task from scx_tasks and exits it, the 4949 * task no longer exists from SCX's POV. However, certain sched_class ops may be 4950 * invoked on these dead tasks leading to failures - e.g. sched_setscheduler() 4951 * may try to switch a task which finished sched_ext_dead() back into SCX 4952 * triggering invalid SCX task state transitions and worse. 4953 * 4954 * Once a task has finished the final switch, sched_ext_dead() is the only thing 4955 * that needs to happen on the task. Use this test to short-circuit sched_class 4956 * operations which may be called on dead tasks. 4957 */ 4958 static bool task_dead_and_done(struct task_struct *p) 4959 { 4960 struct rq *rq = task_rq(p); 4961 4962 lockdep_assert_rq_held(rq); 4963 4964 /* 4965 * In do_task_dead(), a dying task sets %TASK_DEAD with preemption 4966 * disabled and __schedule(). If @p has %TASK_DEAD set and off CPU, @p 4967 * won't ever run again. 4968 */ 4969 return unlikely(READ_ONCE(p->__state) == TASK_DEAD) && 4970 !task_on_cpu(rq, p); 4971 } 4972 4973 void sched_ext_dead(struct task_struct *p) 4974 { 4975 /* 4976 * By the time control reaches here, @p has %TASK_DEAD set, switched out 4977 * for the last time and then dropped the rq lock - task_dead_and_done() 4978 * should be returning %true nullifying the straggling sched_class ops. 4979 * Remove from scx_tasks and exit @p. 4980 */ 4981 scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) { 4982 list_del_init(&p->scx.tasks_node); 4983 if (scx_tid_to_task_enabled()) 4984 rhashtable_remove_fast(&scx_tid_hash, 4985 &p->scx.tid_hash_node, 4986 scx_tid_hash_params); 4987 } 4988 4989 /* 4990 * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY -> 4991 * ENABLED transitions can't race us. Disable ops for @p. 4992 * 4993 * %SCX_TASK_DEAD synchronizes against cgroup task iteration - see 4994 * scx_task_iter_next_locked(). NONE tasks need no marking: cgroup 4995 * iteration is only used from sub-sched paths, which require root 4996 * enabled. Root enable transitions every live task to at least READY. 4997 * 4998 * %INIT_BEGIN means ops.init_task() is running for @p. Don't call 4999 * into ops; transition to %DEAD so the post-init recheck unwinds 5000 * via scx_sub_init_cancel_task(). 5001 */ 5002 if (scx_get_task_state(p) != SCX_TASK_NONE) { 5003 struct rq_flags rf; 5004 struct rq *rq; 5005 5006 rq = task_rq_lock(p, &rf); 5007 if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN) 5008 scx_disable_and_exit_task(scx_task_sched(p), p); 5009 scx_set_task_state(p, SCX_TASK_DEAD); 5010 task_rq_unlock(rq, p, &rf); 5011 } 5012 } 5013 5014 static void reweight_task_scx(struct rq *rq, struct task_struct *p, 5015 const struct load_weight *lw) 5016 { 5017 struct scx_sched *sch = scx_task_sched(p); 5018 5019 lockdep_assert_rq_held(task_rq(p)); 5020 5021 if (task_dead_and_done(p)) 5022 return; 5023 5024 /* 5025 * When switching sched_class away from SCX, reweight_task_scx() 5026 * is called _after_ scx_disable_task(). Skip calling ops.set_weight() 5027 * since the BPF scheduler may have already forgotten the task in 5028 * ops.disable(). 5029 * p->scx.weight will be recalculated in scx_enable_task() if the task 5030 * ever returns to SCX class. 5031 */ 5032 if (scx_get_task_state(p) != SCX_TASK_ENABLED) 5033 return; 5034 5035 p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); 5036 if (SCX_HAS_OP(sch, set_weight)) 5037 SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); 5038 } 5039 5040 static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio) 5041 { 5042 } 5043 5044 static void switching_to_scx(struct rq *rq, struct task_struct *p) 5045 { 5046 struct scx_sched *sch = scx_task_sched(p); 5047 5048 if (task_dead_and_done(p)) 5049 return; 5050 5051 scx_enable_task(sch, p); 5052 5053 /* 5054 * set_cpus_allowed_scx() is not called while @p is associated with a 5055 * different scheduler class. Keep the BPF scheduler up-to-date. The cid 5056 * form gets its mask from scx_enable_task(). 5057 */ 5058 if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) 5059 scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); 5060 } 5061 5062 static void switched_from_scx(struct rq *rq, struct task_struct *p) 5063 { 5064 /* 5065 * A class change of the running task: sched_change_begin() put @p with 5066 * no successor, which leaves rq->scx.sched set. 5067 */ 5068 if (task_current_donor(rq, p)) 5069 scx_cid_sched_update(rq, NULL); 5070 5071 if (task_dead_and_done(p)) 5072 return; 5073 5074 /* 5075 * %NONE means SCX is no longer tracking @p at the task level (e.g. 5076 * scx_fail_parent() handed @p back to the parent at NONE pending the 5077 * parent's own teardown). There is nothing to disable; calling 5078 * scx_disable_task() would WARN on the non-%ENABLED state and trigger a 5079 * NONE -> READY validation failure. 5080 * 5081 * Either way, reset @p after ops.disable() has seen the final values. A 5082 * %NONE @p can still be a rescuee under the parent. 5083 */ 5084 if (scx_get_task_state(p) != SCX_TASK_NONE) 5085 scx_disable_task(scx_task_sched(p), p); 5086 scx_reset_task_fields(rq, p); 5087 } 5088 5089 static void switched_to_scx(struct rq *rq, struct task_struct *p) {} 5090 5091 int scx_check_setscheduler(struct task_struct *p, int policy) 5092 { 5093 lockdep_assert_rq_held(task_rq(p)); 5094 5095 /* If disallow, reject transitioning into SCX. */ 5096 if (scx_enabled() && READ_ONCE(p->scx.disallow) && 5097 p->policy != policy && policy == SCHED_EXT) 5098 return -EACCES; 5099 5100 return 0; 5101 } 5102 5103 /* 5104 * Don't carry a donor retained by another class into sched_ext. The caller 5105 * has updated the rq clock and invokes this immediately before 5106 * sched_change_begin() records the task's queued state. 5107 */ 5108 void scx_prepare_setscheduler(struct task_struct *p, int policy) 5109 { 5110 lockdep_assert_held(&p->pi_lock); 5111 lockdep_assert_rq_held(task_rq(p)); 5112 5113 if (scx_enabled() && p->policy != policy && policy == SCHED_EXT) 5114 scx_block_proxy_donor(task_rq(p), p); 5115 } 5116 5117 static void process_ddsp_deferred_locals(struct rq *rq) 5118 { 5119 struct task_struct *p; 5120 5121 lockdep_assert_rq_held(rq); 5122 5123 /* 5124 * Now that @rq can be unlocked, execute the deferred enqueueing of 5125 * tasks directly dispatched to the local DSQs of other CPUs. See 5126 * direct_dispatch(). Keep popping from the head instead of using 5127 * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq 5128 * temporarily. 5129 */ 5130 while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, 5131 struct task_struct, scx.dsq_list.node))) { 5132 struct scx_sched *sch = scx_task_sched(p); 5133 struct scx_dispatch_q *dsq; 5134 u64 dsq_id = p->scx.ddsp_dsq_id; 5135 u64 enq_flags = p->scx.ddsp_enq_flags; 5136 u64 slice = p->scx.ddsp_slice; 5137 u64 vtime = p->scx.ddsp_vtime; 5138 5139 list_del_init(&p->scx.dsq_list.node); 5140 clear_direct_dispatch(p); 5141 5142 dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); 5143 if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) 5144 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); 5145 } 5146 } 5147 5148 /* 5149 * Determine whether @p should be reenqueued from a local DSQ. 5150 * 5151 * @reenq_flags is mutable and accumulates state across the DSQ walk: 5152 * 5153 * - %SCX_REENQ_TSR_NOT_FIRST: Set after the first task is visited. "First" 5154 * tracks position in the DSQ list, not among IMMED tasks. A non-IMMED task at 5155 * the head consumes the first slot. 5156 * 5157 * - %SCX_REENQ_TSR_RQ_OPEN: Set by reenq_local() before the walk if 5158 * rq_is_open() is true. 5159 * 5160 * An IMMED task is kept (returns %false) only if it's the first task in the DSQ 5161 * AND the current task is done — i.e. it will execute immediately. All other 5162 * IMMED tasks are reenqueued. This means if a non-IMMED task sits at the head, 5163 * every IMMED task behind it gets reenqueued. 5164 * 5165 * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ | 5166 * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local 5167 * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers 5168 * another reenq cycle. Repetitions are bounded by %SCX_REENQ_MAX_REPEAT in 5169 * scx_do_enqueue_task(), which ejects the task's owning scheduler. A bypassing 5170 * scheduler's tasks are placed by the kernel instead and are not counted. 5171 */ 5172 static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, 5173 u64 *reenq_flags, u32 *reason) 5174 { 5175 bool first; 5176 5177 first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); 5178 *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; 5179 5180 if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || scx_task_has_kernel_slice(p))) 5181 return false; 5182 5183 *reason = SCX_TASK_REENQ_KFUNC; 5184 5185 if ((p->scx.flags & SCX_TASK_IMMED) && 5186 (!first || !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) { 5187 __scx_add_event(scx_task_sched(p), SCX_EV_REENQ_IMMED, 1); 5188 *reason = SCX_TASK_REENQ_IMMED; 5189 return true; 5190 } 5191 5192 if ((*reenq_flags & SCX_REENQ_CAP_REVOKE) && 5193 scx_task_reenq_on_cap_revoke(rq, p)) { 5194 *reason = SCX_TASK_REENQ_CAP; 5195 return true; 5196 } 5197 5198 return *reenq_flags & SCX_REENQ_ANY; 5199 } 5200 5201 /* 5202 * The dispatcher runs ops.dequeue() and stores the final ops_state after 5203 * dropping the DSQ lock, so @p can be found on a DSQ while still 5204 * %SCX_OPSS_DISPATCHING. Wait for the dispatch to finish before operating on 5205 * @p. 5206 */ 5207 void scx_wait_dispatching(struct task_struct *p) 5208 { 5209 if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) 5210 wait_ops_state(p, SCX_OPSS_DISPATCHING); 5211 } 5212 5213 static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) 5214 { 5215 LIST_HEAD(tasks); 5216 u32 nr_enqueued = 0; 5217 struct task_struct *p, *n; 5218 5219 lockdep_assert_rq_held(rq); 5220 5221 if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK)) 5222 reenq_flags &= ~__SCX_REENQ_TSR_MASK; 5223 if (rq_is_open(rq, 0)) 5224 reenq_flags |= SCX_REENQ_TSR_RQ_OPEN; 5225 5226 /* 5227 * The BPF scheduler may choose to dispatch tasks back to 5228 * @rq->scx.local_dsq. Move all candidate tasks off to a private list 5229 * first to avoid processing the same tasks repeatedly. 5230 */ 5231 list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, 5232 scx.dsq_list.node) { 5233 struct scx_sched *task_sch = scx_task_sched(p); 5234 u32 reason; 5235 5236 /* 5237 * If @p is being migrated, @p's current CPU may not agree with 5238 * its allowed CPUs and the migration_cpu_stop is about to 5239 * deactivate and re-activate @p anyway. Skip re-enqueueing. 5240 * 5241 * While racing sched property changes may also dequeue and 5242 * re-enqueue a migrating task while its current CPU and allowed 5243 * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to 5244 * the current local DSQ for running tasks and thus are not 5245 * visible to the BPF scheduler. 5246 */ 5247 if (p->migration_pending) 5248 continue; 5249 5250 if (!scx_is_descendant(task_sch, sch)) 5251 continue; 5252 5253 if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) 5254 continue; 5255 5256 scx_wait_dispatching(p); 5257 scx_dispatch_dequeue(rq, p); 5258 5259 WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); 5260 p->scx.flags |= reason; 5261 5262 list_add_tail(&p->scx.dsq_list.node, &tasks); 5263 } 5264 5265 list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { 5266 list_del_init(&p->scx.dsq_list.node); 5267 5268 scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); 5269 5270 nr_enqueued++; 5271 } 5272 5273 /* 5274 * The revoke that scheduled this scan may have raced the pick: donor 5275 * may be a now-capless task, either one that kept running or one 5276 * promoted off the local DSQ between the ecaps sync and this scan. 5277 * Zero the slice to evict it. The enqueue gate blocks new capless 5278 * inserts, so no later pick can slip through after the scan. 5279 */ 5280 if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && 5281 rq->donor->sched_class == &ext_sched_class && 5282 scx_task_reenq_on_cap_revoke(rq, rq->donor)) { 5283 expire_task_slice(rq->donor); 5284 resched_curr(rq); 5285 } 5286 5287 return nr_enqueued; 5288 } 5289 5290 static void process_deferred_reenq_locals(struct rq *rq) 5291 { 5292 lockdep_assert_rq_held(rq); 5293 5294 /* 5295 * A task can be re-queued within this loop when a reenqueued task 5296 * bounces straight back to the local DSQ. That recursion is bounded by 5297 * the per-task reenqueue cap in scx_do_enqueue_task(). 5298 */ 5299 while (true) { 5300 struct scx_sched *sch; 5301 u64 reenq_flags; 5302 5303 scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { 5304 struct scx_deferred_reenq_local *drl = 5305 list_first_entry_or_null(&rq->scx.deferred_reenq_locals, 5306 struct scx_deferred_reenq_local, 5307 node); 5308 struct scx_sched_pcpu *sch_pcpu; 5309 5310 if (!drl) 5311 return; 5312 5313 sch_pcpu = container_of(drl, struct scx_sched_pcpu, 5314 deferred_reenq_local); 5315 sch = sch_pcpu->sch; 5316 5317 reenq_flags = drl->flags; 5318 WRITE_ONCE(drl->flags, 0); 5319 list_del_init(&drl->node); 5320 } 5321 5322 /* see schedule_dsq_reenq() */ 5323 smp_mb(); 5324 5325 reenq_local(sch, rq, reenq_flags); 5326 } 5327 } 5328 5329 static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) 5330 { 5331 *reason = SCX_TASK_REENQ_KFUNC; 5332 if (reenq_flags & SCX_REENQ_ANY) 5333 return true; 5334 return (reenq_flags & SCX_REENQ_MARKED) && (p->scx.dsq_flags & SCX_TASK_DSQ_REENQ); 5335 } 5336 5337 static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) 5338 { 5339 struct rq *locked_rq = rq; 5340 struct scx_sched *sch = dsq->sched; 5341 struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); 5342 struct task_struct *p; 5343 s32 nr_visited = 0; 5344 5345 lockdep_assert_rq_held(rq); 5346 5347 raw_spin_lock(&dsq->lock); 5348 5349 while (likely(!READ_ONCE(sch->bypass_depth))) { 5350 struct rq *task_rq; 5351 u32 reason; 5352 5353 /* 5354 * Count skipped tasks too so that a long walk doesn't hold the 5355 * locks throughout. The cursor keeps the position. 5356 */ 5357 if (!(++nr_visited % SCX_TASK_ITER_BATCH)) { 5358 raw_spin_unlock(&dsq->lock); 5359 if (locked_rq) { 5360 scx_rq_lock_drop(locked_rq); 5361 raw_spin_rq_unlock(locked_rq); 5362 locked_rq = NULL; 5363 } 5364 cpu_relax(); 5365 raw_spin_lock(&dsq->lock); 5366 } 5367 5368 p = nldsq_cursor_next_task(&cursor, dsq); 5369 if (!p) 5370 break; 5371 5372 if (!user_task_should_reenq(p, reenq_flags, &reason)) 5373 continue; 5374 5375 task_rq = task_rq(p); 5376 5377 if (locked_rq != task_rq) { 5378 if (locked_rq) { 5379 scx_rq_lock_drop(locked_rq); 5380 raw_spin_rq_unlock(locked_rq); 5381 } 5382 if (unlikely(!raw_spin_rq_trylock(task_rq))) { 5383 raw_spin_unlock(&dsq->lock); 5384 raw_spin_rq_lock(task_rq); 5385 raw_spin_lock(&dsq->lock); 5386 } 5387 locked_rq = task_rq; 5388 5389 /* did we lose @p while switching locks? */ 5390 if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p)) 5391 continue; 5392 } 5393 5394 /* @p is on @dsq, its rq and @dsq are locked */ 5395 scx_wait_dispatching(p); 5396 dispatch_dequeue_locked(p, dsq); 5397 raw_spin_unlock(&dsq->lock); 5398 5399 WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); 5400 p->scx.flags |= reason; 5401 5402 scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); 5403 5404 raw_spin_lock(&dsq->lock); 5405 } 5406 5407 list_del_init(&cursor.node); 5408 raw_spin_unlock(&dsq->lock); 5409 5410 if (locked_rq != rq) { 5411 if (locked_rq) { 5412 scx_rq_lock_drop(locked_rq); 5413 raw_spin_rq_unlock(locked_rq); 5414 } 5415 raw_spin_rq_lock(rq); 5416 } 5417 } 5418 5419 static void process_deferred_reenq_users(struct rq *rq) 5420 { 5421 lockdep_assert_rq_held(rq); 5422 5423 while (true) { 5424 struct scx_dispatch_q *dsq; 5425 u64 dsq_id, reenq_flags; 5426 5427 scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { 5428 struct scx_deferred_reenq_user *dru = 5429 list_first_entry_or_null(&rq->scx.deferred_reenq_users, 5430 struct scx_deferred_reenq_user, 5431 node); 5432 struct scx_dsq_pcpu *dsq_pcpu; 5433 5434 if (!dru) 5435 return; 5436 5437 dsq_pcpu = container_of(dru, struct scx_dsq_pcpu, 5438 deferred_reenq_user); 5439 dsq = dsq_pcpu->dsq; 5440 reenq_flags = dru->flags; 5441 WRITE_ONCE(dru->flags, 0); 5442 list_del_init(&dru->node); 5443 } 5444 5445 /* see schedule_dsq_reenq() */ 5446 smp_mb(); 5447 5448 /* destroy_dsq() may have raced and invalidated @dsq, nothing to reenq */ 5449 dsq_id = READ_ONCE(dsq->id); 5450 if (unlikely(dsq_id == SCX_DSQ_INVALID)) 5451 continue; 5452 5453 BUG_ON(dsq_id & SCX_DSQ_FLAG_BUILTIN); 5454 reenq_user(rq, dsq, reenq_flags); 5455 } 5456 } 5457 5458 /* 5459 * Drain ready tasks from @rq->scx.reject_dsq and reenqueue them so that their 5460 * owning BPF schedulers choose placement again. Proxy-active tasks remain 5461 * parked and rearm the retry notification for a later proxy resolution. 5462 * 5463 * A task can be re-rejected repeatedly. Reenqueues are bounded per task by 5464 * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning 5465 * scheduler. A bypassing scheduler's tasks are placed by the kernel instead and 5466 * are not counted. 5467 */ 5468 static void scx_reenq_reject(struct rq *rq) 5469 { 5470 LIST_HEAD(tasks); 5471 struct task_struct *p, *n; 5472 5473 lockdep_assert_rq_held(rq); 5474 5475 if (list_empty(&rq->scx.reject_dsq.list)) 5476 return; 5477 5478 /* 5479 * Move ready tasks to a private list so a task re-rejected by 5480 * scx_do_enqueue_task() below isn't revisited this round. 5481 */ 5482 list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { 5483 u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK; 5484 5485 WARN_ON_ONCE(!reason); 5486 5487 if (sched_proxy_exec() && reason == SCX_TASK_REENQ_PROXY) { 5488 /* Affinity machinery will dequeue and reactivate @p. */ 5489 if (p->migration_pending) 5490 continue; 5491 5492 if (task_on_cpu(rq, p) || task_current_donor(rq, p)) { 5493 rq->scx.flags |= SCX_RQ_PROXY_RETRY; 5494 continue; 5495 } 5496 } else { 5497 WARN_ON_ONCE(p->migration_pending); 5498 } 5499 5500 scx_wait_dispatching(p); 5501 scx_dispatch_dequeue(rq, p); 5502 5503 list_add_tail(&p->scx.dsq_list.node, &tasks); 5504 } 5505 5506 list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { 5507 list_del_init(&p->scx.dsq_list.node); 5508 5509 scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); 5510 } 5511 } 5512 5513 static void run_deferred(struct rq *rq) 5514 { 5515 process_ddsp_deferred_locals(rq); 5516 5517 if (!list_empty(&rq->scx.deferred_reenq_locals)) 5518 process_deferred_reenq_locals(rq); 5519 5520 if (!list_empty(&rq->scx.deferred_reenq_users)) 5521 process_deferred_reenq_users(rq); 5522 5523 scx_reenq_reject(rq); 5524 } 5525 5526 #ifdef CONFIG_NO_HZ_FULL 5527 bool scx_can_stop_tick(struct rq *rq) 5528 { 5529 struct task_struct *p = rq->donor; 5530 struct scx_sched *sch = scx_task_sched(p); 5531 5532 /* Proxy execution is conservatively tick-driven for now. */ 5533 if (rq->scx.flags & SCX_RQ_PROXY_TICK) 5534 return false; 5535 5536 if (p->sched_class != &ext_sched_class) 5537 return true; 5538 5539 /* 5540 * @rq->donor may still reference an outgoing EXT task after it has been 5541 * dequeued. If no EXT tasks are accounted on @rq, ignore its stale 5542 * slice state. If another task is dispatched from a DSQ, 5543 * set_next_task_scx() will update the dependency for the incoming task. 5544 */ 5545 if (!rq->scx.nr_running) 5546 return true; 5547 5548 if (scx_bypassing(sch, cpu_of(rq))) 5549 return false; 5550 5551 /* 5552 * A running rescuee's charging and expiry are tick-driven, see 5553 * scx_rescue_charge(). Keep the tick while rescue is in progress. 5554 */ 5555 if (unlikely(p == scx_rescuee(rq))) 5556 return false; 5557 5558 /* 5559 * @rq can dispatch from different DSQs, so we can't tell whether it 5560 * needs the tick or not by looking at nr_running. Allow stopping ticks 5561 * iff set_next_task_scx() determined that the selected scheduling context 5562 * can run tickless. 5563 */ 5564 return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; 5565 } 5566 #endif 5567 5568 #ifdef CONFIG_EXT_GROUP_SCHED 5569 5570 DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem); 5571 5572 void scx_tg_init(struct task_group *tg) 5573 { 5574 tg->scx.weight = CGROUP_WEIGHT_DFL; 5575 tg->scx.bw_period_us = default_bw_period_us(); 5576 tg->scx.bw_quota_us = RUNTIME_INF; 5577 tg->scx.sched_idle = false; 5578 } 5579 5580 /** 5581 * scx_tg_sched - Resolve a task_group's sched 5582 * @tg: task_group of interest 5583 * 5584 * Return the sched that @tg's ops.cgroup_init() succeeded on, %NULL if @tg 5585 * isn't inited. An autogroup tg has no cgroup of its own and resolves to the 5586 * root sched. 5587 * 5588 * When a child sched exits, its task_groups are moved to the parent and 5589 * re-inited on it. A failed re-init fails the parent in turn and leaves the 5590 * task_group without a sched it's inited on, resolving to %NULL. See 5591 * scx_cgroup_return_subtree(). 5592 * 5593 * Safe for callers read-locking the ops rwsem. tg->scx.sched rewrites 5594 * write-lock it, and tg on/offline can't overlap such callers as a css's files 5595 * are created after online and drained before offline. 5596 */ 5597 static struct scx_sched *scx_tg_sched(struct task_group *tg) 5598 { 5599 lockdep_assert(lockdep_is_held(&cgroup_mutex) || 5600 lockdep_is_held(&scx_cgroup_ops_rwsem)); 5601 5602 if (!tg->css.cgroup) 5603 tg = &root_task_group; 5604 /* INITED means ops.cgroup_init() succeeded on @tg->scx.sched */ 5605 return (tg->scx.flags & SCX_TG_INITED) ? tg->scx.sched : NULL; 5606 } 5607 5608 /** 5609 * scx_tg_knob_sched - Resolve the sched receiving a task_group's knob updates 5610 * @tg: task_group of interest 5611 * 5612 * Knobs of a cgroup belong to the parent. Deliver the set_* ops to the 5613 * parent task_group's sched, which equals @tg's own sched everywhere except 5614 * at a sub-scheduler attach point, where the sub's parent sched receives 5615 * them. 5616 * 5617 * Return %NULL if the parent task_group has no sched. That can happen when the 5618 * parent's ops.cgroup_init() fails while a sub-scheduler is being disabled. 5619 * 5620 * The callers sit in @tg's cgroup file writes holding the ops rwsem read 5621 * side. That extends scx_tg_sched()'s file-write argument to the parent's 5622 * sched read: a parent css outlives its children's files. 5623 */ 5624 static struct scx_sched *scx_tg_knob_sched(struct task_group *tg) 5625 { 5626 lockdep_assert(lockdep_is_held(&cgroup_mutex) || 5627 lockdep_is_held(&scx_cgroup_ops_rwsem)); 5628 5629 if (!tg->css.cgroup || !tg->css.parent) 5630 return scx_tg_sched(&root_task_group); 5631 return scx_tg_sched(css_tg(tg->css.parent)); 5632 } 5633 5634 int scx_tg_online(struct task_group *tg) 5635 { 5636 int ret = 0; 5637 5638 WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); 5639 5640 if (scx_cgroup_enabled) { 5641 struct scx_sched *sch; 5642 5643 /* 5644 * The cgroup lifetime notifier populates cgrp->scx_sched before 5645 * css_online, but only on the default hierarchy. Sub-scheds are 5646 * attached to the cgroup2 hierarchy, so a cgroup1 task_group 5647 * always belongs to the root sched. 5648 */ 5649 if (cgroup_on_dfl(tg->css.cgroup)) 5650 sch = scx_cgroup_sched(tg->css.cgroup); 5651 else 5652 sch = scx_tg_sched(&root_task_group); 5653 5654 if (SCX_HAS_OP(sch, cgroup_init)) { 5655 struct scx_cgroup_init_args args = 5656 { .weight = tg->scx.weight, 5657 .bw_period_us = tg->scx.bw_period_us, 5658 .bw_quota_us = tg->scx.bw_quota_us, 5659 .bw_burst_us = tg->scx.bw_burst_us, 5660 .sched_idle = tg->scx.sched_idle }; 5661 5662 ret = SCX_CALL_OP_RET(sch, cgroup_init, 5663 NULL, tg->css.cgroup, &args); 5664 if (ret) 5665 ret = scx_ops_sanitize_err(sch, "cgroup_init", ret); 5666 } 5667 if (ret == 0) { 5668 tg->scx.sched = sch; 5669 tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; 5670 } 5671 } else { 5672 tg->scx.flags |= SCX_TG_ONLINE; 5673 } 5674 5675 return ret; 5676 } 5677 5678 void scx_tg_offline(struct task_group *tg) 5679 { 5680 struct scx_sched *sch = tg->scx.sched; 5681 5682 WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); 5683 5684 /* INITED implies non-NULL @sch, test before SCX_HAS_OP() derefs */ 5685 if (scx_cgroup_enabled && (tg->scx.flags & SCX_TG_INITED) && 5686 SCX_HAS_OP(sch, cgroup_exit)) 5687 SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); 5688 tg->scx.sched = NULL; 5689 tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); 5690 } 5691 5692 /* 5693 * @p's sched for the cgroup migration paths. Stable as re-homes happen either 5694 * at CGROUP_TASK_MIGRATED of the same migration or under scx_cgroup_lock(), 5695 * both while holding cgroup_mutex. 5696 */ 5697 static struct scx_sched *scx_cgroup_task_sched(struct task_struct *p) 5698 { 5699 return rcu_dereference_protected(p->scx.sched, lockdep_is_held(&cgroup_mutex)); 5700 } 5701 5702 int scx_cgroup_can_attach(struct cgroup_taskset *tset) 5703 { 5704 struct cgroup_subsys_state *css; 5705 struct task_struct *p; 5706 int ret; 5707 5708 if (!scx_cgroup_enabled) 5709 return 0; 5710 5711 cgroup_taskset_for_each(p, css, tset) { 5712 struct scx_sched *sch = scx_cgroup_task_sched(p); 5713 struct cgroup *from = tg_cgrp(task_group(p)); 5714 struct cgroup *to = tg_cgrp(css_tg(css)); 5715 5716 WARN_ON_ONCE(p->scx.cgrp_moving_from); 5717 5718 /* 5719 * sched_move_task() omits identity migrations. Let's match the 5720 * behavior so that ops.cgroup_prep_move() and ops.cgroup_move() 5721 * always match one-to-one. 5722 */ 5723 if (from == to) 5724 continue; 5725 5726 /* 5727 * The cgroup_move ops are delivered to @p's sched, and only for 5728 * moves that don't re-home @p. A re-homing move changes the dfl 5729 * cgroup's sched and is reported through the 5730 * exit_task/init_task pair that the re-homing generates. 5731 */ 5732 if (!sch || sch != scx_cgroup_sched(task_css_set(p)->mg_dst_cset->dfl_cgrp)) 5733 continue; 5734 5735 if (SCX_HAS_OP(sch, cgroup_prep_move)) { 5736 ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, 5737 p, from, css->cgroup); 5738 if (ret) { 5739 ret = scx_ops_sanitize_err(sch, "cgroup_prep_move", ret); 5740 goto err; 5741 } 5742 } 5743 5744 p->scx.cgrp_moving_from = from; 5745 } 5746 5747 return 0; 5748 5749 err: 5750 cgroup_taskset_for_each(p, css, tset) { 5751 struct scx_sched *sch = scx_cgroup_task_sched(p); 5752 5753 /* cgrp_moving_from implies non-NULL @sch, test it first */ 5754 if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) 5755 SCX_CALL_OP(sch, cgroup_cancel_move, NULL, 5756 p, p->scx.cgrp_moving_from, css->cgroup); 5757 p->scx.cgrp_moving_from = NULL; 5758 } 5759 5760 return ret; 5761 } 5762 5763 void scx_cgroup_move_task(struct task_struct *p) 5764 { 5765 struct scx_sched *sch; 5766 5767 if (!scx_cgroup_enabled) 5768 return; 5769 5770 /* 5771 * Migration keys off css rather than cgroup identity, so it can hand an 5772 * unchanged-cgroup task here with cgrp_moving_from NULL. Nothing to 5773 * report to the BPF scheduler then, so skip it and keep prep_move and 5774 * move paired. 5775 */ 5776 sch = scx_cgroup_task_sched(p); 5777 if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_move)) 5778 SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), 5779 p, p->scx.cgrp_moving_from, 5780 tg_cgrp(task_group(p))); 5781 p->scx.cgrp_moving_from = NULL; 5782 } 5783 5784 void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) 5785 { 5786 struct cgroup_subsys_state *css; 5787 struct task_struct *p; 5788 5789 if (!scx_cgroup_enabled) 5790 return; 5791 5792 cgroup_taskset_for_each(p, css, tset) { 5793 struct scx_sched *sch = scx_cgroup_task_sched(p); 5794 5795 /* cgrp_moving_from implies non-NULL @sch, test it first */ 5796 if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) 5797 SCX_CALL_OP(sch, cgroup_cancel_move, NULL, 5798 p, p->scx.cgrp_moving_from, css->cgroup); 5799 p->scx.cgrp_moving_from = NULL; 5800 } 5801 } 5802 5803 void scx_group_set_weight(struct task_group *tg, unsigned long weight) 5804 { 5805 struct scx_sched *sch; 5806 5807 percpu_down_read(&scx_cgroup_ops_rwsem); 5808 sch = scx_tg_knob_sched(tg); 5809 5810 if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_weight) && 5811 tg->scx.weight != weight) 5812 SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); 5813 5814 tg->scx.weight = weight; 5815 5816 percpu_up_read(&scx_cgroup_ops_rwsem); 5817 } 5818 5819 void scx_group_set_idle(struct task_group *tg, bool idle) 5820 { 5821 struct scx_sched *sch; 5822 5823 percpu_down_read(&scx_cgroup_ops_rwsem); 5824 sch = scx_tg_knob_sched(tg); 5825 5826 if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) && 5827 tg->scx.sched_idle != idle) 5828 SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); 5829 5830 /* Update the task group's idle state */ 5831 tg->scx.sched_idle = idle; 5832 5833 percpu_up_read(&scx_cgroup_ops_rwsem); 5834 } 5835 5836 void scx_group_set_bandwidth(struct task_group *tg, 5837 u64 period_us, u64 quota_us, u64 burst_us) 5838 { 5839 struct scx_sched *sch; 5840 5841 percpu_down_read(&scx_cgroup_ops_rwsem); 5842 sch = scx_tg_knob_sched(tg); 5843 5844 if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_bandwidth) && 5845 (tg->scx.bw_period_us != period_us || 5846 tg->scx.bw_quota_us != quota_us || 5847 tg->scx.bw_burst_us != burst_us)) 5848 SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL, 5849 tg_cgrp(tg), period_us, quota_us, burst_us); 5850 5851 tg->scx.bw_period_us = period_us; 5852 tg->scx.bw_quota_us = quota_us; 5853 tg->scx.bw_burst_us = burst_us; 5854 5855 percpu_up_read(&scx_cgroup_ops_rwsem); 5856 } 5857 #endif /* CONFIG_EXT_GROUP_SCHED */ 5858 5859 #if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED) 5860 static struct cgroup *root_cgroup(void) 5861 { 5862 return &cgrp_dfl_root.cgrp; 5863 } 5864 5865 /* 5866 * cgroup_lock() must nest outside the rwsem write side: a writer waiting 5867 * for cgroup_mutex deadlocks with cgroup teardown, which holds it while 5868 * draining a set_* file write blocked on the rwsem behind the writer. 5869 */ 5870 void scx_cgroup_lock(void) 5871 { 5872 cgroup_lock(); 5873 #ifdef CONFIG_EXT_GROUP_SCHED 5874 percpu_down_write(&scx_cgroup_ops_rwsem); 5875 #endif 5876 } 5877 5878 void scx_cgroup_unlock(void) 5879 { 5880 #ifdef CONFIG_EXT_GROUP_SCHED 5881 percpu_up_write(&scx_cgroup_ops_rwsem); 5882 #endif 5883 cgroup_unlock(); 5884 } 5885 #else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ 5886 static inline struct cgroup *root_cgroup(void) { return NULL; } 5887 static inline void scx_cgroup_lock(void) {} 5888 static inline void scx_cgroup_unlock(void) {} 5889 #endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ 5890 5891 /* 5892 * Omitted operations: 5893 * 5894 * - migrate_task_rq: Unnecessary as task to cpu mapping is transient. 5895 * 5896 * - task_fork/dead: We need fork/dead notifications for all tasks regardless of 5897 * their current sched_class. Call them directly from sched core instead. 5898 */ 5899 DEFINE_SCHED_CLASS(ext) = { 5900 .enqueue_task = enqueue_task_scx, 5901 .dequeue_task = dequeue_task_scx, 5902 .yield_task = yield_task_scx, 5903 .yield_to_task = yield_to_task_scx, 5904 5905 .wakeup_preempt = wakeup_preempt_scx, 5906 5907 .pick_task = pick_task_scx, 5908 5909 .put_prev_task = put_prev_task_scx, 5910 .set_next_task = set_next_task_scx, 5911 5912 .select_task_rq = select_task_rq_scx, 5913 .task_woken = task_woken_scx, 5914 .set_cpus_allowed = set_cpus_allowed_scx, 5915 5916 .task_tick = task_tick_scx, 5917 5918 .switching_to = switching_to_scx, 5919 .switched_from = switched_from_scx, 5920 .switched_to = switched_to_scx, 5921 .reweight_task = reweight_task_scx, 5922 .prio_changed = prio_changed_scx, 5923 5924 .update_curr = update_curr_scx, 5925 5926 #ifdef CONFIG_UCLAMP_TASK 5927 .uclamp_enabled = 1, 5928 #endif 5929 }; 5930 5931 s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch) 5932 { 5933 s32 cpu; 5934 5935 memset(dsq, 0, sizeof(*dsq)); 5936 5937 raw_spin_lock_init(&dsq->lock); 5938 INIT_LIST_HEAD(&dsq->list); 5939 dsq->id = dsq_id; 5940 dsq->sched = sch; 5941 5942 /* per-DSQ deferred reenq state is only needed for user DSQs */ 5943 if (dsq_id & SCX_DSQ_FLAG_BUILTIN) 5944 return 0; 5945 5946 dsq->pcpu_user = alloc_percpu(struct scx_dsq_pcpu); 5947 if (!dsq->pcpu_user) 5948 return -ENOMEM; 5949 5950 for_each_possible_cpu(cpu) { 5951 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); 5952 5953 pcpu->dsq = dsq; 5954 INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); 5955 } 5956 5957 return 0; 5958 } 5959 5960 static void exit_dsq(struct scx_dispatch_q *dsq) 5961 { 5962 s32 cpu; 5963 5964 if (!dsq->pcpu_user) 5965 return; 5966 5967 for_each_possible_cpu(cpu) { 5968 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); 5969 struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; 5970 struct rq *rq = cpu_rq(cpu); 5971 5972 /* 5973 * free_dsq_irq_workfn() cancelled the reenqs before the grace 5974 * period and schedule_dsq_reenq() queues nothing after that. 5975 */ 5976 if (WARN_ON_ONCE(!list_empty(&dru->node))) { 5977 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); 5978 list_del_init(&dru->node); 5979 } 5980 } 5981 5982 free_percpu(dsq->pcpu_user); 5983 } 5984 5985 static void free_dsq_rcufn(struct rcu_head *rcu) 5986 { 5987 struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu); 5988 5989 exit_dsq(dsq); 5990 kfree(dsq); 5991 } 5992 5993 static void free_dsq_irq_workfn(struct irq_work *irq_work) 5994 { 5995 struct llist_node *to_free = llist_del_all(&dsqs_to_free); 5996 struct scx_dispatch_q *dsq, *tmp_dsq; 5997 5998 llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) { 5999 s32 cpu; 6000 6001 /* 6002 * Cancel pending reenqs. Only a run_deferred() that started 6003 * before the grace period can hold one, and it keeps IRQs off, 6004 * so the grace period waits for it. After this sweep, 6005 * schedule_dsq_reenq() sees the invalid id under the same lock 6006 * and queues nothing. 6007 */ 6008 for_each_possible_cpu(cpu) { 6009 struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); 6010 struct rq *rq = cpu_rq(cpu); 6011 6012 guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); 6013 list_del_init(&pcpu->deferred_reenq_user.node); 6014 } 6015 6016 call_rcu(&dsq->rcu, free_dsq_rcufn); 6017 } 6018 } 6019 6020 static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); 6021 6022 static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) 6023 { 6024 struct scx_dispatch_q *dsq; 6025 unsigned long flags; 6026 6027 rcu_read_lock(); 6028 6029 dsq = find_user_dsq(sch, dsq_id); 6030 if (!dsq) 6031 goto out_unlock_rcu; 6032 6033 raw_spin_lock_irqsave(&dsq->lock, flags); 6034 6035 if (dsq->nr) { 6036 scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)", 6037 dsq->id, dsq->nr); 6038 goto out_unlock_dsq; 6039 } 6040 6041 if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node, 6042 dsq_hash_params)) 6043 goto out_unlock_dsq; 6044 6045 /* 6046 * Mark dead by invalidating ->id to prevent scx_dispatch_enqueue() from 6047 * queueing more tasks. As this function can be called from anywhere, 6048 * freeing is bounced through an irq work to avoid nesting RCU 6049 * operations inside scheduler locks. 6050 */ 6051 dsq->id = SCX_DSQ_INVALID; 6052 if (llist_add(&dsq->free_node, &dsqs_to_free)) 6053 irq_work_queue(&free_dsq_irq_work); 6054 6055 out_unlock_dsq: 6056 raw_spin_unlock_irqrestore(&dsq->lock, flags); 6057 out_unlock_rcu: 6058 rcu_read_unlock(); 6059 } 6060 6061 #ifdef CONFIG_EXT_GROUP_SCHED 6062 static void scx_cgroup_exit(struct scx_sched *sch) 6063 { 6064 struct cgroup_subsys_state *css; 6065 6066 /* 6067 * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk 6068 * cgroups and exit all the inited ones, all online cgroups are exited. 6069 */ 6070 css_for_each_descendant_post(css, &root_task_group.css) { 6071 struct task_group *tg = css_tg(css); 6072 6073 /* also clear the sched of tgs whose ops.cgroup_init() failed */ 6074 tg->scx.sched = NULL; 6075 if (tg->scx.flags & SCX_TG_INITED) { 6076 tg->scx.flags &= ~SCX_TG_INITED; 6077 if (sch->ops.cgroup_exit) 6078 SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); 6079 } 6080 } 6081 } 6082 6083 static int scx_cgroup_init(struct scx_sched *sch) 6084 { 6085 struct cgroup_subsys_state *css; 6086 int ret; 6087 6088 /* 6089 * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk 6090 * cgroups and init, all online cgroups are initialized. 6091 */ 6092 css_for_each_descendant_pre(css, &root_task_group.css) { 6093 struct task_group *tg = css_tg(css); 6094 6095 if ((tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) 6096 continue; 6097 6098 if (sch->ops.cgroup_init) { 6099 struct scx_cgroup_init_args args = { 6100 .weight = tg->scx.weight, 6101 .bw_period_us = tg->scx.bw_period_us, 6102 .bw_quota_us = tg->scx.bw_quota_us, 6103 .bw_burst_us = tg->scx.bw_burst_us, 6104 .sched_idle = tg->scx.sched_idle, 6105 }; 6106 6107 ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); 6108 if (ret) { 6109 scx_error(sch, "ops.cgroup_init() failed (%d)", ret); 6110 return ret; 6111 } 6112 } 6113 6114 tg->scx.sched = sch; 6115 tg->scx.flags |= SCX_TG_INITED; 6116 } 6117 6118 return 0; 6119 } 6120 6121 #else 6122 static void scx_cgroup_exit(struct scx_sched *sch) {} 6123 static int scx_cgroup_init(struct scx_sched *sch) { return 0; } 6124 #endif 6125 6126 6127 /******************************************************************************** 6128 * Sysfs interface and ops enable/disable. 6129 */ 6130 6131 #define SCX_ATTR(_name) \ 6132 static struct kobj_attribute scx_attr_##_name = { \ 6133 .attr = { .name = __stringify(_name), .mode = 0444 }, \ 6134 .show = scx_attr_##_name##_show, \ 6135 } 6136 6137 static ssize_t scx_attr_state_show(struct kobject *kobj, 6138 struct kobj_attribute *ka, char *buf) 6139 { 6140 return sysfs_emit(buf, "%s\n", scx_enable_state_str[scx_enable_state()]); 6141 } 6142 SCX_ATTR(state); 6143 6144 static ssize_t scx_attr_switch_all_show(struct kobject *kobj, 6145 struct kobj_attribute *ka, char *buf) 6146 { 6147 return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all)); 6148 } 6149 SCX_ATTR(switch_all); 6150 6151 static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj, 6152 struct kobj_attribute *ka, char *buf) 6153 { 6154 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected)); 6155 } 6156 SCX_ATTR(nr_rejected); 6157 6158 static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj, 6159 struct kobj_attribute *ka, char *buf) 6160 { 6161 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq)); 6162 } 6163 SCX_ATTR(hotplug_seq); 6164 6165 static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, 6166 struct kobj_attribute *ka, char *buf) 6167 { 6168 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq)); 6169 } 6170 SCX_ATTR(enable_seq); 6171 6172 static struct attribute *scx_global_attrs[] = { 6173 &scx_attr_state.attr, 6174 &scx_attr_switch_all.attr, 6175 &scx_attr_nr_rejected.attr, 6176 &scx_attr_hotplug_seq.attr, 6177 &scx_attr_enable_seq.attr, 6178 NULL, 6179 }; 6180 6181 static const struct attribute_group scx_global_attr_group = { 6182 .attrs = scx_global_attrs, 6183 }; 6184 6185 static void free_pnode(struct scx_sched_pnode *pnode); 6186 static void free_exit_info(struct scx_exit_info *ei); 6187 static const char *scx_exit_reason(enum scx_exit_kind kind); 6188 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind); 6189 6190 s32 scx_alloc_kern_arena_objs(struct scx_sched *sch) 6191 { 6192 size_t size = struct_size_t(struct scx_cmask, bits, 6193 SCX_CMASK_NR_WORDS(num_possible_cpus())); 6194 struct scx_cmask *online; 6195 struct scx_cmask_ref ref; 6196 int cpu; 6197 6198 /* hotplug stays excluded until the online mask is published */ 6199 lockdep_assert_cpus_held(); 6200 6201 if (!sch->is_cid_type || !sch->arena_pool) 6202 return 0; 6203 6204 sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *); 6205 if (!sch->set_cmask_scratch) 6206 return -ENOMEM; 6207 6208 for_each_possible_cpu(cpu) { 6209 struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); 6210 6211 *slot = scx_arena_alloc(sch, size); 6212 if (!*slot) 6213 return -ENOMEM; 6214 scx_cmask_init(*slot, 0, num_possible_cpus()); 6215 } 6216 6217 /* pack the online mask alongside the scratch masks */ 6218 online = scx_arena_alloc(sch, size); 6219 if (!online) 6220 return -ENOMEM; 6221 6222 scoped_guard(rcu) { 6223 scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref); 6224 scx_cmask_ref_from_cpumask(&ref, cpu_active_mask); 6225 } 6226 sch->online_cmask = online; 6227 6228 return 0; 6229 } 6230 6231 static void scx_free_kern_arena_objs(struct scx_sched *sch) 6232 { 6233 size_t size = struct_size_t(struct scx_cmask, bits, 6234 SCX_CMASK_NR_WORDS(num_possible_cpus())); 6235 int cpu; 6236 6237 scx_arena_free(sch, sch->online_cmask, size); 6238 if (!sch->set_cmask_scratch) 6239 return; 6240 6241 for_each_possible_cpu(cpu) { 6242 struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); 6243 6244 scx_arena_free(sch, *slot, size); 6245 } 6246 free_percpu(sch->set_cmask_scratch); 6247 sch->set_cmask_scratch = NULL; 6248 } 6249 6250 static void scx_sched_free_rcu_work(struct work_struct *work) 6251 { 6252 struct rcu_work *rcu_work = to_rcu_work(work); 6253 struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); 6254 struct rhashtable_iter rht_iter; 6255 struct scx_dispatch_q *dsq; 6256 int cpu, node; 6257 6258 free_cpumask_var(sch->bypass_lb_donee_cpumask); 6259 free_cpumask_var(sch->bypass_lb_resched_cpumask); 6260 free_cpumask_var(sch->stall_cpus); 6261 6262 #ifdef CONFIG_EXT_SUB_SCHED 6263 kfree(sch->cgrp_path); 6264 if (sch_cgroup(sch)) 6265 cgroup_put(sch_cgroup(sch)); 6266 if (sch->sub_kset) 6267 kobject_put(&sch->sub_kset->kobj); 6268 if (scx_parent(sch)) 6269 kobject_put(&scx_parent(sch)->kobj); 6270 #endif /* CONFIG_EXT_SUB_SCHED */ 6271 6272 for_each_possible_cpu(cpu) { 6273 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); 6274 6275 free_cpumask_var(pcpu->cpus_to_kick); 6276 free_cpumask_var(pcpu->cpus_to_kick_if_idle); 6277 free_cpumask_var(pcpu->cpus_to_preempt); 6278 free_cpumask_var(pcpu->cpus_to_preempt_lazy); 6279 free_cpumask_var(pcpu->cpus_to_wait); 6280 6281 exit_dsq(scx_bypass_dsq(sch, cpu)); 6282 } 6283 6284 free_percpu(sch->pcpu); 6285 6286 for_each_node_state(node, N_POSSIBLE) 6287 free_pnode(sch->pnode[node]); 6288 kfree(sch->pnode); 6289 6290 scx_free_pshards(sch); 6291 6292 rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); 6293 do { 6294 rhashtable_walk_start(&rht_iter); 6295 6296 while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter)))) 6297 destroy_dsq(sch, dsq->id); 6298 6299 rhashtable_walk_stop(&rht_iter); 6300 } while (dsq == ERR_PTR(-EAGAIN)); 6301 rhashtable_walk_exit(&rht_iter); 6302 6303 rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); 6304 free_exit_info(sch->exit_info); 6305 scx_free_kern_arena_objs(sch); 6306 scx_arena_pool_destroy(sch); 6307 if (sch->arena_map) 6308 bpf_map_put(sch->arena_map); 6309 6310 /* @sch is completely inactive by now */ 6311 scx_dec_has_subs(sch); 6312 6313 kfree(sch); 6314 } 6315 6316 /** 6317 * scx_sched_quiesce_rcu_work - Quiesce a sched and queue its free 6318 * @work: &scx_sched.rcu_work 6319 * 6320 * $sch is freed after two RCU grace periods because of its ecaps sync nodes. 6321 * 6322 * The first grace period runs between scx_kobj_release() and this function. It 6323 * waits for the paths that found $sch before scx_unlink_sched(), such as the 6324 * sub-cap kfuncs and the cpu hotplug hooks. Until they finish, they can queue 6325 * $sch's syncs, and a cpu can take a queued node into a sync batch. The first 6326 * grace period doesn't wait for a batch that started after it began. 6327 * 6328 * Nothing queues $sch's syncs now. Remove the nodes still on the cpus' lists. A 6329 * batch that took a node can still write $sch's pcpu, and a delivery in the 6330 * batch can drop the rq lock before that write. The batch keeps irqs off from 6331 * taking the node to its last pcpu write, so the second grace period waits for 6332 * it before scx_sched_free_rcu_work() frees $sch. 6333 */ 6334 static void scx_sched_quiesce_rcu_work(struct work_struct *work) 6335 { 6336 struct rcu_work *rcu_work = to_rcu_work(work); 6337 struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); 6338 s32 cpu; 6339 6340 irq_work_sync(&sch->propagate_exit_irq_work); 6341 irq_work_sync(&sch->disable_irq_work); 6342 kthread_destroy_worker(sch->helper); 6343 timer_shutdown_sync(&sch->bypass_lb_timer); 6344 6345 for_each_possible_cpu(cpu) { 6346 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); 6347 6348 /* 6349 * $sch would have entered bypass mode before the first RCU 6350 * grace period. As that blocks new deferrals, all 6351 * deferred_reenq_local_node's must be off-list by now. 6352 */ 6353 WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); 6354 6355 /* remove the queued ecaps sync so the pcpu can be freed */ 6356 scx_discard_ecaps_to_sync(cpu, pcpu); 6357 6358 /* 6359 * Bypass blocks new kicks. Flush the kick irq_work so this 6360 * pcpu's to_kick_node is off the list before it is freed. 6361 */ 6362 irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); 6363 WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node)); 6364 } 6365 6366 INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); 6367 queue_rcu_work(system_dfl_wq, &sch->rcu_work); 6368 } 6369 6370 static void scx_kobj_release(struct kobject *kobj) 6371 { 6372 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); 6373 6374 INIT_RCU_WORK(&sch->rcu_work, scx_sched_quiesce_rcu_work); 6375 queue_rcu_work(system_dfl_wq, &sch->rcu_work); 6376 } 6377 6378 static ssize_t scx_attr_ops_show(struct kobject *kobj, 6379 struct kobj_attribute *ka, char *buf) 6380 { 6381 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); 6382 6383 return sysfs_emit(buf, "%s\n", sch->ops.name); 6384 } 6385 SCX_ATTR(ops); 6386 6387 #define scx_attr_event_show(buf, at, events, kind) ({ \ 6388 sysfs_emit_at(buf, at, "%s %llu\n", #kind, (events)->kind); \ 6389 }) 6390 6391 static ssize_t scx_attr_events_show(struct kobject *kobj, 6392 struct kobj_attribute *ka, char *buf) 6393 { 6394 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); 6395 struct scx_event_stats events; 6396 int at = 0; 6397 6398 scx_read_events(sch, &events); 6399 #define SCX_EVENT(name) (at += scx_attr_event_show(buf, at, &events, name)) 6400 SCX_EVENTS_LIST(SCX_EVENT); 6401 #undef SCX_EVENT 6402 return at; 6403 } 6404 SCX_ATTR(events); 6405 6406 #ifdef CONFIG_EXT_SUB_SCHED 6407 static const char *scx_cap_names[__SCX_NR_CAPS] = { 6408 [__SCX_CAP_ENQ_IMMED] = "enq_immed", 6409 [__SCX_CAP_ENQ] = "enq", 6410 [__SCX_CAP_PREEMPT] = "preempt", 6411 [__SCX_CAP_PERF] = "perf", 6412 }; 6413 6414 static ssize_t scx_attr_caps_show(struct kobject *kobj, 6415 struct kobj_attribute *ka, char *buf) 6416 { 6417 struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); 6418 u32 npossible = num_possible_cpus(); 6419 struct scx_cmask *agg __free(kfree) = 6420 kzalloc_flex(*agg, bits, SCX_CMASK_NR_WORDS(npossible)); 6421 unsigned long *agg_bm __free(bitmap) = bitmap_zalloc(npossible, GFP_KERNEL); 6422 ssize_t count = 0; 6423 s32 cap, si; 6424 6425 if (!agg || !agg_bm) 6426 return -ENOMEM; 6427 6428 for (cap = 0; cap < __SCX_NR_CAPS; cap++) { 6429 SCX_CMASK_DEFINE(snap, 0, SCX_CID_SHARD_MAX_CPUS); 6430 6431 scx_cmask_init(agg, 0, npossible); 6432 for (si = 0; si < sch->nr_pshards; si++) { 6433 struct scx_cmask *cm = &sch->pshard[si]->caps[cap].cmask; 6434 6435 scx_cmask_reframe(snap, cm->base, cm->nr_cids); 6436 scx_cmask_copy(snap, cm); 6437 scx_cmask_or(agg, snap); 6438 } 6439 /* %*pbl takes unsigned long bitmap layout, convert from u64 */ 6440 bitmap_from_arr64(agg_bm, agg->bits, npossible); 6441 count += sysfs_emit_at(buf, count, "%s: %*pbl\n", 6442 scx_cap_names[cap], npossible, agg_bm); 6443 } 6444 return count; 6445 } 6446 SCX_ATTR(caps); 6447 #endif /* CONFIG_EXT_SUB_SCHED */ 6448 6449 static struct attribute *scx_sched_attrs[] = { 6450 &scx_attr_ops.attr, 6451 &scx_attr_events.attr, 6452 #ifdef CONFIG_EXT_SUB_SCHED 6453 &scx_attr_caps.attr, 6454 #endif 6455 NULL, 6456 }; 6457 ATTRIBUTE_GROUPS(scx_sched); 6458 6459 static const struct kobj_type scx_ktype = { 6460 .release = scx_kobj_release, 6461 .sysfs_ops = &kobj_sysfs_ops, 6462 .default_groups = scx_sched_groups, 6463 }; 6464 6465 static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) 6466 { 6467 const struct scx_sched *sch; 6468 6469 /* 6470 * scx_uevent() can be reached by both scx_sched kobjects (scx_ktype) 6471 * and sub-scheduler kset kobjects (kset_ktype) through the parent 6472 * chain walk. Filter out the latter to avoid invalid casts. 6473 */ 6474 if (kobj->ktype != &scx_ktype) 6475 return 0; 6476 6477 sch = container_of(kobj, struct scx_sched, kobj); 6478 6479 return add_uevent_var(env, "SCXOPS=%s", sch->ops.name); 6480 } 6481 6482 static const struct kset_uevent_ops scx_uevent_ops = { 6483 .uevent = scx_uevent, 6484 }; 6485 6486 /* 6487 * Used by sched_fork() and __setscheduler_class() to pick the matching 6488 * sched_class. dl/rt are already handled. 6489 */ 6490 bool task_should_scx(int policy) 6491 { 6492 /* if disabled, nothing should be on it */ 6493 if (!scx_enabled()) 6494 return false; 6495 6496 /* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */ 6497 if (READ_ONCE(scx_switching_all)) 6498 return true; 6499 6500 /* 6501 * scx is tearing down - keep new SCHED_EXT tasks out. 6502 * 6503 * Must come after scx_switching_all test, which serves as a proxy 6504 * for __scx_switched_all. While __scx_switched_all is set, we must 6505 * return true via the branch above: a fork routed to fair would 6506 * stall because next_active_class() skips fair. 6507 * 6508 * This can develop into a deadlock - scx holds scx_enable_mutex across 6509 * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is 6510 * the stalled task, the disable path can never grab the mutex to clear 6511 * scx_switching_all. 6512 */ 6513 if (unlikely(scx_enable_state() == SCX_DISABLING)) 6514 return false; 6515 6516 return policy == SCHED_EXT; 6517 } 6518 6519 bool scx_allow_ttwu_queue(const struct task_struct *p) 6520 { 6521 struct scx_sched *sch; 6522 6523 if (!scx_enabled()) 6524 return true; 6525 6526 sch = scx_task_sched(p); 6527 if (unlikely(!sch)) 6528 return true; 6529 6530 if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP) 6531 return true; 6532 6533 if (unlikely(p->sched_class != &ext_sched_class)) 6534 return true; 6535 6536 return false; 6537 } 6538 6539 /** 6540 * handle_lockup - sched_ext common lockup handler 6541 * @exit_cpu: CPU to record in exit_info. Pass the stalled/hung CPU, not current. 6542 * @fmt: format string 6543 * 6544 * Called on system stall or lockup condition and initiates abort of sched_ext 6545 * if enabled, which may resolve the reported lockup. 6546 * 6547 * Returns %true if sched_ext is enabled and abort was initiated, which may 6548 * resolve the lockup. %false if sched_ext is not enabled or abort was already 6549 * initiated by someone else. 6550 */ 6551 static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...) 6552 { 6553 struct scx_sched *sch; 6554 va_list args; 6555 bool ret; 6556 6557 guard(rcu)(); 6558 6559 sch = rcu_dereference(scx_root); 6560 if (unlikely(!sch)) 6561 return false; 6562 6563 switch (scx_enable_state()) { 6564 case SCX_ENABLING: 6565 case SCX_ENABLED: 6566 va_start(args, fmt); 6567 ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args); 6568 va_end(args); 6569 return ret; 6570 default: 6571 return false; 6572 } 6573 } 6574 6575 /** 6576 * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler 6577 * @stalled_mask: bit mask of stalled CPUs 6578 * 6579 * While there are various reasons why RCU CPU stalls can occur on a system 6580 * that may not be caused by the current BPF scheduler, try kicking out the 6581 * current scheduler in an attempt to recover the system to a good state before 6582 * issuing panics. 6583 * 6584 * Returns %true if sched_ext is enabled and abort was initiated, which may 6585 * resolve the reported RCU stall. %false if sched_ext is not enabled or someone 6586 * else already initiated abort. 6587 */ 6588 bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) 6589 { 6590 struct scx_sched *sch; 6591 struct scx_exit_info *ei; 6592 int exit_cpu; 6593 6594 guard(rcu)(); 6595 6596 sch = rcu_dereference(scx_root); 6597 if (unlikely(!sch)) 6598 return false; 6599 6600 switch (scx_enable_state()) { 6601 case SCX_ENABLING: 6602 case SCX_ENABLED: 6603 break; 6604 default: 6605 return false; 6606 } 6607 6608 exit_cpu = cpumask_empty(stalled_mask) ? -1 : (int)cpumask_first(stalled_mask); 6609 ei = sch->exit_info; 6610 6611 guard(preempt)(); 6612 6613 if (!scx_claim_exit(sch, SCX_EXIT_ERROR)) 6614 return false; 6615 6616 #ifdef CONFIG_STACKTRACE 6617 ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); 6618 #endif 6619 scnprintf(ei->msg, SCX_EXIT_MSG_LEN, "RCU CPU stall on CPUs (%*pbl)", 6620 cpumask_pr_args(stalled_mask)); 6621 ei->kind = SCX_EXIT_ERROR; 6622 ei->reason = scx_exit_reason(SCX_EXIT_ERROR); 6623 ei->exit_cpu = exit_cpu; 6624 cpumask_copy(sch->stall_cpus, stalled_mask); 6625 6626 irq_work_queue(&sch->disable_irq_work); 6627 return true; 6628 } 6629 6630 /** 6631 * scx_softlockup - sched_ext softlockup handler 6632 * @dur_s: number of seconds of CPU stuck due to soft lockup 6633 * 6634 * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can 6635 * live-lock the system by making many CPUs target the same DSQ to the point 6636 * where soft-lockup detection triggers. This function is called from 6637 * soft-lockup watchdog when the triggering point is close and tries to unjam 6638 * the system and aborting the BPF scheduler. 6639 */ 6640 void scx_softlockup(u32 dur_s) 6641 { 6642 int cpu = smp_processor_id(); 6643 6644 if (!handle_lockup(cpu, "soft lockup - CPU %d stuck for %us", cpu, dur_s)) 6645 return; 6646 6647 printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", 6648 cpu, dur_s); 6649 } 6650 6651 /** 6652 * scx_hardlockup - sched_ext hardlockup handler 6653 * @cpu: the target CPU 6654 * 6655 * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting 6656 * numerous affinitized tasks in a single queue and directing all CPUs at it. 6657 * Try kicking out the current scheduler in an attempt to recover the system to 6658 * a good state before taking more drastic actions. 6659 * 6660 * Called from NMI. Aborting the scheduler sets ->aborting throughout the 6661 * hierarchy before returning, which is what breaks the dispatch-path live-locks 6662 * that can hard-lock CPUs. 6663 * 6664 * Returns %true if sched_ext is enabled and abort was initiated, which may 6665 * resolve the lockup. %false if sched_ext is not enabled or abort was already 6666 * initiated by someone else. 6667 */ 6668 bool scx_hardlockup(int cpu) 6669 { 6670 if (!handle_lockup(cpu, "hard lockup - CPU %d", cpu)) 6671 return false; 6672 6673 printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", 6674 cpu); 6675 return true; 6676 } 6677 6678 /** 6679 * bypass_lb_reenq_home - Reenqueue a task at its home from the balancer 6680 * @rq: @p's home rq, locked 6681 * @p: task off any DSQ, taken from the bypass DSQ of @dsq_cpu 6682 * @dsq_cpu: cpu whose bypass DSQ gave up @p 6683 * 6684 * @p came off the bypass DSQ of @dsq_cpu with no donee able to take it. A drain 6685 * sends a migration-disabled @p here unless the donee picked for it is its 6686 * home, as every other cpu refuses it. The enqueue sends @p to rescue when the 6687 * host lacks %SCX_CAP_BASE at home and rescue is enabled, and to the home's 6688 * bypass DSQ otherwise. When @p must run at home, migration-disabled or with a 6689 * migration pending, and the host lacks BASE there, it is admitted to the 6690 * home's local DSQ in either case, see bypass_enq_target_dsq(). No DSQ lock may 6691 * be held, as the enqueue takes one. 6692 */ 6693 static void bypass_lb_reenq_home(struct rq *rq, struct task_struct *p, s32 dsq_cpu) 6694 { 6695 lockdep_assert_rq_held(rq); 6696 6697 WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); 6698 p->scx.flags |= SCX_TASK_REENQ_CAP; 6699 p->scx.reenq_reason_caps = SCX_CAP_BASE; 6700 p->scx.reenq_reason_cid = __scx_cpu_to_cid(dsq_cpu); 6701 scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); 6702 } 6703 6704 /** 6705 * bypass_lb_can_move - Test whether @p can move to the bypass DSQ of @donee 6706 * @sch: host sched whose bypass DSQs are balanced 6707 * @p: task on a bypass DSQ of @sch, homed anywhere 6708 * @donee: cpu whose bypass DSQ would receive @p, -1 for none 6709 * 6710 * @p's rq lock pins @p's affinity. Without it the answer is a hint. A task 6711 * whose donee is its own cpu can always run there, and 6712 * task_can_run_on_remote_rq() warns when asked about a task's own cpu. 6713 */ 6714 static bool bypass_lb_can_move(struct scx_sched *sch, struct task_struct *p, s32 donee) 6715 { 6716 return donee >= 0 && (donee == task_cpu(p) || 6717 task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)); 6718 } 6719 6720 /** 6721 * bypass_lb_move - Move one task off a bypass DSQ for the balancer 6722 * @sch: host sched whose bypass DSQs are balanced 6723 * @donor_rq: rq of the cpu whose bypass DSQ gives up @p, locked 6724 * @donor_dsq: @sch's bypass DSQ on @donor_rq's cpu, locked 6725 * @p: task on @donor_dsq, homed anywhere 6726 * @donee: cpu whose bypass DSQ receives @p, -1 for none 6727 * 6728 * Take @p off @donor_dsq and insert it into the bypass DSQ of @donee. The 6729 * caller tests bypass_lb_can_move() without @p's rq lock for a task homed 6730 * elsewhere, and not at all for a drain, so the test is repeated here under 6731 * @p's rq lock. A task that fails it is not put back, as the host may no longer 6732 * consume @donor_dsq. bypass_lb_reenq_home() reenqueues it at its home instead. 6733 * 6734 * The insert and the reenqueue run under @p's rq lock. A task homed on 6735 * @donor_rq is dequeued in place. unlink_dsq_and_switch_rq_lock() takes one 6736 * homed elsewhere off and switches the rq lock from @donor_rq to its own. The 6737 * switch can lose to a dequeue, and @p then belongs to the dequeuer. 6738 * 6739 * @donor_rq and @donor_dsq are locked on entry and on return. @donor_dsq is 6740 * unlocked in between, and @donor_rq too while the lock is switched, so the 6741 * caller's walk must find its place in @donor_dsq again after the call. 6742 * 6743 * Return the cpu that received @p, or -1 if @p was lost to a dequeue. 6744 */ 6745 static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq, 6746 struct scx_dispatch_q *donor_dsq, struct task_struct *p, s32 donee) 6747 { 6748 struct rq *rq = task_rq(p); 6749 s32 ret; 6750 6751 lockdep_assert_rq_held(donor_rq); 6752 lockdep_assert_held(&donor_dsq->lock); 6753 6754 if (rq == donor_rq) { 6755 scx_wait_dispatching(p); 6756 dispatch_dequeue_locked(p, donor_dsq); 6757 raw_spin_unlock(&donor_dsq->lock); 6758 } else if (!unlink_dsq_and_switch_rq_lock(p, donor_dsq, donor_rq, rq)) { 6759 /* lost to a dequeue, @p is the dequeuer's now */ 6760 ret = -1; 6761 goto out; 6762 } 6763 6764 if (bypass_lb_can_move(sch, p, donee)) { 6765 scx_dispatch_enqueue(sch, rq, scx_bypass_dsq(sch, donee), p, 0, 0, 0); 6766 ret = donee; 6767 } else { 6768 /* 6769 * No donee, or @p can't run on it. Reenqueue @p at home 6770 * instead, see bypass_lb_reenq_home(). 6771 */ 6772 bypass_lb_reenq_home(rq, p, cpu_of(donor_rq)); 6773 ret = cpu_of(rq); 6774 } 6775 out: 6776 if (rq != donor_rq) 6777 switch_rq_lock(rq, donor_rq); 6778 raw_spin_lock(&donor_dsq->lock); 6779 return ret; 6780 } 6781 6782 static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, 6783 struct cpumask *donee_mask, struct cpumask *resched_mask, 6784 u32 nr_donor_target, u32 nr_donee_target) 6785 { 6786 struct rq *donor_rq = cpu_rq(donor); 6787 struct scx_dispatch_q *donor_dsq = scx_bypass_dsq(sch, donor); 6788 struct task_struct *p, *n; 6789 struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0); 6790 s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; 6791 u32 nr_balanced = 0, min_delta_us; 6792 6793 /* 6794 * All we want to guarantee is reasonable forward progress. No reason to 6795 * fine tune. Assuming every task on @donor_dsq runs their full slice, 6796 * consider offloading iff the total queued duration is over the 6797 * threshold. A zero target is a drain and empties the DSQ regardless. 6798 */ 6799 min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV; 6800 if (nr_donor_target && 6801 delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) 6802 return 0; 6803 6804 raw_spin_rq_lock_irq(donor_rq); 6805 raw_spin_lock(&donor_dsq->lock); 6806 /* bound the walk to the tasks queued now, see the seq test below */ 6807 cursor.priv = donor_dsq->seq; 6808 list_add(&cursor.node, &donor_dsq->list); 6809 resume: 6810 n = container_of(&cursor, struct task_struct, scx.dsq_list); 6811 n = nldsq_next_task(donor_dsq, n, false); 6812 6813 while ((p = n)) { 6814 s32 donee, cpu; 6815 6816 n = nldsq_next_task(donor_dsq, n, false); 6817 6818 if (donor_dsq->nr <= nr_donor_target) 6819 break; 6820 6821 /* 6822 * Without a donee, a balancing donor has nothing to do. A drain 6823 * still sends its tasks home. 6824 */ 6825 if (cpumask_empty(donee_mask) && nr_donor_target) 6826 break; 6827 6828 /* 6829 * A task queued after the walk began, including one the walk 6830 * itself reenqueued, is left for the next pass. 6831 */ 6832 if (u32_before(cursor.priv, p->scx.dsq_seq)) 6833 continue; 6834 6835 /* 6836 * A balancing donor leaves a task that can't move where it is. 6837 * A home reenqueue would push one homed on the donor to the 6838 * back of @donor_dsq on every pass. A drain sends it home. 6839 */ 6840 donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); 6841 if (donee >= nr_cpu_ids) 6842 donee = -1; 6843 if (nr_donor_target && !bypass_lb_can_move(sch, p, donee)) 6844 continue; 6845 6846 /* 6847 * The move drops @donor_dsq's lock, and the rq lock too when 6848 * @p is homed elsewhere. The donor cpu's dispatch and its 6849 * enqueuers can reach @donor_dsq then. Park the cursor behind 6850 * @p and resume from it afterwards. The seq test above keeps a 6851 * task enqueued meanwhile out of the pass. 6852 */ 6853 list_move(&cursor.node, &p->scx.dsq_list.node); 6854 cpu = bypass_lb_move(sch, donor_rq, donor_dsq, p, donee); 6855 if (cpu >= 0) { 6856 /* 6857 * @cpu might have been idle and need to be woken up. 6858 * No need to be clever. Reschedule every cpu that 6859 * receives tasks. 6860 */ 6861 cpumask_set_cpu(cpu, resched_mask); 6862 6863 if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) >= nr_donee_target) 6864 cpumask_clear_cpu(cpu, donee_mask); 6865 6866 /* 6867 * The rq lock is held across the walk, not cycled per 6868 * move. @donor_dsq can be very long and contended, and 6869 * the rq lock is what keeps its consumer and enqueuers 6870 * away while the walk cycles the DSQ lock. The rare 6871 * move of a task homed elsewhere does switch it away, 6872 * see the cursor comment above. The walk can still take 6873 * a while, so let the cpu breathe every so often. 6874 */ 6875 if (!(++nr_balanced % SCX_BYPASS_LB_BATCH)) { 6876 raw_spin_unlock(&donor_dsq->lock); 6877 scx_rq_lock_drop(donor_rq); 6878 raw_spin_rq_unlock_irq(donor_rq); 6879 cpu_relax(); 6880 raw_spin_rq_lock_irq(donor_rq); 6881 raw_spin_lock(&donor_dsq->lock); 6882 } 6883 } 6884 goto resume; 6885 } 6886 6887 list_del_init(&cursor.node); 6888 raw_spin_unlock(&donor_dsq->lock); 6889 scx_rq_lock_drop(donor_rq); 6890 raw_spin_rq_unlock_irq(donor_rq); 6891 6892 return nr_balanced; 6893 } 6894 6895 static void bypass_lb_node(struct scx_sched *sch, int node) 6896 { 6897 const struct cpumask *node_mask = cpumask_of_node(node); 6898 struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask; 6899 struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask; 6900 u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us); 6901 u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0; 6902 u32 nr_target, nr_donor_target; 6903 u32 before_min = U32_MAX, before_max = 0; 6904 u32 after_min = U32_MAX, after_max = 0; 6905 int cpu; 6906 6907 /* count the target tasks and CPUs */ 6908 for_each_cpu_and(cpu, cpu_online_mask, node_mask) { 6909 u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); 6910 6911 nr_tasks += nr; 6912 /* only a cpu whose dispatch consumes its DSQ is capacity */ 6913 if (!scx_missing_caps(sch, cpu, SCX_CAP_BASE)) 6914 nr_cpus++; 6915 6916 before_min = min(nr, before_min); 6917 before_max = max(nr, before_max); 6918 } 6919 6920 if (!nr_tasks) 6921 return; 6922 6923 /* 6924 * We don't want CPUs to have more than $nr_donor_target tasks and 6925 * balancing to fill donee CPUs up to $nr_target. Once targets are 6926 * calculated, find the donee CPUs. 6927 */ 6928 nr_target = nr_cpus ? DIV_ROUND_UP(nr_tasks, nr_cpus) : 0; 6929 nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100); 6930 6931 cpumask_clear(donee_mask); 6932 for_each_cpu_and(cpu, cpu_online_mask, node_mask) { 6933 /* a donee's DSQ is consumed only if @sch holds %SCX_CAP_BASE */ 6934 if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target && 6935 !scx_missing_caps(sch, cpu, SCX_CAP_BASE)) 6936 cpumask_set_cpu(cpu, donee_mask); 6937 } 6938 6939 /* 6940 * Offload the !donee cpus. A cpu where the host lost %SCX_CAP_BASE no 6941 * longer consumes its DSQ. Drain it as a donor with a target of zero. A 6942 * zero interval disables the balancing, not the drain. 6943 */ 6944 cpumask_clear(resched_mask); 6945 for_each_cpu_and(cpu, cpu_online_mask, node_mask) { 6946 u32 target = nr_donor_target; 6947 6948 if (unlikely(scx_missing_caps(sch, cpu, SCX_CAP_BASE))) { 6949 /* the mask may predate the loss, keep this cpu out */ 6950 cpumask_clear_cpu(cpu, donee_mask); 6951 target = 0; 6952 } else if (!intv_us || cpumask_empty(donee_mask) || 6953 cpumask_test_cpu(cpu, donee_mask)) { 6954 continue; 6955 } 6956 if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= target) 6957 continue; 6958 6959 nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, 6960 target, nr_target); 6961 } 6962 6963 for_each_cpu(cpu, resched_mask) 6964 resched_cpu(cpu); 6965 6966 for_each_cpu_and(cpu, cpu_online_mask, node_mask) { 6967 u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); 6968 6969 after_min = min(nr, after_min); 6970 after_max = max(nr, after_max); 6971 6972 } 6973 6974 trace_sched_ext_bypass_lb(node, nr_cpus, nr_tasks, nr_balanced, 6975 before_min, before_max, after_min, after_max); 6976 } 6977 6978 /* 6979 * In bypass mode, all tasks are put on the per-CPU bypass DSQs. If the machine 6980 * is over-saturated and the BPF scheduler skewed tasks into few CPUs, some 6981 * bypass DSQs can be overloaded. If there are enough tasks to saturate other 6982 * lightly loaded CPUs, such imbalance can lead to very high execution latency 6983 * on the overloaded CPUs and thus to hung tasks and RCU stalls. To avoid such 6984 * outcomes, a simple load balancing mechanism is implemented by the following 6985 * timer which runs periodically while bypass mode is in effect. 6986 */ 6987 static void scx_bypass_lb_timerfn(struct timer_list *timer) 6988 { 6989 struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer); 6990 int node; 6991 u32 intv_us; 6992 6993 if (!scx_bypass_dsp_enabled(sch)) 6994 return; 6995 6996 for_each_node_with_cpus(node) 6997 bypass_lb_node(sch, node); 6998 6999 /* 7000 * A kick during this run can leave the timer pending at now, see 7001 * scx_bypass_lb_kick(). timer_reduce() only brings the expiry forward, 7002 * so that kick is kept. 7003 */ 7004 intv_us = READ_ONCE(scx_bypass_lb_intv_us); 7005 if (intv_us) 7006 timer_reduce(timer, jiffies + usecs_to_jiffies(intv_us)); 7007 } 7008 7009 /** 7010 * scx_bypass_lb_kick - Run the bypass load balancer now 7011 * @sch: sched that lost %SCX_CAP_BASE on a cpu 7012 * 7013 * @sch no longer consumes its bypass DSQ on that cpu. Have the balancer drain 7014 * it without waiting for the next interval, see bypass_lb_node(). A loss during 7015 * a run leaves the timer pending at now, and the run that follows sees it. 7016 */ 7017 void scx_bypass_lb_kick(struct scx_sched *sch) 7018 { 7019 if (scx_bypass_dsp_enabled(sch)) 7020 mod_timer(&sch->bypass_lb_timer, jiffies); 7021 } 7022 7023 static bool inc_bypass_depth(struct scx_sched *sch) 7024 { 7025 lockdep_assert_held(&scx_bypass_lock); 7026 7027 WARN_ON_ONCE(sch->bypass_depth < 0); 7028 WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1); 7029 if (sch->bypass_depth != 1) 7030 return false; 7031 7032 WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC); 7033 sch->bypass_timestamp = ktime_get_ns(); 7034 scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1); 7035 return true; 7036 } 7037 7038 static bool dec_bypass_depth(struct scx_sched *sch) 7039 { 7040 lockdep_assert_held(&scx_bypass_lock); 7041 7042 WARN_ON_ONCE(sch->bypass_depth < 1); 7043 WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1); 7044 if (sch->bypass_depth != 0) 7045 return false; 7046 7047 WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL); 7048 scx_add_event(sch, SCX_EV_BYPASS_DURATION, 7049 ktime_get_ns() - sch->bypass_timestamp); 7050 return true; 7051 } 7052 7053 static void enable_bypass_dsp(struct scx_sched *sch) 7054 { 7055 struct scx_sched *host = scx_parent(sch) ?: sch; 7056 u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us); 7057 s32 ret; 7058 7059 /* 7060 * @sch->bypass_depth transitioning from 0 to 1 triggers enabling. 7061 * Shouldn't stagger. 7062 */ 7063 if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim))) 7064 return; 7065 7066 /* 7067 * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of 7068 * the nearest non-bypassing ancestor or root. The parent hosts them 7069 * unless it is bypassing too. Then they sit further up and the parent 7070 * consumes only its own bypass DSQs, empty until it leaves bypass and 7071 * becomes the host. Either way the nearest non-bypassing ancestor or 7072 * root has bypass dispatch enabled while a descendant is bypassing, 7073 * which is all that's required. 7074 * 7075 * scx_bypass_dsp_enabled() test is used to determine whether to enter 7076 * the bypass dispatch handling path from both bypassing and hosting 7077 * scheds. Bump enable depth on both @sch and bypass dispatch host. 7078 */ 7079 ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); 7080 WARN_ON_ONCE(ret <= 0); 7081 7082 if (host != sch) { 7083 ret = atomic_inc_return(&host->bypass_dsp_enable_depth); 7084 WARN_ON_ONCE(ret <= 0); 7085 } 7086 7087 /* 7088 * The LB timer will stop running if bypass dispatch is disabled. Start 7089 * after enabling bypass dispatch. A kick from a BASE loss can have set 7090 * the timer to expire now in the meantime, see scx_bypass_lb_kick(). 7091 * timer_reduce() keeps that expiry. 7092 */ 7093 if (intv_us) 7094 timer_reduce(&host->bypass_lb_timer, jiffies + usecs_to_jiffies(intv_us)); 7095 } 7096 7097 /* may be called without holding scx_bypass_lock */ 7098 void scx_disable_bypass_dsp(struct scx_sched *sch) 7099 { 7100 s32 ret; 7101 7102 if (!test_and_clear_bit(0, &sch->bypass_dsp_claim)) 7103 return; 7104 7105 ret = atomic_dec_return(&sch->bypass_dsp_enable_depth); 7106 WARN_ON_ONCE(ret < 0); 7107 7108 if (scx_parent(sch)) { 7109 ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth); 7110 WARN_ON_ONCE(ret < 0); 7111 } 7112 } 7113 7114 /** 7115 * unbypass_renotify_idle - Arm an idle re-notify for a sched leaving bypass 7116 * @rq: rq of the cpu leaving bypass 7117 * @pos: scheduler that just left bypass on @rq's cpu 7118 * @pcpu: @pos's per-cpu state for @rq's cpu 7119 * 7120 * A sched leaving bypass is owed the ops.update_idle() calls suppressed while 7121 * bypassing. A cpu that goes idle during the bypass window and stays idle won't 7122 * produce a notification. Arm a re-notify that scx_bypass()'s resched flushes 7123 * on the next idle pick. 7124 * 7125 * An acute case is ops.sub_attach(). If the parent grants the child cids while 7126 * attaching, when attach is complete and bypass is lifted, the child may hold 7127 * idle cids it never saw go idle. 7128 * 7129 * The root is no exception as bypass suppresses its notifications the same way. 7130 * However, the root uses a separate per-rq flag so its re-notify keeps working 7131 * even when !CONFIG_EXT_SUB_SCHED. 7132 */ 7133 static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, 7134 struct scx_sched_pcpu *pcpu) 7135 { 7136 if (!pos->level) { 7137 rq->scx.flags |= SCX_RQ_ROOT_IDLE_RENOTIFY; 7138 return; 7139 } 7140 #ifdef CONFIG_EXT_SUB_SCHED 7141 pcpu->idle_renotify = true; 7142 rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY; 7143 #endif 7144 } 7145 7146 /** 7147 * scx_bypass - [Un]bypass scx_ops and guarantee forward progress 7148 * @sch: sched to bypass 7149 * @bypass: true for bypass, false for unbypass 7150 * 7151 * Bypassing guarantees that all runnable tasks make forward progress without 7152 * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might 7153 * be held by tasks that the BPF scheduler is forgetting to run, which 7154 * unfortunately also excludes toggling the static branches. 7155 * 7156 * Let's work around by overriding a couple ops and modifying behaviors based on 7157 * the DISABLING state and then cycling the queued tasks through dequeue/enqueue 7158 * to force global FIFO scheduling. 7159 * 7160 * - ops.select_cpu() is ignored and the default select_cpu() is used. 7161 * 7162 * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order. 7163 * %SCX_OPS_ENQ_LAST is also ignored. 7164 * 7165 * - ops.dispatch() is ignored. 7166 * 7167 * - ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() are still called. 7168 * The bypassed tasks are on the bypass DSQs, under rescue or admitted to a 7169 * local DSQ, out of their reach. 7170 * 7171 * - dispatch_one() expires prev's slice before its keep test as the slice can't 7172 * be trusted. Each tick does the same and rotates the running task to the 7173 * tail of the queue. A kernel slice is left alone, see 7174 * %SCX_TASK_KERNEL_SLICE. 7175 * 7176 * - pick_next_task() suppresses zero slice warning. 7177 * 7178 * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM 7179 * operations. 7180 * 7181 * - scx_prio_less() reverts to the default runnable_at order. 7182 */ 7183 void scx_bypass(struct scx_sched *sch, bool bypass) 7184 { 7185 struct scx_sched *pos; 7186 unsigned long flags; 7187 int cpu; 7188 bool changed; 7189 7190 raw_spin_lock_irqsave(&scx_bypass_lock, flags); 7191 7192 if (bypass) { 7193 changed = inc_bypass_depth(sch); 7194 if (changed) 7195 enable_bypass_dsp(sch); 7196 } else { 7197 changed = dec_bypass_depth(sch); 7198 } 7199 7200 /* 7201 * Bypass state is propagated to all descendants - an scx_sched bypasses 7202 * if itself or any of its ancestors are in bypass mode. A descendant's 7203 * depth is the sum of its own bypasses and its ancestors', so every 7204 * call propagates whether or not @sch changed state. The descendants' 7205 * claims are released below with @sch's, after their tasks have moved. 7206 */ 7207 raw_spin_lock(&scx_sched_lock); 7208 scx_for_each_descendant_pre(pos, sch) { 7209 if (pos == sch) 7210 continue; 7211 if (bypass) { 7212 if (inc_bypass_depth(pos)) 7213 enable_bypass_dsp(pos); 7214 } else { 7215 dec_bypass_depth(pos); 7216 } 7217 } 7218 raw_spin_unlock(&scx_sched_lock); 7219 7220 /* 7221 * A descendant's depth is at least @sch's, so no bypass state changed 7222 * unless @sch's did. 7223 */ 7224 if (!changed) 7225 goto unlock; 7226 7227 /* 7228 * No task property is changing. We just need to make sure all currently 7229 * queued tasks are re-queued according to the new scx_bypassing() 7230 * state. As an optimization, walk each rq's runnable_list instead of 7231 * the scx_tasks list. 7232 * 7233 * This function can't trust the scheduler and thus can't use 7234 * cpus_read_lock(). Walk all possible CPUs instead of online. 7235 */ 7236 for_each_possible_cpu(cpu) { 7237 struct rq *rq = cpu_rq(cpu); 7238 struct task_struct *p, *n; 7239 7240 raw_spin_rq_lock(rq); 7241 raw_spin_lock(&scx_sched_lock); 7242 7243 scx_for_each_descendant_pre(pos, sch) { 7244 struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); 7245 bool was_bypassing = pcpu->flags & SCX_SCHED_PCPU_BYPASSING; 7246 7247 if (pos->bypass_depth) { 7248 pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; 7249 } else { 7250 pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; 7251 if (was_bypassing) 7252 unbypass_renotify_idle(rq, pos, pcpu); 7253 } 7254 } 7255 7256 raw_spin_unlock(&scx_sched_lock); 7257 7258 /* 7259 * We need to guarantee that no tasks are on the BPF scheduler 7260 * while bypassing. Either we see enabled or the enable path 7261 * sees scx_bypassing() before moving tasks to SCX. 7262 */ 7263 if (!scx_enabled()) { 7264 scx_rq_lock_drop(rq); 7265 raw_spin_rq_unlock(rq); 7266 continue; 7267 } 7268 7269 /* 7270 * Bypass trumps protection. Cycling the queued tasks below 7271 * clears theirs. The current donor isn't on the runnable list, 7272 * so strip its protection here. 7273 * 7274 * When bypass lifts, the current donor keeps running on its 7275 * slice without a cap check. Evict it if it can't stay on the 7276 * cpu. 7277 */ 7278 p = rq->donor; 7279 if (p->sched_class == &ext_sched_class && 7280 scx_is_descendant(scx_task_sched(p), sch)) { 7281 if (bypass) 7282 scx_task_slice_ended(rq, p); 7283 else if (!scx_task_can_stay_on_cpu(rq, p)) 7284 expire_task_slice(p); 7285 } 7286 7287 /* 7288 * The use of list_for_each_entry_safe_reverse() is required 7289 * because each task is going to be removed from and added back 7290 * to the runnable_list during iteration. Because they're added 7291 * to the tail of the list, safe reverse iteration can still 7292 * visit all nodes. 7293 */ 7294 list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, 7295 scx.runnable_node) { 7296 struct scx_sched *owner = scx_task_sched(p); 7297 7298 if (!scx_is_descendant(owner, sch)) 7299 continue; 7300 7301 /* cycling deq/enq is enough, see the function comment */ 7302 scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { 7303 /* 7304 * The requeue can change the sched running @p, 7305 * and the watchdog counts @p's wait against 7306 * that sched. Restart the wait unless @owner is 7307 * the root, which runs its own tasks bypassing 7308 * or not. 7309 * 7310 * A dispatch in flight writes p->scx.flags 7311 * without @p's rq lock. The dequeue waits it 7312 * out, so set the flag after it. 7313 */ 7314 if (scx_parent(owner)) 7315 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; 7316 } 7317 } 7318 7319 /* resched to restore ticks and idle state */ 7320 if (cpu_online(cpu) || cpu == smp_processor_id()) 7321 resched_curr(rq); 7322 7323 scx_rq_lock_drop(rq); 7324 raw_spin_rq_unlock(rq); 7325 } 7326 7327 /* disarming must come after moving all tasks out of the bypass DSQs */ 7328 if (!bypass) { 7329 guard(raw_spinlock)(&scx_sched_lock); 7330 scx_for_each_descendant_pre(pos, sch) 7331 if (!pos->bypass_depth) 7332 scx_disable_bypass_dsp(pos); 7333 } 7334 unlock: 7335 raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); 7336 } 7337 7338 static void free_exit_info(struct scx_exit_info *ei) 7339 { 7340 kvfree(ei->dump); 7341 kfree(ei->msg); 7342 kfree(ei->bt); 7343 kfree(ei); 7344 } 7345 7346 static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len) 7347 { 7348 struct scx_exit_info *ei; 7349 7350 ei = kzalloc_obj(*ei); 7351 if (!ei) 7352 return NULL; 7353 7354 ei->exit_cpu = -1; 7355 ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN); 7356 ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); 7357 ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL); 7358 7359 if (!ei->bt || !ei->msg || !ei->dump) { 7360 free_exit_info(ei); 7361 return NULL; 7362 } 7363 7364 return ei; 7365 } 7366 7367 static const char *scx_exit_reason(enum scx_exit_kind kind) 7368 { 7369 switch (kind) { 7370 case SCX_EXIT_UNREG: 7371 return "unregistered from user space"; 7372 case SCX_EXIT_UNREG_BPF: 7373 return "unregistered from BPF"; 7374 case SCX_EXIT_UNREG_KERN: 7375 return "unregistered from the main kernel"; 7376 case SCX_EXIT_SYSRQ: 7377 return "disabled by sysrq-S"; 7378 case SCX_EXIT_PARENT: 7379 return "parent exiting"; 7380 case SCX_EXIT_PARENT_KILL: 7381 return "killed by parent scheduler"; 7382 case SCX_EXIT_ERROR: 7383 return "runtime error"; 7384 case SCX_EXIT_ERROR_BPF: 7385 return "scx_bpf_error"; 7386 case SCX_EXIT_ERROR_STALL: 7387 return "runnable task stall"; 7388 case SCX_EXIT_ERROR_REENQ: 7389 return "reenqueue limit"; 7390 case SCX_EXIT_ERROR_RESCUE: 7391 return "rescue bandwidth overload"; 7392 default: 7393 return "<UNKNOWN>"; 7394 } 7395 } 7396 7397 static void free_kick_syncs(void) 7398 { 7399 int cpu; 7400 7401 for_each_possible_cpu(cpu) { 7402 struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); 7403 struct scx_kick_syncs *to_free; 7404 7405 /* flush the pending kick before freeing @ksyncs */ 7406 irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); 7407 to_free = rcu_replace_pointer(*ksyncs, NULL, true); 7408 if (to_free) 7409 kvfree_rcu(to_free, rcu); 7410 } 7411 } 7412 7413 static void refresh_watchdog(void) 7414 { 7415 struct scx_sched *sch; 7416 unsigned long intv = ULONG_MAX; 7417 7418 /* take the shortest timeout and use its half for watchdog interval */ 7419 rcu_read_lock(); 7420 list_for_each_entry_rcu(sch, &scx_sched_all, all) 7421 intv = max(min(intv, sch->watchdog_timeout / 2), 1); 7422 rcu_read_unlock(); 7423 7424 WRITE_ONCE(scx_watchdog_timestamp, jiffies); 7425 WRITE_ONCE(scx_watchdog_interval, intv); 7426 7427 if (intv < ULONG_MAX) 7428 mod_delayed_work(system_dfl_wq, &scx_watchdog_work, intv); 7429 else 7430 cancel_delayed_work_sync(&scx_watchdog_work); 7431 } 7432 7433 s32 scx_link_sched(struct scx_sched *sch) 7434 { 7435 scoped_guard(raw_spinlock_irqsave, &scx_bypass_lock) /* for the parent bypass check */ 7436 scoped_guard(raw_spinlock, &scx_sched_lock) { 7437 #ifdef CONFIG_EXT_SUB_SCHED 7438 struct scx_sched *parent = scx_parent(sch); 7439 7440 if (parent) { 7441 s32 ret; 7442 7443 /* 7444 * Bypass state is spread across per-cpu flags and a 7445 * depth count, so inheriting it is tricky and has no 7446 * valid use case. Refuse it. 7447 */ 7448 if (READ_ONCE(parent->bypass_depth)) { 7449 scx_error(sch, "parent bypassing (%d)", -EBUSY); 7450 return -EBUSY; 7451 } 7452 7453 ret = rhashtable_lookup_insert_fast(&scx_sched_hash, 7454 &sch->hash_node, scx_sched_hash_params); 7455 if (ret) { 7456 scx_error(sch, "failed to insert into scx_sched_hash (%d)", 7457 ret); 7458 return ret; 7459 } 7460 7461 list_add_tail_rcu(&sch->sibling, &parent->children); 7462 7463 /* 7464 * Pairs with the mb after the ->aborting assertion in 7465 * scx_claim_exit(). Either we see ->aborting and back 7466 * out, or the exit path sees us and exits us. 7467 */ 7468 smp_mb(); 7469 if (unlikely(READ_ONCE(parent->aborting))) { 7470 rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, 7471 scx_sched_hash_params); 7472 list_del_rcu(&sch->sibling); 7473 scx_error(sch, "parent disabled (%d)", -ENOENT); 7474 return -ENOENT; 7475 } 7476 7477 sch->linked = true; 7478 } 7479 #endif /* CONFIG_EXT_SUB_SCHED */ 7480 7481 list_add_tail_rcu(&sch->all, &scx_sched_all); 7482 } 7483 7484 refresh_watchdog(); 7485 return 0; 7486 } 7487 7488 void scx_unlink_sched(struct scx_sched *sch) 7489 { 7490 scoped_guard(raw_spinlock_irq, &scx_sched_lock) { 7491 #ifdef CONFIG_EXT_SUB_SCHED 7492 if (sch->linked) { 7493 rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, 7494 scx_sched_hash_params); 7495 list_del_rcu(&sch->sibling); 7496 sch->linked = false; 7497 } 7498 #endif /* CONFIG_EXT_SUB_SCHED */ 7499 list_del_rcu(&sch->all); 7500 } 7501 7502 refresh_watchdog(); 7503 } 7504 7505 /* 7506 * Called to disable future dumps and wait for in-progress one while disabling 7507 * @sch. Once @sch becomes empty during disable, there's no point in dumping it. 7508 * This prevents calling dump ops on a dead sch. 7509 */ 7510 void scx_disable_dump(struct scx_sched *sch) 7511 { 7512 guard(raw_spinlock_irqsave)(&scx_dump_lock); 7513 sch->dump_disabled = true; 7514 } 7515 7516 void scx_log_sched_disable(struct scx_sched *sch) 7517 { 7518 struct scx_exit_info *ei = sch->exit_info; 7519 const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; 7520 7521 if (ei->kind >= SCX_EXIT_ERROR) { 7522 pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, 7523 sch->ops.name, ei->reason); 7524 7525 if (ei->msg[0] != '\0') 7526 pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg); 7527 #ifdef CONFIG_STACKTRACE 7528 stack_trace_print(ei->bt, ei->bt_len, 2); 7529 #endif 7530 } else { 7531 pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, 7532 sch->ops.name, ei->reason); 7533 } 7534 } 7535 7536 static void scx_root_disable(struct scx_sched *sch) 7537 { 7538 struct scx_task_iter sti; 7539 struct task_struct *p; 7540 bool was_switched_all; 7541 int cpu; 7542 7543 /* guarantee forward progress and wait for descendants to be disabled */ 7544 scx_bypass(sch, true); 7545 drain_descendants(sch); 7546 7547 switch (scx_set_enable_state(SCX_DISABLING)) { 7548 case SCX_DISABLING: 7549 WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); 7550 break; 7551 case SCX_DISABLED: 7552 pr_warn("sched_ext: ops error detected without ops (%s)\n", 7553 sch->exit_info->msg); 7554 WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); 7555 goto done; 7556 default: 7557 break; 7558 } 7559 7560 /* 7561 * Here, every runnable task is guaranteed to make forward progress and 7562 * we can safely use blocking synchronization constructs. Actually 7563 * disable ops. 7564 */ 7565 mutex_lock(&scx_enable_mutex); 7566 7567 was_switched_all = scx_switched_all(); 7568 7569 static_branch_disable(&__scx_switched_all); 7570 WRITE_ONCE(scx_switching_all, false); 7571 7572 /* 7573 * Shut down cgroup support before tasks so that the cgroup attach and 7574 * migration paths don't race against scx_disable_and_exit_task(). 7575 */ 7576 scx_cgroup_lock(); 7577 scx_cgroup_enabled = false; 7578 scx_cgroup_exit(sch); 7579 scx_cgroup_unlock(); 7580 7581 /* 7582 * The BPF scheduler is going away. All tasks including %TASK_DEAD ones 7583 * must be switched out and exited synchronously. 7584 */ 7585 percpu_down_write(&scx_fork_rwsem); 7586 7587 scx_init_task_enabled = false; 7588 7589 scx_task_iter_start(&sti, NULL); 7590 while ((p = scx_task_iter_next_locked(&sti))) { 7591 unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7592 const struct sched_class *old_class = p->sched_class; 7593 const struct sched_class *new_class = scx_setscheduler_class(p); 7594 7595 update_rq_clock(task_rq(p)); 7596 7597 if (old_class != new_class) 7598 queue_flags |= DEQUEUE_CLASS; 7599 7600 scoped_guard (sched_change, p, queue_flags) { 7601 p->sched_class = new_class; 7602 } 7603 7604 scx_disable_and_exit_task(scx_task_sched(p), p); 7605 } 7606 scx_task_iter_stop(&sti); 7607 7608 scx_disable_dump(sch); 7609 7610 scx_cgroup_lock(); 7611 set_cgroup_sched(sch_cgroup(sch), NULL); 7612 scx_cgroup_unlock(); 7613 7614 percpu_up_write(&scx_fork_rwsem); 7615 7616 /* 7617 * Re-balance the dl_server bandwidth reservations: detach ext_server 7618 * (no more sched_ext tasks) and reinstate fair_server if it was 7619 * previously detached because we were running in full mode. 7620 * 7621 * Unlike the enable path, this runs on a recovery path that cannot 7622 * fail, so we use dl_server_swap_bw() to atomically free ext_server's 7623 * bandwidth and reclaim it for fair_server under the same dl_b lock. 7624 * 7625 * The swap can still fail with -EBUSY if someone bumped ext_server's 7626 * runtime via debugfs between enable and disable; in that narrow case 7627 * both servers end up detached and we just WARN. 7628 */ 7629 for_each_possible_cpu(cpu) { 7630 struct rq *rq = cpu_rq(cpu); 7631 7632 scoped_guard(rq_lock_irqsave, rq) { 7633 update_rq_clock(rq); 7634 if (was_switched_all) { 7635 if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server, 7636 &rq->fair_server))) 7637 pr_warn("failed to re-attach fair_server on CPU %d\n", cpu); 7638 } else { 7639 dl_server_detach_bw(&rq->ext_server); 7640 } 7641 } 7642 } 7643 7644 scx_ops_cid_sched_updated_disable(sch); 7645 7646 /* no task is on scx, turn off all the switches and flush in-progress calls */ 7647 static_branch_disable(&__scx_enabled); 7648 static_branch_disable(&__scx_is_cid_type); 7649 if (sch->ops.flags & SCX_OPS_TID_TO_TASK) 7650 static_branch_disable(&__scx_tid_to_task_enabled); 7651 bitmap_zero(sch->has_op, SCX_OPI_END); 7652 scx_idle_disable(); 7653 synchronize_rcu(); 7654 if (sch->ops.flags & SCX_OPS_TID_TO_TASK) 7655 rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); 7656 7657 scx_log_sched_disable(sch); 7658 7659 if (sch->ops.exit) 7660 SCX_CALL_OP(sch, exit, NULL, sch->exit_info); 7661 7662 /* 7663 * @sch's non-ops programs such as timers and tracers can fire after 7664 * ops.exit(). Now that exit is complete, stop scx_prog_sched() from 7665 * resolving to @sch and drain in-flight resolvers. 7666 */ 7667 WRITE_ONCE(sch->dead, true); 7668 synchronize_rcu(); 7669 7670 scx_unlink_sched(sch); 7671 7672 /* 7673 * scx_root clearing and cid table retirement must be inside 7674 * cpus_read_lock(). See handle_hotplug(). 7675 */ 7676 cpus_read_lock(); 7677 RCU_INIT_POINTER(scx_root, NULL); 7678 scx_cid_retire_tables(); 7679 cpus_read_unlock(); 7680 7681 /* 7682 * Delete the kobject from the hierarchy synchronously. Otherwise, sysfs 7683 * could observe an object of the same name still in the hierarchy when 7684 * the next scheduler is loaded. 7685 */ 7686 #ifdef CONFIG_EXT_SUB_SCHED 7687 if (sch->sub_kset) 7688 kobject_del(&sch->sub_kset->kobj); 7689 #endif 7690 /* not added if enable failed before scx_sched_sysfs_add() */ 7691 if (sch->kobj.state_in_sysfs) 7692 kobject_del(&sch->kobj); 7693 7694 free_kick_syncs(); 7695 7696 mutex_unlock(&scx_enable_mutex); 7697 7698 WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); 7699 done: 7700 scx_bypass(sch, false); 7701 } 7702 7703 /** 7704 * scx_propagate_exit_irq_workfn - Claim SCX_EXIT_PARENT on the exiting subtree 7705 * @irq_work: &scx_sched.propagate_exit_irq_work 7706 * 7707 * Queued by scx_claim_exit() after a non-PARENT claim. Claims SCX_EXIT_PARENT 7708 * on each descendant, giving every one its own disable work - most of disabling 7709 * is serialized but ops.exit() can take arbitrarily long and running them in 7710 * separate helper kthreads parallelizes it. No recursion as only non-PARENT 7711 * claims propagate. 7712 */ 7713 static void scx_propagate_exit_irq_workfn(struct irq_work *irq_work) 7714 { 7715 struct scx_sched *sch = container_of(irq_work, struct scx_sched, 7716 propagate_exit_irq_work); 7717 struct scx_sched *pos; 7718 7719 scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { 7720 scx_for_each_descendant_pre(pos, sch) 7721 scx_disable(pos, SCX_EXIT_PARENT); 7722 } 7723 } 7724 7725 /* 7726 * Claim the exit on @sch. The caller must ensure that the helper kthread work 7727 * is kicked before the current task can be preempted. Once exit_kind is 7728 * claimed, scx_error() can no longer trigger, so if the current task gets 7729 * preempted and the BPF scheduler fails to schedule it back, the helper work 7730 * will never be kicked and the whole system can wedge. 7731 * 7732 * Lock-free and safe to call from any context including NMI. 7733 */ 7734 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) 7735 { 7736 int none = SCX_EXIT_NONE; 7737 7738 lockdep_assert_preemption_disabled(); 7739 7740 if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) 7741 kind = SCX_EXIT_ERROR; 7742 7743 if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) 7744 return false; 7745 7746 if (kind == SCX_EXIT_PARENT) { 7747 /* an ancestor is already sweeping the subtree */ 7748 WRITE_ONCE(sch->aborting, true); 7749 } else { 7750 struct scx_sched *pos; 7751 7752 /* 7753 * CPUs may be live-locked in the dispatch paths of @sch or its 7754 * descendants, which ->aborting breaks. Sweep the subtree 7755 * locklessly so that this works from NMI. smp_store_mb() orders 7756 * each node's ->aborting store before its children are walked - 7757 * either we see a racing scx_link_sched() on ->children or it 7758 * sees ->aborting. 7759 */ 7760 scoped_guard (rcu) { 7761 scx_for_each_descendant_pre(pos, sch) 7762 smp_store_mb(pos->aborting, true); 7763 } 7764 7765 irq_work_queue(&sch->propagate_exit_irq_work); 7766 } 7767 7768 /* fired after ->aborting is set so callbacks can't delay recovery */ 7769 trace_sched_ext_exit(sch, kind); 7770 7771 return true; 7772 } 7773 7774 static void scx_disable_workfn(struct kthread_work *work) 7775 { 7776 struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); 7777 struct scx_exit_info *ei = sch->exit_info; 7778 int kind; 7779 7780 kind = atomic_read(&sch->exit_kind); 7781 while (true) { 7782 if (kind == SCX_EXIT_DONE) /* already disabled? */ 7783 return; 7784 WARN_ON_ONCE(kind == SCX_EXIT_NONE); 7785 if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) 7786 break; 7787 } 7788 ei->kind = kind; 7789 ei->reason = scx_exit_reason(ei->kind); 7790 7791 if (scx_parent(sch)) 7792 scx_sub_disable(sch); 7793 else 7794 scx_root_disable(sch); 7795 } 7796 7797 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) 7798 { 7799 guard(preempt)(); 7800 if (scx_claim_exit(sch, kind)) 7801 irq_work_queue(&sch->disable_irq_work); 7802 } 7803 7804 /** 7805 * scx_flush_disable_work - flush the disable work and wait for it to finish 7806 * @sch: the scheduler 7807 * 7808 * sch->disable_work might still not queued, causing kthread_flush_work() 7809 * as a noop. Syncing the irq_work first is required to guarantee the 7810 * kthread work has been queued before waiting for it. 7811 */ 7812 void scx_flush_disable_work(struct scx_sched *sch) 7813 { 7814 int kind; 7815 7816 /* 7817 * scx_disable_workfn() sets SCX_EXIT_DONE before tearing down. Read 7818 * exit_kind with acquire ordering (paired with atomic_try_cmpxchg() in 7819 * scx_disable_workfn()) before syncing and flushing so that, once DONE 7820 * is seen, the following flush observes the queued work and waits for 7821 * the teardown to finish. 7822 */ 7823 do { 7824 kind = atomic_read_acquire(&sch->exit_kind); 7825 irq_work_sync(&sch->disable_irq_work); 7826 kthread_flush_work(&sch->disable_work); 7827 } while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE); 7828 } 7829 7830 static void dump_newline(struct seq_buf *s) 7831 { 7832 trace_sched_ext_dump(""); 7833 7834 /* @s may be zero sized and seq_buf triggers WARN if so */ 7835 if (s->size) 7836 seq_buf_putc(s, '\n'); 7837 } 7838 7839 __printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...) 7840 { 7841 va_list args; 7842 7843 #ifdef CONFIG_TRACEPOINTS 7844 if (trace_sched_ext_dump_enabled()) { 7845 /* protected by scx_dump_lock */ 7846 static char line_buf[SCX_EXIT_MSG_LEN]; 7847 7848 va_start(args, fmt); 7849 vscnprintf(line_buf, sizeof(line_buf), fmt, args); 7850 va_end(args); 7851 7852 trace_call__sched_ext_dump(line_buf); 7853 } 7854 #endif 7855 /* @s may be zero sized and seq_buf triggers WARN if so */ 7856 if (s->size) { 7857 va_start(args, fmt); 7858 seq_buf_vprintf(s, fmt, args); 7859 va_end(args); 7860 7861 seq_buf_putc(s, '\n'); 7862 } 7863 } 7864 7865 static void dump_stack_trace(struct seq_buf *s, const char *prefix, 7866 const unsigned long *bt, unsigned int len) 7867 { 7868 unsigned int i; 7869 7870 for (i = 0; i < len; i++) 7871 scx_dump_line(s, "%s%pS", prefix, (void *)bt[i]); 7872 } 7873 7874 static void ops_dump_init(struct seq_buf *s, const char *prefix) 7875 { 7876 struct scx_dump_data *dd = &scx_dump_data; 7877 7878 lockdep_assert_irqs_disabled(); 7879 7880 dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */ 7881 dd->first = true; 7882 dd->cursor = 0; 7883 dd->s = s; 7884 dd->prefix = prefix; 7885 } 7886 7887 static void ops_dump_flush(void) 7888 { 7889 struct scx_dump_data *dd = &scx_dump_data; 7890 char *line = dd->buf.line; 7891 7892 if (!dd->cursor) 7893 return; 7894 7895 /* 7896 * There's something to flush and this is the first line. Insert a blank 7897 * line to distinguish ops dump. 7898 */ 7899 if (dd->first) { 7900 dump_newline(dd->s); 7901 dd->first = false; 7902 } 7903 7904 /* 7905 * There may be multiple lines in $line. Scan and emit each line 7906 * separately. 7907 */ 7908 while (true) { 7909 char *end = line; 7910 char c; 7911 7912 while (*end != '\n' && *end != '\0') 7913 end++; 7914 7915 /* 7916 * If $line overflowed, it may not have newline at the end. 7917 * Always emit with a newline. 7918 */ 7919 c = *end; 7920 *end = '\0'; 7921 scx_dump_line(dd->s, "%s%s", dd->prefix, line); 7922 if (c == '\0') 7923 break; 7924 7925 /* move to the next line */ 7926 end++; 7927 if (*end == '\0') 7928 break; 7929 line = end; 7930 } 7931 7932 dd->cursor = 0; 7933 } 7934 7935 static void ops_dump_exit(void) 7936 { 7937 ops_dump_flush(); 7938 scx_dump_data.cpu = -1; 7939 } 7940 7941 static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx, 7942 struct rq *rq, struct task_struct *p, char marker) 7943 { 7944 static unsigned long bt[SCX_EXIT_BT_LEN]; 7945 struct scx_sched *task_sch = scx_task_sched(p); 7946 const char *own_marker; 7947 char sch_id_buf[32]; 7948 char dsq_id_buf[19] = "(n/a)"; 7949 unsigned long ops_state = atomic_long_read(&p->scx.ops_state); 7950 unsigned int bt_len = 0; 7951 7952 own_marker = task_sch == sch ? "*" : ""; 7953 7954 if (task_sch->level == 0) 7955 scnprintf(sch_id_buf, sizeof(sch_id_buf), "root"); 7956 else 7957 scnprintf(sch_id_buf, sizeof(sch_id_buf), "sub%d-%llu", 7958 task_sch->level, task_sch->ops.sub_cgroup_id); 7959 7960 if (p->scx.dsq) 7961 scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx", 7962 (unsigned long long)p->scx.dsq->id); 7963 7964 dump_newline(s); 7965 scx_dump_line(s, " %c%c %s[%d] %s%s %+ldms", 7966 marker, task_state_to_char(p), p->comm, p->pid, own_marker, sch_id_buf, 7967 jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); 7968 scx_dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", 7969 scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, 7970 p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags, 7971 ops_state & SCX_OPSS_STATE_MASK, ops_state >> SCX_OPSS_QSEQ_SHIFT); 7972 scx_dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", 7973 p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); 7974 scx_dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", 7975 p->scx.dsq_vtime, p->scx.slice, p->scx.weight); 7976 scx_dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), 7977 p->migration_disabled); 7978 7979 if (SCX_HAS_OP(sch, dump_task)) { 7980 ops_dump_init(s, " "); 7981 SCX_CALL_OP(sch, dump_task, rq, dctx, p); 7982 ops_dump_exit(); 7983 } 7984 7985 #ifdef CONFIG_STACKTRACE 7986 bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1); 7987 #endif 7988 if (bt_len) { 7989 dump_newline(s); 7990 dump_stack_trace(s, " ", bt, bt_len); 7991 } 7992 } 7993 7994 static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, 7995 struct scx_dump_ctx *dctx, int cpu, 7996 bool dump_all_tasks) 7997 { 7998 struct rq *rq = cpu_rq(cpu); 7999 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); 8000 struct rq_flags rf; 8001 struct task_struct *p; 8002 struct seq_buf ns; 8003 size_t avail, used; 8004 char *buf; 8005 bool idle; 8006 8007 rq_lock_irqsave(rq, &rf); 8008 8009 idle = list_empty(&rq->scx.runnable_list) && 8010 rq->curr->sched_class == &idle_sched_class; 8011 8012 if (idle && !SCX_HAS_OP(sch, dump_cpu)) 8013 goto next; 8014 8015 /* 8016 * We don't yet know whether ops.dump_cpu() will produce output 8017 * and we may want to skip the default CPU dump if it doesn't. 8018 * Use a nested seq_buf to generate the standard dump so that we 8019 * can decide whether to commit later. 8020 */ 8021 avail = seq_buf_get_buf(s, &buf); 8022 seq_buf_init(&ns, buf, avail); 8023 8024 dump_newline(&ns); 8025 scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ksync=%lu", 8026 cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, 8027 rq->scx.kick_sync); 8028 scx_rescue_dump(&ns, rq); 8029 scx_dump_line(&ns, " curr=%s[%d] class=%ps", 8030 rq->curr->comm, rq->curr->pid, rq->curr->sched_class); 8031 scx_dump_line(&ns, " donor=%s[%d] class=%ps", 8032 rq->donor->comm, rq->donor->pid, rq->donor->sched_class); 8033 if (!cpumask_empty(pcpu->cpus_to_kick)) 8034 scx_dump_line(&ns, " cpus_to_kick : %*pb", 8035 cpumask_pr_args(pcpu->cpus_to_kick)); 8036 if (!cpumask_empty(pcpu->cpus_to_kick_if_idle)) 8037 scx_dump_line(&ns, " idle_to_kick : %*pb", 8038 cpumask_pr_args(pcpu->cpus_to_kick_if_idle)); 8039 if (!cpumask_empty(pcpu->cpus_to_preempt)) 8040 scx_dump_line(&ns, " cpus_to_preempt: %*pb", 8041 cpumask_pr_args(pcpu->cpus_to_preempt)); 8042 if (!cpumask_empty(pcpu->cpus_to_preempt_lazy)) 8043 scx_dump_line(&ns, " preempt_lazy : %*pb", 8044 cpumask_pr_args(pcpu->cpus_to_preempt_lazy)); 8045 if (!cpumask_empty(pcpu->cpus_to_wait)) 8046 scx_dump_line(&ns, " cpus_to_wait : %*pb", 8047 cpumask_pr_args(pcpu->cpus_to_wait)); 8048 if (!cpumask_empty(rq->scx.cpus_to_sync)) 8049 scx_dump_line(&ns, " cpus_to_sync : %*pb", 8050 cpumask_pr_args(rq->scx.cpus_to_sync)); 8051 8052 used = seq_buf_used(&ns); 8053 if (SCX_HAS_OP(sch, dump_cpu)) { 8054 ops_dump_init(&ns, " "); 8055 SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle); 8056 ops_dump_exit(); 8057 } 8058 8059 /* 8060 * If idle && nothing generated by ops.dump_cpu(), there's 8061 * nothing interesting. Skip. 8062 */ 8063 if (idle && used == seq_buf_used(&ns)) 8064 goto next; 8065 8066 /* 8067 * $s may already have overflowed when $ns was created. If so, 8068 * calling commit on it will trigger BUG. 8069 */ 8070 if (avail) { 8071 seq_buf_commit(s, seq_buf_used(&ns)); 8072 if (seq_buf_has_overflowed(&ns)) 8073 seq_buf_set_overflow(s); 8074 } 8075 8076 if (rq->curr->sched_class == &ext_sched_class && 8077 (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) 8078 scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); 8079 if (rq->donor != rq->curr && 8080 rq->donor->sched_class == &ext_sched_class && 8081 (dump_all_tasks || scx_task_on_sched(sch, rq->donor))) 8082 scx_dump_task(sch, s, dctx, rq, rq->donor, ' '); 8083 8084 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) 8085 if (dump_all_tasks || scx_task_on_sched(sch, p)) 8086 scx_dump_task(sch, s, dctx, rq, p, ' '); 8087 next: 8088 rq_unlock_irqrestore(rq, &rf); 8089 } 8090 8091 /* 8092 * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless 8093 * of which scheduler they belong to. If false, only dump tasks owned by @sch. 8094 * For SysRq-D dumps, @dump_all_tasks=false since all schedulers are dumped 8095 * separately. For error dumps, @dump_all_tasks=true since only the failing 8096 * scheduler is dumped. 8097 */ 8098 static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, 8099 size_t dump_len, bool dump_all_tasks) 8100 { 8101 static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n"; 8102 struct scx_dump_ctx dctx = { 8103 .kind = ei->kind, 8104 .exit_code = ei->exit_code, 8105 .reason = ei->reason, 8106 .at_ns = ktime_get_ns(), 8107 .at_jiffies = jiffies, 8108 }; 8109 struct seq_buf s; 8110 struct scx_event_stats events; 8111 int cpu; 8112 8113 guard(raw_spinlock_irqsave)(&scx_dump_lock); 8114 8115 if (sch->dump_disabled) 8116 return; 8117 8118 seq_buf_init(&s, ei->dump, dump_len); 8119 8120 #ifdef CONFIG_EXT_SUB_SCHED 8121 if (sch->level == 0) 8122 scx_dump_line(&s, "%s: root", sch->ops.name); 8123 else 8124 scx_dump_line(&s, "%s: sub%d-%llu %s", 8125 sch->ops.name, sch->level, sch->ops.sub_cgroup_id, 8126 sch->cgrp_path); 8127 #endif 8128 if (ei->kind == SCX_EXIT_NONE) { 8129 scx_dump_line(&s, "Debug dump triggered by %s", ei->reason); 8130 } else { 8131 if (ei->exit_cpu >= 0) 8132 scx_dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", 8133 current->comm, current->pid, ei->kind, 8134 ei->exit_cpu); 8135 else 8136 scx_dump_line(&s, "%s[%d] triggered exit kind %d:", 8137 current->comm, current->pid, ei->kind); 8138 scx_dump_line(&s, " %s (%s)", ei->reason, ei->msg); 8139 dump_newline(&s); 8140 scx_dump_line(&s, "Backtrace:"); 8141 dump_stack_trace(&s, " ", ei->bt, ei->bt_len); 8142 } 8143 8144 if (SCX_HAS_OP(sch, dump)) { 8145 ops_dump_init(&s, ""); 8146 SCX_CALL_OP(sch, dump, NULL, &dctx); 8147 ops_dump_exit(); 8148 } 8149 8150 dump_newline(&s); 8151 scx_dump_line(&s, "CPU states"); 8152 scx_dump_line(&s, "----------"); 8153 8154 /* 8155 * Dump stalled CPUs first so they aren't lost to dump truncation, then 8156 * walk the rest in order. Fall back to exit_cpu if no stall mask set. 8157 */ 8158 if (!cpumask_empty(sch->stall_cpus)) { 8159 for_each_cpu(cpu, sch->stall_cpus) 8160 scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); 8161 for_each_possible_cpu(cpu) { 8162 if (!cpumask_test_cpu(cpu, sch->stall_cpus)) 8163 scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); 8164 } 8165 } else { 8166 if (ei->exit_cpu >= 0) 8167 scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); 8168 for_each_possible_cpu(cpu) { 8169 if (cpu != ei->exit_cpu) 8170 scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); 8171 } 8172 } 8173 8174 dump_newline(&s); 8175 scx_dump_line(&s, "Event counters"); 8176 scx_dump_line(&s, "--------------"); 8177 8178 scx_read_events(sch, &events); 8179 #define SCX_EVENT(name) scx_dump_event(s, &events, name) 8180 SCX_EVENTS_LIST(SCX_EVENT); 8181 #undef SCX_EVENT 8182 8183 if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) 8184 memcpy(ei->dump + dump_len - sizeof(trunc_marker), 8185 trunc_marker, sizeof(trunc_marker)); 8186 } 8187 8188 static void scx_disable_irq_workfn(struct irq_work *irq_work) 8189 { 8190 struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work); 8191 struct scx_exit_info *ei = sch->exit_info; 8192 8193 if (ei->kind >= SCX_EXIT_ERROR) 8194 scx_dump_state(sch, ei, sch->ops.exit_dump_len, true); 8195 8196 kthread_queue_work(sch->helper, &sch->disable_work); 8197 } 8198 8199 /* finish exit_info and kick the disable work, ei->msg must already be set */ 8200 static void scx_finish_exit(struct scx_sched *sch, enum scx_exit_kind kind, 8201 s64 exit_code, s32 exit_cpu) 8202 { 8203 struct scx_exit_info *ei = sch->exit_info; 8204 8205 ei->exit_code = exit_code; 8206 #ifdef CONFIG_STACKTRACE 8207 /* 8208 * stack_trace_save()'s NMI-safety is arch-dependent and undocumented. 8209 * Skip the backtrace when exiting from NMI. 8210 */ 8211 if (kind >= SCX_EXIT_ERROR && !in_nmi()) 8212 ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); 8213 #endif 8214 /* 8215 * Set ei->kind and ->reason for scx_dump_state(). They'll be set again 8216 * in scx_disable_workfn(). 8217 */ 8218 ei->kind = kind; 8219 ei->reason = scx_exit_reason(ei->kind); 8220 ei->exit_cpu = exit_cpu; 8221 8222 irq_work_queue(&sch->disable_irq_work); 8223 } 8224 8225 bool scx_vexit(struct scx_sched *sch, 8226 enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, 8227 const char *fmt, va_list args) 8228 { 8229 struct scx_exit_info *ei = sch->exit_info; 8230 8231 guard(preempt)(); 8232 8233 if (!scx_claim_exit(sch, kind)) 8234 return false; 8235 8236 vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); 8237 8238 scx_finish_exit(sch, kind, exit_code, exit_cpu); 8239 return true; 8240 } 8241 8242 static int alloc_kick_syncs(void) 8243 { 8244 int cpu; 8245 8246 /* 8247 * Allocate per-CPU arrays sized by nr_cpu_ids. Use kvzalloc as size 8248 * can exceed percpu allocator limits on large machines. 8249 */ 8250 for_each_possible_cpu(cpu) { 8251 struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); 8252 struct scx_kick_syncs *new_ksyncs; 8253 8254 WARN_ON_ONCE(rcu_access_pointer(*ksyncs)); 8255 8256 new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids), 8257 GFP_KERNEL, cpu_to_node(cpu)); 8258 if (!new_ksyncs) { 8259 free_kick_syncs(); 8260 return -ENOMEM; 8261 } 8262 8263 rcu_assign_pointer(*ksyncs, new_ksyncs); 8264 } 8265 8266 return 0; 8267 } 8268 8269 static void free_pnode(struct scx_sched_pnode *pnode) 8270 { 8271 if (!pnode) 8272 return; 8273 exit_dsq(&pnode->global_dsq); 8274 kfree(pnode); 8275 } 8276 8277 static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) 8278 { 8279 struct scx_sched_pnode *pnode; 8280 8281 pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node); 8282 if (!pnode) 8283 return NULL; 8284 8285 if (scx_init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { 8286 kfree(pnode); 8287 return NULL; 8288 } 8289 8290 return pnode; 8291 } 8292 8293 /* 8294 * Allocate and initialize a new scx_sched. @cgrp's reference is always 8295 * consumed whether the function succeeds or fails. 8296 */ 8297 struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, 8298 struct cgroup *cgrp, 8299 struct scx_sched *parent) 8300 { 8301 struct sched_ext_ops *ops = cmd->ops; 8302 struct scx_sched *sch; 8303 s32 level = parent ? parent->level + 1 : 0; 8304 s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids; 8305 8306 sch = kzalloc_flex(*sch, ancestors, level + 1); 8307 if (!sch) { 8308 ret = -ENOMEM; 8309 goto err_put_cgrp; 8310 } 8311 8312 sch->exit_info = alloc_exit_info(ops->exit_dump_len); 8313 if (!sch->exit_info) { 8314 ret = -ENOMEM; 8315 goto err_free_sch; 8316 } 8317 8318 ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params); 8319 if (ret < 0) 8320 goto err_free_ei; 8321 8322 sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids); 8323 if (!sch->pnode) { 8324 ret = -ENOMEM; 8325 goto err_free_hash; 8326 } 8327 8328 for_each_node_state(node, N_POSSIBLE) { 8329 sch->pnode[node] = alloc_pnode(sch, node); 8330 if (!sch->pnode[node]) { 8331 ret = -ENOMEM; 8332 goto err_free_pnode; 8333 } 8334 } 8335 8336 sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; 8337 sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu, 8338 dsp_ctx.buf, sch->dsp_max_batch), 8339 __alignof__(struct scx_sched_pcpu)); 8340 if (!sch->pcpu) { 8341 ret = -ENOMEM; 8342 goto err_free_pnode; 8343 } 8344 8345 for_each_possible_cpu(cpu) { 8346 ret = scx_init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); 8347 if (ret) { 8348 bypass_fail_cpu = cpu; 8349 goto err_free_pcpu; 8350 } 8351 } 8352 8353 for_each_possible_cpu(cpu) { 8354 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); 8355 8356 node = cpu_to_node(cpu); 8357 pcpu->sch = sch; 8358 INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); 8359 #ifdef CONFIG_EXT_SUB_SCHED 8360 init_llist_node(&pcpu->ecaps_to_sync_node); 8361 #endif 8362 INIT_LIST_HEAD(&pcpu->to_kick_node); 8363 if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) || 8364 !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) || 8365 !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) || 8366 !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt_lazy, GFP_KERNEL, node) || 8367 !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) { 8368 ret = -ENOMEM; 8369 goto err_free_pcpu; 8370 } 8371 } 8372 8373 sch->helper = kthread_run_worker(0, "sched_ext_helper"); 8374 if (IS_ERR(sch->helper)) { 8375 ret = PTR_ERR(sch->helper); 8376 goto err_free_pcpu; 8377 } 8378 8379 sched_set_fifo(sch->helper->task); 8380 8381 if (parent) 8382 memcpy(sch->ancestors, parent->ancestors, 8383 level * sizeof(parent->ancestors[0])); 8384 sch->ancestors[level] = sch; 8385 sch->level = level; 8386 sch->id = atomic64_inc_return(&scx_sched_id_cursor); 8387 8388 if (ops->timeout_ms) 8389 sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); 8390 else 8391 sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; 8392 8393 sch->slice_dfl = SCX_SLICE_DFL; 8394 atomic_set(&sch->exit_kind, SCX_EXIT_NONE); 8395 sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn); 8396 sch->propagate_exit_irq_work = IRQ_WORK_INIT_HARD(scx_propagate_exit_irq_workfn); 8397 kthread_init_work(&sch->disable_work, scx_disable_workfn); 8398 timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0); 8399 8400 if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) { 8401 ret = -ENOMEM; 8402 goto err_stop_helper; 8403 } 8404 if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) { 8405 ret = -ENOMEM; 8406 goto err_free_lb_cpumask; 8407 } 8408 if (!zalloc_cpumask_var(&sch->stall_cpus, GFP_KERNEL)) { 8409 ret = -ENOMEM; 8410 goto err_free_lb_resched_cpumask; 8411 } 8412 /* 8413 * Copy ops through the right union view. For cid-form the source is 8414 * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ 8415 * cpu_release; those stay zero from kzalloc. 8416 */ 8417 if (cmd->is_cid_type) { 8418 sch->ops_cid = *cmd->ops_cid; 8419 sch->is_cid_type = true; 8420 } else { 8421 sch->ops = *cmd->ops; 8422 } 8423 8424 #ifdef CONFIG_EXT_SUB_SCHED 8425 char *buf = kzalloc(PATH_MAX, GFP_KERNEL); 8426 if (!buf) { 8427 ret = -ENOMEM; 8428 goto err_free_lb_resched; 8429 } 8430 cgroup_path(cgrp, buf, PATH_MAX); 8431 sch->cgrp_path = kstrdup(buf, GFP_KERNEL); 8432 kfree(buf); 8433 if (!sch->cgrp_path) { 8434 ret = -ENOMEM; 8435 goto err_free_lb_resched; 8436 } 8437 8438 sch->cgrp = cgrp; 8439 INIT_LIST_HEAD(&sch->children); 8440 INIT_LIST_HEAD(&sch->sibling); 8441 #endif /* CONFIG_EXT_SUB_SCHED */ 8442 8443 /* 8444 * Publishing makes @sch visible to scx_prog_sched() readers. Failure 8445 * paths after this point must free @sch through kobject_put() whose 8446 * release path defers the actual freeing by an RCU grace period. 8447 */ 8448 rcu_assign_pointer(ops->priv, sch); 8449 8450 sch->kobj.kset = scx_kset; 8451 INIT_LIST_HEAD(&sch->all); 8452 8453 #ifdef CONFIG_EXT_SUB_SCHED 8454 if (parent) { 8455 /* 8456 * Pin @parent for @sch's lifetime. The kobject hierarchy pins 8457 * it only via @parent->sub_kset, which is dropped during 8458 * disable. Released in scx_sched_free_rcu_work(). 8459 */ 8460 kobject_get(&parent->kobj); 8461 } 8462 #endif /* CONFIG_EXT_SUB_SCHED */ 8463 8464 /* 8465 * Init the kobj but don't add to sysfs yet. The enable path calls 8466 * scx_sched_sysfs_add() once @sch's sysfs-visible state is initialized. 8467 */ 8468 kobject_init(&sch->kobj, &scx_ktype); 8469 8470 /* 8471 * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so 8472 * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid 8473 * drops the ref. After this point, sch owns the ref and any cleanup 8474 * runs through scx_sched_free_rcu_work() which puts it. 8475 */ 8476 sch->arena_map = cmd->arena_map; 8477 /* BPF arena is only available on MMU && 64BIT */ 8478 #if defined(CONFIG_MMU) && defined(CONFIG_64BIT) 8479 if (sch->arena_map) 8480 sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map); 8481 #endif 8482 cmd->arena_map = NULL; 8483 return sch; 8484 8485 #ifdef CONFIG_EXT_SUB_SCHED 8486 err_free_lb_resched: 8487 free_cpumask_var(sch->stall_cpus); 8488 #endif 8489 err_free_lb_resched_cpumask: 8490 free_cpumask_var(sch->bypass_lb_resched_cpumask); 8491 err_free_lb_cpumask: 8492 free_cpumask_var(sch->bypass_lb_donee_cpumask); 8493 err_stop_helper: 8494 kthread_destroy_worker(sch->helper); 8495 err_free_pcpu: 8496 for_each_possible_cpu(cpu) { 8497 struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); 8498 8499 free_cpumask_var(pcpu->cpus_to_kick); 8500 free_cpumask_var(pcpu->cpus_to_kick_if_idle); 8501 free_cpumask_var(pcpu->cpus_to_preempt); 8502 free_cpumask_var(pcpu->cpus_to_preempt_lazy); 8503 free_cpumask_var(pcpu->cpus_to_wait); 8504 } 8505 for_each_possible_cpu(cpu) { 8506 if (cpu == bypass_fail_cpu) 8507 break; 8508 exit_dsq(scx_bypass_dsq(sch, cpu)); 8509 } 8510 free_percpu(sch->pcpu); 8511 err_free_pnode: 8512 for_each_node_state(node, N_POSSIBLE) 8513 free_pnode(sch->pnode[node]); 8514 kfree(sch->pnode); 8515 err_free_hash: 8516 rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); 8517 err_free_ei: 8518 free_exit_info(sch->exit_info); 8519 err_free_sch: 8520 kfree(sch); 8521 err_put_cgrp: 8522 #ifdef CONFIG_EXT_SUB_SCHED 8523 cgroup_put(cgrp); 8524 #endif 8525 return ERR_PTR(ret); 8526 } 8527 8528 /* 8529 * Add @sch's kobject to sysfs, and create its sub_kset if the scheduler 8530 * implements ops.sub_attach. Called by the enable workfns once @sch's 8531 * sysfs-visible state is initialized. 8532 */ 8533 int scx_sched_sysfs_add(struct scx_sched *sch) 8534 { 8535 #ifdef CONFIG_EXT_SUB_SCHED 8536 struct scx_sched *parent = scx_parent(sch); 8537 int ret; 8538 8539 if (parent) 8540 ret = kobject_add(&sch->kobj, &parent->sub_kset->kobj, 8541 "sub-%llu", cgroup_id(sch_cgroup(sch))); 8542 else 8543 ret = kobject_add(&sch->kobj, NULL, "root"); 8544 if (ret < 0) 8545 return ret; 8546 8547 if (sch->ops.sub_attach) { 8548 sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); 8549 if (!sch->sub_kset) 8550 return -ENOMEM; 8551 } 8552 return 0; 8553 #else 8554 return kobject_add(&sch->kobj, NULL, "root"); 8555 #endif 8556 } 8557 8558 static int check_hotplug_seq(struct scx_sched *sch, 8559 const struct sched_ext_ops *ops) 8560 { 8561 unsigned long long global_hotplug_seq; 8562 8563 /* 8564 * If a hotplug event has occurred between when a scheduler was 8565 * initialized, and when we were able to attach, exit and notify user 8566 * space about it. 8567 */ 8568 if (ops->hotplug_seq) { 8569 global_hotplug_seq = atomic_long_read(&scx_hotplug_seq); 8570 if (ops->hotplug_seq != global_hotplug_seq) { 8571 scx_exit(sch, SCX_EXIT_UNREG_KERN, 8572 SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, 8573 "expected hotplug seq %llu did not match actual %llu", 8574 ops->hotplug_seq, global_hotplug_seq); 8575 return -EBUSY; 8576 } 8577 } 8578 8579 return 0; 8580 } 8581 8582 int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) 8583 { 8584 /* 8585 * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the 8586 * ops.enqueue() callback isn't implemented. 8587 */ 8588 if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) { 8589 scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); 8590 return -EINVAL; 8591 } 8592 8593 if ((ops->flags & SCX_OPS_ENQ_BLOCKED) && !ops->enqueue) { 8594 scx_error(sch, "SCX_OPS_ENQ_BLOCKED requires ops.enqueue() to be implemented"); 8595 return -EINVAL; 8596 } 8597 8598 /* 8599 * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched 8600 * may set it to declare a dependency; reject if the root hasn't 8601 * enabled it. 8602 */ 8603 if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && 8604 !(sch->ancestors[0]->ops.flags & SCX_OPS_TID_TO_TASK)) { 8605 scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); 8606 return -EINVAL; 8607 } 8608 8609 /* 8610 * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle 8611 * selection policy to be enabled. 8612 */ 8613 if ((ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) && 8614 (ops->update_idle && !(ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE))) { 8615 scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled"); 8616 return -EINVAL; 8617 } 8618 8619 /* 8620 * cid-form's struct is shorter and doesn't include the cpu_acquire / 8621 * cpu_release tail; reading those fields off a cid-form @ops would 8622 * run past the BPF allocation. Skip for cid-form. 8623 */ 8624 if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) 8625 pr_warn_ratelimited("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); 8626 8627 /* 8628 * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched 8629 * attaches through a cid-form-only interface (sub_attach/sub_detach), 8630 * and a root that accepts sub-scheds must expose cid-form state to 8631 * them. Reject cpu-form schedulers on either side. 8632 */ 8633 if (!sch->is_cid_type) { 8634 if (scx_parent(sch)) { 8635 scx_error(sch, "sub-sched requires cid-form struct_ops"); 8636 return -EINVAL; 8637 } 8638 if (ops->sub_attach || ops->sub_detach) { 8639 scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops"); 8640 return -EINVAL; 8641 } 8642 } 8643 8644 return 0; 8645 } 8646 8647 static void scx_root_enable_workfn(struct kthread_work *work) 8648 { 8649 struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); 8650 struct sched_ext_ops *ops = cmd->ops; 8651 struct cgroup *cgrp = root_cgroup(); 8652 struct scx_sched *sch; 8653 struct scx_task_iter sti; 8654 struct task_struct *p; 8655 int i, cpu, ret; 8656 8657 mutex_lock(&scx_enable_mutex); 8658 8659 if (scx_enable_state() != SCX_DISABLED) { 8660 ret = -EBUSY; 8661 goto err_unlock; 8662 } 8663 8664 /* 8665 * @ops->priv binds @ops to its scx_sched instance. It is set here by 8666 * scx_alloc_and_add_sched() and cleared at the tail of bpf_scx_unreg(), 8667 * which runs after scx_root_disable() has dropped scx_enable_mutex. If 8668 * it's still non-NULL here, a previous attachment on @ops has not 8669 * finished tearing down; proceeding would let the in-flight unreg's 8670 * RCU_INIT_POINTER(NULL) clobber the @ops->priv we are about to assign. 8671 */ 8672 if (rcu_access_pointer(ops->priv)) { 8673 ret = -EBUSY; 8674 goto err_unlock; 8675 } 8676 8677 ret = alloc_kick_syncs(); 8678 if (ret) 8679 goto err_unlock; 8680 8681 if (ops->flags & SCX_OPS_TID_TO_TASK) { 8682 ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params); 8683 if (ret) 8684 goto err_free_ksyncs; 8685 } 8686 8687 #ifdef CONFIG_EXT_SUB_SCHED 8688 cgroup_get(cgrp); 8689 #endif 8690 /* 8691 * Transition to ENABLING to arm the disable path. Allocation failure 8692 * still unwinds locally. Full disabling on failure applies only after 8693 * scx_alloc_and_add_sched() succeeds. 8694 */ 8695 WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); 8696 WARN_ON_ONCE(scx_root); 8697 8698 sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); 8699 if (IS_ERR(sch)) { 8700 ret = PTR_ERR(sch); 8701 WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING); 8702 goto err_free_tid_hash; 8703 } 8704 8705 if (sch->is_cid_type) 8706 static_branch_enable(&__scx_is_cid_type); 8707 8708 atomic_long_set(&scx_nr_rejected, 0); 8709 8710 for_each_possible_cpu(cpu) { 8711 struct rq *rq = cpu_rq(cpu); 8712 8713 rq->scx.local_dsq.sched = sch; 8714 rq->scx.cpuperf_target = SCX_CPUPERF_ONE; 8715 } 8716 8717 scx_discard_stale_ecaps_syncs(); 8718 scx_rescue_set_knobs(sch); 8719 8720 /* 8721 * Keep CPUs stable during enable so that the BPF scheduler can track 8722 * online CPUs by watching ->on/offline_cpu() after ->init(). 8723 */ 8724 cpus_read_lock(); 8725 8726 /* 8727 * Build the cid mapping into a private under-construction set. It 8728 * becomes visible to readers only through scx_cid_publish_tables() once 8729 * ops.init_cids() has finalized the layout. 8730 */ 8731 ret = scx_cid_init(sch); 8732 if (ret) { 8733 cpus_read_unlock(); 8734 goto err_disable; 8735 } 8736 8737 scx_init_rq_ops_online(); 8738 8739 /* 8740 * Make the scheduler instance visible. Must be inside cpus_read_lock(). 8741 * See handle_hotplug(). 8742 */ 8743 rcu_assign_pointer(scx_root, sch); 8744 8745 ret = scx_link_sched(sch); 8746 if (ret) { 8747 cpus_read_unlock(); 8748 goto err_disable; 8749 } 8750 8751 scx_idle_enable(ops); 8752 8753 /* 8754 * A cid-form scheduler finalizes its cid layout in ops.init_cids(), 8755 * which may call scx_bpf_cid_override(). Run it before the caps and 8756 * shard state are built so the final layout is in effect. 8757 */ 8758 if (sch->is_cid_type && sch->ops_cid.init_cids) { 8759 ret = SCX_CALL_OP_RET(sch, init_cids, NULL); 8760 if (ret) { 8761 ret = scx_ops_sanitize_err(sch, "init_cids", ret); 8762 cpus_read_unlock(); 8763 scx_error(sch, "ops.init_cids() failed (%d)", ret); 8764 goto err_disable; 8765 } 8766 } 8767 8768 /* the cid layout is final, expose it to readers */ 8769 scx_cid_publish_tables(); 8770 8771 ret = scx_arena_pool_init(sch); 8772 if (ret) { 8773 cpus_read_unlock(); 8774 goto err_disable; 8775 } 8776 8777 ret = scx_alloc_kern_arena_objs(sch); 8778 if (ret) { 8779 cpus_read_unlock(); 8780 goto err_disable; 8781 } 8782 8783 ret = scx_alloc_pshards(sch); 8784 if (ret) { 8785 cpus_read_unlock(); 8786 goto err_disable; 8787 } 8788 8789 scx_init_root_caps(sch); 8790 8791 /* the cid caps and shards are live now, so ops.init() can query them */ 8792 if (sch->ops.init) { 8793 ret = SCX_CALL_OP_RET(sch, init, NULL); 8794 if (ret) { 8795 ret = scx_ops_sanitize_err(sch, "init", ret); 8796 cpus_read_unlock(); 8797 scx_error(sch, "ops.init() failed (%d)", ret); 8798 goto err_disable; 8799 } 8800 sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; 8801 } 8802 8803 ret = scx_sched_sysfs_add(sch); 8804 if (ret) { 8805 cpus_read_unlock(); 8806 goto err_disable; 8807 } 8808 8809 for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) 8810 if (((void (**)(void))ops)[i]) 8811 set_bit(i, sch->has_op); 8812 8813 ret = check_hotplug_seq(sch, ops); 8814 if (ret) { 8815 cpus_read_unlock(); 8816 goto err_disable; 8817 } 8818 scx_idle_update_selcpu_topology(ops); 8819 8820 cpus_read_unlock(); 8821 8822 ret = scx_validate_ops(sch, ops); 8823 if (ret) 8824 goto err_disable; 8825 8826 /* 8827 * Attach the ext_server bandwidth reservation before anything is 8828 * committed so that we can fail the enable if the root domain cannot 8829 * accommodate it. The matching fair_server detach is deferred to the 8830 * tail of this function, after the switch is fully committed and can no 8831 * longer fail. 8832 * 8833 * On failure, err_disable funnels into scx_root_disable() which 8834 * detaches ext_server, so partially-attached state is cleaned up 8835 * automatically. 8836 */ 8837 for_each_possible_cpu(cpu) { 8838 struct rq *rq = cpu_rq(cpu); 8839 8840 scoped_guard(rq_lock_irqsave, rq) { 8841 update_rq_clock(rq); 8842 ret = dl_server_attach_bw(&rq->ext_server); 8843 } 8844 if (ret) { 8845 pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n", 8846 cpu, ret); 8847 goto err_disable; 8848 } 8849 } 8850 8851 /* 8852 * Once __scx_enabled is set, %current can be switched to SCX anytime. 8853 * This can lead to stalls as some BPF schedulers (e.g. userspace 8854 * scheduling) may not function correctly before all tasks are switched. 8855 * Init in bypass mode to guarantee forward progress. 8856 */ 8857 scx_bypass(sch, true); 8858 8859 for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) 8860 if (((void (**)(void))ops)[i]) 8861 set_bit(i, sch->has_op); 8862 8863 scx_ops_cid_sched_updated_enable(sch); 8864 8865 if (sch->ops.cpu_acquire || sch->ops.cpu_release) 8866 sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; 8867 8868 /* 8869 * Lock out forks, cgroup on/offlining and moves before opening the 8870 * floodgate so that they don't wander into the operations prematurely. 8871 */ 8872 percpu_down_write(&scx_fork_rwsem); 8873 8874 WARN_ON_ONCE(scx_init_task_enabled); 8875 scx_init_task_enabled = true; 8876 8877 /* flip under fork_rwsem; the iter below covers existing tasks */ 8878 if (ops->flags & SCX_OPS_TID_TO_TASK) 8879 static_branch_enable(&__scx_tid_to_task_enabled); 8880 8881 /* 8882 * Enable ops for every task. Fork is excluded by scx_fork_rwsem 8883 * preventing new tasks from being added. No need to exclude tasks 8884 * leaving as sched_ext_dead() can handle both prepped and enabled 8885 * tasks. Prep all tasks first and then enable them with preemption 8886 * disabled. 8887 * 8888 * All cgroups should be initialized before scx_init_task() so that the 8889 * BPF scheduler can reliably track each task's cgroup membership from 8890 * scx_init_task(). Lock out cgroup on/offlining and task migrations 8891 * while tasks are being initialized so that scx_cgroup_can_attach() 8892 * never sees uninitialized tasks. 8893 */ 8894 scx_cgroup_lock(); 8895 set_cgroup_sched(sch_cgroup(sch), sch); 8896 ret = scx_cgroup_init(sch); 8897 if (ret) 8898 goto err_disable_unlock_all; 8899 8900 WARN_ON_ONCE(scx_cgroup_enabled); 8901 scx_cgroup_enabled = true; 8902 8903 scx_task_iter_start(&sti, NULL); 8904 while ((p = scx_task_iter_next_locked(&sti))) { 8905 /* 8906 * @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD 8907 * tasks are filtered by scx_task_iter_next_locked(). 8908 * sched_ext_dead() removes @p from scx_tasks under the same 8909 * lock before put_task_struct_rcu_user() runs, so @p->usage 8910 * is guaranteed > 0 here. 8911 */ 8912 get_task_struct(p); 8913 8914 /* 8915 * Set %INIT_BEGIN under the iter's rq lock so that a concurrent 8916 * sched_ext_dead() does not call ops.exit_task() on @p while 8917 * ops.init_task() is running. If sched_ext_dead() runs before 8918 * this store, it has already removed @p from scx_tasks and the 8919 * iter won't visit @p; if it runs after, it observes 8920 * %INIT_BEGIN and transitions to %DEAD without calling ops, 8921 * leaving the post-init recheck below to unwind. 8922 */ 8923 scx_set_task_state(p, SCX_TASK_INIT_BEGIN); 8924 scx_task_iter_unlock(&sti); 8925 8926 ret = __scx_init_task(sch, p, NULL, false); 8927 8928 scx_task_iter_relock(&sti, p); 8929 8930 if (unlikely(ret)) { 8931 if (scx_get_task_state(p) != SCX_TASK_DEAD) 8932 scx_set_task_state(p, SCX_TASK_NONE); 8933 scx_task_iter_stop(&sti); 8934 scx_error(sch, "ops.init_task() failed (%d) for %s[%d]", 8935 ret, p->comm, p->pid); 8936 put_task_struct(p); 8937 goto err_disable_unlock_all; 8938 } 8939 8940 if (scx_get_task_state(p) == SCX_TASK_DEAD) { 8941 /* 8942 * sched_ext_dead() observed %INIT_BEGIN and set %DEAD. 8943 * ops.exit_task() is owed to the sched __scx_init_task() 8944 * ran against; call it now. 8945 */ 8946 scx_sub_init_cancel_task(sch, p); 8947 } else { 8948 scx_set_task_state(p, SCX_TASK_INIT); 8949 scx_set_task_sched(p, sch); 8950 scx_set_task_state(p, SCX_TASK_READY); 8951 } 8952 8953 /* 8954 * Insert into the tid hash. scx_tasks_lock is held by the iter; 8955 * list_empty() guards against sched_ext_dead() having taken @p 8956 * off the list while init ran unlocked. 8957 */ 8958 if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node)) 8959 scx_tid_hash_insert(p); 8960 8961 put_task_struct(p); 8962 } 8963 scx_task_iter_stop(&sti); 8964 scx_cgroup_unlock(); 8965 percpu_up_write(&scx_fork_rwsem); 8966 8967 /* 8968 * All tasks are READY. It's safe to turn on scx_enabled() and switch 8969 * all eligible tasks. 8970 */ 8971 WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); 8972 static_branch_enable(&__scx_enabled); 8973 8974 /* 8975 * We're fully committed and can't fail. The task READY -> ENABLED 8976 * transitions here are synchronized against sched_ext_dead() through 8977 * scx_tasks_lock. 8978 */ 8979 percpu_down_write(&scx_fork_rwsem); 8980 scx_task_iter_start(&sti, NULL); 8981 while ((p = scx_task_iter_next_locked(&sti))) { 8982 unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; 8983 const struct sched_class *old_class = p->sched_class; 8984 const struct sched_class *new_class = scx_setscheduler_class(p); 8985 8986 if (scx_get_task_state(p) != SCX_TASK_READY) 8987 continue; 8988 8989 if (old_class != new_class) 8990 queue_flags |= DEQUEUE_CLASS; 8991 if (new_class == &ext_sched_class) { 8992 scx_prepare_task_sched_change(p); 8993 queue_flags |= DEQUEUE_NOCLOCK; 8994 } 8995 8996 scoped_guard (sched_change, p, queue_flags) { 8997 scx_set_task_slice(p, READ_ONCE(sch->slice_dfl)); 8998 p->sched_class = new_class; 8999 } 9000 } 9001 scx_task_iter_stop(&sti); 9002 percpu_up_write(&scx_fork_rwsem); 9003 9004 scx_bypass(sch, false); 9005 9006 if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { 9007 WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); 9008 ret = -EBUSY; 9009 goto err_disable; 9010 } 9011 9012 if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) 9013 static_branch_enable(&__scx_switched_all); 9014 9015 /* 9016 * Detach the fair_server bandwidth reservation now that the switch 9017 * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no 9018 * task will ever run in the fair class, so give that bandwidth 9019 * back to the RT class. The matching ext_server attach already 9020 * happened earlier; this only releases bandwidth and cannot fail. 9021 * 9022 * In partial mode keep fair_server attached. 9023 */ 9024 if (scx_switched_all()) { 9025 for_each_possible_cpu(cpu) { 9026 struct rq *rq = cpu_rq(cpu); 9027 9028 guard(rq_lock_irqsave)(rq); 9029 update_rq_clock(rq); 9030 dl_server_detach_bw(&rq->fair_server); 9031 } 9032 } 9033 9034 pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", 9035 sch->ops.name, scx_switched_all() ? "" : " (partial)"); 9036 kobject_uevent(&sch->kobj, KOBJ_ADD); 9037 mutex_unlock(&scx_enable_mutex); 9038 9039 atomic_long_inc(&scx_enable_seq); 9040 9041 cmd->ret = 0; 9042 return; 9043 9044 err_free_tid_hash: 9045 if (ops->flags & SCX_OPS_TID_TO_TASK) 9046 rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); 9047 err_free_ksyncs: 9048 free_kick_syncs(); 9049 err_unlock: 9050 mutex_unlock(&scx_enable_mutex); 9051 cmd->ret = ret; 9052 return; 9053 9054 err_disable_unlock_all: 9055 scx_cgroup_unlock(); 9056 percpu_up_write(&scx_fork_rwsem); 9057 /* we'll soon enter disable path, keep bypass on */ 9058 err_disable: 9059 mutex_unlock(&scx_enable_mutex); 9060 /* 9061 * Returning an error code here would not pass all the error information 9062 * to userspace. Record errno using scx_error() for cases scx_error() 9063 * wasn't already invoked and exit indicating success so that the error 9064 * is notified through ops.exit() with all the details. 9065 * 9066 * Flush scx_disable_work to ensure that error is reported before init 9067 * completion. sch's base reference will be put by bpf_scx_unreg(). 9068 */ 9069 scx_error(sch, "scx_root_enable() failed (%d)", ret); 9070 scx_flush_disable_work(sch); 9071 cmd->ret = 0; 9072 } 9073 9074 static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link) 9075 { 9076 static struct kthread_worker *helper; 9077 static DEFINE_MUTEX(helper_mutex); 9078 9079 if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) { 9080 pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); 9081 return -EINVAL; 9082 } 9083 9084 if (!READ_ONCE(helper)) { 9085 mutex_lock(&helper_mutex); 9086 if (!helper) { 9087 struct kthread_worker *w = 9088 kthread_run_worker(0, "scx_enable_helper"); 9089 if (IS_ERR_OR_NULL(w)) { 9090 mutex_unlock(&helper_mutex); 9091 return -ENOMEM; 9092 } 9093 sched_set_fifo(w->task); 9094 WRITE_ONCE(helper, w); 9095 } 9096 mutex_unlock(&helper_mutex); 9097 } 9098 9099 #ifdef CONFIG_EXT_SUB_SCHED 9100 if (cmd->ops->sub_cgroup_id > 1) 9101 kthread_init_work(&cmd->work, scx_sub_enable_workfn); 9102 else 9103 #endif /* CONFIG_EXT_SUB_SCHED */ 9104 kthread_init_work(&cmd->work, scx_root_enable_workfn); 9105 9106 kthread_queue_work(READ_ONCE(helper), &cmd->work); 9107 kthread_flush_work(&cmd->work); 9108 return cmd->ret; 9109 } 9110 9111 9112 /******************************************************************************** 9113 * bpf_struct_ops plumbing. 9114 */ 9115 #include <linux/bpf_verifier.h> 9116 #include <linux/bpf.h> 9117 #include <linux/btf.h> 9118 9119 static const struct btf_type *task_struct_type; 9120 9121 static bool bpf_scx_is_valid_access(int off, int size, 9122 enum bpf_access_type type, 9123 const struct bpf_prog *prog, 9124 struct bpf_insn_access_aux *info) 9125 { 9126 if (type != BPF_READ) 9127 return false; 9128 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 9129 return false; 9130 if (off % size != 0) 9131 return false; 9132 9133 return btf_ctx_access(off, size, type, prog, info); 9134 } 9135 9136 /* common to both forms: only scx.disallow is writable */ 9137 static int bpf_scx_btf_struct_access_common(const struct bpf_reg_state *reg, 9138 int off, int size) 9139 { 9140 const struct btf_type *t; 9141 9142 t = btf_type_by_id(reg->btf, reg->btf_id); 9143 if (t == task_struct_type && 9144 off >= offsetof(struct task_struct, scx.disallow) && 9145 off + size <= offsetofend(struct task_struct, scx.disallow)) 9146 return SCALAR_VALUE; 9147 9148 return -EACCES; 9149 } 9150 9151 static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, 9152 const struct bpf_reg_state *reg, int off, 9153 int size) 9154 { 9155 const struct btf_type *t; 9156 9157 t = btf_type_by_id(reg->btf, reg->btf_id); 9158 if (t == task_struct_type) { 9159 if ((off >= offsetof(struct task_struct, scx.slice) && 9160 off + size <= offsetofend(struct task_struct, scx.slice)) || 9161 (off >= offsetof(struct task_struct, scx.dsq_vtime) && 9162 off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) 9163 return SCALAR_VALUE; 9164 } 9165 9166 return bpf_scx_btf_struct_access_common(reg, off, size); 9167 } 9168 9169 /* cid-form rejects direct slice and dsq_vtime writes in favor of the kfuncs */ 9170 static int bpf_scx_cid_btf_struct_access(struct bpf_verifier_log *log, 9171 const struct bpf_reg_state *reg, int off, 9172 int size) 9173 { 9174 return bpf_scx_btf_struct_access_common(reg, off, size); 9175 } 9176 9177 static const struct bpf_verifier_ops bpf_scx_verifier_ops = { 9178 .get_func_proto = bpf_base_func_proto, 9179 .is_valid_access = bpf_scx_is_valid_access, 9180 .btf_struct_access = bpf_scx_btf_struct_access, 9181 }; 9182 9183 static const struct bpf_verifier_ops bpf_scx_cid_verifier_ops = { 9184 .get_func_proto = bpf_base_func_proto, 9185 .is_valid_access = bpf_scx_is_valid_access, 9186 .btf_struct_access = bpf_scx_cid_btf_struct_access, 9187 }; 9188 9189 static int bpf_scx_init_member(const struct btf_type *t, 9190 const struct btf_member *member, 9191 void *kdata, const void *udata) 9192 { 9193 const struct sched_ext_ops *uops = udata; 9194 struct sched_ext_ops *ops = kdata; 9195 u32 moff = __btf_member_bit_offset(t, member) / 8; 9196 int ret; 9197 9198 switch (moff) { 9199 case offsetof(struct sched_ext_ops, dispatch_max_batch): 9200 if (*(u32 *)(udata + moff) > INT_MAX) 9201 return -E2BIG; 9202 ops->dispatch_max_batch = *(u32 *)(udata + moff); 9203 return 1; 9204 case offsetof(struct sched_ext_ops, flags): 9205 if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) 9206 return -EINVAL; 9207 ops->flags = *(u64 *)(udata + moff); 9208 return 1; 9209 case offsetof(struct sched_ext_ops, name): 9210 ret = bpf_obj_name_cpy(ops->name, uops->name, 9211 sizeof(ops->name)); 9212 if (ret < 0) 9213 return ret; 9214 if (ret == 0) 9215 return -EINVAL; 9216 return 1; 9217 case offsetof(struct sched_ext_ops, timeout_ms): 9218 if (msecs_to_jiffies(*(u32 *)(udata + moff)) > 9219 SCX_WATCHDOG_MAX_TIMEOUT) 9220 return -E2BIG; 9221 ops->timeout_ms = *(u32 *)(udata + moff); 9222 return 1; 9223 case offsetof(struct sched_ext_ops, exit_dump_len): 9224 ops->exit_dump_len = 9225 *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN; 9226 return 1; 9227 case offsetof(struct sched_ext_ops, hotplug_seq): 9228 ops->hotplug_seq = *(u64 *)(udata + moff); 9229 return 1; 9230 case offsetof(struct sched_ext_ops, cid_shard_size): 9231 ops->cid_shard_size = *(u32 *)(udata + moff); 9232 return 1; 9233 case offsetof(struct sched_ext_ops, rescue_bandwidth_ppt): { 9234 u32 bw_ppt = *(u32 *)(udata + moff); 9235 9236 if (bw_ppt > SCX_RESCUE_MAX_BW_PPT && bw_ppt != SCX_RESCUE_DISABLE) 9237 return -E2BIG; 9238 ops->rescue_bandwidth_ppt = bw_ppt; 9239 return 1; 9240 } 9241 case offsetof(struct sched_ext_ops, rescue_quantum_us): { 9242 u32 quantum_us = *(u32 *)(udata + moff); 9243 9244 if (quantum_us > SCX_RESCUE_MAX_QUANTUM_US) 9245 return -E2BIG; 9246 if (quantum_us && quantum_us < SCX_RESCUE_MIN_QUANTUM_US) 9247 return -EINVAL; 9248 ops->rescue_quantum_us = quantum_us; 9249 return 1; 9250 } 9251 #ifdef CONFIG_EXT_SUB_SCHED 9252 case offsetof(struct sched_ext_ops, sub_cgroup_id): 9253 ops->sub_cgroup_id = *(u64 *)(udata + moff); 9254 return 1; 9255 #endif /* CONFIG_EXT_SUB_SCHED */ 9256 } 9257 9258 return 0; 9259 } 9260 9261 static int bpf_scx_check_member(const struct btf_type *t, 9262 const struct btf_member *member, 9263 const struct bpf_prog *prog) 9264 { 9265 u32 moff = __btf_member_bit_offset(t, member) / 8; 9266 9267 switch (moff) { 9268 case offsetof(struct sched_ext_ops, init_task): 9269 #ifdef CONFIG_EXT_GROUP_SCHED 9270 case offsetof(struct sched_ext_ops, cgroup_init): 9271 case offsetof(struct sched_ext_ops, cgroup_exit): 9272 case offsetof(struct sched_ext_ops, cgroup_prep_move): 9273 case offsetof(struct sched_ext_ops, cgroup_set_bandwidth): 9274 #endif 9275 case offsetof(struct sched_ext_ops, cpu_online): 9276 case offsetof(struct sched_ext_ops, cpu_offline): 9277 case offsetof(struct sched_ext_ops, init_cids): 9278 case offsetof(struct sched_ext_ops, init): 9279 case offsetof(struct sched_ext_ops, exit): 9280 case offsetof(struct sched_ext_ops, sub_attach): 9281 case offsetof(struct sched_ext_ops, sub_detach): 9282 break; 9283 default: 9284 if (prog->sleepable) 9285 return -EINVAL; 9286 } 9287 9288 #ifdef CONFIG_EXT_SUB_SCHED 9289 /* 9290 * Enable private stack for operations that can nest along the 9291 * hierarchy. 9292 * 9293 * XXX - Ideally, we should only do this for scheds that allow 9294 * sub-scheds and sub-scheds themselves but I don't know how to access 9295 * struct_ops from here. 9296 */ 9297 switch (moff) { 9298 case offsetof(struct sched_ext_ops, dispatch): 9299 prog->aux->priv_stack_requested = true; 9300 prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch; 9301 break; 9302 case offsetof(struct sched_ext_ops, sub_caps_updated): 9303 prog->aux->priv_stack_requested = true; 9304 prog->aux->recursion_detected = scx_pstack_recursion_on_caps_updated; 9305 break; 9306 } 9307 #endif /* CONFIG_EXT_SUB_SCHED */ 9308 9309 return 0; 9310 } 9311 9312 static int bpf_scx_reg(void *kdata, struct bpf_link *link) 9313 { 9314 struct scx_enable_cmd cmd = { .ops = kdata }; 9315 9316 return scx_enable(&cmd, link); 9317 } 9318 9319 struct scx_arena_scan { 9320 struct bpf_map *arena; 9321 int err; 9322 }; 9323 9324 /* 9325 * The verifier enforces one arena per BPF program, so each struct_ops 9326 * member prog contributes at most one arena via bpf_prog_arena(). 9327 * Require all non-NULL contributions to match. 9328 */ 9329 static int scx_arena_scan_prog(struct bpf_prog *prog, void *data) 9330 { 9331 struct scx_arena_scan *s = data; 9332 struct bpf_map *arena = NULL; 9333 9334 /* arena.o, which defines these, is built only on MMU && 64BIT */ 9335 #if defined(CONFIG_MMU) && defined(CONFIG_64BIT) 9336 arena = bpf_prog_arena(prog); 9337 #endif 9338 if (!arena) 9339 return 0; 9340 if (s->arena && s->arena != arena) { 9341 s->err = -EINVAL; 9342 return 1; 9343 } 9344 s->arena = arena; 9345 return 0; 9346 } 9347 9348 static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link) 9349 { 9350 struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true }; 9351 struct scx_arena_scan scan = {}; 9352 int ret; 9353 9354 bpf_struct_ops_for_each_prog(kdata, scx_arena_scan_prog, &scan); 9355 if (scan.err) { 9356 pr_err("sched_ext: cid-form scheduler uses multiple arena maps\n"); 9357 return scan.err; 9358 } 9359 if (!scan.arena) { 9360 pr_err("sched_ext: cid-form scheduler must use a BPF arena map\n"); 9361 return -EINVAL; 9362 } 9363 9364 bpf_map_inc(scan.arena); 9365 cmd.arena_map = scan.arena; 9366 ret = scx_enable(&cmd, link); 9367 if (cmd.arena_map) /* not consumed by scx_alloc_and_add_sched() */ 9368 bpf_map_put(cmd.arena_map); 9369 return ret; 9370 } 9371 9372 static void bpf_scx_unreg(void *kdata, struct bpf_link *link) 9373 { 9374 struct sched_ext_ops *ops = kdata; 9375 struct scx_sched *sch = rcu_dereference_protected(ops->priv, true); 9376 9377 scx_disable(sch, SCX_EXIT_UNREG); 9378 scx_flush_disable_work(sch); 9379 RCU_INIT_POINTER(ops->priv, NULL); 9380 kobject_put(&sch->kobj); 9381 } 9382 9383 static int bpf_scx_init(struct btf *btf) 9384 { 9385 task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]); 9386 9387 return 0; 9388 } 9389 9390 static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link) 9391 { 9392 /* 9393 * sched_ext does not support updating the actively-loaded BPF 9394 * scheduler, as registering a BPF scheduler can always fail if the 9395 * scheduler returns an error code for e.g. ops.init(), ops.init_task(), 9396 * etc. Similarly, we can always race with unregistration happening 9397 * elsewhere, such as with sysrq. 9398 */ 9399 return -EOPNOTSUPP; 9400 } 9401 9402 static int bpf_scx_validate(void *kdata) 9403 { 9404 return 0; 9405 } 9406 9407 static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; } 9408 static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {} 9409 static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {} 9410 static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {} 9411 static void sched_ext_ops__tick(struct task_struct *p) {} 9412 static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {} 9413 static void sched_ext_ops__running(struct task_struct *p) {} 9414 static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {} 9415 static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {} 9416 static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; } 9417 static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; } 9418 static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {} 9419 static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {} 9420 static void sched_ext_ops__update_idle(s32 cpu, bool idle) {} 9421 static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {} 9422 static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {} 9423 static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; } 9424 static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {} 9425 static void sched_ext_ops__enable(struct task_struct *p) {} 9426 static void sched_ext_ops__disable(struct task_struct *p) {} 9427 #ifdef CONFIG_EXT_GROUP_SCHED 9428 static s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; } 9429 static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {} 9430 static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; } 9431 static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} 9432 static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} 9433 static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {} 9434 static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {} 9435 static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {} 9436 #endif /* CONFIG_EXT_GROUP_SCHED */ 9437 static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return -EINVAL; } 9438 static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {} 9439 static void sched_ext_ops__cpu_online(s32 cpu) {} 9440 static void sched_ext_ops__cpu_offline(s32 cpu) {} 9441 static s32 sched_ext_ops__init_cids(void) { return -EINVAL; } 9442 static s32 sched_ext_ops__init(void) { return -EINVAL; } 9443 static void sched_ext_ops__exit(struct scx_exit_info *info) {} 9444 static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {} 9445 static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {} 9446 static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {} 9447 9448 static struct sched_ext_ops __bpf_ops_sched_ext_ops = { 9449 .select_cpu = sched_ext_ops__select_cpu, 9450 .enqueue = sched_ext_ops__enqueue, 9451 .dequeue = sched_ext_ops__dequeue, 9452 .dispatch = sched_ext_ops__dispatch, 9453 .tick = sched_ext_ops__tick, 9454 .runnable = sched_ext_ops__runnable, 9455 .running = sched_ext_ops__running, 9456 .stopping = sched_ext_ops__stopping, 9457 .quiescent = sched_ext_ops__quiescent, 9458 .yield = sched_ext_ops__yield, 9459 .core_sched_before = sched_ext_ops__core_sched_before, 9460 .set_weight = sched_ext_ops__set_weight, 9461 .set_cpumask = sched_ext_ops__set_cpumask, 9462 .update_idle = sched_ext_ops__update_idle, 9463 .cpu_acquire = sched_ext_ops__cpu_acquire, 9464 .cpu_release = sched_ext_ops__cpu_release, 9465 .init_task = sched_ext_ops__init_task, 9466 .exit_task = sched_ext_ops__exit_task, 9467 .enable = sched_ext_ops__enable, 9468 .disable = sched_ext_ops__disable, 9469 #ifdef CONFIG_EXT_GROUP_SCHED 9470 .cgroup_init = sched_ext_ops__cgroup_init, 9471 .cgroup_exit = sched_ext_ops__cgroup_exit, 9472 .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, 9473 .cgroup_move = sched_ext_ops__cgroup_move, 9474 .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, 9475 .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, 9476 .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, 9477 .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, 9478 #endif 9479 .sub_attach = sched_ext_ops__sub_attach, 9480 .sub_detach = sched_ext_ops__sub_detach, 9481 .cpu_online = sched_ext_ops__cpu_online, 9482 .cpu_offline = sched_ext_ops__cpu_offline, 9483 .init_cids = sched_ext_ops__init_cids, 9484 .init = sched_ext_ops__init, 9485 .exit = sched_ext_ops__exit, 9486 .dump = sched_ext_ops__dump, 9487 .dump_cpu = sched_ext_ops__dump_cpu, 9488 .dump_task = sched_ext_ops__dump_task, 9489 }; 9490 9491 static struct bpf_struct_ops bpf_sched_ext_ops = { 9492 .verifier_ops = &bpf_scx_verifier_ops, 9493 .reg = bpf_scx_reg, 9494 .unreg = bpf_scx_unreg, 9495 .check_member = bpf_scx_check_member, 9496 .init_member = bpf_scx_init_member, 9497 .init = bpf_scx_init, 9498 .update = bpf_scx_update, 9499 .validate = bpf_scx_validate, 9500 .name = "sched_ext_ops", 9501 .owner = THIS_MODULE, 9502 .cfi_stubs = &__bpf_ops_sched_ext_ops 9503 }; 9504 9505 /* 9506 * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types 9507 * identical, only param names differ across structs) are reused. Some need 9508 * fresh stubs, set_cmask and enable due to argument differences and the 9509 * sub-sched notifiers because no cpu-form stub exists to reuse. 9510 */ 9511 static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {} 9512 static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {} 9513 static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} 9514 static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} 9515 static void sched_ext_ops__sub_cid_sched_updated(s32 cid, u64 sched) {} 9516 static void sched_ext_ops__sub_child_ecaps_updated(u64 cgroup_id, s32 cid, u64 before, u64 after) {} 9517 9518 static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { 9519 .select_cid = sched_ext_ops__select_cpu, 9520 .enqueue = sched_ext_ops__enqueue, 9521 .dequeue = sched_ext_ops__dequeue, 9522 .dispatch = sched_ext_ops__dispatch, 9523 .tick = sched_ext_ops__tick, 9524 .runnable = sched_ext_ops__runnable, 9525 .running = sched_ext_ops__running, 9526 .stopping = sched_ext_ops__stopping, 9527 .quiescent = sched_ext_ops__quiescent, 9528 .yield = sched_ext_ops__yield, 9529 .core_sched_before = sched_ext_ops__core_sched_before, 9530 .set_weight = sched_ext_ops__set_weight, 9531 .set_cmask = sched_ext_ops_cid__set_cmask, 9532 .update_idle = sched_ext_ops__update_idle, 9533 .init_task = sched_ext_ops__init_task, 9534 .exit_task = sched_ext_ops__exit_task, 9535 .enable = sched_ext_ops_cid__enable, 9536 .disable = sched_ext_ops__disable, 9537 #ifdef CONFIG_EXT_GROUP_SCHED 9538 .cpuctl_init = sched_ext_ops__cgroup_init, 9539 .cpuctl_exit = sched_ext_ops__cgroup_exit, 9540 .cpuctl_prep_move = sched_ext_ops__cgroup_prep_move, 9541 .cpuctl_move = sched_ext_ops__cgroup_move, 9542 .cpuctl_cancel_move = sched_ext_ops__cgroup_cancel_move, 9543 .cpuctl_set_weight = sched_ext_ops__cgroup_set_weight, 9544 .cpuctl_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, 9545 .cpuctl_set_idle = sched_ext_ops__cgroup_set_idle, 9546 #endif 9547 .sub_attach = sched_ext_ops__sub_attach, 9548 .sub_detach = sched_ext_ops__sub_detach, 9549 .sub_caps_updated = sched_ext_ops__sub_caps_updated, 9550 .sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated, 9551 .sub_cid_sched_updated = sched_ext_ops__sub_cid_sched_updated, 9552 .sub_child_ecaps_updated = sched_ext_ops__sub_child_ecaps_updated, 9553 .cid_online = sched_ext_ops__cpu_online, 9554 .cid_offline = sched_ext_ops__cpu_offline, 9555 .init_cids = sched_ext_ops__init_cids, 9556 .init = sched_ext_ops__init, 9557 .exit = sched_ext_ops__exit, 9558 .dump = sched_ext_ops__dump, 9559 .dump_cid = sched_ext_ops__dump_cpu, 9560 .dump_task = sched_ext_ops__dump_task, 9561 }; 9562 9563 /* 9564 * The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form. 9565 * init_member, check_member, reg, unreg, etc. process kdata as the byte block 9566 * verified to match by the BUILD_BUG_ON checks in scx_init(). 9567 */ 9568 static struct bpf_struct_ops bpf_sched_ext_ops_cid = { 9569 .verifier_ops = &bpf_scx_cid_verifier_ops, 9570 .reg = bpf_scx_reg_cid, 9571 .unreg = bpf_scx_unreg, 9572 .check_member = bpf_scx_check_member, 9573 .init_member = bpf_scx_init_member, 9574 .init = bpf_scx_init, 9575 .update = bpf_scx_update, 9576 .validate = bpf_scx_validate, 9577 .name = "sched_ext_ops_cid", 9578 .owner = THIS_MODULE, 9579 .cfi_stubs = &__bpf_ops_sched_ext_ops_cid 9580 }; 9581 9582 9583 /******************************************************************************** 9584 * System integration and init. 9585 */ 9586 9587 static void sysrq_handle_sched_ext_reset(u8 key) 9588 { 9589 struct scx_sched *sch; 9590 9591 sch = rcu_dereference(scx_root); 9592 if (likely(sch)) 9593 scx_disable(sch, SCX_EXIT_SYSRQ); 9594 else 9595 pr_info("sched_ext: BPF schedulers not loaded\n"); 9596 } 9597 9598 static const struct sysrq_key_op sysrq_sched_ext_reset_op = { 9599 .handler = sysrq_handle_sched_ext_reset, 9600 .help_msg = "reset-sched-ext(S)", 9601 .action_msg = "Disable sched_ext and revert all tasks to CFS", 9602 .enable_mask = SYSRQ_ENABLE_RTNICE, 9603 }; 9604 9605 static void sysrq_handle_sched_ext_dump(u8 key) 9606 { 9607 struct scx_exit_info ei = { 9608 .kind = SCX_EXIT_NONE, 9609 .exit_cpu = -1, 9610 .reason = "SysRq-D", 9611 }; 9612 struct scx_sched *sch; 9613 9614 list_for_each_entry_rcu(sch, &scx_sched_all, all) 9615 scx_dump_state(sch, &ei, 0, false); 9616 } 9617 9618 static const struct sysrq_key_op sysrq_sched_ext_dump_op = { 9619 .handler = sysrq_handle_sched_ext_dump, 9620 .help_msg = "dump-sched-ext(D)", 9621 .action_msg = "Trigger sched_ext debug dump", 9622 .enable_mask = SYSRQ_ENABLE_RTNICE, 9623 }; 9624 9625 static bool can_skip_idle_kick(struct rq *rq) 9626 { 9627 lockdep_assert_rq_held(rq); 9628 9629 /* 9630 * We can skip idle kicking if @rq is going to go through at least one 9631 * full SCX scheduling cycle before going idle. Just checking whether 9632 * curr is not idle is insufficient because we could be racing 9633 * dispatch_one() trying to pull the next task from a remote rq, which 9634 * may fail, and @rq may become idle afterwards. 9635 * 9636 * The race window is small and we don't and can't guarantee that @rq is 9637 * only kicked while idle anyway. Skip only when sure. 9638 */ 9639 return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_DISPATCH); 9640 } 9641 9642 static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_rq, 9643 unsigned long *ksyncs) 9644 { 9645 struct rq *rq = cpu_rq(cpu); 9646 struct scx_rq *this_scx = &this_rq->scx; 9647 struct rq_flags rf; 9648 const struct sched_class *cur_class; 9649 bool should_wait = false; 9650 bool kickable; 9651 bool preempt, preempt_lazy, wait, immediate; 9652 9653 rq_lock_irqsave(rq, &rf); 9654 cur_class = rq->donor->sched_class; 9655 preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt); 9656 preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy); 9657 wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait); 9658 /* 9659 * Immediate preemption, waiting and a plain kick take precedence over 9660 * lazy preemption. The lazy request still clears the slice, so all 9661 * accumulated requests are served. 9662 */ 9663 immediate = preempt || wait || cpumask_test_cpu(cpu, pcpu->cpus_to_kick); 9664 9665 /* 9666 * During CPU hotplug, a CPU may depend on kicking itself to make 9667 * forward progress. Allow kicking self regardless of online state. If 9668 * @cpu is running a higher class task, we have no control over @cpu. 9669 * Skip kicking. A sub-sched lacking baseline access on @cid has no 9670 * business forcing a reschedule there - skip. This is the authoritative 9671 * cap check: ecaps is read here under @rq's lock. 9672 */ 9673 kickable = (cpu_online(cpu) || cpu == cpu_of(this_rq)) && 9674 !sched_class_above(cur_class, &ext_sched_class); 9675 9676 if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { 9677 if (preempt || preempt_lazy) { 9678 if (cur_class == &ext_sched_class) { 9679 u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); 9680 9681 if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) { 9682 __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); 9683 /* degrade to a plain, immediate kick */ 9684 immediate = true; 9685 } else if (unlikely(!scx_set_task_slice(rq->donor, 0))) { 9686 __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); 9687 } 9688 } 9689 cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); 9690 cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); 9691 } 9692 9693 if (wait) { 9694 if (cur_class == &ext_sched_class) { 9695 cpumask_set_cpu(cpu, this_scx->cpus_to_sync); 9696 ksyncs[cpu] = rq->scx.kick_sync; 9697 should_wait = true; 9698 } 9699 cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); 9700 } 9701 9702 if (immediate) 9703 resched_curr(rq); 9704 else 9705 scx_resched_curr_lazy(rq); 9706 } else { 9707 /* a kickable cpu was skipped solely for the missing caps */ 9708 if (kickable) 9709 __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); 9710 cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); 9711 cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); 9712 cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); 9713 } 9714 9715 scx_rq_lock_drop(rq); 9716 rq_unlock_irqrestore(rq, &rf); 9717 9718 return should_wait; 9719 } 9720 9721 static void kick_one_cpu_if_idle(s32 cpu, struct scx_sched_pcpu *pcpu, 9722 struct rq *this_rq) 9723 { 9724 struct rq *rq = cpu_rq(cpu); 9725 unsigned long flags; 9726 9727 raw_spin_rq_lock_irqsave(rq, flags); 9728 9729 /* idle kicks need baseline access too, see kick_one_cpu() */ 9730 if (!can_skip_idle_kick(rq) && 9731 (cpu_online(cpu) || cpu == cpu_of(this_rq))) { 9732 if (likely(!scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE))) 9733 resched_curr(rq); 9734 else 9735 __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); 9736 } 9737 9738 scx_rq_lock_drop(rq); 9739 raw_spin_rq_unlock_irqrestore(rq, flags); 9740 } 9741 9742 static void kick_cpus_irq_workfn(struct irq_work *irq_work) 9743 { 9744 struct rq *this_rq = this_rq(); 9745 struct scx_rq *this_scx = &this_rq->scx; 9746 struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); 9747 struct scx_sched_pcpu *pcpu, *tmp; 9748 bool should_wait = false; 9749 unsigned long *ksyncs; 9750 s32 cpu; 9751 9752 /* can race with free_kick_syncs() during scheduler disable */ 9753 if (unlikely(!ksyncs_pcpu)) 9754 return; 9755 9756 ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs; 9757 9758 /* 9759 * Walk scheds with pending kicks on this cpu. scx_kick_cpu() adds to 9760 * the list under local_irq_save() and only this irq_work consumes it. 9761 * A plain list without locking is sufficient. 9762 */ 9763 list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) { 9764 list_del_init(&pcpu->to_kick_node); 9765 9766 for_each_cpu_or(cpu, pcpu->cpus_to_kick, pcpu->cpus_to_preempt_lazy) { 9767 should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs); 9768 cpumask_clear_cpu(cpu, pcpu->cpus_to_kick); 9769 cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); 9770 } 9771 9772 for_each_cpu(cpu, pcpu->cpus_to_kick_if_idle) { 9773 kick_one_cpu_if_idle(cpu, pcpu, this_rq); 9774 cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); 9775 } 9776 } 9777 9778 /* 9779 * Can't wait in hardirq — kick_sync can't advance, deadlocking if 9780 * CPUs wait for each other. Defer to kick_sync_wait_bal_cb(). 9781 */ 9782 if (should_wait) { 9783 raw_spin_rq_lock(this_rq); 9784 this_scx->kick_sync_pending = true; 9785 resched_curr(this_rq); 9786 scx_rq_lock_drop(this_rq); 9787 raw_spin_rq_unlock(this_rq); 9788 } 9789 } 9790 9791 /** 9792 * print_scx_info - print out sched_ext scheduler state 9793 * @log_lvl: the log level to use when printing 9794 * @p: target task 9795 * 9796 * If a sched_ext scheduler is enabled, print the name and state of the 9797 * scheduler. If @p is on sched_ext, print further information about the task. 9798 * 9799 * This function can be safely called on any task as long as the task_struct 9800 * itself is accessible. While safe, this function isn't synchronized and may 9801 * print out mixups or garbages of limited length. 9802 */ 9803 void print_scx_info(const char *log_lvl, struct task_struct *p) 9804 { 9805 struct scx_sched *sch; 9806 enum scx_enable_state state = scx_enable_state(); 9807 const char *all = READ_ONCE(scx_switching_all) ? "+all" : ""; 9808 char runnable_at_buf[22] = "?"; 9809 struct sched_class *class; 9810 unsigned long runnable_at; 9811 9812 guard(rcu)(); 9813 9814 sch = scx_task_sched_rcu(p); 9815 9816 if (!sch) 9817 return; 9818 9819 /* 9820 * Carefully check if the task was running on sched_ext, and then 9821 * carefully copy the time it's been runnable, and its state. 9822 */ 9823 if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) || 9824 class != &ext_sched_class) { 9825 printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name, 9826 scx_enable_state_str[state], all); 9827 return; 9828 } 9829 9830 if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, 9831 sizeof(runnable_at))) 9832 scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms", 9833 jiffies_delta_msecs(runnable_at, jiffies)); 9834 9835 /* print everything onto one line to conserve console space */ 9836 printk("%sSched_ext: %s (%s%s), task: runnable_at=%s", 9837 log_lvl, sch->ops.name, scx_enable_state_str[state], all, 9838 runnable_at_buf); 9839 } 9840 9841 static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr) 9842 { 9843 struct scx_sched *sch; 9844 9845 /* track the PM cycle whether or not a root is loaded */ 9846 switch (event) { 9847 case PM_HIBERNATION_PREPARE: 9848 case PM_SUSPEND_PREPARE: 9849 case PM_RESTORE_PREPARE: 9850 scx_sub_pm_begin(); 9851 break; 9852 case PM_POST_HIBERNATION: 9853 case PM_POST_SUSPEND: 9854 case PM_POST_RESTORE: 9855 scx_sub_pm_end(); 9856 break; 9857 } 9858 9859 guard(rcu)(); 9860 9861 sch = rcu_dereference(scx_root); 9862 if (!sch) 9863 return NOTIFY_OK; 9864 9865 /* 9866 * SCX schedulers often have userspace components which are sometimes 9867 * involved in critial scheduling paths. PM operations involve freezing 9868 * userspace which can lead to scheduling misbehaviors including stalls. 9869 * Let's bypass while PM operations are in progress. 9870 */ 9871 switch (event) { 9872 case PM_HIBERNATION_PREPARE: 9873 case PM_SUSPEND_PREPARE: 9874 case PM_RESTORE_PREPARE: 9875 scx_bypass(sch, true); 9876 break; 9877 case PM_POST_HIBERNATION: 9878 case PM_POST_SUSPEND: 9879 case PM_POST_RESTORE: 9880 scx_bypass(sch, false); 9881 break; 9882 } 9883 9884 return NOTIFY_OK; 9885 } 9886 9887 static struct notifier_block scx_pm_notifier = { 9888 .notifier_call = scx_pm_handler, 9889 }; 9890 9891 void __init init_sched_ext_class(void) 9892 { 9893 s32 cpu, v; 9894 9895 /* 9896 * The following is to prevent the compiler from optimizing out the enum 9897 * definitions so that BPF scheduler implementations can use them 9898 * through the generated vmlinux.h. 9899 */ 9900 WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT | 9901 SCX_TG_ONLINE); 9902 9903 scx_idle_init_masks(); 9904 9905 for_each_possible_cpu(cpu) { 9906 struct rq *rq = cpu_rq(cpu); 9907 int n = cpu_to_node(cpu); 9908 9909 /* local_dsq's sch will be set during scx_root_enable() */ 9910 BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); 9911 BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); 9912 #ifdef CONFIG_EXT_SUB_SCHED 9913 scx_rescue_init(rq); 9914 #endif 9915 9916 INIT_LIST_HEAD(&rq->scx.runnable_list); 9917 INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); 9918 9919 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); 9920 INIT_LIST_HEAD(&rq->scx.sched_pcpus_to_kick); 9921 raw_spin_lock_init(&rq->scx.deferred_reenq_lock); 9922 INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals); 9923 INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); 9924 rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); 9925 rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); 9926 rq->scx.resched_irq_work = IRQ_WORK_INIT_HARD(resched_irq_workfn); 9927 9928 if (cpu_online(cpu)) 9929 cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; 9930 } 9931 9932 register_sysrq_key('S', &sysrq_sched_ext_reset_op); 9933 register_sysrq_key('D', &sysrq_sched_ext_dump_op); 9934 INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); 9935 9936 #ifdef CONFIG_EXT_SUB_SCHED 9937 BUG_ON(rhashtable_init(&scx_sched_hash, &scx_sched_hash_params)); 9938 #endif /* CONFIG_EXT_SUB_SCHED */ 9939 } 9940 9941 9942 /******************************************************************************** 9943 * Helpers that can be called from the BPF scheduler. 9944 */ 9945 static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) 9946 { 9947 bool is_local = dsq_id == SCX_DSQ_LOCAL || 9948 (dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON; 9949 9950 if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { 9951 scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); 9952 return false; 9953 } 9954 9955 if (*enq_flags & SCX_ENQ_IMMED) { 9956 if (unlikely(!is_local)) { 9957 scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id); 9958 return false; 9959 } 9960 } else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) { 9961 *enq_flags |= SCX_ENQ_IMMED; 9962 } 9963 9964 if (unlikely((*enq_flags & SCX_ENQ_RESCUE) && !is_local)) { 9965 scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id); 9966 return false; 9967 } 9968 9969 return true; 9970 } 9971 9972 static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p, 9973 u64 dsq_id, u64 *enq_flags) 9974 { 9975 lockdep_assert_irqs_disabled(); 9976 9977 if (unlikely(!p)) { 9978 scx_error(sch, "called with NULL task"); 9979 return false; 9980 } 9981 9982 /* see SCX_EV_INSERT_NOT_OWNED definition */ 9983 if (unlikely(!scx_task_on_sched(sch, p))) { 9984 __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); 9985 return false; 9986 } 9987 9988 if (!scx_vet_enq_flags(sch, dsq_id, enq_flags)) 9989 return false; 9990 9991 return true; 9992 } 9993 9994 static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, 9995 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) 9996 { 9997 struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; 9998 struct task_struct *ddsp_task; 9999 10000 ddsp_task = __this_cpu_read(direct_dispatch_task); 10001 if (ddsp_task) { 10002 mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags); 10003 return; 10004 } 10005 10006 if (unlikely(dspc->cursor >= sch->dsp_max_batch)) { 10007 scx_error(sch, "dispatch buffer overflow"); 10008 return; 10009 } 10010 10011 dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){ 10012 .task = p, 10013 .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, 10014 .dsq_id = dsq_id, 10015 .slice = slice, 10016 .vtime = vtime, 10017 .enq_flags = enq_flags, 10018 }; 10019 } 10020 10021 __bpf_kfunc_start_defs(); 10022 10023 /** 10024 * scx_bpf_dsq_insert___v2 - Insert a task into the FIFO queue of a DSQ 10025 * @p: task_struct to insert 10026 * @dsq_id: DSQ to insert into 10027 * @slice: duration @p can run for in nsecs, 0 to keep the current value 10028 * @enq_flags: SCX_ENQ_* 10029 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10030 * 10031 * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to 10032 * call this function spuriously. Can be called from ops.enqueue(), 10033 * ops.select_cpu(), and ops.dispatch(). 10034 * 10035 * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch 10036 * and @p must match the task being enqueued. 10037 * 10038 * When called from ops.select_cpu(), @enq_flags and @dsq_id are stored, and @p 10039 * will be directly inserted into the corresponding dispatch queue after 10040 * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be 10041 * inserted into the local DSQ of the CPU returned by ops.select_cpu(). 10042 * @enq_flags are OR'd with the enqueue flags on the enqueue path before the 10043 * task is inserted. 10044 * 10045 * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id 10046 * and this function can be called up to ops.dispatch_max_batch times to insert 10047 * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the 10048 * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the 10049 * counter. 10050 * 10051 * This function doesn't have any locking restrictions and may be called under 10052 * BPF locks (in the future when BPF introduces more flexible locking). 10053 * 10054 * @p is allowed to run for @slice. The scheduling path is triggered on slice 10055 * exhaustion. If zero, the current residual slice is maintained. If 10056 * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with 10057 * scx_bpf_kick_cpu() to trigger scheduling. 10058 * 10059 * Returns %true on successful insertion, %false on failure. On the root 10060 * scheduler, %false return triggers scheduler abort and the caller doesn't need 10061 * to check the return value. 10062 */ 10063 __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, 10064 u64 slice, u64 enq_flags, 10065 const struct bpf_prog_aux *aux) 10066 { 10067 struct scx_sched *sch; 10068 10069 guard(rcu)(); 10070 sch = scx_prog_sched(aux); 10071 if (unlikely(!sch)) 10072 return false; 10073 10074 if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) 10075 return false; 10076 10077 scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags); 10078 10079 return true; 10080 } 10081 10082 /* 10083 * COMPAT: Will be removed in v6.23 along with the ___v2 suffix. 10084 */ 10085 __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, 10086 u64 slice, u64 enq_flags, 10087 const struct bpf_prog_aux *aux) 10088 { 10089 scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux); 10090 } 10091 10092 static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, 10093 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) 10094 { 10095 if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) 10096 return false; 10097 10098 scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ); 10099 10100 return true; 10101 } 10102 10103 struct scx_bpf_dsq_insert_vtime_args { 10104 /* @p can't be packed together as KF_RCU is not transitive */ 10105 u64 dsq_id; 10106 u64 slice; 10107 u64 vtime; 10108 u64 enq_flags; 10109 }; 10110 10111 /** 10112 * __scx_bpf_dsq_insert_vtime - Arg-wrapped vtime DSQ insertion 10113 * @p: task_struct to insert 10114 * @args: struct containing the rest of the arguments 10115 * @args->dsq_id: DSQ to insert into 10116 * @args->slice: duration @p can run for in nsecs, 0 to keep the current value 10117 * @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ 10118 * @args->enq_flags: SCX_ENQ_* 10119 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10120 * 10121 * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument 10122 * limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided 10123 * as an inline wrapper in common.bpf.h. 10124 * 10125 * Insert @p into the vtime priority queue of the DSQ identified by 10126 * @args->dsq_id. Tasks queued into the priority queue are ordered by 10127 * @args->vtime. All other aspects are identical to scx_bpf_dsq_insert(). 10128 * 10129 * @args->vtime ordering is according to time_before64() which considers 10130 * wrapping. A numerically larger vtime may indicate an earlier position in the 10131 * ordering and vice-versa. vtime is a rolling cursor and values used for 10132 * ordering within a given DSQ should stay less than 2^63 apart for 10133 * time_before64() ordering to remain well-defined. 10134 * 10135 * A DSQ can only be used as a FIFO or priority queue at any given time and this 10136 * function must not be called on a DSQ which already has one or more FIFO tasks 10137 * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and 10138 * SCX_DSQ_GLOBAL) cannot be used as priority queues. 10139 * 10140 * Returns %true on successful insertion, %false on failure. On the root 10141 * scheduler, %false return triggers scheduler abort and the caller doesn't need 10142 * to check the return value. 10143 */ 10144 __bpf_kfunc bool 10145 __scx_bpf_dsq_insert_vtime(struct task_struct *p, 10146 struct scx_bpf_dsq_insert_vtime_args *args, 10147 const struct bpf_prog_aux *aux) 10148 { 10149 struct scx_sched *sch; 10150 10151 guard(rcu)(); 10152 10153 sch = scx_prog_sched(aux); 10154 if (unlikely(!sch)) 10155 return false; 10156 10157 return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice, 10158 args->vtime, args->enq_flags); 10159 } 10160 10161 /* 10162 * COMPAT: Will be removed in v6.23. 10163 */ 10164 __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, 10165 u64 slice, u64 vtime, u64 enq_flags) 10166 { 10167 struct scx_sched *sch; 10168 10169 guard(rcu)(); 10170 10171 sch = rcu_dereference(scx_root); 10172 if (unlikely(!sch)) 10173 return; 10174 10175 #ifdef CONFIG_EXT_SUB_SCHED 10176 /* 10177 * Disallow if any sub-scheds are attached. There is no way to tell 10178 * which scheduler called us, so error out @p's scheduler -- read it 10179 * under RCU as @p's locks aren't necessarily held here. @p may be a 10180 * task past sched_ext_dead() or an idle task, in which case its 10181 * scheduler can't be determined and there is nothing obviously wrong 10182 * to report; just refuse the call. 10183 */ 10184 if (unlikely(!list_empty(&sch->children))) { 10185 struct scx_sched *tsch = scx_task_sched_rcu(p); 10186 10187 if (tsch) 10188 scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used"); 10189 return; 10190 } 10191 #endif 10192 10193 scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags); 10194 } 10195 10196 __bpf_kfunc_end_defs(); 10197 10198 BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch) 10199 BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_IMPLICIT_ARGS | KF_RCU) 10200 BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_IMPLICIT_ARGS | KF_RCU) 10201 BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_IMPLICIT_ARGS | KF_RCU) 10202 BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU) 10203 BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch) 10204 10205 static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { 10206 .owner = THIS_MODULE, 10207 .set = &scx_kfunc_ids_enqueue_dispatch, 10208 .filter = scx_kfunc_context_filter, 10209 }; 10210 10211 /* 10212 * Lock @rq from a kfunc called either from ops.dispatch(), which holds the 10213 * dispatched rq's lock, or from a context holding no rq lock. Under core 10214 * scheduling, dispatch can run for a sibling rq, so the locked rq is not 10215 * necessarily this CPU's. Return the rq locked on entry, NULL if none. 10216 */ 10217 static struct rq *kf_switch_rq_lock(struct rq *rq, unsigned long *flags) 10218 { 10219 struct rq *entry_rq; 10220 10221 local_irq_save(*flags); 10222 10223 entry_rq = scx_locked_rq(); 10224 if (entry_rq) { 10225 if (entry_rq != rq) 10226 switch_rq_lock(entry_rq, rq); 10227 } else { 10228 raw_spin_rq_lock(rq); 10229 } 10230 return entry_rq; 10231 } 10232 10233 /* undo kf_switch_rq_lock(), @rq is the rq locked now */ 10234 static void kf_restore_rq_lock(struct rq *rq, struct rq *entry_rq, unsigned long flags) 10235 { 10236 if (entry_rq) { 10237 if (entry_rq != rq) 10238 switch_rq_lock(rq, entry_rq); 10239 } else { 10240 scx_rq_lock_drop(rq); 10241 raw_spin_rq_unlock_irqrestore(rq, flags); 10242 } 10243 } 10244 10245 static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, 10246 struct task_struct *p, u64 dsq_id, u64 enq_flags, 10247 bool priq) 10248 { 10249 struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; 10250 struct scx_sched *sch; 10251 struct rq *p_rq, *src_rq, *locked_rq; 10252 bool dispatched = false; 10253 unsigned long flags; 10254 10255 /* 10256 * The verifier considers an iterator slot initialized on any 10257 * KF_ITER_NEW return, so a BPF program may legally reach here after 10258 * bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL. 10259 */ 10260 if (unlikely(!src_dsq)) 10261 return false; 10262 10263 sch = src_dsq->sched; 10264 10265 if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) 10266 return false; 10267 10268 /* internal bit, can only go in after @enq_flags is vetted */ 10269 if (priq) 10270 enq_flags |= SCX_ENQ_DSQ_PRIQ; 10271 10272 /* 10273 * If the BPF scheduler keeps calling this function repeatedly, it can 10274 * cause similar live-lock conditions as scx_consume_dispatch_q(). 10275 */ 10276 if (unlikely(READ_ONCE(sch->aborting))) 10277 return false; 10278 10279 src_rq = task_rq(p); 10280 locked_rq = kf_switch_rq_lock(src_rq, &flags); 10281 10282 p_rq = src_rq; 10283 raw_spin_lock(&src_dsq->lock); 10284 10285 /* did someone else get to it while we dropped the locks? */ 10286 if (nldsq_cursor_lost_task(&kit->cursor, src_rq, src_dsq, p)) { 10287 raw_spin_unlock(&src_dsq->lock); 10288 goto out; 10289 } 10290 10291 /* 10292 * @p has been on $src_dsq and can't move anymore. If @p is not on @sch, 10293 * the caller didn't have authority over @p at the time of the call. 10294 */ 10295 if (unlikely(!scx_task_on_sched(sch, p))) { 10296 scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", 10297 p->comm, p->pid); 10298 raw_spin_unlock(&src_dsq->lock); 10299 goto out; 10300 } 10301 10302 /* @p is still on $src_dsq and stable, determine the destination */ 10303 dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p)); 10304 10305 /* 10306 * Apply vtime and slice updates before moving. @p is still on $src_dsq 10307 * with both $src_dsq and its task_rq locked, satisfying the write 10308 * rules, and the PRIQ insertion into $dst_dsq reads the new vtime. 10309 */ 10310 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) 10311 p->scx.dsq_vtime = kit->vtime; 10312 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) 10313 scx_set_task_slice(p, kit->slice); 10314 10315 /* execute move */ 10316 p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); 10317 dispatched = true; 10318 out: 10319 kf_restore_rq_lock(p_rq, locked_rq, flags); 10320 10321 kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | 10322 __SCX_DSQ_ITER_HAS_VTIME); 10323 return dispatched; 10324 } 10325 10326 __bpf_kfunc_start_defs(); 10327 10328 /** 10329 * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots 10330 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10331 * 10332 * Can only be called from ops.dispatch(). 10333 */ 10334 __bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux) 10335 { 10336 struct scx_sched *sch; 10337 10338 guard(rcu)(); 10339 10340 sch = scx_prog_sched(aux); 10341 if (unlikely(!sch)) 10342 return 0; 10343 10344 return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor); 10345 } 10346 10347 /** 10348 * scx_bpf_dispatch_cancel - Cancel the latest dispatch 10349 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10350 * 10351 * Cancel the latest dispatch. Can be called multiple times to cancel further 10352 * dispatches. Can only be called from ops.dispatch(). 10353 */ 10354 __bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux) 10355 { 10356 struct scx_sched *sch; 10357 struct scx_dsp_ctx *dspc; 10358 10359 guard(rcu)(); 10360 10361 sch = scx_prog_sched(aux); 10362 if (unlikely(!sch)) 10363 return; 10364 10365 dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; 10366 10367 if (dspc->cursor > 0) 10368 dspc->cursor--; 10369 else 10370 scx_error(sch, "dispatch buffer underflow"); 10371 } 10372 10373 /** 10374 * scx_bpf_dsq_move_to_local___v2 - move a task from a DSQ to the current CPU's local DSQ 10375 * @dsq_id: DSQ to move task from. Must be a user-created DSQ 10376 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10377 * @enq_flags: %SCX_ENQ_* 10378 * 10379 * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's 10380 * local DSQ for execution with @enq_flags applied. Can only be called from 10381 * ops.dispatch(). 10382 * 10383 * Built-in DSQs (%SCX_DSQ_GLOBAL and %SCX_DSQ_LOCAL*) are not supported as 10384 * sources. Local DSQs support reenqueueing (a task can be picked up for 10385 * execution, dequeued for property changes, or reenqueued), but the BPF 10386 * scheduler cannot directly iterate or move tasks from them. %SCX_DSQ_GLOBAL 10387 * is similar but also doesn't support reenqueueing, as it maps to multiple 10388 * per-node DSQs making the scope difficult to define; this may change in the 10389 * future. 10390 * 10391 * This function flushes the in-flight dispatches from scx_bpf_dsq_insert() 10392 * before trying to move from the specified DSQ. It may also grab rq locks and 10393 * thus can't be called under any BPF locks. 10394 * 10395 * Returns %true if a task has been moved, %false if there isn't any task to 10396 * move. 10397 */ 10398 __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, 10399 const struct bpf_prog_aux *aux) 10400 { 10401 struct scx_dispatch_q *dsq; 10402 struct scx_sched *sch; 10403 struct scx_dsp_ctx *dspc; 10404 10405 guard(rcu)(); 10406 10407 sch = scx_prog_sched(aux); 10408 if (unlikely(!sch)) 10409 return false; 10410 10411 if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags)) 10412 return false; 10413 10414 dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; 10415 10416 scx_flush_dispatch_buf(sch, dspc->rq); 10417 10418 dsq = find_user_dsq(sch, dsq_id); 10419 if (unlikely(!dsq)) { 10420 scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id); 10421 return false; 10422 } 10423 10424 if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { 10425 /* 10426 * A successfully consumed task can be dequeued before it starts 10427 * running while the CPU is trying to migrate other dispatched 10428 * tasks. Bump nr_tasks to tell dispatch_one() to retry on empty 10429 * local DSQ. 10430 */ 10431 dspc->nr_tasks++; 10432 return true; 10433 } else { 10434 return false; 10435 } 10436 } 10437 10438 /* 10439 * COMPAT: ___v2 was introduced in v7.1. Remove this and ___v2 tag in the future. 10440 */ 10441 __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux) 10442 { 10443 return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux); 10444 } 10445 10446 /** 10447 * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs 10448 * @it__iter: DSQ iterator in progress 10449 * @slice: duration the moved task can run for in nsecs 10450 * 10451 * Override the slice of the next task that will be moved from @it__iter using 10452 * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous 10453 * slice duration is kept. 10454 */ 10455 __bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, 10456 u64 slice) 10457 { 10458 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; 10459 10460 kit->slice = slice; 10461 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE; 10462 } 10463 10464 /** 10465 * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs 10466 * @it__iter: DSQ iterator in progress 10467 * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ 10468 * 10469 * Override the vtime of the next task that will be moved from @it__iter using 10470 * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice 10471 * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the 10472 * override is ignored and cleared. 10473 */ 10474 __bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, 10475 u64 vtime) 10476 { 10477 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; 10478 10479 kit->vtime = vtime; 10480 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME; 10481 } 10482 10483 /** 10484 * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ 10485 * @it__iter: DSQ iterator in progress 10486 * @p: task to transfer 10487 * @dsq_id: DSQ to move @p to 10488 * @enq_flags: SCX_ENQ_* 10489 * 10490 * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ 10491 * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can 10492 * be the destination. 10493 * 10494 * For the transfer to be successful, @p must still be on the DSQ and have been 10495 * queued before the DSQ iteration started. This function doesn't care whether 10496 * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have 10497 * been queued before the iteration started. 10498 * 10499 * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update. 10500 * 10501 * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq 10502 * lock (e.g. BPF timers or SYSCALL programs). 10503 * 10504 * Returns %true if @p has been consumed, %false if @p had already been 10505 * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local 10506 * DSQ. 10507 */ 10508 __bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, 10509 struct task_struct *p, u64 dsq_id, 10510 u64 enq_flags) 10511 { 10512 return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, 10513 p, dsq_id, enq_flags, false); 10514 } 10515 10516 /** 10517 * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ 10518 * @it__iter: DSQ iterator in progress 10519 * @p: task to transfer 10520 * @dsq_id: DSQ to move @p to 10521 * @enq_flags: SCX_ENQ_* 10522 * 10523 * Transfer @p which is on the DSQ currently iterated by @it__iter to the 10524 * priority queue of the DSQ specified by @dsq_id. The destination must be a 10525 * user DSQ as only user DSQs support priority queue. 10526 * 10527 * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice() 10528 * and scx_bpf_dsq_move_set_vtime() to update. 10529 * 10530 * All other aspects are identical to scx_bpf_dsq_move(). See 10531 * scx_bpf_dsq_insert_vtime() for more information on @vtime. 10532 */ 10533 __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, 10534 struct task_struct *p, u64 dsq_id, 10535 u64 enq_flags) 10536 { 10537 return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, 10538 p, dsq_id, enq_flags, true); 10539 } 10540 10541 __bpf_kfunc_end_defs(); 10542 10543 BTF_KFUNCS_START(scx_kfunc_ids_dispatch) 10544 BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots, KF_IMPLICIT_ARGS) 10545 BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel, KF_IMPLICIT_ARGS) 10546 BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local, KF_IMPLICIT_ARGS) 10547 BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local___v2, KF_IMPLICIT_ARGS) 10548 /* scx_bpf_dsq_move*() also in scx_kfunc_ids_unlocked: callable from unlocked contexts */ 10549 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) 10550 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) 10551 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) 10552 BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) 10553 /* also in scx_kfunc_ids_unlocked, same contexts as scx_bpf_dsq_move() */ 10554 BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) 10555 #ifdef CONFIG_EXT_SUB_SCHED 10556 BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS) 10557 #endif 10558 BTF_KFUNCS_END(scx_kfunc_ids_dispatch) 10559 10560 static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { 10561 .owner = THIS_MODULE, 10562 .set = &scx_kfunc_ids_dispatch, 10563 .filter = scx_kfunc_context_filter, 10564 }; 10565 10566 __bpf_kfunc_start_defs(); 10567 10568 /** 10569 * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ 10570 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10571 * 10572 * Iterate over all of the tasks currently enqueued on the local DSQ of the 10573 * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of 10574 * processed tasks. Can only be called from ops.cpu_release(). 10575 */ 10576 __bpf_kfunc u32 scx_bpf_reenqueue_local(const struct bpf_prog_aux *aux) 10577 { 10578 struct scx_sched *sch; 10579 struct rq *rq; 10580 10581 guard(rcu)(); 10582 sch = scx_prog_sched(aux); 10583 if (unlikely(!sch)) 10584 return 0; 10585 10586 rq = cpu_rq(smp_processor_id()); 10587 lockdep_assert_rq_held(rq); 10588 10589 return reenq_local(sch, rq, SCX_REENQ_ANY); 10590 } 10591 10592 __bpf_kfunc_end_defs(); 10593 10594 BTF_KFUNCS_START(scx_kfunc_ids_cpu_release) 10595 BTF_ID_FLAGS(func, scx_bpf_reenqueue_local, KF_IMPLICIT_ARGS) 10596 BTF_KFUNCS_END(scx_kfunc_ids_cpu_release) 10597 10598 static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { 10599 .owner = THIS_MODULE, 10600 .set = &scx_kfunc_ids_cpu_release, 10601 .filter = scx_kfunc_context_filter, 10602 }; 10603 10604 __bpf_kfunc_start_defs(); 10605 10606 /** 10607 * scx_bpf_create_dsq - Create a custom DSQ 10608 * @dsq_id: DSQ to create 10609 * @node: NUMA node to allocate from 10610 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10611 * 10612 * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable 10613 * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. 10614 */ 10615 __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux) 10616 { 10617 struct scx_dispatch_q *dsq; 10618 struct scx_sched *sch; 10619 s32 ret; 10620 10621 if (unlikely(node >= (int)nr_node_ids || 10622 (node < 0 && node != NUMA_NO_NODE))) 10623 return -EINVAL; 10624 10625 if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) 10626 return -EINVAL; 10627 10628 dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); 10629 if (!dsq) 10630 return -ENOMEM; 10631 10632 /* 10633 * scx_init_dsq() must be called in GFP_KERNEL context. Init it with 10634 * NULL @sch and update afterwards. 10635 */ 10636 ret = scx_init_dsq(dsq, dsq_id, NULL); 10637 if (ret) { 10638 kfree(dsq); 10639 return ret; 10640 } 10641 10642 rcu_read_lock(); 10643 10644 sch = scx_prog_sched(aux); 10645 if (sch) { 10646 dsq->sched = sch; 10647 ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node, 10648 dsq_hash_params); 10649 } else { 10650 ret = -ENODEV; 10651 } 10652 10653 rcu_read_unlock(); 10654 if (ret) { 10655 exit_dsq(dsq); 10656 kfree(dsq); 10657 } 10658 return ret; 10659 } 10660 10661 __bpf_kfunc_end_defs(); 10662 10663 BTF_KFUNCS_START(scx_kfunc_ids_unlocked) 10664 BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_IMPLICIT_ARGS | KF_SLEEPABLE) 10665 /* also in scx_kfunc_ids_dispatch: also callable from ops.dispatch() */ 10666 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) 10667 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) 10668 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) 10669 BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) 10670 BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) 10671 /* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */ 10672 BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) 10673 BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) 10674 BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) 10675 BTF_KFUNCS_END(scx_kfunc_ids_unlocked) 10676 10677 static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = { 10678 .owner = THIS_MODULE, 10679 .set = &scx_kfunc_ids_unlocked, 10680 .filter = scx_kfunc_context_filter, 10681 }; 10682 10683 __bpf_kfunc_start_defs(); 10684 10685 /** 10686 * scx_bpf_task_set_slice - Set task's time slice 10687 * @p: task of interest 10688 * @slice: time slice to set in nsecs 10689 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10690 * 10691 * Set @p's time slice. @p must be on the calling scheduler. The value is 10692 * applied whether or not the caller holds @p's rq lock - see the slice write 10693 * rules above for the ownership model. 10694 * 10695 * Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on 10696 * @p's cpu, and never on a kernel slice, see %SCX_TASK_KERNEL_SLICE. Shortening 10697 * is honored unless @p is protected, see %SCX_TASK_PROTECTED, which refuses 10698 * every write. A refused write is counted in %SCX_EV_SLICE_DENIED. On the 10699 * stashed path the slice is packed into an atomic64_t with the scheduler id and 10700 * a flag bit, so a slice too large to fit is clamped and counted in 10701 * %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved. 10702 * 10703 * Return %true on success, %false if @p is not on the calling scheduler. 10704 */ 10705 __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, 10706 const struct bpf_prog_aux *aux) 10707 { 10708 struct scx_sched *sch; 10709 struct rq *locked_rq; 10710 10711 guard(rcu)(); 10712 sch = scx_prog_sched(aux); 10713 if (unlikely(!sch || !scx_task_on_sched(sch, p))) 10714 return false; 10715 10716 /* 10717 * Directly write only when we hold the lock of the rq @p is queued on or 10718 * provides the current scheduling context for. Under proxy execution, 10719 * rq->donor owns and consumes the slice while rq->curr executes on its 10720 * behalf. See the slice write rules above. 10721 * 10722 * While @p is queued on a user DSQ or in the BPF scheduler, 10723 * synchronization is the scheduler's responsibility. This write can 10724 * race a concurrent dispatch's commit, see apply_slice_vtime(). 10725 * 10726 * Making this kfunc always go through the oob stash would leave the 10727 * commit as the only direct writer and close the race, but that would 10728 * require two more oob application points - the dispatch keep-prev test 10729 * and the tick-time expiry check. 10730 */ 10731 locked_rq = scx_locked_rq(); 10732 if (!locked_rq || 10733 (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) && 10734 !task_current_donor(locked_rq, p))) { 10735 set_task_slice_oob(sch, p, slice); 10736 return true; 10737 } 10738 10739 /* 10740 * @p's rq is locked, so apply now. An extension needs baseline access 10741 * and is refused on a kernel slice. 10742 */ 10743 if (slice > p->scx.slice && 10744 unlikely(scx_task_has_kernel_slice(p) || 10745 scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) { 10746 __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); 10747 return true; 10748 } 10749 10750 if (unlikely(!scx_set_task_slice(p, slice))) 10751 __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); 10752 10753 return true; 10754 } 10755 10756 /** 10757 * scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering 10758 * @p: task of interest 10759 * @vtime: virtual time to set 10760 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10761 * 10762 * Set @p's virtual time to @vtime. Returns %true on success, %false if the 10763 * calling scheduler doesn't have authority over @p. 10764 */ 10765 __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, 10766 const struct bpf_prog_aux *aux) 10767 { 10768 struct scx_sched *sch; 10769 10770 guard(rcu)(); 10771 sch = scx_prog_sched(aux); 10772 if (unlikely(!sch || !scx_task_on_sched(sch, p))) 10773 return false; 10774 10775 p->scx.dsq_vtime = vtime; 10776 return true; 10777 } 10778 10779 /** 10780 * scx_bpf_task_set_lazy_resched - Set task's tick reschedule mode 10781 * @p: task of interest 10782 * @lazy: whether slice expiry should request lazy rescheduling 10783 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10784 * 10785 * Choose whether depletion of @p's slice at the scheduler tick requests lazy or 10786 * immediate rescheduling. This is a persistent per-task policy and does not 10787 * itself clear @p's slice or request rescheduling, unlike the 10788 * %SCX_ENQ_PREEMPT_LAZY and %SCX_KICK_PREEMPT_LAZY one-shot operations. 10789 * 10790 * With lazy rescheduling enabled, a task interrupted in user space still 10791 * reschedules before returning to user space. A task interrupted in the kernel 10792 * may continue until its next return to user space or the next scheduler tick, 10793 * which promotes the request. If lazy preemption is disabled at runtime, the 10794 * request behaves like an immediate reschedule. 10795 * 10796 * @p must be on the calling scheduler. 10797 * 10798 * Return %true on success, %false if @p is not on the calling scheduler. 10799 */ 10800 __bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy, 10801 const struct bpf_prog_aux *aux) 10802 { 10803 struct scx_sched *sch; 10804 10805 /* 10806 * clang's register locations for @lazy make pahole 1.32 drop this kfunc 10807 * from BTF. Keep @lazy in memory so its location names no register. 10808 */ 10809 barrier_data(&lazy); 10810 guard(rcu)(); 10811 sch = scx_prog_sched(aux); 10812 if (unlikely(!sch || !scx_task_on_sched(sch, p))) 10813 return false; 10814 10815 WRITE_ONCE(p->scx.lazy_resched, lazy); 10816 return true; 10817 } 10818 10819 void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) 10820 { 10821 struct scx_sched_pcpu *pcpu; 10822 struct rq *this_rq; 10823 unsigned long irq_flags; 10824 10825 if (!scx_kf_allowed_ctx(sch)) 10826 return; 10827 if (unlikely(flags & ~SCX_KICK_ALL_FLAGS)) { 10828 scx_error(sch, "invalid kick flags 0x%llx", flags); 10829 return; 10830 } 10831 10832 local_irq_save(irq_flags); 10833 10834 this_rq = this_rq(); 10835 pcpu = this_cpu_ptr(sch->pcpu); 10836 10837 /* 10838 * While bypassing for PM ops, IRQ handling may not be online which can 10839 * lead to irq_work_queue() malfunction such as infinite busy wait for 10840 * IRQ status update. Suppress kicking. 10841 */ 10842 if (scx_bypassing(sch, cpu_of(this_rq))) 10843 goto out; 10844 10845 /* 10846 * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting 10847 * rq locks. We can probably be smarter and avoid bouncing if called 10848 * from ops which don't hold a rq lock. 10849 * 10850 * The kick masks are owned by @sch->pcpu, so that a preempt kick can be 10851 * attributed to @sch. 10852 */ 10853 if (flags & SCX_KICK_IDLE) { 10854 struct rq *target_rq = cpu_rq(cpu); 10855 10856 if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY | 10857 SCX_KICK_WAIT))) 10858 scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); 10859 10860 if (raw_spin_rq_trylock(target_rq)) { 10861 if (can_skip_idle_kick(target_rq)) { 10862 scx_rq_lock_drop(target_rq); 10863 raw_spin_rq_unlock(target_rq); 10864 goto out; 10865 } 10866 scx_rq_lock_drop(target_rq); 10867 raw_spin_rq_unlock(target_rq); 10868 } 10869 cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle); 10870 } else { 10871 /* 10872 * Accumulate requests and resolve their precedence at delivery. 10873 */ 10874 if (flags & SCX_KICK_PREEMPT) 10875 cpumask_set_cpu(cpu, pcpu->cpus_to_preempt); 10876 if (flags & SCX_KICK_PREEMPT_LAZY) 10877 cpumask_set_cpu(cpu, pcpu->cpus_to_preempt_lazy); 10878 if (flags & SCX_KICK_WAIT) 10879 cpumask_set_cpu(cpu, pcpu->cpus_to_wait); 10880 if (!(flags & SCX_KICK_PREEMPT_LAZY)) 10881 cpumask_set_cpu(cpu, pcpu->cpus_to_kick); 10882 } 10883 10884 if (list_empty(&pcpu->to_kick_node)) 10885 list_add_tail(&pcpu->to_kick_node, &this_rq->scx.sched_pcpus_to_kick); 10886 irq_work_queue(&this_rq->scx.kick_cpus_irq_work); 10887 out: 10888 local_irq_restore(irq_flags); 10889 } 10890 10891 /** 10892 * scx_bpf_kick_cpu - Trigger reschedule on a CPU 10893 * @cpu: cpu to kick 10894 * @flags: %SCX_KICK_* flags 10895 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10896 * 10897 * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or 10898 * trigger rescheduling on a busy CPU. This can be called from any online 10899 * scx_ops operation and the actual kicking is performed asynchronously through 10900 * an irq work. 10901 */ 10902 __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux *aux) 10903 { 10904 struct scx_sched *sch; 10905 10906 guard(rcu)(); 10907 sch = scx_prog_sched(aux); 10908 if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) 10909 scx_kick_cpu(sch, cpu, flags); 10910 } 10911 10912 /** 10913 * scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid 10914 * @cid: cid to kick 10915 * @flags: %SCX_KICK_* flags 10916 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10917 * 10918 * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the 10919 * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a 10920 * kick is dropped if the caller lacks baseline access on @cid, and a 10921 * %SCX_KICK_PREEMPT or %SCX_KICK_PREEMPT_LAZY request degrades to a plain 10922 * reschedule if the caller lacks %SCX_CAP_PREEMPT for a task outside its 10923 * subtree. 10924 */ 10925 __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) 10926 { 10927 struct scx_sched *sch; 10928 s32 cpu; 10929 10930 guard(rcu)(); 10931 sch = scx_prog_sched(aux); 10932 if (unlikely(!sch)) 10933 return; 10934 cpu = scx_cid_to_cpu(sch, cid); 10935 if (cpu < 0) 10936 return; 10937 scx_kick_cpu(sch, cpu, flags); 10938 } 10939 10940 /** 10941 * scx_bpf_dsq_nr_queued - Return the number of queued tasks 10942 * @dsq_id: id of the DSQ 10943 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10944 * 10945 * Return the number of tasks in the DSQ matching @dsq_id. If not found, 10946 * -%ENOENT is returned. 10947 * 10948 * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler 10949 * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - 10950 * or the calling CPU's when no rq is locked. 10951 */ 10952 __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux) 10953 { 10954 struct scx_sched *sch; 10955 struct scx_dispatch_q *dsq; 10956 s32 ret; 10957 10958 preempt_disable(); 10959 10960 sch = scx_prog_sched(aux); 10961 if (unlikely(!sch)) { 10962 ret = -ENODEV; 10963 goto out; 10964 } 10965 10966 if (dsq_id == SCX_DSQ_LOCAL) { 10967 ret = READ_ONCE((scx_locked_rq() ?: this_rq())->scx.local_dsq.nr); 10968 goto out; 10969 } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { 10970 s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); 10971 10972 if (scx_cpu_valid(sch, cpu, NULL)) { 10973 ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); 10974 goto out; 10975 } 10976 } else { 10977 dsq = find_user_dsq(sch, dsq_id); 10978 if (dsq) { 10979 ret = READ_ONCE(dsq->nr); 10980 goto out; 10981 } 10982 } 10983 ret = -ENOENT; 10984 out: 10985 preempt_enable(); 10986 return ret; 10987 } 10988 10989 /** 10990 * scx_bpf_destroy_dsq - Destroy a custom DSQ 10991 * @dsq_id: DSQ to destroy 10992 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 10993 * 10994 * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with 10995 * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is 10996 * empty and no further tasks are dispatched to it. Ignored if called on a DSQ 10997 * which doesn't exist. Can be called from any online scx_ops operations. 10998 */ 10999 __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux) 11000 { 11001 struct scx_sched *sch; 11002 11003 guard(rcu)(); 11004 sch = scx_prog_sched(aux); 11005 if (unlikely(!sch)) 11006 return; 11007 11008 if (!scx_kf_allowed_ctx(sch)) 11009 return; 11010 11011 destroy_dsq(sch, dsq_id); 11012 } 11013 11014 /** 11015 * bpf_iter_scx_dsq_new - Create a DSQ iterator 11016 * @it: iterator to initialize 11017 * @dsq_id: DSQ to iterate 11018 * @flags: %SCX_DSQ_ITER_* 11019 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11020 * 11021 * Initialize BPF iterator @it which can be used with bpf_for_each() to walk 11022 * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes 11023 * tasks which are already queued when this function is invoked. 11024 */ 11025 __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, 11026 u64 flags, const struct bpf_prog_aux *aux) 11027 { 11028 struct bpf_iter_scx_dsq_kern *kit = (void *)it; 11029 struct scx_sched *sch; 11030 11031 BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) > 11032 sizeof(struct bpf_iter_scx_dsq)); 11033 BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != 11034 __alignof__(struct bpf_iter_scx_dsq)); 11035 BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & 11036 ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1)); 11037 11038 /* 11039 * next() and destroy() will be called regardless of the return value. 11040 * Always clear $kit->dsq. 11041 */ 11042 kit->dsq = NULL; 11043 11044 sch = scx_prog_sched(aux); 11045 if (unlikely(!sch)) 11046 return -ENODEV; 11047 11048 if (!scx_kf_allowed_ctx(sch)) 11049 return -EDEADLK; 11050 11051 if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) 11052 return -EINVAL; 11053 11054 kit->dsq = find_user_dsq(sch, dsq_id); 11055 if (!kit->dsq) 11056 return -ENOENT; 11057 11058 kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags); 11059 11060 return 0; 11061 } 11062 11063 /** 11064 * bpf_iter_scx_dsq_next - Progress a DSQ iterator 11065 * @it: iterator to progress 11066 * 11067 * Return the next task. See bpf_iter_scx_dsq_new(). 11068 */ 11069 __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) 11070 { 11071 struct bpf_iter_scx_dsq_kern *kit = (void *)it; 11072 struct task_struct *p; 11073 11074 if (!kit->dsq || (kit->cursor.flags & __SCX_DSQ_ITER_DRAINED)) 11075 return NULL; 11076 11077 guard(raw_spinlock_irqsave)(&kit->dsq->lock); 11078 11079 p = nldsq_cursor_next_task(&kit->cursor, kit->dsq); 11080 /* 11081 * nldsq_cursor_next_task() unlinks the cursor and starts from the DSQ 11082 * head when it is called again. The verifier relies on an iterator 11083 * which returned NULL to keep returning NULL (see 11084 * process_iter_next_call()), so remember the drained state. 11085 */ 11086 if (!p) 11087 kit->cursor.flags |= __SCX_DSQ_ITER_DRAINED; 11088 11089 return p; 11090 } 11091 11092 /** 11093 * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator 11094 * @it: iterator to destroy 11095 * 11096 * Undo bpf_iter_scx_dsq_new(). If the iteration marked tasks with 11097 * scx_bpf_dsq_reenq_task(), schedule their re-enqueue. 11098 */ 11099 __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) 11100 { 11101 struct bpf_iter_scx_dsq_kern *kit = (void *)it; 11102 11103 if (!kit->dsq) 11104 return; 11105 11106 if (!list_empty(&kit->cursor.node)) { 11107 unsigned long flags; 11108 11109 raw_spin_lock_irqsave(&kit->dsq->lock, flags); 11110 list_del_init(&kit->cursor.node); 11111 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); 11112 } 11113 11114 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_REENQ) { 11115 guard(preempt)(); 11116 schedule_dsq_reenq(kit->dsq->sched, kit->dsq, SCX_REENQ_MARKED, 11117 scx_locked_rq()); 11118 } 11119 11120 kit->dsq = NULL; 11121 } 11122 11123 /** 11124 * scx_bpf_dsq_peek - Lockless peek at the first element. 11125 * @dsq_id: DSQ to examine. 11126 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11127 * 11128 * Read the first element in the DSQ. This is semantically equivalent to using 11129 * the DSQ iterator, but is lockfree. Of course, like any lockless operation, 11130 * this provides only a point-in-time snapshot, and the contents may change 11131 * by the time any subsequent locking operation reads the queue. 11132 * 11133 * Returns the pointer, or NULL indicates an empty queue OR internal error. 11134 */ 11135 __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, 11136 const struct bpf_prog_aux *aux) 11137 { 11138 struct scx_sched *sch; 11139 struct scx_dispatch_q *dsq; 11140 11141 sch = scx_prog_sched(aux); 11142 if (unlikely(!sch)) 11143 return NULL; 11144 11145 if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) { 11146 scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id); 11147 return NULL; 11148 } 11149 11150 dsq = find_user_dsq(sch, dsq_id); 11151 if (unlikely(!dsq)) { 11152 scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id); 11153 return NULL; 11154 } 11155 11156 return rcu_dereference(dsq->first_task); 11157 } 11158 11159 /** 11160 * scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ 11161 * @dsq_id: DSQ to re-enqueue 11162 * @reenq_flags: %SCX_REENQ_* 11163 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11164 * 11165 * Iterate over all of the tasks currently enqueued on the DSQ identified by 11166 * @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are 11167 * supported: 11168 * 11169 * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) 11170 * - User DSQs 11171 * 11172 * Re-enqueues are performed asynchronously. Can be called from any context 11173 * except NMI. A call from NMI aborts the scheduler. A re-enqueue of an inactive 11174 * cpu's local DSQ is ignored. 11175 * 11176 * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler 11177 * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - 11178 * or the calling CPU's when no rq is locked. 11179 */ 11180 __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, 11181 const struct bpf_prog_aux *aux) 11182 { 11183 struct rq *locked_rq = scx_locked_rq(); 11184 struct scx_sched *sch; 11185 struct scx_dispatch_q *dsq; 11186 11187 guard(preempt)(); 11188 11189 sch = scx_prog_sched(aux); 11190 if (unlikely(!sch)) 11191 return; 11192 11193 if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) { 11194 scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags); 11195 return; 11196 } 11197 11198 if (!scx_kf_allowed_ctx(sch)) 11199 return; 11200 11201 /* not specifying any filter bits is the same as %SCX_REENQ_ANY */ 11202 if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) 11203 reenq_flags |= SCX_REENQ_ANY; 11204 11205 dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id()); 11206 schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq); 11207 } 11208 11209 /** 11210 * scx_bpf_dsq_reenq_task - Re-enqueue a task from a DSQ iteration 11211 * @it__iter: DSQ iterator in progress 11212 * @p: task to re-enqueue 11213 * 11214 * Mark @p, which is on the DSQ iterated by @it__iter and was queued before the 11215 * iteration started, for re-enqueue. When the iteration ends, the DSQ's marked 11216 * tasks go through ops.enqueue() with %SCX_ENQ_REENQ in DSQ order and the other 11217 * tasks on the DSQ keep their positions. A marked task that leaves the DSQ in 11218 * the meantime is not re-enqueued. The mark belongs to @p, so a re-enqueue 11219 * scheduled by another iteration of the DSQ may re-enqueue @p before this 11220 * iteration ends. @p must be on the DSQ's scheduler. 11221 * 11222 * Re-enqueues are performed asynchronously. Can be called from ops.dispatch() 11223 * or any BPF context which doesn't hold a rq lock (e.g. BPF timers or SYSCALL 11224 * programs). 11225 * 11226 * Return %true if @p was marked, %false if @p left the DSQ or was queued after 11227 * the iteration started, @p is on another scheduler, or the iterator holds no 11228 * DSQ. 11229 */ 11230 __bpf_kfunc bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it__iter, 11231 struct task_struct *p) 11232 { 11233 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; 11234 struct scx_dispatch_q *dsq = kit->dsq; 11235 struct rq *src_rq, *entry_rq; 11236 unsigned long flags; 11237 bool marked = false; 11238 11239 /* NULL after a failed bpf_iter_scx_dsq_new(), see scx_dsq_move() */ 11240 if (unlikely(!dsq)) 11241 return false; 11242 11243 /* the cursor check needs @p's rq locked */ 11244 src_rq = task_rq(p); 11245 entry_rq = kf_switch_rq_lock(src_rq, &flags); 11246 11247 scoped_guard (raw_spinlock, &dsq->lock) { 11248 if (nldsq_cursor_lost_task(&kit->cursor, src_rq, dsq, p)) 11249 break; 11250 11251 if (unlikely(!scx_task_on_sched(dsq->sched, p))) { 11252 scx_error(dsq->sched, "scx_bpf_dsq_reenq_task() on %s[%d] but the task belongs to a different scheduler", 11253 p->comm, p->pid); 11254 break; 11255 } 11256 11257 p->scx.dsq_flags |= SCX_TASK_DSQ_REENQ; 11258 /* the walk is scheduled once, by bpf_iter_scx_dsq_destroy() */ 11259 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_REENQ; 11260 marked = true; 11261 } 11262 11263 kf_restore_rq_lock(src_rq, entry_rq, flags); 11264 return marked; 11265 } 11266 11267 /** 11268 * scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ 11269 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11270 * 11271 * Iterate over all of the tasks currently enqueued on the local DSQ of the rq 11272 * the current scheduler operation is locked to, or the calling CPU's when no rq 11273 * is locked, and re-enqueue them in the BPF scheduler. Can be called from any 11274 * context except NMI. A call from NMI aborts the scheduler. 11275 * 11276 * This is now a special case of scx_bpf_dsq_reenq() and may be removed in the 11277 * future. 11278 */ 11279 __bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux) 11280 { 11281 scx_bpf_dsq_reenq(SCX_DSQ_LOCAL, 0, aux); 11282 } 11283 11284 __bpf_kfunc_end_defs(); 11285 11286 __printf(5, 0) 11287 static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, 11288 size_t line_size, char *fmt, unsigned long long *data, 11289 u32 data__sz) 11290 { 11291 struct bpf_bprintf_data bprintf_data = { .get_bin_args = true }; 11292 s32 ret; 11293 11294 if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || 11295 (data__sz && !data)) { 11296 scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz); 11297 return -EINVAL; 11298 } 11299 11300 ret = copy_from_kernel_nofault(data_buf, data, data__sz); 11301 if (ret < 0) { 11302 scx_error(sch, "failed to read data fields (%d)", ret); 11303 return ret; 11304 } 11305 11306 ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8, 11307 &bprintf_data); 11308 if (ret < 0) { 11309 scx_error(sch, "format preparation failed (%d)", ret); 11310 return ret; 11311 } 11312 11313 ret = bstr_printf(line_buf, line_size, fmt, 11314 bprintf_data.bin_args); 11315 bpf_bprintf_cleanup(&bprintf_data); 11316 if (ret < 0) { 11317 scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz); 11318 return ret; 11319 } 11320 11321 return ret; 11322 } 11323 11324 /* 11325 * Exit @sch with the reason formatted from a BPF-supplied bstr format. The exit 11326 * is claimed first and the reason is formatted directly into the winner-owned 11327 * exit_info buffer, which allows use from any context including NMI. 11328 * 11329 * @fmt_blame is the sched blamed for formatting failures through the 11330 * scx_error() calls in __bstr_format() and differs from @sch when a parent 11331 * supplies the kill reason for a child. A formatting failure doesn't revert the 11332 * claim - @sch still exits with the claimed kind and a fallback message. 11333 */ 11334 __printf(5, 0) 11335 bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind, 11336 s64 exit_code, struct scx_sched *fmt_blame, char *fmt, 11337 unsigned long long *data, u32 data__sz) 11338 { 11339 struct scx_exit_info *ei = sch->exit_info; 11340 u64 data_buf[MAX_BPRINTF_VARARGS]; 11341 s32 ret; 11342 11343 guard(preempt)(); 11344 11345 if (!scx_claim_exit(sch, kind)) 11346 return false; 11347 11348 ret = __bstr_format(fmt_blame, data_buf, ei->msg, SCX_EXIT_MSG_LEN, 11349 fmt, data, data__sz); 11350 if (ret < 0) 11351 scnprintf(ei->msg, SCX_EXIT_MSG_LEN, 11352 "exit message formatting failed (%d)", ret); 11353 11354 scx_finish_exit(sch, kind, exit_code, raw_smp_processor_id()); 11355 return true; 11356 } 11357 11358 __bpf_kfunc_start_defs(); 11359 11360 /** 11361 * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler. 11362 * @exit_code: Exit value to pass to user space via struct scx_exit_info. 11363 * @fmt: error message format string 11364 * @data: format string parameters packaged using ___bpf_fill() macro 11365 * @data__sz: @data len, must end in '__sz' for the verifier 11366 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11367 * 11368 * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops 11369 * disabling. 11370 */ 11371 __printf(2, 0) 11372 __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, 11373 unsigned long long *data, u32 data__sz, 11374 const struct bpf_prog_aux *aux) 11375 { 11376 struct scx_sched *sch; 11377 11378 guard(rcu)(); 11379 11380 sch = scx_prog_sched(aux); 11381 if (likely(sch)) 11382 scx_exit_bstr(sch, SCX_EXIT_UNREG_BPF, exit_code, sch, fmt, 11383 data, data__sz); 11384 } 11385 11386 /** 11387 * scx_bpf_error_bstr - Indicate fatal error 11388 * @fmt: error message format string 11389 * @data: format string parameters packaged using ___bpf_fill() macro 11390 * @data__sz: @data len, must end in '__sz' for the verifier 11391 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11392 * 11393 * Indicate that the BPF scheduler encountered a fatal error and initiate ops 11394 * disabling. 11395 */ 11396 __printf(1, 0) 11397 __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, 11398 u32 data__sz, const struct bpf_prog_aux *aux) 11399 { 11400 struct scx_sched *sch; 11401 11402 guard(rcu)(); 11403 11404 sch = scx_prog_sched(aux); 11405 if (likely(sch)) 11406 scx_exit_bstr(sch, SCX_EXIT_ERROR_BPF, 0, sch, fmt, data, 11407 data__sz); 11408 } 11409 11410 /** 11411 * scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler 11412 * @fmt: format string 11413 * @data: format string parameters packaged using ___bpf_fill() macro 11414 * @data__sz: @data len, must end in '__sz' for the verifier 11415 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11416 * 11417 * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and 11418 * dump_task() to generate extra debug dump specific to the BPF scheduler. 11419 * 11420 * The extra dump may be multiple lines. A single line may be split over 11421 * multiple calls. The last line is automatically terminated. 11422 */ 11423 __printf(1, 0) 11424 __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, 11425 u32 data__sz, const struct bpf_prog_aux *aux) 11426 { 11427 struct scx_sched *sch; 11428 struct scx_dump_data *dd = &scx_dump_data; 11429 struct scx_bstr_buf *buf = &dd->buf; 11430 s32 ret; 11431 11432 guard(rcu)(); 11433 11434 sch = scx_prog_sched(aux); 11435 if (unlikely(!sch)) 11436 return; 11437 11438 if (raw_smp_processor_id() != dd->cpu) { 11439 scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends"); 11440 return; 11441 } 11442 11443 /* append the formatted string to the line buf */ 11444 ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, 11445 sizeof(buf->line) - dd->cursor, fmt, data, data__sz); 11446 if (ret < 0) { 11447 scx_dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", 11448 dd->prefix, fmt, data, data__sz, ret); 11449 return; 11450 } 11451 11452 dd->cursor += ret; 11453 dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line)); 11454 11455 if (!dd->cursor) 11456 return; 11457 11458 /* 11459 * If the line buf overflowed or ends in a newline, flush it into the 11460 * dump. This is to allow the caller to generate a single line over 11461 * multiple calls. As ops_dump_flush() can also handle multiple lines in 11462 * the line buf, the only case which can lead to an unexpected 11463 * truncation is when the caller keeps generating newlines in the middle 11464 * instead of the end consecutively. Don't do that. 11465 */ 11466 if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n') 11467 ops_dump_flush(); 11468 } 11469 11470 /** 11471 * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU 11472 * @cpu: CPU of interest 11473 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11474 * 11475 * Return the maximum relative capacity of @cpu in relation to the most 11476 * performant CPU in the system. The return value is in the range [1, 11477 * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur(). 11478 */ 11479 __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux) 11480 { 11481 struct scx_sched *sch; 11482 11483 guard(rcu)(); 11484 11485 sch = scx_prog_sched(aux); 11486 if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) 11487 return arch_scale_cpu_capacity(cpu); 11488 else 11489 return SCX_CPUPERF_ONE; 11490 } 11491 11492 /** 11493 * scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid 11494 * @cid: cid of the CPU to query 11495 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11496 * 11497 * cid-addressed equivalent of scx_bpf_cpuperf_cap(). 11498 */ 11499 __bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux) 11500 { 11501 struct scx_sched *sch; 11502 s32 cpu; 11503 11504 guard(rcu)(); 11505 11506 sch = scx_prog_sched(aux); 11507 if (unlikely(!sch)) 11508 return SCX_CPUPERF_ONE; 11509 cpu = scx_cid_to_cpu(sch, cid); 11510 if (cpu < 0) 11511 return SCX_CPUPERF_ONE; 11512 return arch_scale_cpu_capacity(cpu); 11513 } 11514 11515 /** 11516 * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU 11517 * @cpu: CPU of interest 11518 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11519 * 11520 * Return the current relative performance of @cpu in relation to its maximum. 11521 * The return value is in the range [1, %SCX_CPUPERF_ONE]. 11522 * 11523 * The current performance level of a CPU in relation to the maximum performance 11524 * available in the system can be calculated as follows: 11525 * 11526 * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE 11527 * 11528 * The result is in the range [1, %SCX_CPUPERF_ONE]. 11529 */ 11530 __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux) 11531 { 11532 struct scx_sched *sch; 11533 11534 guard(rcu)(); 11535 11536 sch = scx_prog_sched(aux); 11537 if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) 11538 return arch_scale_freq_capacity(cpu); 11539 else 11540 return SCX_CPUPERF_ONE; 11541 } 11542 11543 /** 11544 * scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid 11545 * @cid: cid of the CPU to query 11546 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11547 * 11548 * cid-addressed equivalent of scx_bpf_cpuperf_cur(). 11549 */ 11550 __bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux) 11551 { 11552 struct scx_sched *sch; 11553 s32 cpu; 11554 11555 guard(rcu)(); 11556 11557 sch = scx_prog_sched(aux); 11558 if (unlikely(!sch)) 11559 return SCX_CPUPERF_ONE; 11560 cpu = scx_cid_to_cpu(sch, cid); 11561 if (cpu < 0) 11562 return SCX_CPUPERF_ONE; 11563 return arch_scale_freq_capacity(cpu); 11564 } 11565 11566 /* validate and apply a cpuperf target, see scx_bpf_cpuperf_set() */ 11567 static s32 scx_cpuperf_set(struct scx_sched *sch, s32 cpu, u32 perf) 11568 { 11569 struct rq *rq, *locked_rq; 11570 struct rq_flags rf; 11571 s32 ret; 11572 11573 if (unlikely(perf > SCX_CPUPERF_ONE)) { 11574 scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); 11575 return -EINVAL; 11576 } 11577 11578 if (!scx_cpu_valid(sch, cpu, NULL)) 11579 return -EINVAL; 11580 11581 rq = cpu_rq(cpu); 11582 locked_rq = scx_locked_rq(); 11583 11584 /* 11585 * When called with an rq lock held, restrict the operation to the 11586 * corresponding CPU to prevent ABBA deadlocks. 11587 */ 11588 if (locked_rq && rq != locked_rq) { 11589 scx_error(sch, "Invalid target CPU %d", cpu); 11590 return -EINVAL; 11591 } 11592 11593 /* 11594 * If no rq lock is held, allow to operate on any CPU by acquiring 11595 * the corresponding rq lock. 11596 */ 11597 if (!locked_rq) { 11598 rq_lock_irqsave(rq, &rf); 11599 update_rq_clock(rq); 11600 } 11601 11602 /* 11603 * ecaps updates are folded under the rq lock, making this test 11604 * authoritative: a write can never land after a revoke has taken 11605 * effect on @cpu. 11606 */ 11607 if (likely(!scx_missing_caps(sch, cpu, SCX_CAP_PERF))) { 11608 rq->scx.cpuperf_target = perf; 11609 cpufreq_update_util(rq, 0); 11610 ret = 0; 11611 } else { 11612 __scx_add_event(sch, SCX_EV_SUB_CIDPERF_DENIED, 1); 11613 ret = -EACCES; 11614 } 11615 11616 if (!locked_rq) 11617 rq_unlock_irqrestore(rq, &rf); 11618 11619 return ret; 11620 } 11621 11622 /** 11623 * scx_bpf_cpuperf_set - Set the relative performance target of a CPU 11624 * @cpu: CPU of interest 11625 * @perf: target performance level [0, %SCX_CPUPERF_ONE] 11626 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11627 * 11628 * Set the target performance level of @cpu to @perf. @perf is in linear 11629 * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the 11630 * schedutil cpufreq governor chooses the target frequency. 11631 * 11632 * The actual performance level chosen, CPU grouping, and the overhead and 11633 * latency of the operations are dependent on the hardware and cpufreq driver in 11634 * use. Consult hardware and cpufreq documentation for more information. The 11635 * current performance level can be monitored using scx_bpf_cpuperf_cur(). 11636 */ 11637 __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_aux *aux) 11638 { 11639 struct scx_sched *sch; 11640 11641 guard(rcu)(); 11642 11643 sch = scx_prog_sched(aux); 11644 if (unlikely(!sch)) 11645 return; 11646 11647 if (!scx_kf_allowed_ctx(sch)) 11648 return; 11649 11650 scx_cpuperf_set(sch, cpu, perf); 11651 } 11652 11653 /** 11654 * scx_bpf_cidperf_set - Set the performance target of the CPU at @cid 11655 * @cid: cid of the CPU to target 11656 * @perf: target performance level [0, %SCX_CPUPERF_ONE] 11657 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11658 * 11659 * cid-addressed equivalent of scx_bpf_cpuperf_set(). A sub-sched needs 11660 * SCX_CAP_PERF on @cid. Returns 0 if the target was applied, -%EACCES if 11661 * the write was denied for missing caps, other -errnos if @cid didn't 11662 * resolve. 11663 */ 11664 __bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf, 11665 const struct bpf_prog_aux *aux) 11666 { 11667 struct scx_sched *sch; 11668 s32 cpu; 11669 11670 guard(rcu)(); 11671 11672 sch = scx_prog_sched(aux); 11673 if (unlikely(!sch)) 11674 return -ENODEV; 11675 11676 if (!scx_kf_allowed_ctx(sch)) 11677 return -EDEADLK; 11678 11679 cpu = scx_cid_to_cpu(sch, cid); 11680 if (cpu < 0) 11681 return cpu; 11682 11683 return scx_cpuperf_set(sch, cpu, perf); 11684 } 11685 11686 /** 11687 * scx_bpf_nr_node_ids - Return the number of possible node IDs 11688 * 11689 * All valid node IDs in the system are smaller than the returned value. 11690 */ 11691 __bpf_kfunc u32 scx_bpf_nr_node_ids(void) 11692 { 11693 return nr_node_ids; 11694 } 11695 11696 /** 11697 * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs 11698 * 11699 * All valid CPU IDs in the system are smaller than the returned value. 11700 */ 11701 __bpf_kfunc u32 scx_bpf_nr_cpu_ids(void) 11702 { 11703 return nr_cpu_ids; 11704 } 11705 11706 /** 11707 * scx_bpf_nr_cids - Return the size of the cid space 11708 * 11709 * Equals num_possible_cpus(). All valid cids are in [0, return value). 11710 */ 11711 __bpf_kfunc u32 scx_bpf_nr_cids(void) 11712 { 11713 return num_possible_cpus(); 11714 } 11715 11716 /** 11717 * scx_bpf_nr_online_cids - Return current count of online CPUs in cid space 11718 * 11719 * Return num_online_cpus(). The standard model restarts the scheduler on 11720 * hotplug, which lets schedulers treat [0, nr_online_cids) as the online 11721 * range. Schedulers that prefer to handle hotplug without a restart should 11722 * install a custom mapping via scx_bpf_cid_override() and track onlining 11723 * through the ops.cid_online / ops.cid_offline callbacks, starting from the 11724 * mask scx_bpf_online_cmask() returns. 11725 */ 11726 __bpf_kfunc u32 scx_bpf_nr_online_cids(void) 11727 { 11728 return num_online_cpus(); 11729 } 11730 11731 /** 11732 * scx_bpf_online_cmask - Return the online cid mask in the scheduler arena 11733 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11734 * 11735 * Return a kernel-maintained cmask covering [0, scx_bpf_nr_cids()), or NULL if 11736 * the calling program is not associated with a live cid-form scheduler or the 11737 * mask is not allocated yet, as in ops.init_cids(). Treat the mask as read-only 11738 * even though arena memory stays writable by the BPF scheduler. The mask 11739 * follows the SCX hotplug notifications: a cid's bit is updated before 11740 * ops.cid_online/offline() runs for it. The pointer is valid from ops.init() 11741 * through ops.exit(). Root ops.init() runs with hotplug excluded. Other 11742 * contexts can observe concurrent updates. 11743 */ 11744 __bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux) 11745 { 11746 struct scx_sched *sch; 11747 struct scx_cmask *online; 11748 11749 guard(rcu)(); 11750 11751 sch = scx_prog_sched(aux); 11752 if (unlikely(!sch)) 11753 return NULL; 11754 online = sch->online_cmask; 11755 if (unlikely(!online)) 11756 return NULL; 11757 11758 return (void *)scx_kaddr_to_arena(sch, online); 11759 } 11760 11761 /** 11762 * scx_bpf_this_cid - Return the cid of the CPU this program is running on 11763 * 11764 * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. 11765 * The current cpu is trivially valid, so this is just a table lookup. Return 11766 * -EINVAL if called before any scheduler has ever published its cid tables. 11767 */ 11768 __bpf_kfunc s32 scx_bpf_this_cid(void) 11769 { 11770 s16 *tbl; 11771 11772 guard(rcu)(); 11773 11774 tbl = rcu_dereference(scx_cpu_to_cid_tbl); 11775 if (!tbl) 11776 return -EINVAL; 11777 return tbl[raw_smp_processor_id()]; 11778 } 11779 11780 /** 11781 * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask 11782 */ 11783 __bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void) 11784 { 11785 return cpu_possible_mask; 11786 } 11787 11788 /** 11789 * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask 11790 */ 11791 __bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void) 11792 { 11793 return cpu_online_mask; 11794 } 11795 11796 /** 11797 * scx_bpf_put_cpumask - Release a possible/online cpumask 11798 * @cpumask: cpumask to release 11799 */ 11800 __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) 11801 { 11802 /* 11803 * Empty function body because we aren't actually acquiring or releasing 11804 * a reference to a global cpumask, which is read-only in the caller and 11805 * is never released. The acquire / release semantics here are just used 11806 * to make the cpumask is a trusted pointer in the caller. 11807 */ 11808 } 11809 11810 /** 11811 * scx_bpf_task_running - Is task the current scheduling context? 11812 * @p: task of interest 11813 * 11814 * Under proxy execution, this reports the donor rather than the task whose 11815 * code is physically executing. The physical execution context is intentionally 11816 * not exposed to the BPF scheduler, which continues to observe the donor as the 11817 * running scheduling context. 11818 */ 11819 __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) 11820 { 11821 return rcu_access_pointer(task_rq(p)->donor) == p; 11822 } 11823 11824 /** 11825 * scx_bpf_task_cpu - CPU a task is currently associated with 11826 * @p: task of interest 11827 */ 11828 __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) 11829 { 11830 return task_cpu(p); 11831 } 11832 11833 /** 11834 * scx_bpf_task_cid - cid a task is currently associated with 11835 * @p: task of interest 11836 * 11837 * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a 11838 * valid cpu, so this is just a table lookup. Return -EINVAL if called before 11839 * any scheduler has ever published its cid tables. 11840 */ 11841 __bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) 11842 { 11843 s16 *tbl; 11844 11845 /* KF_RCU covers only @p - a sleepable program holds no RCU lock */ 11846 guard(rcu)(); 11847 11848 tbl = rcu_dereference(scx_cpu_to_cid_tbl); 11849 if (!tbl) 11850 return -EINVAL; 11851 return tbl[task_cpu(p)]; 11852 } 11853 11854 /** 11855 * scx_bpf_locked_rq - Return the rq currently locked by SCX 11856 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11857 * 11858 * Returns the rq if a rq lock is currently held by SCX. 11859 * Otherwise emits an error and returns NULL. 11860 */ 11861 __bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux) 11862 { 11863 struct scx_sched *sch; 11864 struct rq *rq; 11865 11866 guard(preempt)(); 11867 11868 sch = scx_prog_sched(aux); 11869 if (unlikely(!sch)) 11870 return NULL; 11871 11872 rq = scx_locked_rq(); 11873 if (!rq) { 11874 scx_error(sch, "accessing rq without holding rq lock"); 11875 return NULL; 11876 } 11877 11878 return rq; 11879 } 11880 11881 /** 11882 * scx_bpf_cpu_curr - Return remote CPU's current scheduling context 11883 * @cpu: CPU of interest 11884 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11885 * 11886 * Under proxy execution, this returns the donor, which supplies the scheduling 11887 * policy and runtime budget, rather than the task whose code is physically 11888 * executing. The physical execution context is intentionally not exposed to 11889 * the BPF scheduler. 11890 * 11891 * Callers must hold RCU read lock (KF_RCU). 11892 */ 11893 __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux) 11894 { 11895 struct scx_sched *sch; 11896 11897 guard(rcu)(); 11898 11899 sch = scx_prog_sched(aux); 11900 if (unlikely(!sch)) 11901 return NULL; 11902 11903 if (!scx_cpu_valid(sch, cpu, NULL)) 11904 return NULL; 11905 11906 return rcu_dereference(cpu_rq(cpu)->donor); 11907 } 11908 11909 /** 11910 * scx_bpf_cid_curr - Return the curr task on the CPU at @cid 11911 * @cid: cid of interest 11912 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 11913 * 11914 * cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU 11915 * read lock (KF_RCU). 11916 */ 11917 __bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux) 11918 { 11919 struct scx_sched *sch; 11920 s32 cpu; 11921 11922 guard(rcu)(); 11923 11924 sch = scx_prog_sched(aux); 11925 if (unlikely(!sch)) 11926 return NULL; 11927 cpu = scx_cid_to_cpu(sch, cid); 11928 if (cpu < 0) 11929 return NULL; 11930 return rcu_dereference(cpu_rq(cpu)->donor); 11931 } 11932 11933 /** 11934 * scx_bpf_tid_to_task - Look up a task by its scx tid 11935 * @tid: task ID previously read from p->scx.tid 11936 * 11937 * Returns the task with the given tid, or NULL if no such task exists. The 11938 * returned pointer is valid until the end of the current RCU read section 11939 * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root 11940 * scheduler; otherwise an error is raised and NULL returned. 11941 */ 11942 __bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) 11943 { 11944 struct sched_ext_entity *scx; 11945 11946 if (!scx_tid_to_task_enabled()) { 11947 struct scx_sched *sch = rcu_dereference(scx_root); 11948 11949 if (sch) 11950 scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK"); 11951 return NULL; 11952 } 11953 11954 scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params); 11955 if (!scx) 11956 return NULL; 11957 11958 return container_of(scx, struct task_struct, scx); 11959 } 11960 11961 u64 __scx_bpf_now(struct rq *rq) 11962 { 11963 /* the caller must be on @rq's cpu or hold its lock */ 11964 lockdep_assert((rq == this_rq() && !preemptible()) || 11965 lockdep_is_held(__rq_lockp(rq))); 11966 11967 if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { 11968 /* if the rq clock is valid, use the cached rq clock */ 11969 return READ_ONCE(rq->scx.clock); 11970 } else { 11971 /* 11972 * Otherwise, return a fresh rq clock. 11973 * 11974 * The rq clock is updated outside of the rq lock. 11975 * In this case, keep the updated rq clock invalid so the next 11976 * read outside the rq lock gets a fresh rq clock. 11977 */ 11978 return sched_clock_cpu(cpu_of(rq)); 11979 } 11980 } 11981 11982 /** 11983 * scx_bpf_now - Returns a high-performance monotonically non-decreasing 11984 * clock for the current CPU. The clock returned is in nanoseconds. 11985 * 11986 * It provides the following properties: 11987 * 11988 * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently 11989 * to account for execution time and track tasks' runtime properties. 11990 * Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which 11991 * eventually reads a hardware timestamp counter -- is neither performant nor 11992 * scalable. scx_bpf_now() aims to provide a high-performance clock by 11993 * using the rq clock in the scheduler core whenever possible. 11994 * 11995 * 2) High enough resolution for the BPF scheduler use cases: In most BPF 11996 * scheduler use cases, the required clock resolution is lower than the most 11997 * accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically 11998 * uses the rq clock in the scheduler core whenever it is valid. It considers 11999 * that the rq clock is valid from the time the rq clock is updated 12000 * (update_rq_clock) until the rq is unlocked (rq_unpin_lock). 12001 * 12002 * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now() 12003 * guarantees the clock never goes backward when comparing them in the same 12004 * CPU. On the other hand, when comparing clocks in different CPUs, there 12005 * is no such guarantee -- the clock can go backward. It provides a 12006 * monotonically *non-decreasing* clock so that it would provide the same 12007 * clock values in two different scx_bpf_now() calls in the same CPU 12008 * during the same period of when the rq clock is valid. 12009 */ 12010 __bpf_kfunc u64 scx_bpf_now(void) 12011 { 12012 /* 12013 * Note that scx_bpf_now() is re-entrant between a process context and 12014 * an interrupt context (e.g., timer interrupt). However, we don't need 12015 * to consider the race between them because such race is not observable 12016 * from a caller. 12017 */ 12018 guard(preempt)(); 12019 return __scx_bpf_now(this_rq()); 12020 } 12021 12022 static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) 12023 { 12024 int cpu; 12025 12026 /* Aggregate per-CPU event counters into @events. */ 12027 memset(events, 0, sizeof(*events)); 12028 for_each_possible_cpu(cpu) { 12029 struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; 12030 #define SCX_EVENT(name) (events->name += READ_ONCE(e_cpu->name)) 12031 SCX_EVENTS_LIST(SCX_EVENT); 12032 #undef SCX_EVENT 12033 } 12034 } 12035 12036 /** 12037 * scx_bpf_events - Read the event counters of the calling scheduler 12038 * @events: output buffer from a BPF program 12039 * @events__sz: @events len, must end in '__sz' for the verifier 12040 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 12041 * 12042 * Read the event counters of the scheduler associated with the calling program. 12043 * @events is zeroed when no scheduler can be resolved. 12044 */ 12045 __bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, size_t events__sz, 12046 const struct bpf_prog_aux *aux) 12047 { 12048 struct scx_sched *sch; 12049 struct scx_event_stats e_sys; 12050 12051 rcu_read_lock(); 12052 sch = scx_prog_sched(aux); 12053 if (sch) 12054 scx_read_events(sch, &e_sys); 12055 else 12056 memset(&e_sys, 0, sizeof(e_sys)); 12057 rcu_read_unlock(); 12058 12059 /* 12060 * We cannot entirely trust a BPF-provided size since a BPF program 12061 * might be compiled against a different vmlinux.h, of which 12062 * scx_event_stats would be larger (a newer vmlinux.h) or smaller 12063 * (an older vmlinux.h). Hence, we use the smaller size to avoid 12064 * memory corruption. 12065 */ 12066 events__sz = min(events__sz, sizeof(*events)); 12067 memcpy(events, &e_sys, events__sz); 12068 } 12069 12070 #ifdef CONFIG_CGROUP_SCHED 12071 /** 12072 * scx_bpf_task_cgroup - Return the sched cgroup of a task 12073 * @p: task of interest 12074 * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs 12075 * 12076 * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with 12077 * from the scheduler's POV. SCX operations should use this function to 12078 * determine @p's current cgroup as, unlike following @p->cgroups, 12079 * @p->sched_task_group is stable for the duration of the SCX op. See 12080 * SCX_CALL_OP_TASK() for details. 12081 */ 12082 __bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p, 12083 const struct bpf_prog_aux *aux) 12084 { 12085 struct task_group *tg = p->sched_task_group; 12086 struct cgroup *cgrp = &cgrp_dfl_root.cgrp; 12087 struct scx_sched *sch; 12088 12089 guard(rcu)(); 12090 12091 sch = scx_prog_sched(aux); 12092 if (unlikely(!sch)) 12093 goto out; 12094 12095 if (!scx_kf_arg_task_ok(sch, p)) 12096 goto out; 12097 12098 cgrp = tg_cgrp(tg); 12099 12100 out: 12101 cgroup_get(cgrp); 12102 return cgrp; 12103 } 12104 12105 /** 12106 * scx_bpf_cgroup_nr_cpus - Return the number of CPUs in a cgroup's cpuset 12107 * @cgrp: cgroup of interest 12108 * 12109 * Return the number of CPUs in @cgrp's effective cpuset, which is inherited 12110 * from the nearest ancestor with the cpuset controller enabled. This matches 12111 * the count used by fair's group share calculation and lets hierarchical BPF 12112 * schedulers bound a group's weight by the CPUs it can actually run on. 12113 * 12114 * The value is a snapshot and may change at any time through cpuset updates or 12115 * CPU hotplug. Schedulers should re-read it when recomputing group state. 12116 */ 12117 __bpf_kfunc u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) 12118 { 12119 return cpuset_num_cpus(cgrp); 12120 } 12121 #endif /* CONFIG_CGROUP_SCHED */ 12122 12123 __bpf_kfunc_end_defs(); 12124 12125 BTF_KFUNCS_START(scx_kfunc_ids_any) 12126 BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); 12127 BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); 12128 BTF_ID_FLAGS(func, scx_bpf_task_set_lazy_resched, KF_IMPLICIT_ARGS | KF_RCU); 12129 BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) 12130 BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) 12131 BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) 12132 BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS) 12133 BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL) 12134 BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS) 12135 BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS) 12136 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_IMPLICIT_ARGS | KF_ITER_NEW | KF_RCU_PROTECTED) 12137 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL) 12138 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY) 12139 BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_IMPLICIT_ARGS) 12140 BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_IMPLICIT_ARGS) 12141 BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS) 12142 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) 12143 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) 12144 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) 12145 BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS) 12146 BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS) 12147 BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) 12148 BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) 12149 BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) 12150 BTF_ID_FLAGS(func, scx_bpf_nr_cids) 12151 BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) 12152 BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS | KF_ARENA_RET) 12153 BTF_ID_FLAGS(func, scx_bpf_this_cid) 12154 BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) 12155 BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) 12156 BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) 12157 BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) 12158 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) 12159 BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) 12160 BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) 12161 BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) 12162 BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) 12163 BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) 12164 BTF_ID_FLAGS(func, scx_bpf_now) 12165 BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS) 12166 #ifdef CONFIG_CGROUP_SCHED 12167 BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) 12168 BTF_ID_FLAGS(func, scx_bpf_cgroup_nr_cpus, KF_RCU) 12169 #endif 12170 BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) 12171 BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) 12172 BTF_ID_FLAGS(func, scx_bpf_sub_caps, KF_IMPLICIT_ARGS) 12173 BTF_ID_FLAGS(func, scx_bpf_sub_kill_bstr, KF_IMPLICIT_ARGS) 12174 BTF_KFUNCS_END(scx_kfunc_ids_any) 12175 12176 static const struct btf_kfunc_id_set scx_kfunc_set_any = { 12177 .owner = THIS_MODULE, 12178 .set = &scx_kfunc_ids_any, 12179 .filter = scx_kfunc_context_filter, 12180 }; 12181 12182 /* 12183 * Kfuncs that tracing programs can't call. A tracing program can run in any 12184 * kernel context, including with an rq lock held. scx_locked_rq() only tracks 12185 * rq locks held across SCX callbacks and returns NULL there. 12186 * 12187 * scx_bpf_cpuperf_set() and scx_bpf_cidperf_set() take the target cpu's rq lock 12188 * when scx_locked_rq() returns NULL. Taking it while the context already holds 12189 * an rq lock, the same one or another cpu's, can deadlock. 12190 * 12191 * scx_bpf_destroy_dsq() takes a user DSQ lock. So do 12192 * bpf_iter_scx_dsq_next() and bpf_iter_scx_dsq_destroy(), the latter only 12193 * after a successful next() has linked the cursor. A tracing program can 12194 * run while that lock is already held and recursively acquire it. 12195 * 12196 * Before returning, scx_bpf_sub_grant() and scx_bpf_sub_revoke() invoke 12197 * ops.sub_caps_updated() of the sub-schedulers whose caps changed, so those ops 12198 * run in the tracing program's context. No SCX op expects to run from an 12199 * arbitrary tracing context. One that calls scx_bpf_cidperf_set() hits the 12200 * deadlock above. 12201 * 12202 * Members must also stay in scx_kfunc_ids_any so that struct_ops and syscall 12203 * programs keep their access. 12204 */ 12205 BTF_KFUNCS_START(scx_kfunc_ids_no_tracing) 12206 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) 12207 BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) 12208 BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) 12209 BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) 12210 BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS) 12211 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL) 12212 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY) 12213 BTF_KFUNCS_END(scx_kfunc_ids_no_tracing) 12214 12215 /* 12216 * cpu-form kfuncs that are forbidden from cid-form schedulers 12217 * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must 12218 * use the cid-form alternative (cid/cmask kfuncs). 12219 * 12220 * Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter 12221 * tests this set independently and rejects matches before the per-op 12222 * allow-list check runs. 12223 * 12224 * pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and 12225 * intersects flags across duplicate entries, so each entry must carry the 12226 * same flags as the kfunc's primary declaration; otherwise the flags get 12227 * dropped globally. 12228 */ 12229 BTF_KFUNCS_START(scx_kfunc_ids_cpu_only) 12230 BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) 12231 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) 12232 BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) 12233 BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) 12234 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) 12235 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) 12236 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) 12237 BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) 12238 BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) 12239 BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) 12240 BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) 12241 BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) 12242 BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) 12243 BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) 12244 BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) 12245 BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) 12246 BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) 12247 BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) 12248 BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) 12249 BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) 12250 BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) 12251 BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) 12252 BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) 12253 BTF_KFUNCS_END(scx_kfunc_ids_cpu_only) 12254 12255 /* 12256 * Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc 12257 * group; an op may permit zero or more groups, with the union expressed in 12258 * scx_kf_allow_flags[]. The verifier-time filter (scx_kfunc_context_filter()) 12259 * consults this table to decide whether a context-sensitive kfunc is callable 12260 * from a given SCX op. 12261 */ 12262 enum scx_kf_allow_flags { 12263 SCX_KF_ALLOW_UNLOCKED = 1 << 0, 12264 SCX_KF_ALLOW_INIT_CIDS = 1 << 1, 12265 SCX_KF_ALLOW_CPU_RELEASE = 1 << 2, 12266 SCX_KF_ALLOW_DISPATCH = 1 << 3, 12267 SCX_KF_ALLOW_ENQUEUE = 1 << 4, 12268 SCX_KF_ALLOW_SELECT_CPU = 1 << 5, 12269 }; 12270 12271 /* 12272 * Map each SCX op to the union of kfunc groups it permits, indexed by 12273 * SCX_OP_IDX(op). Ops not listed only permit kfuncs that are not 12274 * context-sensitive. 12275 */ 12276 static const u32 scx_kf_allow_flags[] = { 12277 [SCX_OP_IDX(select_cpu)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, 12278 [SCX_OP_IDX(enqueue)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, 12279 [SCX_OP_IDX(dispatch)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, 12280 [SCX_OP_IDX(cpu_release)] = SCX_KF_ALLOW_CPU_RELEASE, 12281 [SCX_OP_IDX(init_task)] = SCX_KF_ALLOW_UNLOCKED, 12282 [SCX_OP_IDX(dump)] = SCX_KF_ALLOW_UNLOCKED, 12283 #ifdef CONFIG_EXT_GROUP_SCHED 12284 [SCX_OP_IDX(cgroup_init)] = SCX_KF_ALLOW_UNLOCKED, 12285 [SCX_OP_IDX(cgroup_exit)] = SCX_KF_ALLOW_UNLOCKED, 12286 [SCX_OP_IDX(cgroup_prep_move)] = SCX_KF_ALLOW_UNLOCKED, 12287 [SCX_OP_IDX(cgroup_cancel_move)] = SCX_KF_ALLOW_UNLOCKED, 12288 [SCX_OP_IDX(cgroup_set_weight)] = SCX_KF_ALLOW_UNLOCKED, 12289 [SCX_OP_IDX(cgroup_set_bandwidth)] = SCX_KF_ALLOW_UNLOCKED, 12290 [SCX_OP_IDX(cgroup_set_idle)] = SCX_KF_ALLOW_UNLOCKED, 12291 #endif /* CONFIG_EXT_GROUP_SCHED */ 12292 [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, 12293 [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, 12294 [SCX_OP_IDX(sub_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, 12295 [SCX_OP_IDX(sub_child_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, 12296 [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, 12297 [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, 12298 [SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS, 12299 [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED, 12300 [SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED, 12301 }; 12302 12303 /* 12304 * Verifier-time filter for SCX kfuncs. Registered via the .filter field on 12305 * each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc 12306 * call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or 12307 * BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the 12308 * kfunc - so the filter must short-circuit on kfuncs it doesn't govern by 12309 * falling through to "allow" when none of the SCX sets contain the kfunc. 12310 */ 12311 int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) 12312 { 12313 bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id); 12314 bool in_init_cids = btf_id_set8_contains(&scx_kfunc_ids_init_cids, kfunc_id); 12315 bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id); 12316 bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id); 12317 bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id); 12318 bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id); 12319 bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id); 12320 bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); 12321 bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); 12322 bool in_cid = btf_id_set8_contains(&scx_kfunc_ids_cid, kfunc_id); 12323 bool in_no_tracing = btf_id_set8_contains(&scx_kfunc_ids_no_tracing, kfunc_id); 12324 u32 moff, flags; 12325 12326 /* Not an SCX kfunc - allow. */ 12327 if (!(in_unlocked || in_init_cids || in_select_cpu || in_enqueue || in_dispatch || 12328 in_cpu_release || in_idle || in_any || in_cid)) 12329 return 0; 12330 12331 /* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */ 12332 if (prog->type == BPF_PROG_TYPE_SYSCALL) 12333 return (in_unlocked || in_select_cpu || in_idle || in_any || in_cid) ? 0 : -EACCES; 12334 12335 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) 12336 return (!in_no_tracing && (in_any || in_idle || in_cid)) ? 0 : -EACCES; 12337 12338 /* 12339 * add_subprog_and_kfunc() collects all kfunc calls, including dead code 12340 * guarded by bpf_ksym_exists(), before check_attach_btf_id() sets 12341 * prog->aux->st_ops. Allow all kfuncs when st_ops is not yet set; 12342 * do_check_main() re-runs the filter with st_ops set and enforces the 12343 * actual restrictions. 12344 */ 12345 if (!prog->aux->st_ops) 12346 return 0; 12347 12348 /* 12349 * Non-SCX struct_ops: SCX kfuncs are not permitted. 12350 * 12351 * Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid 12352 * (cid-form) are valid SCX struct_ops. Member offsets match between 12353 * the two (verified by BUILD_BUG_ON in scx_init()), so the shared 12354 * scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to 12355 * both. 12356 */ 12357 if (prog->aux->st_ops != &bpf_sched_ext_ops && 12358 prog->aux->st_ops != &bpf_sched_ext_ops_cid) 12359 return -EACCES; 12360 12361 /* 12362 * cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both 12363 * small s32s and trivially confused, so cpu-only kfuncs are rejected at 12364 * load time. The reverse (cpu-form calling cid-form kfuncs) is 12365 * intentionally permissive to ease gradual cpumask -> cid migration. 12366 */ 12367 if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only) 12368 return -EACCES; 12369 12370 /* SCX struct_ops: check the per-op allow list. */ 12371 if (in_any || in_idle || in_cid) 12372 return 0; 12373 12374 moff = prog->aux->attach_st_ops_member_off; 12375 flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)]; 12376 12377 if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked) 12378 return 0; 12379 if ((flags & SCX_KF_ALLOW_INIT_CIDS) && in_init_cids) 12380 return 0; 12381 if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release) 12382 return 0; 12383 if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch) 12384 return 0; 12385 if ((flags & SCX_KF_ALLOW_ENQUEUE) && in_enqueue) 12386 return 0; 12387 if ((flags & SCX_KF_ALLOW_SELECT_CPU) && in_select_cpu) 12388 return 0; 12389 12390 return -EACCES; 12391 } 12392 12393 static int __init scx_init(void) 12394 { 12395 int ret; 12396 12397 /* 12398 * sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv. 12399 * Both bpf_scx_init_member() and bpf_scx_check_member() use offsets 12400 * from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets 12401 * matching for the shared fields. Catch any drift at boot. 12402 */ 12403 #define CID_OFFSET_MATCH(cpu_field, cid_field) \ 12404 BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) != \ 12405 offsetof(struct sched_ext_ops_cid, cid_field)) 12406 /* data fields used by bpf_scx_init_member() */ 12407 CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch); 12408 CID_OFFSET_MATCH(flags, flags); 12409 CID_OFFSET_MATCH(name, name); 12410 CID_OFFSET_MATCH(timeout_ms, timeout_ms); 12411 CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); 12412 CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); 12413 CID_OFFSET_MATCH(cid_shard_size, cid_shard_size); 12414 CID_OFFSET_MATCH(rescue_bandwidth_ppt, rescue_bandwidth_ppt); 12415 CID_OFFSET_MATCH(rescue_quantum_us, rescue_quantum_us); 12416 CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); 12417 /* shared callbacks: the union view requires byte-for-byte offset match */ 12418 CID_OFFSET_MATCH(enqueue, enqueue); 12419 CID_OFFSET_MATCH(dequeue, dequeue); 12420 CID_OFFSET_MATCH(dispatch, dispatch); 12421 CID_OFFSET_MATCH(tick, tick); 12422 CID_OFFSET_MATCH(runnable, runnable); 12423 CID_OFFSET_MATCH(running, running); 12424 CID_OFFSET_MATCH(stopping, stopping); 12425 CID_OFFSET_MATCH(quiescent, quiescent); 12426 CID_OFFSET_MATCH(yield, yield); 12427 CID_OFFSET_MATCH(core_sched_before, core_sched_before); 12428 CID_OFFSET_MATCH(set_weight, set_weight); 12429 CID_OFFSET_MATCH(update_idle, update_idle); 12430 CID_OFFSET_MATCH(init_task, init_task); 12431 CID_OFFSET_MATCH(exit_task, exit_task); 12432 CID_OFFSET_MATCH(enable, enable); 12433 CID_OFFSET_MATCH(disable, disable); 12434 CID_OFFSET_MATCH(dump, dump); 12435 CID_OFFSET_MATCH(dump_task, dump_task); 12436 CID_OFFSET_MATCH(sub_attach, sub_attach); 12437 CID_OFFSET_MATCH(sub_detach, sub_detach); 12438 CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); 12439 CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated); 12440 CID_OFFSET_MATCH(sub_cid_sched_updated, sub_cid_sched_updated); 12441 CID_OFFSET_MATCH(sub_child_ecaps_updated, sub_child_ecaps_updated); 12442 CID_OFFSET_MATCH(init_cids, init_cids); 12443 CID_OFFSET_MATCH(init, init); 12444 CID_OFFSET_MATCH(exit, exit); 12445 /* renamed callbacks must occupy the same slot as their cpu-form sibling */ 12446 CID_OFFSET_MATCH(select_cpu, select_cid); 12447 CID_OFFSET_MATCH(set_cpumask, set_cmask); 12448 CID_OFFSET_MATCH(cpu_online, cid_online); 12449 CID_OFFSET_MATCH(cpu_offline, cid_offline); 12450 CID_OFFSET_MATCH(dump_cpu, dump_cid); 12451 #ifdef CONFIG_EXT_GROUP_SCHED 12452 CID_OFFSET_MATCH(cgroup_init, cpuctl_init); 12453 CID_OFFSET_MATCH(cgroup_exit, cpuctl_exit); 12454 CID_OFFSET_MATCH(cgroup_prep_move, cpuctl_prep_move); 12455 CID_OFFSET_MATCH(cgroup_move, cpuctl_move); 12456 CID_OFFSET_MATCH(cgroup_cancel_move, cpuctl_cancel_move); 12457 CID_OFFSET_MATCH(cgroup_set_weight, cpuctl_set_weight); 12458 CID_OFFSET_MATCH(cgroup_set_bandwidth, cpuctl_set_bandwidth); 12459 CID_OFFSET_MATCH(cgroup_set_idle, cpuctl_set_idle); 12460 #endif 12461 /* @priv tail must align since both share the same data block */ 12462 CID_OFFSET_MATCH(priv, priv); 12463 /* 12464 * cid-form must end exactly at @priv - scx_validate_ops() skips 12465 * cpu_acquire/cpu_release for cid-form because reading those fields 12466 * past the BPF allocation would be UB. 12467 */ 12468 BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) != 12469 offsetofend(struct sched_ext_ops, priv)); 12470 #undef CID_OFFSET_MATCH 12471 12472 /* 12473 * kfunc registration can't be done from init_sched_ext_class() as 12474 * register_btf_kfunc_id_set() needs most of the system to be up. 12475 * 12476 * Some kfuncs are context-sensitive and can only be called from 12477 * specific SCX ops. They are grouped into per-context BTF sets, each 12478 * registered with scx_kfunc_context_filter as its .filter callback. The 12479 * BPF core dedups identical filter pointers per hook 12480 * (btf_populate_kfunc_set()), so the filter is invoked exactly once per 12481 * kfunc lookup; it consults scx_kf_allow_flags[] to enforce per-op 12482 * restrictions at verify time. 12483 */ 12484 if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 12485 &scx_kfunc_set_enqueue_dispatch)) || 12486 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 12487 &scx_kfunc_set_dispatch)) || 12488 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 12489 &scx_kfunc_set_cpu_release)) || 12490 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 12491 &scx_kfunc_set_unlocked)) || 12492 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, 12493 &scx_kfunc_set_unlocked)) || 12494 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 12495 &scx_kfunc_set_any)) || 12496 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, 12497 &scx_kfunc_set_any)) || 12498 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, 12499 &scx_kfunc_set_any))) { 12500 pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret); 12501 return ret; 12502 } 12503 12504 ret = scx_idle_init(); 12505 if (ret) { 12506 pr_err("sched_ext: Failed to initialize idle tracking (%d)\n", ret); 12507 return ret; 12508 } 12509 12510 ret = scx_cid_kfunc_init(); 12511 if (ret) { 12512 pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret); 12513 return ret; 12514 } 12515 12516 ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); 12517 if (ret) { 12518 pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); 12519 return ret; 12520 } 12521 12522 ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid); 12523 if (ret) { 12524 pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret); 12525 return ret; 12526 } 12527 12528 ret = register_pm_notifier(&scx_pm_notifier); 12529 if (ret) { 12530 pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); 12531 return ret; 12532 } 12533 12534 scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj); 12535 if (!scx_kset) { 12536 pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n"); 12537 return -ENOMEM; 12538 } 12539 12540 ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group); 12541 if (ret < 0) { 12542 pr_err("sched_ext: Failed to add global attributes\n"); 12543 return ret; 12544 } 12545 12546 return 0; 12547 } 12548 __initcall(scx_init); 12549 12550 /* 12551 * Compatibility markers for userspace. Existence of a marker function 12552 * represents that the kernel supports that sched-ext feature. 12553 */ 12554 12555 /* 12556 * scx_compat_marker_cgroup_set_bandwidth_may_sleep: advertises that 12557 * ops.cgroup_set_bandwidth() may be implemented as a sleepable callback. 12558 */ 12559 #ifdef CONFIG_EXT_GROUP_SCHED 12560 DEFINE_SCX_COMPAT_MARKER(cgroup_set_bandwidth_may_sleep); 12561 #endif /* CONFIG_EXT_GROUP_SCHED */ 12562