Lines Matching defs:slice
251 MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)");
379 * $curr's slice. We say yes here and resolve it at the end of dispatch.
386 * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch,
394 * A protected slice refuses the preemption and the cpu stays
635 u64 slice;
1168 * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero
1173 * 42-0 requested slice duration in nsecs
1198 * update_curr_scx() RMWs the slice and the cap check for slice extension is
1212 * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration.
1214 * While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is
1222 /* clear a pending slice request */
1270 * set_task_slice_keep_oob - Set @p's slice, leaving any pending oob request
1272 * @slice: slice to set
1274 * While %SCX_TASK_PROTECTED is set, BPF schedulers may not modify the slice.
1277 static bool set_task_slice_keep_oob(struct task_struct *p, u64 slice)
1284 p->scx.slice = slice;
1288 /* set @p's slice, superseding any pending out-of-band request */
1289 bool scx_set_task_slice(struct task_struct *p, u64 slice)
1291 if (!set_task_slice_keep_oob(p, slice))
1298 * scx_task_slice_ended - @p's slice is consumed or given up
1302 * End what rides on the slice - the protection, and the rescue if @p is being
1305 * A dequeue normally ends the slice too. The exception is a save/restore pair
1307 * dequeue and enqueue while it keeps executing, so the slice continues. A
1308 * queued task instead loses its DSQ position on any dequeue and the slice ends
1320 /* request @p's slice to be set to @slice, see the write rules above */
1321 static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice)
1325 if (slice == SCX_SLICE_INF) {
1327 } else if (unlikely(slice >= SCX_SLICE_OOB_DUR_MASK)) {
1331 dur = slice;
1339 * Apply a pending out-of-band slice request under @rq's lock. A request whose
1342 * protected slice refuses both. %SCX_EV_SLICE_DENIED counts the denials. See
1347 u64 oob, dur, slice;
1364 slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur;
1366 if (slice > p->scx.slice &&
1372 if (unlikely(!set_task_slice_keep_oob(p, slice)))
1377 * A dsq insert kfunc doesn't write slice or dsq_vtime. The verdict carries them
1378 * and they are committed here, at the insertion. A zero @slice keeps the
1381 static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags)
1383 if (slice) {
1384 p->scx.slice = slice;
1386 * An explicit slice supersedes a pending oob request. A carried
1391 } else if (!p->scx.slice) {
1392 p->scx.slice = 1;
1404 /* apply even on 0 delta_exec, callers may still act on the slice */
1411 if (curr->scx.slice != SCX_SLICE_INF)
1412 curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec);
1442 * through e.g. SAVE/RESTORE cycles and slice extensions.
1594 u64 slice, u64 vtime, u64 enq_flags)
1634 * @dsq is locked and @enq_flags is sanitized. Commit the carried slice
1638 apply_slice_vtime(p, slice, vtime, enq_flags);
1869 u64 slice, u64 vtime, u64 enq_flags)
1893 p->scx.ddsp_slice = slice;
1926 u64 ddsp_enq_flags, slice, vtime;
1966 slice = p->scx.ddsp_slice;
1970 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
2356 /* a yield gives the slice up */
2370 /* like a plain yield, giving the slice up ends the protection */
2383 * task's slice to 0 and triggering reschedule on the target CPU.
2735 * @slice: slice carried by the insert verdict, 0 keeps the current value
2748 u64 slice, u64 vtime, u64 enq_flags)
2761 scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime,
2800 slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE);
2805 p, slice, vtime,
2809 apply_slice_vtime(p, slice, vtime, enq_flags);
2832 * @slice: slice carried by the insert verdict, 0 keeps the current value
2849 u64 slice, u64 vtime, u64 enq_flags)
2908 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
2910 scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
2923 ent->slice, ent->vtime, ent->enq_flags);
2971 * If @prev is runnable & has slice left, it has priority and
2980 if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice &&
3020 * - rq_is_open() can't reliably tell when and how slice is going to be
3055 /* apply any pending out-of-band slice request before the tick decision */
3060 * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be
3067 if (p->scx.slice == SCX_SLICE_INF) {
3071 * can't have an infinite slice while bypassing.
3088 * slice is sufficient by itself to require the tick.
3158 * If the slice is consumed, protection ends with it. A rescuee
3161 if (!p->scx.slice) {
3176 * If @p has slice left and is being put, @p is getting
3183 * scheduler zeroed the slice in ops.stopping() above.
3185 if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) &&
3419 * If we're keeping @prev, replenish slice if necessary and keep running
3424 if (!p->scx.slice) {
3425 /* the slice is consumed, protection ends */
3434 if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) {
3439 printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n",
3617 * Carry the slice refill and let the insertion commit
3861 * While disabling, always resched as we can't trust the slice
3869 if (!curr->scx.slice)
4127 scx->slice = SCX_SLICE_DFL;
4408 u64 slice = p->scx.ddsp_slice;
4416 dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
4549 * Zero the slice to evict it. The enqueue gate blocks new capless
4748 * slice state. If another task is dispatched from a DSQ,
5839 * fine tune. Assuming every task on @donor_dsq runs their full slice,
6172 * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice
6176 * - pick_next_task() suppresses zero slice warning.
6927 scx_dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u",
6928 p->scx.dsq_vtime, p->scx.slice, p->scx.weight);
8088 if ((off >= offsetof(struct task_struct, scx.slice) &&
8089 off + size <= offsetofend(struct task_struct, scx.slice)) ||
8098 /* cid-form rejects direct slice and dsq_vtime writes in favor of the kfuncs */
8887 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
8894 mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags);
8907 .slice = slice,
8919 * @slice: duration @p can run for in nsecs, 0 to keep the current value
8946 * @p is allowed to run for @slice. The scheduling path is triggered on slice
8947 * exhaustion. If zero, the current residual slice is maintained. If
8956 u64 slice, u64 enq_flags,
8969 scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags);
8978 u64 slice, u64 enq_flags,
8981 scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux);
8985 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
8990 scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ);
8998 u64 slice;
9008 * @args->slice: duration @p can run for in nsecs, 0 to keep the current value
9047 return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice,
9055 u64 slice, u64 vtime, u64 enq_flags)
9083 scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags);
9181 * Apply vtime and slice updates before moving. @p is still on $src_dsq
9188 scx_set_task_slice(p, kit->slice);
9328 * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs
9330 * @slice: duration the moved task can run for in nsecs
9332 * Override the slice of the next task that will be moved from @it__iter using
9334 * slice duration is kept.
9337 u64 slice)
9341 kit->slice = slice;
9351 * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice
9380 * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update.
9408 * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice()
9564 * scx_bpf_task_set_slice - Set task's time slice
9566 * @slice: time slice to set in nsecs
9569 * Set @p's time slice. @p must be on the calling scheduler. The value is
9570 * applied whether or not the caller holds @p's rq lock - see the slice write
9573 * Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on
9575 * always allowed. On the stashed path the slice is packed into an atomic64_t
9576 * with the scheduler id and a flag bit, so a slice too large to fit is clamped
9581 __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
9609 set_task_slice_oob(sch, p, slice);
9614 if (slice > p->scx.slice &&
9620 if (unlikely(!scx_set_task_slice(p, slice)))