1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/ftrace.h> 3 #include <linux/tracepoint.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/init.h> 7 #include <linux/rv.h> 8 #include <rv/instrumentation.h> 9 10 #define MODULE_NAME "nomiss" 11 12 #include <uapi/linux/sched/types.h> 13 #include <trace/events/syscalls.h> 14 #include <trace/events/sched.h> 15 #include <trace/events/task.h> 16 #include <rv_trace.h> 17 18 #define RV_MON_TYPE RV_MON_PER_OBJ 19 #define HA_TIMER_TYPE HA_TIMER_WHEEL 20 /* The start condition is on sched_switch, it's dangerous to allocate there */ 21 #define DA_SKIP_AUTO_ALLOC 22 typedef struct sched_dl_entity *monitor_target; 23 #include "nomiss.h" 24 #include <rv/ha_monitor.h> 25 #include <monitors/deadline/deadline.h> 26 27 /* 28 * User configurable deadline threshold. If the total utilisation of deadline 29 * tasks is larger than 1, they are only guaranteed bounded tardiness. See 30 * Documentation/scheduler/sched-deadline.rst for more details. 31 * The minimum tardiness without sched_feat(HRTICK_DL) is 1 tick to accommodate 32 * for throttle enforced on the next tick. 33 */ 34 static u64 deadline_thresh = TICK_NSEC; 35 module_param(deadline_thresh, ullong, 0644); 36 #define DEADLINE_NS(ha_mon) (ha_get_target(ha_mon)->dl_deadline + deadline_thresh) 37 38 static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns) 39 { 40 if (env == clk_nomiss) 41 return ha_get_clk_ns(ha_mon, env, time_ns); 42 else if (env == is_constr_dl_nomiss) 43 return !dl_is_implicit(ha_get_target(ha_mon)); 44 else if (env == is_defer_nomiss) 45 return ha_get_target(ha_mon)->dl_defer; 46 return ENV_INVALID_VALUE; 47 } 48 49 static void ha_reset_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns) 50 { 51 if (env == clk_nomiss) 52 ha_reset_clk_ns(ha_mon, env, time_ns); 53 } 54 55 static inline bool ha_verify_invariants(struct ha_monitor *ha_mon, 56 enum states curr_state, enum events event, 57 enum states next_state, u64 time_ns) 58 { 59 if (curr_state == ready_nomiss) 60 return ha_check_invariant_ns(ha_mon, clk_nomiss, time_ns); 61 else if (curr_state == running_nomiss) 62 return ha_check_invariant_ns(ha_mon, clk_nomiss, time_ns); 63 return true; 64 } 65 66 static inline void ha_convert_inv_guard(struct ha_monitor *ha_mon, 67 enum states curr_state, enum events event, 68 enum states next_state, u64 time_ns) 69 { 70 if (curr_state == next_state) 71 return; 72 if (curr_state == ready_nomiss) 73 ha_inv_to_guard(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns); 74 else if (curr_state == running_nomiss) 75 ha_inv_to_guard(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns); 76 } 77 78 static inline bool ha_verify_guards(struct ha_monitor *ha_mon, 79 enum states curr_state, enum events event, 80 enum states next_state, u64 time_ns) 81 { 82 bool res = true; 83 84 if (curr_state == ready_nomiss && event == dl_replenish_nomiss) 85 ha_reset_env(ha_mon, clk_nomiss, time_ns); 86 else if (curr_state == ready_nomiss && event == dl_throttle_nomiss) 87 res = ha_get_env(ha_mon, is_defer_nomiss, time_ns) == 1ull; 88 else if (curr_state == idle_nomiss && event == dl_replenish_nomiss) 89 ha_reset_env(ha_mon, clk_nomiss, time_ns); 90 else if (curr_state == running_nomiss && event == dl_replenish_nomiss) 91 ha_reset_env(ha_mon, clk_nomiss, time_ns); 92 else if (curr_state == sleeping_nomiss && event == dl_replenish_nomiss) 93 ha_reset_env(ha_mon, clk_nomiss, time_ns); 94 else if (curr_state == sleeping_nomiss && event == dl_throttle_nomiss) 95 res = ha_get_env(ha_mon, is_constr_dl_nomiss, time_ns) == 1ull || 96 ha_get_env(ha_mon, is_defer_nomiss, time_ns) == 1ull; 97 else if (curr_state == throttled_nomiss && event == dl_replenish_nomiss) 98 ha_reset_env(ha_mon, clk_nomiss, time_ns); 99 return res; 100 } 101 102 static inline void ha_setup_invariants(struct ha_monitor *ha_mon, 103 enum states curr_state, enum events event, 104 enum states next_state, u64 time_ns) 105 { 106 if (next_state == curr_state && event != dl_replenish_nomiss) 107 return; 108 if (next_state == ready_nomiss) 109 ha_start_timer_ns(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns); 110 else if (next_state == running_nomiss) 111 ha_start_timer_ns(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns); 112 else if (curr_state == ready_nomiss) 113 ha_cancel_timer(ha_mon); 114 else if (curr_state == running_nomiss) 115 ha_cancel_timer(ha_mon); 116 } 117 118 static bool ha_verify_constraint(struct ha_monitor *ha_mon, 119 enum states curr_state, enum events event, 120 enum states next_state, u64 time_ns) 121 { 122 if (!ha_verify_invariants(ha_mon, curr_state, event, next_state, time_ns)) 123 return false; 124 125 ha_convert_inv_guard(ha_mon, curr_state, event, next_state, time_ns); 126 127 if (!ha_verify_guards(ha_mon, curr_state, event, next_state, time_ns)) 128 return false; 129 130 ha_setup_invariants(ha_mon, curr_state, event, next_state, time_ns); 131 132 return true; 133 } 134 135 static void handle_dl_replenish(void *data, struct sched_dl_entity *dl_se, 136 int cpu, u8 type) 137 { 138 if (is_supported_type(type)) 139 da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_replenish_nomiss); 140 } 141 142 static void handle_dl_throttle(void *data, struct sched_dl_entity *dl_se, 143 int cpu, u8 type) 144 { 145 if (is_supported_type(type)) 146 da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_throttle_nomiss); 147 } 148 149 static void handle_dl_server_stop(void *data, struct sched_dl_entity *dl_se, 150 int cpu, u8 type) 151 { 152 /* 153 * This isn't the standard use of da_handle_start_run_event since this 154 * event cannot only occur from the initial state. 155 * It is fine to use here because it always brings to a known state and 156 * the fact we "pretend" the transition starts from the initial state 157 * has no side effect. 158 */ 159 if (is_supported_type(type)) 160 da_handle_start_run_event(EXPAND_ID(dl_se, cpu, type), dl_server_stop_nomiss); 161 } 162 163 static inline void handle_server_switch(struct task_struct *next, int cpu, u8 type) 164 { 165 struct sched_dl_entity *dl_se = get_server(next, type); 166 167 if (dl_se && is_idle_task(next)) 168 da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_server_idle_nomiss); 169 } 170 171 static void handle_sched_switch(void *data, bool preempt, 172 struct task_struct *prev, 173 struct task_struct *next, 174 unsigned int prev_state) 175 { 176 int cpu = task_cpu(next); 177 178 if (prev_state != TASK_RUNNING && !preempt && prev->policy == SCHED_DEADLINE) 179 da_handle_event(EXPAND_ID_TASK(prev), sched_switch_suspend_nomiss); 180 if (next->policy == SCHED_DEADLINE) 181 da_handle_start_run_event(EXPAND_ID_TASK(next), sched_switch_in_nomiss); 182 183 /* 184 * The server is available in next only if the next task is boosted, 185 * otherwise we need to retrieve it. 186 * Here the server continues in the state running/armed until actually 187 * stopped, this works since we continue expecting a throttle. 188 */ 189 if (next->dl_server) 190 da_handle_start_event(EXPAND_ID(next->dl_server, cpu, 191 get_server_type(next)), 192 sched_switch_in_nomiss); 193 else { 194 handle_server_switch(next, cpu, DL_SERVER_FAIR); 195 if (IS_ENABLED(CONFIG_SCHED_CLASS_EXT)) 196 handle_server_switch(next, cpu, DL_SERVER_EXT); 197 } 198 } 199 200 static void handle_sys_enter(void *data, struct pt_regs *regs, long id) 201 { 202 struct task_struct *p; 203 int new_policy = -1; 204 pid_t pid = 0; 205 206 new_policy = extract_params(regs, id, &pid); 207 if (new_policy < 0) 208 return; 209 guard(rcu)(); 210 p = pid ? find_task_by_vpid(pid) : current; 211 if (unlikely(!p) || new_policy == p->policy) 212 return; 213 214 if (p->policy == SCHED_DEADLINE) 215 da_reset(EXPAND_ID_TASK(p)); 216 else if (new_policy == SCHED_DEADLINE) 217 da_create_or_get(EXPAND_ID_TASK(p)); 218 } 219 220 static void handle_sched_wakeup(void *data, struct task_struct *tsk) 221 { 222 if (tsk->policy == SCHED_DEADLINE) 223 da_handle_event(EXPAND_ID_TASK(tsk), sched_wakeup_nomiss); 224 } 225 226 static int enable_nomiss(void) 227 { 228 int retval; 229 230 retval = da_monitor_init(); 231 if (retval) 232 return retval; 233 234 retval = init_storage(false); 235 if (retval) 236 return retval; 237 rv_attach_trace_probe("nomiss", sched_dl_replenish_tp, handle_dl_replenish); 238 rv_attach_trace_probe("nomiss", sched_dl_throttle_tp, handle_dl_throttle); 239 rv_attach_trace_probe("nomiss", sched_dl_server_stop_tp, handle_dl_server_stop); 240 rv_attach_trace_probe("nomiss", sched_switch, handle_sched_switch); 241 rv_attach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup); 242 if (!should_skip_syscall_handle()) 243 rv_attach_trace_probe("nomiss", sys_enter, handle_sys_enter); 244 rv_attach_trace_probe("nomiss", task_newtask, handle_newtask); 245 rv_attach_trace_probe("nomiss", sched_process_exit, handle_exit); 246 247 return 0; 248 } 249 250 static void disable_nomiss(void) 251 { 252 rv_this.enabled = 0; 253 254 /* Those are RCU writers, detach earlier hoping to close a bit faster */ 255 rv_detach_trace_probe("nomiss", task_newtask, handle_newtask); 256 rv_detach_trace_probe("nomiss", sched_process_exit, handle_exit); 257 if (!should_skip_syscall_handle()) 258 rv_detach_trace_probe("nomiss", sys_enter, handle_sys_enter); 259 260 rv_detach_trace_probe("nomiss", sched_dl_replenish_tp, handle_dl_replenish); 261 rv_detach_trace_probe("nomiss", sched_dl_throttle_tp, handle_dl_throttle); 262 rv_detach_trace_probe("nomiss", sched_dl_server_stop_tp, handle_dl_server_stop); 263 rv_detach_trace_probe("nomiss", sched_switch, handle_sched_switch); 264 rv_detach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup); 265 266 da_monitor_destroy(); 267 } 268 269 static struct rv_monitor rv_this = { 270 .name = "nomiss", 271 .description = "dl entities run to completion before their deadline.", 272 .enable = enable_nomiss, 273 .disable = disable_nomiss, 274 .reset = da_monitor_reset_all, 275 .enabled = 0, 276 }; 277 278 static int __init register_nomiss(void) 279 { 280 return rv_register_monitor(&rv_this, &rv_deadline); 281 } 282 283 static void __exit unregister_nomiss(void) 284 { 285 rv_unregister_monitor(&rv_this); 286 } 287 288 module_init(register_nomiss); 289 module_exit(unregister_nomiss); 290 291 MODULE_LICENSE("GPL"); 292 MODULE_AUTHOR("Gabriele Monaco <gmonaco@redhat.com>"); 293 MODULE_DESCRIPTION("nomiss: dl entities run to completion before their deadline."); 294