1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Auto-group scheduling implementation: 5 */ 6 7 #include "autogroup.h" 8 #include "sched.h" 9 10 unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1; 11 static struct autogroup autogroup_default; 12 static atomic_t autogroup_seq_nr; 13 14 #ifdef CONFIG_SYSCTL 15 static const struct ctl_table sched_autogroup_sysctls[] = { 16 { 17 .procname = "sched_autogroup_enabled", 18 .data = &sysctl_sched_autogroup_enabled, 19 .maxlen = sizeof(unsigned int), 20 .mode = 0644, 21 .proc_handler = proc_dointvec_minmax, 22 .extra1 = SYSCTL_ZERO, 23 .extra2 = SYSCTL_ONE, 24 }, 25 }; 26 27 static void __init sched_autogroup_sysctl_init(void) 28 { 29 register_sysctl_init("kernel", sched_autogroup_sysctls); 30 } 31 #else /* !CONFIG_SYSCTL: */ 32 #define sched_autogroup_sysctl_init() do { } while (0) 33 #endif /* !CONFIG_SYSCTL */ 34 35 void __init autogroup_init(struct task_struct *init_task) 36 { 37 autogroup_default.tg = &root_task_group; 38 kref_init(&autogroup_default.kref); 39 init_rwsem(&autogroup_default.lock); 40 init_task->signal->autogroup = &autogroup_default; 41 sched_autogroup_sysctl_init(); 42 } 43 44 void autogroup_free(struct task_group *tg) 45 { 46 kfree(tg->autogroup); 47 } 48 49 static inline void autogroup_destroy(struct kref *kref) 50 { 51 struct autogroup *ag = container_of(kref, struct autogroup, kref); 52 53 #ifdef CONFIG_RT_GROUP_SCHED 54 /* We've redirected RT tasks to the root task group... */ 55 ag->tg->rt_se = NULL; 56 ag->tg->rt_rq = NULL; 57 #endif 58 sched_release_group(ag->tg); 59 sched_destroy_group(ag->tg); 60 } 61 62 static inline void autogroup_kref_put(struct autogroup *ag) 63 { 64 kref_put(&ag->kref, autogroup_destroy); 65 } 66 67 static inline struct autogroup *autogroup_kref_get(struct autogroup *ag) 68 { 69 kref_get(&ag->kref); 70 return ag; 71 } 72 73 static inline struct autogroup *autogroup_task_get(struct task_struct *p) 74 { 75 struct autogroup *ag; 76 unsigned long flags; 77 78 if (!lock_task_sighand(p, &flags)) 79 return autogroup_kref_get(&autogroup_default); 80 81 ag = autogroup_kref_get(p->signal->autogroup); 82 unlock_task_sighand(p, &flags); 83 84 return ag; 85 } 86 87 static inline struct autogroup *autogroup_create(void) 88 { 89 struct autogroup *ag = kzalloc(sizeof(*ag), GFP_KERNEL); 90 struct task_group *tg; 91 92 if (!ag) 93 goto out_fail; 94 95 tg = sched_create_group(&root_task_group); 96 if (IS_ERR(tg)) 97 goto out_free; 98 99 kref_init(&ag->kref); 100 init_rwsem(&ag->lock); 101 ag->id = atomic_inc_return(&autogroup_seq_nr); 102 ag->tg = tg; 103 #ifdef CONFIG_RT_GROUP_SCHED 104 /* 105 * Autogroup RT tasks are redirected to the root task group 106 * so we don't have to move tasks around upon policy change, 107 * or flail around trying to allocate bandwidth on the fly. 108 * A bandwidth exception in __sched_setscheduler() allows 109 * the policy change to proceed. 110 */ 111 free_rt_sched_group(tg); 112 tg->rt_se = root_task_group.rt_se; 113 tg->rt_rq = root_task_group.rt_rq; 114 #endif /* CONFIG_RT_GROUP_SCHED */ 115 tg->autogroup = ag; 116 117 sched_online_group(tg, &root_task_group); 118 return ag; 119 120 out_free: 121 kfree(ag); 122 out_fail: 123 if (printk_ratelimit()) { 124 printk(KERN_WARNING "autogroup_create: %s failure.\n", 125 ag ? "sched_create_group()" : "kzalloc()"); 126 } 127 128 return autogroup_kref_get(&autogroup_default); 129 } 130 131 bool task_wants_autogroup(struct task_struct *p, struct task_group *tg) 132 { 133 if (tg != &root_task_group) 134 return false; 135 /* 136 * If we race with autogroup_move_group() the caller can use the old 137 * value of signal->autogroup but in this case sched_move_task() will 138 * be called again before autogroup_kref_put(). 139 * 140 * However, there is no way sched_autogroup_exit_task() could tell us 141 * to avoid autogroup->tg, so we abuse PF_EXITING flag for this case. 142 */ 143 if (p->flags & PF_EXITING) 144 return false; 145 146 return true; 147 } 148 149 void sched_autogroup_exit_task(struct task_struct *p) 150 { 151 /* 152 * We are going to call exit_notify() and autogroup_move_group() can't 153 * see this thread after that: we can no longer use signal->autogroup. 154 * See the PF_EXITING check in task_wants_autogroup(). 155 */ 156 sched_move_task(p, true); 157 } 158 159 static void 160 autogroup_move_group(struct task_struct *p, struct autogroup *ag) 161 { 162 struct autogroup *prev; 163 struct task_struct *t; 164 unsigned long flags; 165 166 if (WARN_ON_ONCE(!lock_task_sighand(p, &flags))) 167 return; 168 169 prev = p->signal->autogroup; 170 if (prev == ag) { 171 unlock_task_sighand(p, &flags); 172 return; 173 } 174 175 p->signal->autogroup = autogroup_kref_get(ag); 176 /* 177 * We can't avoid sched_move_task() after we changed signal->autogroup, 178 * this process can already run with task_group() == prev->tg or we can 179 * race with cgroup code which can read autogroup = prev under rq->lock. 180 * In the latter case for_each_thread() can not miss a migrating thread, 181 * cpu_cgroup_attach() must not be possible after cgroup_exit() and it 182 * can't be removed from thread list, we hold ->siglock. 183 * 184 * If an exiting thread was already removed from thread list we rely on 185 * sched_autogroup_exit_task(). 186 */ 187 for_each_thread(p, t) 188 sched_move_task(t, true); 189 190 unlock_task_sighand(p, &flags); 191 autogroup_kref_put(prev); 192 } 193 194 /* Allocates GFP_KERNEL, cannot be called under any spinlock: */ 195 void sched_autogroup_create_attach(struct task_struct *p) 196 { 197 struct autogroup *ag = autogroup_create(); 198 199 autogroup_move_group(p, ag); 200 201 /* Drop extra reference added by autogroup_create(): */ 202 autogroup_kref_put(ag); 203 } 204 EXPORT_SYMBOL(sched_autogroup_create_attach); 205 206 /* Cannot be called under siglock. Currently has no users: */ 207 void sched_autogroup_detach(struct task_struct *p) 208 { 209 autogroup_move_group(p, &autogroup_default); 210 } 211 EXPORT_SYMBOL(sched_autogroup_detach); 212 213 void sched_autogroup_fork(struct signal_struct *sig) 214 { 215 sig->autogroup = autogroup_task_get(current); 216 } 217 218 void sched_autogroup_exit(struct signal_struct *sig) 219 { 220 autogroup_kref_put(sig->autogroup); 221 } 222 223 static int __init setup_autogroup(char *str) 224 { 225 sysctl_sched_autogroup_enabled = 0; 226 227 return 1; 228 } 229 __setup("noautogroup", setup_autogroup); 230 231 #ifdef CONFIG_PROC_FS 232 233 int proc_sched_autogroup_set_nice(struct task_struct *p, int nice) 234 { 235 static unsigned long next = INITIAL_JIFFIES; 236 struct autogroup *ag; 237 unsigned long shares; 238 int err, idx; 239 240 if (nice < MIN_NICE || nice > MAX_NICE) 241 return -EINVAL; 242 243 err = security_task_setnice(current, nice); 244 if (err) 245 return err; 246 247 if (nice < 0 && !can_nice(current, nice)) 248 return -EPERM; 249 250 /* This is a heavy operation, taking global locks.. */ 251 if (!capable(CAP_SYS_ADMIN) && time_before(jiffies, next)) 252 return -EAGAIN; 253 254 next = HZ / 10 + jiffies; 255 ag = autogroup_task_get(p); 256 257 idx = array_index_nospec(nice + 20, 40); 258 shares = scale_load(sched_prio_to_weight[idx]); 259 260 down_write(&ag->lock); 261 err = sched_group_set_shares(ag->tg, shares); 262 if (!err) 263 ag->nice = nice; 264 up_write(&ag->lock); 265 266 autogroup_kref_put(ag); 267 268 return err; 269 } 270 271 void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m) 272 { 273 struct autogroup *ag = autogroup_task_get(p); 274 275 if (!task_group_is_autogroup(ag->tg)) 276 goto out; 277 278 down_read(&ag->lock); 279 seq_printf(m, "/autogroup-%ld nice %d\n", ag->id, ag->nice); 280 up_read(&ag->lock); 281 282 out: 283 autogroup_kref_put(ag); 284 } 285 #endif /* CONFIG_PROC_FS */ 286 287 int autogroup_path(struct task_group *tg, char *buf, int buflen) 288 { 289 if (!task_group_is_autogroup(tg)) 290 return 0; 291 292 return snprintf(buf, buflen, "%s-%ld", "/autogroup", tg->autogroup->id); 293 } 294