1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2801c1419SIngo Molnar
3325ea10cSIngo Molnar /*
4325ea10cSIngo Molnar * Auto-group scheduling implementation:
5325ea10cSIngo Molnar */
625493e5fSSergey Senozhatsky
71051408fSIngo Molnar unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1;
81051408fSIngo Molnar static struct autogroup autogroup_default;
91051408fSIngo Molnar static atomic_t autogroup_seq_nr;
101051408fSIngo Molnar
11c8eaf6acSZhen Ni #ifdef CONFIG_SYSCTL
12c8eaf6acSZhen Ni static struct ctl_table sched_autogroup_sysctls[] = {
13c8eaf6acSZhen Ni {
14c8eaf6acSZhen Ni .procname = "sched_autogroup_enabled",
15c8eaf6acSZhen Ni .data = &sysctl_sched_autogroup_enabled,
16c8eaf6acSZhen Ni .maxlen = sizeof(unsigned int),
17c8eaf6acSZhen Ni .mode = 0644,
18c8eaf6acSZhen Ni .proc_handler = proc_dointvec_minmax,
19c8eaf6acSZhen Ni .extra1 = SYSCTL_ZERO,
20c8eaf6acSZhen Ni .extra2 = SYSCTL_ONE,
21c8eaf6acSZhen Ni },
22c8eaf6acSZhen Ni };
23c8eaf6acSZhen Ni
sched_autogroup_sysctl_init(void)24c8eaf6acSZhen Ni static void __init sched_autogroup_sysctl_init(void)
25c8eaf6acSZhen Ni {
26c8eaf6acSZhen Ni register_sysctl_init("kernel", sched_autogroup_sysctls);
27c8eaf6acSZhen Ni }
28c8eaf6acSZhen Ni #else
29c8eaf6acSZhen Ni #define sched_autogroup_sysctl_init() do { } while (0)
30c8eaf6acSZhen Ni #endif
31c8eaf6acSZhen Ni
autogroup_init(struct task_struct * init_task)321051408fSIngo Molnar void __init autogroup_init(struct task_struct *init_task)
331051408fSIngo Molnar {
341051408fSIngo Molnar autogroup_default.tg = &root_task_group;
351051408fSIngo Molnar kref_init(&autogroup_default.kref);
361051408fSIngo Molnar init_rwsem(&autogroup_default.lock);
371051408fSIngo Molnar init_task->signal->autogroup = &autogroup_default;
3882f586f9SPeter Zijlstra sched_autogroup_sysctl_init();
391051408fSIngo Molnar }
401051408fSIngo Molnar
autogroup_free(struct task_group * tg)411051408fSIngo Molnar void autogroup_free(struct task_group *tg)
421051408fSIngo Molnar {
431051408fSIngo Molnar kfree(tg->autogroup);
441051408fSIngo Molnar }
451051408fSIngo Molnar
autogroup_destroy(struct kref * kref)461051408fSIngo Molnar static inline void autogroup_destroy(struct kref *kref)
471051408fSIngo Molnar {
481051408fSIngo Molnar struct autogroup *ag = container_of(kref, struct autogroup, kref);
491051408fSIngo Molnar
501051408fSIngo Molnar #ifdef CONFIG_RT_GROUP_SCHED
511051408fSIngo Molnar /* We've redirected RT tasks to the root task group... */
521051408fSIngo Molnar ag->tg->rt_se = NULL;
531051408fSIngo Molnar ag->tg->rt_rq = NULL;
541051408fSIngo Molnar #endif
55b027789eSMathias Krause sched_release_group(ag->tg);
561051408fSIngo Molnar sched_destroy_group(ag->tg);
571051408fSIngo Molnar }
581051408fSIngo Molnar
autogroup_kref_put(struct autogroup * ag)591051408fSIngo Molnar static inline void autogroup_kref_put(struct autogroup *ag)
601051408fSIngo Molnar {
611051408fSIngo Molnar kref_put(&ag->kref, autogroup_destroy);
621051408fSIngo Molnar }
631051408fSIngo Molnar
autogroup_kref_get(struct autogroup * ag)641051408fSIngo Molnar static inline struct autogroup *autogroup_kref_get(struct autogroup *ag)
651051408fSIngo Molnar {
661051408fSIngo Molnar kref_get(&ag->kref);
671051408fSIngo Molnar return ag;
681051408fSIngo Molnar }
691051408fSIngo Molnar
autogroup_task_get(struct task_struct * p)701051408fSIngo Molnar static inline struct autogroup *autogroup_task_get(struct task_struct *p)
711051408fSIngo Molnar {
721051408fSIngo Molnar struct autogroup *ag;
731051408fSIngo Molnar unsigned long flags;
741051408fSIngo Molnar
751051408fSIngo Molnar if (!lock_task_sighand(p, &flags))
761051408fSIngo Molnar return autogroup_kref_get(&autogroup_default);
771051408fSIngo Molnar
781051408fSIngo Molnar ag = autogroup_kref_get(p->signal->autogroup);
791051408fSIngo Molnar unlock_task_sighand(p, &flags);
801051408fSIngo Molnar
811051408fSIngo Molnar return ag;
821051408fSIngo Molnar }
831051408fSIngo Molnar
autogroup_create(void)841051408fSIngo Molnar static inline struct autogroup *autogroup_create(void)
851051408fSIngo Molnar {
861051408fSIngo Molnar struct autogroup *ag = kzalloc(sizeof(*ag), GFP_KERNEL);
871051408fSIngo Molnar struct task_group *tg;
881051408fSIngo Molnar
891051408fSIngo Molnar if (!ag)
901051408fSIngo Molnar goto out_fail;
911051408fSIngo Molnar
921051408fSIngo Molnar tg = sched_create_group(&root_task_group);
931051408fSIngo Molnar if (IS_ERR(tg))
941051408fSIngo Molnar goto out_free;
951051408fSIngo Molnar
961051408fSIngo Molnar kref_init(&ag->kref);
971051408fSIngo Molnar init_rwsem(&ag->lock);
981051408fSIngo Molnar ag->id = atomic_inc_return(&autogroup_seq_nr);
991051408fSIngo Molnar ag->tg = tg;
1001051408fSIngo Molnar #ifdef CONFIG_RT_GROUP_SCHED
1011051408fSIngo Molnar /*
1021051408fSIngo Molnar * Autogroup RT tasks are redirected to the root task group
1031051408fSIngo Molnar * so we don't have to move tasks around upon policy change,
1041051408fSIngo Molnar * or flail around trying to allocate bandwidth on the fly.
1051051408fSIngo Molnar * A bandwidth exception in __sched_setscheduler() allows
1061051408fSIngo Molnar * the policy change to proceed.
1071051408fSIngo Molnar */
1081051408fSIngo Molnar free_rt_sched_group(tg);
1091051408fSIngo Molnar tg->rt_se = root_task_group.rt_se;
1101051408fSIngo Molnar tg->rt_rq = root_task_group.rt_rq;
1111051408fSIngo Molnar #endif
1121051408fSIngo Molnar tg->autogroup = ag;
1131051408fSIngo Molnar
1141051408fSIngo Molnar sched_online_group(tg, &root_task_group);
1151051408fSIngo Molnar return ag;
1161051408fSIngo Molnar
1171051408fSIngo Molnar out_free:
1181051408fSIngo Molnar kfree(ag);
1191051408fSIngo Molnar out_fail:
1201051408fSIngo Molnar if (printk_ratelimit()) {
1211051408fSIngo Molnar printk(KERN_WARNING "autogroup_create: %s failure.\n",
1221e58565eSAnshuman Khandual ag ? "sched_create_group()" : "kzalloc()");
1231051408fSIngo Molnar }
1241051408fSIngo Molnar
1251051408fSIngo Molnar return autogroup_kref_get(&autogroup_default);
1261051408fSIngo Molnar }
1271051408fSIngo Molnar
task_wants_autogroup(struct task_struct * p,struct task_group * tg)1281051408fSIngo Molnar bool task_wants_autogroup(struct task_struct *p, struct task_group *tg)
1291051408fSIngo Molnar {
1301051408fSIngo Molnar if (tg != &root_task_group)
1311051408fSIngo Molnar return false;
1321051408fSIngo Molnar /*
1331051408fSIngo Molnar * If we race with autogroup_move_group() the caller can use the old
1341051408fSIngo Molnar * value of signal->autogroup but in this case sched_move_task() will
1351051408fSIngo Molnar * be called again before autogroup_kref_put().
1361051408fSIngo Molnar *
1371051408fSIngo Molnar * However, there is no way sched_autogroup_exit_task() could tell us
1381051408fSIngo Molnar * to avoid autogroup->tg, so we abuse PF_EXITING flag for this case.
1391051408fSIngo Molnar */
1401051408fSIngo Molnar if (p->flags & PF_EXITING)
1411051408fSIngo Molnar return false;
1421051408fSIngo Molnar
1431051408fSIngo Molnar return true;
1441051408fSIngo Molnar }
1451051408fSIngo Molnar
sched_autogroup_exit_task(struct task_struct * p)1461051408fSIngo Molnar void sched_autogroup_exit_task(struct task_struct *p)
1471051408fSIngo Molnar {
1481051408fSIngo Molnar /*
1491051408fSIngo Molnar * We are going to call exit_notify() and autogroup_move_group() can't
1501051408fSIngo Molnar * see this thread after that: we can no longer use signal->autogroup.
1511051408fSIngo Molnar * See the PF_EXITING check in task_wants_autogroup().
1521051408fSIngo Molnar */
1531051408fSIngo Molnar sched_move_task(p);
1541051408fSIngo Molnar }
1551051408fSIngo Molnar
1561051408fSIngo Molnar static void
autogroup_move_group(struct task_struct * p,struct autogroup * ag)1571051408fSIngo Molnar autogroup_move_group(struct task_struct *p, struct autogroup *ag)
1581051408fSIngo Molnar {
1591051408fSIngo Molnar struct autogroup *prev;
1601051408fSIngo Molnar struct task_struct *t;
1611051408fSIngo Molnar unsigned long flags;
1621051408fSIngo Molnar
163*09348d75SIngo Molnar if (WARN_ON_ONCE(!lock_task_sighand(p, &flags)))
164*09348d75SIngo Molnar return;
1651051408fSIngo Molnar
1661051408fSIngo Molnar prev = p->signal->autogroup;
1671051408fSIngo Molnar if (prev == ag) {
1681051408fSIngo Molnar unlock_task_sighand(p, &flags);
1691051408fSIngo Molnar return;
1701051408fSIngo Molnar }
1711051408fSIngo Molnar
1721051408fSIngo Molnar p->signal->autogroup = autogroup_kref_get(ag);
1731051408fSIngo Molnar /*
1741051408fSIngo Molnar * We can't avoid sched_move_task() after we changed signal->autogroup,
1751051408fSIngo Molnar * this process can already run with task_group() == prev->tg or we can
1761051408fSIngo Molnar * race with cgroup code which can read autogroup = prev under rq->lock.
1771051408fSIngo Molnar * In the latter case for_each_thread() can not miss a migrating thread,
1781051408fSIngo Molnar * cpu_cgroup_attach() must not be possible after cgroup_exit() and it
1791051408fSIngo Molnar * can't be removed from thread list, we hold ->siglock.
1801051408fSIngo Molnar *
1811051408fSIngo Molnar * If an exiting thread was already removed from thread list we rely on
1821051408fSIngo Molnar * sched_autogroup_exit_task().
1831051408fSIngo Molnar */
1841051408fSIngo Molnar for_each_thread(p, t)
1851051408fSIngo Molnar sched_move_task(t);
1861051408fSIngo Molnar
1871051408fSIngo Molnar unlock_task_sighand(p, &flags);
1881051408fSIngo Molnar autogroup_kref_put(prev);
1891051408fSIngo Molnar }
1901051408fSIngo Molnar
19197fb7a0aSIngo Molnar /* Allocates GFP_KERNEL, cannot be called under any spinlock: */
sched_autogroup_create_attach(struct task_struct * p)1921051408fSIngo Molnar void sched_autogroup_create_attach(struct task_struct *p)
1931051408fSIngo Molnar {
1941051408fSIngo Molnar struct autogroup *ag = autogroup_create();
1951051408fSIngo Molnar
1961051408fSIngo Molnar autogroup_move_group(p, ag);
19797fb7a0aSIngo Molnar
19897fb7a0aSIngo Molnar /* Drop extra reference added by autogroup_create(): */
1991051408fSIngo Molnar autogroup_kref_put(ag);
2001051408fSIngo Molnar }
2011051408fSIngo Molnar EXPORT_SYMBOL(sched_autogroup_create_attach);
2021051408fSIngo Molnar
20397fb7a0aSIngo Molnar /* Cannot be called under siglock. Currently has no users: */
sched_autogroup_detach(struct task_struct * p)2041051408fSIngo Molnar void sched_autogroup_detach(struct task_struct *p)
2051051408fSIngo Molnar {
2061051408fSIngo Molnar autogroup_move_group(p, &autogroup_default);
2071051408fSIngo Molnar }
2081051408fSIngo Molnar EXPORT_SYMBOL(sched_autogroup_detach);
2091051408fSIngo Molnar
sched_autogroup_fork(struct signal_struct * sig)2101051408fSIngo Molnar void sched_autogroup_fork(struct signal_struct *sig)
2111051408fSIngo Molnar {
2121051408fSIngo Molnar sig->autogroup = autogroup_task_get(current);
2131051408fSIngo Molnar }
2141051408fSIngo Molnar
sched_autogroup_exit(struct signal_struct * sig)2151051408fSIngo Molnar void sched_autogroup_exit(struct signal_struct *sig)
2161051408fSIngo Molnar {
2171051408fSIngo Molnar autogroup_kref_put(sig->autogroup);
2181051408fSIngo Molnar }
2191051408fSIngo Molnar
setup_autogroup(char * str)2201051408fSIngo Molnar static int __init setup_autogroup(char *str)
2211051408fSIngo Molnar {
2221051408fSIngo Molnar sysctl_sched_autogroup_enabled = 0;
2231051408fSIngo Molnar
2241051408fSIngo Molnar return 1;
2251051408fSIngo Molnar }
2261051408fSIngo Molnar __setup("noautogroup", setup_autogroup);
2271051408fSIngo Molnar
2281051408fSIngo Molnar #ifdef CONFIG_PROC_FS
2291051408fSIngo Molnar
proc_sched_autogroup_set_nice(struct task_struct * p,int nice)2301051408fSIngo Molnar int proc_sched_autogroup_set_nice(struct task_struct *p, int nice)
2311051408fSIngo Molnar {
2321051408fSIngo Molnar static unsigned long next = INITIAL_JIFFIES;
2331051408fSIngo Molnar struct autogroup *ag;
2341051408fSIngo Molnar unsigned long shares;
235354d7793SPeter Zijlstra int err, idx;
2361051408fSIngo Molnar
2371051408fSIngo Molnar if (nice < MIN_NICE || nice > MAX_NICE)
2381051408fSIngo Molnar return -EINVAL;
2391051408fSIngo Molnar
2401051408fSIngo Molnar err = security_task_setnice(current, nice);
2411051408fSIngo Molnar if (err)
2421051408fSIngo Molnar return err;
2431051408fSIngo Molnar
2441051408fSIngo Molnar if (nice < 0 && !can_nice(current, nice))
2451051408fSIngo Molnar return -EPERM;
2461051408fSIngo Molnar
24797fb7a0aSIngo Molnar /* This is a heavy operation, taking global locks.. */
2481051408fSIngo Molnar if (!capable(CAP_SYS_ADMIN) && time_before(jiffies, next))
2491051408fSIngo Molnar return -EAGAIN;
2501051408fSIngo Molnar
2511051408fSIngo Molnar next = HZ / 10 + jiffies;
2521051408fSIngo Molnar ag = autogroup_task_get(p);
253354d7793SPeter Zijlstra
254354d7793SPeter Zijlstra idx = array_index_nospec(nice + 20, 40);
255354d7793SPeter Zijlstra shares = scale_load(sched_prio_to_weight[idx]);
2561051408fSIngo Molnar
2571051408fSIngo Molnar down_write(&ag->lock);
2581051408fSIngo Molnar err = sched_group_set_shares(ag->tg, shares);
2591051408fSIngo Molnar if (!err)
2601051408fSIngo Molnar ag->nice = nice;
2611051408fSIngo Molnar up_write(&ag->lock);
2621051408fSIngo Molnar
2631051408fSIngo Molnar autogroup_kref_put(ag);
2641051408fSIngo Molnar
2651051408fSIngo Molnar return err;
2661051408fSIngo Molnar }
2671051408fSIngo Molnar
proc_sched_autogroup_show_task(struct task_struct * p,struct seq_file * m)2681051408fSIngo Molnar void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m)
2691051408fSIngo Molnar {
2701051408fSIngo Molnar struct autogroup *ag = autogroup_task_get(p);
2711051408fSIngo Molnar
2721051408fSIngo Molnar if (!task_group_is_autogroup(ag->tg))
2731051408fSIngo Molnar goto out;
2741051408fSIngo Molnar
2751051408fSIngo Molnar down_read(&ag->lock);
2761051408fSIngo Molnar seq_printf(m, "/autogroup-%ld nice %d\n", ag->id, ag->nice);
2771051408fSIngo Molnar up_read(&ag->lock);
2781051408fSIngo Molnar
2791051408fSIngo Molnar out:
2801051408fSIngo Molnar autogroup_kref_put(ag);
2811051408fSIngo Molnar }
2821051408fSIngo Molnar #endif /* CONFIG_PROC_FS */
2831051408fSIngo Molnar
autogroup_path(struct task_group * tg,char * buf,int buflen)2841051408fSIngo Molnar int autogroup_path(struct task_group *tg, char *buf, int buflen)
2851051408fSIngo Molnar {
2861051408fSIngo Molnar if (!task_group_is_autogroup(tg))
2871051408fSIngo Molnar return 0;
2881051408fSIngo Molnar
2891051408fSIngo Molnar return snprintf(buf, buflen, "%s-%ld", "/autogroup", tg->autogroup->id);
2901051408fSIngo Molnar }
291