1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 #include <linux/sched/isolation.h> 11 #include <linux/llist.h> 12 #include <linux/pci.h> 13 #include "sched.h" 14 15 enum hk_flags { 16 HK_FLAG_DOMAIN_BOOT = BIT(HK_TYPE_DOMAIN_BOOT), 17 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), 18 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), 19 HK_FLAG_KERNEL_NOISE = BIT(HK_TYPE_KERNEL_NOISE), 20 }; 21 22 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 23 EXPORT_SYMBOL_GPL(housekeeping_overridden); 24 25 struct housekeeping { 26 struct cpumask __rcu *cpumasks[HK_TYPE_MAX]; 27 unsigned long flags; 28 }; 29 30 static struct housekeeping housekeeping; 31 static __initdata LLIST_HEAD(memblock_freelist); 32 33 bool housekeeping_enabled(enum hk_type type) 34 { 35 return !!(READ_ONCE(housekeeping.flags) & BIT(type)); 36 } 37 EXPORT_SYMBOL_GPL(housekeeping_enabled); 38 39 static bool housekeeping_dereference_check(enum hk_type type) 40 { 41 if (IS_ENABLED(CONFIG_LOCKDEP) && type == HK_TYPE_DOMAIN) { 42 /* Cpuset isn't even writable yet? */ 43 if (system_state <= SYSTEM_SCHEDULING) 44 return true; 45 46 /* CPU hotplug write locked, so cpuset partition can't be overwritten */ 47 if (IS_ENABLED(CONFIG_HOTPLUG_CPU) && lockdep_is_cpus_write_held()) 48 return true; 49 50 /* Cpuset lock held, partitions not writable */ 51 if (IS_ENABLED(CONFIG_CPUSETS) && lockdep_is_cpuset_held()) 52 return true; 53 54 return false; 55 } 56 57 return true; 58 } 59 60 static inline struct cpumask *housekeeping_cpumask_dereference(enum hk_type type) 61 { 62 return rcu_dereference_all_check(housekeeping.cpumasks[type], 63 housekeeping_dereference_check(type)); 64 } 65 66 const struct cpumask *housekeeping_cpumask(enum hk_type type) 67 { 68 const struct cpumask *mask = NULL; 69 70 if (static_branch_unlikely(&housekeeping_overridden)) { 71 if (READ_ONCE(housekeeping.flags) & BIT(type)) 72 mask = housekeeping_cpumask_dereference(type); 73 } 74 if (!mask) 75 mask = cpu_possible_mask; 76 return mask; 77 } 78 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 79 80 int housekeeping_any_cpu(enum hk_type type) 81 { 82 int cpu; 83 84 if (static_branch_unlikely(&housekeeping_overridden)) { 85 if (housekeeping.flags & BIT(type)) { 86 cpu = sched_numa_find_closest(housekeeping_cpumask(type), smp_processor_id()); 87 if (cpu < nr_cpu_ids) 88 return cpu; 89 90 cpu = cpumask_any_and_distribute(housekeeping_cpumask(type), cpu_online_mask); 91 if (likely(cpu < nr_cpu_ids)) 92 return cpu; 93 /* 94 * Unless we have another problem this can only happen 95 * at boot time before start_secondary() brings the 1st 96 * housekeeping CPU up. 97 */ 98 WARN_ON_ONCE(system_state == SYSTEM_RUNNING || 99 type != HK_TYPE_TIMER); 100 } 101 } 102 return smp_processor_id(); 103 } 104 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 105 106 void housekeeping_affine(struct task_struct *t, enum hk_type type) 107 { 108 if (static_branch_unlikely(&housekeeping_overridden)) 109 if (housekeeping.flags & BIT(type)) 110 set_cpus_allowed_ptr(t, housekeeping_cpumask(type)); 111 } 112 EXPORT_SYMBOL_GPL(housekeeping_affine); 113 114 bool housekeeping_test_cpu(int cpu, enum hk_type type) 115 { 116 if (static_branch_unlikely(&housekeeping_overridden) && 117 READ_ONCE(housekeeping.flags) & BIT(type)) 118 return cpumask_test_cpu(cpu, housekeeping_cpumask(type)); 119 return true; 120 } 121 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 122 123 int housekeeping_update(struct cpumask *isol_mask) 124 { 125 struct cpumask *trial, *old = NULL; 126 int err; 127 128 trial = kmalloc(cpumask_size(), GFP_KERNEL); 129 if (!trial) 130 return -ENOMEM; 131 132 cpumask_andnot(trial, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT), isol_mask); 133 if (!cpumask_intersects(trial, cpu_online_mask)) { 134 kfree(trial); 135 return -EINVAL; 136 } 137 138 if (!housekeeping.flags) 139 static_branch_enable(&housekeeping_overridden); 140 141 if (housekeeping.flags & HK_FLAG_DOMAIN) 142 old = housekeeping_cpumask_dereference(HK_TYPE_DOMAIN); 143 else 144 WRITE_ONCE(housekeeping.flags, housekeeping.flags | HK_FLAG_DOMAIN); 145 rcu_assign_pointer(housekeeping.cpumasks[HK_TYPE_DOMAIN], trial); 146 147 synchronize_rcu(); 148 149 pci_probe_flush_workqueue(); 150 mem_cgroup_flush_workqueue(); 151 vmstat_flush_workqueue(); 152 153 err = workqueue_unbound_housekeeping_update(housekeeping_cpumask(HK_TYPE_DOMAIN)); 154 WARN_ON_ONCE(err < 0); 155 156 err = tmigr_isolated_exclude_cpumask(isol_mask); 157 WARN_ON_ONCE(err < 0); 158 159 err = kthreads_update_housekeeping(); 160 WARN_ON_ONCE(err < 0); 161 162 kfree(old); 163 164 return 0; 165 } 166 167 void __init housekeeping_init(void) 168 { 169 enum hk_type type; 170 171 if (!housekeeping.flags) 172 return; 173 174 static_branch_enable(&housekeeping_overridden); 175 176 if (housekeeping.flags & HK_FLAG_KERNEL_NOISE) 177 sched_tick_offload_init(); 178 /* 179 * Realloc with a proper allocator so that any cpumask update 180 * can indifferently free the old version with kfree(). 181 */ 182 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { 183 struct cpumask *omask, *nmask = kmalloc(cpumask_size(), GFP_KERNEL); 184 185 if (WARN_ON_ONCE(!nmask)) 186 return; 187 188 omask = rcu_dereference(housekeeping.cpumasks[type]); 189 190 /* We need at least one CPU to handle housekeeping work */ 191 WARN_ON_ONCE(cpumask_empty(omask)); 192 cpumask_copy(nmask, omask); 193 RCU_INIT_POINTER(housekeeping.cpumasks[type], nmask); 194 __llist_add((struct llist_node *)omask, &memblock_freelist); 195 } 196 } 197 198 static int __init housekeeping_late_init(void) 199 { 200 struct llist_node *llnode, *pos, *t; 201 202 /* Free allocated memblock memory, if any */ 203 llnode = __llist_del_all(&memblock_freelist); 204 llist_for_each_safe(pos, t, llnode) 205 memblock_free(pos, cpumask_size()); 206 return 0; 207 } 208 pure_initcall(housekeeping_late_init); 209 210 static void __init housekeeping_setup_type(enum hk_type type, 211 cpumask_var_t housekeeping_staging) 212 { 213 struct cpumask *mask = memblock_alloc_or_panic(cpumask_size(), SMP_CACHE_BYTES); 214 215 cpumask_copy(mask, housekeeping_staging); 216 RCU_INIT_POINTER(housekeeping.cpumasks[type], mask); 217 } 218 219 static int __init housekeeping_setup(char *str, unsigned long flags) 220 { 221 cpumask_var_t non_housekeeping_mask, housekeeping_staging; 222 unsigned int first_cpu; 223 int err = 0; 224 225 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) { 226 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { 227 pr_warn("Housekeeping: nohz unsupported." 228 " Build with CONFIG_NO_HZ_FULL\n"); 229 return 0; 230 } 231 } 232 233 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 234 if (cpulist_parse(str, non_housekeeping_mask) < 0) { 235 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 236 goto free_non_housekeeping_mask; 237 } 238 239 alloc_bootmem_cpumask_var(&housekeeping_staging); 240 cpumask_andnot(housekeeping_staging, 241 cpu_possible_mask, non_housekeeping_mask); 242 243 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging); 244 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) { 245 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging); 246 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 247 if (!housekeeping.flags) { 248 pr_warn("Housekeeping: must include one present CPU, " 249 "using boot CPU:%d\n", smp_processor_id()); 250 } 251 } 252 253 if (cpumask_empty(non_housekeeping_mask)) 254 goto free_housekeeping_staging; 255 256 if (!housekeeping.flags) { 257 /* First setup call ("nohz_full=" or "isolcpus=") */ 258 enum hk_type type; 259 260 for_each_set_bit(type, &flags, HK_TYPE_MAX) 261 housekeeping_setup_type(type, housekeeping_staging); 262 } else { 263 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */ 264 enum hk_type type; 265 unsigned long iter_flags = flags & housekeeping.flags; 266 267 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { 268 if (!cpumask_equal(housekeeping_staging, 269 housekeeping_cpumask(type))) { 270 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 271 goto free_housekeeping_staging; 272 } 273 } 274 275 /* 276 * Check the combination of nohz_full and isolcpus=domain, 277 * necessary to avoid problems with the timer migration 278 * hierarchy. managed_irq is ignored by this check since it 279 * isn't considered in the timer migration logic. 280 */ 281 iter_flags = housekeeping.flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN); 282 type = find_first_bit(&iter_flags, HK_TYPE_MAX); 283 /* 284 * Pass the check if none of these flags were previously set or 285 * are not in the current selection. 286 */ 287 iter_flags = flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN); 288 first_cpu = (type == HK_TYPE_MAX || !iter_flags) ? 0 : 289 cpumask_first_and_and(cpu_present_mask, 290 housekeeping_staging, housekeeping_cpumask(type)); 291 if (first_cpu >= min(nr_cpu_ids, setup_max_cpus)) { 292 pr_warn("Housekeeping: must include one present CPU " 293 "neither in nohz_full= nor in isolcpus=domain, " 294 "ignoring setting %s\n", str); 295 goto free_housekeeping_staging; 296 } 297 298 iter_flags = flags & ~housekeeping.flags; 299 300 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) 301 housekeeping_setup_type(type, housekeeping_staging); 302 } 303 304 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) 305 tick_nohz_full_setup(non_housekeeping_mask); 306 307 housekeeping.flags |= flags; 308 err = 1; 309 310 free_housekeeping_staging: 311 free_bootmem_cpumask_var(housekeeping_staging); 312 free_non_housekeeping_mask: 313 free_bootmem_cpumask_var(non_housekeeping_mask); 314 315 return err; 316 } 317 318 static int __init housekeeping_nohz_full_setup(char *str) 319 { 320 unsigned long flags; 321 322 flags = HK_FLAG_KERNEL_NOISE; 323 324 return housekeeping_setup(str, flags); 325 } 326 __setup("nohz_full=", housekeeping_nohz_full_setup); 327 328 static int __init housekeeping_isolcpus_setup(char *str) 329 { 330 unsigned long flags = 0; 331 bool illegal = false; 332 char *par; 333 int len; 334 335 while (isalpha(*str)) { 336 /* 337 * isolcpus=nohz is equivalent to nohz_full. 338 */ 339 if (!strncmp(str, "nohz,", 5)) { 340 str += 5; 341 flags |= HK_FLAG_KERNEL_NOISE; 342 continue; 343 } 344 345 if (!strncmp(str, "domain,", 7)) { 346 str += 7; 347 flags |= HK_FLAG_DOMAIN | HK_FLAG_DOMAIN_BOOT; 348 continue; 349 } 350 351 if (!strncmp(str, "managed_irq,", 12)) { 352 str += 12; 353 flags |= HK_FLAG_MANAGED_IRQ; 354 continue; 355 } 356 357 /* 358 * Skip unknown sub-parameter and validate that it is not 359 * containing an invalid character. 360 */ 361 for (par = str, len = 0; *str && *str != ','; str++, len++) { 362 if (!isalpha(*str) && *str != '_') 363 illegal = true; 364 } 365 366 if (illegal) { 367 pr_warn("isolcpus: Invalid flag %.*s\n", len, par); 368 return 0; 369 } 370 371 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); 372 str++; 373 } 374 375 /* Default behaviour for isolcpus without flags */ 376 if (!flags) 377 flags |= HK_FLAG_DOMAIN | HK_FLAG_DOMAIN_BOOT; 378 379 return housekeeping_setup(str, flags); 380 } 381 __setup("isolcpus=", housekeeping_isolcpus_setup); 382