1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 #include <linux/sched/isolation.h> 11 #include "sched.h" 12 13 enum hk_flags { 14 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), 15 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), 16 HK_FLAG_KERNEL_NOISE = BIT(HK_TYPE_KERNEL_NOISE), 17 }; 18 19 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 20 EXPORT_SYMBOL_GPL(housekeeping_overridden); 21 22 struct housekeeping { 23 cpumask_var_t cpumasks[HK_TYPE_MAX]; 24 unsigned long flags; 25 }; 26 27 static struct housekeeping housekeeping; 28 29 bool housekeeping_enabled(enum hk_type type) 30 { 31 return !!(housekeeping.flags & BIT(type)); 32 } 33 EXPORT_SYMBOL_GPL(housekeeping_enabled); 34 35 int housekeeping_any_cpu(enum hk_type type) 36 { 37 int cpu; 38 39 if (static_branch_unlikely(&housekeeping_overridden)) { 40 if (housekeeping.flags & BIT(type)) { 41 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id()); 42 if (cpu < nr_cpu_ids) 43 return cpu; 44 45 cpu = cpumask_any_and_distribute(housekeeping.cpumasks[type], cpu_online_mask); 46 if (likely(cpu < nr_cpu_ids)) 47 return cpu; 48 /* 49 * Unless we have another problem this can only happen 50 * at boot time before start_secondary() brings the 1st 51 * housekeeping CPU up. 52 */ 53 WARN_ON_ONCE(system_state == SYSTEM_RUNNING || 54 type != HK_TYPE_TIMER); 55 } 56 } 57 return smp_processor_id(); 58 } 59 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 60 61 const struct cpumask *housekeeping_cpumask(enum hk_type type) 62 { 63 if (static_branch_unlikely(&housekeeping_overridden)) 64 if (housekeeping.flags & BIT(type)) 65 return housekeeping.cpumasks[type]; 66 return cpu_possible_mask; 67 } 68 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 69 70 void housekeeping_affine(struct task_struct *t, enum hk_type type) 71 { 72 if (static_branch_unlikely(&housekeeping_overridden)) 73 if (housekeeping.flags & BIT(type)) 74 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]); 75 } 76 EXPORT_SYMBOL_GPL(housekeeping_affine); 77 78 bool housekeeping_test_cpu(int cpu, enum hk_type type) 79 { 80 if (static_branch_unlikely(&housekeeping_overridden)) 81 if (housekeeping.flags & BIT(type)) 82 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]); 83 return true; 84 } 85 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 86 87 void __init housekeeping_init(void) 88 { 89 enum hk_type type; 90 91 if (!housekeeping.flags) 92 return; 93 94 static_branch_enable(&housekeeping_overridden); 95 96 if (housekeeping.flags & HK_FLAG_KERNEL_NOISE) 97 sched_tick_offload_init(); 98 99 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { 100 /* We need at least one CPU to handle housekeeping work */ 101 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); 102 } 103 } 104 105 static void __init housekeeping_setup_type(enum hk_type type, 106 cpumask_var_t housekeeping_staging) 107 { 108 109 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]); 110 cpumask_copy(housekeeping.cpumasks[type], 111 housekeeping_staging); 112 } 113 114 static int __init housekeeping_setup(char *str, unsigned long flags) 115 { 116 cpumask_var_t non_housekeeping_mask, housekeeping_staging; 117 unsigned int first_cpu; 118 int err = 0; 119 120 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) { 121 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { 122 pr_warn("Housekeeping: nohz unsupported." 123 " Build with CONFIG_NO_HZ_FULL\n"); 124 return 0; 125 } 126 } 127 128 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 129 if (cpulist_parse(str, non_housekeeping_mask) < 0) { 130 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 131 goto free_non_housekeeping_mask; 132 } 133 134 alloc_bootmem_cpumask_var(&housekeeping_staging); 135 cpumask_andnot(housekeeping_staging, 136 cpu_possible_mask, non_housekeeping_mask); 137 138 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging); 139 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) { 140 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging); 141 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 142 if (!housekeeping.flags) { 143 pr_warn("Housekeeping: must include one present CPU, " 144 "using boot CPU:%d\n", smp_processor_id()); 145 } 146 } 147 148 if (cpumask_empty(non_housekeeping_mask)) 149 goto free_housekeeping_staging; 150 151 if (!housekeeping.flags) { 152 /* First setup call ("nohz_full=" or "isolcpus=") */ 153 enum hk_type type; 154 155 for_each_set_bit(type, &flags, HK_TYPE_MAX) 156 housekeeping_setup_type(type, housekeeping_staging); 157 } else { 158 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */ 159 enum hk_type type; 160 unsigned long iter_flags = flags & housekeeping.flags; 161 162 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { 163 if (!cpumask_equal(housekeeping_staging, 164 housekeeping.cpumasks[type])) { 165 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 166 goto free_housekeeping_staging; 167 } 168 } 169 170 /* 171 * Check the combination of nohz_full and isolcpus=domain, 172 * necessary to avoid problems with the timer migration 173 * hierarchy. managed_irq is ignored by this check since it 174 * isn't considered in the timer migration logic. 175 */ 176 iter_flags = housekeeping.flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN); 177 type = find_first_bit(&iter_flags, HK_TYPE_MAX); 178 /* 179 * Pass the check if none of these flags were previously set or 180 * are not in the current selection. 181 */ 182 iter_flags = flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN); 183 first_cpu = (type == HK_TYPE_MAX || !iter_flags) ? 0 : 184 cpumask_first_and_and(cpu_present_mask, 185 housekeeping_staging, housekeeping.cpumasks[type]); 186 if (first_cpu >= min(nr_cpu_ids, setup_max_cpus)) { 187 pr_warn("Housekeeping: must include one present CPU " 188 "neither in nohz_full= nor in isolcpus=domain, " 189 "ignoring setting %s\n", str); 190 goto free_housekeeping_staging; 191 } 192 193 iter_flags = flags & ~housekeeping.flags; 194 195 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) 196 housekeeping_setup_type(type, housekeeping_staging); 197 } 198 199 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) 200 tick_nohz_full_setup(non_housekeeping_mask); 201 202 housekeeping.flags |= flags; 203 err = 1; 204 205 free_housekeeping_staging: 206 free_bootmem_cpumask_var(housekeeping_staging); 207 free_non_housekeeping_mask: 208 free_bootmem_cpumask_var(non_housekeeping_mask); 209 210 return err; 211 } 212 213 static int __init housekeeping_nohz_full_setup(char *str) 214 { 215 unsigned long flags; 216 217 flags = HK_FLAG_KERNEL_NOISE; 218 219 return housekeeping_setup(str, flags); 220 } 221 __setup("nohz_full=", housekeeping_nohz_full_setup); 222 223 static int __init housekeeping_isolcpus_setup(char *str) 224 { 225 unsigned long flags = 0; 226 bool illegal = false; 227 char *par; 228 int len; 229 230 while (isalpha(*str)) { 231 /* 232 * isolcpus=nohz is equivalent to nohz_full. 233 */ 234 if (!strncmp(str, "nohz,", 5)) { 235 str += 5; 236 flags |= HK_FLAG_KERNEL_NOISE; 237 continue; 238 } 239 240 if (!strncmp(str, "domain,", 7)) { 241 str += 7; 242 flags |= HK_FLAG_DOMAIN; 243 continue; 244 } 245 246 if (!strncmp(str, "managed_irq,", 12)) { 247 str += 12; 248 flags |= HK_FLAG_MANAGED_IRQ; 249 continue; 250 } 251 252 /* 253 * Skip unknown sub-parameter and validate that it is not 254 * containing an invalid character. 255 */ 256 for (par = str, len = 0; *str && *str != ','; str++, len++) { 257 if (!isalpha(*str) && *str != '_') 258 illegal = true; 259 } 260 261 if (illegal) { 262 pr_warn("isolcpus: Invalid flag %.*s\n", len, par); 263 return 0; 264 } 265 266 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); 267 str++; 268 } 269 270 /* Default behaviour for isolcpus without flags */ 271 if (!flags) 272 flags |= HK_FLAG_DOMAIN; 273 274 return housekeeping_setup(str, flags); 275 } 276 __setup("isolcpus=", housekeeping_isolcpus_setup); 277