1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 11 enum hk_flags { 12 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), 13 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), 14 HK_FLAG_KERNEL_NOISE = BIT(HK_TYPE_KERNEL_NOISE), 15 }; 16 17 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 18 EXPORT_SYMBOL_GPL(housekeeping_overridden); 19 20 struct housekeeping { 21 cpumask_var_t cpumasks[HK_TYPE_MAX]; 22 unsigned long flags; 23 }; 24 25 static struct housekeeping housekeeping; 26 27 bool housekeeping_enabled(enum hk_type type) 28 { 29 return !!(housekeeping.flags & BIT(type)); 30 } 31 EXPORT_SYMBOL_GPL(housekeeping_enabled); 32 33 int housekeeping_any_cpu(enum hk_type type) 34 { 35 int cpu; 36 37 if (static_branch_unlikely(&housekeeping_overridden)) { 38 if (housekeeping.flags & BIT(type)) { 39 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id()); 40 if (cpu < nr_cpu_ids) 41 return cpu; 42 43 cpu = cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask); 44 if (likely(cpu < nr_cpu_ids)) 45 return cpu; 46 /* 47 * Unless we have another problem this can only happen 48 * at boot time before start_secondary() brings the 1st 49 * housekeeping CPU up. 50 */ 51 WARN_ON_ONCE(system_state == SYSTEM_RUNNING || 52 type != HK_TYPE_TIMER); 53 } 54 } 55 return smp_processor_id(); 56 } 57 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 58 59 const struct cpumask *housekeeping_cpumask(enum hk_type type) 60 { 61 if (static_branch_unlikely(&housekeeping_overridden)) 62 if (housekeeping.flags & BIT(type)) 63 return housekeeping.cpumasks[type]; 64 return cpu_possible_mask; 65 } 66 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 67 68 void housekeeping_affine(struct task_struct *t, enum hk_type type) 69 { 70 if (static_branch_unlikely(&housekeeping_overridden)) 71 if (housekeeping.flags & BIT(type)) 72 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]); 73 } 74 EXPORT_SYMBOL_GPL(housekeeping_affine); 75 76 bool housekeeping_test_cpu(int cpu, enum hk_type type) 77 { 78 if (static_branch_unlikely(&housekeeping_overridden)) 79 if (housekeeping.flags & BIT(type)) 80 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]); 81 return true; 82 } 83 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 84 85 void __init housekeeping_init(void) 86 { 87 enum hk_type type; 88 89 if (!housekeeping.flags) 90 return; 91 92 static_branch_enable(&housekeeping_overridden); 93 94 if (housekeeping.flags & HK_FLAG_KERNEL_NOISE) 95 sched_tick_offload_init(); 96 97 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { 98 /* We need at least one CPU to handle housekeeping work */ 99 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); 100 } 101 } 102 103 static void __init housekeeping_setup_type(enum hk_type type, 104 cpumask_var_t housekeeping_staging) 105 { 106 107 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]); 108 cpumask_copy(housekeeping.cpumasks[type], 109 housekeeping_staging); 110 } 111 112 static int __init housekeeping_setup(char *str, unsigned long flags) 113 { 114 cpumask_var_t non_housekeeping_mask, housekeeping_staging; 115 unsigned int first_cpu; 116 int err = 0; 117 118 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) { 119 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { 120 pr_warn("Housekeeping: nohz unsupported." 121 " Build with CONFIG_NO_HZ_FULL\n"); 122 return 0; 123 } 124 } 125 126 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 127 if (cpulist_parse(str, non_housekeeping_mask) < 0) { 128 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 129 goto free_non_housekeeping_mask; 130 } 131 132 alloc_bootmem_cpumask_var(&housekeeping_staging); 133 cpumask_andnot(housekeeping_staging, 134 cpu_possible_mask, non_housekeeping_mask); 135 136 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging); 137 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) { 138 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging); 139 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 140 if (!housekeeping.flags) { 141 pr_warn("Housekeeping: must include one present CPU, " 142 "using boot CPU:%d\n", smp_processor_id()); 143 } 144 } 145 146 if (cpumask_empty(non_housekeeping_mask)) 147 goto free_housekeeping_staging; 148 149 if (!housekeeping.flags) { 150 /* First setup call ("nohz_full=" or "isolcpus=") */ 151 enum hk_type type; 152 153 for_each_set_bit(type, &flags, HK_TYPE_MAX) 154 housekeeping_setup_type(type, housekeeping_staging); 155 } else { 156 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */ 157 enum hk_type type; 158 unsigned long iter_flags = flags & housekeeping.flags; 159 160 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { 161 if (!cpumask_equal(housekeeping_staging, 162 housekeeping.cpumasks[type])) { 163 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 164 goto free_housekeeping_staging; 165 } 166 } 167 168 iter_flags = flags & ~housekeeping.flags; 169 170 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) 171 housekeeping_setup_type(type, housekeeping_staging); 172 } 173 174 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) 175 tick_nohz_full_setup(non_housekeeping_mask); 176 177 housekeeping.flags |= flags; 178 err = 1; 179 180 free_housekeeping_staging: 181 free_bootmem_cpumask_var(housekeeping_staging); 182 free_non_housekeeping_mask: 183 free_bootmem_cpumask_var(non_housekeeping_mask); 184 185 return err; 186 } 187 188 static int __init housekeeping_nohz_full_setup(char *str) 189 { 190 unsigned long flags; 191 192 flags = HK_FLAG_KERNEL_NOISE; 193 194 return housekeeping_setup(str, flags); 195 } 196 __setup("nohz_full=", housekeeping_nohz_full_setup); 197 198 static int __init housekeeping_isolcpus_setup(char *str) 199 { 200 unsigned long flags = 0; 201 bool illegal = false; 202 char *par; 203 int len; 204 205 while (isalpha(*str)) { 206 /* 207 * isolcpus=nohz is equivalent to nohz_full. 208 */ 209 if (!strncmp(str, "nohz,", 5)) { 210 str += 5; 211 flags |= HK_FLAG_KERNEL_NOISE; 212 continue; 213 } 214 215 if (!strncmp(str, "domain,", 7)) { 216 str += 7; 217 flags |= HK_FLAG_DOMAIN; 218 continue; 219 } 220 221 if (!strncmp(str, "managed_irq,", 12)) { 222 str += 12; 223 flags |= HK_FLAG_MANAGED_IRQ; 224 continue; 225 } 226 227 /* 228 * Skip unknown sub-parameter and validate that it is not 229 * containing an invalid character. 230 */ 231 for (par = str, len = 0; *str && *str != ','; str++, len++) { 232 if (!isalpha(*str) && *str != '_') 233 illegal = true; 234 } 235 236 if (illegal) { 237 pr_warn("isolcpus: Invalid flag %.*s\n", len, par); 238 return 0; 239 } 240 241 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); 242 str++; 243 } 244 245 /* Default behaviour for isolcpus without flags */ 246 if (!flags) 247 flags |= HK_FLAG_DOMAIN; 248 249 return housekeeping_setup(str, flags); 250 } 251 __setup("isolcpus=", housekeeping_isolcpus_setup); 252