1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Housekeeping management. Manage the targets for routine code that can run on 4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work. 5 * 6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker 7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker 8 * 9 */ 10 11 enum hk_flags { 12 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER), 13 HK_FLAG_RCU = BIT(HK_TYPE_RCU), 14 HK_FLAG_MISC = BIT(HK_TYPE_MISC), 15 HK_FLAG_SCHED = BIT(HK_TYPE_SCHED), 16 HK_FLAG_TICK = BIT(HK_TYPE_TICK), 17 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), 18 HK_FLAG_WQ = BIT(HK_TYPE_WQ), 19 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), 20 HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD), 21 }; 22 23 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); 24 EXPORT_SYMBOL_GPL(housekeeping_overridden); 25 26 struct housekeeping { 27 cpumask_var_t cpumasks[HK_TYPE_MAX]; 28 unsigned long flags; 29 }; 30 31 static struct housekeeping housekeeping; 32 33 bool housekeeping_enabled(enum hk_type type) 34 { 35 return !!(housekeeping.flags & BIT(type)); 36 } 37 EXPORT_SYMBOL_GPL(housekeeping_enabled); 38 39 int housekeeping_any_cpu(enum hk_type type) 40 { 41 int cpu; 42 43 if (static_branch_unlikely(&housekeeping_overridden)) { 44 if (housekeeping.flags & BIT(type)) { 45 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id()); 46 if (cpu < nr_cpu_ids) 47 return cpu; 48 49 cpu = cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask); 50 if (likely(cpu < nr_cpu_ids)) 51 return cpu; 52 /* 53 * Unless we have another problem this can only happen 54 * at boot time before start_secondary() brings the 1st 55 * housekeeping CPU up. 56 */ 57 WARN_ON_ONCE(system_state == SYSTEM_RUNNING || 58 type != HK_TYPE_TIMER); 59 } 60 } 61 return smp_processor_id(); 62 } 63 EXPORT_SYMBOL_GPL(housekeeping_any_cpu); 64 65 const struct cpumask *housekeeping_cpumask(enum hk_type type) 66 { 67 if (static_branch_unlikely(&housekeeping_overridden)) 68 if (housekeeping.flags & BIT(type)) 69 return housekeeping.cpumasks[type]; 70 return cpu_possible_mask; 71 } 72 EXPORT_SYMBOL_GPL(housekeeping_cpumask); 73 74 void housekeeping_affine(struct task_struct *t, enum hk_type type) 75 { 76 if (static_branch_unlikely(&housekeeping_overridden)) 77 if (housekeeping.flags & BIT(type)) 78 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]); 79 } 80 EXPORT_SYMBOL_GPL(housekeeping_affine); 81 82 bool housekeeping_test_cpu(int cpu, enum hk_type type) 83 { 84 if (static_branch_unlikely(&housekeeping_overridden)) 85 if (housekeeping.flags & BIT(type)) 86 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]); 87 return true; 88 } 89 EXPORT_SYMBOL_GPL(housekeeping_test_cpu); 90 91 void __init housekeeping_init(void) 92 { 93 enum hk_type type; 94 95 if (!housekeeping.flags) 96 return; 97 98 static_branch_enable(&housekeeping_overridden); 99 100 if (housekeeping.flags & HK_FLAG_TICK) 101 sched_tick_offload_init(); 102 103 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { 104 /* We need at least one CPU to handle housekeeping work */ 105 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); 106 } 107 } 108 109 static void __init housekeeping_setup_type(enum hk_type type, 110 cpumask_var_t housekeeping_staging) 111 { 112 113 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]); 114 cpumask_copy(housekeeping.cpumasks[type], 115 housekeeping_staging); 116 } 117 118 static int __init housekeeping_setup(char *str, unsigned long flags) 119 { 120 cpumask_var_t non_housekeeping_mask, housekeeping_staging; 121 unsigned int first_cpu; 122 int err = 0; 123 124 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) { 125 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { 126 pr_warn("Housekeeping: nohz unsupported." 127 " Build with CONFIG_NO_HZ_FULL\n"); 128 return 0; 129 } 130 } 131 132 alloc_bootmem_cpumask_var(&non_housekeeping_mask); 133 if (cpulist_parse(str, non_housekeeping_mask) < 0) { 134 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); 135 goto free_non_housekeeping_mask; 136 } 137 138 alloc_bootmem_cpumask_var(&housekeeping_staging); 139 cpumask_andnot(housekeeping_staging, 140 cpu_possible_mask, non_housekeeping_mask); 141 142 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging); 143 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) { 144 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging); 145 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); 146 if (!housekeeping.flags) { 147 pr_warn("Housekeeping: must include one present CPU, " 148 "using boot CPU:%d\n", smp_processor_id()); 149 } 150 } 151 152 if (cpumask_empty(non_housekeeping_mask)) 153 goto free_housekeeping_staging; 154 155 if (!housekeeping.flags) { 156 /* First setup call ("nohz_full=" or "isolcpus=") */ 157 enum hk_type type; 158 159 for_each_set_bit(type, &flags, HK_TYPE_MAX) 160 housekeeping_setup_type(type, housekeeping_staging); 161 } else { 162 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */ 163 enum hk_type type; 164 unsigned long iter_flags = flags & housekeeping.flags; 165 166 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { 167 if (!cpumask_equal(housekeeping_staging, 168 housekeeping.cpumasks[type])) { 169 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); 170 goto free_housekeeping_staging; 171 } 172 } 173 174 iter_flags = flags & ~housekeeping.flags; 175 176 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) 177 housekeeping_setup_type(type, housekeeping_staging); 178 } 179 180 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) 181 tick_nohz_full_setup(non_housekeeping_mask); 182 183 housekeeping.flags |= flags; 184 err = 1; 185 186 free_housekeeping_staging: 187 free_bootmem_cpumask_var(housekeeping_staging); 188 free_non_housekeeping_mask: 189 free_bootmem_cpumask_var(non_housekeeping_mask); 190 191 return err; 192 } 193 194 static int __init housekeeping_nohz_full_setup(char *str) 195 { 196 unsigned long flags; 197 198 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | 199 HK_FLAG_MISC | HK_FLAG_KTHREAD; 200 201 return housekeeping_setup(str, flags); 202 } 203 __setup("nohz_full=", housekeeping_nohz_full_setup); 204 205 static int __init housekeeping_isolcpus_setup(char *str) 206 { 207 unsigned long flags = 0; 208 bool illegal = false; 209 char *par; 210 int len; 211 212 while (isalpha(*str)) { 213 if (!strncmp(str, "nohz,", 5)) { 214 str += 5; 215 flags |= HK_FLAG_TICK; 216 continue; 217 } 218 219 if (!strncmp(str, "domain,", 7)) { 220 str += 7; 221 flags |= HK_FLAG_DOMAIN; 222 continue; 223 } 224 225 if (!strncmp(str, "managed_irq,", 12)) { 226 str += 12; 227 flags |= HK_FLAG_MANAGED_IRQ; 228 continue; 229 } 230 231 /* 232 * Skip unknown sub-parameter and validate that it is not 233 * containing an invalid character. 234 */ 235 for (par = str, len = 0; *str && *str != ','; str++, len++) { 236 if (!isalpha(*str) && *str != '_') 237 illegal = true; 238 } 239 240 if (illegal) { 241 pr_warn("isolcpus: Invalid flag %.*s\n", len, par); 242 return 0; 243 } 244 245 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); 246 str++; 247 } 248 249 /* Default behaviour for isolcpus without flags */ 250 if (!flags) 251 flags |= HK_FLAG_DOMAIN; 252 253 return housekeeping_setup(str, flags); 254 } 255 __setup("isolcpus=", housekeeping_isolcpus_setup); 256