1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2024 4 */ 5 6 #define pr_fmt(fmt) "hd: " fmt 7 8 /* 9 * Hiperdispatch: 10 * Dynamically calculates the optimum number of high capacity COREs 11 * by considering the state the system is in. When hiperdispatch decides 12 * that a capacity update is necessary, it schedules a topology update. 13 * During topology updates the CPU capacities are always re-adjusted. 14 * 15 * There is two places where CPU capacities are being accessed within 16 * hiperdispatch. 17 * -> hiperdispatch's reoccuring work function reads CPU capacities to 18 * determine high capacity CPU count. 19 * -> during a topology update hiperdispatch's adjustment function 20 * updates CPU capacities. 21 * These two can run on different CPUs in parallel which can cause 22 * hiperdispatch to make wrong decisions. This can potentially cause 23 * some overhead by leading to extra rebuild_sched_domains() calls 24 * for correction. Access to capacities within hiperdispatch has to be 25 * serialized to prevent the overhead. 26 * 27 * Hiperdispatch decision making revolves around steal time. 28 * HD_STEAL_THRESHOLD value is taken as reference. Whenever steal time 29 * crosses the threshold value hiperdispatch falls back to giving high 30 * capacities to entitled CPUs. When steal time drops below the 31 * threshold boundary, hiperdispatch utilizes all CPUs by giving all 32 * of them high capacity. 33 * 34 * The theory behind HD_STEAL_THRESHOLD is related to the SMP thread 35 * performance. Comparing the throughput of; 36 * - single CORE, with N threads, running N tasks 37 * - N separate COREs running N tasks, 38 * using individual COREs for individual tasks yield better 39 * performance. This performance difference is roughly ~30% (can change 40 * between machine generations) 41 * 42 * Hiperdispatch tries to hint scheduler to use individual COREs for 43 * each task, as long as steal time on those COREs are less than 30%, 44 * therefore delaying the throughput loss caused by using SMP threads. 45 */ 46 47 #include <linux/cpufeature.h> 48 #include <linux/cpumask.h> 49 #include <linux/debugfs.h> 50 #include <linux/device.h> 51 #include <linux/kernel_stat.h> 52 #include <linux/kstrtox.h> 53 #include <linux/ktime.h> 54 #include <linux/sysctl.h> 55 #include <linux/types.h> 56 #include <linux/workqueue.h> 57 #include <asm/hiperdispatch.h> 58 #include <asm/setup.h> 59 #include <asm/smp.h> 60 #include <asm/topology.h> 61 62 #define CREATE_TRACE_POINTS 63 #include <asm/trace/hiperdispatch.h> 64 65 #define HD_DELAY_FACTOR (4) 66 #define HD_DELAY_INTERVAL (HZ / 4) 67 #define HD_STEAL_THRESHOLD 10 68 #define HD_STEAL_AVG_WEIGHT 16 69 70 static cpumask_t hd_vl_coremask; /* Mask containing all vertical low COREs */ 71 static cpumask_t hd_vmvl_cpumask; /* Mask containing vertical medium and low CPUs */ 72 static int hd_high_capacity_cores; /* Current CORE count with high capacity */ 73 static int hd_entitled_cores; /* Total vertical high and medium CORE count */ 74 static int hd_online_cores; /* Current online CORE count */ 75 76 static unsigned long hd_previous_steal; /* Previous iteration's CPU steal timer total */ 77 static unsigned long hd_high_time; /* Total time spent while all cpus have high capacity */ 78 static unsigned long hd_low_time; /* Total time spent while vl cpus have low capacity */ 79 static atomic64_t hd_adjustments; /* Total occurrence count of hiperdispatch adjustments */ 80 81 static unsigned int hd_steal_threshold = HD_STEAL_THRESHOLD; 82 static unsigned int hd_delay_factor = HD_DELAY_FACTOR; 83 static int hd_enabled; 84 85 static void hd_capacity_work_fn(struct work_struct *work); 86 static DECLARE_DELAYED_WORK(hd_capacity_work, hd_capacity_work_fn); 87 88 static int hd_set_hiperdispatch_mode(int enable) 89 { 90 if (!cpu_has_topology()) 91 enable = 0; 92 if (hd_enabled == enable) 93 return 0; 94 hd_enabled = enable; 95 return 1; 96 } 97 98 void hd_reset_state(void) 99 { 100 cpumask_clear(&hd_vl_coremask); 101 cpumask_clear(&hd_vmvl_cpumask); 102 hd_entitled_cores = 0; 103 hd_online_cores = 0; 104 } 105 106 void hd_add_core(int cpu) 107 { 108 const struct cpumask *siblings; 109 int polarization; 110 111 hd_online_cores++; 112 polarization = smp_cpu_get_polarization(cpu); 113 siblings = topology_sibling_cpumask(cpu); 114 switch (polarization) { 115 case POLARIZATION_VH: 116 hd_entitled_cores++; 117 break; 118 case POLARIZATION_VM: 119 hd_entitled_cores++; 120 cpumask_or(&hd_vmvl_cpumask, &hd_vmvl_cpumask, siblings); 121 break; 122 case POLARIZATION_VL: 123 cpumask_set_cpu(cpu, &hd_vl_coremask); 124 cpumask_or(&hd_vmvl_cpumask, &hd_vmvl_cpumask, siblings); 125 break; 126 } 127 } 128 129 /* Serialize update and read operations of debug counters. */ 130 static DEFINE_MUTEX(hd_counter_mutex); 131 132 static void hd_update_times(void) 133 { 134 static ktime_t prev; 135 ktime_t now; 136 137 /* 138 * Check if hiperdispatch is active, if not set the prev to 0. 139 * This way it is possible to differentiate the first update iteration after 140 * enabling hiperdispatch. 141 */ 142 if (hd_entitled_cores == 0 || hd_enabled == 0) { 143 prev = ktime_set(0, 0); 144 return; 145 } 146 now = ktime_get(); 147 if (ktime_after(prev, 0)) { 148 if (hd_high_capacity_cores == hd_online_cores) 149 hd_high_time += ktime_ms_delta(now, prev); 150 else 151 hd_low_time += ktime_ms_delta(now, prev); 152 } 153 prev = now; 154 } 155 156 static void hd_update_capacities(void) 157 { 158 int cpu, upscaling_cores; 159 unsigned long capacity; 160 161 upscaling_cores = hd_high_capacity_cores - hd_entitled_cores; 162 capacity = upscaling_cores > 0 ? CPU_CAPACITY_HIGH : CPU_CAPACITY_LOW; 163 hd_high_capacity_cores = hd_entitled_cores; 164 for_each_cpu(cpu, &hd_vl_coremask) { 165 smp_set_core_capacity(cpu, capacity); 166 if (capacity != CPU_CAPACITY_HIGH) 167 continue; 168 hd_high_capacity_cores++; 169 upscaling_cores--; 170 if (upscaling_cores == 0) 171 capacity = CPU_CAPACITY_LOW; 172 } 173 } 174 175 void hd_disable_hiperdispatch(void) 176 { 177 cancel_delayed_work_sync(&hd_capacity_work); 178 hd_high_capacity_cores = hd_online_cores; 179 hd_previous_steal = 0; 180 } 181 182 int hd_enable_hiperdispatch(void) 183 { 184 mutex_lock(&hd_counter_mutex); 185 hd_update_times(); 186 mutex_unlock(&hd_counter_mutex); 187 if (hd_enabled == 0) 188 return 0; 189 if (hd_entitled_cores == 0) 190 return 0; 191 if (hd_online_cores <= hd_entitled_cores) 192 return 0; 193 mod_delayed_work(system_dfl_wq, &hd_capacity_work, HD_DELAY_INTERVAL * hd_delay_factor); 194 hd_update_capacities(); 195 return 1; 196 } 197 198 static unsigned long hd_steal_avg(unsigned long new) 199 { 200 static unsigned long steal; 201 202 steal = (steal * (HD_STEAL_AVG_WEIGHT - 1) + new) / HD_STEAL_AVG_WEIGHT; 203 return steal; 204 } 205 206 static unsigned long hd_calculate_steal_percentage(void) 207 { 208 unsigned long time_delta, steal_delta, steal, percentage; 209 static ktime_t prev; 210 int cpus; 211 ktime_t now; 212 213 percentage = 0; 214 steal = kcpustat_field_total(CPUTIME_STEAL, &hd_vmvl_cpumask); 215 cpus = cpumask_weight(&hd_vmvl_cpumask); 216 /* 217 * If there is no vertical medium and low CPUs steal time 218 * is 0 as vertical high CPUs shouldn't experience steal time. 219 */ 220 if (cpus == 0) 221 return percentage; 222 now = ktime_get(); 223 time_delta = ktime_to_ns(ktime_sub(now, prev)); 224 if (steal > hd_previous_steal && hd_previous_steal != 0) { 225 steal_delta = (steal - hd_previous_steal) * 100 / time_delta; 226 percentage = steal_delta / cpus; 227 } 228 hd_previous_steal = steal; 229 prev = now; 230 return percentage; 231 } 232 233 static void hd_capacity_work_fn(struct work_struct *work) 234 { 235 unsigned long steal_percentage, new_cores; 236 237 mutex_lock(&smp_cpu_state_mutex); 238 /* 239 * If online cores are less or equal to entitled cores hiperdispatch 240 * does not need to make any adjustments, call a topology update to 241 * disable hiperdispatch. 242 * Normally this check is handled on topology update, but during cpu 243 * unhotplug, topology and cpu mask updates are done in reverse 244 * order, causing hd_enable_hiperdispatch() to get stale data. 245 */ 246 if (hd_online_cores <= hd_entitled_cores) { 247 topology_schedule_update(); 248 mutex_unlock(&smp_cpu_state_mutex); 249 return; 250 } 251 steal_percentage = hd_steal_avg(hd_calculate_steal_percentage()); 252 if (steal_percentage < hd_steal_threshold) 253 new_cores = hd_online_cores; 254 else 255 new_cores = hd_entitled_cores; 256 if (hd_high_capacity_cores != new_cores) { 257 trace_s390_hd_rebuild_domains(hd_high_capacity_cores, new_cores); 258 hd_high_capacity_cores = new_cores; 259 atomic64_inc(&hd_adjustments); 260 topology_schedule_update(); 261 } 262 trace_s390_hd_work_fn(steal_percentage, hd_entitled_cores, hd_high_capacity_cores); 263 mutex_unlock(&smp_cpu_state_mutex); 264 queue_delayed_work(system_dfl_wq, &hd_capacity_work, HD_DELAY_INTERVAL); 265 } 266 267 static int hiperdispatch_ctl_handler(const struct ctl_table *ctl, int write, 268 void *buffer, size_t *lenp, loff_t *ppos) 269 { 270 int hiperdispatch; 271 int rc; 272 struct ctl_table ctl_entry = { 273 .procname = ctl->procname, 274 .data = &hiperdispatch, 275 .maxlen = sizeof(int), 276 .extra1 = SYSCTL_ZERO, 277 .extra2 = SYSCTL_ONE, 278 }; 279 280 hiperdispatch = hd_enabled; 281 rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos); 282 if (rc < 0 || !write) 283 return rc; 284 mutex_lock(&smp_cpu_state_mutex); 285 if (hd_set_hiperdispatch_mode(hiperdispatch)) 286 topology_schedule_update(); 287 mutex_unlock(&smp_cpu_state_mutex); 288 return 0; 289 } 290 291 static const struct ctl_table hiperdispatch_ctl_table[] = { 292 { 293 .procname = "hiperdispatch", 294 .mode = 0644, 295 .proc_handler = hiperdispatch_ctl_handler, 296 }, 297 }; 298 299 static ssize_t hd_steal_threshold_show(struct device *dev, 300 struct device_attribute *attr, 301 char *buf) 302 { 303 return sysfs_emit(buf, "%u\n", hd_steal_threshold); 304 } 305 306 static ssize_t hd_steal_threshold_store(struct device *dev, 307 struct device_attribute *attr, 308 const char *buf, 309 size_t count) 310 { 311 unsigned int val; 312 int rc; 313 314 rc = kstrtouint(buf, 0, &val); 315 if (rc) 316 return rc; 317 if (val > 100) 318 return -ERANGE; 319 hd_steal_threshold = val; 320 return count; 321 } 322 323 static DEVICE_ATTR_RW(hd_steal_threshold); 324 325 static ssize_t hd_delay_factor_show(struct device *dev, 326 struct device_attribute *attr, 327 char *buf) 328 { 329 return sysfs_emit(buf, "%u\n", hd_delay_factor); 330 } 331 332 static ssize_t hd_delay_factor_store(struct device *dev, 333 struct device_attribute *attr, 334 const char *buf, 335 size_t count) 336 { 337 unsigned int val; 338 int rc; 339 340 rc = kstrtouint(buf, 0, &val); 341 if (rc) 342 return rc; 343 if (!val) 344 return -ERANGE; 345 hd_delay_factor = val; 346 return count; 347 } 348 349 static DEVICE_ATTR_RW(hd_delay_factor); 350 351 static struct attribute *hd_attrs[] = { 352 &dev_attr_hd_steal_threshold.attr, 353 &dev_attr_hd_delay_factor.attr, 354 NULL, 355 }; 356 357 static const struct attribute_group hd_attr_group = { 358 .name = "hiperdispatch", 359 .attrs = hd_attrs, 360 }; 361 362 static int hd_greedy_time_get(void *unused, u64 *val) 363 { 364 mutex_lock(&hd_counter_mutex); 365 hd_update_times(); 366 *val = hd_high_time; 367 mutex_unlock(&hd_counter_mutex); 368 return 0; 369 } 370 371 DEFINE_SIMPLE_ATTRIBUTE(hd_greedy_time_fops, hd_greedy_time_get, NULL, "%llu\n"); 372 373 static int hd_conservative_time_get(void *unused, u64 *val) 374 { 375 mutex_lock(&hd_counter_mutex); 376 hd_update_times(); 377 *val = hd_low_time; 378 mutex_unlock(&hd_counter_mutex); 379 return 0; 380 } 381 382 DEFINE_SIMPLE_ATTRIBUTE(hd_conservative_time_fops, hd_conservative_time_get, NULL, "%llu\n"); 383 384 static int hd_adjustment_count_get(void *unused, u64 *val) 385 { 386 *val = atomic64_read(&hd_adjustments); 387 return 0; 388 } 389 390 DEFINE_SIMPLE_ATTRIBUTE(hd_adjustments_fops, hd_adjustment_count_get, NULL, "%llu\n"); 391 392 static void __init hd_create_debugfs_counters(void) 393 { 394 struct dentry *dir; 395 396 dir = debugfs_create_dir("hiperdispatch", arch_debugfs_dir); 397 debugfs_create_file("conservative_time_ms", 0400, dir, NULL, &hd_conservative_time_fops); 398 debugfs_create_file("greedy_time_ms", 0400, dir, NULL, &hd_greedy_time_fops); 399 debugfs_create_file("adjustment_count", 0400, dir, NULL, &hd_adjustments_fops); 400 } 401 402 static void __init hd_create_attributes(void) 403 { 404 struct device *dev; 405 406 dev = bus_get_dev_root(&cpu_subsys); 407 if (!dev) 408 return; 409 if (sysfs_create_group(&dev->kobj, &hd_attr_group)) 410 pr_warn("Unable to create hiperdispatch attribute group\n"); 411 put_device(dev); 412 } 413 414 static int __init hd_init(void) 415 { 416 if (IS_ENABLED(CONFIG_HIPERDISPATCH_ON)) { 417 hd_set_hiperdispatch_mode(1); 418 topology_schedule_update(); 419 } 420 if (!register_sysctl("s390", hiperdispatch_ctl_table)) 421 pr_warn("Failed to register s390.hiperdispatch sysctl attribute\n"); 422 hd_create_debugfs_counters(); 423 hd_create_attributes(); 424 return 0; 425 } 426 late_initcall(hd_init); 427