1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2013, Michael (Ellerman|Neuling), IBM Corporation. 4 */ 5 6 #define pr_fmt(fmt) "powernv: " fmt 7 8 #include <linux/kernel.h> 9 #include <linux/cpu.h> 10 #include <linux/cpumask.h> 11 #include <linux/device.h> 12 #include <linux/gfp.h> 13 #include <linux/smp.h> 14 #include <linux/stop_machine.h> 15 #include <linux/sysfs.h> 16 17 #include <asm/cputhreads.h> 18 #include <asm/cpuidle.h> 19 #include <asm/kvm_ppc.h> 20 #include <asm/machdep.h> 21 #include <asm/opal.h> 22 #include <asm/smp.h> 23 24 #include <trace/events/ipi.h> 25 26 #include "subcore.h" 27 #include "powernv.h" 28 29 30 /* 31 * Split/unsplit procedure: 32 * 33 * A core can be in one of three states, unsplit, 2-way split, and 4-way split. 34 * 35 * The mapping to subcores_per_core is simple: 36 * 37 * State | subcores_per_core 38 * ------------|------------------ 39 * Unsplit | 1 40 * 2-way split | 2 41 * 4-way split | 4 42 * 43 * The core is split along thread boundaries, the mapping between subcores and 44 * threads is as follows: 45 * 46 * Unsplit: 47 * ---------------------------- 48 * Subcore | 0 | 49 * ---------------------------- 50 * Thread | 0 1 2 3 4 5 6 7 | 51 * ---------------------------- 52 * 53 * 2-way split: 54 * ------------------------------------- 55 * Subcore | 0 | 1 | 56 * ------------------------------------- 57 * Thread | 0 1 2 3 | 4 5 6 7 | 58 * ------------------------------------- 59 * 60 * 4-way split: 61 * ----------------------------------------- 62 * Subcore | 0 | 1 | 2 | 3 | 63 * ----------------------------------------- 64 * Thread | 0 1 | 2 3 | 4 5 | 6 7 | 65 * ----------------------------------------- 66 * 67 * 68 * Transitions 69 * ----------- 70 * 71 * It is not possible to transition between either of the split states, the 72 * core must first be unsplit. The legal transitions are: 73 * 74 * ----------- --------------- 75 * | | <----> | 2-way split | 76 * | | --------------- 77 * | Unsplit | 78 * | | --------------- 79 * | | <----> | 4-way split | 80 * ----------- --------------- 81 * 82 * Unsplitting 83 * ----------- 84 * 85 * Unsplitting is the simpler procedure. It requires thread 0 to request the 86 * unsplit while all other threads NAP. 87 * 88 * Thread 0 clears HID0_POWER8_DYNLPARDIS (Dynamic LPAR Disable). This tells 89 * the hardware that if all threads except 0 are napping, the hardware should 90 * unsplit the core. 91 * 92 * Non-zero threads are sent to a NAP loop, they don't exit the loop until they 93 * see the core unsplit. 94 * 95 * Core 0 spins waiting for the hardware to see all the other threads napping 96 * and perform the unsplit. 97 * 98 * Once thread 0 sees the unsplit, it IPIs the secondary threads to wake them 99 * out of NAP. They will then see the core unsplit and exit the NAP loop. 100 * 101 * Splitting 102 * --------- 103 * 104 * The basic splitting procedure is fairly straight forward. However it is 105 * complicated by the fact that after the split occurs, the newly created 106 * subcores are not in a fully initialised state. 107 * 108 * Most notably the subcores do not have the correct value for SDR1, which 109 * means they must not be running in virtual mode when the split occurs. The 110 * subcores have separate timebases SPRs but these are pre-synchronised by 111 * opal. 112 * 113 * To begin with secondary threads are sent to an assembly routine. There they 114 * switch to real mode, so they are immune to the uninitialised SDR1 value. 115 * Once in real mode they indicate that they are in real mode, and spin waiting 116 * to see the core split. 117 * 118 * Thread 0 waits to see that all secondaries are in real mode, and then begins 119 * the splitting procedure. It firstly sets HID0_POWER8_DYNLPARDIS, which 120 * prevents the hardware from unsplitting. Then it sets the appropriate HID bit 121 * to request the split, and spins waiting to see that the split has happened. 122 * 123 * Concurrently the secondaries will notice the split. When they do they set up 124 * their SPRs, notably SDR1, and then they can return to virtual mode and exit 125 * the procedure. 126 */ 127 128 /* Initialised at boot by subcore_init() */ 129 static int subcores_per_core; 130 131 /* 132 * Used to communicate to offline cpus that we want them to pop out of the 133 * offline loop and do a split or unsplit. 134 * 135 * 0 - no split happening 136 * 1 - unsplit in progress 137 * 2 - split to 2 in progress 138 * 4 - split to 4 in progress 139 */ 140 static int new_split_mode; 141 142 static cpumask_var_t cpu_offline_mask; 143 144 struct split_state { 145 u8 step; 146 u8 master; 147 }; 148 149 static DEFINE_PER_CPU(struct split_state, split_state); 150 151 static void wait_for_sync_step(int step) 152 { 153 int i, cpu = smp_processor_id(); 154 155 for (i = cpu + 1; i < cpu + threads_per_core; i++) 156 while(per_cpu(split_state, i).step < step) 157 barrier(); 158 159 /* Order the wait loop vs any subsequent loads/stores. */ 160 mb(); 161 } 162 163 static void update_hid_in_slw(u64 hid0) 164 { 165 u64 idle_states = pnv_get_supported_cpuidle_states(); 166 167 if (idle_states & OPAL_PM_WINKLE_ENABLED) { 168 /* OPAL call to patch slw with the new HID0 value */ 169 u64 cpu_pir = hard_smp_processor_id(); 170 171 opal_slw_set_reg(cpu_pir, SPRN_HID0, hid0); 172 } 173 } 174 175 static inline void update_power8_hid0(unsigned long hid0) 176 { 177 /* 178 * The HID0 update on Power8 should at the very least be 179 * preceded by a SYNC instruction followed by an ISYNC 180 * instruction 181 */ 182 asm volatile("sync; mtspr %0,%1; isync":: "i"(SPRN_HID0), "r"(hid0)); 183 } 184 185 static void unsplit_core(void) 186 { 187 u64 hid0, mask; 188 int i, cpu; 189 190 mask = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; 191 192 cpu = smp_processor_id(); 193 if (cpu_thread_in_core(cpu) != 0) { 194 while (mfspr(SPRN_HID0) & mask) 195 power7_idle_type(PNV_THREAD_NAP); 196 197 per_cpu(split_state, cpu).step = SYNC_STEP_UNSPLIT; 198 return; 199 } 200 201 hid0 = mfspr(SPRN_HID0); 202 hid0 &= ~HID0_POWER8_DYNLPARDIS; 203 update_power8_hid0(hid0); 204 update_hid_in_slw(hid0); 205 206 while (mfspr(SPRN_HID0) & mask) 207 cpu_relax(); 208 209 /* Wake secondaries out of NAP */ 210 for (i = cpu + 1; i < cpu + threads_per_core; i++) 211 smp_send_reschedule(i); 212 213 wait_for_sync_step(SYNC_STEP_UNSPLIT); 214 } 215 216 static void split_core(int new_mode) 217 { 218 struct { u64 value; u64 mask; } split_parms[2] = { 219 { HID0_POWER8_1TO2LPAR, HID0_POWER8_2LPARMODE }, 220 { HID0_POWER8_1TO4LPAR, HID0_POWER8_4LPARMODE } 221 }; 222 int i, cpu; 223 u64 hid0; 224 225 /* Convert new_mode (2 or 4) into an index into our parms array */ 226 i = (new_mode >> 1) - 1; 227 BUG_ON(i < 0 || i > 1); 228 229 cpu = smp_processor_id(); 230 if (cpu_thread_in_core(cpu) != 0) { 231 split_core_secondary_loop(&per_cpu(split_state, cpu).step); 232 return; 233 } 234 235 wait_for_sync_step(SYNC_STEP_REAL_MODE); 236 237 /* Write new mode */ 238 hid0 = mfspr(SPRN_HID0); 239 hid0 |= HID0_POWER8_DYNLPARDIS | split_parms[i].value; 240 update_power8_hid0(hid0); 241 update_hid_in_slw(hid0); 242 243 /* Wait for it to happen */ 244 while (!(mfspr(SPRN_HID0) & split_parms[i].mask)) 245 cpu_relax(); 246 } 247 248 static void cpu_do_split(int new_mode) 249 { 250 /* 251 * At boot subcores_per_core will be 0, so we will always unsplit at 252 * boot. In the usual case where the core is already unsplit it's a 253 * nop, and this just ensures the kernel's notion of the mode is 254 * consistent with the hardware. 255 */ 256 if (subcores_per_core != 1) 257 unsplit_core(); 258 259 if (new_mode != 1) 260 split_core(new_mode); 261 262 mb(); 263 per_cpu(split_state, smp_processor_id()).step = SYNC_STEP_FINISHED; 264 } 265 266 bool cpu_core_split_required(void) 267 { 268 smp_rmb(); 269 270 if (!new_split_mode) 271 return false; 272 273 cpu_do_split(new_split_mode); 274 275 return true; 276 } 277 278 void update_subcore_sibling_mask(void) 279 { 280 int cpu; 281 /* 282 * sibling mask for the first cpu. Left shift this by required bits 283 * to get sibling mask for the rest of the cpus. 284 */ 285 int sibling_mask_first_cpu = (1 << threads_per_subcore) - 1; 286 287 for_each_possible_cpu(cpu) { 288 int tid = cpu_thread_in_core(cpu); 289 int offset = (tid / threads_per_subcore) * threads_per_subcore; 290 int mask = sibling_mask_first_cpu << offset; 291 292 paca_ptrs[cpu]->subcore_sibling_mask = mask; 293 294 } 295 } 296 297 static int cpu_update_split_mode(void *data) 298 { 299 int cpu, new_mode = *(int *)data; 300 301 if (this_cpu_ptr(&split_state)->master) { 302 new_split_mode = new_mode; 303 smp_wmb(); 304 305 cpumask_andnot(cpu_offline_mask, cpu_present_mask, 306 cpu_online_mask); 307 308 /* This should work even though the cpu is offline */ 309 for_each_cpu(cpu, cpu_offline_mask) 310 smp_send_reschedule(cpu); 311 } 312 313 cpu_do_split(new_mode); 314 315 if (this_cpu_ptr(&split_state)->master) { 316 /* Wait for all cpus to finish before we touch subcores_per_core */ 317 for_each_present_cpu(cpu) { 318 if (cpu >= setup_max_cpus) 319 break; 320 321 while(per_cpu(split_state, cpu).step < SYNC_STEP_FINISHED) 322 barrier(); 323 } 324 325 new_split_mode = 0; 326 327 /* Make the new mode public */ 328 subcores_per_core = new_mode; 329 threads_per_subcore = threads_per_core / subcores_per_core; 330 update_subcore_sibling_mask(); 331 332 /* Make sure the new mode is written before we exit */ 333 mb(); 334 } 335 336 return 0; 337 } 338 339 static int set_subcores_per_core(int new_mode) 340 { 341 struct split_state *state; 342 int cpu; 343 344 if (kvm_hv_mode_active()) { 345 pr_err("Unable to change split core mode while KVM active.\n"); 346 return -EBUSY; 347 } 348 349 /* 350 * We are only called at boot, or from the sysfs write. If that ever 351 * changes we'll need a lock here. 352 */ 353 BUG_ON(new_mode < 1 || new_mode > 4 || new_mode == 3); 354 355 for_each_present_cpu(cpu) { 356 state = &per_cpu(split_state, cpu); 357 state->step = SYNC_STEP_INITIAL; 358 state->master = 0; 359 } 360 361 cpus_read_lock(); 362 363 /* This cpu will update the globals before exiting stop machine */ 364 this_cpu_ptr(&split_state)->master = 1; 365 366 /* Ensure state is consistent before we call the other cpus */ 367 mb(); 368 369 stop_machine_cpuslocked(cpu_update_split_mode, &new_mode, 370 cpu_online_mask); 371 372 cpus_read_unlock(); 373 374 return 0; 375 } 376 377 static ssize_t __used store_subcores_per_core(struct device *dev, 378 struct device_attribute *attr, const char *buf, 379 size_t count) 380 { 381 unsigned long val; 382 int rc; 383 384 /* We are serialised by the attribute lock */ 385 386 rc = sscanf(buf, "%lx", &val); 387 if (rc != 1) 388 return -EINVAL; 389 390 switch (val) { 391 case 1: 392 case 2: 393 case 4: 394 if (subcores_per_core == val) 395 /* Nothing to do */ 396 goto out; 397 break; 398 default: 399 return -EINVAL; 400 } 401 402 rc = set_subcores_per_core(val); 403 if (rc) 404 return rc; 405 406 out: 407 return count; 408 } 409 410 static ssize_t show_subcores_per_core(struct device *dev, 411 struct device_attribute *attr, char *buf) 412 { 413 return sysfs_emit(buf, "%x\n", subcores_per_core); 414 } 415 416 static DEVICE_ATTR(subcores_per_core, 0644, 417 show_subcores_per_core, store_subcores_per_core); 418 419 static int subcore_init(void) 420 { 421 struct device *dev_root; 422 unsigned pvr_ver; 423 int rc = 0; 424 425 pvr_ver = PVR_VER(mfspr(SPRN_PVR)); 426 427 if (pvr_ver != PVR_POWER8 && 428 pvr_ver != PVR_POWER8E && 429 pvr_ver != PVR_POWER8NVL && 430 pvr_ver != PVR_HX_C2000) 431 return 0; 432 433 /* 434 * We need all threads in a core to be present to split/unsplit so 435 * continue only if max_cpus are aligned to threads_per_core. 436 */ 437 if (setup_max_cpus % threads_per_core) 438 return 0; 439 440 BUG_ON(!alloc_cpumask_var(&cpu_offline_mask, GFP_KERNEL)); 441 442 set_subcores_per_core(1); 443 444 dev_root = bus_get_dev_root(&cpu_subsys); 445 if (dev_root) { 446 rc = device_create_file(dev_root, &dev_attr_subcores_per_core); 447 put_device(dev_root); 448 } 449 return rc; 450 } 451 machine_device_initcall(powernv, subcore_init); 452