1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2014 Imagination Technologies 4 * Author: Paul Burton <paul.burton@mips.com> 5 */ 6 7 #include <linux/cpuhotplug.h> 8 #include <linux/init.h> 9 #include <linux/percpu.h> 10 #include <linux/slab.h> 11 #include <linux/suspend.h> 12 13 #include <asm/asm-offsets.h> 14 #include <asm/cacheflush.h> 15 #include <asm/cacheops.h> 16 #include <asm/idle.h> 17 #include <asm/mips-cps.h> 18 #include <asm/mipsmtregs.h> 19 #include <asm/pm.h> 20 #include <asm/pm-cps.h> 21 #include <asm/regdef.h> 22 #include <asm/smp-cps.h> 23 #include <asm/uasm.h> 24 25 /* 26 * cps_nc_entry_fn - type of a generated non-coherent state entry function 27 * @online: the count of online coupled VPEs 28 * @nc_ready_count: pointer to a non-coherent mapping of the core ready_count 29 * 30 * The code entering & exiting non-coherent states is generated at runtime 31 * using uasm, in order to ensure that the compiler cannot insert a stray 32 * memory access at an unfortunate time and to allow the generation of optimal 33 * core-specific code particularly for cache routines. If coupled_coherence 34 * is non-zero and this is the entry function for the CPS_PM_NC_WAIT state, 35 * returns the number of VPEs that were in the wait state at the point this 36 * VPE left it. Returns garbage if coupled_coherence is zero or this is not 37 * the entry function for CPS_PM_NC_WAIT. 38 */ 39 typedef unsigned (*cps_nc_entry_fn)(unsigned online, u32 *nc_ready_count); 40 41 /* 42 * The entry point of the generated non-coherent idle state entry/exit 43 * functions. Actually per-core rather than per-CPU. 44 */ 45 static DEFINE_PER_CPU_READ_MOSTLY(cps_nc_entry_fn[CPS_PM_STATE_COUNT], 46 nc_asm_enter); 47 48 /* Bitmap indicating which states are supported by the system */ 49 static DECLARE_BITMAP(state_support, CPS_PM_STATE_COUNT); 50 51 /* 52 * Indicates the number of coupled VPEs ready to operate in a non-coherent 53 * state. Actually per-core rather than per-CPU. 54 */ 55 static DEFINE_PER_CPU_ALIGNED(u32*, ready_count); 56 57 /* Indicates online CPUs coupled with the current CPU */ 58 static DEFINE_PER_CPU_ALIGNED(cpumask_t, online_coupled); 59 60 /* Used to synchronize entry to deep idle states */ 61 static DEFINE_PER_CPU_ALIGNED(atomic_t, pm_barrier); 62 63 /* Saved CPU state across the CPS_PM_POWER_GATED state */ 64 DEFINE_PER_CPU_ALIGNED(struct mips_static_suspend_state, cps_cpu_state); 65 66 /* A somewhat arbitrary number of labels & relocs for uasm */ 67 static struct uasm_label labels[32]; 68 static struct uasm_reloc relocs[32]; 69 70 bool cps_pm_support_state(enum cps_pm_state state) 71 { 72 return test_bit(state, state_support); 73 } 74 75 static void coupled_barrier(atomic_t *a, unsigned online) 76 { 77 /* 78 * This function is effectively the same as 79 * cpuidle_coupled_parallel_barrier, which can't be used here since 80 * there's no cpuidle device. 81 */ 82 83 if (!coupled_coherence) 84 return; 85 86 smp_mb__before_atomic(); 87 atomic_inc(a); 88 89 while (atomic_read(a) < online) 90 cpu_relax(); 91 92 if (atomic_inc_return(a) == online * 2) { 93 atomic_set(a, 0); 94 return; 95 } 96 97 while (atomic_read(a) > online) 98 cpu_relax(); 99 } 100 101 int cps_pm_enter_state(enum cps_pm_state state) 102 { 103 unsigned cpu = smp_processor_id(); 104 unsigned int cluster = cpu_cluster(¤t_cpu_data); 105 unsigned core = cpu_core(¤t_cpu_data); 106 unsigned online, left; 107 cpumask_t *coupled_mask = this_cpu_ptr(&online_coupled); 108 u32 *core_ready_count, *nc_core_ready_count; 109 void *nc_addr; 110 cps_nc_entry_fn entry; 111 struct cluster_boot_config *cluster_cfg; 112 struct core_boot_config *core_cfg; 113 struct vpe_boot_config *vpe_cfg; 114 atomic_t *barrier; 115 116 /* Check that there is an entry function for this state */ 117 entry = per_cpu(nc_asm_enter, cpu)[state]; 118 if (!entry) 119 return -EINVAL; 120 121 /* Calculate which coupled CPUs (VPEs) are online */ 122 #if defined(CONFIG_MIPS_MT) || defined(CONFIG_CPU_MIPSR6) 123 if (cpu_online(cpu)) { 124 cpumask_and(coupled_mask, cpu_online_mask, 125 &cpu_sibling_map[cpu]); 126 online = cpumask_weight(coupled_mask); 127 cpumask_clear_cpu(cpu, coupled_mask); 128 } else 129 #endif 130 { 131 cpumask_clear(coupled_mask); 132 online = 1; 133 } 134 135 /* Setup the VPE to run mips_cps_pm_restore when started again */ 136 if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) { 137 /* Power gating relies upon CPS SMP */ 138 if (!mips_cps_smp_in_use()) 139 return -EINVAL; 140 141 cluster_cfg = &mips_cps_cluster_bootcfg[cluster]; 142 core_cfg = &cluster_cfg->core_config[core]; 143 vpe_cfg = &core_cfg->vpe_config[cpu_vpe_id(¤t_cpu_data)]; 144 vpe_cfg->pc = (unsigned long)mips_cps_pm_restore; 145 vpe_cfg->gp = (unsigned long)current_thread_info(); 146 vpe_cfg->sp = 0; 147 } 148 149 /* Indicate that this CPU might not be coherent */ 150 cpumask_clear_cpu(cpu, &cpu_coherent_mask); 151 smp_mb__after_atomic(); 152 153 /* Create a non-coherent mapping of the core ready_count */ 154 core_ready_count = per_cpu(ready_count, cpu); 155 nc_addr = kmap_noncoherent(virt_to_page(core_ready_count), 156 (unsigned long)core_ready_count); 157 nc_addr += ((unsigned long)core_ready_count & ~PAGE_MASK); 158 nc_core_ready_count = nc_addr; 159 160 /* Ensure ready_count is zero-initialised before the assembly runs */ 161 WRITE_ONCE(*nc_core_ready_count, 0); 162 barrier = &per_cpu(pm_barrier, cpumask_first(&cpu_sibling_map[cpu])); 163 coupled_barrier(barrier, online); 164 165 /* Run the generated entry code */ 166 left = entry(online, nc_core_ready_count); 167 168 /* Remove the non-coherent mapping of ready_count */ 169 kunmap_noncoherent(); 170 171 /* Indicate that this CPU is definitely coherent */ 172 cpumask_set_cpu(cpu, &cpu_coherent_mask); 173 174 /* 175 * If this VPE is the first to leave the non-coherent wait state then 176 * it needs to wake up any coupled VPEs still running their wait 177 * instruction so that they return to cpuidle, which can then complete 178 * coordination between the coupled VPEs & provide the governor with 179 * a chance to reflect on the length of time the VPEs were in the 180 * idle state. 181 */ 182 if (coupled_coherence && (state == CPS_PM_NC_WAIT) && (left == online)) 183 arch_send_call_function_ipi_mask(coupled_mask); 184 185 return 0; 186 } 187 188 static void cps_gen_cache_routine(u32 **pp, struct uasm_label **pl, 189 struct uasm_reloc **pr, 190 const struct cache_desc *cache, 191 unsigned op, int lbl) 192 { 193 unsigned cache_size = cache->ways << cache->waybit; 194 unsigned i; 195 const unsigned unroll_lines = 32; 196 197 /* If the cache isn't present this function has it easy */ 198 if (cache->flags & MIPS_CACHE_NOT_PRESENT) 199 return; 200 201 /* Load base address */ 202 UASM_i_LA(pp, GPR_T0, (long)CKSEG0); 203 204 /* Calculate end address */ 205 if (cache_size < 0x8000) 206 uasm_i_addiu(pp, GPR_T1, GPR_T0, cache_size); 207 else 208 UASM_i_LA(pp, GPR_T1, (long)(CKSEG0 + cache_size)); 209 210 /* Start of cache op loop */ 211 uasm_build_label(pl, *pp, lbl); 212 213 /* Generate the cache ops */ 214 for (i = 0; i < unroll_lines; i++) { 215 if (cpu_has_mips_r6) { 216 uasm_i_cache(pp, op, 0, GPR_T0); 217 uasm_i_addiu(pp, GPR_T0, GPR_T0, cache->linesz); 218 } else { 219 uasm_i_cache(pp, op, i * cache->linesz, GPR_T0); 220 } 221 } 222 223 if (!cpu_has_mips_r6) 224 /* Update the base address */ 225 uasm_i_addiu(pp, GPR_T0, GPR_T0, unroll_lines * cache->linesz); 226 227 /* Loop if we haven't reached the end address yet */ 228 uasm_il_bne(pp, pr, GPR_T0, GPR_T1, lbl); 229 uasm_i_nop(pp); 230 } 231 232 static int cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, 233 struct uasm_reloc **pr, 234 const struct cpuinfo_mips *cpu_info, 235 int lbl) 236 { 237 unsigned i, fsb_size = 8; 238 unsigned num_loads = (fsb_size * 3) / 2; 239 unsigned line_stride = 2; 240 unsigned line_size = cpu_info->dcache.linesz; 241 unsigned perf_counter, perf_event; 242 unsigned revision = cpu_info->processor_id & PRID_REV_MASK; 243 244 /* 245 * Determine whether this CPU requires an FSB flush, and if so which 246 * performance counter/event reflect stalls due to a full FSB. 247 */ 248 switch (__get_cpu_type(cpu_info->cputype)) { 249 case CPU_INTERAPTIV: 250 perf_counter = 1; 251 perf_event = 51; 252 break; 253 254 case CPU_PROAPTIV: 255 /* Newer proAptiv cores don't require this workaround */ 256 if (revision >= PRID_REV_ENCODE_332(1, 1, 0)) 257 return 0; 258 259 /* On older ones it's unavailable */ 260 return -1; 261 262 default: 263 /* Assume that the CPU does not need this workaround */ 264 return 0; 265 } 266 267 /* 268 * Ensure that the fill/store buffer (FSB) is not holding the results 269 * of a prefetch, since if it is then the CPC sequencer may become 270 * stuck in the D3 (ClrBus) state whilst entering a low power state. 271 */ 272 273 /* Preserve perf counter setup */ 274 uasm_i_mfc0(pp, GPR_T2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ 275 uasm_i_mfc0(pp, GPR_T3, 25, (perf_counter * 2) + 1); /* PerfCntN */ 276 277 /* Setup perf counter to count FSB full pipeline stalls */ 278 uasm_i_addiu(pp, GPR_T0, GPR_ZERO, (perf_event << 5) | 0xf); 279 uasm_i_mtc0(pp, GPR_T0, 25, (perf_counter * 2) + 0); /* PerfCtlN */ 280 uasm_i_ehb(pp); 281 uasm_i_mtc0(pp, GPR_ZERO, 25, (perf_counter * 2) + 1); /* PerfCntN */ 282 uasm_i_ehb(pp); 283 284 /* Base address for loads */ 285 UASM_i_LA(pp, GPR_T0, (long)CKSEG0); 286 287 /* Start of clear loop */ 288 uasm_build_label(pl, *pp, lbl); 289 290 /* Perform some loads to fill the FSB */ 291 for (i = 0; i < num_loads; i++) 292 uasm_i_lw(pp, GPR_ZERO, i * line_size * line_stride, GPR_T0); 293 294 /* 295 * Invalidate the new D-cache entries so that the cache will need 296 * refilling (via the FSB) if the loop is executed again. 297 */ 298 for (i = 0; i < num_loads; i++) { 299 uasm_i_cache(pp, Hit_Invalidate_D, 300 i * line_size * line_stride, GPR_T0); 301 uasm_i_cache(pp, Hit_Writeback_Inv_SD, 302 i * line_size * line_stride, GPR_T0); 303 } 304 305 /* Barrier ensuring previous cache invalidates are complete */ 306 uasm_i_sync(pp, __SYNC_full); 307 uasm_i_ehb(pp); 308 309 /* Check whether the pipeline stalled due to the FSB being full */ 310 uasm_i_mfc0(pp, GPR_T1, 25, (perf_counter * 2) + 1); /* PerfCntN */ 311 312 /* Loop if it didn't */ 313 uasm_il_beqz(pp, pr, GPR_T1, lbl); 314 uasm_i_nop(pp); 315 316 /* Restore perf counter 1. The count may well now be wrong... */ 317 uasm_i_mtc0(pp, GPR_T2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ 318 uasm_i_ehb(pp); 319 uasm_i_mtc0(pp, GPR_T3, 25, (perf_counter * 2) + 1); /* PerfCntN */ 320 uasm_i_ehb(pp); 321 322 return 0; 323 } 324 325 static void cps_gen_set_top_bit(u32 **pp, struct uasm_label **pl, 326 struct uasm_reloc **pr, 327 unsigned r_addr, int lbl) 328 { 329 uasm_i_lui(pp, GPR_T0, uasm_rel_hi(0x80000000)); 330 uasm_build_label(pl, *pp, lbl); 331 uasm_i_ll(pp, GPR_T1, 0, r_addr); 332 uasm_i_or(pp, GPR_T1, GPR_T1, GPR_T0); 333 uasm_i_sc(pp, GPR_T1, 0, r_addr); 334 uasm_il_beqz(pp, pr, GPR_T1, lbl); 335 uasm_i_nop(pp); 336 } 337 338 static void *cps_gen_entry_code(unsigned cpu, enum cps_pm_state state) 339 { 340 struct uasm_label *l = labels; 341 struct uasm_reloc *r = relocs; 342 u32 *buf, *p; 343 const unsigned r_online = GPR_A0; 344 const unsigned r_nc_count = GPR_A1; 345 const unsigned r_pcohctl = GPR_T8; 346 const unsigned max_instrs = 256; 347 unsigned cpc_cmd; 348 int err; 349 enum { 350 lbl_incready = 1, 351 lbl_poll_cont, 352 lbl_secondary_hang, 353 lbl_disable_coherence, 354 lbl_flush_fsb, 355 lbl_invicache, 356 lbl_flushdcache, 357 lbl_hang, 358 lbl_set_cont, 359 lbl_secondary_cont, 360 lbl_decready, 361 }; 362 363 /* Allocate a buffer to hold the generated code */ 364 p = buf = kcalloc(max_instrs, sizeof(u32), GFP_KERNEL); 365 if (!buf) 366 return NULL; 367 368 /* Clear labels & relocs ready for (re)use */ 369 memset(labels, 0, sizeof(labels)); 370 memset(relocs, 0, sizeof(relocs)); 371 372 if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) { 373 /* Power gating relies upon CPS SMP */ 374 if (!mips_cps_smp_in_use()) 375 goto out_err; 376 377 /* 378 * Save CPU state. Note the non-standard calling convention 379 * with the return address placed in v0 to avoid clobbering 380 * the ra register before it is saved. 381 */ 382 UASM_i_LA(&p, GPR_T0, (long)mips_cps_pm_save); 383 uasm_i_jalr(&p, GPR_V0, GPR_T0); 384 uasm_i_nop(&p); 385 } 386 387 /* 388 * Load addresses of required CM & CPC registers. This is done early 389 * because they're needed in both the enable & disable coherence steps 390 * but in the coupled case the enable step will only run on one VPE. 391 */ 392 UASM_i_LA(&p, r_pcohctl, (long)addr_gcr_cl_coherence()); 393 394 if (coupled_coherence) { 395 /* Increment ready_count */ 396 uasm_i_sync(&p, __SYNC_mb); 397 uasm_build_label(&l, p, lbl_incready); 398 uasm_i_ll(&p, GPR_T1, 0, r_nc_count); 399 uasm_i_addiu(&p, GPR_T2, GPR_T1, 1); 400 uasm_i_sc(&p, GPR_T2, 0, r_nc_count); 401 uasm_il_beqz(&p, &r, GPR_T2, lbl_incready); 402 uasm_i_addiu(&p, GPR_T1, GPR_T1, 1); 403 404 /* Barrier ensuring all CPUs see the updated r_nc_count value */ 405 uasm_i_sync(&p, __SYNC_mb); 406 407 /* 408 * If this is the last VPE to become ready for non-coherence 409 * then it should branch below. 410 */ 411 uasm_il_beq(&p, &r, GPR_T1, r_online, lbl_disable_coherence); 412 uasm_i_nop(&p); 413 414 if (state < CPS_PM_POWER_GATED) { 415 /* 416 * Otherwise this is not the last VPE to become ready 417 * for non-coherence. It needs to wait until coherence 418 * has been disabled before proceeding, which it will do 419 * by polling for the top bit of ready_count being set. 420 */ 421 uasm_i_addiu(&p, GPR_T1, GPR_ZERO, -1); 422 uasm_build_label(&l, p, lbl_poll_cont); 423 uasm_i_lw(&p, GPR_T0, 0, r_nc_count); 424 uasm_il_bltz(&p, &r, GPR_T0, lbl_secondary_cont); 425 uasm_i_ehb(&p); 426 if (cpu_has_mipsmt) 427 uasm_i_yield(&p, GPR_ZERO, GPR_T1); 428 uasm_il_b(&p, &r, lbl_poll_cont); 429 uasm_i_nop(&p); 430 } else { 431 /* 432 * The core will lose power & this VPE will not continue 433 * so it can simply halt here. 434 */ 435 if (cpu_has_mipsmt) { 436 /* Halt the VPE via C0 tchalt register */ 437 uasm_i_addiu(&p, GPR_T0, GPR_ZERO, TCHALT_H); 438 uasm_i_mtc0(&p, GPR_T0, 2, 4); 439 } else if (cpu_has_vp) { 440 /* Halt the VP via the CPC VP_STOP register */ 441 unsigned int vpe_id; 442 443 vpe_id = cpu_vpe_id(&cpu_data[cpu]); 444 uasm_i_addiu(&p, GPR_T0, GPR_ZERO, 1 << vpe_id); 445 UASM_i_LA(&p, GPR_T1, (long)addr_cpc_cl_vp_stop()); 446 uasm_i_sw(&p, GPR_T0, 0, GPR_T1); 447 } else { 448 BUG(); 449 } 450 uasm_build_label(&l, p, lbl_secondary_hang); 451 uasm_il_b(&p, &r, lbl_secondary_hang); 452 uasm_i_nop(&p); 453 } 454 } 455 456 /* 457 * This is the point of no return - this VPE will now proceed to 458 * disable coherence. At this point we *must* be sure that no other 459 * VPE within the core will interfere with the L1 dcache. 460 */ 461 uasm_build_label(&l, p, lbl_disable_coherence); 462 463 /* Invalidate the L1 icache */ 464 cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].icache, 465 Index_Invalidate_I, lbl_invicache); 466 467 /* Writeback & invalidate the L1 dcache */ 468 cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].dcache, 469 Index_Writeback_Inv_D, lbl_flushdcache); 470 471 /* Barrier ensuring previous cache invalidates are complete */ 472 uasm_i_sync(&p, __SYNC_full); 473 uasm_i_ehb(&p); 474 475 if (mips_cm_revision() < CM_REV_CM3) { 476 /* 477 * Disable all but self interventions. The load from COHCTL is 478 * defined by the interAptiv & proAptiv SUMs as ensuring that the 479 * operation resulting from the preceding store is complete. 480 */ 481 uasm_i_addiu(&p, GPR_T0, GPR_ZERO, 1 << cpu_core(&cpu_data[cpu])); 482 uasm_i_sw(&p, GPR_T0, 0, r_pcohctl); 483 uasm_i_lw(&p, GPR_T0, 0, r_pcohctl); 484 485 /* Barrier to ensure write to coherence control is complete */ 486 uasm_i_sync(&p, __SYNC_full); 487 uasm_i_ehb(&p); 488 } 489 490 /* Disable coherence */ 491 uasm_i_sw(&p, GPR_ZERO, 0, r_pcohctl); 492 uasm_i_lw(&p, GPR_T0, 0, r_pcohctl); 493 494 if (state >= CPS_PM_CLOCK_GATED) { 495 err = cps_gen_flush_fsb(&p, &l, &r, &cpu_data[cpu], 496 lbl_flush_fsb); 497 if (err) 498 goto out_err; 499 500 /* Determine the CPC command to issue */ 501 switch (state) { 502 case CPS_PM_CLOCK_GATED: 503 cpc_cmd = CPC_Cx_CMD_CLOCKOFF; 504 break; 505 case CPS_PM_POWER_GATED: 506 cpc_cmd = CPC_Cx_CMD_PWRDOWN; 507 break; 508 default: 509 BUG(); 510 goto out_err; 511 } 512 513 /* Issue the CPC command */ 514 UASM_i_LA(&p, GPR_T0, (long)addr_cpc_cl_cmd()); 515 uasm_i_addiu(&p, GPR_T1, GPR_ZERO, cpc_cmd); 516 uasm_i_sw(&p, GPR_T1, 0, GPR_T0); 517 518 if (state == CPS_PM_POWER_GATED) { 519 /* If anything goes wrong just hang */ 520 uasm_build_label(&l, p, lbl_hang); 521 uasm_il_b(&p, &r, lbl_hang); 522 uasm_i_nop(&p); 523 524 /* 525 * There's no point generating more code, the core is 526 * powered down & if powered back up will run from the 527 * reset vector not from here. 528 */ 529 goto gen_done; 530 } 531 532 /* Barrier to ensure write to CPC command is complete */ 533 uasm_i_sync(&p, __SYNC_full); 534 uasm_i_ehb(&p); 535 } 536 537 if (state == CPS_PM_NC_WAIT) { 538 /* 539 * At this point it is safe for all VPEs to proceed with 540 * execution. This VPE will set the top bit of ready_count 541 * to indicate to the other VPEs that they may continue. 542 */ 543 if (coupled_coherence) 544 cps_gen_set_top_bit(&p, &l, &r, r_nc_count, 545 lbl_set_cont); 546 547 /* 548 * VPEs which did not disable coherence will continue 549 * executing, after coherence has been disabled, from this 550 * point. 551 */ 552 uasm_build_label(&l, p, lbl_secondary_cont); 553 554 /* Now perform our wait */ 555 uasm_i_wait(&p, 0); 556 } 557 558 /* 559 * Re-enable coherence. Note that for CPS_PM_NC_WAIT all coupled VPEs 560 * will run this. The first will actually re-enable coherence & the 561 * rest will just be performing a rather unusual nop. 562 */ 563 uasm_i_addiu(&p, GPR_T0, GPR_ZERO, mips_cm_revision() < CM_REV_CM3 564 ? CM_GCR_Cx_COHERENCE_COHDOMAINEN 565 : CM3_GCR_Cx_COHERENCE_COHEN); 566 567 uasm_i_sw(&p, GPR_T0, 0, r_pcohctl); 568 uasm_i_lw(&p, GPR_T0, 0, r_pcohctl); 569 570 /* Barrier to ensure write to coherence control is complete */ 571 uasm_i_sync(&p, __SYNC_full); 572 uasm_i_ehb(&p); 573 574 if (coupled_coherence && (state == CPS_PM_NC_WAIT)) { 575 /* Decrement ready_count */ 576 uasm_build_label(&l, p, lbl_decready); 577 uasm_i_sync(&p, __SYNC_mb); 578 uasm_i_ll(&p, GPR_T1, 0, r_nc_count); 579 uasm_i_addiu(&p, GPR_T2, GPR_T1, -1); 580 uasm_i_sc(&p, GPR_T2, 0, r_nc_count); 581 uasm_il_beqz(&p, &r, GPR_T2, lbl_decready); 582 uasm_i_andi(&p, GPR_V0, GPR_T1, (1 << fls(smp_num_siblings)) - 1); 583 584 /* Barrier ensuring all CPUs see the updated r_nc_count value */ 585 uasm_i_sync(&p, __SYNC_mb); 586 } 587 588 if (coupled_coherence && (state == CPS_PM_CLOCK_GATED)) { 589 /* 590 * At this point it is safe for all VPEs to proceed with 591 * execution. This VPE will set the top bit of ready_count 592 * to indicate to the other VPEs that they may continue. 593 */ 594 cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont); 595 596 /* 597 * This core will be reliant upon another core sending a 598 * power-up command to the CPC in order to resume operation. 599 * Thus an arbitrary VPE can't trigger the core leaving the 600 * idle state and the one that disables coherence might as well 601 * be the one to re-enable it. The rest will continue from here 602 * after that has been done. 603 */ 604 uasm_build_label(&l, p, lbl_secondary_cont); 605 606 /* Barrier ensuring all CPUs see the updated r_nc_count value */ 607 uasm_i_sync(&p, __SYNC_mb); 608 } 609 610 /* The core is coherent, time to return to C code */ 611 uasm_i_jr(&p, GPR_RA); 612 uasm_i_nop(&p); 613 614 gen_done: 615 /* Ensure the code didn't exceed the resources allocated for it */ 616 BUG_ON((p - buf) > max_instrs); 617 BUG_ON((l - labels) > ARRAY_SIZE(labels)); 618 BUG_ON((r - relocs) > ARRAY_SIZE(relocs)); 619 620 /* Patch branch offsets */ 621 uasm_resolve_relocs(relocs, labels); 622 623 /* Flush the icache */ 624 local_flush_icache_range((unsigned long)buf, (unsigned long)p); 625 626 return buf; 627 out_err: 628 kfree(buf); 629 return NULL; 630 } 631 632 static int cps_pm_online_cpu(unsigned int cpu) 633 { 634 unsigned int sibling, core; 635 void *entry_fn, *core_rc; 636 enum cps_pm_state state; 637 638 core = cpu_core(&cpu_data[cpu]); 639 640 for (state = CPS_PM_NC_WAIT; state < CPS_PM_STATE_COUNT; state++) { 641 if (per_cpu(nc_asm_enter, cpu)[state]) 642 continue; 643 if (!test_bit(state, state_support)) 644 continue; 645 646 entry_fn = cps_gen_entry_code(cpu, state); 647 if (!entry_fn) { 648 pr_err("Failed to generate core %u state %u entry\n", 649 core, state); 650 clear_bit(state, state_support); 651 } 652 653 for_each_cpu(sibling, &cpu_sibling_map[cpu]) 654 per_cpu(nc_asm_enter, sibling)[state] = entry_fn; 655 } 656 657 if (!per_cpu(ready_count, cpu)) { 658 core_rc = kmalloc(sizeof(u32), GFP_KERNEL); 659 if (!core_rc) { 660 pr_err("Failed allocate core %u ready_count\n", core); 661 return -ENOMEM; 662 } 663 664 for_each_cpu(sibling, &cpu_sibling_map[cpu]) 665 per_cpu(ready_count, sibling) = core_rc; 666 } 667 668 return 0; 669 } 670 671 static int cps_pm_power_notifier(struct notifier_block *this, 672 unsigned long event, void *ptr) 673 { 674 unsigned int stat; 675 676 switch (event) { 677 case PM_SUSPEND_PREPARE: 678 stat = read_cpc_cl_stat_conf(); 679 /* 680 * If we're attempting to suspend the system and power down all 681 * of the cores, the JTAG detect bit indicates that the CPC will 682 * instead put the cores into clock-off state. In this state 683 * a connected debugger can cause the CPU to attempt 684 * interactions with the powered down system. At best this will 685 * fail. At worst, it can hang the NoC, requiring a hard reset. 686 * To avoid this, just block system suspend if a JTAG probe 687 * is detected. 688 */ 689 if (stat & CPC_Cx_STAT_CONF_EJTAG_PROBE) { 690 pr_warn("JTAG probe is connected - abort suspend\n"); 691 return NOTIFY_BAD; 692 } 693 return NOTIFY_DONE; 694 default: 695 return NOTIFY_DONE; 696 } 697 } 698 699 static int __init cps_pm_init(void) 700 { 701 /* A CM is required for all non-coherent states */ 702 if (!mips_cm_present()) { 703 pr_warn("pm-cps: no CM, non-coherent states unavailable\n"); 704 return 0; 705 } 706 707 /* 708 * If interrupts were enabled whilst running a wait instruction on a 709 * non-coherent core then the VPE may end up processing interrupts 710 * whilst non-coherent. That would be bad. 711 */ 712 if (cpu_wait == r4k_wait_irqoff) 713 set_bit(CPS_PM_NC_WAIT, state_support); 714 else 715 pr_warn("pm-cps: non-coherent wait unavailable\n"); 716 717 /* Detect whether a CPC is present */ 718 if (mips_cpc_present()) { 719 /* Detect whether clock gating is implemented */ 720 if (read_cpc_cl_stat_conf() & CPC_Cx_STAT_CONF_CLKGAT_IMPL) 721 set_bit(CPS_PM_CLOCK_GATED, state_support); 722 else 723 pr_warn("pm-cps: CPC does not support clock gating\n"); 724 725 /* Power gating is available with CPS SMP & any CPC */ 726 if (mips_cps_smp_in_use()) 727 set_bit(CPS_PM_POWER_GATED, state_support); 728 else 729 pr_warn("pm-cps: CPS SMP not in use, power gating unavailable\n"); 730 } else { 731 pr_warn("pm-cps: no CPC, clock & power gating unavailable\n"); 732 } 733 734 pm_notifier(cps_pm_power_notifier, 0); 735 736 return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mips/cps_pm:online", 737 cps_pm_online_cpu, NULL); 738 } 739 arch_initcall(cps_pm_init); 740