1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/machsystm.h> 27 #include <sys/archsystm.h> 28 #include <sys/vm.h> 29 #include <sys/cpu.h> 30 #include <sys/cpupart.h> 31 #include <sys/cmt.h> 32 #include <sys/bitset.h> 33 #include <sys/reboot.h> 34 #include <sys/kdi.h> 35 #include <sys/bootconf.h> 36 #include <sys/memlist_plat.h> 37 #include <sys/memlist_impl.h> 38 #include <sys/prom_plat.h> 39 #include <sys/prom_isa.h> 40 #include <sys/autoconf.h> 41 #include <sys/intreg.h> 42 #include <sys/ivintr.h> 43 #include <sys/fpu/fpusystm.h> 44 #include <sys/iommutsb.h> 45 #include <vm/vm_dep.h> 46 #include <vm/seg_kmem.h> 47 #include <vm/seg_kpm.h> 48 #include <vm/seg_map.h> 49 #include <vm/seg_kp.h> 50 #include <sys/sysconf.h> 51 #include <vm/hat_sfmmu.h> 52 #include <sys/kobj.h> 53 #include <sys/sun4asi.h> 54 #include <sys/clconf.h> 55 #include <sys/platform_module.h> 56 #include <sys/panic.h> 57 #include <sys/cpu_sgnblk_defs.h> 58 #include <sys/clock.h> 59 #include <sys/fpras_impl.h> 60 #include <sys/prom_debug.h> 61 #include <sys/traptrace.h> 62 #include <sys/memnode.h> 63 #include <sys/mem_cage.h> 64 65 /* 66 * fpRAS implementation structures. 67 */ 68 struct fpras_chkfn *fpras_chkfnaddrs[FPRAS_NCOPYOPS]; 69 struct fpras_chkfngrp *fpras_chkfngrps; 70 struct fpras_chkfngrp *fpras_chkfngrps_base; 71 int fpras_frequency = -1; 72 int64_t fpras_interval = -1; 73 74 /* 75 * Halt idling cpus optimization 76 * 77 * This optimation is only enabled in platforms that have 78 * the CPU halt support. The cpu_halt_cpu() support is provided 79 * in the cpu module and it is referenced here with a pragma weak. 80 * The presence of this routine automatically enable the halt idling 81 * cpus functionality if the global switch enable_halt_idle_cpus 82 * is set (default is set). 83 * 84 */ 85 #pragma weak cpu_halt_cpu 86 extern void cpu_halt_cpu(); 87 88 /* 89 * Defines for the idle_state_transition DTrace probe 90 * 91 * The probe fires when the CPU undergoes an idle state change (e.g. halting) 92 * The agument passed is the state to which the CPU is transitioning. 93 * 94 * The states are defined here. 95 */ 96 #define IDLE_STATE_NORMAL 0 97 #define IDLE_STATE_HALTED 1 98 99 int enable_halt_idle_cpus = 1; /* global switch */ 100 101 void 102 setup_trap_table(void) 103 { 104 intr_init(CPU); /* init interrupt request free list */ 105 setwstate(WSTATE_KERN); 106 prom_set_traptable(&trap_table); 107 } 108 109 void 110 mach_fpras() 111 { 112 if (fpras_implemented && !fpras_disable) { 113 int i; 114 struct fpras_chkfngrp *fcgp; 115 size_t chkfngrpsallocsz; 116 117 /* 118 * Note that we size off of NCPU and setup for 119 * all those possibilities regardless of whether 120 * the cpu id is present or not. We do this so that 121 * we don't have any construction or destruction 122 * activity to perform at DR time, and it's not 123 * costly in memory. We require block alignment. 124 */ 125 chkfngrpsallocsz = NCPU * sizeof (struct fpras_chkfngrp); 126 fpras_chkfngrps_base = kmem_alloc(chkfngrpsallocsz, KM_SLEEP); 127 if (IS_P2ALIGNED((uintptr_t)fpras_chkfngrps_base, 64)) { 128 fpras_chkfngrps = fpras_chkfngrps_base; 129 } else { 130 kmem_free(fpras_chkfngrps_base, chkfngrpsallocsz); 131 chkfngrpsallocsz += 64; 132 fpras_chkfngrps_base = kmem_alloc(chkfngrpsallocsz, 133 KM_SLEEP); 134 fpras_chkfngrps = (struct fpras_chkfngrp *) 135 P2ROUNDUP((uintptr_t)fpras_chkfngrps_base, 64); 136 } 137 138 /* 139 * Copy our check function into place for each copy operation 140 * and each cpu id. 141 */ 142 fcgp = &fpras_chkfngrps[0]; 143 for (i = 0; i < FPRAS_NCOPYOPS; ++i) 144 bcopy((void *)fpras_chkfn_type1, &fcgp->fpras_fn[i], 145 sizeof (struct fpras_chkfn)); 146 for (i = 1; i < NCPU; ++i) 147 *(&fpras_chkfngrps[i]) = *fcgp; 148 149 /* 150 * At definition fpras_frequency is set to -1, and it will 151 * still have that value unless changed in /etc/system (not 152 * strictly supported, but not preventable). The following 153 * both sets the default and sanity checks anything from 154 * /etc/system. 155 */ 156 if (fpras_frequency < 0) 157 fpras_frequency = FPRAS_DEFAULT_FREQUENCY; 158 159 /* 160 * Now calculate fpras_interval. When fpras_interval 161 * becomes non-negative fpras checks will commence 162 * (copies before this point in boot will bypass fpras). 163 * Our stores of instructions must be visible; no need 164 * to flush as they're never been executed before. 165 */ 166 membar_producer(); 167 fpras_interval = (fpras_frequency == 0) ? 168 0 : sys_tick_freq / fpras_frequency; 169 } 170 } 171 172 void 173 mach_hw_copy_limit(void) 174 { 175 if (!fpu_exists) { 176 use_hw_bcopy = 0; 177 hw_copy_limit_1 = 0; 178 hw_copy_limit_2 = 0; 179 hw_copy_limit_4 = 0; 180 hw_copy_limit_8 = 0; 181 use_hw_bzero = 0; 182 } 183 } 184 185 void 186 load_tod_module() 187 { 188 /* 189 * Load tod driver module for the tod part found on this system. 190 * Recompute the cpu frequency/delays based on tod as tod part 191 * tends to keep time more accurately. 192 */ 193 if (tod_module_name == NULL || modload("tod", tod_module_name) == -1) 194 halt("Can't load tod module"); 195 } 196 197 void 198 mach_memscrub(void) 199 { 200 /* 201 * Startup memory scrubber, if not running fpu emulation code. 202 */ 203 204 #ifndef _HW_MEMSCRUB_SUPPORT 205 if (fpu_exists) { 206 if (memscrub_init()) { 207 cmn_err(CE_WARN, 208 "Memory scrubber failed to initialize"); 209 } 210 } 211 #endif /* _HW_MEMSCRUB_SUPPORT */ 212 } 213 214 /* 215 * Halt the present CPU until awoken via an interrupt. 216 * This routine should only be invoked if cpu_halt_cpu() 217 * exists and is supported, see mach_cpu_halt_idle() 218 */ 219 void 220 cpu_halt(void) 221 { 222 cpu_t *cpup = CPU; 223 processorid_t cpu_sid = cpup->cpu_seqid; 224 cpupart_t *cp = cpup->cpu_part; 225 int hset_update = 1; 226 volatile int *p = &cpup->cpu_disp->disp_nrunnable; 227 uint_t s; 228 229 /* 230 * If this CPU is online then we should notate our halting 231 * by adding ourselves to the partition's halted CPU 232 * bitset. This allows other CPUs to find/awaken us when 233 * work becomes available. 234 */ 235 if (CPU->cpu_flags & CPU_OFFLINE) 236 hset_update = 0; 237 238 /* 239 * Add ourselves to the partition's halted CPUs bitset 240 * and set our HALTED flag, if necessary. 241 * 242 * When a thread becomes runnable, it is placed on the queue 243 * and then the halted cpu bitset is checked to determine who 244 * (if anyone) should be awoken. We therefore need to first 245 * add ourselves to the halted bitset, and then check if there 246 * is any work available. The order is important to prevent a race 247 * that can lead to work languishing on a run queue somewhere while 248 * this CPU remains halted. 249 * 250 * Either the producing CPU will see we're halted and will awaken us, 251 * or this CPU will see the work available in disp_anywork() 252 */ 253 if (hset_update) { 254 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 255 membar_producer(); 256 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 257 } 258 259 /* 260 * Check to make sure there's really nothing to do. 261 * Work destined for this CPU may become available after 262 * this check. We'll be notified through the clearing of our 263 * bit in the halted CPU bitset, and a poke. 264 */ 265 if (disp_anywork()) { 266 if (hset_update) { 267 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 268 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 269 } 270 return; 271 } 272 273 /* 274 * We're on our way to being halted. Wait until something becomes 275 * runnable locally or we are awaken (i.e. removed from the halt set). 276 * Note that the call to hv_cpu_yield() can return even if we have 277 * nothing to do. 278 * 279 * Disable interrupts now, so that we'll awaken immediately 280 * after halting if someone tries to poke us between now and 281 * the time we actually halt. 282 * 283 * We check for the presence of our bit after disabling interrupts. 284 * If it's cleared, we'll return. If the bit is cleared after 285 * we check then the poke will pop us out of the halted state. 286 * Also, if the offlined CPU has been brought back on-line, then 287 * we return as well. 288 * 289 * The ordering of the poke and the clearing of the bit by cpu_wakeup 290 * is important. 291 * cpu_wakeup() must clear, then poke. 292 * cpu_halt() must disable interrupts, then check for the bit. 293 * 294 * The check for anything locally runnable is here for performance 295 * and isn't needed for correctness. disp_nrunnable ought to be 296 * in our cache still, so it's inexpensive to check, and if there 297 * is anything runnable we won't have to wait for the poke. 298 * 299 * Any interrupt will awaken the cpu from halt. Looping here 300 * will filter spurious interrupts that wake us up, but don't 301 * represent a need for us to head back out to idle(). This 302 * will enable the idle loop to be more efficient and sleep in 303 * the processor pipeline for a larger percent of the time, 304 * which returns useful cycles to the peer hardware strand 305 * that shares the pipeline. 306 */ 307 s = disable_vec_intr(); 308 while (*p == 0 && 309 ((hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid)) || 310 (!hset_update && (CPU->cpu_flags & CPU_OFFLINE)))) { 311 312 DTRACE_PROBE1(idle__state__transition, 313 uint_t, IDLE_STATE_HALTED); 314 (void) cpu_halt_cpu(); 315 DTRACE_PROBE1(idle__state__transition, 316 uint_t, IDLE_STATE_NORMAL); 317 318 enable_vec_intr(s); 319 s = disable_vec_intr(); 320 } 321 322 /* 323 * We're no longer halted 324 */ 325 enable_vec_intr(s); 326 if (hset_update) { 327 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 328 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 329 } 330 } 331 332 /* 333 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 334 * Otherwise, see if other CPUs in the cpu partition are halted and need to 335 * be woken up so that they can steal the thread we placed on this CPU. 336 * This function is only used on MP systems. 337 * This function should only be invoked if cpu_halt_cpu() 338 * exists and is supported, see mach_cpu_halt_idle() 339 */ 340 static void 341 cpu_wakeup(cpu_t *cpu, int bound) 342 { 343 uint_t cpu_found; 344 processorid_t cpu_sid; 345 cpupart_t *cp; 346 347 cp = cpu->cpu_part; 348 cpu_sid = cpu->cpu_seqid; 349 if (bitset_in_set(&cp->cp_haltset, cpu_sid)) { 350 /* 351 * Clear the halted bit for that CPU since it will be 352 * poked in a moment. 353 */ 354 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 355 /* 356 * We may find the current CPU present in the halted cpu bitset 357 * if we're in the context of an interrupt that occurred 358 * before we had a chance to clear our bit in cpu_halt(). 359 * Poking ourself is obviously unnecessary, since if 360 * we're here, we're not halted. 361 */ 362 if (cpu != CPU) 363 poke_cpu(cpu->cpu_id); 364 return; 365 } else { 366 /* 367 * This cpu isn't halted, but it's idle or undergoing a 368 * context switch. No need to awaken anyone else. 369 */ 370 if (cpu->cpu_thread == cpu->cpu_idle_thread || 371 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 372 return; 373 } 374 375 /* 376 * No need to wake up other CPUs if this is for a bound thread. 377 */ 378 if (bound) 379 return; 380 381 /* 382 * The CPU specified for wakeup isn't currently halted, so check 383 * to see if there are any other halted CPUs in the partition, 384 * and if there are then awaken one. 385 * 386 * If possible, try to select a CPU close to the target, since this 387 * will likely trigger a migration. 388 */ 389 do { 390 cpu_found = bitset_find(&cp->cp_haltset); 391 if (cpu_found == (uint_t)-1) 392 return; 393 } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0); 394 395 if (cpu_found != CPU->cpu_seqid) 396 poke_cpu(cpu_seq[cpu_found]->cpu_id); 397 } 398 399 void 400 mach_cpu_halt_idle(void) 401 { 402 if (enable_halt_idle_cpus) { 403 if (&cpu_halt_cpu) { 404 idle_cpu = cpu_halt; 405 disp_enq_thread = cpu_wakeup; 406 } 407 } 408 } 409 410 /*ARGSUSED*/ 411 int 412 cpu_intrq_setup(struct cpu *cp) 413 { 414 /* Interrupt mondo queues not applicable to sun4u */ 415 return (0); 416 } 417 418 /*ARGSUSED*/ 419 void 420 cpu_intrq_cleanup(struct cpu *cp) 421 { 422 /* Interrupt mondo queues not applicable to sun4u */ 423 } 424 425 /*ARGSUSED*/ 426 void 427 cpu_intrq_register(struct cpu *cp) 428 { 429 /* Interrupt/error queues not applicable to sun4u */ 430 } 431 432 /*ARGSUSED*/ 433 void 434 mach_htraptrace_setup(int cpuid) 435 { 436 /* Setup hypervisor traptrace buffer, not applicable to sun4u */ 437 } 438 439 /*ARGSUSED*/ 440 void 441 mach_htraptrace_configure(int cpuid) 442 { 443 /* enable/ disable hypervisor traptracing, not applicable to sun4u */ 444 } 445 446 /*ARGSUSED*/ 447 void 448 mach_htraptrace_cleanup(int cpuid) 449 { 450 /* cleanup hypervisor traptrace buffer, not applicable to sun4u */ 451 } 452 453 void 454 mach_descrip_startup_init(void) 455 { 456 /* 457 * Only for sun4v. 458 * Initialize Machine description framework during startup. 459 */ 460 } 461 void 462 mach_descrip_startup_fini(void) 463 { 464 /* 465 * Only for sun4v. 466 * Clean up Machine Description framework during startup. 467 */ 468 } 469 470 void 471 mach_descrip_init(void) 472 { 473 /* 474 * Only for sun4v. 475 * Initialize Machine description framework. 476 */ 477 } 478 479 void 480 hsvc_setup(void) 481 { 482 /* Setup hypervisor services, not applicable to sun4u */ 483 } 484 485 void 486 load_mach_drivers(void) 487 { 488 /* Currently no machine class (sun4u) specific drivers to load */ 489 } 490 491 /* 492 * Return true if the machine we're running on is a Positron. 493 * (Positron is an unsupported developers platform.) 494 */ 495 int 496 iam_positron(void) 497 { 498 char model[32]; 499 const char proto_model[] = "SUNW,501-2732"; 500 pnode_t root = prom_rootnode(); 501 502 if (prom_getproplen(root, "model") != sizeof (proto_model)) 503 return (0); 504 505 (void) prom_getprop(root, "model", model); 506 if (strcmp(model, proto_model) == 0) 507 return (1); 508 return (0); 509 } 510 511 /* 512 * Find a physically contiguous area of twice the largest ecache size 513 * to be used while doing displacement flush of ecaches. 514 */ 515 uint64_t 516 ecache_flush_address(void) 517 { 518 struct memlist *pmem; 519 uint64_t flush_size; 520 uint64_t ret_val; 521 522 flush_size = ecache_size * 2; 523 for (pmem = phys_install; pmem; pmem = pmem->next) { 524 ret_val = P2ROUNDUP(pmem->address, ecache_size); 525 if (ret_val + flush_size <= pmem->address + pmem->size) 526 return (ret_val); 527 } 528 return ((uint64_t)-1); 529 } 530 531 /* 532 * Called with the memlist lock held to say that phys_install has 533 * changed. 534 */ 535 void 536 phys_install_has_changed(void) 537 { 538 /* 539 * Get the new address into a temporary just in case panicking 540 * involves use of ecache_flushaddr. 541 */ 542 uint64_t new_addr; 543 544 new_addr = ecache_flush_address(); 545 if (new_addr == (uint64_t)-1) { 546 cmn_err(CE_PANIC, 547 "ecache_flush_address(): failed, ecache_size=%x", 548 ecache_size); 549 /*NOTREACHED*/ 550 } 551 ecache_flushaddr = new_addr; 552 membar_producer(); 553 } 554