1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/systm.h> 31 #include <sys/archsystm.h> 32 #include <sys/machparam.h> 33 #include <sys/machsystm.h> 34 #include <sys/cpu.h> 35 #include <sys/elf_SPARC.h> 36 #include <vm/hat_sfmmu.h> 37 #include <vm/page.h> 38 #include <vm/vm_dep.h> 39 #include <sys/cpuvar.h> 40 #include <sys/async.h> 41 #include <sys/cmn_err.h> 42 #include <sys/debug.h> 43 #include <sys/dditypes.h> 44 #include <sys/sunddi.h> 45 #include <sys/cpu_module.h> 46 #include <sys/prom_debug.h> 47 #include <sys/vmsystm.h> 48 #include <sys/prom_plat.h> 49 #include <sys/sysmacros.h> 50 #include <sys/intreg.h> 51 #include <sys/machtrap.h> 52 #include <sys/ontrap.h> 53 #include <sys/ivintr.h> 54 #include <sys/atomic.h> 55 #include <sys/panic.h> 56 #include <sys/dtrace.h> 57 #include <sys/simulate.h> 58 #include <sys/fault.h> 59 #include <sys/niagara2regs.h> 60 #include <sys/hsvc.h> 61 #include <sys/trapstat.h> 62 63 uint_t root_phys_addr_lo_mask = 0xffffffffU; 64 #if defined(NIAGARA2_IMPL) 65 char cpu_module_name[] = "SUNW,UltraSPARC-T2"; 66 #elif defined(VFALLS_IMPL) 67 char cpu_module_name[] = "SUNW,UltraSPARC-T2+"; 68 #endif 69 70 /* 71 * Hypervisor services information for the NIAGARA2 and Victoria Falls 72 * CPU module 73 */ 74 static boolean_t cpu_hsvc_available = B_TRUE; 75 static uint64_t cpu_sup_minor; /* Supported minor number */ 76 #if defined(NIAGARA2_IMPL) 77 static hsvc_info_t cpu_hsvc = { 78 HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, 79 NIAGARA2_HSVC_MINOR, cpu_module_name 80 }; 81 #elif defined(VFALLS_IMPL) 82 static hsvc_info_t cpu_hsvc = { 83 HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR, 84 VFALLS_HSVC_MINOR, cpu_module_name 85 }; 86 #endif 87 88 void 89 cpu_setup(void) 90 { 91 extern int mmu_exported_pagesize_mask; 92 extern int cpc_has_overflow_intr; 93 extern size_t contig_mem_prealloc_base; 94 int status; 95 96 /* 97 * Negotiate the API version for Niagara2 specific hypervisor 98 * services. 99 */ 100 status = hsvc_register(&cpu_hsvc, &cpu_sup_minor); 101 if (status != 0) { 102 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 103 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", 104 cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group, 105 cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status); 106 cpu_hsvc_available = B_FALSE; 107 } 108 109 /* 110 * The setup common to all CPU modules is done in cpu_setup_common 111 * routine. 112 */ 113 cpu_setup_common(NULL); 114 115 cache |= (CACHE_PTAG | CACHE_IOCOHERENT); 116 117 if ((mmu_exported_pagesize_mask & 118 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != 119 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) 120 cmn_err(CE_PANIC, "machine description" 121 " does not have required sun4v page sizes" 122 " 8K, 64K and 4M: MD mask is 0x%x", 123 mmu_exported_pagesize_mask); 124 125 cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT; 126 127 /* 128 * Niagara2 supports a 48-bit subset of the full 64-bit virtual 129 * address space. Virtual addresses between 0x0000800000000000 130 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" 131 * and must never be mapped. In addition, software must not use 132 * pages within 4GB of the VA hole as instruction pages to 133 * avoid problems with prefetching into the VA hole. 134 */ 135 hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); 136 hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); 137 138 /* 139 * Niagara2 has a performance counter overflow interrupt 140 */ 141 cpc_has_overflow_intr = 1; 142 143 /* 144 * Enable 4M pages for OOB. 145 */ 146 max_uheap_lpsize = MMU_PAGESIZE4M; 147 max_ustack_lpsize = MMU_PAGESIZE4M; 148 max_privmap_lpsize = MMU_PAGESIZE4M; 149 150 contig_mem_prealloc_base = NIAGARA2_PREALLOC_BASE; 151 } 152 153 /* 154 * Set the magic constants of the implementation. 155 */ 156 void 157 cpu_fiximp(struct cpu_node *cpunode) 158 { 159 /* 160 * The Cache node is optional in MD. Therefore in case "Cache" 161 * node does not exists in MD, set the default L2 cache associativity, 162 * size, linesize. 163 */ 164 if (cpunode->ecache_size == 0) 165 cpunode->ecache_size = L2CACHE_SIZE; 166 if (cpunode->ecache_linesize == 0) 167 cpunode->ecache_linesize = L2CACHE_LINESIZE; 168 if (cpunode->ecache_associativity == 0) 169 cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; 170 } 171 172 void 173 cpu_map_exec_units(struct cpu *cp) 174 { 175 ASSERT(MUTEX_HELD(&cpu_lock)); 176 177 /* 178 * The cpu_ipipe and cpu_fpu fields are initialized based on 179 * the execution unit sharing information from the MD. They 180 * default to the CPU id in the absence of such information. 181 */ 182 cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; 183 if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) 184 cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); 185 186 cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping; 187 if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND) 188 cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id); 189 190 /* 191 * Niagara 2 defines the core to be at the FPU level 192 */ 193 cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu; 194 195 /* 196 * The cpu_chip field is initialized based on the information 197 * in the MD and assume that all cpus within a chip 198 * share the same L2 cache. If no such info is available, we 199 * set the cpu to belong to the defacto chip 0. 200 */ 201 cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping; 202 if (cp->cpu_m.cpu_chip == NO_CHIP_MAPPING_FOUND) 203 cp->cpu_m.cpu_chip = 0; 204 } 205 206 static int cpucnt; 207 208 void 209 cpu_init_private(struct cpu *cp) 210 { 211 extern void niagara_kstat_init(void); 212 213 ASSERT(MUTEX_HELD(&cpu_lock)); 214 215 cpu_map_exec_units(cp); 216 217 if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE)) 218 (void) niagara_kstat_init(); 219 } 220 221 /*ARGSUSED*/ 222 void 223 cpu_uninit_private(struct cpu *cp) 224 { 225 extern void niagara_kstat_fini(void); 226 227 ASSERT(MUTEX_HELD(&cpu_lock)); 228 if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE)) 229 (void) niagara_kstat_fini(); 230 } 231 232 /* 233 * On Niagara2, any flush will cause all preceding stores to be 234 * synchronized wrt the i$, regardless of address or ASI. In fact, 235 * the address is ignored, so we always flush address 0. 236 */ 237 /*ARGSUSED*/ 238 void 239 dtrace_flush_sec(uintptr_t addr) 240 { 241 doflush(0); 242 } 243 244 /* 245 * Trapstat support for Niagara2 processor 246 * The Niagara2 provides HWTW support for TSB lookup and with HWTW 247 * enabled no TSB hit information will be available. Therefore setting 248 * the time spent in TLB miss handler for TSB hits to 0. 249 */ 250 int 251 cpu_trapstat_conf(int cmd) 252 { 253 int status = 0; 254 255 switch (cmd) { 256 case CPU_TSTATCONF_INIT: 257 case CPU_TSTATCONF_FINI: 258 case CPU_TSTATCONF_ENABLE: 259 case CPU_TSTATCONF_DISABLE: 260 break; 261 default: 262 status = EINVAL; 263 break; 264 } 265 return (status); 266 } 267 268 void 269 cpu_trapstat_data(void *buf, uint_t tstat_pgszs) 270 { 271 tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; 272 int i; 273 274 for (i = 0; i < tstat_pgszs; i++, tstatp++) { 275 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; 276 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; 277 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; 278 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; 279 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 280 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 281 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 282 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 283 } 284 } 285 286 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ 287 uint_t 288 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc) 289 { 290 uint_t color; 291 292 ASSERT(szc <= TTE256M); 293 294 pfn = PFN_BASE(pfn, szc); 295 color = ((pfn >> 15) ^ pfn) & 0x1f; 296 if (szc >= TTE4M) 297 return (color); 298 299 color = (color << 2) | ((pfn >> 5) & 0x3); 300 301 return (szc <= TTE64K ? color : (color >> 1)); 302 } 303 304 #if TTE256M != 5 305 #error TTE256M is not 5 306 #endif 307 308 uint_t 309 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) 310 { 311 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; 312 ASSERT(szc < TTE256M); 313 314 mask &= ni2_color_masks[szc]; 315 return ((szc == TTE64K || szc == TTE512K) ? (mask >> 1) : mask); 316 } 317 318 uint_t 319 page_get_nsz_color_cpu(uchar_t szc, uint_t color) 320 { 321 ASSERT(szc < TTE256M); 322 return ((szc == TTE64K || szc == TTE512K) ? (color >> 1) : color); 323 } 324 325 uint_t 326 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) 327 { 328 ASSERT(nszc >= szc); 329 ASSERT(nszc <= TTE256M); 330 331 if (szc == nszc) 332 return (0); 333 if (szc <= TTE64K) 334 return ((nszc >= TTE4M) ? 2 : ((nszc >= TTE512K) ? 1 : 0)); 335 if (szc == TTE512K) 336 return (1); 337 338 return (0); 339 } 340 341 /*ARGSUSED*/ 342 pfn_t 343 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, 344 uint_t ceq_mask, uint_t color_mask) 345 { 346 pfn_t pstep = PNUM_SIZE(szc); 347 pfn_t npfn, pfn_ceq_mask, pfn_color; 348 pfn_t tmpmask, mask = (pfn_t)-1; 349 350 ASSERT((color & ~ceq_mask) == 0); 351 352 if (((page_pfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0) { 353 354 /* we start from the page with correct color */ 355 if (szc >= TTE512K) { 356 if (szc >= TTE4M) { 357 /* page color is PA[32:28] */ 358 pfn_ceq_mask = ceq_mask << 15; 359 } else { 360 /* page color is PA[32:28].PA[19:19] */ 361 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 362 ((ceq_mask >> 1) << 15); 363 } 364 pfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 365 return (pfn); 366 } else { 367 /* 368 * We deal 64K or 8K page. Check if we could the 369 * satisfy the request without changing PA[32:28] 370 */ 371 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 372 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 373 374 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 375 return (npfn); 376 377 /* 378 * for next pfn we have to change bits PA[32:28] 379 * set PA[63:28] and PA[19:18] of the next pfn 380 */ 381 npfn = (pfn >> 15) << 15; 382 npfn |= (ceq_mask & color & 3) << 5; 383 pfn_ceq_mask = (szc == TTE8K) ? 0 : 384 (ceq_mask & 0x1c) << 13; 385 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); 386 387 /* 388 * set bits PA[17:13] to match the color 389 */ 390 ceq_mask >>= 2; 391 color = (color >> 2) & ceq_mask; 392 npfn |= ((npfn >> 15) ^ color) & ceq_mask; 393 return (npfn); 394 } 395 } 396 397 /* 398 * we start from the page with incorrect color - rare case 399 */ 400 if (szc >= TTE512K) { 401 if (szc >= TTE4M) { 402 /* page color is in bits PA[32:28] */ 403 npfn = ((pfn >> 20) << 20) | (color << 15); 404 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; 405 } else { 406 /* try get the right color by changing bit PA[19:19] */ 407 npfn = pfn + pstep; 408 if (((page_pfn_2_color_cpu(npfn, szc) ^ color) & 409 ceq_mask) == 0) 410 return (npfn); 411 412 /* page color is PA[32:28].PA[19:19] */ 413 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 414 ((ceq_mask >> 1) << 15) | (0xff << 7); 415 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); 416 npfn = ((pfn >> 20) << 20) | pfn_color; 417 } 418 419 while (npfn <= pfn) { 420 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 421 } 422 return (npfn); 423 } 424 425 /* 426 * We deal 64K or 8K page of incorrect color. 427 * Try correcting color without changing PA[32:28] 428 */ 429 430 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 431 pfn_color = ((color & 3) << 5) | (color >> 2); 432 npfn = (pfn & ~(pfn_t)0x7f); 433 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 434 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 435 436 if (((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0) { 437 438 /* the color is fixed - find the next page */ 439 while (npfn <= pfn) { 440 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 441 } 442 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 443 return (npfn); 444 } 445 446 /* to fix the color need to touch PA[32:28] */ 447 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : 448 (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); 449 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; 450 451 while (npfn <= pfn) { 452 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); 453 } 454 455 /* set bits PA[19:13] to match the color */ 456 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 457 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 458 459 ASSERT(((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); 460 461 return (npfn); 462 } 463 464 /* 465 * init page coloring 466 */ 467 void 468 page_coloring_init_cpu() 469 { 470 int i; 471 472 hw_page_array[0].hp_colors = 1 << 7; 473 hw_page_array[1].hp_colors = 1 << 7; 474 hw_page_array[2].hp_colors = 1 << 6; 475 476 for (i = 3; i < mmu_page_sizes; i++) { 477 hw_page_array[i].hp_colors = 1 << 5; 478 } 479 } 480 481 /* 482 * group colorequiv colors on N2 by low order bits of the color first 483 */ 484 void 485 page_set_colorequiv_arr_cpu(void) 486 { 487 static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0}; 488 489 if (colorequiv > 1) { 490 int i; 491 uint_t sv_a = lowbit(colorequiv) - 1; 492 493 if (sv_a > 15) 494 sv_a = 15; 495 496 for (i = 0; i < MMU_PAGE_SIZES; i++) { 497 uint_t colors; 498 uint_t a = sv_a; 499 500 if ((colors = hw_page_array[i].hp_colors) <= 1) 501 continue; 502 while ((colors >> a) == 0) 503 a--; 504 if (a > (colorequivszc[i] & 0xf) + 505 (colorequivszc[i] >> 4)) { 506 if (a <= nequiv_shades_log2[i]) { 507 colorequivszc[i] = (uchar_t)a; 508 } else { 509 colorequivszc[i] = 510 ((a - nequiv_shades_log2[i]) << 4) | 511 nequiv_shades_log2[i]; 512 } 513 } 514 } 515 } 516 } 517