1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/systm.h> 31 #include <sys/archsystm.h> 32 #include <sys/machparam.h> 33 #include <sys/machsystm.h> 34 #include <sys/cpu.h> 35 #include <sys/elf_SPARC.h> 36 #include <vm/hat_sfmmu.h> 37 #include <vm/page.h> 38 #include <vm/vm_dep.h> 39 #include <sys/cpuvar.h> 40 #include <sys/async.h> 41 #include <sys/cmn_err.h> 42 #include <sys/debug.h> 43 #include <sys/dditypes.h> 44 #include <sys/sunddi.h> 45 #include <sys/cpu_module.h> 46 #include <sys/prom_debug.h> 47 #include <sys/vmsystm.h> 48 #include <sys/prom_plat.h> 49 #include <sys/sysmacros.h> 50 #include <sys/intreg.h> 51 #include <sys/machtrap.h> 52 #include <sys/ontrap.h> 53 #include <sys/ivintr.h> 54 #include <sys/atomic.h> 55 #include <sys/panic.h> 56 #include <sys/dtrace.h> 57 #include <sys/simulate.h> 58 #include <sys/fault.h> 59 #include <sys/niagara2regs.h> 60 #include <sys/hsvc.h> 61 #include <sys/trapstat.h> 62 #include <sys/mutex_impl.h> 63 64 uint_t root_phys_addr_lo_mask = 0xffffffffU; 65 #if defined(NIAGARA2_IMPL) 66 char cpu_module_name[] = "SUNW,UltraSPARC-T2"; 67 #elif defined(VFALLS_IMPL) 68 char cpu_module_name[] = "SUNW,UltraSPARC-T2+"; 69 #endif 70 71 /* 72 * Hypervisor services information for the NIAGARA2 and Victoria Falls 73 * CPU module 74 */ 75 static boolean_t cpu_hsvc_available = B_TRUE; 76 static uint64_t cpu_sup_minor; /* Supported minor number */ 77 #if defined(NIAGARA2_IMPL) 78 static hsvc_info_t cpu_hsvc = { 79 HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, 80 NIAGARA2_HSVC_MINOR, cpu_module_name 81 }; 82 #elif defined(VFALLS_IMPL) 83 static hsvc_info_t cpu_hsvc = { 84 HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR, 85 VFALLS_HSVC_MINOR, cpu_module_name 86 }; 87 #endif 88 89 void 90 cpu_setup(void) 91 { 92 extern int mmu_exported_pagesize_mask; 93 extern int cpc_has_overflow_intr; 94 extern size_t contig_mem_prealloc_base_size; 95 int status; 96 97 /* 98 * Negotiate the API version for Niagara2 specific hypervisor 99 * services. 100 */ 101 status = hsvc_register(&cpu_hsvc, &cpu_sup_minor); 102 if (status != 0) { 103 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 104 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", 105 cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group, 106 cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status); 107 cpu_hsvc_available = B_FALSE; 108 } 109 110 /* 111 * The setup common to all CPU modules is done in cpu_setup_common 112 * routine. 113 */ 114 cpu_setup_common(NULL); 115 116 cache |= (CACHE_PTAG | CACHE_IOCOHERENT); 117 118 if ((mmu_exported_pagesize_mask & 119 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != 120 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) 121 cmn_err(CE_PANIC, "machine description" 122 " does not have required sun4v page sizes" 123 " 8K, 64K and 4M: MD mask is 0x%x", 124 mmu_exported_pagesize_mask); 125 126 cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT; 127 128 /* 129 * Niagara2 supports a 48-bit subset of the full 64-bit virtual 130 * address space. Virtual addresses between 0x0000800000000000 131 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" 132 * and must never be mapped. In addition, software must not use 133 * pages within 4GB of the VA hole as instruction pages to 134 * avoid problems with prefetching into the VA hole. 135 */ 136 hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); 137 hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); 138 139 /* 140 * Niagara2 has a performance counter overflow interrupt 141 */ 142 cpc_has_overflow_intr = 1; 143 144 /* 145 * Enable 4M pages for OOB. 146 */ 147 max_uheap_lpsize = MMU_PAGESIZE4M; 148 max_ustack_lpsize = MMU_PAGESIZE4M; 149 max_privmap_lpsize = MMU_PAGESIZE4M; 150 151 #ifdef SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB 152 /* 153 * Use CPU Makefile specific compile time define (if exists) 154 * to add to the contig preallocation size. 155 */ 156 contig_mem_prealloc_base_size = MB(SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB); 157 #endif 158 } 159 160 /* 161 * Set the magic constants of the implementation. 162 */ 163 void 164 cpu_fiximp(struct cpu_node *cpunode) 165 { 166 /* 167 * The Cache node is optional in MD. Therefore in case "Cache" 168 * node does not exists in MD, set the default L2 cache associativity, 169 * size, linesize. 170 */ 171 if (cpunode->ecache_size == 0) 172 cpunode->ecache_size = L2CACHE_SIZE; 173 if (cpunode->ecache_linesize == 0) 174 cpunode->ecache_linesize = L2CACHE_LINESIZE; 175 if (cpunode->ecache_associativity == 0) 176 cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; 177 } 178 179 void 180 cpu_map_exec_units(struct cpu *cp) 181 { 182 ASSERT(MUTEX_HELD(&cpu_lock)); 183 184 /* 185 * The cpu_ipipe and cpu_fpu fields are initialized based on 186 * the execution unit sharing information from the MD. They 187 * default to the CPU id in the absence of such information. 188 */ 189 cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; 190 if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) 191 cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); 192 193 cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping; 194 if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND) 195 cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id); 196 197 /* 198 * Niagara 2 defines the core to be at the FPU level 199 */ 200 cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu; 201 202 /* 203 * The cpu_chip field is initialized based on the information 204 * in the MD and assume that all cpus within a chip 205 * share the same L2 cache. If no such info is available, we 206 * set the cpu to belong to the defacto chip 0. 207 */ 208 cp->cpu_m.cpu_mpipe = cpunodes[cp->cpu_id].l2_cache_mapping; 209 if (cp->cpu_m.cpu_mpipe == NO_L2_CACHE_MAPPING_FOUND) 210 cp->cpu_m.cpu_mpipe = CPU_L2_CACHEID_INVALID; 211 212 cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping; 213 if (cp->cpu_m.cpu_chip == NO_L2_CACHE_MAPPING_FOUND) 214 cp->cpu_m.cpu_chip = CPU_CHIPID_INVALID; 215 } 216 217 static int cpucnt; 218 219 void 220 cpu_init_private(struct cpu *cp) 221 { 222 extern void niagara_kstat_init(void); 223 224 ASSERT(MUTEX_HELD(&cpu_lock)); 225 226 cpu_map_exec_units(cp); 227 228 if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE)) 229 (void) niagara_kstat_init(); 230 231 mutex_delay = rdccr_delay; 232 } 233 234 /*ARGSUSED*/ 235 void 236 cpu_uninit_private(struct cpu *cp) 237 { 238 extern void niagara_kstat_fini(void); 239 240 ASSERT(MUTEX_HELD(&cpu_lock)); 241 if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE)) 242 (void) niagara_kstat_fini(); 243 } 244 245 /* 246 * On Niagara2, any flush will cause all preceding stores to be 247 * synchronized wrt the i$, regardless of address or ASI. In fact, 248 * the address is ignored, so we always flush address 0. 249 */ 250 /*ARGSUSED*/ 251 void 252 dtrace_flush_sec(uintptr_t addr) 253 { 254 doflush(0); 255 } 256 257 /* 258 * Trapstat support for Niagara2 processor 259 * The Niagara2 provides HWTW support for TSB lookup and with HWTW 260 * enabled no TSB hit information will be available. Therefore setting 261 * the time spent in TLB miss handler for TSB hits to 0. 262 */ 263 int 264 cpu_trapstat_conf(int cmd) 265 { 266 int status = 0; 267 268 switch (cmd) { 269 case CPU_TSTATCONF_INIT: 270 case CPU_TSTATCONF_FINI: 271 case CPU_TSTATCONF_ENABLE: 272 case CPU_TSTATCONF_DISABLE: 273 break; 274 default: 275 status = EINVAL; 276 break; 277 } 278 return (status); 279 } 280 281 void 282 cpu_trapstat_data(void *buf, uint_t tstat_pgszs) 283 { 284 tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; 285 int i; 286 287 for (i = 0; i < tstat_pgszs; i++, tstatp++) { 288 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; 289 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; 290 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; 291 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; 292 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 293 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 294 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 295 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 296 } 297 } 298 299 /* 300 * Page coloring support for hashed cache index mode 301 */ 302 303 /* 304 * Node id bits from machine description (MD). Node id distinguishes 305 * local versus remote memory. Because of MPO, page allocation does 306 * not cross node boundaries. Therefore, remove the node id bits from 307 * the color, since they are fixed. Either bit 30, or 31:30 in 308 * Victoria Falls processors. 309 * The number of node id bits is always 0 in Niagara2. 310 */ 311 typedef struct n2color { 312 uchar_t nnbits; /* number of node id bits */ 313 uchar_t nnmask; /* mask for node id bits */ 314 uchar_t lomask; /* mask for bits below node id */ 315 uchar_t lobits; /* number of bits below node id */ 316 } n2color_t; 317 318 n2color_t n2color[MMU_PAGE_SIZES]; 319 static uchar_t nhbits[] = {7, 7, 6, 5, 5, 5}; 320 321 /* 322 * Remove node id bits from color bits 32:28. 323 * This will reduce the number of colors. 324 * No change if number of node bits is zero. 325 */ 326 static inline uint_t 327 n2_hash2color(uint_t color, uchar_t szc) 328 { 329 n2color_t m = n2color[szc]; 330 331 if (m.nnbits > 0) { 332 color = ((color >> m.nnbits) & ~m.lomask) | (color & m.lomask); 333 ASSERT((color & ~(hw_page_array[szc].hp_colors - 1)) == 0); 334 } 335 336 return (color); 337 } 338 339 /* 340 * Restore node id bits into page color. 341 * This will increase the number of colors to match N2. 342 * No change if number of node bits is zero. 343 */ 344 static inline uint_t 345 n2_color2hash(uint_t color, uchar_t szc, uint_t node) 346 { 347 n2color_t m = n2color[szc]; 348 349 if (m.nnbits > 0) { 350 color = ((color & ~m.lomask) << m.nnbits) | (color & m.lomask); 351 color |= (node & m.nnmask) << m.lobits; 352 } 353 354 return (color); 355 } 356 357 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ 358 359 /* 360 * iterator NULL means pfn is VA, do not adjust ra_to_pa 361 * iterator (-1) means pfn is RA, need to convert to PA 362 * iterator non-null means pfn is RA, use ra_to_pa 363 */ 364 uint_t 365 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc, void *cookie) 366 { 367 mem_node_iterator_t *it = cookie; 368 uint_t color; 369 370 ASSERT(szc <= TTE256M); 371 372 if (it == ((mem_node_iterator_t *)(-1))) { 373 pfn = plat_rapfn_to_papfn(pfn); 374 } else if (it != NULL) { 375 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 376 pfn = pfn + it->mi_ra_to_pa; 377 } 378 pfn = PFN_BASE(pfn, szc); 379 color = ((pfn >> 15) ^ pfn) & 0x1f; 380 if (szc < TTE4M) { 381 /* 19:18 */ 382 color = (color << 2) | ((pfn >> 5) & 0x3); 383 if (szc > TTE64K) 384 color >>= 1; /* 19 */ 385 } 386 return (n2_hash2color(color, szc)); 387 } 388 389 static uint_t 390 page_papfn_2_color_cpu(pfn_t papfn, uchar_t szc) 391 { 392 uint_t color; 393 394 ASSERT(szc <= TTE256M); 395 396 papfn = PFN_BASE(papfn, szc); 397 color = ((papfn >> 15) ^ papfn) & 0x1f; 398 if (szc < TTE4M) { 399 /* 19:18 */ 400 color = (color << 2) | ((papfn >> 5) & 0x3); 401 if (szc > TTE64K) 402 color >>= 1; /* 19 */ 403 } 404 return (color); 405 } 406 407 #if TTE256M != 5 408 #error TTE256M is not 5 409 #endif 410 411 uint_t 412 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) 413 { 414 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; 415 ASSERT(szc < TTE256M); 416 mask = n2_color2hash(mask, szc, 0); 417 mask &= ni2_color_masks[szc]; 418 if (szc == TTE64K || szc == TTE512K) 419 mask >>= 1; 420 return (n2_hash2color(mask, szc + 1)); 421 } 422 423 uint_t 424 page_get_nsz_color_cpu(uchar_t szc, uint_t color) 425 { 426 ASSERT(szc < TTE256M); 427 color = n2_color2hash(color, szc, 0); 428 if (szc == TTE64K || szc == TTE512K) 429 color >>= 1; 430 return (n2_hash2color(color, szc + 1)); 431 } 432 433 uint_t 434 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) 435 { 436 uint_t s; 437 ASSERT(nszc >= szc); 438 ASSERT(nszc <= TTE256M); 439 440 s = nhbits[szc] - n2color[szc].nnbits; 441 s -= nhbits[nszc] - n2color[nszc].nnbits; 442 443 return (s); 444 } 445 446 uint_t 447 page_convert_color_cpu(uint_t ncolor, uchar_t szc, uchar_t nszc) 448 { 449 uint_t color; 450 451 ASSERT(nszc > szc); 452 ASSERT(nszc <= TTE256M); 453 ncolor = n2_color2hash(ncolor, nszc, 0); 454 color = ncolor << (nhbits[szc] - nhbits[nszc]); 455 color = n2_hash2color(color, szc); 456 return (color); 457 } 458 459 #define PAPFN_2_MNODE(pfn) \ 460 (((pfn) & it->mi_mnode_pfn_mask) >> it->mi_mnode_pfn_shift) 461 462 /*ARGSUSED*/ 463 pfn_t 464 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, 465 uint_t ceq_mask, uint_t color_mask, void *cookie) 466 { 467 mem_node_iterator_t *it = cookie; 468 pfn_t pstep = PNUM_SIZE(szc); 469 pfn_t npfn, pfn_ceq_mask, pfn_color; 470 pfn_t tmpmask, mask = (pfn_t)-1; 471 uint_t pfnmn; 472 473 ASSERT((color & ~ceq_mask) == 0); 474 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 475 476 /* convert RA to PA for accurate color calculation */ 477 if (it->mi_init) { 478 /* first call after it, so cache these values */ 479 it->mi_hash_ceq_mask = 480 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask); 481 it->mi_hash_color = 482 n2_color2hash(color, szc, it->mi_mnode); 483 it->mi_init = 0; 484 } else { 485 ASSERT(it->mi_hash_ceq_mask == 486 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask)); 487 ASSERT(it->mi_hash_color == 488 n2_color2hash(color, szc, it->mi_mnode)); 489 } 490 ceq_mask = it->mi_hash_ceq_mask; 491 color = it->mi_hash_color; 492 pfn += it->mi_ra_to_pa; 493 494 /* restart here when we switch memblocks */ 495 next_mem_block: 496 if (szc <= TTE64K) { 497 pfnmn = PAPFN_2_MNODE(pfn); 498 } 499 if (((page_papfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0 && 500 (szc > TTE64K || pfnmn == it->mi_mnode)) { 501 502 /* we start from the page with correct color */ 503 if (szc >= TTE512K) { 504 if (szc >= TTE4M) { 505 /* page color is PA[32:28] */ 506 pfn_ceq_mask = ceq_mask << 15; 507 } else { 508 /* page color is PA[32:28].PA[19:19] */ 509 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 510 ((ceq_mask >> 1) << 15); 511 } 512 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 513 goto done; 514 } else { 515 /* 516 * We deal 64K or 8K page. Check if we could the 517 * satisfy the request without changing PA[32:28] 518 */ 519 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 520 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 521 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 522 523 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 524 goto done; 525 526 /* 527 * for next pfn we have to change bits PA[32:28] 528 * set PA[63:28] and PA[19:18] of the next pfn 529 */ 530 npfn = (pfn >> 15) << 15; 531 npfn |= (ceq_mask & color & 3) << 5; 532 pfn_ceq_mask = (szc == TTE8K) ? 0 : 533 (ceq_mask & 0x1c) << 13; 534 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 535 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); 536 537 /* 538 * set bits PA[17:13] to match the color 539 */ 540 npfn |= ((npfn >> 15) ^ (color >> 2)) & (ceq_mask >> 2); 541 goto done; 542 } 543 } 544 545 /* 546 * we start from the page with incorrect color - rare case 547 */ 548 if (szc >= TTE512K) { 549 if (szc >= TTE4M) { 550 /* page color is in bits PA[32:28] */ 551 npfn = ((pfn >> 20) << 20) | (color << 15); 552 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; 553 } else { 554 /* try get the right color by changing bit PA[19:19] */ 555 npfn = pfn + pstep; 556 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 557 ceq_mask) == 0) 558 goto done; 559 560 /* page color is PA[32:28].PA[19:19] */ 561 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 562 ((ceq_mask >> 1) << 15) | (0xff << 7); 563 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); 564 npfn = ((pfn >> 20) << 20) | pfn_color; 565 } 566 567 while (npfn <= pfn) { 568 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 569 } 570 goto done; 571 } 572 573 /* 574 * We deal 64K or 8K page of incorrect color. 575 * Try correcting color without changing PA[32:28] 576 */ 577 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 578 pfn_color = ((color & 3) << 5) | (color >> 2); 579 if (pfnmn == it->mi_mnode) { 580 npfn = (pfn & ~(pfn_t)0x7f); 581 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 582 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 583 584 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 585 ceq_mask) == 0) { 586 /* the color is fixed - find the next page */ 587 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 588 while (npfn <= pfn) { 589 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, 590 mask); 591 } 592 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 593 goto done; 594 } 595 } 596 597 /* to fix the color need to touch PA[32:28] */ 598 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : 599 (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); 600 601 /* fix mnode if input pfn is in the wrong mnode. */ 602 if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) { 603 npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) << 604 it->mi_mnode_pfn_shift; 605 } 606 607 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; 608 tmpmask |= it->mi_mnode_pfn_mask; 609 610 while (npfn <= pfn) { 611 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); 612 } 613 614 /* set bits PA[19:13] to match the color */ 615 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 616 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 617 618 done: 619 ASSERT(((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); 620 ASSERT(PAPFN_2_MNODE(npfn) == it->mi_mnode); 621 622 /* PA to RA */ 623 npfn -= it->mi_ra_to_pa; 624 625 /* check for possible memblock switch */ 626 if (npfn > it->mi_mblock_end) { 627 pfn = plat_mem_node_iterator_init(npfn, it->mi_mnode, szc, it, 628 0); 629 if (pfn == (pfn_t)-1) 630 return (pfn); 631 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 632 pfn += it->mi_ra_to_pa; 633 goto next_mem_block; 634 } 635 636 return (npfn); 637 } 638 639 /* 640 * init page coloring 641 * VF encodes node_id for an L-group in either bit 30 or 31:30, 642 * which effectively reduces the number of colors available per mnode. 643 */ 644 void 645 page_coloring_init_cpu() 646 { 647 int i; 648 uchar_t id; 649 uchar_t lo; 650 uchar_t hi; 651 n2color_t m; 652 mem_node_iterator_t it; 653 static uchar_t idmask[] = {0, 0x7, 0x1f, 0x1f, 0x1f, 0x1f}; 654 655 for (i = 0; i < max_mem_nodes; i++) { 656 memset(&it, 0, sizeof (it)); 657 if (plat_mem_node_iterator_init(0, i, 0, &it, 1) != (pfn_t)-1) 658 break; 659 } 660 ASSERT(i < max_mem_nodes); 661 for (i = 0; i < mmu_page_sizes; i++) { 662 (void) memset(&m, 0, sizeof (m)); 663 id = it.mi_mnode_pfn_mask >> 15; /* node id mask */ 664 id &= idmask[i]; 665 lo = lowbit(id); 666 if (lo > 0) { 667 hi = highbit(id); 668 m.nnbits = hi - lo + 1; 669 m.nnmask = (1 << m.nnbits) - 1; 670 lo += nhbits[i] - 5; 671 m.lomask = (1 << (lo - 1)) - 1; 672 m.lobits = lo - 1; 673 } 674 hw_page_array[i].hp_colors = 1 << (nhbits[i] - m.nnbits); 675 n2color[i] = m; 676 } 677 } 678 679 /* 680 * group colorequiv colors on N2 by low order bits of the color first 681 */ 682 void 683 page_set_colorequiv_arr_cpu(void) 684 { 685 static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0}; 686 687 nequiv_shades_log2[1] -= n2color[1].nnbits; 688 if (colorequiv > 1) { 689 int i; 690 uint_t sv_a = lowbit(colorequiv) - 1; 691 692 if (sv_a > 15) 693 sv_a = 15; 694 695 for (i = 0; i < MMU_PAGE_SIZES; i++) { 696 uint_t colors; 697 uint_t a = sv_a; 698 699 if ((colors = hw_page_array[i].hp_colors) <= 1) 700 continue; 701 while ((colors >> a) == 0) 702 a--; 703 if (a > (colorequivszc[i] & 0xf) + 704 (colorequivszc[i] >> 4)) { 705 if (a <= nequiv_shades_log2[i]) { 706 colorequivszc[i] = (uchar_t)a; 707 } else { 708 colorequivszc[i] = 709 ((a - nequiv_shades_log2[i]) << 4) | 710 nequiv_shades_log2[i]; 711 } 712 } 713 } 714 } 715 } 716