1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/systm.h> 31 #include <sys/archsystm.h> 32 #include <sys/machparam.h> 33 #include <sys/machsystm.h> 34 #include <sys/cpu.h> 35 #include <sys/elf_SPARC.h> 36 #include <vm/hat_sfmmu.h> 37 #include <vm/page.h> 38 #include <vm/vm_dep.h> 39 #include <sys/cpuvar.h> 40 #include <sys/async.h> 41 #include <sys/cmn_err.h> 42 #include <sys/debug.h> 43 #include <sys/dditypes.h> 44 #include <sys/sunddi.h> 45 #include <sys/cpu_module.h> 46 #include <sys/prom_debug.h> 47 #include <sys/vmsystm.h> 48 #include <sys/prom_plat.h> 49 #include <sys/sysmacros.h> 50 #include <sys/intreg.h> 51 #include <sys/machtrap.h> 52 #include <sys/ontrap.h> 53 #include <sys/ivintr.h> 54 #include <sys/atomic.h> 55 #include <sys/panic.h> 56 #include <sys/dtrace.h> 57 #include <sys/simulate.h> 58 #include <sys/fault.h> 59 #include <sys/niagara2regs.h> 60 #include <sys/hsvc.h> 61 #include <sys/trapstat.h> 62 63 uint_t root_phys_addr_lo_mask = 0xffffffffU; 64 #if defined(NIAGARA2_IMPL) 65 char cpu_module_name[] = "SUNW,UltraSPARC-T2"; 66 #elif defined(VFALLS_IMPL) 67 char cpu_module_name[] = "SUNW,UltraSPARC-T2+"; 68 #endif 69 70 /* 71 * Hypervisor services information for the NIAGARA2 and Victoria Falls 72 * CPU module 73 */ 74 static boolean_t cpu_hsvc_available = B_TRUE; 75 static uint64_t cpu_sup_minor; /* Supported minor number */ 76 #if defined(NIAGARA2_IMPL) 77 static hsvc_info_t cpu_hsvc = { 78 HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, 79 NIAGARA2_HSVC_MINOR, cpu_module_name 80 }; 81 #elif defined(VFALLS_IMPL) 82 static hsvc_info_t cpu_hsvc = { 83 HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR, 84 VFALLS_HSVC_MINOR, cpu_module_name 85 }; 86 #endif 87 88 void 89 cpu_setup(void) 90 { 91 extern int mmu_exported_pagesize_mask; 92 extern int cpc_has_overflow_intr; 93 extern size_t contig_mem_prealloc_base_size; 94 int status; 95 96 /* 97 * Negotiate the API version for Niagara2 specific hypervisor 98 * services. 99 */ 100 status = hsvc_register(&cpu_hsvc, &cpu_sup_minor); 101 if (status != 0) { 102 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 103 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", 104 cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group, 105 cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status); 106 cpu_hsvc_available = B_FALSE; 107 } 108 109 /* 110 * The setup common to all CPU modules is done in cpu_setup_common 111 * routine. 112 */ 113 cpu_setup_common(NULL); 114 115 cache |= (CACHE_PTAG | CACHE_IOCOHERENT); 116 117 if ((mmu_exported_pagesize_mask & 118 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != 119 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) 120 cmn_err(CE_PANIC, "machine description" 121 " does not have required sun4v page sizes" 122 " 8K, 64K and 4M: MD mask is 0x%x", 123 mmu_exported_pagesize_mask); 124 125 cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT; 126 127 /* 128 * Niagara2 supports a 48-bit subset of the full 64-bit virtual 129 * address space. Virtual addresses between 0x0000800000000000 130 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" 131 * and must never be mapped. In addition, software must not use 132 * pages within 4GB of the VA hole as instruction pages to 133 * avoid problems with prefetching into the VA hole. 134 */ 135 hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); 136 hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); 137 138 /* 139 * Niagara2 has a performance counter overflow interrupt 140 */ 141 cpc_has_overflow_intr = 1; 142 143 /* 144 * Enable 4M pages for OOB. 145 */ 146 max_uheap_lpsize = MMU_PAGESIZE4M; 147 max_ustack_lpsize = MMU_PAGESIZE4M; 148 max_privmap_lpsize = MMU_PAGESIZE4M; 149 150 #ifdef SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB 151 /* 152 * Use CPU Makefile specific compile time define (if exists) 153 * to add to the contig preallocation size. 154 */ 155 contig_mem_prealloc_base_size = MB(SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB); 156 #endif 157 } 158 159 /* 160 * Set the magic constants of the implementation. 161 */ 162 void 163 cpu_fiximp(struct cpu_node *cpunode) 164 { 165 /* 166 * The Cache node is optional in MD. Therefore in case "Cache" 167 * node does not exists in MD, set the default L2 cache associativity, 168 * size, linesize. 169 */ 170 if (cpunode->ecache_size == 0) 171 cpunode->ecache_size = L2CACHE_SIZE; 172 if (cpunode->ecache_linesize == 0) 173 cpunode->ecache_linesize = L2CACHE_LINESIZE; 174 if (cpunode->ecache_associativity == 0) 175 cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; 176 } 177 178 void 179 cpu_map_exec_units(struct cpu *cp) 180 { 181 ASSERT(MUTEX_HELD(&cpu_lock)); 182 183 /* 184 * The cpu_ipipe and cpu_fpu fields are initialized based on 185 * the execution unit sharing information from the MD. They 186 * default to the CPU id in the absence of such information. 187 */ 188 cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; 189 if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) 190 cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); 191 192 cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping; 193 if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND) 194 cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id); 195 196 /* 197 * Niagara 2 defines the core to be at the FPU level 198 */ 199 cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu; 200 201 /* 202 * The cpu_chip field is initialized based on the information 203 * in the MD and assume that all cpus within a chip 204 * share the same L2 cache. If no such info is available, we 205 * set the cpu to belong to the defacto chip 0. 206 */ 207 cp->cpu_m.cpu_mpipe = cpunodes[cp->cpu_id].l2_cache_mapping; 208 if (cp->cpu_m.cpu_mpipe == NO_L2_CACHE_MAPPING_FOUND) 209 cp->cpu_m.cpu_mpipe = CPU_L2_CACHEID_INVALID; 210 211 cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping; 212 if (cp->cpu_m.cpu_chip == NO_L2_CACHE_MAPPING_FOUND) 213 cp->cpu_m.cpu_chip = CPU_CHIPID_INVALID; 214 } 215 216 static int cpucnt; 217 218 void 219 cpu_init_private(struct cpu *cp) 220 { 221 extern void niagara_kstat_init(void); 222 223 ASSERT(MUTEX_HELD(&cpu_lock)); 224 225 cpu_map_exec_units(cp); 226 227 if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE)) 228 (void) niagara_kstat_init(); 229 } 230 231 /*ARGSUSED*/ 232 void 233 cpu_uninit_private(struct cpu *cp) 234 { 235 extern void niagara_kstat_fini(void); 236 237 ASSERT(MUTEX_HELD(&cpu_lock)); 238 if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE)) 239 (void) niagara_kstat_fini(); 240 } 241 242 /* 243 * On Niagara2, any flush will cause all preceding stores to be 244 * synchronized wrt the i$, regardless of address or ASI. In fact, 245 * the address is ignored, so we always flush address 0. 246 */ 247 /*ARGSUSED*/ 248 void 249 dtrace_flush_sec(uintptr_t addr) 250 { 251 doflush(0); 252 } 253 254 /* 255 * Trapstat support for Niagara2 processor 256 * The Niagara2 provides HWTW support for TSB lookup and with HWTW 257 * enabled no TSB hit information will be available. Therefore setting 258 * the time spent in TLB miss handler for TSB hits to 0. 259 */ 260 int 261 cpu_trapstat_conf(int cmd) 262 { 263 int status = 0; 264 265 switch (cmd) { 266 case CPU_TSTATCONF_INIT: 267 case CPU_TSTATCONF_FINI: 268 case CPU_TSTATCONF_ENABLE: 269 case CPU_TSTATCONF_DISABLE: 270 break; 271 default: 272 status = EINVAL; 273 break; 274 } 275 return (status); 276 } 277 278 void 279 cpu_trapstat_data(void *buf, uint_t tstat_pgszs) 280 { 281 tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; 282 int i; 283 284 for (i = 0; i < tstat_pgszs; i++, tstatp++) { 285 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; 286 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; 287 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; 288 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; 289 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 290 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 291 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 292 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 293 } 294 } 295 296 /* 297 * Page coloring support for hashed cache index mode 298 */ 299 300 /* 301 * Node id bits from machine description (MD). Node id distinguishes 302 * local versus remote memory. Because of MPO, page allocation does 303 * not cross node boundaries. Therefore, remove the node id bits from 304 * the color, since they are fixed. Either bit 30, or 31:30 in 305 * Victoria Falls processors. 306 * The number of node id bits is always 0 in Niagara2. 307 */ 308 typedef struct n2color { 309 uchar_t nnbits; /* number of node id bits */ 310 uchar_t nnmask; /* mask for node id bits */ 311 uchar_t lomask; /* mask for bits below node id */ 312 uchar_t lobits; /* number of bits below node id */ 313 } n2color_t; 314 315 n2color_t n2color[MMU_PAGE_SIZES]; 316 static uchar_t nhbits[] = {7, 7, 6, 5, 5, 5}; 317 318 /* 319 * Remove node id bits from color bits 32:28. 320 * This will reduce the number of colors. 321 * No change if number of node bits is zero. 322 */ 323 static inline uint_t 324 n2_hash2color(uint_t color, uchar_t szc) 325 { 326 n2color_t m = n2color[szc]; 327 328 if (m.nnbits > 0) { 329 color = ((color >> m.nnbits) & ~m.lomask) | (color & m.lomask); 330 ASSERT((color & ~(hw_page_array[szc].hp_colors - 1)) == 0); 331 } 332 333 return (color); 334 } 335 336 /* 337 * Restore node id bits into page color. 338 * This will increase the number of colors to match N2. 339 * No change if number of node bits is zero. 340 */ 341 static inline uint_t 342 n2_color2hash(uint_t color, uchar_t szc, uint_t node) 343 { 344 n2color_t m = n2color[szc]; 345 346 if (m.nnbits > 0) { 347 color = ((color & ~m.lomask) << m.nnbits) | (color & m.lomask); 348 color |= (node & m.nnmask) << m.lobits; 349 } 350 351 return (color); 352 } 353 354 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ 355 356 /* 357 * iterator NULL means pfn is VA, do not adjust ra_to_pa 358 * iterator (-1) means pfn is RA, need to convert to PA 359 * iterator non-null means pfn is RA, use ra_to_pa 360 */ 361 uint_t 362 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc, void *cookie) 363 { 364 mem_node_iterator_t *it = cookie; 365 uint_t color; 366 367 ASSERT(szc <= TTE256M); 368 369 if (it == ((mem_node_iterator_t *)(-1))) { 370 pfn = plat_rapfn_to_papfn(pfn); 371 } else if (it != NULL) { 372 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 373 pfn = pfn + it->mi_ra_to_pa; 374 } 375 pfn = PFN_BASE(pfn, szc); 376 color = ((pfn >> 15) ^ pfn) & 0x1f; 377 if (szc < TTE4M) { 378 /* 19:18 */ 379 color = (color << 2) | ((pfn >> 5) & 0x3); 380 if (szc > TTE64K) 381 color >>= 1; /* 19 */ 382 } 383 return (n2_hash2color(color, szc)); 384 } 385 386 static uint_t 387 page_papfn_2_color_cpu(pfn_t papfn, uchar_t szc) 388 { 389 uint_t color; 390 391 ASSERT(szc <= TTE256M); 392 393 papfn = PFN_BASE(papfn, szc); 394 color = ((papfn >> 15) ^ papfn) & 0x1f; 395 if (szc < TTE4M) { 396 /* 19:18 */ 397 color = (color << 2) | ((papfn >> 5) & 0x3); 398 if (szc > TTE64K) 399 color >>= 1; /* 19 */ 400 } 401 return (color); 402 } 403 404 #if TTE256M != 5 405 #error TTE256M is not 5 406 #endif 407 408 uint_t 409 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) 410 { 411 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; 412 ASSERT(szc < TTE256M); 413 mask = n2_color2hash(mask, szc, 0); 414 mask &= ni2_color_masks[szc]; 415 if (szc == TTE64K || szc == TTE512K) 416 mask >>= 1; 417 return (n2_hash2color(mask, szc + 1)); 418 } 419 420 uint_t 421 page_get_nsz_color_cpu(uchar_t szc, uint_t color) 422 { 423 ASSERT(szc < TTE256M); 424 color = n2_color2hash(color, szc, 0); 425 if (szc == TTE64K || szc == TTE512K) 426 color >>= 1; 427 return (n2_hash2color(color, szc + 1)); 428 } 429 430 uint_t 431 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) 432 { 433 uint_t s; 434 ASSERT(nszc >= szc); 435 ASSERT(nszc <= TTE256M); 436 437 s = nhbits[szc] - n2color[szc].nnbits; 438 s -= nhbits[nszc] - n2color[nszc].nnbits; 439 440 return (s); 441 } 442 443 uint_t 444 page_convert_color_cpu(uint_t ncolor, uchar_t szc, uchar_t nszc) 445 { 446 uint_t color; 447 448 ASSERT(nszc > szc); 449 ASSERT(nszc <= TTE256M); 450 ncolor = n2_color2hash(ncolor, nszc, 0); 451 color = ncolor << (nhbits[szc] - nhbits[nszc]); 452 color = n2_hash2color(color, szc); 453 return (color); 454 } 455 456 #define PAPFN_2_MNODE(pfn) \ 457 (((pfn) & it->mi_mnode_pfn_mask) >> it->mi_mnode_pfn_shift) 458 459 /*ARGSUSED*/ 460 pfn_t 461 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, 462 uint_t ceq_mask, uint_t color_mask, void *cookie) 463 { 464 mem_node_iterator_t *it = cookie; 465 pfn_t pstep = PNUM_SIZE(szc); 466 pfn_t npfn, pfn_ceq_mask, pfn_color; 467 pfn_t tmpmask, mask = (pfn_t)-1; 468 uint_t pfnmn; 469 470 ASSERT((color & ~ceq_mask) == 0); 471 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 472 473 /* convert RA to PA for accurate color calculation */ 474 if (it->mi_init) { 475 /* first call after it, so cache these values */ 476 it->mi_hash_ceq_mask = 477 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask); 478 it->mi_hash_color = 479 n2_color2hash(color, szc, it->mi_mnode); 480 it->mi_init = 0; 481 } else { 482 ASSERT(it->mi_hash_ceq_mask == 483 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask)); 484 ASSERT(it->mi_hash_color == 485 n2_color2hash(color, szc, it->mi_mnode)); 486 } 487 ceq_mask = it->mi_hash_ceq_mask; 488 color = it->mi_hash_color; 489 pfn += it->mi_ra_to_pa; 490 491 /* restart here when we switch memblocks */ 492 next_mem_block: 493 if (szc <= TTE64K) { 494 pfnmn = PAPFN_2_MNODE(pfn); 495 } 496 if (((page_papfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0 && 497 (szc > TTE64K || pfnmn == it->mi_mnode)) { 498 499 /* we start from the page with correct color */ 500 if (szc >= TTE512K) { 501 if (szc >= TTE4M) { 502 /* page color is PA[32:28] */ 503 pfn_ceq_mask = ceq_mask << 15; 504 } else { 505 /* page color is PA[32:28].PA[19:19] */ 506 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 507 ((ceq_mask >> 1) << 15); 508 } 509 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 510 goto done; 511 } else { 512 /* 513 * We deal 64K or 8K page. Check if we could the 514 * satisfy the request without changing PA[32:28] 515 */ 516 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 517 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 518 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 519 520 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 521 goto done; 522 523 /* 524 * for next pfn we have to change bits PA[32:28] 525 * set PA[63:28] and PA[19:18] of the next pfn 526 */ 527 npfn = (pfn >> 15) << 15; 528 npfn |= (ceq_mask & color & 3) << 5; 529 pfn_ceq_mask = (szc == TTE8K) ? 0 : 530 (ceq_mask & 0x1c) << 13; 531 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 532 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); 533 534 /* 535 * set bits PA[17:13] to match the color 536 */ 537 npfn |= ((npfn >> 15) ^ (color >> 2)) & (ceq_mask >> 2); 538 goto done; 539 } 540 } 541 542 /* 543 * we start from the page with incorrect color - rare case 544 */ 545 if (szc >= TTE512K) { 546 if (szc >= TTE4M) { 547 /* page color is in bits PA[32:28] */ 548 npfn = ((pfn >> 20) << 20) | (color << 15); 549 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; 550 } else { 551 /* try get the right color by changing bit PA[19:19] */ 552 npfn = pfn + pstep; 553 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 554 ceq_mask) == 0) 555 goto done; 556 557 /* page color is PA[32:28].PA[19:19] */ 558 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 559 ((ceq_mask >> 1) << 15) | (0xff << 7); 560 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); 561 npfn = ((pfn >> 20) << 20) | pfn_color; 562 } 563 564 while (npfn <= pfn) { 565 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 566 } 567 goto done; 568 } 569 570 /* 571 * We deal 64K or 8K page of incorrect color. 572 * Try correcting color without changing PA[32:28] 573 */ 574 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 575 pfn_color = ((color & 3) << 5) | (color >> 2); 576 if (pfnmn == it->mi_mnode) { 577 npfn = (pfn & ~(pfn_t)0x7f); 578 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 579 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 580 581 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 582 ceq_mask) == 0) { 583 /* the color is fixed - find the next page */ 584 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 585 while (npfn <= pfn) { 586 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, 587 mask); 588 } 589 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 590 goto done; 591 } 592 } 593 594 /* to fix the color need to touch PA[32:28] */ 595 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : 596 (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); 597 598 /* fix mnode if input pfn is in the wrong mnode. */ 599 if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) { 600 npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) << 601 it->mi_mnode_pfn_shift; 602 } 603 604 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; 605 tmpmask |= it->mi_mnode_pfn_mask; 606 607 while (npfn <= pfn) { 608 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); 609 } 610 611 /* set bits PA[19:13] to match the color */ 612 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 613 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 614 615 done: 616 ASSERT(((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); 617 ASSERT(PAPFN_2_MNODE(npfn) == it->mi_mnode); 618 619 /* PA to RA */ 620 npfn -= it->mi_ra_to_pa; 621 622 /* check for possible memblock switch */ 623 if (npfn > it->mi_mblock_end) { 624 pfn = plat_mem_node_iterator_init(npfn, it->mi_mnode, it, 0); 625 if (pfn == (pfn_t)-1) 626 return (pfn); 627 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 628 pfn += it->mi_ra_to_pa; 629 goto next_mem_block; 630 } 631 632 return (npfn); 633 } 634 635 /* 636 * init page coloring 637 * VF encodes node_id for an L-group in either bit 30 or 31:30, 638 * which effectively reduces the number of colors available per mnode. 639 */ 640 void 641 page_coloring_init_cpu() 642 { 643 int i; 644 uchar_t id; 645 uchar_t lo; 646 uchar_t hi; 647 n2color_t m; 648 mem_node_iterator_t it; 649 static uchar_t idmask[] = {0, 0x7, 0x1f, 0x1f, 0x1f, 0x1f}; 650 651 for (i = 0; i < max_mem_nodes; i++) { 652 memset(&it, 0, sizeof (it)); 653 if (plat_mem_node_iterator_init(0, i, &it, 1) != (pfn_t)-1) 654 break; 655 } 656 ASSERT(i < max_mem_nodes); 657 for (i = 0; i < mmu_page_sizes; i++) { 658 (void) memset(&m, 0, sizeof (m)); 659 id = it.mi_mnode_pfn_mask >> 15; /* node id mask */ 660 id &= idmask[i]; 661 lo = lowbit(id); 662 if (lo > 0) { 663 hi = highbit(id); 664 m.nnbits = hi - lo + 1; 665 m.nnmask = (1 << m.nnbits) - 1; 666 lo += nhbits[i] - 5; 667 m.lomask = (1 << (lo - 1)) - 1; 668 m.lobits = lo - 1; 669 } 670 hw_page_array[i].hp_colors = 1 << (nhbits[i] - m.nnbits); 671 n2color[i] = m; 672 } 673 } 674 675 /* 676 * group colorequiv colors on N2 by low order bits of the color first 677 */ 678 void 679 page_set_colorequiv_arr_cpu(void) 680 { 681 static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0}; 682 683 nequiv_shades_log2[1] -= n2color[1].nnbits; 684 if (colorequiv > 1) { 685 int i; 686 uint_t sv_a = lowbit(colorequiv) - 1; 687 688 if (sv_a > 15) 689 sv_a = 15; 690 691 for (i = 0; i < MMU_PAGE_SIZES; i++) { 692 uint_t colors; 693 uint_t a = sv_a; 694 695 if ((colors = hw_page_array[i].hp_colors) <= 1) 696 continue; 697 while ((colors >> a) == 0) 698 a--; 699 if (a > (colorequivszc[i] & 0xf) + 700 (colorequivszc[i] >> 4)) { 701 if (a <= nequiv_shades_log2[i]) { 702 colorequivszc[i] = (uchar_t)a; 703 } else { 704 colorequivszc[i] = 705 ((a - nequiv_shades_log2[i]) << 4) | 706 nequiv_shades_log2[i]; 707 } 708 } 709 } 710 } 711 } 712