144961713Sgirish /* 244961713Sgirish * CDDL HEADER START 344961713Sgirish * 444961713Sgirish * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 744961713Sgirish * 844961713Sgirish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 944961713Sgirish * or http://www.opensolaris.org/os/licensing. 1044961713Sgirish * See the License for the specific language governing permissions 1144961713Sgirish * and limitations under the License. 1244961713Sgirish * 1344961713Sgirish * When distributing Covered Code, include this CDDL HEADER in each 1444961713Sgirish * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1544961713Sgirish * If applicable, add the following below this CDDL HEADER, with the 1644961713Sgirish * fields enclosed by brackets "[]" replaced with your own identifying 1744961713Sgirish * information: Portions Copyright [yyyy] [name of copyright owner] 1844961713Sgirish * 1944961713Sgirish * CDDL HEADER END 2044961713Sgirish */ 21459190a5Srsmaeda 2244961713Sgirish /* 23575a7426Spt157919 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 2444961713Sgirish * Use is subject to license terms. 2544961713Sgirish */ 2644961713Sgirish 2744961713Sgirish #pragma ident "%Z%%M% %I% %E% SMI" 2844961713Sgirish 2944961713Sgirish #include <sys/types.h> 3044961713Sgirish #include <sys/systm.h> 3144961713Sgirish #include <sys/archsystm.h> 3244961713Sgirish #include <sys/machparam.h> 3344961713Sgirish #include <sys/machsystm.h> 3444961713Sgirish #include <sys/cpu.h> 3544961713Sgirish #include <sys/elf_SPARC.h> 3644961713Sgirish #include <vm/hat_sfmmu.h> 3744961713Sgirish #include <vm/page.h> 38102033aaSdp78419 #include <vm/vm_dep.h> 3944961713Sgirish #include <sys/cpuvar.h> 4044961713Sgirish #include <sys/async.h> 4144961713Sgirish #include <sys/cmn_err.h> 4244961713Sgirish #include <sys/debug.h> 4344961713Sgirish #include <sys/dditypes.h> 4444961713Sgirish #include <sys/sunddi.h> 4544961713Sgirish #include <sys/cpu_module.h> 4644961713Sgirish #include <sys/prom_debug.h> 4744961713Sgirish #include <sys/vmsystm.h> 4844961713Sgirish #include <sys/prom_plat.h> 4944961713Sgirish #include <sys/sysmacros.h> 5044961713Sgirish #include <sys/intreg.h> 5144961713Sgirish #include <sys/machtrap.h> 5244961713Sgirish #include <sys/ontrap.h> 5344961713Sgirish #include <sys/ivintr.h> 5444961713Sgirish #include <sys/atomic.h> 5544961713Sgirish #include <sys/panic.h> 5644961713Sgirish #include <sys/dtrace.h> 5744961713Sgirish #include <sys/simulate.h> 5844961713Sgirish #include <sys/fault.h> 5944961713Sgirish #include <sys/niagara2regs.h> 6044961713Sgirish #include <sys/hsvc.h> 6144961713Sgirish #include <sys/trapstat.h> 62575a7426Spt157919 #include <sys/mutex_impl.h> 6344961713Sgirish 6444961713Sgirish uint_t root_phys_addr_lo_mask = 0xffffffffU; 6559ac0c16Sdavemq #if defined(NIAGARA2_IMPL) 6644961713Sgirish char cpu_module_name[] = "SUNW,UltraSPARC-T2"; 6759ac0c16Sdavemq #elif defined(VFALLS_IMPL) 6859ac0c16Sdavemq char cpu_module_name[] = "SUNW,UltraSPARC-T2+"; 6959ac0c16Sdavemq #endif 7044961713Sgirish 7144961713Sgirish /* 7259ac0c16Sdavemq * Hypervisor services information for the NIAGARA2 and Victoria Falls 7359ac0c16Sdavemq * CPU module 7444961713Sgirish */ 7559ac0c16Sdavemq static boolean_t cpu_hsvc_available = B_TRUE; 7659ac0c16Sdavemq static uint64_t cpu_sup_minor; /* Supported minor number */ 7759ac0c16Sdavemq #if defined(NIAGARA2_IMPL) 7859ac0c16Sdavemq static hsvc_info_t cpu_hsvc = { 7944961713Sgirish HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, 8044961713Sgirish NIAGARA2_HSVC_MINOR, cpu_module_name 8144961713Sgirish }; 8259ac0c16Sdavemq #elif defined(VFALLS_IMPL) 8359ac0c16Sdavemq static hsvc_info_t cpu_hsvc = { 8459ac0c16Sdavemq HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR, 8559ac0c16Sdavemq VFALLS_HSVC_MINOR, cpu_module_name 8659ac0c16Sdavemq }; 8759ac0c16Sdavemq #endif 8844961713Sgirish 8944961713Sgirish void 9044961713Sgirish cpu_setup(void) 9144961713Sgirish { 9244961713Sgirish extern int mmu_exported_pagesize_mask; 9344961713Sgirish extern int cpc_has_overflow_intr; 947ec363dcSwh94709 extern size_t contig_mem_prealloc_base_size; 9544961713Sgirish int status; 9644961713Sgirish 9744961713Sgirish /* 9844961713Sgirish * Negotiate the API version for Niagara2 specific hypervisor 9944961713Sgirish * services. 10044961713Sgirish */ 10159ac0c16Sdavemq status = hsvc_register(&cpu_hsvc, &cpu_sup_minor); 10244961713Sgirish if (status != 0) { 10344961713Sgirish cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 10444961713Sgirish "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", 10559ac0c16Sdavemq cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group, 10659ac0c16Sdavemq cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status); 10759ac0c16Sdavemq cpu_hsvc_available = B_FALSE; 10844961713Sgirish } 10944961713Sgirish 11044961713Sgirish /* 11144961713Sgirish * The setup common to all CPU modules is done in cpu_setup_common 11244961713Sgirish * routine. 11344961713Sgirish */ 11444961713Sgirish cpu_setup_common(NULL); 11544961713Sgirish 11644961713Sgirish cache |= (CACHE_PTAG | CACHE_IOCOHERENT); 11744961713Sgirish 11844961713Sgirish if ((mmu_exported_pagesize_mask & 11944961713Sgirish DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != 12044961713Sgirish DEFAULT_SUN4V_MMU_PAGESIZE_MASK) 12144961713Sgirish cmn_err(CE_PANIC, "machine description" 12244961713Sgirish " does not have required sun4v page sizes" 12344961713Sgirish " 8K, 64K and 4M: MD mask is 0x%x", 12444961713Sgirish mmu_exported_pagesize_mask); 12544961713Sgirish 12644961713Sgirish cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT; 12744961713Sgirish 12844961713Sgirish /* 12944961713Sgirish * Niagara2 supports a 48-bit subset of the full 64-bit virtual 13044961713Sgirish * address space. Virtual addresses between 0x0000800000000000 13144961713Sgirish * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" 13244961713Sgirish * and must never be mapped. In addition, software must not use 13344961713Sgirish * pages within 4GB of the VA hole as instruction pages to 13444961713Sgirish * avoid problems with prefetching into the VA hole. 13544961713Sgirish */ 13644961713Sgirish hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); 13744961713Sgirish hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); 13844961713Sgirish 13944961713Sgirish /* 14044961713Sgirish * Niagara2 has a performance counter overflow interrupt 14144961713Sgirish */ 14244961713Sgirish cpc_has_overflow_intr = 1; 143102033aaSdp78419 144102033aaSdp78419 /* 145102033aaSdp78419 * Enable 4M pages for OOB. 146102033aaSdp78419 */ 147102033aaSdp78419 max_uheap_lpsize = MMU_PAGESIZE4M; 148102033aaSdp78419 max_ustack_lpsize = MMU_PAGESIZE4M; 149102033aaSdp78419 max_privmap_lpsize = MMU_PAGESIZE4M; 150aaa10e67Sha137994 1517ec363dcSwh94709 #ifdef SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB 1527ec363dcSwh94709 /* 1537ec363dcSwh94709 * Use CPU Makefile specific compile time define (if exists) 1547ec363dcSwh94709 * to add to the contig preallocation size. 1557ec363dcSwh94709 */ 1567ec363dcSwh94709 contig_mem_prealloc_base_size = MB(SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB); 1577ec363dcSwh94709 #endif 15844961713Sgirish } 15944961713Sgirish 16044961713Sgirish /* 16144961713Sgirish * Set the magic constants of the implementation. 16244961713Sgirish */ 16344961713Sgirish void 16444961713Sgirish cpu_fiximp(struct cpu_node *cpunode) 16544961713Sgirish { 16644961713Sgirish /* 16744961713Sgirish * The Cache node is optional in MD. Therefore in case "Cache" 16844961713Sgirish * node does not exists in MD, set the default L2 cache associativity, 16944961713Sgirish * size, linesize. 17044961713Sgirish */ 17144961713Sgirish if (cpunode->ecache_size == 0) 17244961713Sgirish cpunode->ecache_size = L2CACHE_SIZE; 17344961713Sgirish if (cpunode->ecache_linesize == 0) 17444961713Sgirish cpunode->ecache_linesize = L2CACHE_LINESIZE; 17544961713Sgirish if (cpunode->ecache_associativity == 0) 17644961713Sgirish cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; 17744961713Sgirish } 17844961713Sgirish 17944961713Sgirish void 180459190a5Srsmaeda cpu_map_exec_units(struct cpu *cp) 18144961713Sgirish { 182459190a5Srsmaeda ASSERT(MUTEX_HELD(&cpu_lock)); 18344961713Sgirish 18444961713Sgirish /* 185fb2f18f8Sesaxe * The cpu_ipipe and cpu_fpu fields are initialized based on 186459190a5Srsmaeda * the execution unit sharing information from the MD. They 187459190a5Srsmaeda * default to the CPU id in the absence of such information. 18844961713Sgirish */ 18944961713Sgirish cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; 19044961713Sgirish if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) 19144961713Sgirish cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); 19244961713Sgirish 193fb2f18f8Sesaxe cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping; 194fb2f18f8Sesaxe if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND) 195fb2f18f8Sesaxe cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id); 196fb2f18f8Sesaxe 197fb2f18f8Sesaxe /* 198fb2f18f8Sesaxe * Niagara 2 defines the core to be at the FPU level 199fb2f18f8Sesaxe */ 200fb2f18f8Sesaxe cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu; 20159ac0c16Sdavemq 20259ac0c16Sdavemq /* 20359ac0c16Sdavemq * The cpu_chip field is initialized based on the information 20459ac0c16Sdavemq * in the MD and assume that all cpus within a chip 20559ac0c16Sdavemq * share the same L2 cache. If no such info is available, we 20659ac0c16Sdavemq * set the cpu to belong to the defacto chip 0. 20759ac0c16Sdavemq */ 208ce8eb11aSdp78419 cp->cpu_m.cpu_mpipe = cpunodes[cp->cpu_id].l2_cache_mapping; 209ce8eb11aSdp78419 if (cp->cpu_m.cpu_mpipe == NO_L2_CACHE_MAPPING_FOUND) 210ce8eb11aSdp78419 cp->cpu_m.cpu_mpipe = CPU_L2_CACHEID_INVALID; 211e853d8c3Sjc25722 212e853d8c3Sjc25722 cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping; 213e853d8c3Sjc25722 if (cp->cpu_m.cpu_chip == NO_L2_CACHE_MAPPING_FOUND) 214e853d8c3Sjc25722 cp->cpu_m.cpu_chip = CPU_CHIPID_INVALID; 215459190a5Srsmaeda } 216459190a5Srsmaeda 21759ac0c16Sdavemq static int cpucnt; 218459190a5Srsmaeda 219459190a5Srsmaeda void 220459190a5Srsmaeda cpu_init_private(struct cpu *cp) 221459190a5Srsmaeda { 222459190a5Srsmaeda extern void niagara_kstat_init(void); 223fb2f18f8Sesaxe 22444961713Sgirish ASSERT(MUTEX_HELD(&cpu_lock)); 225459190a5Srsmaeda 226459190a5Srsmaeda cpu_map_exec_units(cp); 227459190a5Srsmaeda 22859ac0c16Sdavemq if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE)) 22959ac0c16Sdavemq (void) niagara_kstat_init(); 230575a7426Spt157919 231575a7426Spt157919 mutex_delay = rdccr_delay; 23244961713Sgirish } 23344961713Sgirish 23444961713Sgirish /*ARGSUSED*/ 23544961713Sgirish void 23644961713Sgirish cpu_uninit_private(struct cpu *cp) 23744961713Sgirish { 238459190a5Srsmaeda extern void niagara_kstat_fini(void); 23944961713Sgirish 24044961713Sgirish ASSERT(MUTEX_HELD(&cpu_lock)); 24159ac0c16Sdavemq if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE)) 24259ac0c16Sdavemq (void) niagara_kstat_fini(); 24344961713Sgirish } 24444961713Sgirish 24544961713Sgirish /* 24644961713Sgirish * On Niagara2, any flush will cause all preceding stores to be 24744961713Sgirish * synchronized wrt the i$, regardless of address or ASI. In fact, 24844961713Sgirish * the address is ignored, so we always flush address 0. 24944961713Sgirish */ 25044961713Sgirish /*ARGSUSED*/ 25144961713Sgirish void 25244961713Sgirish dtrace_flush_sec(uintptr_t addr) 25344961713Sgirish { 25444961713Sgirish doflush(0); 25544961713Sgirish } 25644961713Sgirish 25744961713Sgirish /* 25844961713Sgirish * Trapstat support for Niagara2 processor 25944961713Sgirish * The Niagara2 provides HWTW support for TSB lookup and with HWTW 26044961713Sgirish * enabled no TSB hit information will be available. Therefore setting 26144961713Sgirish * the time spent in TLB miss handler for TSB hits to 0. 26244961713Sgirish */ 26344961713Sgirish int 26444961713Sgirish cpu_trapstat_conf(int cmd) 26544961713Sgirish { 26644961713Sgirish int status = 0; 26744961713Sgirish 26844961713Sgirish switch (cmd) { 26944961713Sgirish case CPU_TSTATCONF_INIT: 27044961713Sgirish case CPU_TSTATCONF_FINI: 27144961713Sgirish case CPU_TSTATCONF_ENABLE: 27244961713Sgirish case CPU_TSTATCONF_DISABLE: 27344961713Sgirish break; 27444961713Sgirish default: 27544961713Sgirish status = EINVAL; 27644961713Sgirish break; 27744961713Sgirish } 27844961713Sgirish return (status); 27944961713Sgirish } 28044961713Sgirish 28144961713Sgirish void 28244961713Sgirish cpu_trapstat_data(void *buf, uint_t tstat_pgszs) 28344961713Sgirish { 28444961713Sgirish tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; 28544961713Sgirish int i; 28644961713Sgirish 28744961713Sgirish for (i = 0; i < tstat_pgszs; i++, tstatp++) { 28844961713Sgirish tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; 28944961713Sgirish tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; 29044961713Sgirish tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; 29144961713Sgirish tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; 29244961713Sgirish tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 29344961713Sgirish tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 29444961713Sgirish tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 29544961713Sgirish tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 29644961713Sgirish } 29744961713Sgirish } 298102033aaSdp78419 299ce8eb11aSdp78419 /* 300ce8eb11aSdp78419 * Page coloring support for hashed cache index mode 301ce8eb11aSdp78419 */ 302ce8eb11aSdp78419 303ce8eb11aSdp78419 /* 304ce8eb11aSdp78419 * Node id bits from machine description (MD). Node id distinguishes 305ce8eb11aSdp78419 * local versus remote memory. Because of MPO, page allocation does 306ce8eb11aSdp78419 * not cross node boundaries. Therefore, remove the node id bits from 307ce8eb11aSdp78419 * the color, since they are fixed. Either bit 30, or 31:30 in 308ce8eb11aSdp78419 * Victoria Falls processors. 309ce8eb11aSdp78419 * The number of node id bits is always 0 in Niagara2. 310ce8eb11aSdp78419 */ 311ce8eb11aSdp78419 typedef struct n2color { 312ce8eb11aSdp78419 uchar_t nnbits; /* number of node id bits */ 313ce8eb11aSdp78419 uchar_t nnmask; /* mask for node id bits */ 314ce8eb11aSdp78419 uchar_t lomask; /* mask for bits below node id */ 315ce8eb11aSdp78419 uchar_t lobits; /* number of bits below node id */ 316ce8eb11aSdp78419 } n2color_t; 317ce8eb11aSdp78419 318ce8eb11aSdp78419 n2color_t n2color[MMU_PAGE_SIZES]; 319ce8eb11aSdp78419 static uchar_t nhbits[] = {7, 7, 6, 5, 5, 5}; 320ce8eb11aSdp78419 321ce8eb11aSdp78419 /* 322ce8eb11aSdp78419 * Remove node id bits from color bits 32:28. 323ce8eb11aSdp78419 * This will reduce the number of colors. 324ce8eb11aSdp78419 * No change if number of node bits is zero. 325ce8eb11aSdp78419 */ 326ce8eb11aSdp78419 static inline uint_t 327ce8eb11aSdp78419 n2_hash2color(uint_t color, uchar_t szc) 328ce8eb11aSdp78419 { 329ce8eb11aSdp78419 n2color_t m = n2color[szc]; 330ce8eb11aSdp78419 331ce8eb11aSdp78419 if (m.nnbits > 0) { 332ce8eb11aSdp78419 color = ((color >> m.nnbits) & ~m.lomask) | (color & m.lomask); 333ce8eb11aSdp78419 ASSERT((color & ~(hw_page_array[szc].hp_colors - 1)) == 0); 334ce8eb11aSdp78419 } 335ce8eb11aSdp78419 336ce8eb11aSdp78419 return (color); 337ce8eb11aSdp78419 } 338ce8eb11aSdp78419 339ce8eb11aSdp78419 /* 340ce8eb11aSdp78419 * Restore node id bits into page color. 341ce8eb11aSdp78419 * This will increase the number of colors to match N2. 342ce8eb11aSdp78419 * No change if number of node bits is zero. 343ce8eb11aSdp78419 */ 344ce8eb11aSdp78419 static inline uint_t 345ce8eb11aSdp78419 n2_color2hash(uint_t color, uchar_t szc, uint_t node) 346ce8eb11aSdp78419 { 347ce8eb11aSdp78419 n2color_t m = n2color[szc]; 348ce8eb11aSdp78419 349ce8eb11aSdp78419 if (m.nnbits > 0) { 350ce8eb11aSdp78419 color = ((color & ~m.lomask) << m.nnbits) | (color & m.lomask); 351ce8eb11aSdp78419 color |= (node & m.nnmask) << m.lobits; 352ce8eb11aSdp78419 } 353ce8eb11aSdp78419 354ce8eb11aSdp78419 return (color); 355ce8eb11aSdp78419 } 356ce8eb11aSdp78419 357102033aaSdp78419 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ 358ce8eb11aSdp78419 359ce8eb11aSdp78419 /* 360ce8eb11aSdp78419 * iterator NULL means pfn is VA, do not adjust ra_to_pa 361ce8eb11aSdp78419 * iterator (-1) means pfn is RA, need to convert to PA 362ce8eb11aSdp78419 * iterator non-null means pfn is RA, use ra_to_pa 363ce8eb11aSdp78419 */ 364102033aaSdp78419 uint_t 365ce8eb11aSdp78419 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc, void *cookie) 366ce8eb11aSdp78419 { 367ce8eb11aSdp78419 mem_node_iterator_t *it = cookie; 368ce8eb11aSdp78419 uint_t color; 369ce8eb11aSdp78419 370ce8eb11aSdp78419 ASSERT(szc <= TTE256M); 371ce8eb11aSdp78419 372ce8eb11aSdp78419 if (it == ((mem_node_iterator_t *)(-1))) { 373ce8eb11aSdp78419 pfn = plat_rapfn_to_papfn(pfn); 374ce8eb11aSdp78419 } else if (it != NULL) { 375ce8eb11aSdp78419 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 376ce8eb11aSdp78419 pfn = pfn + it->mi_ra_to_pa; 377ce8eb11aSdp78419 } 378ce8eb11aSdp78419 pfn = PFN_BASE(pfn, szc); 379ce8eb11aSdp78419 color = ((pfn >> 15) ^ pfn) & 0x1f; 380ce8eb11aSdp78419 if (szc < TTE4M) { 381ce8eb11aSdp78419 /* 19:18 */ 382ce8eb11aSdp78419 color = (color << 2) | ((pfn >> 5) & 0x3); 383ce8eb11aSdp78419 if (szc > TTE64K) 384ce8eb11aSdp78419 color >>= 1; /* 19 */ 385ce8eb11aSdp78419 } 386ce8eb11aSdp78419 return (n2_hash2color(color, szc)); 387ce8eb11aSdp78419 } 388ce8eb11aSdp78419 389ce8eb11aSdp78419 static uint_t 390ce8eb11aSdp78419 page_papfn_2_color_cpu(pfn_t papfn, uchar_t szc) 391102033aaSdp78419 { 392102033aaSdp78419 uint_t color; 393102033aaSdp78419 394102033aaSdp78419 ASSERT(szc <= TTE256M); 395102033aaSdp78419 396ce8eb11aSdp78419 papfn = PFN_BASE(papfn, szc); 397ce8eb11aSdp78419 color = ((papfn >> 15) ^ papfn) & 0x1f; 398ce8eb11aSdp78419 if (szc < TTE4M) { 399ce8eb11aSdp78419 /* 19:18 */ 400ce8eb11aSdp78419 color = (color << 2) | ((papfn >> 5) & 0x3); 401ce8eb11aSdp78419 if (szc > TTE64K) 402ce8eb11aSdp78419 color >>= 1; /* 19 */ 403ce8eb11aSdp78419 } 404102033aaSdp78419 return (color); 405102033aaSdp78419 } 406102033aaSdp78419 407102033aaSdp78419 #if TTE256M != 5 408102033aaSdp78419 #error TTE256M is not 5 409102033aaSdp78419 #endif 410102033aaSdp78419 411102033aaSdp78419 uint_t 412102033aaSdp78419 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) 413102033aaSdp78419 { 414102033aaSdp78419 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; 415102033aaSdp78419 ASSERT(szc < TTE256M); 416ce8eb11aSdp78419 mask = n2_color2hash(mask, szc, 0); 417102033aaSdp78419 mask &= ni2_color_masks[szc]; 418ce8eb11aSdp78419 if (szc == TTE64K || szc == TTE512K) 419ce8eb11aSdp78419 mask >>= 1; 420ce8eb11aSdp78419 return (n2_hash2color(mask, szc + 1)); 421102033aaSdp78419 } 422102033aaSdp78419 423102033aaSdp78419 uint_t 424102033aaSdp78419 page_get_nsz_color_cpu(uchar_t szc, uint_t color) 425102033aaSdp78419 { 426102033aaSdp78419 ASSERT(szc < TTE256M); 427ce8eb11aSdp78419 color = n2_color2hash(color, szc, 0); 428ce8eb11aSdp78419 if (szc == TTE64K || szc == TTE512K) 429ce8eb11aSdp78419 color >>= 1; 430ce8eb11aSdp78419 return (n2_hash2color(color, szc + 1)); 431102033aaSdp78419 } 432102033aaSdp78419 433102033aaSdp78419 uint_t 434102033aaSdp78419 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) 435102033aaSdp78419 { 436ce8eb11aSdp78419 uint_t s; 437ef29e907Ssusans ASSERT(nszc >= szc); 438102033aaSdp78419 ASSERT(nszc <= TTE256M); 439102033aaSdp78419 440ce8eb11aSdp78419 s = nhbits[szc] - n2color[szc].nnbits; 441ce8eb11aSdp78419 s -= nhbits[nszc] - n2color[nszc].nnbits; 442102033aaSdp78419 443ce8eb11aSdp78419 return (s); 444102033aaSdp78419 } 445102033aaSdp78419 446ce8eb11aSdp78419 uint_t 447ce8eb11aSdp78419 page_convert_color_cpu(uint_t ncolor, uchar_t szc, uchar_t nszc) 448ce8eb11aSdp78419 { 449ce8eb11aSdp78419 uint_t color; 450ce8eb11aSdp78419 451ce8eb11aSdp78419 ASSERT(nszc > szc); 452ce8eb11aSdp78419 ASSERT(nszc <= TTE256M); 453ce8eb11aSdp78419 ncolor = n2_color2hash(ncolor, nszc, 0); 454ce8eb11aSdp78419 color = ncolor << (nhbits[szc] - nhbits[nszc]); 455ce8eb11aSdp78419 color = n2_hash2color(color, szc); 456ce8eb11aSdp78419 return (color); 457ce8eb11aSdp78419 } 458ce8eb11aSdp78419 459ce8eb11aSdp78419 #define PAPFN_2_MNODE(pfn) \ 460ce8eb11aSdp78419 (((pfn) & it->mi_mnode_pfn_mask) >> it->mi_mnode_pfn_shift) 461ce8eb11aSdp78419 462102033aaSdp78419 /*ARGSUSED*/ 463102033aaSdp78419 pfn_t 464102033aaSdp78419 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, 465ce8eb11aSdp78419 uint_t ceq_mask, uint_t color_mask, void *cookie) 466102033aaSdp78419 { 467ce8eb11aSdp78419 mem_node_iterator_t *it = cookie; 468102033aaSdp78419 pfn_t pstep = PNUM_SIZE(szc); 469102033aaSdp78419 pfn_t npfn, pfn_ceq_mask, pfn_color; 470102033aaSdp78419 pfn_t tmpmask, mask = (pfn_t)-1; 471ce8eb11aSdp78419 uint_t pfnmn; 472102033aaSdp78419 473102033aaSdp78419 ASSERT((color & ~ceq_mask) == 0); 474ce8eb11aSdp78419 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 475102033aaSdp78419 476ce8eb11aSdp78419 /* convert RA to PA for accurate color calculation */ 477ce8eb11aSdp78419 if (it->mi_init) { 478ce8eb11aSdp78419 /* first call after it, so cache these values */ 479ce8eb11aSdp78419 it->mi_hash_ceq_mask = 480ce8eb11aSdp78419 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask); 481ce8eb11aSdp78419 it->mi_hash_color = 482ce8eb11aSdp78419 n2_color2hash(color, szc, it->mi_mnode); 483ce8eb11aSdp78419 it->mi_init = 0; 484ce8eb11aSdp78419 } else { 485ce8eb11aSdp78419 ASSERT(it->mi_hash_ceq_mask == 486ce8eb11aSdp78419 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask)); 487ce8eb11aSdp78419 ASSERT(it->mi_hash_color == 488ce8eb11aSdp78419 n2_color2hash(color, szc, it->mi_mnode)); 489ce8eb11aSdp78419 } 490ce8eb11aSdp78419 ceq_mask = it->mi_hash_ceq_mask; 491ce8eb11aSdp78419 color = it->mi_hash_color; 492ce8eb11aSdp78419 pfn += it->mi_ra_to_pa; 493ce8eb11aSdp78419 494ce8eb11aSdp78419 /* restart here when we switch memblocks */ 495ce8eb11aSdp78419 next_mem_block: 496ce8eb11aSdp78419 if (szc <= TTE64K) { 497ce8eb11aSdp78419 pfnmn = PAPFN_2_MNODE(pfn); 498ce8eb11aSdp78419 } 499ce8eb11aSdp78419 if (((page_papfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0 && 500ce8eb11aSdp78419 (szc > TTE64K || pfnmn == it->mi_mnode)) { 501102033aaSdp78419 502102033aaSdp78419 /* we start from the page with correct color */ 503102033aaSdp78419 if (szc >= TTE512K) { 504102033aaSdp78419 if (szc >= TTE4M) { 505102033aaSdp78419 /* page color is PA[32:28] */ 506102033aaSdp78419 pfn_ceq_mask = ceq_mask << 15; 507102033aaSdp78419 } else { 508102033aaSdp78419 /* page color is PA[32:28].PA[19:19] */ 509102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 510102033aaSdp78419 ((ceq_mask >> 1) << 15); 511102033aaSdp78419 } 512ce8eb11aSdp78419 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 513ce8eb11aSdp78419 goto done; 514102033aaSdp78419 } else { 515102033aaSdp78419 /* 516102033aaSdp78419 * We deal 64K or 8K page. Check if we could the 517102033aaSdp78419 * satisfy the request without changing PA[32:28] 518102033aaSdp78419 */ 519102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 520ce8eb11aSdp78419 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 521102033aaSdp78419 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 522102033aaSdp78419 523102033aaSdp78419 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 524ce8eb11aSdp78419 goto done; 525102033aaSdp78419 526102033aaSdp78419 /* 527102033aaSdp78419 * for next pfn we have to change bits PA[32:28] 528102033aaSdp78419 * set PA[63:28] and PA[19:18] of the next pfn 529102033aaSdp78419 */ 530102033aaSdp78419 npfn = (pfn >> 15) << 15; 531102033aaSdp78419 npfn |= (ceq_mask & color & 3) << 5; 532102033aaSdp78419 pfn_ceq_mask = (szc == TTE8K) ? 0 : 533102033aaSdp78419 (ceq_mask & 0x1c) << 13; 534ce8eb11aSdp78419 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 535102033aaSdp78419 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); 536102033aaSdp78419 537102033aaSdp78419 /* 538102033aaSdp78419 * set bits PA[17:13] to match the color 539102033aaSdp78419 */ 540ce8eb11aSdp78419 npfn |= ((npfn >> 15) ^ (color >> 2)) & (ceq_mask >> 2); 541ce8eb11aSdp78419 goto done; 542102033aaSdp78419 } 543102033aaSdp78419 } 544102033aaSdp78419 545102033aaSdp78419 /* 546102033aaSdp78419 * we start from the page with incorrect color - rare case 547102033aaSdp78419 */ 548102033aaSdp78419 if (szc >= TTE512K) { 549102033aaSdp78419 if (szc >= TTE4M) { 550102033aaSdp78419 /* page color is in bits PA[32:28] */ 551102033aaSdp78419 npfn = ((pfn >> 20) << 20) | (color << 15); 552102033aaSdp78419 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; 553102033aaSdp78419 } else { 554102033aaSdp78419 /* try get the right color by changing bit PA[19:19] */ 555102033aaSdp78419 npfn = pfn + pstep; 556ce8eb11aSdp78419 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 557102033aaSdp78419 ceq_mask) == 0) 558ce8eb11aSdp78419 goto done; 559102033aaSdp78419 560102033aaSdp78419 /* page color is PA[32:28].PA[19:19] */ 561102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 562102033aaSdp78419 ((ceq_mask >> 1) << 15) | (0xff << 7); 563102033aaSdp78419 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); 564102033aaSdp78419 npfn = ((pfn >> 20) << 20) | pfn_color; 565102033aaSdp78419 } 566102033aaSdp78419 567102033aaSdp78419 while (npfn <= pfn) { 568102033aaSdp78419 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 569102033aaSdp78419 } 570ce8eb11aSdp78419 goto done; 571102033aaSdp78419 } 572102033aaSdp78419 573102033aaSdp78419 /* 574102033aaSdp78419 * We deal 64K or 8K page of incorrect color. 575102033aaSdp78419 * Try correcting color without changing PA[32:28] 576102033aaSdp78419 */ 577102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 578102033aaSdp78419 pfn_color = ((color & 3) << 5) | (color >> 2); 579ce8eb11aSdp78419 if (pfnmn == it->mi_mnode) { 580102033aaSdp78419 npfn = (pfn & ~(pfn_t)0x7f); 581102033aaSdp78419 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 582102033aaSdp78419 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 583102033aaSdp78419 584ce8eb11aSdp78419 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 585ce8eb11aSdp78419 ceq_mask) == 0) { 586102033aaSdp78419 /* the color is fixed - find the next page */ 587ce8eb11aSdp78419 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 588102033aaSdp78419 while (npfn <= pfn) { 589ce8eb11aSdp78419 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, 590ce8eb11aSdp78419 mask); 591102033aaSdp78419 } 592102033aaSdp78419 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 593ce8eb11aSdp78419 goto done; 594ce8eb11aSdp78419 } 595102033aaSdp78419 } 596102033aaSdp78419 597102033aaSdp78419 /* to fix the color need to touch PA[32:28] */ 598102033aaSdp78419 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : 599102033aaSdp78419 (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); 600ce8eb11aSdp78419 601ce8eb11aSdp78419 /* fix mnode if input pfn is in the wrong mnode. */ 602ce8eb11aSdp78419 if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) { 603ce8eb11aSdp78419 npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) << 604ce8eb11aSdp78419 it->mi_mnode_pfn_shift; 605ce8eb11aSdp78419 } 606ce8eb11aSdp78419 607102033aaSdp78419 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; 608ce8eb11aSdp78419 tmpmask |= it->mi_mnode_pfn_mask; 609102033aaSdp78419 610102033aaSdp78419 while (npfn <= pfn) { 611102033aaSdp78419 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); 612102033aaSdp78419 } 613102033aaSdp78419 614102033aaSdp78419 /* set bits PA[19:13] to match the color */ 615102033aaSdp78419 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 616102033aaSdp78419 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 617102033aaSdp78419 618ce8eb11aSdp78419 done: 619ce8eb11aSdp78419 ASSERT(((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); 620ce8eb11aSdp78419 ASSERT(PAPFN_2_MNODE(npfn) == it->mi_mnode); 621ce8eb11aSdp78419 622ce8eb11aSdp78419 /* PA to RA */ 623ce8eb11aSdp78419 npfn -= it->mi_ra_to_pa; 624ce8eb11aSdp78419 625ce8eb11aSdp78419 /* check for possible memblock switch */ 626ce8eb11aSdp78419 if (npfn > it->mi_mblock_end) { 627*b779d3e0Sdp78419 pfn = plat_mem_node_iterator_init(npfn, it->mi_mnode, szc, it, 628*b779d3e0Sdp78419 0); 629ce8eb11aSdp78419 if (pfn == (pfn_t)-1) 630ce8eb11aSdp78419 return (pfn); 631ce8eb11aSdp78419 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 632ce8eb11aSdp78419 pfn += it->mi_ra_to_pa; 633ce8eb11aSdp78419 goto next_mem_block; 634ce8eb11aSdp78419 } 635102033aaSdp78419 636102033aaSdp78419 return (npfn); 637102033aaSdp78419 } 638102033aaSdp78419 639102033aaSdp78419 /* 640102033aaSdp78419 * init page coloring 641ce8eb11aSdp78419 * VF encodes node_id for an L-group in either bit 30 or 31:30, 642ce8eb11aSdp78419 * which effectively reduces the number of colors available per mnode. 643102033aaSdp78419 */ 644102033aaSdp78419 void 645102033aaSdp78419 page_coloring_init_cpu() 646102033aaSdp78419 { 647102033aaSdp78419 int i; 648ce8eb11aSdp78419 uchar_t id; 649ce8eb11aSdp78419 uchar_t lo; 650ce8eb11aSdp78419 uchar_t hi; 651ce8eb11aSdp78419 n2color_t m; 652ce8eb11aSdp78419 mem_node_iterator_t it; 653ce8eb11aSdp78419 static uchar_t idmask[] = {0, 0x7, 0x1f, 0x1f, 0x1f, 0x1f}; 654102033aaSdp78419 655284953f5Sdp78419 for (i = 0; i < max_mem_nodes; i++) { 656284953f5Sdp78419 memset(&it, 0, sizeof (it)); 657*b779d3e0Sdp78419 if (plat_mem_node_iterator_init(0, i, 0, &it, 1) != (pfn_t)-1) 658284953f5Sdp78419 break; 659284953f5Sdp78419 } 660284953f5Sdp78419 ASSERT(i < max_mem_nodes); 661ce8eb11aSdp78419 for (i = 0; i < mmu_page_sizes; i++) { 662b02e9a2dSsvemuri (void) memset(&m, 0, sizeof (m)); 663ce8eb11aSdp78419 id = it.mi_mnode_pfn_mask >> 15; /* node id mask */ 664ce8eb11aSdp78419 id &= idmask[i]; 665ce8eb11aSdp78419 lo = lowbit(id); 666ce8eb11aSdp78419 if (lo > 0) { 667ce8eb11aSdp78419 hi = highbit(id); 668ce8eb11aSdp78419 m.nnbits = hi - lo + 1; 669ce8eb11aSdp78419 m.nnmask = (1 << m.nnbits) - 1; 670ce8eb11aSdp78419 lo += nhbits[i] - 5; 671ce8eb11aSdp78419 m.lomask = (1 << (lo - 1)) - 1; 672ce8eb11aSdp78419 m.lobits = lo - 1; 673ce8eb11aSdp78419 } 674ce8eb11aSdp78419 hw_page_array[i].hp_colors = 1 << (nhbits[i] - m.nnbits); 675ce8eb11aSdp78419 n2color[i] = m; 676102033aaSdp78419 } 677102033aaSdp78419 } 678fe70c9cfSdp78419 679fe70c9cfSdp78419 /* 680fe70c9cfSdp78419 * group colorequiv colors on N2 by low order bits of the color first 681fe70c9cfSdp78419 */ 682fe70c9cfSdp78419 void 683fe70c9cfSdp78419 page_set_colorequiv_arr_cpu(void) 684fe70c9cfSdp78419 { 685fe70c9cfSdp78419 static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0}; 686fe70c9cfSdp78419 687ce8eb11aSdp78419 nequiv_shades_log2[1] -= n2color[1].nnbits; 688fe70c9cfSdp78419 if (colorequiv > 1) { 689fe70c9cfSdp78419 int i; 690fe70c9cfSdp78419 uint_t sv_a = lowbit(colorequiv) - 1; 691fe70c9cfSdp78419 692fe70c9cfSdp78419 if (sv_a > 15) 693fe70c9cfSdp78419 sv_a = 15; 694fe70c9cfSdp78419 695fe70c9cfSdp78419 for (i = 0; i < MMU_PAGE_SIZES; i++) { 696fe70c9cfSdp78419 uint_t colors; 697fe70c9cfSdp78419 uint_t a = sv_a; 698fe70c9cfSdp78419 699fe70c9cfSdp78419 if ((colors = hw_page_array[i].hp_colors) <= 1) 700fe70c9cfSdp78419 continue; 701fe70c9cfSdp78419 while ((colors >> a) == 0) 702fe70c9cfSdp78419 a--; 703fe70c9cfSdp78419 if (a > (colorequivszc[i] & 0xf) + 704fe70c9cfSdp78419 (colorequivszc[i] >> 4)) { 705fe70c9cfSdp78419 if (a <= nequiv_shades_log2[i]) { 70659ac0c16Sdavemq colorequivszc[i] = (uchar_t)a; 707fe70c9cfSdp78419 } else { 708fe70c9cfSdp78419 colorequivszc[i] = 709fe70c9cfSdp78419 ((a - nequiv_shades_log2[i]) << 4) | 710fe70c9cfSdp78419 nequiv_shades_log2[i]; 711fe70c9cfSdp78419 } 712fe70c9cfSdp78419 } 713fe70c9cfSdp78419 } 714fe70c9cfSdp78419 } 715fe70c9cfSdp78419 } 716