144961713Sgirish /* 244961713Sgirish * CDDL HEADER START 344961713Sgirish * 444961713Sgirish * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 744961713Sgirish * 844961713Sgirish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 944961713Sgirish * or http://www.opensolaris.org/os/licensing. 1044961713Sgirish * See the License for the specific language governing permissions 1144961713Sgirish * and limitations under the License. 1244961713Sgirish * 1344961713Sgirish * When distributing Covered Code, include this CDDL HEADER in each 1444961713Sgirish * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1544961713Sgirish * If applicable, add the following below this CDDL HEADER, with the 1644961713Sgirish * fields enclosed by brackets "[]" replaced with your own identifying 1744961713Sgirish * information: Portions Copyright [yyyy] [name of copyright owner] 1844961713Sgirish * 1944961713Sgirish * CDDL HEADER END 2044961713Sgirish */ 21459190a5Srsmaeda 2244961713Sgirish /* 23fb2f18f8Sesaxe * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 2444961713Sgirish * Use is subject to license terms. 2544961713Sgirish */ 2644961713Sgirish 2744961713Sgirish #pragma ident "%Z%%M% %I% %E% SMI" 2844961713Sgirish 2944961713Sgirish #include <sys/types.h> 3044961713Sgirish #include <sys/systm.h> 3144961713Sgirish #include <sys/archsystm.h> 3244961713Sgirish #include <sys/machparam.h> 3344961713Sgirish #include <sys/machsystm.h> 3444961713Sgirish #include <sys/cpu.h> 3544961713Sgirish #include <sys/elf_SPARC.h> 3644961713Sgirish #include <vm/hat_sfmmu.h> 3744961713Sgirish #include <vm/page.h> 38102033aaSdp78419 #include <vm/vm_dep.h> 3944961713Sgirish #include <sys/cpuvar.h> 4044961713Sgirish #include <sys/async.h> 4144961713Sgirish #include <sys/cmn_err.h> 4244961713Sgirish #include <sys/debug.h> 4344961713Sgirish #include <sys/dditypes.h> 4444961713Sgirish #include <sys/sunddi.h> 4544961713Sgirish #include <sys/cpu_module.h> 4644961713Sgirish #include <sys/prom_debug.h> 4744961713Sgirish #include <sys/vmsystm.h> 4844961713Sgirish #include <sys/prom_plat.h> 4944961713Sgirish #include <sys/sysmacros.h> 5044961713Sgirish #include <sys/intreg.h> 5144961713Sgirish #include <sys/machtrap.h> 5244961713Sgirish #include <sys/ontrap.h> 5344961713Sgirish #include <sys/ivintr.h> 5444961713Sgirish #include <sys/atomic.h> 5544961713Sgirish #include <sys/panic.h> 5644961713Sgirish #include <sys/dtrace.h> 5744961713Sgirish #include <sys/simulate.h> 5844961713Sgirish #include <sys/fault.h> 5944961713Sgirish #include <sys/niagara2regs.h> 6044961713Sgirish #include <sys/hsvc.h> 6144961713Sgirish #include <sys/trapstat.h> 6244961713Sgirish 6344961713Sgirish uint_t root_phys_addr_lo_mask = 0xffffffffU; 6459ac0c16Sdavemq #if defined(NIAGARA2_IMPL) 6544961713Sgirish char cpu_module_name[] = "SUNW,UltraSPARC-T2"; 6659ac0c16Sdavemq #elif defined(VFALLS_IMPL) 6759ac0c16Sdavemq char cpu_module_name[] = "SUNW,UltraSPARC-T2+"; 6859ac0c16Sdavemq #endif 6944961713Sgirish 7044961713Sgirish /* 7159ac0c16Sdavemq * Hypervisor services information for the NIAGARA2 and Victoria Falls 7259ac0c16Sdavemq * CPU module 7344961713Sgirish */ 7459ac0c16Sdavemq static boolean_t cpu_hsvc_available = B_TRUE; 7559ac0c16Sdavemq static uint64_t cpu_sup_minor; /* Supported minor number */ 7659ac0c16Sdavemq #if defined(NIAGARA2_IMPL) 7759ac0c16Sdavemq static hsvc_info_t cpu_hsvc = { 7844961713Sgirish HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, 7944961713Sgirish NIAGARA2_HSVC_MINOR, cpu_module_name 8044961713Sgirish }; 8159ac0c16Sdavemq #elif defined(VFALLS_IMPL) 8259ac0c16Sdavemq static hsvc_info_t cpu_hsvc = { 8359ac0c16Sdavemq HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR, 8459ac0c16Sdavemq VFALLS_HSVC_MINOR, cpu_module_name 8559ac0c16Sdavemq }; 8659ac0c16Sdavemq #endif 8744961713Sgirish 8844961713Sgirish void 8944961713Sgirish cpu_setup(void) 9044961713Sgirish { 9144961713Sgirish extern int mmu_exported_pagesize_mask; 9244961713Sgirish extern int cpc_has_overflow_intr; 93*7ec363dcSwh94709 extern size_t contig_mem_prealloc_base_size; 9444961713Sgirish int status; 9544961713Sgirish 9644961713Sgirish /* 9744961713Sgirish * Negotiate the API version for Niagara2 specific hypervisor 9844961713Sgirish * services. 9944961713Sgirish */ 10059ac0c16Sdavemq status = hsvc_register(&cpu_hsvc, &cpu_sup_minor); 10144961713Sgirish if (status != 0) { 10244961713Sgirish cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 10344961713Sgirish "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", 10459ac0c16Sdavemq cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group, 10559ac0c16Sdavemq cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status); 10659ac0c16Sdavemq cpu_hsvc_available = B_FALSE; 10744961713Sgirish } 10844961713Sgirish 10944961713Sgirish /* 11044961713Sgirish * The setup common to all CPU modules is done in cpu_setup_common 11144961713Sgirish * routine. 11244961713Sgirish */ 11344961713Sgirish cpu_setup_common(NULL); 11444961713Sgirish 11544961713Sgirish cache |= (CACHE_PTAG | CACHE_IOCOHERENT); 11644961713Sgirish 11744961713Sgirish if ((mmu_exported_pagesize_mask & 11844961713Sgirish DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != 11944961713Sgirish DEFAULT_SUN4V_MMU_PAGESIZE_MASK) 12044961713Sgirish cmn_err(CE_PANIC, "machine description" 12144961713Sgirish " does not have required sun4v page sizes" 12244961713Sgirish " 8K, 64K and 4M: MD mask is 0x%x", 12344961713Sgirish mmu_exported_pagesize_mask); 12444961713Sgirish 12544961713Sgirish cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT; 12644961713Sgirish 12744961713Sgirish /* 12844961713Sgirish * Niagara2 supports a 48-bit subset of the full 64-bit virtual 12944961713Sgirish * address space. Virtual addresses between 0x0000800000000000 13044961713Sgirish * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" 13144961713Sgirish * and must never be mapped. In addition, software must not use 13244961713Sgirish * pages within 4GB of the VA hole as instruction pages to 13344961713Sgirish * avoid problems with prefetching into the VA hole. 13444961713Sgirish */ 13544961713Sgirish hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); 13644961713Sgirish hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); 13744961713Sgirish 13844961713Sgirish /* 13944961713Sgirish * Niagara2 has a performance counter overflow interrupt 14044961713Sgirish */ 14144961713Sgirish cpc_has_overflow_intr = 1; 142102033aaSdp78419 143102033aaSdp78419 /* 144102033aaSdp78419 * Enable 4M pages for OOB. 145102033aaSdp78419 */ 146102033aaSdp78419 max_uheap_lpsize = MMU_PAGESIZE4M; 147102033aaSdp78419 max_ustack_lpsize = MMU_PAGESIZE4M; 148102033aaSdp78419 max_privmap_lpsize = MMU_PAGESIZE4M; 149aaa10e67Sha137994 150*7ec363dcSwh94709 #ifdef SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB 151*7ec363dcSwh94709 /* 152*7ec363dcSwh94709 * Use CPU Makefile specific compile time define (if exists) 153*7ec363dcSwh94709 * to add to the contig preallocation size. 154*7ec363dcSwh94709 */ 155*7ec363dcSwh94709 contig_mem_prealloc_base_size = MB(SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB); 156*7ec363dcSwh94709 #endif 15744961713Sgirish } 15844961713Sgirish 15944961713Sgirish /* 16044961713Sgirish * Set the magic constants of the implementation. 16144961713Sgirish */ 16244961713Sgirish void 16344961713Sgirish cpu_fiximp(struct cpu_node *cpunode) 16444961713Sgirish { 16544961713Sgirish /* 16644961713Sgirish * The Cache node is optional in MD. Therefore in case "Cache" 16744961713Sgirish * node does not exists in MD, set the default L2 cache associativity, 16844961713Sgirish * size, linesize. 16944961713Sgirish */ 17044961713Sgirish if (cpunode->ecache_size == 0) 17144961713Sgirish cpunode->ecache_size = L2CACHE_SIZE; 17244961713Sgirish if (cpunode->ecache_linesize == 0) 17344961713Sgirish cpunode->ecache_linesize = L2CACHE_LINESIZE; 17444961713Sgirish if (cpunode->ecache_associativity == 0) 17544961713Sgirish cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; 17644961713Sgirish } 17744961713Sgirish 17844961713Sgirish void 179459190a5Srsmaeda cpu_map_exec_units(struct cpu *cp) 18044961713Sgirish { 181459190a5Srsmaeda ASSERT(MUTEX_HELD(&cpu_lock)); 18244961713Sgirish 18344961713Sgirish /* 184fb2f18f8Sesaxe * The cpu_ipipe and cpu_fpu fields are initialized based on 185459190a5Srsmaeda * the execution unit sharing information from the MD. They 186459190a5Srsmaeda * default to the CPU id in the absence of such information. 18744961713Sgirish */ 18844961713Sgirish cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; 18944961713Sgirish if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) 19044961713Sgirish cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); 19144961713Sgirish 192fb2f18f8Sesaxe cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping; 193fb2f18f8Sesaxe if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND) 194fb2f18f8Sesaxe cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id); 195fb2f18f8Sesaxe 196fb2f18f8Sesaxe /* 197fb2f18f8Sesaxe * Niagara 2 defines the core to be at the FPU level 198fb2f18f8Sesaxe */ 199fb2f18f8Sesaxe cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu; 20059ac0c16Sdavemq 20159ac0c16Sdavemq /* 20259ac0c16Sdavemq * The cpu_chip field is initialized based on the information 20359ac0c16Sdavemq * in the MD and assume that all cpus within a chip 20459ac0c16Sdavemq * share the same L2 cache. If no such info is available, we 20559ac0c16Sdavemq * set the cpu to belong to the defacto chip 0. 20659ac0c16Sdavemq */ 207ce8eb11aSdp78419 cp->cpu_m.cpu_mpipe = cpunodes[cp->cpu_id].l2_cache_mapping; 208ce8eb11aSdp78419 if (cp->cpu_m.cpu_mpipe == NO_L2_CACHE_MAPPING_FOUND) 209ce8eb11aSdp78419 cp->cpu_m.cpu_mpipe = CPU_L2_CACHEID_INVALID; 210e853d8c3Sjc25722 211e853d8c3Sjc25722 cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping; 212e853d8c3Sjc25722 if (cp->cpu_m.cpu_chip == NO_L2_CACHE_MAPPING_FOUND) 213e853d8c3Sjc25722 cp->cpu_m.cpu_chip = CPU_CHIPID_INVALID; 214459190a5Srsmaeda } 215459190a5Srsmaeda 21659ac0c16Sdavemq static int cpucnt; 217459190a5Srsmaeda 218459190a5Srsmaeda void 219459190a5Srsmaeda cpu_init_private(struct cpu *cp) 220459190a5Srsmaeda { 221459190a5Srsmaeda extern void niagara_kstat_init(void); 222fb2f18f8Sesaxe 22344961713Sgirish ASSERT(MUTEX_HELD(&cpu_lock)); 224459190a5Srsmaeda 225459190a5Srsmaeda cpu_map_exec_units(cp); 226459190a5Srsmaeda 22759ac0c16Sdavemq if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE)) 22859ac0c16Sdavemq (void) niagara_kstat_init(); 22944961713Sgirish } 23044961713Sgirish 23144961713Sgirish /*ARGSUSED*/ 23244961713Sgirish void 23344961713Sgirish cpu_uninit_private(struct cpu *cp) 23444961713Sgirish { 235459190a5Srsmaeda extern void niagara_kstat_fini(void); 23644961713Sgirish 23744961713Sgirish ASSERT(MUTEX_HELD(&cpu_lock)); 23859ac0c16Sdavemq if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE)) 23959ac0c16Sdavemq (void) niagara_kstat_fini(); 24044961713Sgirish } 24144961713Sgirish 24244961713Sgirish /* 24344961713Sgirish * On Niagara2, any flush will cause all preceding stores to be 24444961713Sgirish * synchronized wrt the i$, regardless of address or ASI. In fact, 24544961713Sgirish * the address is ignored, so we always flush address 0. 24644961713Sgirish */ 24744961713Sgirish /*ARGSUSED*/ 24844961713Sgirish void 24944961713Sgirish dtrace_flush_sec(uintptr_t addr) 25044961713Sgirish { 25144961713Sgirish doflush(0); 25244961713Sgirish } 25344961713Sgirish 25444961713Sgirish /* 25544961713Sgirish * Trapstat support for Niagara2 processor 25644961713Sgirish * The Niagara2 provides HWTW support for TSB lookup and with HWTW 25744961713Sgirish * enabled no TSB hit information will be available. Therefore setting 25844961713Sgirish * the time spent in TLB miss handler for TSB hits to 0. 25944961713Sgirish */ 26044961713Sgirish int 26144961713Sgirish cpu_trapstat_conf(int cmd) 26244961713Sgirish { 26344961713Sgirish int status = 0; 26444961713Sgirish 26544961713Sgirish switch (cmd) { 26644961713Sgirish case CPU_TSTATCONF_INIT: 26744961713Sgirish case CPU_TSTATCONF_FINI: 26844961713Sgirish case CPU_TSTATCONF_ENABLE: 26944961713Sgirish case CPU_TSTATCONF_DISABLE: 27044961713Sgirish break; 27144961713Sgirish default: 27244961713Sgirish status = EINVAL; 27344961713Sgirish break; 27444961713Sgirish } 27544961713Sgirish return (status); 27644961713Sgirish } 27744961713Sgirish 27844961713Sgirish void 27944961713Sgirish cpu_trapstat_data(void *buf, uint_t tstat_pgszs) 28044961713Sgirish { 28144961713Sgirish tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; 28244961713Sgirish int i; 28344961713Sgirish 28444961713Sgirish for (i = 0; i < tstat_pgszs; i++, tstatp++) { 28544961713Sgirish tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; 28644961713Sgirish tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; 28744961713Sgirish tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; 28844961713Sgirish tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; 28944961713Sgirish tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 29044961713Sgirish tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 29144961713Sgirish tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 29244961713Sgirish tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 29344961713Sgirish } 29444961713Sgirish } 295102033aaSdp78419 296ce8eb11aSdp78419 /* 297ce8eb11aSdp78419 * Page coloring support for hashed cache index mode 298ce8eb11aSdp78419 */ 299ce8eb11aSdp78419 300ce8eb11aSdp78419 /* 301ce8eb11aSdp78419 * Node id bits from machine description (MD). Node id distinguishes 302ce8eb11aSdp78419 * local versus remote memory. Because of MPO, page allocation does 303ce8eb11aSdp78419 * not cross node boundaries. Therefore, remove the node id bits from 304ce8eb11aSdp78419 * the color, since they are fixed. Either bit 30, or 31:30 in 305ce8eb11aSdp78419 * Victoria Falls processors. 306ce8eb11aSdp78419 * The number of node id bits is always 0 in Niagara2. 307ce8eb11aSdp78419 */ 308ce8eb11aSdp78419 typedef struct n2color { 309ce8eb11aSdp78419 uchar_t nnbits; /* number of node id bits */ 310ce8eb11aSdp78419 uchar_t nnmask; /* mask for node id bits */ 311ce8eb11aSdp78419 uchar_t lomask; /* mask for bits below node id */ 312ce8eb11aSdp78419 uchar_t lobits; /* number of bits below node id */ 313ce8eb11aSdp78419 } n2color_t; 314ce8eb11aSdp78419 315ce8eb11aSdp78419 n2color_t n2color[MMU_PAGE_SIZES]; 316ce8eb11aSdp78419 static uchar_t nhbits[] = {7, 7, 6, 5, 5, 5}; 317ce8eb11aSdp78419 318ce8eb11aSdp78419 /* 319ce8eb11aSdp78419 * Remove node id bits from color bits 32:28. 320ce8eb11aSdp78419 * This will reduce the number of colors. 321ce8eb11aSdp78419 * No change if number of node bits is zero. 322ce8eb11aSdp78419 */ 323ce8eb11aSdp78419 static inline uint_t 324ce8eb11aSdp78419 n2_hash2color(uint_t color, uchar_t szc) 325ce8eb11aSdp78419 { 326ce8eb11aSdp78419 n2color_t m = n2color[szc]; 327ce8eb11aSdp78419 328ce8eb11aSdp78419 if (m.nnbits > 0) { 329ce8eb11aSdp78419 color = ((color >> m.nnbits) & ~m.lomask) | (color & m.lomask); 330ce8eb11aSdp78419 ASSERT((color & ~(hw_page_array[szc].hp_colors - 1)) == 0); 331ce8eb11aSdp78419 } 332ce8eb11aSdp78419 333ce8eb11aSdp78419 return (color); 334ce8eb11aSdp78419 } 335ce8eb11aSdp78419 336ce8eb11aSdp78419 /* 337ce8eb11aSdp78419 * Restore node id bits into page color. 338ce8eb11aSdp78419 * This will increase the number of colors to match N2. 339ce8eb11aSdp78419 * No change if number of node bits is zero. 340ce8eb11aSdp78419 */ 341ce8eb11aSdp78419 static inline uint_t 342ce8eb11aSdp78419 n2_color2hash(uint_t color, uchar_t szc, uint_t node) 343ce8eb11aSdp78419 { 344ce8eb11aSdp78419 n2color_t m = n2color[szc]; 345ce8eb11aSdp78419 346ce8eb11aSdp78419 if (m.nnbits > 0) { 347ce8eb11aSdp78419 color = ((color & ~m.lomask) << m.nnbits) | (color & m.lomask); 348ce8eb11aSdp78419 color |= (node & m.nnmask) << m.lobits; 349ce8eb11aSdp78419 } 350ce8eb11aSdp78419 351ce8eb11aSdp78419 return (color); 352ce8eb11aSdp78419 } 353ce8eb11aSdp78419 354102033aaSdp78419 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ 355ce8eb11aSdp78419 356ce8eb11aSdp78419 /* 357ce8eb11aSdp78419 * iterator NULL means pfn is VA, do not adjust ra_to_pa 358ce8eb11aSdp78419 * iterator (-1) means pfn is RA, need to convert to PA 359ce8eb11aSdp78419 * iterator non-null means pfn is RA, use ra_to_pa 360ce8eb11aSdp78419 */ 361102033aaSdp78419 uint_t 362ce8eb11aSdp78419 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc, void *cookie) 363ce8eb11aSdp78419 { 364ce8eb11aSdp78419 mem_node_iterator_t *it = cookie; 365ce8eb11aSdp78419 uint_t color; 366ce8eb11aSdp78419 367ce8eb11aSdp78419 ASSERT(szc <= TTE256M); 368ce8eb11aSdp78419 369ce8eb11aSdp78419 if (it == ((mem_node_iterator_t *)(-1))) { 370ce8eb11aSdp78419 pfn = plat_rapfn_to_papfn(pfn); 371ce8eb11aSdp78419 } else if (it != NULL) { 372ce8eb11aSdp78419 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 373ce8eb11aSdp78419 pfn = pfn + it->mi_ra_to_pa; 374ce8eb11aSdp78419 } 375ce8eb11aSdp78419 pfn = PFN_BASE(pfn, szc); 376ce8eb11aSdp78419 color = ((pfn >> 15) ^ pfn) & 0x1f; 377ce8eb11aSdp78419 if (szc < TTE4M) { 378ce8eb11aSdp78419 /* 19:18 */ 379ce8eb11aSdp78419 color = (color << 2) | ((pfn >> 5) & 0x3); 380ce8eb11aSdp78419 if (szc > TTE64K) 381ce8eb11aSdp78419 color >>= 1; /* 19 */ 382ce8eb11aSdp78419 } 383ce8eb11aSdp78419 return (n2_hash2color(color, szc)); 384ce8eb11aSdp78419 } 385ce8eb11aSdp78419 386ce8eb11aSdp78419 static uint_t 387ce8eb11aSdp78419 page_papfn_2_color_cpu(pfn_t papfn, uchar_t szc) 388102033aaSdp78419 { 389102033aaSdp78419 uint_t color; 390102033aaSdp78419 391102033aaSdp78419 ASSERT(szc <= TTE256M); 392102033aaSdp78419 393ce8eb11aSdp78419 papfn = PFN_BASE(papfn, szc); 394ce8eb11aSdp78419 color = ((papfn >> 15) ^ papfn) & 0x1f; 395ce8eb11aSdp78419 if (szc < TTE4M) { 396ce8eb11aSdp78419 /* 19:18 */ 397ce8eb11aSdp78419 color = (color << 2) | ((papfn >> 5) & 0x3); 398ce8eb11aSdp78419 if (szc > TTE64K) 399ce8eb11aSdp78419 color >>= 1; /* 19 */ 400ce8eb11aSdp78419 } 401102033aaSdp78419 return (color); 402102033aaSdp78419 } 403102033aaSdp78419 404102033aaSdp78419 #if TTE256M != 5 405102033aaSdp78419 #error TTE256M is not 5 406102033aaSdp78419 #endif 407102033aaSdp78419 408102033aaSdp78419 uint_t 409102033aaSdp78419 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) 410102033aaSdp78419 { 411102033aaSdp78419 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; 412102033aaSdp78419 ASSERT(szc < TTE256M); 413ce8eb11aSdp78419 mask = n2_color2hash(mask, szc, 0); 414102033aaSdp78419 mask &= ni2_color_masks[szc]; 415ce8eb11aSdp78419 if (szc == TTE64K || szc == TTE512K) 416ce8eb11aSdp78419 mask >>= 1; 417ce8eb11aSdp78419 return (n2_hash2color(mask, szc + 1)); 418102033aaSdp78419 } 419102033aaSdp78419 420102033aaSdp78419 uint_t 421102033aaSdp78419 page_get_nsz_color_cpu(uchar_t szc, uint_t color) 422102033aaSdp78419 { 423102033aaSdp78419 ASSERT(szc < TTE256M); 424ce8eb11aSdp78419 color = n2_color2hash(color, szc, 0); 425ce8eb11aSdp78419 if (szc == TTE64K || szc == TTE512K) 426ce8eb11aSdp78419 color >>= 1; 427ce8eb11aSdp78419 return (n2_hash2color(color, szc + 1)); 428102033aaSdp78419 } 429102033aaSdp78419 430102033aaSdp78419 uint_t 431102033aaSdp78419 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) 432102033aaSdp78419 { 433ce8eb11aSdp78419 uint_t s; 434ef29e907Ssusans ASSERT(nszc >= szc); 435102033aaSdp78419 ASSERT(nszc <= TTE256M); 436102033aaSdp78419 437ce8eb11aSdp78419 s = nhbits[szc] - n2color[szc].nnbits; 438ce8eb11aSdp78419 s -= nhbits[nszc] - n2color[nszc].nnbits; 439102033aaSdp78419 440ce8eb11aSdp78419 return (s); 441102033aaSdp78419 } 442102033aaSdp78419 443ce8eb11aSdp78419 uint_t 444ce8eb11aSdp78419 page_convert_color_cpu(uint_t ncolor, uchar_t szc, uchar_t nszc) 445ce8eb11aSdp78419 { 446ce8eb11aSdp78419 uint_t color; 447ce8eb11aSdp78419 448ce8eb11aSdp78419 ASSERT(nszc > szc); 449ce8eb11aSdp78419 ASSERT(nszc <= TTE256M); 450ce8eb11aSdp78419 ncolor = n2_color2hash(ncolor, nszc, 0); 451ce8eb11aSdp78419 color = ncolor << (nhbits[szc] - nhbits[nszc]); 452ce8eb11aSdp78419 color = n2_hash2color(color, szc); 453ce8eb11aSdp78419 return (color); 454ce8eb11aSdp78419 } 455ce8eb11aSdp78419 456ce8eb11aSdp78419 #define PAPFN_2_MNODE(pfn) \ 457ce8eb11aSdp78419 (((pfn) & it->mi_mnode_pfn_mask) >> it->mi_mnode_pfn_shift) 458ce8eb11aSdp78419 459102033aaSdp78419 /*ARGSUSED*/ 460102033aaSdp78419 pfn_t 461102033aaSdp78419 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, 462ce8eb11aSdp78419 uint_t ceq_mask, uint_t color_mask, void *cookie) 463102033aaSdp78419 { 464ce8eb11aSdp78419 mem_node_iterator_t *it = cookie; 465102033aaSdp78419 pfn_t pstep = PNUM_SIZE(szc); 466102033aaSdp78419 pfn_t npfn, pfn_ceq_mask, pfn_color; 467102033aaSdp78419 pfn_t tmpmask, mask = (pfn_t)-1; 468ce8eb11aSdp78419 uint_t pfnmn; 469102033aaSdp78419 470102033aaSdp78419 ASSERT((color & ~ceq_mask) == 0); 471ce8eb11aSdp78419 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 472102033aaSdp78419 473ce8eb11aSdp78419 /* convert RA to PA for accurate color calculation */ 474ce8eb11aSdp78419 if (it->mi_init) { 475ce8eb11aSdp78419 /* first call after it, so cache these values */ 476ce8eb11aSdp78419 it->mi_hash_ceq_mask = 477ce8eb11aSdp78419 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask); 478ce8eb11aSdp78419 it->mi_hash_color = 479ce8eb11aSdp78419 n2_color2hash(color, szc, it->mi_mnode); 480ce8eb11aSdp78419 it->mi_init = 0; 481ce8eb11aSdp78419 } else { 482ce8eb11aSdp78419 ASSERT(it->mi_hash_ceq_mask == 483ce8eb11aSdp78419 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask)); 484ce8eb11aSdp78419 ASSERT(it->mi_hash_color == 485ce8eb11aSdp78419 n2_color2hash(color, szc, it->mi_mnode)); 486ce8eb11aSdp78419 } 487ce8eb11aSdp78419 ceq_mask = it->mi_hash_ceq_mask; 488ce8eb11aSdp78419 color = it->mi_hash_color; 489ce8eb11aSdp78419 pfn += it->mi_ra_to_pa; 490ce8eb11aSdp78419 491ce8eb11aSdp78419 /* restart here when we switch memblocks */ 492ce8eb11aSdp78419 next_mem_block: 493ce8eb11aSdp78419 if (szc <= TTE64K) { 494ce8eb11aSdp78419 pfnmn = PAPFN_2_MNODE(pfn); 495ce8eb11aSdp78419 } 496ce8eb11aSdp78419 if (((page_papfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0 && 497ce8eb11aSdp78419 (szc > TTE64K || pfnmn == it->mi_mnode)) { 498102033aaSdp78419 499102033aaSdp78419 /* we start from the page with correct color */ 500102033aaSdp78419 if (szc >= TTE512K) { 501102033aaSdp78419 if (szc >= TTE4M) { 502102033aaSdp78419 /* page color is PA[32:28] */ 503102033aaSdp78419 pfn_ceq_mask = ceq_mask << 15; 504102033aaSdp78419 } else { 505102033aaSdp78419 /* page color is PA[32:28].PA[19:19] */ 506102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 507102033aaSdp78419 ((ceq_mask >> 1) << 15); 508102033aaSdp78419 } 509ce8eb11aSdp78419 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 510ce8eb11aSdp78419 goto done; 511102033aaSdp78419 } else { 512102033aaSdp78419 /* 513102033aaSdp78419 * We deal 64K or 8K page. Check if we could the 514102033aaSdp78419 * satisfy the request without changing PA[32:28] 515102033aaSdp78419 */ 516102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 517ce8eb11aSdp78419 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 518102033aaSdp78419 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 519102033aaSdp78419 520102033aaSdp78419 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 521ce8eb11aSdp78419 goto done; 522102033aaSdp78419 523102033aaSdp78419 /* 524102033aaSdp78419 * for next pfn we have to change bits PA[32:28] 525102033aaSdp78419 * set PA[63:28] and PA[19:18] of the next pfn 526102033aaSdp78419 */ 527102033aaSdp78419 npfn = (pfn >> 15) << 15; 528102033aaSdp78419 npfn |= (ceq_mask & color & 3) << 5; 529102033aaSdp78419 pfn_ceq_mask = (szc == TTE8K) ? 0 : 530102033aaSdp78419 (ceq_mask & 0x1c) << 13; 531ce8eb11aSdp78419 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 532102033aaSdp78419 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); 533102033aaSdp78419 534102033aaSdp78419 /* 535102033aaSdp78419 * set bits PA[17:13] to match the color 536102033aaSdp78419 */ 537ce8eb11aSdp78419 npfn |= ((npfn >> 15) ^ (color >> 2)) & (ceq_mask >> 2); 538ce8eb11aSdp78419 goto done; 539102033aaSdp78419 } 540102033aaSdp78419 } 541102033aaSdp78419 542102033aaSdp78419 /* 543102033aaSdp78419 * we start from the page with incorrect color - rare case 544102033aaSdp78419 */ 545102033aaSdp78419 if (szc >= TTE512K) { 546102033aaSdp78419 if (szc >= TTE4M) { 547102033aaSdp78419 /* page color is in bits PA[32:28] */ 548102033aaSdp78419 npfn = ((pfn >> 20) << 20) | (color << 15); 549102033aaSdp78419 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; 550102033aaSdp78419 } else { 551102033aaSdp78419 /* try get the right color by changing bit PA[19:19] */ 552102033aaSdp78419 npfn = pfn + pstep; 553ce8eb11aSdp78419 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 554102033aaSdp78419 ceq_mask) == 0) 555ce8eb11aSdp78419 goto done; 556102033aaSdp78419 557102033aaSdp78419 /* page color is PA[32:28].PA[19:19] */ 558102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 559102033aaSdp78419 ((ceq_mask >> 1) << 15) | (0xff << 7); 560102033aaSdp78419 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); 561102033aaSdp78419 npfn = ((pfn >> 20) << 20) | pfn_color; 562102033aaSdp78419 } 563102033aaSdp78419 564102033aaSdp78419 while (npfn <= pfn) { 565102033aaSdp78419 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 566102033aaSdp78419 } 567ce8eb11aSdp78419 goto done; 568102033aaSdp78419 } 569102033aaSdp78419 570102033aaSdp78419 /* 571102033aaSdp78419 * We deal 64K or 8K page of incorrect color. 572102033aaSdp78419 * Try correcting color without changing PA[32:28] 573102033aaSdp78419 */ 574102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 575102033aaSdp78419 pfn_color = ((color & 3) << 5) | (color >> 2); 576ce8eb11aSdp78419 if (pfnmn == it->mi_mnode) { 577102033aaSdp78419 npfn = (pfn & ~(pfn_t)0x7f); 578102033aaSdp78419 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 579102033aaSdp78419 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 580102033aaSdp78419 581ce8eb11aSdp78419 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 582ce8eb11aSdp78419 ceq_mask) == 0) { 583102033aaSdp78419 /* the color is fixed - find the next page */ 584ce8eb11aSdp78419 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 585102033aaSdp78419 while (npfn <= pfn) { 586ce8eb11aSdp78419 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, 587ce8eb11aSdp78419 mask); 588102033aaSdp78419 } 589102033aaSdp78419 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 590ce8eb11aSdp78419 goto done; 591ce8eb11aSdp78419 } 592102033aaSdp78419 } 593102033aaSdp78419 594102033aaSdp78419 /* to fix the color need to touch PA[32:28] */ 595102033aaSdp78419 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : 596102033aaSdp78419 (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); 597ce8eb11aSdp78419 598ce8eb11aSdp78419 /* fix mnode if input pfn is in the wrong mnode. */ 599ce8eb11aSdp78419 if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) { 600ce8eb11aSdp78419 npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) << 601ce8eb11aSdp78419 it->mi_mnode_pfn_shift; 602ce8eb11aSdp78419 } 603ce8eb11aSdp78419 604102033aaSdp78419 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; 605ce8eb11aSdp78419 tmpmask |= it->mi_mnode_pfn_mask; 606102033aaSdp78419 607102033aaSdp78419 while (npfn <= pfn) { 608102033aaSdp78419 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); 609102033aaSdp78419 } 610102033aaSdp78419 611102033aaSdp78419 /* set bits PA[19:13] to match the color */ 612102033aaSdp78419 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 613102033aaSdp78419 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 614102033aaSdp78419 615ce8eb11aSdp78419 done: 616ce8eb11aSdp78419 ASSERT(((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); 617ce8eb11aSdp78419 ASSERT(PAPFN_2_MNODE(npfn) == it->mi_mnode); 618ce8eb11aSdp78419 619ce8eb11aSdp78419 /* PA to RA */ 620ce8eb11aSdp78419 npfn -= it->mi_ra_to_pa; 621ce8eb11aSdp78419 622ce8eb11aSdp78419 /* check for possible memblock switch */ 623ce8eb11aSdp78419 if (npfn > it->mi_mblock_end) { 624ce8eb11aSdp78419 pfn = plat_mem_node_iterator_init(npfn, it->mi_mnode, it, 0); 625ce8eb11aSdp78419 if (pfn == (pfn_t)-1) 626ce8eb11aSdp78419 return (pfn); 627ce8eb11aSdp78419 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 628ce8eb11aSdp78419 pfn += it->mi_ra_to_pa; 629ce8eb11aSdp78419 goto next_mem_block; 630ce8eb11aSdp78419 } 631102033aaSdp78419 632102033aaSdp78419 return (npfn); 633102033aaSdp78419 } 634102033aaSdp78419 635102033aaSdp78419 /* 636102033aaSdp78419 * init page coloring 637ce8eb11aSdp78419 * VF encodes node_id for an L-group in either bit 30 or 31:30, 638ce8eb11aSdp78419 * which effectively reduces the number of colors available per mnode. 639102033aaSdp78419 */ 640102033aaSdp78419 void 641102033aaSdp78419 page_coloring_init_cpu() 642102033aaSdp78419 { 643102033aaSdp78419 int i; 644ce8eb11aSdp78419 uchar_t id; 645ce8eb11aSdp78419 uchar_t lo; 646ce8eb11aSdp78419 uchar_t hi; 647ce8eb11aSdp78419 n2color_t m; 648ce8eb11aSdp78419 mem_node_iterator_t it; 649ce8eb11aSdp78419 static uchar_t idmask[] = {0, 0x7, 0x1f, 0x1f, 0x1f, 0x1f}; 650102033aaSdp78419 651284953f5Sdp78419 for (i = 0; i < max_mem_nodes; i++) { 652284953f5Sdp78419 memset(&it, 0, sizeof (it)); 653284953f5Sdp78419 if (plat_mem_node_iterator_init(0, i, &it, 1) != (pfn_t)-1) 654284953f5Sdp78419 break; 655284953f5Sdp78419 } 656284953f5Sdp78419 ASSERT(i < max_mem_nodes); 657ce8eb11aSdp78419 for (i = 0; i < mmu_page_sizes; i++) { 658b02e9a2dSsvemuri (void) memset(&m, 0, sizeof (m)); 659ce8eb11aSdp78419 id = it.mi_mnode_pfn_mask >> 15; /* node id mask */ 660ce8eb11aSdp78419 id &= idmask[i]; 661ce8eb11aSdp78419 lo = lowbit(id); 662ce8eb11aSdp78419 if (lo > 0) { 663ce8eb11aSdp78419 hi = highbit(id); 664ce8eb11aSdp78419 m.nnbits = hi - lo + 1; 665ce8eb11aSdp78419 m.nnmask = (1 << m.nnbits) - 1; 666ce8eb11aSdp78419 lo += nhbits[i] - 5; 667ce8eb11aSdp78419 m.lomask = (1 << (lo - 1)) - 1; 668ce8eb11aSdp78419 m.lobits = lo - 1; 669ce8eb11aSdp78419 } 670ce8eb11aSdp78419 hw_page_array[i].hp_colors = 1 << (nhbits[i] - m.nnbits); 671ce8eb11aSdp78419 n2color[i] = m; 672102033aaSdp78419 } 673102033aaSdp78419 } 674fe70c9cfSdp78419 675fe70c9cfSdp78419 /* 676fe70c9cfSdp78419 * group colorequiv colors on N2 by low order bits of the color first 677fe70c9cfSdp78419 */ 678fe70c9cfSdp78419 void 679fe70c9cfSdp78419 page_set_colorequiv_arr_cpu(void) 680fe70c9cfSdp78419 { 681fe70c9cfSdp78419 static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0}; 682fe70c9cfSdp78419 683ce8eb11aSdp78419 nequiv_shades_log2[1] -= n2color[1].nnbits; 684fe70c9cfSdp78419 if (colorequiv > 1) { 685fe70c9cfSdp78419 int i; 686fe70c9cfSdp78419 uint_t sv_a = lowbit(colorequiv) - 1; 687fe70c9cfSdp78419 688fe70c9cfSdp78419 if (sv_a > 15) 689fe70c9cfSdp78419 sv_a = 15; 690fe70c9cfSdp78419 691fe70c9cfSdp78419 for (i = 0; i < MMU_PAGE_SIZES; i++) { 692fe70c9cfSdp78419 uint_t colors; 693fe70c9cfSdp78419 uint_t a = sv_a; 694fe70c9cfSdp78419 695fe70c9cfSdp78419 if ((colors = hw_page_array[i].hp_colors) <= 1) 696fe70c9cfSdp78419 continue; 697fe70c9cfSdp78419 while ((colors >> a) == 0) 698fe70c9cfSdp78419 a--; 699fe70c9cfSdp78419 if (a > (colorequivszc[i] & 0xf) + 700fe70c9cfSdp78419 (colorequivszc[i] >> 4)) { 701fe70c9cfSdp78419 if (a <= nequiv_shades_log2[i]) { 70259ac0c16Sdavemq colorequivszc[i] = (uchar_t)a; 703fe70c9cfSdp78419 } else { 704fe70c9cfSdp78419 colorequivszc[i] = 705fe70c9cfSdp78419 ((a - nequiv_shades_log2[i]) << 4) | 706fe70c9cfSdp78419 nequiv_shades_log2[i]; 707fe70c9cfSdp78419 } 708fe70c9cfSdp78419 } 709fe70c9cfSdp78419 } 710fe70c9cfSdp78419 } 711fe70c9cfSdp78419 } 712