144961713Sgirish /* 244961713Sgirish * CDDL HEADER START 344961713Sgirish * 444961713Sgirish * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 744961713Sgirish * 844961713Sgirish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 944961713Sgirish * or http://www.opensolaris.org/os/licensing. 1044961713Sgirish * See the License for the specific language governing permissions 1144961713Sgirish * and limitations under the License. 1244961713Sgirish * 1344961713Sgirish * When distributing Covered Code, include this CDDL HEADER in each 1444961713Sgirish * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1544961713Sgirish * If applicable, add the following below this CDDL HEADER, with the 1644961713Sgirish * fields enclosed by brackets "[]" replaced with your own identifying 1744961713Sgirish * information: Portions Copyright [yyyy] [name of copyright owner] 1844961713Sgirish * 1944961713Sgirish * CDDL HEADER END 2044961713Sgirish */ 21459190a5Srsmaeda 2244961713Sgirish /* 23fb2f18f8Sesaxe * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 2444961713Sgirish * Use is subject to license terms. 2544961713Sgirish */ 2644961713Sgirish 2744961713Sgirish #pragma ident "%Z%%M% %I% %E% SMI" 2844961713Sgirish 2944961713Sgirish #include <sys/types.h> 3044961713Sgirish #include <sys/systm.h> 3144961713Sgirish #include <sys/archsystm.h> 3244961713Sgirish #include <sys/machparam.h> 3344961713Sgirish #include <sys/machsystm.h> 3444961713Sgirish #include <sys/cpu.h> 3544961713Sgirish #include <sys/elf_SPARC.h> 3644961713Sgirish #include <vm/hat_sfmmu.h> 3744961713Sgirish #include <vm/page.h> 38102033aaSdp78419 #include <vm/vm_dep.h> 3944961713Sgirish #include <sys/cpuvar.h> 4044961713Sgirish #include <sys/async.h> 4144961713Sgirish #include <sys/cmn_err.h> 4244961713Sgirish #include <sys/debug.h> 4344961713Sgirish #include <sys/dditypes.h> 4444961713Sgirish #include <sys/sunddi.h> 4544961713Sgirish #include <sys/cpu_module.h> 4644961713Sgirish #include <sys/prom_debug.h> 4744961713Sgirish #include <sys/vmsystm.h> 4844961713Sgirish #include <sys/prom_plat.h> 4944961713Sgirish #include <sys/sysmacros.h> 5044961713Sgirish #include <sys/intreg.h> 5144961713Sgirish #include <sys/machtrap.h> 5244961713Sgirish #include <sys/ontrap.h> 5344961713Sgirish #include <sys/ivintr.h> 5444961713Sgirish #include <sys/atomic.h> 5544961713Sgirish #include <sys/panic.h> 5644961713Sgirish #include <sys/dtrace.h> 5744961713Sgirish #include <sys/simulate.h> 5844961713Sgirish #include <sys/fault.h> 5944961713Sgirish #include <sys/niagara2regs.h> 6044961713Sgirish #include <sys/hsvc.h> 6144961713Sgirish #include <sys/trapstat.h> 6244961713Sgirish 6344961713Sgirish uint_t root_phys_addr_lo_mask = 0xffffffffU; 6459ac0c16Sdavemq #if defined(NIAGARA2_IMPL) 6544961713Sgirish char cpu_module_name[] = "SUNW,UltraSPARC-T2"; 6659ac0c16Sdavemq #elif defined(VFALLS_IMPL) 6759ac0c16Sdavemq char cpu_module_name[] = "SUNW,UltraSPARC-T2+"; 6859ac0c16Sdavemq #endif 6944961713Sgirish 7044961713Sgirish /* 7159ac0c16Sdavemq * Hypervisor services information for the NIAGARA2 and Victoria Falls 7259ac0c16Sdavemq * CPU module 7344961713Sgirish */ 7459ac0c16Sdavemq static boolean_t cpu_hsvc_available = B_TRUE; 7559ac0c16Sdavemq static uint64_t cpu_sup_minor; /* Supported minor number */ 7659ac0c16Sdavemq #if defined(NIAGARA2_IMPL) 7759ac0c16Sdavemq static hsvc_info_t cpu_hsvc = { 7844961713Sgirish HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, 7944961713Sgirish NIAGARA2_HSVC_MINOR, cpu_module_name 8044961713Sgirish }; 8159ac0c16Sdavemq #elif defined(VFALLS_IMPL) 8259ac0c16Sdavemq static hsvc_info_t cpu_hsvc = { 8359ac0c16Sdavemq HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR, 8459ac0c16Sdavemq VFALLS_HSVC_MINOR, cpu_module_name 8559ac0c16Sdavemq }; 8659ac0c16Sdavemq #endif 8744961713Sgirish 8844961713Sgirish void 8944961713Sgirish cpu_setup(void) 9044961713Sgirish { 9144961713Sgirish extern int mmu_exported_pagesize_mask; 9244961713Sgirish extern int cpc_has_overflow_intr; 93aaa10e67Sha137994 extern size_t contig_mem_prealloc_base; 9444961713Sgirish int status; 9544961713Sgirish 9644961713Sgirish /* 9744961713Sgirish * Negotiate the API version for Niagara2 specific hypervisor 9844961713Sgirish * services. 9944961713Sgirish */ 10059ac0c16Sdavemq status = hsvc_register(&cpu_hsvc, &cpu_sup_minor); 10144961713Sgirish if (status != 0) { 10244961713Sgirish cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 10344961713Sgirish "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", 10459ac0c16Sdavemq cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group, 10559ac0c16Sdavemq cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status); 10659ac0c16Sdavemq cpu_hsvc_available = B_FALSE; 10744961713Sgirish } 10844961713Sgirish 10944961713Sgirish /* 11044961713Sgirish * The setup common to all CPU modules is done in cpu_setup_common 11144961713Sgirish * routine. 11244961713Sgirish */ 11344961713Sgirish cpu_setup_common(NULL); 11444961713Sgirish 11544961713Sgirish cache |= (CACHE_PTAG | CACHE_IOCOHERENT); 11644961713Sgirish 11744961713Sgirish if ((mmu_exported_pagesize_mask & 11844961713Sgirish DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != 11944961713Sgirish DEFAULT_SUN4V_MMU_PAGESIZE_MASK) 12044961713Sgirish cmn_err(CE_PANIC, "machine description" 12144961713Sgirish " does not have required sun4v page sizes" 12244961713Sgirish " 8K, 64K and 4M: MD mask is 0x%x", 12344961713Sgirish mmu_exported_pagesize_mask); 12444961713Sgirish 12544961713Sgirish cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT; 12644961713Sgirish 12744961713Sgirish /* 12844961713Sgirish * Niagara2 supports a 48-bit subset of the full 64-bit virtual 12944961713Sgirish * address space. Virtual addresses between 0x0000800000000000 13044961713Sgirish * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" 13144961713Sgirish * and must never be mapped. In addition, software must not use 13244961713Sgirish * pages within 4GB of the VA hole as instruction pages to 13344961713Sgirish * avoid problems with prefetching into the VA hole. 13444961713Sgirish */ 13544961713Sgirish hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); 13644961713Sgirish hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); 13744961713Sgirish 13844961713Sgirish /* 13944961713Sgirish * Niagara2 has a performance counter overflow interrupt 14044961713Sgirish */ 14144961713Sgirish cpc_has_overflow_intr = 1; 142102033aaSdp78419 143102033aaSdp78419 /* 144102033aaSdp78419 * Enable 4M pages for OOB. 145102033aaSdp78419 */ 146102033aaSdp78419 max_uheap_lpsize = MMU_PAGESIZE4M; 147102033aaSdp78419 max_ustack_lpsize = MMU_PAGESIZE4M; 148102033aaSdp78419 max_privmap_lpsize = MMU_PAGESIZE4M; 149aaa10e67Sha137994 150aaa10e67Sha137994 contig_mem_prealloc_base = NIAGARA2_PREALLOC_BASE; 15144961713Sgirish } 15244961713Sgirish 15344961713Sgirish /* 15444961713Sgirish * Set the magic constants of the implementation. 15544961713Sgirish */ 15644961713Sgirish void 15744961713Sgirish cpu_fiximp(struct cpu_node *cpunode) 15844961713Sgirish { 15944961713Sgirish /* 16044961713Sgirish * The Cache node is optional in MD. Therefore in case "Cache" 16144961713Sgirish * node does not exists in MD, set the default L2 cache associativity, 16244961713Sgirish * size, linesize. 16344961713Sgirish */ 16444961713Sgirish if (cpunode->ecache_size == 0) 16544961713Sgirish cpunode->ecache_size = L2CACHE_SIZE; 16644961713Sgirish if (cpunode->ecache_linesize == 0) 16744961713Sgirish cpunode->ecache_linesize = L2CACHE_LINESIZE; 16844961713Sgirish if (cpunode->ecache_associativity == 0) 16944961713Sgirish cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; 17044961713Sgirish } 17144961713Sgirish 17244961713Sgirish void 173459190a5Srsmaeda cpu_map_exec_units(struct cpu *cp) 17444961713Sgirish { 175459190a5Srsmaeda ASSERT(MUTEX_HELD(&cpu_lock)); 17644961713Sgirish 17744961713Sgirish /* 178fb2f18f8Sesaxe * The cpu_ipipe and cpu_fpu fields are initialized based on 179459190a5Srsmaeda * the execution unit sharing information from the MD. They 180459190a5Srsmaeda * default to the CPU id in the absence of such information. 18144961713Sgirish */ 18244961713Sgirish cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; 18344961713Sgirish if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) 18444961713Sgirish cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); 18544961713Sgirish 186fb2f18f8Sesaxe cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping; 187fb2f18f8Sesaxe if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND) 188fb2f18f8Sesaxe cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id); 189fb2f18f8Sesaxe 190fb2f18f8Sesaxe /* 191fb2f18f8Sesaxe * Niagara 2 defines the core to be at the FPU level 192fb2f18f8Sesaxe */ 193fb2f18f8Sesaxe cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu; 19459ac0c16Sdavemq 19559ac0c16Sdavemq /* 19659ac0c16Sdavemq * The cpu_chip field is initialized based on the information 19759ac0c16Sdavemq * in the MD and assume that all cpus within a chip 19859ac0c16Sdavemq * share the same L2 cache. If no such info is available, we 19959ac0c16Sdavemq * set the cpu to belong to the defacto chip 0. 20059ac0c16Sdavemq */ 201ce8eb11aSdp78419 cp->cpu_m.cpu_mpipe = cpunodes[cp->cpu_id].l2_cache_mapping; 202ce8eb11aSdp78419 if (cp->cpu_m.cpu_mpipe == NO_L2_CACHE_MAPPING_FOUND) 203ce8eb11aSdp78419 cp->cpu_m.cpu_mpipe = CPU_L2_CACHEID_INVALID; 204e853d8c3Sjc25722 205e853d8c3Sjc25722 cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping; 206e853d8c3Sjc25722 if (cp->cpu_m.cpu_chip == NO_L2_CACHE_MAPPING_FOUND) 207e853d8c3Sjc25722 cp->cpu_m.cpu_chip = CPU_CHIPID_INVALID; 208459190a5Srsmaeda } 209459190a5Srsmaeda 21059ac0c16Sdavemq static int cpucnt; 211459190a5Srsmaeda 212459190a5Srsmaeda void 213459190a5Srsmaeda cpu_init_private(struct cpu *cp) 214459190a5Srsmaeda { 215459190a5Srsmaeda extern void niagara_kstat_init(void); 216fb2f18f8Sesaxe 21744961713Sgirish ASSERT(MUTEX_HELD(&cpu_lock)); 218459190a5Srsmaeda 219459190a5Srsmaeda cpu_map_exec_units(cp); 220459190a5Srsmaeda 22159ac0c16Sdavemq if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE)) 22259ac0c16Sdavemq (void) niagara_kstat_init(); 22344961713Sgirish } 22444961713Sgirish 22544961713Sgirish /*ARGSUSED*/ 22644961713Sgirish void 22744961713Sgirish cpu_uninit_private(struct cpu *cp) 22844961713Sgirish { 229459190a5Srsmaeda extern void niagara_kstat_fini(void); 23044961713Sgirish 23144961713Sgirish ASSERT(MUTEX_HELD(&cpu_lock)); 23259ac0c16Sdavemq if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE)) 23359ac0c16Sdavemq (void) niagara_kstat_fini(); 23444961713Sgirish } 23544961713Sgirish 23644961713Sgirish /* 23744961713Sgirish * On Niagara2, any flush will cause all preceding stores to be 23844961713Sgirish * synchronized wrt the i$, regardless of address or ASI. In fact, 23944961713Sgirish * the address is ignored, so we always flush address 0. 24044961713Sgirish */ 24144961713Sgirish /*ARGSUSED*/ 24244961713Sgirish void 24344961713Sgirish dtrace_flush_sec(uintptr_t addr) 24444961713Sgirish { 24544961713Sgirish doflush(0); 24644961713Sgirish } 24744961713Sgirish 24844961713Sgirish /* 24944961713Sgirish * Trapstat support for Niagara2 processor 25044961713Sgirish * The Niagara2 provides HWTW support for TSB lookup and with HWTW 25144961713Sgirish * enabled no TSB hit information will be available. Therefore setting 25244961713Sgirish * the time spent in TLB miss handler for TSB hits to 0. 25344961713Sgirish */ 25444961713Sgirish int 25544961713Sgirish cpu_trapstat_conf(int cmd) 25644961713Sgirish { 25744961713Sgirish int status = 0; 25844961713Sgirish 25944961713Sgirish switch (cmd) { 26044961713Sgirish case CPU_TSTATCONF_INIT: 26144961713Sgirish case CPU_TSTATCONF_FINI: 26244961713Sgirish case CPU_TSTATCONF_ENABLE: 26344961713Sgirish case CPU_TSTATCONF_DISABLE: 26444961713Sgirish break; 26544961713Sgirish default: 26644961713Sgirish status = EINVAL; 26744961713Sgirish break; 26844961713Sgirish } 26944961713Sgirish return (status); 27044961713Sgirish } 27144961713Sgirish 27244961713Sgirish void 27344961713Sgirish cpu_trapstat_data(void *buf, uint_t tstat_pgszs) 27444961713Sgirish { 27544961713Sgirish tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; 27644961713Sgirish int i; 27744961713Sgirish 27844961713Sgirish for (i = 0; i < tstat_pgszs; i++, tstatp++) { 27944961713Sgirish tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; 28044961713Sgirish tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; 28144961713Sgirish tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; 28244961713Sgirish tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; 28344961713Sgirish tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 28444961713Sgirish tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 28544961713Sgirish tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 28644961713Sgirish tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 28744961713Sgirish } 28844961713Sgirish } 289102033aaSdp78419 290ce8eb11aSdp78419 /* 291ce8eb11aSdp78419 * Page coloring support for hashed cache index mode 292ce8eb11aSdp78419 */ 293ce8eb11aSdp78419 294ce8eb11aSdp78419 /* 295ce8eb11aSdp78419 * Node id bits from machine description (MD). Node id distinguishes 296ce8eb11aSdp78419 * local versus remote memory. Because of MPO, page allocation does 297ce8eb11aSdp78419 * not cross node boundaries. Therefore, remove the node id bits from 298ce8eb11aSdp78419 * the color, since they are fixed. Either bit 30, or 31:30 in 299ce8eb11aSdp78419 * Victoria Falls processors. 300ce8eb11aSdp78419 * The number of node id bits is always 0 in Niagara2. 301ce8eb11aSdp78419 */ 302ce8eb11aSdp78419 typedef struct n2color { 303ce8eb11aSdp78419 uchar_t nnbits; /* number of node id bits */ 304ce8eb11aSdp78419 uchar_t nnmask; /* mask for node id bits */ 305ce8eb11aSdp78419 uchar_t lomask; /* mask for bits below node id */ 306ce8eb11aSdp78419 uchar_t lobits; /* number of bits below node id */ 307ce8eb11aSdp78419 } n2color_t; 308ce8eb11aSdp78419 309ce8eb11aSdp78419 n2color_t n2color[MMU_PAGE_SIZES]; 310ce8eb11aSdp78419 static uchar_t nhbits[] = {7, 7, 6, 5, 5, 5}; 311ce8eb11aSdp78419 312ce8eb11aSdp78419 /* 313ce8eb11aSdp78419 * Remove node id bits from color bits 32:28. 314ce8eb11aSdp78419 * This will reduce the number of colors. 315ce8eb11aSdp78419 * No change if number of node bits is zero. 316ce8eb11aSdp78419 */ 317ce8eb11aSdp78419 static inline uint_t 318ce8eb11aSdp78419 n2_hash2color(uint_t color, uchar_t szc) 319ce8eb11aSdp78419 { 320ce8eb11aSdp78419 n2color_t m = n2color[szc]; 321ce8eb11aSdp78419 322ce8eb11aSdp78419 if (m.nnbits > 0) { 323ce8eb11aSdp78419 color = ((color >> m.nnbits) & ~m.lomask) | (color & m.lomask); 324ce8eb11aSdp78419 ASSERT((color & ~(hw_page_array[szc].hp_colors - 1)) == 0); 325ce8eb11aSdp78419 } 326ce8eb11aSdp78419 327ce8eb11aSdp78419 return (color); 328ce8eb11aSdp78419 } 329ce8eb11aSdp78419 330ce8eb11aSdp78419 /* 331ce8eb11aSdp78419 * Restore node id bits into page color. 332ce8eb11aSdp78419 * This will increase the number of colors to match N2. 333ce8eb11aSdp78419 * No change if number of node bits is zero. 334ce8eb11aSdp78419 */ 335ce8eb11aSdp78419 static inline uint_t 336ce8eb11aSdp78419 n2_color2hash(uint_t color, uchar_t szc, uint_t node) 337ce8eb11aSdp78419 { 338ce8eb11aSdp78419 n2color_t m = n2color[szc]; 339ce8eb11aSdp78419 340ce8eb11aSdp78419 if (m.nnbits > 0) { 341ce8eb11aSdp78419 color = ((color & ~m.lomask) << m.nnbits) | (color & m.lomask); 342ce8eb11aSdp78419 color |= (node & m.nnmask) << m.lobits; 343ce8eb11aSdp78419 } 344ce8eb11aSdp78419 345ce8eb11aSdp78419 return (color); 346ce8eb11aSdp78419 } 347ce8eb11aSdp78419 348102033aaSdp78419 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ 349ce8eb11aSdp78419 350ce8eb11aSdp78419 /* 351ce8eb11aSdp78419 * iterator NULL means pfn is VA, do not adjust ra_to_pa 352ce8eb11aSdp78419 * iterator (-1) means pfn is RA, need to convert to PA 353ce8eb11aSdp78419 * iterator non-null means pfn is RA, use ra_to_pa 354ce8eb11aSdp78419 */ 355102033aaSdp78419 uint_t 356ce8eb11aSdp78419 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc, void *cookie) 357ce8eb11aSdp78419 { 358ce8eb11aSdp78419 mem_node_iterator_t *it = cookie; 359ce8eb11aSdp78419 uint_t color; 360ce8eb11aSdp78419 361ce8eb11aSdp78419 ASSERT(szc <= TTE256M); 362ce8eb11aSdp78419 363ce8eb11aSdp78419 if (it == ((mem_node_iterator_t *)(-1))) { 364ce8eb11aSdp78419 pfn = plat_rapfn_to_papfn(pfn); 365ce8eb11aSdp78419 } else if (it != NULL) { 366ce8eb11aSdp78419 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 367ce8eb11aSdp78419 pfn = pfn + it->mi_ra_to_pa; 368ce8eb11aSdp78419 } 369ce8eb11aSdp78419 pfn = PFN_BASE(pfn, szc); 370ce8eb11aSdp78419 color = ((pfn >> 15) ^ pfn) & 0x1f; 371ce8eb11aSdp78419 if (szc < TTE4M) { 372ce8eb11aSdp78419 /* 19:18 */ 373ce8eb11aSdp78419 color = (color << 2) | ((pfn >> 5) & 0x3); 374ce8eb11aSdp78419 if (szc > TTE64K) 375ce8eb11aSdp78419 color >>= 1; /* 19 */ 376ce8eb11aSdp78419 } 377ce8eb11aSdp78419 return (n2_hash2color(color, szc)); 378ce8eb11aSdp78419 } 379ce8eb11aSdp78419 380ce8eb11aSdp78419 static uint_t 381ce8eb11aSdp78419 page_papfn_2_color_cpu(pfn_t papfn, uchar_t szc) 382102033aaSdp78419 { 383102033aaSdp78419 uint_t color; 384102033aaSdp78419 385102033aaSdp78419 ASSERT(szc <= TTE256M); 386102033aaSdp78419 387ce8eb11aSdp78419 papfn = PFN_BASE(papfn, szc); 388ce8eb11aSdp78419 color = ((papfn >> 15) ^ papfn) & 0x1f; 389ce8eb11aSdp78419 if (szc < TTE4M) { 390ce8eb11aSdp78419 /* 19:18 */ 391ce8eb11aSdp78419 color = (color << 2) | ((papfn >> 5) & 0x3); 392ce8eb11aSdp78419 if (szc > TTE64K) 393ce8eb11aSdp78419 color >>= 1; /* 19 */ 394ce8eb11aSdp78419 } 395102033aaSdp78419 return (color); 396102033aaSdp78419 } 397102033aaSdp78419 398102033aaSdp78419 #if TTE256M != 5 399102033aaSdp78419 #error TTE256M is not 5 400102033aaSdp78419 #endif 401102033aaSdp78419 402102033aaSdp78419 uint_t 403102033aaSdp78419 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) 404102033aaSdp78419 { 405102033aaSdp78419 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; 406102033aaSdp78419 ASSERT(szc < TTE256M); 407ce8eb11aSdp78419 mask = n2_color2hash(mask, szc, 0); 408102033aaSdp78419 mask &= ni2_color_masks[szc]; 409ce8eb11aSdp78419 if (szc == TTE64K || szc == TTE512K) 410ce8eb11aSdp78419 mask >>= 1; 411ce8eb11aSdp78419 return (n2_hash2color(mask, szc + 1)); 412102033aaSdp78419 } 413102033aaSdp78419 414102033aaSdp78419 uint_t 415102033aaSdp78419 page_get_nsz_color_cpu(uchar_t szc, uint_t color) 416102033aaSdp78419 { 417102033aaSdp78419 ASSERT(szc < TTE256M); 418ce8eb11aSdp78419 color = n2_color2hash(color, szc, 0); 419ce8eb11aSdp78419 if (szc == TTE64K || szc == TTE512K) 420ce8eb11aSdp78419 color >>= 1; 421ce8eb11aSdp78419 return (n2_hash2color(color, szc + 1)); 422102033aaSdp78419 } 423102033aaSdp78419 424102033aaSdp78419 uint_t 425102033aaSdp78419 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) 426102033aaSdp78419 { 427ce8eb11aSdp78419 uint_t s; 428ef29e907Ssusans ASSERT(nszc >= szc); 429102033aaSdp78419 ASSERT(nszc <= TTE256M); 430102033aaSdp78419 431ce8eb11aSdp78419 s = nhbits[szc] - n2color[szc].nnbits; 432ce8eb11aSdp78419 s -= nhbits[nszc] - n2color[nszc].nnbits; 433102033aaSdp78419 434ce8eb11aSdp78419 return (s); 435102033aaSdp78419 } 436102033aaSdp78419 437ce8eb11aSdp78419 uint_t 438ce8eb11aSdp78419 page_convert_color_cpu(uint_t ncolor, uchar_t szc, uchar_t nszc) 439ce8eb11aSdp78419 { 440ce8eb11aSdp78419 uint_t color; 441ce8eb11aSdp78419 442ce8eb11aSdp78419 ASSERT(nszc > szc); 443ce8eb11aSdp78419 ASSERT(nszc <= TTE256M); 444ce8eb11aSdp78419 ncolor = n2_color2hash(ncolor, nszc, 0); 445ce8eb11aSdp78419 color = ncolor << (nhbits[szc] - nhbits[nszc]); 446ce8eb11aSdp78419 color = n2_hash2color(color, szc); 447ce8eb11aSdp78419 return (color); 448ce8eb11aSdp78419 } 449ce8eb11aSdp78419 450ce8eb11aSdp78419 #define PAPFN_2_MNODE(pfn) \ 451ce8eb11aSdp78419 (((pfn) & it->mi_mnode_pfn_mask) >> it->mi_mnode_pfn_shift) 452ce8eb11aSdp78419 453102033aaSdp78419 /*ARGSUSED*/ 454102033aaSdp78419 pfn_t 455102033aaSdp78419 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, 456ce8eb11aSdp78419 uint_t ceq_mask, uint_t color_mask, void *cookie) 457102033aaSdp78419 { 458ce8eb11aSdp78419 mem_node_iterator_t *it = cookie; 459102033aaSdp78419 pfn_t pstep = PNUM_SIZE(szc); 460102033aaSdp78419 pfn_t npfn, pfn_ceq_mask, pfn_color; 461102033aaSdp78419 pfn_t tmpmask, mask = (pfn_t)-1; 462ce8eb11aSdp78419 uint_t pfnmn; 463102033aaSdp78419 464102033aaSdp78419 ASSERT((color & ~ceq_mask) == 0); 465ce8eb11aSdp78419 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 466102033aaSdp78419 467ce8eb11aSdp78419 /* convert RA to PA for accurate color calculation */ 468ce8eb11aSdp78419 if (it->mi_init) { 469ce8eb11aSdp78419 /* first call after it, so cache these values */ 470ce8eb11aSdp78419 it->mi_hash_ceq_mask = 471ce8eb11aSdp78419 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask); 472ce8eb11aSdp78419 it->mi_hash_color = 473ce8eb11aSdp78419 n2_color2hash(color, szc, it->mi_mnode); 474ce8eb11aSdp78419 it->mi_init = 0; 475ce8eb11aSdp78419 } else { 476ce8eb11aSdp78419 ASSERT(it->mi_hash_ceq_mask == 477ce8eb11aSdp78419 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask)); 478ce8eb11aSdp78419 ASSERT(it->mi_hash_color == 479ce8eb11aSdp78419 n2_color2hash(color, szc, it->mi_mnode)); 480ce8eb11aSdp78419 } 481ce8eb11aSdp78419 ceq_mask = it->mi_hash_ceq_mask; 482ce8eb11aSdp78419 color = it->mi_hash_color; 483ce8eb11aSdp78419 pfn += it->mi_ra_to_pa; 484ce8eb11aSdp78419 485ce8eb11aSdp78419 /* restart here when we switch memblocks */ 486ce8eb11aSdp78419 next_mem_block: 487ce8eb11aSdp78419 if (szc <= TTE64K) { 488ce8eb11aSdp78419 pfnmn = PAPFN_2_MNODE(pfn); 489ce8eb11aSdp78419 } 490ce8eb11aSdp78419 if (((page_papfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0 && 491ce8eb11aSdp78419 (szc > TTE64K || pfnmn == it->mi_mnode)) { 492102033aaSdp78419 493102033aaSdp78419 /* we start from the page with correct color */ 494102033aaSdp78419 if (szc >= TTE512K) { 495102033aaSdp78419 if (szc >= TTE4M) { 496102033aaSdp78419 /* page color is PA[32:28] */ 497102033aaSdp78419 pfn_ceq_mask = ceq_mask << 15; 498102033aaSdp78419 } else { 499102033aaSdp78419 /* page color is PA[32:28].PA[19:19] */ 500102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 501102033aaSdp78419 ((ceq_mask >> 1) << 15); 502102033aaSdp78419 } 503ce8eb11aSdp78419 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 504ce8eb11aSdp78419 goto done; 505102033aaSdp78419 } else { 506102033aaSdp78419 /* 507102033aaSdp78419 * We deal 64K or 8K page. Check if we could the 508102033aaSdp78419 * satisfy the request without changing PA[32:28] 509102033aaSdp78419 */ 510102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 511ce8eb11aSdp78419 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 512102033aaSdp78419 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 513102033aaSdp78419 514102033aaSdp78419 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 515ce8eb11aSdp78419 goto done; 516102033aaSdp78419 517102033aaSdp78419 /* 518102033aaSdp78419 * for next pfn we have to change bits PA[32:28] 519102033aaSdp78419 * set PA[63:28] and PA[19:18] of the next pfn 520102033aaSdp78419 */ 521102033aaSdp78419 npfn = (pfn >> 15) << 15; 522102033aaSdp78419 npfn |= (ceq_mask & color & 3) << 5; 523102033aaSdp78419 pfn_ceq_mask = (szc == TTE8K) ? 0 : 524102033aaSdp78419 (ceq_mask & 0x1c) << 13; 525ce8eb11aSdp78419 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 526102033aaSdp78419 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); 527102033aaSdp78419 528102033aaSdp78419 /* 529102033aaSdp78419 * set bits PA[17:13] to match the color 530102033aaSdp78419 */ 531ce8eb11aSdp78419 npfn |= ((npfn >> 15) ^ (color >> 2)) & (ceq_mask >> 2); 532ce8eb11aSdp78419 goto done; 533102033aaSdp78419 } 534102033aaSdp78419 } 535102033aaSdp78419 536102033aaSdp78419 /* 537102033aaSdp78419 * we start from the page with incorrect color - rare case 538102033aaSdp78419 */ 539102033aaSdp78419 if (szc >= TTE512K) { 540102033aaSdp78419 if (szc >= TTE4M) { 541102033aaSdp78419 /* page color is in bits PA[32:28] */ 542102033aaSdp78419 npfn = ((pfn >> 20) << 20) | (color << 15); 543102033aaSdp78419 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; 544102033aaSdp78419 } else { 545102033aaSdp78419 /* try get the right color by changing bit PA[19:19] */ 546102033aaSdp78419 npfn = pfn + pstep; 547ce8eb11aSdp78419 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 548102033aaSdp78419 ceq_mask) == 0) 549ce8eb11aSdp78419 goto done; 550102033aaSdp78419 551102033aaSdp78419 /* page color is PA[32:28].PA[19:19] */ 552102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 553102033aaSdp78419 ((ceq_mask >> 1) << 15) | (0xff << 7); 554102033aaSdp78419 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); 555102033aaSdp78419 npfn = ((pfn >> 20) << 20) | pfn_color; 556102033aaSdp78419 } 557102033aaSdp78419 558102033aaSdp78419 while (npfn <= pfn) { 559102033aaSdp78419 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 560102033aaSdp78419 } 561ce8eb11aSdp78419 goto done; 562102033aaSdp78419 } 563102033aaSdp78419 564102033aaSdp78419 /* 565102033aaSdp78419 * We deal 64K or 8K page of incorrect color. 566102033aaSdp78419 * Try correcting color without changing PA[32:28] 567102033aaSdp78419 */ 568102033aaSdp78419 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 569102033aaSdp78419 pfn_color = ((color & 3) << 5) | (color >> 2); 570ce8eb11aSdp78419 if (pfnmn == it->mi_mnode) { 571102033aaSdp78419 npfn = (pfn & ~(pfn_t)0x7f); 572102033aaSdp78419 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 573102033aaSdp78419 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 574102033aaSdp78419 575ce8eb11aSdp78419 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 576ce8eb11aSdp78419 ceq_mask) == 0) { 577102033aaSdp78419 /* the color is fixed - find the next page */ 578ce8eb11aSdp78419 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 579102033aaSdp78419 while (npfn <= pfn) { 580ce8eb11aSdp78419 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, 581ce8eb11aSdp78419 mask); 582102033aaSdp78419 } 583102033aaSdp78419 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 584ce8eb11aSdp78419 goto done; 585ce8eb11aSdp78419 } 586102033aaSdp78419 } 587102033aaSdp78419 588102033aaSdp78419 /* to fix the color need to touch PA[32:28] */ 589102033aaSdp78419 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : 590102033aaSdp78419 (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); 591ce8eb11aSdp78419 592ce8eb11aSdp78419 /* fix mnode if input pfn is in the wrong mnode. */ 593ce8eb11aSdp78419 if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) { 594ce8eb11aSdp78419 npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) << 595ce8eb11aSdp78419 it->mi_mnode_pfn_shift; 596ce8eb11aSdp78419 } 597ce8eb11aSdp78419 598102033aaSdp78419 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; 599ce8eb11aSdp78419 tmpmask |= it->mi_mnode_pfn_mask; 600102033aaSdp78419 601102033aaSdp78419 while (npfn <= pfn) { 602102033aaSdp78419 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); 603102033aaSdp78419 } 604102033aaSdp78419 605102033aaSdp78419 /* set bits PA[19:13] to match the color */ 606102033aaSdp78419 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 607102033aaSdp78419 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 608102033aaSdp78419 609ce8eb11aSdp78419 done: 610ce8eb11aSdp78419 ASSERT(((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); 611ce8eb11aSdp78419 ASSERT(PAPFN_2_MNODE(npfn) == it->mi_mnode); 612ce8eb11aSdp78419 613ce8eb11aSdp78419 /* PA to RA */ 614ce8eb11aSdp78419 npfn -= it->mi_ra_to_pa; 615ce8eb11aSdp78419 616ce8eb11aSdp78419 /* check for possible memblock switch */ 617ce8eb11aSdp78419 if (npfn > it->mi_mblock_end) { 618ce8eb11aSdp78419 pfn = plat_mem_node_iterator_init(npfn, it->mi_mnode, it, 0); 619ce8eb11aSdp78419 if (pfn == (pfn_t)-1) 620ce8eb11aSdp78419 return (pfn); 621ce8eb11aSdp78419 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 622ce8eb11aSdp78419 pfn += it->mi_ra_to_pa; 623ce8eb11aSdp78419 goto next_mem_block; 624ce8eb11aSdp78419 } 625102033aaSdp78419 626102033aaSdp78419 return (npfn); 627102033aaSdp78419 } 628102033aaSdp78419 629102033aaSdp78419 /* 630102033aaSdp78419 * init page coloring 631ce8eb11aSdp78419 * VF encodes node_id for an L-group in either bit 30 or 31:30, 632ce8eb11aSdp78419 * which effectively reduces the number of colors available per mnode. 633102033aaSdp78419 */ 634102033aaSdp78419 void 635102033aaSdp78419 page_coloring_init_cpu() 636102033aaSdp78419 { 637102033aaSdp78419 int i; 638ce8eb11aSdp78419 uchar_t id; 639ce8eb11aSdp78419 uchar_t lo; 640ce8eb11aSdp78419 uchar_t hi; 641ce8eb11aSdp78419 n2color_t m; 642ce8eb11aSdp78419 mem_node_iterator_t it; 643ce8eb11aSdp78419 static uchar_t idmask[] = {0, 0x7, 0x1f, 0x1f, 0x1f, 0x1f}; 644102033aaSdp78419 645ce8eb11aSdp78419 (void) plat_mem_node_iterator_init(0, 0, &it, 1); 646ce8eb11aSdp78419 for (i = 0; i < mmu_page_sizes; i++) { 647*b02e9a2dSsvemuri (void) memset(&m, 0, sizeof (m)); 648ce8eb11aSdp78419 id = it.mi_mnode_pfn_mask >> 15; /* node id mask */ 649ce8eb11aSdp78419 id &= idmask[i]; 650ce8eb11aSdp78419 lo = lowbit(id); 651ce8eb11aSdp78419 if (lo > 0) { 652ce8eb11aSdp78419 hi = highbit(id); 653ce8eb11aSdp78419 m.nnbits = hi - lo + 1; 654ce8eb11aSdp78419 m.nnmask = (1 << m.nnbits) - 1; 655ce8eb11aSdp78419 lo += nhbits[i] - 5; 656ce8eb11aSdp78419 m.lomask = (1 << (lo - 1)) - 1; 657ce8eb11aSdp78419 m.lobits = lo - 1; 658ce8eb11aSdp78419 } 659ce8eb11aSdp78419 hw_page_array[i].hp_colors = 1 << (nhbits[i] - m.nnbits); 660ce8eb11aSdp78419 n2color[i] = m; 661102033aaSdp78419 } 662102033aaSdp78419 } 663fe70c9cfSdp78419 664fe70c9cfSdp78419 /* 665fe70c9cfSdp78419 * group colorequiv colors on N2 by low order bits of the color first 666fe70c9cfSdp78419 */ 667fe70c9cfSdp78419 void 668fe70c9cfSdp78419 page_set_colorequiv_arr_cpu(void) 669fe70c9cfSdp78419 { 670fe70c9cfSdp78419 static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0}; 671fe70c9cfSdp78419 672ce8eb11aSdp78419 nequiv_shades_log2[1] -= n2color[1].nnbits; 673fe70c9cfSdp78419 if (colorequiv > 1) { 674fe70c9cfSdp78419 int i; 675fe70c9cfSdp78419 uint_t sv_a = lowbit(colorequiv) - 1; 676fe70c9cfSdp78419 677fe70c9cfSdp78419 if (sv_a > 15) 678fe70c9cfSdp78419 sv_a = 15; 679fe70c9cfSdp78419 680fe70c9cfSdp78419 for (i = 0; i < MMU_PAGE_SIZES; i++) { 681fe70c9cfSdp78419 uint_t colors; 682fe70c9cfSdp78419 uint_t a = sv_a; 683fe70c9cfSdp78419 684fe70c9cfSdp78419 if ((colors = hw_page_array[i].hp_colors) <= 1) 685fe70c9cfSdp78419 continue; 686fe70c9cfSdp78419 while ((colors >> a) == 0) 687fe70c9cfSdp78419 a--; 688fe70c9cfSdp78419 if (a > (colorequivszc[i] & 0xf) + 689fe70c9cfSdp78419 (colorequivszc[i] >> 4)) { 690fe70c9cfSdp78419 if (a <= nequiv_shades_log2[i]) { 69159ac0c16Sdavemq colorequivszc[i] = (uchar_t)a; 692fe70c9cfSdp78419 } else { 693fe70c9cfSdp78419 colorequivszc[i] = 694fe70c9cfSdp78419 ((a - nequiv_shades_log2[i]) << 4) | 695fe70c9cfSdp78419 nequiv_shades_log2[i]; 696fe70c9cfSdp78419 } 697fe70c9cfSdp78419 } 698fe70c9cfSdp78419 } 699fe70c9cfSdp78419 } 700fe70c9cfSdp78419 } 701