1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/types.h> 29 #include <sys/systm.h> 30 #include <sys/archsystm.h> 31 #include <sys/machparam.h> 32 #include <sys/machsystm.h> 33 #include <sys/cpu.h> 34 #include <sys/elf_SPARC.h> 35 #include <vm/hat_sfmmu.h> 36 #include <vm/page.h> 37 #include <vm/vm_dep.h> 38 #include <sys/cpuvar.h> 39 #include <sys/async.h> 40 #include <sys/cmn_err.h> 41 #include <sys/debug.h> 42 #include <sys/dditypes.h> 43 #include <sys/sunddi.h> 44 #include <sys/cpu_module.h> 45 #include <sys/prom_debug.h> 46 #include <sys/vmsystm.h> 47 #include <sys/prom_plat.h> 48 #include <sys/sysmacros.h> 49 #include <sys/intreg.h> 50 #include <sys/machtrap.h> 51 #include <sys/ontrap.h> 52 #include <sys/ivintr.h> 53 #include <sys/atomic.h> 54 #include <sys/panic.h> 55 #include <sys/dtrace.h> 56 #include <sys/simulate.h> 57 #include <sys/fault.h> 58 #include <sys/niagara2regs.h> 59 #include <sys/hsvc.h> 60 #include <sys/trapstat.h> 61 62 uint_t root_phys_addr_lo_mask = 0xffffffffU; 63 char cpu_module_name[] = "SUNW,UltraSPARC-T2"; 64 65 /* 66 * Hypervisor services information for the NIAGARA2 CPU module 67 */ 68 static boolean_t niagara2_hsvc_available = B_TRUE; 69 static uint64_t niagara2_sup_minor; /* Supported minor number */ 70 static hsvc_info_t niagara2_hsvc = { 71 HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, 72 NIAGARA2_HSVC_MINOR, cpu_module_name 73 }; 74 75 void 76 cpu_setup(void) 77 { 78 extern int mmu_exported_pagesize_mask; 79 extern int cpc_has_overflow_intr; 80 int status; 81 82 /* 83 * Negotiate the API version for Niagara2 specific hypervisor 84 * services. 85 */ 86 status = hsvc_register(&niagara2_hsvc, &niagara2_sup_minor); 87 if (status != 0) { 88 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 89 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", 90 niagara2_hsvc.hsvc_modname, niagara2_hsvc.hsvc_group, 91 niagara2_hsvc.hsvc_major, niagara2_hsvc.hsvc_minor, status); 92 niagara2_hsvc_available = B_FALSE; 93 } 94 95 /* 96 * The setup common to all CPU modules is done in cpu_setup_common 97 * routine. 98 */ 99 cpu_setup_common(NULL); 100 101 cache |= (CACHE_PTAG | CACHE_IOCOHERENT); 102 103 if ((mmu_exported_pagesize_mask & 104 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != 105 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) 106 cmn_err(CE_PANIC, "machine description" 107 " does not have required sun4v page sizes" 108 " 8K, 64K and 4M: MD mask is 0x%x", 109 mmu_exported_pagesize_mask); 110 111 cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT; 112 113 /* 114 * Niagara2 supports a 48-bit subset of the full 64-bit virtual 115 * address space. Virtual addresses between 0x0000800000000000 116 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" 117 * and must never be mapped. In addition, software must not use 118 * pages within 4GB of the VA hole as instruction pages to 119 * avoid problems with prefetching into the VA hole. 120 */ 121 hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); 122 hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); 123 124 /* 125 * Niagara2 has a performance counter overflow interrupt 126 */ 127 cpc_has_overflow_intr = 1; 128 129 /* 130 * Enable 4M pages for OOB. 131 */ 132 max_uheap_lpsize = MMU_PAGESIZE4M; 133 max_ustack_lpsize = MMU_PAGESIZE4M; 134 max_privmap_lpsize = MMU_PAGESIZE4M; 135 } 136 137 /* 138 * Set the magic constants of the implementation. 139 */ 140 void 141 cpu_fiximp(struct cpu_node *cpunode) 142 { 143 /* 144 * The Cache node is optional in MD. Therefore in case "Cache" 145 * node does not exists in MD, set the default L2 cache associativity, 146 * size, linesize. 147 */ 148 if (cpunode->ecache_size == 0) 149 cpunode->ecache_size = L2CACHE_SIZE; 150 if (cpunode->ecache_linesize == 0) 151 cpunode->ecache_linesize = L2CACHE_LINESIZE; 152 if (cpunode->ecache_associativity == 0) 153 cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; 154 } 155 156 static int niagara2_cpucnt; 157 158 void 159 cpu_init_private(struct cpu *cp) 160 { 161 extern int niagara_kstat_init(void); 162 163 /* 164 * The cpu_ipipe field is initialized based on the execution 165 * unit sharing information from the MD. It defaults to the 166 * virtual CPU id in the absence of such information. 167 */ 168 cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; 169 if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) 170 cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); 171 172 ASSERT(MUTEX_HELD(&cpu_lock)); 173 if ((niagara2_cpucnt++ == 0) && (niagara2_hsvc_available == B_TRUE)) 174 (void) niagara_kstat_init(); 175 } 176 177 /*ARGSUSED*/ 178 void 179 cpu_uninit_private(struct cpu *cp) 180 { 181 extern int niagara_kstat_fini(void); 182 183 ASSERT(MUTEX_HELD(&cpu_lock)); 184 if ((--niagara2_cpucnt == 0) && (niagara2_hsvc_available == B_TRUE)) 185 (void) niagara_kstat_fini(); 186 } 187 188 /* 189 * On Niagara2, any flush will cause all preceding stores to be 190 * synchronized wrt the i$, regardless of address or ASI. In fact, 191 * the address is ignored, so we always flush address 0. 192 */ 193 /*ARGSUSED*/ 194 void 195 dtrace_flush_sec(uintptr_t addr) 196 { 197 doflush(0); 198 } 199 200 /* 201 * Trapstat support for Niagara2 processor 202 * The Niagara2 provides HWTW support for TSB lookup and with HWTW 203 * enabled no TSB hit information will be available. Therefore setting 204 * the time spent in TLB miss handler for TSB hits to 0. 205 */ 206 int 207 cpu_trapstat_conf(int cmd) 208 { 209 int status = 0; 210 211 switch (cmd) { 212 case CPU_TSTATCONF_INIT: 213 case CPU_TSTATCONF_FINI: 214 case CPU_TSTATCONF_ENABLE: 215 case CPU_TSTATCONF_DISABLE: 216 break; 217 default: 218 status = EINVAL; 219 break; 220 } 221 return (status); 222 } 223 224 void 225 cpu_trapstat_data(void *buf, uint_t tstat_pgszs) 226 { 227 tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; 228 int i; 229 230 for (i = 0; i < tstat_pgszs; i++, tstatp++) { 231 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; 232 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; 233 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; 234 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; 235 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 236 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 237 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 238 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 239 } 240 } 241 242 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ 243 uint_t 244 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc) 245 { 246 uint_t color; 247 248 ASSERT(szc <= TTE256M); 249 250 pfn = PFN_BASE(pfn, szc); 251 color = ((pfn >> 15) ^ pfn) & 0x1f; 252 if (szc >= TTE4M) 253 return (color); 254 255 color = (color << 2) | ((pfn >> 5) & 0x3); 256 257 return (szc <= TTE64K ? color : (color >> 1)); 258 } 259 260 #if TTE256M != 5 261 #error TTE256M is not 5 262 #endif 263 264 uint_t 265 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) 266 { 267 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; 268 ASSERT(szc < TTE256M); 269 270 mask &= ni2_color_masks[szc]; 271 return ((szc == TTE64K || szc == TTE512K) ? (mask >> 1) : mask); 272 } 273 274 uint_t 275 page_get_nsz_color_cpu(uchar_t szc, uint_t color) 276 { 277 ASSERT(szc < TTE256M); 278 return ((szc == TTE64K || szc == TTE512K) ? (color >> 1) : color); 279 } 280 281 uint_t 282 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) 283 { 284 ASSERT(nszc > szc); 285 ASSERT(nszc <= TTE256M); 286 287 if (szc <= TTE64K) 288 return ((nszc >= TTE4M) ? 2 : ((nszc >= TTE512K) ? 1 : 0)); 289 if (szc == TTE512K) 290 return (1); 291 292 return (0); 293 } 294 295 /*ARGSUSED*/ 296 pfn_t 297 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, 298 uint_t ceq_mask, uint_t color_mask) 299 { 300 pfn_t pstep = PNUM_SIZE(szc); 301 pfn_t npfn, pfn_ceq_mask, pfn_color; 302 pfn_t tmpmask, mask = (pfn_t)-1; 303 304 ASSERT((color & ~ceq_mask) == 0); 305 306 if (((page_pfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0) { 307 308 /* we start from the page with correct color */ 309 if (szc >= TTE512K) { 310 if (szc >= TTE4M) { 311 /* page color is PA[32:28] */ 312 pfn_ceq_mask = ceq_mask << 15; 313 } else { 314 /* page color is PA[32:28].PA[19:19] */ 315 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 316 ((ceq_mask >> 1) << 15); 317 } 318 pfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 319 return (pfn); 320 } else { 321 /* 322 * We deal 64K or 8K page. Check if we could the 323 * satisfy the request without changing PA[32:28] 324 */ 325 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 326 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 327 328 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 329 return (npfn); 330 331 /* 332 * for next pfn we have to change bits PA[32:28] 333 * set PA[63:28] and PA[19:18] of the next pfn 334 */ 335 npfn = (pfn >> 15) << 15; 336 npfn |= (ceq_mask & color & 3) << 5; 337 pfn_ceq_mask = (szc == TTE8K) ? 0 : 338 (ceq_mask & 0x1c) << 13; 339 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); 340 341 /* 342 * set bits PA[17:13] to match the color 343 */ 344 ceq_mask >>= 2; 345 color = (color >> 2) & ceq_mask; 346 npfn |= ((npfn >> 15) ^ color) & ceq_mask; 347 return (npfn); 348 } 349 } 350 351 /* 352 * we start from the page with incorrect color - rare case 353 */ 354 if (szc >= TTE512K) { 355 if (szc >= TTE4M) { 356 /* page color is in bits PA[32:28] */ 357 npfn = ((pfn >> 20) << 20) | (color << 15); 358 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; 359 } else { 360 /* try get the right color by changing bit PA[19:19] */ 361 npfn = pfn + pstep; 362 if (((page_pfn_2_color_cpu(npfn, szc) ^ color) & 363 ceq_mask) == 0) 364 return (npfn); 365 366 /* page color is PA[32:28].PA[19:19] */ 367 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 368 ((ceq_mask >> 1) << 15) | (0xff << 7); 369 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); 370 npfn = ((pfn >> 20) << 20) | pfn_color; 371 } 372 373 while (npfn <= pfn) { 374 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 375 } 376 return (npfn); 377 } 378 379 /* 380 * We deal 64K or 8K page of incorrect color. 381 * Try correcting color without changing PA[32:28] 382 */ 383 384 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 385 pfn_color = ((color & 3) << 5) | (color >> 2); 386 npfn = (pfn & ~(pfn_t)0x7f); 387 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 388 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 389 390 if (((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0) { 391 392 /* the color is fixed - find the next page */ 393 while (npfn <= pfn) { 394 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 395 } 396 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 397 return (npfn); 398 } 399 400 /* to fix the color need to touch PA[32:28] */ 401 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : 402 (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); 403 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; 404 405 while (npfn <= pfn) { 406 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); 407 } 408 409 /* set bits PA[19:13] to match the color */ 410 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 411 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 412 413 ASSERT(((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); 414 415 return (npfn); 416 } 417 418 /* 419 * init page coloring 420 */ 421 void 422 page_coloring_init_cpu() 423 { 424 int i; 425 uint_t colors; 426 427 hw_page_array[0].hp_colors = 1 << 7; 428 hw_page_array[1].hp_colors = 1 << 7; 429 hw_page_array[2].hp_colors = 1 << 6; 430 431 for (i = 3; i < mmu_page_sizes; i++) { 432 hw_page_array[i].hp_colors = 1 << 5; 433 } 434 435 if (colorequiv > 1) { 436 int a = lowbit(colorequiv) - 1; 437 438 if (a > 15) 439 a = 15; 440 441 for (i = 0; i < mmu_page_sizes; i++) { 442 if ((colors = hw_page_array[i].hp_colors) <= 1) { 443 continue; 444 } 445 while ((colors >> a) == 0) 446 a--; 447 colorequivszc[i] = (a << 4); 448 } 449 } 450 } 451