1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26/* 27 * SFMMU primitives. These primitives should only be used by sfmmu 28 * routines. 29 */ 30 31#include "assym.h" 32 33#include <sys/asm_linkage.h> 34#include <sys/machtrap.h> 35#include <sys/machasi.h> 36#include <sys/sun4asi.h> 37#include <sys/pte.h> 38#include <sys/mmu.h> 39#include <vm/hat_sfmmu.h> 40#include <vm/seg_spt.h> 41#include <sys/machparam.h> 42#include <sys/privregs.h> 43#include <sys/scb.h> 44#include <sys/intreg.h> 45#include <sys/machthread.h> 46#include <sys/clock.h> 47#include <sys/trapstat.h> 48 49/* 50 * sfmmu related subroutines 51 */ 52 53/* 54 * Invalidate either the context of a specific victim or any process 55 * currently running on this CPU. 56 * 57 * %g1 = sfmmup whose ctx is being stolen (victim) 58 * when called from sfmmu_wrap_around, %g1 == INVALID_CONTEXT. 59 * Note %g1 is the only input argument used by this xcall handler. 60 */ 61 62 ENTRY(sfmmu_raise_tsb_exception) 63 ! 64 ! if (victim == INVALID_CONTEXT) { 65 ! if (sec-ctx > INVALID_CONTEXT) 66 ! write INVALID_CONTEXT to sec-ctx 67 ! if (pri-ctx > INVALID_CONTEXT) 68 ! write INVALID_CONTEXT to pri-ctx 69 ! 70 ! } else if (current CPU tsbmiss->usfmmup != victim sfmmup) { 71 ! return 72 ! } else { 73 ! if (sec-ctx > INVALID_CONTEXT) 74 ! write INVALID_CONTEXT to sec-ctx 75 ! 76 ! if (pri-ctx > INVALID_CONTEXT) 77 ! write INVALID_CONTEXT to pri-ctx 78 ! } 79 ! 80 81 sethi %hi(ksfmmup), %g3 82 ldx [%g3 + %lo(ksfmmup)], %g3 83 cmp %g1, %g3 84 be,a,pn %xcc, ptl1_panic /* can't invalidate kernel ctx */ 85 mov PTL1_BAD_RAISE_TSBEXCP, %g1 86 87 set INVALID_CONTEXT, %g2 88 89 cmp %g1, INVALID_CONTEXT 90 bne,pt %xcc, 1f /* called from wrap_around? */ 91 mov MMU_SCONTEXT, %g3 92 93 ldxa [%g3]ASI_MMU_CTX, %g5 /* %g5 = sec-ctx */ 94 cmp %g5, INVALID_CONTEXT /* kernel or invalid ctx ? */ 95 ble,pn %xcc, 0f /* yes, no need to change */ 96 mov MMU_PCONTEXT, %g7 97 98 stxa %g2, [%g3]ASI_MMU_CTX /* set invalid ctx */ 99 membar #Sync 100 1010: 102 ldxa [%g7]ASI_MMU_CTX, %g5 /* %g5 = pri-ctx */ 103 cmp %g5, INVALID_CONTEXT /* kernel or invalid ctx? */ 104 ble,pn %xcc, 6f /* yes, no need to change */ 105 nop 106 107 stxa %g2, [%g7]ASI_MMU_CTX /* set pri-ctx to invalid */ 108 membar #Sync 109 1106: /* flushall tlb */ 111 mov %o0, %g3 112 mov %o1, %g4 113 mov %o2, %g6 114 mov %o5, %g7 115 116 mov %g0, %o0 ! XXX no cpu list yet 117 mov %g0, %o1 ! XXX no cpu list yet 118 mov MAP_ITLB | MAP_DTLB, %o2 119 mov MMU_DEMAP_ALL, %o5 120 ta FAST_TRAP 121 brz,pt %o0, 5f 122 nop 123 ba ptl1_panic /* bad HV call */ 124 mov PTL1_BAD_RAISE_TSBEXCP, %g1 1255: 126 mov %g3, %o0 127 mov %g4, %o1 128 mov %g6, %o2 129 mov %g7, %o5 130 131 ba 3f 132 nop 1331: 134 /* 135 * %g1 = sfmmup 136 * %g2 = INVALID_CONTEXT 137 * %g3 = MMU_SCONTEXT 138 */ 139 CPU_TSBMISS_AREA(%g5, %g6) /* load cpu tsbmiss area */ 140 ldx [%g5 + TSBMISS_UHATID], %g5 /* load usfmmup */ 141 142 cmp %g5, %g1 /* is it the victim? */ 143 bne,pt %xcc, 2f /* is our sec-ctx a victim? */ 144 nop 145 146 ldxa [%g3]ASI_MMU_CTX, %g5 /* %g5 = sec-ctx */ 147 cmp %g5, INVALID_CONTEXT /* kernel or invalid ctx ? */ 148 ble,pn %xcc, 0f /* yes, no need to change */ 149 mov MMU_PCONTEXT, %g7 150 151 stxa %g2, [%g3]ASI_MMU_CTX /* set sec-ctx to invalid */ 152 membar #Sync 153 1540: 155 ldxa [%g7]ASI_MMU_CTX, %g4 /* %g4 = pri-ctx */ 156 cmp %g4, INVALID_CONTEXT /* is pri-ctx the victim? */ 157 ble %icc, 3f /* no need to change pri-ctx */ 158 nop 159 stxa %g2, [%g7]ASI_MMU_CTX /* set pri-ctx to invalid */ 160 membar #Sync 161 1623: 163 /* TSB program must be cleared - walkers do not check a context. */ 164 mov %o0, %g3 165 mov %o1, %g4 166 mov %o5, %g7 167 clr %o0 168 clr %o1 169 mov MMU_TSB_CTXNON0, %o5 170 ta FAST_TRAP 171 brnz,a,pn %o0, ptl1_panic 172 mov PTL1_BAD_HCALL, %g1 173 mov %g3, %o0 174 mov %g4, %o1 175 mov %g7, %o5 1762: 177 retry 178 SET_SIZE(sfmmu_raise_tsb_exception) 179 180 ENTRY_NP(sfmmu_getctx_pri) 181 set MMU_PCONTEXT, %o0 182 retl 183 ldxa [%o0]ASI_MMU_CTX, %o0 184 SET_SIZE(sfmmu_getctx_pri) 185 186 ENTRY_NP(sfmmu_getctx_sec) 187 set MMU_SCONTEXT, %o0 188 retl 189 ldxa [%o0]ASI_MMU_CTX, %o0 190 SET_SIZE(sfmmu_getctx_sec) 191 192 /* 193 * Set the secondary context register for this process. 194 * %o0 = context number 195 */ 196 ENTRY_NP(sfmmu_setctx_sec) 197 /* 198 * From resume we call sfmmu_setctx_sec with interrupts disabled. 199 * But we can also get called from C with interrupts enabled. So, 200 * we need to check first. 201 */ 202 203 /* If interrupts are not disabled, then disable them */ 204 rdpr %pstate, %g1 205 btst PSTATE_IE, %g1 206 bnz,a,pt %icc, 1f 207 wrpr %g1, PSTATE_IE, %pstate /* disable interrupts */ 2081: 209 mov MMU_SCONTEXT, %o1 210 stxa %o0, [%o1]ASI_MMU_CTX /* set 2nd context reg. */ 211 membar #Sync 212 /* 213 * if the routine is entered with intr enabled, then enable intr now. 214 * otherwise, keep intr disabled, return without enabing intr. 215 * %g1 - old intr state 216 */ 217 btst PSTATE_IE, %g1 218 bnz,a,pt %icc, 2f 219 wrpr %g0, %g1, %pstate /* enable interrupts */ 2202: retl 221 nop 222 SET_SIZE(sfmmu_setctx_sec) 223 224 /* 225 * set ktsb_phys to 1 if the processor supports ASI_QUAD_LDD_PHYS. 226 * returns the detection value in %o0. 227 */ 228 ENTRY_NP(sfmmu_setup_4lp) 229 set ktsb_phys, %o2 230 mov 1, %o1 231 st %o1, [%o2] 232 retl 233 mov %o1, %o0 234 SET_SIZE(sfmmu_setup_4lp) 235 236 /* 237 * Called to load MMU registers and tsbmiss area 238 * for the active process. This function should 239 * only be called from TL=0. 240 * 241 * %o0 - hat pointer 242 */ 243 ENTRY_NP(sfmmu_load_mmustate) 244 245#ifdef DEBUG 246 PANIC_IF_INTR_ENABLED_PSTR(msfmmu_ei_l1, %g1) 247#endif /* DEBUG */ 248 249 sethi %hi(ksfmmup), %o3 250 ldx [%o3 + %lo(ksfmmup)], %o3 251 cmp %o3, %o0 252 be,pn %xcc, 7f ! if kernel as, do nothing 253 nop 254 255 set MMU_SCONTEXT, %o3 256 ldxa [%o3]ASI_MMU_CTX, %o5 257 258 cmp %o5, INVALID_CONTEXT ! ctx is invalid? 259 bne,pt %icc, 1f 260 nop 261 262 CPU_TSBMISS_AREA(%o2, %o3) ! %o2 = tsbmiss area 263 stx %o0, [%o2 + TSBMISS_UHATID] 264 stx %g0, [%o2 + TSBMISS_SHARED_UHATID] 265#ifdef DEBUG 266 /* check if hypervisor/hardware should handle user TSB */ 267 sethi %hi(hv_use_non0_tsb), %o2 268 ld [%o2 + %lo(hv_use_non0_tsb)], %o2 269 brz,pn %o2, 0f 270 nop 271#endif /* DEBUG */ 272 clr %o0 ! ntsb = 0 for invalid ctx 273 clr %o1 ! HV_TSB_INFO_PA = 0 if inv ctx 274 mov MMU_TSB_CTXNON0, %o5 275 ta FAST_TRAP ! set TSB info for user process 276 brnz,a,pn %o0, panic_bad_hcall 277 mov MMU_TSB_CTXNON0, %o1 2780: 279 retl 280 nop 2811: 282 /* 283 * We need to set up the TSB base register, tsbmiss 284 * area, and pass the TSB information into the hypervisor 285 */ 286 ldx [%o0 + SFMMU_TSB], %o1 ! %o1 = first tsbinfo 287 ldx [%o1 + TSBINFO_NEXTPTR], %g2 ! %g2 = second tsbinfo 288 289 /* create/set first UTSBREG */ 290 MAKE_UTSBREG(%o1, %o2, %o3) ! %o2 = user tsbreg 291 SET_UTSBREG(SCRATCHPAD_UTSBREG1, %o2, %o3) 292 293 brz,pt %g2, 2f 294 mov -1, %o2 ! use -1 if no second TSB 295 296 /* make 2nd UTSBREG */ 297 MAKE_UTSBREG(%g2, %o2, %o3) ! %o2 = user tsbreg 2982: 299 SET_UTSBREG(SCRATCHPAD_UTSBREG2, %o2, %o3) 300 301 /* make 3rd and 4th TSB */ 302 CPU_TSBMISS_AREA(%o4, %o3) ! %o4 = tsbmiss area 303 304 ldx [%o0 + SFMMU_SCDP], %g2 ! %g2 = sfmmu_scd 305 brz,pt %g2, 3f 306 mov -1, %o2 ! use -1 if no third TSB 307 308 ldx [%g2 + SCD_SFMMUP], %g3 ! %g3 = scdp->scd_sfmmup 309 ldx [%g3 + SFMMU_TSB], %o1 ! %o1 = first scd tsbinfo 310 brz,pn %o1, 9f 311 nop ! panic if no third TSB 312 313 /* make 3rd UTSBREG */ 314 MAKE_UTSBREG(%o1, %o2, %o3) ! %o2 = user tsbreg 3153: 316 SET_UTSBREG_SHCTX(%o4, TSBMISS_TSBSCDPTR, %o2) 317 318 brz,pt %g2, 4f 319 mov -1, %o2 ! use -1 if no 3rd or 4th TSB 320 321 brz,pt %o1, 4f 322 mov -1, %o2 ! use -1 if no 3rd or 4th TSB 323 ldx [%o1 + TSBINFO_NEXTPTR], %g2 ! %g2 = second scd tsbinfo 324 brz,pt %g2, 4f 325 mov -1, %o2 ! use -1 if no 4th TSB 326 327 /* make 4th UTSBREG */ 328 MAKE_UTSBREG(%g2, %o2, %o3) ! %o2 = user tsbreg 3294: 330 SET_UTSBREG_SHCTX(%o4, TSBMISS_TSBSCDPTR4M, %o2) 331 332#ifdef DEBUG 333 /* check if hypervisor/hardware should handle user TSB */ 334 sethi %hi(hv_use_non0_tsb), %o2 335 ld [%o2 + %lo(hv_use_non0_tsb)], %o2 336 brz,pn %o2, 6f 337 nop 338#endif /* DEBUG */ 339 CPU_ADDR(%o2, %o4) ! load CPU struct addr to %o2 using %o4 340 ldub [%o2 + CPU_TSTAT_FLAGS], %o1 ! load cpu_tstat_flag to %o1 341 342 mov %o0, %o3 ! preserve %o0 343 btst TSTAT_TLB_STATS, %o1 344 bnz,a,pn %icc, 5f ! ntsb = 0 if TLB stats enabled 345 clr %o0 346 347 ldx [%o3 + SFMMU_HVBLOCK + HV_TSB_INFO_CNT], %o0 3485: 349 ldx [%o3 + SFMMU_HVBLOCK + HV_TSB_INFO_PA], %o1 350 mov MMU_TSB_CTXNON0, %o5 351 ta FAST_TRAP ! set TSB info for user process 352 brnz,a,pn %o0, panic_bad_hcall 353 mov MMU_TSB_CTXNON0, %o1 354 mov %o3, %o0 ! restore %o0 3556: 356 ldx [%o0 + SFMMU_ISMBLKPA], %o1 ! copy members of sfmmu 357 CPU_TSBMISS_AREA(%o2, %o3) ! %o2 = tsbmiss area 358 stx %o1, [%o2 + TSBMISS_ISMBLKPA] ! sfmmu_tsb_miss into the 359 ldub [%o0 + SFMMU_TTEFLAGS], %o3 ! per-CPU tsbmiss area. 360 ldub [%o0 + SFMMU_RTTEFLAGS], %o4 361 ldx [%o0 + SFMMU_SRDP], %o1 362 stx %o0, [%o2 + TSBMISS_UHATID] 363 stub %o3, [%o2 + TSBMISS_UTTEFLAGS] 364 stub %o4, [%o2 + TSBMISS_URTTEFLAGS] 365 stx %o1, [%o2 + TSBMISS_SHARED_UHATID] 366 brz,pn %o1, 7f ! check for sfmmu_srdp 367 add %o0, SFMMU_HMERMAP, %o1 368 add %o2, TSBMISS_SHMERMAP, %o2 369 mov SFMMU_HMERGNMAP_WORDS, %o3 370 ! set tsbmiss shmermap 371 SET_REGION_MAP(%o1, %o2, %o3, %o4, load_shme_mmustate) 372 373 ldx [%o0 + SFMMU_SCDP], %o4 ! %o4 = sfmmu_scd 374 CPU_TSBMISS_AREA(%o2, %o3) ! %o2 = tsbmiss area 375 mov SFMMU_HMERGNMAP_WORDS, %o3 376 brnz,pt %o4, 8f ! check for sfmmu_scdp else 377 add %o2, TSBMISS_SCDSHMERMAP, %o2 ! zero tsbmiss scd_shmermap 378 ZERO_REGION_MAP(%o2, %o3, zero_scd_mmustate) 3797: 380 retl 381 nop 3828: ! set tsbmiss scd_shmermap 383 add %o4, SCD_HMERMAP, %o1 384 SET_REGION_MAP(%o1, %o2, %o3, %o4, load_scd_mmustate) 385 retl 386 nop 3879: 388 sethi %hi(panicstr), %g1 ! panic if no 3rd TSB 389 ldx [%g1 + %lo(panicstr)], %g1 390 tst %g1 391 392 bnz,pn %xcc, 7b 393 nop 394 395 sethi %hi(sfmmu_panic10), %o0 396 call panic 397 or %o0, %lo(sfmmu_panic10), %o0 398 399 SET_SIZE(sfmmu_load_mmustate) 400 401 ENTRY(prefetch_tsbe_read) 402 retl 403 nop 404 SET_SIZE(prefetch_tsbe_read) 405 406 ENTRY(prefetch_tsbe_write) 407 retl 408 nop 409 SET_SIZE(prefetch_tsbe_write) 410