1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * VM - Hardware Address Translation management. 28 * 29 * This file describes the contents of the sun-reference-mmu(sfmmu)- 30 * specific hat data structures and the sfmmu-specific hat procedures. 31 * The machine-independent interface is described in <vm/hat.h>. 32 */ 33 34 #ifndef _VM_HAT_SFMMU_H 35 #define _VM_HAT_SFMMU_H 36 37 #pragma ident "%Z%%M% %I% %E% SMI" 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 #ifndef _ASM 44 45 #include <sys/types.h> 46 47 #endif /* _ASM */ 48 49 #ifdef _KERNEL 50 51 #include <sys/pte.h> 52 #include <vm/mach_sfmmu.h> 53 #include <sys/mmu.h> 54 55 /* 56 * Don't alter these without considering changes to ism_map_t. 57 */ 58 #define DEFAULT_ISM_PAGESIZE MMU_PAGESIZE4M 59 #define ISM_PG_SIZE(ism_vbshift) (1 << ism_vbshift) 60 #define ISM_SZ_MASK(ism_vbshift) (ISM_PG_SIZE(ism_vbshift) - 1) 61 #define ISM_MAP_SLOTS 8 /* Change this carefully. */ 62 63 #ifndef _ASM 64 65 #include <sys/t_lock.h> 66 #include <vm/hat.h> 67 #include <vm/seg.h> 68 #include <sys/machparam.h> 69 #include <sys/systm.h> 70 #include <sys/x_call.h> 71 #include <vm/page.h> 72 #include <sys/ksynch.h> 73 74 typedef struct hat sfmmu_t; 75 76 /* 77 * SFMMU attributes for hat_memload/hat_devload 78 */ 79 #define SFMMU_UNCACHEPTTE 0x01000000 /* unencache in physical $ */ 80 #define SFMMU_UNCACHEVTTE 0x02000000 /* unencache in virtual $ */ 81 #define SFMMU_SIDEFFECT 0x04000000 /* set side effect bit */ 82 #define SFMMU_LOAD_ALLATTR (HAT_PROT_MASK | HAT_ORDER_MASK | \ 83 HAT_ENDIAN_MASK | HAT_NOFAULT | HAT_NOSYNC | \ 84 SFMMU_UNCACHEPTTE | SFMMU_UNCACHEVTTE | SFMMU_SIDEFFECT) 85 86 87 /* 88 * sfmmu flags for hat_memload/hat_devload 89 */ 90 #define SFMMU_NO_TSBLOAD 0x08000000 /* do not preload tsb */ 91 #define SFMMU_LOAD_ALLFLAG (HAT_LOAD | HAT_LOAD_LOCK | \ 92 HAT_LOAD_ADV | HAT_LOAD_CONTIG | HAT_LOAD_NOCONSIST | \ 93 HAT_LOAD_SHARE | HAT_LOAD_REMAP | SFMMU_NO_TSBLOAD | \ 94 HAT_RELOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_TEXT) 95 96 /* 97 * sfmmu internal flag to hat_pageunload that spares locked mappings 98 */ 99 #define SFMMU_KERNEL_RELOC 0x8000 100 101 /* 102 * mode for sfmmu_chgattr 103 */ 104 #define SFMMU_SETATTR 0x0 105 #define SFMMU_CLRATTR 0x1 106 #define SFMMU_CHGATTR 0x2 107 108 /* 109 * sfmmu specific flags for page_t 110 */ 111 #define P_PNC 0x8 /* non-caching is permanent bit */ 112 #define P_TNC 0x10 /* non-caching is temporary bit */ 113 #define P_KPMS 0x20 /* kpm mapped small (vac alias prevention) */ 114 #define P_KPMC 0x40 /* kpm conflict page (vac alias prevention) */ 115 116 #define PP_GENERIC_ATTR(pp) ((pp)->p_nrm & (P_MOD | P_REF | P_RO)) 117 #define PP_ISMOD(pp) ((pp)->p_nrm & P_MOD) 118 #define PP_ISREF(pp) ((pp)->p_nrm & P_REF) 119 #define PP_ISRO(pp) ((pp)->p_nrm & P_RO) 120 #define PP_ISNC(pp) ((pp)->p_nrm & (P_PNC|P_TNC)) 121 #define PP_ISPNC(pp) ((pp)->p_nrm & P_PNC) 122 #define PP_ISTNC(pp) ((pp)->p_nrm & P_TNC) 123 #define PP_ISKPMS(pp) ((pp)->p_nrm & P_KPMS) 124 #define PP_ISKPMC(pp) ((pp)->p_nrm & P_KPMC) 125 126 #define PP_SETMOD(pp) ((pp)->p_nrm |= P_MOD) 127 #define PP_SETREF(pp) ((pp)->p_nrm |= P_REF) 128 #define PP_SETREFMOD(pp) ((pp)->p_nrm |= (P_REF|P_MOD)) 129 #define PP_SETRO(pp) ((pp)->p_nrm |= P_RO) 130 #define PP_SETREFRO(pp) ((pp)->p_nrm |= (P_REF|P_RO)) 131 #define PP_SETPNC(pp) ((pp)->p_nrm |= P_PNC) 132 #define PP_SETTNC(pp) ((pp)->p_nrm |= P_TNC) 133 #define PP_SETKPMS(pp) ((pp)->p_nrm |= P_KPMS) 134 #define PP_SETKPMC(pp) ((pp)->p_nrm |= P_KPMC) 135 136 #define PP_CLRMOD(pp) ((pp)->p_nrm &= ~P_MOD) 137 #define PP_CLRREF(pp) ((pp)->p_nrm &= ~P_REF) 138 #define PP_CLRREFMOD(pp) ((pp)->p_nrm &= ~(P_REF|P_MOD)) 139 #define PP_CLRRO(pp) ((pp)->p_nrm &= ~P_RO) 140 #define PP_CLRPNC(pp) ((pp)->p_nrm &= ~P_PNC) 141 #define PP_CLRTNC(pp) ((pp)->p_nrm &= ~P_TNC) 142 #define PP_CLRKPMS(pp) ((pp)->p_nrm &= ~P_KPMS) 143 #define PP_CLRKPMC(pp) ((pp)->p_nrm &= ~P_KPMC) 144 145 /* 146 * All shared memory segments attached with the SHM_SHARE_MMU flag (ISM) 147 * will be constrained to a 4M, 32M or 256M alignment. Also since every newly- 148 * created ISM segment is created out of a new address space at base va 149 * of 0 we don't need to store it. 150 */ 151 #define ISM_ALIGN(shift) (1 << shift) /* base va aligned to <n>M */ 152 #define ISM_ALIGNED(shift, va) (((uintptr_t)va & (ISM_ALIGN(shift) - 1)) == 0) 153 #define ISM_SHIFT(shift, x) ((uintptr_t)x >> (shift)) 154 155 /* 156 * Pad locks out to cache sub-block boundaries to prevent 157 * false sharing, so several processes don't contend for 158 * the same line if they aren't using the same lock. Since 159 * this is a typedef we also have a bit of freedom in 160 * changing lock implementations later if we decide it 161 * is necessary. 162 */ 163 typedef struct hat_lock { 164 kmutex_t hl_mutex; 165 uchar_t hl_pad[64 - sizeof (kmutex_t)]; 166 } hatlock_t; 167 168 #define HATLOCK_MUTEXP(hatlockp) (&((hatlockp)->hl_mutex)) 169 170 /* 171 * All segments mapped with ISM are guaranteed to be 4M, 32M or 256M aligned. 172 * Also size is guaranteed to be in 4M, 32M or 256M chunks. 173 * ism_seg consists of the following members: 174 * [XX..22] base address of ism segment. XX is 63 or 31 depending whether 175 * caddr_t is 64 bits or 32 bits. 176 * [21..0] size of segment. 177 * 178 * NOTE: Don't alter this structure without changing defines above and 179 * the tsb_miss and protection handlers. 180 */ 181 typedef struct ism_map { 182 uintptr_t imap_seg; /* base va + sz of ISM segment */ 183 ushort_t imap_vb_shift; /* mmu_pageshift for ism page size */ 184 ushort_t imap_hatflags; /* primary ism page size */ 185 uint_t imap_sz_mask; /* mmu_pagemask for ism page size */ 186 sfmmu_t *imap_ismhat; /* hat id of dummy ISM as */ 187 struct ism_ment *imap_ment; /* pointer to mapping list entry */ 188 } ism_map_t; 189 190 #define ism_start(map) ((caddr_t)((map).imap_seg & \ 191 ~ISM_SZ_MASK((map).imap_vb_shift))) 192 #define ism_size(map) ((map).imap_seg & ISM_SZ_MASK((map).imap_vb_shift)) 193 #define ism_end(map) ((caddr_t)(ism_start(map) + (ism_size(map) * \ 194 ISM_PG_SIZE((map).imap_vb_shift)))) 195 /* 196 * ISM mapping entry. Used to link all hat's sharing a ism_hat. 197 * Same function as the p_mapping list for a page. 198 */ 199 typedef struct ism_ment { 200 sfmmu_t *iment_hat; /* back pointer to hat_share() hat */ 201 caddr_t iment_base_va; /* hat's va base for this ism seg */ 202 struct ism_ment *iment_next; /* next ism map entry */ 203 struct ism_ment *iment_prev; /* prev ism map entry */ 204 } ism_ment_t; 205 206 /* 207 * ISM segment block. One will be hung off the sfmmu structure if a 208 * a process uses ISM. More will be linked using ismblk_next if more 209 * than ISM_MAP_SLOTS segments are attached to this proc. 210 * 211 * All modifications to fields in this structure will be protected 212 * by the hat mutex. In order to avoid grabbing this lock in low level 213 * routines (tsb miss/protection handlers and vatopfn) while not 214 * introducing any race conditions with hat_unshare, we will set 215 * CTX_ISM_BUSY bit in the ctx struct. Any mmu traps that occur 216 * for this ctx while this bit is set will be handled in sfmmu_tsb_excption 217 * where it will synchronize behind the hat mutex. 218 */ 219 typedef struct ism_blk { 220 ism_map_t iblk_maps[ISM_MAP_SLOTS]; 221 struct ism_blk *iblk_next; 222 uint64_t iblk_nextpa; 223 } ism_blk_t; 224 225 /* 226 * TSB access information. All fields are protected by the process's 227 * hat lock. 228 */ 229 230 struct tsb_info { 231 caddr_t tsb_va; /* tsb base virtual address */ 232 uint64_t tsb_pa; /* tsb base physical address */ 233 struct tsb_info *tsb_next; /* next tsb used by this process */ 234 uint16_t tsb_szc; /* tsb size code */ 235 uint16_t tsb_flags; /* flags for this tsb; see below */ 236 uint_t tsb_ttesz_mask; /* page size masks; see below */ 237 238 tte_t tsb_tte; /* tte to lock into DTLB */ 239 sfmmu_t *tsb_sfmmu; /* sfmmu */ 240 kmem_cache_t *tsb_cache; /* cache from which mem allocated */ 241 vmem_t *tsb_vmp; /* vmem arena from which mem alloc'd */ 242 }; 243 244 /* 245 * Values for "tsb_ttesz_mask" bitmask. 246 */ 247 #define TSB8K (1 << TTE8K) 248 #define TSB64K (1 << TTE64K) 249 #define TSB512K (1 << TTE512K) 250 #define TSB4M (1 << TTE4M) 251 #define TSB32M (1 << TTE32M) 252 #define TSB256M (1 << TTE256M) 253 254 /* 255 * Values for "tsb_flags" field. 256 */ 257 #define TSB_RELOC_FLAG 0x1 258 #define TSB_FLUSH_NEEDED 0x2 259 #define TSB_SWAPPED 0x4 260 261 /* 262 * Per-MMU context domain kstats. 263 * 264 * TSB Miss Exceptions 265 * Number of times a TSB miss exception is handled in an MMU. See 266 * sfmmu_tsbmiss_exception() for more details. 267 * TSB Raise Exception 268 * Number of times the CPUs within an MMU are cross-called 269 * to invalidate either a specific process context (when the process 270 * switches MMU contexts) or the context of any process that is 271 * running on those CPUs (as part of the MMU context wrap-around). 272 * Wrap Around 273 * The number of times a wrap-around of MMU context happens. 274 */ 275 typedef enum mmu_ctx_stat_types { 276 MMU_CTX_TSB_EXCEPTIONS, /* TSB miss exceptions handled */ 277 MMU_CTX_TSB_RAISE_EXCEPTION, /* ctx invalidation cross calls */ 278 MMU_CTX_WRAP_AROUND, /* wraparounds */ 279 MMU_CTX_NUM_STATS 280 } mmu_ctx_stat_t; 281 282 /* 283 * Per-MMU context domain structure. This is instantiated the first time a CPU 284 * belonging to the MMU context domain is configured into the system, at boot 285 * time or at DR time. 286 * 287 * mmu_gnum 288 * The current generation number for the context IDs on this MMU context 289 * domain. It is protected by mmu_lock. 290 * mmu_cnum 291 * The current cnum to be allocated on this MMU context domain. It 292 * is protected via CAS. 293 * mmu_nctxs 294 * The max number of context IDs supported on every CPU in this 295 * MMU context domain. It is 8K except for Rock where it is 64K. 296 * This is needed here in case the system supports mixed type of 297 * processors/MMUs. It also helps to make ctx switch code access 298 * fewer cache lines i.e. no need to retrieve it from some global nctxs. 299 * mmu_lock 300 * The mutex spin lock used to serialize context ID wrap around 301 * mmu_idx 302 * The index for this MMU context domain structure in the global array 303 * mmu_ctxdoms. 304 * mmu_ncpus 305 * The actual number of CPUs that have been configured in this 306 * MMU context domain. This also acts as a reference count for the 307 * structure. When the last CPU in an MMU context domain is unconfigured, 308 * the structure is freed. It is protected by mmu_lock. 309 * mmu_cpuset 310 * The CPU set of configured CPUs for this MMU context domain. Used 311 * to cross-call all the CPUs in the MMU context domain to invalidate 312 * context IDs during a wraparound operation. It is protected by mmu_lock. 313 */ 314 315 typedef struct mmu_ctx { 316 uint64_t mmu_gnum; 317 uint_t mmu_cnum; 318 uint_t mmu_nctxs; 319 kmutex_t mmu_lock; 320 uint_t mmu_idx; 321 uint_t mmu_ncpus; 322 cpuset_t mmu_cpuset; 323 kstat_t *mmu_kstat; 324 kstat_named_t mmu_kstat_data[MMU_CTX_NUM_STATS]; 325 } mmu_ctx_t; 326 327 #define mmu_tsb_exceptions \ 328 mmu_kstat_data[MMU_CTX_TSB_EXCEPTIONS].value.ui64 329 #define mmu_tsb_raise_exception \ 330 mmu_kstat_data[MMU_CTX_TSB_RAISE_EXCEPTION].value.ui64 331 #define mmu_wrap_around \ 332 mmu_kstat_data[MMU_CTX_WRAP_AROUND].value.ui64 333 334 extern uint_t max_mmu_ctxdoms; 335 extern mmu_ctx_t **mmu_ctxs_tbl; 336 337 extern void sfmmu_cpu_init(cpu_t *); 338 extern void sfmmu_cpu_cleanup(cpu_t *); 339 340 /* 341 * The following structure is used to get MMU context domain information for 342 * a CPU from the platform. 343 * 344 * mmu_idx 345 * The MMU context domain index within the global array mmu_ctxs 346 * mmu_nctxs 347 * The number of context IDs supported in the MMU context domain 348 * (64K for Rock) 349 */ 350 typedef struct mmu_ctx_info { 351 uint_t mmu_idx; 352 uint_t mmu_nctxs; 353 } mmu_ctx_info_t; 354 355 #pragma weak plat_cpuid_to_mmu_ctx_info 356 357 extern void plat_cpuid_to_mmu_ctx_info(processorid_t, mmu_ctx_info_t *); 358 359 /* 360 * Each address space has an array of sfmmu_ctx_t structures, one structure 361 * per MMU context domain. 362 * 363 * cnum 364 * The context ID allocated for an address space on an MMU context domain 365 * gnum 366 * The generation number for the context ID in the MMU context domain. 367 * 368 * This structure needs to be a power-of-two in size. 369 */ 370 typedef struct sfmmu_ctx { 371 uint64_t gnum:48; 372 uint64_t cnum:16; 373 } sfmmu_ctx_t; 374 375 376 /* 377 * The platform dependent hat structure. 378 * tte counts should be protected by cas. 379 * cpuset is protected by cas. 380 * 381 * Note that sfmmu_xhat_provider MUST be the first element. 382 */ 383 struct hat { 384 void *sfmmu_xhat_provider; /* NULL for CPU hat */ 385 cpuset_t sfmmu_cpusran; /* cpu bit mask for efficient xcalls */ 386 struct as *sfmmu_as; /* as this hat provides mapping for */ 387 ulong_t sfmmu_ttecnt[MMU_PAGE_SIZES]; /* per sz tte counts */ 388 ulong_t sfmmu_ismttecnt[MMU_PAGE_SIZES]; /* est. ism ttes */ 389 union _h_un { 390 ism_blk_t *sfmmu_iblkp; /* maps to ismhat(s) */ 391 ism_ment_t *sfmmu_imentp; /* ism hat's mapping list */ 392 } h_un; 393 uint_t sfmmu_free:1; /* hat to be freed - set on as_free */ 394 uint_t sfmmu_ismhat:1; /* hat is dummy ism hatid */ 395 uint_t sfmmu_ctxflushed:1; /* ctx has been flushed */ 396 uchar_t sfmmu_rmstat; /* refmod stats refcnt */ 397 uchar_t sfmmu_clrstart; /* start color bin for page coloring */ 398 ushort_t sfmmu_clrbin; /* per as phys page coloring bin */ 399 ushort_t sfmmu_flags; /* flags */ 400 struct tsb_info *sfmmu_tsb; /* list of per as tsbs */ 401 uint64_t sfmmu_ismblkpa; /* pa of sfmmu_iblkp, or -1 */ 402 lock_t sfmmu_ctx_lock; /* sync ctx alloc and invalidation */ 403 kcondvar_t sfmmu_tsb_cv; /* signals TSB swapin or relocation */ 404 uchar_t sfmmu_cext; /* context page size encoding */ 405 uint8_t sfmmu_pgsz[MMU_PAGE_SIZES]; /* ranking for MMU */ 406 #ifdef sun4v 407 struct hv_tsb_block sfmmu_hvblock; 408 #endif 409 /* 410 * sfmmu_ctxs is a variable length array of max_mmu_ctxdoms # of 411 * elements. max_mmu_ctxdoms is determined at run-time. 412 * sfmmu_ctxs[1] is just the fist element of an array, it always 413 * has to be the last field to ensure that the memory allocated 414 * for sfmmu_ctxs is consecutive with the memory of the rest of 415 * the hat data structure. 416 */ 417 sfmmu_ctx_t sfmmu_ctxs[1]; 418 419 }; 420 421 #define sfmmu_iblk h_un.sfmmu_iblkp 422 #define sfmmu_iment h_un.sfmmu_imentp 423 424 /* 425 * bit mask for managing vac conflicts on large pages. 426 * bit 1 is for uncache flag. 427 * bits 2 through min(num of cache colors + 1,31) are 428 * for cache colors that have already been flushed. 429 */ 430 #define CACHE_UNCACHE 1 431 #define CACHE_NUM_COLOR (shm_alignment >> MMU_PAGESHIFT) 432 433 #define CACHE_VCOLOR_MASK(vcolor) (2 << (vcolor & (CACHE_NUM_COLOR - 1))) 434 435 #define CacheColor_IsFlushed(flag, vcolor) \ 436 ((flag) & CACHE_VCOLOR_MASK(vcolor)) 437 438 #define CacheColor_SetFlushed(flag, vcolor) \ 439 ((flag) |= CACHE_VCOLOR_MASK(vcolor)) 440 /* 441 * Flags passed to sfmmu_page_cache to flush page from vac or not. 442 */ 443 #define CACHE_FLUSH 0 444 #define CACHE_NO_FLUSH 1 445 446 /* 447 * Flags passed to sfmmu_tlbcache_demap 448 */ 449 #define FLUSH_NECESSARY_CPUS 0 450 #define FLUSH_ALL_CPUS 1 451 452 #ifdef DEBUG 453 /* 454 * For debugging purpose only. Maybe removed later. 455 */ 456 struct ctx_trace { 457 sfmmu_t *sc_sfmmu_stolen; 458 sfmmu_t *sc_sfmmu_stealing; 459 clock_t sc_time; 460 ushort_t sc_type; 461 ushort_t sc_cnum; 462 }; 463 #define CTX_TRC_STEAL 0x1 464 #define CTX_TRC_FREE 0x0 465 #define TRSIZE 0x400 466 #define NEXT_CTXTR(ptr) (((ptr) >= ctx_trace_last) ? \ 467 ctx_trace_first : ((ptr) + 1)) 468 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) \ 469 mutex_enter(mutex); \ 470 (ptr)->sc_sfmmu_stolen = (stolen_sfmmu); \ 471 (ptr)->sc_sfmmu_stealing = (stealing_sfmmu); \ 472 (ptr)->sc_cnum = (cnum); \ 473 (ptr)->sc_type = (type); \ 474 (ptr)->sc_time = lbolt; \ 475 (ptr) = NEXT_CTXTR(ptr); \ 476 num_ctx_stolen += (type); \ 477 mutex_exit(mutex); 478 #else 479 480 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) 481 482 #endif /* DEBUG */ 483 484 #endif /* !_ASM */ 485 486 /* 487 * Macros for sfmmup->sfmmu_flags access. The macros that change the flags 488 * ASSERT() that we're holding the HAT lock before changing the flags; 489 * however callers that read the flags may do so without acquiring the lock 490 * in a fast path, and then recheck the flag after acquiring the lock in 491 * a slow path. 492 */ 493 #define SFMMU_FLAGS_ISSET(sfmmup, flags) \ 494 (((sfmmup)->sfmmu_flags & (flags)) == (flags)) 495 496 #define SFMMU_FLAGS_CLEAR(sfmmup, flags) \ 497 (ASSERT(sfmmu_hat_lock_held((sfmmup))), \ 498 (sfmmup)->sfmmu_flags &= ~(flags)) 499 500 #define SFMMU_FLAGS_SET(sfmmup, flags) \ 501 (ASSERT(sfmmu_hat_lock_held((sfmmup))), \ 502 (sfmmup)->sfmmu_flags |= (flags)) 503 504 /* 505 * sfmmu HAT flags 506 */ 507 #define HAT_64K_FLAG 0x01 508 #define HAT_512K_FLAG 0x02 509 #define HAT_4M_FLAG 0x04 510 #define HAT_32M_FLAG 0x08 511 #define HAT_256M_FLAG 0x10 512 #define HAT_4MTEXT_FLAG 0x80 513 #define HAT_SWAPPED 0x100 /* swapped out */ 514 #define HAT_SWAPIN 0x200 /* swapping in */ 515 #define HAT_BUSY 0x400 /* replacing TSB(s) */ 516 #define HAT_ISMBUSY 0x800 /* adding/removing/traversing ISM maps */ 517 518 #define HAT_LGPG_FLAGS \ 519 (HAT_64K_FLAG | HAT_512K_FLAG | HAT_4M_FLAG | \ 520 HAT_32M_FLAG | HAT_256M_FLAG) 521 522 #define HAT_FLAGS_MASK \ 523 (HAT_LGPG_FLAGS | HAT_4MTEXT_FLAG | HAT_SWAPPED | \ 524 HAT_SWAPIN | HAT_BUSY | HAT_ISMBUSY) 525 526 /* 527 * Context flags 528 */ 529 #define CTX_FREE_FLAG 0x1 530 #define CTX_FLAGS_MASK 0x1 531 532 #define CTX_SET_FLAGS(ctx, flag) \ 533 { \ 534 uint32_t old, new; \ 535 \ 536 do { \ 537 new = old = (ctx)->ctx_flags; \ 538 new &= CTX_FLAGS_MASK; \ 539 new |= flag; \ 540 new = cas32(&(ctx)->ctx_flags, old, new); \ 541 } while (new != old); \ 542 } 543 544 #define CTX_CLEAR_FLAGS(ctx, flag) \ 545 { \ 546 uint32_t old, new; \ 547 \ 548 do { \ 549 new = old = (ctx)->ctx_flags; \ 550 new &= CTX_FLAGS_MASK & ~(flag); \ 551 new = cas32(&(ctx)->ctx_flags, old, new); \ 552 } while (new != old); \ 553 } 554 555 #define ctxtoctxnum(ctx) ((ushort_t)((ctx) - ctxs)) 556 557 /* 558 * Defines needed for ctx stealing. 559 */ 560 #define GET_CTX_RETRY_CNT 100 561 562 /* 563 * Starting with context 0, the first NUM_LOCKED_CTXS contexts 564 * are locked so that sfmmu_getctx can't steal any of these 565 * contexts. At the time this software was being developed, the 566 * only context that needs to be locked is context 0 (the kernel 567 * context), and context 1 (reserved for stolen context). So this constant 568 * was originally defined to be 2. 569 */ 570 #define NUM_LOCKED_CTXS 2 571 #define INVALID_CONTEXT 1 572 573 #ifndef _ASM 574 575 /* 576 * Kernel page relocation stuff. 577 */ 578 struct sfmmu_callback { 579 int key; 580 int (*prehandler)(caddr_t, uint_t, uint_t, void *); 581 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t); 582 int (*errhandler)(caddr_t, uint_t, uint_t, void *); 583 int capture_cpus; 584 }; 585 586 extern int sfmmu_max_cb_id; 587 extern struct sfmmu_callback *sfmmu_cb_table; 588 589 extern int hat_kpr_enabled; 590 591 struct pa_hment; 592 593 /* 594 * RFE: With multihat gone we gain back an int. We could use this to 595 * keep ref bits on a per cpu basis to eliminate xcalls. 596 */ 597 struct sf_hment { 598 tte_t hme_tte; /* tte for this hment */ 599 600 union { 601 struct page *page; /* what page this maps */ 602 struct pa_hment *data; /* pa_hment */ 603 } sf_hment_un; 604 605 struct sf_hment *hme_next; /* next hment */ 606 struct sf_hment *hme_prev; /* prev hment */ 607 }; 608 609 struct pa_hment { 610 caddr_t addr; /* va */ 611 uint_t len; /* bytes */ 612 ushort_t flags; /* internal flags */ 613 ushort_t refcnt; /* reference count */ 614 id_t cb_id; /* callback id, table index */ 615 void *pvt; /* handler's private data */ 616 struct sf_hment sfment; /* corresponding dummy sf_hment */ 617 }; 618 619 #define hme_page sf_hment_un.page 620 #define hme_data sf_hment_un.data 621 #define hme_size(sfhmep) ((int)(TTE_CSZ(&(sfhmep)->hme_tte))) 622 #define PAHME_SZ (sizeof (struct pa_hment)) 623 #define SFHME_SZ (sizeof (struct sf_hment)) 624 625 #define IS_PAHME(hme) ((hme)->hme_tte.ll == 0) 626 627 /* 628 * hmeblk_tag structure 629 * structure used to obtain a match on a hme_blk. Currently consists of 630 * the address of the sfmmu struct (or hatid), the base page address of the 631 * hme_blk, and the rehash count. The rehash count is actually only 2 bits 632 * and has the following meaning: 633 * 1 = 8k or 64k hash sequence. 634 * 2 = 512k hash sequence. 635 * 3 = 4M hash sequence. 636 * We require this count because we don't want to get a false hit on a 512K or 637 * 4M rehash with a base address corresponding to a 8k or 64k hmeblk. 638 * Note: The ordering and size of the hmeblk_tag members are implictly known 639 * by the tsb miss handlers written in assembly. Do not change this structure 640 * without checking those routines. See HTAG_SFMMUPSZ define. 641 */ 642 643 typedef union { 644 struct { 645 uint64_t hblk_basepg: 51, /* hme_blk base pg # */ 646 hblk_rehash: 13; /* rehash number */ 647 sfmmu_t *sfmmup; 648 } hblk_tag_un; 649 uint64_t htag_tag[2]; 650 } hmeblk_tag; 651 652 #define htag_id hblk_tag_un.sfmmup 653 #define htag_bspage hblk_tag_un.hblk_basepg 654 #define htag_rehash hblk_tag_un.hblk_rehash 655 656 #define HTAGS_EQ(tag1, tag2) (((tag1.htag_tag[0] ^ tag2.htag_tag[0]) | \ 657 (tag1.htag_tag[1] ^ tag2.htag_tag[1])) == 0) 658 #define HME_REHASH(sfmmup) \ 659 ((sfmmup)->sfmmu_ttecnt[TTE512K] != 0 || \ 660 (sfmmup)->sfmmu_ttecnt[TTE4M] != 0 || \ 661 (sfmmup)->sfmmu_ttecnt[TTE32M] != 0 || \ 662 (sfmmup)->sfmmu_ttecnt[TTE256M] != 0) 663 664 #endif /* !_ASM */ 665 666 #define NHMENTS 8 /* # of hments in an 8k hme_blk */ 667 /* needs to be multiple of 2 */ 668 #ifndef _ASM 669 670 #ifdef HBLK_TRACE 671 672 #define HBLK_LOCK 1 673 #define HBLK_UNLOCK 0 674 #define HBLK_STACK_DEPTH 6 675 #define HBLK_AUDIT_CACHE_SIZE 16 676 #define HBLK_LOCK_PATTERN 0xaaaaaaaa 677 #define HBLK_UNLOCK_PATTERN 0xbbbbbbbb 678 679 struct hblk_lockcnt_audit { 680 int flag; /* lock or unlock */ 681 kthread_id_t thread; 682 int depth; 683 pc_t stack[HBLK_STACK_DEPTH]; 684 }; 685 686 #endif /* HBLK_TRACE */ 687 688 689 /* 690 * Hment block structure. 691 * The hme_blk is the node data structure which the hash structure 692 * mantains. An hme_blk can have 2 different sizes depending on the 693 * number of hments it implicitly contains. When dealing with 64K, 512K, 694 * or 4M hments there is one hment per hme_blk. When dealing with 695 * 8k hments we allocate an hme_blk plus an additional 7 hments to 696 * give us a total of 8 (NHMENTS) hments that can be referenced through a 697 * hme_blk. 698 * 699 * The hmeblk structure contains 2 tte reference counters used to determine if 700 * it is ok to free up the hmeblk. Both counters have to be zero in order 701 * to be able to free up hmeblk. They are protected by cas. 702 * hblk_hmecnt is the number of hments present on pp mapping lists. 703 * hblk_vcnt reflects number of valid ttes in hmeblk. 704 * 705 * The hmeblk now also has per tte lock cnts. This is required because 706 * the counts can be high and there are not enough bits in the tte. When 707 * physio is fixed to not lock the translations we should be able to move 708 * the lock cnt back to the tte. See bug id 1198554. 709 * 710 * Note that xhat_hme_blk's layout follows this structure: hme_blk_misc 711 * and sf_hment are at the same offsets in both structures. Whenever 712 * hme_blk is changed, xhat_hme_blk may need to be updated as well. 713 */ 714 715 struct hme_blk_misc { 716 ushort_t locked_cnt; /* HAT_LOAD_LOCK ref cnt */ 717 uint_t notused:10; 718 uint_t xhat_bit:1; /* set for an xhat hme_blk */ 719 uint_t shadow_bit:1; /* set for a shadow hme_blk */ 720 uint_t nucleus_bit:1; /* set for a nucleus hme_blk */ 721 uint_t ttesize:3; /* contains ttesz of hmeblk */ 722 }; 723 724 struct hme_blk { 725 uint64_t hblk_nextpa; /* physical address for hash list */ 726 727 hmeblk_tag hblk_tag; /* tag used to obtain an hmeblk match */ 728 729 struct hme_blk *hblk_next; /* on free list or on hash list */ 730 /* protected by hash lock */ 731 732 struct hme_blk *hblk_shadow; /* pts to shadow hblk */ 733 /* protected by hash lock */ 734 uint_t hblk_span; /* span of memory hmeblk maps */ 735 736 struct hme_blk_misc hblk_misc; 737 738 union { 739 struct { 740 ushort_t hblk_hmecount; /* hment on mlists counter */ 741 ushort_t hblk_validcnt; /* valid tte reference count */ 742 } hblk_counts; 743 uint_t hblk_shadow_mask; 744 } hblk_un; 745 746 #ifdef HBLK_TRACE 747 kmutex_t hblk_audit_lock; /* lock to protect index */ 748 uint_t hblk_audit_index; /* index into audit_cache */ 749 struct hblk_lockcnt_audit hblk_audit_cache[HBLK_AUDIT_CACHE_SIZE]; 750 #endif /* HBLK_AUDIT */ 751 752 struct sf_hment hblk_hme[1]; /* hment array */ 753 }; 754 755 #define hblk_lckcnt hblk_misc.locked_cnt 756 #define hblk_xhat_bit hblk_misc.xhat_bit 757 #define hblk_shw_bit hblk_misc.shadow_bit 758 #define hblk_nuc_bit hblk_misc.nucleus_bit 759 #define hblk_ttesz hblk_misc.ttesize 760 #define hblk_hmecnt hblk_un.hblk_counts.hblk_hmecount 761 #define hblk_vcnt hblk_un.hblk_counts.hblk_validcnt 762 #define hblk_shw_mask hblk_un.hblk_shadow_mask 763 764 #define MAX_HBLK_LCKCNT 0xFFFF 765 #define HMEBLK_ALIGN 0x8 /* hmeblk has to be double aligned */ 766 767 #ifdef HBLK_TRACE 768 769 #define HBLK_STACK_TRACE(hmeblkp, lock) \ 770 { \ 771 int flag = lock; /* to pacify lint */ \ 772 int audit_index; \ 773 \ 774 mutex_enter(&hmeblkp->hblk_audit_lock); \ 775 audit_index = hmeblkp->hblk_audit_index; \ 776 hmeblkp->hblk_audit_index = ((hmeblkp->hblk_audit_index + 1) & \ 777 (HBLK_AUDIT_CACHE_SIZE - 1)); \ 778 mutex_exit(&hmeblkp->hblk_audit_lock); \ 779 \ 780 if (flag) \ 781 hmeblkp->hblk_audit_cache[audit_index].flag = \ 782 HBLK_LOCK_PATTERN; \ 783 else \ 784 hmeblkp->hblk_audit_cache[audit_index].flag = \ 785 HBLK_UNLOCK_PATTERN; \ 786 \ 787 hmeblkp->hblk_audit_cache[audit_index].thread = curthread; \ 788 hmeblkp->hblk_audit_cache[audit_index].depth = \ 789 getpcstack(hmeblkp->hblk_audit_cache[audit_index].stack, \ 790 HBLK_STACK_DEPTH); \ 791 } 792 793 #else 794 795 #define HBLK_STACK_TRACE(hmeblkp, lock) 796 797 #endif /* HBLK_TRACE */ 798 799 #define HMEHASH_FACTOR 16 /* used to calc # of buckets in hme hash */ 800 801 /* 802 * A maximum number of user hmeblks is defined in order to place an upper 803 * limit on how much nucleus memory is required and to avoid overflowing the 804 * tsbmiss uhashsz and khashsz data areas. The number below corresponds to 805 * the number of buckets required, for an average hash chain length of 4 on 806 * a 16TB machine. 807 */ 808 809 #define MAX_UHME_BUCKETS (0x1 << 30) 810 #define MAX_KHME_BUCKETS (0x1 << 30) 811 812 /* 813 * The minimum number of kernel hash buckets. 814 */ 815 #define MIN_KHME_BUCKETS 0x800 816 817 /* 818 * The number of hash buckets must be a power of 2. If the initial calculated 819 * value is less than USER_BUCKETS_THRESHOLD we round up to the next greater 820 * power of 2, otherwise we round down to avoid huge over allocations. 821 */ 822 #define USER_BUCKETS_THRESHOLD (1<<22) 823 824 #define MAX_NUCUHME_BUCKETS 0x4000 825 #define MAX_NUCKHME_BUCKETS 0x2000 826 827 /* 828 * There are 2 locks in the hmehash bucket. The hmehash_mutex is 829 * a regular mutex used to make sure operations on a hash link are only 830 * done by one thread. Any operation which comes into the hat with 831 * a <vaddr, as> will grab the hmehash_mutex. Normally one would expect 832 * the tsb miss handlers to grab the hash lock to make sure the hash list 833 * is consistent while we traverse it. Unfortunately this can lead to 834 * deadlocks or recursive mutex enters since it is possible for 835 * someone holding the lock to take a tlb/tsb miss. 836 * To solve this problem we have added the hmehash_listlock. This lock 837 * is only grabbed by the tsb miss handlers, vatopfn, and while 838 * adding/removing a hmeblk from the hash list. The code is written to 839 * guarantee we won't take a tlb miss while holding this lock. 840 */ 841 struct hmehash_bucket { 842 kmutex_t hmehash_mutex; 843 uint64_t hmeh_nextpa; /* physical address for hash list */ 844 struct hme_blk *hmeblkp; 845 uint_t hmeh_listlock; 846 }; 847 848 #endif /* !_ASM */ 849 850 /* Proc Count Project */ 851 #define SFMMU_PGCNT_MASK 0x3f 852 #define SFMMU_PGCNT_SHIFT 6 853 #define INVALID_MMU_ID -1 854 #define SFMMU_MMU_GNUM_RSHIFT 16 855 #define SFMMU_MMU_CNUM_LSHIFT (64 - SFMMU_MMU_GNUM_RSHIFT) 856 #define MAX_SFMMU_CTX_VAL ((1 << 16) - 1) /* for sanity check */ 857 #define MAX_SFMMU_GNUM_VAL ((0x1UL << 48) - 1) 858 859 /* 860 * The tsb miss handlers written in assembly know that sfmmup 861 * is a 64 bit ptr. 862 * 863 * The bspage and re-hash part is 64 bits, with the sfmmup being another 64 864 * bits. 865 */ 866 #define HTAG_SFMMUPSZ 0 /* Not really used for LP64 */ 867 #define HTAG_REHASHSZ 13 868 869 /* 870 * Assembly routines need to be able to get to ttesz 871 */ 872 #define HBLK_SZMASK 0x7 873 874 #ifndef _ASM 875 876 /* 877 * Returns the number of bytes that an hmeblk spans given its tte size 878 */ 879 #define get_hblk_span(hmeblkp) ((hmeblkp)->hblk_span) 880 #define get_hblk_ttesz(hmeblkp) ((hmeblkp)->hblk_ttesz) 881 #define get_hblk_cache(hmeblkp) (((hmeblkp)->hblk_ttesz == TTE8K) ? \ 882 sfmmu8_cache : sfmmu1_cache) 883 #define HMEBLK_SPAN(ttesz) \ 884 ((ttesz == TTE8K)? (TTEBYTES(ttesz) * NHMENTS) : TTEBYTES(ttesz)) 885 886 #define set_hblk_sz(hmeblkp, ttesz) \ 887 (hmeblkp)->hblk_ttesz = (ttesz); \ 888 (hmeblkp)->hblk_span = HMEBLK_SPAN(ttesz) 889 890 #define get_hblk_base(hmeblkp) \ 891 ((uintptr_t)(hmeblkp)->hblk_tag.htag_bspage << MMU_PAGESHIFT) 892 893 #define get_hblk_endaddr(hmeblkp) \ 894 ((caddr_t)(get_hblk_base(hmeblkp) + get_hblk_span(hmeblkp))) 895 896 #define in_hblk_range(hmeblkp, vaddr) \ 897 (((uintptr_t)(vaddr) >= get_hblk_base(hmeblkp)) && \ 898 ((uintptr_t)(vaddr) < (get_hblk_base(hmeblkp) + \ 899 get_hblk_span(hmeblkp)))) 900 901 #define tte_to_vaddr(hmeblkp, tte) ((caddr_t)(get_hblk_base(hmeblkp) \ 902 + (TTEBYTES(TTE_CSZ(&tte)) * (tte).tte_hmenum))) 903 904 #define vaddr_to_vshift(hblktag, vaddr, shwsz) \ 905 ((((uintptr_t)(vaddr) >> MMU_PAGESHIFT) - (hblktag.htag_bspage)) >>\ 906 TTE_BSZS_SHIFT((shwsz) - 1)) 907 908 #define HME8BLK_SZ (sizeof (struct hme_blk) + \ 909 (NHMENTS - 1) * sizeof (struct sf_hment)) 910 #define HME1BLK_SZ (sizeof (struct hme_blk)) 911 #define H8TOH1 (MMU_PAGESIZE4M / MMU_PAGESIZE) 912 #define H1MIN (2 + MAX_BIGKTSB_TTES) /* nucleus text+data, ktsb */ 913 914 /* 915 * Hme_blk hash structure 916 * Active mappings are kept in a hash structure of hme_blks. The hash 917 * function is based on (ctx, vaddr) The size of the hash table size is a 918 * power of 2 such that the average hash chain lenth is HMENT_HASHAVELEN. 919 * The hash actually consists of 2 separate hashes. One hash is for the user 920 * address space and the other hash is for the kernel address space. 921 * The number of buckets are calculated at boot time and stored in the global 922 * variables "uhmehash_num" and "khmehash_num". By making the hash table size 923 * a power of 2 we can use a simply & function to derive an index instead of 924 * a divide. 925 * 926 * HME_HASH_FUNCTION(hatid, vaddr, shift) returns a pointer to a hme_hash 927 * bucket. 928 * An hme hash bucket contains a pointer to an hme_blk and the mutex that 929 * protects the link list. 930 * Spitfire supports 4 page sizes. 8k and 64K pages only need one hash. 931 * 512K pages need 2 hashes and 4M pages need 3 hashes. 932 * The 'shift' parameter controls how many bits the vaddr will be shifted in 933 * the hash function. It is calculated in the HME_HASH_SHIFT(ttesz) function 934 * and it varies depending on the page size as follows: 935 * 8k pages: HBLK_RANGE_SHIFT 936 * 64k pages: MMU_PAGESHIFT64K 937 * 512K pages: MMU_PAGESHIFT512K 938 * 4M pages: MMU_PAGESHIFT4M 939 * An assembly version of the hash function exists in sfmmu_ktsb_miss(). All 940 * changes should be reflected in both versions. This function and the TSB 941 * miss handlers are the only places which know about the two hashes. 942 * 943 * HBLK_RANGE_SHIFT controls range of virtual addresses that will fall 944 * into the same bucket for a particular process. It is currently set to 945 * be equivalent to 64K range or one hme_blk. 946 * 947 * The hme_blks in the hash are protected by a per hash bucket mutex 948 * known as SFMMU_HASH_LOCK. 949 * You need to acquire this lock before traversing the hash bucket link 950 * list, while adding/removing a hme_blk to the list, and while 951 * modifying an hme_blk. A possible optimization is to replace these 952 * mutexes by readers/writer lock but right now it is not clear whether 953 * this is a win or not. 954 * 955 * The HME_HASH_TABLE_SEARCH will search the hash table for the 956 * hme_blk that contains the hment that corresponds to the passed 957 * ctx and vaddr. It assumed the SFMMU_HASH_LOCK is held. 958 */ 959 960 #endif /* ! _ASM */ 961 962 #define KHATID ksfmmup 963 #define UHMEHASH_SZ uhmehash_num 964 #define KHMEHASH_SZ khmehash_num 965 #define HMENT_HASHAVELEN 4 966 #define HBLK_RANGE_SHIFT MMU_PAGESHIFT64K /* shift for HBLK_BS_MASK */ 967 #define MAX_HASHCNT 5 968 #define DEFAULT_MAX_HASHCNT 3 969 970 #ifndef _ASM 971 972 #define HASHADDR_MASK(hashno) TTE_PAGEMASK(hashno) 973 974 #define HME_HASH_SHIFT(ttesz) \ 975 ((ttesz == TTE8K)? HBLK_RANGE_SHIFT : TTE_PAGE_SHIFT(ttesz)) \ 976 977 #define HME_HASH_ADDR(vaddr, hmeshift) \ 978 ((caddr_t)(((uintptr_t)(vaddr) >> (hmeshift)) << (hmeshift))) 979 980 #define HME_HASH_BSPAGE(vaddr, hmeshift) \ 981 (((uintptr_t)(vaddr) >> (hmeshift)) << ((hmeshift) - MMU_PAGESHIFT)) 982 983 #define HME_HASH_REHASH(ttesz) \ 984 (((ttesz) < TTE512K)? 1 : (ttesz)) 985 986 #define HME_HASH_FUNCTION(hatid, vaddr, shift) \ 987 ((hatid != KHATID)? \ 988 (&uhme_hash[ (((uintptr_t)(hatid) ^ \ 989 ((uintptr_t)vaddr >> (shift))) & UHMEHASH_SZ) ]): \ 990 (&khme_hash[ (((uintptr_t)(hatid) ^ \ 991 ((uintptr_t)vaddr >> (shift))) & KHMEHASH_SZ) ])) 992 993 /* 994 * This macro will traverse a hmeblk hash link list looking for an hme_blk 995 * that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp 996 * will be set to NULL, otherwise it will point to the correct hme_blk. 997 * This macro also cleans empty hblks. 998 */ 999 #define HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, \ 1000 pr_hblk, prevpa, listp) \ 1001 { \ 1002 struct hme_blk *nx_hblk; \ 1003 uint64_t nx_pa; \ 1004 \ 1005 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \ 1006 hblkp = hmebp->hmeblkp; \ 1007 hblkpa = hmebp->hmeh_nextpa; \ 1008 prevpa = 0; \ 1009 pr_hblk = NULL; \ 1010 while (hblkp) { \ 1011 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \ 1012 /* found hme_blk */ \ 1013 break; \ 1014 } \ 1015 nx_hblk = hblkp->hblk_next; \ 1016 nx_pa = hblkp->hblk_nextpa; \ 1017 if (!hblkp->hblk_vcnt && !hblkp->hblk_hmecnt) { \ 1018 sfmmu_hblk_hash_rm(hmebp, hblkp, prevpa, pr_hblk); \ 1019 sfmmu_hblk_free(hmebp, hblkp, hblkpa, listp); \ 1020 } else { \ 1021 pr_hblk = hblkp; \ 1022 prevpa = hblkpa; \ 1023 } \ 1024 hblkp = nx_hblk; \ 1025 hblkpa = nx_pa; \ 1026 } \ 1027 } 1028 1029 #define HME_HASH_SEARCH(hmebp, hblktag, hblkp, listp) \ 1030 { \ 1031 struct hme_blk *pr_hblk; \ 1032 uint64_t hblkpa, prevpa; \ 1033 \ 1034 HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, pr_hblk, \ 1035 prevpa, listp); \ 1036 } 1037 1038 /* 1039 * This macro will traverse a hmeblk hash link list looking for an hme_blk 1040 * that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp 1041 * will be set to NULL, otherwise it will point to the correct hme_blk. 1042 * It doesn't remove empty hblks. 1043 */ 1044 #define HME_HASH_FAST_SEARCH(hmebp, hblktag, hblkp) \ 1045 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \ 1046 for (hblkp = hmebp->hmeblkp; hblkp; \ 1047 hblkp = hblkp->hblk_next) { \ 1048 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \ 1049 /* found hme_blk */ \ 1050 break; \ 1051 } \ 1052 } \ 1053 1054 1055 #define SFMMU_HASH_LOCK(hmebp) \ 1056 (mutex_enter(&hmebp->hmehash_mutex)) 1057 1058 #define SFMMU_HASH_UNLOCK(hmebp) \ 1059 (mutex_exit(&hmebp->hmehash_mutex)) 1060 1061 #define SFMMU_HASH_LOCK_TRYENTER(hmebp) \ 1062 (mutex_tryenter(&hmebp->hmehash_mutex)) 1063 1064 #define SFMMU_HASH_LOCK_ISHELD(hmebp) \ 1065 (mutex_owned(&hmebp->hmehash_mutex)) 1066 1067 #define SFMMU_XCALL_STATS(sfmmup) \ 1068 { \ 1069 if (sfmmup == ksfmmup) { \ 1070 SFMMU_STAT(sf_kernel_xcalls); \ 1071 } else { \ 1072 SFMMU_STAT(sf_user_xcalls); \ 1073 } \ 1074 } 1075 1076 #define astosfmmu(as) ((as)->a_hat) 1077 #define hblktosfmmu(hmeblkp) ((sfmmu_t *)(hmeblkp)->hblk_tag.htag_id) 1078 #define sfmmutoas(sfmmup) ((sfmmup)->sfmmu_as) 1079 /* 1080 * We use the sfmmu data structure to keep the per as page coloring info. 1081 */ 1082 #define as_color_bin(as) (astosfmmu(as)->sfmmu_clrbin) 1083 #define as_color_start(as) (astosfmmu(as)->sfmmu_clrstart) 1084 1085 typedef struct { 1086 char h8[HME8BLK_SZ]; 1087 } hblk8_t; 1088 1089 typedef struct { 1090 char h1[HME1BLK_SZ]; 1091 } hblk1_t; 1092 1093 typedef struct { 1094 ulong_t index; 1095 ulong_t len; 1096 hblk8_t *list; 1097 } nucleus_hblk8_info_t; 1098 1099 typedef struct { 1100 ulong_t index; 1101 ulong_t len; 1102 hblk1_t *list; 1103 } nucleus_hblk1_info_t; 1104 1105 /* 1106 * This struct is used for accumlating information about a range 1107 * of pages that are unloading so that a single xcall can flush 1108 * the entire range from remote tlbs. A function that must demap 1109 * a range of virtual addresses declares one of these structures 1110 * and initializes using DEMP_RANGE_INIT(). It then passes a pointer to this 1111 * struct to the appropriate sfmmu_hblk_* level function which does 1112 * all the bookkeeping using the other macros. When the function has 1113 * finished the virtual address range, it needs to call DEMAP_RANGE_FLUSH() 1114 * macro to take care of any remaining unflushed mappings. 1115 * 1116 * The maximum range this struct can represent is the number of bits 1117 * in the dmr_bitvec field times the pagesize in dmr_pgsz. Currently, only 1118 * MMU_PAGESIZE pages are supported. 1119 * 1120 * Since there are now cases where it's no longer necessary to do 1121 * flushes (e.g. when the process isn't runnable because it's swapping 1122 * out or exiting) we allow these macros to take a NULL dmr input and do 1123 * nothing in that case. 1124 */ 1125 typedef struct { 1126 sfmmu_t *dmr_sfmmup; /* relevent hat */ 1127 caddr_t dmr_addr; /* beginning address */ 1128 caddr_t dmr_endaddr; /* ending address */ 1129 ulong_t dmr_bitvec; /* valid pages found */ 1130 ulong_t dmr_bit; /* next page to examine */ 1131 ulong_t dmr_maxbit; /* highest page in range */ 1132 ulong_t dmr_pgsz; /* page size in range */ 1133 } demap_range_t; 1134 1135 #define DMR_MAXBIT ((ulong_t)1<<63) /* dmr_bit high bit */ 1136 1137 #define DEMAP_RANGE_INIT(sfmmup, dmrp) \ 1138 if ((dmrp) != NULL) { \ 1139 (dmrp)->dmr_sfmmup = (sfmmup); \ 1140 (dmrp)->dmr_bitvec = 0; \ 1141 (dmrp)->dmr_maxbit = sfmmu_dmr_maxbit; \ 1142 (dmrp)->dmr_pgsz = MMU_PAGESIZE; \ 1143 } 1144 1145 #define DEMAP_RANGE_PGSZ(dmrp) ((dmrp)? (dmrp)->dmr_pgsz : MMU_PAGESIZE) 1146 1147 #define DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr) \ 1148 if ((dmrp) != NULL) { \ 1149 if ((dmrp)->dmr_bitvec != 0 && (dmrp)->dmr_endaddr != (addr)) \ 1150 sfmmu_tlb_range_demap(dmrp); \ 1151 (dmrp)->dmr_endaddr = (endaddr); \ 1152 } 1153 1154 #define DEMAP_RANGE_FLUSH(dmrp) \ 1155 if ((dmrp) != NULL) { \ 1156 if ((dmrp)->dmr_bitvec != 0) \ 1157 sfmmu_tlb_range_demap(dmrp); \ 1158 } 1159 1160 #define DEMAP_RANGE_MARKPG(dmrp, addr) \ 1161 if ((dmrp) != NULL) { \ 1162 if ((dmrp)->dmr_bitvec == 0) { \ 1163 (dmrp)->dmr_addr = (addr); \ 1164 (dmrp)->dmr_bit = 1; \ 1165 } \ 1166 (dmrp)->dmr_bitvec |= (dmrp)->dmr_bit; \ 1167 } 1168 1169 #define DEMAP_RANGE_NEXTPG(dmrp) \ 1170 if ((dmrp) != NULL && (dmrp)->dmr_bitvec != 0) { \ 1171 if ((dmrp)->dmr_bit & (dmrp)->dmr_maxbit) { \ 1172 sfmmu_tlb_range_demap(dmrp); \ 1173 } else { \ 1174 (dmrp)->dmr_bit <<= 1; \ 1175 } \ 1176 } 1177 1178 /* 1179 * TSB related structures 1180 * 1181 * The TSB is made up of tte entries. Both the tag and data are present 1182 * in the TSB. The TSB locking is managed as follows: 1183 * A software bit in the tsb tag is used to indicate that entry is locked. 1184 * If a cpu servicing a tsb miss reads a locked entry the tag compare will 1185 * fail forcing the cpu to go to the hat hash for the translation. 1186 * The cpu who holds the lock can then modify the data side, and the tag side. 1187 * The last write should be to the word containing the lock bit which will 1188 * clear the lock and allow the tsb entry to be read. It is assumed that all 1189 * cpus reading the tsb will do so with atomic 128-bit loads. An atomic 128 1190 * bit load is required to prevent the following from happening: 1191 * 1192 * cpu 0 cpu 1 comments 1193 * 1194 * ldx tag tag unlocked 1195 * ldstub lock set lock 1196 * stx data 1197 * stx tag unlock 1198 * ldx tag incorrect tte!!! 1199 * 1200 * The software also maintains a bit in the tag to indicate an invalid 1201 * tsb entry. The purpose of this bit is to allow the tsb invalidate code 1202 * to invalidate a tsb entry with a single cas. See code for details. 1203 */ 1204 1205 union tsb_tag { 1206 struct { 1207 uint32_t tag_res0:16; /* reserved - context area */ 1208 uint32_t tag_inv:1; /* sw - invalid tsb entry */ 1209 uint32_t tag_lock:1; /* sw - locked tsb entry */ 1210 uint32_t tag_res1:4; /* reserved */ 1211 uint32_t tag_va_hi:10; /* va[63:54] */ 1212 uint32_t tag_va_lo; /* va[53:22] */ 1213 } tagbits; 1214 struct tsb_tagints { 1215 uint32_t inthi; 1216 uint32_t intlo; 1217 } tagints; 1218 }; 1219 #define tag_invalid tagbits.tag_inv 1220 #define tag_locked tagbits.tag_lock 1221 #define tag_vahi tagbits.tag_va_hi 1222 #define tag_valo tagbits.tag_va_lo 1223 #define tag_inthi tagints.inthi 1224 #define tag_intlo tagints.intlo 1225 1226 struct tsbe { 1227 union tsb_tag tte_tag; 1228 tte_t tte_data; 1229 }; 1230 1231 /* 1232 * A per cpu struct is kept that duplicates some info 1233 * used by the tl>0 tsb miss handlers plus it provides 1234 * a scratch area. Its purpose is to minimize cache misses 1235 * in the tsb miss handler and is 128 bytes (2 e$ lines). 1236 * 1237 * There should be one allocated per cpu in nucleus memory 1238 * and should be aligned on an ecache line boundary. 1239 */ 1240 struct tsbmiss { 1241 sfmmu_t *ksfmmup; /* kernel hat id */ 1242 sfmmu_t *usfmmup; /* user hat id */ 1243 struct tsbe *tsbptr; /* hardware computed ptr */ 1244 struct tsbe *tsbptr4m; /* hardware computed ptr */ 1245 uint64_t ismblkpa; 1246 struct hmehash_bucket *khashstart; 1247 struct hmehash_bucket *uhashstart; 1248 uint_t khashsz; 1249 uint_t uhashsz; 1250 uint16_t dcache_line_mask; /* used to flush dcache */ 1251 uint16_t hat_flags; 1252 uint32_t itlb_misses; 1253 uint32_t dtlb_misses; 1254 uint32_t utsb_misses; 1255 uint32_t ktsb_misses; 1256 uint16_t uprot_traps; 1257 uint16_t kprot_traps; 1258 1259 /* 1260 * scratch[0] -> TSB_TAGACC 1261 * scratch[1] -> TSBMISS_HMEBP 1262 * scratch[2] -> TSBMISS_HATID 1263 */ 1264 uintptr_t scratch[3]; 1265 uint8_t pad[0x10]; 1266 }; 1267 1268 /* 1269 * A per cpu struct is kept for the use within the tl>0 kpm tsb 1270 * miss handler. Some members are duplicates of common data or 1271 * the physical addresses of common data. A few members are also 1272 * written by the tl>0 kpm tsb miss handler. Its purpose is to 1273 * minimize cache misses in the kpm tsb miss handler and occupies 1274 * one ecache line. There should be one allocated per cpu in 1275 * nucleus memory and it should be aligned on an ecache line 1276 * boundary. It is not merged w/ struct tsbmiss since there is 1277 * not much to share and the tsbmiss pathes are different, so 1278 * a kpm tlbmiss/tsbmiss only touches one cacheline, except for 1279 * (DEBUG || SFMMU_STAT_GATHER) where the dtlb_misses counter 1280 * of struct tsbmiss is used on every dtlb miss. 1281 */ 1282 struct kpmtsbm { 1283 caddr_t vbase; /* start of address kpm range */ 1284 caddr_t vend; /* end of address kpm range */ 1285 uchar_t flags; /* flags needed in TL tsbmiss handler */ 1286 uchar_t sz_shift; /* for single kpm window */ 1287 uchar_t kpmp_shift; /* hash lock shift */ 1288 uchar_t kpmp2pshft; /* kpm page to page shift */ 1289 uint_t kpmp_table_sz; /* size of kpmp_table or kpmp_stable */ 1290 uint64_t kpmp_tablepa; /* paddr of kpmp_table or kpmp_stable */ 1291 uint64_t msegphashpa; /* paddr of memseg_phash */ 1292 struct tsbe *tsbptr; /* saved ktsb pointer */ 1293 uint_t kpm_dtlb_misses; /* kpm tlbmiss counter */ 1294 uint_t kpm_tsb_misses; /* kpm tsbmiss counter */ 1295 uintptr_t pad[1]; 1296 }; 1297 1298 extern uint_t tsb_slab_size; 1299 extern uint_t tsb_slab_shift; 1300 extern uint_t tsb_slab_ttesz; 1301 extern uint_t tsb_slab_pamask; 1302 1303 #endif /* !_ASM */ 1304 1305 /* 1306 * Flags for TL kpm tsbmiss handler 1307 */ 1308 #define KPMTSBM_ENABLE_FLAG 0x01 /* bit copy of kpm_enable */ 1309 #define KPMTSBM_TLTSBM_FLAG 0x02 /* use TL tsbmiss handler */ 1310 #define KPMTSBM_TSBPHYS_FLAG 0x04 /* use ASI_MEM for TSB update */ 1311 1312 /* 1313 * The TSB 1314 * All TSB sizes supported by the hardware are now supported (8K - 1M). 1315 * For kernel TSBs we may go beyond the hardware supported sizes and support 1316 * larger TSBs via software. 1317 * All TTE sizes are supported in the TSB; the manner in which this is 1318 * done is cpu dependent. 1319 */ 1320 #define TSB_MIN_SZCODE TSB_8K_SZCODE /* min. supported TSB size */ 1321 #define TSB_MIN_OFFSET_MASK (TSB_OFFSET_MASK(TSB_MIN_SZCODE)) 1322 1323 #define UTSB_MAX_SZCODE TSB_1M_SZCODE /* max. supported TSB size */ 1324 #define UTSB_MAX_OFFSET_MASK (TSB_OFFSET_MASK(UTSB_MAX_SZCODE)) 1325 1326 #define TSB_FREEMEM_MIN 0x1000 /* 32 mb */ 1327 #define TSB_FREEMEM_LARGE 0x10000 /* 512 mb */ 1328 #define TSB_8K_SZCODE 0 /* 512 entries */ 1329 #define TSB_16K_SZCODE 1 /* 1k entries */ 1330 #define TSB_32K_SZCODE 2 /* 2k entries */ 1331 #define TSB_64K_SZCODE 3 /* 4k entries */ 1332 #define TSB_128K_SZCODE 4 /* 8k entries */ 1333 #define TSB_256K_SZCODE 5 /* 16k entries */ 1334 #define TSB_512K_SZCODE 6 /* 32k entries */ 1335 #define TSB_1M_SZCODE 7 /* 64k entries */ 1336 #define TSB_2M_SZCODE 8 /* 128k entries */ 1337 #define TSB_4M_SZCODE 9 /* 256k entries */ 1338 #define TSB_ENTRY_SHIFT 4 /* each entry = 128 bits = 16 bytes */ 1339 #define TSB_ENTRY_SIZE (1 << 4) 1340 #define TSB_START_SIZE 9 1341 #define TSB_ENTRIES(tsbsz) (1 << (TSB_START_SIZE + tsbsz)) 1342 #define TSB_BYTES(tsbsz) (TSB_ENTRIES(tsbsz) << TSB_ENTRY_SHIFT) 1343 #define TSB_OFFSET_MASK(tsbsz) (TSB_ENTRIES(tsbsz) - 1) 1344 #define TSB_BASEADDR_MASK ((1 << 12) - 1) 1345 1346 /* 1347 * sun4u platforms 1348 * --------------- 1349 * We now support two user TSBs with one TSB base register. 1350 * Hence the TSB base register is split up as follows: 1351 * 1352 * When only one TSB present: 1353 * [63 62..42 41..13 12..4 3..0] 1354 * ^ ^ ^ ^ ^ 1355 * | | | | | 1356 * | | | | |_ TSB size code 1357 * | | | | 1358 * | | | |_ Reserved 0 1359 * | | | 1360 * | | |_ TSB VA[41..13] 1361 * | | 1362 * | |_ VA hole (Spitfire), zeros (Cheetah and beyond) 1363 * | 1364 * |_ 0 1365 * 1366 * When second TSB present: 1367 * [63 62..42 41..33 32..29 28..22 21..13 12..4 3..0] 1368 * ^ ^ ^ ^ ^ ^ ^ ^ 1369 * | | | | | | | | 1370 * | | | | | | | |_ First TSB size code 1371 * | | | | | | | 1372 * | | | | | | |_ Reserved 0 1373 * | | | | | | 1374 * | | | | | |_ First TSB's VA[21..13] 1375 * | | | | | 1376 * | | | | |_ Reserved for future use 1377 * | | | | 1378 * | | | |_ Second TSB's size code 1379 * | | | 1380 * | | |_ Second TSB's VA[21..13] 1381 * | | 1382 * | |_ VA hole (Spitfire) / ones (Cheetah and beyond) 1383 * | 1384 * |_ 1 1385 * 1386 * Note that since we store 21..13 of each TSB's VA, TSBs and their slabs 1387 * may be up to 4M in size. For now, only hardware supported TSB sizes 1388 * are supported, though the slabs are usually 4M in size. 1389 * 1390 * sun4u platforms that define UTSB_PHYS use physical addressing to access 1391 * the user TSBs at TL>0. The first user TSB base is in the MMU I/D TSB Base 1392 * registers. The second TSB base uses a dedicated scratchpad register which 1393 * requires a definition of SCRATCHPAD_UTSBREG in mach_sfmmu.h. The layout for 1394 * both registers is equivalent to sun4v below, except the TSB PA range is 1395 * [46..13] for sun4u. 1396 * 1397 * sun4v platforms 1398 * --------------- 1399 * On sun4v platforms, we use two dedicated scratchpad registers as pseudo 1400 * hardware TSB base registers to hold up to two different user TSBs. 1401 * 1402 * Each register contains TSB's physical base and size code information 1403 * as follows: 1404 * 1405 * [63..56 55..13 12..4 3..0] 1406 * ^ ^ ^ ^ 1407 * | | | | 1408 * | | | |_ TSB size code 1409 * | | | 1410 * | | |_ Reserved 0 1411 * | | 1412 * | |_ TSB PA[55..13] 1413 * | 1414 * | 1415 * | 1416 * |_ 0 for valid TSB 1417 * 1418 * Absence of a user TSB (primarily the second user TSB) is indicated by 1419 * storing a negative value in the TSB base register. This allows us to 1420 * check for presence of a user TSB by simply checking bit# 63. 1421 */ 1422 #define TSBREG_MSB_SHIFT 32 /* set upper bits */ 1423 #define TSBREG_MSB_CONST 0xfffff800 /* set bits 63..43 */ 1424 #define TSBREG_FIRTSB_SHIFT 42 /* to clear bits 63:22 */ 1425 #define TSBREG_SECTSB_MKSHIFT 20 /* 21:13 --> 41:33 */ 1426 #define TSBREG_SECTSB_LSHIFT 22 /* to clear bits 63:42 */ 1427 #define TSBREG_SECTSB_RSHIFT (TSBREG_SECTSB_MKSHIFT + TSBREG_SECTSB_LSHIFT) 1428 /* sectsb va -> bits 21:13 */ 1429 /* after clearing upper bits */ 1430 #define TSBREG_SECSZ_SHIFT 29 /* to get sectsb szc to 3:0 */ 1431 #define TSBREG_VAMASK_SHIFT 13 /* set up VA mask */ 1432 1433 #define BIGKTSB_SZ_MASK 0xf 1434 #define TSB_SOFTSZ_MASK BIGKTSB_SZ_MASK 1435 #define MIN_BIGKTSB_SZCODE 9 /* 256k entries */ 1436 #define MAX_BIGKTSB_SZCODE 11 /* 1024k entries */ 1437 #define MAX_BIGKTSB_TTES (TSB_BYTES(MAX_BIGKTSB_SZCODE) / MMU_PAGESIZE4M) 1438 1439 #define TAG_VALO_SHIFT 22 /* tag's va are bits 63-22 */ 1440 /* 1441 * sw bits used on tsb_tag - bit masks used only in assembly 1442 * use only a sethi for these fields. 1443 */ 1444 #define TSBTAG_INVALID 0x00008000 /* tsb_tag.tag_invalid */ 1445 #define TSBTAG_LOCKED 0x00004000 /* tsb_tag.tag_locked */ 1446 1447 #ifdef _ASM 1448 1449 /* 1450 * Marker to indicate that this instruction will be hot patched at runtime 1451 * to some other value. 1452 * This value must be zero since it fills in the imm bits of the target 1453 * instructions to be patched 1454 */ 1455 #define RUNTIME_PATCH (0) 1456 1457 /* 1458 * V9 defines nop instruction as the following, which we use 1459 * at runtime to nullify some instructions we don't want to 1460 * execute in the trap handlers on certain platforms. 1461 */ 1462 #define MAKE_NOP_INSTR(reg) \ 1463 sethi %hi(0x1000000), reg 1464 1465 /* 1466 * Macro to get hat per-MMU cnum on this CPU. 1467 * sfmmu - In, pass in "sfmmup" from the caller. 1468 * cnum - Out, return 'cnum' to the caller 1469 * scr - scratch 1470 */ 1471 #define SFMMU_CPU_CNUM(sfmmu, cnum, scr) \ 1472 CPU_ADDR(scr, cnum); /* scr = load CPU struct addr */ \ 1473 ld [scr + CPU_MMU_IDX], cnum; /* cnum = mmuid */ \ 1474 add sfmmu, SFMMU_CTXS, scr; /* scr = sfmmup->sfmmu_ctxs[] */ \ 1475 sllx cnum, SFMMU_MMU_CTX_SHIFT, cnum; \ 1476 add scr, cnum, scr; /* scr = sfmmup->sfmmu_ctxs[id] */ \ 1477 ldx [scr + SFMMU_MMU_GC_NUM], scr; /* sfmmu_ctxs[id].gcnum */ \ 1478 sllx scr, SFMMU_MMU_CNUM_LSHIFT, scr; \ 1479 srlx scr, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */ 1480 1481 /* 1482 * Macro to get hat gnum & cnum assocaited with sfmmu_ctx[mmuid] entry 1483 * entry - In, pass in (&sfmmu_ctxs[mmuid] - SFMMU_CTXS) from the caller. 1484 * gnum - Out, return sfmmu gnum 1485 * cnum - Out, return sfmmu cnum 1486 * reg - scratch 1487 */ 1488 #define SFMMU_MMUID_GNUM_CNUM(entry, gnum, cnum, reg) \ 1489 ldx [entry + SFMMU_CTXS], reg; /* reg = sfmmu (gnum | cnum) */ \ 1490 srlx reg, SFMMU_MMU_GNUM_RSHIFT, gnum; /* gnum = sfmmu gnum */ \ 1491 sllx reg, SFMMU_MMU_CNUM_LSHIFT, cnum; \ 1492 srlx cnum, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */ 1493 1494 /* 1495 * Macro to get this CPU's tsbmiss area. 1496 */ 1497 #define CPU_TSBMISS_AREA(tsbmiss, tmp1) \ 1498 CPU_INDEX(tmp1, tsbmiss); /* tmp1 = cpu idx */ \ 1499 sethi %hi(tsbmiss_area), tsbmiss; /* tsbmiss base ptr */ \ 1500 sllx tmp1, TSBMISS_SHIFT, tmp1; /* byte offset */ \ 1501 or tsbmiss, %lo(tsbmiss_area), tsbmiss; \ 1502 add tsbmiss, tmp1, tsbmiss /* tsbmiss area of CPU */ 1503 1504 1505 /* 1506 * Macro to set kernel context + page size codes in DMMU primary context 1507 * register. It is only necessary for sun4u because sun4v does not need 1508 * page size codes 1509 */ 1510 #ifdef sun4v 1511 1512 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) 1513 1514 #else 1515 1516 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) \ 1517 sethi %hi(kcontextreg), reg0; \ 1518 ldx [reg0 + %lo(kcontextreg)], reg0; \ 1519 mov MMU_PCONTEXT, reg1; \ 1520 ldxa [reg1]ASI_MMU_CTX, reg2; \ 1521 xor reg0, reg2, reg2; \ 1522 brz reg2, label3; \ 1523 srlx reg2, CTXREG_NEXT_SHIFT, reg2; \ 1524 rdpr %pstate, reg3; /* disable interrupts */ \ 1525 btst PSTATE_IE, reg3; \ 1526 /*CSTYLED*/ \ 1527 bnz,a,pt %icc, label1; \ 1528 wrpr reg3, PSTATE_IE, %pstate; \ 1529 /*CSTYLED*/ \ 1530 label1:; \ 1531 brz reg2, label2; /* need demap if N_pgsz0/1 change */ \ 1532 sethi %hi(FLUSH_ADDR), reg4; \ 1533 mov DEMAP_ALL_TYPE, reg2; \ 1534 stxa %g0, [reg2]ASI_DTLB_DEMAP; \ 1535 stxa %g0, [reg2]ASI_ITLB_DEMAP; \ 1536 /*CSTYLED*/ \ 1537 label2:; \ 1538 stxa reg0, [reg1]ASI_MMU_CTX; \ 1539 flush reg4; \ 1540 btst PSTATE_IE, reg3; \ 1541 /*CSTYLED*/ \ 1542 bnz,a,pt %icc, label3; \ 1543 wrpr %g0, reg3, %pstate; /* restore interrupt state */ \ 1544 label3:; 1545 1546 #endif 1547 1548 /* 1549 * Macro to setup arguments with kernel sfmmup context + page size before 1550 * calling sfmmu_setctx_sec() 1551 */ 1552 #ifdef sun4v 1553 #define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \ 1554 set KCONTEXT, arg0; \ 1555 set 0, arg1; 1556 #else 1557 #define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \ 1558 ldub [sfmmup + SFMMU_CEXT], arg1; \ 1559 set KCONTEXT, arg0; \ 1560 sll arg1, CTXREG_EXT_SHIFT, arg1; 1561 #endif 1562 1563 #define PANIC_IF_INTR_DISABLED_PSTR(pstatereg, label, scr) \ 1564 andcc pstatereg, PSTATE_IE, %g0; /* panic if intrs */ \ 1565 /*CSTYLED*/ \ 1566 bnz,pt %icc, label; /* already disabled */ \ 1567 nop; \ 1568 \ 1569 sethi %hi(panicstr), scr; \ 1570 ldx [scr + %lo(panicstr)], scr; \ 1571 tst scr; \ 1572 /*CSTYLED*/ \ 1573 bnz,pt %xcc, label; \ 1574 nop; \ 1575 \ 1576 save %sp, -SA(MINFRAME), %sp; \ 1577 sethi %hi(sfmmu_panic1), %o0; \ 1578 call panic; \ 1579 or %o0, %lo(sfmmu_panic1), %o0; \ 1580 /*CSTYLED*/ \ 1581 label: 1582 1583 #define PANIC_IF_INTR_ENABLED_PSTR(label, scr) \ 1584 /* \ 1585 * The caller must have disabled interrupts. \ 1586 * If interrupts are not disabled, panic \ 1587 */ \ 1588 rdpr %pstate, scr; \ 1589 andcc scr, PSTATE_IE, %g0; \ 1590 /*CSTYLED*/ \ 1591 bz,pt %icc, label; \ 1592 nop; \ 1593 \ 1594 sethi %hi(panicstr), scr; \ 1595 ldx [scr + %lo(panicstr)], scr; \ 1596 tst scr; \ 1597 /*CSTYLED*/ \ 1598 bnz,pt %xcc, label; \ 1599 nop; \ 1600 \ 1601 sethi %hi(sfmmu_panic6), %o0; \ 1602 call panic; \ 1603 or %o0, %lo(sfmmu_panic6), %o0; \ 1604 /*CSTYLED*/ \ 1605 label: 1606 1607 #endif /* _ASM */ 1608 1609 #ifndef _ASM 1610 1611 /* 1612 * Page coloring 1613 * The p_vcolor field of the page struct (1 byte) is used to store the 1614 * virtual page color. This provides for 255 colors. The value zero is 1615 * used to mean the page has no color - never been mapped or somehow 1616 * purified. 1617 */ 1618 1619 #define PP_GET_VCOLOR(pp) (((pp)->p_vcolor) - 1) 1620 #define PP_NEWPAGE(pp) (!(pp)->p_vcolor) 1621 #define PP_SET_VCOLOR(pp, color) \ 1622 ((pp)->p_vcolor = ((color) + 1)) 1623 1624 /* 1625 * As mentioned p_vcolor == 0 means there is no color for this page. 1626 * But PP_SET_VCOLOR(pp, color) expects 'color' to be real color minus 1627 * one so we define this constant. 1628 */ 1629 #define NO_VCOLOR (-1) 1630 1631 #define addr_to_vcolor(addr) \ 1632 (((uint_t)(uintptr_t)(addr) >> MMU_PAGESHIFT) & vac_colors_mask) 1633 1634 /* 1635 * The field p_index in the psm page structure is for large pages support. 1636 * P_index is a bit-vector of the different mapping sizes that a given page 1637 * is part of. An hme structure for a large mapping is only added in the 1638 * group leader page (first page). All pages covered by a given large mapping 1639 * have the corrosponding mapping bit set in their p_index field. This allows 1640 * us to only store an explicit hme structure in the leading page which 1641 * simplifies the mapping link list management. Furthermore, it provides us 1642 * a fast mechanism for determining the largest mapping a page is part of. For 1643 * exmaple, a page with a 64K and a 4M mappings has a p_index value of 0x0A. 1644 * 1645 * Implementation note: even though the first bit in p_index is reserved 1646 * for 8K mappings, it is NOT USED by the code and SHOULD NOT be set. 1647 * In addition, the upper four bits of the p_index field are used by the 1648 * code as temporaries 1649 */ 1650 1651 /* 1652 * Defines for psm page struct fields and large page support 1653 */ 1654 #define SFMMU_INDEX_SHIFT 6 1655 #define SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1) 1656 1657 /* Return the mapping index */ 1658 #define PP_MAPINDEX(pp) ((pp)->p_index & SFMMU_INDEX_MASK) 1659 1660 /* 1661 * These macros rely on the following property: 1662 * All pages constituting a large page are covered by a virtually 1663 * contiguous set of page_t's. 1664 */ 1665 1666 /* Return the leader for this mapping size */ 1667 #define PP_GROUPLEADER(pp, sz) \ 1668 (&(pp)[-(int)(pp->p_pagenum & (TTEPAGES(sz)-1))]) 1669 1670 /* Return the root page for this page based on p_szc */ 1671 #define PP_PAGEROOT(pp) ((pp)->p_szc == 0 ? (pp) : \ 1672 PP_GROUPLEADER((pp), (pp)->p_szc)) 1673 1674 #define PP_PAGENEXT_N(pp, n) ((pp) + (n)) 1675 #define PP_PAGENEXT(pp) PP_PAGENEXT_N((pp), 1) 1676 1677 #define PP_PAGEPREV_N(pp, n) ((pp) - (n)) 1678 #define PP_PAGEPREV(pp) PP_PAGEPREV_N((pp), 1) 1679 1680 #define PP_ISMAPPED_LARGE(pp) (PP_MAPINDEX(pp) != 0) 1681 1682 /* Need function to test the page mappping which takes p_index into account */ 1683 #define PP_ISMAPPED(pp) ((pp)->p_mapping || PP_ISMAPPED_LARGE(pp)) 1684 1685 /* 1686 * Don't call this macro with sz equal to zero. 8K mappings SHOULD NOT 1687 * set p_index field. 1688 */ 1689 #define PAGESZ_TO_INDEX(sz) (1 << (sz)) 1690 1691 1692 /* 1693 * prototypes for hat assembly routines. Some of these are 1694 * known to machine dependent VM code. 1695 */ 1696 extern uint64_t sfmmu_make_tsbtag(caddr_t); 1697 extern struct tsbe * 1698 sfmmu_get_tsbe(uint64_t, caddr_t, int, int); 1699 extern void sfmmu_load_tsbe(struct tsbe *, uint64_t, tte_t *, int); 1700 extern void sfmmu_unload_tsbe(struct tsbe *, uint64_t, int); 1701 extern void sfmmu_load_mmustate(sfmmu_t *); 1702 extern void sfmmu_raise_tsb_exception(uint64_t, uint64_t); 1703 #ifndef sun4v 1704 extern void sfmmu_itlb_ld_kva(caddr_t, tte_t *); 1705 extern void sfmmu_dtlb_ld_kva(caddr_t, tte_t *); 1706 #endif /* sun4v */ 1707 extern void sfmmu_copytte(tte_t *, tte_t *); 1708 extern int sfmmu_modifytte(tte_t *, tte_t *, tte_t *); 1709 extern int sfmmu_modifytte_try(tte_t *, tte_t *, tte_t *); 1710 extern pfn_t sfmmu_ttetopfn(tte_t *, caddr_t); 1711 extern void sfmmu_hblk_hash_rm(struct hmehash_bucket *, 1712 struct hme_blk *, uint64_t, struct hme_blk *); 1713 extern void sfmmu_hblk_hash_add(struct hmehash_bucket *, struct hme_blk *, 1714 uint64_t); 1715 extern uint_t sfmmu_disable_intrs(void); 1716 extern void sfmmu_enable_intrs(uint_t); 1717 /* 1718 * functions exported to machine dependent VM code 1719 */ 1720 extern void sfmmu_patch_ktsb(void); 1721 #ifndef UTSB_PHYS 1722 extern void sfmmu_patch_utsb(void); 1723 #endif /* UTSB_PHYS */ 1724 extern pfn_t sfmmu_vatopfn(caddr_t, sfmmu_t *, tte_t *); 1725 extern void sfmmu_vatopfn_suspended(caddr_t, sfmmu_t *, tte_t *); 1726 #ifdef DEBUG 1727 extern void sfmmu_check_kpfn(pfn_t); 1728 #else 1729 #define sfmmu_check_kpfn(pfn) /* disabled */ 1730 #endif /* DEBUG */ 1731 extern void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 1732 extern void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, uint_t); 1733 extern void sfmmu_tsbmiss_exception(struct regs *, uintptr_t, uint_t); 1734 extern void sfmmu_init_tsbs(void); 1735 extern caddr_t sfmmu_ktsb_alloc(caddr_t); 1736 extern int sfmmu_getctx_pri(void); 1737 extern int sfmmu_getctx_sec(void); 1738 extern void sfmmu_setctx_sec(int); 1739 extern void sfmmu_inv_tsb(caddr_t, uint_t); 1740 extern void sfmmu_init_ktsbinfo(void); 1741 extern int sfmmu_setup_4lp(void); 1742 extern void sfmmu_patch_mmu_asi(int); 1743 extern void sfmmu_init_nucleus_hblks(caddr_t, size_t, int, int); 1744 extern void sfmmu_cache_flushall(void); 1745 extern pgcnt_t sfmmu_tte_cnt(sfmmu_t *, uint_t); 1746 extern void *sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int); 1747 extern void sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t); 1748 extern void sfmmu_reprog_pgsz_arr(sfmmu_t *, uint8_t *); 1749 1750 extern void hat_kern_setup(void); 1751 extern int hat_page_relocate(page_t **, page_t **, spgcnt_t *); 1752 extern uint_t hat_preferred_pgsz(struct hat *, caddr_t, size_t, int); 1753 extern int sfmmu_get_ppvcolor(struct page *); 1754 extern int sfmmu_get_addrvcolor(caddr_t); 1755 extern int sfmmu_hat_lock_held(sfmmu_t *); 1756 extern void sfmmu_alloc_ctx(sfmmu_t *, int, struct cpu *); 1757 1758 /* 1759 * Functions exported to xhat_sfmmu.c 1760 */ 1761 extern kmutex_t *sfmmu_mlist_enter(page_t *); 1762 extern void sfmmu_mlist_exit(kmutex_t *); 1763 extern int sfmmu_mlist_held(struct page *); 1764 extern struct hme_blk *sfmmu_hmetohblk(struct sf_hment *); 1765 1766 /* 1767 * MMU-specific functions optionally imported from the CPU module 1768 */ 1769 #pragma weak mmu_large_pages_disabled 1770 #pragma weak mmu_set_ctx_page_sizes 1771 #pragma weak mmu_preferred_pgsz 1772 #pragma weak mmu_check_page_sizes 1773 1774 extern int mmu_large_pages_disabled(uint_t); 1775 extern void mmu_set_ctx_page_sizes(sfmmu_t *); 1776 extern uint_t mmu_preferred_pgsz(sfmmu_t *, caddr_t, size_t); 1777 extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *); 1778 1779 extern sfmmu_t *ksfmmup; 1780 extern caddr_t ktsb_base; 1781 extern uint64_t ktsb_pbase; 1782 extern int ktsb_sz; 1783 extern int ktsb_szcode; 1784 extern caddr_t ktsb4m_base; 1785 extern uint64_t ktsb4m_pbase; 1786 extern int ktsb4m_sz; 1787 extern int ktsb4m_szcode; 1788 extern uint64_t kpm_tsbbase; 1789 extern int kpm_tsbsz; 1790 extern int ktsb_phys; 1791 extern int enable_bigktsb; 1792 #ifndef sun4v 1793 extern int utsb_dtlb_ttenum; 1794 extern int utsb4m_dtlb_ttenum; 1795 #endif /* sun4v */ 1796 extern int uhmehash_num; 1797 extern int khmehash_num; 1798 extern struct hmehash_bucket *uhme_hash; 1799 extern struct hmehash_bucket *khme_hash; 1800 extern kmutex_t *mml_table; 1801 extern uint_t mml_table_sz; 1802 extern uint_t mml_shift; 1803 extern uint_t hblk_alloc_dynamic; 1804 extern struct tsbmiss tsbmiss_area[NCPU]; 1805 extern struct kpmtsbm kpmtsbm_area[NCPU]; 1806 extern int tsb_max_growsize; 1807 #ifndef sun4v 1808 extern int dtlb_resv_ttenum; 1809 extern caddr_t utsb_vabase; 1810 extern caddr_t utsb4m_vabase; 1811 #endif /* sun4v */ 1812 extern vmem_t *kmem_tsb_default_arena[]; 1813 extern int tsb_lgrp_affinity; 1814 1815 /* kpm externals */ 1816 extern pfn_t sfmmu_kpm_vatopfn(caddr_t); 1817 extern void sfmmu_kpm_patch_tlbm(void); 1818 extern void sfmmu_kpm_patch_tsbm(void); 1819 extern void sfmmu_kpm_load_tsb(caddr_t, tte_t *, int); 1820 extern void sfmmu_kpm_unload_tsb(caddr_t, int); 1821 extern void sfmmu_kpm_tsbmtl(short *, uint_t *, int); 1822 extern int sfmmu_kpm_stsbmtl(char *, uint_t *, int); 1823 extern caddr_t kpm_vbase; 1824 extern size_t kpm_size; 1825 extern struct memseg *memseg_hash[]; 1826 extern uint64_t memseg_phash[]; 1827 extern kpm_hlk_t *kpmp_table; 1828 extern kpm_shlk_t *kpmp_stable; 1829 extern uint_t kpmp_table_sz; 1830 extern uint_t kpmp_stable_sz; 1831 extern uchar_t kpmp_shift; 1832 1833 #define PP_ISMAPPED_KPM(pp) ((pp)->p_kpmref > 0) 1834 1835 #define IS_KPM_ALIAS_RANGE(vaddr) \ 1836 (((vaddr) - kpm_vbase) >> (uintptr_t)kpm_size_shift > 0) 1837 1838 #endif /* !_ASM */ 1839 1840 /* sfmmu_kpm_tsbmtl flags */ 1841 #define KPMTSBM_STOP 0 1842 #define KPMTSBM_START 1 1843 1844 /* kpm_smallpages kp_mapped values */ 1845 #define KPM_MAPPEDS -1 /* small mapping valid, no conflict */ 1846 #define KPM_MAPPEDSC 1 /* small mapping valid, conflict */ 1847 1848 /* Physical memseg address NULL marker */ 1849 #define MSEG_NULLPTR_PA -1 1850 1851 /* 1852 * Memseg hash defines for kpm trap level tsbmiss handler. 1853 * Must be in sync w/ page.h . 1854 */ 1855 #define SFMMU_MEM_HASH_SHIFT 0x9 1856 #define SFMMU_N_MEM_SLOTS 0x200 1857 #define SFMMU_MEM_HASH_ENTRY_SHIFT 3 1858 1859 #ifndef _ASM 1860 #if (SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT) 1861 #error SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT 1862 #endif 1863 #if (SFMMU_N_MEM_SLOTS != N_MEM_SLOTS) 1864 #error SFMMU_N_MEM_SLOTS != N_MEM_SLOTS 1865 #endif 1866 1867 /* Physical memseg address NULL marker */ 1868 #define SFMMU_MEMSEG_NULLPTR_PA -1 1869 1870 /* 1871 * Check KCONTEXT to be zero, asm parts depend on that assumption. 1872 */ 1873 #if (KCONTEXT != 0) 1874 #error KCONTEXT != 0 1875 #endif 1876 #endif /* !_ASM */ 1877 1878 1879 #endif /* _KERNEL */ 1880 1881 #ifndef _ASM 1882 /* 1883 * ctx, hmeblk, mlistlock and other stats for sfmmu 1884 */ 1885 struct sfmmu_global_stat { 1886 int sf_tsb_exceptions; /* # of tsb exceptions */ 1887 int sf_tsb_raise_exception; /* # tsb exc. w/o TLB flush */ 1888 1889 int sf_pagefaults; /* # of pagefaults */ 1890 1891 int sf_uhash_searches; /* # of user hash searches */ 1892 int sf_uhash_links; /* # of user hash links */ 1893 int sf_khash_searches; /* # of kernel hash searches */ 1894 int sf_khash_links; /* # of kernel hash links */ 1895 1896 int sf_swapout; /* # times hat swapped out */ 1897 1898 int sf_tsb_alloc; /* # TSB allocations */ 1899 int sf_tsb_allocfail; /* # times TSB alloc fail */ 1900 int sf_tsb_sectsb_create; /* # times second TSB added */ 1901 1902 int sf_tteload8k; /* calls to sfmmu_tteload */ 1903 int sf_tteload64k; /* calls to sfmmu_tteload */ 1904 int sf_tteload512k; /* calls to sfmmu_tteload */ 1905 int sf_tteload4m; /* calls to sfmmu_tteload */ 1906 int sf_tteload32m; /* calls to sfmmu_tteload */ 1907 int sf_tteload256m; /* calls to sfmmu_tteload */ 1908 1909 int sf_tsb_load8k; /* # times loaded 8K tsbent */ 1910 int sf_tsb_load4m; /* # times loaded 4M tsbent */ 1911 1912 int sf_hblk_hit; /* found hblk during tteload */ 1913 int sf_hblk8_ncreate; /* static hblk8's created */ 1914 int sf_hblk8_nalloc; /* static hblk8's allocated */ 1915 int sf_hblk1_ncreate; /* static hblk1's created */ 1916 int sf_hblk1_nalloc; /* static hblk1's allocated */ 1917 int sf_hblk_slab_cnt; /* sfmmu8_cache slab creates */ 1918 int sf_hblk_reserve_cnt; /* hblk_reserve usage */ 1919 int sf_hblk_recurse_cnt; /* hblk_reserve owner reqs */ 1920 int sf_hblk_reserve_hit; /* hblk_reserve hash hits */ 1921 int sf_get_free_success; /* reserve list allocs */ 1922 int sf_get_free_throttle; /* fails due to throttling */ 1923 int sf_get_free_fail; /* fails due to empty list */ 1924 int sf_put_free_success; /* reserve list frees */ 1925 int sf_put_free_fail; /* fails due to full list */ 1926 1927 int sf_pgcolor_conflict; /* VAC conflict resolution */ 1928 int sf_uncache_conflict; /* VAC conflict resolution */ 1929 int sf_unload_conflict; /* VAC unload resolution */ 1930 int sf_ism_uncache; /* VAC conflict resolution */ 1931 int sf_ism_recache; /* VAC conflict resolution */ 1932 int sf_recache; /* VAC conflict resolution */ 1933 1934 int sf_steal_count; /* # of hblks stolen */ 1935 1936 int sf_pagesync; /* # of pagesyncs */ 1937 int sf_clrwrt; /* # of clear write perms */ 1938 int sf_pagesync_invalid; /* pagesync with inv tte */ 1939 1940 int sf_kernel_xcalls; /* # of kernel cross calls */ 1941 int sf_user_xcalls; /* # of user cross calls */ 1942 1943 int sf_tsb_grow; /* # of user tsb grows */ 1944 int sf_tsb_shrink; /* # of user tsb shrinks */ 1945 int sf_tsb_resize_failures; /* # of user tsb resize */ 1946 int sf_tsb_reloc; /* # of user tsb relocations */ 1947 1948 int sf_user_vtop; /* # of user vatopfn calls */ 1949 1950 int sf_ctx_inv; /* #times invalidate MMU ctx */ 1951 1952 int sf_tlb_reprog_pgsz; /* # times switch TLB pgsz */ 1953 }; 1954 1955 struct sfmmu_tsbsize_stat { 1956 int sf_tsbsz_8k; 1957 int sf_tsbsz_16k; 1958 int sf_tsbsz_32k; 1959 int sf_tsbsz_64k; 1960 int sf_tsbsz_128k; 1961 int sf_tsbsz_256k; 1962 int sf_tsbsz_512k; 1963 int sf_tsbsz_1m; 1964 int sf_tsbsz_2m; 1965 int sf_tsbsz_4m; 1966 }; 1967 1968 struct sfmmu_percpu_stat { 1969 int sf_itlb_misses; /* # of itlb misses */ 1970 int sf_dtlb_misses; /* # of dtlb misses */ 1971 int sf_utsb_misses; /* # of user tsb misses */ 1972 int sf_ktsb_misses; /* # of kernel tsb misses */ 1973 int sf_tsb_hits; /* # of tsb hits */ 1974 int sf_umod_faults; /* # of mod (prot viol) flts */ 1975 int sf_kmod_faults; /* # of mod (prot viol) flts */ 1976 }; 1977 1978 #define SFMMU_STAT(stat) sfmmu_global_stat.stat++ 1979 #define SFMMU_STAT_ADD(stat, amount) sfmmu_global_stat.stat += (amount) 1980 #define SFMMU_STAT_SET(stat, count) sfmmu_global_stat.stat = (count) 1981 1982 #define SFMMU_MMU_STAT(stat) CPU->cpu_m.cpu_mmu_ctxp->stat++ 1983 1984 #endif /* !_ASM */ 1985 1986 #ifdef __cplusplus 1987 } 1988 #endif 1989 1990 #endif /* _VM_HAT_SFMMU_H */ 1991