1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * VM - Hardware Address Translation management. 28 * 29 * This file describes the contents of the sun-reference-mmu(sfmmu)- 30 * specific hat data structures and the sfmmu-specific hat procedures. 31 * The machine-independent interface is described in <vm/hat.h>. 32 */ 33 34 #ifndef _VM_HAT_SFMMU_H 35 #define _VM_HAT_SFMMU_H 36 37 #pragma ident "%Z%%M% %I% %E% SMI" 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 #ifndef _ASM 44 45 #include <sys/types.h> 46 47 #endif /* _ASM */ 48 49 #ifdef _KERNEL 50 51 #include <sys/pte.h> 52 #include <vm/mach_sfmmu.h> 53 #include <sys/mmu.h> 54 55 /* 56 * Don't alter these without considering changes to ism_map_t. 57 */ 58 #define DEFAULT_ISM_PAGESIZE MMU_PAGESIZE4M 59 #define ISM_PG_SIZE(ism_vbshift) (1 << ism_vbshift) 60 #define ISM_SZ_MASK(ism_vbshift) (ISM_PG_SIZE(ism_vbshift) - 1) 61 #define ISM_MAP_SLOTS 8 /* Change this carefully. */ 62 63 #ifndef _ASM 64 65 #include <sys/t_lock.h> 66 #include <vm/hat.h> 67 #include <vm/seg.h> 68 #include <sys/machparam.h> 69 #include <sys/systm.h> 70 #include <sys/x_call.h> 71 #include <vm/page.h> 72 #include <sys/ksynch.h> 73 74 typedef struct hat sfmmu_t; 75 76 /* 77 * SFMMU attributes for hat_memload/hat_devload 78 */ 79 #define SFMMU_UNCACHEPTTE 0x01000000 /* unencache in physical $ */ 80 #define SFMMU_UNCACHEVTTE 0x02000000 /* unencache in virtual $ */ 81 #define SFMMU_SIDEFFECT 0x04000000 /* set side effect bit */ 82 #define SFMMU_LOAD_ALLATTR (HAT_PROT_MASK | HAT_ORDER_MASK | \ 83 HAT_ENDIAN_MASK | HAT_NOFAULT | HAT_NOSYNC | \ 84 SFMMU_UNCACHEPTTE | SFMMU_UNCACHEVTTE | SFMMU_SIDEFFECT) 85 86 87 /* 88 * sfmmu flags for hat_memload/hat_devload 89 */ 90 #define SFMMU_NO_TSBLOAD 0x08000000 /* do not preload tsb */ 91 #define SFMMU_LOAD_ALLFLAG (HAT_LOAD | HAT_LOAD_LOCK | \ 92 HAT_LOAD_ADV | HAT_LOAD_CONTIG | HAT_LOAD_NOCONSIST | \ 93 HAT_LOAD_SHARE | HAT_LOAD_REMAP | SFMMU_NO_TSBLOAD | \ 94 HAT_RELOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_TEXT) 95 96 /* 97 * sfmmu internal flag to hat_pageunload that spares locked mappings 98 */ 99 #define SFMMU_KERNEL_RELOC 0x8000 100 101 /* 102 * mode for sfmmu_chgattr 103 */ 104 #define SFMMU_SETATTR 0x0 105 #define SFMMU_CLRATTR 0x1 106 #define SFMMU_CHGATTR 0x2 107 108 /* 109 * sfmmu specific flags for page_t 110 */ 111 #define P_PNC 0x8 /* non-caching is permanent bit */ 112 #define P_TNC 0x10 /* non-caching is temporary bit */ 113 #define P_KPMS 0x20 /* kpm mapped small (vac alias prevention) */ 114 #define P_KPMC 0x40 /* kpm conflict page (vac alias prevention) */ 115 116 #define PP_GENERIC_ATTR(pp) ((pp)->p_nrm & (P_MOD | P_REF | P_RO)) 117 #define PP_ISMOD(pp) ((pp)->p_nrm & P_MOD) 118 #define PP_ISREF(pp) ((pp)->p_nrm & P_REF) 119 #define PP_ISRO(pp) ((pp)->p_nrm & P_RO) 120 #define PP_ISNC(pp) ((pp)->p_nrm & (P_PNC|P_TNC)) 121 #define PP_ISPNC(pp) ((pp)->p_nrm & P_PNC) 122 #ifdef VAC 123 #define PP_ISTNC(pp) ((pp)->p_nrm & P_TNC) 124 #endif 125 #define PP_ISKPMS(pp) ((pp)->p_nrm & P_KPMS) 126 #define PP_ISKPMC(pp) ((pp)->p_nrm & P_KPMC) 127 128 #define PP_SETMOD(pp) ((pp)->p_nrm |= P_MOD) 129 #define PP_SETREF(pp) ((pp)->p_nrm |= P_REF) 130 #define PP_SETREFMOD(pp) ((pp)->p_nrm |= (P_REF|P_MOD)) 131 #define PP_SETRO(pp) ((pp)->p_nrm |= P_RO) 132 #define PP_SETREFRO(pp) ((pp)->p_nrm |= (P_REF|P_RO)) 133 #define PP_SETPNC(pp) ((pp)->p_nrm |= P_PNC) 134 #ifdef VAC 135 #define PP_SETTNC(pp) ((pp)->p_nrm |= P_TNC) 136 #endif 137 #define PP_SETKPMS(pp) ((pp)->p_nrm |= P_KPMS) 138 #define PP_SETKPMC(pp) ((pp)->p_nrm |= P_KPMC) 139 140 #define PP_CLRMOD(pp) ((pp)->p_nrm &= ~P_MOD) 141 #define PP_CLRREF(pp) ((pp)->p_nrm &= ~P_REF) 142 #define PP_CLRREFMOD(pp) ((pp)->p_nrm &= ~(P_REF|P_MOD)) 143 #define PP_CLRRO(pp) ((pp)->p_nrm &= ~P_RO) 144 #define PP_CLRPNC(pp) ((pp)->p_nrm &= ~P_PNC) 145 #ifdef VAC 146 #define PP_CLRTNC(pp) ((pp)->p_nrm &= ~P_TNC) 147 #endif 148 #define PP_CLRKPMS(pp) ((pp)->p_nrm &= ~P_KPMS) 149 #define PP_CLRKPMC(pp) ((pp)->p_nrm &= ~P_KPMC) 150 151 /* 152 * All shared memory segments attached with the SHM_SHARE_MMU flag (ISM) 153 * will be constrained to a 4M, 32M or 256M alignment. Also since every newly- 154 * created ISM segment is created out of a new address space at base va 155 * of 0 we don't need to store it. 156 */ 157 #define ISM_ALIGN(shift) (1 << shift) /* base va aligned to <n>M */ 158 #define ISM_ALIGNED(shift, va) (((uintptr_t)va & (ISM_ALIGN(shift) - 1)) == 0) 159 #define ISM_SHIFT(shift, x) ((uintptr_t)x >> (shift)) 160 161 /* 162 * Pad locks out to cache sub-block boundaries to prevent 163 * false sharing, so several processes don't contend for 164 * the same line if they aren't using the same lock. Since 165 * this is a typedef we also have a bit of freedom in 166 * changing lock implementations later if we decide it 167 * is necessary. 168 */ 169 typedef struct hat_lock { 170 kmutex_t hl_mutex; 171 uchar_t hl_pad[64 - sizeof (kmutex_t)]; 172 } hatlock_t; 173 174 #define HATLOCK_MUTEXP(hatlockp) (&((hatlockp)->hl_mutex)) 175 176 /* 177 * All segments mapped with ISM are guaranteed to be 4M, 32M or 256M aligned. 178 * Also size is guaranteed to be in 4M, 32M or 256M chunks. 179 * ism_seg consists of the following members: 180 * [XX..22] base address of ism segment. XX is 63 or 31 depending whether 181 * caddr_t is 64 bits or 32 bits. 182 * [21..0] size of segment. 183 * 184 * NOTE: Don't alter this structure without changing defines above and 185 * the tsb_miss and protection handlers. 186 */ 187 typedef struct ism_map { 188 uintptr_t imap_seg; /* base va + sz of ISM segment */ 189 ushort_t imap_vb_shift; /* mmu_pageshift for ism page size */ 190 ushort_t imap_hatflags; /* primary ism page size */ 191 uint_t imap_sz_mask; /* mmu_pagemask for ism page size */ 192 sfmmu_t *imap_ismhat; /* hat id of dummy ISM as */ 193 struct ism_ment *imap_ment; /* pointer to mapping list entry */ 194 } ism_map_t; 195 196 #define ism_start(map) ((caddr_t)((map).imap_seg & \ 197 ~ISM_SZ_MASK((map).imap_vb_shift))) 198 #define ism_size(map) ((map).imap_seg & ISM_SZ_MASK((map).imap_vb_shift)) 199 #define ism_end(map) ((caddr_t)(ism_start(map) + (ism_size(map) * \ 200 ISM_PG_SIZE((map).imap_vb_shift)))) 201 /* 202 * ISM mapping entry. Used to link all hat's sharing a ism_hat. 203 * Same function as the p_mapping list for a page. 204 */ 205 typedef struct ism_ment { 206 sfmmu_t *iment_hat; /* back pointer to hat_share() hat */ 207 caddr_t iment_base_va; /* hat's va base for this ism seg */ 208 struct ism_ment *iment_next; /* next ism map entry */ 209 struct ism_ment *iment_prev; /* prev ism map entry */ 210 } ism_ment_t; 211 212 /* 213 * ISM segment block. One will be hung off the sfmmu structure if a 214 * a process uses ISM. More will be linked using ismblk_next if more 215 * than ISM_MAP_SLOTS segments are attached to this proc. 216 * 217 * All modifications to fields in this structure will be protected 218 * by the hat mutex. In order to avoid grabbing this lock in low level 219 * routines (tsb miss/protection handlers and vatopfn) while not 220 * introducing any race conditions with hat_unshare, we will set 221 * CTX_ISM_BUSY bit in the ctx struct. Any mmu traps that occur 222 * for this ctx while this bit is set will be handled in sfmmu_tsb_excption 223 * where it will synchronize behind the hat mutex. 224 */ 225 typedef struct ism_blk { 226 ism_map_t iblk_maps[ISM_MAP_SLOTS]; 227 struct ism_blk *iblk_next; 228 uint64_t iblk_nextpa; 229 } ism_blk_t; 230 231 /* 232 * TSB access information. All fields are protected by the process's 233 * hat lock. 234 */ 235 236 struct tsb_info { 237 caddr_t tsb_va; /* tsb base virtual address */ 238 uint64_t tsb_pa; /* tsb base physical address */ 239 struct tsb_info *tsb_next; /* next tsb used by this process */ 240 uint16_t tsb_szc; /* tsb size code */ 241 uint16_t tsb_flags; /* flags for this tsb; see below */ 242 uint_t tsb_ttesz_mask; /* page size masks; see below */ 243 244 tte_t tsb_tte; /* tte to lock into DTLB */ 245 sfmmu_t *tsb_sfmmu; /* sfmmu */ 246 kmem_cache_t *tsb_cache; /* cache from which mem allocated */ 247 vmem_t *tsb_vmp; /* vmem arena from which mem alloc'd */ 248 }; 249 250 /* 251 * Values for "tsb_ttesz_mask" bitmask. 252 */ 253 #define TSB8K (1 << TTE8K) 254 #define TSB64K (1 << TTE64K) 255 #define TSB512K (1 << TTE512K) 256 #define TSB4M (1 << TTE4M) 257 #define TSB32M (1 << TTE32M) 258 #define TSB256M (1 << TTE256M) 259 260 /* 261 * Values for "tsb_flags" field. 262 */ 263 #define TSB_RELOC_FLAG 0x1 264 #define TSB_FLUSH_NEEDED 0x2 265 #define TSB_SWAPPED 0x4 266 267 /* 268 * Per-MMU context domain kstats. 269 * 270 * TSB Miss Exceptions 271 * Number of times a TSB miss exception is handled in an MMU. See 272 * sfmmu_tsbmiss_exception() for more details. 273 * TSB Raise Exception 274 * Number of times the CPUs within an MMU are cross-called 275 * to invalidate either a specific process context (when the process 276 * switches MMU contexts) or the context of any process that is 277 * running on those CPUs (as part of the MMU context wrap-around). 278 * Wrap Around 279 * The number of times a wrap-around of MMU context happens. 280 */ 281 typedef enum mmu_ctx_stat_types { 282 MMU_CTX_TSB_EXCEPTIONS, /* TSB miss exceptions handled */ 283 MMU_CTX_TSB_RAISE_EXCEPTION, /* ctx invalidation cross calls */ 284 MMU_CTX_WRAP_AROUND, /* wraparounds */ 285 MMU_CTX_NUM_STATS 286 } mmu_ctx_stat_t; 287 288 /* 289 * Per-MMU context domain structure. This is instantiated the first time a CPU 290 * belonging to the MMU context domain is configured into the system, at boot 291 * time or at DR time. 292 * 293 * mmu_gnum 294 * The current generation number for the context IDs on this MMU context 295 * domain. It is protected by mmu_lock. 296 * mmu_cnum 297 * The current cnum to be allocated on this MMU context domain. It 298 * is protected via CAS. 299 * mmu_nctxs 300 * The max number of context IDs supported on every CPU in this 301 * MMU context domain. It is 8K except for Rock where it is 64K. 302 * This is needed here in case the system supports mixed type of 303 * processors/MMUs. It also helps to make ctx switch code access 304 * fewer cache lines i.e. no need to retrieve it from some global nctxs. 305 * mmu_lock 306 * The mutex spin lock used to serialize context ID wrap around 307 * mmu_idx 308 * The index for this MMU context domain structure in the global array 309 * mmu_ctxdoms. 310 * mmu_ncpus 311 * The actual number of CPUs that have been configured in this 312 * MMU context domain. This also acts as a reference count for the 313 * structure. When the last CPU in an MMU context domain is unconfigured, 314 * the structure is freed. It is protected by mmu_lock. 315 * mmu_cpuset 316 * The CPU set of configured CPUs for this MMU context domain. Used 317 * to cross-call all the CPUs in the MMU context domain to invalidate 318 * context IDs during a wraparound operation. It is protected by mmu_lock. 319 */ 320 321 typedef struct mmu_ctx { 322 uint64_t mmu_gnum; 323 uint_t mmu_cnum; 324 uint_t mmu_nctxs; 325 kmutex_t mmu_lock; 326 uint_t mmu_idx; 327 uint_t mmu_ncpus; 328 cpuset_t mmu_cpuset; 329 kstat_t *mmu_kstat; 330 kstat_named_t mmu_kstat_data[MMU_CTX_NUM_STATS]; 331 } mmu_ctx_t; 332 333 #define mmu_tsb_exceptions \ 334 mmu_kstat_data[MMU_CTX_TSB_EXCEPTIONS].value.ui64 335 #define mmu_tsb_raise_exception \ 336 mmu_kstat_data[MMU_CTX_TSB_RAISE_EXCEPTION].value.ui64 337 #define mmu_wrap_around \ 338 mmu_kstat_data[MMU_CTX_WRAP_AROUND].value.ui64 339 340 extern uint_t max_mmu_ctxdoms; 341 extern mmu_ctx_t **mmu_ctxs_tbl; 342 343 extern void sfmmu_cpu_init(cpu_t *); 344 extern void sfmmu_cpu_cleanup(cpu_t *); 345 346 /* 347 * The following structure is used to get MMU context domain information for 348 * a CPU from the platform. 349 * 350 * mmu_idx 351 * The MMU context domain index within the global array mmu_ctxs 352 * mmu_nctxs 353 * The number of context IDs supported in the MMU context domain 354 * (64K for Rock) 355 */ 356 typedef struct mmu_ctx_info { 357 uint_t mmu_idx; 358 uint_t mmu_nctxs; 359 } mmu_ctx_info_t; 360 361 #pragma weak plat_cpuid_to_mmu_ctx_info 362 363 extern void plat_cpuid_to_mmu_ctx_info(processorid_t, mmu_ctx_info_t *); 364 365 /* 366 * Each address space has an array of sfmmu_ctx_t structures, one structure 367 * per MMU context domain. 368 * 369 * cnum 370 * The context ID allocated for an address space on an MMU context domain 371 * gnum 372 * The generation number for the context ID in the MMU context domain. 373 * 374 * This structure needs to be a power-of-two in size. 375 */ 376 typedef struct sfmmu_ctx { 377 uint64_t gnum:48; 378 uint64_t cnum:16; 379 } sfmmu_ctx_t; 380 381 382 /* 383 * The platform dependent hat structure. 384 * tte counts should be protected by cas. 385 * cpuset is protected by cas. 386 * 387 * Note that sfmmu_xhat_provider MUST be the first element. 388 */ 389 struct hat { 390 void *sfmmu_xhat_provider; /* NULL for CPU hat */ 391 cpuset_t sfmmu_cpusran; /* cpu bit mask for efficient xcalls */ 392 struct as *sfmmu_as; /* as this hat provides mapping for */ 393 ulong_t sfmmu_ttecnt[MMU_PAGE_SIZES]; /* per sz tte counts */ 394 ulong_t sfmmu_ismttecnt[MMU_PAGE_SIZES]; /* est. ism ttes */ 395 union _h_un { 396 ism_blk_t *sfmmu_iblkp; /* maps to ismhat(s) */ 397 ism_ment_t *sfmmu_imentp; /* ism hat's mapping list */ 398 } h_un; 399 uint_t sfmmu_free:1; /* hat to be freed - set on as_free */ 400 uint_t sfmmu_ismhat:1; /* hat is dummy ism hatid */ 401 uint_t sfmmu_ctxflushed:1; /* ctx has been flushed */ 402 uchar_t sfmmu_rmstat; /* refmod stats refcnt */ 403 ushort_t sfmmu_clrstart; /* start color bin for page coloring */ 404 ushort_t sfmmu_clrbin; /* per as phys page coloring bin */ 405 ushort_t sfmmu_flags; /* flags */ 406 struct tsb_info *sfmmu_tsb; /* list of per as tsbs */ 407 uint64_t sfmmu_ismblkpa; /* pa of sfmmu_iblkp, or -1 */ 408 lock_t sfmmu_ctx_lock; /* sync ctx alloc and invalidation */ 409 kcondvar_t sfmmu_tsb_cv; /* signals TSB swapin or relocation */ 410 uchar_t sfmmu_cext; /* context page size encoding */ 411 uint8_t sfmmu_pgsz[MMU_PAGE_SIZES]; /* ranking for MMU */ 412 #ifdef sun4v 413 struct hv_tsb_block sfmmu_hvblock; 414 #endif 415 /* 416 * sfmmu_ctxs is a variable length array of max_mmu_ctxdoms # of 417 * elements. max_mmu_ctxdoms is determined at run-time. 418 * sfmmu_ctxs[1] is just the fist element of an array, it always 419 * has to be the last field to ensure that the memory allocated 420 * for sfmmu_ctxs is consecutive with the memory of the rest of 421 * the hat data structure. 422 */ 423 sfmmu_ctx_t sfmmu_ctxs[1]; 424 425 }; 426 427 #define sfmmu_iblk h_un.sfmmu_iblkp 428 #define sfmmu_iment h_un.sfmmu_imentp 429 430 /* 431 * bit mask for managing vac conflicts on large pages. 432 * bit 1 is for uncache flag. 433 * bits 2 through min(num of cache colors + 1,31) are 434 * for cache colors that have already been flushed. 435 */ 436 #ifdef VAC 437 #define CACHE_NUM_COLOR (shm_alignment >> MMU_PAGESHIFT) 438 #else 439 #define CACHE_NUM_COLOR 1 440 #endif 441 442 #define CACHE_VCOLOR_MASK(vcolor) (2 << (vcolor & (CACHE_NUM_COLOR - 1))) 443 444 #define CacheColor_IsFlushed(flag, vcolor) \ 445 ((flag) & CACHE_VCOLOR_MASK(vcolor)) 446 447 #define CacheColor_SetFlushed(flag, vcolor) \ 448 ((flag) |= CACHE_VCOLOR_MASK(vcolor)) 449 /* 450 * Flags passed to sfmmu_page_cache to flush page from vac or not. 451 */ 452 #define CACHE_FLUSH 0 453 #define CACHE_NO_FLUSH 1 454 455 /* 456 * Flags passed to sfmmu_tlbcache_demap 457 */ 458 #define FLUSH_NECESSARY_CPUS 0 459 #define FLUSH_ALL_CPUS 1 460 461 #ifdef DEBUG 462 /* 463 * For debugging purpose only. Maybe removed later. 464 */ 465 struct ctx_trace { 466 sfmmu_t *sc_sfmmu_stolen; 467 sfmmu_t *sc_sfmmu_stealing; 468 clock_t sc_time; 469 ushort_t sc_type; 470 ushort_t sc_cnum; 471 }; 472 #define CTX_TRC_STEAL 0x1 473 #define CTX_TRC_FREE 0x0 474 #define TRSIZE 0x400 475 #define NEXT_CTXTR(ptr) (((ptr) >= ctx_trace_last) ? \ 476 ctx_trace_first : ((ptr) + 1)) 477 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) \ 478 mutex_enter(mutex); \ 479 (ptr)->sc_sfmmu_stolen = (stolen_sfmmu); \ 480 (ptr)->sc_sfmmu_stealing = (stealing_sfmmu); \ 481 (ptr)->sc_cnum = (cnum); \ 482 (ptr)->sc_type = (type); \ 483 (ptr)->sc_time = lbolt; \ 484 (ptr) = NEXT_CTXTR(ptr); \ 485 num_ctx_stolen += (type); \ 486 mutex_exit(mutex); 487 #else 488 489 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) 490 491 #endif /* DEBUG */ 492 493 #endif /* !_ASM */ 494 495 /* 496 * Macros for sfmmup->sfmmu_flags access. The macros that change the flags 497 * ASSERT() that we're holding the HAT lock before changing the flags; 498 * however callers that read the flags may do so without acquiring the lock 499 * in a fast path, and then recheck the flag after acquiring the lock in 500 * a slow path. 501 */ 502 #define SFMMU_FLAGS_ISSET(sfmmup, flags) \ 503 (((sfmmup)->sfmmu_flags & (flags)) == (flags)) 504 505 #define SFMMU_FLAGS_CLEAR(sfmmup, flags) \ 506 (ASSERT(sfmmu_hat_lock_held((sfmmup))), \ 507 (sfmmup)->sfmmu_flags &= ~(flags)) 508 509 #define SFMMU_FLAGS_SET(sfmmup, flags) \ 510 (ASSERT(sfmmu_hat_lock_held((sfmmup))), \ 511 (sfmmup)->sfmmu_flags |= (flags)) 512 513 /* 514 * sfmmu HAT flags 515 */ 516 #define HAT_64K_FLAG 0x01 517 #define HAT_512K_FLAG 0x02 518 #define HAT_4M_FLAG 0x04 519 #define HAT_32M_FLAG 0x08 520 #define HAT_256M_FLAG 0x10 521 #define HAT_4MTEXT_FLAG 0x80 522 #define HAT_SWAPPED 0x100 /* swapped out */ 523 #define HAT_SWAPIN 0x200 /* swapping in */ 524 #define HAT_BUSY 0x400 /* replacing TSB(s) */ 525 #define HAT_ISMBUSY 0x800 /* adding/removing/traversing ISM maps */ 526 527 #define HAT_LGPG_FLAGS \ 528 (HAT_64K_FLAG | HAT_512K_FLAG | HAT_4M_FLAG | \ 529 HAT_32M_FLAG | HAT_256M_FLAG) 530 531 #define HAT_FLAGS_MASK \ 532 (HAT_LGPG_FLAGS | HAT_4MTEXT_FLAG | HAT_SWAPPED | \ 533 HAT_SWAPIN | HAT_BUSY | HAT_ISMBUSY) 534 535 /* 536 * Context flags 537 */ 538 #define CTX_FREE_FLAG 0x1 539 #define CTX_FLAGS_MASK 0x1 540 541 #define CTX_SET_FLAGS(ctx, flag) \ 542 { \ 543 uint32_t old, new; \ 544 \ 545 do { \ 546 new = old = (ctx)->ctx_flags; \ 547 new &= CTX_FLAGS_MASK; \ 548 new |= flag; \ 549 new = cas32(&(ctx)->ctx_flags, old, new); \ 550 } while (new != old); \ 551 } 552 553 #define CTX_CLEAR_FLAGS(ctx, flag) \ 554 { \ 555 uint32_t old, new; \ 556 \ 557 do { \ 558 new = old = (ctx)->ctx_flags; \ 559 new &= CTX_FLAGS_MASK & ~(flag); \ 560 new = cas32(&(ctx)->ctx_flags, old, new); \ 561 } while (new != old); \ 562 } 563 564 #define ctxtoctxnum(ctx) ((ushort_t)((ctx) - ctxs)) 565 566 /* 567 * Defines needed for ctx stealing. 568 */ 569 #define GET_CTX_RETRY_CNT 100 570 571 /* 572 * Starting with context 0, the first NUM_LOCKED_CTXS contexts 573 * are locked so that sfmmu_getctx can't steal any of these 574 * contexts. At the time this software was being developed, the 575 * only context that needs to be locked is context 0 (the kernel 576 * context), and context 1 (reserved for stolen context). So this constant 577 * was originally defined to be 2. 578 * 579 * For sun4v only, USER_CONTEXT_TYPE represents any user context. Many 580 * routines only care whether the context is kernel, invalid or user. 581 */ 582 583 #define NUM_LOCKED_CTXS 2 584 #define INVALID_CONTEXT 1 585 586 #ifdef sun4v 587 #define USER_CONTEXT_TYPE NUM_LOCKED_CTXS 588 #endif 589 590 #ifndef _ASM 591 592 /* 593 * Kernel page relocation stuff. 594 */ 595 struct sfmmu_callback { 596 int key; 597 int (*prehandler)(caddr_t, uint_t, uint_t, void *); 598 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t); 599 int (*errhandler)(caddr_t, uint_t, uint_t, void *); 600 int capture_cpus; 601 }; 602 603 extern int sfmmu_max_cb_id; 604 extern struct sfmmu_callback *sfmmu_cb_table; 605 606 extern int hat_kpr_enabled; 607 608 struct pa_hment; 609 610 /* 611 * RFE: With multihat gone we gain back an int. We could use this to 612 * keep ref bits on a per cpu basis to eliminate xcalls. 613 */ 614 struct sf_hment { 615 tte_t hme_tte; /* tte for this hment */ 616 617 union { 618 struct page *page; /* what page this maps */ 619 struct pa_hment *data; /* pa_hment */ 620 } sf_hment_un; 621 622 struct sf_hment *hme_next; /* next hment */ 623 struct sf_hment *hme_prev; /* prev hment */ 624 }; 625 626 struct pa_hment { 627 caddr_t addr; /* va */ 628 uint_t len; /* bytes */ 629 ushort_t flags; /* internal flags */ 630 ushort_t refcnt; /* reference count */ 631 id_t cb_id; /* callback id, table index */ 632 void *pvt; /* handler's private data */ 633 struct sf_hment sfment; /* corresponding dummy sf_hment */ 634 }; 635 636 #define hme_page sf_hment_un.page 637 #define hme_data sf_hment_un.data 638 #define hme_size(sfhmep) ((int)(TTE_CSZ(&(sfhmep)->hme_tte))) 639 #define PAHME_SZ (sizeof (struct pa_hment)) 640 #define SFHME_SZ (sizeof (struct sf_hment)) 641 642 #define IS_PAHME(hme) ((hme)->hme_tte.ll == 0) 643 644 /* 645 * hmeblk_tag structure 646 * structure used to obtain a match on a hme_blk. Currently consists of 647 * the address of the sfmmu struct (or hatid), the base page address of the 648 * hme_blk, and the rehash count. The rehash count is actually only 2 bits 649 * and has the following meaning: 650 * 1 = 8k or 64k hash sequence. 651 * 2 = 512k hash sequence. 652 * 3 = 4M hash sequence. 653 * We require this count because we don't want to get a false hit on a 512K or 654 * 4M rehash with a base address corresponding to a 8k or 64k hmeblk. 655 * Note: The ordering and size of the hmeblk_tag members are implictly known 656 * by the tsb miss handlers written in assembly. Do not change this structure 657 * without checking those routines. See HTAG_SFMMUPSZ define. 658 */ 659 660 typedef union { 661 struct { 662 uint64_t hblk_basepg: 51, /* hme_blk base pg # */ 663 hblk_rehash: 13; /* rehash number */ 664 sfmmu_t *sfmmup; 665 } hblk_tag_un; 666 uint64_t htag_tag[2]; 667 } hmeblk_tag; 668 669 #define htag_id hblk_tag_un.sfmmup 670 #define htag_bspage hblk_tag_un.hblk_basepg 671 #define htag_rehash hblk_tag_un.hblk_rehash 672 673 #define HTAGS_EQ(tag1, tag2) (((tag1.htag_tag[0] ^ tag2.htag_tag[0]) | \ 674 (tag1.htag_tag[1] ^ tag2.htag_tag[1])) == 0) 675 #define HME_REHASH(sfmmup) \ 676 ((sfmmup)->sfmmu_ttecnt[TTE512K] != 0 || \ 677 (sfmmup)->sfmmu_ttecnt[TTE4M] != 0 || \ 678 (sfmmup)->sfmmu_ttecnt[TTE32M] != 0 || \ 679 (sfmmup)->sfmmu_ttecnt[TTE256M] != 0) 680 681 #endif /* !_ASM */ 682 683 #define NHMENTS 8 /* # of hments in an 8k hme_blk */ 684 /* needs to be multiple of 2 */ 685 #ifndef _ASM 686 687 #ifdef HBLK_TRACE 688 689 #define HBLK_LOCK 1 690 #define HBLK_UNLOCK 0 691 #define HBLK_STACK_DEPTH 6 692 #define HBLK_AUDIT_CACHE_SIZE 16 693 #define HBLK_LOCK_PATTERN 0xaaaaaaaa 694 #define HBLK_UNLOCK_PATTERN 0xbbbbbbbb 695 696 struct hblk_lockcnt_audit { 697 int flag; /* lock or unlock */ 698 kthread_id_t thread; 699 int depth; 700 pc_t stack[HBLK_STACK_DEPTH]; 701 }; 702 703 #endif /* HBLK_TRACE */ 704 705 706 /* 707 * Hment block structure. 708 * The hme_blk is the node data structure which the hash structure 709 * mantains. An hme_blk can have 2 different sizes depending on the 710 * number of hments it implicitly contains. When dealing with 64K, 512K, 711 * or 4M hments there is one hment per hme_blk. When dealing with 712 * 8k hments we allocate an hme_blk plus an additional 7 hments to 713 * give us a total of 8 (NHMENTS) hments that can be referenced through a 714 * hme_blk. 715 * 716 * The hmeblk structure contains 2 tte reference counters used to determine if 717 * it is ok to free up the hmeblk. Both counters have to be zero in order 718 * to be able to free up hmeblk. They are protected by cas. 719 * hblk_hmecnt is the number of hments present on pp mapping lists. 720 * hblk_vcnt reflects number of valid ttes in hmeblk. 721 * 722 * The hmeblk now also has per tte lock cnts. This is required because 723 * the counts can be high and there are not enough bits in the tte. When 724 * physio is fixed to not lock the translations we should be able to move 725 * the lock cnt back to the tte. See bug id 1198554. 726 * 727 * Note that xhat_hme_blk's layout follows this structure: hme_blk_misc 728 * and sf_hment are at the same offsets in both structures. Whenever 729 * hme_blk is changed, xhat_hme_blk may need to be updated as well. 730 */ 731 732 struct hme_blk_misc { 733 ushort_t locked_cnt; /* HAT_LOAD_LOCK ref cnt */ 734 uint_t notused:10; 735 uint_t xhat_bit:1; /* set for an xhat hme_blk */ 736 uint_t shadow_bit:1; /* set for a shadow hme_blk */ 737 uint_t nucleus_bit:1; /* set for a nucleus hme_blk */ 738 uint_t ttesize:3; /* contains ttesz of hmeblk */ 739 }; 740 741 struct hme_blk { 742 uint64_t hblk_nextpa; /* physical address for hash list */ 743 744 hmeblk_tag hblk_tag; /* tag used to obtain an hmeblk match */ 745 746 struct hme_blk *hblk_next; /* on free list or on hash list */ 747 /* protected by hash lock */ 748 749 struct hme_blk *hblk_shadow; /* pts to shadow hblk */ 750 /* protected by hash lock */ 751 uint_t hblk_span; /* span of memory hmeblk maps */ 752 753 struct hme_blk_misc hblk_misc; 754 755 union { 756 struct { 757 ushort_t hblk_hmecount; /* hment on mlists counter */ 758 ushort_t hblk_validcnt; /* valid tte reference count */ 759 } hblk_counts; 760 uint_t hblk_shadow_mask; 761 } hblk_un; 762 763 #ifdef HBLK_TRACE 764 kmutex_t hblk_audit_lock; /* lock to protect index */ 765 uint_t hblk_audit_index; /* index into audit_cache */ 766 struct hblk_lockcnt_audit hblk_audit_cache[HBLK_AUDIT_CACHE_SIZE]; 767 #endif /* HBLK_AUDIT */ 768 769 struct sf_hment hblk_hme[1]; /* hment array */ 770 }; 771 772 #define hblk_lckcnt hblk_misc.locked_cnt 773 #define hblk_xhat_bit hblk_misc.xhat_bit 774 #define hblk_shw_bit hblk_misc.shadow_bit 775 #define hblk_nuc_bit hblk_misc.nucleus_bit 776 #define hblk_ttesz hblk_misc.ttesize 777 #define hblk_hmecnt hblk_un.hblk_counts.hblk_hmecount 778 #define hblk_vcnt hblk_un.hblk_counts.hblk_validcnt 779 #define hblk_shw_mask hblk_un.hblk_shadow_mask 780 781 #define MAX_HBLK_LCKCNT 0xFFFF 782 #define HMEBLK_ALIGN 0x8 /* hmeblk has to be double aligned */ 783 784 #ifdef HBLK_TRACE 785 786 #define HBLK_STACK_TRACE(hmeblkp, lock) \ 787 { \ 788 int flag = lock; /* to pacify lint */ \ 789 int audit_index; \ 790 \ 791 mutex_enter(&hmeblkp->hblk_audit_lock); \ 792 audit_index = hmeblkp->hblk_audit_index; \ 793 hmeblkp->hblk_audit_index = ((hmeblkp->hblk_audit_index + 1) & \ 794 (HBLK_AUDIT_CACHE_SIZE - 1)); \ 795 mutex_exit(&hmeblkp->hblk_audit_lock); \ 796 \ 797 if (flag) \ 798 hmeblkp->hblk_audit_cache[audit_index].flag = \ 799 HBLK_LOCK_PATTERN; \ 800 else \ 801 hmeblkp->hblk_audit_cache[audit_index].flag = \ 802 HBLK_UNLOCK_PATTERN; \ 803 \ 804 hmeblkp->hblk_audit_cache[audit_index].thread = curthread; \ 805 hmeblkp->hblk_audit_cache[audit_index].depth = \ 806 getpcstack(hmeblkp->hblk_audit_cache[audit_index].stack, \ 807 HBLK_STACK_DEPTH); \ 808 } 809 810 #else 811 812 #define HBLK_STACK_TRACE(hmeblkp, lock) 813 814 #endif /* HBLK_TRACE */ 815 816 #define HMEHASH_FACTOR 16 /* used to calc # of buckets in hme hash */ 817 818 /* 819 * A maximum number of user hmeblks is defined in order to place an upper 820 * limit on how much nucleus memory is required and to avoid overflowing the 821 * tsbmiss uhashsz and khashsz data areas. The number below corresponds to 822 * the number of buckets required, for an average hash chain length of 4 on 823 * a 16TB machine. 824 */ 825 826 #define MAX_UHME_BUCKETS (0x1 << 30) 827 #define MAX_KHME_BUCKETS (0x1 << 30) 828 829 /* 830 * The minimum number of kernel hash buckets. 831 */ 832 #define MIN_KHME_BUCKETS 0x800 833 834 /* 835 * The number of hash buckets must be a power of 2. If the initial calculated 836 * value is less than USER_BUCKETS_THRESHOLD we round up to the next greater 837 * power of 2, otherwise we round down to avoid huge over allocations. 838 */ 839 #define USER_BUCKETS_THRESHOLD (1<<22) 840 841 #define MAX_NUCUHME_BUCKETS 0x4000 842 #define MAX_NUCKHME_BUCKETS 0x2000 843 844 /* 845 * There are 2 locks in the hmehash bucket. The hmehash_mutex is 846 * a regular mutex used to make sure operations on a hash link are only 847 * done by one thread. Any operation which comes into the hat with 848 * a <vaddr, as> will grab the hmehash_mutex. Normally one would expect 849 * the tsb miss handlers to grab the hash lock to make sure the hash list 850 * is consistent while we traverse it. Unfortunately this can lead to 851 * deadlocks or recursive mutex enters since it is possible for 852 * someone holding the lock to take a tlb/tsb miss. 853 * To solve this problem we have added the hmehash_listlock. This lock 854 * is only grabbed by the tsb miss handlers, vatopfn, and while 855 * adding/removing a hmeblk from the hash list. The code is written to 856 * guarantee we won't take a tlb miss while holding this lock. 857 */ 858 struct hmehash_bucket { 859 kmutex_t hmehash_mutex; 860 uint64_t hmeh_nextpa; /* physical address for hash list */ 861 struct hme_blk *hmeblkp; 862 uint_t hmeh_listlock; 863 }; 864 865 #endif /* !_ASM */ 866 867 /* Proc Count Project */ 868 #define SFMMU_PGCNT_MASK 0x3f 869 #define SFMMU_PGCNT_SHIFT 6 870 #define INVALID_MMU_ID -1 871 #define SFMMU_MMU_GNUM_RSHIFT 16 872 #define SFMMU_MMU_CNUM_LSHIFT (64 - SFMMU_MMU_GNUM_RSHIFT) 873 #define MAX_SFMMU_CTX_VAL ((1 << 16) - 1) /* for sanity check */ 874 #define MAX_SFMMU_GNUM_VAL ((0x1UL << 48) - 1) 875 876 /* 877 * The tsb miss handlers written in assembly know that sfmmup 878 * is a 64 bit ptr. 879 * 880 * The bspage and re-hash part is 64 bits, with the sfmmup being another 64 881 * bits. 882 */ 883 #define HTAG_SFMMUPSZ 0 /* Not really used for LP64 */ 884 #define HTAG_REHASHSZ 13 885 886 /* 887 * Assembly routines need to be able to get to ttesz 888 */ 889 #define HBLK_SZMASK 0x7 890 891 #ifndef _ASM 892 893 /* 894 * Returns the number of bytes that an hmeblk spans given its tte size 895 */ 896 #define get_hblk_span(hmeblkp) ((hmeblkp)->hblk_span) 897 #define get_hblk_ttesz(hmeblkp) ((hmeblkp)->hblk_ttesz) 898 #define get_hblk_cache(hmeblkp) (((hmeblkp)->hblk_ttesz == TTE8K) ? \ 899 sfmmu8_cache : sfmmu1_cache) 900 #define HMEBLK_SPAN(ttesz) \ 901 ((ttesz == TTE8K)? (TTEBYTES(ttesz) * NHMENTS) : TTEBYTES(ttesz)) 902 903 #define set_hblk_sz(hmeblkp, ttesz) \ 904 (hmeblkp)->hblk_ttesz = (ttesz); \ 905 (hmeblkp)->hblk_span = HMEBLK_SPAN(ttesz) 906 907 #define get_hblk_base(hmeblkp) \ 908 ((uintptr_t)(hmeblkp)->hblk_tag.htag_bspage << MMU_PAGESHIFT) 909 910 #define get_hblk_endaddr(hmeblkp) \ 911 ((caddr_t)(get_hblk_base(hmeblkp) + get_hblk_span(hmeblkp))) 912 913 #define in_hblk_range(hmeblkp, vaddr) \ 914 (((uintptr_t)(vaddr) >= get_hblk_base(hmeblkp)) && \ 915 ((uintptr_t)(vaddr) < (get_hblk_base(hmeblkp) + \ 916 get_hblk_span(hmeblkp)))) 917 918 #define tte_to_vaddr(hmeblkp, tte) ((caddr_t)(get_hblk_base(hmeblkp) \ 919 + (TTEBYTES(TTE_CSZ(&tte)) * (tte).tte_hmenum))) 920 921 #define vaddr_to_vshift(hblktag, vaddr, shwsz) \ 922 ((((uintptr_t)(vaddr) >> MMU_PAGESHIFT) - (hblktag.htag_bspage)) >>\ 923 TTE_BSZS_SHIFT((shwsz) - 1)) 924 925 #define HME8BLK_SZ (sizeof (struct hme_blk) + \ 926 (NHMENTS - 1) * sizeof (struct sf_hment)) 927 #define HME1BLK_SZ (sizeof (struct hme_blk)) 928 #define H1MIN (2 + MAX_BIGKTSB_TTES) /* nucleus text+data, ktsb */ 929 930 /* 931 * Hme_blk hash structure 932 * Active mappings are kept in a hash structure of hme_blks. The hash 933 * function is based on (ctx, vaddr) The size of the hash table size is a 934 * power of 2 such that the average hash chain lenth is HMENT_HASHAVELEN. 935 * The hash actually consists of 2 separate hashes. One hash is for the user 936 * address space and the other hash is for the kernel address space. 937 * The number of buckets are calculated at boot time and stored in the global 938 * variables "uhmehash_num" and "khmehash_num". By making the hash table size 939 * a power of 2 we can use a simply & function to derive an index instead of 940 * a divide. 941 * 942 * HME_HASH_FUNCTION(hatid, vaddr, shift) returns a pointer to a hme_hash 943 * bucket. 944 * An hme hash bucket contains a pointer to an hme_blk and the mutex that 945 * protects the link list. 946 * Spitfire supports 4 page sizes. 8k and 64K pages only need one hash. 947 * 512K pages need 2 hashes and 4M pages need 3 hashes. 948 * The 'shift' parameter controls how many bits the vaddr will be shifted in 949 * the hash function. It is calculated in the HME_HASH_SHIFT(ttesz) function 950 * and it varies depending on the page size as follows: 951 * 8k pages: HBLK_RANGE_SHIFT 952 * 64k pages: MMU_PAGESHIFT64K 953 * 512K pages: MMU_PAGESHIFT512K 954 * 4M pages: MMU_PAGESHIFT4M 955 * An assembly version of the hash function exists in sfmmu_ktsb_miss(). All 956 * changes should be reflected in both versions. This function and the TSB 957 * miss handlers are the only places which know about the two hashes. 958 * 959 * HBLK_RANGE_SHIFT controls range of virtual addresses that will fall 960 * into the same bucket for a particular process. It is currently set to 961 * be equivalent to 64K range or one hme_blk. 962 * 963 * The hme_blks in the hash are protected by a per hash bucket mutex 964 * known as SFMMU_HASH_LOCK. 965 * You need to acquire this lock before traversing the hash bucket link 966 * list, while adding/removing a hme_blk to the list, and while 967 * modifying an hme_blk. A possible optimization is to replace these 968 * mutexes by readers/writer lock but right now it is not clear whether 969 * this is a win or not. 970 * 971 * The HME_HASH_TABLE_SEARCH will search the hash table for the 972 * hme_blk that contains the hment that corresponds to the passed 973 * ctx and vaddr. It assumed the SFMMU_HASH_LOCK is held. 974 */ 975 976 #endif /* ! _ASM */ 977 978 #define KHATID ksfmmup 979 #define UHMEHASH_SZ uhmehash_num 980 #define KHMEHASH_SZ khmehash_num 981 #define HMENT_HASHAVELEN 4 982 #define HBLK_RANGE_SHIFT MMU_PAGESHIFT64K /* shift for HBLK_BS_MASK */ 983 #define MAX_HASHCNT 5 984 #define DEFAULT_MAX_HASHCNT 3 985 986 #ifndef _ASM 987 988 #define HASHADDR_MASK(hashno) TTE_PAGEMASK(hashno) 989 990 #define HME_HASH_SHIFT(ttesz) \ 991 ((ttesz == TTE8K)? HBLK_RANGE_SHIFT : TTE_PAGE_SHIFT(ttesz)) 992 993 #define HME_HASH_ADDR(vaddr, hmeshift) \ 994 ((caddr_t)(((uintptr_t)(vaddr) >> (hmeshift)) << (hmeshift))) 995 996 #define HME_HASH_BSPAGE(vaddr, hmeshift) \ 997 (((uintptr_t)(vaddr) >> (hmeshift)) << ((hmeshift) - MMU_PAGESHIFT)) 998 999 #define HME_HASH_REHASH(ttesz) \ 1000 (((ttesz) < TTE512K)? 1 : (ttesz)) 1001 1002 #define HME_HASH_FUNCTION(hatid, vaddr, shift) \ 1003 ((hatid != KHATID)? \ 1004 (&uhme_hash[ (((uintptr_t)(hatid) ^ \ 1005 ((uintptr_t)vaddr >> (shift))) & UHMEHASH_SZ) ]): \ 1006 (&khme_hash[ (((uintptr_t)(hatid) ^ \ 1007 ((uintptr_t)vaddr >> (shift))) & KHMEHASH_SZ) ])) 1008 1009 /* 1010 * This macro will traverse a hmeblk hash link list looking for an hme_blk 1011 * that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp 1012 * will be set to NULL, otherwise it will point to the correct hme_blk. 1013 * This macro also cleans empty hblks. 1014 */ 1015 #define HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, \ 1016 pr_hblk, prevpa, listp) \ 1017 { \ 1018 struct hme_blk *nx_hblk; \ 1019 uint64_t nx_pa; \ 1020 \ 1021 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \ 1022 hblkp = hmebp->hmeblkp; \ 1023 hblkpa = hmebp->hmeh_nextpa; \ 1024 prevpa = 0; \ 1025 pr_hblk = NULL; \ 1026 while (hblkp) { \ 1027 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \ 1028 /* found hme_blk */ \ 1029 break; \ 1030 } \ 1031 nx_hblk = hblkp->hblk_next; \ 1032 nx_pa = hblkp->hblk_nextpa; \ 1033 if (!hblkp->hblk_vcnt && !hblkp->hblk_hmecnt) { \ 1034 sfmmu_hblk_hash_rm(hmebp, hblkp, prevpa, pr_hblk); \ 1035 sfmmu_hblk_free(hmebp, hblkp, hblkpa, listp); \ 1036 } else { \ 1037 pr_hblk = hblkp; \ 1038 prevpa = hblkpa; \ 1039 } \ 1040 hblkp = nx_hblk; \ 1041 hblkpa = nx_pa; \ 1042 } \ 1043 } 1044 1045 #define HME_HASH_SEARCH(hmebp, hblktag, hblkp, listp) \ 1046 { \ 1047 struct hme_blk *pr_hblk; \ 1048 uint64_t hblkpa, prevpa; \ 1049 \ 1050 HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, pr_hblk, \ 1051 prevpa, listp); \ 1052 } 1053 1054 /* 1055 * This macro will traverse a hmeblk hash link list looking for an hme_blk 1056 * that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp 1057 * will be set to NULL, otherwise it will point to the correct hme_blk. 1058 * It doesn't remove empty hblks. 1059 */ 1060 #define HME_HASH_FAST_SEARCH(hmebp, hblktag, hblkp) \ 1061 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \ 1062 for (hblkp = hmebp->hmeblkp; hblkp; \ 1063 hblkp = hblkp->hblk_next) { \ 1064 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \ 1065 /* found hme_blk */ \ 1066 break; \ 1067 } \ 1068 } 1069 1070 1071 #define SFMMU_HASH_LOCK(hmebp) \ 1072 (mutex_enter(&hmebp->hmehash_mutex)) 1073 1074 #define SFMMU_HASH_UNLOCK(hmebp) \ 1075 (mutex_exit(&hmebp->hmehash_mutex)) 1076 1077 #define SFMMU_HASH_LOCK_TRYENTER(hmebp) \ 1078 (mutex_tryenter(&hmebp->hmehash_mutex)) 1079 1080 #define SFMMU_HASH_LOCK_ISHELD(hmebp) \ 1081 (mutex_owned(&hmebp->hmehash_mutex)) 1082 1083 #define SFMMU_XCALL_STATS(sfmmup) \ 1084 { \ 1085 if (sfmmup == ksfmmup) { \ 1086 SFMMU_STAT(sf_kernel_xcalls); \ 1087 } else { \ 1088 SFMMU_STAT(sf_user_xcalls); \ 1089 } \ 1090 } 1091 1092 #define astosfmmu(as) ((as)->a_hat) 1093 #define hblktosfmmu(hmeblkp) ((sfmmu_t *)(hmeblkp)->hblk_tag.htag_id) 1094 #define sfmmutoas(sfmmup) ((sfmmup)->sfmmu_as) 1095 /* 1096 * We use the sfmmu data structure to keep the per as page coloring info. 1097 */ 1098 #define as_color_bin(as) (astosfmmu(as)->sfmmu_clrbin) 1099 #define as_color_start(as) (astosfmmu(as)->sfmmu_clrstart) 1100 1101 typedef struct { 1102 char h8[HME8BLK_SZ]; 1103 } hblk8_t; 1104 1105 typedef struct { 1106 char h1[HME1BLK_SZ]; 1107 } hblk1_t; 1108 1109 typedef struct { 1110 ulong_t index; 1111 ulong_t len; 1112 hblk8_t *list; 1113 } nucleus_hblk8_info_t; 1114 1115 typedef struct { 1116 ulong_t index; 1117 ulong_t len; 1118 hblk1_t *list; 1119 } nucleus_hblk1_info_t; 1120 1121 /* 1122 * This struct is used for accumlating information about a range 1123 * of pages that are unloading so that a single xcall can flush 1124 * the entire range from remote tlbs. A function that must demap 1125 * a range of virtual addresses declares one of these structures 1126 * and initializes using DEMP_RANGE_INIT(). It then passes a pointer to this 1127 * struct to the appropriate sfmmu_hblk_* level function which does 1128 * all the bookkeeping using the other macros. When the function has 1129 * finished the virtual address range, it needs to call DEMAP_RANGE_FLUSH() 1130 * macro to take care of any remaining unflushed mappings. 1131 * 1132 * The maximum range this struct can represent is the number of bits 1133 * in the dmr_bitvec field times the pagesize in dmr_pgsz. Currently, only 1134 * MMU_PAGESIZE pages are supported. 1135 * 1136 * Since there are now cases where it's no longer necessary to do 1137 * flushes (e.g. when the process isn't runnable because it's swapping 1138 * out or exiting) we allow these macros to take a NULL dmr input and do 1139 * nothing in that case. 1140 */ 1141 typedef struct { 1142 sfmmu_t *dmr_sfmmup; /* relevant hat */ 1143 caddr_t dmr_addr; /* beginning address */ 1144 caddr_t dmr_endaddr; /* ending address */ 1145 ulong_t dmr_bitvec; /* valid pages found */ 1146 ulong_t dmr_bit; /* next page to examine */ 1147 ulong_t dmr_maxbit; /* highest page in range */ 1148 ulong_t dmr_pgsz; /* page size in range */ 1149 } demap_range_t; 1150 1151 #define DMR_MAXBIT ((ulong_t)1<<63) /* dmr_bit high bit */ 1152 1153 #define DEMAP_RANGE_INIT(sfmmup, dmrp) \ 1154 if ((dmrp) != NULL) { \ 1155 (dmrp)->dmr_sfmmup = (sfmmup); \ 1156 (dmrp)->dmr_bitvec = 0; \ 1157 (dmrp)->dmr_maxbit = sfmmu_dmr_maxbit; \ 1158 (dmrp)->dmr_pgsz = MMU_PAGESIZE; \ 1159 } 1160 1161 #define DEMAP_RANGE_PGSZ(dmrp) ((dmrp)? (dmrp)->dmr_pgsz : MMU_PAGESIZE) 1162 1163 #define DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr) \ 1164 if ((dmrp) != NULL) { \ 1165 if ((dmrp)->dmr_bitvec != 0 && (dmrp)->dmr_endaddr != (addr)) \ 1166 sfmmu_tlb_range_demap(dmrp); \ 1167 (dmrp)->dmr_endaddr = (endaddr); \ 1168 } 1169 1170 #define DEMAP_RANGE_FLUSH(dmrp) \ 1171 if ((dmrp) != NULL) { \ 1172 if ((dmrp)->dmr_bitvec != 0) \ 1173 sfmmu_tlb_range_demap(dmrp); \ 1174 } 1175 1176 #define DEMAP_RANGE_MARKPG(dmrp, addr) \ 1177 if ((dmrp) != NULL) { \ 1178 if ((dmrp)->dmr_bitvec == 0) { \ 1179 (dmrp)->dmr_addr = (addr); \ 1180 (dmrp)->dmr_bit = 1; \ 1181 } \ 1182 (dmrp)->dmr_bitvec |= (dmrp)->dmr_bit; \ 1183 } 1184 1185 #define DEMAP_RANGE_NEXTPG(dmrp) \ 1186 if ((dmrp) != NULL && (dmrp)->dmr_bitvec != 0) { \ 1187 if ((dmrp)->dmr_bit & (dmrp)->dmr_maxbit) { \ 1188 sfmmu_tlb_range_demap(dmrp); \ 1189 } else { \ 1190 (dmrp)->dmr_bit <<= 1; \ 1191 } \ 1192 } 1193 1194 /* 1195 * TSB related structures 1196 * 1197 * The TSB is made up of tte entries. Both the tag and data are present 1198 * in the TSB. The TSB locking is managed as follows: 1199 * A software bit in the tsb tag is used to indicate that entry is locked. 1200 * If a cpu servicing a tsb miss reads a locked entry the tag compare will 1201 * fail forcing the cpu to go to the hat hash for the translation. 1202 * The cpu who holds the lock can then modify the data side, and the tag side. 1203 * The last write should be to the word containing the lock bit which will 1204 * clear the lock and allow the tsb entry to be read. It is assumed that all 1205 * cpus reading the tsb will do so with atomic 128-bit loads. An atomic 128 1206 * bit load is required to prevent the following from happening: 1207 * 1208 * cpu 0 cpu 1 comments 1209 * 1210 * ldx tag tag unlocked 1211 * ldstub lock set lock 1212 * stx data 1213 * stx tag unlock 1214 * ldx tag incorrect tte!!! 1215 * 1216 * The software also maintains a bit in the tag to indicate an invalid 1217 * tsb entry. The purpose of this bit is to allow the tsb invalidate code 1218 * to invalidate a tsb entry with a single cas. See code for details. 1219 */ 1220 1221 union tsb_tag { 1222 struct { 1223 uint32_t tag_res0:16; /* reserved - context area */ 1224 uint32_t tag_inv:1; /* sw - invalid tsb entry */ 1225 uint32_t tag_lock:1; /* sw - locked tsb entry */ 1226 uint32_t tag_res1:4; /* reserved */ 1227 uint32_t tag_va_hi:10; /* va[63:54] */ 1228 uint32_t tag_va_lo; /* va[53:22] */ 1229 } tagbits; 1230 struct tsb_tagints { 1231 uint32_t inthi; 1232 uint32_t intlo; 1233 } tagints; 1234 }; 1235 #define tag_invalid tagbits.tag_inv 1236 #define tag_locked tagbits.tag_lock 1237 #define tag_vahi tagbits.tag_va_hi 1238 #define tag_valo tagbits.tag_va_lo 1239 #define tag_inthi tagints.inthi 1240 #define tag_intlo tagints.intlo 1241 1242 struct tsbe { 1243 union tsb_tag tte_tag; 1244 tte_t tte_data; 1245 }; 1246 1247 /* 1248 * A per cpu struct is kept that duplicates some info 1249 * used by the tl>0 tsb miss handlers plus it provides 1250 * a scratch area. Its purpose is to minimize cache misses 1251 * in the tsb miss handler and is 128 bytes (2 e$ lines). 1252 * 1253 * There should be one allocated per cpu in nucleus memory 1254 * and should be aligned on an ecache line boundary. 1255 */ 1256 struct tsbmiss { 1257 sfmmu_t *ksfmmup; /* kernel hat id */ 1258 sfmmu_t *usfmmup; /* user hat id */ 1259 struct tsbe *tsbptr; /* hardware computed ptr */ 1260 struct tsbe *tsbptr4m; /* hardware computed ptr */ 1261 uint64_t ismblkpa; 1262 struct hmehash_bucket *khashstart; 1263 struct hmehash_bucket *uhashstart; 1264 uint_t khashsz; 1265 uint_t uhashsz; 1266 uint16_t dcache_line_mask; /* used to flush dcache */ 1267 uint16_t hat_flags; 1268 uint32_t itlb_misses; 1269 uint32_t dtlb_misses; 1270 uint32_t utsb_misses; 1271 uint32_t ktsb_misses; 1272 uint16_t uprot_traps; 1273 uint16_t kprot_traps; 1274 1275 /* 1276 * scratch[0] -> TSB_TAGACC 1277 * scratch[1] -> TSBMISS_HMEBP 1278 * scratch[2] -> TSBMISS_HATID 1279 */ 1280 uintptr_t scratch[3]; 1281 uint8_t pad[0x10]; 1282 }; 1283 1284 /* 1285 * A per cpu struct is kept for the use within the tl>0 kpm tsb 1286 * miss handler. Some members are duplicates of common data or 1287 * the physical addresses of common data. A few members are also 1288 * written by the tl>0 kpm tsb miss handler. Its purpose is to 1289 * minimize cache misses in the kpm tsb miss handler and occupies 1290 * one ecache line. There should be one allocated per cpu in 1291 * nucleus memory and it should be aligned on an ecache line 1292 * boundary. It is not merged w/ struct tsbmiss since there is 1293 * not much to share and the tsbmiss pathes are different, so 1294 * a kpm tlbmiss/tsbmiss only touches one cacheline, except for 1295 * (DEBUG || SFMMU_STAT_GATHER) where the dtlb_misses counter 1296 * of struct tsbmiss is used on every dtlb miss. 1297 */ 1298 struct kpmtsbm { 1299 caddr_t vbase; /* start of address kpm range */ 1300 caddr_t vend; /* end of address kpm range */ 1301 uchar_t flags; /* flags needed in TL tsbmiss handler */ 1302 uchar_t sz_shift; /* for single kpm window */ 1303 uchar_t kpmp_shift; /* hash lock shift */ 1304 uchar_t kpmp2pshft; /* kpm page to page shift */ 1305 uint_t kpmp_table_sz; /* size of kpmp_table or kpmp_stable */ 1306 uint64_t kpmp_tablepa; /* paddr of kpmp_table or kpmp_stable */ 1307 uint64_t msegphashpa; /* paddr of memseg_phash */ 1308 struct tsbe *tsbptr; /* saved ktsb pointer */ 1309 uint_t kpm_dtlb_misses; /* kpm tlbmiss counter */ 1310 uint_t kpm_tsb_misses; /* kpm tsbmiss counter */ 1311 uintptr_t pad[1]; 1312 }; 1313 1314 extern uint_t tsb_slab_size; 1315 extern uint_t tsb_slab_shift; 1316 extern uint_t tsb_slab_ttesz; 1317 extern uint_t tsb_slab_pamask; 1318 1319 #endif /* !_ASM */ 1320 1321 /* 1322 * Flags for TL kpm tsbmiss handler 1323 */ 1324 #define KPMTSBM_ENABLE_FLAG 0x01 /* bit copy of kpm_enable */ 1325 #define KPMTSBM_TLTSBM_FLAG 0x02 /* use TL tsbmiss handler */ 1326 #define KPMTSBM_TSBPHYS_FLAG 0x04 /* use ASI_MEM for TSB update */ 1327 1328 /* 1329 * The TSB 1330 * All TSB sizes supported by the hardware are now supported (8K - 1M). 1331 * For kernel TSBs we may go beyond the hardware supported sizes and support 1332 * larger TSBs via software. 1333 * All TTE sizes are supported in the TSB; the manner in which this is 1334 * done is cpu dependent. 1335 */ 1336 #define TSB_MIN_SZCODE TSB_8K_SZCODE /* min. supported TSB size */ 1337 #define TSB_MIN_OFFSET_MASK (TSB_OFFSET_MASK(TSB_MIN_SZCODE)) 1338 1339 #define UTSB_MAX_SZCODE TSB_1M_SZCODE /* max. supported TSB size */ 1340 #define UTSB_MAX_OFFSET_MASK (TSB_OFFSET_MASK(UTSB_MAX_SZCODE)) 1341 1342 #define TSB_FREEMEM_MIN 0x1000 /* 32 mb */ 1343 #define TSB_FREEMEM_LARGE 0x10000 /* 512 mb */ 1344 #define TSB_8K_SZCODE 0 /* 512 entries */ 1345 #define TSB_16K_SZCODE 1 /* 1k entries */ 1346 #define TSB_32K_SZCODE 2 /* 2k entries */ 1347 #define TSB_64K_SZCODE 3 /* 4k entries */ 1348 #define TSB_128K_SZCODE 4 /* 8k entries */ 1349 #define TSB_256K_SZCODE 5 /* 16k entries */ 1350 #define TSB_512K_SZCODE 6 /* 32k entries */ 1351 #define TSB_1M_SZCODE 7 /* 64k entries */ 1352 #define TSB_2M_SZCODE 8 /* 128k entries */ 1353 #define TSB_4M_SZCODE 9 /* 256k entries */ 1354 #define TSB_ENTRY_SHIFT 4 /* each entry = 128 bits = 16 bytes */ 1355 #define TSB_ENTRY_SIZE (1 << 4) 1356 #define TSB_START_SIZE 9 1357 #define TSB_ENTRIES(tsbsz) (1 << (TSB_START_SIZE + tsbsz)) 1358 #define TSB_BYTES(tsbsz) (TSB_ENTRIES(tsbsz) << TSB_ENTRY_SHIFT) 1359 #define TSB_OFFSET_MASK(tsbsz) (TSB_ENTRIES(tsbsz) - 1) 1360 #define TSB_BASEADDR_MASK ((1 << 12) - 1) 1361 1362 /* 1363 * sun4u platforms 1364 * --------------- 1365 * We now support two user TSBs with one TSB base register. 1366 * Hence the TSB base register is split up as follows: 1367 * 1368 * When only one TSB present: 1369 * [63 62..42 41..13 12..4 3..0] 1370 * ^ ^ ^ ^ ^ 1371 * | | | | | 1372 * | | | | |_ TSB size code 1373 * | | | | 1374 * | | | |_ Reserved 0 1375 * | | | 1376 * | | |_ TSB VA[41..13] 1377 * | | 1378 * | |_ VA hole (Spitfire), zeros (Cheetah and beyond) 1379 * | 1380 * |_ 0 1381 * 1382 * When second TSB present: 1383 * [63 62..42 41..33 32..29 28..22 21..13 12..4 3..0] 1384 * ^ ^ ^ ^ ^ ^ ^ ^ 1385 * | | | | | | | | 1386 * | | | | | | | |_ First TSB size code 1387 * | | | | | | | 1388 * | | | | | | |_ Reserved 0 1389 * | | | | | | 1390 * | | | | | |_ First TSB's VA[21..13] 1391 * | | | | | 1392 * | | | | |_ Reserved for future use 1393 * | | | | 1394 * | | | |_ Second TSB's size code 1395 * | | | 1396 * | | |_ Second TSB's VA[21..13] 1397 * | | 1398 * | |_ VA hole (Spitfire) / ones (Cheetah and beyond) 1399 * | 1400 * |_ 1 1401 * 1402 * Note that since we store 21..13 of each TSB's VA, TSBs and their slabs 1403 * may be up to 4M in size. For now, only hardware supported TSB sizes 1404 * are supported, though the slabs are usually 4M in size. 1405 * 1406 * sun4u platforms that define UTSB_PHYS use physical addressing to access 1407 * the user TSBs at TL>0. The first user TSB base is in the MMU I/D TSB Base 1408 * registers. The second TSB base uses a dedicated scratchpad register which 1409 * requires a definition of SCRATCHPAD_UTSBREG in mach_sfmmu.h. The layout for 1410 * both registers is equivalent to sun4v below, except the TSB PA range is 1411 * [46..13] for sun4u. 1412 * 1413 * sun4v platforms 1414 * --------------- 1415 * On sun4v platforms, we use two dedicated scratchpad registers as pseudo 1416 * hardware TSB base registers to hold up to two different user TSBs. 1417 * 1418 * Each register contains TSB's physical base and size code information 1419 * as follows: 1420 * 1421 * [63..56 55..13 12..4 3..0] 1422 * ^ ^ ^ ^ 1423 * | | | | 1424 * | | | |_ TSB size code 1425 * | | | 1426 * | | |_ Reserved 0 1427 * | | 1428 * | |_ TSB PA[55..13] 1429 * | 1430 * | 1431 * | 1432 * |_ 0 for valid TSB 1433 * 1434 * Absence of a user TSB (primarily the second user TSB) is indicated by 1435 * storing a negative value in the TSB base register. This allows us to 1436 * check for presence of a user TSB by simply checking bit# 63. 1437 */ 1438 #define TSBREG_MSB_SHIFT 32 /* set upper bits */ 1439 #define TSBREG_MSB_CONST 0xfffff800 /* set bits 63..43 */ 1440 #define TSBREG_FIRTSB_SHIFT 42 /* to clear bits 63:22 */ 1441 #define TSBREG_SECTSB_MKSHIFT 20 /* 21:13 --> 41:33 */ 1442 #define TSBREG_SECTSB_LSHIFT 22 /* to clear bits 63:42 */ 1443 #define TSBREG_SECTSB_RSHIFT (TSBREG_SECTSB_MKSHIFT + TSBREG_SECTSB_LSHIFT) 1444 /* sectsb va -> bits 21:13 */ 1445 /* after clearing upper bits */ 1446 #define TSBREG_SECSZ_SHIFT 29 /* to get sectsb szc to 3:0 */ 1447 #define TSBREG_VAMASK_SHIFT 13 /* set up VA mask */ 1448 1449 #define BIGKTSB_SZ_MASK 0xf 1450 #define TSB_SOFTSZ_MASK BIGKTSB_SZ_MASK 1451 #define MIN_BIGKTSB_SZCODE 9 /* 256k entries */ 1452 #define MAX_BIGKTSB_SZCODE 11 /* 1024k entries */ 1453 #define MAX_BIGKTSB_TTES (TSB_BYTES(MAX_BIGKTSB_SZCODE) / MMU_PAGESIZE4M) 1454 1455 #define TAG_VALO_SHIFT 22 /* tag's va are bits 63-22 */ 1456 /* 1457 * sw bits used on tsb_tag - bit masks used only in assembly 1458 * use only a sethi for these fields. 1459 */ 1460 #define TSBTAG_INVALID 0x00008000 /* tsb_tag.tag_invalid */ 1461 #define TSBTAG_LOCKED 0x00004000 /* tsb_tag.tag_locked */ 1462 1463 #ifdef _ASM 1464 1465 /* 1466 * Marker to indicate that this instruction will be hot patched at runtime 1467 * to some other value. 1468 * This value must be zero since it fills in the imm bits of the target 1469 * instructions to be patched 1470 */ 1471 #define RUNTIME_PATCH (0) 1472 1473 /* 1474 * V9 defines nop instruction as the following, which we use 1475 * at runtime to nullify some instructions we don't want to 1476 * execute in the trap handlers on certain platforms. 1477 */ 1478 #define MAKE_NOP_INSTR(reg) \ 1479 sethi %hi(0x1000000), reg 1480 1481 /* 1482 * Macro to get hat per-MMU cnum on this CPU. 1483 * sfmmu - In, pass in "sfmmup" from the caller. 1484 * cnum - Out, return 'cnum' to the caller 1485 * scr - scratch 1486 */ 1487 #define SFMMU_CPU_CNUM(sfmmu, cnum, scr) \ 1488 CPU_ADDR(scr, cnum); /* scr = load CPU struct addr */ \ 1489 ld [scr + CPU_MMU_IDX], cnum; /* cnum = mmuid */ \ 1490 add sfmmu, SFMMU_CTXS, scr; /* scr = sfmmup->sfmmu_ctxs[] */ \ 1491 sllx cnum, SFMMU_MMU_CTX_SHIFT, cnum; \ 1492 add scr, cnum, scr; /* scr = sfmmup->sfmmu_ctxs[id] */ \ 1493 ldx [scr + SFMMU_MMU_GC_NUM], scr; /* sfmmu_ctxs[id].gcnum */ \ 1494 sllx scr, SFMMU_MMU_CNUM_LSHIFT, scr; \ 1495 srlx scr, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */ 1496 1497 /* 1498 * Macro to get hat gnum & cnum assocaited with sfmmu_ctx[mmuid] entry 1499 * entry - In, pass in (&sfmmu_ctxs[mmuid] - SFMMU_CTXS) from the caller. 1500 * gnum - Out, return sfmmu gnum 1501 * cnum - Out, return sfmmu cnum 1502 * reg - scratch 1503 */ 1504 #define SFMMU_MMUID_GNUM_CNUM(entry, gnum, cnum, reg) \ 1505 ldx [entry + SFMMU_CTXS], reg; /* reg = sfmmu (gnum | cnum) */ \ 1506 srlx reg, SFMMU_MMU_GNUM_RSHIFT, gnum; /* gnum = sfmmu gnum */ \ 1507 sllx reg, SFMMU_MMU_CNUM_LSHIFT, cnum; \ 1508 srlx cnum, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */ 1509 1510 /* 1511 * Macro to get this CPU's tsbmiss area. 1512 */ 1513 #define CPU_TSBMISS_AREA(tsbmiss, tmp1) \ 1514 CPU_INDEX(tmp1, tsbmiss); /* tmp1 = cpu idx */ \ 1515 sethi %hi(tsbmiss_area), tsbmiss; /* tsbmiss base ptr */ \ 1516 sllx tmp1, TSBMISS_SHIFT, tmp1; /* byte offset */ \ 1517 or tsbmiss, %lo(tsbmiss_area), tsbmiss; \ 1518 add tsbmiss, tmp1, tsbmiss /* tsbmiss area of CPU */ 1519 1520 1521 /* 1522 * Macro to set kernel context + page size codes in DMMU primary context 1523 * register. It is only necessary for sun4u because sun4v does not need 1524 * page size codes 1525 */ 1526 #ifdef sun4v 1527 1528 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) 1529 1530 #else 1531 1532 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) \ 1533 sethi %hi(kcontextreg), reg0; \ 1534 ldx [reg0 + %lo(kcontextreg)], reg0; \ 1535 mov MMU_PCONTEXT, reg1; \ 1536 ldxa [reg1]ASI_MMU_CTX, reg2; \ 1537 xor reg0, reg2, reg2; \ 1538 brz reg2, label3; \ 1539 srlx reg2, CTXREG_NEXT_SHIFT, reg2; \ 1540 rdpr %pstate, reg3; /* disable interrupts */ \ 1541 btst PSTATE_IE, reg3; \ 1542 /*CSTYLED*/ \ 1543 bnz,a,pt %icc, label1; \ 1544 wrpr reg3, PSTATE_IE, %pstate; \ 1545 /*CSTYLED*/ \ 1546 label1:; \ 1547 brz reg2, label2; /* need demap if N_pgsz0/1 change */ \ 1548 sethi %hi(FLUSH_ADDR), reg4; \ 1549 mov DEMAP_ALL_TYPE, reg2; \ 1550 stxa %g0, [reg2]ASI_DTLB_DEMAP; \ 1551 stxa %g0, [reg2]ASI_ITLB_DEMAP; \ 1552 /*CSTYLED*/ \ 1553 label2:; \ 1554 stxa reg0, [reg1]ASI_MMU_CTX; \ 1555 flush reg4; \ 1556 btst PSTATE_IE, reg3; \ 1557 /*CSTYLED*/ \ 1558 bnz,a,pt %icc, label3; \ 1559 wrpr %g0, reg3, %pstate; /* restore interrupt state */ \ 1560 label3:; 1561 1562 #endif 1563 1564 /* 1565 * Macro to setup arguments with kernel sfmmup context + page size before 1566 * calling sfmmu_setctx_sec() 1567 */ 1568 #ifdef sun4v 1569 #define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \ 1570 set KCONTEXT, arg0; \ 1571 set 0, arg1; 1572 #else 1573 #define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \ 1574 ldub [sfmmup + SFMMU_CEXT], arg1; \ 1575 set KCONTEXT, arg0; \ 1576 sll arg1, CTXREG_EXT_SHIFT, arg1; 1577 #endif 1578 1579 #define PANIC_IF_INTR_DISABLED_PSTR(pstatereg, label, scr) \ 1580 andcc pstatereg, PSTATE_IE, %g0; /* panic if intrs */ \ 1581 /*CSTYLED*/ \ 1582 bnz,pt %icc, label; /* already disabled */ \ 1583 nop; \ 1584 \ 1585 sethi %hi(panicstr), scr; \ 1586 ldx [scr + %lo(panicstr)], scr; \ 1587 tst scr; \ 1588 /*CSTYLED*/ \ 1589 bnz,pt %xcc, label; \ 1590 nop; \ 1591 \ 1592 save %sp, -SA(MINFRAME), %sp; \ 1593 sethi %hi(sfmmu_panic1), %o0; \ 1594 call panic; \ 1595 or %o0, %lo(sfmmu_panic1), %o0; \ 1596 /*CSTYLED*/ \ 1597 label: 1598 1599 #define PANIC_IF_INTR_ENABLED_PSTR(label, scr) \ 1600 /* \ 1601 * The caller must have disabled interrupts. \ 1602 * If interrupts are not disabled, panic \ 1603 */ \ 1604 rdpr %pstate, scr; \ 1605 andcc scr, PSTATE_IE, %g0; \ 1606 /*CSTYLED*/ \ 1607 bz,pt %icc, label; \ 1608 nop; \ 1609 \ 1610 sethi %hi(panicstr), scr; \ 1611 ldx [scr + %lo(panicstr)], scr; \ 1612 tst scr; \ 1613 /*CSTYLED*/ \ 1614 bnz,pt %xcc, label; \ 1615 nop; \ 1616 \ 1617 sethi %hi(sfmmu_panic6), %o0; \ 1618 call panic; \ 1619 or %o0, %lo(sfmmu_panic6), %o0; \ 1620 /*CSTYLED*/ \ 1621 label: 1622 1623 #endif /* _ASM */ 1624 1625 #ifndef _ASM 1626 1627 #ifdef VAC 1628 /* 1629 * Page coloring 1630 * The p_vcolor field of the page struct (1 byte) is used to store the 1631 * virtual page color. This provides for 255 colors. The value zero is 1632 * used to mean the page has no color - never been mapped or somehow 1633 * purified. 1634 */ 1635 1636 #define PP_GET_VCOLOR(pp) (((pp)->p_vcolor) - 1) 1637 #define PP_NEWPAGE(pp) (!(pp)->p_vcolor) 1638 #define PP_SET_VCOLOR(pp, color) \ 1639 ((pp)->p_vcolor = ((color) + 1)) 1640 1641 /* 1642 * As mentioned p_vcolor == 0 means there is no color for this page. 1643 * But PP_SET_VCOLOR(pp, color) expects 'color' to be real color minus 1644 * one so we define this constant. 1645 */ 1646 #define NO_VCOLOR (-1) 1647 1648 #define addr_to_vcolor(addr) \ 1649 (((uint_t)(uintptr_t)(addr) >> MMU_PAGESHIFT) & vac_colors_mask) 1650 #else /* VAC */ 1651 #define addr_to_vcolor(addr) (0) 1652 #endif /* VAC */ 1653 1654 /* 1655 * The field p_index in the psm page structure is for large pages support. 1656 * P_index is a bit-vector of the different mapping sizes that a given page 1657 * is part of. An hme structure for a large mapping is only added in the 1658 * group leader page (first page). All pages covered by a given large mapping 1659 * have the corrosponding mapping bit set in their p_index field. This allows 1660 * us to only store an explicit hme structure in the leading page which 1661 * simplifies the mapping link list management. Furthermore, it provides us 1662 * a fast mechanism for determining the largest mapping a page is part of. For 1663 * exmaple, a page with a 64K and a 4M mappings has a p_index value of 0x0A. 1664 * 1665 * Implementation note: even though the first bit in p_index is reserved 1666 * for 8K mappings, it is NOT USED by the code and SHOULD NOT be set. 1667 * In addition, the upper four bits of the p_index field are used by the 1668 * code as temporaries 1669 */ 1670 1671 /* 1672 * Defines for psm page struct fields and large page support 1673 */ 1674 #define SFMMU_INDEX_SHIFT 6 1675 #define SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1) 1676 1677 /* Return the mapping index */ 1678 #define PP_MAPINDEX(pp) ((pp)->p_index & SFMMU_INDEX_MASK) 1679 1680 /* 1681 * These macros rely on the following property: 1682 * All pages constituting a large page are covered by a virtually 1683 * contiguous set of page_t's. 1684 */ 1685 1686 /* Return the leader for this mapping size */ 1687 #define PP_GROUPLEADER(pp, sz) \ 1688 (&(pp)[-(int)(pp->p_pagenum & (TTEPAGES(sz)-1))]) 1689 1690 /* Return the root page for this page based on p_szc */ 1691 #define PP_PAGEROOT(pp) ((pp)->p_szc == 0 ? (pp) : \ 1692 PP_GROUPLEADER((pp), (pp)->p_szc)) 1693 1694 #define PP_PAGENEXT_N(pp, n) ((pp) + (n)) 1695 #define PP_PAGENEXT(pp) PP_PAGENEXT_N((pp), 1) 1696 1697 #define PP_PAGEPREV_N(pp, n) ((pp) - (n)) 1698 #define PP_PAGEPREV(pp) PP_PAGEPREV_N((pp), 1) 1699 1700 #define PP_ISMAPPED_LARGE(pp) (PP_MAPINDEX(pp) != 0) 1701 1702 /* Need function to test the page mappping which takes p_index into account */ 1703 #define PP_ISMAPPED(pp) ((pp)->p_mapping || PP_ISMAPPED_LARGE(pp)) 1704 1705 /* 1706 * Don't call this macro with sz equal to zero. 8K mappings SHOULD NOT 1707 * set p_index field. 1708 */ 1709 #define PAGESZ_TO_INDEX(sz) (1 << (sz)) 1710 1711 1712 /* 1713 * prototypes for hat assembly routines. Some of these are 1714 * known to machine dependent VM code. 1715 */ 1716 extern uint64_t sfmmu_make_tsbtag(caddr_t); 1717 extern struct tsbe * 1718 sfmmu_get_tsbe(uint64_t, caddr_t, int, int); 1719 extern void sfmmu_load_tsbe(struct tsbe *, uint64_t, tte_t *, int); 1720 extern void sfmmu_unload_tsbe(struct tsbe *, uint64_t, int); 1721 extern void sfmmu_load_mmustate(sfmmu_t *); 1722 extern void sfmmu_raise_tsb_exception(uint64_t, uint64_t); 1723 #ifndef sun4v 1724 extern void sfmmu_itlb_ld_kva(caddr_t, tte_t *); 1725 extern void sfmmu_dtlb_ld_kva(caddr_t, tte_t *); 1726 #endif /* sun4v */ 1727 extern void sfmmu_copytte(tte_t *, tte_t *); 1728 extern int sfmmu_modifytte(tte_t *, tte_t *, tte_t *); 1729 extern int sfmmu_modifytte_try(tte_t *, tte_t *, tte_t *); 1730 extern pfn_t sfmmu_ttetopfn(tte_t *, caddr_t); 1731 extern void sfmmu_hblk_hash_rm(struct hmehash_bucket *, 1732 struct hme_blk *, uint64_t, struct hme_blk *); 1733 extern void sfmmu_hblk_hash_add(struct hmehash_bucket *, struct hme_blk *, 1734 uint64_t); 1735 extern uint_t sfmmu_disable_intrs(void); 1736 extern void sfmmu_enable_intrs(uint_t); 1737 /* 1738 * functions exported to machine dependent VM code 1739 */ 1740 extern void sfmmu_patch_ktsb(void); 1741 #ifndef UTSB_PHYS 1742 extern void sfmmu_patch_utsb(void); 1743 #endif /* UTSB_PHYS */ 1744 extern pfn_t sfmmu_vatopfn(caddr_t, sfmmu_t *, tte_t *); 1745 extern void sfmmu_vatopfn_suspended(caddr_t, sfmmu_t *, tte_t *); 1746 extern pfn_t sfmmu_kvaszc2pfn(caddr_t, int); 1747 #ifdef DEBUG 1748 extern void sfmmu_check_kpfn(pfn_t); 1749 #else 1750 #define sfmmu_check_kpfn(pfn) /* disabled */ 1751 #endif /* DEBUG */ 1752 extern void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 1753 extern void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, uint_t); 1754 extern void sfmmu_tsbmiss_exception(struct regs *, uintptr_t, uint_t); 1755 extern void sfmmu_init_tsbs(void); 1756 extern caddr_t sfmmu_ktsb_alloc(caddr_t); 1757 extern int sfmmu_getctx_pri(void); 1758 extern int sfmmu_getctx_sec(void); 1759 extern void sfmmu_setctx_sec(int); 1760 extern void sfmmu_inv_tsb(caddr_t, uint_t); 1761 extern void sfmmu_init_ktsbinfo(void); 1762 extern int sfmmu_setup_4lp(void); 1763 extern void sfmmu_patch_mmu_asi(int); 1764 extern void sfmmu_init_nucleus_hblks(caddr_t, size_t, int, int); 1765 extern void sfmmu_cache_flushall(void); 1766 extern pgcnt_t sfmmu_tte_cnt(sfmmu_t *, uint_t); 1767 extern void *sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int); 1768 extern void sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t); 1769 extern void sfmmu_reprog_pgsz_arr(sfmmu_t *, uint8_t *); 1770 1771 extern void hat_kern_setup(void); 1772 extern int hat_page_relocate(page_t **, page_t **, spgcnt_t *); 1773 extern int sfmmu_get_ppvcolor(struct page *); 1774 extern int sfmmu_get_addrvcolor(caddr_t); 1775 extern int sfmmu_hat_lock_held(sfmmu_t *); 1776 extern void sfmmu_alloc_ctx(sfmmu_t *, int, struct cpu *); 1777 1778 /* 1779 * Functions exported to xhat_sfmmu.c 1780 */ 1781 extern kmutex_t *sfmmu_mlist_enter(page_t *); 1782 extern void sfmmu_mlist_exit(kmutex_t *); 1783 extern int sfmmu_mlist_held(struct page *); 1784 extern struct hme_blk *sfmmu_hmetohblk(struct sf_hment *); 1785 1786 /* 1787 * MMU-specific functions optionally imported from the CPU module 1788 */ 1789 #pragma weak mmu_large_pages_disabled 1790 #pragma weak mmu_set_ctx_page_sizes 1791 #pragma weak mmu_check_page_sizes 1792 1793 extern uint_t mmu_large_pages_disabled(uint_t); 1794 extern void mmu_set_ctx_page_sizes(sfmmu_t *); 1795 extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *); 1796 1797 extern sfmmu_t *ksfmmup; 1798 extern caddr_t ktsb_base; 1799 extern uint64_t ktsb_pbase; 1800 extern int ktsb_sz; 1801 extern int ktsb_szcode; 1802 extern caddr_t ktsb4m_base; 1803 extern uint64_t ktsb4m_pbase; 1804 extern int ktsb4m_sz; 1805 extern int ktsb4m_szcode; 1806 extern uint64_t kpm_tsbbase; 1807 extern int kpm_tsbsz; 1808 extern int ktsb_phys; 1809 extern int enable_bigktsb; 1810 #ifndef sun4v 1811 extern int utsb_dtlb_ttenum; 1812 extern int utsb4m_dtlb_ttenum; 1813 #endif /* sun4v */ 1814 extern int uhmehash_num; 1815 extern int khmehash_num; 1816 extern struct hmehash_bucket *uhme_hash; 1817 extern struct hmehash_bucket *khme_hash; 1818 extern kmutex_t *mml_table; 1819 extern uint_t mml_table_sz; 1820 extern uint_t mml_shift; 1821 extern uint_t hblk_alloc_dynamic; 1822 extern struct tsbmiss tsbmiss_area[NCPU]; 1823 extern struct kpmtsbm kpmtsbm_area[NCPU]; 1824 extern int tsb_max_growsize; 1825 #ifndef sun4v 1826 extern int dtlb_resv_ttenum; 1827 extern caddr_t utsb_vabase; 1828 extern caddr_t utsb4m_vabase; 1829 #endif /* sun4v */ 1830 extern vmem_t *kmem_tsb_default_arena[]; 1831 extern int tsb_lgrp_affinity; 1832 1833 extern uint_t disable_large_pages; 1834 extern uint_t disable_ism_large_pages; 1835 extern uint_t disable_auto_data_large_pages; 1836 extern uint_t disable_auto_text_large_pages; 1837 1838 /* kpm externals */ 1839 extern pfn_t sfmmu_kpm_vatopfn(caddr_t); 1840 extern void sfmmu_kpm_patch_tlbm(void); 1841 extern void sfmmu_kpm_patch_tsbm(void); 1842 extern void sfmmu_kpm_load_tsb(caddr_t, tte_t *, int); 1843 extern void sfmmu_kpm_unload_tsb(caddr_t, int); 1844 extern void sfmmu_kpm_tsbmtl(short *, uint_t *, int); 1845 extern int sfmmu_kpm_stsbmtl(char *, uint_t *, int); 1846 extern caddr_t kpm_vbase; 1847 extern size_t kpm_size; 1848 extern struct memseg *memseg_hash[]; 1849 extern uint64_t memseg_phash[]; 1850 extern kpm_hlk_t *kpmp_table; 1851 extern kpm_shlk_t *kpmp_stable; 1852 extern uint_t kpmp_table_sz; 1853 extern uint_t kpmp_stable_sz; 1854 extern uchar_t kpmp_shift; 1855 1856 #define PP_ISMAPPED_KPM(pp) ((pp)->p_kpmref > 0) 1857 1858 #define IS_KPM_ALIAS_RANGE(vaddr) \ 1859 (((vaddr) - kpm_vbase) >> (uintptr_t)kpm_size_shift > 0) 1860 1861 #endif /* !_ASM */ 1862 1863 /* sfmmu_kpm_tsbmtl flags */ 1864 #define KPMTSBM_STOP 0 1865 #define KPMTSBM_START 1 1866 1867 /* kpm_smallpages kp_mapped values */ 1868 #define KPM_MAPPEDS -1 /* small mapping valid, no conflict */ 1869 #define KPM_MAPPEDSC 1 /* small mapping valid, conflict */ 1870 1871 /* Physical memseg address NULL marker */ 1872 #define MSEG_NULLPTR_PA -1 1873 1874 /* 1875 * Memseg hash defines for kpm trap level tsbmiss handler. 1876 * Must be in sync w/ page.h . 1877 */ 1878 #define SFMMU_MEM_HASH_SHIFT 0x9 1879 #define SFMMU_N_MEM_SLOTS 0x200 1880 #define SFMMU_MEM_HASH_ENTRY_SHIFT 3 1881 1882 #ifndef _ASM 1883 #if (SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT) 1884 #error SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT 1885 #endif 1886 #if (SFMMU_N_MEM_SLOTS != N_MEM_SLOTS) 1887 #error SFMMU_N_MEM_SLOTS != N_MEM_SLOTS 1888 #endif 1889 1890 /* Physical memseg address NULL marker */ 1891 #define SFMMU_MEMSEG_NULLPTR_PA -1 1892 1893 /* 1894 * Check KCONTEXT to be zero, asm parts depend on that assumption. 1895 */ 1896 #if (KCONTEXT != 0) 1897 #error KCONTEXT != 0 1898 #endif 1899 #endif /* !_ASM */ 1900 1901 1902 #endif /* _KERNEL */ 1903 1904 #ifndef _ASM 1905 /* 1906 * ctx, hmeblk, mlistlock and other stats for sfmmu 1907 */ 1908 struct sfmmu_global_stat { 1909 int sf_tsb_exceptions; /* # of tsb exceptions */ 1910 int sf_tsb_raise_exception; /* # tsb exc. w/o TLB flush */ 1911 1912 int sf_pagefaults; /* # of pagefaults */ 1913 1914 int sf_uhash_searches; /* # of user hash searches */ 1915 int sf_uhash_links; /* # of user hash links */ 1916 int sf_khash_searches; /* # of kernel hash searches */ 1917 int sf_khash_links; /* # of kernel hash links */ 1918 1919 int sf_swapout; /* # times hat swapped out */ 1920 1921 int sf_tsb_alloc; /* # TSB allocations */ 1922 int sf_tsb_allocfail; /* # times TSB alloc fail */ 1923 int sf_tsb_sectsb_create; /* # times second TSB added */ 1924 1925 int sf_tteload8k; /* calls to sfmmu_tteload */ 1926 int sf_tteload64k; /* calls to sfmmu_tteload */ 1927 int sf_tteload512k; /* calls to sfmmu_tteload */ 1928 int sf_tteload4m; /* calls to sfmmu_tteload */ 1929 int sf_tteload32m; /* calls to sfmmu_tteload */ 1930 int sf_tteload256m; /* calls to sfmmu_tteload */ 1931 1932 int sf_tsb_load8k; /* # times loaded 8K tsbent */ 1933 int sf_tsb_load4m; /* # times loaded 4M tsbent */ 1934 1935 int sf_hblk_hit; /* found hblk during tteload */ 1936 int sf_hblk8_ncreate; /* static hblk8's created */ 1937 int sf_hblk8_nalloc; /* static hblk8's allocated */ 1938 int sf_hblk1_ncreate; /* static hblk1's created */ 1939 int sf_hblk1_nalloc; /* static hblk1's allocated */ 1940 int sf_hblk_slab_cnt; /* sfmmu8_cache slab creates */ 1941 int sf_hblk_reserve_cnt; /* hblk_reserve usage */ 1942 int sf_hblk_recurse_cnt; /* hblk_reserve owner reqs */ 1943 int sf_hblk_reserve_hit; /* hblk_reserve hash hits */ 1944 int sf_get_free_success; /* reserve list allocs */ 1945 int sf_get_free_throttle; /* fails due to throttling */ 1946 int sf_get_free_fail; /* fails due to empty list */ 1947 int sf_put_free_success; /* reserve list frees */ 1948 int sf_put_free_fail; /* fails due to full list */ 1949 1950 int sf_pgcolor_conflict; /* VAC conflict resolution */ 1951 int sf_uncache_conflict; /* VAC conflict resolution */ 1952 int sf_unload_conflict; /* VAC unload resolution */ 1953 int sf_ism_uncache; /* VAC conflict resolution */ 1954 int sf_ism_recache; /* VAC conflict resolution */ 1955 int sf_recache; /* VAC conflict resolution */ 1956 1957 int sf_steal_count; /* # of hblks stolen */ 1958 1959 int sf_pagesync; /* # of pagesyncs */ 1960 int sf_clrwrt; /* # of clear write perms */ 1961 int sf_pagesync_invalid; /* pagesync with inv tte */ 1962 1963 int sf_kernel_xcalls; /* # of kernel cross calls */ 1964 int sf_user_xcalls; /* # of user cross calls */ 1965 1966 int sf_tsb_grow; /* # of user tsb grows */ 1967 int sf_tsb_shrink; /* # of user tsb shrinks */ 1968 int sf_tsb_resize_failures; /* # of user tsb resize */ 1969 int sf_tsb_reloc; /* # of user tsb relocations */ 1970 1971 int sf_user_vtop; /* # of user vatopfn calls */ 1972 1973 int sf_ctx_inv; /* #times invalidate MMU ctx */ 1974 1975 int sf_tlb_reprog_pgsz; /* # times switch TLB pgsz */ 1976 }; 1977 1978 struct sfmmu_tsbsize_stat { 1979 int sf_tsbsz_8k; 1980 int sf_tsbsz_16k; 1981 int sf_tsbsz_32k; 1982 int sf_tsbsz_64k; 1983 int sf_tsbsz_128k; 1984 int sf_tsbsz_256k; 1985 int sf_tsbsz_512k; 1986 int sf_tsbsz_1m; 1987 int sf_tsbsz_2m; 1988 int sf_tsbsz_4m; 1989 }; 1990 1991 struct sfmmu_percpu_stat { 1992 int sf_itlb_misses; /* # of itlb misses */ 1993 int sf_dtlb_misses; /* # of dtlb misses */ 1994 int sf_utsb_misses; /* # of user tsb misses */ 1995 int sf_ktsb_misses; /* # of kernel tsb misses */ 1996 int sf_tsb_hits; /* # of tsb hits */ 1997 int sf_umod_faults; /* # of mod (prot viol) flts */ 1998 int sf_kmod_faults; /* # of mod (prot viol) flts */ 1999 }; 2000 2001 #define SFMMU_STAT(stat) sfmmu_global_stat.stat++ 2002 #define SFMMU_STAT_ADD(stat, amount) sfmmu_global_stat.stat += (amount) 2003 #define SFMMU_STAT_SET(stat, count) sfmmu_global_stat.stat = (count) 2004 2005 #define SFMMU_MMU_STAT(stat) CPU->cpu_m.cpu_mmu_ctxp->stat++ 2006 2007 #endif /* !_ASM */ 2008 2009 #ifdef __cplusplus 2010 } 2011 #endif 2012 2013 #endif /* _VM_HAT_SFMMU_H */ 2014