1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * VM - Hardware Address Translation management. 28 * 29 * This file describes the contents of the sun-reference-mmu(sfmmu)- 30 * specific hat data structures and the sfmmu-specific hat procedures. 31 * The machine-independent interface is described in <vm/hat.h>. 32 */ 33 34 #ifndef _VM_HAT_SFMMU_H 35 #define _VM_HAT_SFMMU_H 36 37 #pragma ident "%Z%%M% %I% %E% SMI" 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 #ifndef _ASM 44 45 #include <sys/types.h> 46 47 #endif /* _ASM */ 48 49 #ifdef _KERNEL 50 51 #include <sys/pte.h> 52 #include <vm/mach_sfmmu.h> 53 #include <sys/mmu.h> 54 55 /* 56 * Don't alter these without considering changes to ism_map_t. 57 */ 58 #define DEFAULT_ISM_PAGESIZE MMU_PAGESIZE4M 59 #define ISM_PG_SIZE(ism_vbshift) (1 << ism_vbshift) 60 #define ISM_SZ_MASK(ism_vbshift) (ISM_PG_SIZE(ism_vbshift) - 1) 61 #define ISM_MAP_SLOTS 8 /* Change this carefully. */ 62 63 #ifndef _ASM 64 65 #include <sys/t_lock.h> 66 #include <vm/hat.h> 67 #include <vm/seg.h> 68 #include <sys/machparam.h> 69 #include <sys/systm.h> 70 #include <sys/x_call.h> 71 #include <vm/page.h> 72 #include <sys/ksynch.h> 73 74 typedef struct hat sfmmu_t; 75 typedef struct sf_scd sf_scd_t; 76 77 /* 78 * SFMMU attributes for hat_memload/hat_devload 79 */ 80 #define SFMMU_UNCACHEPTTE 0x01000000 /* unencache in physical $ */ 81 #define SFMMU_UNCACHEVTTE 0x02000000 /* unencache in virtual $ */ 82 #define SFMMU_SIDEFFECT 0x04000000 /* set side effect bit */ 83 #define SFMMU_LOAD_ALLATTR (HAT_PROT_MASK | HAT_ORDER_MASK | \ 84 HAT_ENDIAN_MASK | HAT_NOFAULT | HAT_NOSYNC | \ 85 SFMMU_UNCACHEPTTE | SFMMU_UNCACHEVTTE | SFMMU_SIDEFFECT) 86 87 88 /* 89 * sfmmu flags for hat_memload/hat_devload 90 */ 91 #define SFMMU_NO_TSBLOAD 0x08000000 /* do not preload tsb */ 92 #define SFMMU_LOAD_ALLFLAG (HAT_LOAD | HAT_LOAD_LOCK | \ 93 HAT_LOAD_ADV | HAT_LOAD_CONTIG | HAT_LOAD_NOCONSIST | \ 94 HAT_LOAD_SHARE | HAT_LOAD_REMAP | SFMMU_NO_TSBLOAD | \ 95 HAT_RELOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_TEXT) 96 97 /* 98 * sfmmu internal flag to hat_pageunload that spares locked mappings 99 */ 100 #define SFMMU_KERNEL_RELOC 0x8000 101 102 /* 103 * mode for sfmmu_chgattr 104 */ 105 #define SFMMU_SETATTR 0x0 106 #define SFMMU_CLRATTR 0x1 107 #define SFMMU_CHGATTR 0x2 108 109 /* 110 * sfmmu specific flags for page_t 111 */ 112 #define P_PNC 0x8 /* non-caching is permanent bit */ 113 #define P_TNC 0x10 /* non-caching is temporary bit */ 114 #define P_KPMS 0x20 /* kpm mapped small (vac alias prevention) */ 115 #define P_KPMC 0x40 /* kpm conflict page (vac alias prevention) */ 116 117 #define PP_GENERIC_ATTR(pp) ((pp)->p_nrm & (P_MOD | P_REF | P_RO)) 118 #define PP_ISMOD(pp) ((pp)->p_nrm & P_MOD) 119 #define PP_ISREF(pp) ((pp)->p_nrm & P_REF) 120 #define PP_ISRO(pp) ((pp)->p_nrm & P_RO) 121 #define PP_ISNC(pp) ((pp)->p_nrm & (P_PNC|P_TNC)) 122 #define PP_ISPNC(pp) ((pp)->p_nrm & P_PNC) 123 #ifdef VAC 124 #define PP_ISTNC(pp) ((pp)->p_nrm & P_TNC) 125 #endif 126 #define PP_ISKPMS(pp) ((pp)->p_nrm & P_KPMS) 127 #define PP_ISKPMC(pp) ((pp)->p_nrm & P_KPMC) 128 129 #define PP_SETMOD(pp) ((pp)->p_nrm |= P_MOD) 130 #define PP_SETREF(pp) ((pp)->p_nrm |= P_REF) 131 #define PP_SETREFMOD(pp) ((pp)->p_nrm |= (P_REF|P_MOD)) 132 #define PP_SETRO(pp) ((pp)->p_nrm |= P_RO) 133 #define PP_SETREFRO(pp) ((pp)->p_nrm |= (P_REF|P_RO)) 134 #define PP_SETPNC(pp) ((pp)->p_nrm |= P_PNC) 135 #ifdef VAC 136 #define PP_SETTNC(pp) ((pp)->p_nrm |= P_TNC) 137 #endif 138 #define PP_SETKPMS(pp) ((pp)->p_nrm |= P_KPMS) 139 #define PP_SETKPMC(pp) ((pp)->p_nrm |= P_KPMC) 140 141 #define PP_CLRMOD(pp) ((pp)->p_nrm &= ~P_MOD) 142 #define PP_CLRREF(pp) ((pp)->p_nrm &= ~P_REF) 143 #define PP_CLRREFMOD(pp) ((pp)->p_nrm &= ~(P_REF|P_MOD)) 144 #define PP_CLRRO(pp) ((pp)->p_nrm &= ~P_RO) 145 #define PP_CLRPNC(pp) ((pp)->p_nrm &= ~P_PNC) 146 #ifdef VAC 147 #define PP_CLRTNC(pp) ((pp)->p_nrm &= ~P_TNC) 148 #endif 149 #define PP_CLRKPMS(pp) ((pp)->p_nrm &= ~P_KPMS) 150 #define PP_CLRKPMC(pp) ((pp)->p_nrm &= ~P_KPMC) 151 152 /* 153 * All shared memory segments attached with the SHM_SHARE_MMU flag (ISM) 154 * will be constrained to a 4M, 32M or 256M alignment. Also since every newly- 155 * created ISM segment is created out of a new address space at base va 156 * of 0 we don't need to store it. 157 */ 158 #define ISM_ALIGN(shift) (1 << shift) /* base va aligned to <n>M */ 159 #define ISM_ALIGNED(shift, va) (((uintptr_t)va & (ISM_ALIGN(shift) - 1)) == 0) 160 #define ISM_SHIFT(shift, x) ((uintptr_t)x >> (shift)) 161 162 /* 163 * Pad locks out to cache sub-block boundaries to prevent 164 * false sharing, so several processes don't contend for 165 * the same line if they aren't using the same lock. Since 166 * this is a typedef we also have a bit of freedom in 167 * changing lock implementations later if we decide it 168 * is necessary. 169 */ 170 typedef struct hat_lock { 171 kmutex_t hl_mutex; 172 uchar_t hl_pad[64 - sizeof (kmutex_t)]; 173 } hatlock_t; 174 175 #define HATLOCK_MUTEXP(hatlockp) (&((hatlockp)->hl_mutex)) 176 177 /* 178 * All segments mapped with ISM are guaranteed to be 4M, 32M or 256M aligned. 179 * Also size is guaranteed to be in 4M, 32M or 256M chunks. 180 * ism_seg consists of the following members: 181 * [XX..22] base address of ism segment. XX is 63 or 31 depending whether 182 * caddr_t is 64 bits or 32 bits. 183 * [21..0] size of segment. 184 * 185 * NOTE: Don't alter this structure without changing defines above and 186 * the tsb_miss and protection handlers. 187 */ 188 typedef struct ism_map { 189 uintptr_t imap_seg; /* base va + sz of ISM segment */ 190 uchar_t imap_vb_shift; /* mmu_pageshift for ism page size */ 191 uchar_t imap_rid; /* region id for ism */ 192 ushort_t imap_hatflags; /* primary ism page size */ 193 uint_t imap_sz_mask; /* mmu_pagemask for ism page size */ 194 sfmmu_t *imap_ismhat; /* hat id of dummy ISM as */ 195 struct ism_ment *imap_ment; /* pointer to mapping list entry */ 196 } ism_map_t; 197 198 #define ism_start(map) ((caddr_t)((map).imap_seg & \ 199 ~ISM_SZ_MASK((map).imap_vb_shift))) 200 #define ism_size(map) ((map).imap_seg & ISM_SZ_MASK((map).imap_vb_shift)) 201 #define ism_end(map) ((caddr_t)(ism_start(map) + (ism_size(map) * \ 202 ISM_PG_SIZE((map).imap_vb_shift)))) 203 /* 204 * ISM mapping entry. Used to link all hat's sharing a ism_hat. 205 * Same function as the p_mapping list for a page. 206 */ 207 typedef struct ism_ment { 208 sfmmu_t *iment_hat; /* back pointer to hat_share() hat */ 209 caddr_t iment_base_va; /* hat's va base for this ism seg */ 210 struct ism_ment *iment_next; /* next ism map entry */ 211 struct ism_ment *iment_prev; /* prev ism map entry */ 212 } ism_ment_t; 213 214 /* 215 * ISM segment block. One will be hung off the sfmmu structure if a 216 * a process uses ISM. More will be linked using ismblk_next if more 217 * than ISM_MAP_SLOTS segments are attached to this proc. 218 * 219 * All modifications to fields in this structure will be protected 220 * by the hat mutex. In order to avoid grabbing this lock in low level 221 * routines (tsb miss/protection handlers and vatopfn) while not 222 * introducing any race conditions with hat_unshare, we will set 223 * CTX_ISM_BUSY bit in the ctx struct. Any mmu traps that occur 224 * for this ctx while this bit is set will be handled in sfmmu_tsb_excption 225 * where it will synchronize behind the hat mutex. 226 */ 227 typedef struct ism_blk { 228 ism_map_t iblk_maps[ISM_MAP_SLOTS]; 229 struct ism_blk *iblk_next; 230 uint64_t iblk_nextpa; 231 } ism_blk_t; 232 233 /* 234 * TSB access information. All fields are protected by the process's 235 * hat lock. 236 */ 237 238 struct tsb_info { 239 caddr_t tsb_va; /* tsb base virtual address */ 240 uint64_t tsb_pa; /* tsb base physical address */ 241 struct tsb_info *tsb_next; /* next tsb used by this process */ 242 uint16_t tsb_szc; /* tsb size code */ 243 uint16_t tsb_flags; /* flags for this tsb; see below */ 244 uint_t tsb_ttesz_mask; /* page size masks; see below */ 245 246 tte_t tsb_tte; /* tte to lock into DTLB */ 247 sfmmu_t *tsb_sfmmu; /* sfmmu */ 248 kmem_cache_t *tsb_cache; /* cache from which mem allocated */ 249 vmem_t *tsb_vmp; /* vmem arena from which mem alloc'd */ 250 }; 251 252 /* 253 * Values for "tsb_ttesz_mask" bitmask. 254 */ 255 #define TSB8K (1 << TTE8K) 256 #define TSB64K (1 << TTE64K) 257 #define TSB512K (1 << TTE512K) 258 #define TSB4M (1 << TTE4M) 259 #define TSB32M (1 << TTE32M) 260 #define TSB256M (1 << TTE256M) 261 262 /* 263 * Values for "tsb_flags" field. 264 */ 265 #define TSB_RELOC_FLAG 0x1 266 #define TSB_FLUSH_NEEDED 0x2 267 #define TSB_SWAPPED 0x4 268 #define TSB_SHAREDCTX 0x8 269 270 #endif /* !_ASM */ 271 272 /* 273 * Data structures for shared hmeblk support. 274 */ 275 276 /* 277 * Do not increase the maximum number of ism/hme regions without checking first 278 * the impact on ism_map_t, TSB miss area, hblk tag and region id type in 279 * sf_region structure. 280 * Initially, shared hmes will only be used for the main text segment 281 * therefore this value will be set to 64, it will be increased when shared 282 * libraries are included. 283 */ 284 285 #define SFMMU_MAX_HME_REGIONS (64) 286 #define SFMMU_HMERGNMAP_WORDS BT_BITOUL(SFMMU_MAX_HME_REGIONS) 287 288 #define SFMMU_PRIVATE 0 289 #define SFMMU_SHARED 1 290 291 #ifndef _ASM 292 293 #define SFMMU_MAX_ISM_REGIONS (64) 294 #define SFMMU_ISMRGNMAP_WORDS BT_BITOUL(SFMMU_MAX_ISM_REGIONS) 295 296 #define SFMMU_RGNMAP_WORDS (SFMMU_HMERGNMAP_WORDS + SFMMU_ISMRGNMAP_WORDS) 297 298 #define SFMMU_MAX_REGION_BUCKETS (128) 299 #define SFMMU_MAX_SRD_BUCKETS (2048) 300 301 typedef struct sf_hmeregion_map { 302 ulong_t bitmap[SFMMU_HMERGNMAP_WORDS]; 303 } sf_hmeregion_map_t; 304 305 typedef struct sf_ismregion_map { 306 ulong_t bitmap[SFMMU_ISMRGNMAP_WORDS]; 307 } sf_ismregion_map_t; 308 309 typedef union sf_region_map_u { 310 struct _h_rmap_s { 311 sf_hmeregion_map_t hmeregion_map; 312 sf_ismregion_map_t ismregion_map; 313 } h_rmap_s; 314 ulong_t bitmap[SFMMU_RGNMAP_WORDS]; 315 } sf_region_map_t; 316 317 #define SF_RGNMAP_ZERO(map) { \ 318 int _i; \ 319 for (_i = 0; _i < SFMMU_RGNMAP_WORDS; _i++) { \ 320 (map).bitmap[_i] = 0; \ 321 } \ 322 } 323 324 /* 325 * Returns 1 if map1 and map2 are equal. 326 */ 327 #define SF_RGNMAP_EQUAL(map1, map2, rval) { \ 328 int _i; \ 329 for (_i = 0; _i < SFMMU_RGNMAP_WORDS; _i++) { \ 330 if ((map1)->bitmap[_i] != (map2)->bitmap[_i]) \ 331 break; \ 332 } \ 333 if (_i < SFMMU_RGNMAP_WORDS) \ 334 rval = 0; \ 335 else \ 336 rval = 1; \ 337 } 338 339 #define SF_RGNMAP_ADD(map, r) BT_SET((map).bitmap, r) 340 #define SF_RGNMAP_DEL(map, r) BT_CLEAR((map).bitmap, r) 341 #define SF_RGNMAP_TEST(map, r) BT_TEST((map).bitmap, r) 342 343 /* 344 * Tests whether map2 is a subset of map1, returns 1 if 345 * this assertion is true. 346 */ 347 #define SF_RGNMAP_IS_SUBSET(map1, map2, rval) { \ 348 int _i; \ 349 for (_i = 0; _i < SFMMU_RGNMAP_WORDS; _i++) { \ 350 if (((map1)->bitmap[_i] & (map2)->bitmap[_i]) \ 351 != (map2)->bitmap[_i]) { \ 352 break; \ 353 } \ 354 } \ 355 if (_i < SFMMU_RGNMAP_WORDS) \ 356 rval = 0; \ 357 else \ 358 rval = 1; \ 359 } 360 361 #define SF_SCD_INCR_REF(scdp) { \ 362 atomic_add_32((volatile uint32_t *)&(scdp)->scd_refcnt, 1); \ 363 } 364 365 #define SF_SCD_DECR_REF(srdp, scdp) { \ 366 sf_region_map_t _scd_rmap = (scdp)->scd_region_map; \ 367 if (!atomic_add_32_nv( \ 368 (volatile uint32_t *)&(scdp)->scd_refcnt, -1)) { \ 369 sfmmu_destroy_scd((srdp), (scdp), &_scd_rmap); \ 370 } \ 371 } 372 373 /* 374 * A sfmmup link in the link list of sfmmups that share the same region. 375 */ 376 typedef struct sf_rgn_link { 377 sfmmu_t *next; 378 sfmmu_t *prev; 379 } sf_rgn_link_t; 380 381 /* 382 * rgn_flags values. 383 */ 384 #define SFMMU_REGION_HME 0x1 385 #define SFMMU_REGION_ISM 0x2 386 #define SFMMU_REGION_FREE 0x8 387 388 #define SFMMU_REGION_TYPE_MASK (0x3) 389 390 /* 391 * sf_region defines a text or (D)ISM segment which map 392 * the same underlying physical object. 393 */ 394 typedef struct sf_region { 395 caddr_t rgn_saddr; /* base addr of attached seg */ 396 size_t rgn_size; /* size of attached seg */ 397 void *rgn_obj; /* the underlying object id */ 398 u_offset_t rgn_objoff; /* offset in the object mapped */ 399 uchar_t rgn_perm; /* PROT_READ/WRITE/EXEC */ 400 uchar_t rgn_pgszc; /* page size of the region */ 401 uchar_t rgn_flags; /* region type, free flag */ 402 uchar_t rgn_id; 403 int rgn_refcnt; /* # of hats sharing the region */ 404 /* callback function for hat_unload_callback */ 405 hat_rgn_cb_func_t rgn_cb_function; 406 struct sf_region *rgn_hash; /* hash chain linking the rgns */ 407 kmutex_t rgn_mutex; /* protect region sfmmu list */ 408 /* A link list of processes attached to this region */ 409 sfmmu_t *rgn_sfmmu_head; 410 ulong_t rgn_ttecnt[MMU_PAGE_SIZES]; 411 uint16_t rgn_hmeflags; /* rgn tte size flags */ 412 } sf_region_t; 413 414 #define rgn_next rgn_hash 415 416 /* srd */ 417 typedef struct sf_shared_region_domain { 418 vnode_t *srd_evp; /* executable vnode */ 419 /* hme region table */ 420 sf_region_t *srd_hmergnp[SFMMU_MAX_HME_REGIONS]; 421 /* ism region table */ 422 sf_region_t *srd_ismrgnp[SFMMU_MAX_ISM_REGIONS]; 423 /* hash chain linking srds */ 424 struct sf_shared_region_domain *srd_hash; 425 /* pointer to the next free hme region */ 426 sf_region_t *srd_hmergnfree; 427 /* pointer to the next free ism region */ 428 sf_region_t *srd_ismrgnfree; 429 /* id of next ism region created */ 430 uint16_t srd_next_ismrid; 431 /* id of next hme region created */ 432 uint16_t srd_next_hmerid; 433 uint16_t srd_ismbusyrgns; /* # of ism rgns in use */ 434 uint16_t srd_hmebusyrgns; /* # of hme rgns in use */ 435 int srd_refcnt; /* # of procs in the srd */ 436 kmutex_t srd_mutex; /* sync add/remove rgns */ 437 kmutex_t srd_scd_mutex; 438 sf_scd_t *srd_scdp; /* list of scds in srd */ 439 /* hash of regions associated with the same executable */ 440 sf_region_t *srd_rgnhash[SFMMU_MAX_REGION_BUCKETS]; 441 } sf_srd_t; 442 443 typedef struct sf_srd_bucket { 444 kmutex_t srdb_lock; 445 sf_srd_t *srdb_srdp; 446 } sf_srd_bucket_t; 447 448 /* 449 * The value of SFMMU_L1_HMERLINKS and SFMMU_L2_HMERLINKS will be increased 450 * to 16 when the use of shared hmes for shared libraries is enabled. 451 */ 452 453 #define SFMMU_L1_HMERLINKS (8) 454 #define SFMMU_L2_HMERLINKS (8) 455 #define SFMMU_L1_HMERLINKS_SHIFT (3) 456 #define SFMMU_L1_HMERLINKS_MASK (SFMMU_L1_HMERLINKS - 1) 457 #define SFMMU_L2_HMERLINKS_MASK (SFMMU_L2_HMERLINKS - 1) 458 #define SFMMU_L1_HMERLINKS_SIZE \ 459 (SFMMU_L1_HMERLINKS * sizeof (sf_rgn_link_t *)) 460 #define SFMMU_L2_HMERLINKS_SIZE \ 461 (SFMMU_L2_HMERLINKS * sizeof (sf_rgn_link_t)) 462 463 #if (SFMMU_L1_HMERLINKS * SFMMU_L2_HMERLINKS < SFMMU_MAX_HME_REGIONS) 464 #error Not Enough HMERLINKS 465 #endif 466 467 /* 468 * This macro grabs hat lock and allocates level 2 hat chain 469 * associated with a shme rgn. In the majority of cases, the macro 470 * is called with alloc = 0, and lock = 0. 471 * A pointer to the level 2 sf_rgn_link_t structure is returned in the lnkp 472 * parameter. 473 */ 474 #define SFMMU_HMERID2RLINKP(sfmmup, rid, lnkp, alloc, lock) \ 475 { \ 476 int _l1ix = ((rid) >> SFMMU_L1_HMERLINKS_SHIFT) & \ 477 SFMMU_L1_HMERLINKS_MASK; \ 478 int _l2ix = ((rid) & SFMMU_L2_HMERLINKS_MASK); \ 479 hatlock_t *_hatlockp; \ 480 lnkp = (sfmmup)->sfmmu_hmeregion_links[_l1ix]; \ 481 if (lnkp != NULL) { \ 482 lnkp = &lnkp[_l2ix]; \ 483 } else if (alloc && lock) { \ 484 lnkp = kmem_zalloc(SFMMU_L2_HMERLINKS_SIZE, KM_SLEEP); \ 485 _hatlockp = sfmmu_hat_enter(sfmmup); \ 486 if ((sfmmup)->sfmmu_hmeregion_links[_l1ix] != NULL) { \ 487 sfmmu_hat_exit(_hatlockp); \ 488 kmem_free(lnkp, SFMMU_L2_HMERLINKS_SIZE); \ 489 lnkp = (sfmmup)->sfmmu_hmeregion_links[_l1ix]; \ 490 ASSERT(lnkp != NULL); \ 491 } else { \ 492 (sfmmup)->sfmmu_hmeregion_links[_l1ix] = lnkp; \ 493 sfmmu_hat_exit(_hatlockp); \ 494 } \ 495 lnkp = &lnkp[_l2ix]; \ 496 } else if (alloc) { \ 497 lnkp = kmem_zalloc(SFMMU_L2_HMERLINKS_SIZE, KM_SLEEP); \ 498 ASSERT((sfmmup)->sfmmu_hmeregion_links[_l1ix] == NULL); \ 499 (sfmmup)->sfmmu_hmeregion_links[_l1ix] = lnkp; \ 500 lnkp = &lnkp[_l2ix]; \ 501 } \ 502 } 503 504 /* 505 * Per-MMU context domain kstats. 506 * 507 * TSB Miss Exceptions 508 * Number of times a TSB miss exception is handled in an MMU. See 509 * sfmmu_tsbmiss_exception() for more details. 510 * TSB Raise Exception 511 * Number of times the CPUs within an MMU are cross-called 512 * to invalidate either a specific process context (when the process 513 * switches MMU contexts) or the context of any process that is 514 * running on those CPUs (as part of the MMU context wrap-around). 515 * Wrap Around 516 * The number of times a wrap-around of MMU context happens. 517 */ 518 typedef enum mmu_ctx_stat_types { 519 MMU_CTX_TSB_EXCEPTIONS, /* TSB miss exceptions handled */ 520 MMU_CTX_TSB_RAISE_EXCEPTION, /* ctx invalidation cross calls */ 521 MMU_CTX_WRAP_AROUND, /* wraparounds */ 522 MMU_CTX_NUM_STATS 523 } mmu_ctx_stat_t; 524 525 /* 526 * Per-MMU context domain structure. This is instantiated the first time a CPU 527 * belonging to the MMU context domain is configured into the system, at boot 528 * time or at DR time. 529 * 530 * mmu_gnum 531 * The current generation number for the context IDs on this MMU context 532 * domain. It is protected by mmu_lock. 533 * mmu_cnum 534 * The current cnum to be allocated on this MMU context domain. It 535 * is protected via CAS. 536 * mmu_nctxs 537 * The max number of context IDs supported on every CPU in this 538 * MMU context domain. It is 8K except for Rock where it is 64K. 539 * This is needed here in case the system supports mixed type of 540 * processors/MMUs. It also helps to make ctx switch code access 541 * fewer cache lines i.e. no need to retrieve it from some global nctxs. 542 * mmu_lock 543 * The mutex spin lock used to serialize context ID wrap around 544 * mmu_idx 545 * The index for this MMU context domain structure in the global array 546 * mmu_ctxdoms. 547 * mmu_ncpus 548 * The actual number of CPUs that have been configured in this 549 * MMU context domain. This also acts as a reference count for the 550 * structure. When the last CPU in an MMU context domain is unconfigured, 551 * the structure is freed. It is protected by mmu_lock. 552 * mmu_cpuset 553 * The CPU set of configured CPUs for this MMU context domain. Used 554 * to cross-call all the CPUs in the MMU context domain to invalidate 555 * context IDs during a wraparound operation. It is protected by mmu_lock. 556 */ 557 558 typedef struct mmu_ctx { 559 uint64_t mmu_gnum; 560 uint_t mmu_cnum; 561 uint_t mmu_nctxs; 562 kmutex_t mmu_lock; 563 uint_t mmu_idx; 564 uint_t mmu_ncpus; 565 cpuset_t mmu_cpuset; 566 kstat_t *mmu_kstat; 567 kstat_named_t mmu_kstat_data[MMU_CTX_NUM_STATS]; 568 } mmu_ctx_t; 569 570 #define mmu_tsb_exceptions \ 571 mmu_kstat_data[MMU_CTX_TSB_EXCEPTIONS].value.ui64 572 #define mmu_tsb_raise_exception \ 573 mmu_kstat_data[MMU_CTX_TSB_RAISE_EXCEPTION].value.ui64 574 #define mmu_wrap_around \ 575 mmu_kstat_data[MMU_CTX_WRAP_AROUND].value.ui64 576 577 extern uint_t max_mmu_ctxdoms; 578 extern mmu_ctx_t **mmu_ctxs_tbl; 579 580 extern void sfmmu_cpu_init(cpu_t *); 581 extern void sfmmu_cpu_cleanup(cpu_t *); 582 583 /* 584 * The following structure is used to get MMU context domain information for 585 * a CPU from the platform. 586 * 587 * mmu_idx 588 * The MMU context domain index within the global array mmu_ctxs 589 * mmu_nctxs 590 * The number of context IDs supported in the MMU context domain 591 * (64K for Rock) 592 */ 593 typedef struct mmu_ctx_info { 594 uint_t mmu_idx; 595 uint_t mmu_nctxs; 596 } mmu_ctx_info_t; 597 598 #pragma weak plat_cpuid_to_mmu_ctx_info 599 600 extern void plat_cpuid_to_mmu_ctx_info(processorid_t, mmu_ctx_info_t *); 601 602 /* 603 * Each address space has an array of sfmmu_ctx_t structures, one structure 604 * per MMU context domain. 605 * 606 * cnum 607 * The context ID allocated for an address space on an MMU context domain 608 * gnum 609 * The generation number for the context ID in the MMU context domain. 610 * 611 * This structure needs to be a power-of-two in size. 612 */ 613 typedef struct sfmmu_ctx { 614 uint64_t gnum:48; 615 uint64_t cnum:16; 616 } sfmmu_ctx_t; 617 618 619 /* 620 * The platform dependent hat structure. 621 * tte counts should be protected by cas. 622 * cpuset is protected by cas. 623 * 624 * ttecnt accounting for mappings which do not use shared hme is carried out 625 * during pagefault handling. In the shared hme case, only the first process 626 * to access a mapping generates a pagefault, subsequent processes simply 627 * find the shared hme entry during trap handling and therefore there is no 628 * corresponding event to initiate ttecnt accounting. Currently, as shared 629 * hmes are only used for text segments, when joining a region we assume the 630 * worst case and add the the number of ttes required to map the entire region 631 * to the ttecnt corresponding to the region pagesize. However, if the region 632 * has a 4M pagesize, and memory is low, the allocation of 4M pages may fail 633 * then 8K pages will be allocated instead and the first TSB which stores 8K 634 * mappings will potentially be undersized. To compensate for the potential 635 * underaccounting in this case we always add 1/4 of the region size to the 8K 636 * ttecnt. 637 * 638 * Note that sfmmu_xhat_provider MUST be the first element. 639 */ 640 641 struct hat { 642 void *sfmmu_xhat_provider; /* NULL for CPU hat */ 643 cpuset_t sfmmu_cpusran; /* cpu bit mask for efficient xcalls */ 644 struct as *sfmmu_as; /* as this hat provides mapping for */ 645 /* per pgsz private ttecnt + shme rgns ttecnt for rgns not in SCD */ 646 ulong_t sfmmu_ttecnt[MMU_PAGE_SIZES]; 647 /* shme rgns ttecnt for rgns in SCD */ 648 ulong_t sfmmu_scdrttecnt[MMU_PAGE_SIZES]; 649 /* est. ism ttes that are NOT in a SCD */ 650 ulong_t sfmmu_ismttecnt[MMU_PAGE_SIZES]; 651 /* ttecnt for isms that are in a SCD */ 652 ulong_t sfmmu_scdismttecnt[MMU_PAGE_SIZES]; 653 /* inflate tsb0 to allow for large page alloc failure in region */ 654 ulong_t sfmmu_tsb0_4minflcnt; 655 union _h_un { 656 ism_blk_t *sfmmu_iblkp; /* maps to ismhat(s) */ 657 ism_ment_t *sfmmu_imentp; /* ism hat's mapping list */ 658 } h_un; 659 uint_t sfmmu_free:1; /* hat to be freed - set on as_free */ 660 uint_t sfmmu_ismhat:1; /* hat is dummy ism hatid */ 661 uint_t sfmmu_scdhat:1; /* hat is dummy scd hatid */ 662 uchar_t sfmmu_rmstat; /* refmod stats refcnt */ 663 ushort_t sfmmu_clrstart; /* start color bin for page coloring */ 664 ushort_t sfmmu_clrbin; /* per as phys page coloring bin */ 665 ushort_t sfmmu_flags; /* flags */ 666 uchar_t sfmmu_tteflags; /* pgsz flags */ 667 uchar_t sfmmu_rtteflags; /* pgsz flags for SRD hmes */ 668 struct tsb_info *sfmmu_tsb; /* list of per as tsbs */ 669 uint64_t sfmmu_ismblkpa; /* pa of sfmmu_iblkp, or -1 */ 670 lock_t sfmmu_ctx_lock; /* sync ctx alloc and invalidation */ 671 kcondvar_t sfmmu_tsb_cv; /* signals TSB swapin or relocation */ 672 uchar_t sfmmu_cext; /* context page size encoding */ 673 uint8_t sfmmu_pgsz[MMU_PAGE_SIZES]; /* ranking for MMU */ 674 sf_srd_t *sfmmu_srdp; 675 sf_scd_t *sfmmu_scdp; /* scd this address space belongs to */ 676 sf_region_map_t sfmmu_region_map; 677 sf_rgn_link_t *sfmmu_hmeregion_links[SFMMU_L1_HMERLINKS]; 678 sf_rgn_link_t sfmmu_scd_link; /* link to scd or pending queue */ 679 #ifdef sun4v 680 struct hv_tsb_block sfmmu_hvblock; 681 #endif 682 /* 683 * sfmmu_ctxs is a variable length array of max_mmu_ctxdoms # of 684 * elements. max_mmu_ctxdoms is determined at run-time. 685 * sfmmu_ctxs[1] is just the fist element of an array, it always 686 * has to be the last field to ensure that the memory allocated 687 * for sfmmu_ctxs is consecutive with the memory of the rest of 688 * the hat data structure. 689 */ 690 sfmmu_ctx_t sfmmu_ctxs[1]; 691 692 }; 693 694 #define sfmmu_iblk h_un.sfmmu_iblkp 695 #define sfmmu_iment h_un.sfmmu_imentp 696 697 #define sfmmu_hmeregion_map sfmmu_region_map.h_rmap_s.hmeregion_map 698 #define sfmmu_ismregion_map sfmmu_region_map.h_rmap_s.ismregion_map 699 700 #define SF_RGNMAP_ISNULL(sfmmup) \ 701 (sfrgnmap_isnull(&(sfmmup)->sfmmu_region_map)) 702 #define SF_HMERGNMAP_ISNULL(sfmmup) \ 703 (sfhmergnmap_isnull(&(sfmmup)->sfmmu_hmeregion_map)) 704 705 struct sf_scd { 706 sfmmu_t *scd_sfmmup; /* shared context hat */ 707 /* per pgsz ttecnt for shme rgns in SCD */ 708 ulong_t scd_rttecnt[MMU_PAGE_SIZES]; 709 uint_t scd_refcnt; /* address spaces attached to scd */ 710 sf_region_map_t scd_region_map; /* bit mask of attached segments */ 711 sf_scd_t *scd_next; /* link pointers for srd_scd list */ 712 sf_scd_t *scd_prev; 713 sfmmu_t *scd_sf_list; /* list of doubly linked hat structs */ 714 kmutex_t scd_mutex; 715 /* 716 * Link used to add an scd to the sfmmu_iment list. 717 */ 718 ism_ment_t scd_ism_links[SFMMU_MAX_ISM_REGIONS]; 719 }; 720 721 #define scd_hmeregion_map scd_region_map.h_rmap_s.hmeregion_map 722 #define scd_ismregion_map scd_region_map.h_rmap_s.ismregion_map 723 724 extern int disable_shctx; 725 extern int shctx_on; 726 727 /* 728 * bit mask for managing vac conflicts on large pages. 729 * bit 1 is for uncache flag. 730 * bits 2 through min(num of cache colors + 1,31) are 731 * for cache colors that have already been flushed. 732 */ 733 #ifdef VAC 734 #define CACHE_NUM_COLOR (shm_alignment >> MMU_PAGESHIFT) 735 #else 736 #define CACHE_NUM_COLOR 1 737 #endif 738 739 #define CACHE_VCOLOR_MASK(vcolor) (2 << (vcolor & (CACHE_NUM_COLOR - 1))) 740 741 #define CacheColor_IsFlushed(flag, vcolor) \ 742 ((flag) & CACHE_VCOLOR_MASK(vcolor)) 743 744 #define CacheColor_SetFlushed(flag, vcolor) \ 745 ((flag) |= CACHE_VCOLOR_MASK(vcolor)) 746 /* 747 * Flags passed to sfmmu_page_cache to flush page from vac or not. 748 */ 749 #define CACHE_FLUSH 0 750 #define CACHE_NO_FLUSH 1 751 752 /* 753 * Flags passed to sfmmu_tlbcache_demap 754 */ 755 #define FLUSH_NECESSARY_CPUS 0 756 #define FLUSH_ALL_CPUS 1 757 758 #ifdef DEBUG 759 /* 760 * For debugging purpose only. Maybe removed later. 761 */ 762 struct ctx_trace { 763 sfmmu_t *sc_sfmmu_stolen; 764 sfmmu_t *sc_sfmmu_stealing; 765 clock_t sc_time; 766 ushort_t sc_type; 767 ushort_t sc_cnum; 768 }; 769 #define CTX_TRC_STEAL 0x1 770 #define CTX_TRC_FREE 0x0 771 #define TRSIZE 0x400 772 #define NEXT_CTXTR(ptr) (((ptr) >= ctx_trace_last) ? \ 773 ctx_trace_first : ((ptr) + 1)) 774 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) \ 775 mutex_enter(mutex); \ 776 (ptr)->sc_sfmmu_stolen = (stolen_sfmmu); \ 777 (ptr)->sc_sfmmu_stealing = (stealing_sfmmu); \ 778 (ptr)->sc_cnum = (cnum); \ 779 (ptr)->sc_type = (type); \ 780 (ptr)->sc_time = lbolt; \ 781 (ptr) = NEXT_CTXTR(ptr); \ 782 num_ctx_stolen += (type); \ 783 mutex_exit(mutex); 784 #else 785 786 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) 787 788 #endif /* DEBUG */ 789 790 #endif /* !_ASM */ 791 792 /* 793 * Macros for sfmmup->sfmmu_flags access. The macros that change the flags 794 * ASSERT() that we're holding the HAT lock before changing the flags; 795 * however callers that read the flags may do so without acquiring the lock 796 * in a fast path, and then recheck the flag after acquiring the lock in 797 * a slow path. 798 */ 799 #define SFMMU_FLAGS_ISSET(sfmmup, flags) \ 800 (((sfmmup)->sfmmu_flags & (flags)) == (flags)) 801 802 #define SFMMU_FLAGS_CLEAR(sfmmup, flags) \ 803 (ASSERT(sfmmu_hat_lock_held((sfmmup))), \ 804 (sfmmup)->sfmmu_flags &= ~(flags)) 805 806 #define SFMMU_FLAGS_SET(sfmmup, flags) \ 807 (ASSERT(sfmmu_hat_lock_held((sfmmup))), \ 808 (sfmmup)->sfmmu_flags |= (flags)) 809 810 #define SFMMU_TTEFLAGS_ISSET(sfmmup, flags) \ 811 ((((sfmmup)->sfmmu_tteflags | (sfmmup)->sfmmu_rtteflags) & (flags)) == \ 812 (flags)) 813 814 815 /* 816 * sfmmu tte HAT flags, must fit in 8 bits 817 */ 818 #define HAT_CHKCTX1_FLAG 0x1 819 #define HAT_64K_FLAG (0x1 << TTE64K) 820 #define HAT_512K_FLAG (0x1 << TTE512K) 821 #define HAT_4M_FLAG (0x1 << TTE4M) 822 #define HAT_32M_FLAG (0x1 << TTE32M) 823 #define HAT_256M_FLAG (0x1 << TTE256M) 824 825 /* 826 * sfmmu HAT flags, 16 bits at the moment. 827 */ 828 #define HAT_4MTEXT_FLAG 0x01 829 #define HAT_32M_ISM 0x02 830 #define HAT_256M_ISM 0x04 831 #define HAT_SWAPPED 0x08 /* swapped out */ 832 #define HAT_SWAPIN 0x10 /* swapping in */ 833 #define HAT_BUSY 0x20 /* replacing TSB(s) */ 834 #define HAT_ISMBUSY 0x40 /* adding/removing/traversing ISM maps */ 835 836 #define HAT_CTX1_FLAG 0x100 /* ISM imap hatflag for ctx1 */ 837 #define HAT_JOIN_SCD 0x200 /* region is joining scd */ 838 #define HAT_ALLCTX_INVALID 0x400 /* all per-MMU ctxs are invalidated */ 839 840 #define SFMMU_LGPGS_INUSE(sfmmup) \ 841 (((sfmmup)->sfmmu_tteflags | (sfmmup)->sfmmu_rtteflags) || \ 842 ((sfmmup)->sfmmu_iblk != NULL)) 843 844 /* 845 * Starting with context 0, the first NUM_LOCKED_CTXS contexts 846 * are locked so that sfmmu_getctx can't steal any of these 847 * contexts. At the time this software was being developed, the 848 * only context that needs to be locked is context 0 (the kernel 849 * context), and context 1 (reserved for stolen context). So this constant 850 * was originally defined to be 2. 851 * 852 * For sun4v only, USER_CONTEXT_TYPE represents any user context. Many 853 * routines only care whether the context is kernel, invalid or user. 854 */ 855 856 #define NUM_LOCKED_CTXS 2 857 #define INVALID_CONTEXT 1 858 859 #ifdef sun4v 860 #define USER_CONTEXT_TYPE NUM_LOCKED_CTXS 861 #endif 862 863 #ifndef _ASM 864 865 /* 866 * Kernel page relocation stuff. 867 */ 868 struct sfmmu_callback { 869 int key; 870 int (*prehandler)(caddr_t, uint_t, uint_t, void *); 871 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t); 872 int (*errhandler)(caddr_t, uint_t, uint_t, void *); 873 int capture_cpus; 874 }; 875 876 extern int sfmmu_max_cb_id; 877 extern struct sfmmu_callback *sfmmu_cb_table; 878 879 extern int hat_kpr_enabled; 880 881 struct pa_hment; 882 883 /* 884 * RFE: With multihat gone we gain back an int. We could use this to 885 * keep ref bits on a per cpu basis to eliminate xcalls. 886 */ 887 struct sf_hment { 888 tte_t hme_tte; /* tte for this hment */ 889 890 union { 891 struct page *page; /* what page this maps */ 892 struct pa_hment *data; /* pa_hment */ 893 } sf_hment_un; 894 895 struct sf_hment *hme_next; /* next hment */ 896 struct sf_hment *hme_prev; /* prev hment */ 897 }; 898 899 struct pa_hment { 900 caddr_t addr; /* va */ 901 uint_t len; /* bytes */ 902 ushort_t flags; /* internal flags */ 903 ushort_t refcnt; /* reference count */ 904 id_t cb_id; /* callback id, table index */ 905 void *pvt; /* handler's private data */ 906 struct sf_hment sfment; /* corresponding dummy sf_hment */ 907 }; 908 909 #define hme_page sf_hment_un.page 910 #define hme_data sf_hment_un.data 911 #define hme_size(sfhmep) ((int)(TTE_CSZ(&(sfhmep)->hme_tte))) 912 #define PAHME_SZ (sizeof (struct pa_hment)) 913 #define SFHME_SZ (sizeof (struct sf_hment)) 914 915 #define IS_PAHME(hme) ((hme)->hme_tte.ll == 0) 916 917 /* 918 * hmeblk_tag structure 919 * structure used to obtain a match on a hme_blk. Currently consists of 920 * the address of the sfmmu struct (or hatid), the base page address of the 921 * hme_blk, and the rehash count. The rehash count is actually only 2 bits 922 * and has the following meaning: 923 * 1 = 8k or 64k hash sequence. 924 * 2 = 512k hash sequence. 925 * 3 = 4M hash sequence. 926 * We require this count because we don't want to get a false hit on a 512K or 927 * 4M rehash with a base address corresponding to a 8k or 64k hmeblk. 928 * Note: The ordering and size of the hmeblk_tag members are implictly known 929 * by the tsb miss handlers written in assembly. Do not change this structure 930 * without checking those routines. See HTAG_SFMMUPSZ define. 931 */ 932 933 /* 934 * In private hmeblks hblk_rid field must be SFMMU_INVALID_RID. 935 */ 936 typedef union { 937 struct { 938 uint64_t hblk_basepg: 51, /* hme_blk base pg # */ 939 hblk_rehash: 3, /* rehash number */ 940 hblk_rid: 10; /* hme_blk region id */ 941 void *hblk_id; 942 } hblk_tag_un; 943 uint64_t htag_tag[2]; 944 } hmeblk_tag; 945 946 #define htag_id hblk_tag_un.hblk_id 947 #define htag_bspage hblk_tag_un.hblk_basepg 948 #define htag_rehash hblk_tag_un.hblk_rehash 949 #define htag_rid hblk_tag_un.hblk_rid 950 951 #endif /* !_ASM */ 952 953 #define HTAG_REHASH_SHIFT 10 954 #define HTAG_MAX_RID (((0x1 << HTAG_REHASH_SHIFT) - 1)) 955 #define HTAG_RID_MASK HTAG_MAX_RID 956 957 /* used for tagging all per sfmmu (i.e. non SRD) private hmeblks */ 958 #define SFMMU_INVALID_SHMERID HTAG_MAX_RID 959 960 #if SFMMU_INVALID_SHMERID < SFMMU_MAX_HME_REGIONS 961 #error SFMMU_INVALID_SHMERID < SFMMU_MAX_HME_REGIONS 962 #endif 963 964 #define SFMMU_IS_SHMERID_VALID(rid) ((rid) != SFMMU_INVALID_SHMERID) 965 966 /* ISM regions */ 967 #define SFMMU_INVALID_ISMRID 0xff 968 969 #if SFMMU_INVALID_ISMRID < SFMMU_MAX_ISM_REGIONS 970 #error SFMMU_INVALID_ISMRID < SFMMU_MAX_ISM_REGIONS 971 #endif 972 973 #define SFMMU_IS_ISMRID_VALID(rid) ((rid) != SFMMU_INVALID_ISMRID) 974 975 976 #define HTAGS_EQ(tag1, tag2) (((tag1.htag_tag[0] ^ tag2.htag_tag[0]) | \ 977 (tag1.htag_tag[1] ^ tag2.htag_tag[1])) == 0) 978 979 /* 980 * this macro must only be used for comparing tags in shared hmeblks. 981 */ 982 #define HTAGS_EQ_SHME(hmetag, tag, hrmap) \ 983 (((hmetag).htag_rid != SFMMU_INVALID_SHMERID) && \ 984 (((((hmetag).htag_tag[0] ^ (tag).htag_tag[0]) & \ 985 ~HTAG_RID_MASK) | \ 986 ((hmetag).htag_tag[1] ^ (tag).htag_tag[1])) == 0) && \ 987 SF_RGNMAP_TEST(hrmap, hmetag.htag_rid)) 988 989 #define HME_REHASH(sfmmup) \ 990 ((sfmmup)->sfmmu_ttecnt[TTE512K] != 0 || \ 991 (sfmmup)->sfmmu_ttecnt[TTE4M] != 0 || \ 992 (sfmmup)->sfmmu_ttecnt[TTE32M] != 0 || \ 993 (sfmmup)->sfmmu_ttecnt[TTE256M] != 0) 994 995 #define NHMENTS 8 /* # of hments in an 8k hme_blk */ 996 /* needs to be multiple of 2 */ 997 998 #ifndef _ASM 999 1000 #ifdef HBLK_TRACE 1001 1002 #define HBLK_LOCK 1 1003 #define HBLK_UNLOCK 0 1004 #define HBLK_STACK_DEPTH 6 1005 #define HBLK_AUDIT_CACHE_SIZE 16 1006 #define HBLK_LOCK_PATTERN 0xaaaaaaaa 1007 #define HBLK_UNLOCK_PATTERN 0xbbbbbbbb 1008 1009 struct hblk_lockcnt_audit { 1010 int flag; /* lock or unlock */ 1011 kthread_id_t thread; 1012 int depth; 1013 pc_t stack[HBLK_STACK_DEPTH]; 1014 }; 1015 1016 #endif /* HBLK_TRACE */ 1017 1018 1019 /* 1020 * Hment block structure. 1021 * The hme_blk is the node data structure which the hash structure 1022 * mantains. An hme_blk can have 2 different sizes depending on the 1023 * number of hments it implicitly contains. When dealing with 64K, 512K, 1024 * or 4M hments there is one hment per hme_blk. When dealing with 1025 * 8k hments we allocate an hme_blk plus an additional 7 hments to 1026 * give us a total of 8 (NHMENTS) hments that can be referenced through a 1027 * hme_blk. 1028 * 1029 * The hmeblk structure contains 2 tte reference counters used to determine if 1030 * it is ok to free up the hmeblk. Both counters have to be zero in order 1031 * to be able to free up hmeblk. They are protected by cas. 1032 * hblk_hmecnt is the number of hments present on pp mapping lists. 1033 * hblk_vcnt reflects number of valid ttes in hmeblk. 1034 * 1035 * The hmeblk now also has per tte lock cnts. This is required because 1036 * the counts can be high and there are not enough bits in the tte. When 1037 * physio is fixed to not lock the translations we should be able to move 1038 * the lock cnt back to the tte. See bug id 1198554. 1039 * 1040 * Note that xhat_hme_blk's layout follows this structure: hme_blk_misc 1041 * and sf_hment are at the same offsets in both structures. Whenever 1042 * hme_blk is changed, xhat_hme_blk may need to be updated as well. 1043 */ 1044 1045 struct hme_blk_misc { 1046 uint_t notused:25; 1047 uint_t shared_bit:1; /* set for SRD shared hmeblk */ 1048 uint_t xhat_bit:1; /* set for an xhat hme_blk */ 1049 uint_t shadow_bit:1; /* set for a shadow hme_blk */ 1050 uint_t nucleus_bit:1; /* set for a nucleus hme_blk */ 1051 uint_t ttesize:3; /* contains ttesz of hmeblk */ 1052 }; 1053 1054 struct hme_blk { 1055 uint64_t hblk_nextpa; /* physical address for hash list */ 1056 1057 hmeblk_tag hblk_tag; /* tag used to obtain an hmeblk match */ 1058 1059 struct hme_blk *hblk_next; /* on free list or on hash list */ 1060 /* protected by hash lock */ 1061 1062 struct hme_blk *hblk_shadow; /* pts to shadow hblk */ 1063 /* protected by hash lock */ 1064 uint_t hblk_span; /* span of memory hmeblk maps */ 1065 1066 struct hme_blk_misc hblk_misc; 1067 1068 union { 1069 struct { 1070 ushort_t hblk_hmecount; /* hment on mlists counter */ 1071 ushort_t hblk_validcnt; /* valid tte reference count */ 1072 } hblk_counts; 1073 uint_t hblk_shadow_mask; 1074 } hblk_un; 1075 1076 uint_t hblk_lckcnt; 1077 1078 #ifdef HBLK_TRACE 1079 kmutex_t hblk_audit_lock; /* lock to protect index */ 1080 uint_t hblk_audit_index; /* index into audit_cache */ 1081 struct hblk_lockcnt_audit hblk_audit_cache[HBLK_AUDIT_CACHE_SIZE]; 1082 #endif /* HBLK_AUDIT */ 1083 1084 struct sf_hment hblk_hme[1]; /* hment array */ 1085 }; 1086 1087 #define hblk_shared hblk_misc.shared_bit 1088 #define hblk_xhat_bit hblk_misc.xhat_bit 1089 #define hblk_shw_bit hblk_misc.shadow_bit 1090 #define hblk_nuc_bit hblk_misc.nucleus_bit 1091 #define hblk_ttesz hblk_misc.ttesize 1092 #define hblk_hmecnt hblk_un.hblk_counts.hblk_hmecount 1093 #define hblk_vcnt hblk_un.hblk_counts.hblk_validcnt 1094 #define hblk_shw_mask hblk_un.hblk_shadow_mask 1095 1096 #define MAX_HBLK_LCKCNT 0xFFFFFFFF 1097 #define HMEBLK_ALIGN 0x8 /* hmeblk has to be double aligned */ 1098 1099 #ifdef HBLK_TRACE 1100 1101 #define HBLK_STACK_TRACE(hmeblkp, lock) \ 1102 { \ 1103 int flag = lock; /* to pacify lint */ \ 1104 int audit_index; \ 1105 \ 1106 mutex_enter(&hmeblkp->hblk_audit_lock); \ 1107 audit_index = hmeblkp->hblk_audit_index; \ 1108 hmeblkp->hblk_audit_index = ((hmeblkp->hblk_audit_index + 1) & \ 1109 (HBLK_AUDIT_CACHE_SIZE - 1)); \ 1110 mutex_exit(&hmeblkp->hblk_audit_lock); \ 1111 \ 1112 if (flag) \ 1113 hmeblkp->hblk_audit_cache[audit_index].flag = \ 1114 HBLK_LOCK_PATTERN; \ 1115 else \ 1116 hmeblkp->hblk_audit_cache[audit_index].flag = \ 1117 HBLK_UNLOCK_PATTERN; \ 1118 \ 1119 hmeblkp->hblk_audit_cache[audit_index].thread = curthread; \ 1120 hmeblkp->hblk_audit_cache[audit_index].depth = \ 1121 getpcstack(hmeblkp->hblk_audit_cache[audit_index].stack, \ 1122 HBLK_STACK_DEPTH); \ 1123 } 1124 1125 #else 1126 1127 #define HBLK_STACK_TRACE(hmeblkp, lock) 1128 1129 #endif /* HBLK_TRACE */ 1130 1131 #define HMEHASH_FACTOR 16 /* used to calc # of buckets in hme hash */ 1132 1133 /* 1134 * A maximum number of user hmeblks is defined in order to place an upper 1135 * limit on how much nucleus memory is required and to avoid overflowing the 1136 * tsbmiss uhashsz and khashsz data areas. The number below corresponds to 1137 * the number of buckets required, for an average hash chain length of 4 on 1138 * a 16TB machine. 1139 */ 1140 1141 #define MAX_UHME_BUCKETS (0x1 << 30) 1142 #define MAX_KHME_BUCKETS (0x1 << 30) 1143 1144 /* 1145 * The minimum number of kernel hash buckets. 1146 */ 1147 #define MIN_KHME_BUCKETS 0x800 1148 1149 /* 1150 * The number of hash buckets must be a power of 2. If the initial calculated 1151 * value is less than USER_BUCKETS_THRESHOLD we round up to the next greater 1152 * power of 2, otherwise we round down to avoid huge over allocations. 1153 */ 1154 #define USER_BUCKETS_THRESHOLD (1<<22) 1155 1156 #define MAX_NUCUHME_BUCKETS 0x4000 1157 #define MAX_NUCKHME_BUCKETS 0x2000 1158 1159 /* 1160 * There are 2 locks in the hmehash bucket. The hmehash_mutex is 1161 * a regular mutex used to make sure operations on a hash link are only 1162 * done by one thread. Any operation which comes into the hat with 1163 * a <vaddr, as> will grab the hmehash_mutex. Normally one would expect 1164 * the tsb miss handlers to grab the hash lock to make sure the hash list 1165 * is consistent while we traverse it. Unfortunately this can lead to 1166 * deadlocks or recursive mutex enters since it is possible for 1167 * someone holding the lock to take a tlb/tsb miss. 1168 * To solve this problem we have added the hmehash_listlock. This lock 1169 * is only grabbed by the tsb miss handlers, vatopfn, and while 1170 * adding/removing a hmeblk from the hash list. The code is written to 1171 * guarantee we won't take a tlb miss while holding this lock. 1172 */ 1173 struct hmehash_bucket { 1174 kmutex_t hmehash_mutex; 1175 uint64_t hmeh_nextpa; /* physical address for hash list */ 1176 struct hme_blk *hmeblkp; 1177 uint_t hmeh_listlock; 1178 }; 1179 1180 #endif /* !_ASM */ 1181 1182 #define SFMMU_PGCNT_MASK 0x3f 1183 #define SFMMU_PGCNT_SHIFT 6 1184 #define INVALID_MMU_ID -1 1185 #define SFMMU_MMU_GNUM_RSHIFT 16 1186 #define SFMMU_MMU_CNUM_LSHIFT (64 - SFMMU_MMU_GNUM_RSHIFT) 1187 #define MAX_SFMMU_CTX_VAL ((1 << 16) - 1) /* for sanity check */ 1188 #define MAX_SFMMU_GNUM_VAL ((0x1UL << 48) - 1) 1189 1190 /* 1191 * The tsb miss handlers written in assembly know that sfmmup 1192 * is a 64 bit ptr. 1193 * 1194 * The bspage and re-hash part is 64 bits, with the sfmmup being another 64 1195 * bits. 1196 */ 1197 #define HTAG_SFMMUPSZ 0 /* Not really used for LP64 */ 1198 #define HTAG_BSPAGE_SHIFT 13 1199 1200 /* 1201 * Assembly routines need to be able to get to ttesz 1202 */ 1203 #define HBLK_SZMASK 0x7 1204 1205 #ifndef _ASM 1206 1207 /* 1208 * Returns the number of bytes that an hmeblk spans given its tte size 1209 */ 1210 #define get_hblk_span(hmeblkp) ((hmeblkp)->hblk_span) 1211 #define get_hblk_ttesz(hmeblkp) ((hmeblkp)->hblk_ttesz) 1212 #define get_hblk_cache(hmeblkp) (((hmeblkp)->hblk_ttesz == TTE8K) ? \ 1213 sfmmu8_cache : sfmmu1_cache) 1214 #define HMEBLK_SPAN(ttesz) \ 1215 ((ttesz == TTE8K)? (TTEBYTES(ttesz) * NHMENTS) : TTEBYTES(ttesz)) 1216 1217 #define set_hblk_sz(hmeblkp, ttesz) \ 1218 (hmeblkp)->hblk_ttesz = (ttesz); \ 1219 (hmeblkp)->hblk_span = HMEBLK_SPAN(ttesz) 1220 1221 #define get_hblk_base(hmeblkp) \ 1222 ((uintptr_t)(hmeblkp)->hblk_tag.htag_bspage << MMU_PAGESHIFT) 1223 1224 #define get_hblk_endaddr(hmeblkp) \ 1225 ((caddr_t)(get_hblk_base(hmeblkp) + get_hblk_span(hmeblkp))) 1226 1227 #define in_hblk_range(hmeblkp, vaddr) \ 1228 (((uintptr_t)(vaddr) >= get_hblk_base(hmeblkp)) && \ 1229 ((uintptr_t)(vaddr) < (get_hblk_base(hmeblkp) + \ 1230 get_hblk_span(hmeblkp)))) 1231 1232 #define tte_to_vaddr(hmeblkp, tte) ((caddr_t)(get_hblk_base(hmeblkp) \ 1233 + (TTEBYTES(TTE_CSZ(&tte)) * (tte).tte_hmenum))) 1234 1235 #define tte_to_evaddr(hmeblkp, ttep) ((caddr_t)(get_hblk_base(hmeblkp) \ 1236 + (TTEBYTES(TTE_CSZ(ttep)) * ((ttep)->tte_hmenum + 1)))) 1237 1238 #define vaddr_to_vshift(hblktag, vaddr, shwsz) \ 1239 ((((uintptr_t)(vaddr) >> MMU_PAGESHIFT) - (hblktag.htag_bspage)) >>\ 1240 TTE_BSZS_SHIFT((shwsz) - 1)) 1241 1242 #define HME8BLK_SZ (sizeof (struct hme_blk) + \ 1243 (NHMENTS - 1) * sizeof (struct sf_hment)) 1244 #define HME1BLK_SZ (sizeof (struct hme_blk)) 1245 #define H1MIN (2 + MAX_BIGKTSB_TTES) /* nucleus text+data, ktsb */ 1246 1247 /* 1248 * Hme_blk hash structure 1249 * Active mappings are kept in a hash structure of hme_blks. The hash 1250 * function is based on (ctx, vaddr) The size of the hash table size is a 1251 * power of 2 such that the average hash chain lenth is HMENT_HASHAVELEN. 1252 * The hash actually consists of 2 separate hashes. One hash is for the user 1253 * address space and the other hash is for the kernel address space. 1254 * The number of buckets are calculated at boot time and stored in the global 1255 * variables "uhmehash_num" and "khmehash_num". By making the hash table size 1256 * a power of 2 we can use a simply & function to derive an index instead of 1257 * a divide. 1258 * 1259 * HME_HASH_FUNCTION(hatid, vaddr, shift) returns a pointer to a hme_hash 1260 * bucket. 1261 * An hme hash bucket contains a pointer to an hme_blk and the mutex that 1262 * protects the link list. 1263 * Spitfire supports 4 page sizes. 8k and 64K pages only need one hash. 1264 * 512K pages need 2 hashes and 4M pages need 3 hashes. 1265 * The 'shift' parameter controls how many bits the vaddr will be shifted in 1266 * the hash function. It is calculated in the HME_HASH_SHIFT(ttesz) function 1267 * and it varies depending on the page size as follows: 1268 * 8k pages: HBLK_RANGE_SHIFT 1269 * 64k pages: MMU_PAGESHIFT64K 1270 * 512K pages: MMU_PAGESHIFT512K 1271 * 4M pages: MMU_PAGESHIFT4M 1272 * An assembly version of the hash function exists in sfmmu_ktsb_miss(). All 1273 * changes should be reflected in both versions. This function and the TSB 1274 * miss handlers are the only places which know about the two hashes. 1275 * 1276 * HBLK_RANGE_SHIFT controls range of virtual addresses that will fall 1277 * into the same bucket for a particular process. It is currently set to 1278 * be equivalent to 64K range or one hme_blk. 1279 * 1280 * The hme_blks in the hash are protected by a per hash bucket mutex 1281 * known as SFMMU_HASH_LOCK. 1282 * You need to acquire this lock before traversing the hash bucket link 1283 * list, while adding/removing a hme_blk to the list, and while 1284 * modifying an hme_blk. A possible optimization is to replace these 1285 * mutexes by readers/writer lock but right now it is not clear whether 1286 * this is a win or not. 1287 * 1288 * The HME_HASH_TABLE_SEARCH will search the hash table for the 1289 * hme_blk that contains the hment that corresponds to the passed 1290 * ctx and vaddr. It assumed the SFMMU_HASH_LOCK is held. 1291 */ 1292 1293 #endif /* ! _ASM */ 1294 1295 #define KHATID ksfmmup 1296 #define UHMEHASH_SZ uhmehash_num 1297 #define KHMEHASH_SZ khmehash_num 1298 #define HMENT_HASHAVELEN 4 1299 #define HBLK_RANGE_SHIFT MMU_PAGESHIFT64K /* shift for HBLK_BS_MASK */ 1300 #define HBLK_MIN_TTESZ 1 1301 #define HBLK_MIN_BYTES MMU_PAGESIZE64K 1302 #define HBLK_MIN_SHIFT MMU_PAGESHIFT64K 1303 #define MAX_HASHCNT 5 1304 #define DEFAULT_MAX_HASHCNT 3 1305 1306 #ifndef _ASM 1307 1308 #define HASHADDR_MASK(hashno) TTE_PAGEMASK(hashno) 1309 1310 #define HME_HASH_SHIFT(ttesz) \ 1311 ((ttesz == TTE8K)? HBLK_RANGE_SHIFT : TTE_PAGE_SHIFT(ttesz)) 1312 1313 #define HME_HASH_ADDR(vaddr, hmeshift) \ 1314 ((caddr_t)(((uintptr_t)(vaddr) >> (hmeshift)) << (hmeshift))) 1315 1316 #define HME_HASH_BSPAGE(vaddr, hmeshift) \ 1317 (((uintptr_t)(vaddr) >> (hmeshift)) << ((hmeshift) - MMU_PAGESHIFT)) 1318 1319 #define HME_HASH_REHASH(ttesz) \ 1320 (((ttesz) < TTE512K)? 1 : (ttesz)) 1321 1322 #define HME_HASH_FUNCTION(hatid, vaddr, shift) \ 1323 ((((void *)hatid) != ((void *)KHATID)) ? \ 1324 (&uhme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \ 1325 UHMEHASH_SZ) ]): \ 1326 (&khme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \ 1327 KHMEHASH_SZ) ])) 1328 1329 /* 1330 * This macro will traverse a hmeblk hash link list looking for an hme_blk 1331 * that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp 1332 * will be set to NULL, otherwise it will point to the correct hme_blk. 1333 * This macro also cleans empty hblks. 1334 */ 1335 #define HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, \ 1336 pr_hblk, prevpa, listp) \ 1337 { \ 1338 struct hme_blk *nx_hblk; \ 1339 uint64_t nx_pa; \ 1340 \ 1341 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \ 1342 hblkp = hmebp->hmeblkp; \ 1343 hblkpa = hmebp->hmeh_nextpa; \ 1344 prevpa = 0; \ 1345 pr_hblk = NULL; \ 1346 while (hblkp) { \ 1347 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \ 1348 /* found hme_blk */ \ 1349 break; \ 1350 } \ 1351 nx_hblk = hblkp->hblk_next; \ 1352 nx_pa = hblkp->hblk_nextpa; \ 1353 if (!hblkp->hblk_vcnt && !hblkp->hblk_hmecnt) { \ 1354 sfmmu_hblk_hash_rm(hmebp, hblkp, prevpa, pr_hblk); \ 1355 sfmmu_hblk_free(hmebp, hblkp, hblkpa, listp); \ 1356 } else { \ 1357 pr_hblk = hblkp; \ 1358 prevpa = hblkpa; \ 1359 } \ 1360 hblkp = nx_hblk; \ 1361 hblkpa = nx_pa; \ 1362 } \ 1363 } 1364 1365 #define HME_HASH_SEARCH(hmebp, hblktag, hblkp, listp) \ 1366 { \ 1367 struct hme_blk *pr_hblk; \ 1368 uint64_t hblkpa, prevpa; \ 1369 \ 1370 HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, pr_hblk, \ 1371 prevpa, listp); \ 1372 } 1373 1374 /* 1375 * This macro will traverse a hmeblk hash link list looking for an hme_blk 1376 * that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp 1377 * will be set to NULL, otherwise it will point to the correct hme_blk. 1378 * It doesn't remove empty hblks. 1379 */ 1380 #define HME_HASH_FAST_SEARCH(hmebp, hblktag, hblkp) \ 1381 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \ 1382 for (hblkp = hmebp->hmeblkp; hblkp; \ 1383 hblkp = hblkp->hblk_next) { \ 1384 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \ 1385 /* found hme_blk */ \ 1386 break; \ 1387 } \ 1388 } 1389 1390 #define SFMMU_HASH_LOCK(hmebp) \ 1391 (mutex_enter(&hmebp->hmehash_mutex)) 1392 1393 #define SFMMU_HASH_UNLOCK(hmebp) \ 1394 (mutex_exit(&hmebp->hmehash_mutex)) 1395 1396 #define SFMMU_HASH_LOCK_TRYENTER(hmebp) \ 1397 (mutex_tryenter(&hmebp->hmehash_mutex)) 1398 1399 #define SFMMU_HASH_LOCK_ISHELD(hmebp) \ 1400 (mutex_owned(&hmebp->hmehash_mutex)) 1401 1402 #define SFMMU_XCALL_STATS(sfmmup) \ 1403 { \ 1404 if (sfmmup == ksfmmup) { \ 1405 SFMMU_STAT(sf_kernel_xcalls); \ 1406 } else { \ 1407 SFMMU_STAT(sf_user_xcalls); \ 1408 } \ 1409 } 1410 1411 #define astosfmmu(as) ((as)->a_hat) 1412 #define hblktosfmmu(hmeblkp) ((sfmmu_t *)(hmeblkp)->hblk_tag.htag_id) 1413 #define hblktosrd(hmeblkp) ((sf_srd_t *)(hmeblkp)->hblk_tag.htag_id) 1414 #define sfmmutoas(sfmmup) ((sfmmup)->sfmmu_as) 1415 1416 #define sfmmutohtagid(sfmmup, rid) \ 1417 (((rid) == SFMMU_INVALID_SHMERID) ? (void *)(sfmmup) : \ 1418 (void *)((sfmmup)->sfmmu_srdp)) 1419 1420 /* 1421 * We use the sfmmu data structure to keep the per as page coloring info. 1422 */ 1423 #define as_color_bin(as) (astosfmmu(as)->sfmmu_clrbin) 1424 #define as_color_start(as) (astosfmmu(as)->sfmmu_clrstart) 1425 1426 typedef struct { 1427 char h8[HME8BLK_SZ]; 1428 } hblk8_t; 1429 1430 typedef struct { 1431 char h1[HME1BLK_SZ]; 1432 } hblk1_t; 1433 1434 typedef struct { 1435 ulong_t index; 1436 ulong_t len; 1437 hblk8_t *list; 1438 } nucleus_hblk8_info_t; 1439 1440 typedef struct { 1441 ulong_t index; 1442 ulong_t len; 1443 hblk1_t *list; 1444 } nucleus_hblk1_info_t; 1445 1446 /* 1447 * This struct is used for accumlating information about a range 1448 * of pages that are unloading so that a single xcall can flush 1449 * the entire range from remote tlbs. A function that must demap 1450 * a range of virtual addresses declares one of these structures 1451 * and initializes using DEMP_RANGE_INIT(). It then passes a pointer to this 1452 * struct to the appropriate sfmmu_hblk_* level function which does 1453 * all the bookkeeping using the other macros. When the function has 1454 * finished the virtual address range, it needs to call DEMAP_RANGE_FLUSH() 1455 * macro to take care of any remaining unflushed mappings. 1456 * 1457 * The maximum range this struct can represent is the number of bits 1458 * in the dmr_bitvec field times the pagesize in dmr_pgsz. Currently, only 1459 * MMU_PAGESIZE pages are supported. 1460 * 1461 * Since there are now cases where it's no longer necessary to do 1462 * flushes (e.g. when the process isn't runnable because it's swapping 1463 * out or exiting) we allow these macros to take a NULL dmr input and do 1464 * nothing in that case. 1465 */ 1466 typedef struct { 1467 sfmmu_t *dmr_sfmmup; /* relevant hat */ 1468 caddr_t dmr_addr; /* beginning address */ 1469 caddr_t dmr_endaddr; /* ending address */ 1470 ulong_t dmr_bitvec; /* valid pages found */ 1471 ulong_t dmr_bit; /* next page to examine */ 1472 ulong_t dmr_maxbit; /* highest page in range */ 1473 ulong_t dmr_pgsz; /* page size in range */ 1474 } demap_range_t; 1475 1476 #define DMR_MAXBIT ((ulong_t)1<<63) /* dmr_bit high bit */ 1477 1478 #define DEMAP_RANGE_INIT(sfmmup, dmrp) \ 1479 if ((dmrp) != NULL) { \ 1480 (dmrp)->dmr_sfmmup = (sfmmup); \ 1481 (dmrp)->dmr_bitvec = 0; \ 1482 (dmrp)->dmr_maxbit = sfmmu_dmr_maxbit; \ 1483 (dmrp)->dmr_pgsz = MMU_PAGESIZE; \ 1484 } 1485 1486 #define DEMAP_RANGE_PGSZ(dmrp) ((dmrp)? (dmrp)->dmr_pgsz : MMU_PAGESIZE) 1487 1488 #define DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr) \ 1489 if ((dmrp) != NULL) { \ 1490 if ((dmrp)->dmr_bitvec != 0 && (dmrp)->dmr_endaddr != (addr)) \ 1491 sfmmu_tlb_range_demap(dmrp); \ 1492 (dmrp)->dmr_endaddr = (endaddr); \ 1493 } 1494 1495 #define DEMAP_RANGE_FLUSH(dmrp) \ 1496 if ((dmrp) != NULL) { \ 1497 if ((dmrp)->dmr_bitvec != 0) \ 1498 sfmmu_tlb_range_demap(dmrp); \ 1499 } 1500 1501 #define DEMAP_RANGE_MARKPG(dmrp, addr) \ 1502 if ((dmrp) != NULL) { \ 1503 if ((dmrp)->dmr_bitvec == 0) { \ 1504 (dmrp)->dmr_addr = (addr); \ 1505 (dmrp)->dmr_bit = 1; \ 1506 } \ 1507 (dmrp)->dmr_bitvec |= (dmrp)->dmr_bit; \ 1508 } 1509 1510 #define DEMAP_RANGE_NEXTPG(dmrp) \ 1511 if ((dmrp) != NULL && (dmrp)->dmr_bitvec != 0) { \ 1512 if ((dmrp)->dmr_bit & (dmrp)->dmr_maxbit) { \ 1513 sfmmu_tlb_range_demap(dmrp); \ 1514 } else { \ 1515 (dmrp)->dmr_bit <<= 1; \ 1516 } \ 1517 } 1518 1519 /* 1520 * TSB related structures 1521 * 1522 * The TSB is made up of tte entries. Both the tag and data are present 1523 * in the TSB. The TSB locking is managed as follows: 1524 * A software bit in the tsb tag is used to indicate that entry is locked. 1525 * If a cpu servicing a tsb miss reads a locked entry the tag compare will 1526 * fail forcing the cpu to go to the hat hash for the translation. 1527 * The cpu who holds the lock can then modify the data side, and the tag side. 1528 * The last write should be to the word containing the lock bit which will 1529 * clear the lock and allow the tsb entry to be read. It is assumed that all 1530 * cpus reading the tsb will do so with atomic 128-bit loads. An atomic 128 1531 * bit load is required to prevent the following from happening: 1532 * 1533 * cpu 0 cpu 1 comments 1534 * 1535 * ldx tag tag unlocked 1536 * ldstub lock set lock 1537 * stx data 1538 * stx tag unlock 1539 * ldx tag incorrect tte!!! 1540 * 1541 * The software also maintains a bit in the tag to indicate an invalid 1542 * tsb entry. The purpose of this bit is to allow the tsb invalidate code 1543 * to invalidate a tsb entry with a single cas. See code for details. 1544 */ 1545 1546 union tsb_tag { 1547 struct { 1548 uint32_t tag_res0:16; /* reserved - context area */ 1549 uint32_t tag_inv:1; /* sw - invalid tsb entry */ 1550 uint32_t tag_lock:1; /* sw - locked tsb entry */ 1551 uint32_t tag_res1:4; /* reserved */ 1552 uint32_t tag_va_hi:10; /* va[63:54] */ 1553 uint32_t tag_va_lo; /* va[53:22] */ 1554 } tagbits; 1555 struct tsb_tagints { 1556 uint32_t inthi; 1557 uint32_t intlo; 1558 } tagints; 1559 }; 1560 #define tag_invalid tagbits.tag_inv 1561 #define tag_locked tagbits.tag_lock 1562 #define tag_vahi tagbits.tag_va_hi 1563 #define tag_valo tagbits.tag_va_lo 1564 #define tag_inthi tagints.inthi 1565 #define tag_intlo tagints.intlo 1566 1567 struct tsbe { 1568 union tsb_tag tte_tag; 1569 tte_t tte_data; 1570 }; 1571 1572 /* 1573 * A per cpu struct is kept that duplicates some info 1574 * used by the tl>0 tsb miss handlers plus it provides 1575 * a scratch area. Its purpose is to minimize cache misses 1576 * in the tsb miss handler and is 128 bytes (2 e$ lines). 1577 * 1578 * There should be one allocated per cpu in nucleus memory 1579 * and should be aligned on an ecache line boundary. 1580 */ 1581 struct tsbmiss { 1582 sfmmu_t *ksfmmup; /* kernel hat id */ 1583 sfmmu_t *usfmmup; /* user hat id */ 1584 sf_srd_t *usrdp; /* user's SRD hat id */ 1585 struct tsbe *tsbptr; /* hardware computed ptr */ 1586 struct tsbe *tsbptr4m; /* hardware computed ptr */ 1587 struct tsbe *tsbscdptr; /* hardware computed ptr */ 1588 struct tsbe *tsbscdptr4m; /* hardware computed ptr */ 1589 uint64_t ismblkpa; 1590 struct hmehash_bucket *khashstart; 1591 struct hmehash_bucket *uhashstart; 1592 uint_t khashsz; 1593 uint_t uhashsz; 1594 uint16_t dcache_line_mask; /* used to flush dcache */ 1595 uchar_t uhat_tteflags; /* private page sizes */ 1596 uchar_t uhat_rtteflags; /* SHME pagesizes */ 1597 uint32_t utsb_misses; 1598 uint32_t ktsb_misses; 1599 uint16_t uprot_traps; 1600 uint16_t kprot_traps; 1601 /* 1602 * scratch[0] -> TSB_TAGACC 1603 * scratch[1] -> TSBMISS_HMEBP 1604 * scratch[2] -> TSBMISS_HATID 1605 */ 1606 uintptr_t scratch[3]; 1607 ulong_t shmermap[SFMMU_HMERGNMAP_WORDS]; /* 8 bytes */ 1608 ulong_t scd_shmermap[SFMMU_HMERGNMAP_WORDS]; /* 8 bytes */ 1609 uint8_t pad[48]; /* pad to 64 bytes */ 1610 }; 1611 1612 /* 1613 * A per cpu struct is kept for the use within the tl>0 kpm tsb 1614 * miss handler. Some members are duplicates of common data or 1615 * the physical addresses of common data. A few members are also 1616 * written by the tl>0 kpm tsb miss handler. Its purpose is to 1617 * minimize cache misses in the kpm tsb miss handler and occupies 1618 * one ecache line. There should be one allocated per cpu in 1619 * nucleus memory and it should be aligned on an ecache line 1620 * boundary. It is not merged w/ struct tsbmiss since there is 1621 * not much to share and the tsbmiss pathes are different, so 1622 * a kpm tlbmiss/tsbmiss only touches one cacheline, except for 1623 * (DEBUG || SFMMU_STAT_GATHER) where the dtlb_misses counter 1624 * of struct tsbmiss is used on every dtlb miss. 1625 */ 1626 struct kpmtsbm { 1627 caddr_t vbase; /* start of address kpm range */ 1628 caddr_t vend; /* end of address kpm range */ 1629 uchar_t flags; /* flags needed in TL tsbmiss handler */ 1630 uchar_t sz_shift; /* for single kpm window */ 1631 uchar_t kpmp_shift; /* hash lock shift */ 1632 uchar_t kpmp2pshft; /* kpm page to page shift */ 1633 uint_t kpmp_table_sz; /* size of kpmp_table or kpmp_stable */ 1634 uint64_t kpmp_tablepa; /* paddr of kpmp_table or kpmp_stable */ 1635 uint64_t msegphashpa; /* paddr of memseg_phash */ 1636 struct tsbe *tsbptr; /* saved ktsb pointer */ 1637 uint_t kpm_dtlb_misses; /* kpm tlbmiss counter */ 1638 uint_t kpm_tsb_misses; /* kpm tsbmiss counter */ 1639 uintptr_t pad[1]; 1640 }; 1641 1642 extern size_t tsb_slab_size; 1643 extern uint_t tsb_slab_shift; 1644 extern size_t tsb_slab_mask; 1645 1646 #endif /* !_ASM */ 1647 1648 /* 1649 * Flags for TL kpm tsbmiss handler 1650 */ 1651 #define KPMTSBM_ENABLE_FLAG 0x01 /* bit copy of kpm_enable */ 1652 #define KPMTSBM_TLTSBM_FLAG 0x02 /* use TL tsbmiss handler */ 1653 #define KPMTSBM_TSBPHYS_FLAG 0x04 /* use ASI_MEM for TSB update */ 1654 1655 /* 1656 * The TSB 1657 * All TSB sizes supported by the hardware are now supported (8K - 1M). 1658 * For kernel TSBs we may go beyond the hardware supported sizes and support 1659 * larger TSBs via software. 1660 * All TTE sizes are supported in the TSB; the manner in which this is 1661 * done is cpu dependent. 1662 */ 1663 #define TSB_MIN_SZCODE TSB_8K_SZCODE /* min. supported TSB size */ 1664 #define TSB_MIN_OFFSET_MASK (TSB_OFFSET_MASK(TSB_MIN_SZCODE)) 1665 1666 #ifdef sun4v 1667 #define UTSB_MAX_SZCODE TSB_256M_SZCODE /* max. supported TSB size */ 1668 #else /* sun4u */ 1669 #define UTSB_MAX_SZCODE TSB_1M_SZCODE /* max. supported TSB size */ 1670 #endif /* sun4v */ 1671 1672 #define UTSB_MAX_OFFSET_MASK (TSB_OFFSET_MASK(UTSB_MAX_SZCODE)) 1673 1674 #define TSB_FREEMEM_MIN 0x1000 /* 32 mb */ 1675 #define TSB_FREEMEM_LARGE 0x10000 /* 512 mb */ 1676 #define TSB_8K_SZCODE 0 /* 512 entries */ 1677 #define TSB_16K_SZCODE 1 /* 1k entries */ 1678 #define TSB_32K_SZCODE 2 /* 2k entries */ 1679 #define TSB_64K_SZCODE 3 /* 4k entries */ 1680 #define TSB_128K_SZCODE 4 /* 8k entries */ 1681 #define TSB_256K_SZCODE 5 /* 16k entries */ 1682 #define TSB_512K_SZCODE 6 /* 32k entries */ 1683 #define TSB_1M_SZCODE 7 /* 64k entries */ 1684 #define TSB_2M_SZCODE 8 /* 128k entries */ 1685 #define TSB_4M_SZCODE 9 /* 256k entries */ 1686 #define TSB_8M_SZCODE 10 /* 512k entries */ 1687 #define TSB_16M_SZCODE 11 /* 1M entries */ 1688 #define TSB_32M_SZCODE 12 /* 2M entries */ 1689 #define TSB_64M_SZCODE 13 /* 4M entries */ 1690 #define TSB_128M_SZCODE 14 /* 8M entries */ 1691 #define TSB_256M_SZCODE 15 /* 16M entries */ 1692 #define TSB_ENTRY_SHIFT 4 /* each entry = 128 bits = 16 bytes */ 1693 #define TSB_ENTRY_SIZE (1 << 4) 1694 #define TSB_START_SIZE 9 1695 #define TSB_ENTRIES(tsbsz) (1 << (TSB_START_SIZE + tsbsz)) 1696 #define TSB_BYTES(tsbsz) (TSB_ENTRIES(tsbsz) << TSB_ENTRY_SHIFT) 1697 #define TSB_OFFSET_MASK(tsbsz) (TSB_ENTRIES(tsbsz) - 1) 1698 #define TSB_BASEADDR_MASK ((1 << 12) - 1) 1699 1700 /* 1701 * sun4u platforms 1702 * --------------- 1703 * We now support two user TSBs with one TSB base register. 1704 * Hence the TSB base register is split up as follows: 1705 * 1706 * When only one TSB present: 1707 * [63 62..42 41..13 12..4 3..0] 1708 * ^ ^ ^ ^ ^ 1709 * | | | | | 1710 * | | | | |_ TSB size code 1711 * | | | | 1712 * | | | |_ Reserved 0 1713 * | | | 1714 * | | |_ TSB VA[41..13] 1715 * | | 1716 * | |_ VA hole (Spitfire), zeros (Cheetah and beyond) 1717 * | 1718 * |_ 0 1719 * 1720 * When second TSB present: 1721 * [63 62..42 41..33 32..29 28..22 21..13 12..4 3..0] 1722 * ^ ^ ^ ^ ^ ^ ^ ^ 1723 * | | | | | | | | 1724 * | | | | | | | |_ First TSB size code 1725 * | | | | | | | 1726 * | | | | | | |_ Reserved 0 1727 * | | | | | | 1728 * | | | | | |_ First TSB's VA[21..13] 1729 * | | | | | 1730 * | | | | |_ Reserved for future use 1731 * | | | | 1732 * | | | |_ Second TSB's size code 1733 * | | | 1734 * | | |_ Second TSB's VA[21..13] 1735 * | | 1736 * | |_ VA hole (Spitfire) / ones (Cheetah and beyond) 1737 * | 1738 * |_ 1 1739 * 1740 * Note that since we store 21..13 of each TSB's VA, TSBs and their slabs 1741 * may be up to 4M in size. For now, only hardware supported TSB sizes 1742 * are supported, though the slabs are usually 4M in size. 1743 * 1744 * sun4u platforms that define UTSB_PHYS use physical addressing to access 1745 * the user TSBs at TL>0. The first user TSB base is in the MMU I/D TSB Base 1746 * registers. The second TSB base uses a dedicated scratchpad register which 1747 * requires a definition of SCRATCHPAD_UTSBREG in mach_sfmmu.h. The layout for 1748 * both registers is equivalent to sun4v below, except the TSB PA range is 1749 * [46..13] for sun4u. 1750 * 1751 * sun4v platforms 1752 * --------------- 1753 * On sun4v platforms, we use two dedicated scratchpad registers as pseudo 1754 * hardware TSB base registers to hold up to two different user TSBs. 1755 * 1756 * Each register contains TSB's physical base and size code information 1757 * as follows: 1758 * 1759 * [63..56 55..13 12..4 3..0] 1760 * ^ ^ ^ ^ 1761 * | | | | 1762 * | | | |_ TSB size code 1763 * | | | 1764 * | | |_ Reserved 0 1765 * | | 1766 * | |_ TSB PA[55..13] 1767 * | 1768 * | 1769 * | 1770 * |_ 0 for valid TSB 1771 * 1772 * Absence of a user TSB (primarily the second user TSB) is indicated by 1773 * storing a negative value in the TSB base register. This allows us to 1774 * check for presence of a user TSB by simply checking bit# 63. 1775 */ 1776 #define TSBREG_MSB_SHIFT 32 /* set upper bits */ 1777 #define TSBREG_MSB_CONST 0xfffff800 /* set bits 63..43 */ 1778 #define TSBREG_FIRTSB_SHIFT 42 /* to clear bits 63:22 */ 1779 #define TSBREG_SECTSB_MKSHIFT 20 /* 21:13 --> 41:33 */ 1780 #define TSBREG_SECTSB_LSHIFT 22 /* to clear bits 63:42 */ 1781 #define TSBREG_SECTSB_RSHIFT (TSBREG_SECTSB_MKSHIFT + TSBREG_SECTSB_LSHIFT) 1782 /* sectsb va -> bits 21:13 */ 1783 /* after clearing upper bits */ 1784 #define TSBREG_SECSZ_SHIFT 29 /* to get sectsb szc to 3:0 */ 1785 #define TSBREG_VAMASK_SHIFT 13 /* set up VA mask */ 1786 1787 #define BIGKTSB_SZ_MASK 0xf 1788 #define TSB_SOFTSZ_MASK BIGKTSB_SZ_MASK 1789 #define MIN_BIGKTSB_SZCODE 9 /* 256k entries */ 1790 #define MAX_BIGKTSB_SZCODE 11 /* 1024k entries */ 1791 #define MAX_BIGKTSB_TTES (TSB_BYTES(MAX_BIGKTSB_SZCODE) / MMU_PAGESIZE4M) 1792 1793 #define TAG_VALO_SHIFT 22 /* tag's va are bits 63-22 */ 1794 /* 1795 * sw bits used on tsb_tag - bit masks used only in assembly 1796 * use only a sethi for these fields. 1797 */ 1798 #define TSBTAG_INVALID 0x00008000 /* tsb_tag.tag_invalid */ 1799 #define TSBTAG_LOCKED 0x00004000 /* tsb_tag.tag_locked */ 1800 1801 #ifdef _ASM 1802 1803 /* 1804 * Marker to indicate that this instruction will be hot patched at runtime 1805 * to some other value. 1806 * This value must be zero since it fills in the imm bits of the target 1807 * instructions to be patched 1808 */ 1809 #define RUNTIME_PATCH (0) 1810 1811 /* 1812 * V9 defines nop instruction as the following, which we use 1813 * at runtime to nullify some instructions we don't want to 1814 * execute in the trap handlers on certain platforms. 1815 */ 1816 #define MAKE_NOP_INSTR(reg) \ 1817 sethi %hi(0x1000000), reg 1818 1819 /* 1820 * This macro constructs a SPARC V9 "jmpl <source reg>, %g0" 1821 * instruction, with the source register specified by the jump_reg_number. 1822 * The jmp opcode [24:19] = 11 1000 and source register is bits [18:14]. 1823 * The instruction is returned in reg. The macro is used to patch in a jmpl 1824 * instruction at runtime. 1825 */ 1826 #define MAKE_JMP_INSTR(jump_reg_number, reg, tmp) \ 1827 sethi %hi(0x81c00000), reg; \ 1828 mov jump_reg_number, tmp; \ 1829 sll tmp, 14, tmp; \ 1830 or reg, tmp, reg 1831 1832 /* 1833 * Macro to get hat per-MMU cnum on this CPU. 1834 * sfmmu - In, pass in "sfmmup" from the caller. 1835 * cnum - Out, return 'cnum' to the caller 1836 * scr - scratch 1837 */ 1838 #define SFMMU_CPU_CNUM(sfmmu, cnum, scr) \ 1839 CPU_ADDR(scr, cnum); /* scr = load CPU struct addr */ \ 1840 ld [scr + CPU_MMU_IDX], cnum; /* cnum = mmuid */ \ 1841 add sfmmu, SFMMU_CTXS, scr; /* scr = sfmmup->sfmmu_ctxs[] */ \ 1842 sllx cnum, SFMMU_MMU_CTX_SHIFT, cnum; \ 1843 add scr, cnum, scr; /* scr = sfmmup->sfmmu_ctxs[id] */ \ 1844 ldx [scr + SFMMU_MMU_GC_NUM], scr; /* sfmmu_ctxs[id].gcnum */ \ 1845 sllx scr, SFMMU_MMU_CNUM_LSHIFT, scr; \ 1846 srlx scr, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */ 1847 1848 /* 1849 * Macro to get hat gnum & cnum assocaited with sfmmu_ctx[mmuid] entry 1850 * entry - In, pass in (&sfmmu_ctxs[mmuid] - SFMMU_CTXS) from the caller. 1851 * gnum - Out, return sfmmu gnum 1852 * cnum - Out, return sfmmu cnum 1853 * reg - scratch 1854 */ 1855 #define SFMMU_MMUID_GNUM_CNUM(entry, gnum, cnum, reg) \ 1856 ldx [entry + SFMMU_CTXS], reg; /* reg = sfmmu (gnum | cnum) */ \ 1857 srlx reg, SFMMU_MMU_GNUM_RSHIFT, gnum; /* gnum = sfmmu gnum */ \ 1858 sllx reg, SFMMU_MMU_CNUM_LSHIFT, cnum; \ 1859 srlx cnum, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */ 1860 1861 /* 1862 * Macro to get this CPU's tsbmiss area. 1863 */ 1864 #define CPU_TSBMISS_AREA(tsbmiss, tmp1) \ 1865 CPU_INDEX(tmp1, tsbmiss); /* tmp1 = cpu idx */ \ 1866 sethi %hi(tsbmiss_area), tsbmiss; /* tsbmiss base ptr */ \ 1867 mulx tmp1, TSBMISS_SIZE, tmp1; /* byte offset */ \ 1868 or tsbmiss, %lo(tsbmiss_area), tsbmiss; \ 1869 add tsbmiss, tmp1, tsbmiss /* tsbmiss area of CPU */ 1870 1871 1872 /* 1873 * Macro to set kernel context + page size codes in DMMU primary context 1874 * register. It is only necessary for sun4u because sun4v does not need 1875 * page size codes 1876 */ 1877 #ifdef sun4v 1878 1879 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) 1880 1881 #else 1882 1883 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) \ 1884 sethi %hi(kcontextreg), reg0; \ 1885 ldx [reg0 + %lo(kcontextreg)], reg0; \ 1886 mov MMU_PCONTEXT, reg1; \ 1887 ldxa [reg1]ASI_MMU_CTX, reg2; \ 1888 xor reg0, reg2, reg2; \ 1889 brz reg2, label3; \ 1890 srlx reg2, CTXREG_NEXT_SHIFT, reg2; \ 1891 rdpr %pstate, reg3; /* disable interrupts */ \ 1892 btst PSTATE_IE, reg3; \ 1893 /*CSTYLED*/ \ 1894 bnz,a,pt %icc, label1; \ 1895 wrpr reg3, PSTATE_IE, %pstate; \ 1896 /*CSTYLED*/ \ 1897 label1:; \ 1898 brz reg2, label2; /* need demap if N_pgsz0/1 change */ \ 1899 sethi %hi(FLUSH_ADDR), reg4; \ 1900 mov DEMAP_ALL_TYPE, reg2; \ 1901 stxa %g0, [reg2]ASI_DTLB_DEMAP; \ 1902 stxa %g0, [reg2]ASI_ITLB_DEMAP; \ 1903 /*CSTYLED*/ \ 1904 label2:; \ 1905 stxa reg0, [reg1]ASI_MMU_CTX; \ 1906 flush reg4; \ 1907 btst PSTATE_IE, reg3; \ 1908 /*CSTYLED*/ \ 1909 bnz,a,pt %icc, label3; \ 1910 wrpr %g0, reg3, %pstate; /* restore interrupt state */ \ 1911 label3:; 1912 1913 #endif 1914 1915 /* 1916 * Macro to setup arguments with kernel sfmmup context + page size before 1917 * calling sfmmu_setctx_sec() 1918 */ 1919 #ifdef sun4v 1920 #define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \ 1921 set KCONTEXT, arg0; \ 1922 set 0, arg1; 1923 #else 1924 #define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \ 1925 ldub [sfmmup + SFMMU_CEXT], arg1; \ 1926 set KCONTEXT, arg0; \ 1927 sll arg1, CTXREG_EXT_SHIFT, arg1; 1928 #endif 1929 1930 #define PANIC_IF_INTR_DISABLED_PSTR(pstatereg, label, scr) \ 1931 andcc pstatereg, PSTATE_IE, %g0; /* panic if intrs */ \ 1932 /*CSTYLED*/ \ 1933 bnz,pt %icc, label; /* already disabled */ \ 1934 nop; \ 1935 \ 1936 sethi %hi(panicstr), scr; \ 1937 ldx [scr + %lo(panicstr)], scr; \ 1938 tst scr; \ 1939 /*CSTYLED*/ \ 1940 bnz,pt %xcc, label; \ 1941 nop; \ 1942 \ 1943 save %sp, -SA(MINFRAME), %sp; \ 1944 sethi %hi(sfmmu_panic1), %o0; \ 1945 call panic; \ 1946 or %o0, %lo(sfmmu_panic1), %o0; \ 1947 /*CSTYLED*/ \ 1948 label: 1949 1950 #define PANIC_IF_INTR_ENABLED_PSTR(label, scr) \ 1951 /* \ 1952 * The caller must have disabled interrupts. \ 1953 * If interrupts are not disabled, panic \ 1954 */ \ 1955 rdpr %pstate, scr; \ 1956 andcc scr, PSTATE_IE, %g0; \ 1957 /*CSTYLED*/ \ 1958 bz,pt %icc, label; \ 1959 nop; \ 1960 \ 1961 sethi %hi(panicstr), scr; \ 1962 ldx [scr + %lo(panicstr)], scr; \ 1963 tst scr; \ 1964 /*CSTYLED*/ \ 1965 bnz,pt %xcc, label; \ 1966 nop; \ 1967 \ 1968 sethi %hi(sfmmu_panic6), %o0; \ 1969 call panic; \ 1970 or %o0, %lo(sfmmu_panic6), %o0; \ 1971 /*CSTYLED*/ \ 1972 label: 1973 1974 #endif /* _ASM */ 1975 1976 #ifndef _ASM 1977 1978 #ifdef VAC 1979 /* 1980 * Page coloring 1981 * The p_vcolor field of the page struct (1 byte) is used to store the 1982 * virtual page color. This provides for 255 colors. The value zero is 1983 * used to mean the page has no color - never been mapped or somehow 1984 * purified. 1985 */ 1986 1987 #define PP_GET_VCOLOR(pp) (((pp)->p_vcolor) - 1) 1988 #define PP_NEWPAGE(pp) (!(pp)->p_vcolor) 1989 #define PP_SET_VCOLOR(pp, color) \ 1990 ((pp)->p_vcolor = ((color) + 1)) 1991 1992 /* 1993 * As mentioned p_vcolor == 0 means there is no color for this page. 1994 * But PP_SET_VCOLOR(pp, color) expects 'color' to be real color minus 1995 * one so we define this constant. 1996 */ 1997 #define NO_VCOLOR (-1) 1998 1999 #define addr_to_vcolor(addr) \ 2000 (((uint_t)(uintptr_t)(addr) >> MMU_PAGESHIFT) & vac_colors_mask) 2001 #else /* VAC */ 2002 #define addr_to_vcolor(addr) (0) 2003 #endif /* VAC */ 2004 2005 /* 2006 * The field p_index in the psm page structure is for large pages support. 2007 * P_index is a bit-vector of the different mapping sizes that a given page 2008 * is part of. An hme structure for a large mapping is only added in the 2009 * group leader page (first page). All pages covered by a given large mapping 2010 * have the corrosponding mapping bit set in their p_index field. This allows 2011 * us to only store an explicit hme structure in the leading page which 2012 * simplifies the mapping link list management. Furthermore, it provides us 2013 * a fast mechanism for determining the largest mapping a page is part of. For 2014 * exmaple, a page with a 64K and a 4M mappings has a p_index value of 0x0A. 2015 * 2016 * Implementation note: even though the first bit in p_index is reserved 2017 * for 8K mappings, it is NOT USED by the code and SHOULD NOT be set. 2018 * In addition, the upper four bits of the p_index field are used by the 2019 * code as temporaries 2020 */ 2021 2022 /* 2023 * Defines for psm page struct fields and large page support 2024 */ 2025 #define SFMMU_INDEX_SHIFT 6 2026 #define SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1) 2027 2028 /* Return the mapping index */ 2029 #define PP_MAPINDEX(pp) ((pp)->p_index & SFMMU_INDEX_MASK) 2030 2031 /* 2032 * These macros rely on the following property: 2033 * All pages constituting a large page are covered by a virtually 2034 * contiguous set of page_t's. 2035 */ 2036 2037 /* Return the leader for this mapping size */ 2038 #define PP_GROUPLEADER(pp, sz) \ 2039 (&(pp)[-(int)(pp->p_pagenum & (TTEPAGES(sz)-1))]) 2040 2041 /* Return the root page for this page based on p_szc */ 2042 #define PP_PAGEROOT(pp) ((pp)->p_szc == 0 ? (pp) : \ 2043 PP_GROUPLEADER((pp), (pp)->p_szc)) 2044 2045 #define PP_PAGENEXT_N(pp, n) ((pp) + (n)) 2046 #define PP_PAGENEXT(pp) PP_PAGENEXT_N((pp), 1) 2047 2048 #define PP_PAGEPREV_N(pp, n) ((pp) - (n)) 2049 #define PP_PAGEPREV(pp) PP_PAGEPREV_N((pp), 1) 2050 2051 #define PP_ISMAPPED_LARGE(pp) (PP_MAPINDEX(pp) != 0) 2052 2053 /* Need function to test the page mappping which takes p_index into account */ 2054 #define PP_ISMAPPED(pp) ((pp)->p_mapping || PP_ISMAPPED_LARGE(pp)) 2055 2056 /* 2057 * Don't call this macro with sz equal to zero. 8K mappings SHOULD NOT 2058 * set p_index field. 2059 */ 2060 #define PAGESZ_TO_INDEX(sz) (1 << (sz)) 2061 2062 2063 /* 2064 * prototypes for hat assembly routines. Some of these are 2065 * known to machine dependent VM code. 2066 */ 2067 extern uint64_t sfmmu_make_tsbtag(caddr_t); 2068 extern struct tsbe * 2069 sfmmu_get_tsbe(uint64_t, caddr_t, int, int); 2070 extern void sfmmu_load_tsbe(struct tsbe *, uint64_t, tte_t *, int); 2071 extern void sfmmu_unload_tsbe(struct tsbe *, uint64_t, int); 2072 extern void sfmmu_load_mmustate(sfmmu_t *); 2073 extern void sfmmu_raise_tsb_exception(uint64_t, uint64_t); 2074 #ifndef sun4v 2075 extern void sfmmu_itlb_ld_kva(caddr_t, tte_t *); 2076 extern void sfmmu_dtlb_ld_kva(caddr_t, tte_t *); 2077 #endif /* sun4v */ 2078 extern void sfmmu_copytte(tte_t *, tte_t *); 2079 extern int sfmmu_modifytte(tte_t *, tte_t *, tte_t *); 2080 extern int sfmmu_modifytte_try(tte_t *, tte_t *, tte_t *); 2081 extern pfn_t sfmmu_ttetopfn(tte_t *, caddr_t); 2082 extern void sfmmu_hblk_hash_rm(struct hmehash_bucket *, 2083 struct hme_blk *, uint64_t, struct hme_blk *); 2084 extern void sfmmu_hblk_hash_add(struct hmehash_bucket *, struct hme_blk *, 2085 uint64_t); 2086 extern uint_t sfmmu_disable_intrs(void); 2087 extern void sfmmu_enable_intrs(uint_t); 2088 /* 2089 * functions exported to machine dependent VM code 2090 */ 2091 extern void sfmmu_patch_ktsb(void); 2092 #ifndef UTSB_PHYS 2093 extern void sfmmu_patch_utsb(void); 2094 #endif /* UTSB_PHYS */ 2095 extern pfn_t sfmmu_vatopfn(caddr_t, sfmmu_t *, tte_t *); 2096 extern void sfmmu_vatopfn_suspended(caddr_t, sfmmu_t *, tte_t *); 2097 extern pfn_t sfmmu_kvaszc2pfn(caddr_t, int); 2098 #ifdef DEBUG 2099 extern void sfmmu_check_kpfn(pfn_t); 2100 #else 2101 #define sfmmu_check_kpfn(pfn) /* disabled */ 2102 #endif /* DEBUG */ 2103 extern void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 2104 extern void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, uint_t); 2105 extern void sfmmu_tsbmiss_exception(struct regs *, uintptr_t, uint_t); 2106 extern void sfmmu_init_tsbs(void); 2107 extern caddr_t sfmmu_ktsb_alloc(caddr_t); 2108 extern int sfmmu_getctx_pri(void); 2109 extern int sfmmu_getctx_sec(void); 2110 extern void sfmmu_setctx_sec(uint_t); 2111 extern void sfmmu_inv_tsb(caddr_t, uint_t); 2112 extern void sfmmu_init_ktsbinfo(void); 2113 extern int sfmmu_setup_4lp(void); 2114 extern void sfmmu_patch_mmu_asi(int); 2115 extern void sfmmu_init_nucleus_hblks(caddr_t, size_t, int, int); 2116 extern void sfmmu_cache_flushall(void); 2117 extern pgcnt_t sfmmu_tte_cnt(sfmmu_t *, uint_t); 2118 extern void *sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int); 2119 extern void sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t); 2120 extern void sfmmu_reprog_pgsz_arr(sfmmu_t *, uint8_t *); 2121 2122 extern void hat_kern_setup(void); 2123 extern int hat_page_relocate(page_t **, page_t **, spgcnt_t *); 2124 extern int sfmmu_get_ppvcolor(struct page *); 2125 extern int sfmmu_get_addrvcolor(caddr_t); 2126 extern int sfmmu_hat_lock_held(sfmmu_t *); 2127 extern int sfmmu_alloc_ctx(sfmmu_t *, int, struct cpu *, int); 2128 2129 /* 2130 * Functions exported to xhat_sfmmu.c 2131 */ 2132 extern kmutex_t *sfmmu_mlist_enter(page_t *); 2133 extern void sfmmu_mlist_exit(kmutex_t *); 2134 extern int sfmmu_mlist_held(struct page *); 2135 extern struct hme_blk *sfmmu_hmetohblk(struct sf_hment *); 2136 2137 /* 2138 * MMU-specific functions optionally imported from the CPU module 2139 */ 2140 #pragma weak mmu_large_pages_disabled 2141 #pragma weak mmu_set_ctx_page_sizes 2142 #pragma weak mmu_check_page_sizes 2143 2144 extern uint_t mmu_large_pages_disabled(uint_t); 2145 extern void mmu_set_ctx_page_sizes(sfmmu_t *); 2146 extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *); 2147 2148 extern sfmmu_t *ksfmmup; 2149 extern caddr_t ktsb_base; 2150 extern uint64_t ktsb_pbase; 2151 extern int ktsb_sz; 2152 extern int ktsb_szcode; 2153 extern caddr_t ktsb4m_base; 2154 extern uint64_t ktsb4m_pbase; 2155 extern int ktsb4m_sz; 2156 extern int ktsb4m_szcode; 2157 extern uint64_t kpm_tsbbase; 2158 extern int kpm_tsbsz; 2159 extern int ktsb_phys; 2160 extern int enable_bigktsb; 2161 #ifndef sun4v 2162 extern int utsb_dtlb_ttenum; 2163 extern int utsb4m_dtlb_ttenum; 2164 #endif /* sun4v */ 2165 extern int uhmehash_num; 2166 extern int khmehash_num; 2167 extern struct hmehash_bucket *uhme_hash; 2168 extern struct hmehash_bucket *khme_hash; 2169 extern kmutex_t *mml_table; 2170 extern uint_t mml_table_sz; 2171 extern uint_t mml_shift; 2172 extern uint_t hblk_alloc_dynamic; 2173 extern struct tsbmiss tsbmiss_area[NCPU]; 2174 extern struct kpmtsbm kpmtsbm_area[NCPU]; 2175 2176 #ifndef sun4v 2177 extern int dtlb_resv_ttenum; 2178 extern caddr_t utsb_vabase; 2179 extern caddr_t utsb4m_vabase; 2180 #endif /* sun4v */ 2181 extern vmem_t *kmem_tsb_default_arena[]; 2182 extern int tsb_lgrp_affinity; 2183 2184 extern uint_t disable_large_pages; 2185 extern uint_t disable_ism_large_pages; 2186 extern uint_t disable_auto_data_large_pages; 2187 extern uint_t disable_auto_text_large_pages; 2188 2189 /* kpm externals */ 2190 extern pfn_t sfmmu_kpm_vatopfn(caddr_t); 2191 extern void sfmmu_kpm_patch_tlbm(void); 2192 extern void sfmmu_kpm_patch_tsbm(void); 2193 extern void sfmmu_patch_shctx(void); 2194 extern void sfmmu_kpm_load_tsb(caddr_t, tte_t *, int); 2195 extern void sfmmu_kpm_unload_tsb(caddr_t, int); 2196 extern void sfmmu_kpm_tsbmtl(short *, uint_t *, int); 2197 extern int sfmmu_kpm_stsbmtl(char *, uint_t *, int); 2198 extern caddr_t kpm_vbase; 2199 extern size_t kpm_size; 2200 extern struct memseg *memseg_hash[]; 2201 extern uint64_t memseg_phash[]; 2202 extern kpm_hlk_t *kpmp_table; 2203 extern kpm_shlk_t *kpmp_stable; 2204 extern uint_t kpmp_table_sz; 2205 extern uint_t kpmp_stable_sz; 2206 extern uchar_t kpmp_shift; 2207 2208 #define PP_ISMAPPED_KPM(pp) ((pp)->p_kpmref > 0) 2209 2210 #define IS_KPM_ALIAS_RANGE(vaddr) \ 2211 (((vaddr) - kpm_vbase) >> (uintptr_t)kpm_size_shift > 0) 2212 2213 #endif /* !_ASM */ 2214 2215 /* sfmmu_kpm_tsbmtl flags */ 2216 #define KPMTSBM_STOP 0 2217 #define KPMTSBM_START 1 2218 2219 /* kpm_smallpages kp_mapped values */ 2220 #define KPM_MAPPEDS -1 /* small mapping valid, no conflict */ 2221 #define KPM_MAPPEDSC 1 /* small mapping valid, conflict */ 2222 2223 /* Physical memseg address NULL marker */ 2224 #define MSEG_NULLPTR_PA -1 2225 2226 /* 2227 * Memseg hash defines for kpm trap level tsbmiss handler. 2228 * Must be in sync w/ page.h . 2229 */ 2230 #define SFMMU_MEM_HASH_SHIFT 0x9 2231 #define SFMMU_N_MEM_SLOTS 0x200 2232 #define SFMMU_MEM_HASH_ENTRY_SHIFT 3 2233 2234 #ifndef _ASM 2235 #if (SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT) 2236 #error SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT 2237 #endif 2238 #if (SFMMU_N_MEM_SLOTS != N_MEM_SLOTS) 2239 #error SFMMU_N_MEM_SLOTS != N_MEM_SLOTS 2240 #endif 2241 2242 /* Physical memseg address NULL marker */ 2243 #define SFMMU_MEMSEG_NULLPTR_PA -1 2244 2245 /* 2246 * Check KCONTEXT to be zero, asm parts depend on that assumption. 2247 */ 2248 #if (KCONTEXT != 0) 2249 #error KCONTEXT != 0 2250 #endif 2251 #endif /* !_ASM */ 2252 2253 2254 #endif /* _KERNEL */ 2255 2256 #ifndef _ASM 2257 /* 2258 * ctx, hmeblk, mlistlock and other stats for sfmmu 2259 */ 2260 struct sfmmu_global_stat { 2261 int sf_tsb_exceptions; /* # of tsb exceptions */ 2262 int sf_tsb_raise_exception; /* # tsb exc. w/o TLB flush */ 2263 2264 int sf_pagefaults; /* # of pagefaults */ 2265 2266 int sf_uhash_searches; /* # of user hash searches */ 2267 int sf_uhash_links; /* # of user hash links */ 2268 int sf_khash_searches; /* # of kernel hash searches */ 2269 int sf_khash_links; /* # of kernel hash links */ 2270 2271 int sf_swapout; /* # times hat swapped out */ 2272 2273 int sf_tsb_alloc; /* # TSB allocations */ 2274 int sf_tsb_allocfail; /* # times TSB alloc fail */ 2275 int sf_tsb_sectsb_create; /* # times second TSB added */ 2276 2277 int sf_scd_1sttsb_alloc; /* # SCD 1st TSB allocations */ 2278 int sf_scd_2ndtsb_alloc; /* # SCD 2nd TSB allocations */ 2279 int sf_scd_1sttsb_allocfail; /* # SCD 1st TSB alloc fail */ 2280 int sf_scd_2ndtsb_allocfail; /* # SCD 2nd TSB alloc fail */ 2281 2282 2283 int sf_tteload8k; /* calls to sfmmu_tteload */ 2284 int sf_tteload64k; /* calls to sfmmu_tteload */ 2285 int sf_tteload512k; /* calls to sfmmu_tteload */ 2286 int sf_tteload4m; /* calls to sfmmu_tteload */ 2287 int sf_tteload32m; /* calls to sfmmu_tteload */ 2288 int sf_tteload256m; /* calls to sfmmu_tteload */ 2289 2290 int sf_tsb_load8k; /* # times loaded 8K tsbent */ 2291 int sf_tsb_load4m; /* # times loaded 4M tsbent */ 2292 2293 int sf_hblk_hit; /* found hblk during tteload */ 2294 int sf_hblk8_ncreate; /* static hblk8's created */ 2295 int sf_hblk8_nalloc; /* static hblk8's allocated */ 2296 int sf_hblk1_ncreate; /* static hblk1's created */ 2297 int sf_hblk1_nalloc; /* static hblk1's allocated */ 2298 int sf_hblk_slab_cnt; /* sfmmu8_cache slab creates */ 2299 int sf_hblk_reserve_cnt; /* hblk_reserve usage */ 2300 int sf_hblk_recurse_cnt; /* hblk_reserve owner reqs */ 2301 int sf_hblk_reserve_hit; /* hblk_reserve hash hits */ 2302 int sf_get_free_success; /* reserve list allocs */ 2303 int sf_get_free_throttle; /* fails due to throttling */ 2304 int sf_get_free_fail; /* fails due to empty list */ 2305 int sf_put_free_success; /* reserve list frees */ 2306 int sf_put_free_fail; /* fails due to full list */ 2307 2308 int sf_pgcolor_conflict; /* VAC conflict resolution */ 2309 int sf_uncache_conflict; /* VAC conflict resolution */ 2310 int sf_unload_conflict; /* VAC unload resolution */ 2311 int sf_ism_uncache; /* VAC conflict resolution */ 2312 int sf_ism_recache; /* VAC conflict resolution */ 2313 int sf_recache; /* VAC conflict resolution */ 2314 2315 int sf_steal_count; /* # of hblks stolen */ 2316 2317 int sf_pagesync; /* # of pagesyncs */ 2318 int sf_clrwrt; /* # of clear write perms */ 2319 int sf_pagesync_invalid; /* pagesync with inv tte */ 2320 2321 int sf_kernel_xcalls; /* # of kernel cross calls */ 2322 int sf_user_xcalls; /* # of user cross calls */ 2323 2324 int sf_tsb_grow; /* # of user tsb grows */ 2325 int sf_tsb_shrink; /* # of user tsb shrinks */ 2326 int sf_tsb_resize_failures; /* # of user tsb resize */ 2327 int sf_tsb_reloc; /* # of user tsb relocations */ 2328 2329 int sf_user_vtop; /* # of user vatopfn calls */ 2330 2331 int sf_ctx_inv; /* #times invalidate MMU ctx */ 2332 2333 int sf_tlb_reprog_pgsz; /* # times switch TLB pgsz */ 2334 2335 int sf_region_remap_demap; /* # times shme remap demap */ 2336 2337 int sf_create_scd; /* # times SCD is created */ 2338 int sf_join_scd; /* # process joined scd */ 2339 int sf_leave_scd; /* # process left scd */ 2340 int sf_destroy_scd; /* # times SCD is destroyed */ 2341 }; 2342 2343 struct sfmmu_tsbsize_stat { 2344 int sf_tsbsz_8k; 2345 int sf_tsbsz_16k; 2346 int sf_tsbsz_32k; 2347 int sf_tsbsz_64k; 2348 int sf_tsbsz_128k; 2349 int sf_tsbsz_256k; 2350 int sf_tsbsz_512k; 2351 int sf_tsbsz_1m; 2352 int sf_tsbsz_2m; 2353 int sf_tsbsz_4m; 2354 int sf_tsbsz_8m; 2355 int sf_tsbsz_16m; 2356 int sf_tsbsz_32m; 2357 int sf_tsbsz_64m; 2358 int sf_tsbsz_128m; 2359 int sf_tsbsz_256m; 2360 }; 2361 2362 struct sfmmu_percpu_stat { 2363 int sf_itlb_misses; /* # of itlb misses */ 2364 int sf_dtlb_misses; /* # of dtlb misses */ 2365 int sf_utsb_misses; /* # of user tsb misses */ 2366 int sf_ktsb_misses; /* # of kernel tsb misses */ 2367 int sf_tsb_hits; /* # of tsb hits */ 2368 int sf_umod_faults; /* # of mod (prot viol) flts */ 2369 int sf_kmod_faults; /* # of mod (prot viol) flts */ 2370 }; 2371 2372 #define SFMMU_STAT(stat) sfmmu_global_stat.stat++ 2373 #define SFMMU_STAT_ADD(stat, amount) sfmmu_global_stat.stat += (amount) 2374 #define SFMMU_STAT_SET(stat, count) sfmmu_global_stat.stat = (count) 2375 2376 #define SFMMU_MMU_STAT(stat) CPU->cpu_m.cpu_mmu_ctxp->stat++ 2377 2378 #endif /* !_ASM */ 2379 2380 #ifdef __cplusplus 2381 } 2382 #endif 2383 2384 #endif /* _VM_HAT_SFMMU_H */ 2385