1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * VM - Hardware Address Translation management for Spitfire MMU. 30 * 31 * This file implements the machine specific hardware translation 32 * needed by the VM system. The machine independent interface is 33 * described in <vm/hat.h> while the machine dependent interface 34 * and data structures are described in <vm/hat_sfmmu.h>. 35 * 36 * The hat layer manages the address translation hardware as a cache 37 * driven by calls from the higher levels in the VM system. 38 */ 39 40 #include <sys/types.h> 41 #include <sys/kstat.h> 42 #include <vm/hat.h> 43 #include <vm/hat_sfmmu.h> 44 #include <vm/page.h> 45 #include <sys/pte.h> 46 #include <sys/systm.h> 47 #include <sys/mman.h> 48 #include <sys/sysmacros.h> 49 #include <sys/machparam.h> 50 #include <sys/vtrace.h> 51 #include <sys/kmem.h> 52 #include <sys/mmu.h> 53 #include <sys/cmn_err.h> 54 #include <sys/cpu.h> 55 #include <sys/cpuvar.h> 56 #include <sys/debug.h> 57 #include <sys/lgrp.h> 58 #include <sys/archsystm.h> 59 #include <sys/machsystm.h> 60 #include <sys/vmsystm.h> 61 #include <vm/as.h> 62 #include <vm/seg.h> 63 #include <vm/seg_kp.h> 64 #include <vm/seg_kmem.h> 65 #include <vm/seg_kpm.h> 66 #include <vm/rm.h> 67 #include <sys/t_lock.h> 68 #include <sys/obpdefs.h> 69 #include <sys/vm_machparam.h> 70 #include <sys/var.h> 71 #include <sys/trap.h> 72 #include <sys/machtrap.h> 73 #include <sys/scb.h> 74 #include <sys/bitmap.h> 75 #include <sys/machlock.h> 76 #include <sys/membar.h> 77 #include <sys/atomic.h> 78 #include <sys/cpu_module.h> 79 #include <sys/prom_debug.h> 80 #include <sys/ksynch.h> 81 #include <sys/mem_config.h> 82 #include <sys/mem_cage.h> 83 #include <sys/dtrace.h> 84 #include <vm/vm_dep.h> 85 #include <vm/xhat_sfmmu.h> 86 #include <sys/fpu/fpusystm.h> 87 #include <vm/mach_kpm.h> 88 89 #if defined(SF_ERRATA_57) 90 extern caddr_t errata57_limit; 91 #endif 92 93 #define HME8BLK_SZ_RND ((roundup(HME8BLK_SZ, sizeof (int64_t))) / \ 94 (sizeof (int64_t))) 95 #define HBLK_RESERVE ((struct hme_blk *)hblk_reserve) 96 97 #define HBLK_RESERVE_CNT 128 98 #define HBLK_RESERVE_MIN 20 99 100 static struct hme_blk *freehblkp; 101 static kmutex_t freehblkp_lock; 102 static int freehblkcnt; 103 104 static int64_t hblk_reserve[HME8BLK_SZ_RND]; 105 static kmutex_t hblk_reserve_lock; 106 static kthread_t *hblk_reserve_thread; 107 108 static nucleus_hblk8_info_t nucleus_hblk8; 109 static nucleus_hblk1_info_t nucleus_hblk1; 110 111 /* 112 * SFMMU specific hat functions 113 */ 114 void hat_pagecachectl(struct page *, int); 115 116 /* flags for hat_pagecachectl */ 117 #define HAT_CACHE 0x1 118 #define HAT_UNCACHE 0x2 119 #define HAT_TMPNC 0x4 120 121 /* 122 * Flag to allow the creation of non-cacheable translations 123 * to system memory. It is off by default. At the moment this 124 * flag is used by the ecache error injector. The error injector 125 * will turn it on when creating such a translation then shut it 126 * off when it's finished. 127 */ 128 129 int sfmmu_allow_nc_trans = 0; 130 131 /* 132 * Flag to disable large page support. 133 * value of 1 => disable all large pages. 134 * bits 1, 2, and 3 are to disable 64K, 512K and 4M pages respectively. 135 * 136 * For example, use the value 0x4 to disable 512K pages. 137 * 138 */ 139 #define LARGE_PAGES_OFF 0x1 140 141 /* 142 * The disable_large_pages and disable_ism_large_pages variables control 143 * hat_memload_array and the page sizes to be used by ISM and the kernel. 144 * 145 * The disable_auto_data_large_pages and disable_auto_text_large_pages variables 146 * are only used to control which OOB pages to use at upper VM segment creation 147 * time, and are set in hat_init_pagesizes and used in the map_pgsz* routines. 148 * Their values may come from platform or CPU specific code to disable page 149 * sizes that should not be used. 150 * 151 * WARNING: 512K pages are currently not supported for ISM/DISM. 152 */ 153 uint_t disable_large_pages = 0; 154 uint_t disable_ism_large_pages = (1 << TTE512K); 155 uint_t disable_auto_data_large_pages = 0; 156 uint_t disable_auto_text_large_pages = 0; 157 158 /* 159 * Private sfmmu data structures for hat management 160 */ 161 static struct kmem_cache *sfmmuid_cache; 162 static struct kmem_cache *mmuctxdom_cache; 163 164 /* 165 * Private sfmmu data structures for tsb management 166 */ 167 static struct kmem_cache *sfmmu_tsbinfo_cache; 168 static struct kmem_cache *sfmmu_tsb8k_cache; 169 static struct kmem_cache *sfmmu_tsb_cache[NLGRPS_MAX]; 170 static vmem_t *kmem_tsb_arena; 171 172 /* 173 * sfmmu static variables for hmeblk resource management. 174 */ 175 static vmem_t *hat_memload1_arena; /* HAT translation arena for sfmmu1_cache */ 176 static struct kmem_cache *sfmmu8_cache; 177 static struct kmem_cache *sfmmu1_cache; 178 static struct kmem_cache *pa_hment_cache; 179 180 static kmutex_t ism_mlist_lock; /* mutex for ism mapping list */ 181 /* 182 * private data for ism 183 */ 184 static struct kmem_cache *ism_blk_cache; 185 static struct kmem_cache *ism_ment_cache; 186 #define ISMID_STARTADDR NULL 187 188 /* 189 * Whether to delay TLB flushes and use Cheetah's flush-all support 190 * when removing contexts from the dirty list. 191 */ 192 int delay_tlb_flush; 193 int disable_delay_tlb_flush; 194 195 /* 196 * ``hat_lock'' is a hashed mutex lock for protecting sfmmu TSB lists, 197 * HAT flags, synchronizing TLB/TSB coherency, and context management. 198 * The lock is hashed on the sfmmup since the case where we need to lock 199 * all processes is rare but does occur (e.g. we need to unload a shared 200 * mapping from all processes using the mapping). We have a lot of buckets, 201 * and each slab of sfmmu_t's can use about a quarter of them, giving us 202 * a fairly good distribution without wasting too much space and overhead 203 * when we have to grab them all. 204 */ 205 #define SFMMU_NUM_LOCK 128 /* must be power of two */ 206 hatlock_t hat_lock[SFMMU_NUM_LOCK]; 207 208 /* 209 * Hash algorithm optimized for a small number of slabs. 210 * 7 is (highbit((sizeof sfmmu_t)) - 1) 211 * This hash algorithm is based upon the knowledge that sfmmu_t's come from a 212 * kmem_cache, and thus they will be sequential within that cache. In 213 * addition, each new slab will have a different "color" up to cache_maxcolor 214 * which will skew the hashing for each successive slab which is allocated. 215 * If the size of sfmmu_t changed to a larger size, this algorithm may need 216 * to be revisited. 217 */ 218 #define TSB_HASH_SHIFT_BITS (7) 219 #define PTR_HASH(x) ((uintptr_t)x >> TSB_HASH_SHIFT_BITS) 220 221 #ifdef DEBUG 222 int tsb_hash_debug = 0; 223 #define TSB_HASH(sfmmup) \ 224 (tsb_hash_debug ? &hat_lock[0] : \ 225 &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)]) 226 #else /* DEBUG */ 227 #define TSB_HASH(sfmmup) &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)] 228 #endif /* DEBUG */ 229 230 231 /* sfmmu_replace_tsb() return codes. */ 232 typedef enum tsb_replace_rc { 233 TSB_SUCCESS, 234 TSB_ALLOCFAIL, 235 TSB_LOSTRACE, 236 TSB_ALREADY_SWAPPED, 237 TSB_CANTGROW 238 } tsb_replace_rc_t; 239 240 /* 241 * Flags for TSB allocation routines. 242 */ 243 #define TSB_ALLOC 0x01 244 #define TSB_FORCEALLOC 0x02 245 #define TSB_GROW 0x04 246 #define TSB_SHRINK 0x08 247 #define TSB_SWAPIN 0x10 248 249 /* 250 * Support for HAT callbacks. 251 */ 252 #define SFMMU_MAX_RELOC_CALLBACKS 10 253 int sfmmu_max_cb_id = SFMMU_MAX_RELOC_CALLBACKS; 254 static id_t sfmmu_cb_nextid = 0; 255 static id_t sfmmu_tsb_cb_id; 256 struct sfmmu_callback *sfmmu_cb_table; 257 258 /* 259 * Kernel page relocation is enabled by default for non-caged 260 * kernel pages. This has little effect unless segkmem_reloc is 261 * set, since by default kernel memory comes from inside the 262 * kernel cage. 263 */ 264 int hat_kpr_enabled = 1; 265 266 kmutex_t kpr_mutex; 267 kmutex_t kpr_suspendlock; 268 kthread_t *kreloc_thread; 269 270 /* 271 * Enable VA->PA translation sanity checking on DEBUG kernels. 272 * Disabled by default. This is incompatible with some 273 * drivers (error injector, RSM) so if it breaks you get 274 * to keep both pieces. 275 */ 276 int hat_check_vtop = 0; 277 278 /* 279 * Private sfmmu routines (prototypes) 280 */ 281 static struct hme_blk *sfmmu_shadow_hcreate(sfmmu_t *, caddr_t, int, uint_t); 282 static struct hme_blk *sfmmu_hblk_alloc(sfmmu_t *, caddr_t, 283 struct hmehash_bucket *, uint_t, hmeblk_tag, uint_t); 284 static caddr_t sfmmu_hblk_unload(struct hat *, struct hme_blk *, caddr_t, 285 caddr_t, demap_range_t *, uint_t); 286 static caddr_t sfmmu_hblk_sync(struct hat *, struct hme_blk *, caddr_t, 287 caddr_t, int); 288 static void sfmmu_hblk_free(struct hmehash_bucket *, struct hme_blk *, 289 uint64_t, struct hme_blk **); 290 static void sfmmu_hblks_list_purge(struct hme_blk **); 291 static uint_t sfmmu_get_free_hblk(struct hme_blk **, uint_t); 292 static uint_t sfmmu_put_free_hblk(struct hme_blk *, uint_t); 293 static struct hme_blk *sfmmu_hblk_steal(int); 294 static int sfmmu_steal_this_hblk(struct hmehash_bucket *, 295 struct hme_blk *, uint64_t, uint64_t, 296 struct hme_blk *); 297 static caddr_t sfmmu_hblk_unlock(struct hme_blk *, caddr_t, caddr_t); 298 299 static void sfmmu_memload_batchsmall(struct hat *, caddr_t, page_t **, 300 uint_t, uint_t, pgcnt_t); 301 void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, 302 uint_t); 303 static int sfmmu_tteload_array(sfmmu_t *, tte_t *, caddr_t, page_t **, 304 uint_t); 305 static struct hmehash_bucket *sfmmu_tteload_acquire_hashbucket(sfmmu_t *, 306 caddr_t, int); 307 static struct hme_blk *sfmmu_tteload_find_hmeblk(sfmmu_t *, 308 struct hmehash_bucket *, caddr_t, uint_t, uint_t); 309 static int sfmmu_tteload_addentry(sfmmu_t *, struct hme_blk *, tte_t *, 310 caddr_t, page_t **, uint_t); 311 static void sfmmu_tteload_release_hashbucket(struct hmehash_bucket *); 312 313 static int sfmmu_pagearray_setup(caddr_t, page_t **, tte_t *, int); 314 pfn_t sfmmu_uvatopfn(caddr_t, sfmmu_t *); 315 void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 316 #ifdef VAC 317 static void sfmmu_vac_conflict(struct hat *, caddr_t, page_t *); 318 static int sfmmu_vacconflict_array(caddr_t, page_t *, int *); 319 int tst_tnc(page_t *pp, pgcnt_t); 320 void conv_tnc(page_t *pp, int); 321 #endif 322 323 static void sfmmu_get_ctx(sfmmu_t *); 324 static void sfmmu_free_sfmmu(sfmmu_t *); 325 326 static void sfmmu_gettte(struct hat *, caddr_t, tte_t *); 327 static void sfmmu_ttesync(struct hat *, caddr_t, tte_t *, page_t *); 328 static void sfmmu_chgattr(struct hat *, caddr_t, size_t, uint_t, int); 329 330 cpuset_t sfmmu_pageunload(page_t *, struct sf_hment *, int); 331 static void hat_pagereload(struct page *, struct page *); 332 static cpuset_t sfmmu_pagesync(page_t *, struct sf_hment *, uint_t); 333 #ifdef VAC 334 void sfmmu_page_cache_array(page_t *, int, int, pgcnt_t); 335 static void sfmmu_page_cache(page_t *, int, int, int); 336 #endif 337 338 static void sfmmu_tlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 339 pfn_t, int, int, int, int); 340 static void sfmmu_ismtlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 341 pfn_t, int); 342 static void sfmmu_tlb_demap(caddr_t, sfmmu_t *, struct hme_blk *, int, int); 343 static void sfmmu_tlb_range_demap(demap_range_t *); 344 static void sfmmu_invalidate_ctx(sfmmu_t *); 345 static void sfmmu_sync_mmustate(sfmmu_t *); 346 347 static void sfmmu_tsbinfo_setup_phys(struct tsb_info *, pfn_t); 348 static int sfmmu_tsbinfo_alloc(struct tsb_info **, int, int, uint_t, 349 sfmmu_t *); 350 static void sfmmu_tsb_free(struct tsb_info *); 351 static void sfmmu_tsbinfo_free(struct tsb_info *); 352 static int sfmmu_init_tsbinfo(struct tsb_info *, int, int, uint_t, 353 sfmmu_t *); 354 355 static void sfmmu_tsb_swapin(sfmmu_t *, hatlock_t *); 356 static int sfmmu_select_tsb_szc(pgcnt_t); 357 static void sfmmu_mod_tsb(sfmmu_t *, caddr_t, tte_t *, int); 358 #define sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \ 359 sfmmu_mod_tsb(sfmmup, vaddr, tte, szc) 360 #define sfmmu_unload_tsb(sfmmup, vaddr, szc) \ 361 sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc) 362 static void sfmmu_copy_tsb(struct tsb_info *, struct tsb_info *); 363 static tsb_replace_rc_t sfmmu_replace_tsb(sfmmu_t *, struct tsb_info *, uint_t, 364 hatlock_t *, uint_t); 365 static void sfmmu_size_tsb(sfmmu_t *, int, uint64_t, uint64_t, int); 366 367 #ifdef VAC 368 void sfmmu_cache_flush(pfn_t, int); 369 void sfmmu_cache_flushcolor(int, pfn_t); 370 #endif 371 static caddr_t sfmmu_hblk_chgattr(sfmmu_t *, struct hme_blk *, caddr_t, 372 caddr_t, demap_range_t *, uint_t, int); 373 374 static uint64_t sfmmu_vtop_attr(uint_t, int mode, tte_t *); 375 static uint_t sfmmu_ptov_attr(tte_t *); 376 static caddr_t sfmmu_hblk_chgprot(sfmmu_t *, struct hme_blk *, caddr_t, 377 caddr_t, demap_range_t *, uint_t); 378 static uint_t sfmmu_vtop_prot(uint_t, uint_t *); 379 static int sfmmu_idcache_constructor(void *, void *, int); 380 static void sfmmu_idcache_destructor(void *, void *); 381 static int sfmmu_hblkcache_constructor(void *, void *, int); 382 static void sfmmu_hblkcache_destructor(void *, void *); 383 static void sfmmu_hblkcache_reclaim(void *); 384 static void sfmmu_shadow_hcleanup(sfmmu_t *, struct hme_blk *, 385 struct hmehash_bucket *); 386 static void sfmmu_free_hblks(sfmmu_t *, caddr_t, caddr_t, int); 387 static void sfmmu_rm_large_mappings(page_t *, int); 388 389 static void hat_lock_init(void); 390 static void hat_kstat_init(void); 391 static int sfmmu_kstat_percpu_update(kstat_t *ksp, int rw); 392 static void sfmmu_check_page_sizes(sfmmu_t *, int); 393 int fnd_mapping_sz(page_t *); 394 static void iment_add(struct ism_ment *, struct hat *); 395 static void iment_sub(struct ism_ment *, struct hat *); 396 static pgcnt_t ism_tsb_entries(sfmmu_t *, int szc); 397 extern void sfmmu_setup_tsbinfo(sfmmu_t *); 398 #ifdef sun4v 399 extern void sfmmu_invalidate_tsbinfo(sfmmu_t *); 400 #endif /* sun4v */ 401 extern void sfmmu_clear_utsbinfo(void); 402 403 static void sfmmu_ctx_wrap_around(mmu_ctx_t *); 404 405 /* kpm globals */ 406 #ifdef DEBUG 407 /* 408 * Enable trap level tsbmiss handling 409 */ 410 int kpm_tsbmtl = 1; 411 412 /* 413 * Flush the TLB on kpm mapout. Note: Xcalls are used (again) for the 414 * required TLB shootdowns in this case, so handle w/ care. Off by default. 415 */ 416 int kpm_tlb_flush; 417 #endif /* DEBUG */ 418 419 static void *sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *, size_t, int); 420 421 #ifdef DEBUG 422 static void sfmmu_check_hblk_flist(); 423 #endif 424 425 /* 426 * Semi-private sfmmu data structures. Some of them are initialize in 427 * startup or in hat_init. Some of them are private but accessed by 428 * assembly code or mach_sfmmu.c 429 */ 430 struct hmehash_bucket *uhme_hash; /* user hmeblk hash table */ 431 struct hmehash_bucket *khme_hash; /* kernel hmeblk hash table */ 432 uint64_t uhme_hash_pa; /* PA of uhme_hash */ 433 uint64_t khme_hash_pa; /* PA of khme_hash */ 434 int uhmehash_num; /* # of buckets in user hash table */ 435 int khmehash_num; /* # of buckets in kernel hash table */ 436 437 uint_t max_mmu_ctxdoms = 0; /* max context domains in the system */ 438 mmu_ctx_t **mmu_ctxs_tbl; /* global array of context domains */ 439 uint64_t mmu_saved_gnum = 0; /* to init incoming MMUs' gnums */ 440 441 #define DEFAULT_NUM_CTXS_PER_MMU 8192 442 static uint_t nctxs = DEFAULT_NUM_CTXS_PER_MMU; 443 444 int cache; /* describes system cache */ 445 446 caddr_t ktsb_base; /* kernel 8k-indexed tsb base address */ 447 uint64_t ktsb_pbase; /* kernel 8k-indexed tsb phys address */ 448 int ktsb_szcode; /* kernel 8k-indexed tsb size code */ 449 int ktsb_sz; /* kernel 8k-indexed tsb size */ 450 451 caddr_t ktsb4m_base; /* kernel 4m-indexed tsb base address */ 452 uint64_t ktsb4m_pbase; /* kernel 4m-indexed tsb phys address */ 453 int ktsb4m_szcode; /* kernel 4m-indexed tsb size code */ 454 int ktsb4m_sz; /* kernel 4m-indexed tsb size */ 455 456 uint64_t kpm_tsbbase; /* kernel seg_kpm 4M TSB base address */ 457 int kpm_tsbsz; /* kernel seg_kpm 4M TSB size code */ 458 uint64_t kpmsm_tsbbase; /* kernel seg_kpm 8K TSB base address */ 459 int kpmsm_tsbsz; /* kernel seg_kpm 8K TSB size code */ 460 461 #ifndef sun4v 462 int utsb_dtlb_ttenum = -1; /* index in TLB for utsb locked TTE */ 463 int utsb4m_dtlb_ttenum = -1; /* index in TLB for 4M TSB TTE */ 464 int dtlb_resv_ttenum; /* index in TLB of first reserved TTE */ 465 caddr_t utsb_vabase; /* reserved kernel virtual memory */ 466 caddr_t utsb4m_vabase; /* for trap handler TSB accesses */ 467 #endif /* sun4v */ 468 uint64_t tsb_alloc_bytes = 0; /* bytes allocated to TSBs */ 469 vmem_t *kmem_tsb_default_arena[NLGRPS_MAX]; /* For dynamic TSBs */ 470 471 /* 472 * Size to use for TSB slabs. Future platforms that support page sizes 473 * larger than 4M may wish to change these values, and provide their own 474 * assembly macros for building and decoding the TSB base register contents. 475 * Note disable_large_pages will override the value set here. 476 */ 477 uint_t tsb_slab_ttesz = TTE4M; 478 uint_t tsb_slab_size; 479 uint_t tsb_slab_shift; 480 uint_t tsb_slab_mask; /* PFN mask for TTE */ 481 482 /* largest TSB size to grow to, will be smaller on smaller memory systems */ 483 int tsb_max_growsize = UTSB_MAX_SZCODE; 484 485 /* 486 * Tunable parameters dealing with TSB policies. 487 */ 488 489 /* 490 * This undocumented tunable forces all 8K TSBs to be allocated from 491 * the kernel heap rather than from the kmem_tsb_default_arena arenas. 492 */ 493 #ifdef DEBUG 494 int tsb_forceheap = 0; 495 #endif /* DEBUG */ 496 497 /* 498 * Decide whether to use per-lgroup arenas, or one global set of 499 * TSB arenas. The default is not to break up per-lgroup, since 500 * most platforms don't recognize any tangible benefit from it. 501 */ 502 int tsb_lgrp_affinity = 0; 503 504 /* 505 * Used for growing the TSB based on the process RSS. 506 * tsb_rss_factor is based on the smallest TSB, and is 507 * shifted by the TSB size to determine if we need to grow. 508 * The default will grow the TSB if the number of TTEs for 509 * this page size exceeds 75% of the number of TSB entries, 510 * which should _almost_ eliminate all conflict misses 511 * (at the expense of using up lots and lots of memory). 512 */ 513 #define TSB_RSS_FACTOR (TSB_ENTRIES(TSB_MIN_SZCODE) * 0.75) 514 #define SFMMU_RSS_TSBSIZE(tsbszc) (tsb_rss_factor << tsbszc) 515 #define SELECT_TSB_SIZECODE(pgcnt) ( \ 516 (enable_tsb_rss_sizing)? sfmmu_select_tsb_szc(pgcnt) : \ 517 default_tsb_size) 518 #define TSB_OK_SHRINK() \ 519 (tsb_alloc_bytes > tsb_alloc_hiwater || freemem < desfree) 520 #define TSB_OK_GROW() \ 521 (tsb_alloc_bytes < tsb_alloc_hiwater && freemem > desfree) 522 523 int enable_tsb_rss_sizing = 1; 524 int tsb_rss_factor = (int)TSB_RSS_FACTOR; 525 526 /* which TSB size code to use for new address spaces or if rss sizing off */ 527 int default_tsb_size = TSB_8K_SZCODE; 528 529 static uint64_t tsb_alloc_hiwater; /* limit TSB reserved memory */ 530 uint64_t tsb_alloc_hiwater_factor; /* tsb_alloc_hiwater = physmem / this */ 531 #define TSB_ALLOC_HIWATER_FACTOR_DEFAULT 32 532 533 #ifdef DEBUG 534 static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */ 535 static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */ 536 static int tsb_alloc_mtbf = 0; /* fail allocation every n attempts */ 537 static int tsb_alloc_fail_mtbf = 0; 538 static int tsb_alloc_count = 0; 539 #endif /* DEBUG */ 540 541 /* if set to 1, will remap valid TTEs when growing TSB. */ 542 int tsb_remap_ttes = 1; 543 544 /* 545 * If we have more than this many mappings, allocate a second TSB. 546 * This default is chosen because the I/D fully associative TLBs are 547 * assumed to have at least 8 available entries. Platforms with a 548 * larger fully-associative TLB could probably override the default. 549 */ 550 int tsb_sectsb_threshold = 8; 551 552 /* 553 * kstat data 554 */ 555 struct sfmmu_global_stat sfmmu_global_stat; 556 struct sfmmu_tsbsize_stat sfmmu_tsbsize_stat; 557 558 /* 559 * Global data 560 */ 561 sfmmu_t *ksfmmup; /* kernel's hat id */ 562 563 #ifdef DEBUG 564 static void chk_tte(tte_t *, tte_t *, tte_t *, struct hme_blk *); 565 #endif 566 567 /* sfmmu locking operations */ 568 static kmutex_t *sfmmu_mlspl_enter(struct page *, int); 569 static int sfmmu_mlspl_held(struct page *, int); 570 571 kmutex_t *sfmmu_page_enter(page_t *); 572 void sfmmu_page_exit(kmutex_t *); 573 int sfmmu_page_spl_held(struct page *); 574 575 /* sfmmu internal locking operations - accessed directly */ 576 static void sfmmu_mlist_reloc_enter(page_t *, page_t *, 577 kmutex_t **, kmutex_t **); 578 static void sfmmu_mlist_reloc_exit(kmutex_t *, kmutex_t *); 579 static hatlock_t * 580 sfmmu_hat_enter(sfmmu_t *); 581 static hatlock_t * 582 sfmmu_hat_tryenter(sfmmu_t *); 583 static void sfmmu_hat_exit(hatlock_t *); 584 static void sfmmu_hat_lock_all(void); 585 static void sfmmu_hat_unlock_all(void); 586 static void sfmmu_ismhat_enter(sfmmu_t *, int); 587 static void sfmmu_ismhat_exit(sfmmu_t *, int); 588 589 /* 590 * Array of mutexes protecting a page's mapping list and p_nrm field. 591 * 592 * The hash function looks complicated, but is made up so that: 593 * 594 * "pp" not shifted, so adjacent pp values will hash to different cache lines 595 * (8 byte alignment * 8 bytes/mutes == 64 byte coherency subblock) 596 * 597 * "pp" >> mml_shift, incorporates more source bits into the hash result 598 * 599 * "& (mml_table_size - 1), should be faster than using remainder "%" 600 * 601 * Hopefully, mml_table, mml_table_size and mml_shift are all in the same 602 * cacheline, since they get declared next to each other below. We'll trust 603 * ld not to do something random. 604 */ 605 #ifdef DEBUG 606 int mlist_hash_debug = 0; 607 #define MLIST_HASH(pp) (mlist_hash_debug ? &mml_table[0] : \ 608 &mml_table[((uintptr_t)(pp) + \ 609 ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)]) 610 #else /* !DEBUG */ 611 #define MLIST_HASH(pp) &mml_table[ \ 612 ((uintptr_t)(pp) + ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)] 613 #endif /* !DEBUG */ 614 615 kmutex_t *mml_table; 616 uint_t mml_table_sz; /* must be a power of 2 */ 617 uint_t mml_shift; /* log2(mml_table_sz) + 3 for align */ 618 619 kpm_hlk_t *kpmp_table; 620 uint_t kpmp_table_sz; /* must be a power of 2 */ 621 uchar_t kpmp_shift; 622 623 kpm_shlk_t *kpmp_stable; 624 uint_t kpmp_stable_sz; /* must be a power of 2 */ 625 626 /* 627 * SPL_HASH was improved to avoid false cache line sharing 628 */ 629 #define SPL_TABLE_SIZE 128 630 #define SPL_MASK (SPL_TABLE_SIZE - 1) 631 #define SPL_SHIFT 7 /* log2(SPL_TABLE_SIZE) */ 632 633 #define SPL_INDEX(pp) \ 634 ((((uintptr_t)(pp) >> SPL_SHIFT) ^ \ 635 ((uintptr_t)(pp) >> (SPL_SHIFT << 1))) & \ 636 (SPL_TABLE_SIZE - 1)) 637 638 #define SPL_HASH(pp) \ 639 (&sfmmu_page_lock[SPL_INDEX(pp) & SPL_MASK].pad_mutex) 640 641 static pad_mutex_t sfmmu_page_lock[SPL_TABLE_SIZE]; 642 643 644 /* 645 * hat_unload_callback() will group together callbacks in order 646 * to avoid xt_sync() calls. This is the maximum size of the group. 647 */ 648 #define MAX_CB_ADDR 32 649 650 tte_t hw_tte; 651 static ulong_t sfmmu_dmr_maxbit = DMR_MAXBIT; 652 653 static char *mmu_ctx_kstat_names[] = { 654 "mmu_ctx_tsb_exceptions", 655 "mmu_ctx_tsb_raise_exception", 656 "mmu_ctx_wrap_around", 657 }; 658 659 /* 660 * Wrapper for vmem_xalloc since vmem_create only allows limited 661 * parameters for vm_source_alloc functions. This function allows us 662 * to specify alignment consistent with the size of the object being 663 * allocated. 664 */ 665 static void * 666 sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *vmp, size_t size, int vmflag) 667 { 668 return (vmem_xalloc(vmp, size, size, 0, 0, NULL, NULL, vmflag)); 669 } 670 671 /* Common code for setting tsb_alloc_hiwater. */ 672 #define SFMMU_SET_TSB_ALLOC_HIWATER(pages) tsb_alloc_hiwater = \ 673 ptob(pages) / tsb_alloc_hiwater_factor 674 675 /* 676 * Set tsb_max_growsize to allow at most all of physical memory to be mapped by 677 * a single TSB. physmem is the number of physical pages so we need physmem 8K 678 * TTEs to represent all those physical pages. We round this up by using 679 * 1<<highbit(). To figure out which size code to use, remember that the size 680 * code is just an amount to shift the smallest TSB size to get the size of 681 * this TSB. So we subtract that size, TSB_START_SIZE, from highbit() (or 682 * highbit() - 1) to get the size code for the smallest TSB that can represent 683 * all of physical memory, while erring on the side of too much. 684 * 685 * If the computed size code is less than the current tsb_max_growsize, we set 686 * tsb_max_growsize to the computed size code. In the case where the computed 687 * size code is greater than tsb_max_growsize, we have these restrictions that 688 * apply to increasing tsb_max_growsize: 689 * 1) TSBs can't grow larger than the TSB slab size 690 * 2) TSBs can't grow larger than UTSB_MAX_SZCODE. 691 */ 692 #define SFMMU_SET_TSB_MAX_GROWSIZE(pages) { \ 693 int i, szc; \ 694 \ 695 i = highbit(pages); \ 696 if ((1 << (i - 1)) == (pages)) \ 697 i--; /* 2^n case, round down */ \ 698 szc = i - TSB_START_SIZE; \ 699 if (szc < tsb_max_growsize) \ 700 tsb_max_growsize = szc; \ 701 else if ((szc > tsb_max_growsize) && \ 702 (szc <= tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT))) \ 703 tsb_max_growsize = MIN(szc, UTSB_MAX_SZCODE); \ 704 } 705 706 /* 707 * Given a pointer to an sfmmu and a TTE size code, return a pointer to the 708 * tsb_info which handles that TTE size. 709 */ 710 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) \ 711 (tsbinfop) = (sfmmup)->sfmmu_tsb; \ 712 ASSERT(sfmmu_hat_lock_held(sfmmup)); \ 713 if ((tte_szc) >= TTE4M) \ 714 (tsbinfop) = (tsbinfop)->tsb_next; 715 716 /* 717 * Return the number of mappings present in the HAT 718 * for a particular process and page size. 719 */ 720 #define SFMMU_TTE_CNT(sfmmup, szc) \ 721 (sfmmup)->sfmmu_iblk? \ 722 (sfmmup)->sfmmu_ismttecnt[(szc)] + \ 723 (sfmmup)->sfmmu_ttecnt[(szc)] : \ 724 (sfmmup)->sfmmu_ttecnt[(szc)]; 725 726 /* 727 * Macro to use to unload entries from the TSB. 728 * It has knowledge of which page sizes get replicated in the TSB 729 * and will call the appropriate unload routine for the appropriate size. 730 */ 731 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp) \ 732 { \ 733 int ttesz = get_hblk_ttesz(hmeblkp); \ 734 if (ttesz == TTE8K || ttesz == TTE4M) { \ 735 sfmmu_unload_tsb(sfmmup, addr, ttesz); \ 736 } else { \ 737 caddr_t sva = (caddr_t)get_hblk_base(hmeblkp); \ 738 caddr_t eva = sva + get_hblk_span(hmeblkp); \ 739 ASSERT(addr >= sva && addr < eva); \ 740 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); \ 741 } \ 742 } 743 744 745 /* Update tsb_alloc_hiwater after memory is configured. */ 746 /*ARGSUSED*/ 747 static void 748 sfmmu_update_tsb_post_add(void *arg, pgcnt_t delta_pages) 749 { 750 /* Assumes physmem has already been updated. */ 751 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 752 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 753 } 754 755 /* 756 * Update tsb_alloc_hiwater before memory is deleted. We'll do nothing here 757 * and update tsb_alloc_hiwater and tsb_max_growsize after the memory is 758 * deleted. 759 */ 760 /*ARGSUSED*/ 761 static int 762 sfmmu_update_tsb_pre_del(void *arg, pgcnt_t delta_pages) 763 { 764 return (0); 765 } 766 767 /* Update tsb_alloc_hiwater after memory fails to be unconfigured. */ 768 /*ARGSUSED*/ 769 static void 770 sfmmu_update_tsb_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 771 { 772 /* 773 * Whether the delete was cancelled or not, just go ahead and update 774 * tsb_alloc_hiwater and tsb_max_growsize. 775 */ 776 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 777 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 778 } 779 780 static kphysm_setup_vector_t sfmmu_update_tsb_vec = { 781 KPHYSM_SETUP_VECTOR_VERSION, /* version */ 782 sfmmu_update_tsb_post_add, /* post_add */ 783 sfmmu_update_tsb_pre_del, /* pre_del */ 784 sfmmu_update_tsb_post_del /* post_del */ 785 }; 786 787 788 /* 789 * HME_BLK HASH PRIMITIVES 790 */ 791 792 /* 793 * Enter a hme on the mapping list for page pp. 794 * When large pages are more prevalent in the system we might want to 795 * keep the mapping list in ascending order by the hment size. For now, 796 * small pages are more frequent, so don't slow it down. 797 */ 798 #define HME_ADD(hme, pp) \ 799 { \ 800 ASSERT(sfmmu_mlist_held(pp)); \ 801 \ 802 hme->hme_prev = NULL; \ 803 hme->hme_next = pp->p_mapping; \ 804 hme->hme_page = pp; \ 805 if (pp->p_mapping) { \ 806 ((struct sf_hment *)(pp->p_mapping))->hme_prev = hme;\ 807 ASSERT(pp->p_share > 0); \ 808 } else { \ 809 /* EMPTY */ \ 810 ASSERT(pp->p_share == 0); \ 811 } \ 812 pp->p_mapping = hme; \ 813 pp->p_share++; \ 814 } 815 816 /* 817 * Enter a hme on the mapping list for page pp. 818 * If we are unmapping a large translation, we need to make sure that the 819 * change is reflect in the corresponding bit of the p_index field. 820 */ 821 #define HME_SUB(hme, pp) \ 822 { \ 823 ASSERT(sfmmu_mlist_held(pp)); \ 824 ASSERT(hme->hme_page == pp || IS_PAHME(hme)); \ 825 \ 826 if (pp->p_mapping == NULL) { \ 827 panic("hme_remove - no mappings"); \ 828 } \ 829 \ 830 membar_stst(); /* ensure previous stores finish */ \ 831 \ 832 ASSERT(pp->p_share > 0); \ 833 pp->p_share--; \ 834 \ 835 if (hme->hme_prev) { \ 836 ASSERT(pp->p_mapping != hme); \ 837 ASSERT(hme->hme_prev->hme_page == pp || \ 838 IS_PAHME(hme->hme_prev)); \ 839 hme->hme_prev->hme_next = hme->hme_next; \ 840 } else { \ 841 ASSERT(pp->p_mapping == hme); \ 842 pp->p_mapping = hme->hme_next; \ 843 ASSERT((pp->p_mapping == NULL) ? \ 844 (pp->p_share == 0) : 1); \ 845 } \ 846 \ 847 if (hme->hme_next) { \ 848 ASSERT(hme->hme_next->hme_page == pp || \ 849 IS_PAHME(hme->hme_next)); \ 850 hme->hme_next->hme_prev = hme->hme_prev; \ 851 } \ 852 \ 853 /* zero out the entry */ \ 854 hme->hme_next = NULL; \ 855 hme->hme_prev = NULL; \ 856 hme->hme_page = NULL; \ 857 \ 858 if (hme_size(hme) > TTE8K) { \ 859 /* remove mappings for remainder of large pg */ \ 860 sfmmu_rm_large_mappings(pp, hme_size(hme)); \ 861 } \ 862 } 863 864 /* 865 * This function returns the hment given the hme_blk and a vaddr. 866 * It assumes addr has already been checked to belong to hme_blk's 867 * range. 868 */ 869 #define HBLKTOHME(hment, hmeblkp, addr) \ 870 { \ 871 int index; \ 872 HBLKTOHME_IDX(hment, hmeblkp, addr, index) \ 873 } 874 875 /* 876 * Version of HBLKTOHME that also returns the index in hmeblkp 877 * of the hment. 878 */ 879 #define HBLKTOHME_IDX(hment, hmeblkp, addr, idx) \ 880 { \ 881 ASSERT(in_hblk_range((hmeblkp), (addr))); \ 882 \ 883 if (get_hblk_ttesz(hmeblkp) == TTE8K) { \ 884 idx = (((uintptr_t)(addr) >> MMU_PAGESHIFT) & (NHMENTS-1)); \ 885 } else \ 886 idx = 0; \ 887 \ 888 (hment) = &(hmeblkp)->hblk_hme[idx]; \ 889 } 890 891 /* 892 * Disable any page sizes not supported by the CPU 893 */ 894 void 895 hat_init_pagesizes() 896 { 897 int i; 898 899 mmu_exported_page_sizes = 0; 900 for (i = TTE8K; i < max_mmu_page_sizes; i++) { 901 902 szc_2_userszc[i] = (uint_t)-1; 903 userszc_2_szc[i] = (uint_t)-1; 904 905 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) { 906 disable_large_pages |= (1 << i); 907 } else { 908 szc_2_userszc[i] = mmu_exported_page_sizes; 909 userszc_2_szc[mmu_exported_page_sizes] = i; 910 mmu_exported_page_sizes++; 911 } 912 } 913 914 disable_ism_large_pages |= disable_large_pages; 915 disable_auto_data_large_pages = disable_large_pages; 916 disable_auto_text_large_pages = disable_large_pages; 917 918 /* 919 * Initialize mmu-specific large page sizes. 920 */ 921 if (&mmu_large_pages_disabled) { 922 disable_large_pages |= mmu_large_pages_disabled(HAT_LOAD); 923 disable_ism_large_pages |= 924 mmu_large_pages_disabled(HAT_LOAD_SHARE); 925 disable_auto_data_large_pages |= 926 mmu_large_pages_disabled(HAT_AUTO_DATA); 927 disable_auto_text_large_pages |= 928 mmu_large_pages_disabled(HAT_AUTO_TEXT); 929 } 930 } 931 932 /* 933 * Initialize the hardware address translation structures. 934 */ 935 void 936 hat_init(void) 937 { 938 int i; 939 uint_t sz; 940 uint_t maxtsb; 941 size_t size; 942 943 hat_lock_init(); 944 hat_kstat_init(); 945 946 /* 947 * Hardware-only bits in a TTE 948 */ 949 MAKE_TTE_MASK(&hw_tte); 950 951 hat_init_pagesizes(); 952 953 /* Initialize the hash locks */ 954 for (i = 0; i < khmehash_num; i++) { 955 mutex_init(&khme_hash[i].hmehash_mutex, NULL, 956 MUTEX_DEFAULT, NULL); 957 } 958 for (i = 0; i < uhmehash_num; i++) { 959 mutex_init(&uhme_hash[i].hmehash_mutex, NULL, 960 MUTEX_DEFAULT, NULL); 961 } 962 khmehash_num--; /* make sure counter starts from 0 */ 963 uhmehash_num--; /* make sure counter starts from 0 */ 964 965 /* 966 * Allocate context domain structures. 967 * 968 * A platform may choose to modify max_mmu_ctxdoms in 969 * set_platform_defaults(). If a platform does not define 970 * a set_platform_defaults() or does not choose to modify 971 * max_mmu_ctxdoms, it gets one MMU context domain for every CPU. 972 * 973 * For sun4v, there will be one global context domain, this is to 974 * avoid the ldom cpu substitution problem. 975 * 976 * For all platforms that have CPUs sharing MMUs, this 977 * value must be defined. 978 */ 979 if (max_mmu_ctxdoms == 0) { 980 #ifndef sun4v 981 max_mmu_ctxdoms = max_ncpus; 982 #else /* sun4v */ 983 max_mmu_ctxdoms = 1; 984 #endif /* sun4v */ 985 } 986 987 size = max_mmu_ctxdoms * sizeof (mmu_ctx_t *); 988 mmu_ctxs_tbl = kmem_zalloc(size, KM_SLEEP); 989 990 /* mmu_ctx_t is 64 bytes aligned */ 991 mmuctxdom_cache = kmem_cache_create("mmuctxdom_cache", 992 sizeof (mmu_ctx_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 993 /* 994 * MMU context domain initialization for the Boot CPU. 995 * This needs the context domains array allocated above. 996 */ 997 mutex_enter(&cpu_lock); 998 sfmmu_cpu_init(CPU); 999 mutex_exit(&cpu_lock); 1000 1001 /* 1002 * Intialize ism mapping list lock. 1003 */ 1004 1005 mutex_init(&ism_mlist_lock, NULL, MUTEX_DEFAULT, NULL); 1006 1007 /* 1008 * Each sfmmu structure carries an array of MMU context info 1009 * structures, one per context domain. The size of this array depends 1010 * on the maximum number of context domains. So, the size of the 1011 * sfmmu structure varies per platform. 1012 * 1013 * sfmmu is allocated from static arena, because trap 1014 * handler at TL > 0 is not allowed to touch kernel relocatable 1015 * memory. sfmmu's alignment is changed to 64 bytes from 1016 * default 8 bytes, as the lower 6 bits will be used to pass 1017 * pgcnt to vtag_flush_pgcnt_tl1. 1018 */ 1019 size = sizeof (sfmmu_t) + sizeof (sfmmu_ctx_t) * (max_mmu_ctxdoms - 1); 1020 1021 sfmmuid_cache = kmem_cache_create("sfmmuid_cache", size, 1022 64, sfmmu_idcache_constructor, sfmmu_idcache_destructor, 1023 NULL, NULL, static_arena, 0); 1024 1025 sfmmu_tsbinfo_cache = kmem_cache_create("sfmmu_tsbinfo_cache", 1026 sizeof (struct tsb_info), 0, NULL, NULL, NULL, NULL, NULL, 0); 1027 1028 /* 1029 * Since we only use the tsb8k cache to "borrow" pages for TSBs 1030 * from the heap when low on memory or when TSB_FORCEALLOC is 1031 * specified, don't use magazines to cache them--we want to return 1032 * them to the system as quickly as possible. 1033 */ 1034 sfmmu_tsb8k_cache = kmem_cache_create("sfmmu_tsb8k_cache", 1035 MMU_PAGESIZE, MMU_PAGESIZE, NULL, NULL, NULL, NULL, 1036 static_arena, KMC_NOMAGAZINE); 1037 1038 /* 1039 * Set tsb_alloc_hiwater to 1/tsb_alloc_hiwater_factor of physical 1040 * memory, which corresponds to the old static reserve for TSBs. 1041 * tsb_alloc_hiwater_factor defaults to 32. This caps the amount of 1042 * memory we'll allocate for TSB slabs; beyond this point TSB 1043 * allocations will be taken from the kernel heap (via 1044 * sfmmu_tsb8k_cache) and will be throttled as would any other kmem 1045 * consumer. 1046 */ 1047 if (tsb_alloc_hiwater_factor == 0) { 1048 tsb_alloc_hiwater_factor = TSB_ALLOC_HIWATER_FACTOR_DEFAULT; 1049 } 1050 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 1051 1052 /* Set tsb_max_growsize. */ 1053 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 1054 1055 /* 1056 * On smaller memory systems, allocate TSB memory in smaller chunks 1057 * than the default 4M slab size. We also honor disable_large_pages 1058 * here. 1059 * 1060 * The trap handlers need to be patched with the final slab shift, 1061 * since they need to be able to construct the TSB pointer at runtime. 1062 */ 1063 if (tsb_max_growsize <= TSB_512K_SZCODE) 1064 tsb_slab_ttesz = TTE512K; 1065 1066 for (sz = tsb_slab_ttesz; sz > 0; sz--) { 1067 if (!(disable_large_pages & (1 << sz))) 1068 break; 1069 } 1070 1071 tsb_slab_ttesz = sz; 1072 tsb_slab_shift = MMU_PAGESHIFT + (sz << 1) + sz; 1073 tsb_slab_size = 1 << tsb_slab_shift; 1074 tsb_slab_mask = (1 << (tsb_slab_shift - MMU_PAGESHIFT)) - 1; 1075 1076 maxtsb = tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT); 1077 if (tsb_max_growsize > maxtsb) 1078 tsb_max_growsize = maxtsb; 1079 1080 /* 1081 * Set up memory callback to update tsb_alloc_hiwater and 1082 * tsb_max_growsize. 1083 */ 1084 i = kphysm_setup_func_register(&sfmmu_update_tsb_vec, (void *) 0); 1085 ASSERT(i == 0); 1086 1087 /* 1088 * kmem_tsb_arena is the source from which large TSB slabs are 1089 * drawn. The quantum of this arena corresponds to the largest 1090 * TSB size we can dynamically allocate for user processes. 1091 * Currently it must also be a supported page size since we 1092 * use exactly one translation entry to map each slab page. 1093 * 1094 * The per-lgroup kmem_tsb_default_arena arenas are the arenas from 1095 * which most TSBs are allocated. Since most TSB allocations are 1096 * typically 8K we have a kmem cache we stack on top of each 1097 * kmem_tsb_default_arena to speed up those allocations. 1098 * 1099 * Note the two-level scheme of arenas is required only 1100 * because vmem_create doesn't allow us to specify alignment 1101 * requirements. If this ever changes the code could be 1102 * simplified to use only one level of arenas. 1103 */ 1104 kmem_tsb_arena = vmem_create("kmem_tsb", NULL, 0, tsb_slab_size, 1105 sfmmu_vmem_xalloc_aligned_wrapper, vmem_xfree, heap_arena, 1106 0, VM_SLEEP); 1107 1108 if (tsb_lgrp_affinity) { 1109 char s[50]; 1110 for (i = 0; i < NLGRPS_MAX; i++) { 1111 (void) sprintf(s, "kmem_tsb_lgrp%d", i); 1112 kmem_tsb_default_arena[i] = 1113 vmem_create(s, NULL, 0, PAGESIZE, 1114 sfmmu_tsb_segkmem_alloc, sfmmu_tsb_segkmem_free, 1115 kmem_tsb_arena, 0, VM_SLEEP | VM_BESTFIT); 1116 (void) sprintf(s, "sfmmu_tsb_lgrp%d_cache", i); 1117 sfmmu_tsb_cache[i] = kmem_cache_create(s, PAGESIZE, 1118 PAGESIZE, NULL, NULL, NULL, NULL, 1119 kmem_tsb_default_arena[i], 0); 1120 } 1121 } else { 1122 kmem_tsb_default_arena[0] = vmem_create("kmem_tsb_default", 1123 NULL, 0, PAGESIZE, sfmmu_tsb_segkmem_alloc, 1124 sfmmu_tsb_segkmem_free, kmem_tsb_arena, 0, 1125 VM_SLEEP | VM_BESTFIT); 1126 1127 sfmmu_tsb_cache[0] = kmem_cache_create("sfmmu_tsb_cache", 1128 PAGESIZE, PAGESIZE, NULL, NULL, NULL, NULL, 1129 kmem_tsb_default_arena[0], 0); 1130 } 1131 1132 sfmmu8_cache = kmem_cache_create("sfmmu8_cache", HME8BLK_SZ, 1133 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1134 sfmmu_hblkcache_destructor, 1135 sfmmu_hblkcache_reclaim, (void *)HME8BLK_SZ, 1136 hat_memload_arena, KMC_NOHASH); 1137 1138 hat_memload1_arena = vmem_create("hat_memload1", NULL, 0, PAGESIZE, 1139 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); 1140 1141 sfmmu1_cache = kmem_cache_create("sfmmu1_cache", HME1BLK_SZ, 1142 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1143 sfmmu_hblkcache_destructor, 1144 NULL, (void *)HME1BLK_SZ, 1145 hat_memload1_arena, KMC_NOHASH); 1146 1147 pa_hment_cache = kmem_cache_create("pa_hment_cache", PAHME_SZ, 1148 0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH); 1149 1150 ism_blk_cache = kmem_cache_create("ism_blk_cache", 1151 sizeof (ism_blk_t), ecache_alignsize, NULL, NULL, 1152 NULL, NULL, static_arena, KMC_NOHASH); 1153 1154 ism_ment_cache = kmem_cache_create("ism_ment_cache", 1155 sizeof (ism_ment_t), 0, NULL, NULL, 1156 NULL, NULL, NULL, 0); 1157 1158 /* 1159 * We grab the first hat for the kernel, 1160 */ 1161 AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER); 1162 kas.a_hat = hat_alloc(&kas); 1163 AS_LOCK_EXIT(&kas, &kas.a_lock); 1164 1165 /* 1166 * Initialize hblk_reserve. 1167 */ 1168 ((struct hme_blk *)hblk_reserve)->hblk_nextpa = 1169 va_to_pa((caddr_t)hblk_reserve); 1170 1171 #ifndef UTSB_PHYS 1172 /* 1173 * Reserve some kernel virtual address space for the locked TTEs 1174 * that allow us to probe the TSB from TL>0. 1175 */ 1176 utsb_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1177 0, 0, NULL, NULL, VM_SLEEP); 1178 utsb4m_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1179 0, 0, NULL, NULL, VM_SLEEP); 1180 #endif 1181 1182 #ifdef VAC 1183 /* 1184 * The big page VAC handling code assumes VAC 1185 * will not be bigger than the smallest big 1186 * page- which is 64K. 1187 */ 1188 if (TTEPAGES(TTE64K) < CACHE_NUM_COLOR) { 1189 cmn_err(CE_PANIC, "VAC too big!"); 1190 } 1191 #endif 1192 1193 (void) xhat_init(); 1194 1195 uhme_hash_pa = va_to_pa(uhme_hash); 1196 khme_hash_pa = va_to_pa(khme_hash); 1197 1198 /* 1199 * Initialize relocation locks. kpr_suspendlock is held 1200 * at PIL_MAX to prevent interrupts from pinning the holder 1201 * of a suspended TTE which may access it leading to a 1202 * deadlock condition. 1203 */ 1204 mutex_init(&kpr_mutex, NULL, MUTEX_DEFAULT, NULL); 1205 mutex_init(&kpr_suspendlock, NULL, MUTEX_SPIN, (void *)PIL_MAX); 1206 1207 /* 1208 * Pre-allocate hrm_hashtab before enabling the collection of 1209 * refmod statistics. Allocating on the fly would mean us 1210 * running the risk of suffering recursive mutex enters or 1211 * deadlocks. 1212 */ 1213 hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *), 1214 KM_SLEEP); 1215 } 1216 1217 /* 1218 * Initialize locking for the hat layer, called early during boot. 1219 */ 1220 static void 1221 hat_lock_init() 1222 { 1223 int i; 1224 1225 /* 1226 * initialize the array of mutexes protecting a page's mapping 1227 * list and p_nrm field. 1228 */ 1229 for (i = 0; i < mml_table_sz; i++) 1230 mutex_init(&mml_table[i], NULL, MUTEX_DEFAULT, NULL); 1231 1232 if (kpm_enable) { 1233 for (i = 0; i < kpmp_table_sz; i++) { 1234 mutex_init(&kpmp_table[i].khl_mutex, NULL, 1235 MUTEX_DEFAULT, NULL); 1236 } 1237 } 1238 1239 /* 1240 * Initialize array of mutex locks that protects sfmmu fields and 1241 * TSB lists. 1242 */ 1243 for (i = 0; i < SFMMU_NUM_LOCK; i++) 1244 mutex_init(HATLOCK_MUTEXP(&hat_lock[i]), NULL, MUTEX_DEFAULT, 1245 NULL); 1246 } 1247 1248 extern caddr_t kmem64_base, kmem64_end; 1249 1250 #define SFMMU_KERNEL_MAXVA \ 1251 (kmem64_base ? (uintptr_t)kmem64_end : (SYSLIMIT)) 1252 1253 /* 1254 * Allocate a hat structure. 1255 * Called when an address space first uses a hat. 1256 */ 1257 struct hat * 1258 hat_alloc(struct as *as) 1259 { 1260 sfmmu_t *sfmmup; 1261 int i; 1262 uint64_t cnum; 1263 extern uint_t get_color_start(struct as *); 1264 1265 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 1266 sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP); 1267 sfmmup->sfmmu_as = as; 1268 sfmmup->sfmmu_flags = 0; 1269 LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock); 1270 1271 if (as == &kas) { 1272 ksfmmup = sfmmup; 1273 sfmmup->sfmmu_cext = 0; 1274 cnum = KCONTEXT; 1275 1276 sfmmup->sfmmu_clrstart = 0; 1277 sfmmup->sfmmu_tsb = NULL; 1278 /* 1279 * hat_kern_setup() will call sfmmu_init_ktsbinfo() 1280 * to setup tsb_info for ksfmmup. 1281 */ 1282 } else { 1283 1284 /* 1285 * Just set to invalid ctx. When it faults, it will 1286 * get a valid ctx. This would avoid the situation 1287 * where we get a ctx, but it gets stolen and then 1288 * we fault when we try to run and so have to get 1289 * another ctx. 1290 */ 1291 sfmmup->sfmmu_cext = 0; 1292 cnum = INVALID_CONTEXT; 1293 1294 /* initialize original physical page coloring bin */ 1295 sfmmup->sfmmu_clrstart = get_color_start(as); 1296 #ifdef DEBUG 1297 if (tsb_random_size) { 1298 uint32_t randval = (uint32_t)gettick() >> 4; 1299 int size = randval % (tsb_max_growsize + 1); 1300 1301 /* chose a random tsb size for stress testing */ 1302 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size, 1303 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1304 } else 1305 #endif /* DEBUG */ 1306 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, 1307 default_tsb_size, 1308 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1309 sfmmup->sfmmu_flags = HAT_SWAPPED; 1310 ASSERT(sfmmup->sfmmu_tsb != NULL); 1311 } 1312 1313 ASSERT(max_mmu_ctxdoms > 0); 1314 for (i = 0; i < max_mmu_ctxdoms; i++) { 1315 sfmmup->sfmmu_ctxs[i].cnum = cnum; 1316 sfmmup->sfmmu_ctxs[i].gnum = 0; 1317 } 1318 1319 sfmmu_setup_tsbinfo(sfmmup); 1320 for (i = 0; i < max_mmu_page_sizes; i++) { 1321 sfmmup->sfmmu_ttecnt[i] = 0; 1322 sfmmup->sfmmu_ismttecnt[i] = 0; 1323 sfmmup->sfmmu_pgsz[i] = TTE8K; 1324 } 1325 1326 sfmmup->sfmmu_iblk = NULL; 1327 sfmmup->sfmmu_ismhat = 0; 1328 sfmmup->sfmmu_ismblkpa = (uint64_t)-1; 1329 if (sfmmup == ksfmmup) { 1330 CPUSET_ALL(sfmmup->sfmmu_cpusran); 1331 } else { 1332 CPUSET_ZERO(sfmmup->sfmmu_cpusran); 1333 } 1334 sfmmup->sfmmu_free = 0; 1335 sfmmup->sfmmu_rmstat = 0; 1336 sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart; 1337 sfmmup->sfmmu_xhat_provider = NULL; 1338 cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL); 1339 return (sfmmup); 1340 } 1341 1342 /* 1343 * Create per-MMU context domain kstats for a given MMU ctx. 1344 */ 1345 static void 1346 sfmmu_mmu_kstat_create(mmu_ctx_t *mmu_ctxp) 1347 { 1348 mmu_ctx_stat_t stat; 1349 kstat_t *mmu_kstat; 1350 1351 ASSERT(MUTEX_HELD(&cpu_lock)); 1352 ASSERT(mmu_ctxp->mmu_kstat == NULL); 1353 1354 mmu_kstat = kstat_create("unix", mmu_ctxp->mmu_idx, "mmu_ctx", 1355 "hat", KSTAT_TYPE_NAMED, MMU_CTX_NUM_STATS, KSTAT_FLAG_VIRTUAL); 1356 1357 if (mmu_kstat == NULL) { 1358 cmn_err(CE_WARN, "kstat_create for MMU %d failed", 1359 mmu_ctxp->mmu_idx); 1360 } else { 1361 mmu_kstat->ks_data = mmu_ctxp->mmu_kstat_data; 1362 for (stat = 0; stat < MMU_CTX_NUM_STATS; stat++) 1363 kstat_named_init(&mmu_ctxp->mmu_kstat_data[stat], 1364 mmu_ctx_kstat_names[stat], KSTAT_DATA_INT64); 1365 mmu_ctxp->mmu_kstat = mmu_kstat; 1366 kstat_install(mmu_kstat); 1367 } 1368 } 1369 1370 /* 1371 * plat_cpuid_to_mmu_ctx_info() is a platform interface that returns MMU 1372 * context domain information for a given CPU. If a platform does not 1373 * specify that interface, then the function below is used instead to return 1374 * default information. The defaults are as follows: 1375 * 1376 * - For sun4u systems there's one MMU context domain per CPU. 1377 * This default is used by all sun4u systems except OPL. OPL systems 1378 * provide platform specific interface to map CPU ids to MMU ids 1379 * because on OPL more than 1 CPU shares a single MMU. 1380 * Note that on sun4v, there is one global context domain for 1381 * the entire system. This is to avoid running into potential problem 1382 * with ldom physical cpu substitution feature. 1383 * - The number of MMU context IDs supported on any CPU in the 1384 * system is 8K. 1385 */ 1386 /*ARGSUSED*/ 1387 static void 1388 sfmmu_cpuid_to_mmu_ctx_info(processorid_t cpuid, mmu_ctx_info_t *infop) 1389 { 1390 infop->mmu_nctxs = nctxs; 1391 #ifndef sun4v 1392 infop->mmu_idx = cpu[cpuid]->cpu_seqid; 1393 #else /* sun4v */ 1394 infop->mmu_idx = 0; 1395 #endif /* sun4v */ 1396 } 1397 1398 /* 1399 * Called during CPU initialization to set the MMU context-related information 1400 * for a CPU. 1401 * 1402 * cpu_lock serializes accesses to mmu_ctxs and mmu_saved_gnum. 1403 */ 1404 void 1405 sfmmu_cpu_init(cpu_t *cp) 1406 { 1407 mmu_ctx_info_t info; 1408 mmu_ctx_t *mmu_ctxp; 1409 1410 ASSERT(MUTEX_HELD(&cpu_lock)); 1411 1412 if (&plat_cpuid_to_mmu_ctx_info == NULL) 1413 sfmmu_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1414 else 1415 plat_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1416 1417 ASSERT(info.mmu_idx < max_mmu_ctxdoms); 1418 1419 if ((mmu_ctxp = mmu_ctxs_tbl[info.mmu_idx]) == NULL) { 1420 /* Each mmu_ctx is cacheline aligned. */ 1421 mmu_ctxp = kmem_cache_alloc(mmuctxdom_cache, KM_SLEEP); 1422 bzero(mmu_ctxp, sizeof (mmu_ctx_t)); 1423 1424 mutex_init(&mmu_ctxp->mmu_lock, NULL, MUTEX_SPIN, 1425 (void *)ipltospl(DISP_LEVEL)); 1426 mmu_ctxp->mmu_idx = info.mmu_idx; 1427 mmu_ctxp->mmu_nctxs = info.mmu_nctxs; 1428 /* 1429 * Globally for lifetime of a system, 1430 * gnum must always increase. 1431 * mmu_saved_gnum is protected by the cpu_lock. 1432 */ 1433 mmu_ctxp->mmu_gnum = mmu_saved_gnum + 1; 1434 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 1435 1436 sfmmu_mmu_kstat_create(mmu_ctxp); 1437 1438 mmu_ctxs_tbl[info.mmu_idx] = mmu_ctxp; 1439 } else { 1440 ASSERT(mmu_ctxp->mmu_idx == info.mmu_idx); 1441 } 1442 1443 /* 1444 * The mmu_lock is acquired here to prevent races with 1445 * the wrap-around code. 1446 */ 1447 mutex_enter(&mmu_ctxp->mmu_lock); 1448 1449 1450 mmu_ctxp->mmu_ncpus++; 1451 CPUSET_ADD(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1452 CPU_MMU_IDX(cp) = info.mmu_idx; 1453 CPU_MMU_CTXP(cp) = mmu_ctxp; 1454 1455 mutex_exit(&mmu_ctxp->mmu_lock); 1456 } 1457 1458 /* 1459 * Called to perform MMU context-related cleanup for a CPU. 1460 */ 1461 void 1462 sfmmu_cpu_cleanup(cpu_t *cp) 1463 { 1464 mmu_ctx_t *mmu_ctxp; 1465 1466 ASSERT(MUTEX_HELD(&cpu_lock)); 1467 1468 mmu_ctxp = CPU_MMU_CTXP(cp); 1469 ASSERT(mmu_ctxp != NULL); 1470 1471 /* 1472 * The mmu_lock is acquired here to prevent races with 1473 * the wrap-around code. 1474 */ 1475 mutex_enter(&mmu_ctxp->mmu_lock); 1476 1477 CPU_MMU_CTXP(cp) = NULL; 1478 1479 CPUSET_DEL(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1480 if (--mmu_ctxp->mmu_ncpus == 0) { 1481 mmu_ctxs_tbl[mmu_ctxp->mmu_idx] = NULL; 1482 mutex_exit(&mmu_ctxp->mmu_lock); 1483 mutex_destroy(&mmu_ctxp->mmu_lock); 1484 1485 if (mmu_ctxp->mmu_kstat) 1486 kstat_delete(mmu_ctxp->mmu_kstat); 1487 1488 /* mmu_saved_gnum is protected by the cpu_lock. */ 1489 if (mmu_saved_gnum < mmu_ctxp->mmu_gnum) 1490 mmu_saved_gnum = mmu_ctxp->mmu_gnum; 1491 1492 kmem_cache_free(mmuctxdom_cache, mmu_ctxp); 1493 1494 return; 1495 } 1496 1497 mutex_exit(&mmu_ctxp->mmu_lock); 1498 } 1499 1500 /* 1501 * Hat_setup, makes an address space context the current active one. 1502 * In sfmmu this translates to setting the secondary context with the 1503 * corresponding context. 1504 */ 1505 void 1506 hat_setup(struct hat *sfmmup, int allocflag) 1507 { 1508 hatlock_t *hatlockp; 1509 1510 /* Init needs some special treatment. */ 1511 if (allocflag == HAT_INIT) { 1512 /* 1513 * Make sure that we have 1514 * 1. a TSB 1515 * 2. a valid ctx that doesn't get stolen after this point. 1516 */ 1517 hatlockp = sfmmu_hat_enter(sfmmup); 1518 1519 /* 1520 * Swap in the TSB. hat_init() allocates tsbinfos without 1521 * TSBs, but we need one for init, since the kernel does some 1522 * special things to set up its stack and needs the TSB to 1523 * resolve page faults. 1524 */ 1525 sfmmu_tsb_swapin(sfmmup, hatlockp); 1526 1527 sfmmu_get_ctx(sfmmup); 1528 1529 sfmmu_hat_exit(hatlockp); 1530 } else { 1531 ASSERT(allocflag == HAT_ALLOC); 1532 1533 hatlockp = sfmmu_hat_enter(sfmmup); 1534 kpreempt_disable(); 1535 1536 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); 1537 1538 /* 1539 * sfmmu_setctx_sec takes <pgsz|cnum> as a parameter, 1540 * pagesize bits don't matter in this case since we are passing 1541 * INVALID_CONTEXT to it. 1542 */ 1543 sfmmu_setctx_sec(INVALID_CONTEXT); 1544 sfmmu_clear_utsbinfo(); 1545 1546 kpreempt_enable(); 1547 sfmmu_hat_exit(hatlockp); 1548 } 1549 } 1550 1551 /* 1552 * Free all the translation resources for the specified address space. 1553 * Called from as_free when an address space is being destroyed. 1554 */ 1555 void 1556 hat_free_start(struct hat *sfmmup) 1557 { 1558 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 1559 ASSERT(sfmmup != ksfmmup); 1560 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1561 1562 sfmmup->sfmmu_free = 1; 1563 } 1564 1565 void 1566 hat_free_end(struct hat *sfmmup) 1567 { 1568 int i; 1569 1570 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1571 if (sfmmup->sfmmu_ismhat) { 1572 for (i = 0; i < mmu_page_sizes; i++) { 1573 sfmmup->sfmmu_ttecnt[i] = 0; 1574 sfmmup->sfmmu_ismttecnt[i] = 0; 1575 } 1576 } else { 1577 /* EMPTY */ 1578 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 1579 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 1580 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 1581 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 1582 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 1583 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 1584 } 1585 1586 if (sfmmup->sfmmu_rmstat) { 1587 hat_freestat(sfmmup->sfmmu_as, NULL); 1588 } 1589 1590 while (sfmmup->sfmmu_tsb != NULL) { 1591 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next; 1592 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb); 1593 sfmmup->sfmmu_tsb = next; 1594 } 1595 sfmmu_free_sfmmu(sfmmup); 1596 1597 kmem_cache_free(sfmmuid_cache, sfmmup); 1598 } 1599 1600 /* 1601 * Set up any translation structures, for the specified address space, 1602 * that are needed or preferred when the process is being swapped in. 1603 */ 1604 /* ARGSUSED */ 1605 void 1606 hat_swapin(struct hat *hat) 1607 { 1608 ASSERT(hat->sfmmu_xhat_provider == NULL); 1609 } 1610 1611 /* 1612 * Free all of the translation resources, for the specified address space, 1613 * that can be freed while the process is swapped out. Called from as_swapout. 1614 * Also, free up the ctx that this process was using. 1615 */ 1616 void 1617 hat_swapout(struct hat *sfmmup) 1618 { 1619 struct hmehash_bucket *hmebp; 1620 struct hme_blk *hmeblkp; 1621 struct hme_blk *pr_hblk = NULL; 1622 struct hme_blk *nx_hblk; 1623 int i; 1624 uint64_t hblkpa, prevpa, nx_pa; 1625 struct hme_blk *list = NULL; 1626 hatlock_t *hatlockp; 1627 struct tsb_info *tsbinfop; 1628 struct free_tsb { 1629 struct free_tsb *next; 1630 struct tsb_info *tsbinfop; 1631 }; /* free list of TSBs */ 1632 struct free_tsb *freelist, *last, *next; 1633 1634 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1635 SFMMU_STAT(sf_swapout); 1636 1637 /* 1638 * There is no way to go from an as to all its translations in sfmmu. 1639 * Here is one of the times when we take the big hit and traverse 1640 * the hash looking for hme_blks to free up. Not only do we free up 1641 * this as hme_blks but all those that are free. We are obviously 1642 * swapping because we need memory so let's free up as much 1643 * as we can. 1644 * 1645 * Note that we don't flush TLB/TSB here -- it's not necessary 1646 * because: 1647 * 1) we free the ctx we're using and throw away the TSB(s); 1648 * 2) processes aren't runnable while being swapped out. 1649 */ 1650 ASSERT(sfmmup != KHATID); 1651 for (i = 0; i <= UHMEHASH_SZ; i++) { 1652 hmebp = &uhme_hash[i]; 1653 SFMMU_HASH_LOCK(hmebp); 1654 hmeblkp = hmebp->hmeblkp; 1655 hblkpa = hmebp->hmeh_nextpa; 1656 prevpa = 0; 1657 pr_hblk = NULL; 1658 while (hmeblkp) { 1659 1660 ASSERT(!hmeblkp->hblk_xhat_bit); 1661 1662 if ((hmeblkp->hblk_tag.htag_id == sfmmup) && 1663 !hmeblkp->hblk_shw_bit && !hmeblkp->hblk_lckcnt) { 1664 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 1665 (caddr_t)get_hblk_base(hmeblkp), 1666 get_hblk_endaddr(hmeblkp), 1667 NULL, HAT_UNLOAD); 1668 } 1669 nx_hblk = hmeblkp->hblk_next; 1670 nx_pa = hmeblkp->hblk_nextpa; 1671 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 1672 ASSERT(!hmeblkp->hblk_lckcnt); 1673 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 1674 prevpa, pr_hblk); 1675 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 1676 } else { 1677 pr_hblk = hmeblkp; 1678 prevpa = hblkpa; 1679 } 1680 hmeblkp = nx_hblk; 1681 hblkpa = nx_pa; 1682 } 1683 SFMMU_HASH_UNLOCK(hmebp); 1684 } 1685 1686 sfmmu_hblks_list_purge(&list); 1687 1688 /* 1689 * Now free up the ctx so that others can reuse it. 1690 */ 1691 hatlockp = sfmmu_hat_enter(sfmmup); 1692 1693 sfmmu_invalidate_ctx(sfmmup); 1694 1695 /* 1696 * Free TSBs, but not tsbinfos, and set SWAPPED flag. 1697 * If TSBs were never swapped in, just return. 1698 * This implies that we don't support partial swapping 1699 * of TSBs -- either all are swapped out, or none are. 1700 * 1701 * We must hold the HAT lock here to prevent racing with another 1702 * thread trying to unmap TTEs from the TSB or running the post- 1703 * relocator after relocating the TSB's memory. Unfortunately, we 1704 * can't free memory while holding the HAT lock or we could 1705 * deadlock, so we build a list of TSBs to be freed after marking 1706 * the tsbinfos as swapped out and free them after dropping the 1707 * lock. 1708 */ 1709 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 1710 sfmmu_hat_exit(hatlockp); 1711 return; 1712 } 1713 1714 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED); 1715 last = freelist = NULL; 1716 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 1717 tsbinfop = tsbinfop->tsb_next) { 1718 ASSERT((tsbinfop->tsb_flags & TSB_SWAPPED) == 0); 1719 1720 /* 1721 * Cast the TSB into a struct free_tsb and put it on the free 1722 * list. 1723 */ 1724 if (freelist == NULL) { 1725 last = freelist = (struct free_tsb *)tsbinfop->tsb_va; 1726 } else { 1727 last->next = (struct free_tsb *)tsbinfop->tsb_va; 1728 last = last->next; 1729 } 1730 last->next = NULL; 1731 last->tsbinfop = tsbinfop; 1732 tsbinfop->tsb_flags |= TSB_SWAPPED; 1733 /* 1734 * Zero out the TTE to clear the valid bit. 1735 * Note we can't use a value like 0xbad because we want to 1736 * ensure diagnostic bits are NEVER set on TTEs that might 1737 * be loaded. The intent is to catch any invalid access 1738 * to the swapped TSB, such as a thread running with a valid 1739 * context without first calling sfmmu_tsb_swapin() to 1740 * allocate TSB memory. 1741 */ 1742 tsbinfop->tsb_tte.ll = 0; 1743 } 1744 1745 #ifdef sun4v 1746 if (freelist) 1747 sfmmu_invalidate_tsbinfo(sfmmup); 1748 #endif /* sun4v */ 1749 1750 /* Now we can drop the lock and free the TSB memory. */ 1751 sfmmu_hat_exit(hatlockp); 1752 for (; freelist != NULL; freelist = next) { 1753 next = freelist->next; 1754 sfmmu_tsb_free(freelist->tsbinfop); 1755 } 1756 } 1757 1758 /* 1759 * Duplicate the translations of an as into another newas 1760 */ 1761 /* ARGSUSED */ 1762 int 1763 hat_dup(struct hat *hat, struct hat *newhat, caddr_t addr, size_t len, 1764 uint_t flag) 1765 { 1766 ASSERT(hat->sfmmu_xhat_provider == NULL); 1767 ASSERT((flag == 0) || (flag == HAT_DUP_ALL) || (flag == HAT_DUP_COW)); 1768 1769 if (flag == HAT_DUP_COW) { 1770 panic("hat_dup: HAT_DUP_COW not supported"); 1771 } 1772 return (0); 1773 } 1774 1775 /* 1776 * Set up addr to map to page pp with protection prot. 1777 * As an optimization we also load the TSB with the 1778 * corresponding tte but it is no big deal if the tte gets kicked out. 1779 */ 1780 void 1781 hat_memload(struct hat *hat, caddr_t addr, struct page *pp, 1782 uint_t attr, uint_t flags) 1783 { 1784 tte_t tte; 1785 1786 1787 ASSERT(hat != NULL); 1788 ASSERT(PAGE_LOCKED(pp)); 1789 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 1790 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1791 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1792 1793 if (PP_ISFREE(pp)) { 1794 panic("hat_memload: loading a mapping to free page %p", 1795 (void *)pp); 1796 } 1797 1798 if (hat->sfmmu_xhat_provider) { 1799 XHAT_MEMLOAD(hat, addr, pp, attr, flags); 1800 return; 1801 } 1802 1803 ASSERT((hat == ksfmmup) || 1804 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1805 1806 if (flags & ~SFMMU_LOAD_ALLFLAG) 1807 cmn_err(CE_NOTE, "hat_memload: unsupported flags %d", 1808 flags & ~SFMMU_LOAD_ALLFLAG); 1809 1810 if (hat->sfmmu_rmstat) 1811 hat_resvstat(MMU_PAGESIZE, hat->sfmmu_as, addr); 1812 1813 #if defined(SF_ERRATA_57) 1814 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1815 (addr < errata57_limit) && (attr & PROT_EXEC) && 1816 !(flags & HAT_LOAD_SHARE)) { 1817 cmn_err(CE_WARN, "hat_memload: illegal attempt to make user " 1818 " page executable"); 1819 attr &= ~PROT_EXEC; 1820 } 1821 #endif 1822 1823 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 1824 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, flags); 1825 1826 /* 1827 * Check TSB and TLB page sizes. 1828 */ 1829 if ((flags & HAT_LOAD_SHARE) == 0) { 1830 sfmmu_check_page_sizes(hat, 1); 1831 } 1832 } 1833 1834 /* 1835 * hat_devload can be called to map real memory (e.g. 1836 * /dev/kmem) and even though hat_devload will determine pf is 1837 * for memory, it will be unable to get a shared lock on the 1838 * page (because someone else has it exclusively) and will 1839 * pass dp = NULL. If tteload doesn't get a non-NULL 1840 * page pointer it can't cache memory. 1841 */ 1842 void 1843 hat_devload(struct hat *hat, caddr_t addr, size_t len, pfn_t pfn, 1844 uint_t attr, int flags) 1845 { 1846 tte_t tte; 1847 struct page *pp = NULL; 1848 int use_lgpg = 0; 1849 1850 ASSERT(hat != NULL); 1851 1852 if (hat->sfmmu_xhat_provider) { 1853 XHAT_DEVLOAD(hat, addr, len, pfn, attr, flags); 1854 return; 1855 } 1856 1857 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1858 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1859 ASSERT((hat == ksfmmup) || 1860 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1861 if (len == 0) 1862 panic("hat_devload: zero len"); 1863 if (flags & ~SFMMU_LOAD_ALLFLAG) 1864 cmn_err(CE_NOTE, "hat_devload: unsupported flags %d", 1865 flags & ~SFMMU_LOAD_ALLFLAG); 1866 1867 #if defined(SF_ERRATA_57) 1868 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1869 (addr < errata57_limit) && (attr & PROT_EXEC) && 1870 !(flags & HAT_LOAD_SHARE)) { 1871 cmn_err(CE_WARN, "hat_devload: illegal attempt to make user " 1872 " page executable"); 1873 attr &= ~PROT_EXEC; 1874 } 1875 #endif 1876 1877 /* 1878 * If it's a memory page find its pp 1879 */ 1880 if (!(flags & HAT_LOAD_NOCONSIST) && pf_is_memory(pfn)) { 1881 pp = page_numtopp_nolock(pfn); 1882 if (pp == NULL) { 1883 flags |= HAT_LOAD_NOCONSIST; 1884 } else { 1885 if (PP_ISFREE(pp)) { 1886 panic("hat_memload: loading " 1887 "a mapping to free page %p", 1888 (void *)pp); 1889 } 1890 if (!PAGE_LOCKED(pp) && !PP_ISNORELOC(pp)) { 1891 panic("hat_memload: loading a mapping " 1892 "to unlocked relocatable page %p", 1893 (void *)pp); 1894 } 1895 ASSERT(len == MMU_PAGESIZE); 1896 } 1897 } 1898 1899 if (hat->sfmmu_rmstat) 1900 hat_resvstat(len, hat->sfmmu_as, addr); 1901 1902 if (flags & HAT_LOAD_NOCONSIST) { 1903 attr |= SFMMU_UNCACHEVTTE; 1904 use_lgpg = 1; 1905 } 1906 if (!pf_is_memory(pfn)) { 1907 attr |= SFMMU_UNCACHEPTTE | HAT_NOSYNC; 1908 use_lgpg = 1; 1909 switch (attr & HAT_ORDER_MASK) { 1910 case HAT_STRICTORDER: 1911 case HAT_UNORDERED_OK: 1912 /* 1913 * we set the side effect bit for all non 1914 * memory mappings unless merging is ok 1915 */ 1916 attr |= SFMMU_SIDEFFECT; 1917 break; 1918 case HAT_MERGING_OK: 1919 case HAT_LOADCACHING_OK: 1920 case HAT_STORECACHING_OK: 1921 break; 1922 default: 1923 panic("hat_devload: bad attr"); 1924 break; 1925 } 1926 } 1927 while (len) { 1928 if (!use_lgpg) { 1929 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1930 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1931 flags); 1932 len -= MMU_PAGESIZE; 1933 addr += MMU_PAGESIZE; 1934 pfn++; 1935 continue; 1936 } 1937 /* 1938 * try to use large pages, check va/pa alignments 1939 * Note that 32M/256M page sizes are not (yet) supported. 1940 */ 1941 if ((len >= MMU_PAGESIZE4M) && 1942 !((uintptr_t)addr & MMU_PAGEOFFSET4M) && 1943 !(disable_large_pages & (1 << TTE4M)) && 1944 !(mmu_ptob(pfn) & MMU_PAGEOFFSET4M)) { 1945 sfmmu_memtte(&tte, pfn, attr, TTE4M); 1946 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1947 flags); 1948 len -= MMU_PAGESIZE4M; 1949 addr += MMU_PAGESIZE4M; 1950 pfn += MMU_PAGESIZE4M / MMU_PAGESIZE; 1951 } else if ((len >= MMU_PAGESIZE512K) && 1952 !((uintptr_t)addr & MMU_PAGEOFFSET512K) && 1953 !(disable_large_pages & (1 << TTE512K)) && 1954 !(mmu_ptob(pfn) & MMU_PAGEOFFSET512K)) { 1955 sfmmu_memtte(&tte, pfn, attr, TTE512K); 1956 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1957 flags); 1958 len -= MMU_PAGESIZE512K; 1959 addr += MMU_PAGESIZE512K; 1960 pfn += MMU_PAGESIZE512K / MMU_PAGESIZE; 1961 } else if ((len >= MMU_PAGESIZE64K) && 1962 !((uintptr_t)addr & MMU_PAGEOFFSET64K) && 1963 !(disable_large_pages & (1 << TTE64K)) && 1964 !(mmu_ptob(pfn) & MMU_PAGEOFFSET64K)) { 1965 sfmmu_memtte(&tte, pfn, attr, TTE64K); 1966 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1967 flags); 1968 len -= MMU_PAGESIZE64K; 1969 addr += MMU_PAGESIZE64K; 1970 pfn += MMU_PAGESIZE64K / MMU_PAGESIZE; 1971 } else { 1972 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1973 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1974 flags); 1975 len -= MMU_PAGESIZE; 1976 addr += MMU_PAGESIZE; 1977 pfn++; 1978 } 1979 } 1980 1981 /* 1982 * Check TSB and TLB page sizes. 1983 */ 1984 if ((flags & HAT_LOAD_SHARE) == 0) { 1985 sfmmu_check_page_sizes(hat, 1); 1986 } 1987 } 1988 1989 /* 1990 * Map the largest extend possible out of the page array. The array may NOT 1991 * be in order. The largest possible mapping a page can have 1992 * is specified in the p_szc field. The p_szc field 1993 * cannot change as long as there any mappings (large or small) 1994 * to any of the pages that make up the large page. (ie. any 1995 * promotion/demotion of page size is not up to the hat but up to 1996 * the page free list manager). The array 1997 * should consist of properly aligned contigous pages that are 1998 * part of a big page for a large mapping to be created. 1999 */ 2000 void 2001 hat_memload_array(struct hat *hat, caddr_t addr, size_t len, 2002 struct page **pps, uint_t attr, uint_t flags) 2003 { 2004 int ttesz; 2005 size_t mapsz; 2006 pgcnt_t numpg, npgs; 2007 tte_t tte; 2008 page_t *pp; 2009 uint_t large_pages_disable; 2010 2011 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 2012 2013 if (hat->sfmmu_xhat_provider) { 2014 XHAT_MEMLOAD_ARRAY(hat, addr, len, pps, attr, flags); 2015 return; 2016 } 2017 2018 if (hat->sfmmu_rmstat) 2019 hat_resvstat(len, hat->sfmmu_as, addr); 2020 2021 #if defined(SF_ERRATA_57) 2022 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 2023 (addr < errata57_limit) && (attr & PROT_EXEC) && 2024 !(flags & HAT_LOAD_SHARE)) { 2025 cmn_err(CE_WARN, "hat_memload_array: illegal attempt to make " 2026 "user page executable"); 2027 attr &= ~PROT_EXEC; 2028 } 2029 #endif 2030 2031 /* Get number of pages */ 2032 npgs = len >> MMU_PAGESHIFT; 2033 2034 if (flags & HAT_LOAD_SHARE) { 2035 large_pages_disable = disable_ism_large_pages; 2036 } else { 2037 large_pages_disable = disable_large_pages; 2038 } 2039 2040 if (npgs < NHMENTS || large_pages_disable == LARGE_PAGES_OFF) { 2041 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2042 return; 2043 } 2044 2045 while (npgs >= NHMENTS) { 2046 pp = *pps; 2047 for (ttesz = pp->p_szc; ttesz != TTE8K; ttesz--) { 2048 /* 2049 * Check if this page size is disabled. 2050 */ 2051 if (large_pages_disable & (1 << ttesz)) 2052 continue; 2053 2054 numpg = TTEPAGES(ttesz); 2055 mapsz = numpg << MMU_PAGESHIFT; 2056 if ((npgs >= numpg) && 2057 IS_P2ALIGNED(addr, mapsz) && 2058 IS_P2ALIGNED(pp->p_pagenum, numpg)) { 2059 /* 2060 * At this point we have enough pages and 2061 * we know the virtual address and the pfn 2062 * are properly aligned. We still need 2063 * to check for physical contiguity but since 2064 * it is very likely that this is the case 2065 * we will assume they are so and undo 2066 * the request if necessary. It would 2067 * be great if we could get a hint flag 2068 * like HAT_CONTIG which would tell us 2069 * the pages are contigous for sure. 2070 */ 2071 sfmmu_memtte(&tte, (*pps)->p_pagenum, 2072 attr, ttesz); 2073 if (!sfmmu_tteload_array(hat, &tte, addr, 2074 pps, flags)) { 2075 break; 2076 } 2077 } 2078 } 2079 if (ttesz == TTE8K) { 2080 /* 2081 * We were not able to map array using a large page 2082 * batch a hmeblk or fraction at a time. 2083 */ 2084 numpg = ((uintptr_t)addr >> MMU_PAGESHIFT) 2085 & (NHMENTS-1); 2086 numpg = NHMENTS - numpg; 2087 ASSERT(numpg <= npgs); 2088 mapsz = numpg * MMU_PAGESIZE; 2089 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, 2090 numpg); 2091 } 2092 addr += mapsz; 2093 npgs -= numpg; 2094 pps += numpg; 2095 } 2096 2097 if (npgs) { 2098 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2099 } 2100 2101 /* 2102 * Check TSB and TLB page sizes. 2103 */ 2104 if ((flags & HAT_LOAD_SHARE) == 0) { 2105 sfmmu_check_page_sizes(hat, 1); 2106 } 2107 } 2108 2109 /* 2110 * Function tries to batch 8K pages into the same hme blk. 2111 */ 2112 static void 2113 sfmmu_memload_batchsmall(struct hat *hat, caddr_t vaddr, page_t **pps, 2114 uint_t attr, uint_t flags, pgcnt_t npgs) 2115 { 2116 tte_t tte; 2117 page_t *pp; 2118 struct hmehash_bucket *hmebp; 2119 struct hme_blk *hmeblkp; 2120 int index; 2121 2122 while (npgs) { 2123 /* 2124 * Acquire the hash bucket. 2125 */ 2126 hmebp = sfmmu_tteload_acquire_hashbucket(hat, vaddr, TTE8K); 2127 ASSERT(hmebp); 2128 2129 /* 2130 * Find the hment block. 2131 */ 2132 hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr, 2133 TTE8K, flags); 2134 ASSERT(hmeblkp); 2135 2136 do { 2137 /* 2138 * Make the tte. 2139 */ 2140 pp = *pps; 2141 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 2142 2143 /* 2144 * Add the translation. 2145 */ 2146 (void) sfmmu_tteload_addentry(hat, hmeblkp, &tte, 2147 vaddr, pps, flags); 2148 2149 /* 2150 * Goto next page. 2151 */ 2152 pps++; 2153 npgs--; 2154 2155 /* 2156 * Goto next address. 2157 */ 2158 vaddr += MMU_PAGESIZE; 2159 2160 /* 2161 * Don't crossover into a different hmentblk. 2162 */ 2163 index = (int)(((uintptr_t)vaddr >> MMU_PAGESHIFT) & 2164 (NHMENTS-1)); 2165 2166 } while (index != 0 && npgs != 0); 2167 2168 /* 2169 * Release the hash bucket. 2170 */ 2171 2172 sfmmu_tteload_release_hashbucket(hmebp); 2173 } 2174 } 2175 2176 /* 2177 * Construct a tte for a page: 2178 * 2179 * tte_valid = 1 2180 * tte_size2 = size & TTE_SZ2_BITS (Panther and Olympus-C only) 2181 * tte_size = size 2182 * tte_nfo = attr & HAT_NOFAULT 2183 * tte_ie = attr & HAT_STRUCTURE_LE 2184 * tte_hmenum = hmenum 2185 * tte_pahi = pp->p_pagenum >> TTE_PASHIFT; 2186 * tte_palo = pp->p_pagenum & TTE_PALOMASK; 2187 * tte_ref = 1 (optimization) 2188 * tte_wr_perm = attr & PROT_WRITE; 2189 * tte_no_sync = attr & HAT_NOSYNC 2190 * tte_lock = attr & SFMMU_LOCKTTE 2191 * tte_cp = !(attr & SFMMU_UNCACHEPTTE) 2192 * tte_cv = !(attr & SFMMU_UNCACHEVTTE) 2193 * tte_e = attr & SFMMU_SIDEFFECT 2194 * tte_priv = !(attr & PROT_USER) 2195 * tte_hwwr = if nosync is set and it is writable we set the mod bit (opt) 2196 * tte_glb = 0 2197 */ 2198 void 2199 sfmmu_memtte(tte_t *ttep, pfn_t pfn, uint_t attr, int tte_sz) 2200 { 2201 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 2202 2203 ttep->tte_inthi = MAKE_TTE_INTHI(pfn, attr, tte_sz, 0 /* hmenum */); 2204 ttep->tte_intlo = MAKE_TTE_INTLO(pfn, attr, tte_sz, 0 /* hmenum */); 2205 2206 if (TTE_IS_NOSYNC(ttep)) { 2207 TTE_SET_REF(ttep); 2208 if (TTE_IS_WRITABLE(ttep)) { 2209 TTE_SET_MOD(ttep); 2210 } 2211 } 2212 if (TTE_IS_NFO(ttep) && TTE_IS_EXECUTABLE(ttep)) { 2213 panic("sfmmu_memtte: can't set both NFO and EXEC bits"); 2214 } 2215 } 2216 2217 /* 2218 * This function will add a translation to the hme_blk and allocate the 2219 * hme_blk if one does not exist. 2220 * If a page structure is specified then it will add the 2221 * corresponding hment to the mapping list. 2222 * It will also update the hmenum field for the tte. 2223 */ 2224 void 2225 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp, 2226 uint_t flags) 2227 { 2228 (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags); 2229 } 2230 2231 /* 2232 * Load (ttep != NULL) or unload (ttep == NULL) one entry in the TSB. 2233 * Assumes that a particular page size may only be resident in one TSB. 2234 */ 2235 static void 2236 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz) 2237 { 2238 struct tsb_info *tsbinfop = NULL; 2239 uint64_t tag; 2240 struct tsbe *tsbe_addr; 2241 uint64_t tsb_base; 2242 uint_t tsb_size; 2243 int vpshift = MMU_PAGESHIFT; 2244 int phys = 0; 2245 2246 if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */ 2247 phys = ktsb_phys; 2248 if (ttesz >= TTE4M) { 2249 #ifndef sun4v 2250 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2251 #endif 2252 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2253 tsb_size = ktsb4m_szcode; 2254 } else { 2255 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2256 tsb_size = ktsb_szcode; 2257 } 2258 } else { 2259 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2260 2261 /* 2262 * If there isn't a TSB for this page size, or the TSB is 2263 * swapped out, there is nothing to do. Note that the latter 2264 * case seems impossible but can occur if hat_pageunload() 2265 * is called on an ISM mapping while the process is swapped 2266 * out. 2267 */ 2268 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2269 return; 2270 2271 /* 2272 * If another thread is in the middle of relocating a TSB 2273 * we can't unload the entry so set a flag so that the 2274 * TSB will be flushed before it can be accessed by the 2275 * process. 2276 */ 2277 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2278 if (ttep == NULL) 2279 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2280 return; 2281 } 2282 #if defined(UTSB_PHYS) 2283 phys = 1; 2284 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2285 #else 2286 tsb_base = (uint64_t)tsbinfop->tsb_va; 2287 #endif 2288 tsb_size = tsbinfop->tsb_szc; 2289 } 2290 if (ttesz >= TTE4M) 2291 vpshift = MMU_PAGESHIFT4M; 2292 2293 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2294 tag = sfmmu_make_tsbtag(vaddr); 2295 2296 if (ttep == NULL) { 2297 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2298 } else { 2299 if (ttesz >= TTE4M) { 2300 SFMMU_STAT(sf_tsb_load4m); 2301 } else { 2302 SFMMU_STAT(sf_tsb_load8k); 2303 } 2304 2305 sfmmu_load_tsbe(tsbe_addr, tag, ttep, phys); 2306 } 2307 } 2308 2309 /* 2310 * Unmap all entries from [start, end) matching the given page size. 2311 * 2312 * This function is used primarily to unmap replicated 64K or 512K entries 2313 * from the TSB that are inserted using the base page size TSB pointer, but 2314 * it may also be called to unmap a range of addresses from the TSB. 2315 */ 2316 void 2317 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz) 2318 { 2319 struct tsb_info *tsbinfop; 2320 uint64_t tag; 2321 struct tsbe *tsbe_addr; 2322 caddr_t vaddr; 2323 uint64_t tsb_base; 2324 int vpshift, vpgsz; 2325 uint_t tsb_size; 2326 int phys = 0; 2327 2328 /* 2329 * Assumptions: 2330 * If ttesz == 8K, 64K or 512K, we walk through the range 8K 2331 * at a time shooting down any valid entries we encounter. 2332 * 2333 * If ttesz >= 4M we walk the range 4M at a time shooting 2334 * down any valid mappings we find. 2335 */ 2336 if (sfmmup == ksfmmup) { 2337 phys = ktsb_phys; 2338 if (ttesz >= TTE4M) { 2339 #ifndef sun4v 2340 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2341 #endif 2342 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2343 tsb_size = ktsb4m_szcode; 2344 } else { 2345 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2346 tsb_size = ktsb_szcode; 2347 } 2348 } else { 2349 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2350 2351 /* 2352 * If there isn't a TSB for this page size, or the TSB is 2353 * swapped out, there is nothing to do. Note that the latter 2354 * case seems impossible but can occur if hat_pageunload() 2355 * is called on an ISM mapping while the process is swapped 2356 * out. 2357 */ 2358 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2359 return; 2360 2361 /* 2362 * If another thread is in the middle of relocating a TSB 2363 * we can't unload the entry so set a flag so that the 2364 * TSB will be flushed before it can be accessed by the 2365 * process. 2366 */ 2367 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2368 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2369 return; 2370 } 2371 #if defined(UTSB_PHYS) 2372 phys = 1; 2373 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2374 #else 2375 tsb_base = (uint64_t)tsbinfop->tsb_va; 2376 #endif 2377 tsb_size = tsbinfop->tsb_szc; 2378 } 2379 if (ttesz >= TTE4M) { 2380 vpshift = MMU_PAGESHIFT4M; 2381 vpgsz = MMU_PAGESIZE4M; 2382 } else { 2383 vpshift = MMU_PAGESHIFT; 2384 vpgsz = MMU_PAGESIZE; 2385 } 2386 2387 for (vaddr = start; vaddr < end; vaddr += vpgsz) { 2388 tag = sfmmu_make_tsbtag(vaddr); 2389 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2390 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2391 } 2392 } 2393 2394 /* 2395 * Select the optimum TSB size given the number of mappings 2396 * that need to be cached. 2397 */ 2398 static int 2399 sfmmu_select_tsb_szc(pgcnt_t pgcnt) 2400 { 2401 int szc = 0; 2402 2403 #ifdef DEBUG 2404 if (tsb_grow_stress) { 2405 uint32_t randval = (uint32_t)gettick() >> 4; 2406 return (randval % (tsb_max_growsize + 1)); 2407 } 2408 #endif /* DEBUG */ 2409 2410 while ((szc < tsb_max_growsize) && (pgcnt > SFMMU_RSS_TSBSIZE(szc))) 2411 szc++; 2412 return (szc); 2413 } 2414 2415 /* 2416 * This function will add a translation to the hme_blk and allocate the 2417 * hme_blk if one does not exist. 2418 * If a page structure is specified then it will add the 2419 * corresponding hment to the mapping list. 2420 * It will also update the hmenum field for the tte. 2421 * Furthermore, it attempts to create a large page translation 2422 * for <addr,hat> at page array pps. It assumes addr and first 2423 * pp is correctly aligned. It returns 0 if successful and 1 otherwise. 2424 */ 2425 static int 2426 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr, 2427 page_t **pps, uint_t flags) 2428 { 2429 struct hmehash_bucket *hmebp; 2430 struct hme_blk *hmeblkp; 2431 int ret; 2432 uint_t size; 2433 2434 /* 2435 * Get mapping size. 2436 */ 2437 size = TTE_CSZ(ttep); 2438 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2439 2440 /* 2441 * Acquire the hash bucket. 2442 */ 2443 hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size); 2444 ASSERT(hmebp); 2445 2446 /* 2447 * Find the hment block. 2448 */ 2449 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags); 2450 ASSERT(hmeblkp); 2451 2452 /* 2453 * Add the translation. 2454 */ 2455 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags); 2456 2457 /* 2458 * Release the hash bucket. 2459 */ 2460 sfmmu_tteload_release_hashbucket(hmebp); 2461 2462 return (ret); 2463 } 2464 2465 /* 2466 * Function locks and returns a pointer to the hash bucket for vaddr and size. 2467 */ 2468 static struct hmehash_bucket * 2469 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size) 2470 { 2471 struct hmehash_bucket *hmebp; 2472 int hmeshift; 2473 2474 hmeshift = HME_HASH_SHIFT(size); 2475 2476 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 2477 2478 SFMMU_HASH_LOCK(hmebp); 2479 2480 return (hmebp); 2481 } 2482 2483 /* 2484 * Function returns a pointer to an hmeblk in the hash bucket, hmebp. If the 2485 * hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is 2486 * allocated. 2487 */ 2488 static struct hme_blk * 2489 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp, 2490 caddr_t vaddr, uint_t size, uint_t flags) 2491 { 2492 hmeblk_tag hblktag; 2493 int hmeshift; 2494 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 2495 uint64_t hblkpa, prevpa; 2496 struct kmem_cache *sfmmu_cache; 2497 uint_t forcefree; 2498 2499 hblktag.htag_id = sfmmup; 2500 hmeshift = HME_HASH_SHIFT(size); 2501 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 2502 hblktag.htag_rehash = HME_HASH_REHASH(size); 2503 2504 ttearray_realloc: 2505 2506 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, 2507 pr_hblk, prevpa, &list); 2508 2509 /* 2510 * We block until hblk_reserve_lock is released; it's held by 2511 * the thread, temporarily using hblk_reserve, until hblk_reserve is 2512 * replaced by a hblk from sfmmu8_cache. 2513 */ 2514 if (hmeblkp == (struct hme_blk *)hblk_reserve && 2515 hblk_reserve_thread != curthread) { 2516 SFMMU_HASH_UNLOCK(hmebp); 2517 mutex_enter(&hblk_reserve_lock); 2518 mutex_exit(&hblk_reserve_lock); 2519 SFMMU_STAT(sf_hblk_reserve_hit); 2520 SFMMU_HASH_LOCK(hmebp); 2521 goto ttearray_realloc; 2522 } 2523 2524 if (hmeblkp == NULL) { 2525 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 2526 hblktag, flags); 2527 } else { 2528 /* 2529 * It is possible for 8k and 64k hblks to collide since they 2530 * have the same rehash value. This is because we 2531 * lazily free hblks and 8K/64K blks could be lingering. 2532 * If we find size mismatch we free the block and & try again. 2533 */ 2534 if (get_hblk_ttesz(hmeblkp) != size) { 2535 ASSERT(!hmeblkp->hblk_vcnt); 2536 ASSERT(!hmeblkp->hblk_hmecnt); 2537 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 2538 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 2539 goto ttearray_realloc; 2540 } 2541 if (hmeblkp->hblk_shw_bit) { 2542 /* 2543 * if the hblk was previously used as a shadow hblk then 2544 * we will change it to a normal hblk 2545 */ 2546 if (hmeblkp->hblk_shw_mask) { 2547 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp); 2548 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2549 goto ttearray_realloc; 2550 } else { 2551 hmeblkp->hblk_shw_bit = 0; 2552 } 2553 } 2554 SFMMU_STAT(sf_hblk_hit); 2555 } 2556 2557 /* 2558 * hat_memload() should never call kmem_cache_free(); see block 2559 * comment showing the stacktrace in sfmmu_hblk_alloc(); 2560 * enqueue each hblk in the list to reserve list if it's created 2561 * from sfmmu8_cache *and* sfmmup == KHATID. 2562 */ 2563 forcefree = (sfmmup == KHATID) ? 1 : 0; 2564 while ((pr_hblk = list) != NULL) { 2565 list = pr_hblk->hblk_next; 2566 sfmmu_cache = get_hblk_cache(pr_hblk); 2567 if ((sfmmu_cache == sfmmu8_cache) && 2568 sfmmu_put_free_hblk(pr_hblk, forcefree)) 2569 continue; 2570 2571 ASSERT(sfmmup != KHATID); 2572 kmem_cache_free(sfmmu_cache, pr_hblk); 2573 } 2574 2575 ASSERT(get_hblk_ttesz(hmeblkp) == size); 2576 ASSERT(!hmeblkp->hblk_shw_bit); 2577 2578 return (hmeblkp); 2579 } 2580 2581 /* 2582 * Function adds a tte entry into the hmeblk. It returns 0 if successful and 1 2583 * otherwise. 2584 */ 2585 static int 2586 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep, 2587 caddr_t vaddr, page_t **pps, uint_t flags) 2588 { 2589 page_t *pp = *pps; 2590 int hmenum, size, remap; 2591 tte_t tteold, flush_tte; 2592 #ifdef DEBUG 2593 tte_t orig_old; 2594 #endif /* DEBUG */ 2595 struct sf_hment *sfhme; 2596 kmutex_t *pml, *pmtx; 2597 hatlock_t *hatlockp; 2598 2599 /* 2600 * remove this panic when we decide to let user virtual address 2601 * space be >= USERLIMIT. 2602 */ 2603 if (!TTE_IS_PRIVILEGED(ttep) && vaddr >= (caddr_t)USERLIMIT) 2604 panic("user addr %p in kernel space", vaddr); 2605 #if defined(TTE_IS_GLOBAL) 2606 if (TTE_IS_GLOBAL(ttep)) 2607 panic("sfmmu_tteload: creating global tte"); 2608 #endif 2609 2610 #ifdef DEBUG 2611 if (pf_is_memory(sfmmu_ttetopfn(ttep, vaddr)) && 2612 !TTE_IS_PCACHEABLE(ttep) && !sfmmu_allow_nc_trans) 2613 panic("sfmmu_tteload: non cacheable memory tte"); 2614 #endif /* DEBUG */ 2615 2616 if ((flags & HAT_LOAD_SHARE) || !TTE_IS_REF(ttep) || 2617 !TTE_IS_MOD(ttep)) { 2618 /* 2619 * Don't load TSB for dummy as in ISM. Also don't preload 2620 * the TSB if the TTE isn't writable since we're likely to 2621 * fault on it again -- preloading can be fairly expensive. 2622 */ 2623 flags |= SFMMU_NO_TSBLOAD; 2624 } 2625 2626 size = TTE_CSZ(ttep); 2627 switch (size) { 2628 case TTE8K: 2629 SFMMU_STAT(sf_tteload8k); 2630 break; 2631 case TTE64K: 2632 SFMMU_STAT(sf_tteload64k); 2633 break; 2634 case TTE512K: 2635 SFMMU_STAT(sf_tteload512k); 2636 break; 2637 case TTE4M: 2638 SFMMU_STAT(sf_tteload4m); 2639 break; 2640 case (TTE32M): 2641 SFMMU_STAT(sf_tteload32m); 2642 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2643 break; 2644 case (TTE256M): 2645 SFMMU_STAT(sf_tteload256m); 2646 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2647 break; 2648 } 2649 2650 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2651 2652 HBLKTOHME_IDX(sfhme, hmeblkp, vaddr, hmenum); 2653 2654 /* 2655 * Need to grab mlist lock here so that pageunload 2656 * will not change tte behind us. 2657 */ 2658 if (pp) { 2659 pml = sfmmu_mlist_enter(pp); 2660 } 2661 2662 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2663 /* 2664 * Look for corresponding hment and if valid verify 2665 * pfns are equal. 2666 */ 2667 remap = TTE_IS_VALID(&tteold); 2668 if (remap) { 2669 pfn_t new_pfn, old_pfn; 2670 2671 old_pfn = TTE_TO_PFN(vaddr, &tteold); 2672 new_pfn = TTE_TO_PFN(vaddr, ttep); 2673 2674 if (flags & HAT_LOAD_REMAP) { 2675 /* make sure we are remapping same type of pages */ 2676 if (pf_is_memory(old_pfn) != pf_is_memory(new_pfn)) { 2677 panic("sfmmu_tteload - tte remap io<->memory"); 2678 } 2679 if (old_pfn != new_pfn && 2680 (pp != NULL || sfhme->hme_page != NULL)) { 2681 panic("sfmmu_tteload - tte remap pp != NULL"); 2682 } 2683 } else if (old_pfn != new_pfn) { 2684 panic("sfmmu_tteload - tte remap, hmeblkp 0x%p", 2685 (void *)hmeblkp); 2686 } 2687 ASSERT(TTE_CSZ(&tteold) == TTE_CSZ(ttep)); 2688 } 2689 2690 if (pp) { 2691 if (size == TTE8K) { 2692 #ifdef VAC 2693 /* 2694 * Handle VAC consistency 2695 */ 2696 if (!remap && (cache & CACHE_VAC) && !PP_ISNC(pp)) { 2697 sfmmu_vac_conflict(sfmmup, vaddr, pp); 2698 } 2699 #endif 2700 2701 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2702 pmtx = sfmmu_page_enter(pp); 2703 PP_CLRRO(pp); 2704 sfmmu_page_exit(pmtx); 2705 } else if (!PP_ISMAPPED(pp) && 2706 (!TTE_IS_WRITABLE(ttep)) && !(PP_ISMOD(pp))) { 2707 pmtx = sfmmu_page_enter(pp); 2708 if (!(PP_ISMOD(pp))) { 2709 PP_SETRO(pp); 2710 } 2711 sfmmu_page_exit(pmtx); 2712 } 2713 2714 } else if (sfmmu_pagearray_setup(vaddr, pps, ttep, remap)) { 2715 /* 2716 * sfmmu_pagearray_setup failed so return 2717 */ 2718 sfmmu_mlist_exit(pml); 2719 return (1); 2720 } 2721 } 2722 2723 /* 2724 * Make sure hment is not on a mapping list. 2725 */ 2726 ASSERT(remap || (sfhme->hme_page == NULL)); 2727 2728 /* if it is not a remap then hme->next better be NULL */ 2729 ASSERT((!remap) ? sfhme->hme_next == NULL : 1); 2730 2731 if (flags & HAT_LOAD_LOCK) { 2732 if (((int)hmeblkp->hblk_lckcnt + 1) >= MAX_HBLK_LCKCNT) { 2733 panic("too high lckcnt-hmeblk %p", 2734 (void *)hmeblkp); 2735 } 2736 atomic_add_16(&hmeblkp->hblk_lckcnt, 1); 2737 2738 HBLK_STACK_TRACE(hmeblkp, HBLK_LOCK); 2739 } 2740 2741 #ifdef VAC 2742 if (pp && PP_ISNC(pp)) { 2743 /* 2744 * If the physical page is marked to be uncacheable, like 2745 * by a vac conflict, make sure the new mapping is also 2746 * uncacheable. 2747 */ 2748 TTE_CLR_VCACHEABLE(ttep); 2749 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 2750 } 2751 #endif 2752 ttep->tte_hmenum = hmenum; 2753 2754 #ifdef DEBUG 2755 orig_old = tteold; 2756 #endif /* DEBUG */ 2757 2758 while (sfmmu_modifytte_try(&tteold, ttep, &sfhme->hme_tte) < 0) { 2759 if ((sfmmup == KHATID) && 2760 (flags & (HAT_LOAD_LOCK | HAT_LOAD_REMAP))) { 2761 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2762 } 2763 #ifdef DEBUG 2764 chk_tte(&orig_old, &tteold, ttep, hmeblkp); 2765 #endif /* DEBUG */ 2766 } 2767 2768 if (!TTE_IS_VALID(&tteold)) { 2769 2770 atomic_add_16(&hmeblkp->hblk_vcnt, 1); 2771 atomic_add_long(&sfmmup->sfmmu_ttecnt[size], 1); 2772 2773 /* 2774 * HAT_RELOAD_SHARE has been deprecated with lpg DISM. 2775 */ 2776 2777 if (size > TTE8K && (flags & HAT_LOAD_SHARE) == 0 && 2778 sfmmup != ksfmmup) { 2779 /* 2780 * If this is the first large mapping for the process 2781 * we must force any CPUs running this process to TL=0 2782 * where they will reload the HAT flags from the 2783 * tsbmiss area. This is necessary to make the large 2784 * mappings we are about to load visible to those CPUs; 2785 * otherwise they'll loop forever calling pagefault() 2786 * since we don't search large hash chains by default. 2787 */ 2788 hatlockp = sfmmu_hat_enter(sfmmup); 2789 if (size == TTE512K && 2790 !SFMMU_FLAGS_ISSET(sfmmup, HAT_512K_FLAG)) { 2791 SFMMU_FLAGS_SET(sfmmup, HAT_512K_FLAG); 2792 sfmmu_sync_mmustate(sfmmup); 2793 } else if (size == TTE4M && 2794 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 2795 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 2796 sfmmu_sync_mmustate(sfmmup); 2797 } else if (size == TTE64K && 2798 !SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) { 2799 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 2800 /* no sync mmustate; 64K shares 8K hashes */ 2801 } else if (mmu_page_sizes == max_mmu_page_sizes) { 2802 if (size == TTE32M && 2803 !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 2804 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 2805 sfmmu_sync_mmustate(sfmmup); 2806 } else if (size == TTE256M && 2807 !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 2808 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 2809 sfmmu_sync_mmustate(sfmmup); 2810 } 2811 } 2812 if (size >= TTE4M && (flags & HAT_LOAD_TEXT) && 2813 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) { 2814 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG); 2815 } 2816 sfmmu_hat_exit(hatlockp); 2817 } 2818 } 2819 ASSERT(TTE_IS_VALID(&sfhme->hme_tte)); 2820 2821 flush_tte.tte_intlo = (tteold.tte_intlo ^ ttep->tte_intlo) & 2822 hw_tte.tte_intlo; 2823 flush_tte.tte_inthi = (tteold.tte_inthi ^ ttep->tte_inthi) & 2824 hw_tte.tte_inthi; 2825 2826 if (remap && (flush_tte.tte_inthi || flush_tte.tte_intlo)) { 2827 /* 2828 * If remap and new tte differs from old tte we need 2829 * to sync the mod bit and flush TLB/TSB. We don't 2830 * need to sync ref bit because we currently always set 2831 * ref bit in tteload. 2832 */ 2833 ASSERT(TTE_IS_REF(ttep)); 2834 if (TTE_IS_MOD(&tteold)) { 2835 sfmmu_ttesync(sfmmup, vaddr, &tteold, pp); 2836 } 2837 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0); 2838 xt_sync(sfmmup->sfmmu_cpusran); 2839 } 2840 2841 if ((flags & SFMMU_NO_TSBLOAD) == 0) { 2842 /* 2843 * We only preload 8K and 4M mappings into the TSB, since 2844 * 64K and 512K mappings are replicated and hence don't 2845 * have a single, unique TSB entry. Ditto for 32M/256M. 2846 */ 2847 if (size == TTE8K || size == TTE4M) { 2848 hatlockp = sfmmu_hat_enter(sfmmup); 2849 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte, size); 2850 sfmmu_hat_exit(hatlockp); 2851 } 2852 } 2853 if (pp) { 2854 if (!remap) { 2855 HME_ADD(sfhme, pp); 2856 atomic_add_16(&hmeblkp->hblk_hmecnt, 1); 2857 ASSERT(hmeblkp->hblk_hmecnt > 0); 2858 2859 /* 2860 * Cannot ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 2861 * see pageunload() for comment. 2862 */ 2863 } 2864 sfmmu_mlist_exit(pml); 2865 } 2866 2867 return (0); 2868 } 2869 /* 2870 * Function unlocks hash bucket. 2871 */ 2872 static void 2873 sfmmu_tteload_release_hashbucket(struct hmehash_bucket *hmebp) 2874 { 2875 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2876 SFMMU_HASH_UNLOCK(hmebp); 2877 } 2878 2879 /* 2880 * function which checks and sets up page array for a large 2881 * translation. Will set p_vcolor, p_index, p_ro fields. 2882 * Assumes addr and pfnum of first page are properly aligned. 2883 * Will check for physical contiguity. If check fails it return 2884 * non null. 2885 */ 2886 static int 2887 sfmmu_pagearray_setup(caddr_t addr, page_t **pps, tte_t *ttep, int remap) 2888 { 2889 int i, index, ttesz; 2890 pfn_t pfnum; 2891 pgcnt_t npgs; 2892 page_t *pp, *pp1; 2893 kmutex_t *pmtx; 2894 #ifdef VAC 2895 int osz; 2896 int cflags = 0; 2897 int vac_err = 0; 2898 #endif 2899 int newidx = 0; 2900 2901 ttesz = TTE_CSZ(ttep); 2902 2903 ASSERT(ttesz > TTE8K); 2904 2905 npgs = TTEPAGES(ttesz); 2906 index = PAGESZ_TO_INDEX(ttesz); 2907 2908 pfnum = (*pps)->p_pagenum; 2909 ASSERT(IS_P2ALIGNED(pfnum, npgs)); 2910 2911 /* 2912 * Save the first pp so we can do HAT_TMPNC at the end. 2913 */ 2914 pp1 = *pps; 2915 #ifdef VAC 2916 osz = fnd_mapping_sz(pp1); 2917 #endif 2918 2919 for (i = 0; i < npgs; i++, pps++) { 2920 pp = *pps; 2921 ASSERT(PAGE_LOCKED(pp)); 2922 ASSERT(pp->p_szc >= ttesz); 2923 ASSERT(pp->p_szc == pp1->p_szc); 2924 ASSERT(sfmmu_mlist_held(pp)); 2925 2926 /* 2927 * XXX is it possible to maintain P_RO on the root only? 2928 */ 2929 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2930 pmtx = sfmmu_page_enter(pp); 2931 PP_CLRRO(pp); 2932 sfmmu_page_exit(pmtx); 2933 } else if (!PP_ISMAPPED(pp) && !TTE_IS_WRITABLE(ttep) && 2934 !PP_ISMOD(pp)) { 2935 pmtx = sfmmu_page_enter(pp); 2936 if (!(PP_ISMOD(pp))) { 2937 PP_SETRO(pp); 2938 } 2939 sfmmu_page_exit(pmtx); 2940 } 2941 2942 /* 2943 * If this is a remap we skip vac & contiguity checks. 2944 */ 2945 if (remap) 2946 continue; 2947 2948 /* 2949 * set p_vcolor and detect any vac conflicts. 2950 */ 2951 #ifdef VAC 2952 if (vac_err == 0) { 2953 vac_err = sfmmu_vacconflict_array(addr, pp, &cflags); 2954 2955 } 2956 #endif 2957 2958 /* 2959 * Save current index in case we need to undo it. 2960 * Note: "PAGESZ_TO_INDEX(sz) (1 << (sz))" 2961 * "SFMMU_INDEX_SHIFT 6" 2962 * "SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1)" 2963 * "PP_MAPINDEX(p_index) (p_index & SFMMU_INDEX_MASK)" 2964 * 2965 * So: index = PAGESZ_TO_INDEX(ttesz); 2966 * if ttesz == 1 then index = 0x2 2967 * 2 then index = 0x4 2968 * 3 then index = 0x8 2969 * 4 then index = 0x10 2970 * 5 then index = 0x20 2971 * The code below checks if it's a new pagesize (ie, newidx) 2972 * in case we need to take it back out of p_index, 2973 * and then or's the new index into the existing index. 2974 */ 2975 if ((PP_MAPINDEX(pp) & index) == 0) 2976 newidx = 1; 2977 pp->p_index = (PP_MAPINDEX(pp) | index); 2978 2979 /* 2980 * contiguity check 2981 */ 2982 if (pp->p_pagenum != pfnum) { 2983 /* 2984 * If we fail the contiguity test then 2985 * the only thing we need to fix is the p_index field. 2986 * We might get a few extra flushes but since this 2987 * path is rare that is ok. The p_ro field will 2988 * get automatically fixed on the next tteload to 2989 * the page. NO TNC bit is set yet. 2990 */ 2991 while (i >= 0) { 2992 pp = *pps; 2993 if (newidx) 2994 pp->p_index = (PP_MAPINDEX(pp) & 2995 ~index); 2996 pps--; 2997 i--; 2998 } 2999 return (1); 3000 } 3001 pfnum++; 3002 addr += MMU_PAGESIZE; 3003 } 3004 3005 #ifdef VAC 3006 if (vac_err) { 3007 if (ttesz > osz) { 3008 /* 3009 * There are some smaller mappings that causes vac 3010 * conflicts. Convert all existing small mappings to 3011 * TNC. 3012 */ 3013 SFMMU_STAT_ADD(sf_uncache_conflict, npgs); 3014 sfmmu_page_cache_array(pp1, HAT_TMPNC, CACHE_FLUSH, 3015 npgs); 3016 } else { 3017 /* EMPTY */ 3018 /* 3019 * If there exists an big page mapping, 3020 * that means the whole existing big page 3021 * has TNC setting already. No need to covert to 3022 * TNC again. 3023 */ 3024 ASSERT(PP_ISTNC(pp1)); 3025 } 3026 } 3027 #endif /* VAC */ 3028 3029 return (0); 3030 } 3031 3032 #ifdef VAC 3033 /* 3034 * Routine that detects vac consistency for a large page. It also 3035 * sets virtual color for all pp's for this big mapping. 3036 */ 3037 static int 3038 sfmmu_vacconflict_array(caddr_t addr, page_t *pp, int *cflags) 3039 { 3040 int vcolor, ocolor; 3041 3042 ASSERT(sfmmu_mlist_held(pp)); 3043 3044 if (PP_ISNC(pp)) { 3045 return (HAT_TMPNC); 3046 } 3047 3048 vcolor = addr_to_vcolor(addr); 3049 if (PP_NEWPAGE(pp)) { 3050 PP_SET_VCOLOR(pp, vcolor); 3051 return (0); 3052 } 3053 3054 ocolor = PP_GET_VCOLOR(pp); 3055 if (ocolor == vcolor) { 3056 return (0); 3057 } 3058 3059 if (!PP_ISMAPPED(pp)) { 3060 /* 3061 * Previous user of page had a differnet color 3062 * but since there are no current users 3063 * we just flush the cache and change the color. 3064 * As an optimization for large pages we flush the 3065 * entire cache of that color and set a flag. 3066 */ 3067 SFMMU_STAT(sf_pgcolor_conflict); 3068 if (!CacheColor_IsFlushed(*cflags, ocolor)) { 3069 CacheColor_SetFlushed(*cflags, ocolor); 3070 sfmmu_cache_flushcolor(ocolor, pp->p_pagenum); 3071 } 3072 PP_SET_VCOLOR(pp, vcolor); 3073 return (0); 3074 } 3075 3076 /* 3077 * We got a real conflict with a current mapping. 3078 * set flags to start unencaching all mappings 3079 * and return failure so we restart looping 3080 * the pp array from the beginning. 3081 */ 3082 return (HAT_TMPNC); 3083 } 3084 #endif /* VAC */ 3085 3086 /* 3087 * creates a large page shadow hmeblk for a tte. 3088 * The purpose of this routine is to allow us to do quick unloads because 3089 * the vm layer can easily pass a very large but sparsely populated range. 3090 */ 3091 static struct hme_blk * 3092 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags) 3093 { 3094 struct hmehash_bucket *hmebp; 3095 hmeblk_tag hblktag; 3096 int hmeshift, size, vshift; 3097 uint_t shw_mask, newshw_mask; 3098 struct hme_blk *hmeblkp; 3099 3100 ASSERT(sfmmup != KHATID); 3101 if (mmu_page_sizes == max_mmu_page_sizes) { 3102 ASSERT(ttesz < TTE256M); 3103 } else { 3104 ASSERT(ttesz < TTE4M); 3105 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 3106 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 3107 } 3108 3109 if (ttesz == TTE8K) { 3110 size = TTE512K; 3111 } else { 3112 size = ++ttesz; 3113 } 3114 3115 hblktag.htag_id = sfmmup; 3116 hmeshift = HME_HASH_SHIFT(size); 3117 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 3118 hblktag.htag_rehash = HME_HASH_REHASH(size); 3119 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 3120 3121 SFMMU_HASH_LOCK(hmebp); 3122 3123 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3124 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 3125 if (hmeblkp == NULL) { 3126 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 3127 hblktag, flags); 3128 } 3129 ASSERT(hmeblkp); 3130 if (!hmeblkp->hblk_shw_mask) { 3131 /* 3132 * if this is a unused hblk it was just allocated or could 3133 * potentially be a previous large page hblk so we need to 3134 * set the shadow bit. 3135 */ 3136 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt); 3137 hmeblkp->hblk_shw_bit = 1; 3138 } else if (hmeblkp->hblk_shw_bit == 0) { 3139 panic("sfmmu_shadow_hcreate: shw bit not set in hmeblkp 0x%p", 3140 (void *)hmeblkp); 3141 } 3142 3143 vshift = vaddr_to_vshift(hblktag, vaddr, size); 3144 ASSERT(vshift < 8); 3145 /* 3146 * Atomically set shw mask bit 3147 */ 3148 do { 3149 shw_mask = hmeblkp->hblk_shw_mask; 3150 newshw_mask = shw_mask | (1 << vshift); 3151 newshw_mask = cas32(&hmeblkp->hblk_shw_mask, shw_mask, 3152 newshw_mask); 3153 } while (newshw_mask != shw_mask); 3154 3155 SFMMU_HASH_UNLOCK(hmebp); 3156 3157 return (hmeblkp); 3158 } 3159 3160 /* 3161 * This routine cleanup a previous shadow hmeblk and changes it to 3162 * a regular hblk. This happens rarely but it is possible 3163 * when a process wants to use large pages and there are hblks still 3164 * lying around from the previous as that used these hmeblks. 3165 * The alternative was to cleanup the shadow hblks at unload time 3166 * but since so few user processes actually use large pages, it is 3167 * better to be lazy and cleanup at this time. 3168 */ 3169 static void 3170 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 3171 struct hmehash_bucket *hmebp) 3172 { 3173 caddr_t addr, endaddr; 3174 int hashno, size; 3175 3176 ASSERT(hmeblkp->hblk_shw_bit); 3177 3178 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 3179 3180 if (!hmeblkp->hblk_shw_mask) { 3181 hmeblkp->hblk_shw_bit = 0; 3182 return; 3183 } 3184 addr = (caddr_t)get_hblk_base(hmeblkp); 3185 endaddr = get_hblk_endaddr(hmeblkp); 3186 size = get_hblk_ttesz(hmeblkp); 3187 hashno = size - 1; 3188 ASSERT(hashno > 0); 3189 SFMMU_HASH_UNLOCK(hmebp); 3190 3191 sfmmu_free_hblks(sfmmup, addr, endaddr, hashno); 3192 3193 SFMMU_HASH_LOCK(hmebp); 3194 } 3195 3196 static void 3197 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr, 3198 int hashno) 3199 { 3200 int hmeshift, shadow = 0; 3201 hmeblk_tag hblktag; 3202 struct hmehash_bucket *hmebp; 3203 struct hme_blk *hmeblkp; 3204 struct hme_blk *nx_hblk, *pr_hblk, *list = NULL; 3205 uint64_t hblkpa, prevpa, nx_pa; 3206 3207 ASSERT(hashno > 0); 3208 hblktag.htag_id = sfmmup; 3209 hblktag.htag_rehash = hashno; 3210 3211 hmeshift = HME_HASH_SHIFT(hashno); 3212 3213 while (addr < endaddr) { 3214 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3215 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3216 SFMMU_HASH_LOCK(hmebp); 3217 /* inline HME_HASH_SEARCH */ 3218 hmeblkp = hmebp->hmeblkp; 3219 hblkpa = hmebp->hmeh_nextpa; 3220 prevpa = 0; 3221 pr_hblk = NULL; 3222 while (hmeblkp) { 3223 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 3224 if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) { 3225 /* found hme_blk */ 3226 if (hmeblkp->hblk_shw_bit) { 3227 if (hmeblkp->hblk_shw_mask) { 3228 shadow = 1; 3229 sfmmu_shadow_hcleanup(sfmmup, 3230 hmeblkp, hmebp); 3231 break; 3232 } else { 3233 hmeblkp->hblk_shw_bit = 0; 3234 } 3235 } 3236 3237 /* 3238 * Hblk_hmecnt and hblk_vcnt could be non zero 3239 * since hblk_unload() does not gurantee that. 3240 * 3241 * XXX - this could cause tteload() to spin 3242 * where sfmmu_shadow_hcleanup() is called. 3243 */ 3244 } 3245 3246 nx_hblk = hmeblkp->hblk_next; 3247 nx_pa = hmeblkp->hblk_nextpa; 3248 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 3249 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 3250 pr_hblk); 3251 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 3252 } else { 3253 pr_hblk = hmeblkp; 3254 prevpa = hblkpa; 3255 } 3256 hmeblkp = nx_hblk; 3257 hblkpa = nx_pa; 3258 } 3259 3260 SFMMU_HASH_UNLOCK(hmebp); 3261 3262 if (shadow) { 3263 /* 3264 * We found another shadow hblk so cleaned its 3265 * children. We need to go back and cleanup 3266 * the original hblk so we don't change the 3267 * addr. 3268 */ 3269 shadow = 0; 3270 } else { 3271 addr = (caddr_t)roundup((uintptr_t)addr + 1, 3272 (1 << hmeshift)); 3273 } 3274 } 3275 sfmmu_hblks_list_purge(&list); 3276 } 3277 3278 /* 3279 * Release one hardware address translation lock on the given address range. 3280 */ 3281 void 3282 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len) 3283 { 3284 struct hmehash_bucket *hmebp; 3285 hmeblk_tag hblktag; 3286 int hmeshift, hashno = 1; 3287 struct hme_blk *hmeblkp, *list = NULL; 3288 caddr_t endaddr; 3289 3290 ASSERT(sfmmup != NULL); 3291 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3292 3293 ASSERT((sfmmup == ksfmmup) || 3294 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 3295 ASSERT((len & MMU_PAGEOFFSET) == 0); 3296 endaddr = addr + len; 3297 hblktag.htag_id = sfmmup; 3298 3299 /* 3300 * Spitfire supports 4 page sizes. 3301 * Most pages are expected to be of the smallest page size (8K) and 3302 * these will not need to be rehashed. 64K pages also don't need to be 3303 * rehashed because an hmeblk spans 64K of address space. 512K pages 3304 * might need 1 rehash and and 4M pages might need 2 rehashes. 3305 */ 3306 while (addr < endaddr) { 3307 hmeshift = HME_HASH_SHIFT(hashno); 3308 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3309 hblktag.htag_rehash = hashno; 3310 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3311 3312 SFMMU_HASH_LOCK(hmebp); 3313 3314 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 3315 if (hmeblkp != NULL) { 3316 /* 3317 * If we encounter a shadow hmeblk then 3318 * we know there are no valid hmeblks mapping 3319 * this address at this size or larger. 3320 * Just increment address by the smallest 3321 * page size. 3322 */ 3323 if (hmeblkp->hblk_shw_bit) { 3324 addr += MMU_PAGESIZE; 3325 } else { 3326 addr = sfmmu_hblk_unlock(hmeblkp, addr, 3327 endaddr); 3328 } 3329 SFMMU_HASH_UNLOCK(hmebp); 3330 hashno = 1; 3331 continue; 3332 } 3333 SFMMU_HASH_UNLOCK(hmebp); 3334 3335 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 3336 /* 3337 * We have traversed the whole list and rehashed 3338 * if necessary without finding the address to unlock 3339 * which should never happen. 3340 */ 3341 panic("sfmmu_unlock: addr not found. " 3342 "addr %p hat %p", (void *)addr, (void *)sfmmup); 3343 } else { 3344 hashno++; 3345 } 3346 } 3347 3348 sfmmu_hblks_list_purge(&list); 3349 } 3350 3351 /* 3352 * Function to unlock a range of addresses in an hmeblk. It returns the 3353 * next address that needs to be unlocked. 3354 * Should be called with the hash lock held. 3355 */ 3356 static caddr_t 3357 sfmmu_hblk_unlock(struct hme_blk *hmeblkp, caddr_t addr, caddr_t endaddr) 3358 { 3359 struct sf_hment *sfhme; 3360 tte_t tteold, ttemod; 3361 int ttesz, ret; 3362 3363 ASSERT(in_hblk_range(hmeblkp, addr)); 3364 ASSERT(hmeblkp->hblk_shw_bit == 0); 3365 3366 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 3367 ttesz = get_hblk_ttesz(hmeblkp); 3368 3369 HBLKTOHME(sfhme, hmeblkp, addr); 3370 while (addr < endaddr) { 3371 readtte: 3372 sfmmu_copytte(&sfhme->hme_tte, &tteold); 3373 if (TTE_IS_VALID(&tteold)) { 3374 3375 ttemod = tteold; 3376 3377 ret = sfmmu_modifytte_try(&tteold, &ttemod, 3378 &sfhme->hme_tte); 3379 3380 if (ret < 0) 3381 goto readtte; 3382 3383 if (hmeblkp->hblk_lckcnt == 0) 3384 panic("zero hblk lckcnt"); 3385 3386 if (((uintptr_t)addr + TTEBYTES(ttesz)) > 3387 (uintptr_t)endaddr) 3388 panic("can't unlock large tte"); 3389 3390 ASSERT(hmeblkp->hblk_lckcnt > 0); 3391 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 3392 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 3393 } else { 3394 panic("sfmmu_hblk_unlock: invalid tte"); 3395 } 3396 addr += TTEBYTES(ttesz); 3397 sfhme++; 3398 } 3399 return (addr); 3400 } 3401 3402 /* 3403 * Physical Address Mapping Framework 3404 * 3405 * General rules: 3406 * 3407 * (1) Applies only to seg_kmem memory pages. To make things easier, 3408 * seg_kpm addresses are also accepted by the routines, but nothing 3409 * is done with them since by definition their PA mappings are static. 3410 * (2) hat_add_callback() may only be called while holding the page lock 3411 * SE_SHARED or SE_EXCL of the underlying page (e.g., as_pagelock()), 3412 * or passing HAC_PAGELOCK flag. 3413 * (3) prehandler() and posthandler() may not call hat_add_callback() or 3414 * hat_delete_callback(), nor should they allocate memory. Post quiesce 3415 * callbacks may not sleep or acquire adaptive mutex locks. 3416 * (4) Either prehandler() or posthandler() (but not both) may be specified 3417 * as being NULL. Specifying an errhandler() is optional. 3418 * 3419 * Details of using the framework: 3420 * 3421 * registering a callback (hat_register_callback()) 3422 * 3423 * Pass prehandler, posthandler, errhandler addresses 3424 * as described below. If capture_cpus argument is nonzero, 3425 * suspend callback to the prehandler will occur with CPUs 3426 * captured and executing xc_loop() and CPUs will remain 3427 * captured until after the posthandler suspend callback 3428 * occurs. 3429 * 3430 * adding a callback (hat_add_callback()) 3431 * 3432 * as_pagelock(); 3433 * hat_add_callback(); 3434 * save returned pfn in private data structures or program registers; 3435 * as_pageunlock(); 3436 * 3437 * prehandler() 3438 * 3439 * Stop all accesses by physical address to this memory page. 3440 * Called twice: the first, PRESUSPEND, is a context safe to acquire 3441 * adaptive locks. The second, SUSPEND, is called at high PIL with 3442 * CPUs captured so adaptive locks may NOT be acquired (and all spin 3443 * locks must be XCALL_PIL or higher locks). 3444 * 3445 * May return the following errors: 3446 * EIO: A fatal error has occurred. This will result in panic. 3447 * EAGAIN: The page cannot be suspended. This will fail the 3448 * relocation. 3449 * 0: Success. 3450 * 3451 * posthandler() 3452 * 3453 * Save new pfn in private data structures or program registers; 3454 * not allowed to fail (non-zero return values will result in panic). 3455 * 3456 * errhandler() 3457 * 3458 * called when an error occurs related to the callback. Currently 3459 * the only such error is HAT_CB_ERR_LEAKED which indicates that 3460 * a page is being freed, but there are still outstanding callback(s) 3461 * registered on the page. 3462 * 3463 * removing a callback (hat_delete_callback(); e.g., prior to freeing memory) 3464 * 3465 * stop using physical address 3466 * hat_delete_callback(); 3467 * 3468 */ 3469 3470 /* 3471 * Register a callback class. Each subsystem should do this once and 3472 * cache the id_t returned for use in setting up and tearing down callbacks. 3473 * 3474 * There is no facility for removing callback IDs once they are created; 3475 * the "key" should be unique for each module, so in case a module is unloaded 3476 * and subsequently re-loaded, we can recycle the module's previous entry. 3477 */ 3478 id_t 3479 hat_register_callback(int key, 3480 int (*prehandler)(caddr_t, uint_t, uint_t, void *), 3481 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t), 3482 int (*errhandler)(caddr_t, uint_t, uint_t, void *), 3483 int capture_cpus) 3484 { 3485 id_t id; 3486 3487 /* 3488 * Search the table for a pre-existing callback associated with 3489 * the identifier "key". If one exists, we re-use that entry in 3490 * the table for this instance, otherwise we assign the next 3491 * available table slot. 3492 */ 3493 for (id = 0; id < sfmmu_max_cb_id; id++) { 3494 if (sfmmu_cb_table[id].key == key) 3495 break; 3496 } 3497 3498 if (id == sfmmu_max_cb_id) { 3499 id = sfmmu_cb_nextid++; 3500 if (id >= sfmmu_max_cb_id) 3501 panic("hat_register_callback: out of callback IDs"); 3502 } 3503 3504 ASSERT(prehandler != NULL || posthandler != NULL); 3505 3506 sfmmu_cb_table[id].key = key; 3507 sfmmu_cb_table[id].prehandler = prehandler; 3508 sfmmu_cb_table[id].posthandler = posthandler; 3509 sfmmu_cb_table[id].errhandler = errhandler; 3510 sfmmu_cb_table[id].capture_cpus = capture_cpus; 3511 3512 return (id); 3513 } 3514 3515 #define HAC_COOKIE_NONE (void *)-1 3516 3517 /* 3518 * Add relocation callbacks to the specified addr/len which will be called 3519 * when relocating the associated page. See the description of pre and 3520 * posthandler above for more details. 3521 * 3522 * If HAC_PAGELOCK is included in flags, the underlying memory page is 3523 * locked internally so the caller must be able to deal with the callback 3524 * running even before this function has returned. If HAC_PAGELOCK is not 3525 * set, it is assumed that the underlying memory pages are locked. 3526 * 3527 * Since the caller must track the individual page boundaries anyway, 3528 * we only allow a callback to be added to a single page (large 3529 * or small). Thus [addr, addr + len) MUST be contained within a single 3530 * page. 3531 * 3532 * Registering multiple callbacks on the same [addr, addr+len) is supported, 3533 * _provided_that_ a unique parameter is specified for each callback. 3534 * If multiple callbacks are registered on the same range the callback will 3535 * be invoked with each unique parameter. Registering the same callback with 3536 * the same argument more than once will result in corrupted kernel state. 3537 * 3538 * Returns the pfn of the underlying kernel page in *rpfn 3539 * on success, or PFN_INVALID on failure. 3540 * 3541 * cookiep (if passed) provides storage space for an opaque cookie 3542 * to return later to hat_delete_callback(). This cookie makes the callback 3543 * deletion significantly quicker by avoiding a potentially lengthy hash 3544 * search. 3545 * 3546 * Returns values: 3547 * 0: success 3548 * ENOMEM: memory allocation failure (e.g. flags was passed as HAC_NOSLEEP) 3549 * EINVAL: callback ID is not valid 3550 * ENXIO: ["vaddr", "vaddr" + len) is not mapped in the kernel's address 3551 * space 3552 * ERANGE: ["vaddr", "vaddr" + len) crosses a page boundary 3553 */ 3554 int 3555 hat_add_callback(id_t callback_id, caddr_t vaddr, uint_t len, uint_t flags, 3556 void *pvt, pfn_t *rpfn, void **cookiep) 3557 { 3558 struct hmehash_bucket *hmebp; 3559 hmeblk_tag hblktag; 3560 struct hme_blk *hmeblkp; 3561 int hmeshift, hashno; 3562 caddr_t saddr, eaddr, baseaddr; 3563 struct pa_hment *pahmep; 3564 struct sf_hment *sfhmep, *osfhmep; 3565 kmutex_t *pml; 3566 tte_t tte; 3567 page_t *pp; 3568 vnode_t *vp; 3569 u_offset_t off; 3570 pfn_t pfn; 3571 int kmflags = (flags & HAC_SLEEP)? KM_SLEEP : KM_NOSLEEP; 3572 int locked = 0; 3573 3574 /* 3575 * For KPM mappings, just return the physical address since we 3576 * don't need to register any callbacks. 3577 */ 3578 if (IS_KPM_ADDR(vaddr)) { 3579 uint64_t paddr; 3580 SFMMU_KPM_VTOP(vaddr, paddr); 3581 *rpfn = btop(paddr); 3582 if (cookiep != NULL) 3583 *cookiep = HAC_COOKIE_NONE; 3584 return (0); 3585 } 3586 3587 if (callback_id < (id_t)0 || callback_id >= sfmmu_cb_nextid) { 3588 *rpfn = PFN_INVALID; 3589 return (EINVAL); 3590 } 3591 3592 if ((pahmep = kmem_cache_alloc(pa_hment_cache, kmflags)) == NULL) { 3593 *rpfn = PFN_INVALID; 3594 return (ENOMEM); 3595 } 3596 3597 sfhmep = &pahmep->sfment; 3598 3599 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3600 eaddr = saddr + len; 3601 3602 rehash: 3603 /* Find the mapping(s) for this page */ 3604 for (hashno = TTE64K, hmeblkp = NULL; 3605 hmeblkp == NULL && hashno <= mmu_hashcnt; 3606 hashno++) { 3607 hmeshift = HME_HASH_SHIFT(hashno); 3608 hblktag.htag_id = ksfmmup; 3609 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3610 hblktag.htag_rehash = hashno; 3611 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3612 3613 SFMMU_HASH_LOCK(hmebp); 3614 3615 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3616 3617 if (hmeblkp == NULL) 3618 SFMMU_HASH_UNLOCK(hmebp); 3619 } 3620 3621 if (hmeblkp == NULL) { 3622 kmem_cache_free(pa_hment_cache, pahmep); 3623 *rpfn = PFN_INVALID; 3624 return (ENXIO); 3625 } 3626 3627 HBLKTOHME(osfhmep, hmeblkp, saddr); 3628 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3629 3630 if (!TTE_IS_VALID(&tte)) { 3631 SFMMU_HASH_UNLOCK(hmebp); 3632 kmem_cache_free(pa_hment_cache, pahmep); 3633 *rpfn = PFN_INVALID; 3634 return (ENXIO); 3635 } 3636 3637 /* 3638 * Make sure the boundaries for the callback fall within this 3639 * single mapping. 3640 */ 3641 baseaddr = (caddr_t)get_hblk_base(hmeblkp); 3642 ASSERT(saddr >= baseaddr); 3643 if (eaddr > saddr + TTEBYTES(TTE_CSZ(&tte))) { 3644 SFMMU_HASH_UNLOCK(hmebp); 3645 kmem_cache_free(pa_hment_cache, pahmep); 3646 *rpfn = PFN_INVALID; 3647 return (ERANGE); 3648 } 3649 3650 pfn = sfmmu_ttetopfn(&tte, vaddr); 3651 3652 /* 3653 * The pfn may not have a page_t underneath in which case we 3654 * just return it. This can happen if we are doing I/O to a 3655 * static portion of the kernel's address space, for instance. 3656 */ 3657 pp = osfhmep->hme_page; 3658 if (pp == NULL) { 3659 SFMMU_HASH_UNLOCK(hmebp); 3660 kmem_cache_free(pa_hment_cache, pahmep); 3661 *rpfn = pfn; 3662 if (cookiep) 3663 *cookiep = HAC_COOKIE_NONE; 3664 return (0); 3665 } 3666 ASSERT(pp == PP_PAGEROOT(pp)); 3667 3668 vp = pp->p_vnode; 3669 off = pp->p_offset; 3670 3671 pml = sfmmu_mlist_enter(pp); 3672 3673 if (flags & HAC_PAGELOCK) { 3674 if (!page_trylock(pp, SE_SHARED)) { 3675 /* 3676 * Somebody is holding SE_EXCL lock. Might 3677 * even be hat_page_relocate(). Drop all 3678 * our locks, lookup the page in &kvp, and 3679 * retry. If it doesn't exist in &kvp and &zvp, 3680 * then we must be dealing with a kernel mapped 3681 * page which doesn't actually belong to 3682 * segkmem so we punt. 3683 */ 3684 sfmmu_mlist_exit(pml); 3685 SFMMU_HASH_UNLOCK(hmebp); 3686 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3687 3688 /* check zvp before giving up */ 3689 if (pp == NULL) 3690 pp = page_lookup(&zvp, (u_offset_t)saddr, 3691 SE_SHARED); 3692 3693 /* Okay, we didn't find it, give up */ 3694 if (pp == NULL) { 3695 kmem_cache_free(pa_hment_cache, pahmep); 3696 *rpfn = pfn; 3697 if (cookiep) 3698 *cookiep = HAC_COOKIE_NONE; 3699 return (0); 3700 } 3701 page_unlock(pp); 3702 goto rehash; 3703 } 3704 locked = 1; 3705 } 3706 3707 if (!PAGE_LOCKED(pp) && !panicstr) 3708 panic("hat_add_callback: page 0x%p not locked", pp); 3709 3710 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3711 pp->p_offset != off) { 3712 /* 3713 * The page moved before we got our hands on it. Drop 3714 * all the locks and try again. 3715 */ 3716 ASSERT((flags & HAC_PAGELOCK) != 0); 3717 sfmmu_mlist_exit(pml); 3718 SFMMU_HASH_UNLOCK(hmebp); 3719 page_unlock(pp); 3720 locked = 0; 3721 goto rehash; 3722 } 3723 3724 if (!VN_ISKAS(vp)) { 3725 /* 3726 * This is not a segkmem page but another page which 3727 * has been kernel mapped. It had better have at least 3728 * a share lock on it. Return the pfn. 3729 */ 3730 sfmmu_mlist_exit(pml); 3731 SFMMU_HASH_UNLOCK(hmebp); 3732 if (locked) 3733 page_unlock(pp); 3734 kmem_cache_free(pa_hment_cache, pahmep); 3735 ASSERT(PAGE_LOCKED(pp)); 3736 *rpfn = pfn; 3737 if (cookiep) 3738 *cookiep = HAC_COOKIE_NONE; 3739 return (0); 3740 } 3741 3742 /* 3743 * Setup this pa_hment and link its embedded dummy sf_hment into 3744 * the mapping list. 3745 */ 3746 pp->p_share++; 3747 pahmep->cb_id = callback_id; 3748 pahmep->addr = vaddr; 3749 pahmep->len = len; 3750 pahmep->refcnt = 1; 3751 pahmep->flags = 0; 3752 pahmep->pvt = pvt; 3753 3754 sfhmep->hme_tte.ll = 0; 3755 sfhmep->hme_data = pahmep; 3756 sfhmep->hme_prev = osfhmep; 3757 sfhmep->hme_next = osfhmep->hme_next; 3758 3759 if (osfhmep->hme_next) 3760 osfhmep->hme_next->hme_prev = sfhmep; 3761 3762 osfhmep->hme_next = sfhmep; 3763 3764 sfmmu_mlist_exit(pml); 3765 SFMMU_HASH_UNLOCK(hmebp); 3766 3767 if (locked) 3768 page_unlock(pp); 3769 3770 *rpfn = pfn; 3771 if (cookiep) 3772 *cookiep = (void *)pahmep; 3773 3774 return (0); 3775 } 3776 3777 /* 3778 * Remove the relocation callbacks from the specified addr/len. 3779 */ 3780 void 3781 hat_delete_callback(caddr_t vaddr, uint_t len, void *pvt, uint_t flags, 3782 void *cookie) 3783 { 3784 struct hmehash_bucket *hmebp; 3785 hmeblk_tag hblktag; 3786 struct hme_blk *hmeblkp; 3787 int hmeshift, hashno; 3788 caddr_t saddr; 3789 struct pa_hment *pahmep; 3790 struct sf_hment *sfhmep, *osfhmep; 3791 kmutex_t *pml; 3792 tte_t tte; 3793 page_t *pp; 3794 vnode_t *vp; 3795 u_offset_t off; 3796 int locked = 0; 3797 3798 /* 3799 * If the cookie is HAC_COOKIE_NONE then there is no pa_hment to 3800 * remove so just return. 3801 */ 3802 if (cookie == HAC_COOKIE_NONE || IS_KPM_ADDR(vaddr)) 3803 return; 3804 3805 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3806 3807 rehash: 3808 /* Find the mapping(s) for this page */ 3809 for (hashno = TTE64K, hmeblkp = NULL; 3810 hmeblkp == NULL && hashno <= mmu_hashcnt; 3811 hashno++) { 3812 hmeshift = HME_HASH_SHIFT(hashno); 3813 hblktag.htag_id = ksfmmup; 3814 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3815 hblktag.htag_rehash = hashno; 3816 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3817 3818 SFMMU_HASH_LOCK(hmebp); 3819 3820 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3821 3822 if (hmeblkp == NULL) 3823 SFMMU_HASH_UNLOCK(hmebp); 3824 } 3825 3826 if (hmeblkp == NULL) 3827 return; 3828 3829 HBLKTOHME(osfhmep, hmeblkp, saddr); 3830 3831 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3832 if (!TTE_IS_VALID(&tte)) { 3833 SFMMU_HASH_UNLOCK(hmebp); 3834 return; 3835 } 3836 3837 pp = osfhmep->hme_page; 3838 if (pp == NULL) { 3839 SFMMU_HASH_UNLOCK(hmebp); 3840 ASSERT(cookie == NULL); 3841 return; 3842 } 3843 3844 vp = pp->p_vnode; 3845 off = pp->p_offset; 3846 3847 pml = sfmmu_mlist_enter(pp); 3848 3849 if (flags & HAC_PAGELOCK) { 3850 if (!page_trylock(pp, SE_SHARED)) { 3851 /* 3852 * Somebody is holding SE_EXCL lock. Might 3853 * even be hat_page_relocate(). Drop all 3854 * our locks, lookup the page in &kvp, and 3855 * retry. If it doesn't exist in &kvp and &zvp, 3856 * then we must be dealing with a kernel mapped 3857 * page which doesn't actually belong to 3858 * segkmem so we punt. 3859 */ 3860 sfmmu_mlist_exit(pml); 3861 SFMMU_HASH_UNLOCK(hmebp); 3862 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3863 /* check zvp before giving up */ 3864 if (pp == NULL) 3865 pp = page_lookup(&zvp, (u_offset_t)saddr, 3866 SE_SHARED); 3867 3868 if (pp == NULL) { 3869 ASSERT(cookie == NULL); 3870 return; 3871 } 3872 page_unlock(pp); 3873 goto rehash; 3874 } 3875 locked = 1; 3876 } 3877 3878 ASSERT(PAGE_LOCKED(pp)); 3879 3880 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3881 pp->p_offset != off) { 3882 /* 3883 * The page moved before we got our hands on it. Drop 3884 * all the locks and try again. 3885 */ 3886 ASSERT((flags & HAC_PAGELOCK) != 0); 3887 sfmmu_mlist_exit(pml); 3888 SFMMU_HASH_UNLOCK(hmebp); 3889 page_unlock(pp); 3890 locked = 0; 3891 goto rehash; 3892 } 3893 3894 if (!VN_ISKAS(vp)) { 3895 /* 3896 * This is not a segkmem page but another page which 3897 * has been kernel mapped. 3898 */ 3899 sfmmu_mlist_exit(pml); 3900 SFMMU_HASH_UNLOCK(hmebp); 3901 if (locked) 3902 page_unlock(pp); 3903 ASSERT(cookie == NULL); 3904 return; 3905 } 3906 3907 if (cookie != NULL) { 3908 pahmep = (struct pa_hment *)cookie; 3909 sfhmep = &pahmep->sfment; 3910 } else { 3911 for (sfhmep = pp->p_mapping; sfhmep != NULL; 3912 sfhmep = sfhmep->hme_next) { 3913 3914 /* 3915 * skip va<->pa mappings 3916 */ 3917 if (!IS_PAHME(sfhmep)) 3918 continue; 3919 3920 pahmep = sfhmep->hme_data; 3921 ASSERT(pahmep != NULL); 3922 3923 /* 3924 * if pa_hment matches, remove it 3925 */ 3926 if ((pahmep->pvt == pvt) && 3927 (pahmep->addr == vaddr) && 3928 (pahmep->len == len)) { 3929 break; 3930 } 3931 } 3932 } 3933 3934 if (sfhmep == NULL) { 3935 if (!panicstr) { 3936 panic("hat_delete_callback: pa_hment not found, pp %p", 3937 (void *)pp); 3938 } 3939 return; 3940 } 3941 3942 /* 3943 * Note: at this point a valid kernel mapping must still be 3944 * present on this page. 3945 */ 3946 pp->p_share--; 3947 if (pp->p_share <= 0) 3948 panic("hat_delete_callback: zero p_share"); 3949 3950 if (--pahmep->refcnt == 0) { 3951 if (pahmep->flags != 0) 3952 panic("hat_delete_callback: pa_hment is busy"); 3953 3954 /* 3955 * Remove sfhmep from the mapping list for the page. 3956 */ 3957 if (sfhmep->hme_prev) { 3958 sfhmep->hme_prev->hme_next = sfhmep->hme_next; 3959 } else { 3960 pp->p_mapping = sfhmep->hme_next; 3961 } 3962 3963 if (sfhmep->hme_next) 3964 sfhmep->hme_next->hme_prev = sfhmep->hme_prev; 3965 3966 sfmmu_mlist_exit(pml); 3967 SFMMU_HASH_UNLOCK(hmebp); 3968 3969 if (locked) 3970 page_unlock(pp); 3971 3972 kmem_cache_free(pa_hment_cache, pahmep); 3973 return; 3974 } 3975 3976 sfmmu_mlist_exit(pml); 3977 SFMMU_HASH_UNLOCK(hmebp); 3978 if (locked) 3979 page_unlock(pp); 3980 } 3981 3982 /* 3983 * hat_probe returns 1 if the translation for the address 'addr' is 3984 * loaded, zero otherwise. 3985 * 3986 * hat_probe should be used only for advisorary purposes because it may 3987 * occasionally return the wrong value. The implementation must guarantee that 3988 * returning the wrong value is a very rare event. hat_probe is used 3989 * to implement optimizations in the segment drivers. 3990 * 3991 */ 3992 int 3993 hat_probe(struct hat *sfmmup, caddr_t addr) 3994 { 3995 pfn_t pfn; 3996 tte_t tte; 3997 3998 ASSERT(sfmmup != NULL); 3999 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4000 4001 ASSERT((sfmmup == ksfmmup) || 4002 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4003 4004 if (sfmmup == ksfmmup) { 4005 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte)) 4006 == PFN_SUSPENDED) { 4007 sfmmu_vatopfn_suspended(addr, sfmmup, &tte); 4008 } 4009 } else { 4010 pfn = sfmmu_uvatopfn(addr, sfmmup); 4011 } 4012 4013 if (pfn != PFN_INVALID) 4014 return (1); 4015 else 4016 return (0); 4017 } 4018 4019 ssize_t 4020 hat_getpagesize(struct hat *sfmmup, caddr_t addr) 4021 { 4022 tte_t tte; 4023 4024 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4025 4026 sfmmu_gettte(sfmmup, addr, &tte); 4027 if (TTE_IS_VALID(&tte)) { 4028 return (TTEBYTES(TTE_CSZ(&tte))); 4029 } 4030 return (-1); 4031 } 4032 4033 static void 4034 sfmmu_gettte(struct hat *sfmmup, caddr_t addr, tte_t *ttep) 4035 { 4036 struct hmehash_bucket *hmebp; 4037 hmeblk_tag hblktag; 4038 int hmeshift, hashno = 1; 4039 struct hme_blk *hmeblkp, *list = NULL; 4040 struct sf_hment *sfhmep; 4041 4042 /* support for ISM */ 4043 ism_map_t *ism_map; 4044 ism_blk_t *ism_blkp; 4045 int i; 4046 sfmmu_t *ism_hatid = NULL; 4047 sfmmu_t *locked_hatid = NULL; 4048 4049 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 4050 4051 ism_blkp = sfmmup->sfmmu_iblk; 4052 if (ism_blkp) { 4053 sfmmu_ismhat_enter(sfmmup, 0); 4054 locked_hatid = sfmmup; 4055 } 4056 while (ism_blkp && ism_hatid == NULL) { 4057 ism_map = ism_blkp->iblk_maps; 4058 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 4059 if (addr >= ism_start(ism_map[i]) && 4060 addr < ism_end(ism_map[i])) { 4061 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 4062 addr = (caddr_t)(addr - 4063 ism_start(ism_map[i])); 4064 break; 4065 } 4066 } 4067 ism_blkp = ism_blkp->iblk_next; 4068 } 4069 if (locked_hatid) { 4070 sfmmu_ismhat_exit(locked_hatid, 0); 4071 } 4072 4073 hblktag.htag_id = sfmmup; 4074 ttep->ll = 0; 4075 4076 do { 4077 hmeshift = HME_HASH_SHIFT(hashno); 4078 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4079 hblktag.htag_rehash = hashno; 4080 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4081 4082 SFMMU_HASH_LOCK(hmebp); 4083 4084 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4085 if (hmeblkp != NULL) { 4086 HBLKTOHME(sfhmep, hmeblkp, addr); 4087 sfmmu_copytte(&sfhmep->hme_tte, ttep); 4088 SFMMU_HASH_UNLOCK(hmebp); 4089 break; 4090 } 4091 SFMMU_HASH_UNLOCK(hmebp); 4092 hashno++; 4093 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 4094 4095 sfmmu_hblks_list_purge(&list); 4096 } 4097 4098 uint_t 4099 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr) 4100 { 4101 tte_t tte; 4102 4103 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4104 4105 sfmmu_gettte(sfmmup, addr, &tte); 4106 if (TTE_IS_VALID(&tte)) { 4107 *attr = sfmmu_ptov_attr(&tte); 4108 return (0); 4109 } 4110 *attr = 0; 4111 return ((uint_t)0xffffffff); 4112 } 4113 4114 /* 4115 * Enables more attributes on specified address range (ie. logical OR) 4116 */ 4117 void 4118 hat_setattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4119 { 4120 if (hat->sfmmu_xhat_provider) { 4121 XHAT_SETATTR(hat, addr, len, attr); 4122 return; 4123 } else { 4124 /* 4125 * This must be a CPU HAT. If the address space has 4126 * XHATs attached, change attributes for all of them, 4127 * just in case 4128 */ 4129 ASSERT(hat->sfmmu_as != NULL); 4130 if (hat->sfmmu_as->a_xhat != NULL) 4131 xhat_setattr_all(hat->sfmmu_as, addr, len, attr); 4132 } 4133 4134 sfmmu_chgattr(hat, addr, len, attr, SFMMU_SETATTR); 4135 } 4136 4137 /* 4138 * Assigns attributes to the specified address range. All the attributes 4139 * are specified. 4140 */ 4141 void 4142 hat_chgattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4143 { 4144 if (hat->sfmmu_xhat_provider) { 4145 XHAT_CHGATTR(hat, addr, len, attr); 4146 return; 4147 } else { 4148 /* 4149 * This must be a CPU HAT. If the address space has 4150 * XHATs attached, change attributes for all of them, 4151 * just in case 4152 */ 4153 ASSERT(hat->sfmmu_as != NULL); 4154 if (hat->sfmmu_as->a_xhat != NULL) 4155 xhat_chgattr_all(hat->sfmmu_as, addr, len, attr); 4156 } 4157 4158 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CHGATTR); 4159 } 4160 4161 /* 4162 * Remove attributes on the specified address range (ie. loginal NAND) 4163 */ 4164 void 4165 hat_clrattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4166 { 4167 if (hat->sfmmu_xhat_provider) { 4168 XHAT_CLRATTR(hat, addr, len, attr); 4169 return; 4170 } else { 4171 /* 4172 * This must be a CPU HAT. If the address space has 4173 * XHATs attached, change attributes for all of them, 4174 * just in case 4175 */ 4176 ASSERT(hat->sfmmu_as != NULL); 4177 if (hat->sfmmu_as->a_xhat != NULL) 4178 xhat_clrattr_all(hat->sfmmu_as, addr, len, attr); 4179 } 4180 4181 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CLRATTR); 4182 } 4183 4184 /* 4185 * Change attributes on an address range to that specified by attr and mode. 4186 */ 4187 static void 4188 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr, 4189 int mode) 4190 { 4191 struct hmehash_bucket *hmebp; 4192 hmeblk_tag hblktag; 4193 int hmeshift, hashno = 1; 4194 struct hme_blk *hmeblkp, *list = NULL; 4195 caddr_t endaddr; 4196 cpuset_t cpuset; 4197 demap_range_t dmr; 4198 4199 CPUSET_ZERO(cpuset); 4200 4201 ASSERT((sfmmup == ksfmmup) || 4202 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4203 ASSERT((len & MMU_PAGEOFFSET) == 0); 4204 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4205 4206 if ((attr & PROT_USER) && (mode != SFMMU_CLRATTR) && 4207 ((addr + len) > (caddr_t)USERLIMIT)) { 4208 panic("user addr %p in kernel space", 4209 (void *)addr); 4210 } 4211 4212 endaddr = addr + len; 4213 hblktag.htag_id = sfmmup; 4214 DEMAP_RANGE_INIT(sfmmup, &dmr); 4215 4216 while (addr < endaddr) { 4217 hmeshift = HME_HASH_SHIFT(hashno); 4218 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4219 hblktag.htag_rehash = hashno; 4220 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4221 4222 SFMMU_HASH_LOCK(hmebp); 4223 4224 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4225 if (hmeblkp != NULL) { 4226 /* 4227 * We've encountered a shadow hmeblk so skip the range 4228 * of the next smaller mapping size. 4229 */ 4230 if (hmeblkp->hblk_shw_bit) { 4231 ASSERT(sfmmup != ksfmmup); 4232 ASSERT(hashno > 1); 4233 addr = (caddr_t)P2END((uintptr_t)addr, 4234 TTEBYTES(hashno - 1)); 4235 } else { 4236 addr = sfmmu_hblk_chgattr(sfmmup, 4237 hmeblkp, addr, endaddr, &dmr, attr, mode); 4238 } 4239 SFMMU_HASH_UNLOCK(hmebp); 4240 hashno = 1; 4241 continue; 4242 } 4243 SFMMU_HASH_UNLOCK(hmebp); 4244 4245 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4246 /* 4247 * We have traversed the whole list and rehashed 4248 * if necessary without finding the address to chgattr. 4249 * This is ok, so we increment the address by the 4250 * smallest hmeblk range for kernel mappings or for 4251 * user mappings with no large pages, and the largest 4252 * hmeblk range, to account for shadow hmeblks, for 4253 * user mappings with large pages and continue. 4254 */ 4255 if (sfmmup == ksfmmup) 4256 addr = (caddr_t)P2END((uintptr_t)addr, 4257 TTEBYTES(1)); 4258 else 4259 addr = (caddr_t)P2END((uintptr_t)addr, 4260 TTEBYTES(hashno)); 4261 hashno = 1; 4262 } else { 4263 hashno++; 4264 } 4265 } 4266 4267 sfmmu_hblks_list_purge(&list); 4268 DEMAP_RANGE_FLUSH(&dmr); 4269 cpuset = sfmmup->sfmmu_cpusran; 4270 xt_sync(cpuset); 4271 } 4272 4273 /* 4274 * This function chgattr on a range of addresses in an hmeblk. It returns the 4275 * next addres that needs to be chgattr. 4276 * It should be called with the hash lock held. 4277 * XXX It should be possible to optimize chgattr by not flushing every time but 4278 * on the other hand: 4279 * 1. do one flush crosscall. 4280 * 2. only flush if we are increasing permissions (make sure this will work) 4281 */ 4282 static caddr_t 4283 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4284 caddr_t endaddr, demap_range_t *dmrp, uint_t attr, int mode) 4285 { 4286 tte_t tte, tteattr, tteflags, ttemod; 4287 struct sf_hment *sfhmep; 4288 int ttesz; 4289 struct page *pp = NULL; 4290 kmutex_t *pml, *pmtx; 4291 int ret; 4292 int use_demap_range; 4293 #if defined(SF_ERRATA_57) 4294 int check_exec; 4295 #endif 4296 4297 ASSERT(in_hblk_range(hmeblkp, addr)); 4298 ASSERT(hmeblkp->hblk_shw_bit == 0); 4299 4300 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4301 ttesz = get_hblk_ttesz(hmeblkp); 4302 4303 /* 4304 * Flush the current demap region if addresses have been 4305 * skipped or the page size doesn't match. 4306 */ 4307 use_demap_range = (TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp)); 4308 if (use_demap_range) { 4309 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4310 } else { 4311 DEMAP_RANGE_FLUSH(dmrp); 4312 } 4313 4314 tteattr.ll = sfmmu_vtop_attr(attr, mode, &tteflags); 4315 #if defined(SF_ERRATA_57) 4316 check_exec = (sfmmup != ksfmmup) && 4317 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4318 TTE_IS_EXECUTABLE(&tteattr); 4319 #endif 4320 HBLKTOHME(sfhmep, hmeblkp, addr); 4321 while (addr < endaddr) { 4322 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4323 if (TTE_IS_VALID(&tte)) { 4324 if ((tte.ll & tteflags.ll) == tteattr.ll) { 4325 /* 4326 * if the new attr is the same as old 4327 * continue 4328 */ 4329 goto next_addr; 4330 } 4331 if (!TTE_IS_WRITABLE(&tteattr)) { 4332 /* 4333 * make sure we clear hw modify bit if we 4334 * removing write protections 4335 */ 4336 tteflags.tte_intlo |= TTE_HWWR_INT; 4337 } 4338 4339 pml = NULL; 4340 pp = sfhmep->hme_page; 4341 if (pp) { 4342 pml = sfmmu_mlist_enter(pp); 4343 } 4344 4345 if (pp != sfhmep->hme_page) { 4346 /* 4347 * tte must have been unloaded. 4348 */ 4349 ASSERT(pml); 4350 sfmmu_mlist_exit(pml); 4351 continue; 4352 } 4353 4354 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4355 4356 ttemod = tte; 4357 ttemod.ll = (ttemod.ll & ~tteflags.ll) | tteattr.ll; 4358 ASSERT(TTE_TO_TTEPFN(&ttemod) == TTE_TO_TTEPFN(&tte)); 4359 4360 #if defined(SF_ERRATA_57) 4361 if (check_exec && addr < errata57_limit) 4362 ttemod.tte_exec_perm = 0; 4363 #endif 4364 ret = sfmmu_modifytte_try(&tte, &ttemod, 4365 &sfhmep->hme_tte); 4366 4367 if (ret < 0) { 4368 /* tte changed underneath us */ 4369 if (pml) { 4370 sfmmu_mlist_exit(pml); 4371 } 4372 continue; 4373 } 4374 4375 if (tteflags.tte_intlo & TTE_HWWR_INT) { 4376 /* 4377 * need to sync if we are clearing modify bit. 4378 */ 4379 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4380 } 4381 4382 if (pp && PP_ISRO(pp)) { 4383 if (tteattr.tte_intlo & TTE_WRPRM_INT) { 4384 pmtx = sfmmu_page_enter(pp); 4385 PP_CLRRO(pp); 4386 sfmmu_page_exit(pmtx); 4387 } 4388 } 4389 4390 if (ret > 0 && use_demap_range) { 4391 DEMAP_RANGE_MARKPG(dmrp, addr); 4392 } else if (ret > 0) { 4393 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4394 } 4395 4396 if (pml) { 4397 sfmmu_mlist_exit(pml); 4398 } 4399 } 4400 next_addr: 4401 addr += TTEBYTES(ttesz); 4402 sfhmep++; 4403 DEMAP_RANGE_NEXTPG(dmrp); 4404 } 4405 return (addr); 4406 } 4407 4408 /* 4409 * This routine converts virtual attributes to physical ones. It will 4410 * update the tteflags field with the tte mask corresponding to the attributes 4411 * affected and it returns the new attributes. It will also clear the modify 4412 * bit if we are taking away write permission. This is necessary since the 4413 * modify bit is the hardware permission bit and we need to clear it in order 4414 * to detect write faults. 4415 */ 4416 static uint64_t 4417 sfmmu_vtop_attr(uint_t attr, int mode, tte_t *ttemaskp) 4418 { 4419 tte_t ttevalue; 4420 4421 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 4422 4423 switch (mode) { 4424 case SFMMU_CHGATTR: 4425 /* all attributes specified */ 4426 ttevalue.tte_inthi = MAKE_TTEATTR_INTHI(attr); 4427 ttevalue.tte_intlo = MAKE_TTEATTR_INTLO(attr); 4428 ttemaskp->tte_inthi = TTEINTHI_ATTR; 4429 ttemaskp->tte_intlo = TTEINTLO_ATTR; 4430 break; 4431 case SFMMU_SETATTR: 4432 ASSERT(!(attr & ~HAT_PROT_MASK)); 4433 ttemaskp->ll = 0; 4434 ttevalue.ll = 0; 4435 /* 4436 * a valid tte implies exec and read for sfmmu 4437 * so no need to do anything about them. 4438 * since priviledged access implies user access 4439 * PROT_USER doesn't make sense either. 4440 */ 4441 if (attr & PROT_WRITE) { 4442 ttemaskp->tte_intlo |= TTE_WRPRM_INT; 4443 ttevalue.tte_intlo |= TTE_WRPRM_INT; 4444 } 4445 break; 4446 case SFMMU_CLRATTR: 4447 /* attributes will be nand with current ones */ 4448 if (attr & ~(PROT_WRITE | PROT_USER)) { 4449 panic("sfmmu: attr %x not supported", attr); 4450 } 4451 ttemaskp->ll = 0; 4452 ttevalue.ll = 0; 4453 if (attr & PROT_WRITE) { 4454 /* clear both writable and modify bit */ 4455 ttemaskp->tte_intlo |= TTE_WRPRM_INT | TTE_HWWR_INT; 4456 } 4457 if (attr & PROT_USER) { 4458 ttemaskp->tte_intlo |= TTE_PRIV_INT; 4459 ttevalue.tte_intlo |= TTE_PRIV_INT; 4460 } 4461 break; 4462 default: 4463 panic("sfmmu_vtop_attr: bad mode %x", mode); 4464 } 4465 ASSERT(TTE_TO_TTEPFN(&ttevalue) == 0); 4466 return (ttevalue.ll); 4467 } 4468 4469 static uint_t 4470 sfmmu_ptov_attr(tte_t *ttep) 4471 { 4472 uint_t attr; 4473 4474 ASSERT(TTE_IS_VALID(ttep)); 4475 4476 attr = PROT_READ; 4477 4478 if (TTE_IS_WRITABLE(ttep)) { 4479 attr |= PROT_WRITE; 4480 } 4481 if (TTE_IS_EXECUTABLE(ttep)) { 4482 attr |= PROT_EXEC; 4483 } 4484 if (!TTE_IS_PRIVILEGED(ttep)) { 4485 attr |= PROT_USER; 4486 } 4487 if (TTE_IS_NFO(ttep)) { 4488 attr |= HAT_NOFAULT; 4489 } 4490 if (TTE_IS_NOSYNC(ttep)) { 4491 attr |= HAT_NOSYNC; 4492 } 4493 if (TTE_IS_SIDEFFECT(ttep)) { 4494 attr |= SFMMU_SIDEFFECT; 4495 } 4496 if (!TTE_IS_VCACHEABLE(ttep)) { 4497 attr |= SFMMU_UNCACHEVTTE; 4498 } 4499 if (!TTE_IS_PCACHEABLE(ttep)) { 4500 attr |= SFMMU_UNCACHEPTTE; 4501 } 4502 return (attr); 4503 } 4504 4505 /* 4506 * hat_chgprot is a deprecated hat call. New segment drivers 4507 * should store all attributes and use hat_*attr calls. 4508 * 4509 * Change the protections in the virtual address range 4510 * given to the specified virtual protection. If vprot is ~PROT_WRITE, 4511 * then remove write permission, leaving the other 4512 * permissions unchanged. If vprot is ~PROT_USER, remove user permissions. 4513 * 4514 */ 4515 void 4516 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot) 4517 { 4518 struct hmehash_bucket *hmebp; 4519 hmeblk_tag hblktag; 4520 int hmeshift, hashno = 1; 4521 struct hme_blk *hmeblkp, *list = NULL; 4522 caddr_t endaddr; 4523 cpuset_t cpuset; 4524 demap_range_t dmr; 4525 4526 ASSERT((len & MMU_PAGEOFFSET) == 0); 4527 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4528 4529 if (sfmmup->sfmmu_xhat_provider) { 4530 XHAT_CHGPROT(sfmmup, addr, len, vprot); 4531 return; 4532 } else { 4533 /* 4534 * This must be a CPU HAT. If the address space has 4535 * XHATs attached, change attributes for all of them, 4536 * just in case 4537 */ 4538 ASSERT(sfmmup->sfmmu_as != NULL); 4539 if (sfmmup->sfmmu_as->a_xhat != NULL) 4540 xhat_chgprot_all(sfmmup->sfmmu_as, addr, len, vprot); 4541 } 4542 4543 CPUSET_ZERO(cpuset); 4544 4545 if ((vprot != (uint_t)~PROT_WRITE) && (vprot & PROT_USER) && 4546 ((addr + len) > (caddr_t)USERLIMIT)) { 4547 panic("user addr %p vprot %x in kernel space", 4548 (void *)addr, vprot); 4549 } 4550 endaddr = addr + len; 4551 hblktag.htag_id = sfmmup; 4552 DEMAP_RANGE_INIT(sfmmup, &dmr); 4553 4554 while (addr < endaddr) { 4555 hmeshift = HME_HASH_SHIFT(hashno); 4556 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4557 hblktag.htag_rehash = hashno; 4558 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4559 4560 SFMMU_HASH_LOCK(hmebp); 4561 4562 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4563 if (hmeblkp != NULL) { 4564 /* 4565 * We've encountered a shadow hmeblk so skip the range 4566 * of the next smaller mapping size. 4567 */ 4568 if (hmeblkp->hblk_shw_bit) { 4569 ASSERT(sfmmup != ksfmmup); 4570 ASSERT(hashno > 1); 4571 addr = (caddr_t)P2END((uintptr_t)addr, 4572 TTEBYTES(hashno - 1)); 4573 } else { 4574 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp, 4575 addr, endaddr, &dmr, vprot); 4576 } 4577 SFMMU_HASH_UNLOCK(hmebp); 4578 hashno = 1; 4579 continue; 4580 } 4581 SFMMU_HASH_UNLOCK(hmebp); 4582 4583 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4584 /* 4585 * We have traversed the whole list and rehashed 4586 * if necessary without finding the address to chgprot. 4587 * This is ok so we increment the address by the 4588 * smallest hmeblk range for kernel mappings and the 4589 * largest hmeblk range, to account for shadow hmeblks, 4590 * for user mappings and continue. 4591 */ 4592 if (sfmmup == ksfmmup) 4593 addr = (caddr_t)P2END((uintptr_t)addr, 4594 TTEBYTES(1)); 4595 else 4596 addr = (caddr_t)P2END((uintptr_t)addr, 4597 TTEBYTES(hashno)); 4598 hashno = 1; 4599 } else { 4600 hashno++; 4601 } 4602 } 4603 4604 sfmmu_hblks_list_purge(&list); 4605 DEMAP_RANGE_FLUSH(&dmr); 4606 cpuset = sfmmup->sfmmu_cpusran; 4607 xt_sync(cpuset); 4608 } 4609 4610 /* 4611 * This function chgprots a range of addresses in an hmeblk. It returns the 4612 * next addres that needs to be chgprot. 4613 * It should be called with the hash lock held. 4614 * XXX It shold be possible to optimize chgprot by not flushing every time but 4615 * on the other hand: 4616 * 1. do one flush crosscall. 4617 * 2. only flush if we are increasing permissions (make sure this will work) 4618 */ 4619 static caddr_t 4620 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4621 caddr_t endaddr, demap_range_t *dmrp, uint_t vprot) 4622 { 4623 uint_t pprot; 4624 tte_t tte, ttemod; 4625 struct sf_hment *sfhmep; 4626 uint_t tteflags; 4627 int ttesz; 4628 struct page *pp = NULL; 4629 kmutex_t *pml, *pmtx; 4630 int ret; 4631 int use_demap_range; 4632 #if defined(SF_ERRATA_57) 4633 int check_exec; 4634 #endif 4635 4636 ASSERT(in_hblk_range(hmeblkp, addr)); 4637 ASSERT(hmeblkp->hblk_shw_bit == 0); 4638 4639 #ifdef DEBUG 4640 if (get_hblk_ttesz(hmeblkp) != TTE8K && 4641 (endaddr < get_hblk_endaddr(hmeblkp))) { 4642 panic("sfmmu_hblk_chgprot: partial chgprot of large page"); 4643 } 4644 #endif /* DEBUG */ 4645 4646 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4647 ttesz = get_hblk_ttesz(hmeblkp); 4648 4649 pprot = sfmmu_vtop_prot(vprot, &tteflags); 4650 #if defined(SF_ERRATA_57) 4651 check_exec = (sfmmup != ksfmmup) && 4652 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4653 ((vprot & PROT_EXEC) == PROT_EXEC); 4654 #endif 4655 HBLKTOHME(sfhmep, hmeblkp, addr); 4656 4657 /* 4658 * Flush the current demap region if addresses have been 4659 * skipped or the page size doesn't match. 4660 */ 4661 use_demap_range = (TTEBYTES(ttesz) == MMU_PAGESIZE); 4662 if (use_demap_range) { 4663 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4664 } else { 4665 DEMAP_RANGE_FLUSH(dmrp); 4666 } 4667 4668 while (addr < endaddr) { 4669 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4670 if (TTE_IS_VALID(&tte)) { 4671 if (TTE_GET_LOFLAGS(&tte, tteflags) == pprot) { 4672 /* 4673 * if the new protection is the same as old 4674 * continue 4675 */ 4676 goto next_addr; 4677 } 4678 pml = NULL; 4679 pp = sfhmep->hme_page; 4680 if (pp) { 4681 pml = sfmmu_mlist_enter(pp); 4682 } 4683 if (pp != sfhmep->hme_page) { 4684 /* 4685 * tte most have been unloaded 4686 * underneath us. Recheck 4687 */ 4688 ASSERT(pml); 4689 sfmmu_mlist_exit(pml); 4690 continue; 4691 } 4692 4693 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4694 4695 ttemod = tte; 4696 TTE_SET_LOFLAGS(&ttemod, tteflags, pprot); 4697 #if defined(SF_ERRATA_57) 4698 if (check_exec && addr < errata57_limit) 4699 ttemod.tte_exec_perm = 0; 4700 #endif 4701 ret = sfmmu_modifytte_try(&tte, &ttemod, 4702 &sfhmep->hme_tte); 4703 4704 if (ret < 0) { 4705 /* tte changed underneath us */ 4706 if (pml) { 4707 sfmmu_mlist_exit(pml); 4708 } 4709 continue; 4710 } 4711 4712 if (tteflags & TTE_HWWR_INT) { 4713 /* 4714 * need to sync if we are clearing modify bit. 4715 */ 4716 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4717 } 4718 4719 if (pp && PP_ISRO(pp)) { 4720 if (pprot & TTE_WRPRM_INT) { 4721 pmtx = sfmmu_page_enter(pp); 4722 PP_CLRRO(pp); 4723 sfmmu_page_exit(pmtx); 4724 } 4725 } 4726 4727 if (ret > 0 && use_demap_range) { 4728 DEMAP_RANGE_MARKPG(dmrp, addr); 4729 } else if (ret > 0) { 4730 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4731 } 4732 4733 if (pml) { 4734 sfmmu_mlist_exit(pml); 4735 } 4736 } 4737 next_addr: 4738 addr += TTEBYTES(ttesz); 4739 sfhmep++; 4740 DEMAP_RANGE_NEXTPG(dmrp); 4741 } 4742 return (addr); 4743 } 4744 4745 /* 4746 * This routine is deprecated and should only be used by hat_chgprot. 4747 * The correct routine is sfmmu_vtop_attr. 4748 * This routine converts virtual page protections to physical ones. It will 4749 * update the tteflags field with the tte mask corresponding to the protections 4750 * affected and it returns the new protections. It will also clear the modify 4751 * bit if we are taking away write permission. This is necessary since the 4752 * modify bit is the hardware permission bit and we need to clear it in order 4753 * to detect write faults. 4754 * It accepts the following special protections: 4755 * ~PROT_WRITE = remove write permissions. 4756 * ~PROT_USER = remove user permissions. 4757 */ 4758 static uint_t 4759 sfmmu_vtop_prot(uint_t vprot, uint_t *tteflagsp) 4760 { 4761 if (vprot == (uint_t)~PROT_WRITE) { 4762 *tteflagsp = TTE_WRPRM_INT | TTE_HWWR_INT; 4763 return (0); /* will cause wrprm to be cleared */ 4764 } 4765 if (vprot == (uint_t)~PROT_USER) { 4766 *tteflagsp = TTE_PRIV_INT; 4767 return (0); /* will cause privprm to be cleared */ 4768 } 4769 if ((vprot == 0) || (vprot == PROT_USER) || 4770 ((vprot & PROT_ALL) != vprot)) { 4771 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4772 } 4773 4774 switch (vprot) { 4775 case (PROT_READ): 4776 case (PROT_EXEC): 4777 case (PROT_EXEC | PROT_READ): 4778 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4779 return (TTE_PRIV_INT); /* set prv and clr wrt */ 4780 case (PROT_WRITE): 4781 case (PROT_WRITE | PROT_READ): 4782 case (PROT_EXEC | PROT_WRITE): 4783 case (PROT_EXEC | PROT_WRITE | PROT_READ): 4784 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4785 return (TTE_PRIV_INT | TTE_WRPRM_INT); /* set prv and wrt */ 4786 case (PROT_USER | PROT_READ): 4787 case (PROT_USER | PROT_EXEC): 4788 case (PROT_USER | PROT_EXEC | PROT_READ): 4789 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4790 return (0); /* clr prv and wrt */ 4791 case (PROT_USER | PROT_WRITE): 4792 case (PROT_USER | PROT_WRITE | PROT_READ): 4793 case (PROT_USER | PROT_EXEC | PROT_WRITE): 4794 case (PROT_USER | PROT_EXEC | PROT_WRITE | PROT_READ): 4795 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4796 return (TTE_WRPRM_INT); /* clr prv and set wrt */ 4797 default: 4798 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4799 } 4800 return (0); 4801 } 4802 4803 /* 4804 * Alternate unload for very large virtual ranges. With a true 64 bit VA, 4805 * the normal algorithm would take too long for a very large VA range with 4806 * few real mappings. This routine just walks thru all HMEs in the global 4807 * hash table to find and remove mappings. 4808 */ 4809 static void 4810 hat_unload_large_virtual( 4811 struct hat *sfmmup, 4812 caddr_t startaddr, 4813 size_t len, 4814 uint_t flags, 4815 hat_callback_t *callback) 4816 { 4817 struct hmehash_bucket *hmebp; 4818 struct hme_blk *hmeblkp; 4819 struct hme_blk *pr_hblk = NULL; 4820 struct hme_blk *nx_hblk; 4821 struct hme_blk *list = NULL; 4822 int i; 4823 uint64_t hblkpa, prevpa, nx_pa; 4824 demap_range_t dmr, *dmrp; 4825 cpuset_t cpuset; 4826 caddr_t endaddr = startaddr + len; 4827 caddr_t sa; 4828 caddr_t ea; 4829 caddr_t cb_sa[MAX_CB_ADDR]; 4830 caddr_t cb_ea[MAX_CB_ADDR]; 4831 int addr_cnt = 0; 4832 int a = 0; 4833 4834 if (sfmmup->sfmmu_free) { 4835 dmrp = NULL; 4836 } else { 4837 dmrp = &dmr; 4838 DEMAP_RANGE_INIT(sfmmup, dmrp); 4839 } 4840 4841 /* 4842 * Loop through all the hash buckets of HME blocks looking for matches. 4843 */ 4844 for (i = 0; i <= UHMEHASH_SZ; i++) { 4845 hmebp = &uhme_hash[i]; 4846 SFMMU_HASH_LOCK(hmebp); 4847 hmeblkp = hmebp->hmeblkp; 4848 hblkpa = hmebp->hmeh_nextpa; 4849 prevpa = 0; 4850 pr_hblk = NULL; 4851 while (hmeblkp) { 4852 nx_hblk = hmeblkp->hblk_next; 4853 nx_pa = hmeblkp->hblk_nextpa; 4854 4855 /* 4856 * skip if not this context, if a shadow block or 4857 * if the mapping is not in the requested range 4858 */ 4859 if (hmeblkp->hblk_tag.htag_id != sfmmup || 4860 hmeblkp->hblk_shw_bit || 4861 (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr || 4862 (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) { 4863 pr_hblk = hmeblkp; 4864 prevpa = hblkpa; 4865 goto next_block; 4866 } 4867 4868 /* 4869 * unload if there are any current valid mappings 4870 */ 4871 if (hmeblkp->hblk_vcnt != 0 || 4872 hmeblkp->hblk_hmecnt != 0) 4873 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 4874 sa, ea, dmrp, flags); 4875 4876 /* 4877 * on unmap we also release the HME block itself, once 4878 * all mappings are gone. 4879 */ 4880 if ((flags & HAT_UNLOAD_UNMAP) != 0 && 4881 !hmeblkp->hblk_vcnt && 4882 !hmeblkp->hblk_hmecnt) { 4883 ASSERT(!hmeblkp->hblk_lckcnt); 4884 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 4885 prevpa, pr_hblk); 4886 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 4887 } else { 4888 pr_hblk = hmeblkp; 4889 prevpa = hblkpa; 4890 } 4891 4892 if (callback == NULL) 4893 goto next_block; 4894 4895 /* 4896 * HME blocks may span more than one page, but we may be 4897 * unmapping only one page, so check for a smaller range 4898 * for the callback 4899 */ 4900 if (sa < startaddr) 4901 sa = startaddr; 4902 if (--ea > endaddr) 4903 ea = endaddr - 1; 4904 4905 cb_sa[addr_cnt] = sa; 4906 cb_ea[addr_cnt] = ea; 4907 if (++addr_cnt == MAX_CB_ADDR) { 4908 if (dmrp != NULL) { 4909 DEMAP_RANGE_FLUSH(dmrp); 4910 cpuset = sfmmup->sfmmu_cpusran; 4911 xt_sync(cpuset); 4912 } 4913 4914 for (a = 0; a < MAX_CB_ADDR; ++a) { 4915 callback->hcb_start_addr = cb_sa[a]; 4916 callback->hcb_end_addr = cb_ea[a]; 4917 callback->hcb_function(callback); 4918 } 4919 addr_cnt = 0; 4920 } 4921 4922 next_block: 4923 hmeblkp = nx_hblk; 4924 hblkpa = nx_pa; 4925 } 4926 SFMMU_HASH_UNLOCK(hmebp); 4927 } 4928 4929 sfmmu_hblks_list_purge(&list); 4930 if (dmrp != NULL) { 4931 DEMAP_RANGE_FLUSH(dmrp); 4932 cpuset = sfmmup->sfmmu_cpusran; 4933 xt_sync(cpuset); 4934 } 4935 4936 for (a = 0; a < addr_cnt; ++a) { 4937 callback->hcb_start_addr = cb_sa[a]; 4938 callback->hcb_end_addr = cb_ea[a]; 4939 callback->hcb_function(callback); 4940 } 4941 4942 /* 4943 * Check TSB and TLB page sizes if the process isn't exiting. 4944 */ 4945 if (!sfmmup->sfmmu_free) 4946 sfmmu_check_page_sizes(sfmmup, 0); 4947 } 4948 4949 /* 4950 * Unload all the mappings in the range [addr..addr+len). addr and len must 4951 * be MMU_PAGESIZE aligned. 4952 */ 4953 4954 extern struct seg *segkmap; 4955 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \ 4956 segkmap->s_base <= (addr) && (addr) < (segkmap->s_base + segkmap->s_size)) 4957 4958 4959 void 4960 hat_unload_callback( 4961 struct hat *sfmmup, 4962 caddr_t addr, 4963 size_t len, 4964 uint_t flags, 4965 hat_callback_t *callback) 4966 { 4967 struct hmehash_bucket *hmebp; 4968 hmeblk_tag hblktag; 4969 int hmeshift, hashno, iskernel; 4970 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 4971 caddr_t endaddr; 4972 cpuset_t cpuset; 4973 uint64_t hblkpa, prevpa; 4974 int addr_count = 0; 4975 int a; 4976 caddr_t cb_start_addr[MAX_CB_ADDR]; 4977 caddr_t cb_end_addr[MAX_CB_ADDR]; 4978 int issegkmap = ISSEGKMAP(sfmmup, addr); 4979 demap_range_t dmr, *dmrp; 4980 4981 if (sfmmup->sfmmu_xhat_provider) { 4982 XHAT_UNLOAD_CALLBACK(sfmmup, addr, len, flags, callback); 4983 return; 4984 } else { 4985 /* 4986 * This must be a CPU HAT. If the address space has 4987 * XHATs attached, unload the mappings for all of them, 4988 * just in case 4989 */ 4990 ASSERT(sfmmup->sfmmu_as != NULL); 4991 if (sfmmup->sfmmu_as->a_xhat != NULL) 4992 xhat_unload_callback_all(sfmmup->sfmmu_as, addr, 4993 len, flags, callback); 4994 } 4995 4996 ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \ 4997 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4998 4999 ASSERT(sfmmup != NULL); 5000 ASSERT((len & MMU_PAGEOFFSET) == 0); 5001 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 5002 5003 /* 5004 * Probing through a large VA range (say 63 bits) will be slow, even 5005 * at 4 Meg steps between the probes. So, when the virtual address range 5006 * is very large, search the HME entries for what to unload. 5007 * 5008 * len >> TTE_PAGE_SHIFT(TTE4M) is the # of 4Meg probes we'd need 5009 * 5010 * UHMEHASH_SZ is number of hash buckets to examine 5011 * 5012 */ 5013 if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) { 5014 hat_unload_large_virtual(sfmmup, addr, len, flags, callback); 5015 return; 5016 } 5017 5018 CPUSET_ZERO(cpuset); 5019 5020 /* 5021 * If the process is exiting, we can save a lot of fuss since 5022 * we'll flush the TLB when we free the ctx anyway. 5023 */ 5024 if (sfmmup->sfmmu_free) 5025 dmrp = NULL; 5026 else 5027 dmrp = &dmr; 5028 5029 DEMAP_RANGE_INIT(sfmmup, dmrp); 5030 endaddr = addr + len; 5031 hblktag.htag_id = sfmmup; 5032 5033 /* 5034 * It is likely for the vm to call unload over a wide range of 5035 * addresses that are actually very sparsely populated by 5036 * translations. In order to speed this up the sfmmu hat supports 5037 * the concept of shadow hmeblks. Dummy large page hmeblks that 5038 * correspond to actual small translations are allocated at tteload 5039 * time and are referred to as shadow hmeblks. Now, during unload 5040 * time, we first check if we have a shadow hmeblk for that 5041 * translation. The absence of one means the corresponding address 5042 * range is empty and can be skipped. 5043 * 5044 * The kernel is an exception to above statement and that is why 5045 * we don't use shadow hmeblks and hash starting from the smallest 5046 * page size. 5047 */ 5048 if (sfmmup == KHATID) { 5049 iskernel = 1; 5050 hashno = TTE64K; 5051 } else { 5052 iskernel = 0; 5053 if (mmu_page_sizes == max_mmu_page_sizes) { 5054 hashno = TTE256M; 5055 } else { 5056 hashno = TTE4M; 5057 } 5058 } 5059 while (addr < endaddr) { 5060 hmeshift = HME_HASH_SHIFT(hashno); 5061 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5062 hblktag.htag_rehash = hashno; 5063 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5064 5065 SFMMU_HASH_LOCK(hmebp); 5066 5067 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, pr_hblk, 5068 prevpa, &list); 5069 if (hmeblkp == NULL) { 5070 /* 5071 * didn't find an hmeblk. skip the appropiate 5072 * address range. 5073 */ 5074 SFMMU_HASH_UNLOCK(hmebp); 5075 if (iskernel) { 5076 if (hashno < mmu_hashcnt) { 5077 hashno++; 5078 continue; 5079 } else { 5080 hashno = TTE64K; 5081 addr = (caddr_t)roundup((uintptr_t)addr 5082 + 1, MMU_PAGESIZE64K); 5083 continue; 5084 } 5085 } 5086 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5087 (1 << hmeshift)); 5088 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5089 ASSERT(hashno == TTE64K); 5090 continue; 5091 } 5092 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5093 hashno = TTE512K; 5094 continue; 5095 } 5096 if (mmu_page_sizes == max_mmu_page_sizes) { 5097 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5098 hashno = TTE4M; 5099 continue; 5100 } 5101 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5102 hashno = TTE32M; 5103 continue; 5104 } 5105 hashno = TTE256M; 5106 continue; 5107 } else { 5108 hashno = TTE4M; 5109 continue; 5110 } 5111 } 5112 ASSERT(hmeblkp); 5113 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5114 /* 5115 * If the valid count is zero we can skip the range 5116 * mapped by this hmeblk. 5117 * We free hblks in the case of HAT_UNMAP. HAT_UNMAP 5118 * is used by segment drivers as a hint 5119 * that the mapping resource won't be used any longer. 5120 * The best example of this is during exit(). 5121 */ 5122 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5123 get_hblk_span(hmeblkp)); 5124 if ((flags & HAT_UNLOAD_UNMAP) || 5125 (iskernel && !issegkmap)) { 5126 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5127 pr_hblk); 5128 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5129 } 5130 SFMMU_HASH_UNLOCK(hmebp); 5131 5132 if (iskernel) { 5133 hashno = TTE64K; 5134 continue; 5135 } 5136 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5137 ASSERT(hashno == TTE64K); 5138 continue; 5139 } 5140 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5141 hashno = TTE512K; 5142 continue; 5143 } 5144 if (mmu_page_sizes == max_mmu_page_sizes) { 5145 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5146 hashno = TTE4M; 5147 continue; 5148 } 5149 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5150 hashno = TTE32M; 5151 continue; 5152 } 5153 hashno = TTE256M; 5154 continue; 5155 } else { 5156 hashno = TTE4M; 5157 continue; 5158 } 5159 } 5160 if (hmeblkp->hblk_shw_bit) { 5161 /* 5162 * If we encounter a shadow hmeblk we know there is 5163 * smaller sized hmeblks mapping the same address space. 5164 * Decrement the hash size and rehash. 5165 */ 5166 ASSERT(sfmmup != KHATID); 5167 hashno--; 5168 SFMMU_HASH_UNLOCK(hmebp); 5169 continue; 5170 } 5171 5172 /* 5173 * track callback address ranges. 5174 * only start a new range when it's not contiguous 5175 */ 5176 if (callback != NULL) { 5177 if (addr_count > 0 && 5178 addr == cb_end_addr[addr_count - 1]) 5179 --addr_count; 5180 else 5181 cb_start_addr[addr_count] = addr; 5182 } 5183 5184 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr, 5185 dmrp, flags); 5186 5187 if (callback != NULL) 5188 cb_end_addr[addr_count++] = addr; 5189 5190 if (((flags & HAT_UNLOAD_UNMAP) || (iskernel && !issegkmap)) && 5191 !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5192 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5193 pr_hblk); 5194 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5195 } 5196 SFMMU_HASH_UNLOCK(hmebp); 5197 5198 /* 5199 * Notify our caller as to exactly which pages 5200 * have been unloaded. We do these in clumps, 5201 * to minimize the number of xt_sync()s that need to occur. 5202 */ 5203 if (callback != NULL && addr_count == MAX_CB_ADDR) { 5204 DEMAP_RANGE_FLUSH(dmrp); 5205 if (dmrp != NULL) { 5206 cpuset = sfmmup->sfmmu_cpusran; 5207 xt_sync(cpuset); 5208 } 5209 5210 for (a = 0; a < MAX_CB_ADDR; ++a) { 5211 callback->hcb_start_addr = cb_start_addr[a]; 5212 callback->hcb_end_addr = cb_end_addr[a]; 5213 callback->hcb_function(callback); 5214 } 5215 addr_count = 0; 5216 } 5217 if (iskernel) { 5218 hashno = TTE64K; 5219 continue; 5220 } 5221 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5222 ASSERT(hashno == TTE64K); 5223 continue; 5224 } 5225 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5226 hashno = TTE512K; 5227 continue; 5228 } 5229 if (mmu_page_sizes == max_mmu_page_sizes) { 5230 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5231 hashno = TTE4M; 5232 continue; 5233 } 5234 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5235 hashno = TTE32M; 5236 continue; 5237 } 5238 hashno = TTE256M; 5239 } else { 5240 hashno = TTE4M; 5241 } 5242 } 5243 5244 sfmmu_hblks_list_purge(&list); 5245 DEMAP_RANGE_FLUSH(dmrp); 5246 if (dmrp != NULL) { 5247 cpuset = sfmmup->sfmmu_cpusran; 5248 xt_sync(cpuset); 5249 } 5250 if (callback && addr_count != 0) { 5251 for (a = 0; a < addr_count; ++a) { 5252 callback->hcb_start_addr = cb_start_addr[a]; 5253 callback->hcb_end_addr = cb_end_addr[a]; 5254 callback->hcb_function(callback); 5255 } 5256 } 5257 5258 /* 5259 * Check TSB and TLB page sizes if the process isn't exiting. 5260 */ 5261 if (!sfmmup->sfmmu_free) 5262 sfmmu_check_page_sizes(sfmmup, 0); 5263 } 5264 5265 /* 5266 * Unload all the mappings in the range [addr..addr+len). addr and len must 5267 * be MMU_PAGESIZE aligned. 5268 */ 5269 void 5270 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags) 5271 { 5272 if (sfmmup->sfmmu_xhat_provider) { 5273 XHAT_UNLOAD(sfmmup, addr, len, flags); 5274 return; 5275 } 5276 hat_unload_callback(sfmmup, addr, len, flags, NULL); 5277 } 5278 5279 5280 /* 5281 * Find the largest mapping size for this page. 5282 */ 5283 int 5284 fnd_mapping_sz(page_t *pp) 5285 { 5286 int sz; 5287 int p_index; 5288 5289 p_index = PP_MAPINDEX(pp); 5290 5291 sz = 0; 5292 p_index >>= 1; /* don't care about 8K bit */ 5293 for (; p_index; p_index >>= 1) { 5294 sz++; 5295 } 5296 5297 return (sz); 5298 } 5299 5300 /* 5301 * This function unloads a range of addresses for an hmeblk. 5302 * It returns the next address to be unloaded. 5303 * It should be called with the hash lock held. 5304 */ 5305 static caddr_t 5306 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5307 caddr_t endaddr, demap_range_t *dmrp, uint_t flags) 5308 { 5309 tte_t tte, ttemod; 5310 struct sf_hment *sfhmep; 5311 int ttesz; 5312 long ttecnt; 5313 page_t *pp; 5314 kmutex_t *pml; 5315 int ret; 5316 int use_demap_range; 5317 5318 ASSERT(in_hblk_range(hmeblkp, addr)); 5319 ASSERT(!hmeblkp->hblk_shw_bit); 5320 #ifdef DEBUG 5321 if (get_hblk_ttesz(hmeblkp) != TTE8K && 5322 (endaddr < get_hblk_endaddr(hmeblkp))) { 5323 panic("sfmmu_hblk_unload: partial unload of large page"); 5324 } 5325 #endif /* DEBUG */ 5326 5327 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5328 ttesz = get_hblk_ttesz(hmeblkp); 5329 5330 use_demap_range = (do_virtual_coloring && 5331 ((dmrp == NULL) || TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp))); 5332 if (use_demap_range) { 5333 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 5334 } else { 5335 DEMAP_RANGE_FLUSH(dmrp); 5336 } 5337 ttecnt = 0; 5338 HBLKTOHME(sfhmep, hmeblkp, addr); 5339 5340 while (addr < endaddr) { 5341 pml = NULL; 5342 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5343 if (TTE_IS_VALID(&tte)) { 5344 pp = sfhmep->hme_page; 5345 if (pp != NULL) { 5346 pml = sfmmu_mlist_enter(pp); 5347 } 5348 5349 /* 5350 * Verify if hme still points to 'pp' now that 5351 * we have p_mapping lock. 5352 */ 5353 if (sfhmep->hme_page != pp) { 5354 if (pp != NULL && sfhmep->hme_page != NULL) { 5355 ASSERT(pml != NULL); 5356 sfmmu_mlist_exit(pml); 5357 /* Re-start this iteration. */ 5358 continue; 5359 } 5360 ASSERT((pp != NULL) && 5361 (sfhmep->hme_page == NULL)); 5362 goto tte_unloaded; 5363 } 5364 5365 /* 5366 * This point on we have both HASH and p_mapping 5367 * lock. 5368 */ 5369 ASSERT(pp == sfhmep->hme_page); 5370 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5371 5372 /* 5373 * We need to loop on modify tte because it is 5374 * possible for pagesync to come along and 5375 * change the software bits beneath us. 5376 * 5377 * Page_unload can also invalidate the tte after 5378 * we read tte outside of p_mapping lock. 5379 */ 5380 again: 5381 ttemod = tte; 5382 5383 TTE_SET_INVALID(&ttemod); 5384 ret = sfmmu_modifytte_try(&tte, &ttemod, 5385 &sfhmep->hme_tte); 5386 5387 if (ret <= 0) { 5388 if (TTE_IS_VALID(&tte)) { 5389 ASSERT(ret < 0); 5390 goto again; 5391 } 5392 if (pp != NULL) { 5393 panic("sfmmu_hblk_unload: pp = 0x%p " 5394 "tte became invalid under mlist" 5395 " lock = 0x%p", pp, pml); 5396 } 5397 continue; 5398 } 5399 5400 if (!(flags & HAT_UNLOAD_NOSYNC)) { 5401 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5402 } 5403 5404 /* 5405 * Ok- we invalidated the tte. Do the rest of the job. 5406 */ 5407 ttecnt++; 5408 5409 if (flags & HAT_UNLOAD_UNLOCK) { 5410 ASSERT(hmeblkp->hblk_lckcnt > 0); 5411 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 5412 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 5413 } 5414 5415 /* 5416 * Normally we would need to flush the page 5417 * from the virtual cache at this point in 5418 * order to prevent a potential cache alias 5419 * inconsistency. 5420 * The particular scenario we need to worry 5421 * about is: 5422 * Given: va1 and va2 are two virtual address 5423 * that alias and map the same physical 5424 * address. 5425 * 1. mapping exists from va1 to pa and data 5426 * has been read into the cache. 5427 * 2. unload va1. 5428 * 3. load va2 and modify data using va2. 5429 * 4 unload va2. 5430 * 5. load va1 and reference data. Unless we 5431 * flush the data cache when we unload we will 5432 * get stale data. 5433 * Fortunately, page coloring eliminates the 5434 * above scenario by remembering the color a 5435 * physical page was last or is currently 5436 * mapped to. Now, we delay the flush until 5437 * the loading of translations. Only when the 5438 * new translation is of a different color 5439 * are we forced to flush. 5440 */ 5441 if (use_demap_range) { 5442 /* 5443 * Mark this page as needing a demap. 5444 */ 5445 DEMAP_RANGE_MARKPG(dmrp, addr); 5446 } else { 5447 if (do_virtual_coloring) { 5448 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 5449 sfmmup->sfmmu_free, 0); 5450 } else { 5451 pfn_t pfnum; 5452 5453 pfnum = TTE_TO_PFN(addr, &tte); 5454 sfmmu_tlbcache_demap(addr, sfmmup, 5455 hmeblkp, pfnum, sfmmup->sfmmu_free, 5456 FLUSH_NECESSARY_CPUS, 5457 CACHE_FLUSH, 0); 5458 } 5459 } 5460 5461 if (pp) { 5462 /* 5463 * Remove the hment from the mapping list 5464 */ 5465 ASSERT(hmeblkp->hblk_hmecnt > 0); 5466 5467 /* 5468 * Again, we cannot 5469 * ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS); 5470 */ 5471 HME_SUB(sfhmep, pp); 5472 membar_stst(); 5473 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 5474 } 5475 5476 ASSERT(hmeblkp->hblk_vcnt > 0); 5477 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 5478 5479 ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 5480 !hmeblkp->hblk_lckcnt); 5481 5482 #ifdef VAC 5483 if (pp && (pp->p_nrm & (P_KPMC | P_KPMS | P_TNC))) { 5484 if (PP_ISTNC(pp)) { 5485 /* 5486 * If page was temporary 5487 * uncached, try to recache 5488 * it. Note that HME_SUB() was 5489 * called above so p_index and 5490 * mlist had been updated. 5491 */ 5492 conv_tnc(pp, ttesz); 5493 } else if (pp->p_mapping == NULL) { 5494 ASSERT(kpm_enable); 5495 /* 5496 * Page is marked to be in VAC conflict 5497 * to an existing kpm mapping and/or is 5498 * kpm mapped using only the regular 5499 * pagesize. 5500 */ 5501 sfmmu_kpm_hme_unload(pp); 5502 } 5503 } 5504 #endif /* VAC */ 5505 } else if ((pp = sfhmep->hme_page) != NULL) { 5506 /* 5507 * TTE is invalid but the hme 5508 * still exists. let pageunload 5509 * complete its job. 5510 */ 5511 ASSERT(pml == NULL); 5512 pml = sfmmu_mlist_enter(pp); 5513 if (sfhmep->hme_page != NULL) { 5514 sfmmu_mlist_exit(pml); 5515 continue; 5516 } 5517 ASSERT(sfhmep->hme_page == NULL); 5518 } else if (hmeblkp->hblk_hmecnt != 0) { 5519 /* 5520 * pageunload may have not finished decrementing 5521 * hblk_vcnt and hblk_hmecnt. Find page_t if any and 5522 * wait for pageunload to finish. Rely on pageunload 5523 * to decrement hblk_hmecnt after hblk_vcnt. 5524 */ 5525 pfn_t pfn = TTE_TO_TTEPFN(&tte); 5526 ASSERT(pml == NULL); 5527 if (pf_is_memory(pfn)) { 5528 pp = page_numtopp_nolock(pfn); 5529 if (pp != NULL) { 5530 pml = sfmmu_mlist_enter(pp); 5531 sfmmu_mlist_exit(pml); 5532 pml = NULL; 5533 } 5534 } 5535 } 5536 5537 tte_unloaded: 5538 /* 5539 * At this point, the tte we are looking at 5540 * should be unloaded, and hme has been unlinked 5541 * from page too. This is important because in 5542 * pageunload, it does ttesync() then HME_SUB. 5543 * We need to make sure HME_SUB has been completed 5544 * so we know ttesync() has been completed. Otherwise, 5545 * at exit time, after return from hat layer, VM will 5546 * release as structure which hat_setstat() (called 5547 * by ttesync()) needs. 5548 */ 5549 #ifdef DEBUG 5550 { 5551 tte_t dtte; 5552 5553 ASSERT(sfhmep->hme_page == NULL); 5554 5555 sfmmu_copytte(&sfhmep->hme_tte, &dtte); 5556 ASSERT(!TTE_IS_VALID(&dtte)); 5557 } 5558 #endif 5559 5560 if (pml) { 5561 sfmmu_mlist_exit(pml); 5562 } 5563 5564 addr += TTEBYTES(ttesz); 5565 sfhmep++; 5566 DEMAP_RANGE_NEXTPG(dmrp); 5567 } 5568 if (ttecnt > 0) 5569 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt); 5570 return (addr); 5571 } 5572 5573 /* 5574 * Synchronize all the mappings in the range [addr..addr+len). 5575 * Can be called with clearflag having two states: 5576 * HAT_SYNC_DONTZERO means just return the rm stats 5577 * HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats 5578 */ 5579 void 5580 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag) 5581 { 5582 struct hmehash_bucket *hmebp; 5583 hmeblk_tag hblktag; 5584 int hmeshift, hashno = 1; 5585 struct hme_blk *hmeblkp, *list = NULL; 5586 caddr_t endaddr; 5587 cpuset_t cpuset; 5588 5589 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 5590 ASSERT((sfmmup == ksfmmup) || 5591 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 5592 ASSERT((len & MMU_PAGEOFFSET) == 0); 5593 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 5594 (clearflag == HAT_SYNC_ZERORM)); 5595 5596 CPUSET_ZERO(cpuset); 5597 5598 endaddr = addr + len; 5599 hblktag.htag_id = sfmmup; 5600 /* 5601 * Spitfire supports 4 page sizes. 5602 * Most pages are expected to be of the smallest page 5603 * size (8K) and these will not need to be rehashed. 64K 5604 * pages also don't need to be rehashed because the an hmeblk 5605 * spans 64K of address space. 512K pages might need 1 rehash and 5606 * and 4M pages 2 rehashes. 5607 */ 5608 while (addr < endaddr) { 5609 hmeshift = HME_HASH_SHIFT(hashno); 5610 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5611 hblktag.htag_rehash = hashno; 5612 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5613 5614 SFMMU_HASH_LOCK(hmebp); 5615 5616 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 5617 if (hmeblkp != NULL) { 5618 /* 5619 * We've encountered a shadow hmeblk so skip the range 5620 * of the next smaller mapping size. 5621 */ 5622 if (hmeblkp->hblk_shw_bit) { 5623 ASSERT(sfmmup != ksfmmup); 5624 ASSERT(hashno > 1); 5625 addr = (caddr_t)P2END((uintptr_t)addr, 5626 TTEBYTES(hashno - 1)); 5627 } else { 5628 addr = sfmmu_hblk_sync(sfmmup, hmeblkp, 5629 addr, endaddr, clearflag); 5630 } 5631 SFMMU_HASH_UNLOCK(hmebp); 5632 hashno = 1; 5633 continue; 5634 } 5635 SFMMU_HASH_UNLOCK(hmebp); 5636 5637 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 5638 /* 5639 * We have traversed the whole list and rehashed 5640 * if necessary without finding the address to sync. 5641 * This is ok so we increment the address by the 5642 * smallest hmeblk range for kernel mappings and the 5643 * largest hmeblk range, to account for shadow hmeblks, 5644 * for user mappings and continue. 5645 */ 5646 if (sfmmup == ksfmmup) 5647 addr = (caddr_t)P2END((uintptr_t)addr, 5648 TTEBYTES(1)); 5649 else 5650 addr = (caddr_t)P2END((uintptr_t)addr, 5651 TTEBYTES(hashno)); 5652 hashno = 1; 5653 } else { 5654 hashno++; 5655 } 5656 } 5657 sfmmu_hblks_list_purge(&list); 5658 cpuset = sfmmup->sfmmu_cpusran; 5659 xt_sync(cpuset); 5660 } 5661 5662 static caddr_t 5663 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5664 caddr_t endaddr, int clearflag) 5665 { 5666 tte_t tte, ttemod; 5667 struct sf_hment *sfhmep; 5668 int ttesz; 5669 struct page *pp; 5670 kmutex_t *pml; 5671 int ret; 5672 5673 ASSERT(hmeblkp->hblk_shw_bit == 0); 5674 5675 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5676 5677 ttesz = get_hblk_ttesz(hmeblkp); 5678 HBLKTOHME(sfhmep, hmeblkp, addr); 5679 5680 while (addr < endaddr) { 5681 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5682 if (TTE_IS_VALID(&tte)) { 5683 pml = NULL; 5684 pp = sfhmep->hme_page; 5685 if (pp) { 5686 pml = sfmmu_mlist_enter(pp); 5687 } 5688 if (pp != sfhmep->hme_page) { 5689 /* 5690 * tte most have been unloaded 5691 * underneath us. Recheck 5692 */ 5693 ASSERT(pml); 5694 sfmmu_mlist_exit(pml); 5695 continue; 5696 } 5697 5698 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5699 5700 if (clearflag == HAT_SYNC_ZERORM) { 5701 ttemod = tte; 5702 TTE_CLR_RM(&ttemod); 5703 ret = sfmmu_modifytte_try(&tte, &ttemod, 5704 &sfhmep->hme_tte); 5705 if (ret < 0) { 5706 if (pml) { 5707 sfmmu_mlist_exit(pml); 5708 } 5709 continue; 5710 } 5711 5712 if (ret > 0) { 5713 sfmmu_tlb_demap(addr, sfmmup, 5714 hmeblkp, 0, 0); 5715 } 5716 } 5717 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5718 if (pml) { 5719 sfmmu_mlist_exit(pml); 5720 } 5721 } 5722 addr += TTEBYTES(ttesz); 5723 sfhmep++; 5724 } 5725 return (addr); 5726 } 5727 5728 /* 5729 * This function will sync a tte to the page struct and it will 5730 * update the hat stats. Currently it allows us to pass a NULL pp 5731 * and we will simply update the stats. We may want to change this 5732 * so we only keep stats for pages backed by pp's. 5733 */ 5734 static void 5735 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp) 5736 { 5737 uint_t rm = 0; 5738 int sz; 5739 pgcnt_t npgs; 5740 5741 ASSERT(TTE_IS_VALID(ttep)); 5742 5743 if (TTE_IS_NOSYNC(ttep)) { 5744 return; 5745 } 5746 5747 if (TTE_IS_REF(ttep)) { 5748 rm = P_REF; 5749 } 5750 if (TTE_IS_MOD(ttep)) { 5751 rm |= P_MOD; 5752 } 5753 5754 if (rm == 0) { 5755 return; 5756 } 5757 5758 sz = TTE_CSZ(ttep); 5759 if (sfmmup->sfmmu_rmstat) { 5760 int i; 5761 caddr_t vaddr = addr; 5762 5763 for (i = 0; i < TTEPAGES(sz); i++, vaddr += MMU_PAGESIZE) { 5764 hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm); 5765 } 5766 5767 } 5768 5769 /* 5770 * XXX I want to use cas to update nrm bits but they 5771 * currently belong in common/vm and not in hat where 5772 * they should be. 5773 * The nrm bits are protected by the same mutex as 5774 * the one that protects the page's mapping list. 5775 */ 5776 if (!pp) 5777 return; 5778 ASSERT(sfmmu_mlist_held(pp)); 5779 /* 5780 * If the tte is for a large page, we need to sync all the 5781 * pages covered by the tte. 5782 */ 5783 if (sz != TTE8K) { 5784 ASSERT(pp->p_szc != 0); 5785 pp = PP_GROUPLEADER(pp, sz); 5786 ASSERT(sfmmu_mlist_held(pp)); 5787 } 5788 5789 /* Get number of pages from tte size. */ 5790 npgs = TTEPAGES(sz); 5791 5792 do { 5793 ASSERT(pp); 5794 ASSERT(sfmmu_mlist_held(pp)); 5795 if (((rm & P_REF) != 0 && !PP_ISREF(pp)) || 5796 ((rm & P_MOD) != 0 && !PP_ISMOD(pp))) 5797 hat_page_setattr(pp, rm); 5798 5799 /* 5800 * Are we done? If not, we must have a large mapping. 5801 * For large mappings we need to sync the rest of the pages 5802 * covered by this tte; goto the next page. 5803 */ 5804 } while (--npgs > 0 && (pp = PP_PAGENEXT(pp))); 5805 } 5806 5807 /* 5808 * Execute pre-callback handler of each pa_hment linked to pp 5809 * 5810 * Inputs: 5811 * flag: either HAT_PRESUSPEND or HAT_SUSPEND. 5812 * capture_cpus: pointer to return value (below) 5813 * 5814 * Returns: 5815 * Propagates the subsystem callback return values back to the caller; 5816 * returns 0 on success. If capture_cpus is non-NULL, the value returned 5817 * is zero if all of the pa_hments are of a type that do not require 5818 * capturing CPUs prior to suspending the mapping, else it is 1. 5819 */ 5820 static int 5821 hat_pageprocess_precallbacks(struct page *pp, uint_t flag, int *capture_cpus) 5822 { 5823 struct sf_hment *sfhmep; 5824 struct pa_hment *pahmep; 5825 int (*f)(caddr_t, uint_t, uint_t, void *); 5826 int ret; 5827 id_t id; 5828 int locked = 0; 5829 kmutex_t *pml; 5830 5831 ASSERT(PAGE_EXCL(pp)); 5832 if (!sfmmu_mlist_held(pp)) { 5833 pml = sfmmu_mlist_enter(pp); 5834 locked = 1; 5835 } 5836 5837 if (capture_cpus) 5838 *capture_cpus = 0; 5839 5840 top: 5841 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5842 /* 5843 * skip sf_hments corresponding to VA<->PA mappings; 5844 * for pa_hment's, hme_tte.ll is zero 5845 */ 5846 if (!IS_PAHME(sfhmep)) 5847 continue; 5848 5849 pahmep = sfhmep->hme_data; 5850 ASSERT(pahmep != NULL); 5851 5852 /* 5853 * skip if pre-handler has been called earlier in this loop 5854 */ 5855 if (pahmep->flags & flag) 5856 continue; 5857 5858 id = pahmep->cb_id; 5859 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5860 if (capture_cpus && sfmmu_cb_table[id].capture_cpus != 0) 5861 *capture_cpus = 1; 5862 if ((f = sfmmu_cb_table[id].prehandler) == NULL) { 5863 pahmep->flags |= flag; 5864 continue; 5865 } 5866 5867 /* 5868 * Drop the mapping list lock to avoid locking order issues. 5869 */ 5870 if (locked) 5871 sfmmu_mlist_exit(pml); 5872 5873 ret = f(pahmep->addr, pahmep->len, flag, pahmep->pvt); 5874 if (ret != 0) 5875 return (ret); /* caller must do the cleanup */ 5876 5877 if (locked) { 5878 pml = sfmmu_mlist_enter(pp); 5879 pahmep->flags |= flag; 5880 goto top; 5881 } 5882 5883 pahmep->flags |= flag; 5884 } 5885 5886 if (locked) 5887 sfmmu_mlist_exit(pml); 5888 5889 return (0); 5890 } 5891 5892 /* 5893 * Execute post-callback handler of each pa_hment linked to pp 5894 * 5895 * Same overall assumptions and restrictions apply as for 5896 * hat_pageprocess_precallbacks(). 5897 */ 5898 static void 5899 hat_pageprocess_postcallbacks(struct page *pp, uint_t flag) 5900 { 5901 pfn_t pgpfn = pp->p_pagenum; 5902 pfn_t pgmask = btop(page_get_pagesize(pp->p_szc)) - 1; 5903 pfn_t newpfn; 5904 struct sf_hment *sfhmep; 5905 struct pa_hment *pahmep; 5906 int (*f)(caddr_t, uint_t, uint_t, void *, pfn_t); 5907 id_t id; 5908 int locked = 0; 5909 kmutex_t *pml; 5910 5911 ASSERT(PAGE_EXCL(pp)); 5912 if (!sfmmu_mlist_held(pp)) { 5913 pml = sfmmu_mlist_enter(pp); 5914 locked = 1; 5915 } 5916 5917 top: 5918 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5919 /* 5920 * skip sf_hments corresponding to VA<->PA mappings; 5921 * for pa_hment's, hme_tte.ll is zero 5922 */ 5923 if (!IS_PAHME(sfhmep)) 5924 continue; 5925 5926 pahmep = sfhmep->hme_data; 5927 ASSERT(pahmep != NULL); 5928 5929 if ((pahmep->flags & flag) == 0) 5930 continue; 5931 5932 pahmep->flags &= ~flag; 5933 5934 id = pahmep->cb_id; 5935 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5936 if ((f = sfmmu_cb_table[id].posthandler) == NULL) 5937 continue; 5938 5939 /* 5940 * Convert the base page PFN into the constituent PFN 5941 * which is needed by the callback handler. 5942 */ 5943 newpfn = pgpfn | (btop((uintptr_t)pahmep->addr) & pgmask); 5944 5945 /* 5946 * Drop the mapping list lock to avoid locking order issues. 5947 */ 5948 if (locked) 5949 sfmmu_mlist_exit(pml); 5950 5951 if (f(pahmep->addr, pahmep->len, flag, pahmep->pvt, newpfn) 5952 != 0) 5953 panic("sfmmu: posthandler failed"); 5954 5955 if (locked) { 5956 pml = sfmmu_mlist_enter(pp); 5957 goto top; 5958 } 5959 } 5960 5961 if (locked) 5962 sfmmu_mlist_exit(pml); 5963 } 5964 5965 /* 5966 * Suspend locked kernel mapping 5967 */ 5968 void 5969 hat_pagesuspend(struct page *pp) 5970 { 5971 struct sf_hment *sfhmep; 5972 sfmmu_t *sfmmup; 5973 tte_t tte, ttemod; 5974 struct hme_blk *hmeblkp; 5975 caddr_t addr; 5976 int index, cons; 5977 cpuset_t cpuset; 5978 5979 ASSERT(PAGE_EXCL(pp)); 5980 ASSERT(sfmmu_mlist_held(pp)); 5981 5982 mutex_enter(&kpr_suspendlock); 5983 5984 /* 5985 * Call into dtrace to tell it we're about to suspend a 5986 * kernel mapping. This prevents us from running into issues 5987 * with probe context trying to touch a suspended page 5988 * in the relocation codepath itself. 5989 */ 5990 if (dtrace_kreloc_init) 5991 (*dtrace_kreloc_init)(); 5992 5993 index = PP_MAPINDEX(pp); 5994 cons = TTE8K; 5995 5996 retry: 5997 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5998 5999 if (IS_PAHME(sfhmep)) 6000 continue; 6001 6002 if (get_hblk_ttesz(sfmmu_hmetohblk(sfhmep)) != cons) 6003 continue; 6004 6005 /* 6006 * Loop until we successfully set the suspend bit in 6007 * the TTE. 6008 */ 6009 again: 6010 sfmmu_copytte(&sfhmep->hme_tte, &tte); 6011 ASSERT(TTE_IS_VALID(&tte)); 6012 6013 ttemod = tte; 6014 TTE_SET_SUSPEND(&ttemod); 6015 if (sfmmu_modifytte_try(&tte, &ttemod, 6016 &sfhmep->hme_tte) < 0) 6017 goto again; 6018 6019 /* 6020 * Invalidate TSB entry 6021 */ 6022 hmeblkp = sfmmu_hmetohblk(sfhmep); 6023 6024 sfmmup = hblktosfmmu(hmeblkp); 6025 ASSERT(sfmmup == ksfmmup); 6026 6027 addr = tte_to_vaddr(hmeblkp, tte); 6028 6029 /* 6030 * No need to make sure that the TSB for this sfmmu is 6031 * not being relocated since it is ksfmmup and thus it 6032 * will never be relocated. 6033 */ 6034 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 6035 6036 /* 6037 * Update xcall stats 6038 */ 6039 cpuset = cpu_ready_set; 6040 CPUSET_DEL(cpuset, CPU->cpu_id); 6041 6042 /* LINTED: constant in conditional context */ 6043 SFMMU_XCALL_STATS(ksfmmup); 6044 6045 /* 6046 * Flush TLB entry on remote CPU's 6047 */ 6048 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 6049 (uint64_t)ksfmmup); 6050 xt_sync(cpuset); 6051 6052 /* 6053 * Flush TLB entry on local CPU 6054 */ 6055 vtag_flushpage(addr, (uint64_t)ksfmmup); 6056 } 6057 6058 while (index != 0) { 6059 index = index >> 1; 6060 if (index != 0) 6061 cons++; 6062 if (index & 0x1) { 6063 pp = PP_GROUPLEADER(pp, cons); 6064 goto retry; 6065 } 6066 } 6067 } 6068 6069 #ifdef DEBUG 6070 6071 #define N_PRLE 1024 6072 struct prle { 6073 page_t *targ; 6074 page_t *repl; 6075 int status; 6076 int pausecpus; 6077 hrtime_t whence; 6078 }; 6079 6080 static struct prle page_relocate_log[N_PRLE]; 6081 static int prl_entry; 6082 static kmutex_t prl_mutex; 6083 6084 #define PAGE_RELOCATE_LOG(t, r, s, p) \ 6085 mutex_enter(&prl_mutex); \ 6086 page_relocate_log[prl_entry].targ = *(t); \ 6087 page_relocate_log[prl_entry].repl = *(r); \ 6088 page_relocate_log[prl_entry].status = (s); \ 6089 page_relocate_log[prl_entry].pausecpus = (p); \ 6090 page_relocate_log[prl_entry].whence = gethrtime(); \ 6091 prl_entry = (prl_entry == (N_PRLE - 1))? 0 : prl_entry + 1; \ 6092 mutex_exit(&prl_mutex); 6093 6094 #else /* !DEBUG */ 6095 #define PAGE_RELOCATE_LOG(t, r, s, p) 6096 #endif 6097 6098 /* 6099 * Core Kernel Page Relocation Algorithm 6100 * 6101 * Input: 6102 * 6103 * target : constituent pages are SE_EXCL locked. 6104 * replacement: constituent pages are SE_EXCL locked. 6105 * 6106 * Output: 6107 * 6108 * nrelocp: number of pages relocated 6109 */ 6110 int 6111 hat_page_relocate(page_t **target, page_t **replacement, spgcnt_t *nrelocp) 6112 { 6113 page_t *targ, *repl; 6114 page_t *tpp, *rpp; 6115 kmutex_t *low, *high; 6116 spgcnt_t npages, i; 6117 page_t *pl = NULL; 6118 int old_pil; 6119 cpuset_t cpuset; 6120 int cap_cpus; 6121 int ret; 6122 6123 if (hat_kpr_enabled == 0 || !kcage_on || PP_ISNORELOC(*target)) { 6124 PAGE_RELOCATE_LOG(target, replacement, EAGAIN, -1); 6125 return (EAGAIN); 6126 } 6127 6128 mutex_enter(&kpr_mutex); 6129 kreloc_thread = curthread; 6130 6131 targ = *target; 6132 repl = *replacement; 6133 ASSERT(repl != NULL); 6134 ASSERT(targ->p_szc == repl->p_szc); 6135 6136 npages = page_get_pagecnt(targ->p_szc); 6137 6138 /* 6139 * unload VA<->PA mappings that are not locked 6140 */ 6141 tpp = targ; 6142 for (i = 0; i < npages; i++) { 6143 (void) hat_pageunload(tpp, SFMMU_KERNEL_RELOC); 6144 tpp++; 6145 } 6146 6147 /* 6148 * Do "presuspend" callbacks, in a context from which we can still 6149 * block as needed. Note that we don't hold the mapping list lock 6150 * of "targ" at this point due to potential locking order issues; 6151 * we assume that between the hat_pageunload() above and holding 6152 * the SE_EXCL lock that the mapping list *cannot* change at this 6153 * point. 6154 */ 6155 ret = hat_pageprocess_precallbacks(targ, HAT_PRESUSPEND, &cap_cpus); 6156 if (ret != 0) { 6157 /* 6158 * EIO translates to fatal error, for all others cleanup 6159 * and return EAGAIN. 6160 */ 6161 ASSERT(ret != EIO); 6162 hat_pageprocess_postcallbacks(targ, HAT_POSTUNSUSPEND); 6163 PAGE_RELOCATE_LOG(target, replacement, ret, -1); 6164 kreloc_thread = NULL; 6165 mutex_exit(&kpr_mutex); 6166 return (EAGAIN); 6167 } 6168 6169 /* 6170 * acquire p_mapping list lock for both the target and replacement 6171 * root pages. 6172 * 6173 * low and high refer to the need to grab the mlist locks in a 6174 * specific order in order to prevent race conditions. Thus the 6175 * lower lock must be grabbed before the higher lock. 6176 * 6177 * This will block hat_unload's accessing p_mapping list. Since 6178 * we have SE_EXCL lock, hat_memload and hat_pageunload will be 6179 * blocked. Thus, no one else will be accessing the p_mapping list 6180 * while we suspend and reload the locked mapping below. 6181 */ 6182 tpp = targ; 6183 rpp = repl; 6184 sfmmu_mlist_reloc_enter(tpp, rpp, &low, &high); 6185 6186 kpreempt_disable(); 6187 6188 #ifdef VAC 6189 /* 6190 * If the replacement page is of a different virtual color 6191 * than the page it is replacing, we need to handle the VAC 6192 * consistency for it just as we would if we were setting up 6193 * a new mapping to a page. 6194 */ 6195 if ((tpp->p_szc == 0) && (PP_GET_VCOLOR(rpp) != NO_VCOLOR)) { 6196 if (tpp->p_vcolor != rpp->p_vcolor) { 6197 sfmmu_cache_flushcolor(PP_GET_VCOLOR(rpp), 6198 rpp->p_pagenum); 6199 } 6200 } 6201 #endif 6202 6203 /* 6204 * We raise our PIL to 13 so that we don't get captured by 6205 * another CPU or pinned by an interrupt thread. We can't go to 6206 * PIL 14 since the nexus driver(s) may need to interrupt at 6207 * that level in the case of IOMMU pseudo mappings. 6208 */ 6209 cpuset = cpu_ready_set; 6210 CPUSET_DEL(cpuset, CPU->cpu_id); 6211 if (!cap_cpus || CPUSET_ISNULL(cpuset)) { 6212 old_pil = splr(XCALL_PIL); 6213 } else { 6214 old_pil = -1; 6215 xc_attention(cpuset); 6216 } 6217 ASSERT(getpil() == XCALL_PIL); 6218 6219 /* 6220 * Now do suspend callbacks. In the case of an IOMMU mapping 6221 * this will suspend all DMA activity to the page while it is 6222 * being relocated. Since we are well above LOCK_LEVEL and CPUs 6223 * may be captured at this point we should have acquired any needed 6224 * locks in the presuspend callback. 6225 */ 6226 ret = hat_pageprocess_precallbacks(targ, HAT_SUSPEND, NULL); 6227 if (ret != 0) { 6228 repl = targ; 6229 goto suspend_fail; 6230 } 6231 6232 /* 6233 * Raise the PIL yet again, this time to block all high-level 6234 * interrupts on this CPU. This is necessary to prevent an 6235 * interrupt routine from pinning the thread which holds the 6236 * mapping suspended and then touching the suspended page. 6237 * 6238 * Once the page is suspended we also need to be careful to 6239 * avoid calling any functions which touch any seg_kmem memory 6240 * since that memory may be backed by the very page we are 6241 * relocating in here! 6242 */ 6243 hat_pagesuspend(targ); 6244 6245 /* 6246 * Now that we are confident everybody has stopped using this page, 6247 * copy the page contents. Note we use a physical copy to prevent 6248 * locking issues and to avoid fpRAS because we can't handle it in 6249 * this context. 6250 */ 6251 for (i = 0; i < npages; i++, tpp++, rpp++) { 6252 /* 6253 * Copy the contents of the page. 6254 */ 6255 ppcopy_kernel(tpp, rpp); 6256 } 6257 6258 tpp = targ; 6259 rpp = repl; 6260 for (i = 0; i < npages; i++, tpp++, rpp++) { 6261 /* 6262 * Copy attributes. VAC consistency was handled above, 6263 * if required. 6264 */ 6265 rpp->p_nrm = tpp->p_nrm; 6266 tpp->p_nrm = 0; 6267 rpp->p_index = tpp->p_index; 6268 tpp->p_index = 0; 6269 #ifdef VAC 6270 rpp->p_vcolor = tpp->p_vcolor; 6271 #endif 6272 } 6273 6274 /* 6275 * First, unsuspend the page, if we set the suspend bit, and transfer 6276 * the mapping list from the target page to the replacement page. 6277 * Next process postcallbacks; since pa_hment's are linked only to the 6278 * p_mapping list of root page, we don't iterate over the constituent 6279 * pages. 6280 */ 6281 hat_pagereload(targ, repl); 6282 6283 suspend_fail: 6284 hat_pageprocess_postcallbacks(repl, HAT_UNSUSPEND); 6285 6286 /* 6287 * Now lower our PIL and release any captured CPUs since we 6288 * are out of the "danger zone". After this it will again be 6289 * safe to acquire adaptive mutex locks, or to drop them... 6290 */ 6291 if (old_pil != -1) { 6292 splx(old_pil); 6293 } else { 6294 xc_dismissed(cpuset); 6295 } 6296 6297 kpreempt_enable(); 6298 6299 sfmmu_mlist_reloc_exit(low, high); 6300 6301 /* 6302 * Postsuspend callbacks should drop any locks held across 6303 * the suspend callbacks. As before, we don't hold the mapping 6304 * list lock at this point.. our assumption is that the mapping 6305 * list still can't change due to our holding SE_EXCL lock and 6306 * there being no unlocked mappings left. Hence the restriction 6307 * on calling context to hat_delete_callback() 6308 */ 6309 hat_pageprocess_postcallbacks(repl, HAT_POSTUNSUSPEND); 6310 if (ret != 0) { 6311 /* 6312 * The second presuspend call failed: we got here through 6313 * the suspend_fail label above. 6314 */ 6315 ASSERT(ret != EIO); 6316 PAGE_RELOCATE_LOG(target, replacement, ret, cap_cpus); 6317 kreloc_thread = NULL; 6318 mutex_exit(&kpr_mutex); 6319 return (EAGAIN); 6320 } 6321 6322 /* 6323 * Now that we're out of the performance critical section we can 6324 * take care of updating the hash table, since we still 6325 * hold all the pages locked SE_EXCL at this point we 6326 * needn't worry about things changing out from under us. 6327 */ 6328 tpp = targ; 6329 rpp = repl; 6330 for (i = 0; i < npages; i++, tpp++, rpp++) { 6331 6332 /* 6333 * replace targ with replacement in page_hash table 6334 */ 6335 targ = tpp; 6336 page_relocate_hash(rpp, targ); 6337 6338 /* 6339 * concatenate target; caller of platform_page_relocate() 6340 * expects target to be concatenated after returning. 6341 */ 6342 ASSERT(targ->p_next == targ); 6343 ASSERT(targ->p_prev == targ); 6344 page_list_concat(&pl, &targ); 6345 } 6346 6347 ASSERT(*target == pl); 6348 *nrelocp = npages; 6349 PAGE_RELOCATE_LOG(target, replacement, 0, cap_cpus); 6350 kreloc_thread = NULL; 6351 mutex_exit(&kpr_mutex); 6352 return (0); 6353 } 6354 6355 /* 6356 * Called when stray pa_hments are found attached to a page which is 6357 * being freed. Notify the subsystem which attached the pa_hment of 6358 * the error if it registered a suitable handler, else panic. 6359 */ 6360 static void 6361 sfmmu_pahment_leaked(struct pa_hment *pahmep) 6362 { 6363 id_t cb_id = pahmep->cb_id; 6364 6365 ASSERT(cb_id >= (id_t)0 && cb_id < sfmmu_cb_nextid); 6366 if (sfmmu_cb_table[cb_id].errhandler != NULL) { 6367 if (sfmmu_cb_table[cb_id].errhandler(pahmep->addr, pahmep->len, 6368 HAT_CB_ERR_LEAKED, pahmep->pvt) == 0) 6369 return; /* non-fatal */ 6370 } 6371 panic("pa_hment leaked: 0x%p", pahmep); 6372 } 6373 6374 /* 6375 * Remove all mappings to page 'pp'. 6376 */ 6377 int 6378 hat_pageunload(struct page *pp, uint_t forceflag) 6379 { 6380 struct page *origpp = pp; 6381 struct sf_hment *sfhme, *tmphme; 6382 struct hme_blk *hmeblkp; 6383 kmutex_t *pml; 6384 #ifdef VAC 6385 kmutex_t *pmtx; 6386 #endif 6387 cpuset_t cpuset, tset; 6388 int index, cons; 6389 int xhme_blks; 6390 int pa_hments; 6391 6392 ASSERT(PAGE_EXCL(pp)); 6393 6394 retry_xhat: 6395 tmphme = NULL; 6396 xhme_blks = 0; 6397 pa_hments = 0; 6398 CPUSET_ZERO(cpuset); 6399 6400 pml = sfmmu_mlist_enter(pp); 6401 6402 #ifdef VAC 6403 if (pp->p_kpmref) 6404 sfmmu_kpm_pageunload(pp); 6405 ASSERT(!PP_ISMAPPED_KPM(pp)); 6406 #endif 6407 6408 index = PP_MAPINDEX(pp); 6409 cons = TTE8K; 6410 retry: 6411 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6412 tmphme = sfhme->hme_next; 6413 6414 if (IS_PAHME(sfhme)) { 6415 ASSERT(sfhme->hme_data != NULL); 6416 pa_hments++; 6417 continue; 6418 } 6419 6420 hmeblkp = sfmmu_hmetohblk(sfhme); 6421 if (hmeblkp->hblk_xhat_bit) { 6422 struct xhat_hme_blk *xblk = 6423 (struct xhat_hme_blk *)hmeblkp; 6424 6425 (void) XHAT_PAGEUNLOAD(xblk->xhat_hme_blk_hat, 6426 pp, forceflag, XBLK2PROVBLK(xblk)); 6427 6428 xhme_blks = 1; 6429 continue; 6430 } 6431 6432 /* 6433 * If there are kernel mappings don't unload them, they will 6434 * be suspended. 6435 */ 6436 if (forceflag == SFMMU_KERNEL_RELOC && hmeblkp->hblk_lckcnt && 6437 hmeblkp->hblk_tag.htag_id == ksfmmup) 6438 continue; 6439 6440 tset = sfmmu_pageunload(pp, sfhme, cons); 6441 CPUSET_OR(cpuset, tset); 6442 } 6443 6444 while (index != 0) { 6445 index = index >> 1; 6446 if (index != 0) 6447 cons++; 6448 if (index & 0x1) { 6449 /* Go to leading page */ 6450 pp = PP_GROUPLEADER(pp, cons); 6451 ASSERT(sfmmu_mlist_held(pp)); 6452 goto retry; 6453 } 6454 } 6455 6456 /* 6457 * cpuset may be empty if the page was only mapped by segkpm, 6458 * in which case we won't actually cross-trap. 6459 */ 6460 xt_sync(cpuset); 6461 6462 /* 6463 * The page should have no mappings at this point, unless 6464 * we were called from hat_page_relocate() in which case we 6465 * leave the locked mappings which will be suspended later. 6466 */ 6467 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks || pa_hments || 6468 (forceflag == SFMMU_KERNEL_RELOC)); 6469 6470 #ifdef VAC 6471 if (PP_ISTNC(pp)) { 6472 if (cons == TTE8K) { 6473 pmtx = sfmmu_page_enter(pp); 6474 PP_CLRTNC(pp); 6475 sfmmu_page_exit(pmtx); 6476 } else { 6477 conv_tnc(pp, cons); 6478 } 6479 } 6480 #endif /* VAC */ 6481 6482 if (pa_hments && forceflag != SFMMU_KERNEL_RELOC) { 6483 /* 6484 * Unlink any pa_hments and free them, calling back 6485 * the responsible subsystem to notify it of the error. 6486 * This can occur in situations such as drivers leaking 6487 * DMA handles: naughty, but common enough that we'd like 6488 * to keep the system running rather than bringing it 6489 * down with an obscure error like "pa_hment leaked" 6490 * which doesn't aid the user in debugging their driver. 6491 */ 6492 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6493 tmphme = sfhme->hme_next; 6494 if (IS_PAHME(sfhme)) { 6495 struct pa_hment *pahmep = sfhme->hme_data; 6496 sfmmu_pahment_leaked(pahmep); 6497 HME_SUB(sfhme, pp); 6498 kmem_cache_free(pa_hment_cache, pahmep); 6499 } 6500 } 6501 6502 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks); 6503 } 6504 6505 sfmmu_mlist_exit(pml); 6506 6507 /* 6508 * XHAT may not have finished unloading pages 6509 * because some other thread was waiting for 6510 * mlist lock and XHAT_PAGEUNLOAD let it do 6511 * the job. 6512 */ 6513 if (xhme_blks) { 6514 pp = origpp; 6515 goto retry_xhat; 6516 } 6517 6518 return (0); 6519 } 6520 6521 cpuset_t 6522 sfmmu_pageunload(page_t *pp, struct sf_hment *sfhme, int cons) 6523 { 6524 struct hme_blk *hmeblkp; 6525 sfmmu_t *sfmmup; 6526 tte_t tte, ttemod; 6527 #ifdef DEBUG 6528 tte_t orig_old; 6529 #endif /* DEBUG */ 6530 caddr_t addr; 6531 int ttesz; 6532 int ret; 6533 cpuset_t cpuset; 6534 6535 ASSERT(pp != NULL); 6536 ASSERT(sfmmu_mlist_held(pp)); 6537 ASSERT(!PP_ISKAS(pp)); 6538 6539 CPUSET_ZERO(cpuset); 6540 6541 hmeblkp = sfmmu_hmetohblk(sfhme); 6542 6543 readtte: 6544 sfmmu_copytte(&sfhme->hme_tte, &tte); 6545 if (TTE_IS_VALID(&tte)) { 6546 sfmmup = hblktosfmmu(hmeblkp); 6547 ttesz = get_hblk_ttesz(hmeblkp); 6548 /* 6549 * Only unload mappings of 'cons' size. 6550 */ 6551 if (ttesz != cons) 6552 return (cpuset); 6553 6554 /* 6555 * Note that we have p_mapping lock, but no hash lock here. 6556 * hblk_unload() has to have both hash lock AND p_mapping 6557 * lock before it tries to modify tte. So, the tte could 6558 * not become invalid in the sfmmu_modifytte_try() below. 6559 */ 6560 ttemod = tte; 6561 #ifdef DEBUG 6562 orig_old = tte; 6563 #endif /* DEBUG */ 6564 6565 TTE_SET_INVALID(&ttemod); 6566 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6567 if (ret < 0) { 6568 #ifdef DEBUG 6569 /* only R/M bits can change. */ 6570 chk_tte(&orig_old, &tte, &ttemod, hmeblkp); 6571 #endif /* DEBUG */ 6572 goto readtte; 6573 } 6574 6575 if (ret == 0) { 6576 panic("pageunload: cas failed?"); 6577 } 6578 6579 addr = tte_to_vaddr(hmeblkp, tte); 6580 6581 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6582 6583 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -1); 6584 6585 /* 6586 * We need to flush the page from the virtual cache 6587 * in order to prevent a virtual cache alias 6588 * inconsistency. The particular scenario we need 6589 * to worry about is: 6590 * Given: va1 and va2 are two virtual address that 6591 * alias and will map the same physical address. 6592 * 1. mapping exists from va1 to pa and data has 6593 * been read into the cache. 6594 * 2. unload va1. 6595 * 3. load va2 and modify data using va2. 6596 * 4 unload va2. 6597 * 5. load va1 and reference data. Unless we flush 6598 * the data cache when we unload we will get 6599 * stale data. 6600 * This scenario is taken care of by using virtual 6601 * page coloring. 6602 */ 6603 if (sfmmup->sfmmu_ismhat) { 6604 /* 6605 * Flush TSBs, TLBs and caches 6606 * of every process 6607 * sharing this ism segment. 6608 */ 6609 sfmmu_hat_lock_all(); 6610 mutex_enter(&ism_mlist_lock); 6611 kpreempt_disable(); 6612 if (do_virtual_coloring) 6613 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6614 pp->p_pagenum, CACHE_NO_FLUSH); 6615 else 6616 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6617 pp->p_pagenum, CACHE_FLUSH); 6618 kpreempt_enable(); 6619 mutex_exit(&ism_mlist_lock); 6620 sfmmu_hat_unlock_all(); 6621 cpuset = cpu_ready_set; 6622 } else if (do_virtual_coloring) { 6623 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6624 cpuset = sfmmup->sfmmu_cpusran; 6625 } else { 6626 sfmmu_tlbcache_demap(addr, sfmmup, hmeblkp, 6627 pp->p_pagenum, 0, FLUSH_NECESSARY_CPUS, 6628 CACHE_FLUSH, 0); 6629 cpuset = sfmmup->sfmmu_cpusran; 6630 } 6631 6632 /* 6633 * Hme_sub has to run after ttesync() and a_rss update. 6634 * See hblk_unload(). 6635 */ 6636 HME_SUB(sfhme, pp); 6637 membar_stst(); 6638 6639 /* 6640 * We can not make ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 6641 * since pteload may have done a HME_ADD() right after 6642 * we did the HME_SUB() above. Hmecnt is now maintained 6643 * by cas only. no lock guranteed its value. The only 6644 * gurantee we have is the hmecnt should not be less than 6645 * what it should be so the hblk will not be taken away. 6646 * It's also important that we decremented the hmecnt after 6647 * we are done with hmeblkp so that this hmeblk won't be 6648 * stolen. 6649 */ 6650 ASSERT(hmeblkp->hblk_hmecnt > 0); 6651 ASSERT(hmeblkp->hblk_vcnt > 0); 6652 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 6653 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 6654 /* 6655 * This is bug 4063182. 6656 * XXX: fixme 6657 * ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 6658 * !hmeblkp->hblk_lckcnt); 6659 */ 6660 } else { 6661 panic("invalid tte? pp %p &tte %p", 6662 (void *)pp, (void *)&tte); 6663 } 6664 6665 return (cpuset); 6666 } 6667 6668 /* 6669 * While relocating a kernel page, this function will move the mappings 6670 * from tpp to dpp and modify any associated data with these mappings. 6671 * It also unsuspends the suspended kernel mapping. 6672 */ 6673 static void 6674 hat_pagereload(struct page *tpp, struct page *dpp) 6675 { 6676 struct sf_hment *sfhme; 6677 tte_t tte, ttemod; 6678 int index, cons; 6679 6680 ASSERT(getpil() == PIL_MAX); 6681 ASSERT(sfmmu_mlist_held(tpp)); 6682 ASSERT(sfmmu_mlist_held(dpp)); 6683 6684 index = PP_MAPINDEX(tpp); 6685 cons = TTE8K; 6686 6687 /* Update real mappings to the page */ 6688 retry: 6689 for (sfhme = tpp->p_mapping; sfhme != NULL; sfhme = sfhme->hme_next) { 6690 if (IS_PAHME(sfhme)) 6691 continue; 6692 sfmmu_copytte(&sfhme->hme_tte, &tte); 6693 ttemod = tte; 6694 6695 /* 6696 * replace old pfn with new pfn in TTE 6697 */ 6698 PFN_TO_TTE(ttemod, dpp->p_pagenum); 6699 6700 /* 6701 * clear suspend bit 6702 */ 6703 ASSERT(TTE_IS_SUSPEND(&ttemod)); 6704 TTE_CLR_SUSPEND(&ttemod); 6705 6706 if (sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte) < 0) 6707 panic("hat_pagereload(): sfmmu_modifytte_try() failed"); 6708 6709 /* 6710 * set hme_page point to new page 6711 */ 6712 sfhme->hme_page = dpp; 6713 } 6714 6715 /* 6716 * move p_mapping list from old page to new page 6717 */ 6718 dpp->p_mapping = tpp->p_mapping; 6719 tpp->p_mapping = NULL; 6720 dpp->p_share = tpp->p_share; 6721 tpp->p_share = 0; 6722 6723 while (index != 0) { 6724 index = index >> 1; 6725 if (index != 0) 6726 cons++; 6727 if (index & 0x1) { 6728 tpp = PP_GROUPLEADER(tpp, cons); 6729 dpp = PP_GROUPLEADER(dpp, cons); 6730 goto retry; 6731 } 6732 } 6733 6734 if (dtrace_kreloc_fini) 6735 (*dtrace_kreloc_fini)(); 6736 mutex_exit(&kpr_suspendlock); 6737 } 6738 6739 uint_t 6740 hat_pagesync(struct page *pp, uint_t clearflag) 6741 { 6742 struct sf_hment *sfhme, *tmphme = NULL; 6743 struct hme_blk *hmeblkp; 6744 kmutex_t *pml; 6745 cpuset_t cpuset, tset; 6746 int index, cons; 6747 extern ulong_t po_share; 6748 page_t *save_pp = pp; 6749 6750 CPUSET_ZERO(cpuset); 6751 6752 if (PP_ISRO(pp) && (clearflag & HAT_SYNC_STOPON_MOD)) { 6753 return (PP_GENERIC_ATTR(pp)); 6754 } 6755 6756 if ((clearflag == (HAT_SYNC_STOPON_REF | HAT_SYNC_DONTZERO)) && 6757 PP_ISREF(pp)) { 6758 return (PP_GENERIC_ATTR(pp)); 6759 } 6760 6761 if ((clearflag == (HAT_SYNC_STOPON_MOD | HAT_SYNC_DONTZERO)) && 6762 PP_ISMOD(pp)) { 6763 return (PP_GENERIC_ATTR(pp)); 6764 } 6765 6766 if ((clearflag & HAT_SYNC_STOPON_SHARED) != 0 && 6767 (pp->p_share > po_share) && 6768 !(clearflag & HAT_SYNC_ZERORM)) { 6769 if (PP_ISRO(pp)) 6770 hat_page_setattr(pp, P_REF); 6771 return (PP_GENERIC_ATTR(pp)); 6772 } 6773 6774 clearflag &= ~HAT_SYNC_STOPON_SHARED; 6775 pml = sfmmu_mlist_enter(pp); 6776 index = PP_MAPINDEX(pp); 6777 cons = TTE8K; 6778 retry: 6779 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6780 /* 6781 * We need to save the next hment on the list since 6782 * it is possible for pagesync to remove an invalid hment 6783 * from the list. 6784 */ 6785 tmphme = sfhme->hme_next; 6786 /* 6787 * If we are looking for large mappings and this hme doesn't 6788 * reach the range we are seeking, just ignore its. 6789 */ 6790 hmeblkp = sfmmu_hmetohblk(sfhme); 6791 if (hmeblkp->hblk_xhat_bit) 6792 continue; 6793 6794 if (hme_size(sfhme) < cons) 6795 continue; 6796 tset = sfmmu_pagesync(pp, sfhme, 6797 clearflag & ~HAT_SYNC_STOPON_RM); 6798 CPUSET_OR(cpuset, tset); 6799 /* 6800 * If clearflag is HAT_SYNC_DONTZERO, break out as soon 6801 * as the "ref" or "mod" is set. 6802 */ 6803 if ((clearflag & ~HAT_SYNC_STOPON_RM) == HAT_SYNC_DONTZERO && 6804 ((clearflag & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp)) || 6805 ((clearflag & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp))) { 6806 index = 0; 6807 break; 6808 } 6809 } 6810 6811 while (index) { 6812 index = index >> 1; 6813 cons++; 6814 if (index & 0x1) { 6815 /* Go to leading page */ 6816 pp = PP_GROUPLEADER(pp, cons); 6817 goto retry; 6818 } 6819 } 6820 6821 xt_sync(cpuset); 6822 sfmmu_mlist_exit(pml); 6823 return (PP_GENERIC_ATTR(save_pp)); 6824 } 6825 6826 /* 6827 * Get all the hardware dependent attributes for a page struct 6828 */ 6829 static cpuset_t 6830 sfmmu_pagesync(struct page *pp, struct sf_hment *sfhme, 6831 uint_t clearflag) 6832 { 6833 caddr_t addr; 6834 tte_t tte, ttemod; 6835 struct hme_blk *hmeblkp; 6836 int ret; 6837 sfmmu_t *sfmmup; 6838 cpuset_t cpuset; 6839 6840 ASSERT(pp != NULL); 6841 ASSERT(sfmmu_mlist_held(pp)); 6842 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 6843 (clearflag == HAT_SYNC_ZERORM)); 6844 6845 SFMMU_STAT(sf_pagesync); 6846 6847 CPUSET_ZERO(cpuset); 6848 6849 sfmmu_pagesync_retry: 6850 6851 sfmmu_copytte(&sfhme->hme_tte, &tte); 6852 if (TTE_IS_VALID(&tte)) { 6853 hmeblkp = sfmmu_hmetohblk(sfhme); 6854 sfmmup = hblktosfmmu(hmeblkp); 6855 addr = tte_to_vaddr(hmeblkp, tte); 6856 if (clearflag == HAT_SYNC_ZERORM) { 6857 ttemod = tte; 6858 TTE_CLR_RM(&ttemod); 6859 ret = sfmmu_modifytte_try(&tte, &ttemod, 6860 &sfhme->hme_tte); 6861 if (ret < 0) { 6862 /* 6863 * cas failed and the new value is not what 6864 * we want. 6865 */ 6866 goto sfmmu_pagesync_retry; 6867 } 6868 6869 if (ret > 0) { 6870 /* we win the cas */ 6871 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6872 cpuset = sfmmup->sfmmu_cpusran; 6873 } 6874 } 6875 6876 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6877 } 6878 return (cpuset); 6879 } 6880 6881 /* 6882 * Remove write permission from a mappings to a page, so that 6883 * we can detect the next modification of it. This requires modifying 6884 * the TTE then invalidating (demap) any TLB entry using that TTE. 6885 * This code is similar to sfmmu_pagesync(). 6886 */ 6887 static cpuset_t 6888 sfmmu_pageclrwrt(struct page *pp, struct sf_hment *sfhme) 6889 { 6890 caddr_t addr; 6891 tte_t tte; 6892 tte_t ttemod; 6893 struct hme_blk *hmeblkp; 6894 int ret; 6895 sfmmu_t *sfmmup; 6896 cpuset_t cpuset; 6897 6898 ASSERT(pp != NULL); 6899 ASSERT(sfmmu_mlist_held(pp)); 6900 6901 CPUSET_ZERO(cpuset); 6902 SFMMU_STAT(sf_clrwrt); 6903 6904 retry: 6905 6906 sfmmu_copytte(&sfhme->hme_tte, &tte); 6907 if (TTE_IS_VALID(&tte) && TTE_IS_WRITABLE(&tte)) { 6908 hmeblkp = sfmmu_hmetohblk(sfhme); 6909 6910 /* 6911 * xhat mappings should never be to a VMODSORT page. 6912 */ 6913 ASSERT(hmeblkp->hblk_xhat_bit == 0); 6914 6915 sfmmup = hblktosfmmu(hmeblkp); 6916 addr = tte_to_vaddr(hmeblkp, tte); 6917 6918 ttemod = tte; 6919 TTE_CLR_WRT(&ttemod); 6920 TTE_CLR_MOD(&ttemod); 6921 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6922 6923 /* 6924 * if cas failed and the new value is not what 6925 * we want retry 6926 */ 6927 if (ret < 0) 6928 goto retry; 6929 6930 /* we win the cas */ 6931 if (ret > 0) { 6932 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6933 cpuset = sfmmup->sfmmu_cpusran; 6934 } 6935 } 6936 6937 return (cpuset); 6938 } 6939 6940 /* 6941 * Walk all mappings of a page, removing write permission and clearing the 6942 * ref/mod bits. This code is similar to hat_pagesync() 6943 */ 6944 static void 6945 hat_page_clrwrt(page_t *pp) 6946 { 6947 struct sf_hment *sfhme; 6948 struct sf_hment *tmphme = NULL; 6949 kmutex_t *pml; 6950 cpuset_t cpuset; 6951 cpuset_t tset; 6952 int index; 6953 int cons; 6954 6955 CPUSET_ZERO(cpuset); 6956 6957 pml = sfmmu_mlist_enter(pp); 6958 index = PP_MAPINDEX(pp); 6959 cons = TTE8K; 6960 retry: 6961 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6962 tmphme = sfhme->hme_next; 6963 6964 /* 6965 * If we are looking for large mappings and this hme doesn't 6966 * reach the range we are seeking, just ignore its. 6967 */ 6968 6969 if (hme_size(sfhme) < cons) 6970 continue; 6971 6972 tset = sfmmu_pageclrwrt(pp, sfhme); 6973 CPUSET_OR(cpuset, tset); 6974 } 6975 6976 while (index) { 6977 index = index >> 1; 6978 cons++; 6979 if (index & 0x1) { 6980 /* Go to leading page */ 6981 pp = PP_GROUPLEADER(pp, cons); 6982 goto retry; 6983 } 6984 } 6985 6986 xt_sync(cpuset); 6987 sfmmu_mlist_exit(pml); 6988 } 6989 6990 /* 6991 * Set the given REF/MOD/RO bits for the given page. 6992 * For a vnode with a sorted v_pages list, we need to change 6993 * the attributes and the v_pages list together under page_vnode_mutex. 6994 */ 6995 void 6996 hat_page_setattr(page_t *pp, uint_t flag) 6997 { 6998 vnode_t *vp = pp->p_vnode; 6999 page_t **listp; 7000 kmutex_t *pmtx; 7001 kmutex_t *vphm = NULL; 7002 7003 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7004 7005 /* 7006 * nothing to do if attribute already set 7007 */ 7008 if ((pp->p_nrm & flag) == flag) 7009 return; 7010 7011 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 7012 vphm = page_vnode_mutex(vp); 7013 mutex_enter(vphm); 7014 } 7015 7016 pmtx = sfmmu_page_enter(pp); 7017 pp->p_nrm |= flag; 7018 sfmmu_page_exit(pmtx); 7019 7020 if (vphm != NULL) { 7021 /* 7022 * Some File Systems examine v_pages for NULL w/o 7023 * grabbing the vphm mutex. Must not let it become NULL when 7024 * pp is the only page on the list. 7025 */ 7026 if (pp->p_vpnext != pp) { 7027 page_vpsub(&vp->v_pages, pp); 7028 if (vp->v_pages != NULL) 7029 listp = &vp->v_pages->p_vpprev->p_vpnext; 7030 else 7031 listp = &vp->v_pages; 7032 page_vpadd(listp, pp); 7033 } 7034 mutex_exit(vphm); 7035 } 7036 } 7037 7038 void 7039 hat_page_clrattr(page_t *pp, uint_t flag) 7040 { 7041 vnode_t *vp = pp->p_vnode; 7042 kmutex_t *pmtx; 7043 7044 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7045 7046 pmtx = sfmmu_page_enter(pp); 7047 7048 /* 7049 * Caller is expected to hold page's io lock for VMODSORT to work 7050 * correctly with pvn_vplist_dirty() and pvn_getdirty() when mod 7051 * bit is cleared. 7052 * We don't have assert to avoid tripping some existing third party 7053 * code. The dirty page is moved back to top of the v_page list 7054 * after IO is done in pvn_write_done(). 7055 */ 7056 pp->p_nrm &= ~flag; 7057 sfmmu_page_exit(pmtx); 7058 7059 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 7060 7061 /* 7062 * VMODSORT works by removing write permissions and getting 7063 * a fault when a page is made dirty. At this point 7064 * we need to remove write permission from all mappings 7065 * to this page. 7066 */ 7067 hat_page_clrwrt(pp); 7068 } 7069 } 7070 7071 uint_t 7072 hat_page_getattr(page_t *pp, uint_t flag) 7073 { 7074 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7075 return ((uint_t)(pp->p_nrm & flag)); 7076 } 7077 7078 /* 7079 * DEBUG kernels: verify that a kernel va<->pa translation 7080 * is safe by checking the underlying page_t is in a page 7081 * relocation-safe state. 7082 */ 7083 #ifdef DEBUG 7084 void 7085 sfmmu_check_kpfn(pfn_t pfn) 7086 { 7087 page_t *pp; 7088 int index, cons; 7089 7090 if (hat_check_vtop == 0) 7091 return; 7092 7093 if (hat_kpr_enabled == 0 || kvseg.s_base == NULL || panicstr) 7094 return; 7095 7096 pp = page_numtopp_nolock(pfn); 7097 if (!pp) 7098 return; 7099 7100 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7101 return; 7102 7103 /* 7104 * Handed a large kernel page, we dig up the root page since we 7105 * know the root page might have the lock also. 7106 */ 7107 if (pp->p_szc != 0) { 7108 index = PP_MAPINDEX(pp); 7109 cons = TTE8K; 7110 again: 7111 while (index != 0) { 7112 index >>= 1; 7113 if (index != 0) 7114 cons++; 7115 if (index & 0x1) { 7116 pp = PP_GROUPLEADER(pp, cons); 7117 goto again; 7118 } 7119 } 7120 } 7121 7122 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7123 return; 7124 7125 /* 7126 * Pages need to be locked or allocated "permanent" (either from 7127 * static_arena arena or explicitly setting PG_NORELOC when calling 7128 * page_create_va()) for VA->PA translations to be valid. 7129 */ 7130 if (!PP_ISNORELOC(pp)) 7131 panic("Illegal VA->PA translation, pp 0x%p not permanent", pp); 7132 else 7133 panic("Illegal VA->PA translation, pp 0x%p not locked", pp); 7134 } 7135 #endif /* DEBUG */ 7136 7137 /* 7138 * Returns a page frame number for a given virtual address. 7139 * Returns PFN_INVALID to indicate an invalid mapping 7140 */ 7141 pfn_t 7142 hat_getpfnum(struct hat *hat, caddr_t addr) 7143 { 7144 pfn_t pfn; 7145 tte_t tte; 7146 7147 /* 7148 * We would like to 7149 * ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 7150 * but we can't because the iommu driver will call this 7151 * routine at interrupt time and it can't grab the as lock 7152 * or it will deadlock: A thread could have the as lock 7153 * and be waiting for io. The io can't complete 7154 * because the interrupt thread is blocked trying to grab 7155 * the as lock. 7156 */ 7157 7158 ASSERT(hat->sfmmu_xhat_provider == NULL); 7159 7160 if (hat == ksfmmup) { 7161 if (IS_KMEM_VA_LARGEPAGE(addr)) { 7162 ASSERT(segkmem_lpszc > 0); 7163 pfn = sfmmu_kvaszc2pfn(addr, segkmem_lpszc); 7164 if (pfn != PFN_INVALID) { 7165 sfmmu_check_kpfn(pfn); 7166 return (pfn); 7167 } 7168 } else if (segkpm && IS_KPM_ADDR(addr)) { 7169 return (sfmmu_kpm_vatopfn(addr)); 7170 } 7171 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7172 == PFN_SUSPENDED) { 7173 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7174 } 7175 sfmmu_check_kpfn(pfn); 7176 return (pfn); 7177 } else { 7178 return (sfmmu_uvatopfn(addr, hat)); 7179 } 7180 } 7181 7182 /* 7183 * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged. 7184 * Use hat_getpfnum(kas.a_hat, ...) instead. 7185 * 7186 * We'd like to return PFN_INVALID if the mappings have underlying page_t's 7187 * but can't right now due to the fact that some software has grown to use 7188 * this interface incorrectly. So for now when the interface is misused, 7189 * return a warning to the user that in the future it won't work in the 7190 * way they're abusing it, and carry on (after disabling page relocation). 7191 */ 7192 pfn_t 7193 hat_getkpfnum(caddr_t addr) 7194 { 7195 pfn_t pfn; 7196 tte_t tte; 7197 int badcaller = 0; 7198 extern int segkmem_reloc; 7199 7200 if (segkpm && IS_KPM_ADDR(addr)) { 7201 badcaller = 1; 7202 pfn = sfmmu_kpm_vatopfn(addr); 7203 } else { 7204 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7205 == PFN_SUSPENDED) { 7206 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7207 } 7208 badcaller = pf_is_memory(pfn); 7209 } 7210 7211 if (badcaller) { 7212 /* 7213 * We can't return PFN_INVALID or the caller may panic 7214 * or corrupt the system. The only alternative is to 7215 * disable page relocation at this point for all kernel 7216 * memory. This will impact any callers of page_relocate() 7217 * such as FMA or DR. 7218 * 7219 * RFE: Add junk here to spit out an ereport so the sysadmin 7220 * can be advised that he should upgrade his device driver 7221 * so that this doesn't happen. 7222 */ 7223 hat_getkpfnum_badcall(caller()); 7224 if (hat_kpr_enabled && segkmem_reloc) { 7225 hat_kpr_enabled = 0; 7226 segkmem_reloc = 0; 7227 cmn_err(CE_WARN, "Kernel Page Relocation is DISABLED"); 7228 } 7229 } 7230 return (pfn); 7231 } 7232 7233 pfn_t 7234 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup) 7235 { 7236 struct hmehash_bucket *hmebp; 7237 hmeblk_tag hblktag; 7238 int hmeshift, hashno = 1; 7239 struct hme_blk *hmeblkp = NULL; 7240 7241 struct sf_hment *sfhmep; 7242 tte_t tte; 7243 pfn_t pfn; 7244 7245 /* support for ISM */ 7246 ism_map_t *ism_map; 7247 ism_blk_t *ism_blkp; 7248 int i; 7249 sfmmu_t *ism_hatid = NULL; 7250 sfmmu_t *locked_hatid = NULL; 7251 7252 7253 ASSERT(sfmmup != ksfmmup); 7254 SFMMU_STAT(sf_user_vtop); 7255 /* 7256 * Set ism_hatid if vaddr falls in a ISM segment. 7257 */ 7258 ism_blkp = sfmmup->sfmmu_iblk; 7259 if (ism_blkp) { 7260 sfmmu_ismhat_enter(sfmmup, 0); 7261 locked_hatid = sfmmup; 7262 } 7263 while (ism_blkp && ism_hatid == NULL) { 7264 ism_map = ism_blkp->iblk_maps; 7265 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 7266 if (vaddr >= ism_start(ism_map[i]) && 7267 vaddr < ism_end(ism_map[i])) { 7268 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 7269 vaddr = (caddr_t)(vaddr - 7270 ism_start(ism_map[i])); 7271 break; 7272 } 7273 } 7274 ism_blkp = ism_blkp->iblk_next; 7275 } 7276 if (locked_hatid) { 7277 sfmmu_ismhat_exit(locked_hatid, 0); 7278 } 7279 7280 hblktag.htag_id = sfmmup; 7281 do { 7282 hmeshift = HME_HASH_SHIFT(hashno); 7283 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 7284 hblktag.htag_rehash = hashno; 7285 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 7286 7287 SFMMU_HASH_LOCK(hmebp); 7288 7289 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 7290 if (hmeblkp != NULL) { 7291 HBLKTOHME(sfhmep, hmeblkp, vaddr); 7292 sfmmu_copytte(&sfhmep->hme_tte, &tte); 7293 if (TTE_IS_VALID(&tte)) { 7294 pfn = TTE_TO_PFN(vaddr, &tte); 7295 } else { 7296 pfn = PFN_INVALID; 7297 } 7298 SFMMU_HASH_UNLOCK(hmebp); 7299 return (pfn); 7300 } 7301 SFMMU_HASH_UNLOCK(hmebp); 7302 hashno++; 7303 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 7304 return (PFN_INVALID); 7305 } 7306 7307 7308 /* 7309 * For compatability with AT&T and later optimizations 7310 */ 7311 /* ARGSUSED */ 7312 void 7313 hat_map(struct hat *hat, caddr_t addr, size_t len, uint_t flags) 7314 { 7315 ASSERT(hat != NULL); 7316 ASSERT(hat->sfmmu_xhat_provider == NULL); 7317 } 7318 7319 /* 7320 * Return the number of mappings to a particular page. 7321 * This number is an approximation of the number of 7322 * number of people sharing the page. 7323 */ 7324 ulong_t 7325 hat_page_getshare(page_t *pp) 7326 { 7327 page_t *spp = pp; /* start page */ 7328 kmutex_t *pml; 7329 ulong_t cnt; 7330 int index, sz = TTE64K; 7331 7332 /* 7333 * We need to grab the mlist lock to make sure any outstanding 7334 * load/unloads complete. Otherwise we could return zero 7335 * even though the unload(s) hasn't finished yet. 7336 */ 7337 pml = sfmmu_mlist_enter(spp); 7338 cnt = spp->p_share; 7339 7340 #ifdef VAC 7341 if (kpm_enable) 7342 cnt += spp->p_kpmref; 7343 #endif 7344 7345 /* 7346 * If we have any large mappings, we count the number of 7347 * mappings that this large page is part of. 7348 */ 7349 index = PP_MAPINDEX(spp); 7350 index >>= 1; 7351 while (index) { 7352 pp = PP_GROUPLEADER(spp, sz); 7353 if ((index & 0x1) && pp != spp) { 7354 cnt += pp->p_share; 7355 spp = pp; 7356 } 7357 index >>= 1; 7358 sz++; 7359 } 7360 sfmmu_mlist_exit(pml); 7361 return (cnt); 7362 } 7363 7364 /* 7365 * Unload all large mappings to the pp and reset the p_szc field of every 7366 * constituent page according to the remaining mappings. 7367 * 7368 * pp must be locked SE_EXCL. Even though no other constituent pages are 7369 * locked it's legal to unload the large mappings to the pp because all 7370 * constituent pages of large locked mappings have to be locked SE_SHARED. 7371 * This means if we have SE_EXCL lock on one of constituent pages none of the 7372 * large mappings to pp are locked. 7373 * 7374 * Decrease p_szc field starting from the last constituent page and ending 7375 * with the root page. This method is used because other threads rely on the 7376 * root's p_szc to find the lock to syncronize on. After a root page_t's p_szc 7377 * is demoted then other threads will succeed in sfmmu_mlspl_enter(). This 7378 * ensures that p_szc changes of the constituent pages appears atomic for all 7379 * threads that use sfmmu_mlspl_enter() to examine p_szc field. 7380 * 7381 * This mechanism is only used for file system pages where it's not always 7382 * possible to get SE_EXCL locks on all constituent pages to demote the size 7383 * code (as is done for anonymous or kernel large pages). 7384 * 7385 * See more comments in front of sfmmu_mlspl_enter(). 7386 */ 7387 void 7388 hat_page_demote(page_t *pp) 7389 { 7390 int index; 7391 int sz; 7392 cpuset_t cpuset; 7393 int sync = 0; 7394 page_t *rootpp; 7395 struct sf_hment *sfhme; 7396 struct sf_hment *tmphme = NULL; 7397 struct hme_blk *hmeblkp; 7398 uint_t pszc; 7399 page_t *lastpp; 7400 cpuset_t tset; 7401 pgcnt_t npgs; 7402 kmutex_t *pml; 7403 kmutex_t *pmtx = NULL; 7404 7405 ASSERT(PAGE_EXCL(pp)); 7406 ASSERT(!PP_ISFREE(pp)); 7407 ASSERT(page_szc_lock_assert(pp)); 7408 pml = sfmmu_mlist_enter(pp); 7409 7410 pszc = pp->p_szc; 7411 if (pszc == 0) { 7412 goto out; 7413 } 7414 7415 index = PP_MAPINDEX(pp) >> 1; 7416 7417 if (index) { 7418 CPUSET_ZERO(cpuset); 7419 sz = TTE64K; 7420 sync = 1; 7421 } 7422 7423 while (index) { 7424 if (!(index & 0x1)) { 7425 index >>= 1; 7426 sz++; 7427 continue; 7428 } 7429 ASSERT(sz <= pszc); 7430 rootpp = PP_GROUPLEADER(pp, sz); 7431 for (sfhme = rootpp->p_mapping; sfhme; sfhme = tmphme) { 7432 tmphme = sfhme->hme_next; 7433 hmeblkp = sfmmu_hmetohblk(sfhme); 7434 if (hme_size(sfhme) != sz) { 7435 continue; 7436 } 7437 if (hmeblkp->hblk_xhat_bit) { 7438 cmn_err(CE_PANIC, 7439 "hat_page_demote: xhat hmeblk"); 7440 } 7441 tset = sfmmu_pageunload(rootpp, sfhme, sz); 7442 CPUSET_OR(cpuset, tset); 7443 } 7444 if (index >>= 1) { 7445 sz++; 7446 } 7447 } 7448 7449 ASSERT(!PP_ISMAPPED_LARGE(pp)); 7450 7451 if (sync) { 7452 xt_sync(cpuset); 7453 #ifdef VAC 7454 if (PP_ISTNC(pp)) { 7455 conv_tnc(rootpp, sz); 7456 } 7457 #endif /* VAC */ 7458 } 7459 7460 pmtx = sfmmu_page_enter(pp); 7461 7462 ASSERT(pp->p_szc == pszc); 7463 rootpp = PP_PAGEROOT(pp); 7464 ASSERT(rootpp->p_szc == pszc); 7465 lastpp = PP_PAGENEXT_N(rootpp, TTEPAGES(pszc) - 1); 7466 7467 while (lastpp != rootpp) { 7468 sz = PP_MAPINDEX(lastpp) ? fnd_mapping_sz(lastpp) : 0; 7469 ASSERT(sz < pszc); 7470 npgs = (sz == 0) ? 1 : TTEPAGES(sz); 7471 ASSERT(P2PHASE(lastpp->p_pagenum, npgs) == npgs - 1); 7472 while (--npgs > 0) { 7473 lastpp->p_szc = (uchar_t)sz; 7474 lastpp = PP_PAGEPREV(lastpp); 7475 } 7476 if (sz) { 7477 /* 7478 * make sure before current root's pszc 7479 * is updated all updates to constituent pages pszc 7480 * fields are globally visible. 7481 */ 7482 membar_producer(); 7483 } 7484 lastpp->p_szc = sz; 7485 ASSERT(IS_P2ALIGNED(lastpp->p_pagenum, TTEPAGES(sz))); 7486 if (lastpp != rootpp) { 7487 lastpp = PP_PAGEPREV(lastpp); 7488 } 7489 } 7490 if (sz == 0) { 7491 /* the loop above doesn't cover this case */ 7492 rootpp->p_szc = 0; 7493 } 7494 out: 7495 ASSERT(pp->p_szc == 0); 7496 if (pmtx != NULL) { 7497 sfmmu_page_exit(pmtx); 7498 } 7499 sfmmu_mlist_exit(pml); 7500 } 7501 7502 /* 7503 * Refresh the HAT ismttecnt[] element for size szc. 7504 * Caller must have set ISM busy flag to prevent mapping 7505 * lists from changing while we're traversing them. 7506 */ 7507 pgcnt_t 7508 ism_tsb_entries(sfmmu_t *sfmmup, int szc) 7509 { 7510 ism_blk_t *ism_blkp = sfmmup->sfmmu_iblk; 7511 ism_map_t *ism_map; 7512 pgcnt_t npgs = 0; 7513 int j; 7514 7515 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 7516 for (; ism_blkp != NULL; ism_blkp = ism_blkp->iblk_next) { 7517 ism_map = ism_blkp->iblk_maps; 7518 for (j = 0; ism_map[j].imap_ismhat && j < ISM_MAP_SLOTS; j++) 7519 npgs += ism_map[j].imap_ismhat->sfmmu_ttecnt[szc]; 7520 } 7521 sfmmup->sfmmu_ismttecnt[szc] = npgs; 7522 return (npgs); 7523 } 7524 7525 /* 7526 * Yield the memory claim requirement for an address space. 7527 * 7528 * This is currently implemented as the number of bytes that have active 7529 * hardware translations that have page structures. Therefore, it can 7530 * underestimate the traditional resident set size, eg, if the 7531 * physical page is present and the hardware translation is missing; 7532 * and it can overestimate the rss, eg, if there are active 7533 * translations to a frame buffer with page structs. 7534 * Also, it does not take sharing into account. 7535 * 7536 * Note that we don't acquire locks here since this function is most often 7537 * called from the clock thread. 7538 */ 7539 size_t 7540 hat_get_mapped_size(struct hat *hat) 7541 { 7542 size_t assize = 0; 7543 int i; 7544 7545 if (hat == NULL) 7546 return (0); 7547 7548 ASSERT(hat->sfmmu_xhat_provider == NULL); 7549 7550 for (i = 0; i < mmu_page_sizes; i++) 7551 assize += (pgcnt_t)hat->sfmmu_ttecnt[i] * TTEBYTES(i); 7552 7553 if (hat->sfmmu_iblk == NULL) 7554 return (assize); 7555 7556 for (i = 0; i < mmu_page_sizes; i++) 7557 assize += (pgcnt_t)hat->sfmmu_ismttecnt[i] * TTEBYTES(i); 7558 7559 return (assize); 7560 } 7561 7562 int 7563 hat_stats_enable(struct hat *hat) 7564 { 7565 hatlock_t *hatlockp; 7566 7567 ASSERT(hat->sfmmu_xhat_provider == NULL); 7568 7569 hatlockp = sfmmu_hat_enter(hat); 7570 hat->sfmmu_rmstat++; 7571 sfmmu_hat_exit(hatlockp); 7572 return (1); 7573 } 7574 7575 void 7576 hat_stats_disable(struct hat *hat) 7577 { 7578 hatlock_t *hatlockp; 7579 7580 ASSERT(hat->sfmmu_xhat_provider == NULL); 7581 7582 hatlockp = sfmmu_hat_enter(hat); 7583 hat->sfmmu_rmstat--; 7584 sfmmu_hat_exit(hatlockp); 7585 } 7586 7587 /* 7588 * Routines for entering or removing ourselves from the 7589 * ism_hat's mapping list. 7590 */ 7591 static void 7592 iment_add(struct ism_ment *iment, struct hat *ism_hat) 7593 { 7594 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7595 7596 iment->iment_prev = NULL; 7597 iment->iment_next = ism_hat->sfmmu_iment; 7598 if (ism_hat->sfmmu_iment) { 7599 ism_hat->sfmmu_iment->iment_prev = iment; 7600 } 7601 ism_hat->sfmmu_iment = iment; 7602 } 7603 7604 static void 7605 iment_sub(struct ism_ment *iment, struct hat *ism_hat) 7606 { 7607 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7608 7609 if (ism_hat->sfmmu_iment == NULL) { 7610 panic("ism map entry remove - no entries"); 7611 } 7612 7613 if (iment->iment_prev) { 7614 ASSERT(ism_hat->sfmmu_iment != iment); 7615 iment->iment_prev->iment_next = iment->iment_next; 7616 } else { 7617 ASSERT(ism_hat->sfmmu_iment == iment); 7618 ism_hat->sfmmu_iment = iment->iment_next; 7619 } 7620 7621 if (iment->iment_next) { 7622 iment->iment_next->iment_prev = iment->iment_prev; 7623 } 7624 7625 /* 7626 * zero out the entry 7627 */ 7628 iment->iment_next = NULL; 7629 iment->iment_prev = NULL; 7630 iment->iment_hat = NULL; 7631 } 7632 7633 /* 7634 * Hat_share()/unshare() return an (non-zero) error 7635 * when saddr and daddr are not properly aligned. 7636 * 7637 * The top level mapping element determines the alignment 7638 * requirement for saddr and daddr, depending on different 7639 * architectures. 7640 * 7641 * When hat_share()/unshare() are not supported, 7642 * HATOP_SHARE()/UNSHARE() return 0 7643 */ 7644 int 7645 hat_share(struct hat *sfmmup, caddr_t addr, 7646 struct hat *ism_hatid, caddr_t sptaddr, size_t len, uint_t ismszc) 7647 { 7648 ism_blk_t *ism_blkp; 7649 ism_blk_t *new_iblk; 7650 ism_map_t *ism_map; 7651 ism_ment_t *ism_ment; 7652 int i, added; 7653 hatlock_t *hatlockp; 7654 int reload_mmu = 0; 7655 uint_t ismshift = page_get_shift(ismszc); 7656 size_t ismpgsz = page_get_pagesize(ismszc); 7657 uint_t ismmask = (uint_t)ismpgsz - 1; 7658 size_t sh_size = ISM_SHIFT(ismshift, len); 7659 ushort_t ismhatflag; 7660 7661 #ifdef DEBUG 7662 caddr_t eaddr = addr + len; 7663 #endif /* DEBUG */ 7664 7665 ASSERT(ism_hatid != NULL && sfmmup != NULL); 7666 ASSERT(sptaddr == ISMID_STARTADDR); 7667 /* 7668 * Check the alignment. 7669 */ 7670 if (!ISM_ALIGNED(ismshift, addr) || !ISM_ALIGNED(ismshift, sptaddr)) 7671 return (EINVAL); 7672 7673 /* 7674 * Check size alignment. 7675 */ 7676 if (!ISM_ALIGNED(ismshift, len)) 7677 return (EINVAL); 7678 7679 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 7680 7681 /* 7682 * Allocate ism_ment for the ism_hat's mapping list, and an 7683 * ism map blk in case we need one. We must do our 7684 * allocations before acquiring locks to prevent a deadlock 7685 * in the kmem allocator on the mapping list lock. 7686 */ 7687 new_iblk = kmem_cache_alloc(ism_blk_cache, KM_SLEEP); 7688 ism_ment = kmem_cache_alloc(ism_ment_cache, KM_SLEEP); 7689 7690 /* 7691 * Serialize ISM mappings with the ISM busy flag, and also the 7692 * trap handlers. 7693 */ 7694 sfmmu_ismhat_enter(sfmmup, 0); 7695 7696 /* 7697 * Allocate an ism map blk if necessary. 7698 */ 7699 if (sfmmup->sfmmu_iblk == NULL) { 7700 sfmmup->sfmmu_iblk = new_iblk; 7701 bzero(new_iblk, sizeof (*new_iblk)); 7702 new_iblk->iblk_nextpa = (uint64_t)-1; 7703 membar_stst(); /* make sure next ptr visible to all CPUs */ 7704 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk); 7705 reload_mmu = 1; 7706 new_iblk = NULL; 7707 } 7708 7709 #ifdef DEBUG 7710 /* 7711 * Make sure mapping does not already exist. 7712 */ 7713 ism_blkp = sfmmup->sfmmu_iblk; 7714 while (ism_blkp) { 7715 ism_map = ism_blkp->iblk_maps; 7716 for (i = 0; i < ISM_MAP_SLOTS && ism_map[i].imap_ismhat; i++) { 7717 if ((addr >= ism_start(ism_map[i]) && 7718 addr < ism_end(ism_map[i])) || 7719 eaddr > ism_start(ism_map[i]) && 7720 eaddr <= ism_end(ism_map[i])) { 7721 panic("sfmmu_share: Already mapped!"); 7722 } 7723 } 7724 ism_blkp = ism_blkp->iblk_next; 7725 } 7726 #endif /* DEBUG */ 7727 7728 ASSERT(ismszc >= TTE4M); 7729 if (ismszc == TTE4M) { 7730 ismhatflag = HAT_4M_FLAG; 7731 } else if (ismszc == TTE32M) { 7732 ismhatflag = HAT_32M_FLAG; 7733 } else if (ismszc == TTE256M) { 7734 ismhatflag = HAT_256M_FLAG; 7735 } 7736 /* 7737 * Add mapping to first available mapping slot. 7738 */ 7739 ism_blkp = sfmmup->sfmmu_iblk; 7740 added = 0; 7741 while (!added) { 7742 ism_map = ism_blkp->iblk_maps; 7743 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7744 if (ism_map[i].imap_ismhat == NULL) { 7745 7746 ism_map[i].imap_ismhat = ism_hatid; 7747 ism_map[i].imap_vb_shift = (ushort_t)ismshift; 7748 ism_map[i].imap_hatflags = ismhatflag; 7749 ism_map[i].imap_sz_mask = ismmask; 7750 /* 7751 * imap_seg is checked in ISM_CHECK to see if 7752 * non-NULL, then other info assumed valid. 7753 */ 7754 membar_stst(); 7755 ism_map[i].imap_seg = (uintptr_t)addr | sh_size; 7756 ism_map[i].imap_ment = ism_ment; 7757 7758 /* 7759 * Now add ourselves to the ism_hat's 7760 * mapping list. 7761 */ 7762 ism_ment->iment_hat = sfmmup; 7763 ism_ment->iment_base_va = addr; 7764 ism_hatid->sfmmu_ismhat = 1; 7765 ism_hatid->sfmmu_flags = 0; 7766 mutex_enter(&ism_mlist_lock); 7767 iment_add(ism_ment, ism_hatid); 7768 mutex_exit(&ism_mlist_lock); 7769 added = 1; 7770 break; 7771 } 7772 } 7773 if (!added && ism_blkp->iblk_next == NULL) { 7774 ism_blkp->iblk_next = new_iblk; 7775 new_iblk = NULL; 7776 bzero(ism_blkp->iblk_next, 7777 sizeof (*ism_blkp->iblk_next)); 7778 ism_blkp->iblk_next->iblk_nextpa = (uint64_t)-1; 7779 membar_stst(); 7780 ism_blkp->iblk_nextpa = 7781 va_to_pa((caddr_t)ism_blkp->iblk_next); 7782 } 7783 ism_blkp = ism_blkp->iblk_next; 7784 } 7785 7786 /* 7787 * Update our counters for this sfmmup's ism mappings. 7788 */ 7789 for (i = 0; i <= ismszc; i++) { 7790 if (!(disable_ism_large_pages & (1 << i))) 7791 (void) ism_tsb_entries(sfmmup, i); 7792 } 7793 7794 hatlockp = sfmmu_hat_enter(sfmmup); 7795 7796 /* 7797 * For ISM and DISM we do not support 512K pages, so we only 7798 * only search the 4M and 8K/64K hashes for 4 pagesize cpus, and search 7799 * the 256M or 32M, and 4M and 8K/64K hashes for 6 pagesize cpus. 7800 */ 7801 ASSERT((disable_ism_large_pages & (1 << TTE512K)) != 0); 7802 7803 if (ismszc > TTE4M && !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) 7804 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7805 7806 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) 7807 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 7808 7809 /* 7810 * If we updated the ismblkpa for this HAT or we need 7811 * to start searching the 256M or 32M or 4M hash, we must 7812 * make sure all CPUs running this process reload their 7813 * tsbmiss area. Otherwise they will fail to load the mappings 7814 * in the tsbmiss handler and will loop calling pagefault(). 7815 */ 7816 switch (ismszc) { 7817 case TTE256M: 7818 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 7819 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 7820 sfmmu_sync_mmustate(sfmmup); 7821 } 7822 break; 7823 case TTE32M: 7824 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 7825 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 7826 sfmmu_sync_mmustate(sfmmup); 7827 } 7828 break; 7829 case TTE4M: 7830 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 7831 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7832 sfmmu_sync_mmustate(sfmmup); 7833 } 7834 break; 7835 default: 7836 break; 7837 } 7838 7839 /* 7840 * Now we can drop the locks. 7841 */ 7842 sfmmu_ismhat_exit(sfmmup, 1); 7843 sfmmu_hat_exit(hatlockp); 7844 7845 /* 7846 * Free up ismblk if we didn't use it. 7847 */ 7848 if (new_iblk != NULL) 7849 kmem_cache_free(ism_blk_cache, new_iblk); 7850 7851 /* 7852 * Check TSB and TLB page sizes. 7853 */ 7854 sfmmu_check_page_sizes(sfmmup, 1); 7855 7856 return (0); 7857 } 7858 7859 /* 7860 * hat_unshare removes exactly one ism_map from 7861 * this process's as. It expects multiple calls 7862 * to hat_unshare for multiple shm segments. 7863 */ 7864 void 7865 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) 7866 { 7867 ism_map_t *ism_map; 7868 ism_ment_t *free_ment = NULL; 7869 ism_blk_t *ism_blkp; 7870 struct hat *ism_hatid; 7871 int found, i; 7872 hatlock_t *hatlockp; 7873 struct tsb_info *tsbinfo; 7874 uint_t ismshift = page_get_shift(ismszc); 7875 size_t sh_size = ISM_SHIFT(ismshift, len); 7876 7877 ASSERT(ISM_ALIGNED(ismshift, addr)); 7878 ASSERT(ISM_ALIGNED(ismshift, len)); 7879 ASSERT(sfmmup != NULL); 7880 ASSERT(sfmmup != ksfmmup); 7881 7882 if (sfmmup->sfmmu_xhat_provider) { 7883 XHAT_UNSHARE(sfmmup, addr, len); 7884 return; 7885 } else { 7886 /* 7887 * This must be a CPU HAT. If the address space has 7888 * XHATs attached, inform all XHATs that ISM segment 7889 * is going away 7890 */ 7891 ASSERT(sfmmup->sfmmu_as != NULL); 7892 if (sfmmup->sfmmu_as->a_xhat != NULL) 7893 xhat_unshare_all(sfmmup->sfmmu_as, addr, len); 7894 } 7895 7896 /* 7897 * Make sure that during the entire time ISM mappings are removed, 7898 * the trap handlers serialize behind us, and that no one else 7899 * can be mucking with ISM mappings. This also lets us get away 7900 * with not doing expensive cross calls to flush the TLB -- we 7901 * just discard the context, flush the entire TSB, and call it 7902 * a day. 7903 */ 7904 sfmmu_ismhat_enter(sfmmup, 0); 7905 7906 /* 7907 * Remove the mapping. 7908 * 7909 * We can't have any holes in the ism map. 7910 * The tsb miss code while searching the ism map will 7911 * stop on an empty map slot. So we must move 7912 * everyone past the hole up 1 if any. 7913 * 7914 * Also empty ism map blks are not freed until the 7915 * process exits. This is to prevent a MT race condition 7916 * between sfmmu_unshare() and sfmmu_tsbmiss_exception(). 7917 */ 7918 found = 0; 7919 ism_blkp = sfmmup->sfmmu_iblk; 7920 while (!found && ism_blkp) { 7921 ism_map = ism_blkp->iblk_maps; 7922 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7923 if (addr == ism_start(ism_map[i]) && 7924 sh_size == (size_t)(ism_size(ism_map[i]))) { 7925 found = 1; 7926 break; 7927 } 7928 } 7929 if (!found) 7930 ism_blkp = ism_blkp->iblk_next; 7931 } 7932 7933 if (found) { 7934 ism_hatid = ism_map[i].imap_ismhat; 7935 ASSERT(ism_hatid != NULL); 7936 ASSERT(ism_hatid->sfmmu_ismhat == 1); 7937 7938 /* 7939 * First remove ourselves from the ism mapping list. 7940 */ 7941 mutex_enter(&ism_mlist_lock); 7942 iment_sub(ism_map[i].imap_ment, ism_hatid); 7943 mutex_exit(&ism_mlist_lock); 7944 free_ment = ism_map[i].imap_ment; 7945 7946 /* 7947 * Now gurantee that any other cpu 7948 * that tries to process an ISM miss 7949 * will go to tl=0. 7950 */ 7951 hatlockp = sfmmu_hat_enter(sfmmup); 7952 7953 sfmmu_invalidate_ctx(sfmmup); 7954 7955 sfmmu_hat_exit(hatlockp); 7956 7957 /* 7958 * We delete the ism map by copying 7959 * the next map over the current one. 7960 * We will take the next one in the maps 7961 * array or from the next ism_blk. 7962 */ 7963 while (ism_blkp) { 7964 ism_map = ism_blkp->iblk_maps; 7965 while (i < (ISM_MAP_SLOTS - 1)) { 7966 ism_map[i] = ism_map[i + 1]; 7967 i++; 7968 } 7969 /* i == (ISM_MAP_SLOTS - 1) */ 7970 ism_blkp = ism_blkp->iblk_next; 7971 if (ism_blkp) { 7972 ism_map[i] = ism_blkp->iblk_maps[0]; 7973 i = 0; 7974 } else { 7975 ism_map[i].imap_seg = 0; 7976 ism_map[i].imap_vb_shift = 0; 7977 ism_map[i].imap_hatflags = 0; 7978 ism_map[i].imap_sz_mask = 0; 7979 ism_map[i].imap_ismhat = NULL; 7980 ism_map[i].imap_ment = NULL; 7981 } 7982 } 7983 7984 /* 7985 * Now flush entire TSB for the process, since 7986 * demapping page by page can be too expensive. 7987 * We don't have to flush the TLB here anymore 7988 * since we switch to a new TLB ctx instead. 7989 * Also, there is no need to flush if the process 7990 * is exiting since the TSB will be freed later. 7991 */ 7992 if (!sfmmup->sfmmu_free) { 7993 hatlockp = sfmmu_hat_enter(sfmmup); 7994 for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL; 7995 tsbinfo = tsbinfo->tsb_next) { 7996 if (tsbinfo->tsb_flags & TSB_SWAPPED) 7997 continue; 7998 sfmmu_inv_tsb(tsbinfo->tsb_va, 7999 TSB_BYTES(tsbinfo->tsb_szc)); 8000 } 8001 sfmmu_hat_exit(hatlockp); 8002 } 8003 } 8004 8005 /* 8006 * Update our counters for this sfmmup's ism mappings. 8007 */ 8008 for (i = 0; i <= ismszc; i++) { 8009 if (!(disable_ism_large_pages & (1 << i))) 8010 (void) ism_tsb_entries(sfmmup, i); 8011 } 8012 8013 sfmmu_ismhat_exit(sfmmup, 0); 8014 8015 /* 8016 * We must do our freeing here after dropping locks 8017 * to prevent a deadlock in the kmem allocator on the 8018 * mapping list lock. 8019 */ 8020 if (free_ment != NULL) 8021 kmem_cache_free(ism_ment_cache, free_ment); 8022 8023 /* 8024 * Check TSB and TLB page sizes if the process isn't exiting. 8025 */ 8026 if (!sfmmup->sfmmu_free) 8027 sfmmu_check_page_sizes(sfmmup, 0); 8028 } 8029 8030 /* ARGSUSED */ 8031 static int 8032 sfmmu_idcache_constructor(void *buf, void *cdrarg, int kmflags) 8033 { 8034 /* void *buf is sfmmu_t pointer */ 8035 return (0); 8036 } 8037 8038 /* ARGSUSED */ 8039 static void 8040 sfmmu_idcache_destructor(void *buf, void *cdrarg) 8041 { 8042 /* void *buf is sfmmu_t pointer */ 8043 } 8044 8045 /* 8046 * setup kmem hmeblks by bzeroing all members and initializing the nextpa 8047 * field to be the pa of this hmeblk 8048 */ 8049 /* ARGSUSED */ 8050 static int 8051 sfmmu_hblkcache_constructor(void *buf, void *cdrarg, int kmflags) 8052 { 8053 struct hme_blk *hmeblkp; 8054 8055 bzero(buf, (size_t)cdrarg); 8056 hmeblkp = (struct hme_blk *)buf; 8057 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); 8058 8059 #ifdef HBLK_TRACE 8060 mutex_init(&hmeblkp->hblk_audit_lock, NULL, MUTEX_DEFAULT, NULL); 8061 #endif /* HBLK_TRACE */ 8062 8063 return (0); 8064 } 8065 8066 /* ARGSUSED */ 8067 static void 8068 sfmmu_hblkcache_destructor(void *buf, void *cdrarg) 8069 { 8070 8071 #ifdef HBLK_TRACE 8072 8073 struct hme_blk *hmeblkp; 8074 8075 hmeblkp = (struct hme_blk *)buf; 8076 mutex_destroy(&hmeblkp->hblk_audit_lock); 8077 8078 #endif /* HBLK_TRACE */ 8079 } 8080 8081 #define SFMMU_CACHE_RECLAIM_SCAN_RATIO 8 8082 static int sfmmu_cache_reclaim_scan_ratio = SFMMU_CACHE_RECLAIM_SCAN_RATIO; 8083 /* 8084 * The kmem allocator will callback into our reclaim routine when the system 8085 * is running low in memory. We traverse the hash and free up all unused but 8086 * still cached hme_blks. We also traverse the free list and free them up 8087 * as well. 8088 */ 8089 /*ARGSUSED*/ 8090 static void 8091 sfmmu_hblkcache_reclaim(void *cdrarg) 8092 { 8093 int i; 8094 uint64_t hblkpa, prevpa, nx_pa; 8095 struct hmehash_bucket *hmebp; 8096 struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL; 8097 static struct hmehash_bucket *uhmehash_reclaim_hand; 8098 static struct hmehash_bucket *khmehash_reclaim_hand; 8099 struct hme_blk *list = NULL; 8100 8101 hmebp = uhmehash_reclaim_hand; 8102 if (hmebp == NULL || hmebp > &uhme_hash[UHMEHASH_SZ]) 8103 uhmehash_reclaim_hand = hmebp = uhme_hash; 8104 uhmehash_reclaim_hand += UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8105 8106 for (i = UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8107 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8108 hmeblkp = hmebp->hmeblkp; 8109 hblkpa = hmebp->hmeh_nextpa; 8110 prevpa = 0; 8111 pr_hblk = NULL; 8112 while (hmeblkp) { 8113 nx_hblk = hmeblkp->hblk_next; 8114 nx_pa = hmeblkp->hblk_nextpa; 8115 if (!hmeblkp->hblk_vcnt && 8116 !hmeblkp->hblk_hmecnt) { 8117 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8118 prevpa, pr_hblk); 8119 sfmmu_hblk_free(hmebp, hmeblkp, 8120 hblkpa, &list); 8121 } else { 8122 pr_hblk = hmeblkp; 8123 prevpa = hblkpa; 8124 } 8125 hmeblkp = nx_hblk; 8126 hblkpa = nx_pa; 8127 } 8128 SFMMU_HASH_UNLOCK(hmebp); 8129 } 8130 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 8131 hmebp = uhme_hash; 8132 } 8133 8134 hmebp = khmehash_reclaim_hand; 8135 if (hmebp == NULL || hmebp > &khme_hash[KHMEHASH_SZ]) 8136 khmehash_reclaim_hand = hmebp = khme_hash; 8137 khmehash_reclaim_hand += KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8138 8139 for (i = KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8140 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8141 hmeblkp = hmebp->hmeblkp; 8142 hblkpa = hmebp->hmeh_nextpa; 8143 prevpa = 0; 8144 pr_hblk = NULL; 8145 while (hmeblkp) { 8146 nx_hblk = hmeblkp->hblk_next; 8147 nx_pa = hmeblkp->hblk_nextpa; 8148 if (!hmeblkp->hblk_vcnt && 8149 !hmeblkp->hblk_hmecnt) { 8150 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8151 prevpa, pr_hblk); 8152 sfmmu_hblk_free(hmebp, hmeblkp, 8153 hblkpa, &list); 8154 } else { 8155 pr_hblk = hmeblkp; 8156 prevpa = hblkpa; 8157 } 8158 hmeblkp = nx_hblk; 8159 hblkpa = nx_pa; 8160 } 8161 SFMMU_HASH_UNLOCK(hmebp); 8162 } 8163 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 8164 hmebp = khme_hash; 8165 } 8166 sfmmu_hblks_list_purge(&list); 8167 } 8168 8169 /* 8170 * sfmmu_get_ppvcolor should become a vm_machdep or hatop interface. 8171 * same goes for sfmmu_get_addrvcolor(). 8172 * 8173 * This function will return the virtual color for the specified page. The 8174 * virtual color corresponds to this page current mapping or its last mapping. 8175 * It is used by memory allocators to choose addresses with the correct 8176 * alignment so vac consistency is automatically maintained. If the page 8177 * has no color it returns -1. 8178 */ 8179 /*ARGSUSED*/ 8180 int 8181 sfmmu_get_ppvcolor(struct page *pp) 8182 { 8183 #ifdef VAC 8184 int color; 8185 8186 if (!(cache & CACHE_VAC) || PP_NEWPAGE(pp)) { 8187 return (-1); 8188 } 8189 color = PP_GET_VCOLOR(pp); 8190 ASSERT(color < mmu_btop(shm_alignment)); 8191 return (color); 8192 #else 8193 return (-1); 8194 #endif /* VAC */ 8195 } 8196 8197 /* 8198 * This function will return the desired alignment for vac consistency 8199 * (vac color) given a virtual address. If no vac is present it returns -1. 8200 */ 8201 /*ARGSUSED*/ 8202 int 8203 sfmmu_get_addrvcolor(caddr_t vaddr) 8204 { 8205 #ifdef VAC 8206 if (cache & CACHE_VAC) { 8207 return (addr_to_vcolor(vaddr)); 8208 } else { 8209 return (-1); 8210 } 8211 #else 8212 return (-1); 8213 #endif /* VAC */ 8214 } 8215 8216 #ifdef VAC 8217 /* 8218 * Check for conflicts. 8219 * A conflict exists if the new and existent mappings do not match in 8220 * their "shm_alignment fields. If conflicts exist, the existant mappings 8221 * are flushed unless one of them is locked. If one of them is locked, then 8222 * the mappings are flushed and converted to non-cacheable mappings. 8223 */ 8224 static void 8225 sfmmu_vac_conflict(struct hat *hat, caddr_t addr, page_t *pp) 8226 { 8227 struct hat *tmphat; 8228 struct sf_hment *sfhmep, *tmphme = NULL; 8229 struct hme_blk *hmeblkp; 8230 int vcolor; 8231 tte_t tte; 8232 8233 ASSERT(sfmmu_mlist_held(pp)); 8234 ASSERT(!PP_ISNC(pp)); /* page better be cacheable */ 8235 8236 vcolor = addr_to_vcolor(addr); 8237 if (PP_NEWPAGE(pp)) { 8238 PP_SET_VCOLOR(pp, vcolor); 8239 return; 8240 } 8241 8242 if (PP_GET_VCOLOR(pp) == vcolor) { 8243 return; 8244 } 8245 8246 if (!PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp)) { 8247 /* 8248 * Previous user of page had a different color 8249 * but since there are no current users 8250 * we just flush the cache and change the color. 8251 */ 8252 SFMMU_STAT(sf_pgcolor_conflict); 8253 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8254 PP_SET_VCOLOR(pp, vcolor); 8255 return; 8256 } 8257 8258 /* 8259 * If we get here we have a vac conflict with a current 8260 * mapping. VAC conflict policy is as follows. 8261 * - The default is to unload the other mappings unless: 8262 * - If we have a large mapping we uncache the page. 8263 * We need to uncache the rest of the large page too. 8264 * - If any of the mappings are locked we uncache the page. 8265 * - If the requested mapping is inconsistent 8266 * with another mapping and that mapping 8267 * is in the same address space we have to 8268 * make it non-cached. The default thing 8269 * to do is unload the inconsistent mapping 8270 * but if they are in the same address space 8271 * we run the risk of unmapping the pc or the 8272 * stack which we will use as we return to the user, 8273 * in which case we can then fault on the thing 8274 * we just unloaded and get into an infinite loop. 8275 */ 8276 if (PP_ISMAPPED_LARGE(pp)) { 8277 int sz; 8278 8279 /* 8280 * Existing mapping is for big pages. We don't unload 8281 * existing big mappings to satisfy new mappings. 8282 * Always convert all mappings to TNC. 8283 */ 8284 sz = fnd_mapping_sz(pp); 8285 pp = PP_GROUPLEADER(pp, sz); 8286 SFMMU_STAT_ADD(sf_uncache_conflict, TTEPAGES(sz)); 8287 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 8288 TTEPAGES(sz)); 8289 8290 return; 8291 } 8292 8293 /* 8294 * check if any mapping is in same as or if it is locked 8295 * since in that case we need to uncache. 8296 */ 8297 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8298 tmphme = sfhmep->hme_next; 8299 hmeblkp = sfmmu_hmetohblk(sfhmep); 8300 if (hmeblkp->hblk_xhat_bit) 8301 continue; 8302 tmphat = hblktosfmmu(hmeblkp); 8303 sfmmu_copytte(&sfhmep->hme_tte, &tte); 8304 ASSERT(TTE_IS_VALID(&tte)); 8305 if ((tmphat == hat) || hmeblkp->hblk_lckcnt) { 8306 /* 8307 * We have an uncache conflict 8308 */ 8309 SFMMU_STAT(sf_uncache_conflict); 8310 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1); 8311 return; 8312 } 8313 } 8314 8315 /* 8316 * We have an unload conflict 8317 * We have already checked for LARGE mappings, therefore 8318 * the remaining mapping(s) must be TTE8K. 8319 */ 8320 SFMMU_STAT(sf_unload_conflict); 8321 8322 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8323 tmphme = sfhmep->hme_next; 8324 hmeblkp = sfmmu_hmetohblk(sfhmep); 8325 if (hmeblkp->hblk_xhat_bit) 8326 continue; 8327 (void) sfmmu_pageunload(pp, sfhmep, TTE8K); 8328 } 8329 8330 if (PP_ISMAPPED_KPM(pp)) 8331 sfmmu_kpm_vac_unload(pp, addr); 8332 8333 /* 8334 * Unloads only do TLB flushes so we need to flush the 8335 * cache here. 8336 */ 8337 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8338 PP_SET_VCOLOR(pp, vcolor); 8339 } 8340 8341 /* 8342 * Whenever a mapping is unloaded and the page is in TNC state, 8343 * we see if the page can be made cacheable again. 'pp' is 8344 * the page that we just unloaded a mapping from, the size 8345 * of mapping that was unloaded is 'ottesz'. 8346 * Remark: 8347 * The recache policy for mpss pages can leave a performance problem 8348 * under the following circumstances: 8349 * . A large page in uncached mode has just been unmapped. 8350 * . All constituent pages are TNC due to a conflicting small mapping. 8351 * . There are many other, non conflicting, small mappings around for 8352 * a lot of the constituent pages. 8353 * . We're called w/ the "old" groupleader page and the old ottesz, 8354 * but this is irrelevant, since we're no more "PP_ISMAPPED_LARGE", so 8355 * we end up w/ TTE8K or npages == 1. 8356 * . We call tst_tnc w/ the old groupleader only, and if there is no 8357 * conflict, we re-cache only this page. 8358 * . All other small mappings are not checked and will be left in TNC mode. 8359 * The problem is not very serious because: 8360 * . mpss is actually only defined for heap and stack, so the probability 8361 * is not very high that a large page mapping exists in parallel to a small 8362 * one (this is possible, but seems to be bad programming style in the 8363 * appl). 8364 * . The problem gets a little bit more serious, when those TNC pages 8365 * have to be mapped into kernel space, e.g. for networking. 8366 * . When VAC alias conflicts occur in applications, this is regarded 8367 * as an application bug. So if kstat's show them, the appl should 8368 * be changed anyway. 8369 */ 8370 void 8371 conv_tnc(page_t *pp, int ottesz) 8372 { 8373 int cursz, dosz; 8374 pgcnt_t curnpgs, dopgs; 8375 pgcnt_t pg64k; 8376 page_t *pp2; 8377 8378 /* 8379 * Determine how big a range we check for TNC and find 8380 * leader page. cursz is the size of the biggest 8381 * mapping that still exist on 'pp'. 8382 */ 8383 if (PP_ISMAPPED_LARGE(pp)) { 8384 cursz = fnd_mapping_sz(pp); 8385 } else { 8386 cursz = TTE8K; 8387 } 8388 8389 if (ottesz >= cursz) { 8390 dosz = ottesz; 8391 pp2 = pp; 8392 } else { 8393 dosz = cursz; 8394 pp2 = PP_GROUPLEADER(pp, dosz); 8395 } 8396 8397 pg64k = TTEPAGES(TTE64K); 8398 dopgs = TTEPAGES(dosz); 8399 8400 ASSERT(dopgs == 1 || ((dopgs & (pg64k - 1)) == 0)); 8401 8402 while (dopgs != 0) { 8403 curnpgs = TTEPAGES(cursz); 8404 if (tst_tnc(pp2, curnpgs)) { 8405 SFMMU_STAT_ADD(sf_recache, curnpgs); 8406 sfmmu_page_cache_array(pp2, HAT_CACHE, CACHE_NO_FLUSH, 8407 curnpgs); 8408 } 8409 8410 ASSERT(dopgs >= curnpgs); 8411 dopgs -= curnpgs; 8412 8413 if (dopgs == 0) { 8414 break; 8415 } 8416 8417 pp2 = PP_PAGENEXT_N(pp2, curnpgs); 8418 if (((dopgs & (pg64k - 1)) == 0) && PP_ISMAPPED_LARGE(pp2)) { 8419 cursz = fnd_mapping_sz(pp2); 8420 } else { 8421 cursz = TTE8K; 8422 } 8423 } 8424 } 8425 8426 /* 8427 * Returns 1 if page(s) can be converted from TNC to cacheable setting, 8428 * returns 0 otherwise. Note that oaddr argument is valid for only 8429 * 8k pages. 8430 */ 8431 int 8432 tst_tnc(page_t *pp, pgcnt_t npages) 8433 { 8434 struct sf_hment *sfhme; 8435 struct hme_blk *hmeblkp; 8436 tte_t tte; 8437 caddr_t vaddr; 8438 int clr_valid = 0; 8439 int color, color1, bcolor; 8440 int i, ncolors; 8441 8442 ASSERT(pp != NULL); 8443 ASSERT(!(cache & CACHE_WRITEBACK)); 8444 8445 if (npages > 1) { 8446 ncolors = CACHE_NUM_COLOR; 8447 } 8448 8449 for (i = 0; i < npages; i++) { 8450 ASSERT(sfmmu_mlist_held(pp)); 8451 ASSERT(PP_ISTNC(pp)); 8452 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 8453 8454 if (PP_ISPNC(pp)) { 8455 return (0); 8456 } 8457 8458 clr_valid = 0; 8459 if (PP_ISMAPPED_KPM(pp)) { 8460 caddr_t kpmvaddr; 8461 8462 ASSERT(kpm_enable); 8463 kpmvaddr = hat_kpm_page2va(pp, 1); 8464 ASSERT(!(npages > 1 && IS_KPM_ALIAS_RANGE(kpmvaddr))); 8465 color1 = addr_to_vcolor(kpmvaddr); 8466 clr_valid = 1; 8467 } 8468 8469 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8470 hmeblkp = sfmmu_hmetohblk(sfhme); 8471 if (hmeblkp->hblk_xhat_bit) 8472 continue; 8473 8474 sfmmu_copytte(&sfhme->hme_tte, &tte); 8475 ASSERT(TTE_IS_VALID(&tte)); 8476 8477 vaddr = tte_to_vaddr(hmeblkp, tte); 8478 color = addr_to_vcolor(vaddr); 8479 8480 if (npages > 1) { 8481 /* 8482 * If there is a big mapping, make sure 8483 * 8K mapping is consistent with the big 8484 * mapping. 8485 */ 8486 bcolor = i % ncolors; 8487 if (color != bcolor) { 8488 return (0); 8489 } 8490 } 8491 if (!clr_valid) { 8492 clr_valid = 1; 8493 color1 = color; 8494 } 8495 8496 if (color1 != color) { 8497 return (0); 8498 } 8499 } 8500 8501 pp = PP_PAGENEXT(pp); 8502 } 8503 8504 return (1); 8505 } 8506 8507 void 8508 sfmmu_page_cache_array(page_t *pp, int flags, int cache_flush_flag, 8509 pgcnt_t npages) 8510 { 8511 kmutex_t *pmtx; 8512 int i, ncolors, bcolor; 8513 kpm_hlk_t *kpmp; 8514 cpuset_t cpuset; 8515 8516 ASSERT(pp != NULL); 8517 ASSERT(!(cache & CACHE_WRITEBACK)); 8518 8519 kpmp = sfmmu_kpm_kpmp_enter(pp, npages); 8520 pmtx = sfmmu_page_enter(pp); 8521 8522 /* 8523 * Fast path caching single unmapped page 8524 */ 8525 if (npages == 1 && !PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp) && 8526 flags == HAT_CACHE) { 8527 PP_CLRTNC(pp); 8528 PP_CLRPNC(pp); 8529 sfmmu_page_exit(pmtx); 8530 sfmmu_kpm_kpmp_exit(kpmp); 8531 return; 8532 } 8533 8534 /* 8535 * We need to capture all cpus in order to change cacheability 8536 * because we can't allow one cpu to access the same physical 8537 * page using a cacheable and a non-cachebale mapping at the same 8538 * time. Since we may end up walking the ism mapping list 8539 * have to grab it's lock now since we can't after all the 8540 * cpus have been captured. 8541 */ 8542 sfmmu_hat_lock_all(); 8543 mutex_enter(&ism_mlist_lock); 8544 kpreempt_disable(); 8545 cpuset = cpu_ready_set; 8546 xc_attention(cpuset); 8547 8548 if (npages > 1) { 8549 /* 8550 * Make sure all colors are flushed since the 8551 * sfmmu_page_cache() only flushes one color- 8552 * it does not know big pages. 8553 */ 8554 ncolors = CACHE_NUM_COLOR; 8555 if (flags & HAT_TMPNC) { 8556 for (i = 0; i < ncolors; i++) { 8557 sfmmu_cache_flushcolor(i, pp->p_pagenum); 8558 } 8559 cache_flush_flag = CACHE_NO_FLUSH; 8560 } 8561 } 8562 8563 for (i = 0; i < npages; i++) { 8564 8565 ASSERT(sfmmu_mlist_held(pp)); 8566 8567 if (!(flags == HAT_TMPNC && PP_ISTNC(pp))) { 8568 8569 if (npages > 1) { 8570 bcolor = i % ncolors; 8571 } else { 8572 bcolor = NO_VCOLOR; 8573 } 8574 8575 sfmmu_page_cache(pp, flags, cache_flush_flag, 8576 bcolor); 8577 } 8578 8579 pp = PP_PAGENEXT(pp); 8580 } 8581 8582 xt_sync(cpuset); 8583 xc_dismissed(cpuset); 8584 mutex_exit(&ism_mlist_lock); 8585 sfmmu_hat_unlock_all(); 8586 sfmmu_page_exit(pmtx); 8587 sfmmu_kpm_kpmp_exit(kpmp); 8588 kpreempt_enable(); 8589 } 8590 8591 /* 8592 * This function changes the virtual cacheability of all mappings to a 8593 * particular page. When changing from uncache to cacheable the mappings will 8594 * only be changed if all of them have the same virtual color. 8595 * We need to flush the cache in all cpus. It is possible that 8596 * a process referenced a page as cacheable but has sinced exited 8597 * and cleared the mapping list. We still to flush it but have no 8598 * state so all cpus is the only alternative. 8599 */ 8600 static void 8601 sfmmu_page_cache(page_t *pp, int flags, int cache_flush_flag, int bcolor) 8602 { 8603 struct sf_hment *sfhme; 8604 struct hme_blk *hmeblkp; 8605 sfmmu_t *sfmmup; 8606 tte_t tte, ttemod; 8607 caddr_t vaddr; 8608 int ret, color; 8609 pfn_t pfn; 8610 8611 color = bcolor; 8612 pfn = pp->p_pagenum; 8613 8614 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8615 8616 hmeblkp = sfmmu_hmetohblk(sfhme); 8617 8618 if (hmeblkp->hblk_xhat_bit) 8619 continue; 8620 8621 sfmmu_copytte(&sfhme->hme_tte, &tte); 8622 ASSERT(TTE_IS_VALID(&tte)); 8623 vaddr = tte_to_vaddr(hmeblkp, tte); 8624 color = addr_to_vcolor(vaddr); 8625 8626 #ifdef DEBUG 8627 if ((flags & HAT_CACHE) && bcolor != NO_VCOLOR) { 8628 ASSERT(color == bcolor); 8629 } 8630 #endif 8631 8632 ASSERT(flags != HAT_TMPNC || color == PP_GET_VCOLOR(pp)); 8633 8634 ttemod = tte; 8635 if (flags & (HAT_UNCACHE | HAT_TMPNC)) { 8636 TTE_CLR_VCACHEABLE(&ttemod); 8637 } else { /* flags & HAT_CACHE */ 8638 TTE_SET_VCACHEABLE(&ttemod); 8639 } 8640 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 8641 if (ret < 0) { 8642 /* 8643 * Since all cpus are captured modifytte should not 8644 * fail. 8645 */ 8646 panic("sfmmu_page_cache: write to tte failed"); 8647 } 8648 8649 sfmmup = hblktosfmmu(hmeblkp); 8650 if (cache_flush_flag == CACHE_FLUSH) { 8651 /* 8652 * Flush TSBs, TLBs and caches 8653 */ 8654 if (sfmmup->sfmmu_ismhat) { 8655 if (flags & HAT_CACHE) { 8656 SFMMU_STAT(sf_ism_recache); 8657 } else { 8658 SFMMU_STAT(sf_ism_uncache); 8659 } 8660 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8661 pfn, CACHE_FLUSH); 8662 } else { 8663 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp, 8664 pfn, 0, FLUSH_ALL_CPUS, CACHE_FLUSH, 1); 8665 } 8666 8667 /* 8668 * all cache entries belonging to this pfn are 8669 * now flushed. 8670 */ 8671 cache_flush_flag = CACHE_NO_FLUSH; 8672 } else { 8673 8674 /* 8675 * Flush only TSBs and TLBs. 8676 */ 8677 if (sfmmup->sfmmu_ismhat) { 8678 if (flags & HAT_CACHE) { 8679 SFMMU_STAT(sf_ism_recache); 8680 } else { 8681 SFMMU_STAT(sf_ism_uncache); 8682 } 8683 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8684 pfn, CACHE_NO_FLUSH); 8685 } else { 8686 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1); 8687 } 8688 } 8689 } 8690 8691 if (PP_ISMAPPED_KPM(pp)) 8692 sfmmu_kpm_page_cache(pp, flags, cache_flush_flag); 8693 8694 switch (flags) { 8695 8696 default: 8697 panic("sfmmu_pagecache: unknown flags"); 8698 break; 8699 8700 case HAT_CACHE: 8701 PP_CLRTNC(pp); 8702 PP_CLRPNC(pp); 8703 PP_SET_VCOLOR(pp, color); 8704 break; 8705 8706 case HAT_TMPNC: 8707 PP_SETTNC(pp); 8708 PP_SET_VCOLOR(pp, NO_VCOLOR); 8709 break; 8710 8711 case HAT_UNCACHE: 8712 PP_SETPNC(pp); 8713 PP_CLRTNC(pp); 8714 PP_SET_VCOLOR(pp, NO_VCOLOR); 8715 break; 8716 } 8717 } 8718 #endif /* VAC */ 8719 8720 8721 /* 8722 * Wrapper routine used to return a context. 8723 * 8724 * It's the responsibility of the caller to guarantee that the 8725 * process serializes on calls here by taking the HAT lock for 8726 * the hat. 8727 * 8728 */ 8729 static void 8730 sfmmu_get_ctx(sfmmu_t *sfmmup) 8731 { 8732 mmu_ctx_t *mmu_ctxp; 8733 uint_t pstate_save; 8734 8735 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8736 ASSERT(sfmmup != ksfmmup); 8737 8738 kpreempt_disable(); 8739 8740 mmu_ctxp = CPU_MMU_CTXP(CPU); 8741 ASSERT(mmu_ctxp); 8742 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 8743 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 8744 8745 /* 8746 * Do a wrap-around if cnum reaches the max # cnum supported by a MMU. 8747 */ 8748 if (mmu_ctxp->mmu_cnum == mmu_ctxp->mmu_nctxs) 8749 sfmmu_ctx_wrap_around(mmu_ctxp); 8750 8751 /* 8752 * Let the MMU set up the page sizes to use for 8753 * this context in the TLB. Don't program 2nd dtlb for ism hat. 8754 */ 8755 if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) { 8756 mmu_set_ctx_page_sizes(sfmmup); 8757 } 8758 8759 /* 8760 * sfmmu_alloc_ctx and sfmmu_load_mmustate will be performed with 8761 * interrupts disabled to prevent race condition with wrap-around 8762 * ctx invalidatation. In sun4v, ctx invalidation also involves 8763 * a HV call to set the number of TSBs to 0. If interrupts are not 8764 * disabled until after sfmmu_load_mmustate is complete TSBs may 8765 * become assigned to INVALID_CONTEXT. This is not allowed. 8766 */ 8767 pstate_save = sfmmu_disable_intrs(); 8768 8769 sfmmu_alloc_ctx(sfmmup, 1, CPU); 8770 sfmmu_load_mmustate(sfmmup); 8771 8772 sfmmu_enable_intrs(pstate_save); 8773 8774 kpreempt_enable(); 8775 } 8776 8777 /* 8778 * When all cnums are used up in a MMU, cnum will wrap around to the 8779 * next generation and start from 2. 8780 */ 8781 static void 8782 sfmmu_ctx_wrap_around(mmu_ctx_t *mmu_ctxp) 8783 { 8784 8785 /* caller must have disabled the preemption */ 8786 ASSERT(curthread->t_preempt >= 1); 8787 ASSERT(mmu_ctxp != NULL); 8788 8789 /* acquire Per-MMU (PM) spin lock */ 8790 mutex_enter(&mmu_ctxp->mmu_lock); 8791 8792 /* re-check to see if wrap-around is needed */ 8793 if (mmu_ctxp->mmu_cnum < mmu_ctxp->mmu_nctxs) 8794 goto done; 8795 8796 SFMMU_MMU_STAT(mmu_wrap_around); 8797 8798 /* update gnum */ 8799 ASSERT(mmu_ctxp->mmu_gnum != 0); 8800 mmu_ctxp->mmu_gnum++; 8801 if (mmu_ctxp->mmu_gnum == 0 || 8802 mmu_ctxp->mmu_gnum > MAX_SFMMU_GNUM_VAL) { 8803 cmn_err(CE_PANIC, "mmu_gnum of mmu_ctx 0x%p is out of bound.", 8804 (void *)mmu_ctxp); 8805 } 8806 8807 if (mmu_ctxp->mmu_ncpus > 1) { 8808 cpuset_t cpuset; 8809 8810 membar_enter(); /* make sure updated gnum visible */ 8811 8812 SFMMU_XCALL_STATS(NULL); 8813 8814 /* xcall to others on the same MMU to invalidate ctx */ 8815 cpuset = mmu_ctxp->mmu_cpuset; 8816 ASSERT(CPU_IN_SET(cpuset, CPU->cpu_id)); 8817 CPUSET_DEL(cpuset, CPU->cpu_id); 8818 CPUSET_AND(cpuset, cpu_ready_set); 8819 8820 /* 8821 * Pass in INVALID_CONTEXT as the first parameter to 8822 * sfmmu_raise_tsb_exception, which invalidates the context 8823 * of any process running on the CPUs in the MMU. 8824 */ 8825 xt_some(cpuset, sfmmu_raise_tsb_exception, 8826 INVALID_CONTEXT, INVALID_CONTEXT); 8827 xt_sync(cpuset); 8828 8829 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 8830 } 8831 8832 if (sfmmu_getctx_sec() != INVALID_CONTEXT) { 8833 sfmmu_setctx_sec(INVALID_CONTEXT); 8834 sfmmu_clear_utsbinfo(); 8835 } 8836 8837 /* 8838 * No xcall is needed here. For sun4u systems all CPUs in context 8839 * domain share a single physical MMU therefore it's enough to flush 8840 * TLB on local CPU. On sun4v systems we use 1 global context 8841 * domain and flush all remote TLBs in sfmmu_raise_tsb_exception 8842 * handler. Note that vtag_flushall_uctxs() is called 8843 * for Ultra II machine, where the equivalent flushall functionality 8844 * is implemented in SW, and only user ctx TLB entries are flushed. 8845 */ 8846 if (&vtag_flushall_uctxs != NULL) { 8847 vtag_flushall_uctxs(); 8848 } else { 8849 vtag_flushall(); 8850 } 8851 8852 /* reset mmu cnum, skips cnum 0 and 1 */ 8853 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 8854 8855 done: 8856 mutex_exit(&mmu_ctxp->mmu_lock); 8857 } 8858 8859 8860 /* 8861 * For multi-threaded process, set the process context to INVALID_CONTEXT 8862 * so that it faults and reloads the MMU state from TL=0. For single-threaded 8863 * process, we can just load the MMU state directly without having to 8864 * set context invalid. Caller must hold the hat lock since we don't 8865 * acquire it here. 8866 */ 8867 static void 8868 sfmmu_sync_mmustate(sfmmu_t *sfmmup) 8869 { 8870 uint_t cnum; 8871 uint_t pstate_save; 8872 8873 ASSERT(sfmmup != ksfmmup); 8874 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8875 8876 kpreempt_disable(); 8877 8878 /* 8879 * We check whether the pass'ed-in sfmmup is the same as the 8880 * current running proc. This is to makes sure the current proc 8881 * stays single-threaded if it already is. 8882 */ 8883 if ((sfmmup == curthread->t_procp->p_as->a_hat) && 8884 (curthread->t_procp->p_lwpcnt == 1)) { 8885 /* single-thread */ 8886 cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum; 8887 if (cnum != INVALID_CONTEXT) { 8888 uint_t curcnum; 8889 /* 8890 * Disable interrupts to prevent race condition 8891 * with sfmmu_ctx_wrap_around ctx invalidation. 8892 * In sun4v, ctx invalidation involves setting 8893 * TSB to NULL, hence, interrupts should be disabled 8894 * untill after sfmmu_load_mmustate is completed. 8895 */ 8896 pstate_save = sfmmu_disable_intrs(); 8897 curcnum = sfmmu_getctx_sec(); 8898 if (curcnum == cnum) 8899 sfmmu_load_mmustate(sfmmup); 8900 sfmmu_enable_intrs(pstate_save); 8901 ASSERT(curcnum == cnum || curcnum == INVALID_CONTEXT); 8902 } 8903 } else { 8904 /* 8905 * multi-thread 8906 * or when sfmmup is not the same as the curproc. 8907 */ 8908 sfmmu_invalidate_ctx(sfmmup); 8909 } 8910 8911 kpreempt_enable(); 8912 } 8913 8914 8915 /* 8916 * Replace the specified TSB with a new TSB. This function gets called when 8917 * we grow, shrink or swapin a TSB. When swapping in a TSB (TSB_SWAPIN), the 8918 * TSB_FORCEALLOC flag may be used to force allocation of a minimum-sized TSB 8919 * (8K). 8920 * 8921 * Caller must hold the HAT lock, but should assume any tsb_info 8922 * pointers it has are no longer valid after calling this function. 8923 * 8924 * Return values: 8925 * TSB_ALLOCFAIL Failed to allocate a TSB, due to memory constraints 8926 * TSB_LOSTRACE HAT is busy, i.e. another thread is already doing 8927 * something to this tsbinfo/TSB 8928 * TSB_SUCCESS Operation succeeded 8929 */ 8930 static tsb_replace_rc_t 8931 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, 8932 hatlock_t *hatlockp, uint_t flags) 8933 { 8934 struct tsb_info *new_tsbinfo = NULL; 8935 struct tsb_info *curtsb, *prevtsb; 8936 uint_t tte_sz_mask; 8937 int i; 8938 8939 ASSERT(sfmmup != ksfmmup); 8940 ASSERT(sfmmup->sfmmu_ismhat == 0); 8941 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8942 ASSERT(szc <= tsb_max_growsize); 8943 8944 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY)) 8945 return (TSB_LOSTRACE); 8946 8947 /* 8948 * Find the tsb_info ahead of this one in the list, and 8949 * also make sure that the tsb_info passed in really 8950 * exists! 8951 */ 8952 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 8953 curtsb != old_tsbinfo && curtsb != NULL; 8954 prevtsb = curtsb, curtsb = curtsb->tsb_next); 8955 ASSERT(curtsb != NULL); 8956 8957 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8958 /* 8959 * The process is swapped out, so just set the new size 8960 * code. When it swaps back in, we'll allocate a new one 8961 * of the new chosen size. 8962 */ 8963 curtsb->tsb_szc = szc; 8964 return (TSB_SUCCESS); 8965 } 8966 SFMMU_FLAGS_SET(sfmmup, HAT_BUSY); 8967 8968 tte_sz_mask = old_tsbinfo->tsb_ttesz_mask; 8969 8970 /* 8971 * All initialization is done inside of sfmmu_tsbinfo_alloc(). 8972 * If we fail to allocate a TSB, exit. 8973 */ 8974 sfmmu_hat_exit(hatlockp); 8975 if (sfmmu_tsbinfo_alloc(&new_tsbinfo, szc, tte_sz_mask, 8976 flags, sfmmup)) { 8977 (void) sfmmu_hat_enter(sfmmup); 8978 if (!(flags & TSB_SWAPIN)) 8979 SFMMU_STAT(sf_tsb_resize_failures); 8980 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8981 return (TSB_ALLOCFAIL); 8982 } 8983 (void) sfmmu_hat_enter(sfmmup); 8984 8985 /* 8986 * Re-check to make sure somebody else didn't muck with us while we 8987 * didn't hold the HAT lock. If the process swapped out, fine, just 8988 * exit; this can happen if we try to shrink the TSB from the context 8989 * of another process (such as on an ISM unmap), though it is rare. 8990 */ 8991 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8992 SFMMU_STAT(sf_tsb_resize_failures); 8993 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8994 sfmmu_hat_exit(hatlockp); 8995 sfmmu_tsbinfo_free(new_tsbinfo); 8996 (void) sfmmu_hat_enter(sfmmup); 8997 return (TSB_LOSTRACE); 8998 } 8999 9000 #ifdef DEBUG 9001 /* Reverify that the tsb_info still exists.. for debugging only */ 9002 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 9003 curtsb != old_tsbinfo && curtsb != NULL; 9004 prevtsb = curtsb, curtsb = curtsb->tsb_next); 9005 ASSERT(curtsb != NULL); 9006 #endif /* DEBUG */ 9007 9008 /* 9009 * Quiesce any CPUs running this process on their next TLB miss 9010 * so they atomically see the new tsb_info. We temporarily set the 9011 * context to invalid context so new threads that come on processor 9012 * after we do the xcall to cpusran will also serialize behind the 9013 * HAT lock on TLB miss and will see the new TSB. Since this short 9014 * race with a new thread coming on processor is relatively rare, 9015 * this synchronization mechanism should be cheaper than always 9016 * pausing all CPUs for the duration of the setup, which is what 9017 * the old implementation did. This is particuarly true if we are 9018 * copying a huge chunk of memory around during that window. 9019 * 9020 * The memory barriers are to make sure things stay consistent 9021 * with resume() since it does not hold the HAT lock while 9022 * walking the list of tsb_info structures. 9023 */ 9024 if ((flags & TSB_SWAPIN) != TSB_SWAPIN) { 9025 /* The TSB is either growing or shrinking. */ 9026 sfmmu_invalidate_ctx(sfmmup); 9027 } else { 9028 /* 9029 * It is illegal to swap in TSBs from a process other 9030 * than a process being swapped in. This in turn 9031 * implies we do not have a valid MMU context here 9032 * since a process needs one to resolve translation 9033 * misses. 9034 */ 9035 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup); 9036 } 9037 9038 #ifdef DEBUG 9039 ASSERT(max_mmu_ctxdoms > 0); 9040 9041 /* 9042 * Process should have INVALID_CONTEXT on all MMUs 9043 */ 9044 for (i = 0; i < max_mmu_ctxdoms; i++) { 9045 9046 ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); 9047 } 9048 #endif 9049 9050 new_tsbinfo->tsb_next = old_tsbinfo->tsb_next; 9051 membar_stst(); /* strict ordering required */ 9052 if (prevtsb) 9053 prevtsb->tsb_next = new_tsbinfo; 9054 else 9055 sfmmup->sfmmu_tsb = new_tsbinfo; 9056 membar_enter(); /* make sure new TSB globally visible */ 9057 sfmmu_setup_tsbinfo(sfmmup); 9058 9059 /* 9060 * We need to migrate TSB entries from the old TSB to the new TSB 9061 * if tsb_remap_ttes is set and the TSB is growing. 9062 */ 9063 if (tsb_remap_ttes && ((flags & TSB_GROW) == TSB_GROW)) 9064 sfmmu_copy_tsb(old_tsbinfo, new_tsbinfo); 9065 9066 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 9067 9068 /* 9069 * Drop the HAT lock to free our old tsb_info. 9070 */ 9071 sfmmu_hat_exit(hatlockp); 9072 9073 if ((flags & TSB_GROW) == TSB_GROW) { 9074 SFMMU_STAT(sf_tsb_grow); 9075 } else if ((flags & TSB_SHRINK) == TSB_SHRINK) { 9076 SFMMU_STAT(sf_tsb_shrink); 9077 } 9078 9079 sfmmu_tsbinfo_free(old_tsbinfo); 9080 9081 (void) sfmmu_hat_enter(sfmmup); 9082 return (TSB_SUCCESS); 9083 } 9084 9085 /* 9086 * This function will re-program hat pgsz array, and invalidate the 9087 * process' context, forcing the process to switch to another 9088 * context on the next TLB miss, and therefore start using the 9089 * TLB that is reprogrammed for the new page sizes. 9090 */ 9091 void 9092 sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) 9093 { 9094 int i; 9095 hatlock_t *hatlockp = NULL; 9096 9097 hatlockp = sfmmu_hat_enter(sfmmup); 9098 /* USIII+-IV+ optimization, requires hat lock */ 9099 if (tmp_pgsz) { 9100 for (i = 0; i < mmu_page_sizes; i++) 9101 sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i]; 9102 } 9103 SFMMU_STAT(sf_tlb_reprog_pgsz); 9104 9105 sfmmu_invalidate_ctx(sfmmup); 9106 9107 sfmmu_hat_exit(hatlockp); 9108 } 9109 9110 /* 9111 * This function assumes that there are either four or six supported page 9112 * sizes and at most two programmable TLBs, so we need to decide which 9113 * page sizes are most important and then tell the MMU layer so it 9114 * can adjust the TLB page sizes accordingly (if supported). 9115 * 9116 * If these assumptions change, this function will need to be 9117 * updated to support whatever the new limits are. 9118 * 9119 * The growing flag is nonzero if we are growing the address space, 9120 * and zero if it is shrinking. This allows us to decide whether 9121 * to grow or shrink our TSB, depending upon available memory 9122 * conditions. 9123 */ 9124 static void 9125 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing) 9126 { 9127 uint64_t ttecnt[MMU_PAGE_SIZES]; 9128 uint64_t tte8k_cnt, tte4m_cnt; 9129 uint8_t i; 9130 int sectsb_thresh; 9131 9132 /* 9133 * Kernel threads, processes with small address spaces not using 9134 * large pages, and dummy ISM HATs need not apply. 9135 */ 9136 if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != NULL) 9137 return; 9138 9139 if ((sfmmup->sfmmu_flags & HAT_LGPG_FLAGS) == 0 && 9140 sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor) 9141 return; 9142 9143 for (i = 0; i < mmu_page_sizes; i++) { 9144 ttecnt[i] = SFMMU_TTE_CNT(sfmmup, i); 9145 } 9146 9147 /* Check pagesizes in use, and possibly reprogram DTLB. */ 9148 if (&mmu_check_page_sizes) 9149 mmu_check_page_sizes(sfmmup, ttecnt); 9150 9151 /* 9152 * Calculate the number of 8k ttes to represent the span of these 9153 * pages. 9154 */ 9155 tte8k_cnt = ttecnt[TTE8K] + 9156 (ttecnt[TTE64K] << (MMU_PAGESHIFT64K - MMU_PAGESHIFT)) + 9157 (ttecnt[TTE512K] << (MMU_PAGESHIFT512K - MMU_PAGESHIFT)); 9158 if (mmu_page_sizes == max_mmu_page_sizes) { 9159 tte4m_cnt = ttecnt[TTE4M] + 9160 (ttecnt[TTE32M] << (MMU_PAGESHIFT32M - MMU_PAGESHIFT4M)) + 9161 (ttecnt[TTE256M] << (MMU_PAGESHIFT256M - MMU_PAGESHIFT4M)); 9162 } else { 9163 tte4m_cnt = ttecnt[TTE4M]; 9164 } 9165 9166 /* 9167 * Inflate TSB sizes by a factor of 2 if this process 9168 * uses 4M text pages to minimize extra conflict misses 9169 * in the first TSB since without counting text pages 9170 * 8K TSB may become too small. 9171 * 9172 * Also double the size of the second TSB to minimize 9173 * extra conflict misses due to competition between 4M text pages 9174 * and data pages. 9175 * 9176 * We need to adjust the second TSB allocation threshold by the 9177 * inflation factor, since there is no point in creating a second 9178 * TSB when we know all the mappings can fit in the I/D TLBs. 9179 */ 9180 sectsb_thresh = tsb_sectsb_threshold; 9181 if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) { 9182 tte8k_cnt <<= 1; 9183 tte4m_cnt <<= 1; 9184 sectsb_thresh <<= 1; 9185 } 9186 9187 /* 9188 * Check to see if our TSB is the right size; we may need to 9189 * grow or shrink it. If the process is small, our work is 9190 * finished at this point. 9191 */ 9192 if (tte8k_cnt <= tsb_rss_factor && tte4m_cnt <= sectsb_thresh) { 9193 return; 9194 } 9195 sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh); 9196 } 9197 9198 static void 9199 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt, 9200 uint64_t tte4m_cnt, int sectsb_thresh) 9201 { 9202 int tsb_bits; 9203 uint_t tsb_szc; 9204 struct tsb_info *tsbinfop; 9205 hatlock_t *hatlockp = NULL; 9206 9207 hatlockp = sfmmu_hat_enter(sfmmup); 9208 ASSERT(hatlockp != NULL); 9209 tsbinfop = sfmmup->sfmmu_tsb; 9210 ASSERT(tsbinfop != NULL); 9211 9212 /* 9213 * If we're growing, select the size based on RSS. If we're 9214 * shrinking, leave some room so we don't have to turn around and 9215 * grow again immediately. 9216 */ 9217 if (growing) 9218 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt); 9219 else 9220 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt << 1); 9221 9222 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9223 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9224 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9225 hatlockp, TSB_SHRINK); 9226 } else if (growing && tsb_szc > tsbinfop->tsb_szc && TSB_OK_GROW()) { 9227 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9228 hatlockp, TSB_GROW); 9229 } 9230 tsbinfop = sfmmup->sfmmu_tsb; 9231 9232 /* 9233 * With the TLB and first TSB out of the way, we need to see if 9234 * we need a second TSB for 4M pages. If we managed to reprogram 9235 * the TLB page sizes above, the process will start using this new 9236 * TSB right away; otherwise, it will start using it on the next 9237 * context switch. Either way, it's no big deal so there's no 9238 * synchronization with the trap handlers here unless we grow the 9239 * TSB (in which case it's required to prevent using the old one 9240 * after it's freed). Note: second tsb is required for 32M/256M 9241 * page sizes. 9242 */ 9243 if (tte4m_cnt > sectsb_thresh) { 9244 /* 9245 * If we're growing, select the size based on RSS. If we're 9246 * shrinking, leave some room so we don't have to turn 9247 * around and grow again immediately. 9248 */ 9249 if (growing) 9250 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt); 9251 else 9252 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt << 1); 9253 if (tsbinfop->tsb_next == NULL) { 9254 struct tsb_info *newtsb; 9255 int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)? 9256 0 : TSB_ALLOC; 9257 9258 sfmmu_hat_exit(hatlockp); 9259 9260 /* 9261 * Try to allocate a TSB for 4[32|256]M pages. If we 9262 * can't get the size we want, retry w/a minimum sized 9263 * TSB. If that still didn't work, give up; we can 9264 * still run without one. 9265 */ 9266 tsb_bits = (mmu_page_sizes == max_mmu_page_sizes)? 9267 TSB4M|TSB32M|TSB256M:TSB4M; 9268 if ((sfmmu_tsbinfo_alloc(&newtsb, tsb_szc, tsb_bits, 9269 allocflags, sfmmup) != 0) && 9270 (sfmmu_tsbinfo_alloc(&newtsb, TSB_MIN_SZCODE, 9271 tsb_bits, allocflags, sfmmup) != 0)) { 9272 return; 9273 } 9274 9275 hatlockp = sfmmu_hat_enter(sfmmup); 9276 9277 if (sfmmup->sfmmu_tsb->tsb_next == NULL) { 9278 sfmmup->sfmmu_tsb->tsb_next = newtsb; 9279 SFMMU_STAT(sf_tsb_sectsb_create); 9280 sfmmu_setup_tsbinfo(sfmmup); 9281 sfmmu_hat_exit(hatlockp); 9282 return; 9283 } else { 9284 /* 9285 * It's annoying, but possible for us 9286 * to get here.. we dropped the HAT lock 9287 * because of locking order in the kmem 9288 * allocator, and while we were off getting 9289 * our memory, some other thread decided to 9290 * do us a favor and won the race to get a 9291 * second TSB for this process. Sigh. 9292 */ 9293 sfmmu_hat_exit(hatlockp); 9294 sfmmu_tsbinfo_free(newtsb); 9295 return; 9296 } 9297 } 9298 9299 /* 9300 * We have a second TSB, see if it's big enough. 9301 */ 9302 tsbinfop = tsbinfop->tsb_next; 9303 9304 /* 9305 * Check to see if our second TSB is the right size; 9306 * we may need to grow or shrink it. 9307 * To prevent thrashing (e.g. growing the TSB on a 9308 * subsequent map operation), only try to shrink if 9309 * the TSB reach exceeds twice the virtual address 9310 * space size. 9311 */ 9312 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9313 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9314 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9315 tsb_szc, hatlockp, TSB_SHRINK); 9316 } else if (growing && tsb_szc > tsbinfop->tsb_szc && 9317 TSB_OK_GROW()) { 9318 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9319 tsb_szc, hatlockp, TSB_GROW); 9320 } 9321 } 9322 9323 sfmmu_hat_exit(hatlockp); 9324 } 9325 9326 /* 9327 * Free up a sfmmu 9328 * Since the sfmmu is currently embedded in the hat struct we simply zero 9329 * out our fields and free up the ism map blk list if any. 9330 */ 9331 static void 9332 sfmmu_free_sfmmu(sfmmu_t *sfmmup) 9333 { 9334 ism_blk_t *blkp, *nx_blkp; 9335 #ifdef DEBUG 9336 ism_map_t *map; 9337 int i; 9338 #endif 9339 9340 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 9341 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 9342 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 9343 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 9344 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 9345 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 9346 9347 sfmmup->sfmmu_free = 0; 9348 sfmmup->sfmmu_ismhat = 0; 9349 9350 blkp = sfmmup->sfmmu_iblk; 9351 sfmmup->sfmmu_iblk = NULL; 9352 9353 while (blkp) { 9354 #ifdef DEBUG 9355 map = blkp->iblk_maps; 9356 for (i = 0; i < ISM_MAP_SLOTS; i++) { 9357 ASSERT(map[i].imap_seg == 0); 9358 ASSERT(map[i].imap_ismhat == NULL); 9359 ASSERT(map[i].imap_ment == NULL); 9360 } 9361 #endif 9362 nx_blkp = blkp->iblk_next; 9363 blkp->iblk_next = NULL; 9364 blkp->iblk_nextpa = (uint64_t)-1; 9365 kmem_cache_free(ism_blk_cache, blkp); 9366 blkp = nx_blkp; 9367 } 9368 } 9369 9370 /* 9371 * Locking primitves accessed by HATLOCK macros 9372 */ 9373 9374 #define SFMMU_SPL_MTX (0x0) 9375 #define SFMMU_ML_MTX (0x1) 9376 9377 #define SFMMU_MLSPL_MTX(type, pg) (((type) == SFMMU_SPL_MTX) ? \ 9378 SPL_HASH(pg) : MLIST_HASH(pg)) 9379 9380 kmutex_t * 9381 sfmmu_page_enter(struct page *pp) 9382 { 9383 return (sfmmu_mlspl_enter(pp, SFMMU_SPL_MTX)); 9384 } 9385 9386 void 9387 sfmmu_page_exit(kmutex_t *spl) 9388 { 9389 mutex_exit(spl); 9390 } 9391 9392 int 9393 sfmmu_page_spl_held(struct page *pp) 9394 { 9395 return (sfmmu_mlspl_held(pp, SFMMU_SPL_MTX)); 9396 } 9397 9398 kmutex_t * 9399 sfmmu_mlist_enter(struct page *pp) 9400 { 9401 return (sfmmu_mlspl_enter(pp, SFMMU_ML_MTX)); 9402 } 9403 9404 void 9405 sfmmu_mlist_exit(kmutex_t *mml) 9406 { 9407 mutex_exit(mml); 9408 } 9409 9410 int 9411 sfmmu_mlist_held(struct page *pp) 9412 { 9413 9414 return (sfmmu_mlspl_held(pp, SFMMU_ML_MTX)); 9415 } 9416 9417 /* 9418 * Common code for sfmmu_mlist_enter() and sfmmu_page_enter(). For 9419 * sfmmu_mlist_enter() case mml_table lock array is used and for 9420 * sfmmu_page_enter() sfmmu_page_lock lock array is used. 9421 * 9422 * The lock is taken on a root page so that it protects an operation on all 9423 * constituent pages of a large page pp belongs to. 9424 * 9425 * The routine takes a lock from the appropriate array. The lock is determined 9426 * by hashing the root page. After taking the lock this routine checks if the 9427 * root page has the same size code that was used to determine the root (i.e 9428 * that root hasn't changed). If root page has the expected p_szc field we 9429 * have the right lock and it's returned to the caller. If root's p_szc 9430 * decreased we release the lock and retry from the beginning. This case can 9431 * happen due to hat_page_demote() decreasing p_szc between our load of p_szc 9432 * value and taking the lock. The number of retries due to p_szc decrease is 9433 * limited by the maximum p_szc value. If p_szc is 0 we return the lock 9434 * determined by hashing pp itself. 9435 * 9436 * If our caller doesn't hold a SE_SHARED or SE_EXCL lock on pp it's also 9437 * possible that p_szc can increase. To increase p_szc a thread has to lock 9438 * all constituent pages EXCL and do hat_pageunload() on all of them. All the 9439 * callers that don't hold a page locked recheck if hmeblk through which pp 9440 * was found still maps this pp. If it doesn't map it anymore returned lock 9441 * is immediately dropped. Therefore if sfmmu_mlspl_enter() hits the case of 9442 * p_szc increase after taking the lock it returns this lock without further 9443 * retries because in this case the caller doesn't care about which lock was 9444 * taken. The caller will drop it right away. 9445 * 9446 * After the routine returns it's guaranteed that hat_page_demote() can't 9447 * change p_szc field of any of constituent pages of a large page pp belongs 9448 * to as long as pp was either locked at least SHARED prior to this call or 9449 * the caller finds that hment that pointed to this pp still references this 9450 * pp (this also assumes that the caller holds hme hash bucket lock so that 9451 * the same pp can't be remapped into the same hmeblk after it was unmapped by 9452 * hat_pageunload()). 9453 */ 9454 static kmutex_t * 9455 sfmmu_mlspl_enter(struct page *pp, int type) 9456 { 9457 kmutex_t *mtx; 9458 uint_t prev_rszc = UINT_MAX; 9459 page_t *rootpp; 9460 uint_t szc; 9461 uint_t rszc; 9462 uint_t pszc = pp->p_szc; 9463 9464 ASSERT(pp != NULL); 9465 9466 again: 9467 if (pszc == 0) { 9468 mtx = SFMMU_MLSPL_MTX(type, pp); 9469 mutex_enter(mtx); 9470 return (mtx); 9471 } 9472 9473 /* The lock lives in the root page */ 9474 rootpp = PP_GROUPLEADER(pp, pszc); 9475 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9476 mutex_enter(mtx); 9477 9478 /* 9479 * Return mml in the following 3 cases: 9480 * 9481 * 1) If pp itself is root since if its p_szc decreased before we took 9482 * the lock pp is still the root of smaller szc page. And if its p_szc 9483 * increased it doesn't matter what lock we return (see comment in 9484 * front of this routine). 9485 * 9486 * 2) If pp's not root but rootpp is the root of a rootpp->p_szc size 9487 * large page we have the right lock since any previous potential 9488 * hat_page_demote() is done demoting from greater than current root's 9489 * p_szc because hat_page_demote() changes root's p_szc last. No 9490 * further hat_page_demote() can start or be in progress since it 9491 * would need the same lock we currently hold. 9492 * 9493 * 3) If rootpp's p_szc increased since previous iteration it doesn't 9494 * matter what lock we return (see comment in front of this routine). 9495 */ 9496 if (pp == rootpp || (rszc = rootpp->p_szc) == pszc || 9497 rszc >= prev_rszc) { 9498 return (mtx); 9499 } 9500 9501 /* 9502 * hat_page_demote() could have decreased root's p_szc. 9503 * In this case pp's p_szc must also be smaller than pszc. 9504 * Retry. 9505 */ 9506 if (rszc < pszc) { 9507 szc = pp->p_szc; 9508 if (szc < pszc) { 9509 mutex_exit(mtx); 9510 pszc = szc; 9511 goto again; 9512 } 9513 /* 9514 * pp's p_szc increased after it was decreased. 9515 * page cannot be mapped. Return current lock. The caller 9516 * will drop it right away. 9517 */ 9518 return (mtx); 9519 } 9520 9521 /* 9522 * root's p_szc is greater than pp's p_szc. 9523 * hat_page_demote() is not done with all pages 9524 * yet. Wait for it to complete. 9525 */ 9526 mutex_exit(mtx); 9527 rootpp = PP_GROUPLEADER(rootpp, rszc); 9528 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9529 mutex_enter(mtx); 9530 mutex_exit(mtx); 9531 prev_rszc = rszc; 9532 goto again; 9533 } 9534 9535 static int 9536 sfmmu_mlspl_held(struct page *pp, int type) 9537 { 9538 kmutex_t *mtx; 9539 9540 ASSERT(pp != NULL); 9541 /* The lock lives in the root page */ 9542 pp = PP_PAGEROOT(pp); 9543 ASSERT(pp != NULL); 9544 9545 mtx = SFMMU_MLSPL_MTX(type, pp); 9546 return (MUTEX_HELD(mtx)); 9547 } 9548 9549 static uint_t 9550 sfmmu_get_free_hblk(struct hme_blk **hmeblkpp, uint_t critical) 9551 { 9552 struct hme_blk *hblkp; 9553 9554 if (freehblkp != NULL) { 9555 mutex_enter(&freehblkp_lock); 9556 if (freehblkp != NULL) { 9557 /* 9558 * If the current thread is owning hblk_reserve, 9559 * let it succede even if freehblkcnt is really low. 9560 */ 9561 if (freehblkcnt <= HBLK_RESERVE_MIN && !critical) { 9562 SFMMU_STAT(sf_get_free_throttle); 9563 mutex_exit(&freehblkp_lock); 9564 return (0); 9565 } 9566 freehblkcnt--; 9567 *hmeblkpp = freehblkp; 9568 hblkp = *hmeblkpp; 9569 freehblkp = hblkp->hblk_next; 9570 mutex_exit(&freehblkp_lock); 9571 hblkp->hblk_next = NULL; 9572 SFMMU_STAT(sf_get_free_success); 9573 return (1); 9574 } 9575 mutex_exit(&freehblkp_lock); 9576 } 9577 SFMMU_STAT(sf_get_free_fail); 9578 return (0); 9579 } 9580 9581 static uint_t 9582 sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical) 9583 { 9584 struct hme_blk *hblkp; 9585 9586 /* 9587 * If the current thread is mapping into kernel space, 9588 * let it succede even if freehblkcnt is max 9589 * so that it will avoid freeing it to kmem. 9590 * This will prevent stack overflow due to 9591 * possible recursion since kmem_cache_free() 9592 * might require creation of a slab which 9593 * in turn needs an hmeblk to map that slab; 9594 * let's break this vicious chain at the first 9595 * opportunity. 9596 */ 9597 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9598 mutex_enter(&freehblkp_lock); 9599 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9600 SFMMU_STAT(sf_put_free_success); 9601 freehblkcnt++; 9602 hmeblkp->hblk_next = freehblkp; 9603 freehblkp = hmeblkp; 9604 mutex_exit(&freehblkp_lock); 9605 return (1); 9606 } 9607 mutex_exit(&freehblkp_lock); 9608 } 9609 9610 /* 9611 * Bring down freehblkcnt to HBLK_RESERVE_CNT. We are here 9612 * only if freehblkcnt is at least HBLK_RESERVE_CNT *and* 9613 * we are not in the process of mapping into kernel space. 9614 */ 9615 ASSERT(!critical); 9616 while (freehblkcnt > HBLK_RESERVE_CNT) { 9617 mutex_enter(&freehblkp_lock); 9618 if (freehblkcnt > HBLK_RESERVE_CNT) { 9619 freehblkcnt--; 9620 hblkp = freehblkp; 9621 freehblkp = hblkp->hblk_next; 9622 mutex_exit(&freehblkp_lock); 9623 ASSERT(get_hblk_cache(hblkp) == sfmmu8_cache); 9624 kmem_cache_free(sfmmu8_cache, hblkp); 9625 continue; 9626 } 9627 mutex_exit(&freehblkp_lock); 9628 } 9629 SFMMU_STAT(sf_put_free_fail); 9630 return (0); 9631 } 9632 9633 static void 9634 sfmmu_hblk_swap(struct hme_blk *new) 9635 { 9636 struct hme_blk *old, *hblkp, *prev; 9637 uint64_t hblkpa, prevpa, newpa; 9638 caddr_t base, vaddr, endaddr; 9639 struct hmehash_bucket *hmebp; 9640 struct sf_hment *osfhme, *nsfhme; 9641 page_t *pp; 9642 kmutex_t *pml; 9643 tte_t tte; 9644 9645 #ifdef DEBUG 9646 hmeblk_tag hblktag; 9647 struct hme_blk *found; 9648 #endif 9649 old = HBLK_RESERVE; 9650 9651 /* 9652 * save pa before bcopy clobbers it 9653 */ 9654 newpa = new->hblk_nextpa; 9655 9656 base = (caddr_t)get_hblk_base(old); 9657 endaddr = base + get_hblk_span(old); 9658 9659 /* 9660 * acquire hash bucket lock. 9661 */ 9662 hmebp = sfmmu_tteload_acquire_hashbucket(ksfmmup, base, TTE8K); 9663 9664 /* 9665 * copy contents from old to new 9666 */ 9667 bcopy((void *)old, (void *)new, HME8BLK_SZ); 9668 9669 /* 9670 * add new to hash chain 9671 */ 9672 sfmmu_hblk_hash_add(hmebp, new, newpa); 9673 9674 /* 9675 * search hash chain for hblk_reserve; this needs to be performed 9676 * after adding new, otherwise prevpa and prev won't correspond 9677 * to the hblk which is prior to old in hash chain when we call 9678 * sfmmu_hblk_hash_rm to remove old later. 9679 */ 9680 for (prevpa = 0, prev = NULL, 9681 hblkpa = hmebp->hmeh_nextpa, hblkp = hmebp->hmeblkp; 9682 hblkp != NULL && hblkp != old; 9683 prevpa = hblkpa, prev = hblkp, 9684 hblkpa = hblkp->hblk_nextpa, hblkp = hblkp->hblk_next); 9685 9686 if (hblkp != old) 9687 panic("sfmmu_hblk_swap: hblk_reserve not found"); 9688 9689 /* 9690 * p_mapping list is still pointing to hments in hblk_reserve; 9691 * fix up p_mapping list so that they point to hments in new. 9692 * 9693 * Since all these mappings are created by hblk_reserve_thread 9694 * on the way and it's using at least one of the buffers from each of 9695 * the newly minted slabs, there is no danger of any of these 9696 * mappings getting unloaded by another thread. 9697 * 9698 * tsbmiss could only modify ref/mod bits of hments in old/new. 9699 * Since all of these hments hold mappings established by segkmem 9700 * and mappings in segkmem are setup with HAT_NOSYNC, ref/mod bits 9701 * have no meaning for the mappings in hblk_reserve. hments in 9702 * old and new are identical except for ref/mod bits. 9703 */ 9704 for (vaddr = base; vaddr < endaddr; vaddr += TTEBYTES(TTE8K)) { 9705 9706 HBLKTOHME(osfhme, old, vaddr); 9707 sfmmu_copytte(&osfhme->hme_tte, &tte); 9708 9709 if (TTE_IS_VALID(&tte)) { 9710 if ((pp = osfhme->hme_page) == NULL) 9711 panic("sfmmu_hblk_swap: page not mapped"); 9712 9713 pml = sfmmu_mlist_enter(pp); 9714 9715 if (pp != osfhme->hme_page) 9716 panic("sfmmu_hblk_swap: mapping changed"); 9717 9718 HBLKTOHME(nsfhme, new, vaddr); 9719 9720 HME_ADD(nsfhme, pp); 9721 HME_SUB(osfhme, pp); 9722 9723 sfmmu_mlist_exit(pml); 9724 } 9725 } 9726 9727 /* 9728 * remove old from hash chain 9729 */ 9730 sfmmu_hblk_hash_rm(hmebp, old, prevpa, prev); 9731 9732 #ifdef DEBUG 9733 9734 hblktag.htag_id = ksfmmup; 9735 hblktag.htag_bspage = HME_HASH_BSPAGE(base, HME_HASH_SHIFT(TTE8K)); 9736 hblktag.htag_rehash = HME_HASH_REHASH(TTE8K); 9737 HME_HASH_FAST_SEARCH(hmebp, hblktag, found); 9738 9739 if (found != new) 9740 panic("sfmmu_hblk_swap: new hblk not found"); 9741 #endif 9742 9743 SFMMU_HASH_UNLOCK(hmebp); 9744 9745 /* 9746 * Reset hblk_reserve 9747 */ 9748 bzero((void *)old, HME8BLK_SZ); 9749 old->hblk_nextpa = va_to_pa((caddr_t)old); 9750 } 9751 9752 /* 9753 * Grab the mlist mutex for both pages passed in. 9754 * 9755 * low and high will be returned as pointers to the mutexes for these pages. 9756 * low refers to the mutex residing in the lower bin of the mlist hash, while 9757 * high refers to the mutex residing in the higher bin of the mlist hash. This 9758 * is due to the locking order restrictions on the same thread grabbing 9759 * multiple mlist mutexes. The low lock must be acquired before the high lock. 9760 * 9761 * If both pages hash to the same mutex, only grab that single mutex, and 9762 * high will be returned as NULL 9763 * If the pages hash to different bins in the hash, grab the lower addressed 9764 * lock first and then the higher addressed lock in order to follow the locking 9765 * rules involved with the same thread grabbing multiple mlist mutexes. 9766 * low and high will both have non-NULL values. 9767 */ 9768 static void 9769 sfmmu_mlist_reloc_enter(struct page *targ, struct page *repl, 9770 kmutex_t **low, kmutex_t **high) 9771 { 9772 kmutex_t *mml_targ, *mml_repl; 9773 9774 /* 9775 * no need to do the dance around szc as in sfmmu_mlist_enter() 9776 * because this routine is only called by hat_page_relocate() and all 9777 * targ and repl pages are already locked EXCL so szc can't change. 9778 */ 9779 9780 mml_targ = MLIST_HASH(PP_PAGEROOT(targ)); 9781 mml_repl = MLIST_HASH(PP_PAGEROOT(repl)); 9782 9783 if (mml_targ == mml_repl) { 9784 *low = mml_targ; 9785 *high = NULL; 9786 } else { 9787 if (mml_targ < mml_repl) { 9788 *low = mml_targ; 9789 *high = mml_repl; 9790 } else { 9791 *low = mml_repl; 9792 *high = mml_targ; 9793 } 9794 } 9795 9796 mutex_enter(*low); 9797 if (*high) 9798 mutex_enter(*high); 9799 } 9800 9801 static void 9802 sfmmu_mlist_reloc_exit(kmutex_t *low, kmutex_t *high) 9803 { 9804 if (high) 9805 mutex_exit(high); 9806 mutex_exit(low); 9807 } 9808 9809 static hatlock_t * 9810 sfmmu_hat_enter(sfmmu_t *sfmmup) 9811 { 9812 hatlock_t *hatlockp; 9813 9814 if (sfmmup != ksfmmup) { 9815 hatlockp = TSB_HASH(sfmmup); 9816 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 9817 return (hatlockp); 9818 } 9819 return (NULL); 9820 } 9821 9822 static hatlock_t * 9823 sfmmu_hat_tryenter(sfmmu_t *sfmmup) 9824 { 9825 hatlock_t *hatlockp; 9826 9827 if (sfmmup != ksfmmup) { 9828 hatlockp = TSB_HASH(sfmmup); 9829 if (mutex_tryenter(HATLOCK_MUTEXP(hatlockp)) == 0) 9830 return (NULL); 9831 return (hatlockp); 9832 } 9833 return (NULL); 9834 } 9835 9836 static void 9837 sfmmu_hat_exit(hatlock_t *hatlockp) 9838 { 9839 if (hatlockp != NULL) 9840 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 9841 } 9842 9843 static void 9844 sfmmu_hat_lock_all(void) 9845 { 9846 int i; 9847 for (i = 0; i < SFMMU_NUM_LOCK; i++) 9848 mutex_enter(HATLOCK_MUTEXP(&hat_lock[i])); 9849 } 9850 9851 static void 9852 sfmmu_hat_unlock_all(void) 9853 { 9854 int i; 9855 for (i = SFMMU_NUM_LOCK - 1; i >= 0; i--) 9856 mutex_exit(HATLOCK_MUTEXP(&hat_lock[i])); 9857 } 9858 9859 int 9860 sfmmu_hat_lock_held(sfmmu_t *sfmmup) 9861 { 9862 ASSERT(sfmmup != ksfmmup); 9863 return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup)))); 9864 } 9865 9866 /* 9867 * Locking primitives to provide consistency between ISM unmap 9868 * and other operations. Since ISM unmap can take a long time, we 9869 * use HAT_ISMBUSY flag (protected by the hatlock) to avoid creating 9870 * contention on the hatlock buckets while ISM segments are being 9871 * unmapped. The tradeoff is that the flags don't prevent priority 9872 * inversion from occurring, so we must request kernel priority in 9873 * case we have to sleep to keep from getting buried while holding 9874 * the HAT_ISMBUSY flag set, which in turn could block other kernel 9875 * threads from running (for example, in sfmmu_uvatopfn()). 9876 */ 9877 static void 9878 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held) 9879 { 9880 hatlock_t *hatlockp; 9881 9882 THREAD_KPRI_REQUEST(); 9883 if (!hatlock_held) 9884 hatlockp = sfmmu_hat_enter(sfmmup); 9885 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) 9886 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 9887 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY); 9888 if (!hatlock_held) 9889 sfmmu_hat_exit(hatlockp); 9890 } 9891 9892 static void 9893 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held) 9894 { 9895 hatlock_t *hatlockp; 9896 9897 if (!hatlock_held) 9898 hatlockp = sfmmu_hat_enter(sfmmup); 9899 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 9900 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY); 9901 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 9902 if (!hatlock_held) 9903 sfmmu_hat_exit(hatlockp); 9904 THREAD_KPRI_RELEASE(); 9905 } 9906 9907 /* 9908 * 9909 * Algorithm: 9910 * 9911 * (1) if segkmem is not ready, allocate hblk from an array of pre-alloc'ed 9912 * hblks. 9913 * 9914 * (2) if we are allocating an hblk for mapping a slab in sfmmu_cache, 9915 * 9916 * (a) try to return an hblk from reserve pool of free hblks; 9917 * (b) if the reserve pool is empty, acquire hblk_reserve_lock 9918 * and return hblk_reserve. 9919 * 9920 * (3) call kmem_cache_alloc() to allocate hblk; 9921 * 9922 * (a) if hblk_reserve_lock is held by the current thread, 9923 * atomically replace hblk_reserve by the hblk that is 9924 * returned by kmem_cache_alloc; release hblk_reserve_lock 9925 * and call kmem_cache_alloc() again. 9926 * (b) if reserve pool is not full, add the hblk that is 9927 * returned by kmem_cache_alloc to reserve pool and 9928 * call kmem_cache_alloc again. 9929 * 9930 */ 9931 static struct hme_blk * 9932 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr, 9933 struct hmehash_bucket *hmebp, uint_t size, hmeblk_tag hblktag, 9934 uint_t flags) 9935 { 9936 struct hme_blk *hmeblkp = NULL; 9937 struct hme_blk *newhblkp; 9938 struct hme_blk *shw_hblkp = NULL; 9939 struct kmem_cache *sfmmu_cache = NULL; 9940 uint64_t hblkpa; 9941 ulong_t index; 9942 uint_t owner; /* set to 1 if using hblk_reserve */ 9943 uint_t forcefree; 9944 int sleep; 9945 9946 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 9947 9948 /* 9949 * If segkmem is not created yet, allocate from static hmeblks 9950 * created at the end of startup_modules(). See the block comment 9951 * in startup_modules() describing how we estimate the number of 9952 * static hmeblks that will be needed during re-map. 9953 */ 9954 if (!hblk_alloc_dynamic) { 9955 9956 if (size == TTE8K) { 9957 index = nucleus_hblk8.index; 9958 if (index >= nucleus_hblk8.len) { 9959 /* 9960 * If we panic here, see startup_modules() to 9961 * make sure that we are calculating the 9962 * number of hblk8's that we need correctly. 9963 */ 9964 panic("no nucleus hblk8 to allocate"); 9965 } 9966 hmeblkp = 9967 (struct hme_blk *)&nucleus_hblk8.list[index]; 9968 nucleus_hblk8.index++; 9969 SFMMU_STAT(sf_hblk8_nalloc); 9970 } else { 9971 index = nucleus_hblk1.index; 9972 if (nucleus_hblk1.index >= nucleus_hblk1.len) { 9973 /* 9974 * If we panic here, see startup_modules() 9975 * and H8TOH1; most likely you need to 9976 * update the calculation of the number 9977 * of hblk1's the kernel needs to boot. 9978 */ 9979 panic("no nucleus hblk1 to allocate"); 9980 } 9981 hmeblkp = 9982 (struct hme_blk *)&nucleus_hblk1.list[index]; 9983 nucleus_hblk1.index++; 9984 SFMMU_STAT(sf_hblk1_nalloc); 9985 } 9986 9987 goto hblk_init; 9988 } 9989 9990 SFMMU_HASH_UNLOCK(hmebp); 9991 9992 if (sfmmup != KHATID) { 9993 if (mmu_page_sizes == max_mmu_page_sizes) { 9994 if (size < TTE256M) 9995 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 9996 size, flags); 9997 } else { 9998 if (size < TTE4M) 9999 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 10000 size, flags); 10001 } 10002 } 10003 10004 fill_hblk: 10005 owner = (hblk_reserve_thread == curthread) ? 1 : 0; 10006 10007 if (owner && size == TTE8K) { 10008 10009 /* 10010 * We are really in a tight spot. We already own 10011 * hblk_reserve and we need another hblk. In anticipation 10012 * of this kind of scenario, we specifically set aside 10013 * HBLK_RESERVE_MIN number of hblks to be used exclusively 10014 * by owner of hblk_reserve. 10015 */ 10016 SFMMU_STAT(sf_hblk_recurse_cnt); 10017 10018 if (!sfmmu_get_free_hblk(&hmeblkp, 1)) 10019 panic("sfmmu_hblk_alloc: reserve list is empty"); 10020 10021 goto hblk_verify; 10022 } 10023 10024 ASSERT(!owner); 10025 10026 if ((flags & HAT_NO_KALLOC) == 0) { 10027 10028 sfmmu_cache = ((size == TTE8K) ? sfmmu8_cache : sfmmu1_cache); 10029 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP); 10030 10031 if ((hmeblkp = kmem_cache_alloc(sfmmu_cache, sleep)) == NULL) { 10032 hmeblkp = sfmmu_hblk_steal(size); 10033 } else { 10034 /* 10035 * if we are the owner of hblk_reserve, 10036 * swap hblk_reserve with hmeblkp and 10037 * start a fresh life. Hope things go 10038 * better this time. 10039 */ 10040 if (hblk_reserve_thread == curthread) { 10041 ASSERT(sfmmu_cache == sfmmu8_cache); 10042 sfmmu_hblk_swap(hmeblkp); 10043 hblk_reserve_thread = NULL; 10044 mutex_exit(&hblk_reserve_lock); 10045 goto fill_hblk; 10046 } 10047 /* 10048 * let's donate this hblk to our reserve list if 10049 * we are not mapping kernel range 10050 */ 10051 if (size == TTE8K && sfmmup != KHATID) 10052 if (sfmmu_put_free_hblk(hmeblkp, 0)) 10053 goto fill_hblk; 10054 } 10055 } else { 10056 /* 10057 * We are here to map the slab in sfmmu8_cache; let's 10058 * check if we could tap our reserve list; if successful, 10059 * this will avoid the pain of going thru sfmmu_hblk_swap 10060 */ 10061 SFMMU_STAT(sf_hblk_slab_cnt); 10062 if (!sfmmu_get_free_hblk(&hmeblkp, 0)) { 10063 /* 10064 * let's start hblk_reserve dance 10065 */ 10066 SFMMU_STAT(sf_hblk_reserve_cnt); 10067 owner = 1; 10068 mutex_enter(&hblk_reserve_lock); 10069 hmeblkp = HBLK_RESERVE; 10070 hblk_reserve_thread = curthread; 10071 } 10072 } 10073 10074 hblk_verify: 10075 ASSERT(hmeblkp != NULL); 10076 set_hblk_sz(hmeblkp, size); 10077 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); 10078 SFMMU_HASH_LOCK(hmebp); 10079 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10080 if (newhblkp != NULL) { 10081 SFMMU_HASH_UNLOCK(hmebp); 10082 if (hmeblkp != HBLK_RESERVE) { 10083 /* 10084 * This is really tricky! 10085 * 10086 * vmem_alloc(vmem_seg_arena) 10087 * vmem_alloc(vmem_internal_arena) 10088 * segkmem_alloc(heap_arena) 10089 * vmem_alloc(heap_arena) 10090 * page_create() 10091 * hat_memload() 10092 * kmem_cache_free() 10093 * kmem_cache_alloc() 10094 * kmem_slab_create() 10095 * vmem_alloc(kmem_internal_arena) 10096 * segkmem_alloc(heap_arena) 10097 * vmem_alloc(heap_arena) 10098 * page_create() 10099 * hat_memload() 10100 * kmem_cache_free() 10101 * ... 10102 * 10103 * Thus, hat_memload() could call kmem_cache_free 10104 * for enough number of times that we could easily 10105 * hit the bottom of the stack or run out of reserve 10106 * list of vmem_seg structs. So, we must donate 10107 * this hblk to reserve list if it's allocated 10108 * from sfmmu8_cache *and* mapping kernel range. 10109 * We don't need to worry about freeing hmeblk1's 10110 * to kmem since they don't map any kmem slabs. 10111 * 10112 * Note: When segkmem supports largepages, we must 10113 * free hmeblk1's to reserve list as well. 10114 */ 10115 forcefree = (sfmmup == KHATID) ? 1 : 0; 10116 if (size == TTE8K && 10117 sfmmu_put_free_hblk(hmeblkp, forcefree)) { 10118 goto re_verify; 10119 } 10120 ASSERT(sfmmup != KHATID); 10121 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10122 } else { 10123 /* 10124 * Hey! we don't need hblk_reserve any more. 10125 */ 10126 ASSERT(owner); 10127 hblk_reserve_thread = NULL; 10128 mutex_exit(&hblk_reserve_lock); 10129 owner = 0; 10130 } 10131 re_verify: 10132 /* 10133 * let's check if the goodies are still present 10134 */ 10135 SFMMU_HASH_LOCK(hmebp); 10136 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10137 if (newhblkp != NULL) { 10138 /* 10139 * return newhblkp if it's not hblk_reserve; 10140 * if newhblkp is hblk_reserve, return it 10141 * _only if_ we are the owner of hblk_reserve. 10142 */ 10143 if (newhblkp != HBLK_RESERVE || owner) { 10144 return (newhblkp); 10145 } else { 10146 /* 10147 * we just hit hblk_reserve in the hash and 10148 * we are not the owner of that; 10149 * 10150 * block until hblk_reserve_thread completes 10151 * swapping hblk_reserve and try the dance 10152 * once again. 10153 */ 10154 SFMMU_HASH_UNLOCK(hmebp); 10155 mutex_enter(&hblk_reserve_lock); 10156 mutex_exit(&hblk_reserve_lock); 10157 SFMMU_STAT(sf_hblk_reserve_hit); 10158 goto fill_hblk; 10159 } 10160 } else { 10161 /* 10162 * it's no more! try the dance once again. 10163 */ 10164 SFMMU_HASH_UNLOCK(hmebp); 10165 goto fill_hblk; 10166 } 10167 } 10168 10169 hblk_init: 10170 set_hblk_sz(hmeblkp, size); 10171 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10172 hmeblkp->hblk_next = (struct hme_blk *)NULL; 10173 hmeblkp->hblk_tag = hblktag; 10174 hmeblkp->hblk_shadow = shw_hblkp; 10175 hblkpa = hmeblkp->hblk_nextpa; 10176 hmeblkp->hblk_nextpa = 0; 10177 10178 ASSERT(get_hblk_ttesz(hmeblkp) == size); 10179 ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size)); 10180 ASSERT(hmeblkp->hblk_hmecnt == 0); 10181 ASSERT(hmeblkp->hblk_vcnt == 0); 10182 ASSERT(hmeblkp->hblk_lckcnt == 0); 10183 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10184 sfmmu_hblk_hash_add(hmebp, hmeblkp, hblkpa); 10185 return (hmeblkp); 10186 } 10187 10188 /* 10189 * This function performs any cleanup required on the hme_blk 10190 * and returns it to the free list. 10191 */ 10192 /* ARGSUSED */ 10193 static void 10194 sfmmu_hblk_free(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10195 uint64_t hblkpa, struct hme_blk **listp) 10196 { 10197 int shw_size, vshift; 10198 struct hme_blk *shw_hblkp; 10199 uint_t shw_mask, newshw_mask; 10200 uintptr_t vaddr; 10201 int size; 10202 uint_t critical; 10203 10204 ASSERT(hmeblkp); 10205 ASSERT(!hmeblkp->hblk_hmecnt); 10206 ASSERT(!hmeblkp->hblk_vcnt); 10207 ASSERT(!hmeblkp->hblk_lckcnt); 10208 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10209 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 10210 10211 critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0; 10212 10213 size = get_hblk_ttesz(hmeblkp); 10214 shw_hblkp = hmeblkp->hblk_shadow; 10215 if (shw_hblkp) { 10216 ASSERT(hblktosfmmu(hmeblkp) != KHATID); 10217 if (mmu_page_sizes == max_mmu_page_sizes) { 10218 ASSERT(size < TTE256M); 10219 } else { 10220 ASSERT(size < TTE4M); 10221 } 10222 10223 shw_size = get_hblk_ttesz(shw_hblkp); 10224 vaddr = get_hblk_base(hmeblkp); 10225 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10226 ASSERT(vshift < 8); 10227 /* 10228 * Atomically clear shadow mask bit 10229 */ 10230 do { 10231 shw_mask = shw_hblkp->hblk_shw_mask; 10232 ASSERT(shw_mask & (1 << vshift)); 10233 newshw_mask = shw_mask & ~(1 << vshift); 10234 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10235 shw_mask, newshw_mask); 10236 } while (newshw_mask != shw_mask); 10237 hmeblkp->hblk_shadow = NULL; 10238 } 10239 hmeblkp->hblk_next = NULL; 10240 hmeblkp->hblk_nextpa = hblkpa; 10241 hmeblkp->hblk_shw_bit = 0; 10242 10243 if (hmeblkp->hblk_nuc_bit == 0) { 10244 10245 if (size == TTE8K && sfmmu_put_free_hblk(hmeblkp, critical)) 10246 return; 10247 10248 hmeblkp->hblk_next = *listp; 10249 *listp = hmeblkp; 10250 } 10251 } 10252 10253 static void 10254 sfmmu_hblks_list_purge(struct hme_blk **listp) 10255 { 10256 struct hme_blk *hmeblkp; 10257 10258 while ((hmeblkp = *listp) != NULL) { 10259 *listp = hmeblkp->hblk_next; 10260 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10261 } 10262 } 10263 10264 #define BUCKETS_TO_SEARCH_BEFORE_UNLOAD 30 10265 10266 static uint_t sfmmu_hblk_steal_twice; 10267 static uint_t sfmmu_hblk_steal_count, sfmmu_hblk_steal_unload_count; 10268 10269 /* 10270 * Steal a hmeblk 10271 * Enough hmeblks were allocated at startup (nucleus hmeblks) and also 10272 * hmeblks were added dynamically. We should never ever not be able to 10273 * find one. Look for an unused/unlocked hmeblk in user hash table. 10274 */ 10275 static struct hme_blk * 10276 sfmmu_hblk_steal(int size) 10277 { 10278 static struct hmehash_bucket *uhmehash_steal_hand = NULL; 10279 struct hmehash_bucket *hmebp; 10280 struct hme_blk *hmeblkp = NULL, *pr_hblk; 10281 uint64_t hblkpa, prevpa; 10282 int i; 10283 10284 for (;;) { 10285 hmebp = (uhmehash_steal_hand == NULL) ? uhme_hash : 10286 uhmehash_steal_hand; 10287 ASSERT(hmebp >= uhme_hash && hmebp <= &uhme_hash[UHMEHASH_SZ]); 10288 10289 for (i = 0; hmeblkp == NULL && i <= UHMEHASH_SZ + 10290 BUCKETS_TO_SEARCH_BEFORE_UNLOAD; i++) { 10291 SFMMU_HASH_LOCK(hmebp); 10292 hmeblkp = hmebp->hmeblkp; 10293 hblkpa = hmebp->hmeh_nextpa; 10294 prevpa = 0; 10295 pr_hblk = NULL; 10296 while (hmeblkp) { 10297 /* 10298 * check if it is a hmeblk that is not locked 10299 * and not shared. skip shadow hmeblks with 10300 * shadow_mask set i.e valid count non zero. 10301 */ 10302 if ((get_hblk_ttesz(hmeblkp) == size) && 10303 (hmeblkp->hblk_shw_bit == 0 || 10304 hmeblkp->hblk_vcnt == 0) && 10305 (hmeblkp->hblk_lckcnt == 0)) { 10306 /* 10307 * there is a high probability that we 10308 * will find a free one. search some 10309 * buckets for a free hmeblk initially 10310 * before unloading a valid hmeblk. 10311 */ 10312 if ((hmeblkp->hblk_vcnt == 0 && 10313 hmeblkp->hblk_hmecnt == 0) || (i >= 10314 BUCKETS_TO_SEARCH_BEFORE_UNLOAD)) { 10315 if (sfmmu_steal_this_hblk(hmebp, 10316 hmeblkp, hblkpa, prevpa, 10317 pr_hblk)) { 10318 /* 10319 * Hblk is unloaded 10320 * successfully 10321 */ 10322 break; 10323 } 10324 } 10325 } 10326 pr_hblk = hmeblkp; 10327 prevpa = hblkpa; 10328 hblkpa = hmeblkp->hblk_nextpa; 10329 hmeblkp = hmeblkp->hblk_next; 10330 } 10331 10332 SFMMU_HASH_UNLOCK(hmebp); 10333 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 10334 hmebp = uhme_hash; 10335 } 10336 uhmehash_steal_hand = hmebp; 10337 10338 if (hmeblkp != NULL) 10339 break; 10340 10341 /* 10342 * in the worst case, look for a free one in the kernel 10343 * hash table. 10344 */ 10345 for (i = 0, hmebp = khme_hash; i <= KHMEHASH_SZ; i++) { 10346 SFMMU_HASH_LOCK(hmebp); 10347 hmeblkp = hmebp->hmeblkp; 10348 hblkpa = hmebp->hmeh_nextpa; 10349 prevpa = 0; 10350 pr_hblk = NULL; 10351 while (hmeblkp) { 10352 /* 10353 * check if it is free hmeblk 10354 */ 10355 if ((get_hblk_ttesz(hmeblkp) == size) && 10356 (hmeblkp->hblk_lckcnt == 0) && 10357 (hmeblkp->hblk_vcnt == 0) && 10358 (hmeblkp->hblk_hmecnt == 0)) { 10359 if (sfmmu_steal_this_hblk(hmebp, 10360 hmeblkp, hblkpa, prevpa, pr_hblk)) { 10361 break; 10362 } else { 10363 /* 10364 * Cannot fail since we have 10365 * hash lock. 10366 */ 10367 panic("fail to steal?"); 10368 } 10369 } 10370 10371 pr_hblk = hmeblkp; 10372 prevpa = hblkpa; 10373 hblkpa = hmeblkp->hblk_nextpa; 10374 hmeblkp = hmeblkp->hblk_next; 10375 } 10376 10377 SFMMU_HASH_UNLOCK(hmebp); 10378 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 10379 hmebp = khme_hash; 10380 } 10381 10382 if (hmeblkp != NULL) 10383 break; 10384 sfmmu_hblk_steal_twice++; 10385 } 10386 return (hmeblkp); 10387 } 10388 10389 /* 10390 * This routine does real work to prepare a hblk to be "stolen" by 10391 * unloading the mappings, updating shadow counts .... 10392 * It returns 1 if the block is ready to be reused (stolen), or 0 10393 * means the block cannot be stolen yet- pageunload is still working 10394 * on this hblk. 10395 */ 10396 static int 10397 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10398 uint64_t hblkpa, uint64_t prevpa, struct hme_blk *pr_hblk) 10399 { 10400 int shw_size, vshift; 10401 struct hme_blk *shw_hblkp; 10402 uintptr_t vaddr; 10403 uint_t shw_mask, newshw_mask; 10404 10405 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10406 10407 /* 10408 * check if the hmeblk is free, unload if necessary 10409 */ 10410 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10411 sfmmu_t *sfmmup; 10412 demap_range_t dmr; 10413 10414 sfmmup = hblktosfmmu(hmeblkp); 10415 DEMAP_RANGE_INIT(sfmmup, &dmr); 10416 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 10417 (caddr_t)get_hblk_base(hmeblkp), 10418 get_hblk_endaddr(hmeblkp), &dmr, HAT_UNLOAD); 10419 DEMAP_RANGE_FLUSH(&dmr); 10420 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10421 /* 10422 * Pageunload is working on the same hblk. 10423 */ 10424 return (0); 10425 } 10426 10427 sfmmu_hblk_steal_unload_count++; 10428 } 10429 10430 ASSERT(hmeblkp->hblk_lckcnt == 0); 10431 ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0); 10432 10433 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 10434 hmeblkp->hblk_nextpa = hblkpa; 10435 10436 shw_hblkp = hmeblkp->hblk_shadow; 10437 if (shw_hblkp) { 10438 shw_size = get_hblk_ttesz(shw_hblkp); 10439 vaddr = get_hblk_base(hmeblkp); 10440 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10441 ASSERT(vshift < 8); 10442 /* 10443 * Atomically clear shadow mask bit 10444 */ 10445 do { 10446 shw_mask = shw_hblkp->hblk_shw_mask; 10447 ASSERT(shw_mask & (1 << vshift)); 10448 newshw_mask = shw_mask & ~(1 << vshift); 10449 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10450 shw_mask, newshw_mask); 10451 } while (newshw_mask != shw_mask); 10452 hmeblkp->hblk_shadow = NULL; 10453 } 10454 10455 /* 10456 * remove shadow bit if we are stealing an unused shadow hmeblk. 10457 * sfmmu_hblk_alloc needs it that way, will set shadow bit later if 10458 * we are indeed allocating a shadow hmeblk. 10459 */ 10460 hmeblkp->hblk_shw_bit = 0; 10461 10462 sfmmu_hblk_steal_count++; 10463 SFMMU_STAT(sf_steal_count); 10464 10465 return (1); 10466 } 10467 10468 struct hme_blk * 10469 sfmmu_hmetohblk(struct sf_hment *sfhme) 10470 { 10471 struct hme_blk *hmeblkp; 10472 struct sf_hment *sfhme0; 10473 struct hme_blk *hblk_dummy = 0; 10474 10475 /* 10476 * No dummy sf_hments, please. 10477 */ 10478 ASSERT(sfhme->hme_tte.ll != 0); 10479 10480 sfhme0 = sfhme - sfhme->hme_tte.tte_hmenum; 10481 hmeblkp = (struct hme_blk *)((uintptr_t)sfhme0 - 10482 (uintptr_t)&hblk_dummy->hblk_hme[0]); 10483 10484 return (hmeblkp); 10485 } 10486 10487 /* 10488 * On swapin, get appropriately sized TSB(s) and clear the HAT_SWAPPED flag. 10489 * If we can't get appropriately sized TSB(s), try for 8K TSB(s) using 10490 * KM_SLEEP allocation. 10491 * 10492 * Return 0 on success, -1 otherwise. 10493 */ 10494 static void 10495 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp) 10496 { 10497 struct tsb_info *tsbinfop, *next; 10498 tsb_replace_rc_t rc; 10499 boolean_t gotfirst = B_FALSE; 10500 10501 ASSERT(sfmmup != ksfmmup); 10502 ASSERT(sfmmu_hat_lock_held(sfmmup)); 10503 10504 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) { 10505 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 10506 } 10507 10508 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10509 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN); 10510 } else { 10511 return; 10512 } 10513 10514 ASSERT(sfmmup->sfmmu_tsb != NULL); 10515 10516 /* 10517 * Loop over all tsbinfo's replacing them with ones that actually have 10518 * a TSB. If any of the replacements ever fail, bail out of the loop. 10519 */ 10520 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) { 10521 ASSERT(tsbinfop->tsb_flags & TSB_SWAPPED); 10522 next = tsbinfop->tsb_next; 10523 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc, 10524 hatlockp, TSB_SWAPIN); 10525 if (rc != TSB_SUCCESS) { 10526 break; 10527 } 10528 gotfirst = B_TRUE; 10529 } 10530 10531 switch (rc) { 10532 case TSB_SUCCESS: 10533 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10534 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10535 return; 10536 case TSB_ALLOCFAIL: 10537 break; 10538 default: 10539 panic("sfmmu_replace_tsb returned unrecognized failure code " 10540 "%d", rc); 10541 } 10542 10543 /* 10544 * In this case, we failed to get one of our TSBs. If we failed to 10545 * get the first TSB, get one of minimum size (8KB). Walk the list 10546 * and throw away the tsbinfos, starting where the allocation failed; 10547 * we can get by with just one TSB as long as we don't leave the 10548 * SWAPPED tsbinfo structures lying around. 10549 */ 10550 tsbinfop = sfmmup->sfmmu_tsb; 10551 next = tsbinfop->tsb_next; 10552 tsbinfop->tsb_next = NULL; 10553 10554 sfmmu_hat_exit(hatlockp); 10555 for (tsbinfop = next; tsbinfop != NULL; tsbinfop = next) { 10556 next = tsbinfop->tsb_next; 10557 sfmmu_tsbinfo_free(tsbinfop); 10558 } 10559 hatlockp = sfmmu_hat_enter(sfmmup); 10560 10561 /* 10562 * If we don't have any TSBs, get a single 8K TSB for 8K, 64K and 512K 10563 * pages. 10564 */ 10565 if (!gotfirst) { 10566 tsbinfop = sfmmup->sfmmu_tsb; 10567 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE, 10568 hatlockp, TSB_SWAPIN | TSB_FORCEALLOC); 10569 ASSERT(rc == TSB_SUCCESS); 10570 } else { 10571 /* update machine specific tsbinfo */ 10572 sfmmu_setup_tsbinfo(sfmmup); 10573 } 10574 10575 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10576 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10577 } 10578 10579 /* 10580 * Handle exceptions for low level tsb_handler. 10581 * 10582 * There are many scenarios that could land us here: 10583 * 10584 * If the context is invalid we land here. The context can be invalid 10585 * for 3 reasons: 1) we couldn't allocate a new context and now need to 10586 * perform a wrap around operation in order to allocate a new context. 10587 * 2) Context was invalidated to change pagesize programming 3) ISMs or 10588 * TSBs configuration is changeing for this process and we are forced into 10589 * here to do a syncronization operation. If the context is valid we can 10590 * be here from window trap hanlder. In this case just call trap to handle 10591 * the fault. 10592 * 10593 * Note that the process will run in INVALID_CONTEXT before 10594 * faulting into here and subsequently loading the MMU registers 10595 * (including the TSB base register) associated with this process. 10596 * For this reason, the trap handlers must all test for 10597 * INVALID_CONTEXT before attempting to access any registers other 10598 * than the context registers. 10599 */ 10600 void 10601 sfmmu_tsbmiss_exception(struct regs *rp, uintptr_t tagaccess, uint_t traptype) 10602 { 10603 sfmmu_t *sfmmup; 10604 uint_t ctxnum; 10605 klwp_id_t lwp; 10606 char lwp_save_state; 10607 hatlock_t *hatlockp; 10608 struct tsb_info *tsbinfop; 10609 10610 SFMMU_STAT(sf_tsb_exceptions); 10611 SFMMU_MMU_STAT(mmu_tsb_exceptions); 10612 sfmmup = astosfmmu(curthread->t_procp->p_as); 10613 ctxnum = tagaccess & TAGACC_CTX_MASK; 10614 10615 ASSERT(sfmmup != ksfmmup && ctxnum != KCONTEXT); 10616 ASSERT(sfmmup->sfmmu_ismhat == 0); 10617 /* 10618 * First, make sure we come out of here with a valid ctx, 10619 * since if we don't get one we'll simply loop on the 10620 * faulting instruction. 10621 * 10622 * If the ISM mappings are changing, the TSB is being relocated, or 10623 * the process is swapped out we serialize behind the controlling 10624 * thread with the sfmmu_flags and sfmmu_tsb_cv condition variable. 10625 * Otherwise we synchronize with the context stealer or the thread 10626 * that required us to change out our MMU registers (such 10627 * as a thread changing out our TSB while we were running) by 10628 * locking the HAT and grabbing the rwlock on the context as a 10629 * reader temporarily. 10630 */ 10631 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) || 10632 ctxnum == INVALID_CONTEXT); 10633 10634 if (ctxnum == INVALID_CONTEXT) { 10635 /* 10636 * Must set lwp state to LWP_SYS before 10637 * trying to acquire any adaptive lock 10638 */ 10639 lwp = ttolwp(curthread); 10640 ASSERT(lwp); 10641 lwp_save_state = lwp->lwp_state; 10642 lwp->lwp_state = LWP_SYS; 10643 10644 hatlockp = sfmmu_hat_enter(sfmmup); 10645 retry: 10646 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 10647 tsbinfop = tsbinfop->tsb_next) { 10648 if (tsbinfop->tsb_flags & TSB_RELOC_FLAG) { 10649 cv_wait(&sfmmup->sfmmu_tsb_cv, 10650 HATLOCK_MUTEXP(hatlockp)); 10651 goto retry; 10652 } 10653 } 10654 10655 /* 10656 * Wait for ISM maps to be updated. 10657 */ 10658 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) { 10659 cv_wait(&sfmmup->sfmmu_tsb_cv, 10660 HATLOCK_MUTEXP(hatlockp)); 10661 goto retry; 10662 } 10663 10664 /* 10665 * If we're swapping in, get TSB(s). Note that we must do 10666 * this before we get a ctx or load the MMU state. Once 10667 * we swap in we have to recheck to make sure the TSB(s) and 10668 * ISM mappings didn't change while we slept. 10669 */ 10670 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10671 sfmmu_tsb_swapin(sfmmup, hatlockp); 10672 goto retry; 10673 } 10674 10675 sfmmu_get_ctx(sfmmup); 10676 10677 sfmmu_hat_exit(hatlockp); 10678 /* 10679 * Must restore lwp_state if not calling 10680 * trap() for further processing. Restore 10681 * it anyway. 10682 */ 10683 lwp->lwp_state = lwp_save_state; 10684 if (sfmmup->sfmmu_ttecnt[TTE8K] != 0 || 10685 sfmmup->sfmmu_ttecnt[TTE64K] != 0 || 10686 sfmmup->sfmmu_ttecnt[TTE512K] != 0 || 10687 sfmmup->sfmmu_ttecnt[TTE4M] != 0 || 10688 sfmmup->sfmmu_ttecnt[TTE32M] != 0 || 10689 sfmmup->sfmmu_ttecnt[TTE256M] != 0) { 10690 return; 10691 } 10692 if (traptype == T_DATA_PROT) { 10693 traptype = T_DATA_MMU_MISS; 10694 } 10695 } 10696 trap(rp, (caddr_t)tagaccess, traptype, 0); 10697 } 10698 10699 /* 10700 * sfmmu_vatopfn_suspended is called from GET_TTE when TL=0 and 10701 * TTE_SUSPENDED bit set in tte we block on aquiring a page lock 10702 * rather than spinning to avoid send mondo timeouts with 10703 * interrupts enabled. When the lock is acquired it is immediately 10704 * released and we return back to sfmmu_vatopfn just after 10705 * the GET_TTE call. 10706 */ 10707 void 10708 sfmmu_vatopfn_suspended(caddr_t vaddr, sfmmu_t *sfmmu, tte_t *ttep) 10709 { 10710 struct page **pp; 10711 10712 (void) as_pagelock(sfmmu->sfmmu_as, &pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10713 as_pageunlock(sfmmu->sfmmu_as, pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10714 } 10715 10716 /* 10717 * sfmmu_tsbmiss_suspended is called from GET_TTE when TL>0 and 10718 * TTE_SUSPENDED bit set in tte. We do this so that we can handle 10719 * cross traps which cannot be handled while spinning in the 10720 * trap handlers. Simply enter and exit the kpr_suspendlock spin 10721 * mutex, which is held by the holder of the suspend bit, and then 10722 * retry the trapped instruction after unwinding. 10723 */ 10724 /*ARGSUSED*/ 10725 void 10726 sfmmu_tsbmiss_suspended(struct regs *rp, uintptr_t tagacc, uint_t traptype) 10727 { 10728 ASSERT(curthread != kreloc_thread); 10729 mutex_enter(&kpr_suspendlock); 10730 mutex_exit(&kpr_suspendlock); 10731 } 10732 10733 /* 10734 * Special routine to flush out ism mappings- TSBs, TLBs and D-caches. 10735 * This routine may be called with all cpu's captured. Therefore, the 10736 * caller is responsible for holding all locks and disabling kernel 10737 * preemption. 10738 */ 10739 /* ARGSUSED */ 10740 static void 10741 sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, 10742 struct hme_blk *hmeblkp, pfn_t pfnum, int cache_flush_flag) 10743 { 10744 cpuset_t cpuset; 10745 caddr_t va; 10746 ism_ment_t *ment; 10747 sfmmu_t *sfmmup; 10748 #ifdef VAC 10749 int vcolor; 10750 #endif 10751 int ttesz; 10752 10753 /* 10754 * Walk the ism_hat's mapping list and flush the page 10755 * from every hat sharing this ism_hat. This routine 10756 * may be called while all cpu's have been captured. 10757 * Therefore we can't attempt to grab any locks. For now 10758 * this means we will protect the ism mapping list under 10759 * a single lock which will be grabbed by the caller. 10760 * If hat_share/unshare scalibility becomes a performance 10761 * problem then we may need to re-think ism mapping list locking. 10762 */ 10763 ASSERT(ism_sfmmup->sfmmu_ismhat); 10764 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 10765 addr = addr - ISMID_STARTADDR; 10766 for (ment = ism_sfmmup->sfmmu_iment; ment; ment = ment->iment_next) { 10767 10768 sfmmup = ment->iment_hat; 10769 10770 va = ment->iment_base_va; 10771 va = (caddr_t)((uintptr_t)va + (uintptr_t)addr); 10772 10773 /* 10774 * Flush TSB of ISM mappings. 10775 */ 10776 ttesz = get_hblk_ttesz(hmeblkp); 10777 if (ttesz == TTE8K || ttesz == TTE4M) { 10778 sfmmu_unload_tsb(sfmmup, va, ttesz); 10779 } else { 10780 caddr_t sva = va; 10781 caddr_t eva; 10782 ASSERT(addr == (caddr_t)get_hblk_base(hmeblkp)); 10783 eva = sva + get_hblk_span(hmeblkp); 10784 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); 10785 } 10786 10787 cpuset = sfmmup->sfmmu_cpusran; 10788 CPUSET_AND(cpuset, cpu_ready_set); 10789 CPUSET_DEL(cpuset, CPU->cpu_id); 10790 10791 SFMMU_XCALL_STATS(sfmmup); 10792 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)va, 10793 (uint64_t)sfmmup); 10794 10795 vtag_flushpage(va, (uint64_t)sfmmup); 10796 10797 #ifdef VAC 10798 /* 10799 * Flush D$ 10800 * When flushing D$ we must flush all 10801 * cpu's. See sfmmu_cache_flush(). 10802 */ 10803 if (cache_flush_flag == CACHE_FLUSH) { 10804 cpuset = cpu_ready_set; 10805 CPUSET_DEL(cpuset, CPU->cpu_id); 10806 10807 SFMMU_XCALL_STATS(sfmmup); 10808 vcolor = addr_to_vcolor(va); 10809 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10810 vac_flushpage(pfnum, vcolor); 10811 } 10812 #endif /* VAC */ 10813 } 10814 } 10815 10816 /* 10817 * Demaps the TSB, CPU caches, and flushes all TLBs on all CPUs of 10818 * a particular virtual address and ctx. If noflush is set we do not 10819 * flush the TLB/TSB. This function may or may not be called with the 10820 * HAT lock held. 10821 */ 10822 static void 10823 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10824 pfn_t pfnum, int tlb_noflush, int cpu_flag, int cache_flush_flag, 10825 int hat_lock_held) 10826 { 10827 #ifdef VAC 10828 int vcolor; 10829 #endif 10830 cpuset_t cpuset; 10831 hatlock_t *hatlockp; 10832 10833 #if defined(lint) && !defined(VAC) 10834 pfnum = pfnum; 10835 cpu_flag = cpu_flag; 10836 cache_flush_flag = cache_flush_flag; 10837 #endif 10838 /* 10839 * There is no longer a need to protect against ctx being 10840 * stolen here since we don't store the ctx in the TSB anymore. 10841 */ 10842 #ifdef VAC 10843 vcolor = addr_to_vcolor(addr); 10844 #endif 10845 10846 /* 10847 * We must hold the hat lock during the flush of TLB, 10848 * to avoid a race with sfmmu_invalidate_ctx(), where 10849 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 10850 * causing TLB demap routine to skip flush on that MMU. 10851 * If the context on a MMU has already been set to 10852 * INVALID_CONTEXT, we just get an extra flush on 10853 * that MMU. 10854 */ 10855 if (!hat_lock_held && !tlb_noflush) 10856 hatlockp = sfmmu_hat_enter(sfmmup); 10857 10858 kpreempt_disable(); 10859 if (!tlb_noflush) { 10860 /* 10861 * Flush the TSB and TLB. 10862 */ 10863 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10864 10865 cpuset = sfmmup->sfmmu_cpusran; 10866 CPUSET_AND(cpuset, cpu_ready_set); 10867 CPUSET_DEL(cpuset, CPU->cpu_id); 10868 10869 SFMMU_XCALL_STATS(sfmmup); 10870 10871 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 10872 (uint64_t)sfmmup); 10873 10874 vtag_flushpage(addr, (uint64_t)sfmmup); 10875 } 10876 10877 if (!hat_lock_held && !tlb_noflush) 10878 sfmmu_hat_exit(hatlockp); 10879 10880 #ifdef VAC 10881 /* 10882 * Flush the D$ 10883 * 10884 * Even if the ctx is stolen, we need to flush the 10885 * cache. Our ctx stealer only flushes the TLBs. 10886 */ 10887 if (cache_flush_flag == CACHE_FLUSH) { 10888 if (cpu_flag & FLUSH_ALL_CPUS) { 10889 cpuset = cpu_ready_set; 10890 } else { 10891 cpuset = sfmmup->sfmmu_cpusran; 10892 CPUSET_AND(cpuset, cpu_ready_set); 10893 } 10894 CPUSET_DEL(cpuset, CPU->cpu_id); 10895 SFMMU_XCALL_STATS(sfmmup); 10896 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10897 vac_flushpage(pfnum, vcolor); 10898 } 10899 #endif /* VAC */ 10900 kpreempt_enable(); 10901 } 10902 10903 /* 10904 * Demaps the TSB and flushes all TLBs on all cpus for a particular virtual 10905 * address and ctx. If noflush is set we do not currently do anything. 10906 * This function may or may not be called with the HAT lock held. 10907 */ 10908 static void 10909 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10910 int tlb_noflush, int hat_lock_held) 10911 { 10912 cpuset_t cpuset; 10913 hatlock_t *hatlockp; 10914 10915 /* 10916 * If the process is exiting we have nothing to do. 10917 */ 10918 if (tlb_noflush) 10919 return; 10920 10921 /* 10922 * Flush TSB. 10923 */ 10924 if (!hat_lock_held) 10925 hatlockp = sfmmu_hat_enter(sfmmup); 10926 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10927 10928 kpreempt_disable(); 10929 10930 cpuset = sfmmup->sfmmu_cpusran; 10931 CPUSET_AND(cpuset, cpu_ready_set); 10932 CPUSET_DEL(cpuset, CPU->cpu_id); 10933 10934 SFMMU_XCALL_STATS(sfmmup); 10935 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup); 10936 10937 vtag_flushpage(addr, (uint64_t)sfmmup); 10938 10939 if (!hat_lock_held) 10940 sfmmu_hat_exit(hatlockp); 10941 10942 kpreempt_enable(); 10943 10944 } 10945 10946 /* 10947 * Special case of sfmmu_tlb_demap for MMU_PAGESIZE hblks. Use the xcall 10948 * call handler that can flush a range of pages to save on xcalls. 10949 */ 10950 static int sfmmu_xcall_save; 10951 10952 static void 10953 sfmmu_tlb_range_demap(demap_range_t *dmrp) 10954 { 10955 sfmmu_t *sfmmup = dmrp->dmr_sfmmup; 10956 hatlock_t *hatlockp; 10957 cpuset_t cpuset; 10958 uint64_t sfmmu_pgcnt; 10959 pgcnt_t pgcnt = 0; 10960 int pgunload = 0; 10961 int dirtypg = 0; 10962 caddr_t addr = dmrp->dmr_addr; 10963 caddr_t eaddr; 10964 uint64_t bitvec = dmrp->dmr_bitvec; 10965 10966 ASSERT(bitvec & 1); 10967 10968 /* 10969 * Flush TSB and calculate number of pages to flush. 10970 */ 10971 while (bitvec != 0) { 10972 dirtypg = 0; 10973 /* 10974 * Find the first page to flush and then count how many 10975 * pages there are after it that also need to be flushed. 10976 * This way the number of TSB flushes is minimized. 10977 */ 10978 while ((bitvec & 1) == 0) { 10979 pgcnt++; 10980 addr += MMU_PAGESIZE; 10981 bitvec >>= 1; 10982 } 10983 while (bitvec & 1) { 10984 dirtypg++; 10985 bitvec >>= 1; 10986 } 10987 eaddr = addr + ptob(dirtypg); 10988 hatlockp = sfmmu_hat_enter(sfmmup); 10989 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K); 10990 sfmmu_hat_exit(hatlockp); 10991 pgunload += dirtypg; 10992 addr = eaddr; 10993 pgcnt += dirtypg; 10994 } 10995 10996 ASSERT((pgcnt<<MMU_PAGESHIFT) <= dmrp->dmr_endaddr - dmrp->dmr_addr); 10997 if (sfmmup->sfmmu_free == 0) { 10998 addr = dmrp->dmr_addr; 10999 bitvec = dmrp->dmr_bitvec; 11000 11001 /* 11002 * make sure it has SFMMU_PGCNT_SHIFT bits only, 11003 * as it will be used to pack argument for xt_some 11004 */ 11005 ASSERT((pgcnt > 0) && 11006 (pgcnt <= (1 << SFMMU_PGCNT_SHIFT))); 11007 11008 /* 11009 * Encode pgcnt as (pgcnt -1 ), and pass (pgcnt - 1) in 11010 * the low 6 bits of sfmmup. This is doable since pgcnt 11011 * always >= 1. 11012 */ 11013 ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK)); 11014 sfmmu_pgcnt = (uint64_t)sfmmup | 11015 ((pgcnt - 1) & SFMMU_PGCNT_MASK); 11016 11017 /* 11018 * We must hold the hat lock during the flush of TLB, 11019 * to avoid a race with sfmmu_invalidate_ctx(), where 11020 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 11021 * causing TLB demap routine to skip flush on that MMU. 11022 * If the context on a MMU has already been set to 11023 * INVALID_CONTEXT, we just get an extra flush on 11024 * that MMU. 11025 */ 11026 hatlockp = sfmmu_hat_enter(sfmmup); 11027 kpreempt_disable(); 11028 11029 cpuset = sfmmup->sfmmu_cpusran; 11030 CPUSET_AND(cpuset, cpu_ready_set); 11031 CPUSET_DEL(cpuset, CPU->cpu_id); 11032 11033 SFMMU_XCALL_STATS(sfmmup); 11034 xt_some(cpuset, vtag_flush_pgcnt_tl1, (uint64_t)addr, 11035 sfmmu_pgcnt); 11036 11037 for (; bitvec != 0; bitvec >>= 1) { 11038 if (bitvec & 1) 11039 vtag_flushpage(addr, (uint64_t)sfmmup); 11040 addr += MMU_PAGESIZE; 11041 } 11042 kpreempt_enable(); 11043 sfmmu_hat_exit(hatlockp); 11044 11045 sfmmu_xcall_save += (pgunload-1); 11046 } 11047 dmrp->dmr_bitvec = 0; 11048 } 11049 11050 /* 11051 * In cases where we need to synchronize with TLB/TSB miss trap 11052 * handlers, _and_ need to flush the TLB, it's a lot easier to 11053 * throw away the context from the process than to do a 11054 * special song and dance to keep things consistent for the 11055 * handlers. 11056 * 11057 * Since the process suddenly ends up without a context and our caller 11058 * holds the hat lock, threads that fault after this function is called 11059 * will pile up on the lock. We can then do whatever we need to 11060 * atomically from the context of the caller. The first blocked thread 11061 * to resume executing will get the process a new context, and the 11062 * process will resume executing. 11063 * 11064 * One added advantage of this approach is that on MMUs that 11065 * support a "flush all" operation, we will delay the flush until 11066 * cnum wrap-around, and then flush the TLB one time. This 11067 * is rather rare, so it's a lot less expensive than making 8000 11068 * x-calls to flush the TLB 8000 times. 11069 * 11070 * A per-process (PP) lock is used to synchronize ctx allocations in 11071 * resume() and ctx invalidations here. 11072 */ 11073 static void 11074 sfmmu_invalidate_ctx(sfmmu_t *sfmmup) 11075 { 11076 cpuset_t cpuset; 11077 int cnum, currcnum; 11078 mmu_ctx_t *mmu_ctxp; 11079 int i; 11080 uint_t pstate_save; 11081 11082 SFMMU_STAT(sf_ctx_inv); 11083 11084 ASSERT(sfmmu_hat_lock_held(sfmmup)); 11085 ASSERT(sfmmup != ksfmmup); 11086 11087 kpreempt_disable(); 11088 11089 mmu_ctxp = CPU_MMU_CTXP(CPU); 11090 ASSERT(mmu_ctxp); 11091 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 11092 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 11093 11094 currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; 11095 11096 pstate_save = sfmmu_disable_intrs(); 11097 11098 lock_set(&sfmmup->sfmmu_ctx_lock); /* acquire PP lock */ 11099 /* set HAT cnum invalid across all context domains. */ 11100 for (i = 0; i < max_mmu_ctxdoms; i++) { 11101 11102 cnum = sfmmup->sfmmu_ctxs[i].cnum; 11103 if (cnum == INVALID_CONTEXT) { 11104 continue; 11105 } 11106 11107 sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT; 11108 } 11109 membar_enter(); /* make sure globally visible to all CPUs */ 11110 lock_clear(&sfmmup->sfmmu_ctx_lock); /* release PP lock */ 11111 11112 sfmmu_enable_intrs(pstate_save); 11113 11114 cpuset = sfmmup->sfmmu_cpusran; 11115 CPUSET_DEL(cpuset, CPU->cpu_id); 11116 CPUSET_AND(cpuset, cpu_ready_set); 11117 if (!CPUSET_ISNULL(cpuset)) { 11118 SFMMU_XCALL_STATS(sfmmup); 11119 xt_some(cpuset, sfmmu_raise_tsb_exception, 11120 (uint64_t)sfmmup, INVALID_CONTEXT); 11121 xt_sync(cpuset); 11122 SFMMU_STAT(sf_tsb_raise_exception); 11123 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 11124 } 11125 11126 /* 11127 * If the hat to-be-invalidated is the same as the current 11128 * process on local CPU we need to invalidate 11129 * this CPU context as well. 11130 */ 11131 if ((sfmmu_getctx_sec() == currcnum) && 11132 (currcnum != INVALID_CONTEXT)) { 11133 sfmmu_setctx_sec(INVALID_CONTEXT); 11134 sfmmu_clear_utsbinfo(); 11135 } 11136 11137 kpreempt_enable(); 11138 11139 /* 11140 * we hold the hat lock, so nobody should allocate a context 11141 * for us yet 11142 */ 11143 ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT); 11144 } 11145 11146 #ifdef VAC 11147 /* 11148 * We need to flush the cache in all cpus. It is possible that 11149 * a process referenced a page as cacheable but has sinced exited 11150 * and cleared the mapping list. We still to flush it but have no 11151 * state so all cpus is the only alternative. 11152 */ 11153 void 11154 sfmmu_cache_flush(pfn_t pfnum, int vcolor) 11155 { 11156 cpuset_t cpuset; 11157 11158 kpreempt_disable(); 11159 cpuset = cpu_ready_set; 11160 CPUSET_DEL(cpuset, CPU->cpu_id); 11161 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11162 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 11163 xt_sync(cpuset); 11164 vac_flushpage(pfnum, vcolor); 11165 kpreempt_enable(); 11166 } 11167 11168 void 11169 sfmmu_cache_flushcolor(int vcolor, pfn_t pfnum) 11170 { 11171 cpuset_t cpuset; 11172 11173 ASSERT(vcolor >= 0); 11174 11175 kpreempt_disable(); 11176 cpuset = cpu_ready_set; 11177 CPUSET_DEL(cpuset, CPU->cpu_id); 11178 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11179 xt_some(cpuset, vac_flushcolor_tl1, vcolor, pfnum); 11180 xt_sync(cpuset); 11181 vac_flushcolor(vcolor, pfnum); 11182 kpreempt_enable(); 11183 } 11184 #endif /* VAC */ 11185 11186 /* 11187 * We need to prevent processes from accessing the TSB using a cached physical 11188 * address. It's alright if they try to access the TSB via virtual address 11189 * since they will just fault on that virtual address once the mapping has 11190 * been suspended. 11191 */ 11192 #pragma weak sendmondo_in_recover 11193 11194 /* ARGSUSED */ 11195 static int 11196 sfmmu_tsb_pre_relocator(caddr_t va, uint_t tsbsz, uint_t flags, void *tsbinfo) 11197 { 11198 hatlock_t *hatlockp; 11199 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11200 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11201 extern uint32_t sendmondo_in_recover; 11202 11203 if (flags != HAT_PRESUSPEND) 11204 return (0); 11205 11206 hatlockp = sfmmu_hat_enter(sfmmup); 11207 11208 tsbinfop->tsb_flags |= TSB_RELOC_FLAG; 11209 11210 /* 11211 * For Cheetah+ Erratum 25: 11212 * Wait for any active recovery to finish. We can't risk 11213 * relocating the TSB of the thread running mondo_recover_proc() 11214 * since, if we did that, we would deadlock. The scenario we are 11215 * trying to avoid is as follows: 11216 * 11217 * THIS CPU RECOVER CPU 11218 * -------- ----------- 11219 * Begins recovery, walking through TSB 11220 * hat_pagesuspend() TSB TTE 11221 * TLB miss on TSB TTE, spins at TL1 11222 * xt_sync() 11223 * send_mondo_timeout() 11224 * mondo_recover_proc() 11225 * ((deadlocked)) 11226 * 11227 * The second half of the workaround is that mondo_recover_proc() 11228 * checks to see if the tsb_info has the RELOC flag set, and if it 11229 * does, it skips over that TSB without ever touching tsbinfop->tsb_va 11230 * and hence avoiding the TLB miss that could result in a deadlock. 11231 */ 11232 if (&sendmondo_in_recover) { 11233 membar_enter(); /* make sure RELOC flag visible */ 11234 while (sendmondo_in_recover) { 11235 drv_usecwait(1); 11236 membar_consumer(); 11237 } 11238 } 11239 11240 sfmmu_invalidate_ctx(sfmmup); 11241 sfmmu_hat_exit(hatlockp); 11242 11243 return (0); 11244 } 11245 11246 /* ARGSUSED */ 11247 static int 11248 sfmmu_tsb_post_relocator(caddr_t va, uint_t tsbsz, uint_t flags, 11249 void *tsbinfo, pfn_t newpfn) 11250 { 11251 hatlock_t *hatlockp; 11252 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11253 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11254 11255 if (flags != HAT_POSTUNSUSPEND) 11256 return (0); 11257 11258 hatlockp = sfmmu_hat_enter(sfmmup); 11259 11260 SFMMU_STAT(sf_tsb_reloc); 11261 11262 /* 11263 * The process may have swapped out while we were relocating one 11264 * of its TSBs. If so, don't bother doing the setup since the 11265 * process can't be using the memory anymore. 11266 */ 11267 if ((tsbinfop->tsb_flags & TSB_SWAPPED) == 0) { 11268 ASSERT(va == tsbinfop->tsb_va); 11269 sfmmu_tsbinfo_setup_phys(tsbinfop, newpfn); 11270 sfmmu_setup_tsbinfo(sfmmup); 11271 11272 if (tsbinfop->tsb_flags & TSB_FLUSH_NEEDED) { 11273 sfmmu_inv_tsb(tsbinfop->tsb_va, 11274 TSB_BYTES(tsbinfop->tsb_szc)); 11275 tsbinfop->tsb_flags &= ~TSB_FLUSH_NEEDED; 11276 } 11277 } 11278 11279 membar_exit(); 11280 tsbinfop->tsb_flags &= ~TSB_RELOC_FLAG; 11281 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 11282 11283 sfmmu_hat_exit(hatlockp); 11284 11285 return (0); 11286 } 11287 11288 /* 11289 * Allocate and initialize a tsb_info structure. Note that we may or may not 11290 * allocate a TSB here, depending on the flags passed in. 11291 */ 11292 static int 11293 sfmmu_tsbinfo_alloc(struct tsb_info **tsbinfopp, int tsb_szc, int tte_sz_mask, 11294 uint_t flags, sfmmu_t *sfmmup) 11295 { 11296 int err; 11297 11298 *tsbinfopp = (struct tsb_info *)kmem_cache_alloc( 11299 sfmmu_tsbinfo_cache, KM_SLEEP); 11300 11301 if ((err = sfmmu_init_tsbinfo(*tsbinfopp, tte_sz_mask, 11302 tsb_szc, flags, sfmmup)) != 0) { 11303 kmem_cache_free(sfmmu_tsbinfo_cache, *tsbinfopp); 11304 SFMMU_STAT(sf_tsb_allocfail); 11305 *tsbinfopp = NULL; 11306 return (err); 11307 } 11308 SFMMU_STAT(sf_tsb_alloc); 11309 11310 /* 11311 * Bump the TSB size counters for this TSB size. 11312 */ 11313 (*(((int *)&sfmmu_tsbsize_stat) + tsb_szc))++; 11314 return (0); 11315 } 11316 11317 static void 11318 sfmmu_tsb_free(struct tsb_info *tsbinfo) 11319 { 11320 caddr_t tsbva = tsbinfo->tsb_va; 11321 uint_t tsb_size = TSB_BYTES(tsbinfo->tsb_szc); 11322 struct kmem_cache *kmem_cachep = tsbinfo->tsb_cache; 11323 vmem_t *vmp = tsbinfo->tsb_vmp; 11324 11325 /* 11326 * If we allocated this TSB from relocatable kernel memory, then we 11327 * need to uninstall the callback handler. 11328 */ 11329 if (tsbinfo->tsb_cache != sfmmu_tsb8k_cache) { 11330 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11331 caddr_t slab_vaddr = (caddr_t)((uintptr_t)tsbva & slab_mask); 11332 page_t **ppl; 11333 int ret; 11334 11335 ret = as_pagelock(&kas, &ppl, slab_vaddr, PAGESIZE, S_WRITE); 11336 ASSERT(ret == 0); 11337 hat_delete_callback(tsbva, (uint_t)tsb_size, (void *)tsbinfo, 11338 0, NULL); 11339 as_pageunlock(&kas, ppl, slab_vaddr, PAGESIZE, S_WRITE); 11340 } 11341 11342 if (kmem_cachep != NULL) { 11343 kmem_cache_free(kmem_cachep, tsbva); 11344 } else { 11345 vmem_xfree(vmp, (void *)tsbva, tsb_size); 11346 } 11347 tsbinfo->tsb_va = (caddr_t)0xbad00bad; 11348 atomic_add_64(&tsb_alloc_bytes, -(int64_t)tsb_size); 11349 } 11350 11351 static void 11352 sfmmu_tsbinfo_free(struct tsb_info *tsbinfo) 11353 { 11354 if ((tsbinfo->tsb_flags & TSB_SWAPPED) == 0) { 11355 sfmmu_tsb_free(tsbinfo); 11356 } 11357 kmem_cache_free(sfmmu_tsbinfo_cache, tsbinfo); 11358 11359 } 11360 11361 /* 11362 * Setup all the references to physical memory for this tsbinfo. 11363 * The underlying page(s) must be locked. 11364 */ 11365 static void 11366 sfmmu_tsbinfo_setup_phys(struct tsb_info *tsbinfo, pfn_t pfn) 11367 { 11368 ASSERT(pfn != PFN_INVALID); 11369 ASSERT(pfn == va_to_pfn(tsbinfo->tsb_va)); 11370 11371 #ifndef sun4v 11372 if (tsbinfo->tsb_szc == 0) { 11373 sfmmu_memtte(&tsbinfo->tsb_tte, pfn, 11374 PROT_WRITE|PROT_READ, TTE8K); 11375 } else { 11376 /* 11377 * Round down PA and use a large mapping; the handlers will 11378 * compute the TSB pointer at the correct offset into the 11379 * big virtual page. NOTE: this assumes all TSBs larger 11380 * than 8K must come from physically contiguous slabs of 11381 * size tsb_slab_size. 11382 */ 11383 sfmmu_memtte(&tsbinfo->tsb_tte, pfn & ~tsb_slab_mask, 11384 PROT_WRITE|PROT_READ, tsb_slab_ttesz); 11385 } 11386 tsbinfo->tsb_pa = ptob(pfn); 11387 11388 TTE_SET_LOCKED(&tsbinfo->tsb_tte); /* lock the tte into dtlb */ 11389 TTE_SET_MOD(&tsbinfo->tsb_tte); /* enable writes */ 11390 11391 ASSERT(TTE_IS_PRIVILEGED(&tsbinfo->tsb_tte)); 11392 ASSERT(TTE_IS_LOCKED(&tsbinfo->tsb_tte)); 11393 #else /* sun4v */ 11394 tsbinfo->tsb_pa = ptob(pfn); 11395 #endif /* sun4v */ 11396 } 11397 11398 11399 /* 11400 * Returns zero on success, ENOMEM if over the high water mark, 11401 * or EAGAIN if the caller needs to retry with a smaller TSB 11402 * size (or specify TSB_FORCEALLOC if the allocation can't fail). 11403 * 11404 * This call cannot fail to allocate a TSB if TSB_FORCEALLOC 11405 * is specified and the TSB requested is PAGESIZE, though it 11406 * may sleep waiting for memory if sufficient memory is not 11407 * available. 11408 */ 11409 static int 11410 sfmmu_init_tsbinfo(struct tsb_info *tsbinfo, int tteszmask, 11411 int tsbcode, uint_t flags, sfmmu_t *sfmmup) 11412 { 11413 caddr_t vaddr = NULL; 11414 caddr_t slab_vaddr; 11415 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11416 int tsbbytes = TSB_BYTES(tsbcode); 11417 int lowmem = 0; 11418 struct kmem_cache *kmem_cachep = NULL; 11419 vmem_t *vmp = NULL; 11420 lgrp_id_t lgrpid = LGRP_NONE; 11421 pfn_t pfn; 11422 uint_t cbflags = HAC_SLEEP; 11423 page_t **pplist; 11424 int ret; 11425 11426 if (flags & (TSB_FORCEALLOC | TSB_SWAPIN | TSB_GROW | TSB_SHRINK)) 11427 flags |= TSB_ALLOC; 11428 11429 ASSERT((flags & TSB_FORCEALLOC) == 0 || tsbcode == TSB_MIN_SZCODE); 11430 11431 tsbinfo->tsb_sfmmu = sfmmup; 11432 11433 /* 11434 * If not allocating a TSB, set up the tsbinfo, set TSB_SWAPPED, and 11435 * return. 11436 */ 11437 if ((flags & TSB_ALLOC) == 0) { 11438 tsbinfo->tsb_szc = tsbcode; 11439 tsbinfo->tsb_ttesz_mask = tteszmask; 11440 tsbinfo->tsb_va = (caddr_t)0xbadbadbeef; 11441 tsbinfo->tsb_pa = -1; 11442 tsbinfo->tsb_tte.ll = 0; 11443 tsbinfo->tsb_next = NULL; 11444 tsbinfo->tsb_flags = TSB_SWAPPED; 11445 tsbinfo->tsb_cache = NULL; 11446 tsbinfo->tsb_vmp = NULL; 11447 return (0); 11448 } 11449 11450 #ifdef DEBUG 11451 /* 11452 * For debugging: 11453 * Randomly force allocation failures every tsb_alloc_mtbf 11454 * tries if TSB_FORCEALLOC is not specified. This will 11455 * return ENOMEM if tsb_alloc_mtbf is odd, or EAGAIN if 11456 * it is even, to allow testing of both failure paths... 11457 */ 11458 if (tsb_alloc_mtbf && ((flags & TSB_FORCEALLOC) == 0) && 11459 (tsb_alloc_count++ == tsb_alloc_mtbf)) { 11460 tsb_alloc_count = 0; 11461 tsb_alloc_fail_mtbf++; 11462 return ((tsb_alloc_mtbf & 1)? ENOMEM : EAGAIN); 11463 } 11464 #endif /* DEBUG */ 11465 11466 /* 11467 * Enforce high water mark if we are not doing a forced allocation 11468 * and are not shrinking a process' TSB. 11469 */ 11470 if ((flags & TSB_SHRINK) == 0 && 11471 (tsbbytes + tsb_alloc_bytes) > tsb_alloc_hiwater) { 11472 if ((flags & TSB_FORCEALLOC) == 0) 11473 return (ENOMEM); 11474 lowmem = 1; 11475 } 11476 11477 /* 11478 * Allocate from the correct location based upon the size of the TSB 11479 * compared to the base page size, and what memory conditions dictate. 11480 * Note we always do nonblocking allocations from the TSB arena since 11481 * we don't want memory fragmentation to cause processes to block 11482 * indefinitely waiting for memory; until the kernel algorithms that 11483 * coalesce large pages are improved this is our best option. 11484 * 11485 * Algorithm: 11486 * If allocating a "large" TSB (>8K), allocate from the 11487 * appropriate kmem_tsb_default_arena vmem arena 11488 * else if low on memory or the TSB_FORCEALLOC flag is set or 11489 * tsb_forceheap is set 11490 * Allocate from kernel heap via sfmmu_tsb8k_cache with 11491 * KM_SLEEP (never fails) 11492 * else 11493 * Allocate from appropriate sfmmu_tsb_cache with 11494 * KM_NOSLEEP 11495 * endif 11496 */ 11497 if (tsb_lgrp_affinity) 11498 lgrpid = lgrp_home_id(curthread); 11499 if (lgrpid == LGRP_NONE) 11500 lgrpid = 0; /* use lgrp of boot CPU */ 11501 11502 if (tsbbytes > MMU_PAGESIZE) { 11503 vmp = kmem_tsb_default_arena[lgrpid]; 11504 vaddr = (caddr_t)vmem_xalloc(vmp, tsbbytes, tsbbytes, 0, 0, 11505 NULL, NULL, VM_NOSLEEP); 11506 #ifdef DEBUG 11507 } else if (lowmem || (flags & TSB_FORCEALLOC) || tsb_forceheap) { 11508 #else /* !DEBUG */ 11509 } else if (lowmem || (flags & TSB_FORCEALLOC)) { 11510 #endif /* DEBUG */ 11511 kmem_cachep = sfmmu_tsb8k_cache; 11512 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_SLEEP); 11513 ASSERT(vaddr != NULL); 11514 } else { 11515 kmem_cachep = sfmmu_tsb_cache[lgrpid]; 11516 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_NOSLEEP); 11517 } 11518 11519 tsbinfo->tsb_cache = kmem_cachep; 11520 tsbinfo->tsb_vmp = vmp; 11521 11522 if (vaddr == NULL) { 11523 return (EAGAIN); 11524 } 11525 11526 atomic_add_64(&tsb_alloc_bytes, (int64_t)tsbbytes); 11527 kmem_cachep = tsbinfo->tsb_cache; 11528 11529 /* 11530 * If we are allocating from outside the cage, then we need to 11531 * register a relocation callback handler. Note that for now 11532 * since pseudo mappings always hang off of the slab's root page, 11533 * we need only lock the first 8K of the TSB slab. This is a bit 11534 * hacky but it is good for performance. 11535 */ 11536 if (kmem_cachep != sfmmu_tsb8k_cache) { 11537 slab_vaddr = (caddr_t)((uintptr_t)vaddr & slab_mask); 11538 ret = as_pagelock(&kas, &pplist, slab_vaddr, PAGESIZE, S_WRITE); 11539 ASSERT(ret == 0); 11540 ret = hat_add_callback(sfmmu_tsb_cb_id, vaddr, (uint_t)tsbbytes, 11541 cbflags, (void *)tsbinfo, &pfn, NULL); 11542 11543 /* 11544 * Need to free up resources if we could not successfully 11545 * add the callback function and return an error condition. 11546 */ 11547 if (ret != 0) { 11548 if (kmem_cachep) { 11549 kmem_cache_free(kmem_cachep, vaddr); 11550 } else { 11551 vmem_xfree(vmp, (void *)vaddr, tsbbytes); 11552 } 11553 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, 11554 S_WRITE); 11555 return (EAGAIN); 11556 } 11557 } else { 11558 /* 11559 * Since allocation of 8K TSBs from heap is rare and occurs 11560 * during memory pressure we allocate them from permanent 11561 * memory rather than using callbacks to get the PFN. 11562 */ 11563 pfn = hat_getpfnum(kas.a_hat, vaddr); 11564 } 11565 11566 tsbinfo->tsb_va = vaddr; 11567 tsbinfo->tsb_szc = tsbcode; 11568 tsbinfo->tsb_ttesz_mask = tteszmask; 11569 tsbinfo->tsb_next = NULL; 11570 tsbinfo->tsb_flags = 0; 11571 11572 sfmmu_tsbinfo_setup_phys(tsbinfo, pfn); 11573 11574 if (kmem_cachep != sfmmu_tsb8k_cache) { 11575 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, S_WRITE); 11576 } 11577 11578 sfmmu_inv_tsb(vaddr, tsbbytes); 11579 return (0); 11580 } 11581 11582 /* 11583 * Initialize per cpu tsb and per cpu tsbmiss_area 11584 */ 11585 void 11586 sfmmu_init_tsbs(void) 11587 { 11588 int i; 11589 struct tsbmiss *tsbmissp; 11590 struct kpmtsbm *kpmtsbmp; 11591 #ifndef sun4v 11592 extern int dcache_line_mask; 11593 #endif /* sun4v */ 11594 extern uint_t vac_colors; 11595 11596 /* 11597 * Init. tsb miss area. 11598 */ 11599 tsbmissp = tsbmiss_area; 11600 11601 for (i = 0; i < NCPU; tsbmissp++, i++) { 11602 /* 11603 * initialize the tsbmiss area. 11604 * Do this for all possible CPUs as some may be added 11605 * while the system is running. There is no cost to this. 11606 */ 11607 tsbmissp->ksfmmup = ksfmmup; 11608 #ifndef sun4v 11609 tsbmissp->dcache_line_mask = (uint16_t)dcache_line_mask; 11610 #endif /* sun4v */ 11611 tsbmissp->khashstart = 11612 (struct hmehash_bucket *)va_to_pa((caddr_t)khme_hash); 11613 tsbmissp->uhashstart = 11614 (struct hmehash_bucket *)va_to_pa((caddr_t)uhme_hash); 11615 tsbmissp->khashsz = khmehash_num; 11616 tsbmissp->uhashsz = uhmehash_num; 11617 } 11618 11619 sfmmu_tsb_cb_id = hat_register_callback('T'<<16 | 'S' << 8 | 'B', 11620 sfmmu_tsb_pre_relocator, sfmmu_tsb_post_relocator, NULL, 0); 11621 11622 if (kpm_enable == 0) 11623 return; 11624 11625 /* -- Begin KPM specific init -- */ 11626 11627 if (kpm_smallpages) { 11628 /* 11629 * If we're using base pagesize pages for seg_kpm 11630 * mappings, we use the kernel TSB since we can't afford 11631 * to allocate a second huge TSB for these mappings. 11632 */ 11633 kpm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11634 kpm_tsbsz = ktsb_szcode; 11635 kpmsm_tsbbase = kpm_tsbbase; 11636 kpmsm_tsbsz = kpm_tsbsz; 11637 } else { 11638 /* 11639 * In VAC conflict case, just put the entries in the 11640 * kernel 8K indexed TSB for now so we can find them. 11641 * This could really be changed in the future if we feel 11642 * the need... 11643 */ 11644 kpmsm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11645 kpmsm_tsbsz = ktsb_szcode; 11646 kpm_tsbbase = ktsb_phys? ktsb4m_pbase : (uint64_t)ktsb4m_base; 11647 kpm_tsbsz = ktsb4m_szcode; 11648 } 11649 11650 kpmtsbmp = kpmtsbm_area; 11651 for (i = 0; i < NCPU; kpmtsbmp++, i++) { 11652 /* 11653 * Initialize the kpmtsbm area. 11654 * Do this for all possible CPUs as some may be added 11655 * while the system is running. There is no cost to this. 11656 */ 11657 kpmtsbmp->vbase = kpm_vbase; 11658 kpmtsbmp->vend = kpm_vbase + kpm_size * vac_colors; 11659 kpmtsbmp->sz_shift = kpm_size_shift; 11660 kpmtsbmp->kpmp_shift = kpmp_shift; 11661 kpmtsbmp->kpmp2pshft = (uchar_t)kpmp2pshft; 11662 if (kpm_smallpages == 0) { 11663 kpmtsbmp->kpmp_table_sz = kpmp_table_sz; 11664 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_table); 11665 } else { 11666 kpmtsbmp->kpmp_table_sz = kpmp_stable_sz; 11667 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_stable); 11668 } 11669 kpmtsbmp->msegphashpa = va_to_pa(memseg_phash); 11670 kpmtsbmp->flags = KPMTSBM_ENABLE_FLAG; 11671 #ifdef DEBUG 11672 kpmtsbmp->flags |= (kpm_tsbmtl) ? KPMTSBM_TLTSBM_FLAG : 0; 11673 #endif /* DEBUG */ 11674 if (ktsb_phys) 11675 kpmtsbmp->flags |= KPMTSBM_TSBPHYS_FLAG; 11676 } 11677 11678 /* -- End KPM specific init -- */ 11679 } 11680 11681 /* Avoid using sfmmu_tsbinfo_alloc() to avoid kmem_alloc - no real reason */ 11682 struct tsb_info ktsb_info[2]; 11683 11684 /* 11685 * Called from hat_kern_setup() to setup the tsb_info for ksfmmup. 11686 */ 11687 void 11688 sfmmu_init_ktsbinfo() 11689 { 11690 ASSERT(ksfmmup != NULL); 11691 ASSERT(ksfmmup->sfmmu_tsb == NULL); 11692 /* 11693 * Allocate tsbinfos for kernel and copy in data 11694 * to make debug easier and sun4v setup easier. 11695 */ 11696 ktsb_info[0].tsb_sfmmu = ksfmmup; 11697 ktsb_info[0].tsb_szc = ktsb_szcode; 11698 ktsb_info[0].tsb_ttesz_mask = TSB8K|TSB64K|TSB512K; 11699 ktsb_info[0].tsb_va = ktsb_base; 11700 ktsb_info[0].tsb_pa = ktsb_pbase; 11701 ktsb_info[0].tsb_flags = 0; 11702 ktsb_info[0].tsb_tte.ll = 0; 11703 ktsb_info[0].tsb_cache = NULL; 11704 11705 ktsb_info[1].tsb_sfmmu = ksfmmup; 11706 ktsb_info[1].tsb_szc = ktsb4m_szcode; 11707 ktsb_info[1].tsb_ttesz_mask = TSB4M; 11708 ktsb_info[1].tsb_va = ktsb4m_base; 11709 ktsb_info[1].tsb_pa = ktsb4m_pbase; 11710 ktsb_info[1].tsb_flags = 0; 11711 ktsb_info[1].tsb_tte.ll = 0; 11712 ktsb_info[1].tsb_cache = NULL; 11713 11714 /* Link them into ksfmmup. */ 11715 ktsb_info[0].tsb_next = &ktsb_info[1]; 11716 ktsb_info[1].tsb_next = NULL; 11717 ksfmmup->sfmmu_tsb = &ktsb_info[0]; 11718 11719 sfmmu_setup_tsbinfo(ksfmmup); 11720 } 11721 11722 /* 11723 * Cache the last value returned from va_to_pa(). If the VA specified 11724 * in the current call to cached_va_to_pa() maps to the same Page (as the 11725 * previous call to cached_va_to_pa()), then compute the PA using 11726 * cached info, else call va_to_pa(). 11727 * 11728 * Note: this function is neither MT-safe nor consistent in the presence 11729 * of multiple, interleaved threads. This function was created to enable 11730 * an optimization used during boot (at a point when there's only one thread 11731 * executing on the "boot CPU", and before startup_vm() has been called). 11732 */ 11733 static uint64_t 11734 cached_va_to_pa(void *vaddr) 11735 { 11736 static uint64_t prev_vaddr_base = 0; 11737 static uint64_t prev_pfn = 0; 11738 11739 if ((((uint64_t)vaddr) & MMU_PAGEMASK) == prev_vaddr_base) { 11740 return (prev_pfn | ((uint64_t)vaddr & MMU_PAGEOFFSET)); 11741 } else { 11742 uint64_t pa = va_to_pa(vaddr); 11743 11744 if (pa != ((uint64_t)-1)) { 11745 /* 11746 * Computed physical address is valid. Cache its 11747 * related info for the next cached_va_to_pa() call. 11748 */ 11749 prev_pfn = pa & MMU_PAGEMASK; 11750 prev_vaddr_base = ((uint64_t)vaddr) & MMU_PAGEMASK; 11751 } 11752 11753 return (pa); 11754 } 11755 } 11756 11757 /* 11758 * Carve up our nucleus hblk region. We may allocate more hblks than 11759 * asked due to rounding errors but we are guaranteed to have at least 11760 * enough space to allocate the requested number of hblk8's and hblk1's. 11761 */ 11762 void 11763 sfmmu_init_nucleus_hblks(caddr_t addr, size_t size, int nhblk8, int nhblk1) 11764 { 11765 struct hme_blk *hmeblkp; 11766 size_t hme8blk_sz, hme1blk_sz; 11767 size_t i; 11768 size_t hblk8_bound; 11769 ulong_t j = 0, k = 0; 11770 11771 ASSERT(addr != NULL && size != 0); 11772 11773 /* Need to use proper structure alignment */ 11774 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 11775 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 11776 11777 nucleus_hblk8.list = (void *)addr; 11778 nucleus_hblk8.index = 0; 11779 11780 /* 11781 * Use as much memory as possible for hblk8's since we 11782 * expect all bop_alloc'ed memory to be allocated in 8k chunks. 11783 * We need to hold back enough space for the hblk1's which 11784 * we'll allocate next. 11785 */ 11786 hblk8_bound = size - (nhblk1 * hme1blk_sz) - hme8blk_sz; 11787 for (i = 0; i <= hblk8_bound; i += hme8blk_sz, j++) { 11788 hmeblkp = (struct hme_blk *)addr; 11789 addr += hme8blk_sz; 11790 hmeblkp->hblk_nuc_bit = 1; 11791 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11792 } 11793 nucleus_hblk8.len = j; 11794 ASSERT(j >= nhblk8); 11795 SFMMU_STAT_ADD(sf_hblk8_ncreate, j); 11796 11797 nucleus_hblk1.list = (void *)addr; 11798 nucleus_hblk1.index = 0; 11799 for (; i <= (size - hme1blk_sz); i += hme1blk_sz, k++) { 11800 hmeblkp = (struct hme_blk *)addr; 11801 addr += hme1blk_sz; 11802 hmeblkp->hblk_nuc_bit = 1; 11803 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11804 } 11805 ASSERT(k >= nhblk1); 11806 nucleus_hblk1.len = k; 11807 SFMMU_STAT_ADD(sf_hblk1_ncreate, k); 11808 } 11809 11810 /* 11811 * This function is currently not supported on this platform. For what 11812 * it's supposed to do, see hat.c and hat_srmmu.c 11813 */ 11814 /* ARGSUSED */ 11815 faultcode_t 11816 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp, 11817 uint_t flags) 11818 { 11819 ASSERT(hat->sfmmu_xhat_provider == NULL); 11820 return (FC_NOSUPPORT); 11821 } 11822 11823 /* 11824 * Searchs the mapping list of the page for a mapping of the same size. If not 11825 * found the corresponding bit is cleared in the p_index field. When large 11826 * pages are more prevalent in the system, we can maintain the mapping list 11827 * in order and we don't have to traverse the list each time. Just check the 11828 * next and prev entries, and if both are of different size, we clear the bit. 11829 */ 11830 static void 11831 sfmmu_rm_large_mappings(page_t *pp, int ttesz) 11832 { 11833 struct sf_hment *sfhmep; 11834 struct hme_blk *hmeblkp; 11835 int index; 11836 pgcnt_t npgs; 11837 11838 ASSERT(ttesz > TTE8K); 11839 11840 ASSERT(sfmmu_mlist_held(pp)); 11841 11842 ASSERT(PP_ISMAPPED_LARGE(pp)); 11843 11844 /* 11845 * Traverse mapping list looking for another mapping of same size. 11846 * since we only want to clear index field if all mappings of 11847 * that size are gone. 11848 */ 11849 11850 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 11851 hmeblkp = sfmmu_hmetohblk(sfhmep); 11852 if (hmeblkp->hblk_xhat_bit) 11853 continue; 11854 if (hme_size(sfhmep) == ttesz) { 11855 /* 11856 * another mapping of the same size. don't clear index. 11857 */ 11858 return; 11859 } 11860 } 11861 11862 /* 11863 * Clear the p_index bit for large page. 11864 */ 11865 index = PAGESZ_TO_INDEX(ttesz); 11866 npgs = TTEPAGES(ttesz); 11867 while (npgs-- > 0) { 11868 ASSERT(pp->p_index & index); 11869 pp->p_index &= ~index; 11870 pp = PP_PAGENEXT(pp); 11871 } 11872 } 11873 11874 /* 11875 * return supported features 11876 */ 11877 /* ARGSUSED */ 11878 int 11879 hat_supported(enum hat_features feature, void *arg) 11880 { 11881 switch (feature) { 11882 case HAT_SHARED_PT: 11883 case HAT_DYNAMIC_ISM_UNMAP: 11884 case HAT_VMODSORT: 11885 return (1); 11886 default: 11887 return (0); 11888 } 11889 } 11890 11891 void 11892 hat_enter(struct hat *hat) 11893 { 11894 hatlock_t *hatlockp; 11895 11896 if (hat != ksfmmup) { 11897 hatlockp = TSB_HASH(hat); 11898 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 11899 } 11900 } 11901 11902 void 11903 hat_exit(struct hat *hat) 11904 { 11905 hatlock_t *hatlockp; 11906 11907 if (hat != ksfmmup) { 11908 hatlockp = TSB_HASH(hat); 11909 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 11910 } 11911 } 11912 11913 /*ARGSUSED*/ 11914 void 11915 hat_reserve(struct as *as, caddr_t addr, size_t len) 11916 { 11917 } 11918 11919 static void 11920 hat_kstat_init(void) 11921 { 11922 kstat_t *ksp; 11923 11924 ksp = kstat_create("unix", 0, "sfmmu_global_stat", "hat", 11925 KSTAT_TYPE_RAW, sizeof (struct sfmmu_global_stat), 11926 KSTAT_FLAG_VIRTUAL); 11927 if (ksp) { 11928 ksp->ks_data = (void *) &sfmmu_global_stat; 11929 kstat_install(ksp); 11930 } 11931 ksp = kstat_create("unix", 0, "sfmmu_tsbsize_stat", "hat", 11932 KSTAT_TYPE_RAW, sizeof (struct sfmmu_tsbsize_stat), 11933 KSTAT_FLAG_VIRTUAL); 11934 if (ksp) { 11935 ksp->ks_data = (void *) &sfmmu_tsbsize_stat; 11936 kstat_install(ksp); 11937 } 11938 ksp = kstat_create("unix", 0, "sfmmu_percpu_stat", "hat", 11939 KSTAT_TYPE_RAW, sizeof (struct sfmmu_percpu_stat) * NCPU, 11940 KSTAT_FLAG_WRITABLE); 11941 if (ksp) { 11942 ksp->ks_update = sfmmu_kstat_percpu_update; 11943 kstat_install(ksp); 11944 } 11945 } 11946 11947 /* ARGSUSED */ 11948 static int 11949 sfmmu_kstat_percpu_update(kstat_t *ksp, int rw) 11950 { 11951 struct sfmmu_percpu_stat *cpu_kstat = ksp->ks_data; 11952 struct tsbmiss *tsbm = tsbmiss_area; 11953 struct kpmtsbm *kpmtsbm = kpmtsbm_area; 11954 int i; 11955 11956 ASSERT(cpu_kstat); 11957 if (rw == KSTAT_READ) { 11958 for (i = 0; i < NCPU; cpu_kstat++, tsbm++, kpmtsbm++, i++) { 11959 cpu_kstat->sf_itlb_misses = tsbm->itlb_misses; 11960 cpu_kstat->sf_dtlb_misses = tsbm->dtlb_misses; 11961 cpu_kstat->sf_utsb_misses = tsbm->utsb_misses - 11962 tsbm->uprot_traps; 11963 cpu_kstat->sf_ktsb_misses = tsbm->ktsb_misses + 11964 kpmtsbm->kpm_tsb_misses - tsbm->kprot_traps; 11965 11966 if (tsbm->itlb_misses > 0 && tsbm->dtlb_misses > 0) { 11967 cpu_kstat->sf_tsb_hits = 11968 (tsbm->itlb_misses + tsbm->dtlb_misses) - 11969 (tsbm->utsb_misses + tsbm->ktsb_misses + 11970 kpmtsbm->kpm_tsb_misses); 11971 } else { 11972 cpu_kstat->sf_tsb_hits = 0; 11973 } 11974 cpu_kstat->sf_umod_faults = tsbm->uprot_traps; 11975 cpu_kstat->sf_kmod_faults = tsbm->kprot_traps; 11976 } 11977 } else { 11978 /* KSTAT_WRITE is used to clear stats */ 11979 for (i = 0; i < NCPU; tsbm++, kpmtsbm++, i++) { 11980 tsbm->itlb_misses = 0; 11981 tsbm->dtlb_misses = 0; 11982 tsbm->utsb_misses = 0; 11983 tsbm->ktsb_misses = 0; 11984 tsbm->uprot_traps = 0; 11985 tsbm->kprot_traps = 0; 11986 kpmtsbm->kpm_dtlb_misses = 0; 11987 kpmtsbm->kpm_tsb_misses = 0; 11988 } 11989 } 11990 return (0); 11991 } 11992 11993 #ifdef DEBUG 11994 11995 tte_t *gorig[NCPU], *gcur[NCPU], *gnew[NCPU]; 11996 11997 /* 11998 * A tte checker. *orig_old is the value we read before cas. 11999 * *cur is the value returned by cas. 12000 * *new is the desired value when we do the cas. 12001 * 12002 * *hmeblkp is currently unused. 12003 */ 12004 12005 /* ARGSUSED */ 12006 void 12007 chk_tte(tte_t *orig_old, tte_t *cur, tte_t *new, struct hme_blk *hmeblkp) 12008 { 12009 pfn_t i, j, k; 12010 int cpuid = CPU->cpu_id; 12011 12012 gorig[cpuid] = orig_old; 12013 gcur[cpuid] = cur; 12014 gnew[cpuid] = new; 12015 12016 #ifdef lint 12017 hmeblkp = hmeblkp; 12018 #endif 12019 12020 if (TTE_IS_VALID(orig_old)) { 12021 if (TTE_IS_VALID(cur)) { 12022 i = TTE_TO_TTEPFN(orig_old); 12023 j = TTE_TO_TTEPFN(cur); 12024 k = TTE_TO_TTEPFN(new); 12025 if (i != j) { 12026 /* remap error? */ 12027 panic("chk_tte: bad pfn, 0x%lx, 0x%lx", i, j); 12028 } 12029 12030 if (i != k) { 12031 /* remap error? */ 12032 panic("chk_tte: bad pfn2, 0x%lx, 0x%lx", i, k); 12033 } 12034 } else { 12035 if (TTE_IS_VALID(new)) { 12036 panic("chk_tte: invalid cur? "); 12037 } 12038 12039 i = TTE_TO_TTEPFN(orig_old); 12040 k = TTE_TO_TTEPFN(new); 12041 if (i != k) { 12042 panic("chk_tte: bad pfn3, 0x%lx, 0x%lx", i, k); 12043 } 12044 } 12045 } else { 12046 if (TTE_IS_VALID(cur)) { 12047 j = TTE_TO_TTEPFN(cur); 12048 if (TTE_IS_VALID(new)) { 12049 k = TTE_TO_TTEPFN(new); 12050 if (j != k) { 12051 panic("chk_tte: bad pfn4, 0x%lx, 0x%lx", 12052 j, k); 12053 } 12054 } else { 12055 panic("chk_tte: why here?"); 12056 } 12057 } else { 12058 if (!TTE_IS_VALID(new)) { 12059 panic("chk_tte: why here2 ?"); 12060 } 12061 } 12062 } 12063 } 12064 12065 #endif /* DEBUG */ 12066 12067 extern void prefetch_tsbe_read(struct tsbe *); 12068 extern void prefetch_tsbe_write(struct tsbe *); 12069 12070 12071 /* 12072 * We want to prefetch 7 cache lines ahead for our read prefetch. This gives 12073 * us optimal performance on Cheetah+. You can only have 8 outstanding 12074 * prefetches at any one time, so we opted for 7 read prefetches and 1 write 12075 * prefetch to make the most utilization of the prefetch capability. 12076 */ 12077 #define TSBE_PREFETCH_STRIDE (7) 12078 12079 void 12080 sfmmu_copy_tsb(struct tsb_info *old_tsbinfo, struct tsb_info *new_tsbinfo) 12081 { 12082 int old_bytes = TSB_BYTES(old_tsbinfo->tsb_szc); 12083 int new_bytes = TSB_BYTES(new_tsbinfo->tsb_szc); 12084 int old_entries = TSB_ENTRIES(old_tsbinfo->tsb_szc); 12085 int new_entries = TSB_ENTRIES(new_tsbinfo->tsb_szc); 12086 struct tsbe *old; 12087 struct tsbe *new; 12088 struct tsbe *new_base = (struct tsbe *)new_tsbinfo->tsb_va; 12089 uint64_t va; 12090 int new_offset; 12091 int i; 12092 int vpshift; 12093 int last_prefetch; 12094 12095 if (old_bytes == new_bytes) { 12096 bcopy(old_tsbinfo->tsb_va, new_tsbinfo->tsb_va, new_bytes); 12097 } else { 12098 12099 /* 12100 * A TSBE is 16 bytes which means there are four TSBE's per 12101 * P$ line (64 bytes), thus every 4 TSBE's we prefetch. 12102 */ 12103 old = (struct tsbe *)old_tsbinfo->tsb_va; 12104 last_prefetch = old_entries - (4*(TSBE_PREFETCH_STRIDE+1)); 12105 for (i = 0; i < old_entries; i++, old++) { 12106 if (((i & (4-1)) == 0) && (i < last_prefetch)) 12107 prefetch_tsbe_read(old); 12108 if (!old->tte_tag.tag_invalid) { 12109 /* 12110 * We have a valid TTE to remap. Check the 12111 * size. We won't remap 64K or 512K TTEs 12112 * because they span more than one TSB entry 12113 * and are indexed using an 8K virt. page. 12114 * Ditto for 32M and 256M TTEs. 12115 */ 12116 if (TTE_CSZ(&old->tte_data) == TTE64K || 12117 TTE_CSZ(&old->tte_data) == TTE512K) 12118 continue; 12119 if (mmu_page_sizes == max_mmu_page_sizes) { 12120 if (TTE_CSZ(&old->tte_data) == TTE32M || 12121 TTE_CSZ(&old->tte_data) == TTE256M) 12122 continue; 12123 } 12124 12125 /* clear the lower 22 bits of the va */ 12126 va = *(uint64_t *)old << 22; 12127 /* turn va into a virtual pfn */ 12128 va >>= 22 - TSB_START_SIZE; 12129 /* 12130 * or in bits from the offset in the tsb 12131 * to get the real virtual pfn. These 12132 * correspond to bits [21:13] in the va 12133 */ 12134 vpshift = 12135 TTE_BSZS_SHIFT(TTE_CSZ(&old->tte_data)) & 12136 0x1ff; 12137 va |= (i << vpshift); 12138 va >>= vpshift; 12139 new_offset = va & (new_entries - 1); 12140 new = new_base + new_offset; 12141 prefetch_tsbe_write(new); 12142 *new = *old; 12143 } 12144 } 12145 } 12146 } 12147 12148 /* 12149 * unused in sfmmu 12150 */ 12151 void 12152 hat_dump(void) 12153 { 12154 } 12155 12156 /* 12157 * Called when a thread is exiting and we have switched to the kernel address 12158 * space. Perform the same VM initialization resume() uses when switching 12159 * processes. 12160 * 12161 * Note that sfmmu_load_mmustate() is currently a no-op for kernel threads, but 12162 * we call it anyway in case the semantics change in the future. 12163 */ 12164 /*ARGSUSED*/ 12165 void 12166 hat_thread_exit(kthread_t *thd) 12167 { 12168 uint64_t pgsz_cnum; 12169 uint_t pstate_save; 12170 12171 ASSERT(thd->t_procp->p_as == &kas); 12172 12173 pgsz_cnum = KCONTEXT; 12174 #ifdef sun4u 12175 pgsz_cnum |= (ksfmmup->sfmmu_cext << CTXREG_EXT_SHIFT); 12176 #endif 12177 /* 12178 * Note that sfmmu_load_mmustate() is currently a no-op for 12179 * kernel threads. We need to disable interrupts here, 12180 * simply because otherwise sfmmu_load_mmustate() would panic 12181 * if the caller does not disable interrupts. 12182 */ 12183 pstate_save = sfmmu_disable_intrs(); 12184 sfmmu_setctx_sec(pgsz_cnum); 12185 sfmmu_load_mmustate(ksfmmup); 12186 sfmmu_enable_intrs(pstate_save); 12187 } 12188