1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * VM - Hardware Address Translation management for Spitfire MMU. 30 * 31 * This file implements the machine specific hardware translation 32 * needed by the VM system. The machine independent interface is 33 * described in <vm/hat.h> while the machine dependent interface 34 * and data structures are described in <vm/hat_sfmmu.h>. 35 * 36 * The hat layer manages the address translation hardware as a cache 37 * driven by calls from the higher levels in the VM system. 38 */ 39 40 #include <sys/types.h> 41 #include <sys/kstat.h> 42 #include <vm/hat.h> 43 #include <vm/hat_sfmmu.h> 44 #include <vm/page.h> 45 #include <sys/pte.h> 46 #include <sys/systm.h> 47 #include <sys/mman.h> 48 #include <sys/sysmacros.h> 49 #include <sys/machparam.h> 50 #include <sys/vtrace.h> 51 #include <sys/kmem.h> 52 #include <sys/mmu.h> 53 #include <sys/cmn_err.h> 54 #include <sys/cpu.h> 55 #include <sys/cpuvar.h> 56 #include <sys/debug.h> 57 #include <sys/lgrp.h> 58 #include <sys/archsystm.h> 59 #include <sys/machsystm.h> 60 #include <sys/vmsystm.h> 61 #include <vm/as.h> 62 #include <vm/seg.h> 63 #include <vm/seg_kp.h> 64 #include <vm/seg_kmem.h> 65 #include <vm/seg_kpm.h> 66 #include <vm/rm.h> 67 #include <sys/t_lock.h> 68 #include <sys/obpdefs.h> 69 #include <sys/vm_machparam.h> 70 #include <sys/var.h> 71 #include <sys/trap.h> 72 #include <sys/machtrap.h> 73 #include <sys/scb.h> 74 #include <sys/bitmap.h> 75 #include <sys/machlock.h> 76 #include <sys/membar.h> 77 #include <sys/atomic.h> 78 #include <sys/cpu_module.h> 79 #include <sys/prom_debug.h> 80 #include <sys/ksynch.h> 81 #include <sys/mem_config.h> 82 #include <sys/mem_cage.h> 83 #include <sys/dtrace.h> 84 #include <vm/vm_dep.h> 85 #include <vm/xhat_sfmmu.h> 86 #include <sys/fpu/fpusystm.h> 87 #include <vm/mach_kpm.h> 88 89 #if defined(SF_ERRATA_57) 90 extern caddr_t errata57_limit; 91 #endif 92 93 #define HME8BLK_SZ_RND ((roundup(HME8BLK_SZ, sizeof (int64_t))) / \ 94 (sizeof (int64_t))) 95 #define HBLK_RESERVE ((struct hme_blk *)hblk_reserve) 96 97 #define HBLK_RESERVE_CNT 128 98 #define HBLK_RESERVE_MIN 20 99 100 static struct hme_blk *freehblkp; 101 static kmutex_t freehblkp_lock; 102 static int freehblkcnt; 103 104 static int64_t hblk_reserve[HME8BLK_SZ_RND]; 105 static kmutex_t hblk_reserve_lock; 106 static kthread_t *hblk_reserve_thread; 107 108 static nucleus_hblk8_info_t nucleus_hblk8; 109 static nucleus_hblk1_info_t nucleus_hblk1; 110 111 /* 112 * SFMMU specific hat functions 113 */ 114 void hat_pagecachectl(struct page *, int); 115 116 /* flags for hat_pagecachectl */ 117 #define HAT_CACHE 0x1 118 #define HAT_UNCACHE 0x2 119 #define HAT_TMPNC 0x4 120 121 /* 122 * Flag to allow the creation of non-cacheable translations 123 * to system memory. It is off by default. At the moment this 124 * flag is used by the ecache error injector. The error injector 125 * will turn it on when creating such a translation then shut it 126 * off when it's finished. 127 */ 128 129 int sfmmu_allow_nc_trans = 0; 130 131 /* 132 * Flag to disable large page support. 133 * value of 1 => disable all large pages. 134 * bits 1, 2, and 3 are to disable 64K, 512K and 4M pages respectively. 135 * 136 * For example, use the value 0x4 to disable 512K pages. 137 * 138 */ 139 #define LARGE_PAGES_OFF 0x1 140 141 /* 142 * The disable_large_pages and disable_ism_large_pages variables control 143 * hat_memload_array and the page sizes to be used by ISM and the kernel. 144 * 145 * The disable_auto_data_large_pages and disable_auto_text_large_pages variables 146 * are only used to control which OOB pages to use at upper VM segment creation 147 * time, and are set in hat_init_pagesizes and used in the map_pgsz* routines. 148 * Their values may come from platform or CPU specific code to disable page 149 * sizes that should not be used. 150 * 151 * WARNING: 512K pages are currently not supported for ISM/DISM. 152 */ 153 uint_t disable_large_pages = 0; 154 uint_t disable_ism_large_pages = (1 << TTE512K); 155 uint_t disable_auto_data_large_pages = 0; 156 uint_t disable_auto_text_large_pages = 0; 157 158 /* 159 * Private sfmmu data structures for hat management 160 */ 161 static struct kmem_cache *sfmmuid_cache; 162 static struct kmem_cache *mmuctxdom_cache; 163 164 /* 165 * Private sfmmu data structures for tsb management 166 */ 167 static struct kmem_cache *sfmmu_tsbinfo_cache; 168 static struct kmem_cache *sfmmu_tsb8k_cache; 169 static struct kmem_cache *sfmmu_tsb_cache[NLGRPS_MAX]; 170 static vmem_t *kmem_tsb_arena; 171 172 /* 173 * sfmmu static variables for hmeblk resource management. 174 */ 175 static vmem_t *hat_memload1_arena; /* HAT translation arena for sfmmu1_cache */ 176 static struct kmem_cache *sfmmu8_cache; 177 static struct kmem_cache *sfmmu1_cache; 178 static struct kmem_cache *pa_hment_cache; 179 180 static kmutex_t ism_mlist_lock; /* mutex for ism mapping list */ 181 /* 182 * private data for ism 183 */ 184 static struct kmem_cache *ism_blk_cache; 185 static struct kmem_cache *ism_ment_cache; 186 #define ISMID_STARTADDR NULL 187 188 /* 189 * Whether to delay TLB flushes and use Cheetah's flush-all support 190 * when removing contexts from the dirty list. 191 */ 192 int delay_tlb_flush; 193 int disable_delay_tlb_flush; 194 195 /* 196 * ``hat_lock'' is a hashed mutex lock for protecting sfmmu TSB lists, 197 * HAT flags, synchronizing TLB/TSB coherency, and context management. 198 * The lock is hashed on the sfmmup since the case where we need to lock 199 * all processes is rare but does occur (e.g. we need to unload a shared 200 * mapping from all processes using the mapping). We have a lot of buckets, 201 * and each slab of sfmmu_t's can use about a quarter of them, giving us 202 * a fairly good distribution without wasting too much space and overhead 203 * when we have to grab them all. 204 */ 205 #define SFMMU_NUM_LOCK 128 /* must be power of two */ 206 hatlock_t hat_lock[SFMMU_NUM_LOCK]; 207 208 /* 209 * Hash algorithm optimized for a small number of slabs. 210 * 7 is (highbit((sizeof sfmmu_t)) - 1) 211 * This hash algorithm is based upon the knowledge that sfmmu_t's come from a 212 * kmem_cache, and thus they will be sequential within that cache. In 213 * addition, each new slab will have a different "color" up to cache_maxcolor 214 * which will skew the hashing for each successive slab which is allocated. 215 * If the size of sfmmu_t changed to a larger size, this algorithm may need 216 * to be revisited. 217 */ 218 #define TSB_HASH_SHIFT_BITS (7) 219 #define PTR_HASH(x) ((uintptr_t)x >> TSB_HASH_SHIFT_BITS) 220 221 #ifdef DEBUG 222 int tsb_hash_debug = 0; 223 #define TSB_HASH(sfmmup) \ 224 (tsb_hash_debug ? &hat_lock[0] : \ 225 &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)]) 226 #else /* DEBUG */ 227 #define TSB_HASH(sfmmup) &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)] 228 #endif /* DEBUG */ 229 230 231 /* sfmmu_replace_tsb() return codes. */ 232 typedef enum tsb_replace_rc { 233 TSB_SUCCESS, 234 TSB_ALLOCFAIL, 235 TSB_LOSTRACE, 236 TSB_ALREADY_SWAPPED, 237 TSB_CANTGROW 238 } tsb_replace_rc_t; 239 240 /* 241 * Flags for TSB allocation routines. 242 */ 243 #define TSB_ALLOC 0x01 244 #define TSB_FORCEALLOC 0x02 245 #define TSB_GROW 0x04 246 #define TSB_SHRINK 0x08 247 #define TSB_SWAPIN 0x10 248 249 /* 250 * Support for HAT callbacks. 251 */ 252 #define SFMMU_MAX_RELOC_CALLBACKS 10 253 int sfmmu_max_cb_id = SFMMU_MAX_RELOC_CALLBACKS; 254 static id_t sfmmu_cb_nextid = 0; 255 static id_t sfmmu_tsb_cb_id; 256 struct sfmmu_callback *sfmmu_cb_table; 257 258 /* 259 * Kernel page relocation is enabled by default for non-caged 260 * kernel pages. This has little effect unless segkmem_reloc is 261 * set, since by default kernel memory comes from inside the 262 * kernel cage. 263 */ 264 int hat_kpr_enabled = 1; 265 266 kmutex_t kpr_mutex; 267 kmutex_t kpr_suspendlock; 268 kthread_t *kreloc_thread; 269 270 /* 271 * Enable VA->PA translation sanity checking on DEBUG kernels. 272 * Disabled by default. This is incompatible with some 273 * drivers (error injector, RSM) so if it breaks you get 274 * to keep both pieces. 275 */ 276 int hat_check_vtop = 0; 277 278 /* 279 * Private sfmmu routines (prototypes) 280 */ 281 static struct hme_blk *sfmmu_shadow_hcreate(sfmmu_t *, caddr_t, int, uint_t); 282 static struct hme_blk *sfmmu_hblk_alloc(sfmmu_t *, caddr_t, 283 struct hmehash_bucket *, uint_t, hmeblk_tag, uint_t); 284 static caddr_t sfmmu_hblk_unload(struct hat *, struct hme_blk *, caddr_t, 285 caddr_t, demap_range_t *, uint_t); 286 static caddr_t sfmmu_hblk_sync(struct hat *, struct hme_blk *, caddr_t, 287 caddr_t, int); 288 static void sfmmu_hblk_free(struct hmehash_bucket *, struct hme_blk *, 289 uint64_t, struct hme_blk **); 290 static void sfmmu_hblks_list_purge(struct hme_blk **); 291 static uint_t sfmmu_get_free_hblk(struct hme_blk **, uint_t); 292 static uint_t sfmmu_put_free_hblk(struct hme_blk *, uint_t); 293 static struct hme_blk *sfmmu_hblk_steal(int); 294 static int sfmmu_steal_this_hblk(struct hmehash_bucket *, 295 struct hme_blk *, uint64_t, uint64_t, 296 struct hme_blk *); 297 static caddr_t sfmmu_hblk_unlock(struct hme_blk *, caddr_t, caddr_t); 298 299 static void sfmmu_memload_batchsmall(struct hat *, caddr_t, page_t **, 300 uint_t, uint_t, pgcnt_t); 301 void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, 302 uint_t); 303 static int sfmmu_tteload_array(sfmmu_t *, tte_t *, caddr_t, page_t **, 304 uint_t); 305 static struct hmehash_bucket *sfmmu_tteload_acquire_hashbucket(sfmmu_t *, 306 caddr_t, int); 307 static struct hme_blk *sfmmu_tteload_find_hmeblk(sfmmu_t *, 308 struct hmehash_bucket *, caddr_t, uint_t, uint_t); 309 static int sfmmu_tteload_addentry(sfmmu_t *, struct hme_blk *, tte_t *, 310 caddr_t, page_t **, uint_t); 311 static void sfmmu_tteload_release_hashbucket(struct hmehash_bucket *); 312 313 static int sfmmu_pagearray_setup(caddr_t, page_t **, tte_t *, int); 314 pfn_t sfmmu_uvatopfn(caddr_t, sfmmu_t *); 315 void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 316 #ifdef VAC 317 static void sfmmu_vac_conflict(struct hat *, caddr_t, page_t *); 318 static int sfmmu_vacconflict_array(caddr_t, page_t *, int *); 319 int tst_tnc(page_t *pp, pgcnt_t); 320 void conv_tnc(page_t *pp, int); 321 #endif 322 323 static void sfmmu_get_ctx(sfmmu_t *); 324 static void sfmmu_free_sfmmu(sfmmu_t *); 325 326 static void sfmmu_gettte(struct hat *, caddr_t, tte_t *); 327 static void sfmmu_ttesync(struct hat *, caddr_t, tte_t *, page_t *); 328 static void sfmmu_chgattr(struct hat *, caddr_t, size_t, uint_t, int); 329 330 cpuset_t sfmmu_pageunload(page_t *, struct sf_hment *, int); 331 static void hat_pagereload(struct page *, struct page *); 332 static cpuset_t sfmmu_pagesync(page_t *, struct sf_hment *, uint_t); 333 #ifdef VAC 334 void sfmmu_page_cache_array(page_t *, int, int, pgcnt_t); 335 static void sfmmu_page_cache(page_t *, int, int, int); 336 #endif 337 338 static void sfmmu_tlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 339 pfn_t, int, int, int, int); 340 static void sfmmu_ismtlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 341 pfn_t, int); 342 static void sfmmu_tlb_demap(caddr_t, sfmmu_t *, struct hme_blk *, int, int); 343 static void sfmmu_tlb_range_demap(demap_range_t *); 344 static void sfmmu_invalidate_ctx(sfmmu_t *); 345 static void sfmmu_sync_mmustate(sfmmu_t *); 346 347 static void sfmmu_tsbinfo_setup_phys(struct tsb_info *, pfn_t); 348 static int sfmmu_tsbinfo_alloc(struct tsb_info **, int, int, uint_t, 349 sfmmu_t *); 350 static void sfmmu_tsb_free(struct tsb_info *); 351 static void sfmmu_tsbinfo_free(struct tsb_info *); 352 static int sfmmu_init_tsbinfo(struct tsb_info *, int, int, uint_t, 353 sfmmu_t *); 354 355 static void sfmmu_tsb_swapin(sfmmu_t *, hatlock_t *); 356 static int sfmmu_select_tsb_szc(pgcnt_t); 357 static void sfmmu_mod_tsb(sfmmu_t *, caddr_t, tte_t *, int); 358 #define sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \ 359 sfmmu_mod_tsb(sfmmup, vaddr, tte, szc) 360 #define sfmmu_unload_tsb(sfmmup, vaddr, szc) \ 361 sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc) 362 static void sfmmu_copy_tsb(struct tsb_info *, struct tsb_info *); 363 static tsb_replace_rc_t sfmmu_replace_tsb(sfmmu_t *, struct tsb_info *, uint_t, 364 hatlock_t *, uint_t); 365 static void sfmmu_size_tsb(sfmmu_t *, int, uint64_t, uint64_t, int); 366 367 #ifdef VAC 368 void sfmmu_cache_flush(pfn_t, int); 369 void sfmmu_cache_flushcolor(int, pfn_t); 370 #endif 371 static caddr_t sfmmu_hblk_chgattr(sfmmu_t *, struct hme_blk *, caddr_t, 372 caddr_t, demap_range_t *, uint_t, int); 373 374 static uint64_t sfmmu_vtop_attr(uint_t, int mode, tte_t *); 375 static uint_t sfmmu_ptov_attr(tte_t *); 376 static caddr_t sfmmu_hblk_chgprot(sfmmu_t *, struct hme_blk *, caddr_t, 377 caddr_t, demap_range_t *, uint_t); 378 static uint_t sfmmu_vtop_prot(uint_t, uint_t *); 379 static int sfmmu_idcache_constructor(void *, void *, int); 380 static void sfmmu_idcache_destructor(void *, void *); 381 static int sfmmu_hblkcache_constructor(void *, void *, int); 382 static void sfmmu_hblkcache_destructor(void *, void *); 383 static void sfmmu_hblkcache_reclaim(void *); 384 static void sfmmu_shadow_hcleanup(sfmmu_t *, struct hme_blk *, 385 struct hmehash_bucket *); 386 static void sfmmu_free_hblks(sfmmu_t *, caddr_t, caddr_t, int); 387 static void sfmmu_rm_large_mappings(page_t *, int); 388 389 static void hat_lock_init(void); 390 static void hat_kstat_init(void); 391 static int sfmmu_kstat_percpu_update(kstat_t *ksp, int rw); 392 static void sfmmu_check_page_sizes(sfmmu_t *, int); 393 int fnd_mapping_sz(page_t *); 394 static void iment_add(struct ism_ment *, struct hat *); 395 static void iment_sub(struct ism_ment *, struct hat *); 396 static pgcnt_t ism_tsb_entries(sfmmu_t *, int szc); 397 extern void sfmmu_setup_tsbinfo(sfmmu_t *); 398 #ifdef sun4v 399 extern void sfmmu_invalidate_tsbinfo(sfmmu_t *); 400 #endif /* sun4v */ 401 extern void sfmmu_clear_utsbinfo(void); 402 403 static void sfmmu_ctx_wrap_around(mmu_ctx_t *); 404 405 /* kpm globals */ 406 #ifdef DEBUG 407 /* 408 * Enable trap level tsbmiss handling 409 */ 410 int kpm_tsbmtl = 1; 411 412 /* 413 * Flush the TLB on kpm mapout. Note: Xcalls are used (again) for the 414 * required TLB shootdowns in this case, so handle w/ care. Off by default. 415 */ 416 int kpm_tlb_flush; 417 #endif /* DEBUG */ 418 419 static void *sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *, size_t, int); 420 421 #ifdef DEBUG 422 static void sfmmu_check_hblk_flist(); 423 #endif 424 425 /* 426 * Semi-private sfmmu data structures. Some of them are initialize in 427 * startup or in hat_init. Some of them are private but accessed by 428 * assembly code or mach_sfmmu.c 429 */ 430 struct hmehash_bucket *uhme_hash; /* user hmeblk hash table */ 431 struct hmehash_bucket *khme_hash; /* kernel hmeblk hash table */ 432 uint64_t uhme_hash_pa; /* PA of uhme_hash */ 433 uint64_t khme_hash_pa; /* PA of khme_hash */ 434 int uhmehash_num; /* # of buckets in user hash table */ 435 int khmehash_num; /* # of buckets in kernel hash table */ 436 437 uint_t max_mmu_ctxdoms = 0; /* max context domains in the system */ 438 mmu_ctx_t **mmu_ctxs_tbl; /* global array of context domains */ 439 uint64_t mmu_saved_gnum = 0; /* to init incoming MMUs' gnums */ 440 441 #define DEFAULT_NUM_CTXS_PER_MMU 8192 442 static uint_t nctxs = DEFAULT_NUM_CTXS_PER_MMU; 443 444 int cache; /* describes system cache */ 445 446 caddr_t ktsb_base; /* kernel 8k-indexed tsb base address */ 447 uint64_t ktsb_pbase; /* kernel 8k-indexed tsb phys address */ 448 int ktsb_szcode; /* kernel 8k-indexed tsb size code */ 449 int ktsb_sz; /* kernel 8k-indexed tsb size */ 450 451 caddr_t ktsb4m_base; /* kernel 4m-indexed tsb base address */ 452 uint64_t ktsb4m_pbase; /* kernel 4m-indexed tsb phys address */ 453 int ktsb4m_szcode; /* kernel 4m-indexed tsb size code */ 454 int ktsb4m_sz; /* kernel 4m-indexed tsb size */ 455 456 uint64_t kpm_tsbbase; /* kernel seg_kpm 4M TSB base address */ 457 int kpm_tsbsz; /* kernel seg_kpm 4M TSB size code */ 458 uint64_t kpmsm_tsbbase; /* kernel seg_kpm 8K TSB base address */ 459 int kpmsm_tsbsz; /* kernel seg_kpm 8K TSB size code */ 460 461 #ifndef sun4v 462 int utsb_dtlb_ttenum = -1; /* index in TLB for utsb locked TTE */ 463 int utsb4m_dtlb_ttenum = -1; /* index in TLB for 4M TSB TTE */ 464 int dtlb_resv_ttenum; /* index in TLB of first reserved TTE */ 465 caddr_t utsb_vabase; /* reserved kernel virtual memory */ 466 caddr_t utsb4m_vabase; /* for trap handler TSB accesses */ 467 #endif /* sun4v */ 468 uint64_t tsb_alloc_bytes = 0; /* bytes allocated to TSBs */ 469 vmem_t *kmem_tsb_default_arena[NLGRPS_MAX]; /* For dynamic TSBs */ 470 471 /* 472 * Size to use for TSB slabs. Future platforms that support page sizes 473 * larger than 4M may wish to change these values, and provide their own 474 * assembly macros for building and decoding the TSB base register contents. 475 * Note disable_large_pages will override the value set here. 476 */ 477 uint_t tsb_slab_ttesz = TTE4M; 478 uint_t tsb_slab_size; 479 uint_t tsb_slab_shift; 480 uint_t tsb_slab_mask; /* PFN mask for TTE */ 481 482 /* largest TSB size to grow to, will be smaller on smaller memory systems */ 483 int tsb_max_growsize = UTSB_MAX_SZCODE; 484 485 /* 486 * Tunable parameters dealing with TSB policies. 487 */ 488 489 /* 490 * This undocumented tunable forces all 8K TSBs to be allocated from 491 * the kernel heap rather than from the kmem_tsb_default_arena arenas. 492 */ 493 #ifdef DEBUG 494 int tsb_forceheap = 0; 495 #endif /* DEBUG */ 496 497 /* 498 * Decide whether to use per-lgroup arenas, or one global set of 499 * TSB arenas. The default is not to break up per-lgroup, since 500 * most platforms don't recognize any tangible benefit from it. 501 */ 502 int tsb_lgrp_affinity = 0; 503 504 /* 505 * Used for growing the TSB based on the process RSS. 506 * tsb_rss_factor is based on the smallest TSB, and is 507 * shifted by the TSB size to determine if we need to grow. 508 * The default will grow the TSB if the number of TTEs for 509 * this page size exceeds 75% of the number of TSB entries, 510 * which should _almost_ eliminate all conflict misses 511 * (at the expense of using up lots and lots of memory). 512 */ 513 #define TSB_RSS_FACTOR (TSB_ENTRIES(TSB_MIN_SZCODE) * 0.75) 514 #define SFMMU_RSS_TSBSIZE(tsbszc) (tsb_rss_factor << tsbszc) 515 #define SELECT_TSB_SIZECODE(pgcnt) ( \ 516 (enable_tsb_rss_sizing)? sfmmu_select_tsb_szc(pgcnt) : \ 517 default_tsb_size) 518 #define TSB_OK_SHRINK() \ 519 (tsb_alloc_bytes > tsb_alloc_hiwater || freemem < desfree) 520 #define TSB_OK_GROW() \ 521 (tsb_alloc_bytes < tsb_alloc_hiwater && freemem > desfree) 522 523 int enable_tsb_rss_sizing = 1; 524 int tsb_rss_factor = (int)TSB_RSS_FACTOR; 525 526 /* which TSB size code to use for new address spaces or if rss sizing off */ 527 int default_tsb_size = TSB_8K_SZCODE; 528 529 static uint64_t tsb_alloc_hiwater; /* limit TSB reserved memory */ 530 uint64_t tsb_alloc_hiwater_factor; /* tsb_alloc_hiwater = physmem / this */ 531 #define TSB_ALLOC_HIWATER_FACTOR_DEFAULT 32 532 533 #ifdef DEBUG 534 static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */ 535 static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */ 536 static int tsb_alloc_mtbf = 0; /* fail allocation every n attempts */ 537 static int tsb_alloc_fail_mtbf = 0; 538 static int tsb_alloc_count = 0; 539 #endif /* DEBUG */ 540 541 /* if set to 1, will remap valid TTEs when growing TSB. */ 542 int tsb_remap_ttes = 1; 543 544 /* 545 * If we have more than this many mappings, allocate a second TSB. 546 * This default is chosen because the I/D fully associative TLBs are 547 * assumed to have at least 8 available entries. Platforms with a 548 * larger fully-associative TLB could probably override the default. 549 */ 550 int tsb_sectsb_threshold = 8; 551 552 /* 553 * kstat data 554 */ 555 struct sfmmu_global_stat sfmmu_global_stat; 556 struct sfmmu_tsbsize_stat sfmmu_tsbsize_stat; 557 558 /* 559 * Global data 560 */ 561 sfmmu_t *ksfmmup; /* kernel's hat id */ 562 563 #ifdef DEBUG 564 static void chk_tte(tte_t *, tte_t *, tte_t *, struct hme_blk *); 565 #endif 566 567 /* sfmmu locking operations */ 568 static kmutex_t *sfmmu_mlspl_enter(struct page *, int); 569 static int sfmmu_mlspl_held(struct page *, int); 570 571 kmutex_t *sfmmu_page_enter(page_t *); 572 void sfmmu_page_exit(kmutex_t *); 573 int sfmmu_page_spl_held(struct page *); 574 575 /* sfmmu internal locking operations - accessed directly */ 576 static void sfmmu_mlist_reloc_enter(page_t *, page_t *, 577 kmutex_t **, kmutex_t **); 578 static void sfmmu_mlist_reloc_exit(kmutex_t *, kmutex_t *); 579 static hatlock_t * 580 sfmmu_hat_enter(sfmmu_t *); 581 static hatlock_t * 582 sfmmu_hat_tryenter(sfmmu_t *); 583 static void sfmmu_hat_exit(hatlock_t *); 584 static void sfmmu_hat_lock_all(void); 585 static void sfmmu_hat_unlock_all(void); 586 static void sfmmu_ismhat_enter(sfmmu_t *, int); 587 static void sfmmu_ismhat_exit(sfmmu_t *, int); 588 589 /* 590 * Array of mutexes protecting a page's mapping list and p_nrm field. 591 * 592 * The hash function looks complicated, but is made up so that: 593 * 594 * "pp" not shifted, so adjacent pp values will hash to different cache lines 595 * (8 byte alignment * 8 bytes/mutes == 64 byte coherency subblock) 596 * 597 * "pp" >> mml_shift, incorporates more source bits into the hash result 598 * 599 * "& (mml_table_size - 1), should be faster than using remainder "%" 600 * 601 * Hopefully, mml_table, mml_table_size and mml_shift are all in the same 602 * cacheline, since they get declared next to each other below. We'll trust 603 * ld not to do something random. 604 */ 605 #ifdef DEBUG 606 int mlist_hash_debug = 0; 607 #define MLIST_HASH(pp) (mlist_hash_debug ? &mml_table[0] : \ 608 &mml_table[((uintptr_t)(pp) + \ 609 ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)]) 610 #else /* !DEBUG */ 611 #define MLIST_HASH(pp) &mml_table[ \ 612 ((uintptr_t)(pp) + ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)] 613 #endif /* !DEBUG */ 614 615 kmutex_t *mml_table; 616 uint_t mml_table_sz; /* must be a power of 2 */ 617 uint_t mml_shift; /* log2(mml_table_sz) + 3 for align */ 618 619 kpm_hlk_t *kpmp_table; 620 uint_t kpmp_table_sz; /* must be a power of 2 */ 621 uchar_t kpmp_shift; 622 623 kpm_shlk_t *kpmp_stable; 624 uint_t kpmp_stable_sz; /* must be a power of 2 */ 625 626 /* 627 * SPL_HASH was improved to avoid false cache line sharing 628 */ 629 #define SPL_TABLE_SIZE 128 630 #define SPL_MASK (SPL_TABLE_SIZE - 1) 631 #define SPL_SHIFT 7 /* log2(SPL_TABLE_SIZE) */ 632 633 #define SPL_INDEX(pp) \ 634 ((((uintptr_t)(pp) >> SPL_SHIFT) ^ \ 635 ((uintptr_t)(pp) >> (SPL_SHIFT << 1))) & \ 636 (SPL_TABLE_SIZE - 1)) 637 638 #define SPL_HASH(pp) \ 639 (&sfmmu_page_lock[SPL_INDEX(pp) & SPL_MASK].pad_mutex) 640 641 static pad_mutex_t sfmmu_page_lock[SPL_TABLE_SIZE]; 642 643 644 /* 645 * hat_unload_callback() will group together callbacks in order 646 * to avoid xt_sync() calls. This is the maximum size of the group. 647 */ 648 #define MAX_CB_ADDR 32 649 650 tte_t hw_tte; 651 static ulong_t sfmmu_dmr_maxbit = DMR_MAXBIT; 652 653 static char *mmu_ctx_kstat_names[] = { 654 "mmu_ctx_tsb_exceptions", 655 "mmu_ctx_tsb_raise_exception", 656 "mmu_ctx_wrap_around", 657 }; 658 659 /* 660 * Wrapper for vmem_xalloc since vmem_create only allows limited 661 * parameters for vm_source_alloc functions. This function allows us 662 * to specify alignment consistent with the size of the object being 663 * allocated. 664 */ 665 static void * 666 sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *vmp, size_t size, int vmflag) 667 { 668 return (vmem_xalloc(vmp, size, size, 0, 0, NULL, NULL, vmflag)); 669 } 670 671 /* Common code for setting tsb_alloc_hiwater. */ 672 #define SFMMU_SET_TSB_ALLOC_HIWATER(pages) tsb_alloc_hiwater = \ 673 ptob(pages) / tsb_alloc_hiwater_factor 674 675 /* 676 * Set tsb_max_growsize to allow at most all of physical memory to be mapped by 677 * a single TSB. physmem is the number of physical pages so we need physmem 8K 678 * TTEs to represent all those physical pages. We round this up by using 679 * 1<<highbit(). To figure out which size code to use, remember that the size 680 * code is just an amount to shift the smallest TSB size to get the size of 681 * this TSB. So we subtract that size, TSB_START_SIZE, from highbit() (or 682 * highbit() - 1) to get the size code for the smallest TSB that can represent 683 * all of physical memory, while erring on the side of too much. 684 * 685 * If the computed size code is less than the current tsb_max_growsize, we set 686 * tsb_max_growsize to the computed size code. In the case where the computed 687 * size code is greater than tsb_max_growsize, we have these restrictions that 688 * apply to increasing tsb_max_growsize: 689 * 1) TSBs can't grow larger than the TSB slab size 690 * 2) TSBs can't grow larger than UTSB_MAX_SZCODE. 691 */ 692 #define SFMMU_SET_TSB_MAX_GROWSIZE(pages) { \ 693 int i, szc; \ 694 \ 695 i = highbit(pages); \ 696 if ((1 << (i - 1)) == (pages)) \ 697 i--; /* 2^n case, round down */ \ 698 szc = i - TSB_START_SIZE; \ 699 if (szc < tsb_max_growsize) \ 700 tsb_max_growsize = szc; \ 701 else if ((szc > tsb_max_growsize) && \ 702 (szc <= tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT))) \ 703 tsb_max_growsize = MIN(szc, UTSB_MAX_SZCODE); \ 704 } 705 706 /* 707 * Given a pointer to an sfmmu and a TTE size code, return a pointer to the 708 * tsb_info which handles that TTE size. 709 */ 710 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) \ 711 (tsbinfop) = (sfmmup)->sfmmu_tsb; \ 712 ASSERT(sfmmu_hat_lock_held(sfmmup)); \ 713 if ((tte_szc) >= TTE4M) \ 714 (tsbinfop) = (tsbinfop)->tsb_next; 715 716 /* 717 * Return the number of mappings present in the HAT 718 * for a particular process and page size. 719 */ 720 #define SFMMU_TTE_CNT(sfmmup, szc) \ 721 (sfmmup)->sfmmu_iblk? \ 722 (sfmmup)->sfmmu_ismttecnt[(szc)] + \ 723 (sfmmup)->sfmmu_ttecnt[(szc)] : \ 724 (sfmmup)->sfmmu_ttecnt[(szc)]; 725 726 /* 727 * Macro to use to unload entries from the TSB. 728 * It has knowledge of which page sizes get replicated in the TSB 729 * and will call the appropriate unload routine for the appropriate size. 730 */ 731 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp) \ 732 { \ 733 int ttesz = get_hblk_ttesz(hmeblkp); \ 734 if (ttesz == TTE8K || ttesz == TTE4M) { \ 735 sfmmu_unload_tsb(sfmmup, addr, ttesz); \ 736 } else { \ 737 caddr_t sva = (caddr_t)get_hblk_base(hmeblkp); \ 738 caddr_t eva = sva + get_hblk_span(hmeblkp); \ 739 ASSERT(addr >= sva && addr < eva); \ 740 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); \ 741 } \ 742 } 743 744 745 /* Update tsb_alloc_hiwater after memory is configured. */ 746 /*ARGSUSED*/ 747 static void 748 sfmmu_update_tsb_post_add(void *arg, pgcnt_t delta_pages) 749 { 750 /* Assumes physmem has already been updated. */ 751 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 752 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 753 } 754 755 /* 756 * Update tsb_alloc_hiwater before memory is deleted. We'll do nothing here 757 * and update tsb_alloc_hiwater and tsb_max_growsize after the memory is 758 * deleted. 759 */ 760 /*ARGSUSED*/ 761 static int 762 sfmmu_update_tsb_pre_del(void *arg, pgcnt_t delta_pages) 763 { 764 return (0); 765 } 766 767 /* Update tsb_alloc_hiwater after memory fails to be unconfigured. */ 768 /*ARGSUSED*/ 769 static void 770 sfmmu_update_tsb_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 771 { 772 /* 773 * Whether the delete was cancelled or not, just go ahead and update 774 * tsb_alloc_hiwater and tsb_max_growsize. 775 */ 776 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 777 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 778 } 779 780 static kphysm_setup_vector_t sfmmu_update_tsb_vec = { 781 KPHYSM_SETUP_VECTOR_VERSION, /* version */ 782 sfmmu_update_tsb_post_add, /* post_add */ 783 sfmmu_update_tsb_pre_del, /* pre_del */ 784 sfmmu_update_tsb_post_del /* post_del */ 785 }; 786 787 788 /* 789 * HME_BLK HASH PRIMITIVES 790 */ 791 792 /* 793 * Enter a hme on the mapping list for page pp. 794 * When large pages are more prevalent in the system we might want to 795 * keep the mapping list in ascending order by the hment size. For now, 796 * small pages are more frequent, so don't slow it down. 797 */ 798 #define HME_ADD(hme, pp) \ 799 { \ 800 ASSERT(sfmmu_mlist_held(pp)); \ 801 \ 802 hme->hme_prev = NULL; \ 803 hme->hme_next = pp->p_mapping; \ 804 hme->hme_page = pp; \ 805 if (pp->p_mapping) { \ 806 ((struct sf_hment *)(pp->p_mapping))->hme_prev = hme;\ 807 ASSERT(pp->p_share > 0); \ 808 } else { \ 809 /* EMPTY */ \ 810 ASSERT(pp->p_share == 0); \ 811 } \ 812 pp->p_mapping = hme; \ 813 pp->p_share++; \ 814 } 815 816 /* 817 * Enter a hme on the mapping list for page pp. 818 * If we are unmapping a large translation, we need to make sure that the 819 * change is reflect in the corresponding bit of the p_index field. 820 */ 821 #define HME_SUB(hme, pp) \ 822 { \ 823 ASSERT(sfmmu_mlist_held(pp)); \ 824 ASSERT(hme->hme_page == pp || IS_PAHME(hme)); \ 825 \ 826 if (pp->p_mapping == NULL) { \ 827 panic("hme_remove - no mappings"); \ 828 } \ 829 \ 830 membar_stst(); /* ensure previous stores finish */ \ 831 \ 832 ASSERT(pp->p_share > 0); \ 833 pp->p_share--; \ 834 \ 835 if (hme->hme_prev) { \ 836 ASSERT(pp->p_mapping != hme); \ 837 ASSERT(hme->hme_prev->hme_page == pp || \ 838 IS_PAHME(hme->hme_prev)); \ 839 hme->hme_prev->hme_next = hme->hme_next; \ 840 } else { \ 841 ASSERT(pp->p_mapping == hme); \ 842 pp->p_mapping = hme->hme_next; \ 843 ASSERT((pp->p_mapping == NULL) ? \ 844 (pp->p_share == 0) : 1); \ 845 } \ 846 \ 847 if (hme->hme_next) { \ 848 ASSERT(hme->hme_next->hme_page == pp || \ 849 IS_PAHME(hme->hme_next)); \ 850 hme->hme_next->hme_prev = hme->hme_prev; \ 851 } \ 852 \ 853 /* zero out the entry */ \ 854 hme->hme_next = NULL; \ 855 hme->hme_prev = NULL; \ 856 hme->hme_page = NULL; \ 857 \ 858 if (hme_size(hme) > TTE8K) { \ 859 /* remove mappings for remainder of large pg */ \ 860 sfmmu_rm_large_mappings(pp, hme_size(hme)); \ 861 } \ 862 } 863 864 /* 865 * This function returns the hment given the hme_blk and a vaddr. 866 * It assumes addr has already been checked to belong to hme_blk's 867 * range. 868 */ 869 #define HBLKTOHME(hment, hmeblkp, addr) \ 870 { \ 871 int index; \ 872 HBLKTOHME_IDX(hment, hmeblkp, addr, index) \ 873 } 874 875 /* 876 * Version of HBLKTOHME that also returns the index in hmeblkp 877 * of the hment. 878 */ 879 #define HBLKTOHME_IDX(hment, hmeblkp, addr, idx) \ 880 { \ 881 ASSERT(in_hblk_range((hmeblkp), (addr))); \ 882 \ 883 if (get_hblk_ttesz(hmeblkp) == TTE8K) { \ 884 idx = (((uintptr_t)(addr) >> MMU_PAGESHIFT) & (NHMENTS-1)); \ 885 } else \ 886 idx = 0; \ 887 \ 888 (hment) = &(hmeblkp)->hblk_hme[idx]; \ 889 } 890 891 /* 892 * Disable any page sizes not supported by the CPU 893 */ 894 void 895 hat_init_pagesizes() 896 { 897 int i; 898 899 mmu_exported_page_sizes = 0; 900 for (i = TTE8K; i < max_mmu_page_sizes; i++) { 901 902 szc_2_userszc[i] = (uint_t)-1; 903 userszc_2_szc[i] = (uint_t)-1; 904 905 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) { 906 disable_large_pages |= (1 << i); 907 } else { 908 szc_2_userszc[i] = mmu_exported_page_sizes; 909 userszc_2_szc[mmu_exported_page_sizes] = i; 910 mmu_exported_page_sizes++; 911 } 912 } 913 914 disable_ism_large_pages |= disable_large_pages; 915 disable_auto_data_large_pages = disable_large_pages; 916 disable_auto_text_large_pages = disable_large_pages; 917 918 /* 919 * Initialize mmu-specific large page sizes. 920 */ 921 if (&mmu_large_pages_disabled) { 922 disable_large_pages |= mmu_large_pages_disabled(HAT_LOAD); 923 disable_ism_large_pages |= 924 mmu_large_pages_disabled(HAT_LOAD_SHARE); 925 disable_auto_data_large_pages |= 926 mmu_large_pages_disabled(HAT_AUTO_DATA); 927 disable_auto_text_large_pages |= 928 mmu_large_pages_disabled(HAT_AUTO_TEXT); 929 } 930 } 931 932 /* 933 * Initialize the hardware address translation structures. 934 */ 935 void 936 hat_init(void) 937 { 938 int i; 939 uint_t sz; 940 uint_t maxtsb; 941 size_t size; 942 943 hat_lock_init(); 944 hat_kstat_init(); 945 946 /* 947 * Hardware-only bits in a TTE 948 */ 949 MAKE_TTE_MASK(&hw_tte); 950 951 hat_init_pagesizes(); 952 953 /* Initialize the hash locks */ 954 for (i = 0; i < khmehash_num; i++) { 955 mutex_init(&khme_hash[i].hmehash_mutex, NULL, 956 MUTEX_DEFAULT, NULL); 957 } 958 for (i = 0; i < uhmehash_num; i++) { 959 mutex_init(&uhme_hash[i].hmehash_mutex, NULL, 960 MUTEX_DEFAULT, NULL); 961 } 962 khmehash_num--; /* make sure counter starts from 0 */ 963 uhmehash_num--; /* make sure counter starts from 0 */ 964 965 /* 966 * Allocate context domain structures. 967 * 968 * A platform may choose to modify max_mmu_ctxdoms in 969 * set_platform_defaults(). If a platform does not define 970 * a set_platform_defaults() or does not choose to modify 971 * max_mmu_ctxdoms, it gets one MMU context domain for every CPU. 972 * 973 * For sun4v, there will be one global context domain, this is to 974 * avoid the ldom cpu substitution problem. 975 * 976 * For all platforms that have CPUs sharing MMUs, this 977 * value must be defined. 978 */ 979 if (max_mmu_ctxdoms == 0) { 980 #ifndef sun4v 981 max_mmu_ctxdoms = max_ncpus; 982 #else /* sun4v */ 983 max_mmu_ctxdoms = 1; 984 #endif /* sun4v */ 985 } 986 987 size = max_mmu_ctxdoms * sizeof (mmu_ctx_t *); 988 mmu_ctxs_tbl = kmem_zalloc(size, KM_SLEEP); 989 990 /* mmu_ctx_t is 64 bytes aligned */ 991 mmuctxdom_cache = kmem_cache_create("mmuctxdom_cache", 992 sizeof (mmu_ctx_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 993 /* 994 * MMU context domain initialization for the Boot CPU. 995 * This needs the context domains array allocated above. 996 */ 997 mutex_enter(&cpu_lock); 998 sfmmu_cpu_init(CPU); 999 mutex_exit(&cpu_lock); 1000 1001 /* 1002 * Intialize ism mapping list lock. 1003 */ 1004 1005 mutex_init(&ism_mlist_lock, NULL, MUTEX_DEFAULT, NULL); 1006 1007 /* 1008 * Each sfmmu structure carries an array of MMU context info 1009 * structures, one per context domain. The size of this array depends 1010 * on the maximum number of context domains. So, the size of the 1011 * sfmmu structure varies per platform. 1012 * 1013 * sfmmu is allocated from static arena, because trap 1014 * handler at TL > 0 is not allowed to touch kernel relocatable 1015 * memory. sfmmu's alignment is changed to 64 bytes from 1016 * default 8 bytes, as the lower 6 bits will be used to pass 1017 * pgcnt to vtag_flush_pgcnt_tl1. 1018 */ 1019 size = sizeof (sfmmu_t) + sizeof (sfmmu_ctx_t) * (max_mmu_ctxdoms - 1); 1020 1021 sfmmuid_cache = kmem_cache_create("sfmmuid_cache", size, 1022 64, sfmmu_idcache_constructor, sfmmu_idcache_destructor, 1023 NULL, NULL, static_arena, 0); 1024 1025 sfmmu_tsbinfo_cache = kmem_cache_create("sfmmu_tsbinfo_cache", 1026 sizeof (struct tsb_info), 0, NULL, NULL, NULL, NULL, NULL, 0); 1027 1028 /* 1029 * Since we only use the tsb8k cache to "borrow" pages for TSBs 1030 * from the heap when low on memory or when TSB_FORCEALLOC is 1031 * specified, don't use magazines to cache them--we want to return 1032 * them to the system as quickly as possible. 1033 */ 1034 sfmmu_tsb8k_cache = kmem_cache_create("sfmmu_tsb8k_cache", 1035 MMU_PAGESIZE, MMU_PAGESIZE, NULL, NULL, NULL, NULL, 1036 static_arena, KMC_NOMAGAZINE); 1037 1038 /* 1039 * Set tsb_alloc_hiwater to 1/tsb_alloc_hiwater_factor of physical 1040 * memory, which corresponds to the old static reserve for TSBs. 1041 * tsb_alloc_hiwater_factor defaults to 32. This caps the amount of 1042 * memory we'll allocate for TSB slabs; beyond this point TSB 1043 * allocations will be taken from the kernel heap (via 1044 * sfmmu_tsb8k_cache) and will be throttled as would any other kmem 1045 * consumer. 1046 */ 1047 if (tsb_alloc_hiwater_factor == 0) { 1048 tsb_alloc_hiwater_factor = TSB_ALLOC_HIWATER_FACTOR_DEFAULT; 1049 } 1050 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 1051 1052 /* Set tsb_max_growsize. */ 1053 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 1054 1055 /* 1056 * On smaller memory systems, allocate TSB memory in smaller chunks 1057 * than the default 4M slab size. We also honor disable_large_pages 1058 * here. 1059 * 1060 * The trap handlers need to be patched with the final slab shift, 1061 * since they need to be able to construct the TSB pointer at runtime. 1062 */ 1063 if (tsb_max_growsize <= TSB_512K_SZCODE) 1064 tsb_slab_ttesz = TTE512K; 1065 1066 for (sz = tsb_slab_ttesz; sz > 0; sz--) { 1067 if (!(disable_large_pages & (1 << sz))) 1068 break; 1069 } 1070 1071 tsb_slab_ttesz = sz; 1072 tsb_slab_shift = MMU_PAGESHIFT + (sz << 1) + sz; 1073 tsb_slab_size = 1 << tsb_slab_shift; 1074 tsb_slab_mask = (1 << (tsb_slab_shift - MMU_PAGESHIFT)) - 1; 1075 1076 maxtsb = tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT); 1077 if (tsb_max_growsize > maxtsb) 1078 tsb_max_growsize = maxtsb; 1079 1080 /* 1081 * Set up memory callback to update tsb_alloc_hiwater and 1082 * tsb_max_growsize. 1083 */ 1084 i = kphysm_setup_func_register(&sfmmu_update_tsb_vec, (void *) 0); 1085 ASSERT(i == 0); 1086 1087 /* 1088 * kmem_tsb_arena is the source from which large TSB slabs are 1089 * drawn. The quantum of this arena corresponds to the largest 1090 * TSB size we can dynamically allocate for user processes. 1091 * Currently it must also be a supported page size since we 1092 * use exactly one translation entry to map each slab page. 1093 * 1094 * The per-lgroup kmem_tsb_default_arena arenas are the arenas from 1095 * which most TSBs are allocated. Since most TSB allocations are 1096 * typically 8K we have a kmem cache we stack on top of each 1097 * kmem_tsb_default_arena to speed up those allocations. 1098 * 1099 * Note the two-level scheme of arenas is required only 1100 * because vmem_create doesn't allow us to specify alignment 1101 * requirements. If this ever changes the code could be 1102 * simplified to use only one level of arenas. 1103 */ 1104 kmem_tsb_arena = vmem_create("kmem_tsb", NULL, 0, tsb_slab_size, 1105 sfmmu_vmem_xalloc_aligned_wrapper, vmem_xfree, heap_arena, 1106 0, VM_SLEEP); 1107 1108 if (tsb_lgrp_affinity) { 1109 char s[50]; 1110 for (i = 0; i < NLGRPS_MAX; i++) { 1111 (void) sprintf(s, "kmem_tsb_lgrp%d", i); 1112 kmem_tsb_default_arena[i] = 1113 vmem_create(s, NULL, 0, PAGESIZE, 1114 sfmmu_tsb_segkmem_alloc, sfmmu_tsb_segkmem_free, 1115 kmem_tsb_arena, 0, VM_SLEEP | VM_BESTFIT); 1116 (void) sprintf(s, "sfmmu_tsb_lgrp%d_cache", i); 1117 sfmmu_tsb_cache[i] = kmem_cache_create(s, PAGESIZE, 1118 PAGESIZE, NULL, NULL, NULL, NULL, 1119 kmem_tsb_default_arena[i], 0); 1120 } 1121 } else { 1122 kmem_tsb_default_arena[0] = vmem_create("kmem_tsb_default", 1123 NULL, 0, PAGESIZE, sfmmu_tsb_segkmem_alloc, 1124 sfmmu_tsb_segkmem_free, kmem_tsb_arena, 0, 1125 VM_SLEEP | VM_BESTFIT); 1126 1127 sfmmu_tsb_cache[0] = kmem_cache_create("sfmmu_tsb_cache", 1128 PAGESIZE, PAGESIZE, NULL, NULL, NULL, NULL, 1129 kmem_tsb_default_arena[0], 0); 1130 } 1131 1132 sfmmu8_cache = kmem_cache_create("sfmmu8_cache", HME8BLK_SZ, 1133 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1134 sfmmu_hblkcache_destructor, 1135 sfmmu_hblkcache_reclaim, (void *)HME8BLK_SZ, 1136 hat_memload_arena, KMC_NOHASH); 1137 1138 hat_memload1_arena = vmem_create("hat_memload1", NULL, 0, PAGESIZE, 1139 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); 1140 1141 sfmmu1_cache = kmem_cache_create("sfmmu1_cache", HME1BLK_SZ, 1142 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1143 sfmmu_hblkcache_destructor, 1144 NULL, (void *)HME1BLK_SZ, 1145 hat_memload1_arena, KMC_NOHASH); 1146 1147 pa_hment_cache = kmem_cache_create("pa_hment_cache", PAHME_SZ, 1148 0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH); 1149 1150 ism_blk_cache = kmem_cache_create("ism_blk_cache", 1151 sizeof (ism_blk_t), ecache_alignsize, NULL, NULL, 1152 NULL, NULL, static_arena, KMC_NOHASH); 1153 1154 ism_ment_cache = kmem_cache_create("ism_ment_cache", 1155 sizeof (ism_ment_t), 0, NULL, NULL, 1156 NULL, NULL, NULL, 0); 1157 1158 /* 1159 * We grab the first hat for the kernel, 1160 */ 1161 AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER); 1162 kas.a_hat = hat_alloc(&kas); 1163 AS_LOCK_EXIT(&kas, &kas.a_lock); 1164 1165 /* 1166 * Initialize hblk_reserve. 1167 */ 1168 ((struct hme_blk *)hblk_reserve)->hblk_nextpa = 1169 va_to_pa((caddr_t)hblk_reserve); 1170 1171 #ifndef UTSB_PHYS 1172 /* 1173 * Reserve some kernel virtual address space for the locked TTEs 1174 * that allow us to probe the TSB from TL>0. 1175 */ 1176 utsb_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1177 0, 0, NULL, NULL, VM_SLEEP); 1178 utsb4m_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1179 0, 0, NULL, NULL, VM_SLEEP); 1180 #endif 1181 1182 #ifdef VAC 1183 /* 1184 * The big page VAC handling code assumes VAC 1185 * will not be bigger than the smallest big 1186 * page- which is 64K. 1187 */ 1188 if (TTEPAGES(TTE64K) < CACHE_NUM_COLOR) { 1189 cmn_err(CE_PANIC, "VAC too big!"); 1190 } 1191 #endif 1192 1193 (void) xhat_init(); 1194 1195 uhme_hash_pa = va_to_pa(uhme_hash); 1196 khme_hash_pa = va_to_pa(khme_hash); 1197 1198 /* 1199 * Initialize relocation locks. kpr_suspendlock is held 1200 * at PIL_MAX to prevent interrupts from pinning the holder 1201 * of a suspended TTE which may access it leading to a 1202 * deadlock condition. 1203 */ 1204 mutex_init(&kpr_mutex, NULL, MUTEX_DEFAULT, NULL); 1205 mutex_init(&kpr_suspendlock, NULL, MUTEX_SPIN, (void *)PIL_MAX); 1206 1207 /* 1208 * Pre-allocate hrm_hashtab before enabling the collection of 1209 * refmod statistics. Allocating on the fly would mean us 1210 * running the risk of suffering recursive mutex enters or 1211 * deadlocks. 1212 */ 1213 hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *), 1214 KM_SLEEP); 1215 } 1216 1217 /* 1218 * Initialize locking for the hat layer, called early during boot. 1219 */ 1220 static void 1221 hat_lock_init() 1222 { 1223 int i; 1224 1225 /* 1226 * initialize the array of mutexes protecting a page's mapping 1227 * list and p_nrm field. 1228 */ 1229 for (i = 0; i < mml_table_sz; i++) 1230 mutex_init(&mml_table[i], NULL, MUTEX_DEFAULT, NULL); 1231 1232 if (kpm_enable) { 1233 for (i = 0; i < kpmp_table_sz; i++) { 1234 mutex_init(&kpmp_table[i].khl_mutex, NULL, 1235 MUTEX_DEFAULT, NULL); 1236 } 1237 } 1238 1239 /* 1240 * Initialize array of mutex locks that protects sfmmu fields and 1241 * TSB lists. 1242 */ 1243 for (i = 0; i < SFMMU_NUM_LOCK; i++) 1244 mutex_init(HATLOCK_MUTEXP(&hat_lock[i]), NULL, MUTEX_DEFAULT, 1245 NULL); 1246 } 1247 1248 #define SFMMU_KERNEL_MAXVA \ 1249 (kmem64_base ? (uintptr_t)kmem64_end : (SYSLIMIT)) 1250 1251 /* 1252 * Allocate a hat structure. 1253 * Called when an address space first uses a hat. 1254 */ 1255 struct hat * 1256 hat_alloc(struct as *as) 1257 { 1258 sfmmu_t *sfmmup; 1259 int i; 1260 uint64_t cnum; 1261 extern uint_t get_color_start(struct as *); 1262 1263 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 1264 sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP); 1265 sfmmup->sfmmu_as = as; 1266 sfmmup->sfmmu_flags = 0; 1267 LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock); 1268 1269 if (as == &kas) { 1270 ksfmmup = sfmmup; 1271 sfmmup->sfmmu_cext = 0; 1272 cnum = KCONTEXT; 1273 1274 sfmmup->sfmmu_clrstart = 0; 1275 sfmmup->sfmmu_tsb = NULL; 1276 /* 1277 * hat_kern_setup() will call sfmmu_init_ktsbinfo() 1278 * to setup tsb_info for ksfmmup. 1279 */ 1280 } else { 1281 1282 /* 1283 * Just set to invalid ctx. When it faults, it will 1284 * get a valid ctx. This would avoid the situation 1285 * where we get a ctx, but it gets stolen and then 1286 * we fault when we try to run and so have to get 1287 * another ctx. 1288 */ 1289 sfmmup->sfmmu_cext = 0; 1290 cnum = INVALID_CONTEXT; 1291 1292 /* initialize original physical page coloring bin */ 1293 sfmmup->sfmmu_clrstart = get_color_start(as); 1294 #ifdef DEBUG 1295 if (tsb_random_size) { 1296 uint32_t randval = (uint32_t)gettick() >> 4; 1297 int size = randval % (tsb_max_growsize + 1); 1298 1299 /* chose a random tsb size for stress testing */ 1300 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size, 1301 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1302 } else 1303 #endif /* DEBUG */ 1304 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, 1305 default_tsb_size, 1306 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1307 sfmmup->sfmmu_flags = HAT_SWAPPED; 1308 ASSERT(sfmmup->sfmmu_tsb != NULL); 1309 } 1310 1311 ASSERT(max_mmu_ctxdoms > 0); 1312 for (i = 0; i < max_mmu_ctxdoms; i++) { 1313 sfmmup->sfmmu_ctxs[i].cnum = cnum; 1314 sfmmup->sfmmu_ctxs[i].gnum = 0; 1315 } 1316 1317 sfmmu_setup_tsbinfo(sfmmup); 1318 for (i = 0; i < max_mmu_page_sizes; i++) { 1319 sfmmup->sfmmu_ttecnt[i] = 0; 1320 sfmmup->sfmmu_ismttecnt[i] = 0; 1321 sfmmup->sfmmu_pgsz[i] = TTE8K; 1322 } 1323 1324 sfmmup->sfmmu_iblk = NULL; 1325 sfmmup->sfmmu_ismhat = 0; 1326 sfmmup->sfmmu_ismblkpa = (uint64_t)-1; 1327 if (sfmmup == ksfmmup) { 1328 CPUSET_ALL(sfmmup->sfmmu_cpusran); 1329 } else { 1330 CPUSET_ZERO(sfmmup->sfmmu_cpusran); 1331 } 1332 sfmmup->sfmmu_free = 0; 1333 sfmmup->sfmmu_rmstat = 0; 1334 sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart; 1335 sfmmup->sfmmu_xhat_provider = NULL; 1336 cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL); 1337 return (sfmmup); 1338 } 1339 1340 /* 1341 * Create per-MMU context domain kstats for a given MMU ctx. 1342 */ 1343 static void 1344 sfmmu_mmu_kstat_create(mmu_ctx_t *mmu_ctxp) 1345 { 1346 mmu_ctx_stat_t stat; 1347 kstat_t *mmu_kstat; 1348 1349 ASSERT(MUTEX_HELD(&cpu_lock)); 1350 ASSERT(mmu_ctxp->mmu_kstat == NULL); 1351 1352 mmu_kstat = kstat_create("unix", mmu_ctxp->mmu_idx, "mmu_ctx", 1353 "hat", KSTAT_TYPE_NAMED, MMU_CTX_NUM_STATS, KSTAT_FLAG_VIRTUAL); 1354 1355 if (mmu_kstat == NULL) { 1356 cmn_err(CE_WARN, "kstat_create for MMU %d failed", 1357 mmu_ctxp->mmu_idx); 1358 } else { 1359 mmu_kstat->ks_data = mmu_ctxp->mmu_kstat_data; 1360 for (stat = 0; stat < MMU_CTX_NUM_STATS; stat++) 1361 kstat_named_init(&mmu_ctxp->mmu_kstat_data[stat], 1362 mmu_ctx_kstat_names[stat], KSTAT_DATA_INT64); 1363 mmu_ctxp->mmu_kstat = mmu_kstat; 1364 kstat_install(mmu_kstat); 1365 } 1366 } 1367 1368 /* 1369 * plat_cpuid_to_mmu_ctx_info() is a platform interface that returns MMU 1370 * context domain information for a given CPU. If a platform does not 1371 * specify that interface, then the function below is used instead to return 1372 * default information. The defaults are as follows: 1373 * 1374 * - For sun4u systems there's one MMU context domain per CPU. 1375 * This default is used by all sun4u systems except OPL. OPL systems 1376 * provide platform specific interface to map CPU ids to MMU ids 1377 * because on OPL more than 1 CPU shares a single MMU. 1378 * Note that on sun4v, there is one global context domain for 1379 * the entire system. This is to avoid running into potential problem 1380 * with ldom physical cpu substitution feature. 1381 * - The number of MMU context IDs supported on any CPU in the 1382 * system is 8K. 1383 */ 1384 /*ARGSUSED*/ 1385 static void 1386 sfmmu_cpuid_to_mmu_ctx_info(processorid_t cpuid, mmu_ctx_info_t *infop) 1387 { 1388 infop->mmu_nctxs = nctxs; 1389 #ifndef sun4v 1390 infop->mmu_idx = cpu[cpuid]->cpu_seqid; 1391 #else /* sun4v */ 1392 infop->mmu_idx = 0; 1393 #endif /* sun4v */ 1394 } 1395 1396 /* 1397 * Called during CPU initialization to set the MMU context-related information 1398 * for a CPU. 1399 * 1400 * cpu_lock serializes accesses to mmu_ctxs and mmu_saved_gnum. 1401 */ 1402 void 1403 sfmmu_cpu_init(cpu_t *cp) 1404 { 1405 mmu_ctx_info_t info; 1406 mmu_ctx_t *mmu_ctxp; 1407 1408 ASSERT(MUTEX_HELD(&cpu_lock)); 1409 1410 if (&plat_cpuid_to_mmu_ctx_info == NULL) 1411 sfmmu_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1412 else 1413 plat_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1414 1415 ASSERT(info.mmu_idx < max_mmu_ctxdoms); 1416 1417 if ((mmu_ctxp = mmu_ctxs_tbl[info.mmu_idx]) == NULL) { 1418 /* Each mmu_ctx is cacheline aligned. */ 1419 mmu_ctxp = kmem_cache_alloc(mmuctxdom_cache, KM_SLEEP); 1420 bzero(mmu_ctxp, sizeof (mmu_ctx_t)); 1421 1422 mutex_init(&mmu_ctxp->mmu_lock, NULL, MUTEX_SPIN, 1423 (void *)ipltospl(DISP_LEVEL)); 1424 mmu_ctxp->mmu_idx = info.mmu_idx; 1425 mmu_ctxp->mmu_nctxs = info.mmu_nctxs; 1426 /* 1427 * Globally for lifetime of a system, 1428 * gnum must always increase. 1429 * mmu_saved_gnum is protected by the cpu_lock. 1430 */ 1431 mmu_ctxp->mmu_gnum = mmu_saved_gnum + 1; 1432 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 1433 1434 sfmmu_mmu_kstat_create(mmu_ctxp); 1435 1436 mmu_ctxs_tbl[info.mmu_idx] = mmu_ctxp; 1437 } else { 1438 ASSERT(mmu_ctxp->mmu_idx == info.mmu_idx); 1439 } 1440 1441 /* 1442 * The mmu_lock is acquired here to prevent races with 1443 * the wrap-around code. 1444 */ 1445 mutex_enter(&mmu_ctxp->mmu_lock); 1446 1447 1448 mmu_ctxp->mmu_ncpus++; 1449 CPUSET_ADD(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1450 CPU_MMU_IDX(cp) = info.mmu_idx; 1451 CPU_MMU_CTXP(cp) = mmu_ctxp; 1452 1453 mutex_exit(&mmu_ctxp->mmu_lock); 1454 } 1455 1456 /* 1457 * Called to perform MMU context-related cleanup for a CPU. 1458 */ 1459 void 1460 sfmmu_cpu_cleanup(cpu_t *cp) 1461 { 1462 mmu_ctx_t *mmu_ctxp; 1463 1464 ASSERT(MUTEX_HELD(&cpu_lock)); 1465 1466 mmu_ctxp = CPU_MMU_CTXP(cp); 1467 ASSERT(mmu_ctxp != NULL); 1468 1469 /* 1470 * The mmu_lock is acquired here to prevent races with 1471 * the wrap-around code. 1472 */ 1473 mutex_enter(&mmu_ctxp->mmu_lock); 1474 1475 CPU_MMU_CTXP(cp) = NULL; 1476 1477 CPUSET_DEL(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1478 if (--mmu_ctxp->mmu_ncpus == 0) { 1479 mmu_ctxs_tbl[mmu_ctxp->mmu_idx] = NULL; 1480 mutex_exit(&mmu_ctxp->mmu_lock); 1481 mutex_destroy(&mmu_ctxp->mmu_lock); 1482 1483 if (mmu_ctxp->mmu_kstat) 1484 kstat_delete(mmu_ctxp->mmu_kstat); 1485 1486 /* mmu_saved_gnum is protected by the cpu_lock. */ 1487 if (mmu_saved_gnum < mmu_ctxp->mmu_gnum) 1488 mmu_saved_gnum = mmu_ctxp->mmu_gnum; 1489 1490 kmem_cache_free(mmuctxdom_cache, mmu_ctxp); 1491 1492 return; 1493 } 1494 1495 mutex_exit(&mmu_ctxp->mmu_lock); 1496 } 1497 1498 /* 1499 * Hat_setup, makes an address space context the current active one. 1500 * In sfmmu this translates to setting the secondary context with the 1501 * corresponding context. 1502 */ 1503 void 1504 hat_setup(struct hat *sfmmup, int allocflag) 1505 { 1506 hatlock_t *hatlockp; 1507 1508 /* Init needs some special treatment. */ 1509 if (allocflag == HAT_INIT) { 1510 /* 1511 * Make sure that we have 1512 * 1. a TSB 1513 * 2. a valid ctx that doesn't get stolen after this point. 1514 */ 1515 hatlockp = sfmmu_hat_enter(sfmmup); 1516 1517 /* 1518 * Swap in the TSB. hat_init() allocates tsbinfos without 1519 * TSBs, but we need one for init, since the kernel does some 1520 * special things to set up its stack and needs the TSB to 1521 * resolve page faults. 1522 */ 1523 sfmmu_tsb_swapin(sfmmup, hatlockp); 1524 1525 sfmmu_get_ctx(sfmmup); 1526 1527 sfmmu_hat_exit(hatlockp); 1528 } else { 1529 ASSERT(allocflag == HAT_ALLOC); 1530 1531 hatlockp = sfmmu_hat_enter(sfmmup); 1532 kpreempt_disable(); 1533 1534 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); 1535 1536 /* 1537 * sfmmu_setctx_sec takes <pgsz|cnum> as a parameter, 1538 * pagesize bits don't matter in this case since we are passing 1539 * INVALID_CONTEXT to it. 1540 */ 1541 sfmmu_setctx_sec(INVALID_CONTEXT); 1542 sfmmu_clear_utsbinfo(); 1543 1544 kpreempt_enable(); 1545 sfmmu_hat_exit(hatlockp); 1546 } 1547 } 1548 1549 /* 1550 * Free all the translation resources for the specified address space. 1551 * Called from as_free when an address space is being destroyed. 1552 */ 1553 void 1554 hat_free_start(struct hat *sfmmup) 1555 { 1556 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 1557 ASSERT(sfmmup != ksfmmup); 1558 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1559 1560 sfmmup->sfmmu_free = 1; 1561 } 1562 1563 void 1564 hat_free_end(struct hat *sfmmup) 1565 { 1566 int i; 1567 1568 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1569 if (sfmmup->sfmmu_ismhat) { 1570 for (i = 0; i < mmu_page_sizes; i++) { 1571 sfmmup->sfmmu_ttecnt[i] = 0; 1572 sfmmup->sfmmu_ismttecnt[i] = 0; 1573 } 1574 } else { 1575 /* EMPTY */ 1576 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 1577 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 1578 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 1579 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 1580 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 1581 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 1582 } 1583 1584 if (sfmmup->sfmmu_rmstat) { 1585 hat_freestat(sfmmup->sfmmu_as, NULL); 1586 } 1587 1588 while (sfmmup->sfmmu_tsb != NULL) { 1589 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next; 1590 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb); 1591 sfmmup->sfmmu_tsb = next; 1592 } 1593 sfmmu_free_sfmmu(sfmmup); 1594 1595 kmem_cache_free(sfmmuid_cache, sfmmup); 1596 } 1597 1598 /* 1599 * Set up any translation structures, for the specified address space, 1600 * that are needed or preferred when the process is being swapped in. 1601 */ 1602 /* ARGSUSED */ 1603 void 1604 hat_swapin(struct hat *hat) 1605 { 1606 ASSERT(hat->sfmmu_xhat_provider == NULL); 1607 } 1608 1609 /* 1610 * Free all of the translation resources, for the specified address space, 1611 * that can be freed while the process is swapped out. Called from as_swapout. 1612 * Also, free up the ctx that this process was using. 1613 */ 1614 void 1615 hat_swapout(struct hat *sfmmup) 1616 { 1617 struct hmehash_bucket *hmebp; 1618 struct hme_blk *hmeblkp; 1619 struct hme_blk *pr_hblk = NULL; 1620 struct hme_blk *nx_hblk; 1621 int i; 1622 uint64_t hblkpa, prevpa, nx_pa; 1623 struct hme_blk *list = NULL; 1624 hatlock_t *hatlockp; 1625 struct tsb_info *tsbinfop; 1626 struct free_tsb { 1627 struct free_tsb *next; 1628 struct tsb_info *tsbinfop; 1629 }; /* free list of TSBs */ 1630 struct free_tsb *freelist, *last, *next; 1631 1632 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1633 SFMMU_STAT(sf_swapout); 1634 1635 /* 1636 * There is no way to go from an as to all its translations in sfmmu. 1637 * Here is one of the times when we take the big hit and traverse 1638 * the hash looking for hme_blks to free up. Not only do we free up 1639 * this as hme_blks but all those that are free. We are obviously 1640 * swapping because we need memory so let's free up as much 1641 * as we can. 1642 * 1643 * Note that we don't flush TLB/TSB here -- it's not necessary 1644 * because: 1645 * 1) we free the ctx we're using and throw away the TSB(s); 1646 * 2) processes aren't runnable while being swapped out. 1647 */ 1648 ASSERT(sfmmup != KHATID); 1649 for (i = 0; i <= UHMEHASH_SZ; i++) { 1650 hmebp = &uhme_hash[i]; 1651 SFMMU_HASH_LOCK(hmebp); 1652 hmeblkp = hmebp->hmeblkp; 1653 hblkpa = hmebp->hmeh_nextpa; 1654 prevpa = 0; 1655 pr_hblk = NULL; 1656 while (hmeblkp) { 1657 1658 ASSERT(!hmeblkp->hblk_xhat_bit); 1659 1660 if ((hmeblkp->hblk_tag.htag_id == sfmmup) && 1661 !hmeblkp->hblk_shw_bit && !hmeblkp->hblk_lckcnt) { 1662 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 1663 (caddr_t)get_hblk_base(hmeblkp), 1664 get_hblk_endaddr(hmeblkp), 1665 NULL, HAT_UNLOAD); 1666 } 1667 nx_hblk = hmeblkp->hblk_next; 1668 nx_pa = hmeblkp->hblk_nextpa; 1669 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 1670 ASSERT(!hmeblkp->hblk_lckcnt); 1671 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 1672 prevpa, pr_hblk); 1673 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 1674 } else { 1675 pr_hblk = hmeblkp; 1676 prevpa = hblkpa; 1677 } 1678 hmeblkp = nx_hblk; 1679 hblkpa = nx_pa; 1680 } 1681 SFMMU_HASH_UNLOCK(hmebp); 1682 } 1683 1684 sfmmu_hblks_list_purge(&list); 1685 1686 /* 1687 * Now free up the ctx so that others can reuse it. 1688 */ 1689 hatlockp = sfmmu_hat_enter(sfmmup); 1690 1691 sfmmu_invalidate_ctx(sfmmup); 1692 1693 /* 1694 * Free TSBs, but not tsbinfos, and set SWAPPED flag. 1695 * If TSBs were never swapped in, just return. 1696 * This implies that we don't support partial swapping 1697 * of TSBs -- either all are swapped out, or none are. 1698 * 1699 * We must hold the HAT lock here to prevent racing with another 1700 * thread trying to unmap TTEs from the TSB or running the post- 1701 * relocator after relocating the TSB's memory. Unfortunately, we 1702 * can't free memory while holding the HAT lock or we could 1703 * deadlock, so we build a list of TSBs to be freed after marking 1704 * the tsbinfos as swapped out and free them after dropping the 1705 * lock. 1706 */ 1707 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 1708 sfmmu_hat_exit(hatlockp); 1709 return; 1710 } 1711 1712 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED); 1713 last = freelist = NULL; 1714 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 1715 tsbinfop = tsbinfop->tsb_next) { 1716 ASSERT((tsbinfop->tsb_flags & TSB_SWAPPED) == 0); 1717 1718 /* 1719 * Cast the TSB into a struct free_tsb and put it on the free 1720 * list. 1721 */ 1722 if (freelist == NULL) { 1723 last = freelist = (struct free_tsb *)tsbinfop->tsb_va; 1724 } else { 1725 last->next = (struct free_tsb *)tsbinfop->tsb_va; 1726 last = last->next; 1727 } 1728 last->next = NULL; 1729 last->tsbinfop = tsbinfop; 1730 tsbinfop->tsb_flags |= TSB_SWAPPED; 1731 /* 1732 * Zero out the TTE to clear the valid bit. 1733 * Note we can't use a value like 0xbad because we want to 1734 * ensure diagnostic bits are NEVER set on TTEs that might 1735 * be loaded. The intent is to catch any invalid access 1736 * to the swapped TSB, such as a thread running with a valid 1737 * context without first calling sfmmu_tsb_swapin() to 1738 * allocate TSB memory. 1739 */ 1740 tsbinfop->tsb_tte.ll = 0; 1741 } 1742 1743 #ifdef sun4v 1744 if (freelist) 1745 sfmmu_invalidate_tsbinfo(sfmmup); 1746 #endif /* sun4v */ 1747 1748 /* Now we can drop the lock and free the TSB memory. */ 1749 sfmmu_hat_exit(hatlockp); 1750 for (; freelist != NULL; freelist = next) { 1751 next = freelist->next; 1752 sfmmu_tsb_free(freelist->tsbinfop); 1753 } 1754 } 1755 1756 /* 1757 * Duplicate the translations of an as into another newas 1758 */ 1759 /* ARGSUSED */ 1760 int 1761 hat_dup(struct hat *hat, struct hat *newhat, caddr_t addr, size_t len, 1762 uint_t flag) 1763 { 1764 extern uint_t get_color_start(struct as *); 1765 1766 ASSERT(hat->sfmmu_xhat_provider == NULL); 1767 ASSERT((flag == 0) || (flag == HAT_DUP_ALL) || (flag == HAT_DUP_COW)); 1768 1769 if (flag == HAT_DUP_COW) { 1770 panic("hat_dup: HAT_DUP_COW not supported"); 1771 } 1772 if (flag == HAT_DUP_ALL && consistent_coloring == 0 && 1773 update_proc_pgcolorbase_after_fork != 0) { 1774 hat->sfmmu_clrbin = get_color_start(hat->sfmmu_as); 1775 } 1776 return (0); 1777 } 1778 1779 /* 1780 * Set up addr to map to page pp with protection prot. 1781 * As an optimization we also load the TSB with the 1782 * corresponding tte but it is no big deal if the tte gets kicked out. 1783 */ 1784 void 1785 hat_memload(struct hat *hat, caddr_t addr, struct page *pp, 1786 uint_t attr, uint_t flags) 1787 { 1788 tte_t tte; 1789 1790 1791 ASSERT(hat != NULL); 1792 ASSERT(PAGE_LOCKED(pp)); 1793 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 1794 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1795 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1796 1797 if (PP_ISFREE(pp)) { 1798 panic("hat_memload: loading a mapping to free page %p", 1799 (void *)pp); 1800 } 1801 1802 if (hat->sfmmu_xhat_provider) { 1803 XHAT_MEMLOAD(hat, addr, pp, attr, flags); 1804 return; 1805 } 1806 1807 ASSERT((hat == ksfmmup) || 1808 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1809 1810 if (flags & ~SFMMU_LOAD_ALLFLAG) 1811 cmn_err(CE_NOTE, "hat_memload: unsupported flags %d", 1812 flags & ~SFMMU_LOAD_ALLFLAG); 1813 1814 if (hat->sfmmu_rmstat) 1815 hat_resvstat(MMU_PAGESIZE, hat->sfmmu_as, addr); 1816 1817 #if defined(SF_ERRATA_57) 1818 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1819 (addr < errata57_limit) && (attr & PROT_EXEC) && 1820 !(flags & HAT_LOAD_SHARE)) { 1821 cmn_err(CE_WARN, "hat_memload: illegal attempt to make user " 1822 " page executable"); 1823 attr &= ~PROT_EXEC; 1824 } 1825 #endif 1826 1827 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 1828 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, flags); 1829 1830 /* 1831 * Check TSB and TLB page sizes. 1832 */ 1833 if ((flags & HAT_LOAD_SHARE) == 0) { 1834 sfmmu_check_page_sizes(hat, 1); 1835 } 1836 } 1837 1838 /* 1839 * hat_devload can be called to map real memory (e.g. 1840 * /dev/kmem) and even though hat_devload will determine pf is 1841 * for memory, it will be unable to get a shared lock on the 1842 * page (because someone else has it exclusively) and will 1843 * pass dp = NULL. If tteload doesn't get a non-NULL 1844 * page pointer it can't cache memory. 1845 */ 1846 void 1847 hat_devload(struct hat *hat, caddr_t addr, size_t len, pfn_t pfn, 1848 uint_t attr, int flags) 1849 { 1850 tte_t tte; 1851 struct page *pp = NULL; 1852 int use_lgpg = 0; 1853 1854 ASSERT(hat != NULL); 1855 1856 if (hat->sfmmu_xhat_provider) { 1857 XHAT_DEVLOAD(hat, addr, len, pfn, attr, flags); 1858 return; 1859 } 1860 1861 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1862 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1863 ASSERT((hat == ksfmmup) || 1864 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1865 if (len == 0) 1866 panic("hat_devload: zero len"); 1867 if (flags & ~SFMMU_LOAD_ALLFLAG) 1868 cmn_err(CE_NOTE, "hat_devload: unsupported flags %d", 1869 flags & ~SFMMU_LOAD_ALLFLAG); 1870 1871 #if defined(SF_ERRATA_57) 1872 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1873 (addr < errata57_limit) && (attr & PROT_EXEC) && 1874 !(flags & HAT_LOAD_SHARE)) { 1875 cmn_err(CE_WARN, "hat_devload: illegal attempt to make user " 1876 " page executable"); 1877 attr &= ~PROT_EXEC; 1878 } 1879 #endif 1880 1881 /* 1882 * If it's a memory page find its pp 1883 */ 1884 if (!(flags & HAT_LOAD_NOCONSIST) && pf_is_memory(pfn)) { 1885 pp = page_numtopp_nolock(pfn); 1886 if (pp == NULL) { 1887 flags |= HAT_LOAD_NOCONSIST; 1888 } else { 1889 if (PP_ISFREE(pp)) { 1890 panic("hat_memload: loading " 1891 "a mapping to free page %p", 1892 (void *)pp); 1893 } 1894 if (!PAGE_LOCKED(pp) && !PP_ISNORELOC(pp)) { 1895 panic("hat_memload: loading a mapping " 1896 "to unlocked relocatable page %p", 1897 (void *)pp); 1898 } 1899 ASSERT(len == MMU_PAGESIZE); 1900 } 1901 } 1902 1903 if (hat->sfmmu_rmstat) 1904 hat_resvstat(len, hat->sfmmu_as, addr); 1905 1906 if (flags & HAT_LOAD_NOCONSIST) { 1907 attr |= SFMMU_UNCACHEVTTE; 1908 use_lgpg = 1; 1909 } 1910 if (!pf_is_memory(pfn)) { 1911 attr |= SFMMU_UNCACHEPTTE | HAT_NOSYNC; 1912 use_lgpg = 1; 1913 switch (attr & HAT_ORDER_MASK) { 1914 case HAT_STRICTORDER: 1915 case HAT_UNORDERED_OK: 1916 /* 1917 * we set the side effect bit for all non 1918 * memory mappings unless merging is ok 1919 */ 1920 attr |= SFMMU_SIDEFFECT; 1921 break; 1922 case HAT_MERGING_OK: 1923 case HAT_LOADCACHING_OK: 1924 case HAT_STORECACHING_OK: 1925 break; 1926 default: 1927 panic("hat_devload: bad attr"); 1928 break; 1929 } 1930 } 1931 while (len) { 1932 if (!use_lgpg) { 1933 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1934 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1935 flags); 1936 len -= MMU_PAGESIZE; 1937 addr += MMU_PAGESIZE; 1938 pfn++; 1939 continue; 1940 } 1941 /* 1942 * try to use large pages, check va/pa alignments 1943 * Note that 32M/256M page sizes are not (yet) supported. 1944 */ 1945 if ((len >= MMU_PAGESIZE4M) && 1946 !((uintptr_t)addr & MMU_PAGEOFFSET4M) && 1947 !(disable_large_pages & (1 << TTE4M)) && 1948 !(mmu_ptob(pfn) & MMU_PAGEOFFSET4M)) { 1949 sfmmu_memtte(&tte, pfn, attr, TTE4M); 1950 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1951 flags); 1952 len -= MMU_PAGESIZE4M; 1953 addr += MMU_PAGESIZE4M; 1954 pfn += MMU_PAGESIZE4M / MMU_PAGESIZE; 1955 } else if ((len >= MMU_PAGESIZE512K) && 1956 !((uintptr_t)addr & MMU_PAGEOFFSET512K) && 1957 !(disable_large_pages & (1 << TTE512K)) && 1958 !(mmu_ptob(pfn) & MMU_PAGEOFFSET512K)) { 1959 sfmmu_memtte(&tte, pfn, attr, TTE512K); 1960 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1961 flags); 1962 len -= MMU_PAGESIZE512K; 1963 addr += MMU_PAGESIZE512K; 1964 pfn += MMU_PAGESIZE512K / MMU_PAGESIZE; 1965 } else if ((len >= MMU_PAGESIZE64K) && 1966 !((uintptr_t)addr & MMU_PAGEOFFSET64K) && 1967 !(disable_large_pages & (1 << TTE64K)) && 1968 !(mmu_ptob(pfn) & MMU_PAGEOFFSET64K)) { 1969 sfmmu_memtte(&tte, pfn, attr, TTE64K); 1970 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1971 flags); 1972 len -= MMU_PAGESIZE64K; 1973 addr += MMU_PAGESIZE64K; 1974 pfn += MMU_PAGESIZE64K / MMU_PAGESIZE; 1975 } else { 1976 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1977 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1978 flags); 1979 len -= MMU_PAGESIZE; 1980 addr += MMU_PAGESIZE; 1981 pfn++; 1982 } 1983 } 1984 1985 /* 1986 * Check TSB and TLB page sizes. 1987 */ 1988 if ((flags & HAT_LOAD_SHARE) == 0) { 1989 sfmmu_check_page_sizes(hat, 1); 1990 } 1991 } 1992 1993 /* 1994 * Map the largest extend possible out of the page array. The array may NOT 1995 * be in order. The largest possible mapping a page can have 1996 * is specified in the p_szc field. The p_szc field 1997 * cannot change as long as there any mappings (large or small) 1998 * to any of the pages that make up the large page. (ie. any 1999 * promotion/demotion of page size is not up to the hat but up to 2000 * the page free list manager). The array 2001 * should consist of properly aligned contigous pages that are 2002 * part of a big page for a large mapping to be created. 2003 */ 2004 void 2005 hat_memload_array(struct hat *hat, caddr_t addr, size_t len, 2006 struct page **pps, uint_t attr, uint_t flags) 2007 { 2008 int ttesz; 2009 size_t mapsz; 2010 pgcnt_t numpg, npgs; 2011 tte_t tte; 2012 page_t *pp; 2013 uint_t large_pages_disable; 2014 2015 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 2016 2017 if (hat->sfmmu_xhat_provider) { 2018 XHAT_MEMLOAD_ARRAY(hat, addr, len, pps, attr, flags); 2019 return; 2020 } 2021 2022 if (hat->sfmmu_rmstat) 2023 hat_resvstat(len, hat->sfmmu_as, addr); 2024 2025 #if defined(SF_ERRATA_57) 2026 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 2027 (addr < errata57_limit) && (attr & PROT_EXEC) && 2028 !(flags & HAT_LOAD_SHARE)) { 2029 cmn_err(CE_WARN, "hat_memload_array: illegal attempt to make " 2030 "user page executable"); 2031 attr &= ~PROT_EXEC; 2032 } 2033 #endif 2034 2035 /* Get number of pages */ 2036 npgs = len >> MMU_PAGESHIFT; 2037 2038 if (flags & HAT_LOAD_SHARE) { 2039 large_pages_disable = disable_ism_large_pages; 2040 } else { 2041 large_pages_disable = disable_large_pages; 2042 } 2043 2044 if (npgs < NHMENTS || large_pages_disable == LARGE_PAGES_OFF) { 2045 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2046 return; 2047 } 2048 2049 while (npgs >= NHMENTS) { 2050 pp = *pps; 2051 for (ttesz = pp->p_szc; ttesz != TTE8K; ttesz--) { 2052 /* 2053 * Check if this page size is disabled. 2054 */ 2055 if (large_pages_disable & (1 << ttesz)) 2056 continue; 2057 2058 numpg = TTEPAGES(ttesz); 2059 mapsz = numpg << MMU_PAGESHIFT; 2060 if ((npgs >= numpg) && 2061 IS_P2ALIGNED(addr, mapsz) && 2062 IS_P2ALIGNED(pp->p_pagenum, numpg)) { 2063 /* 2064 * At this point we have enough pages and 2065 * we know the virtual address and the pfn 2066 * are properly aligned. We still need 2067 * to check for physical contiguity but since 2068 * it is very likely that this is the case 2069 * we will assume they are so and undo 2070 * the request if necessary. It would 2071 * be great if we could get a hint flag 2072 * like HAT_CONTIG which would tell us 2073 * the pages are contigous for sure. 2074 */ 2075 sfmmu_memtte(&tte, (*pps)->p_pagenum, 2076 attr, ttesz); 2077 if (!sfmmu_tteload_array(hat, &tte, addr, 2078 pps, flags)) { 2079 break; 2080 } 2081 } 2082 } 2083 if (ttesz == TTE8K) { 2084 /* 2085 * We were not able to map array using a large page 2086 * batch a hmeblk or fraction at a time. 2087 */ 2088 numpg = ((uintptr_t)addr >> MMU_PAGESHIFT) 2089 & (NHMENTS-1); 2090 numpg = NHMENTS - numpg; 2091 ASSERT(numpg <= npgs); 2092 mapsz = numpg * MMU_PAGESIZE; 2093 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, 2094 numpg); 2095 } 2096 addr += mapsz; 2097 npgs -= numpg; 2098 pps += numpg; 2099 } 2100 2101 if (npgs) { 2102 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2103 } 2104 2105 /* 2106 * Check TSB and TLB page sizes. 2107 */ 2108 if ((flags & HAT_LOAD_SHARE) == 0) { 2109 sfmmu_check_page_sizes(hat, 1); 2110 } 2111 } 2112 2113 /* 2114 * Function tries to batch 8K pages into the same hme blk. 2115 */ 2116 static void 2117 sfmmu_memload_batchsmall(struct hat *hat, caddr_t vaddr, page_t **pps, 2118 uint_t attr, uint_t flags, pgcnt_t npgs) 2119 { 2120 tte_t tte; 2121 page_t *pp; 2122 struct hmehash_bucket *hmebp; 2123 struct hme_blk *hmeblkp; 2124 int index; 2125 2126 while (npgs) { 2127 /* 2128 * Acquire the hash bucket. 2129 */ 2130 hmebp = sfmmu_tteload_acquire_hashbucket(hat, vaddr, TTE8K); 2131 ASSERT(hmebp); 2132 2133 /* 2134 * Find the hment block. 2135 */ 2136 hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr, 2137 TTE8K, flags); 2138 ASSERT(hmeblkp); 2139 2140 do { 2141 /* 2142 * Make the tte. 2143 */ 2144 pp = *pps; 2145 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 2146 2147 /* 2148 * Add the translation. 2149 */ 2150 (void) sfmmu_tteload_addentry(hat, hmeblkp, &tte, 2151 vaddr, pps, flags); 2152 2153 /* 2154 * Goto next page. 2155 */ 2156 pps++; 2157 npgs--; 2158 2159 /* 2160 * Goto next address. 2161 */ 2162 vaddr += MMU_PAGESIZE; 2163 2164 /* 2165 * Don't crossover into a different hmentblk. 2166 */ 2167 index = (int)(((uintptr_t)vaddr >> MMU_PAGESHIFT) & 2168 (NHMENTS-1)); 2169 2170 } while (index != 0 && npgs != 0); 2171 2172 /* 2173 * Release the hash bucket. 2174 */ 2175 2176 sfmmu_tteload_release_hashbucket(hmebp); 2177 } 2178 } 2179 2180 /* 2181 * Construct a tte for a page: 2182 * 2183 * tte_valid = 1 2184 * tte_size2 = size & TTE_SZ2_BITS (Panther and Olympus-C only) 2185 * tte_size = size 2186 * tte_nfo = attr & HAT_NOFAULT 2187 * tte_ie = attr & HAT_STRUCTURE_LE 2188 * tte_hmenum = hmenum 2189 * tte_pahi = pp->p_pagenum >> TTE_PASHIFT; 2190 * tte_palo = pp->p_pagenum & TTE_PALOMASK; 2191 * tte_ref = 1 (optimization) 2192 * tte_wr_perm = attr & PROT_WRITE; 2193 * tte_no_sync = attr & HAT_NOSYNC 2194 * tte_lock = attr & SFMMU_LOCKTTE 2195 * tte_cp = !(attr & SFMMU_UNCACHEPTTE) 2196 * tte_cv = !(attr & SFMMU_UNCACHEVTTE) 2197 * tte_e = attr & SFMMU_SIDEFFECT 2198 * tte_priv = !(attr & PROT_USER) 2199 * tte_hwwr = if nosync is set and it is writable we set the mod bit (opt) 2200 * tte_glb = 0 2201 */ 2202 void 2203 sfmmu_memtte(tte_t *ttep, pfn_t pfn, uint_t attr, int tte_sz) 2204 { 2205 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 2206 2207 ttep->tte_inthi = MAKE_TTE_INTHI(pfn, attr, tte_sz, 0 /* hmenum */); 2208 ttep->tte_intlo = MAKE_TTE_INTLO(pfn, attr, tte_sz, 0 /* hmenum */); 2209 2210 if (TTE_IS_NOSYNC(ttep)) { 2211 TTE_SET_REF(ttep); 2212 if (TTE_IS_WRITABLE(ttep)) { 2213 TTE_SET_MOD(ttep); 2214 } 2215 } 2216 if (TTE_IS_NFO(ttep) && TTE_IS_EXECUTABLE(ttep)) { 2217 panic("sfmmu_memtte: can't set both NFO and EXEC bits"); 2218 } 2219 } 2220 2221 /* 2222 * This function will add a translation to the hme_blk and allocate the 2223 * hme_blk if one does not exist. 2224 * If a page structure is specified then it will add the 2225 * corresponding hment to the mapping list. 2226 * It will also update the hmenum field for the tte. 2227 */ 2228 void 2229 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp, 2230 uint_t flags) 2231 { 2232 (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags); 2233 } 2234 2235 /* 2236 * Load (ttep != NULL) or unload (ttep == NULL) one entry in the TSB. 2237 * Assumes that a particular page size may only be resident in one TSB. 2238 */ 2239 static void 2240 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz) 2241 { 2242 struct tsb_info *tsbinfop = NULL; 2243 uint64_t tag; 2244 struct tsbe *tsbe_addr; 2245 uint64_t tsb_base; 2246 uint_t tsb_size; 2247 int vpshift = MMU_PAGESHIFT; 2248 int phys = 0; 2249 2250 if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */ 2251 phys = ktsb_phys; 2252 if (ttesz >= TTE4M) { 2253 #ifndef sun4v 2254 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2255 #endif 2256 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2257 tsb_size = ktsb4m_szcode; 2258 } else { 2259 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2260 tsb_size = ktsb_szcode; 2261 } 2262 } else { 2263 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2264 2265 /* 2266 * If there isn't a TSB for this page size, or the TSB is 2267 * swapped out, there is nothing to do. Note that the latter 2268 * case seems impossible but can occur if hat_pageunload() 2269 * is called on an ISM mapping while the process is swapped 2270 * out. 2271 */ 2272 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2273 return; 2274 2275 /* 2276 * If another thread is in the middle of relocating a TSB 2277 * we can't unload the entry so set a flag so that the 2278 * TSB will be flushed before it can be accessed by the 2279 * process. 2280 */ 2281 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2282 if (ttep == NULL) 2283 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2284 return; 2285 } 2286 #if defined(UTSB_PHYS) 2287 phys = 1; 2288 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2289 #else 2290 tsb_base = (uint64_t)tsbinfop->tsb_va; 2291 #endif 2292 tsb_size = tsbinfop->tsb_szc; 2293 } 2294 if (ttesz >= TTE4M) 2295 vpshift = MMU_PAGESHIFT4M; 2296 2297 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2298 tag = sfmmu_make_tsbtag(vaddr); 2299 2300 if (ttep == NULL) { 2301 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2302 } else { 2303 if (ttesz >= TTE4M) { 2304 SFMMU_STAT(sf_tsb_load4m); 2305 } else { 2306 SFMMU_STAT(sf_tsb_load8k); 2307 } 2308 2309 sfmmu_load_tsbe(tsbe_addr, tag, ttep, phys); 2310 } 2311 } 2312 2313 /* 2314 * Unmap all entries from [start, end) matching the given page size. 2315 * 2316 * This function is used primarily to unmap replicated 64K or 512K entries 2317 * from the TSB that are inserted using the base page size TSB pointer, but 2318 * it may also be called to unmap a range of addresses from the TSB. 2319 */ 2320 void 2321 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz) 2322 { 2323 struct tsb_info *tsbinfop; 2324 uint64_t tag; 2325 struct tsbe *tsbe_addr; 2326 caddr_t vaddr; 2327 uint64_t tsb_base; 2328 int vpshift, vpgsz; 2329 uint_t tsb_size; 2330 int phys = 0; 2331 2332 /* 2333 * Assumptions: 2334 * If ttesz == 8K, 64K or 512K, we walk through the range 8K 2335 * at a time shooting down any valid entries we encounter. 2336 * 2337 * If ttesz >= 4M we walk the range 4M at a time shooting 2338 * down any valid mappings we find. 2339 */ 2340 if (sfmmup == ksfmmup) { 2341 phys = ktsb_phys; 2342 if (ttesz >= TTE4M) { 2343 #ifndef sun4v 2344 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2345 #endif 2346 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2347 tsb_size = ktsb4m_szcode; 2348 } else { 2349 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2350 tsb_size = ktsb_szcode; 2351 } 2352 } else { 2353 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2354 2355 /* 2356 * If there isn't a TSB for this page size, or the TSB is 2357 * swapped out, there is nothing to do. Note that the latter 2358 * case seems impossible but can occur if hat_pageunload() 2359 * is called on an ISM mapping while the process is swapped 2360 * out. 2361 */ 2362 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2363 return; 2364 2365 /* 2366 * If another thread is in the middle of relocating a TSB 2367 * we can't unload the entry so set a flag so that the 2368 * TSB will be flushed before it can be accessed by the 2369 * process. 2370 */ 2371 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2372 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2373 return; 2374 } 2375 #if defined(UTSB_PHYS) 2376 phys = 1; 2377 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2378 #else 2379 tsb_base = (uint64_t)tsbinfop->tsb_va; 2380 #endif 2381 tsb_size = tsbinfop->tsb_szc; 2382 } 2383 if (ttesz >= TTE4M) { 2384 vpshift = MMU_PAGESHIFT4M; 2385 vpgsz = MMU_PAGESIZE4M; 2386 } else { 2387 vpshift = MMU_PAGESHIFT; 2388 vpgsz = MMU_PAGESIZE; 2389 } 2390 2391 for (vaddr = start; vaddr < end; vaddr += vpgsz) { 2392 tag = sfmmu_make_tsbtag(vaddr); 2393 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2394 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2395 } 2396 } 2397 2398 /* 2399 * Select the optimum TSB size given the number of mappings 2400 * that need to be cached. 2401 */ 2402 static int 2403 sfmmu_select_tsb_szc(pgcnt_t pgcnt) 2404 { 2405 int szc = 0; 2406 2407 #ifdef DEBUG 2408 if (tsb_grow_stress) { 2409 uint32_t randval = (uint32_t)gettick() >> 4; 2410 return (randval % (tsb_max_growsize + 1)); 2411 } 2412 #endif /* DEBUG */ 2413 2414 while ((szc < tsb_max_growsize) && (pgcnt > SFMMU_RSS_TSBSIZE(szc))) 2415 szc++; 2416 return (szc); 2417 } 2418 2419 /* 2420 * This function will add a translation to the hme_blk and allocate the 2421 * hme_blk if one does not exist. 2422 * If a page structure is specified then it will add the 2423 * corresponding hment to the mapping list. 2424 * It will also update the hmenum field for the tte. 2425 * Furthermore, it attempts to create a large page translation 2426 * for <addr,hat> at page array pps. It assumes addr and first 2427 * pp is correctly aligned. It returns 0 if successful and 1 otherwise. 2428 */ 2429 static int 2430 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr, 2431 page_t **pps, uint_t flags) 2432 { 2433 struct hmehash_bucket *hmebp; 2434 struct hme_blk *hmeblkp; 2435 int ret; 2436 uint_t size; 2437 2438 /* 2439 * Get mapping size. 2440 */ 2441 size = TTE_CSZ(ttep); 2442 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2443 2444 /* 2445 * Acquire the hash bucket. 2446 */ 2447 hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size); 2448 ASSERT(hmebp); 2449 2450 /* 2451 * Find the hment block. 2452 */ 2453 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags); 2454 ASSERT(hmeblkp); 2455 2456 /* 2457 * Add the translation. 2458 */ 2459 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags); 2460 2461 /* 2462 * Release the hash bucket. 2463 */ 2464 sfmmu_tteload_release_hashbucket(hmebp); 2465 2466 return (ret); 2467 } 2468 2469 /* 2470 * Function locks and returns a pointer to the hash bucket for vaddr and size. 2471 */ 2472 static struct hmehash_bucket * 2473 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size) 2474 { 2475 struct hmehash_bucket *hmebp; 2476 int hmeshift; 2477 2478 hmeshift = HME_HASH_SHIFT(size); 2479 2480 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 2481 2482 SFMMU_HASH_LOCK(hmebp); 2483 2484 return (hmebp); 2485 } 2486 2487 /* 2488 * Function returns a pointer to an hmeblk in the hash bucket, hmebp. If the 2489 * hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is 2490 * allocated. 2491 */ 2492 static struct hme_blk * 2493 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp, 2494 caddr_t vaddr, uint_t size, uint_t flags) 2495 { 2496 hmeblk_tag hblktag; 2497 int hmeshift; 2498 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 2499 uint64_t hblkpa, prevpa; 2500 struct kmem_cache *sfmmu_cache; 2501 uint_t forcefree; 2502 2503 hblktag.htag_id = sfmmup; 2504 hmeshift = HME_HASH_SHIFT(size); 2505 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 2506 hblktag.htag_rehash = HME_HASH_REHASH(size); 2507 2508 ttearray_realloc: 2509 2510 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, 2511 pr_hblk, prevpa, &list); 2512 2513 /* 2514 * We block until hblk_reserve_lock is released; it's held by 2515 * the thread, temporarily using hblk_reserve, until hblk_reserve is 2516 * replaced by a hblk from sfmmu8_cache. 2517 */ 2518 if (hmeblkp == (struct hme_blk *)hblk_reserve && 2519 hblk_reserve_thread != curthread) { 2520 SFMMU_HASH_UNLOCK(hmebp); 2521 mutex_enter(&hblk_reserve_lock); 2522 mutex_exit(&hblk_reserve_lock); 2523 SFMMU_STAT(sf_hblk_reserve_hit); 2524 SFMMU_HASH_LOCK(hmebp); 2525 goto ttearray_realloc; 2526 } 2527 2528 if (hmeblkp == NULL) { 2529 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 2530 hblktag, flags); 2531 } else { 2532 /* 2533 * It is possible for 8k and 64k hblks to collide since they 2534 * have the same rehash value. This is because we 2535 * lazily free hblks and 8K/64K blks could be lingering. 2536 * If we find size mismatch we free the block and & try again. 2537 */ 2538 if (get_hblk_ttesz(hmeblkp) != size) { 2539 ASSERT(!hmeblkp->hblk_vcnt); 2540 ASSERT(!hmeblkp->hblk_hmecnt); 2541 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 2542 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 2543 goto ttearray_realloc; 2544 } 2545 if (hmeblkp->hblk_shw_bit) { 2546 /* 2547 * if the hblk was previously used as a shadow hblk then 2548 * we will change it to a normal hblk 2549 */ 2550 if (hmeblkp->hblk_shw_mask) { 2551 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp); 2552 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2553 goto ttearray_realloc; 2554 } else { 2555 hmeblkp->hblk_shw_bit = 0; 2556 } 2557 } 2558 SFMMU_STAT(sf_hblk_hit); 2559 } 2560 2561 /* 2562 * hat_memload() should never call kmem_cache_free(); see block 2563 * comment showing the stacktrace in sfmmu_hblk_alloc(); 2564 * enqueue each hblk in the list to reserve list if it's created 2565 * from sfmmu8_cache *and* sfmmup == KHATID. 2566 */ 2567 forcefree = (sfmmup == KHATID) ? 1 : 0; 2568 while ((pr_hblk = list) != NULL) { 2569 list = pr_hblk->hblk_next; 2570 sfmmu_cache = get_hblk_cache(pr_hblk); 2571 if ((sfmmu_cache == sfmmu8_cache) && 2572 sfmmu_put_free_hblk(pr_hblk, forcefree)) 2573 continue; 2574 2575 ASSERT(sfmmup != KHATID); 2576 kmem_cache_free(sfmmu_cache, pr_hblk); 2577 } 2578 2579 ASSERT(get_hblk_ttesz(hmeblkp) == size); 2580 ASSERT(!hmeblkp->hblk_shw_bit); 2581 2582 return (hmeblkp); 2583 } 2584 2585 /* 2586 * Function adds a tte entry into the hmeblk. It returns 0 if successful and 1 2587 * otherwise. 2588 */ 2589 static int 2590 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep, 2591 caddr_t vaddr, page_t **pps, uint_t flags) 2592 { 2593 page_t *pp = *pps; 2594 int hmenum, size, remap; 2595 tte_t tteold, flush_tte; 2596 #ifdef DEBUG 2597 tte_t orig_old; 2598 #endif /* DEBUG */ 2599 struct sf_hment *sfhme; 2600 kmutex_t *pml, *pmtx; 2601 hatlock_t *hatlockp; 2602 2603 /* 2604 * remove this panic when we decide to let user virtual address 2605 * space be >= USERLIMIT. 2606 */ 2607 if (!TTE_IS_PRIVILEGED(ttep) && vaddr >= (caddr_t)USERLIMIT) 2608 panic("user addr %p in kernel space", vaddr); 2609 #if defined(TTE_IS_GLOBAL) 2610 if (TTE_IS_GLOBAL(ttep)) 2611 panic("sfmmu_tteload: creating global tte"); 2612 #endif 2613 2614 #ifdef DEBUG 2615 if (pf_is_memory(sfmmu_ttetopfn(ttep, vaddr)) && 2616 !TTE_IS_PCACHEABLE(ttep) && !sfmmu_allow_nc_trans) 2617 panic("sfmmu_tteload: non cacheable memory tte"); 2618 #endif /* DEBUG */ 2619 2620 if ((flags & HAT_LOAD_SHARE) || !TTE_IS_REF(ttep) || 2621 !TTE_IS_MOD(ttep)) { 2622 /* 2623 * Don't load TSB for dummy as in ISM. Also don't preload 2624 * the TSB if the TTE isn't writable since we're likely to 2625 * fault on it again -- preloading can be fairly expensive. 2626 */ 2627 flags |= SFMMU_NO_TSBLOAD; 2628 } 2629 2630 size = TTE_CSZ(ttep); 2631 switch (size) { 2632 case TTE8K: 2633 SFMMU_STAT(sf_tteload8k); 2634 break; 2635 case TTE64K: 2636 SFMMU_STAT(sf_tteload64k); 2637 break; 2638 case TTE512K: 2639 SFMMU_STAT(sf_tteload512k); 2640 break; 2641 case TTE4M: 2642 SFMMU_STAT(sf_tteload4m); 2643 break; 2644 case (TTE32M): 2645 SFMMU_STAT(sf_tteload32m); 2646 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2647 break; 2648 case (TTE256M): 2649 SFMMU_STAT(sf_tteload256m); 2650 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2651 break; 2652 } 2653 2654 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2655 2656 HBLKTOHME_IDX(sfhme, hmeblkp, vaddr, hmenum); 2657 2658 /* 2659 * Need to grab mlist lock here so that pageunload 2660 * will not change tte behind us. 2661 */ 2662 if (pp) { 2663 pml = sfmmu_mlist_enter(pp); 2664 } 2665 2666 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2667 /* 2668 * Look for corresponding hment and if valid verify 2669 * pfns are equal. 2670 */ 2671 remap = TTE_IS_VALID(&tteold); 2672 if (remap) { 2673 pfn_t new_pfn, old_pfn; 2674 2675 old_pfn = TTE_TO_PFN(vaddr, &tteold); 2676 new_pfn = TTE_TO_PFN(vaddr, ttep); 2677 2678 if (flags & HAT_LOAD_REMAP) { 2679 /* make sure we are remapping same type of pages */ 2680 if (pf_is_memory(old_pfn) != pf_is_memory(new_pfn)) { 2681 panic("sfmmu_tteload - tte remap io<->memory"); 2682 } 2683 if (old_pfn != new_pfn && 2684 (pp != NULL || sfhme->hme_page != NULL)) { 2685 panic("sfmmu_tteload - tte remap pp != NULL"); 2686 } 2687 } else if (old_pfn != new_pfn) { 2688 panic("sfmmu_tteload - tte remap, hmeblkp 0x%p", 2689 (void *)hmeblkp); 2690 } 2691 ASSERT(TTE_CSZ(&tteold) == TTE_CSZ(ttep)); 2692 } 2693 2694 if (pp) { 2695 if (size == TTE8K) { 2696 #ifdef VAC 2697 /* 2698 * Handle VAC consistency 2699 */ 2700 if (!remap && (cache & CACHE_VAC) && !PP_ISNC(pp)) { 2701 sfmmu_vac_conflict(sfmmup, vaddr, pp); 2702 } 2703 #endif 2704 2705 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2706 pmtx = sfmmu_page_enter(pp); 2707 PP_CLRRO(pp); 2708 sfmmu_page_exit(pmtx); 2709 } else if (!PP_ISMAPPED(pp) && 2710 (!TTE_IS_WRITABLE(ttep)) && !(PP_ISMOD(pp))) { 2711 pmtx = sfmmu_page_enter(pp); 2712 if (!(PP_ISMOD(pp))) { 2713 PP_SETRO(pp); 2714 } 2715 sfmmu_page_exit(pmtx); 2716 } 2717 2718 } else if (sfmmu_pagearray_setup(vaddr, pps, ttep, remap)) { 2719 /* 2720 * sfmmu_pagearray_setup failed so return 2721 */ 2722 sfmmu_mlist_exit(pml); 2723 return (1); 2724 } 2725 } 2726 2727 /* 2728 * Make sure hment is not on a mapping list. 2729 */ 2730 ASSERT(remap || (sfhme->hme_page == NULL)); 2731 2732 /* if it is not a remap then hme->next better be NULL */ 2733 ASSERT((!remap) ? sfhme->hme_next == NULL : 1); 2734 2735 if (flags & HAT_LOAD_LOCK) { 2736 if (((int)hmeblkp->hblk_lckcnt + 1) >= MAX_HBLK_LCKCNT) { 2737 panic("too high lckcnt-hmeblk %p", 2738 (void *)hmeblkp); 2739 } 2740 atomic_add_16(&hmeblkp->hblk_lckcnt, 1); 2741 2742 HBLK_STACK_TRACE(hmeblkp, HBLK_LOCK); 2743 } 2744 2745 #ifdef VAC 2746 if (pp && PP_ISNC(pp)) { 2747 /* 2748 * If the physical page is marked to be uncacheable, like 2749 * by a vac conflict, make sure the new mapping is also 2750 * uncacheable. 2751 */ 2752 TTE_CLR_VCACHEABLE(ttep); 2753 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 2754 } 2755 #endif 2756 ttep->tte_hmenum = hmenum; 2757 2758 #ifdef DEBUG 2759 orig_old = tteold; 2760 #endif /* DEBUG */ 2761 2762 while (sfmmu_modifytte_try(&tteold, ttep, &sfhme->hme_tte) < 0) { 2763 if ((sfmmup == KHATID) && 2764 (flags & (HAT_LOAD_LOCK | HAT_LOAD_REMAP))) { 2765 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2766 } 2767 #ifdef DEBUG 2768 chk_tte(&orig_old, &tteold, ttep, hmeblkp); 2769 #endif /* DEBUG */ 2770 } 2771 2772 if (!TTE_IS_VALID(&tteold)) { 2773 2774 atomic_add_16(&hmeblkp->hblk_vcnt, 1); 2775 atomic_add_long(&sfmmup->sfmmu_ttecnt[size], 1); 2776 2777 /* 2778 * HAT_RELOAD_SHARE has been deprecated with lpg DISM. 2779 */ 2780 2781 if (size > TTE8K && (flags & HAT_LOAD_SHARE) == 0 && 2782 sfmmup != ksfmmup) { 2783 /* 2784 * If this is the first large mapping for the process 2785 * we must force any CPUs running this process to TL=0 2786 * where they will reload the HAT flags from the 2787 * tsbmiss area. This is necessary to make the large 2788 * mappings we are about to load visible to those CPUs; 2789 * otherwise they'll loop forever calling pagefault() 2790 * since we don't search large hash chains by default. 2791 */ 2792 hatlockp = sfmmu_hat_enter(sfmmup); 2793 if (size == TTE512K && 2794 !SFMMU_FLAGS_ISSET(sfmmup, HAT_512K_FLAG)) { 2795 SFMMU_FLAGS_SET(sfmmup, HAT_512K_FLAG); 2796 sfmmu_sync_mmustate(sfmmup); 2797 } else if (size == TTE4M && 2798 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 2799 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 2800 sfmmu_sync_mmustate(sfmmup); 2801 } else if (size == TTE64K && 2802 !SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) { 2803 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 2804 /* no sync mmustate; 64K shares 8K hashes */ 2805 } else if (mmu_page_sizes == max_mmu_page_sizes) { 2806 if (size == TTE32M && 2807 !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 2808 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 2809 sfmmu_sync_mmustate(sfmmup); 2810 } else if (size == TTE256M && 2811 !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 2812 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 2813 sfmmu_sync_mmustate(sfmmup); 2814 } 2815 } 2816 if (size >= TTE4M && (flags & HAT_LOAD_TEXT) && 2817 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) { 2818 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG); 2819 } 2820 sfmmu_hat_exit(hatlockp); 2821 } 2822 } 2823 ASSERT(TTE_IS_VALID(&sfhme->hme_tte)); 2824 2825 flush_tte.tte_intlo = (tteold.tte_intlo ^ ttep->tte_intlo) & 2826 hw_tte.tte_intlo; 2827 flush_tte.tte_inthi = (tteold.tte_inthi ^ ttep->tte_inthi) & 2828 hw_tte.tte_inthi; 2829 2830 if (remap && (flush_tte.tte_inthi || flush_tte.tte_intlo)) { 2831 /* 2832 * If remap and new tte differs from old tte we need 2833 * to sync the mod bit and flush TLB/TSB. We don't 2834 * need to sync ref bit because we currently always set 2835 * ref bit in tteload. 2836 */ 2837 ASSERT(TTE_IS_REF(ttep)); 2838 if (TTE_IS_MOD(&tteold)) { 2839 sfmmu_ttesync(sfmmup, vaddr, &tteold, pp); 2840 } 2841 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0); 2842 xt_sync(sfmmup->sfmmu_cpusran); 2843 } 2844 2845 if ((flags & SFMMU_NO_TSBLOAD) == 0) { 2846 /* 2847 * We only preload 8K and 4M mappings into the TSB, since 2848 * 64K and 512K mappings are replicated and hence don't 2849 * have a single, unique TSB entry. Ditto for 32M/256M. 2850 */ 2851 if (size == TTE8K || size == TTE4M) { 2852 hatlockp = sfmmu_hat_enter(sfmmup); 2853 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte, size); 2854 sfmmu_hat_exit(hatlockp); 2855 } 2856 } 2857 if (pp) { 2858 if (!remap) { 2859 HME_ADD(sfhme, pp); 2860 atomic_add_16(&hmeblkp->hblk_hmecnt, 1); 2861 ASSERT(hmeblkp->hblk_hmecnt > 0); 2862 2863 /* 2864 * Cannot ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 2865 * see pageunload() for comment. 2866 */ 2867 } 2868 sfmmu_mlist_exit(pml); 2869 } 2870 2871 return (0); 2872 } 2873 /* 2874 * Function unlocks hash bucket. 2875 */ 2876 static void 2877 sfmmu_tteload_release_hashbucket(struct hmehash_bucket *hmebp) 2878 { 2879 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2880 SFMMU_HASH_UNLOCK(hmebp); 2881 } 2882 2883 /* 2884 * function which checks and sets up page array for a large 2885 * translation. Will set p_vcolor, p_index, p_ro fields. 2886 * Assumes addr and pfnum of first page are properly aligned. 2887 * Will check for physical contiguity. If check fails it return 2888 * non null. 2889 */ 2890 static int 2891 sfmmu_pagearray_setup(caddr_t addr, page_t **pps, tte_t *ttep, int remap) 2892 { 2893 int i, index, ttesz; 2894 pfn_t pfnum; 2895 pgcnt_t npgs; 2896 page_t *pp, *pp1; 2897 kmutex_t *pmtx; 2898 #ifdef VAC 2899 int osz; 2900 int cflags = 0; 2901 int vac_err = 0; 2902 #endif 2903 int newidx = 0; 2904 2905 ttesz = TTE_CSZ(ttep); 2906 2907 ASSERT(ttesz > TTE8K); 2908 2909 npgs = TTEPAGES(ttesz); 2910 index = PAGESZ_TO_INDEX(ttesz); 2911 2912 pfnum = (*pps)->p_pagenum; 2913 ASSERT(IS_P2ALIGNED(pfnum, npgs)); 2914 2915 /* 2916 * Save the first pp so we can do HAT_TMPNC at the end. 2917 */ 2918 pp1 = *pps; 2919 #ifdef VAC 2920 osz = fnd_mapping_sz(pp1); 2921 #endif 2922 2923 for (i = 0; i < npgs; i++, pps++) { 2924 pp = *pps; 2925 ASSERT(PAGE_LOCKED(pp)); 2926 ASSERT(pp->p_szc >= ttesz); 2927 ASSERT(pp->p_szc == pp1->p_szc); 2928 ASSERT(sfmmu_mlist_held(pp)); 2929 2930 /* 2931 * XXX is it possible to maintain P_RO on the root only? 2932 */ 2933 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2934 pmtx = sfmmu_page_enter(pp); 2935 PP_CLRRO(pp); 2936 sfmmu_page_exit(pmtx); 2937 } else if (!PP_ISMAPPED(pp) && !TTE_IS_WRITABLE(ttep) && 2938 !PP_ISMOD(pp)) { 2939 pmtx = sfmmu_page_enter(pp); 2940 if (!(PP_ISMOD(pp))) { 2941 PP_SETRO(pp); 2942 } 2943 sfmmu_page_exit(pmtx); 2944 } 2945 2946 /* 2947 * If this is a remap we skip vac & contiguity checks. 2948 */ 2949 if (remap) 2950 continue; 2951 2952 /* 2953 * set p_vcolor and detect any vac conflicts. 2954 */ 2955 #ifdef VAC 2956 if (vac_err == 0) { 2957 vac_err = sfmmu_vacconflict_array(addr, pp, &cflags); 2958 2959 } 2960 #endif 2961 2962 /* 2963 * Save current index in case we need to undo it. 2964 * Note: "PAGESZ_TO_INDEX(sz) (1 << (sz))" 2965 * "SFMMU_INDEX_SHIFT 6" 2966 * "SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1)" 2967 * "PP_MAPINDEX(p_index) (p_index & SFMMU_INDEX_MASK)" 2968 * 2969 * So: index = PAGESZ_TO_INDEX(ttesz); 2970 * if ttesz == 1 then index = 0x2 2971 * 2 then index = 0x4 2972 * 3 then index = 0x8 2973 * 4 then index = 0x10 2974 * 5 then index = 0x20 2975 * The code below checks if it's a new pagesize (ie, newidx) 2976 * in case we need to take it back out of p_index, 2977 * and then or's the new index into the existing index. 2978 */ 2979 if ((PP_MAPINDEX(pp) & index) == 0) 2980 newidx = 1; 2981 pp->p_index = (PP_MAPINDEX(pp) | index); 2982 2983 /* 2984 * contiguity check 2985 */ 2986 if (pp->p_pagenum != pfnum) { 2987 /* 2988 * If we fail the contiguity test then 2989 * the only thing we need to fix is the p_index field. 2990 * We might get a few extra flushes but since this 2991 * path is rare that is ok. The p_ro field will 2992 * get automatically fixed on the next tteload to 2993 * the page. NO TNC bit is set yet. 2994 */ 2995 while (i >= 0) { 2996 pp = *pps; 2997 if (newidx) 2998 pp->p_index = (PP_MAPINDEX(pp) & 2999 ~index); 3000 pps--; 3001 i--; 3002 } 3003 return (1); 3004 } 3005 pfnum++; 3006 addr += MMU_PAGESIZE; 3007 } 3008 3009 #ifdef VAC 3010 if (vac_err) { 3011 if (ttesz > osz) { 3012 /* 3013 * There are some smaller mappings that causes vac 3014 * conflicts. Convert all existing small mappings to 3015 * TNC. 3016 */ 3017 SFMMU_STAT_ADD(sf_uncache_conflict, npgs); 3018 sfmmu_page_cache_array(pp1, HAT_TMPNC, CACHE_FLUSH, 3019 npgs); 3020 } else { 3021 /* EMPTY */ 3022 /* 3023 * If there exists an big page mapping, 3024 * that means the whole existing big page 3025 * has TNC setting already. No need to covert to 3026 * TNC again. 3027 */ 3028 ASSERT(PP_ISTNC(pp1)); 3029 } 3030 } 3031 #endif /* VAC */ 3032 3033 return (0); 3034 } 3035 3036 #ifdef VAC 3037 /* 3038 * Routine that detects vac consistency for a large page. It also 3039 * sets virtual color for all pp's for this big mapping. 3040 */ 3041 static int 3042 sfmmu_vacconflict_array(caddr_t addr, page_t *pp, int *cflags) 3043 { 3044 int vcolor, ocolor; 3045 3046 ASSERT(sfmmu_mlist_held(pp)); 3047 3048 if (PP_ISNC(pp)) { 3049 return (HAT_TMPNC); 3050 } 3051 3052 vcolor = addr_to_vcolor(addr); 3053 if (PP_NEWPAGE(pp)) { 3054 PP_SET_VCOLOR(pp, vcolor); 3055 return (0); 3056 } 3057 3058 ocolor = PP_GET_VCOLOR(pp); 3059 if (ocolor == vcolor) { 3060 return (0); 3061 } 3062 3063 if (!PP_ISMAPPED(pp)) { 3064 /* 3065 * Previous user of page had a differnet color 3066 * but since there are no current users 3067 * we just flush the cache and change the color. 3068 * As an optimization for large pages we flush the 3069 * entire cache of that color and set a flag. 3070 */ 3071 SFMMU_STAT(sf_pgcolor_conflict); 3072 if (!CacheColor_IsFlushed(*cflags, ocolor)) { 3073 CacheColor_SetFlushed(*cflags, ocolor); 3074 sfmmu_cache_flushcolor(ocolor, pp->p_pagenum); 3075 } 3076 PP_SET_VCOLOR(pp, vcolor); 3077 return (0); 3078 } 3079 3080 /* 3081 * We got a real conflict with a current mapping. 3082 * set flags to start unencaching all mappings 3083 * and return failure so we restart looping 3084 * the pp array from the beginning. 3085 */ 3086 return (HAT_TMPNC); 3087 } 3088 #endif /* VAC */ 3089 3090 /* 3091 * creates a large page shadow hmeblk for a tte. 3092 * The purpose of this routine is to allow us to do quick unloads because 3093 * the vm layer can easily pass a very large but sparsely populated range. 3094 */ 3095 static struct hme_blk * 3096 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags) 3097 { 3098 struct hmehash_bucket *hmebp; 3099 hmeblk_tag hblktag; 3100 int hmeshift, size, vshift; 3101 uint_t shw_mask, newshw_mask; 3102 struct hme_blk *hmeblkp; 3103 3104 ASSERT(sfmmup != KHATID); 3105 if (mmu_page_sizes == max_mmu_page_sizes) { 3106 ASSERT(ttesz < TTE256M); 3107 } else { 3108 ASSERT(ttesz < TTE4M); 3109 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 3110 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 3111 } 3112 3113 if (ttesz == TTE8K) { 3114 size = TTE512K; 3115 } else { 3116 size = ++ttesz; 3117 } 3118 3119 hblktag.htag_id = sfmmup; 3120 hmeshift = HME_HASH_SHIFT(size); 3121 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 3122 hblktag.htag_rehash = HME_HASH_REHASH(size); 3123 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 3124 3125 SFMMU_HASH_LOCK(hmebp); 3126 3127 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3128 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 3129 if (hmeblkp == NULL) { 3130 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 3131 hblktag, flags); 3132 } 3133 ASSERT(hmeblkp); 3134 if (!hmeblkp->hblk_shw_mask) { 3135 /* 3136 * if this is a unused hblk it was just allocated or could 3137 * potentially be a previous large page hblk so we need to 3138 * set the shadow bit. 3139 */ 3140 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt); 3141 hmeblkp->hblk_shw_bit = 1; 3142 } else if (hmeblkp->hblk_shw_bit == 0) { 3143 panic("sfmmu_shadow_hcreate: shw bit not set in hmeblkp 0x%p", 3144 (void *)hmeblkp); 3145 } 3146 3147 vshift = vaddr_to_vshift(hblktag, vaddr, size); 3148 ASSERT(vshift < 8); 3149 /* 3150 * Atomically set shw mask bit 3151 */ 3152 do { 3153 shw_mask = hmeblkp->hblk_shw_mask; 3154 newshw_mask = shw_mask | (1 << vshift); 3155 newshw_mask = cas32(&hmeblkp->hblk_shw_mask, shw_mask, 3156 newshw_mask); 3157 } while (newshw_mask != shw_mask); 3158 3159 SFMMU_HASH_UNLOCK(hmebp); 3160 3161 return (hmeblkp); 3162 } 3163 3164 /* 3165 * This routine cleanup a previous shadow hmeblk and changes it to 3166 * a regular hblk. This happens rarely but it is possible 3167 * when a process wants to use large pages and there are hblks still 3168 * lying around from the previous as that used these hmeblks. 3169 * The alternative was to cleanup the shadow hblks at unload time 3170 * but since so few user processes actually use large pages, it is 3171 * better to be lazy and cleanup at this time. 3172 */ 3173 static void 3174 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 3175 struct hmehash_bucket *hmebp) 3176 { 3177 caddr_t addr, endaddr; 3178 int hashno, size; 3179 3180 ASSERT(hmeblkp->hblk_shw_bit); 3181 3182 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 3183 3184 if (!hmeblkp->hblk_shw_mask) { 3185 hmeblkp->hblk_shw_bit = 0; 3186 return; 3187 } 3188 addr = (caddr_t)get_hblk_base(hmeblkp); 3189 endaddr = get_hblk_endaddr(hmeblkp); 3190 size = get_hblk_ttesz(hmeblkp); 3191 hashno = size - 1; 3192 ASSERT(hashno > 0); 3193 SFMMU_HASH_UNLOCK(hmebp); 3194 3195 sfmmu_free_hblks(sfmmup, addr, endaddr, hashno); 3196 3197 SFMMU_HASH_LOCK(hmebp); 3198 } 3199 3200 static void 3201 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr, 3202 int hashno) 3203 { 3204 int hmeshift, shadow = 0; 3205 hmeblk_tag hblktag; 3206 struct hmehash_bucket *hmebp; 3207 struct hme_blk *hmeblkp; 3208 struct hme_blk *nx_hblk, *pr_hblk, *list = NULL; 3209 uint64_t hblkpa, prevpa, nx_pa; 3210 3211 ASSERT(hashno > 0); 3212 hblktag.htag_id = sfmmup; 3213 hblktag.htag_rehash = hashno; 3214 3215 hmeshift = HME_HASH_SHIFT(hashno); 3216 3217 while (addr < endaddr) { 3218 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3219 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3220 SFMMU_HASH_LOCK(hmebp); 3221 /* inline HME_HASH_SEARCH */ 3222 hmeblkp = hmebp->hmeblkp; 3223 hblkpa = hmebp->hmeh_nextpa; 3224 prevpa = 0; 3225 pr_hblk = NULL; 3226 while (hmeblkp) { 3227 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 3228 if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) { 3229 /* found hme_blk */ 3230 if (hmeblkp->hblk_shw_bit) { 3231 if (hmeblkp->hblk_shw_mask) { 3232 shadow = 1; 3233 sfmmu_shadow_hcleanup(sfmmup, 3234 hmeblkp, hmebp); 3235 break; 3236 } else { 3237 hmeblkp->hblk_shw_bit = 0; 3238 } 3239 } 3240 3241 /* 3242 * Hblk_hmecnt and hblk_vcnt could be non zero 3243 * since hblk_unload() does not gurantee that. 3244 * 3245 * XXX - this could cause tteload() to spin 3246 * where sfmmu_shadow_hcleanup() is called. 3247 */ 3248 } 3249 3250 nx_hblk = hmeblkp->hblk_next; 3251 nx_pa = hmeblkp->hblk_nextpa; 3252 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 3253 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 3254 pr_hblk); 3255 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 3256 } else { 3257 pr_hblk = hmeblkp; 3258 prevpa = hblkpa; 3259 } 3260 hmeblkp = nx_hblk; 3261 hblkpa = nx_pa; 3262 } 3263 3264 SFMMU_HASH_UNLOCK(hmebp); 3265 3266 if (shadow) { 3267 /* 3268 * We found another shadow hblk so cleaned its 3269 * children. We need to go back and cleanup 3270 * the original hblk so we don't change the 3271 * addr. 3272 */ 3273 shadow = 0; 3274 } else { 3275 addr = (caddr_t)roundup((uintptr_t)addr + 1, 3276 (1 << hmeshift)); 3277 } 3278 } 3279 sfmmu_hblks_list_purge(&list); 3280 } 3281 3282 /* 3283 * Release one hardware address translation lock on the given address range. 3284 */ 3285 void 3286 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len) 3287 { 3288 struct hmehash_bucket *hmebp; 3289 hmeblk_tag hblktag; 3290 int hmeshift, hashno = 1; 3291 struct hme_blk *hmeblkp, *list = NULL; 3292 caddr_t endaddr; 3293 3294 ASSERT(sfmmup != NULL); 3295 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3296 3297 ASSERT((sfmmup == ksfmmup) || 3298 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 3299 ASSERT((len & MMU_PAGEOFFSET) == 0); 3300 endaddr = addr + len; 3301 hblktag.htag_id = sfmmup; 3302 3303 /* 3304 * Spitfire supports 4 page sizes. 3305 * Most pages are expected to be of the smallest page size (8K) and 3306 * these will not need to be rehashed. 64K pages also don't need to be 3307 * rehashed because an hmeblk spans 64K of address space. 512K pages 3308 * might need 1 rehash and and 4M pages might need 2 rehashes. 3309 */ 3310 while (addr < endaddr) { 3311 hmeshift = HME_HASH_SHIFT(hashno); 3312 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3313 hblktag.htag_rehash = hashno; 3314 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3315 3316 SFMMU_HASH_LOCK(hmebp); 3317 3318 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 3319 if (hmeblkp != NULL) { 3320 /* 3321 * If we encounter a shadow hmeblk then 3322 * we know there are no valid hmeblks mapping 3323 * this address at this size or larger. 3324 * Just increment address by the smallest 3325 * page size. 3326 */ 3327 if (hmeblkp->hblk_shw_bit) { 3328 addr += MMU_PAGESIZE; 3329 } else { 3330 addr = sfmmu_hblk_unlock(hmeblkp, addr, 3331 endaddr); 3332 } 3333 SFMMU_HASH_UNLOCK(hmebp); 3334 hashno = 1; 3335 continue; 3336 } 3337 SFMMU_HASH_UNLOCK(hmebp); 3338 3339 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 3340 /* 3341 * We have traversed the whole list and rehashed 3342 * if necessary without finding the address to unlock 3343 * which should never happen. 3344 */ 3345 panic("sfmmu_unlock: addr not found. " 3346 "addr %p hat %p", (void *)addr, (void *)sfmmup); 3347 } else { 3348 hashno++; 3349 } 3350 } 3351 3352 sfmmu_hblks_list_purge(&list); 3353 } 3354 3355 /* 3356 * Function to unlock a range of addresses in an hmeblk. It returns the 3357 * next address that needs to be unlocked. 3358 * Should be called with the hash lock held. 3359 */ 3360 static caddr_t 3361 sfmmu_hblk_unlock(struct hme_blk *hmeblkp, caddr_t addr, caddr_t endaddr) 3362 { 3363 struct sf_hment *sfhme; 3364 tte_t tteold, ttemod; 3365 int ttesz, ret; 3366 3367 ASSERT(in_hblk_range(hmeblkp, addr)); 3368 ASSERT(hmeblkp->hblk_shw_bit == 0); 3369 3370 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 3371 ttesz = get_hblk_ttesz(hmeblkp); 3372 3373 HBLKTOHME(sfhme, hmeblkp, addr); 3374 while (addr < endaddr) { 3375 readtte: 3376 sfmmu_copytte(&sfhme->hme_tte, &tteold); 3377 if (TTE_IS_VALID(&tteold)) { 3378 3379 ttemod = tteold; 3380 3381 ret = sfmmu_modifytte_try(&tteold, &ttemod, 3382 &sfhme->hme_tte); 3383 3384 if (ret < 0) 3385 goto readtte; 3386 3387 if (hmeblkp->hblk_lckcnt == 0) 3388 panic("zero hblk lckcnt"); 3389 3390 if (((uintptr_t)addr + TTEBYTES(ttesz)) > 3391 (uintptr_t)endaddr) 3392 panic("can't unlock large tte"); 3393 3394 ASSERT(hmeblkp->hblk_lckcnt > 0); 3395 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 3396 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 3397 } else { 3398 panic("sfmmu_hblk_unlock: invalid tte"); 3399 } 3400 addr += TTEBYTES(ttesz); 3401 sfhme++; 3402 } 3403 return (addr); 3404 } 3405 3406 /* 3407 * Physical Address Mapping Framework 3408 * 3409 * General rules: 3410 * 3411 * (1) Applies only to seg_kmem memory pages. To make things easier, 3412 * seg_kpm addresses are also accepted by the routines, but nothing 3413 * is done with them since by definition their PA mappings are static. 3414 * (2) hat_add_callback() may only be called while holding the page lock 3415 * SE_SHARED or SE_EXCL of the underlying page (e.g., as_pagelock()), 3416 * or passing HAC_PAGELOCK flag. 3417 * (3) prehandler() and posthandler() may not call hat_add_callback() or 3418 * hat_delete_callback(), nor should they allocate memory. Post quiesce 3419 * callbacks may not sleep or acquire adaptive mutex locks. 3420 * (4) Either prehandler() or posthandler() (but not both) may be specified 3421 * as being NULL. Specifying an errhandler() is optional. 3422 * 3423 * Details of using the framework: 3424 * 3425 * registering a callback (hat_register_callback()) 3426 * 3427 * Pass prehandler, posthandler, errhandler addresses 3428 * as described below. If capture_cpus argument is nonzero, 3429 * suspend callback to the prehandler will occur with CPUs 3430 * captured and executing xc_loop() and CPUs will remain 3431 * captured until after the posthandler suspend callback 3432 * occurs. 3433 * 3434 * adding a callback (hat_add_callback()) 3435 * 3436 * as_pagelock(); 3437 * hat_add_callback(); 3438 * save returned pfn in private data structures or program registers; 3439 * as_pageunlock(); 3440 * 3441 * prehandler() 3442 * 3443 * Stop all accesses by physical address to this memory page. 3444 * Called twice: the first, PRESUSPEND, is a context safe to acquire 3445 * adaptive locks. The second, SUSPEND, is called at high PIL with 3446 * CPUs captured so adaptive locks may NOT be acquired (and all spin 3447 * locks must be XCALL_PIL or higher locks). 3448 * 3449 * May return the following errors: 3450 * EIO: A fatal error has occurred. This will result in panic. 3451 * EAGAIN: The page cannot be suspended. This will fail the 3452 * relocation. 3453 * 0: Success. 3454 * 3455 * posthandler() 3456 * 3457 * Save new pfn in private data structures or program registers; 3458 * not allowed to fail (non-zero return values will result in panic). 3459 * 3460 * errhandler() 3461 * 3462 * called when an error occurs related to the callback. Currently 3463 * the only such error is HAT_CB_ERR_LEAKED which indicates that 3464 * a page is being freed, but there are still outstanding callback(s) 3465 * registered on the page. 3466 * 3467 * removing a callback (hat_delete_callback(); e.g., prior to freeing memory) 3468 * 3469 * stop using physical address 3470 * hat_delete_callback(); 3471 * 3472 */ 3473 3474 /* 3475 * Register a callback class. Each subsystem should do this once and 3476 * cache the id_t returned for use in setting up and tearing down callbacks. 3477 * 3478 * There is no facility for removing callback IDs once they are created; 3479 * the "key" should be unique for each module, so in case a module is unloaded 3480 * and subsequently re-loaded, we can recycle the module's previous entry. 3481 */ 3482 id_t 3483 hat_register_callback(int key, 3484 int (*prehandler)(caddr_t, uint_t, uint_t, void *), 3485 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t), 3486 int (*errhandler)(caddr_t, uint_t, uint_t, void *), 3487 int capture_cpus) 3488 { 3489 id_t id; 3490 3491 /* 3492 * Search the table for a pre-existing callback associated with 3493 * the identifier "key". If one exists, we re-use that entry in 3494 * the table for this instance, otherwise we assign the next 3495 * available table slot. 3496 */ 3497 for (id = 0; id < sfmmu_max_cb_id; id++) { 3498 if (sfmmu_cb_table[id].key == key) 3499 break; 3500 } 3501 3502 if (id == sfmmu_max_cb_id) { 3503 id = sfmmu_cb_nextid++; 3504 if (id >= sfmmu_max_cb_id) 3505 panic("hat_register_callback: out of callback IDs"); 3506 } 3507 3508 ASSERT(prehandler != NULL || posthandler != NULL); 3509 3510 sfmmu_cb_table[id].key = key; 3511 sfmmu_cb_table[id].prehandler = prehandler; 3512 sfmmu_cb_table[id].posthandler = posthandler; 3513 sfmmu_cb_table[id].errhandler = errhandler; 3514 sfmmu_cb_table[id].capture_cpus = capture_cpus; 3515 3516 return (id); 3517 } 3518 3519 #define HAC_COOKIE_NONE (void *)-1 3520 3521 /* 3522 * Add relocation callbacks to the specified addr/len which will be called 3523 * when relocating the associated page. See the description of pre and 3524 * posthandler above for more details. 3525 * 3526 * If HAC_PAGELOCK is included in flags, the underlying memory page is 3527 * locked internally so the caller must be able to deal with the callback 3528 * running even before this function has returned. If HAC_PAGELOCK is not 3529 * set, it is assumed that the underlying memory pages are locked. 3530 * 3531 * Since the caller must track the individual page boundaries anyway, 3532 * we only allow a callback to be added to a single page (large 3533 * or small). Thus [addr, addr + len) MUST be contained within a single 3534 * page. 3535 * 3536 * Registering multiple callbacks on the same [addr, addr+len) is supported, 3537 * _provided_that_ a unique parameter is specified for each callback. 3538 * If multiple callbacks are registered on the same range the callback will 3539 * be invoked with each unique parameter. Registering the same callback with 3540 * the same argument more than once will result in corrupted kernel state. 3541 * 3542 * Returns the pfn of the underlying kernel page in *rpfn 3543 * on success, or PFN_INVALID on failure. 3544 * 3545 * cookiep (if passed) provides storage space for an opaque cookie 3546 * to return later to hat_delete_callback(). This cookie makes the callback 3547 * deletion significantly quicker by avoiding a potentially lengthy hash 3548 * search. 3549 * 3550 * Returns values: 3551 * 0: success 3552 * ENOMEM: memory allocation failure (e.g. flags was passed as HAC_NOSLEEP) 3553 * EINVAL: callback ID is not valid 3554 * ENXIO: ["vaddr", "vaddr" + len) is not mapped in the kernel's address 3555 * space 3556 * ERANGE: ["vaddr", "vaddr" + len) crosses a page boundary 3557 */ 3558 int 3559 hat_add_callback(id_t callback_id, caddr_t vaddr, uint_t len, uint_t flags, 3560 void *pvt, pfn_t *rpfn, void **cookiep) 3561 { 3562 struct hmehash_bucket *hmebp; 3563 hmeblk_tag hblktag; 3564 struct hme_blk *hmeblkp; 3565 int hmeshift, hashno; 3566 caddr_t saddr, eaddr, baseaddr; 3567 struct pa_hment *pahmep; 3568 struct sf_hment *sfhmep, *osfhmep; 3569 kmutex_t *pml; 3570 tte_t tte; 3571 page_t *pp; 3572 vnode_t *vp; 3573 u_offset_t off; 3574 pfn_t pfn; 3575 int kmflags = (flags & HAC_SLEEP)? KM_SLEEP : KM_NOSLEEP; 3576 int locked = 0; 3577 3578 /* 3579 * For KPM mappings, just return the physical address since we 3580 * don't need to register any callbacks. 3581 */ 3582 if (IS_KPM_ADDR(vaddr)) { 3583 uint64_t paddr; 3584 SFMMU_KPM_VTOP(vaddr, paddr); 3585 *rpfn = btop(paddr); 3586 if (cookiep != NULL) 3587 *cookiep = HAC_COOKIE_NONE; 3588 return (0); 3589 } 3590 3591 if (callback_id < (id_t)0 || callback_id >= sfmmu_cb_nextid) { 3592 *rpfn = PFN_INVALID; 3593 return (EINVAL); 3594 } 3595 3596 if ((pahmep = kmem_cache_alloc(pa_hment_cache, kmflags)) == NULL) { 3597 *rpfn = PFN_INVALID; 3598 return (ENOMEM); 3599 } 3600 3601 sfhmep = &pahmep->sfment; 3602 3603 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3604 eaddr = saddr + len; 3605 3606 rehash: 3607 /* Find the mapping(s) for this page */ 3608 for (hashno = TTE64K, hmeblkp = NULL; 3609 hmeblkp == NULL && hashno <= mmu_hashcnt; 3610 hashno++) { 3611 hmeshift = HME_HASH_SHIFT(hashno); 3612 hblktag.htag_id = ksfmmup; 3613 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3614 hblktag.htag_rehash = hashno; 3615 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3616 3617 SFMMU_HASH_LOCK(hmebp); 3618 3619 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3620 3621 if (hmeblkp == NULL) 3622 SFMMU_HASH_UNLOCK(hmebp); 3623 } 3624 3625 if (hmeblkp == NULL) { 3626 kmem_cache_free(pa_hment_cache, pahmep); 3627 *rpfn = PFN_INVALID; 3628 return (ENXIO); 3629 } 3630 3631 HBLKTOHME(osfhmep, hmeblkp, saddr); 3632 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3633 3634 if (!TTE_IS_VALID(&tte)) { 3635 SFMMU_HASH_UNLOCK(hmebp); 3636 kmem_cache_free(pa_hment_cache, pahmep); 3637 *rpfn = PFN_INVALID; 3638 return (ENXIO); 3639 } 3640 3641 /* 3642 * Make sure the boundaries for the callback fall within this 3643 * single mapping. 3644 */ 3645 baseaddr = (caddr_t)get_hblk_base(hmeblkp); 3646 ASSERT(saddr >= baseaddr); 3647 if (eaddr > saddr + TTEBYTES(TTE_CSZ(&tte))) { 3648 SFMMU_HASH_UNLOCK(hmebp); 3649 kmem_cache_free(pa_hment_cache, pahmep); 3650 *rpfn = PFN_INVALID; 3651 return (ERANGE); 3652 } 3653 3654 pfn = sfmmu_ttetopfn(&tte, vaddr); 3655 3656 /* 3657 * The pfn may not have a page_t underneath in which case we 3658 * just return it. This can happen if we are doing I/O to a 3659 * static portion of the kernel's address space, for instance. 3660 */ 3661 pp = osfhmep->hme_page; 3662 if (pp == NULL) { 3663 SFMMU_HASH_UNLOCK(hmebp); 3664 kmem_cache_free(pa_hment_cache, pahmep); 3665 *rpfn = pfn; 3666 if (cookiep) 3667 *cookiep = HAC_COOKIE_NONE; 3668 return (0); 3669 } 3670 ASSERT(pp == PP_PAGEROOT(pp)); 3671 3672 vp = pp->p_vnode; 3673 off = pp->p_offset; 3674 3675 pml = sfmmu_mlist_enter(pp); 3676 3677 if (flags & HAC_PAGELOCK) { 3678 if (!page_trylock(pp, SE_SHARED)) { 3679 /* 3680 * Somebody is holding SE_EXCL lock. Might 3681 * even be hat_page_relocate(). Drop all 3682 * our locks, lookup the page in &kvp, and 3683 * retry. If it doesn't exist in &kvp and &zvp, 3684 * then we must be dealing with a kernel mapped 3685 * page which doesn't actually belong to 3686 * segkmem so we punt. 3687 */ 3688 sfmmu_mlist_exit(pml); 3689 SFMMU_HASH_UNLOCK(hmebp); 3690 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3691 3692 /* check zvp before giving up */ 3693 if (pp == NULL) 3694 pp = page_lookup(&zvp, (u_offset_t)saddr, 3695 SE_SHARED); 3696 3697 /* Okay, we didn't find it, give up */ 3698 if (pp == NULL) { 3699 kmem_cache_free(pa_hment_cache, pahmep); 3700 *rpfn = pfn; 3701 if (cookiep) 3702 *cookiep = HAC_COOKIE_NONE; 3703 return (0); 3704 } 3705 page_unlock(pp); 3706 goto rehash; 3707 } 3708 locked = 1; 3709 } 3710 3711 if (!PAGE_LOCKED(pp) && !panicstr) 3712 panic("hat_add_callback: page 0x%p not locked", pp); 3713 3714 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3715 pp->p_offset != off) { 3716 /* 3717 * The page moved before we got our hands on it. Drop 3718 * all the locks and try again. 3719 */ 3720 ASSERT((flags & HAC_PAGELOCK) != 0); 3721 sfmmu_mlist_exit(pml); 3722 SFMMU_HASH_UNLOCK(hmebp); 3723 page_unlock(pp); 3724 locked = 0; 3725 goto rehash; 3726 } 3727 3728 if (!VN_ISKAS(vp)) { 3729 /* 3730 * This is not a segkmem page but another page which 3731 * has been kernel mapped. It had better have at least 3732 * a share lock on it. Return the pfn. 3733 */ 3734 sfmmu_mlist_exit(pml); 3735 SFMMU_HASH_UNLOCK(hmebp); 3736 if (locked) 3737 page_unlock(pp); 3738 kmem_cache_free(pa_hment_cache, pahmep); 3739 ASSERT(PAGE_LOCKED(pp)); 3740 *rpfn = pfn; 3741 if (cookiep) 3742 *cookiep = HAC_COOKIE_NONE; 3743 return (0); 3744 } 3745 3746 /* 3747 * Setup this pa_hment and link its embedded dummy sf_hment into 3748 * the mapping list. 3749 */ 3750 pp->p_share++; 3751 pahmep->cb_id = callback_id; 3752 pahmep->addr = vaddr; 3753 pahmep->len = len; 3754 pahmep->refcnt = 1; 3755 pahmep->flags = 0; 3756 pahmep->pvt = pvt; 3757 3758 sfhmep->hme_tte.ll = 0; 3759 sfhmep->hme_data = pahmep; 3760 sfhmep->hme_prev = osfhmep; 3761 sfhmep->hme_next = osfhmep->hme_next; 3762 3763 if (osfhmep->hme_next) 3764 osfhmep->hme_next->hme_prev = sfhmep; 3765 3766 osfhmep->hme_next = sfhmep; 3767 3768 sfmmu_mlist_exit(pml); 3769 SFMMU_HASH_UNLOCK(hmebp); 3770 3771 if (locked) 3772 page_unlock(pp); 3773 3774 *rpfn = pfn; 3775 if (cookiep) 3776 *cookiep = (void *)pahmep; 3777 3778 return (0); 3779 } 3780 3781 /* 3782 * Remove the relocation callbacks from the specified addr/len. 3783 */ 3784 void 3785 hat_delete_callback(caddr_t vaddr, uint_t len, void *pvt, uint_t flags, 3786 void *cookie) 3787 { 3788 struct hmehash_bucket *hmebp; 3789 hmeblk_tag hblktag; 3790 struct hme_blk *hmeblkp; 3791 int hmeshift, hashno; 3792 caddr_t saddr; 3793 struct pa_hment *pahmep; 3794 struct sf_hment *sfhmep, *osfhmep; 3795 kmutex_t *pml; 3796 tte_t tte; 3797 page_t *pp; 3798 vnode_t *vp; 3799 u_offset_t off; 3800 int locked = 0; 3801 3802 /* 3803 * If the cookie is HAC_COOKIE_NONE then there is no pa_hment to 3804 * remove so just return. 3805 */ 3806 if (cookie == HAC_COOKIE_NONE || IS_KPM_ADDR(vaddr)) 3807 return; 3808 3809 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3810 3811 rehash: 3812 /* Find the mapping(s) for this page */ 3813 for (hashno = TTE64K, hmeblkp = NULL; 3814 hmeblkp == NULL && hashno <= mmu_hashcnt; 3815 hashno++) { 3816 hmeshift = HME_HASH_SHIFT(hashno); 3817 hblktag.htag_id = ksfmmup; 3818 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3819 hblktag.htag_rehash = hashno; 3820 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3821 3822 SFMMU_HASH_LOCK(hmebp); 3823 3824 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3825 3826 if (hmeblkp == NULL) 3827 SFMMU_HASH_UNLOCK(hmebp); 3828 } 3829 3830 if (hmeblkp == NULL) 3831 return; 3832 3833 HBLKTOHME(osfhmep, hmeblkp, saddr); 3834 3835 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3836 if (!TTE_IS_VALID(&tte)) { 3837 SFMMU_HASH_UNLOCK(hmebp); 3838 return; 3839 } 3840 3841 pp = osfhmep->hme_page; 3842 if (pp == NULL) { 3843 SFMMU_HASH_UNLOCK(hmebp); 3844 ASSERT(cookie == NULL); 3845 return; 3846 } 3847 3848 vp = pp->p_vnode; 3849 off = pp->p_offset; 3850 3851 pml = sfmmu_mlist_enter(pp); 3852 3853 if (flags & HAC_PAGELOCK) { 3854 if (!page_trylock(pp, SE_SHARED)) { 3855 /* 3856 * Somebody is holding SE_EXCL lock. Might 3857 * even be hat_page_relocate(). Drop all 3858 * our locks, lookup the page in &kvp, and 3859 * retry. If it doesn't exist in &kvp and &zvp, 3860 * then we must be dealing with a kernel mapped 3861 * page which doesn't actually belong to 3862 * segkmem so we punt. 3863 */ 3864 sfmmu_mlist_exit(pml); 3865 SFMMU_HASH_UNLOCK(hmebp); 3866 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3867 /* check zvp before giving up */ 3868 if (pp == NULL) 3869 pp = page_lookup(&zvp, (u_offset_t)saddr, 3870 SE_SHARED); 3871 3872 if (pp == NULL) { 3873 ASSERT(cookie == NULL); 3874 return; 3875 } 3876 page_unlock(pp); 3877 goto rehash; 3878 } 3879 locked = 1; 3880 } 3881 3882 ASSERT(PAGE_LOCKED(pp)); 3883 3884 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3885 pp->p_offset != off) { 3886 /* 3887 * The page moved before we got our hands on it. Drop 3888 * all the locks and try again. 3889 */ 3890 ASSERT((flags & HAC_PAGELOCK) != 0); 3891 sfmmu_mlist_exit(pml); 3892 SFMMU_HASH_UNLOCK(hmebp); 3893 page_unlock(pp); 3894 locked = 0; 3895 goto rehash; 3896 } 3897 3898 if (!VN_ISKAS(vp)) { 3899 /* 3900 * This is not a segkmem page but another page which 3901 * has been kernel mapped. 3902 */ 3903 sfmmu_mlist_exit(pml); 3904 SFMMU_HASH_UNLOCK(hmebp); 3905 if (locked) 3906 page_unlock(pp); 3907 ASSERT(cookie == NULL); 3908 return; 3909 } 3910 3911 if (cookie != NULL) { 3912 pahmep = (struct pa_hment *)cookie; 3913 sfhmep = &pahmep->sfment; 3914 } else { 3915 for (sfhmep = pp->p_mapping; sfhmep != NULL; 3916 sfhmep = sfhmep->hme_next) { 3917 3918 /* 3919 * skip va<->pa mappings 3920 */ 3921 if (!IS_PAHME(sfhmep)) 3922 continue; 3923 3924 pahmep = sfhmep->hme_data; 3925 ASSERT(pahmep != NULL); 3926 3927 /* 3928 * if pa_hment matches, remove it 3929 */ 3930 if ((pahmep->pvt == pvt) && 3931 (pahmep->addr == vaddr) && 3932 (pahmep->len == len)) { 3933 break; 3934 } 3935 } 3936 } 3937 3938 if (sfhmep == NULL) { 3939 if (!panicstr) { 3940 panic("hat_delete_callback: pa_hment not found, pp %p", 3941 (void *)pp); 3942 } 3943 return; 3944 } 3945 3946 /* 3947 * Note: at this point a valid kernel mapping must still be 3948 * present on this page. 3949 */ 3950 pp->p_share--; 3951 if (pp->p_share <= 0) 3952 panic("hat_delete_callback: zero p_share"); 3953 3954 if (--pahmep->refcnt == 0) { 3955 if (pahmep->flags != 0) 3956 panic("hat_delete_callback: pa_hment is busy"); 3957 3958 /* 3959 * Remove sfhmep from the mapping list for the page. 3960 */ 3961 if (sfhmep->hme_prev) { 3962 sfhmep->hme_prev->hme_next = sfhmep->hme_next; 3963 } else { 3964 pp->p_mapping = sfhmep->hme_next; 3965 } 3966 3967 if (sfhmep->hme_next) 3968 sfhmep->hme_next->hme_prev = sfhmep->hme_prev; 3969 3970 sfmmu_mlist_exit(pml); 3971 SFMMU_HASH_UNLOCK(hmebp); 3972 3973 if (locked) 3974 page_unlock(pp); 3975 3976 kmem_cache_free(pa_hment_cache, pahmep); 3977 return; 3978 } 3979 3980 sfmmu_mlist_exit(pml); 3981 SFMMU_HASH_UNLOCK(hmebp); 3982 if (locked) 3983 page_unlock(pp); 3984 } 3985 3986 /* 3987 * hat_probe returns 1 if the translation for the address 'addr' is 3988 * loaded, zero otherwise. 3989 * 3990 * hat_probe should be used only for advisorary purposes because it may 3991 * occasionally return the wrong value. The implementation must guarantee that 3992 * returning the wrong value is a very rare event. hat_probe is used 3993 * to implement optimizations in the segment drivers. 3994 * 3995 */ 3996 int 3997 hat_probe(struct hat *sfmmup, caddr_t addr) 3998 { 3999 pfn_t pfn; 4000 tte_t tte; 4001 4002 ASSERT(sfmmup != NULL); 4003 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4004 4005 ASSERT((sfmmup == ksfmmup) || 4006 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4007 4008 if (sfmmup == ksfmmup) { 4009 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte)) 4010 == PFN_SUSPENDED) { 4011 sfmmu_vatopfn_suspended(addr, sfmmup, &tte); 4012 } 4013 } else { 4014 pfn = sfmmu_uvatopfn(addr, sfmmup); 4015 } 4016 4017 if (pfn != PFN_INVALID) 4018 return (1); 4019 else 4020 return (0); 4021 } 4022 4023 ssize_t 4024 hat_getpagesize(struct hat *sfmmup, caddr_t addr) 4025 { 4026 tte_t tte; 4027 4028 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4029 4030 sfmmu_gettte(sfmmup, addr, &tte); 4031 if (TTE_IS_VALID(&tte)) { 4032 return (TTEBYTES(TTE_CSZ(&tte))); 4033 } 4034 return (-1); 4035 } 4036 4037 static void 4038 sfmmu_gettte(struct hat *sfmmup, caddr_t addr, tte_t *ttep) 4039 { 4040 struct hmehash_bucket *hmebp; 4041 hmeblk_tag hblktag; 4042 int hmeshift, hashno = 1; 4043 struct hme_blk *hmeblkp, *list = NULL; 4044 struct sf_hment *sfhmep; 4045 4046 /* support for ISM */ 4047 ism_map_t *ism_map; 4048 ism_blk_t *ism_blkp; 4049 int i; 4050 sfmmu_t *ism_hatid = NULL; 4051 sfmmu_t *locked_hatid = NULL; 4052 4053 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 4054 4055 ism_blkp = sfmmup->sfmmu_iblk; 4056 if (ism_blkp) { 4057 sfmmu_ismhat_enter(sfmmup, 0); 4058 locked_hatid = sfmmup; 4059 } 4060 while (ism_blkp && ism_hatid == NULL) { 4061 ism_map = ism_blkp->iblk_maps; 4062 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 4063 if (addr >= ism_start(ism_map[i]) && 4064 addr < ism_end(ism_map[i])) { 4065 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 4066 addr = (caddr_t)(addr - 4067 ism_start(ism_map[i])); 4068 break; 4069 } 4070 } 4071 ism_blkp = ism_blkp->iblk_next; 4072 } 4073 if (locked_hatid) { 4074 sfmmu_ismhat_exit(locked_hatid, 0); 4075 } 4076 4077 hblktag.htag_id = sfmmup; 4078 ttep->ll = 0; 4079 4080 do { 4081 hmeshift = HME_HASH_SHIFT(hashno); 4082 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4083 hblktag.htag_rehash = hashno; 4084 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4085 4086 SFMMU_HASH_LOCK(hmebp); 4087 4088 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4089 if (hmeblkp != NULL) { 4090 HBLKTOHME(sfhmep, hmeblkp, addr); 4091 sfmmu_copytte(&sfhmep->hme_tte, ttep); 4092 SFMMU_HASH_UNLOCK(hmebp); 4093 break; 4094 } 4095 SFMMU_HASH_UNLOCK(hmebp); 4096 hashno++; 4097 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 4098 4099 sfmmu_hblks_list_purge(&list); 4100 } 4101 4102 uint_t 4103 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr) 4104 { 4105 tte_t tte; 4106 4107 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4108 4109 sfmmu_gettte(sfmmup, addr, &tte); 4110 if (TTE_IS_VALID(&tte)) { 4111 *attr = sfmmu_ptov_attr(&tte); 4112 return (0); 4113 } 4114 *attr = 0; 4115 return ((uint_t)0xffffffff); 4116 } 4117 4118 /* 4119 * Enables more attributes on specified address range (ie. logical OR) 4120 */ 4121 void 4122 hat_setattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4123 { 4124 if (hat->sfmmu_xhat_provider) { 4125 XHAT_SETATTR(hat, addr, len, attr); 4126 return; 4127 } else { 4128 /* 4129 * This must be a CPU HAT. If the address space has 4130 * XHATs attached, change attributes for all of them, 4131 * just in case 4132 */ 4133 ASSERT(hat->sfmmu_as != NULL); 4134 if (hat->sfmmu_as->a_xhat != NULL) 4135 xhat_setattr_all(hat->sfmmu_as, addr, len, attr); 4136 } 4137 4138 sfmmu_chgattr(hat, addr, len, attr, SFMMU_SETATTR); 4139 } 4140 4141 /* 4142 * Assigns attributes to the specified address range. All the attributes 4143 * are specified. 4144 */ 4145 void 4146 hat_chgattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4147 { 4148 if (hat->sfmmu_xhat_provider) { 4149 XHAT_CHGATTR(hat, addr, len, attr); 4150 return; 4151 } else { 4152 /* 4153 * This must be a CPU HAT. If the address space has 4154 * XHATs attached, change attributes for all of them, 4155 * just in case 4156 */ 4157 ASSERT(hat->sfmmu_as != NULL); 4158 if (hat->sfmmu_as->a_xhat != NULL) 4159 xhat_chgattr_all(hat->sfmmu_as, addr, len, attr); 4160 } 4161 4162 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CHGATTR); 4163 } 4164 4165 /* 4166 * Remove attributes on the specified address range (ie. loginal NAND) 4167 */ 4168 void 4169 hat_clrattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4170 { 4171 if (hat->sfmmu_xhat_provider) { 4172 XHAT_CLRATTR(hat, addr, len, attr); 4173 return; 4174 } else { 4175 /* 4176 * This must be a CPU HAT. If the address space has 4177 * XHATs attached, change attributes for all of them, 4178 * just in case 4179 */ 4180 ASSERT(hat->sfmmu_as != NULL); 4181 if (hat->sfmmu_as->a_xhat != NULL) 4182 xhat_clrattr_all(hat->sfmmu_as, addr, len, attr); 4183 } 4184 4185 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CLRATTR); 4186 } 4187 4188 /* 4189 * Change attributes on an address range to that specified by attr and mode. 4190 */ 4191 static void 4192 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr, 4193 int mode) 4194 { 4195 struct hmehash_bucket *hmebp; 4196 hmeblk_tag hblktag; 4197 int hmeshift, hashno = 1; 4198 struct hme_blk *hmeblkp, *list = NULL; 4199 caddr_t endaddr; 4200 cpuset_t cpuset; 4201 demap_range_t dmr; 4202 4203 CPUSET_ZERO(cpuset); 4204 4205 ASSERT((sfmmup == ksfmmup) || 4206 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4207 ASSERT((len & MMU_PAGEOFFSET) == 0); 4208 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4209 4210 if ((attr & PROT_USER) && (mode != SFMMU_CLRATTR) && 4211 ((addr + len) > (caddr_t)USERLIMIT)) { 4212 panic("user addr %p in kernel space", 4213 (void *)addr); 4214 } 4215 4216 endaddr = addr + len; 4217 hblktag.htag_id = sfmmup; 4218 DEMAP_RANGE_INIT(sfmmup, &dmr); 4219 4220 while (addr < endaddr) { 4221 hmeshift = HME_HASH_SHIFT(hashno); 4222 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4223 hblktag.htag_rehash = hashno; 4224 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4225 4226 SFMMU_HASH_LOCK(hmebp); 4227 4228 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4229 if (hmeblkp != NULL) { 4230 /* 4231 * We've encountered a shadow hmeblk so skip the range 4232 * of the next smaller mapping size. 4233 */ 4234 if (hmeblkp->hblk_shw_bit) { 4235 ASSERT(sfmmup != ksfmmup); 4236 ASSERT(hashno > 1); 4237 addr = (caddr_t)P2END((uintptr_t)addr, 4238 TTEBYTES(hashno - 1)); 4239 } else { 4240 addr = sfmmu_hblk_chgattr(sfmmup, 4241 hmeblkp, addr, endaddr, &dmr, attr, mode); 4242 } 4243 SFMMU_HASH_UNLOCK(hmebp); 4244 hashno = 1; 4245 continue; 4246 } 4247 SFMMU_HASH_UNLOCK(hmebp); 4248 4249 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4250 /* 4251 * We have traversed the whole list and rehashed 4252 * if necessary without finding the address to chgattr. 4253 * This is ok, so we increment the address by the 4254 * smallest hmeblk range for kernel mappings or for 4255 * user mappings with no large pages, and the largest 4256 * hmeblk range, to account for shadow hmeblks, for 4257 * user mappings with large pages and continue. 4258 */ 4259 if (sfmmup == ksfmmup) 4260 addr = (caddr_t)P2END((uintptr_t)addr, 4261 TTEBYTES(1)); 4262 else 4263 addr = (caddr_t)P2END((uintptr_t)addr, 4264 TTEBYTES(hashno)); 4265 hashno = 1; 4266 } else { 4267 hashno++; 4268 } 4269 } 4270 4271 sfmmu_hblks_list_purge(&list); 4272 DEMAP_RANGE_FLUSH(&dmr); 4273 cpuset = sfmmup->sfmmu_cpusran; 4274 xt_sync(cpuset); 4275 } 4276 4277 /* 4278 * This function chgattr on a range of addresses in an hmeblk. It returns the 4279 * next addres that needs to be chgattr. 4280 * It should be called with the hash lock held. 4281 * XXX It should be possible to optimize chgattr by not flushing every time but 4282 * on the other hand: 4283 * 1. do one flush crosscall. 4284 * 2. only flush if we are increasing permissions (make sure this will work) 4285 */ 4286 static caddr_t 4287 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4288 caddr_t endaddr, demap_range_t *dmrp, uint_t attr, int mode) 4289 { 4290 tte_t tte, tteattr, tteflags, ttemod; 4291 struct sf_hment *sfhmep; 4292 int ttesz; 4293 struct page *pp = NULL; 4294 kmutex_t *pml, *pmtx; 4295 int ret; 4296 int use_demap_range; 4297 #if defined(SF_ERRATA_57) 4298 int check_exec; 4299 #endif 4300 4301 ASSERT(in_hblk_range(hmeblkp, addr)); 4302 ASSERT(hmeblkp->hblk_shw_bit == 0); 4303 4304 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4305 ttesz = get_hblk_ttesz(hmeblkp); 4306 4307 /* 4308 * Flush the current demap region if addresses have been 4309 * skipped or the page size doesn't match. 4310 */ 4311 use_demap_range = (TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp)); 4312 if (use_demap_range) { 4313 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4314 } else { 4315 DEMAP_RANGE_FLUSH(dmrp); 4316 } 4317 4318 tteattr.ll = sfmmu_vtop_attr(attr, mode, &tteflags); 4319 #if defined(SF_ERRATA_57) 4320 check_exec = (sfmmup != ksfmmup) && 4321 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4322 TTE_IS_EXECUTABLE(&tteattr); 4323 #endif 4324 HBLKTOHME(sfhmep, hmeblkp, addr); 4325 while (addr < endaddr) { 4326 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4327 if (TTE_IS_VALID(&tte)) { 4328 if ((tte.ll & tteflags.ll) == tteattr.ll) { 4329 /* 4330 * if the new attr is the same as old 4331 * continue 4332 */ 4333 goto next_addr; 4334 } 4335 if (!TTE_IS_WRITABLE(&tteattr)) { 4336 /* 4337 * make sure we clear hw modify bit if we 4338 * removing write protections 4339 */ 4340 tteflags.tte_intlo |= TTE_HWWR_INT; 4341 } 4342 4343 pml = NULL; 4344 pp = sfhmep->hme_page; 4345 if (pp) { 4346 pml = sfmmu_mlist_enter(pp); 4347 } 4348 4349 if (pp != sfhmep->hme_page) { 4350 /* 4351 * tte must have been unloaded. 4352 */ 4353 ASSERT(pml); 4354 sfmmu_mlist_exit(pml); 4355 continue; 4356 } 4357 4358 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4359 4360 ttemod = tte; 4361 ttemod.ll = (ttemod.ll & ~tteflags.ll) | tteattr.ll; 4362 ASSERT(TTE_TO_TTEPFN(&ttemod) == TTE_TO_TTEPFN(&tte)); 4363 4364 #if defined(SF_ERRATA_57) 4365 if (check_exec && addr < errata57_limit) 4366 ttemod.tte_exec_perm = 0; 4367 #endif 4368 ret = sfmmu_modifytte_try(&tte, &ttemod, 4369 &sfhmep->hme_tte); 4370 4371 if (ret < 0) { 4372 /* tte changed underneath us */ 4373 if (pml) { 4374 sfmmu_mlist_exit(pml); 4375 } 4376 continue; 4377 } 4378 4379 if (tteflags.tte_intlo & TTE_HWWR_INT) { 4380 /* 4381 * need to sync if we are clearing modify bit. 4382 */ 4383 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4384 } 4385 4386 if (pp && PP_ISRO(pp)) { 4387 if (tteattr.tte_intlo & TTE_WRPRM_INT) { 4388 pmtx = sfmmu_page_enter(pp); 4389 PP_CLRRO(pp); 4390 sfmmu_page_exit(pmtx); 4391 } 4392 } 4393 4394 if (ret > 0 && use_demap_range) { 4395 DEMAP_RANGE_MARKPG(dmrp, addr); 4396 } else if (ret > 0) { 4397 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4398 } 4399 4400 if (pml) { 4401 sfmmu_mlist_exit(pml); 4402 } 4403 } 4404 next_addr: 4405 addr += TTEBYTES(ttesz); 4406 sfhmep++; 4407 DEMAP_RANGE_NEXTPG(dmrp); 4408 } 4409 return (addr); 4410 } 4411 4412 /* 4413 * This routine converts virtual attributes to physical ones. It will 4414 * update the tteflags field with the tte mask corresponding to the attributes 4415 * affected and it returns the new attributes. It will also clear the modify 4416 * bit if we are taking away write permission. This is necessary since the 4417 * modify bit is the hardware permission bit and we need to clear it in order 4418 * to detect write faults. 4419 */ 4420 static uint64_t 4421 sfmmu_vtop_attr(uint_t attr, int mode, tte_t *ttemaskp) 4422 { 4423 tte_t ttevalue; 4424 4425 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 4426 4427 switch (mode) { 4428 case SFMMU_CHGATTR: 4429 /* all attributes specified */ 4430 ttevalue.tte_inthi = MAKE_TTEATTR_INTHI(attr); 4431 ttevalue.tte_intlo = MAKE_TTEATTR_INTLO(attr); 4432 ttemaskp->tte_inthi = TTEINTHI_ATTR; 4433 ttemaskp->tte_intlo = TTEINTLO_ATTR; 4434 break; 4435 case SFMMU_SETATTR: 4436 ASSERT(!(attr & ~HAT_PROT_MASK)); 4437 ttemaskp->ll = 0; 4438 ttevalue.ll = 0; 4439 /* 4440 * a valid tte implies exec and read for sfmmu 4441 * so no need to do anything about them. 4442 * since priviledged access implies user access 4443 * PROT_USER doesn't make sense either. 4444 */ 4445 if (attr & PROT_WRITE) { 4446 ttemaskp->tte_intlo |= TTE_WRPRM_INT; 4447 ttevalue.tte_intlo |= TTE_WRPRM_INT; 4448 } 4449 break; 4450 case SFMMU_CLRATTR: 4451 /* attributes will be nand with current ones */ 4452 if (attr & ~(PROT_WRITE | PROT_USER)) { 4453 panic("sfmmu: attr %x not supported", attr); 4454 } 4455 ttemaskp->ll = 0; 4456 ttevalue.ll = 0; 4457 if (attr & PROT_WRITE) { 4458 /* clear both writable and modify bit */ 4459 ttemaskp->tte_intlo |= TTE_WRPRM_INT | TTE_HWWR_INT; 4460 } 4461 if (attr & PROT_USER) { 4462 ttemaskp->tte_intlo |= TTE_PRIV_INT; 4463 ttevalue.tte_intlo |= TTE_PRIV_INT; 4464 } 4465 break; 4466 default: 4467 panic("sfmmu_vtop_attr: bad mode %x", mode); 4468 } 4469 ASSERT(TTE_TO_TTEPFN(&ttevalue) == 0); 4470 return (ttevalue.ll); 4471 } 4472 4473 static uint_t 4474 sfmmu_ptov_attr(tte_t *ttep) 4475 { 4476 uint_t attr; 4477 4478 ASSERT(TTE_IS_VALID(ttep)); 4479 4480 attr = PROT_READ; 4481 4482 if (TTE_IS_WRITABLE(ttep)) { 4483 attr |= PROT_WRITE; 4484 } 4485 if (TTE_IS_EXECUTABLE(ttep)) { 4486 attr |= PROT_EXEC; 4487 } 4488 if (!TTE_IS_PRIVILEGED(ttep)) { 4489 attr |= PROT_USER; 4490 } 4491 if (TTE_IS_NFO(ttep)) { 4492 attr |= HAT_NOFAULT; 4493 } 4494 if (TTE_IS_NOSYNC(ttep)) { 4495 attr |= HAT_NOSYNC; 4496 } 4497 if (TTE_IS_SIDEFFECT(ttep)) { 4498 attr |= SFMMU_SIDEFFECT; 4499 } 4500 if (!TTE_IS_VCACHEABLE(ttep)) { 4501 attr |= SFMMU_UNCACHEVTTE; 4502 } 4503 if (!TTE_IS_PCACHEABLE(ttep)) { 4504 attr |= SFMMU_UNCACHEPTTE; 4505 } 4506 return (attr); 4507 } 4508 4509 /* 4510 * hat_chgprot is a deprecated hat call. New segment drivers 4511 * should store all attributes and use hat_*attr calls. 4512 * 4513 * Change the protections in the virtual address range 4514 * given to the specified virtual protection. If vprot is ~PROT_WRITE, 4515 * then remove write permission, leaving the other 4516 * permissions unchanged. If vprot is ~PROT_USER, remove user permissions. 4517 * 4518 */ 4519 void 4520 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot) 4521 { 4522 struct hmehash_bucket *hmebp; 4523 hmeblk_tag hblktag; 4524 int hmeshift, hashno = 1; 4525 struct hme_blk *hmeblkp, *list = NULL; 4526 caddr_t endaddr; 4527 cpuset_t cpuset; 4528 demap_range_t dmr; 4529 4530 ASSERT((len & MMU_PAGEOFFSET) == 0); 4531 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4532 4533 if (sfmmup->sfmmu_xhat_provider) { 4534 XHAT_CHGPROT(sfmmup, addr, len, vprot); 4535 return; 4536 } else { 4537 /* 4538 * This must be a CPU HAT. If the address space has 4539 * XHATs attached, change attributes for all of them, 4540 * just in case 4541 */ 4542 ASSERT(sfmmup->sfmmu_as != NULL); 4543 if (sfmmup->sfmmu_as->a_xhat != NULL) 4544 xhat_chgprot_all(sfmmup->sfmmu_as, addr, len, vprot); 4545 } 4546 4547 CPUSET_ZERO(cpuset); 4548 4549 if ((vprot != (uint_t)~PROT_WRITE) && (vprot & PROT_USER) && 4550 ((addr + len) > (caddr_t)USERLIMIT)) { 4551 panic("user addr %p vprot %x in kernel space", 4552 (void *)addr, vprot); 4553 } 4554 endaddr = addr + len; 4555 hblktag.htag_id = sfmmup; 4556 DEMAP_RANGE_INIT(sfmmup, &dmr); 4557 4558 while (addr < endaddr) { 4559 hmeshift = HME_HASH_SHIFT(hashno); 4560 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4561 hblktag.htag_rehash = hashno; 4562 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4563 4564 SFMMU_HASH_LOCK(hmebp); 4565 4566 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4567 if (hmeblkp != NULL) { 4568 /* 4569 * We've encountered a shadow hmeblk so skip the range 4570 * of the next smaller mapping size. 4571 */ 4572 if (hmeblkp->hblk_shw_bit) { 4573 ASSERT(sfmmup != ksfmmup); 4574 ASSERT(hashno > 1); 4575 addr = (caddr_t)P2END((uintptr_t)addr, 4576 TTEBYTES(hashno - 1)); 4577 } else { 4578 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp, 4579 addr, endaddr, &dmr, vprot); 4580 } 4581 SFMMU_HASH_UNLOCK(hmebp); 4582 hashno = 1; 4583 continue; 4584 } 4585 SFMMU_HASH_UNLOCK(hmebp); 4586 4587 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4588 /* 4589 * We have traversed the whole list and rehashed 4590 * if necessary without finding the address to chgprot. 4591 * This is ok so we increment the address by the 4592 * smallest hmeblk range for kernel mappings and the 4593 * largest hmeblk range, to account for shadow hmeblks, 4594 * for user mappings and continue. 4595 */ 4596 if (sfmmup == ksfmmup) 4597 addr = (caddr_t)P2END((uintptr_t)addr, 4598 TTEBYTES(1)); 4599 else 4600 addr = (caddr_t)P2END((uintptr_t)addr, 4601 TTEBYTES(hashno)); 4602 hashno = 1; 4603 } else { 4604 hashno++; 4605 } 4606 } 4607 4608 sfmmu_hblks_list_purge(&list); 4609 DEMAP_RANGE_FLUSH(&dmr); 4610 cpuset = sfmmup->sfmmu_cpusran; 4611 xt_sync(cpuset); 4612 } 4613 4614 /* 4615 * This function chgprots a range of addresses in an hmeblk. It returns the 4616 * next addres that needs to be chgprot. 4617 * It should be called with the hash lock held. 4618 * XXX It shold be possible to optimize chgprot by not flushing every time but 4619 * on the other hand: 4620 * 1. do one flush crosscall. 4621 * 2. only flush if we are increasing permissions (make sure this will work) 4622 */ 4623 static caddr_t 4624 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4625 caddr_t endaddr, demap_range_t *dmrp, uint_t vprot) 4626 { 4627 uint_t pprot; 4628 tte_t tte, ttemod; 4629 struct sf_hment *sfhmep; 4630 uint_t tteflags; 4631 int ttesz; 4632 struct page *pp = NULL; 4633 kmutex_t *pml, *pmtx; 4634 int ret; 4635 int use_demap_range; 4636 #if defined(SF_ERRATA_57) 4637 int check_exec; 4638 #endif 4639 4640 ASSERT(in_hblk_range(hmeblkp, addr)); 4641 ASSERT(hmeblkp->hblk_shw_bit == 0); 4642 4643 #ifdef DEBUG 4644 if (get_hblk_ttesz(hmeblkp) != TTE8K && 4645 (endaddr < get_hblk_endaddr(hmeblkp))) { 4646 panic("sfmmu_hblk_chgprot: partial chgprot of large page"); 4647 } 4648 #endif /* DEBUG */ 4649 4650 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4651 ttesz = get_hblk_ttesz(hmeblkp); 4652 4653 pprot = sfmmu_vtop_prot(vprot, &tteflags); 4654 #if defined(SF_ERRATA_57) 4655 check_exec = (sfmmup != ksfmmup) && 4656 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4657 ((vprot & PROT_EXEC) == PROT_EXEC); 4658 #endif 4659 HBLKTOHME(sfhmep, hmeblkp, addr); 4660 4661 /* 4662 * Flush the current demap region if addresses have been 4663 * skipped or the page size doesn't match. 4664 */ 4665 use_demap_range = (TTEBYTES(ttesz) == MMU_PAGESIZE); 4666 if (use_demap_range) { 4667 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4668 } else { 4669 DEMAP_RANGE_FLUSH(dmrp); 4670 } 4671 4672 while (addr < endaddr) { 4673 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4674 if (TTE_IS_VALID(&tte)) { 4675 if (TTE_GET_LOFLAGS(&tte, tteflags) == pprot) { 4676 /* 4677 * if the new protection is the same as old 4678 * continue 4679 */ 4680 goto next_addr; 4681 } 4682 pml = NULL; 4683 pp = sfhmep->hme_page; 4684 if (pp) { 4685 pml = sfmmu_mlist_enter(pp); 4686 } 4687 if (pp != sfhmep->hme_page) { 4688 /* 4689 * tte most have been unloaded 4690 * underneath us. Recheck 4691 */ 4692 ASSERT(pml); 4693 sfmmu_mlist_exit(pml); 4694 continue; 4695 } 4696 4697 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4698 4699 ttemod = tte; 4700 TTE_SET_LOFLAGS(&ttemod, tteflags, pprot); 4701 #if defined(SF_ERRATA_57) 4702 if (check_exec && addr < errata57_limit) 4703 ttemod.tte_exec_perm = 0; 4704 #endif 4705 ret = sfmmu_modifytte_try(&tte, &ttemod, 4706 &sfhmep->hme_tte); 4707 4708 if (ret < 0) { 4709 /* tte changed underneath us */ 4710 if (pml) { 4711 sfmmu_mlist_exit(pml); 4712 } 4713 continue; 4714 } 4715 4716 if (tteflags & TTE_HWWR_INT) { 4717 /* 4718 * need to sync if we are clearing modify bit. 4719 */ 4720 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4721 } 4722 4723 if (pp && PP_ISRO(pp)) { 4724 if (pprot & TTE_WRPRM_INT) { 4725 pmtx = sfmmu_page_enter(pp); 4726 PP_CLRRO(pp); 4727 sfmmu_page_exit(pmtx); 4728 } 4729 } 4730 4731 if (ret > 0 && use_demap_range) { 4732 DEMAP_RANGE_MARKPG(dmrp, addr); 4733 } else if (ret > 0) { 4734 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4735 } 4736 4737 if (pml) { 4738 sfmmu_mlist_exit(pml); 4739 } 4740 } 4741 next_addr: 4742 addr += TTEBYTES(ttesz); 4743 sfhmep++; 4744 DEMAP_RANGE_NEXTPG(dmrp); 4745 } 4746 return (addr); 4747 } 4748 4749 /* 4750 * This routine is deprecated and should only be used by hat_chgprot. 4751 * The correct routine is sfmmu_vtop_attr. 4752 * This routine converts virtual page protections to physical ones. It will 4753 * update the tteflags field with the tte mask corresponding to the protections 4754 * affected and it returns the new protections. It will also clear the modify 4755 * bit if we are taking away write permission. This is necessary since the 4756 * modify bit is the hardware permission bit and we need to clear it in order 4757 * to detect write faults. 4758 * It accepts the following special protections: 4759 * ~PROT_WRITE = remove write permissions. 4760 * ~PROT_USER = remove user permissions. 4761 */ 4762 static uint_t 4763 sfmmu_vtop_prot(uint_t vprot, uint_t *tteflagsp) 4764 { 4765 if (vprot == (uint_t)~PROT_WRITE) { 4766 *tteflagsp = TTE_WRPRM_INT | TTE_HWWR_INT; 4767 return (0); /* will cause wrprm to be cleared */ 4768 } 4769 if (vprot == (uint_t)~PROT_USER) { 4770 *tteflagsp = TTE_PRIV_INT; 4771 return (0); /* will cause privprm to be cleared */ 4772 } 4773 if ((vprot == 0) || (vprot == PROT_USER) || 4774 ((vprot & PROT_ALL) != vprot)) { 4775 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4776 } 4777 4778 switch (vprot) { 4779 case (PROT_READ): 4780 case (PROT_EXEC): 4781 case (PROT_EXEC | PROT_READ): 4782 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4783 return (TTE_PRIV_INT); /* set prv and clr wrt */ 4784 case (PROT_WRITE): 4785 case (PROT_WRITE | PROT_READ): 4786 case (PROT_EXEC | PROT_WRITE): 4787 case (PROT_EXEC | PROT_WRITE | PROT_READ): 4788 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4789 return (TTE_PRIV_INT | TTE_WRPRM_INT); /* set prv and wrt */ 4790 case (PROT_USER | PROT_READ): 4791 case (PROT_USER | PROT_EXEC): 4792 case (PROT_USER | PROT_EXEC | PROT_READ): 4793 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4794 return (0); /* clr prv and wrt */ 4795 case (PROT_USER | PROT_WRITE): 4796 case (PROT_USER | PROT_WRITE | PROT_READ): 4797 case (PROT_USER | PROT_EXEC | PROT_WRITE): 4798 case (PROT_USER | PROT_EXEC | PROT_WRITE | PROT_READ): 4799 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4800 return (TTE_WRPRM_INT); /* clr prv and set wrt */ 4801 default: 4802 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4803 } 4804 return (0); 4805 } 4806 4807 /* 4808 * Alternate unload for very large virtual ranges. With a true 64 bit VA, 4809 * the normal algorithm would take too long for a very large VA range with 4810 * few real mappings. This routine just walks thru all HMEs in the global 4811 * hash table to find and remove mappings. 4812 */ 4813 static void 4814 hat_unload_large_virtual( 4815 struct hat *sfmmup, 4816 caddr_t startaddr, 4817 size_t len, 4818 uint_t flags, 4819 hat_callback_t *callback) 4820 { 4821 struct hmehash_bucket *hmebp; 4822 struct hme_blk *hmeblkp; 4823 struct hme_blk *pr_hblk = NULL; 4824 struct hme_blk *nx_hblk; 4825 struct hme_blk *list = NULL; 4826 int i; 4827 uint64_t hblkpa, prevpa, nx_pa; 4828 demap_range_t dmr, *dmrp; 4829 cpuset_t cpuset; 4830 caddr_t endaddr = startaddr + len; 4831 caddr_t sa; 4832 caddr_t ea; 4833 caddr_t cb_sa[MAX_CB_ADDR]; 4834 caddr_t cb_ea[MAX_CB_ADDR]; 4835 int addr_cnt = 0; 4836 int a = 0; 4837 4838 if (sfmmup->sfmmu_free) { 4839 dmrp = NULL; 4840 } else { 4841 dmrp = &dmr; 4842 DEMAP_RANGE_INIT(sfmmup, dmrp); 4843 } 4844 4845 /* 4846 * Loop through all the hash buckets of HME blocks looking for matches. 4847 */ 4848 for (i = 0; i <= UHMEHASH_SZ; i++) { 4849 hmebp = &uhme_hash[i]; 4850 SFMMU_HASH_LOCK(hmebp); 4851 hmeblkp = hmebp->hmeblkp; 4852 hblkpa = hmebp->hmeh_nextpa; 4853 prevpa = 0; 4854 pr_hblk = NULL; 4855 while (hmeblkp) { 4856 nx_hblk = hmeblkp->hblk_next; 4857 nx_pa = hmeblkp->hblk_nextpa; 4858 4859 /* 4860 * skip if not this context, if a shadow block or 4861 * if the mapping is not in the requested range 4862 */ 4863 if (hmeblkp->hblk_tag.htag_id != sfmmup || 4864 hmeblkp->hblk_shw_bit || 4865 (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr || 4866 (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) { 4867 pr_hblk = hmeblkp; 4868 prevpa = hblkpa; 4869 goto next_block; 4870 } 4871 4872 /* 4873 * unload if there are any current valid mappings 4874 */ 4875 if (hmeblkp->hblk_vcnt != 0 || 4876 hmeblkp->hblk_hmecnt != 0) 4877 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 4878 sa, ea, dmrp, flags); 4879 4880 /* 4881 * on unmap we also release the HME block itself, once 4882 * all mappings are gone. 4883 */ 4884 if ((flags & HAT_UNLOAD_UNMAP) != 0 && 4885 !hmeblkp->hblk_vcnt && 4886 !hmeblkp->hblk_hmecnt) { 4887 ASSERT(!hmeblkp->hblk_lckcnt); 4888 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 4889 prevpa, pr_hblk); 4890 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 4891 } else { 4892 pr_hblk = hmeblkp; 4893 prevpa = hblkpa; 4894 } 4895 4896 if (callback == NULL) 4897 goto next_block; 4898 4899 /* 4900 * HME blocks may span more than one page, but we may be 4901 * unmapping only one page, so check for a smaller range 4902 * for the callback 4903 */ 4904 if (sa < startaddr) 4905 sa = startaddr; 4906 if (--ea > endaddr) 4907 ea = endaddr - 1; 4908 4909 cb_sa[addr_cnt] = sa; 4910 cb_ea[addr_cnt] = ea; 4911 if (++addr_cnt == MAX_CB_ADDR) { 4912 if (dmrp != NULL) { 4913 DEMAP_RANGE_FLUSH(dmrp); 4914 cpuset = sfmmup->sfmmu_cpusran; 4915 xt_sync(cpuset); 4916 } 4917 4918 for (a = 0; a < MAX_CB_ADDR; ++a) { 4919 callback->hcb_start_addr = cb_sa[a]; 4920 callback->hcb_end_addr = cb_ea[a]; 4921 callback->hcb_function(callback); 4922 } 4923 addr_cnt = 0; 4924 } 4925 4926 next_block: 4927 hmeblkp = nx_hblk; 4928 hblkpa = nx_pa; 4929 } 4930 SFMMU_HASH_UNLOCK(hmebp); 4931 } 4932 4933 sfmmu_hblks_list_purge(&list); 4934 if (dmrp != NULL) { 4935 DEMAP_RANGE_FLUSH(dmrp); 4936 cpuset = sfmmup->sfmmu_cpusran; 4937 xt_sync(cpuset); 4938 } 4939 4940 for (a = 0; a < addr_cnt; ++a) { 4941 callback->hcb_start_addr = cb_sa[a]; 4942 callback->hcb_end_addr = cb_ea[a]; 4943 callback->hcb_function(callback); 4944 } 4945 4946 /* 4947 * Check TSB and TLB page sizes if the process isn't exiting. 4948 */ 4949 if (!sfmmup->sfmmu_free) 4950 sfmmu_check_page_sizes(sfmmup, 0); 4951 } 4952 4953 /* 4954 * Unload all the mappings in the range [addr..addr+len). addr and len must 4955 * be MMU_PAGESIZE aligned. 4956 */ 4957 4958 extern struct seg *segkmap; 4959 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \ 4960 segkmap->s_base <= (addr) && (addr) < (segkmap->s_base + segkmap->s_size)) 4961 4962 4963 void 4964 hat_unload_callback( 4965 struct hat *sfmmup, 4966 caddr_t addr, 4967 size_t len, 4968 uint_t flags, 4969 hat_callback_t *callback) 4970 { 4971 struct hmehash_bucket *hmebp; 4972 hmeblk_tag hblktag; 4973 int hmeshift, hashno, iskernel; 4974 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 4975 caddr_t endaddr; 4976 cpuset_t cpuset; 4977 uint64_t hblkpa, prevpa; 4978 int addr_count = 0; 4979 int a; 4980 caddr_t cb_start_addr[MAX_CB_ADDR]; 4981 caddr_t cb_end_addr[MAX_CB_ADDR]; 4982 int issegkmap = ISSEGKMAP(sfmmup, addr); 4983 demap_range_t dmr, *dmrp; 4984 4985 if (sfmmup->sfmmu_xhat_provider) { 4986 XHAT_UNLOAD_CALLBACK(sfmmup, addr, len, flags, callback); 4987 return; 4988 } else { 4989 /* 4990 * This must be a CPU HAT. If the address space has 4991 * XHATs attached, unload the mappings for all of them, 4992 * just in case 4993 */ 4994 ASSERT(sfmmup->sfmmu_as != NULL); 4995 if (sfmmup->sfmmu_as->a_xhat != NULL) 4996 xhat_unload_callback_all(sfmmup->sfmmu_as, addr, 4997 len, flags, callback); 4998 } 4999 5000 ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \ 5001 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 5002 5003 ASSERT(sfmmup != NULL); 5004 ASSERT((len & MMU_PAGEOFFSET) == 0); 5005 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 5006 5007 /* 5008 * Probing through a large VA range (say 63 bits) will be slow, even 5009 * at 4 Meg steps between the probes. So, when the virtual address range 5010 * is very large, search the HME entries for what to unload. 5011 * 5012 * len >> TTE_PAGE_SHIFT(TTE4M) is the # of 4Meg probes we'd need 5013 * 5014 * UHMEHASH_SZ is number of hash buckets to examine 5015 * 5016 */ 5017 if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) { 5018 hat_unload_large_virtual(sfmmup, addr, len, flags, callback); 5019 return; 5020 } 5021 5022 CPUSET_ZERO(cpuset); 5023 5024 /* 5025 * If the process is exiting, we can save a lot of fuss since 5026 * we'll flush the TLB when we free the ctx anyway. 5027 */ 5028 if (sfmmup->sfmmu_free) 5029 dmrp = NULL; 5030 else 5031 dmrp = &dmr; 5032 5033 DEMAP_RANGE_INIT(sfmmup, dmrp); 5034 endaddr = addr + len; 5035 hblktag.htag_id = sfmmup; 5036 5037 /* 5038 * It is likely for the vm to call unload over a wide range of 5039 * addresses that are actually very sparsely populated by 5040 * translations. In order to speed this up the sfmmu hat supports 5041 * the concept of shadow hmeblks. Dummy large page hmeblks that 5042 * correspond to actual small translations are allocated at tteload 5043 * time and are referred to as shadow hmeblks. Now, during unload 5044 * time, we first check if we have a shadow hmeblk for that 5045 * translation. The absence of one means the corresponding address 5046 * range is empty and can be skipped. 5047 * 5048 * The kernel is an exception to above statement and that is why 5049 * we don't use shadow hmeblks and hash starting from the smallest 5050 * page size. 5051 */ 5052 if (sfmmup == KHATID) { 5053 iskernel = 1; 5054 hashno = TTE64K; 5055 } else { 5056 iskernel = 0; 5057 if (mmu_page_sizes == max_mmu_page_sizes) { 5058 hashno = TTE256M; 5059 } else { 5060 hashno = TTE4M; 5061 } 5062 } 5063 while (addr < endaddr) { 5064 hmeshift = HME_HASH_SHIFT(hashno); 5065 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5066 hblktag.htag_rehash = hashno; 5067 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5068 5069 SFMMU_HASH_LOCK(hmebp); 5070 5071 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, pr_hblk, 5072 prevpa, &list); 5073 if (hmeblkp == NULL) { 5074 /* 5075 * didn't find an hmeblk. skip the appropiate 5076 * address range. 5077 */ 5078 SFMMU_HASH_UNLOCK(hmebp); 5079 if (iskernel) { 5080 if (hashno < mmu_hashcnt) { 5081 hashno++; 5082 continue; 5083 } else { 5084 hashno = TTE64K; 5085 addr = (caddr_t)roundup((uintptr_t)addr 5086 + 1, MMU_PAGESIZE64K); 5087 continue; 5088 } 5089 } 5090 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5091 (1 << hmeshift)); 5092 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5093 ASSERT(hashno == TTE64K); 5094 continue; 5095 } 5096 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5097 hashno = TTE512K; 5098 continue; 5099 } 5100 if (mmu_page_sizes == max_mmu_page_sizes) { 5101 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5102 hashno = TTE4M; 5103 continue; 5104 } 5105 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5106 hashno = TTE32M; 5107 continue; 5108 } 5109 hashno = TTE256M; 5110 continue; 5111 } else { 5112 hashno = TTE4M; 5113 continue; 5114 } 5115 } 5116 ASSERT(hmeblkp); 5117 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5118 /* 5119 * If the valid count is zero we can skip the range 5120 * mapped by this hmeblk. 5121 * We free hblks in the case of HAT_UNMAP. HAT_UNMAP 5122 * is used by segment drivers as a hint 5123 * that the mapping resource won't be used any longer. 5124 * The best example of this is during exit(). 5125 */ 5126 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5127 get_hblk_span(hmeblkp)); 5128 if ((flags & HAT_UNLOAD_UNMAP) || 5129 (iskernel && !issegkmap)) { 5130 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5131 pr_hblk); 5132 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5133 } 5134 SFMMU_HASH_UNLOCK(hmebp); 5135 5136 if (iskernel) { 5137 hashno = TTE64K; 5138 continue; 5139 } 5140 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5141 ASSERT(hashno == TTE64K); 5142 continue; 5143 } 5144 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5145 hashno = TTE512K; 5146 continue; 5147 } 5148 if (mmu_page_sizes == max_mmu_page_sizes) { 5149 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5150 hashno = TTE4M; 5151 continue; 5152 } 5153 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5154 hashno = TTE32M; 5155 continue; 5156 } 5157 hashno = TTE256M; 5158 continue; 5159 } else { 5160 hashno = TTE4M; 5161 continue; 5162 } 5163 } 5164 if (hmeblkp->hblk_shw_bit) { 5165 /* 5166 * If we encounter a shadow hmeblk we know there is 5167 * smaller sized hmeblks mapping the same address space. 5168 * Decrement the hash size and rehash. 5169 */ 5170 ASSERT(sfmmup != KHATID); 5171 hashno--; 5172 SFMMU_HASH_UNLOCK(hmebp); 5173 continue; 5174 } 5175 5176 /* 5177 * track callback address ranges. 5178 * only start a new range when it's not contiguous 5179 */ 5180 if (callback != NULL) { 5181 if (addr_count > 0 && 5182 addr == cb_end_addr[addr_count - 1]) 5183 --addr_count; 5184 else 5185 cb_start_addr[addr_count] = addr; 5186 } 5187 5188 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr, 5189 dmrp, flags); 5190 5191 if (callback != NULL) 5192 cb_end_addr[addr_count++] = addr; 5193 5194 if (((flags & HAT_UNLOAD_UNMAP) || (iskernel && !issegkmap)) && 5195 !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5196 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5197 pr_hblk); 5198 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5199 } 5200 SFMMU_HASH_UNLOCK(hmebp); 5201 5202 /* 5203 * Notify our caller as to exactly which pages 5204 * have been unloaded. We do these in clumps, 5205 * to minimize the number of xt_sync()s that need to occur. 5206 */ 5207 if (callback != NULL && addr_count == MAX_CB_ADDR) { 5208 DEMAP_RANGE_FLUSH(dmrp); 5209 if (dmrp != NULL) { 5210 cpuset = sfmmup->sfmmu_cpusran; 5211 xt_sync(cpuset); 5212 } 5213 5214 for (a = 0; a < MAX_CB_ADDR; ++a) { 5215 callback->hcb_start_addr = cb_start_addr[a]; 5216 callback->hcb_end_addr = cb_end_addr[a]; 5217 callback->hcb_function(callback); 5218 } 5219 addr_count = 0; 5220 } 5221 if (iskernel) { 5222 hashno = TTE64K; 5223 continue; 5224 } 5225 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5226 ASSERT(hashno == TTE64K); 5227 continue; 5228 } 5229 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5230 hashno = TTE512K; 5231 continue; 5232 } 5233 if (mmu_page_sizes == max_mmu_page_sizes) { 5234 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5235 hashno = TTE4M; 5236 continue; 5237 } 5238 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5239 hashno = TTE32M; 5240 continue; 5241 } 5242 hashno = TTE256M; 5243 } else { 5244 hashno = TTE4M; 5245 } 5246 } 5247 5248 sfmmu_hblks_list_purge(&list); 5249 DEMAP_RANGE_FLUSH(dmrp); 5250 if (dmrp != NULL) { 5251 cpuset = sfmmup->sfmmu_cpusran; 5252 xt_sync(cpuset); 5253 } 5254 if (callback && addr_count != 0) { 5255 for (a = 0; a < addr_count; ++a) { 5256 callback->hcb_start_addr = cb_start_addr[a]; 5257 callback->hcb_end_addr = cb_end_addr[a]; 5258 callback->hcb_function(callback); 5259 } 5260 } 5261 5262 /* 5263 * Check TSB and TLB page sizes if the process isn't exiting. 5264 */ 5265 if (!sfmmup->sfmmu_free) 5266 sfmmu_check_page_sizes(sfmmup, 0); 5267 } 5268 5269 /* 5270 * Unload all the mappings in the range [addr..addr+len). addr and len must 5271 * be MMU_PAGESIZE aligned. 5272 */ 5273 void 5274 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags) 5275 { 5276 if (sfmmup->sfmmu_xhat_provider) { 5277 XHAT_UNLOAD(sfmmup, addr, len, flags); 5278 return; 5279 } 5280 hat_unload_callback(sfmmup, addr, len, flags, NULL); 5281 } 5282 5283 5284 /* 5285 * Find the largest mapping size for this page. 5286 */ 5287 int 5288 fnd_mapping_sz(page_t *pp) 5289 { 5290 int sz; 5291 int p_index; 5292 5293 p_index = PP_MAPINDEX(pp); 5294 5295 sz = 0; 5296 p_index >>= 1; /* don't care about 8K bit */ 5297 for (; p_index; p_index >>= 1) { 5298 sz++; 5299 } 5300 5301 return (sz); 5302 } 5303 5304 /* 5305 * This function unloads a range of addresses for an hmeblk. 5306 * It returns the next address to be unloaded. 5307 * It should be called with the hash lock held. 5308 */ 5309 static caddr_t 5310 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5311 caddr_t endaddr, demap_range_t *dmrp, uint_t flags) 5312 { 5313 tte_t tte, ttemod; 5314 struct sf_hment *sfhmep; 5315 int ttesz; 5316 long ttecnt; 5317 page_t *pp; 5318 kmutex_t *pml; 5319 int ret; 5320 int use_demap_range; 5321 5322 ASSERT(in_hblk_range(hmeblkp, addr)); 5323 ASSERT(!hmeblkp->hblk_shw_bit); 5324 #ifdef DEBUG 5325 if (get_hblk_ttesz(hmeblkp) != TTE8K && 5326 (endaddr < get_hblk_endaddr(hmeblkp))) { 5327 panic("sfmmu_hblk_unload: partial unload of large page"); 5328 } 5329 #endif /* DEBUG */ 5330 5331 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5332 ttesz = get_hblk_ttesz(hmeblkp); 5333 5334 use_demap_range = (do_virtual_coloring && 5335 ((dmrp == NULL) || TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp))); 5336 if (use_demap_range) { 5337 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 5338 } else { 5339 DEMAP_RANGE_FLUSH(dmrp); 5340 } 5341 ttecnt = 0; 5342 HBLKTOHME(sfhmep, hmeblkp, addr); 5343 5344 while (addr < endaddr) { 5345 pml = NULL; 5346 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5347 if (TTE_IS_VALID(&tte)) { 5348 pp = sfhmep->hme_page; 5349 if (pp != NULL) { 5350 pml = sfmmu_mlist_enter(pp); 5351 } 5352 5353 /* 5354 * Verify if hme still points to 'pp' now that 5355 * we have p_mapping lock. 5356 */ 5357 if (sfhmep->hme_page != pp) { 5358 if (pp != NULL && sfhmep->hme_page != NULL) { 5359 ASSERT(pml != NULL); 5360 sfmmu_mlist_exit(pml); 5361 /* Re-start this iteration. */ 5362 continue; 5363 } 5364 ASSERT((pp != NULL) && 5365 (sfhmep->hme_page == NULL)); 5366 goto tte_unloaded; 5367 } 5368 5369 /* 5370 * This point on we have both HASH and p_mapping 5371 * lock. 5372 */ 5373 ASSERT(pp == sfhmep->hme_page); 5374 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5375 5376 /* 5377 * We need to loop on modify tte because it is 5378 * possible for pagesync to come along and 5379 * change the software bits beneath us. 5380 * 5381 * Page_unload can also invalidate the tte after 5382 * we read tte outside of p_mapping lock. 5383 */ 5384 again: 5385 ttemod = tte; 5386 5387 TTE_SET_INVALID(&ttemod); 5388 ret = sfmmu_modifytte_try(&tte, &ttemod, 5389 &sfhmep->hme_tte); 5390 5391 if (ret <= 0) { 5392 if (TTE_IS_VALID(&tte)) { 5393 ASSERT(ret < 0); 5394 goto again; 5395 } 5396 if (pp != NULL) { 5397 panic("sfmmu_hblk_unload: pp = 0x%p " 5398 "tte became invalid under mlist" 5399 " lock = 0x%p", pp, pml); 5400 } 5401 continue; 5402 } 5403 5404 if (!(flags & HAT_UNLOAD_NOSYNC)) { 5405 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5406 } 5407 5408 /* 5409 * Ok- we invalidated the tte. Do the rest of the job. 5410 */ 5411 ttecnt++; 5412 5413 if (flags & HAT_UNLOAD_UNLOCK) { 5414 ASSERT(hmeblkp->hblk_lckcnt > 0); 5415 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 5416 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 5417 } 5418 5419 /* 5420 * Normally we would need to flush the page 5421 * from the virtual cache at this point in 5422 * order to prevent a potential cache alias 5423 * inconsistency. 5424 * The particular scenario we need to worry 5425 * about is: 5426 * Given: va1 and va2 are two virtual address 5427 * that alias and map the same physical 5428 * address. 5429 * 1. mapping exists from va1 to pa and data 5430 * has been read into the cache. 5431 * 2. unload va1. 5432 * 3. load va2 and modify data using va2. 5433 * 4 unload va2. 5434 * 5. load va1 and reference data. Unless we 5435 * flush the data cache when we unload we will 5436 * get stale data. 5437 * Fortunately, page coloring eliminates the 5438 * above scenario by remembering the color a 5439 * physical page was last or is currently 5440 * mapped to. Now, we delay the flush until 5441 * the loading of translations. Only when the 5442 * new translation is of a different color 5443 * are we forced to flush. 5444 */ 5445 if (use_demap_range) { 5446 /* 5447 * Mark this page as needing a demap. 5448 */ 5449 DEMAP_RANGE_MARKPG(dmrp, addr); 5450 } else { 5451 if (do_virtual_coloring) { 5452 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 5453 sfmmup->sfmmu_free, 0); 5454 } else { 5455 pfn_t pfnum; 5456 5457 pfnum = TTE_TO_PFN(addr, &tte); 5458 sfmmu_tlbcache_demap(addr, sfmmup, 5459 hmeblkp, pfnum, sfmmup->sfmmu_free, 5460 FLUSH_NECESSARY_CPUS, 5461 CACHE_FLUSH, 0); 5462 } 5463 } 5464 5465 if (pp) { 5466 /* 5467 * Remove the hment from the mapping list 5468 */ 5469 ASSERT(hmeblkp->hblk_hmecnt > 0); 5470 5471 /* 5472 * Again, we cannot 5473 * ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS); 5474 */ 5475 HME_SUB(sfhmep, pp); 5476 membar_stst(); 5477 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 5478 } 5479 5480 ASSERT(hmeblkp->hblk_vcnt > 0); 5481 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 5482 5483 ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 5484 !hmeblkp->hblk_lckcnt); 5485 5486 #ifdef VAC 5487 if (pp && (pp->p_nrm & (P_KPMC | P_KPMS | P_TNC))) { 5488 if (PP_ISTNC(pp)) { 5489 /* 5490 * If page was temporary 5491 * uncached, try to recache 5492 * it. Note that HME_SUB() was 5493 * called above so p_index and 5494 * mlist had been updated. 5495 */ 5496 conv_tnc(pp, ttesz); 5497 } else if (pp->p_mapping == NULL) { 5498 ASSERT(kpm_enable); 5499 /* 5500 * Page is marked to be in VAC conflict 5501 * to an existing kpm mapping and/or is 5502 * kpm mapped using only the regular 5503 * pagesize. 5504 */ 5505 sfmmu_kpm_hme_unload(pp); 5506 } 5507 } 5508 #endif /* VAC */ 5509 } else if ((pp = sfhmep->hme_page) != NULL) { 5510 /* 5511 * TTE is invalid but the hme 5512 * still exists. let pageunload 5513 * complete its job. 5514 */ 5515 ASSERT(pml == NULL); 5516 pml = sfmmu_mlist_enter(pp); 5517 if (sfhmep->hme_page != NULL) { 5518 sfmmu_mlist_exit(pml); 5519 continue; 5520 } 5521 ASSERT(sfhmep->hme_page == NULL); 5522 } else if (hmeblkp->hblk_hmecnt != 0) { 5523 /* 5524 * pageunload may have not finished decrementing 5525 * hblk_vcnt and hblk_hmecnt. Find page_t if any and 5526 * wait for pageunload to finish. Rely on pageunload 5527 * to decrement hblk_hmecnt after hblk_vcnt. 5528 */ 5529 pfn_t pfn = TTE_TO_TTEPFN(&tte); 5530 ASSERT(pml == NULL); 5531 if (pf_is_memory(pfn)) { 5532 pp = page_numtopp_nolock(pfn); 5533 if (pp != NULL) { 5534 pml = sfmmu_mlist_enter(pp); 5535 sfmmu_mlist_exit(pml); 5536 pml = NULL; 5537 } 5538 } 5539 } 5540 5541 tte_unloaded: 5542 /* 5543 * At this point, the tte we are looking at 5544 * should be unloaded, and hme has been unlinked 5545 * from page too. This is important because in 5546 * pageunload, it does ttesync() then HME_SUB. 5547 * We need to make sure HME_SUB has been completed 5548 * so we know ttesync() has been completed. Otherwise, 5549 * at exit time, after return from hat layer, VM will 5550 * release as structure which hat_setstat() (called 5551 * by ttesync()) needs. 5552 */ 5553 #ifdef DEBUG 5554 { 5555 tte_t dtte; 5556 5557 ASSERT(sfhmep->hme_page == NULL); 5558 5559 sfmmu_copytte(&sfhmep->hme_tte, &dtte); 5560 ASSERT(!TTE_IS_VALID(&dtte)); 5561 } 5562 #endif 5563 5564 if (pml) { 5565 sfmmu_mlist_exit(pml); 5566 } 5567 5568 addr += TTEBYTES(ttesz); 5569 sfhmep++; 5570 DEMAP_RANGE_NEXTPG(dmrp); 5571 } 5572 if (ttecnt > 0) 5573 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt); 5574 return (addr); 5575 } 5576 5577 /* 5578 * Synchronize all the mappings in the range [addr..addr+len). 5579 * Can be called with clearflag having two states: 5580 * HAT_SYNC_DONTZERO means just return the rm stats 5581 * HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats 5582 */ 5583 void 5584 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag) 5585 { 5586 struct hmehash_bucket *hmebp; 5587 hmeblk_tag hblktag; 5588 int hmeshift, hashno = 1; 5589 struct hme_blk *hmeblkp, *list = NULL; 5590 caddr_t endaddr; 5591 cpuset_t cpuset; 5592 5593 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 5594 ASSERT((sfmmup == ksfmmup) || 5595 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 5596 ASSERT((len & MMU_PAGEOFFSET) == 0); 5597 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 5598 (clearflag == HAT_SYNC_ZERORM)); 5599 5600 CPUSET_ZERO(cpuset); 5601 5602 endaddr = addr + len; 5603 hblktag.htag_id = sfmmup; 5604 /* 5605 * Spitfire supports 4 page sizes. 5606 * Most pages are expected to be of the smallest page 5607 * size (8K) and these will not need to be rehashed. 64K 5608 * pages also don't need to be rehashed because the an hmeblk 5609 * spans 64K of address space. 512K pages might need 1 rehash and 5610 * and 4M pages 2 rehashes. 5611 */ 5612 while (addr < endaddr) { 5613 hmeshift = HME_HASH_SHIFT(hashno); 5614 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5615 hblktag.htag_rehash = hashno; 5616 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5617 5618 SFMMU_HASH_LOCK(hmebp); 5619 5620 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 5621 if (hmeblkp != NULL) { 5622 /* 5623 * We've encountered a shadow hmeblk so skip the range 5624 * of the next smaller mapping size. 5625 */ 5626 if (hmeblkp->hblk_shw_bit) { 5627 ASSERT(sfmmup != ksfmmup); 5628 ASSERT(hashno > 1); 5629 addr = (caddr_t)P2END((uintptr_t)addr, 5630 TTEBYTES(hashno - 1)); 5631 } else { 5632 addr = sfmmu_hblk_sync(sfmmup, hmeblkp, 5633 addr, endaddr, clearflag); 5634 } 5635 SFMMU_HASH_UNLOCK(hmebp); 5636 hashno = 1; 5637 continue; 5638 } 5639 SFMMU_HASH_UNLOCK(hmebp); 5640 5641 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 5642 /* 5643 * We have traversed the whole list and rehashed 5644 * if necessary without finding the address to sync. 5645 * This is ok so we increment the address by the 5646 * smallest hmeblk range for kernel mappings and the 5647 * largest hmeblk range, to account for shadow hmeblks, 5648 * for user mappings and continue. 5649 */ 5650 if (sfmmup == ksfmmup) 5651 addr = (caddr_t)P2END((uintptr_t)addr, 5652 TTEBYTES(1)); 5653 else 5654 addr = (caddr_t)P2END((uintptr_t)addr, 5655 TTEBYTES(hashno)); 5656 hashno = 1; 5657 } else { 5658 hashno++; 5659 } 5660 } 5661 sfmmu_hblks_list_purge(&list); 5662 cpuset = sfmmup->sfmmu_cpusran; 5663 xt_sync(cpuset); 5664 } 5665 5666 static caddr_t 5667 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5668 caddr_t endaddr, int clearflag) 5669 { 5670 tte_t tte, ttemod; 5671 struct sf_hment *sfhmep; 5672 int ttesz; 5673 struct page *pp; 5674 kmutex_t *pml; 5675 int ret; 5676 5677 ASSERT(hmeblkp->hblk_shw_bit == 0); 5678 5679 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5680 5681 ttesz = get_hblk_ttesz(hmeblkp); 5682 HBLKTOHME(sfhmep, hmeblkp, addr); 5683 5684 while (addr < endaddr) { 5685 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5686 if (TTE_IS_VALID(&tte)) { 5687 pml = NULL; 5688 pp = sfhmep->hme_page; 5689 if (pp) { 5690 pml = sfmmu_mlist_enter(pp); 5691 } 5692 if (pp != sfhmep->hme_page) { 5693 /* 5694 * tte most have been unloaded 5695 * underneath us. Recheck 5696 */ 5697 ASSERT(pml); 5698 sfmmu_mlist_exit(pml); 5699 continue; 5700 } 5701 5702 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5703 5704 if (clearflag == HAT_SYNC_ZERORM) { 5705 ttemod = tte; 5706 TTE_CLR_RM(&ttemod); 5707 ret = sfmmu_modifytte_try(&tte, &ttemod, 5708 &sfhmep->hme_tte); 5709 if (ret < 0) { 5710 if (pml) { 5711 sfmmu_mlist_exit(pml); 5712 } 5713 continue; 5714 } 5715 5716 if (ret > 0) { 5717 sfmmu_tlb_demap(addr, sfmmup, 5718 hmeblkp, 0, 0); 5719 } 5720 } 5721 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5722 if (pml) { 5723 sfmmu_mlist_exit(pml); 5724 } 5725 } 5726 addr += TTEBYTES(ttesz); 5727 sfhmep++; 5728 } 5729 return (addr); 5730 } 5731 5732 /* 5733 * This function will sync a tte to the page struct and it will 5734 * update the hat stats. Currently it allows us to pass a NULL pp 5735 * and we will simply update the stats. We may want to change this 5736 * so we only keep stats for pages backed by pp's. 5737 */ 5738 static void 5739 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp) 5740 { 5741 uint_t rm = 0; 5742 int sz; 5743 pgcnt_t npgs; 5744 5745 ASSERT(TTE_IS_VALID(ttep)); 5746 5747 if (TTE_IS_NOSYNC(ttep)) { 5748 return; 5749 } 5750 5751 if (TTE_IS_REF(ttep)) { 5752 rm = P_REF; 5753 } 5754 if (TTE_IS_MOD(ttep)) { 5755 rm |= P_MOD; 5756 } 5757 5758 if (rm == 0) { 5759 return; 5760 } 5761 5762 sz = TTE_CSZ(ttep); 5763 if (sfmmup->sfmmu_rmstat) { 5764 int i; 5765 caddr_t vaddr = addr; 5766 5767 for (i = 0; i < TTEPAGES(sz); i++, vaddr += MMU_PAGESIZE) { 5768 hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm); 5769 } 5770 5771 } 5772 5773 /* 5774 * XXX I want to use cas to update nrm bits but they 5775 * currently belong in common/vm and not in hat where 5776 * they should be. 5777 * The nrm bits are protected by the same mutex as 5778 * the one that protects the page's mapping list. 5779 */ 5780 if (!pp) 5781 return; 5782 ASSERT(sfmmu_mlist_held(pp)); 5783 /* 5784 * If the tte is for a large page, we need to sync all the 5785 * pages covered by the tte. 5786 */ 5787 if (sz != TTE8K) { 5788 ASSERT(pp->p_szc != 0); 5789 pp = PP_GROUPLEADER(pp, sz); 5790 ASSERT(sfmmu_mlist_held(pp)); 5791 } 5792 5793 /* Get number of pages from tte size. */ 5794 npgs = TTEPAGES(sz); 5795 5796 do { 5797 ASSERT(pp); 5798 ASSERT(sfmmu_mlist_held(pp)); 5799 if (((rm & P_REF) != 0 && !PP_ISREF(pp)) || 5800 ((rm & P_MOD) != 0 && !PP_ISMOD(pp))) 5801 hat_page_setattr(pp, rm); 5802 5803 /* 5804 * Are we done? If not, we must have a large mapping. 5805 * For large mappings we need to sync the rest of the pages 5806 * covered by this tte; goto the next page. 5807 */ 5808 } while (--npgs > 0 && (pp = PP_PAGENEXT(pp))); 5809 } 5810 5811 /* 5812 * Execute pre-callback handler of each pa_hment linked to pp 5813 * 5814 * Inputs: 5815 * flag: either HAT_PRESUSPEND or HAT_SUSPEND. 5816 * capture_cpus: pointer to return value (below) 5817 * 5818 * Returns: 5819 * Propagates the subsystem callback return values back to the caller; 5820 * returns 0 on success. If capture_cpus is non-NULL, the value returned 5821 * is zero if all of the pa_hments are of a type that do not require 5822 * capturing CPUs prior to suspending the mapping, else it is 1. 5823 */ 5824 static int 5825 hat_pageprocess_precallbacks(struct page *pp, uint_t flag, int *capture_cpus) 5826 { 5827 struct sf_hment *sfhmep; 5828 struct pa_hment *pahmep; 5829 int (*f)(caddr_t, uint_t, uint_t, void *); 5830 int ret; 5831 id_t id; 5832 int locked = 0; 5833 kmutex_t *pml; 5834 5835 ASSERT(PAGE_EXCL(pp)); 5836 if (!sfmmu_mlist_held(pp)) { 5837 pml = sfmmu_mlist_enter(pp); 5838 locked = 1; 5839 } 5840 5841 if (capture_cpus) 5842 *capture_cpus = 0; 5843 5844 top: 5845 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5846 /* 5847 * skip sf_hments corresponding to VA<->PA mappings; 5848 * for pa_hment's, hme_tte.ll is zero 5849 */ 5850 if (!IS_PAHME(sfhmep)) 5851 continue; 5852 5853 pahmep = sfhmep->hme_data; 5854 ASSERT(pahmep != NULL); 5855 5856 /* 5857 * skip if pre-handler has been called earlier in this loop 5858 */ 5859 if (pahmep->flags & flag) 5860 continue; 5861 5862 id = pahmep->cb_id; 5863 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5864 if (capture_cpus && sfmmu_cb_table[id].capture_cpus != 0) 5865 *capture_cpus = 1; 5866 if ((f = sfmmu_cb_table[id].prehandler) == NULL) { 5867 pahmep->flags |= flag; 5868 continue; 5869 } 5870 5871 /* 5872 * Drop the mapping list lock to avoid locking order issues. 5873 */ 5874 if (locked) 5875 sfmmu_mlist_exit(pml); 5876 5877 ret = f(pahmep->addr, pahmep->len, flag, pahmep->pvt); 5878 if (ret != 0) 5879 return (ret); /* caller must do the cleanup */ 5880 5881 if (locked) { 5882 pml = sfmmu_mlist_enter(pp); 5883 pahmep->flags |= flag; 5884 goto top; 5885 } 5886 5887 pahmep->flags |= flag; 5888 } 5889 5890 if (locked) 5891 sfmmu_mlist_exit(pml); 5892 5893 return (0); 5894 } 5895 5896 /* 5897 * Execute post-callback handler of each pa_hment linked to pp 5898 * 5899 * Same overall assumptions and restrictions apply as for 5900 * hat_pageprocess_precallbacks(). 5901 */ 5902 static void 5903 hat_pageprocess_postcallbacks(struct page *pp, uint_t flag) 5904 { 5905 pfn_t pgpfn = pp->p_pagenum; 5906 pfn_t pgmask = btop(page_get_pagesize(pp->p_szc)) - 1; 5907 pfn_t newpfn; 5908 struct sf_hment *sfhmep; 5909 struct pa_hment *pahmep; 5910 int (*f)(caddr_t, uint_t, uint_t, void *, pfn_t); 5911 id_t id; 5912 int locked = 0; 5913 kmutex_t *pml; 5914 5915 ASSERT(PAGE_EXCL(pp)); 5916 if (!sfmmu_mlist_held(pp)) { 5917 pml = sfmmu_mlist_enter(pp); 5918 locked = 1; 5919 } 5920 5921 top: 5922 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5923 /* 5924 * skip sf_hments corresponding to VA<->PA mappings; 5925 * for pa_hment's, hme_tte.ll is zero 5926 */ 5927 if (!IS_PAHME(sfhmep)) 5928 continue; 5929 5930 pahmep = sfhmep->hme_data; 5931 ASSERT(pahmep != NULL); 5932 5933 if ((pahmep->flags & flag) == 0) 5934 continue; 5935 5936 pahmep->flags &= ~flag; 5937 5938 id = pahmep->cb_id; 5939 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5940 if ((f = sfmmu_cb_table[id].posthandler) == NULL) 5941 continue; 5942 5943 /* 5944 * Convert the base page PFN into the constituent PFN 5945 * which is needed by the callback handler. 5946 */ 5947 newpfn = pgpfn | (btop((uintptr_t)pahmep->addr) & pgmask); 5948 5949 /* 5950 * Drop the mapping list lock to avoid locking order issues. 5951 */ 5952 if (locked) 5953 sfmmu_mlist_exit(pml); 5954 5955 if (f(pahmep->addr, pahmep->len, flag, pahmep->pvt, newpfn) 5956 != 0) 5957 panic("sfmmu: posthandler failed"); 5958 5959 if (locked) { 5960 pml = sfmmu_mlist_enter(pp); 5961 goto top; 5962 } 5963 } 5964 5965 if (locked) 5966 sfmmu_mlist_exit(pml); 5967 } 5968 5969 /* 5970 * Suspend locked kernel mapping 5971 */ 5972 void 5973 hat_pagesuspend(struct page *pp) 5974 { 5975 struct sf_hment *sfhmep; 5976 sfmmu_t *sfmmup; 5977 tte_t tte, ttemod; 5978 struct hme_blk *hmeblkp; 5979 caddr_t addr; 5980 int index, cons; 5981 cpuset_t cpuset; 5982 5983 ASSERT(PAGE_EXCL(pp)); 5984 ASSERT(sfmmu_mlist_held(pp)); 5985 5986 mutex_enter(&kpr_suspendlock); 5987 5988 /* 5989 * Call into dtrace to tell it we're about to suspend a 5990 * kernel mapping. This prevents us from running into issues 5991 * with probe context trying to touch a suspended page 5992 * in the relocation codepath itself. 5993 */ 5994 if (dtrace_kreloc_init) 5995 (*dtrace_kreloc_init)(); 5996 5997 index = PP_MAPINDEX(pp); 5998 cons = TTE8K; 5999 6000 retry: 6001 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 6002 6003 if (IS_PAHME(sfhmep)) 6004 continue; 6005 6006 if (get_hblk_ttesz(sfmmu_hmetohblk(sfhmep)) != cons) 6007 continue; 6008 6009 /* 6010 * Loop until we successfully set the suspend bit in 6011 * the TTE. 6012 */ 6013 again: 6014 sfmmu_copytte(&sfhmep->hme_tte, &tte); 6015 ASSERT(TTE_IS_VALID(&tte)); 6016 6017 ttemod = tte; 6018 TTE_SET_SUSPEND(&ttemod); 6019 if (sfmmu_modifytte_try(&tte, &ttemod, 6020 &sfhmep->hme_tte) < 0) 6021 goto again; 6022 6023 /* 6024 * Invalidate TSB entry 6025 */ 6026 hmeblkp = sfmmu_hmetohblk(sfhmep); 6027 6028 sfmmup = hblktosfmmu(hmeblkp); 6029 ASSERT(sfmmup == ksfmmup); 6030 6031 addr = tte_to_vaddr(hmeblkp, tte); 6032 6033 /* 6034 * No need to make sure that the TSB for this sfmmu is 6035 * not being relocated since it is ksfmmup and thus it 6036 * will never be relocated. 6037 */ 6038 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 6039 6040 /* 6041 * Update xcall stats 6042 */ 6043 cpuset = cpu_ready_set; 6044 CPUSET_DEL(cpuset, CPU->cpu_id); 6045 6046 /* LINTED: constant in conditional context */ 6047 SFMMU_XCALL_STATS(ksfmmup); 6048 6049 /* 6050 * Flush TLB entry on remote CPU's 6051 */ 6052 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 6053 (uint64_t)ksfmmup); 6054 xt_sync(cpuset); 6055 6056 /* 6057 * Flush TLB entry on local CPU 6058 */ 6059 vtag_flushpage(addr, (uint64_t)ksfmmup); 6060 } 6061 6062 while (index != 0) { 6063 index = index >> 1; 6064 if (index != 0) 6065 cons++; 6066 if (index & 0x1) { 6067 pp = PP_GROUPLEADER(pp, cons); 6068 goto retry; 6069 } 6070 } 6071 } 6072 6073 #ifdef DEBUG 6074 6075 #define N_PRLE 1024 6076 struct prle { 6077 page_t *targ; 6078 page_t *repl; 6079 int status; 6080 int pausecpus; 6081 hrtime_t whence; 6082 }; 6083 6084 static struct prle page_relocate_log[N_PRLE]; 6085 static int prl_entry; 6086 static kmutex_t prl_mutex; 6087 6088 #define PAGE_RELOCATE_LOG(t, r, s, p) \ 6089 mutex_enter(&prl_mutex); \ 6090 page_relocate_log[prl_entry].targ = *(t); \ 6091 page_relocate_log[prl_entry].repl = *(r); \ 6092 page_relocate_log[prl_entry].status = (s); \ 6093 page_relocate_log[prl_entry].pausecpus = (p); \ 6094 page_relocate_log[prl_entry].whence = gethrtime(); \ 6095 prl_entry = (prl_entry == (N_PRLE - 1))? 0 : prl_entry + 1; \ 6096 mutex_exit(&prl_mutex); 6097 6098 #else /* !DEBUG */ 6099 #define PAGE_RELOCATE_LOG(t, r, s, p) 6100 #endif 6101 6102 /* 6103 * Core Kernel Page Relocation Algorithm 6104 * 6105 * Input: 6106 * 6107 * target : constituent pages are SE_EXCL locked. 6108 * replacement: constituent pages are SE_EXCL locked. 6109 * 6110 * Output: 6111 * 6112 * nrelocp: number of pages relocated 6113 */ 6114 int 6115 hat_page_relocate(page_t **target, page_t **replacement, spgcnt_t *nrelocp) 6116 { 6117 page_t *targ, *repl; 6118 page_t *tpp, *rpp; 6119 kmutex_t *low, *high; 6120 spgcnt_t npages, i; 6121 page_t *pl = NULL; 6122 int old_pil; 6123 cpuset_t cpuset; 6124 int cap_cpus; 6125 int ret; 6126 6127 if (hat_kpr_enabled == 0 || !kcage_on || PP_ISNORELOC(*target)) { 6128 PAGE_RELOCATE_LOG(target, replacement, EAGAIN, -1); 6129 return (EAGAIN); 6130 } 6131 6132 mutex_enter(&kpr_mutex); 6133 kreloc_thread = curthread; 6134 6135 targ = *target; 6136 repl = *replacement; 6137 ASSERT(repl != NULL); 6138 ASSERT(targ->p_szc == repl->p_szc); 6139 6140 npages = page_get_pagecnt(targ->p_szc); 6141 6142 /* 6143 * unload VA<->PA mappings that are not locked 6144 */ 6145 tpp = targ; 6146 for (i = 0; i < npages; i++) { 6147 (void) hat_pageunload(tpp, SFMMU_KERNEL_RELOC); 6148 tpp++; 6149 } 6150 6151 /* 6152 * Do "presuspend" callbacks, in a context from which we can still 6153 * block as needed. Note that we don't hold the mapping list lock 6154 * of "targ" at this point due to potential locking order issues; 6155 * we assume that between the hat_pageunload() above and holding 6156 * the SE_EXCL lock that the mapping list *cannot* change at this 6157 * point. 6158 */ 6159 ret = hat_pageprocess_precallbacks(targ, HAT_PRESUSPEND, &cap_cpus); 6160 if (ret != 0) { 6161 /* 6162 * EIO translates to fatal error, for all others cleanup 6163 * and return EAGAIN. 6164 */ 6165 ASSERT(ret != EIO); 6166 hat_pageprocess_postcallbacks(targ, HAT_POSTUNSUSPEND); 6167 PAGE_RELOCATE_LOG(target, replacement, ret, -1); 6168 kreloc_thread = NULL; 6169 mutex_exit(&kpr_mutex); 6170 return (EAGAIN); 6171 } 6172 6173 /* 6174 * acquire p_mapping list lock for both the target and replacement 6175 * root pages. 6176 * 6177 * low and high refer to the need to grab the mlist locks in a 6178 * specific order in order to prevent race conditions. Thus the 6179 * lower lock must be grabbed before the higher lock. 6180 * 6181 * This will block hat_unload's accessing p_mapping list. Since 6182 * we have SE_EXCL lock, hat_memload and hat_pageunload will be 6183 * blocked. Thus, no one else will be accessing the p_mapping list 6184 * while we suspend and reload the locked mapping below. 6185 */ 6186 tpp = targ; 6187 rpp = repl; 6188 sfmmu_mlist_reloc_enter(tpp, rpp, &low, &high); 6189 6190 kpreempt_disable(); 6191 6192 #ifdef VAC 6193 /* 6194 * If the replacement page is of a different virtual color 6195 * than the page it is replacing, we need to handle the VAC 6196 * consistency for it just as we would if we were setting up 6197 * a new mapping to a page. 6198 */ 6199 if ((tpp->p_szc == 0) && (PP_GET_VCOLOR(rpp) != NO_VCOLOR)) { 6200 if (tpp->p_vcolor != rpp->p_vcolor) { 6201 sfmmu_cache_flushcolor(PP_GET_VCOLOR(rpp), 6202 rpp->p_pagenum); 6203 } 6204 } 6205 #endif 6206 6207 /* 6208 * We raise our PIL to 13 so that we don't get captured by 6209 * another CPU or pinned by an interrupt thread. We can't go to 6210 * PIL 14 since the nexus driver(s) may need to interrupt at 6211 * that level in the case of IOMMU pseudo mappings. 6212 */ 6213 cpuset = cpu_ready_set; 6214 CPUSET_DEL(cpuset, CPU->cpu_id); 6215 if (!cap_cpus || CPUSET_ISNULL(cpuset)) { 6216 old_pil = splr(XCALL_PIL); 6217 } else { 6218 old_pil = -1; 6219 xc_attention(cpuset); 6220 } 6221 ASSERT(getpil() == XCALL_PIL); 6222 6223 /* 6224 * Now do suspend callbacks. In the case of an IOMMU mapping 6225 * this will suspend all DMA activity to the page while it is 6226 * being relocated. Since we are well above LOCK_LEVEL and CPUs 6227 * may be captured at this point we should have acquired any needed 6228 * locks in the presuspend callback. 6229 */ 6230 ret = hat_pageprocess_precallbacks(targ, HAT_SUSPEND, NULL); 6231 if (ret != 0) { 6232 repl = targ; 6233 goto suspend_fail; 6234 } 6235 6236 /* 6237 * Raise the PIL yet again, this time to block all high-level 6238 * interrupts on this CPU. This is necessary to prevent an 6239 * interrupt routine from pinning the thread which holds the 6240 * mapping suspended and then touching the suspended page. 6241 * 6242 * Once the page is suspended we also need to be careful to 6243 * avoid calling any functions which touch any seg_kmem memory 6244 * since that memory may be backed by the very page we are 6245 * relocating in here! 6246 */ 6247 hat_pagesuspend(targ); 6248 6249 /* 6250 * Now that we are confident everybody has stopped using this page, 6251 * copy the page contents. Note we use a physical copy to prevent 6252 * locking issues and to avoid fpRAS because we can't handle it in 6253 * this context. 6254 */ 6255 for (i = 0; i < npages; i++, tpp++, rpp++) { 6256 /* 6257 * Copy the contents of the page. 6258 */ 6259 ppcopy_kernel(tpp, rpp); 6260 } 6261 6262 tpp = targ; 6263 rpp = repl; 6264 for (i = 0; i < npages; i++, tpp++, rpp++) { 6265 /* 6266 * Copy attributes. VAC consistency was handled above, 6267 * if required. 6268 */ 6269 rpp->p_nrm = tpp->p_nrm; 6270 tpp->p_nrm = 0; 6271 rpp->p_index = tpp->p_index; 6272 tpp->p_index = 0; 6273 #ifdef VAC 6274 rpp->p_vcolor = tpp->p_vcolor; 6275 #endif 6276 } 6277 6278 /* 6279 * First, unsuspend the page, if we set the suspend bit, and transfer 6280 * the mapping list from the target page to the replacement page. 6281 * Next process postcallbacks; since pa_hment's are linked only to the 6282 * p_mapping list of root page, we don't iterate over the constituent 6283 * pages. 6284 */ 6285 hat_pagereload(targ, repl); 6286 6287 suspend_fail: 6288 hat_pageprocess_postcallbacks(repl, HAT_UNSUSPEND); 6289 6290 /* 6291 * Now lower our PIL and release any captured CPUs since we 6292 * are out of the "danger zone". After this it will again be 6293 * safe to acquire adaptive mutex locks, or to drop them... 6294 */ 6295 if (old_pil != -1) { 6296 splx(old_pil); 6297 } else { 6298 xc_dismissed(cpuset); 6299 } 6300 6301 kpreempt_enable(); 6302 6303 sfmmu_mlist_reloc_exit(low, high); 6304 6305 /* 6306 * Postsuspend callbacks should drop any locks held across 6307 * the suspend callbacks. As before, we don't hold the mapping 6308 * list lock at this point.. our assumption is that the mapping 6309 * list still can't change due to our holding SE_EXCL lock and 6310 * there being no unlocked mappings left. Hence the restriction 6311 * on calling context to hat_delete_callback() 6312 */ 6313 hat_pageprocess_postcallbacks(repl, HAT_POSTUNSUSPEND); 6314 if (ret != 0) { 6315 /* 6316 * The second presuspend call failed: we got here through 6317 * the suspend_fail label above. 6318 */ 6319 ASSERT(ret != EIO); 6320 PAGE_RELOCATE_LOG(target, replacement, ret, cap_cpus); 6321 kreloc_thread = NULL; 6322 mutex_exit(&kpr_mutex); 6323 return (EAGAIN); 6324 } 6325 6326 /* 6327 * Now that we're out of the performance critical section we can 6328 * take care of updating the hash table, since we still 6329 * hold all the pages locked SE_EXCL at this point we 6330 * needn't worry about things changing out from under us. 6331 */ 6332 tpp = targ; 6333 rpp = repl; 6334 for (i = 0; i < npages; i++, tpp++, rpp++) { 6335 6336 /* 6337 * replace targ with replacement in page_hash table 6338 */ 6339 targ = tpp; 6340 page_relocate_hash(rpp, targ); 6341 6342 /* 6343 * concatenate target; caller of platform_page_relocate() 6344 * expects target to be concatenated after returning. 6345 */ 6346 ASSERT(targ->p_next == targ); 6347 ASSERT(targ->p_prev == targ); 6348 page_list_concat(&pl, &targ); 6349 } 6350 6351 ASSERT(*target == pl); 6352 *nrelocp = npages; 6353 PAGE_RELOCATE_LOG(target, replacement, 0, cap_cpus); 6354 kreloc_thread = NULL; 6355 mutex_exit(&kpr_mutex); 6356 return (0); 6357 } 6358 6359 /* 6360 * Called when stray pa_hments are found attached to a page which is 6361 * being freed. Notify the subsystem which attached the pa_hment of 6362 * the error if it registered a suitable handler, else panic. 6363 */ 6364 static void 6365 sfmmu_pahment_leaked(struct pa_hment *pahmep) 6366 { 6367 id_t cb_id = pahmep->cb_id; 6368 6369 ASSERT(cb_id >= (id_t)0 && cb_id < sfmmu_cb_nextid); 6370 if (sfmmu_cb_table[cb_id].errhandler != NULL) { 6371 if (sfmmu_cb_table[cb_id].errhandler(pahmep->addr, pahmep->len, 6372 HAT_CB_ERR_LEAKED, pahmep->pvt) == 0) 6373 return; /* non-fatal */ 6374 } 6375 panic("pa_hment leaked: 0x%p", pahmep); 6376 } 6377 6378 /* 6379 * Remove all mappings to page 'pp'. 6380 */ 6381 int 6382 hat_pageunload(struct page *pp, uint_t forceflag) 6383 { 6384 struct page *origpp = pp; 6385 struct sf_hment *sfhme, *tmphme; 6386 struct hme_blk *hmeblkp; 6387 kmutex_t *pml; 6388 #ifdef VAC 6389 kmutex_t *pmtx; 6390 #endif 6391 cpuset_t cpuset, tset; 6392 int index, cons; 6393 int xhme_blks; 6394 int pa_hments; 6395 6396 ASSERT(PAGE_EXCL(pp)); 6397 6398 retry_xhat: 6399 tmphme = NULL; 6400 xhme_blks = 0; 6401 pa_hments = 0; 6402 CPUSET_ZERO(cpuset); 6403 6404 pml = sfmmu_mlist_enter(pp); 6405 6406 #ifdef VAC 6407 if (pp->p_kpmref) 6408 sfmmu_kpm_pageunload(pp); 6409 ASSERT(!PP_ISMAPPED_KPM(pp)); 6410 #endif 6411 6412 index = PP_MAPINDEX(pp); 6413 cons = TTE8K; 6414 retry: 6415 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6416 tmphme = sfhme->hme_next; 6417 6418 if (IS_PAHME(sfhme)) { 6419 ASSERT(sfhme->hme_data != NULL); 6420 pa_hments++; 6421 continue; 6422 } 6423 6424 hmeblkp = sfmmu_hmetohblk(sfhme); 6425 if (hmeblkp->hblk_xhat_bit) { 6426 struct xhat_hme_blk *xblk = 6427 (struct xhat_hme_blk *)hmeblkp; 6428 6429 (void) XHAT_PAGEUNLOAD(xblk->xhat_hme_blk_hat, 6430 pp, forceflag, XBLK2PROVBLK(xblk)); 6431 6432 xhme_blks = 1; 6433 continue; 6434 } 6435 6436 /* 6437 * If there are kernel mappings don't unload them, they will 6438 * be suspended. 6439 */ 6440 if (forceflag == SFMMU_KERNEL_RELOC && hmeblkp->hblk_lckcnt && 6441 hmeblkp->hblk_tag.htag_id == ksfmmup) 6442 continue; 6443 6444 tset = sfmmu_pageunload(pp, sfhme, cons); 6445 CPUSET_OR(cpuset, tset); 6446 } 6447 6448 while (index != 0) { 6449 index = index >> 1; 6450 if (index != 0) 6451 cons++; 6452 if (index & 0x1) { 6453 /* Go to leading page */ 6454 pp = PP_GROUPLEADER(pp, cons); 6455 ASSERT(sfmmu_mlist_held(pp)); 6456 goto retry; 6457 } 6458 } 6459 6460 /* 6461 * cpuset may be empty if the page was only mapped by segkpm, 6462 * in which case we won't actually cross-trap. 6463 */ 6464 xt_sync(cpuset); 6465 6466 /* 6467 * The page should have no mappings at this point, unless 6468 * we were called from hat_page_relocate() in which case we 6469 * leave the locked mappings which will be suspended later. 6470 */ 6471 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks || pa_hments || 6472 (forceflag == SFMMU_KERNEL_RELOC)); 6473 6474 #ifdef VAC 6475 if (PP_ISTNC(pp)) { 6476 if (cons == TTE8K) { 6477 pmtx = sfmmu_page_enter(pp); 6478 PP_CLRTNC(pp); 6479 sfmmu_page_exit(pmtx); 6480 } else { 6481 conv_tnc(pp, cons); 6482 } 6483 } 6484 #endif /* VAC */ 6485 6486 if (pa_hments && forceflag != SFMMU_KERNEL_RELOC) { 6487 /* 6488 * Unlink any pa_hments and free them, calling back 6489 * the responsible subsystem to notify it of the error. 6490 * This can occur in situations such as drivers leaking 6491 * DMA handles: naughty, but common enough that we'd like 6492 * to keep the system running rather than bringing it 6493 * down with an obscure error like "pa_hment leaked" 6494 * which doesn't aid the user in debugging their driver. 6495 */ 6496 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6497 tmphme = sfhme->hme_next; 6498 if (IS_PAHME(sfhme)) { 6499 struct pa_hment *pahmep = sfhme->hme_data; 6500 sfmmu_pahment_leaked(pahmep); 6501 HME_SUB(sfhme, pp); 6502 kmem_cache_free(pa_hment_cache, pahmep); 6503 } 6504 } 6505 6506 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks); 6507 } 6508 6509 sfmmu_mlist_exit(pml); 6510 6511 /* 6512 * XHAT may not have finished unloading pages 6513 * because some other thread was waiting for 6514 * mlist lock and XHAT_PAGEUNLOAD let it do 6515 * the job. 6516 */ 6517 if (xhme_blks) { 6518 pp = origpp; 6519 goto retry_xhat; 6520 } 6521 6522 return (0); 6523 } 6524 6525 cpuset_t 6526 sfmmu_pageunload(page_t *pp, struct sf_hment *sfhme, int cons) 6527 { 6528 struct hme_blk *hmeblkp; 6529 sfmmu_t *sfmmup; 6530 tte_t tte, ttemod; 6531 #ifdef DEBUG 6532 tte_t orig_old; 6533 #endif /* DEBUG */ 6534 caddr_t addr; 6535 int ttesz; 6536 int ret; 6537 cpuset_t cpuset; 6538 6539 ASSERT(pp != NULL); 6540 ASSERT(sfmmu_mlist_held(pp)); 6541 ASSERT(!PP_ISKAS(pp)); 6542 6543 CPUSET_ZERO(cpuset); 6544 6545 hmeblkp = sfmmu_hmetohblk(sfhme); 6546 6547 readtte: 6548 sfmmu_copytte(&sfhme->hme_tte, &tte); 6549 if (TTE_IS_VALID(&tte)) { 6550 sfmmup = hblktosfmmu(hmeblkp); 6551 ttesz = get_hblk_ttesz(hmeblkp); 6552 /* 6553 * Only unload mappings of 'cons' size. 6554 */ 6555 if (ttesz != cons) 6556 return (cpuset); 6557 6558 /* 6559 * Note that we have p_mapping lock, but no hash lock here. 6560 * hblk_unload() has to have both hash lock AND p_mapping 6561 * lock before it tries to modify tte. So, the tte could 6562 * not become invalid in the sfmmu_modifytte_try() below. 6563 */ 6564 ttemod = tte; 6565 #ifdef DEBUG 6566 orig_old = tte; 6567 #endif /* DEBUG */ 6568 6569 TTE_SET_INVALID(&ttemod); 6570 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6571 if (ret < 0) { 6572 #ifdef DEBUG 6573 /* only R/M bits can change. */ 6574 chk_tte(&orig_old, &tte, &ttemod, hmeblkp); 6575 #endif /* DEBUG */ 6576 goto readtte; 6577 } 6578 6579 if (ret == 0) { 6580 panic("pageunload: cas failed?"); 6581 } 6582 6583 addr = tte_to_vaddr(hmeblkp, tte); 6584 6585 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6586 6587 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -1); 6588 6589 /* 6590 * We need to flush the page from the virtual cache 6591 * in order to prevent a virtual cache alias 6592 * inconsistency. The particular scenario we need 6593 * to worry about is: 6594 * Given: va1 and va2 are two virtual address that 6595 * alias and will map the same physical address. 6596 * 1. mapping exists from va1 to pa and data has 6597 * been read into the cache. 6598 * 2. unload va1. 6599 * 3. load va2 and modify data using va2. 6600 * 4 unload va2. 6601 * 5. load va1 and reference data. Unless we flush 6602 * the data cache when we unload we will get 6603 * stale data. 6604 * This scenario is taken care of by using virtual 6605 * page coloring. 6606 */ 6607 if (sfmmup->sfmmu_ismhat) { 6608 /* 6609 * Flush TSBs, TLBs and caches 6610 * of every process 6611 * sharing this ism segment. 6612 */ 6613 sfmmu_hat_lock_all(); 6614 mutex_enter(&ism_mlist_lock); 6615 kpreempt_disable(); 6616 if (do_virtual_coloring) 6617 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6618 pp->p_pagenum, CACHE_NO_FLUSH); 6619 else 6620 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6621 pp->p_pagenum, CACHE_FLUSH); 6622 kpreempt_enable(); 6623 mutex_exit(&ism_mlist_lock); 6624 sfmmu_hat_unlock_all(); 6625 cpuset = cpu_ready_set; 6626 } else if (do_virtual_coloring) { 6627 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6628 cpuset = sfmmup->sfmmu_cpusran; 6629 } else { 6630 sfmmu_tlbcache_demap(addr, sfmmup, hmeblkp, 6631 pp->p_pagenum, 0, FLUSH_NECESSARY_CPUS, 6632 CACHE_FLUSH, 0); 6633 cpuset = sfmmup->sfmmu_cpusran; 6634 } 6635 6636 /* 6637 * Hme_sub has to run after ttesync() and a_rss update. 6638 * See hblk_unload(). 6639 */ 6640 HME_SUB(sfhme, pp); 6641 membar_stst(); 6642 6643 /* 6644 * We can not make ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 6645 * since pteload may have done a HME_ADD() right after 6646 * we did the HME_SUB() above. Hmecnt is now maintained 6647 * by cas only. no lock guranteed its value. The only 6648 * gurantee we have is the hmecnt should not be less than 6649 * what it should be so the hblk will not be taken away. 6650 * It's also important that we decremented the hmecnt after 6651 * we are done with hmeblkp so that this hmeblk won't be 6652 * stolen. 6653 */ 6654 ASSERT(hmeblkp->hblk_hmecnt > 0); 6655 ASSERT(hmeblkp->hblk_vcnt > 0); 6656 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 6657 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 6658 /* 6659 * This is bug 4063182. 6660 * XXX: fixme 6661 * ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 6662 * !hmeblkp->hblk_lckcnt); 6663 */ 6664 } else { 6665 panic("invalid tte? pp %p &tte %p", 6666 (void *)pp, (void *)&tte); 6667 } 6668 6669 return (cpuset); 6670 } 6671 6672 /* 6673 * While relocating a kernel page, this function will move the mappings 6674 * from tpp to dpp and modify any associated data with these mappings. 6675 * It also unsuspends the suspended kernel mapping. 6676 */ 6677 static void 6678 hat_pagereload(struct page *tpp, struct page *dpp) 6679 { 6680 struct sf_hment *sfhme; 6681 tte_t tte, ttemod; 6682 int index, cons; 6683 6684 ASSERT(getpil() == PIL_MAX); 6685 ASSERT(sfmmu_mlist_held(tpp)); 6686 ASSERT(sfmmu_mlist_held(dpp)); 6687 6688 index = PP_MAPINDEX(tpp); 6689 cons = TTE8K; 6690 6691 /* Update real mappings to the page */ 6692 retry: 6693 for (sfhme = tpp->p_mapping; sfhme != NULL; sfhme = sfhme->hme_next) { 6694 if (IS_PAHME(sfhme)) 6695 continue; 6696 sfmmu_copytte(&sfhme->hme_tte, &tte); 6697 ttemod = tte; 6698 6699 /* 6700 * replace old pfn with new pfn in TTE 6701 */ 6702 PFN_TO_TTE(ttemod, dpp->p_pagenum); 6703 6704 /* 6705 * clear suspend bit 6706 */ 6707 ASSERT(TTE_IS_SUSPEND(&ttemod)); 6708 TTE_CLR_SUSPEND(&ttemod); 6709 6710 if (sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte) < 0) 6711 panic("hat_pagereload(): sfmmu_modifytte_try() failed"); 6712 6713 /* 6714 * set hme_page point to new page 6715 */ 6716 sfhme->hme_page = dpp; 6717 } 6718 6719 /* 6720 * move p_mapping list from old page to new page 6721 */ 6722 dpp->p_mapping = tpp->p_mapping; 6723 tpp->p_mapping = NULL; 6724 dpp->p_share = tpp->p_share; 6725 tpp->p_share = 0; 6726 6727 while (index != 0) { 6728 index = index >> 1; 6729 if (index != 0) 6730 cons++; 6731 if (index & 0x1) { 6732 tpp = PP_GROUPLEADER(tpp, cons); 6733 dpp = PP_GROUPLEADER(dpp, cons); 6734 goto retry; 6735 } 6736 } 6737 6738 if (dtrace_kreloc_fini) 6739 (*dtrace_kreloc_fini)(); 6740 mutex_exit(&kpr_suspendlock); 6741 } 6742 6743 uint_t 6744 hat_pagesync(struct page *pp, uint_t clearflag) 6745 { 6746 struct sf_hment *sfhme, *tmphme = NULL; 6747 struct hme_blk *hmeblkp; 6748 kmutex_t *pml; 6749 cpuset_t cpuset, tset; 6750 int index, cons; 6751 extern ulong_t po_share; 6752 page_t *save_pp = pp; 6753 6754 CPUSET_ZERO(cpuset); 6755 6756 if (PP_ISRO(pp) && (clearflag & HAT_SYNC_STOPON_MOD)) { 6757 return (PP_GENERIC_ATTR(pp)); 6758 } 6759 6760 if ((clearflag == (HAT_SYNC_STOPON_REF | HAT_SYNC_DONTZERO)) && 6761 PP_ISREF(pp)) { 6762 return (PP_GENERIC_ATTR(pp)); 6763 } 6764 6765 if ((clearflag == (HAT_SYNC_STOPON_MOD | HAT_SYNC_DONTZERO)) && 6766 PP_ISMOD(pp)) { 6767 return (PP_GENERIC_ATTR(pp)); 6768 } 6769 6770 if ((clearflag & HAT_SYNC_STOPON_SHARED) != 0 && 6771 (pp->p_share > po_share) && 6772 !(clearflag & HAT_SYNC_ZERORM)) { 6773 if (PP_ISRO(pp)) 6774 hat_page_setattr(pp, P_REF); 6775 return (PP_GENERIC_ATTR(pp)); 6776 } 6777 6778 clearflag &= ~HAT_SYNC_STOPON_SHARED; 6779 pml = sfmmu_mlist_enter(pp); 6780 index = PP_MAPINDEX(pp); 6781 cons = TTE8K; 6782 retry: 6783 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6784 /* 6785 * We need to save the next hment on the list since 6786 * it is possible for pagesync to remove an invalid hment 6787 * from the list. 6788 */ 6789 tmphme = sfhme->hme_next; 6790 /* 6791 * If we are looking for large mappings and this hme doesn't 6792 * reach the range we are seeking, just ignore its. 6793 */ 6794 hmeblkp = sfmmu_hmetohblk(sfhme); 6795 if (hmeblkp->hblk_xhat_bit) 6796 continue; 6797 6798 if (hme_size(sfhme) < cons) 6799 continue; 6800 tset = sfmmu_pagesync(pp, sfhme, 6801 clearflag & ~HAT_SYNC_STOPON_RM); 6802 CPUSET_OR(cpuset, tset); 6803 /* 6804 * If clearflag is HAT_SYNC_DONTZERO, break out as soon 6805 * as the "ref" or "mod" is set. 6806 */ 6807 if ((clearflag & ~HAT_SYNC_STOPON_RM) == HAT_SYNC_DONTZERO && 6808 ((clearflag & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp)) || 6809 ((clearflag & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp))) { 6810 index = 0; 6811 break; 6812 } 6813 } 6814 6815 while (index) { 6816 index = index >> 1; 6817 cons++; 6818 if (index & 0x1) { 6819 /* Go to leading page */ 6820 pp = PP_GROUPLEADER(pp, cons); 6821 goto retry; 6822 } 6823 } 6824 6825 xt_sync(cpuset); 6826 sfmmu_mlist_exit(pml); 6827 return (PP_GENERIC_ATTR(save_pp)); 6828 } 6829 6830 /* 6831 * Get all the hardware dependent attributes for a page struct 6832 */ 6833 static cpuset_t 6834 sfmmu_pagesync(struct page *pp, struct sf_hment *sfhme, 6835 uint_t clearflag) 6836 { 6837 caddr_t addr; 6838 tte_t tte, ttemod; 6839 struct hme_blk *hmeblkp; 6840 int ret; 6841 sfmmu_t *sfmmup; 6842 cpuset_t cpuset; 6843 6844 ASSERT(pp != NULL); 6845 ASSERT(sfmmu_mlist_held(pp)); 6846 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 6847 (clearflag == HAT_SYNC_ZERORM)); 6848 6849 SFMMU_STAT(sf_pagesync); 6850 6851 CPUSET_ZERO(cpuset); 6852 6853 sfmmu_pagesync_retry: 6854 6855 sfmmu_copytte(&sfhme->hme_tte, &tte); 6856 if (TTE_IS_VALID(&tte)) { 6857 hmeblkp = sfmmu_hmetohblk(sfhme); 6858 sfmmup = hblktosfmmu(hmeblkp); 6859 addr = tte_to_vaddr(hmeblkp, tte); 6860 if (clearflag == HAT_SYNC_ZERORM) { 6861 ttemod = tte; 6862 TTE_CLR_RM(&ttemod); 6863 ret = sfmmu_modifytte_try(&tte, &ttemod, 6864 &sfhme->hme_tte); 6865 if (ret < 0) { 6866 /* 6867 * cas failed and the new value is not what 6868 * we want. 6869 */ 6870 goto sfmmu_pagesync_retry; 6871 } 6872 6873 if (ret > 0) { 6874 /* we win the cas */ 6875 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6876 cpuset = sfmmup->sfmmu_cpusran; 6877 } 6878 } 6879 6880 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6881 } 6882 return (cpuset); 6883 } 6884 6885 /* 6886 * Remove write permission from a mappings to a page, so that 6887 * we can detect the next modification of it. This requires modifying 6888 * the TTE then invalidating (demap) any TLB entry using that TTE. 6889 * This code is similar to sfmmu_pagesync(). 6890 */ 6891 static cpuset_t 6892 sfmmu_pageclrwrt(struct page *pp, struct sf_hment *sfhme) 6893 { 6894 caddr_t addr; 6895 tte_t tte; 6896 tte_t ttemod; 6897 struct hme_blk *hmeblkp; 6898 int ret; 6899 sfmmu_t *sfmmup; 6900 cpuset_t cpuset; 6901 6902 ASSERT(pp != NULL); 6903 ASSERT(sfmmu_mlist_held(pp)); 6904 6905 CPUSET_ZERO(cpuset); 6906 SFMMU_STAT(sf_clrwrt); 6907 6908 retry: 6909 6910 sfmmu_copytte(&sfhme->hme_tte, &tte); 6911 if (TTE_IS_VALID(&tte) && TTE_IS_WRITABLE(&tte)) { 6912 hmeblkp = sfmmu_hmetohblk(sfhme); 6913 6914 /* 6915 * xhat mappings should never be to a VMODSORT page. 6916 */ 6917 ASSERT(hmeblkp->hblk_xhat_bit == 0); 6918 6919 sfmmup = hblktosfmmu(hmeblkp); 6920 addr = tte_to_vaddr(hmeblkp, tte); 6921 6922 ttemod = tte; 6923 TTE_CLR_WRT(&ttemod); 6924 TTE_CLR_MOD(&ttemod); 6925 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6926 6927 /* 6928 * if cas failed and the new value is not what 6929 * we want retry 6930 */ 6931 if (ret < 0) 6932 goto retry; 6933 6934 /* we win the cas */ 6935 if (ret > 0) { 6936 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6937 cpuset = sfmmup->sfmmu_cpusran; 6938 } 6939 } 6940 6941 return (cpuset); 6942 } 6943 6944 /* 6945 * Walk all mappings of a page, removing write permission and clearing the 6946 * ref/mod bits. This code is similar to hat_pagesync() 6947 */ 6948 static void 6949 hat_page_clrwrt(page_t *pp) 6950 { 6951 struct sf_hment *sfhme; 6952 struct sf_hment *tmphme = NULL; 6953 kmutex_t *pml; 6954 cpuset_t cpuset; 6955 cpuset_t tset; 6956 int index; 6957 int cons; 6958 6959 CPUSET_ZERO(cpuset); 6960 6961 pml = sfmmu_mlist_enter(pp); 6962 index = PP_MAPINDEX(pp); 6963 cons = TTE8K; 6964 retry: 6965 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6966 tmphme = sfhme->hme_next; 6967 6968 /* 6969 * If we are looking for large mappings and this hme doesn't 6970 * reach the range we are seeking, just ignore its. 6971 */ 6972 6973 if (hme_size(sfhme) < cons) 6974 continue; 6975 6976 tset = sfmmu_pageclrwrt(pp, sfhme); 6977 CPUSET_OR(cpuset, tset); 6978 } 6979 6980 while (index) { 6981 index = index >> 1; 6982 cons++; 6983 if (index & 0x1) { 6984 /* Go to leading page */ 6985 pp = PP_GROUPLEADER(pp, cons); 6986 goto retry; 6987 } 6988 } 6989 6990 xt_sync(cpuset); 6991 sfmmu_mlist_exit(pml); 6992 } 6993 6994 /* 6995 * Set the given REF/MOD/RO bits for the given page. 6996 * For a vnode with a sorted v_pages list, we need to change 6997 * the attributes and the v_pages list together under page_vnode_mutex. 6998 */ 6999 void 7000 hat_page_setattr(page_t *pp, uint_t flag) 7001 { 7002 vnode_t *vp = pp->p_vnode; 7003 page_t **listp; 7004 kmutex_t *pmtx; 7005 kmutex_t *vphm = NULL; 7006 7007 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7008 7009 /* 7010 * nothing to do if attribute already set 7011 */ 7012 if ((pp->p_nrm & flag) == flag) 7013 return; 7014 7015 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 7016 vphm = page_vnode_mutex(vp); 7017 mutex_enter(vphm); 7018 } 7019 7020 pmtx = sfmmu_page_enter(pp); 7021 pp->p_nrm |= flag; 7022 sfmmu_page_exit(pmtx); 7023 7024 if (vphm != NULL) { 7025 /* 7026 * Some File Systems examine v_pages for NULL w/o 7027 * grabbing the vphm mutex. Must not let it become NULL when 7028 * pp is the only page on the list. 7029 */ 7030 if (pp->p_vpnext != pp) { 7031 page_vpsub(&vp->v_pages, pp); 7032 if (vp->v_pages != NULL) 7033 listp = &vp->v_pages->p_vpprev->p_vpnext; 7034 else 7035 listp = &vp->v_pages; 7036 page_vpadd(listp, pp); 7037 } 7038 mutex_exit(vphm); 7039 } 7040 } 7041 7042 void 7043 hat_page_clrattr(page_t *pp, uint_t flag) 7044 { 7045 vnode_t *vp = pp->p_vnode; 7046 kmutex_t *pmtx; 7047 7048 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7049 7050 pmtx = sfmmu_page_enter(pp); 7051 7052 /* 7053 * Caller is expected to hold page's io lock for VMODSORT to work 7054 * correctly with pvn_vplist_dirty() and pvn_getdirty() when mod 7055 * bit is cleared. 7056 * We don't have assert to avoid tripping some existing third party 7057 * code. The dirty page is moved back to top of the v_page list 7058 * after IO is done in pvn_write_done(). 7059 */ 7060 pp->p_nrm &= ~flag; 7061 sfmmu_page_exit(pmtx); 7062 7063 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 7064 7065 /* 7066 * VMODSORT works by removing write permissions and getting 7067 * a fault when a page is made dirty. At this point 7068 * we need to remove write permission from all mappings 7069 * to this page. 7070 */ 7071 hat_page_clrwrt(pp); 7072 } 7073 } 7074 7075 uint_t 7076 hat_page_getattr(page_t *pp, uint_t flag) 7077 { 7078 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7079 return ((uint_t)(pp->p_nrm & flag)); 7080 } 7081 7082 /* 7083 * DEBUG kernels: verify that a kernel va<->pa translation 7084 * is safe by checking the underlying page_t is in a page 7085 * relocation-safe state. 7086 */ 7087 #ifdef DEBUG 7088 void 7089 sfmmu_check_kpfn(pfn_t pfn) 7090 { 7091 page_t *pp; 7092 int index, cons; 7093 7094 if (hat_check_vtop == 0) 7095 return; 7096 7097 if (hat_kpr_enabled == 0 || kvseg.s_base == NULL || panicstr) 7098 return; 7099 7100 pp = page_numtopp_nolock(pfn); 7101 if (!pp) 7102 return; 7103 7104 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7105 return; 7106 7107 /* 7108 * Handed a large kernel page, we dig up the root page since we 7109 * know the root page might have the lock also. 7110 */ 7111 if (pp->p_szc != 0) { 7112 index = PP_MAPINDEX(pp); 7113 cons = TTE8K; 7114 again: 7115 while (index != 0) { 7116 index >>= 1; 7117 if (index != 0) 7118 cons++; 7119 if (index & 0x1) { 7120 pp = PP_GROUPLEADER(pp, cons); 7121 goto again; 7122 } 7123 } 7124 } 7125 7126 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7127 return; 7128 7129 /* 7130 * Pages need to be locked or allocated "permanent" (either from 7131 * static_arena arena or explicitly setting PG_NORELOC when calling 7132 * page_create_va()) for VA->PA translations to be valid. 7133 */ 7134 if (!PP_ISNORELOC(pp)) 7135 panic("Illegal VA->PA translation, pp 0x%p not permanent", pp); 7136 else 7137 panic("Illegal VA->PA translation, pp 0x%p not locked", pp); 7138 } 7139 #endif /* DEBUG */ 7140 7141 /* 7142 * Returns a page frame number for a given virtual address. 7143 * Returns PFN_INVALID to indicate an invalid mapping 7144 */ 7145 pfn_t 7146 hat_getpfnum(struct hat *hat, caddr_t addr) 7147 { 7148 pfn_t pfn; 7149 tte_t tte; 7150 7151 /* 7152 * We would like to 7153 * ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 7154 * but we can't because the iommu driver will call this 7155 * routine at interrupt time and it can't grab the as lock 7156 * or it will deadlock: A thread could have the as lock 7157 * and be waiting for io. The io can't complete 7158 * because the interrupt thread is blocked trying to grab 7159 * the as lock. 7160 */ 7161 7162 ASSERT(hat->sfmmu_xhat_provider == NULL); 7163 7164 if (hat == ksfmmup) { 7165 if (IS_KMEM_VA_LARGEPAGE(addr)) { 7166 ASSERT(segkmem_lpszc > 0); 7167 pfn = sfmmu_kvaszc2pfn(addr, segkmem_lpszc); 7168 if (pfn != PFN_INVALID) { 7169 sfmmu_check_kpfn(pfn); 7170 return (pfn); 7171 } 7172 } else if (segkpm && IS_KPM_ADDR(addr)) { 7173 return (sfmmu_kpm_vatopfn(addr)); 7174 } 7175 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7176 == PFN_SUSPENDED) { 7177 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7178 } 7179 sfmmu_check_kpfn(pfn); 7180 return (pfn); 7181 } else { 7182 return (sfmmu_uvatopfn(addr, hat)); 7183 } 7184 } 7185 7186 /* 7187 * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged. 7188 * Use hat_getpfnum(kas.a_hat, ...) instead. 7189 * 7190 * We'd like to return PFN_INVALID if the mappings have underlying page_t's 7191 * but can't right now due to the fact that some software has grown to use 7192 * this interface incorrectly. So for now when the interface is misused, 7193 * return a warning to the user that in the future it won't work in the 7194 * way they're abusing it, and carry on (after disabling page relocation). 7195 */ 7196 pfn_t 7197 hat_getkpfnum(caddr_t addr) 7198 { 7199 pfn_t pfn; 7200 tte_t tte; 7201 int badcaller = 0; 7202 extern int segkmem_reloc; 7203 7204 if (segkpm && IS_KPM_ADDR(addr)) { 7205 badcaller = 1; 7206 pfn = sfmmu_kpm_vatopfn(addr); 7207 } else { 7208 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7209 == PFN_SUSPENDED) { 7210 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7211 } 7212 badcaller = pf_is_memory(pfn); 7213 } 7214 7215 if (badcaller) { 7216 /* 7217 * We can't return PFN_INVALID or the caller may panic 7218 * or corrupt the system. The only alternative is to 7219 * disable page relocation at this point for all kernel 7220 * memory. This will impact any callers of page_relocate() 7221 * such as FMA or DR. 7222 * 7223 * RFE: Add junk here to spit out an ereport so the sysadmin 7224 * can be advised that he should upgrade his device driver 7225 * so that this doesn't happen. 7226 */ 7227 hat_getkpfnum_badcall(caller()); 7228 if (hat_kpr_enabled && segkmem_reloc) { 7229 hat_kpr_enabled = 0; 7230 segkmem_reloc = 0; 7231 cmn_err(CE_WARN, "Kernel Page Relocation is DISABLED"); 7232 } 7233 } 7234 return (pfn); 7235 } 7236 7237 pfn_t 7238 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup) 7239 { 7240 struct hmehash_bucket *hmebp; 7241 hmeblk_tag hblktag; 7242 int hmeshift, hashno = 1; 7243 struct hme_blk *hmeblkp = NULL; 7244 7245 struct sf_hment *sfhmep; 7246 tte_t tte; 7247 pfn_t pfn; 7248 7249 /* support for ISM */ 7250 ism_map_t *ism_map; 7251 ism_blk_t *ism_blkp; 7252 int i; 7253 sfmmu_t *ism_hatid = NULL; 7254 sfmmu_t *locked_hatid = NULL; 7255 7256 7257 ASSERT(sfmmup != ksfmmup); 7258 SFMMU_STAT(sf_user_vtop); 7259 /* 7260 * Set ism_hatid if vaddr falls in a ISM segment. 7261 */ 7262 ism_blkp = sfmmup->sfmmu_iblk; 7263 if (ism_blkp) { 7264 sfmmu_ismhat_enter(sfmmup, 0); 7265 locked_hatid = sfmmup; 7266 } 7267 while (ism_blkp && ism_hatid == NULL) { 7268 ism_map = ism_blkp->iblk_maps; 7269 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 7270 if (vaddr >= ism_start(ism_map[i]) && 7271 vaddr < ism_end(ism_map[i])) { 7272 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 7273 vaddr = (caddr_t)(vaddr - 7274 ism_start(ism_map[i])); 7275 break; 7276 } 7277 } 7278 ism_blkp = ism_blkp->iblk_next; 7279 } 7280 if (locked_hatid) { 7281 sfmmu_ismhat_exit(locked_hatid, 0); 7282 } 7283 7284 hblktag.htag_id = sfmmup; 7285 do { 7286 hmeshift = HME_HASH_SHIFT(hashno); 7287 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 7288 hblktag.htag_rehash = hashno; 7289 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 7290 7291 SFMMU_HASH_LOCK(hmebp); 7292 7293 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 7294 if (hmeblkp != NULL) { 7295 HBLKTOHME(sfhmep, hmeblkp, vaddr); 7296 sfmmu_copytte(&sfhmep->hme_tte, &tte); 7297 if (TTE_IS_VALID(&tte)) { 7298 pfn = TTE_TO_PFN(vaddr, &tte); 7299 } else { 7300 pfn = PFN_INVALID; 7301 } 7302 SFMMU_HASH_UNLOCK(hmebp); 7303 return (pfn); 7304 } 7305 SFMMU_HASH_UNLOCK(hmebp); 7306 hashno++; 7307 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 7308 return (PFN_INVALID); 7309 } 7310 7311 7312 /* 7313 * For compatability with AT&T and later optimizations 7314 */ 7315 /* ARGSUSED */ 7316 void 7317 hat_map(struct hat *hat, caddr_t addr, size_t len, uint_t flags) 7318 { 7319 ASSERT(hat != NULL); 7320 ASSERT(hat->sfmmu_xhat_provider == NULL); 7321 } 7322 7323 /* 7324 * Return the number of mappings to a particular page. 7325 * This number is an approximation of the number of 7326 * number of people sharing the page. 7327 */ 7328 ulong_t 7329 hat_page_getshare(page_t *pp) 7330 { 7331 page_t *spp = pp; /* start page */ 7332 kmutex_t *pml; 7333 ulong_t cnt; 7334 int index, sz = TTE64K; 7335 7336 /* 7337 * We need to grab the mlist lock to make sure any outstanding 7338 * load/unloads complete. Otherwise we could return zero 7339 * even though the unload(s) hasn't finished yet. 7340 */ 7341 pml = sfmmu_mlist_enter(spp); 7342 cnt = spp->p_share; 7343 7344 #ifdef VAC 7345 if (kpm_enable) 7346 cnt += spp->p_kpmref; 7347 #endif 7348 7349 /* 7350 * If we have any large mappings, we count the number of 7351 * mappings that this large page is part of. 7352 */ 7353 index = PP_MAPINDEX(spp); 7354 index >>= 1; 7355 while (index) { 7356 pp = PP_GROUPLEADER(spp, sz); 7357 if ((index & 0x1) && pp != spp) { 7358 cnt += pp->p_share; 7359 spp = pp; 7360 } 7361 index >>= 1; 7362 sz++; 7363 } 7364 sfmmu_mlist_exit(pml); 7365 return (cnt); 7366 } 7367 7368 /* 7369 * Unload all large mappings to the pp and reset the p_szc field of every 7370 * constituent page according to the remaining mappings. 7371 * 7372 * pp must be locked SE_EXCL. Even though no other constituent pages are 7373 * locked it's legal to unload the large mappings to the pp because all 7374 * constituent pages of large locked mappings have to be locked SE_SHARED. 7375 * This means if we have SE_EXCL lock on one of constituent pages none of the 7376 * large mappings to pp are locked. 7377 * 7378 * Decrease p_szc field starting from the last constituent page and ending 7379 * with the root page. This method is used because other threads rely on the 7380 * root's p_szc to find the lock to syncronize on. After a root page_t's p_szc 7381 * is demoted then other threads will succeed in sfmmu_mlspl_enter(). This 7382 * ensures that p_szc changes of the constituent pages appears atomic for all 7383 * threads that use sfmmu_mlspl_enter() to examine p_szc field. 7384 * 7385 * This mechanism is only used for file system pages where it's not always 7386 * possible to get SE_EXCL locks on all constituent pages to demote the size 7387 * code (as is done for anonymous or kernel large pages). 7388 * 7389 * See more comments in front of sfmmu_mlspl_enter(). 7390 */ 7391 void 7392 hat_page_demote(page_t *pp) 7393 { 7394 int index; 7395 int sz; 7396 cpuset_t cpuset; 7397 int sync = 0; 7398 page_t *rootpp; 7399 struct sf_hment *sfhme; 7400 struct sf_hment *tmphme = NULL; 7401 struct hme_blk *hmeblkp; 7402 uint_t pszc; 7403 page_t *lastpp; 7404 cpuset_t tset; 7405 pgcnt_t npgs; 7406 kmutex_t *pml; 7407 kmutex_t *pmtx = NULL; 7408 7409 ASSERT(PAGE_EXCL(pp)); 7410 ASSERT(!PP_ISFREE(pp)); 7411 ASSERT(page_szc_lock_assert(pp)); 7412 pml = sfmmu_mlist_enter(pp); 7413 7414 pszc = pp->p_szc; 7415 if (pszc == 0) { 7416 goto out; 7417 } 7418 7419 index = PP_MAPINDEX(pp) >> 1; 7420 7421 if (index) { 7422 CPUSET_ZERO(cpuset); 7423 sz = TTE64K; 7424 sync = 1; 7425 } 7426 7427 while (index) { 7428 if (!(index & 0x1)) { 7429 index >>= 1; 7430 sz++; 7431 continue; 7432 } 7433 ASSERT(sz <= pszc); 7434 rootpp = PP_GROUPLEADER(pp, sz); 7435 for (sfhme = rootpp->p_mapping; sfhme; sfhme = tmphme) { 7436 tmphme = sfhme->hme_next; 7437 hmeblkp = sfmmu_hmetohblk(sfhme); 7438 if (hme_size(sfhme) != sz) { 7439 continue; 7440 } 7441 if (hmeblkp->hblk_xhat_bit) { 7442 cmn_err(CE_PANIC, 7443 "hat_page_demote: xhat hmeblk"); 7444 } 7445 tset = sfmmu_pageunload(rootpp, sfhme, sz); 7446 CPUSET_OR(cpuset, tset); 7447 } 7448 if (index >>= 1) { 7449 sz++; 7450 } 7451 } 7452 7453 ASSERT(!PP_ISMAPPED_LARGE(pp)); 7454 7455 if (sync) { 7456 xt_sync(cpuset); 7457 #ifdef VAC 7458 if (PP_ISTNC(pp)) { 7459 conv_tnc(rootpp, sz); 7460 } 7461 #endif /* VAC */ 7462 } 7463 7464 pmtx = sfmmu_page_enter(pp); 7465 7466 ASSERT(pp->p_szc == pszc); 7467 rootpp = PP_PAGEROOT(pp); 7468 ASSERT(rootpp->p_szc == pszc); 7469 lastpp = PP_PAGENEXT_N(rootpp, TTEPAGES(pszc) - 1); 7470 7471 while (lastpp != rootpp) { 7472 sz = PP_MAPINDEX(lastpp) ? fnd_mapping_sz(lastpp) : 0; 7473 ASSERT(sz < pszc); 7474 npgs = (sz == 0) ? 1 : TTEPAGES(sz); 7475 ASSERT(P2PHASE(lastpp->p_pagenum, npgs) == npgs - 1); 7476 while (--npgs > 0) { 7477 lastpp->p_szc = (uchar_t)sz; 7478 lastpp = PP_PAGEPREV(lastpp); 7479 } 7480 if (sz) { 7481 /* 7482 * make sure before current root's pszc 7483 * is updated all updates to constituent pages pszc 7484 * fields are globally visible. 7485 */ 7486 membar_producer(); 7487 } 7488 lastpp->p_szc = sz; 7489 ASSERT(IS_P2ALIGNED(lastpp->p_pagenum, TTEPAGES(sz))); 7490 if (lastpp != rootpp) { 7491 lastpp = PP_PAGEPREV(lastpp); 7492 } 7493 } 7494 if (sz == 0) { 7495 /* the loop above doesn't cover this case */ 7496 rootpp->p_szc = 0; 7497 } 7498 out: 7499 ASSERT(pp->p_szc == 0); 7500 if (pmtx != NULL) { 7501 sfmmu_page_exit(pmtx); 7502 } 7503 sfmmu_mlist_exit(pml); 7504 } 7505 7506 /* 7507 * Refresh the HAT ismttecnt[] element for size szc. 7508 * Caller must have set ISM busy flag to prevent mapping 7509 * lists from changing while we're traversing them. 7510 */ 7511 pgcnt_t 7512 ism_tsb_entries(sfmmu_t *sfmmup, int szc) 7513 { 7514 ism_blk_t *ism_blkp = sfmmup->sfmmu_iblk; 7515 ism_map_t *ism_map; 7516 pgcnt_t npgs = 0; 7517 int j; 7518 7519 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 7520 for (; ism_blkp != NULL; ism_blkp = ism_blkp->iblk_next) { 7521 ism_map = ism_blkp->iblk_maps; 7522 for (j = 0; ism_map[j].imap_ismhat && j < ISM_MAP_SLOTS; j++) 7523 npgs += ism_map[j].imap_ismhat->sfmmu_ttecnt[szc]; 7524 } 7525 sfmmup->sfmmu_ismttecnt[szc] = npgs; 7526 return (npgs); 7527 } 7528 7529 /* 7530 * Yield the memory claim requirement for an address space. 7531 * 7532 * This is currently implemented as the number of bytes that have active 7533 * hardware translations that have page structures. Therefore, it can 7534 * underestimate the traditional resident set size, eg, if the 7535 * physical page is present and the hardware translation is missing; 7536 * and it can overestimate the rss, eg, if there are active 7537 * translations to a frame buffer with page structs. 7538 * Also, it does not take sharing into account. 7539 * 7540 * Note that we don't acquire locks here since this function is most often 7541 * called from the clock thread. 7542 */ 7543 size_t 7544 hat_get_mapped_size(struct hat *hat) 7545 { 7546 size_t assize = 0; 7547 int i; 7548 7549 if (hat == NULL) 7550 return (0); 7551 7552 ASSERT(hat->sfmmu_xhat_provider == NULL); 7553 7554 for (i = 0; i < mmu_page_sizes; i++) 7555 assize += (pgcnt_t)hat->sfmmu_ttecnt[i] * TTEBYTES(i); 7556 7557 if (hat->sfmmu_iblk == NULL) 7558 return (assize); 7559 7560 for (i = 0; i < mmu_page_sizes; i++) 7561 assize += (pgcnt_t)hat->sfmmu_ismttecnt[i] * TTEBYTES(i); 7562 7563 return (assize); 7564 } 7565 7566 int 7567 hat_stats_enable(struct hat *hat) 7568 { 7569 hatlock_t *hatlockp; 7570 7571 ASSERT(hat->sfmmu_xhat_provider == NULL); 7572 7573 hatlockp = sfmmu_hat_enter(hat); 7574 hat->sfmmu_rmstat++; 7575 sfmmu_hat_exit(hatlockp); 7576 return (1); 7577 } 7578 7579 void 7580 hat_stats_disable(struct hat *hat) 7581 { 7582 hatlock_t *hatlockp; 7583 7584 ASSERT(hat->sfmmu_xhat_provider == NULL); 7585 7586 hatlockp = sfmmu_hat_enter(hat); 7587 hat->sfmmu_rmstat--; 7588 sfmmu_hat_exit(hatlockp); 7589 } 7590 7591 /* 7592 * Routines for entering or removing ourselves from the 7593 * ism_hat's mapping list. 7594 */ 7595 static void 7596 iment_add(struct ism_ment *iment, struct hat *ism_hat) 7597 { 7598 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7599 7600 iment->iment_prev = NULL; 7601 iment->iment_next = ism_hat->sfmmu_iment; 7602 if (ism_hat->sfmmu_iment) { 7603 ism_hat->sfmmu_iment->iment_prev = iment; 7604 } 7605 ism_hat->sfmmu_iment = iment; 7606 } 7607 7608 static void 7609 iment_sub(struct ism_ment *iment, struct hat *ism_hat) 7610 { 7611 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7612 7613 if (ism_hat->sfmmu_iment == NULL) { 7614 panic("ism map entry remove - no entries"); 7615 } 7616 7617 if (iment->iment_prev) { 7618 ASSERT(ism_hat->sfmmu_iment != iment); 7619 iment->iment_prev->iment_next = iment->iment_next; 7620 } else { 7621 ASSERT(ism_hat->sfmmu_iment == iment); 7622 ism_hat->sfmmu_iment = iment->iment_next; 7623 } 7624 7625 if (iment->iment_next) { 7626 iment->iment_next->iment_prev = iment->iment_prev; 7627 } 7628 7629 /* 7630 * zero out the entry 7631 */ 7632 iment->iment_next = NULL; 7633 iment->iment_prev = NULL; 7634 iment->iment_hat = NULL; 7635 } 7636 7637 /* 7638 * Hat_share()/unshare() return an (non-zero) error 7639 * when saddr and daddr are not properly aligned. 7640 * 7641 * The top level mapping element determines the alignment 7642 * requirement for saddr and daddr, depending on different 7643 * architectures. 7644 * 7645 * When hat_share()/unshare() are not supported, 7646 * HATOP_SHARE()/UNSHARE() return 0 7647 */ 7648 int 7649 hat_share(struct hat *sfmmup, caddr_t addr, 7650 struct hat *ism_hatid, caddr_t sptaddr, size_t len, uint_t ismszc) 7651 { 7652 ism_blk_t *ism_blkp; 7653 ism_blk_t *new_iblk; 7654 ism_map_t *ism_map; 7655 ism_ment_t *ism_ment; 7656 int i, added; 7657 hatlock_t *hatlockp; 7658 int reload_mmu = 0; 7659 uint_t ismshift = page_get_shift(ismszc); 7660 size_t ismpgsz = page_get_pagesize(ismszc); 7661 uint_t ismmask = (uint_t)ismpgsz - 1; 7662 size_t sh_size = ISM_SHIFT(ismshift, len); 7663 ushort_t ismhatflag; 7664 7665 #ifdef DEBUG 7666 caddr_t eaddr = addr + len; 7667 #endif /* DEBUG */ 7668 7669 ASSERT(ism_hatid != NULL && sfmmup != NULL); 7670 ASSERT(sptaddr == ISMID_STARTADDR); 7671 /* 7672 * Check the alignment. 7673 */ 7674 if (!ISM_ALIGNED(ismshift, addr) || !ISM_ALIGNED(ismshift, sptaddr)) 7675 return (EINVAL); 7676 7677 /* 7678 * Check size alignment. 7679 */ 7680 if (!ISM_ALIGNED(ismshift, len)) 7681 return (EINVAL); 7682 7683 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 7684 7685 /* 7686 * Allocate ism_ment for the ism_hat's mapping list, and an 7687 * ism map blk in case we need one. We must do our 7688 * allocations before acquiring locks to prevent a deadlock 7689 * in the kmem allocator on the mapping list lock. 7690 */ 7691 new_iblk = kmem_cache_alloc(ism_blk_cache, KM_SLEEP); 7692 ism_ment = kmem_cache_alloc(ism_ment_cache, KM_SLEEP); 7693 7694 /* 7695 * Serialize ISM mappings with the ISM busy flag, and also the 7696 * trap handlers. 7697 */ 7698 sfmmu_ismhat_enter(sfmmup, 0); 7699 7700 /* 7701 * Allocate an ism map blk if necessary. 7702 */ 7703 if (sfmmup->sfmmu_iblk == NULL) { 7704 sfmmup->sfmmu_iblk = new_iblk; 7705 bzero(new_iblk, sizeof (*new_iblk)); 7706 new_iblk->iblk_nextpa = (uint64_t)-1; 7707 membar_stst(); /* make sure next ptr visible to all CPUs */ 7708 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk); 7709 reload_mmu = 1; 7710 new_iblk = NULL; 7711 } 7712 7713 #ifdef DEBUG 7714 /* 7715 * Make sure mapping does not already exist. 7716 */ 7717 ism_blkp = sfmmup->sfmmu_iblk; 7718 while (ism_blkp) { 7719 ism_map = ism_blkp->iblk_maps; 7720 for (i = 0; i < ISM_MAP_SLOTS && ism_map[i].imap_ismhat; i++) { 7721 if ((addr >= ism_start(ism_map[i]) && 7722 addr < ism_end(ism_map[i])) || 7723 eaddr > ism_start(ism_map[i]) && 7724 eaddr <= ism_end(ism_map[i])) { 7725 panic("sfmmu_share: Already mapped!"); 7726 } 7727 } 7728 ism_blkp = ism_blkp->iblk_next; 7729 } 7730 #endif /* DEBUG */ 7731 7732 ASSERT(ismszc >= TTE4M); 7733 if (ismszc == TTE4M) { 7734 ismhatflag = HAT_4M_FLAG; 7735 } else if (ismszc == TTE32M) { 7736 ismhatflag = HAT_32M_FLAG; 7737 } else if (ismszc == TTE256M) { 7738 ismhatflag = HAT_256M_FLAG; 7739 } 7740 /* 7741 * Add mapping to first available mapping slot. 7742 */ 7743 ism_blkp = sfmmup->sfmmu_iblk; 7744 added = 0; 7745 while (!added) { 7746 ism_map = ism_blkp->iblk_maps; 7747 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7748 if (ism_map[i].imap_ismhat == NULL) { 7749 7750 ism_map[i].imap_ismhat = ism_hatid; 7751 ism_map[i].imap_vb_shift = (ushort_t)ismshift; 7752 ism_map[i].imap_hatflags = ismhatflag; 7753 ism_map[i].imap_sz_mask = ismmask; 7754 /* 7755 * imap_seg is checked in ISM_CHECK to see if 7756 * non-NULL, then other info assumed valid. 7757 */ 7758 membar_stst(); 7759 ism_map[i].imap_seg = (uintptr_t)addr | sh_size; 7760 ism_map[i].imap_ment = ism_ment; 7761 7762 /* 7763 * Now add ourselves to the ism_hat's 7764 * mapping list. 7765 */ 7766 ism_ment->iment_hat = sfmmup; 7767 ism_ment->iment_base_va = addr; 7768 ism_hatid->sfmmu_ismhat = 1; 7769 ism_hatid->sfmmu_flags = 0; 7770 mutex_enter(&ism_mlist_lock); 7771 iment_add(ism_ment, ism_hatid); 7772 mutex_exit(&ism_mlist_lock); 7773 added = 1; 7774 break; 7775 } 7776 } 7777 if (!added && ism_blkp->iblk_next == NULL) { 7778 ism_blkp->iblk_next = new_iblk; 7779 new_iblk = NULL; 7780 bzero(ism_blkp->iblk_next, 7781 sizeof (*ism_blkp->iblk_next)); 7782 ism_blkp->iblk_next->iblk_nextpa = (uint64_t)-1; 7783 membar_stst(); 7784 ism_blkp->iblk_nextpa = 7785 va_to_pa((caddr_t)ism_blkp->iblk_next); 7786 } 7787 ism_blkp = ism_blkp->iblk_next; 7788 } 7789 7790 /* 7791 * Update our counters for this sfmmup's ism mappings. 7792 */ 7793 for (i = 0; i <= ismszc; i++) { 7794 if (!(disable_ism_large_pages & (1 << i))) 7795 (void) ism_tsb_entries(sfmmup, i); 7796 } 7797 7798 hatlockp = sfmmu_hat_enter(sfmmup); 7799 7800 /* 7801 * For ISM and DISM we do not support 512K pages, so we only 7802 * only search the 4M and 8K/64K hashes for 4 pagesize cpus, and search 7803 * the 256M or 32M, and 4M and 8K/64K hashes for 6 pagesize cpus. 7804 */ 7805 ASSERT((disable_ism_large_pages & (1 << TTE512K)) != 0); 7806 7807 if (ismszc > TTE4M && !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) 7808 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7809 7810 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) 7811 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 7812 7813 /* 7814 * If we updated the ismblkpa for this HAT or we need 7815 * to start searching the 256M or 32M or 4M hash, we must 7816 * make sure all CPUs running this process reload their 7817 * tsbmiss area. Otherwise they will fail to load the mappings 7818 * in the tsbmiss handler and will loop calling pagefault(). 7819 */ 7820 switch (ismszc) { 7821 case TTE256M: 7822 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 7823 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 7824 sfmmu_sync_mmustate(sfmmup); 7825 } 7826 break; 7827 case TTE32M: 7828 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 7829 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 7830 sfmmu_sync_mmustate(sfmmup); 7831 } 7832 break; 7833 case TTE4M: 7834 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 7835 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7836 sfmmu_sync_mmustate(sfmmup); 7837 } 7838 break; 7839 default: 7840 break; 7841 } 7842 7843 /* 7844 * Now we can drop the locks. 7845 */ 7846 sfmmu_ismhat_exit(sfmmup, 1); 7847 sfmmu_hat_exit(hatlockp); 7848 7849 /* 7850 * Free up ismblk if we didn't use it. 7851 */ 7852 if (new_iblk != NULL) 7853 kmem_cache_free(ism_blk_cache, new_iblk); 7854 7855 /* 7856 * Check TSB and TLB page sizes. 7857 */ 7858 sfmmu_check_page_sizes(sfmmup, 1); 7859 7860 return (0); 7861 } 7862 7863 /* 7864 * hat_unshare removes exactly one ism_map from 7865 * this process's as. It expects multiple calls 7866 * to hat_unshare for multiple shm segments. 7867 */ 7868 void 7869 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) 7870 { 7871 ism_map_t *ism_map; 7872 ism_ment_t *free_ment = NULL; 7873 ism_blk_t *ism_blkp; 7874 struct hat *ism_hatid; 7875 int found, i; 7876 hatlock_t *hatlockp; 7877 struct tsb_info *tsbinfo; 7878 uint_t ismshift = page_get_shift(ismszc); 7879 size_t sh_size = ISM_SHIFT(ismshift, len); 7880 7881 ASSERT(ISM_ALIGNED(ismshift, addr)); 7882 ASSERT(ISM_ALIGNED(ismshift, len)); 7883 ASSERT(sfmmup != NULL); 7884 ASSERT(sfmmup != ksfmmup); 7885 7886 if (sfmmup->sfmmu_xhat_provider) { 7887 XHAT_UNSHARE(sfmmup, addr, len); 7888 return; 7889 } else { 7890 /* 7891 * This must be a CPU HAT. If the address space has 7892 * XHATs attached, inform all XHATs that ISM segment 7893 * is going away 7894 */ 7895 ASSERT(sfmmup->sfmmu_as != NULL); 7896 if (sfmmup->sfmmu_as->a_xhat != NULL) 7897 xhat_unshare_all(sfmmup->sfmmu_as, addr, len); 7898 } 7899 7900 /* 7901 * Make sure that during the entire time ISM mappings are removed, 7902 * the trap handlers serialize behind us, and that no one else 7903 * can be mucking with ISM mappings. This also lets us get away 7904 * with not doing expensive cross calls to flush the TLB -- we 7905 * just discard the context, flush the entire TSB, and call it 7906 * a day. 7907 */ 7908 sfmmu_ismhat_enter(sfmmup, 0); 7909 7910 /* 7911 * Remove the mapping. 7912 * 7913 * We can't have any holes in the ism map. 7914 * The tsb miss code while searching the ism map will 7915 * stop on an empty map slot. So we must move 7916 * everyone past the hole up 1 if any. 7917 * 7918 * Also empty ism map blks are not freed until the 7919 * process exits. This is to prevent a MT race condition 7920 * between sfmmu_unshare() and sfmmu_tsbmiss_exception(). 7921 */ 7922 found = 0; 7923 ism_blkp = sfmmup->sfmmu_iblk; 7924 while (!found && ism_blkp) { 7925 ism_map = ism_blkp->iblk_maps; 7926 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7927 if (addr == ism_start(ism_map[i]) && 7928 sh_size == (size_t)(ism_size(ism_map[i]))) { 7929 found = 1; 7930 break; 7931 } 7932 } 7933 if (!found) 7934 ism_blkp = ism_blkp->iblk_next; 7935 } 7936 7937 if (found) { 7938 ism_hatid = ism_map[i].imap_ismhat; 7939 ASSERT(ism_hatid != NULL); 7940 ASSERT(ism_hatid->sfmmu_ismhat == 1); 7941 7942 /* 7943 * First remove ourselves from the ism mapping list. 7944 */ 7945 mutex_enter(&ism_mlist_lock); 7946 iment_sub(ism_map[i].imap_ment, ism_hatid); 7947 mutex_exit(&ism_mlist_lock); 7948 free_ment = ism_map[i].imap_ment; 7949 7950 /* 7951 * Now gurantee that any other cpu 7952 * that tries to process an ISM miss 7953 * will go to tl=0. 7954 */ 7955 hatlockp = sfmmu_hat_enter(sfmmup); 7956 7957 sfmmu_invalidate_ctx(sfmmup); 7958 7959 sfmmu_hat_exit(hatlockp); 7960 7961 /* 7962 * We delete the ism map by copying 7963 * the next map over the current one. 7964 * We will take the next one in the maps 7965 * array or from the next ism_blk. 7966 */ 7967 while (ism_blkp) { 7968 ism_map = ism_blkp->iblk_maps; 7969 while (i < (ISM_MAP_SLOTS - 1)) { 7970 ism_map[i] = ism_map[i + 1]; 7971 i++; 7972 } 7973 /* i == (ISM_MAP_SLOTS - 1) */ 7974 ism_blkp = ism_blkp->iblk_next; 7975 if (ism_blkp) { 7976 ism_map[i] = ism_blkp->iblk_maps[0]; 7977 i = 0; 7978 } else { 7979 ism_map[i].imap_seg = 0; 7980 ism_map[i].imap_vb_shift = 0; 7981 ism_map[i].imap_hatflags = 0; 7982 ism_map[i].imap_sz_mask = 0; 7983 ism_map[i].imap_ismhat = NULL; 7984 ism_map[i].imap_ment = NULL; 7985 } 7986 } 7987 7988 /* 7989 * Now flush entire TSB for the process, since 7990 * demapping page by page can be too expensive. 7991 * We don't have to flush the TLB here anymore 7992 * since we switch to a new TLB ctx instead. 7993 * Also, there is no need to flush if the process 7994 * is exiting since the TSB will be freed later. 7995 */ 7996 if (!sfmmup->sfmmu_free) { 7997 hatlockp = sfmmu_hat_enter(sfmmup); 7998 for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL; 7999 tsbinfo = tsbinfo->tsb_next) { 8000 if (tsbinfo->tsb_flags & TSB_SWAPPED) 8001 continue; 8002 sfmmu_inv_tsb(tsbinfo->tsb_va, 8003 TSB_BYTES(tsbinfo->tsb_szc)); 8004 } 8005 sfmmu_hat_exit(hatlockp); 8006 } 8007 } 8008 8009 /* 8010 * Update our counters for this sfmmup's ism mappings. 8011 */ 8012 for (i = 0; i <= ismszc; i++) { 8013 if (!(disable_ism_large_pages & (1 << i))) 8014 (void) ism_tsb_entries(sfmmup, i); 8015 } 8016 8017 sfmmu_ismhat_exit(sfmmup, 0); 8018 8019 /* 8020 * We must do our freeing here after dropping locks 8021 * to prevent a deadlock in the kmem allocator on the 8022 * mapping list lock. 8023 */ 8024 if (free_ment != NULL) 8025 kmem_cache_free(ism_ment_cache, free_ment); 8026 8027 /* 8028 * Check TSB and TLB page sizes if the process isn't exiting. 8029 */ 8030 if (!sfmmup->sfmmu_free) 8031 sfmmu_check_page_sizes(sfmmup, 0); 8032 } 8033 8034 /* ARGSUSED */ 8035 static int 8036 sfmmu_idcache_constructor(void *buf, void *cdrarg, int kmflags) 8037 { 8038 /* void *buf is sfmmu_t pointer */ 8039 return (0); 8040 } 8041 8042 /* ARGSUSED */ 8043 static void 8044 sfmmu_idcache_destructor(void *buf, void *cdrarg) 8045 { 8046 /* void *buf is sfmmu_t pointer */ 8047 } 8048 8049 /* 8050 * setup kmem hmeblks by bzeroing all members and initializing the nextpa 8051 * field to be the pa of this hmeblk 8052 */ 8053 /* ARGSUSED */ 8054 static int 8055 sfmmu_hblkcache_constructor(void *buf, void *cdrarg, int kmflags) 8056 { 8057 struct hme_blk *hmeblkp; 8058 8059 bzero(buf, (size_t)cdrarg); 8060 hmeblkp = (struct hme_blk *)buf; 8061 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); 8062 8063 #ifdef HBLK_TRACE 8064 mutex_init(&hmeblkp->hblk_audit_lock, NULL, MUTEX_DEFAULT, NULL); 8065 #endif /* HBLK_TRACE */ 8066 8067 return (0); 8068 } 8069 8070 /* ARGSUSED */ 8071 static void 8072 sfmmu_hblkcache_destructor(void *buf, void *cdrarg) 8073 { 8074 8075 #ifdef HBLK_TRACE 8076 8077 struct hme_blk *hmeblkp; 8078 8079 hmeblkp = (struct hme_blk *)buf; 8080 mutex_destroy(&hmeblkp->hblk_audit_lock); 8081 8082 #endif /* HBLK_TRACE */ 8083 } 8084 8085 #define SFMMU_CACHE_RECLAIM_SCAN_RATIO 8 8086 static int sfmmu_cache_reclaim_scan_ratio = SFMMU_CACHE_RECLAIM_SCAN_RATIO; 8087 /* 8088 * The kmem allocator will callback into our reclaim routine when the system 8089 * is running low in memory. We traverse the hash and free up all unused but 8090 * still cached hme_blks. We also traverse the free list and free them up 8091 * as well. 8092 */ 8093 /*ARGSUSED*/ 8094 static void 8095 sfmmu_hblkcache_reclaim(void *cdrarg) 8096 { 8097 int i; 8098 uint64_t hblkpa, prevpa, nx_pa; 8099 struct hmehash_bucket *hmebp; 8100 struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL; 8101 static struct hmehash_bucket *uhmehash_reclaim_hand; 8102 static struct hmehash_bucket *khmehash_reclaim_hand; 8103 struct hme_blk *list = NULL; 8104 8105 hmebp = uhmehash_reclaim_hand; 8106 if (hmebp == NULL || hmebp > &uhme_hash[UHMEHASH_SZ]) 8107 uhmehash_reclaim_hand = hmebp = uhme_hash; 8108 uhmehash_reclaim_hand += UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8109 8110 for (i = UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8111 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8112 hmeblkp = hmebp->hmeblkp; 8113 hblkpa = hmebp->hmeh_nextpa; 8114 prevpa = 0; 8115 pr_hblk = NULL; 8116 while (hmeblkp) { 8117 nx_hblk = hmeblkp->hblk_next; 8118 nx_pa = hmeblkp->hblk_nextpa; 8119 if (!hmeblkp->hblk_vcnt && 8120 !hmeblkp->hblk_hmecnt) { 8121 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8122 prevpa, pr_hblk); 8123 sfmmu_hblk_free(hmebp, hmeblkp, 8124 hblkpa, &list); 8125 } else { 8126 pr_hblk = hmeblkp; 8127 prevpa = hblkpa; 8128 } 8129 hmeblkp = nx_hblk; 8130 hblkpa = nx_pa; 8131 } 8132 SFMMU_HASH_UNLOCK(hmebp); 8133 } 8134 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 8135 hmebp = uhme_hash; 8136 } 8137 8138 hmebp = khmehash_reclaim_hand; 8139 if (hmebp == NULL || hmebp > &khme_hash[KHMEHASH_SZ]) 8140 khmehash_reclaim_hand = hmebp = khme_hash; 8141 khmehash_reclaim_hand += KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8142 8143 for (i = KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8144 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8145 hmeblkp = hmebp->hmeblkp; 8146 hblkpa = hmebp->hmeh_nextpa; 8147 prevpa = 0; 8148 pr_hblk = NULL; 8149 while (hmeblkp) { 8150 nx_hblk = hmeblkp->hblk_next; 8151 nx_pa = hmeblkp->hblk_nextpa; 8152 if (!hmeblkp->hblk_vcnt && 8153 !hmeblkp->hblk_hmecnt) { 8154 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8155 prevpa, pr_hblk); 8156 sfmmu_hblk_free(hmebp, hmeblkp, 8157 hblkpa, &list); 8158 } else { 8159 pr_hblk = hmeblkp; 8160 prevpa = hblkpa; 8161 } 8162 hmeblkp = nx_hblk; 8163 hblkpa = nx_pa; 8164 } 8165 SFMMU_HASH_UNLOCK(hmebp); 8166 } 8167 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 8168 hmebp = khme_hash; 8169 } 8170 sfmmu_hblks_list_purge(&list); 8171 } 8172 8173 /* 8174 * sfmmu_get_ppvcolor should become a vm_machdep or hatop interface. 8175 * same goes for sfmmu_get_addrvcolor(). 8176 * 8177 * This function will return the virtual color for the specified page. The 8178 * virtual color corresponds to this page current mapping or its last mapping. 8179 * It is used by memory allocators to choose addresses with the correct 8180 * alignment so vac consistency is automatically maintained. If the page 8181 * has no color it returns -1. 8182 */ 8183 /*ARGSUSED*/ 8184 int 8185 sfmmu_get_ppvcolor(struct page *pp) 8186 { 8187 #ifdef VAC 8188 int color; 8189 8190 if (!(cache & CACHE_VAC) || PP_NEWPAGE(pp)) { 8191 return (-1); 8192 } 8193 color = PP_GET_VCOLOR(pp); 8194 ASSERT(color < mmu_btop(shm_alignment)); 8195 return (color); 8196 #else 8197 return (-1); 8198 #endif /* VAC */ 8199 } 8200 8201 /* 8202 * This function will return the desired alignment for vac consistency 8203 * (vac color) given a virtual address. If no vac is present it returns -1. 8204 */ 8205 /*ARGSUSED*/ 8206 int 8207 sfmmu_get_addrvcolor(caddr_t vaddr) 8208 { 8209 #ifdef VAC 8210 if (cache & CACHE_VAC) { 8211 return (addr_to_vcolor(vaddr)); 8212 } else { 8213 return (-1); 8214 } 8215 #else 8216 return (-1); 8217 #endif /* VAC */ 8218 } 8219 8220 #ifdef VAC 8221 /* 8222 * Check for conflicts. 8223 * A conflict exists if the new and existent mappings do not match in 8224 * their "shm_alignment fields. If conflicts exist, the existant mappings 8225 * are flushed unless one of them is locked. If one of them is locked, then 8226 * the mappings are flushed and converted to non-cacheable mappings. 8227 */ 8228 static void 8229 sfmmu_vac_conflict(struct hat *hat, caddr_t addr, page_t *pp) 8230 { 8231 struct hat *tmphat; 8232 struct sf_hment *sfhmep, *tmphme = NULL; 8233 struct hme_blk *hmeblkp; 8234 int vcolor; 8235 tte_t tte; 8236 8237 ASSERT(sfmmu_mlist_held(pp)); 8238 ASSERT(!PP_ISNC(pp)); /* page better be cacheable */ 8239 8240 vcolor = addr_to_vcolor(addr); 8241 if (PP_NEWPAGE(pp)) { 8242 PP_SET_VCOLOR(pp, vcolor); 8243 return; 8244 } 8245 8246 if (PP_GET_VCOLOR(pp) == vcolor) { 8247 return; 8248 } 8249 8250 if (!PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp)) { 8251 /* 8252 * Previous user of page had a different color 8253 * but since there are no current users 8254 * we just flush the cache and change the color. 8255 */ 8256 SFMMU_STAT(sf_pgcolor_conflict); 8257 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8258 PP_SET_VCOLOR(pp, vcolor); 8259 return; 8260 } 8261 8262 /* 8263 * If we get here we have a vac conflict with a current 8264 * mapping. VAC conflict policy is as follows. 8265 * - The default is to unload the other mappings unless: 8266 * - If we have a large mapping we uncache the page. 8267 * We need to uncache the rest of the large page too. 8268 * - If any of the mappings are locked we uncache the page. 8269 * - If the requested mapping is inconsistent 8270 * with another mapping and that mapping 8271 * is in the same address space we have to 8272 * make it non-cached. The default thing 8273 * to do is unload the inconsistent mapping 8274 * but if they are in the same address space 8275 * we run the risk of unmapping the pc or the 8276 * stack which we will use as we return to the user, 8277 * in which case we can then fault on the thing 8278 * we just unloaded and get into an infinite loop. 8279 */ 8280 if (PP_ISMAPPED_LARGE(pp)) { 8281 int sz; 8282 8283 /* 8284 * Existing mapping is for big pages. We don't unload 8285 * existing big mappings to satisfy new mappings. 8286 * Always convert all mappings to TNC. 8287 */ 8288 sz = fnd_mapping_sz(pp); 8289 pp = PP_GROUPLEADER(pp, sz); 8290 SFMMU_STAT_ADD(sf_uncache_conflict, TTEPAGES(sz)); 8291 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 8292 TTEPAGES(sz)); 8293 8294 return; 8295 } 8296 8297 /* 8298 * check if any mapping is in same as or if it is locked 8299 * since in that case we need to uncache. 8300 */ 8301 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8302 tmphme = sfhmep->hme_next; 8303 hmeblkp = sfmmu_hmetohblk(sfhmep); 8304 if (hmeblkp->hblk_xhat_bit) 8305 continue; 8306 tmphat = hblktosfmmu(hmeblkp); 8307 sfmmu_copytte(&sfhmep->hme_tte, &tte); 8308 ASSERT(TTE_IS_VALID(&tte)); 8309 if ((tmphat == hat) || hmeblkp->hblk_lckcnt) { 8310 /* 8311 * We have an uncache conflict 8312 */ 8313 SFMMU_STAT(sf_uncache_conflict); 8314 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1); 8315 return; 8316 } 8317 } 8318 8319 /* 8320 * We have an unload conflict 8321 * We have already checked for LARGE mappings, therefore 8322 * the remaining mapping(s) must be TTE8K. 8323 */ 8324 SFMMU_STAT(sf_unload_conflict); 8325 8326 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8327 tmphme = sfhmep->hme_next; 8328 hmeblkp = sfmmu_hmetohblk(sfhmep); 8329 if (hmeblkp->hblk_xhat_bit) 8330 continue; 8331 (void) sfmmu_pageunload(pp, sfhmep, TTE8K); 8332 } 8333 8334 if (PP_ISMAPPED_KPM(pp)) 8335 sfmmu_kpm_vac_unload(pp, addr); 8336 8337 /* 8338 * Unloads only do TLB flushes so we need to flush the 8339 * cache here. 8340 */ 8341 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8342 PP_SET_VCOLOR(pp, vcolor); 8343 } 8344 8345 /* 8346 * Whenever a mapping is unloaded and the page is in TNC state, 8347 * we see if the page can be made cacheable again. 'pp' is 8348 * the page that we just unloaded a mapping from, the size 8349 * of mapping that was unloaded is 'ottesz'. 8350 * Remark: 8351 * The recache policy for mpss pages can leave a performance problem 8352 * under the following circumstances: 8353 * . A large page in uncached mode has just been unmapped. 8354 * . All constituent pages are TNC due to a conflicting small mapping. 8355 * . There are many other, non conflicting, small mappings around for 8356 * a lot of the constituent pages. 8357 * . We're called w/ the "old" groupleader page and the old ottesz, 8358 * but this is irrelevant, since we're no more "PP_ISMAPPED_LARGE", so 8359 * we end up w/ TTE8K or npages == 1. 8360 * . We call tst_tnc w/ the old groupleader only, and if there is no 8361 * conflict, we re-cache only this page. 8362 * . All other small mappings are not checked and will be left in TNC mode. 8363 * The problem is not very serious because: 8364 * . mpss is actually only defined for heap and stack, so the probability 8365 * is not very high that a large page mapping exists in parallel to a small 8366 * one (this is possible, but seems to be bad programming style in the 8367 * appl). 8368 * . The problem gets a little bit more serious, when those TNC pages 8369 * have to be mapped into kernel space, e.g. for networking. 8370 * . When VAC alias conflicts occur in applications, this is regarded 8371 * as an application bug. So if kstat's show them, the appl should 8372 * be changed anyway. 8373 */ 8374 void 8375 conv_tnc(page_t *pp, int ottesz) 8376 { 8377 int cursz, dosz; 8378 pgcnt_t curnpgs, dopgs; 8379 pgcnt_t pg64k; 8380 page_t *pp2; 8381 8382 /* 8383 * Determine how big a range we check for TNC and find 8384 * leader page. cursz is the size of the biggest 8385 * mapping that still exist on 'pp'. 8386 */ 8387 if (PP_ISMAPPED_LARGE(pp)) { 8388 cursz = fnd_mapping_sz(pp); 8389 } else { 8390 cursz = TTE8K; 8391 } 8392 8393 if (ottesz >= cursz) { 8394 dosz = ottesz; 8395 pp2 = pp; 8396 } else { 8397 dosz = cursz; 8398 pp2 = PP_GROUPLEADER(pp, dosz); 8399 } 8400 8401 pg64k = TTEPAGES(TTE64K); 8402 dopgs = TTEPAGES(dosz); 8403 8404 ASSERT(dopgs == 1 || ((dopgs & (pg64k - 1)) == 0)); 8405 8406 while (dopgs != 0) { 8407 curnpgs = TTEPAGES(cursz); 8408 if (tst_tnc(pp2, curnpgs)) { 8409 SFMMU_STAT_ADD(sf_recache, curnpgs); 8410 sfmmu_page_cache_array(pp2, HAT_CACHE, CACHE_NO_FLUSH, 8411 curnpgs); 8412 } 8413 8414 ASSERT(dopgs >= curnpgs); 8415 dopgs -= curnpgs; 8416 8417 if (dopgs == 0) { 8418 break; 8419 } 8420 8421 pp2 = PP_PAGENEXT_N(pp2, curnpgs); 8422 if (((dopgs & (pg64k - 1)) == 0) && PP_ISMAPPED_LARGE(pp2)) { 8423 cursz = fnd_mapping_sz(pp2); 8424 } else { 8425 cursz = TTE8K; 8426 } 8427 } 8428 } 8429 8430 /* 8431 * Returns 1 if page(s) can be converted from TNC to cacheable setting, 8432 * returns 0 otherwise. Note that oaddr argument is valid for only 8433 * 8k pages. 8434 */ 8435 int 8436 tst_tnc(page_t *pp, pgcnt_t npages) 8437 { 8438 struct sf_hment *sfhme; 8439 struct hme_blk *hmeblkp; 8440 tte_t tte; 8441 caddr_t vaddr; 8442 int clr_valid = 0; 8443 int color, color1, bcolor; 8444 int i, ncolors; 8445 8446 ASSERT(pp != NULL); 8447 ASSERT(!(cache & CACHE_WRITEBACK)); 8448 8449 if (npages > 1) { 8450 ncolors = CACHE_NUM_COLOR; 8451 } 8452 8453 for (i = 0; i < npages; i++) { 8454 ASSERT(sfmmu_mlist_held(pp)); 8455 ASSERT(PP_ISTNC(pp)); 8456 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 8457 8458 if (PP_ISPNC(pp)) { 8459 return (0); 8460 } 8461 8462 clr_valid = 0; 8463 if (PP_ISMAPPED_KPM(pp)) { 8464 caddr_t kpmvaddr; 8465 8466 ASSERT(kpm_enable); 8467 kpmvaddr = hat_kpm_page2va(pp, 1); 8468 ASSERT(!(npages > 1 && IS_KPM_ALIAS_RANGE(kpmvaddr))); 8469 color1 = addr_to_vcolor(kpmvaddr); 8470 clr_valid = 1; 8471 } 8472 8473 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8474 hmeblkp = sfmmu_hmetohblk(sfhme); 8475 if (hmeblkp->hblk_xhat_bit) 8476 continue; 8477 8478 sfmmu_copytte(&sfhme->hme_tte, &tte); 8479 ASSERT(TTE_IS_VALID(&tte)); 8480 8481 vaddr = tte_to_vaddr(hmeblkp, tte); 8482 color = addr_to_vcolor(vaddr); 8483 8484 if (npages > 1) { 8485 /* 8486 * If there is a big mapping, make sure 8487 * 8K mapping is consistent with the big 8488 * mapping. 8489 */ 8490 bcolor = i % ncolors; 8491 if (color != bcolor) { 8492 return (0); 8493 } 8494 } 8495 if (!clr_valid) { 8496 clr_valid = 1; 8497 color1 = color; 8498 } 8499 8500 if (color1 != color) { 8501 return (0); 8502 } 8503 } 8504 8505 pp = PP_PAGENEXT(pp); 8506 } 8507 8508 return (1); 8509 } 8510 8511 void 8512 sfmmu_page_cache_array(page_t *pp, int flags, int cache_flush_flag, 8513 pgcnt_t npages) 8514 { 8515 kmutex_t *pmtx; 8516 int i, ncolors, bcolor; 8517 kpm_hlk_t *kpmp; 8518 cpuset_t cpuset; 8519 8520 ASSERT(pp != NULL); 8521 ASSERT(!(cache & CACHE_WRITEBACK)); 8522 8523 kpmp = sfmmu_kpm_kpmp_enter(pp, npages); 8524 pmtx = sfmmu_page_enter(pp); 8525 8526 /* 8527 * Fast path caching single unmapped page 8528 */ 8529 if (npages == 1 && !PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp) && 8530 flags == HAT_CACHE) { 8531 PP_CLRTNC(pp); 8532 PP_CLRPNC(pp); 8533 sfmmu_page_exit(pmtx); 8534 sfmmu_kpm_kpmp_exit(kpmp); 8535 return; 8536 } 8537 8538 /* 8539 * We need to capture all cpus in order to change cacheability 8540 * because we can't allow one cpu to access the same physical 8541 * page using a cacheable and a non-cachebale mapping at the same 8542 * time. Since we may end up walking the ism mapping list 8543 * have to grab it's lock now since we can't after all the 8544 * cpus have been captured. 8545 */ 8546 sfmmu_hat_lock_all(); 8547 mutex_enter(&ism_mlist_lock); 8548 kpreempt_disable(); 8549 cpuset = cpu_ready_set; 8550 xc_attention(cpuset); 8551 8552 if (npages > 1) { 8553 /* 8554 * Make sure all colors are flushed since the 8555 * sfmmu_page_cache() only flushes one color- 8556 * it does not know big pages. 8557 */ 8558 ncolors = CACHE_NUM_COLOR; 8559 if (flags & HAT_TMPNC) { 8560 for (i = 0; i < ncolors; i++) { 8561 sfmmu_cache_flushcolor(i, pp->p_pagenum); 8562 } 8563 cache_flush_flag = CACHE_NO_FLUSH; 8564 } 8565 } 8566 8567 for (i = 0; i < npages; i++) { 8568 8569 ASSERT(sfmmu_mlist_held(pp)); 8570 8571 if (!(flags == HAT_TMPNC && PP_ISTNC(pp))) { 8572 8573 if (npages > 1) { 8574 bcolor = i % ncolors; 8575 } else { 8576 bcolor = NO_VCOLOR; 8577 } 8578 8579 sfmmu_page_cache(pp, flags, cache_flush_flag, 8580 bcolor); 8581 } 8582 8583 pp = PP_PAGENEXT(pp); 8584 } 8585 8586 xt_sync(cpuset); 8587 xc_dismissed(cpuset); 8588 mutex_exit(&ism_mlist_lock); 8589 sfmmu_hat_unlock_all(); 8590 sfmmu_page_exit(pmtx); 8591 sfmmu_kpm_kpmp_exit(kpmp); 8592 kpreempt_enable(); 8593 } 8594 8595 /* 8596 * This function changes the virtual cacheability of all mappings to a 8597 * particular page. When changing from uncache to cacheable the mappings will 8598 * only be changed if all of them have the same virtual color. 8599 * We need to flush the cache in all cpus. It is possible that 8600 * a process referenced a page as cacheable but has sinced exited 8601 * and cleared the mapping list. We still to flush it but have no 8602 * state so all cpus is the only alternative. 8603 */ 8604 static void 8605 sfmmu_page_cache(page_t *pp, int flags, int cache_flush_flag, int bcolor) 8606 { 8607 struct sf_hment *sfhme; 8608 struct hme_blk *hmeblkp; 8609 sfmmu_t *sfmmup; 8610 tte_t tte, ttemod; 8611 caddr_t vaddr; 8612 int ret, color; 8613 pfn_t pfn; 8614 8615 color = bcolor; 8616 pfn = pp->p_pagenum; 8617 8618 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8619 8620 hmeblkp = sfmmu_hmetohblk(sfhme); 8621 8622 if (hmeblkp->hblk_xhat_bit) 8623 continue; 8624 8625 sfmmu_copytte(&sfhme->hme_tte, &tte); 8626 ASSERT(TTE_IS_VALID(&tte)); 8627 vaddr = tte_to_vaddr(hmeblkp, tte); 8628 color = addr_to_vcolor(vaddr); 8629 8630 #ifdef DEBUG 8631 if ((flags & HAT_CACHE) && bcolor != NO_VCOLOR) { 8632 ASSERT(color == bcolor); 8633 } 8634 #endif 8635 8636 ASSERT(flags != HAT_TMPNC || color == PP_GET_VCOLOR(pp)); 8637 8638 ttemod = tte; 8639 if (flags & (HAT_UNCACHE | HAT_TMPNC)) { 8640 TTE_CLR_VCACHEABLE(&ttemod); 8641 } else { /* flags & HAT_CACHE */ 8642 TTE_SET_VCACHEABLE(&ttemod); 8643 } 8644 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 8645 if (ret < 0) { 8646 /* 8647 * Since all cpus are captured modifytte should not 8648 * fail. 8649 */ 8650 panic("sfmmu_page_cache: write to tte failed"); 8651 } 8652 8653 sfmmup = hblktosfmmu(hmeblkp); 8654 if (cache_flush_flag == CACHE_FLUSH) { 8655 /* 8656 * Flush TSBs, TLBs and caches 8657 */ 8658 if (sfmmup->sfmmu_ismhat) { 8659 if (flags & HAT_CACHE) { 8660 SFMMU_STAT(sf_ism_recache); 8661 } else { 8662 SFMMU_STAT(sf_ism_uncache); 8663 } 8664 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8665 pfn, CACHE_FLUSH); 8666 } else { 8667 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp, 8668 pfn, 0, FLUSH_ALL_CPUS, CACHE_FLUSH, 1); 8669 } 8670 8671 /* 8672 * all cache entries belonging to this pfn are 8673 * now flushed. 8674 */ 8675 cache_flush_flag = CACHE_NO_FLUSH; 8676 } else { 8677 8678 /* 8679 * Flush only TSBs and TLBs. 8680 */ 8681 if (sfmmup->sfmmu_ismhat) { 8682 if (flags & HAT_CACHE) { 8683 SFMMU_STAT(sf_ism_recache); 8684 } else { 8685 SFMMU_STAT(sf_ism_uncache); 8686 } 8687 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8688 pfn, CACHE_NO_FLUSH); 8689 } else { 8690 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1); 8691 } 8692 } 8693 } 8694 8695 if (PP_ISMAPPED_KPM(pp)) 8696 sfmmu_kpm_page_cache(pp, flags, cache_flush_flag); 8697 8698 switch (flags) { 8699 8700 default: 8701 panic("sfmmu_pagecache: unknown flags"); 8702 break; 8703 8704 case HAT_CACHE: 8705 PP_CLRTNC(pp); 8706 PP_CLRPNC(pp); 8707 PP_SET_VCOLOR(pp, color); 8708 break; 8709 8710 case HAT_TMPNC: 8711 PP_SETTNC(pp); 8712 PP_SET_VCOLOR(pp, NO_VCOLOR); 8713 break; 8714 8715 case HAT_UNCACHE: 8716 PP_SETPNC(pp); 8717 PP_CLRTNC(pp); 8718 PP_SET_VCOLOR(pp, NO_VCOLOR); 8719 break; 8720 } 8721 } 8722 #endif /* VAC */ 8723 8724 8725 /* 8726 * Wrapper routine used to return a context. 8727 * 8728 * It's the responsibility of the caller to guarantee that the 8729 * process serializes on calls here by taking the HAT lock for 8730 * the hat. 8731 * 8732 */ 8733 static void 8734 sfmmu_get_ctx(sfmmu_t *sfmmup) 8735 { 8736 mmu_ctx_t *mmu_ctxp; 8737 uint_t pstate_save; 8738 8739 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8740 ASSERT(sfmmup != ksfmmup); 8741 8742 kpreempt_disable(); 8743 8744 mmu_ctxp = CPU_MMU_CTXP(CPU); 8745 ASSERT(mmu_ctxp); 8746 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 8747 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 8748 8749 /* 8750 * Do a wrap-around if cnum reaches the max # cnum supported by a MMU. 8751 */ 8752 if (mmu_ctxp->mmu_cnum == mmu_ctxp->mmu_nctxs) 8753 sfmmu_ctx_wrap_around(mmu_ctxp); 8754 8755 /* 8756 * Let the MMU set up the page sizes to use for 8757 * this context in the TLB. Don't program 2nd dtlb for ism hat. 8758 */ 8759 if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) { 8760 mmu_set_ctx_page_sizes(sfmmup); 8761 } 8762 8763 /* 8764 * sfmmu_alloc_ctx and sfmmu_load_mmustate will be performed with 8765 * interrupts disabled to prevent race condition with wrap-around 8766 * ctx invalidatation. In sun4v, ctx invalidation also involves 8767 * a HV call to set the number of TSBs to 0. If interrupts are not 8768 * disabled until after sfmmu_load_mmustate is complete TSBs may 8769 * become assigned to INVALID_CONTEXT. This is not allowed. 8770 */ 8771 pstate_save = sfmmu_disable_intrs(); 8772 8773 sfmmu_alloc_ctx(sfmmup, 1, CPU); 8774 sfmmu_load_mmustate(sfmmup); 8775 8776 sfmmu_enable_intrs(pstate_save); 8777 8778 kpreempt_enable(); 8779 } 8780 8781 /* 8782 * When all cnums are used up in a MMU, cnum will wrap around to the 8783 * next generation and start from 2. 8784 */ 8785 static void 8786 sfmmu_ctx_wrap_around(mmu_ctx_t *mmu_ctxp) 8787 { 8788 8789 /* caller must have disabled the preemption */ 8790 ASSERT(curthread->t_preempt >= 1); 8791 ASSERT(mmu_ctxp != NULL); 8792 8793 /* acquire Per-MMU (PM) spin lock */ 8794 mutex_enter(&mmu_ctxp->mmu_lock); 8795 8796 /* re-check to see if wrap-around is needed */ 8797 if (mmu_ctxp->mmu_cnum < mmu_ctxp->mmu_nctxs) 8798 goto done; 8799 8800 SFMMU_MMU_STAT(mmu_wrap_around); 8801 8802 /* update gnum */ 8803 ASSERT(mmu_ctxp->mmu_gnum != 0); 8804 mmu_ctxp->mmu_gnum++; 8805 if (mmu_ctxp->mmu_gnum == 0 || 8806 mmu_ctxp->mmu_gnum > MAX_SFMMU_GNUM_VAL) { 8807 cmn_err(CE_PANIC, "mmu_gnum of mmu_ctx 0x%p is out of bound.", 8808 (void *)mmu_ctxp); 8809 } 8810 8811 if (mmu_ctxp->mmu_ncpus > 1) { 8812 cpuset_t cpuset; 8813 8814 membar_enter(); /* make sure updated gnum visible */ 8815 8816 SFMMU_XCALL_STATS(NULL); 8817 8818 /* xcall to others on the same MMU to invalidate ctx */ 8819 cpuset = mmu_ctxp->mmu_cpuset; 8820 ASSERT(CPU_IN_SET(cpuset, CPU->cpu_id)); 8821 CPUSET_DEL(cpuset, CPU->cpu_id); 8822 CPUSET_AND(cpuset, cpu_ready_set); 8823 8824 /* 8825 * Pass in INVALID_CONTEXT as the first parameter to 8826 * sfmmu_raise_tsb_exception, which invalidates the context 8827 * of any process running on the CPUs in the MMU. 8828 */ 8829 xt_some(cpuset, sfmmu_raise_tsb_exception, 8830 INVALID_CONTEXT, INVALID_CONTEXT); 8831 xt_sync(cpuset); 8832 8833 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 8834 } 8835 8836 if (sfmmu_getctx_sec() != INVALID_CONTEXT) { 8837 sfmmu_setctx_sec(INVALID_CONTEXT); 8838 sfmmu_clear_utsbinfo(); 8839 } 8840 8841 /* 8842 * No xcall is needed here. For sun4u systems all CPUs in context 8843 * domain share a single physical MMU therefore it's enough to flush 8844 * TLB on local CPU. On sun4v systems we use 1 global context 8845 * domain and flush all remote TLBs in sfmmu_raise_tsb_exception 8846 * handler. Note that vtag_flushall_uctxs() is called 8847 * for Ultra II machine, where the equivalent flushall functionality 8848 * is implemented in SW, and only user ctx TLB entries are flushed. 8849 */ 8850 if (&vtag_flushall_uctxs != NULL) { 8851 vtag_flushall_uctxs(); 8852 } else { 8853 vtag_flushall(); 8854 } 8855 8856 /* reset mmu cnum, skips cnum 0 and 1 */ 8857 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 8858 8859 done: 8860 mutex_exit(&mmu_ctxp->mmu_lock); 8861 } 8862 8863 8864 /* 8865 * For multi-threaded process, set the process context to INVALID_CONTEXT 8866 * so that it faults and reloads the MMU state from TL=0. For single-threaded 8867 * process, we can just load the MMU state directly without having to 8868 * set context invalid. Caller must hold the hat lock since we don't 8869 * acquire it here. 8870 */ 8871 static void 8872 sfmmu_sync_mmustate(sfmmu_t *sfmmup) 8873 { 8874 uint_t cnum; 8875 uint_t pstate_save; 8876 8877 ASSERT(sfmmup != ksfmmup); 8878 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8879 8880 kpreempt_disable(); 8881 8882 /* 8883 * We check whether the pass'ed-in sfmmup is the same as the 8884 * current running proc. This is to makes sure the current proc 8885 * stays single-threaded if it already is. 8886 */ 8887 if ((sfmmup == curthread->t_procp->p_as->a_hat) && 8888 (curthread->t_procp->p_lwpcnt == 1)) { 8889 /* single-thread */ 8890 cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum; 8891 if (cnum != INVALID_CONTEXT) { 8892 uint_t curcnum; 8893 /* 8894 * Disable interrupts to prevent race condition 8895 * with sfmmu_ctx_wrap_around ctx invalidation. 8896 * In sun4v, ctx invalidation involves setting 8897 * TSB to NULL, hence, interrupts should be disabled 8898 * untill after sfmmu_load_mmustate is completed. 8899 */ 8900 pstate_save = sfmmu_disable_intrs(); 8901 curcnum = sfmmu_getctx_sec(); 8902 if (curcnum == cnum) 8903 sfmmu_load_mmustate(sfmmup); 8904 sfmmu_enable_intrs(pstate_save); 8905 ASSERT(curcnum == cnum || curcnum == INVALID_CONTEXT); 8906 } 8907 } else { 8908 /* 8909 * multi-thread 8910 * or when sfmmup is not the same as the curproc. 8911 */ 8912 sfmmu_invalidate_ctx(sfmmup); 8913 } 8914 8915 kpreempt_enable(); 8916 } 8917 8918 8919 /* 8920 * Replace the specified TSB with a new TSB. This function gets called when 8921 * we grow, shrink or swapin a TSB. When swapping in a TSB (TSB_SWAPIN), the 8922 * TSB_FORCEALLOC flag may be used to force allocation of a minimum-sized TSB 8923 * (8K). 8924 * 8925 * Caller must hold the HAT lock, but should assume any tsb_info 8926 * pointers it has are no longer valid after calling this function. 8927 * 8928 * Return values: 8929 * TSB_ALLOCFAIL Failed to allocate a TSB, due to memory constraints 8930 * TSB_LOSTRACE HAT is busy, i.e. another thread is already doing 8931 * something to this tsbinfo/TSB 8932 * TSB_SUCCESS Operation succeeded 8933 */ 8934 static tsb_replace_rc_t 8935 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, 8936 hatlock_t *hatlockp, uint_t flags) 8937 { 8938 struct tsb_info *new_tsbinfo = NULL; 8939 struct tsb_info *curtsb, *prevtsb; 8940 uint_t tte_sz_mask; 8941 int i; 8942 8943 ASSERT(sfmmup != ksfmmup); 8944 ASSERT(sfmmup->sfmmu_ismhat == 0); 8945 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8946 ASSERT(szc <= tsb_max_growsize); 8947 8948 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY)) 8949 return (TSB_LOSTRACE); 8950 8951 /* 8952 * Find the tsb_info ahead of this one in the list, and 8953 * also make sure that the tsb_info passed in really 8954 * exists! 8955 */ 8956 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 8957 curtsb != old_tsbinfo && curtsb != NULL; 8958 prevtsb = curtsb, curtsb = curtsb->tsb_next); 8959 ASSERT(curtsb != NULL); 8960 8961 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8962 /* 8963 * The process is swapped out, so just set the new size 8964 * code. When it swaps back in, we'll allocate a new one 8965 * of the new chosen size. 8966 */ 8967 curtsb->tsb_szc = szc; 8968 return (TSB_SUCCESS); 8969 } 8970 SFMMU_FLAGS_SET(sfmmup, HAT_BUSY); 8971 8972 tte_sz_mask = old_tsbinfo->tsb_ttesz_mask; 8973 8974 /* 8975 * All initialization is done inside of sfmmu_tsbinfo_alloc(). 8976 * If we fail to allocate a TSB, exit. 8977 */ 8978 sfmmu_hat_exit(hatlockp); 8979 if (sfmmu_tsbinfo_alloc(&new_tsbinfo, szc, tte_sz_mask, 8980 flags, sfmmup)) { 8981 (void) sfmmu_hat_enter(sfmmup); 8982 if (!(flags & TSB_SWAPIN)) 8983 SFMMU_STAT(sf_tsb_resize_failures); 8984 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8985 return (TSB_ALLOCFAIL); 8986 } 8987 (void) sfmmu_hat_enter(sfmmup); 8988 8989 /* 8990 * Re-check to make sure somebody else didn't muck with us while we 8991 * didn't hold the HAT lock. If the process swapped out, fine, just 8992 * exit; this can happen if we try to shrink the TSB from the context 8993 * of another process (such as on an ISM unmap), though it is rare. 8994 */ 8995 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8996 SFMMU_STAT(sf_tsb_resize_failures); 8997 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8998 sfmmu_hat_exit(hatlockp); 8999 sfmmu_tsbinfo_free(new_tsbinfo); 9000 (void) sfmmu_hat_enter(sfmmup); 9001 return (TSB_LOSTRACE); 9002 } 9003 9004 #ifdef DEBUG 9005 /* Reverify that the tsb_info still exists.. for debugging only */ 9006 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 9007 curtsb != old_tsbinfo && curtsb != NULL; 9008 prevtsb = curtsb, curtsb = curtsb->tsb_next); 9009 ASSERT(curtsb != NULL); 9010 #endif /* DEBUG */ 9011 9012 /* 9013 * Quiesce any CPUs running this process on their next TLB miss 9014 * so they atomically see the new tsb_info. We temporarily set the 9015 * context to invalid context so new threads that come on processor 9016 * after we do the xcall to cpusran will also serialize behind the 9017 * HAT lock on TLB miss and will see the new TSB. Since this short 9018 * race with a new thread coming on processor is relatively rare, 9019 * this synchronization mechanism should be cheaper than always 9020 * pausing all CPUs for the duration of the setup, which is what 9021 * the old implementation did. This is particuarly true if we are 9022 * copying a huge chunk of memory around during that window. 9023 * 9024 * The memory barriers are to make sure things stay consistent 9025 * with resume() since it does not hold the HAT lock while 9026 * walking the list of tsb_info structures. 9027 */ 9028 if ((flags & TSB_SWAPIN) != TSB_SWAPIN) { 9029 /* The TSB is either growing or shrinking. */ 9030 sfmmu_invalidate_ctx(sfmmup); 9031 } else { 9032 /* 9033 * It is illegal to swap in TSBs from a process other 9034 * than a process being swapped in. This in turn 9035 * implies we do not have a valid MMU context here 9036 * since a process needs one to resolve translation 9037 * misses. 9038 */ 9039 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup); 9040 } 9041 9042 #ifdef DEBUG 9043 ASSERT(max_mmu_ctxdoms > 0); 9044 9045 /* 9046 * Process should have INVALID_CONTEXT on all MMUs 9047 */ 9048 for (i = 0; i < max_mmu_ctxdoms; i++) { 9049 9050 ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); 9051 } 9052 #endif 9053 9054 new_tsbinfo->tsb_next = old_tsbinfo->tsb_next; 9055 membar_stst(); /* strict ordering required */ 9056 if (prevtsb) 9057 prevtsb->tsb_next = new_tsbinfo; 9058 else 9059 sfmmup->sfmmu_tsb = new_tsbinfo; 9060 membar_enter(); /* make sure new TSB globally visible */ 9061 sfmmu_setup_tsbinfo(sfmmup); 9062 9063 /* 9064 * We need to migrate TSB entries from the old TSB to the new TSB 9065 * if tsb_remap_ttes is set and the TSB is growing. 9066 */ 9067 if (tsb_remap_ttes && ((flags & TSB_GROW) == TSB_GROW)) 9068 sfmmu_copy_tsb(old_tsbinfo, new_tsbinfo); 9069 9070 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 9071 9072 /* 9073 * Drop the HAT lock to free our old tsb_info. 9074 */ 9075 sfmmu_hat_exit(hatlockp); 9076 9077 if ((flags & TSB_GROW) == TSB_GROW) { 9078 SFMMU_STAT(sf_tsb_grow); 9079 } else if ((flags & TSB_SHRINK) == TSB_SHRINK) { 9080 SFMMU_STAT(sf_tsb_shrink); 9081 } 9082 9083 sfmmu_tsbinfo_free(old_tsbinfo); 9084 9085 (void) sfmmu_hat_enter(sfmmup); 9086 return (TSB_SUCCESS); 9087 } 9088 9089 /* 9090 * This function will re-program hat pgsz array, and invalidate the 9091 * process' context, forcing the process to switch to another 9092 * context on the next TLB miss, and therefore start using the 9093 * TLB that is reprogrammed for the new page sizes. 9094 */ 9095 void 9096 sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) 9097 { 9098 int i; 9099 hatlock_t *hatlockp = NULL; 9100 9101 hatlockp = sfmmu_hat_enter(sfmmup); 9102 /* USIII+-IV+ optimization, requires hat lock */ 9103 if (tmp_pgsz) { 9104 for (i = 0; i < mmu_page_sizes; i++) 9105 sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i]; 9106 } 9107 SFMMU_STAT(sf_tlb_reprog_pgsz); 9108 9109 sfmmu_invalidate_ctx(sfmmup); 9110 9111 sfmmu_hat_exit(hatlockp); 9112 } 9113 9114 /* 9115 * This function assumes that there are either four or six supported page 9116 * sizes and at most two programmable TLBs, so we need to decide which 9117 * page sizes are most important and then tell the MMU layer so it 9118 * can adjust the TLB page sizes accordingly (if supported). 9119 * 9120 * If these assumptions change, this function will need to be 9121 * updated to support whatever the new limits are. 9122 * 9123 * The growing flag is nonzero if we are growing the address space, 9124 * and zero if it is shrinking. This allows us to decide whether 9125 * to grow or shrink our TSB, depending upon available memory 9126 * conditions. 9127 */ 9128 static void 9129 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing) 9130 { 9131 uint64_t ttecnt[MMU_PAGE_SIZES]; 9132 uint64_t tte8k_cnt, tte4m_cnt; 9133 uint8_t i; 9134 int sectsb_thresh; 9135 9136 /* 9137 * Kernel threads, processes with small address spaces not using 9138 * large pages, and dummy ISM HATs need not apply. 9139 */ 9140 if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != NULL) 9141 return; 9142 9143 if ((sfmmup->sfmmu_flags & HAT_LGPG_FLAGS) == 0 && 9144 sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor) 9145 return; 9146 9147 for (i = 0; i < mmu_page_sizes; i++) { 9148 ttecnt[i] = SFMMU_TTE_CNT(sfmmup, i); 9149 } 9150 9151 /* Check pagesizes in use, and possibly reprogram DTLB. */ 9152 if (&mmu_check_page_sizes) 9153 mmu_check_page_sizes(sfmmup, ttecnt); 9154 9155 /* 9156 * Calculate the number of 8k ttes to represent the span of these 9157 * pages. 9158 */ 9159 tte8k_cnt = ttecnt[TTE8K] + 9160 (ttecnt[TTE64K] << (MMU_PAGESHIFT64K - MMU_PAGESHIFT)) + 9161 (ttecnt[TTE512K] << (MMU_PAGESHIFT512K - MMU_PAGESHIFT)); 9162 if (mmu_page_sizes == max_mmu_page_sizes) { 9163 tte4m_cnt = ttecnt[TTE4M] + 9164 (ttecnt[TTE32M] << (MMU_PAGESHIFT32M - MMU_PAGESHIFT4M)) + 9165 (ttecnt[TTE256M] << (MMU_PAGESHIFT256M - MMU_PAGESHIFT4M)); 9166 } else { 9167 tte4m_cnt = ttecnt[TTE4M]; 9168 } 9169 9170 /* 9171 * Inflate TSB sizes by a factor of 2 if this process 9172 * uses 4M text pages to minimize extra conflict misses 9173 * in the first TSB since without counting text pages 9174 * 8K TSB may become too small. 9175 * 9176 * Also double the size of the second TSB to minimize 9177 * extra conflict misses due to competition between 4M text pages 9178 * and data pages. 9179 * 9180 * We need to adjust the second TSB allocation threshold by the 9181 * inflation factor, since there is no point in creating a second 9182 * TSB when we know all the mappings can fit in the I/D TLBs. 9183 */ 9184 sectsb_thresh = tsb_sectsb_threshold; 9185 if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) { 9186 tte8k_cnt <<= 1; 9187 tte4m_cnt <<= 1; 9188 sectsb_thresh <<= 1; 9189 } 9190 9191 /* 9192 * Check to see if our TSB is the right size; we may need to 9193 * grow or shrink it. If the process is small, our work is 9194 * finished at this point. 9195 */ 9196 if (tte8k_cnt <= tsb_rss_factor && tte4m_cnt <= sectsb_thresh) { 9197 return; 9198 } 9199 sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh); 9200 } 9201 9202 static void 9203 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt, 9204 uint64_t tte4m_cnt, int sectsb_thresh) 9205 { 9206 int tsb_bits; 9207 uint_t tsb_szc; 9208 struct tsb_info *tsbinfop; 9209 hatlock_t *hatlockp = NULL; 9210 9211 hatlockp = sfmmu_hat_enter(sfmmup); 9212 ASSERT(hatlockp != NULL); 9213 tsbinfop = sfmmup->sfmmu_tsb; 9214 ASSERT(tsbinfop != NULL); 9215 9216 /* 9217 * If we're growing, select the size based on RSS. If we're 9218 * shrinking, leave some room so we don't have to turn around and 9219 * grow again immediately. 9220 */ 9221 if (growing) 9222 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt); 9223 else 9224 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt << 1); 9225 9226 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9227 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9228 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9229 hatlockp, TSB_SHRINK); 9230 } else if (growing && tsb_szc > tsbinfop->tsb_szc && TSB_OK_GROW()) { 9231 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9232 hatlockp, TSB_GROW); 9233 } 9234 tsbinfop = sfmmup->sfmmu_tsb; 9235 9236 /* 9237 * With the TLB and first TSB out of the way, we need to see if 9238 * we need a second TSB for 4M pages. If we managed to reprogram 9239 * the TLB page sizes above, the process will start using this new 9240 * TSB right away; otherwise, it will start using it on the next 9241 * context switch. Either way, it's no big deal so there's no 9242 * synchronization with the trap handlers here unless we grow the 9243 * TSB (in which case it's required to prevent using the old one 9244 * after it's freed). Note: second tsb is required for 32M/256M 9245 * page sizes. 9246 */ 9247 if (tte4m_cnt > sectsb_thresh) { 9248 /* 9249 * If we're growing, select the size based on RSS. If we're 9250 * shrinking, leave some room so we don't have to turn 9251 * around and grow again immediately. 9252 */ 9253 if (growing) 9254 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt); 9255 else 9256 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt << 1); 9257 if (tsbinfop->tsb_next == NULL) { 9258 struct tsb_info *newtsb; 9259 int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)? 9260 0 : TSB_ALLOC; 9261 9262 sfmmu_hat_exit(hatlockp); 9263 9264 /* 9265 * Try to allocate a TSB for 4[32|256]M pages. If we 9266 * can't get the size we want, retry w/a minimum sized 9267 * TSB. If that still didn't work, give up; we can 9268 * still run without one. 9269 */ 9270 tsb_bits = (mmu_page_sizes == max_mmu_page_sizes)? 9271 TSB4M|TSB32M|TSB256M:TSB4M; 9272 if ((sfmmu_tsbinfo_alloc(&newtsb, tsb_szc, tsb_bits, 9273 allocflags, sfmmup) != 0) && 9274 (sfmmu_tsbinfo_alloc(&newtsb, TSB_MIN_SZCODE, 9275 tsb_bits, allocflags, sfmmup) != 0)) { 9276 return; 9277 } 9278 9279 hatlockp = sfmmu_hat_enter(sfmmup); 9280 9281 if (sfmmup->sfmmu_tsb->tsb_next == NULL) { 9282 sfmmup->sfmmu_tsb->tsb_next = newtsb; 9283 SFMMU_STAT(sf_tsb_sectsb_create); 9284 sfmmu_setup_tsbinfo(sfmmup); 9285 sfmmu_hat_exit(hatlockp); 9286 return; 9287 } else { 9288 /* 9289 * It's annoying, but possible for us 9290 * to get here.. we dropped the HAT lock 9291 * because of locking order in the kmem 9292 * allocator, and while we were off getting 9293 * our memory, some other thread decided to 9294 * do us a favor and won the race to get a 9295 * second TSB for this process. Sigh. 9296 */ 9297 sfmmu_hat_exit(hatlockp); 9298 sfmmu_tsbinfo_free(newtsb); 9299 return; 9300 } 9301 } 9302 9303 /* 9304 * We have a second TSB, see if it's big enough. 9305 */ 9306 tsbinfop = tsbinfop->tsb_next; 9307 9308 /* 9309 * Check to see if our second TSB is the right size; 9310 * we may need to grow or shrink it. 9311 * To prevent thrashing (e.g. growing the TSB on a 9312 * subsequent map operation), only try to shrink if 9313 * the TSB reach exceeds twice the virtual address 9314 * space size. 9315 */ 9316 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9317 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9318 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9319 tsb_szc, hatlockp, TSB_SHRINK); 9320 } else if (growing && tsb_szc > tsbinfop->tsb_szc && 9321 TSB_OK_GROW()) { 9322 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9323 tsb_szc, hatlockp, TSB_GROW); 9324 } 9325 } 9326 9327 sfmmu_hat_exit(hatlockp); 9328 } 9329 9330 /* 9331 * Free up a sfmmu 9332 * Since the sfmmu is currently embedded in the hat struct we simply zero 9333 * out our fields and free up the ism map blk list if any. 9334 */ 9335 static void 9336 sfmmu_free_sfmmu(sfmmu_t *sfmmup) 9337 { 9338 ism_blk_t *blkp, *nx_blkp; 9339 #ifdef DEBUG 9340 ism_map_t *map; 9341 int i; 9342 #endif 9343 9344 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 9345 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 9346 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 9347 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 9348 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 9349 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 9350 9351 sfmmup->sfmmu_free = 0; 9352 sfmmup->sfmmu_ismhat = 0; 9353 9354 blkp = sfmmup->sfmmu_iblk; 9355 sfmmup->sfmmu_iblk = NULL; 9356 9357 while (blkp) { 9358 #ifdef DEBUG 9359 map = blkp->iblk_maps; 9360 for (i = 0; i < ISM_MAP_SLOTS; i++) { 9361 ASSERT(map[i].imap_seg == 0); 9362 ASSERT(map[i].imap_ismhat == NULL); 9363 ASSERT(map[i].imap_ment == NULL); 9364 } 9365 #endif 9366 nx_blkp = blkp->iblk_next; 9367 blkp->iblk_next = NULL; 9368 blkp->iblk_nextpa = (uint64_t)-1; 9369 kmem_cache_free(ism_blk_cache, blkp); 9370 blkp = nx_blkp; 9371 } 9372 } 9373 9374 /* 9375 * Locking primitves accessed by HATLOCK macros 9376 */ 9377 9378 #define SFMMU_SPL_MTX (0x0) 9379 #define SFMMU_ML_MTX (0x1) 9380 9381 #define SFMMU_MLSPL_MTX(type, pg) (((type) == SFMMU_SPL_MTX) ? \ 9382 SPL_HASH(pg) : MLIST_HASH(pg)) 9383 9384 kmutex_t * 9385 sfmmu_page_enter(struct page *pp) 9386 { 9387 return (sfmmu_mlspl_enter(pp, SFMMU_SPL_MTX)); 9388 } 9389 9390 void 9391 sfmmu_page_exit(kmutex_t *spl) 9392 { 9393 mutex_exit(spl); 9394 } 9395 9396 int 9397 sfmmu_page_spl_held(struct page *pp) 9398 { 9399 return (sfmmu_mlspl_held(pp, SFMMU_SPL_MTX)); 9400 } 9401 9402 kmutex_t * 9403 sfmmu_mlist_enter(struct page *pp) 9404 { 9405 return (sfmmu_mlspl_enter(pp, SFMMU_ML_MTX)); 9406 } 9407 9408 void 9409 sfmmu_mlist_exit(kmutex_t *mml) 9410 { 9411 mutex_exit(mml); 9412 } 9413 9414 int 9415 sfmmu_mlist_held(struct page *pp) 9416 { 9417 9418 return (sfmmu_mlspl_held(pp, SFMMU_ML_MTX)); 9419 } 9420 9421 /* 9422 * Common code for sfmmu_mlist_enter() and sfmmu_page_enter(). For 9423 * sfmmu_mlist_enter() case mml_table lock array is used and for 9424 * sfmmu_page_enter() sfmmu_page_lock lock array is used. 9425 * 9426 * The lock is taken on a root page so that it protects an operation on all 9427 * constituent pages of a large page pp belongs to. 9428 * 9429 * The routine takes a lock from the appropriate array. The lock is determined 9430 * by hashing the root page. After taking the lock this routine checks if the 9431 * root page has the same size code that was used to determine the root (i.e 9432 * that root hasn't changed). If root page has the expected p_szc field we 9433 * have the right lock and it's returned to the caller. If root's p_szc 9434 * decreased we release the lock and retry from the beginning. This case can 9435 * happen due to hat_page_demote() decreasing p_szc between our load of p_szc 9436 * value and taking the lock. The number of retries due to p_szc decrease is 9437 * limited by the maximum p_szc value. If p_szc is 0 we return the lock 9438 * determined by hashing pp itself. 9439 * 9440 * If our caller doesn't hold a SE_SHARED or SE_EXCL lock on pp it's also 9441 * possible that p_szc can increase. To increase p_szc a thread has to lock 9442 * all constituent pages EXCL and do hat_pageunload() on all of them. All the 9443 * callers that don't hold a page locked recheck if hmeblk through which pp 9444 * was found still maps this pp. If it doesn't map it anymore returned lock 9445 * is immediately dropped. Therefore if sfmmu_mlspl_enter() hits the case of 9446 * p_szc increase after taking the lock it returns this lock without further 9447 * retries because in this case the caller doesn't care about which lock was 9448 * taken. The caller will drop it right away. 9449 * 9450 * After the routine returns it's guaranteed that hat_page_demote() can't 9451 * change p_szc field of any of constituent pages of a large page pp belongs 9452 * to as long as pp was either locked at least SHARED prior to this call or 9453 * the caller finds that hment that pointed to this pp still references this 9454 * pp (this also assumes that the caller holds hme hash bucket lock so that 9455 * the same pp can't be remapped into the same hmeblk after it was unmapped by 9456 * hat_pageunload()). 9457 */ 9458 static kmutex_t * 9459 sfmmu_mlspl_enter(struct page *pp, int type) 9460 { 9461 kmutex_t *mtx; 9462 uint_t prev_rszc = UINT_MAX; 9463 page_t *rootpp; 9464 uint_t szc; 9465 uint_t rszc; 9466 uint_t pszc = pp->p_szc; 9467 9468 ASSERT(pp != NULL); 9469 9470 again: 9471 if (pszc == 0) { 9472 mtx = SFMMU_MLSPL_MTX(type, pp); 9473 mutex_enter(mtx); 9474 return (mtx); 9475 } 9476 9477 /* The lock lives in the root page */ 9478 rootpp = PP_GROUPLEADER(pp, pszc); 9479 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9480 mutex_enter(mtx); 9481 9482 /* 9483 * Return mml in the following 3 cases: 9484 * 9485 * 1) If pp itself is root since if its p_szc decreased before we took 9486 * the lock pp is still the root of smaller szc page. And if its p_szc 9487 * increased it doesn't matter what lock we return (see comment in 9488 * front of this routine). 9489 * 9490 * 2) If pp's not root but rootpp is the root of a rootpp->p_szc size 9491 * large page we have the right lock since any previous potential 9492 * hat_page_demote() is done demoting from greater than current root's 9493 * p_szc because hat_page_demote() changes root's p_szc last. No 9494 * further hat_page_demote() can start or be in progress since it 9495 * would need the same lock we currently hold. 9496 * 9497 * 3) If rootpp's p_szc increased since previous iteration it doesn't 9498 * matter what lock we return (see comment in front of this routine). 9499 */ 9500 if (pp == rootpp || (rszc = rootpp->p_szc) == pszc || 9501 rszc >= prev_rszc) { 9502 return (mtx); 9503 } 9504 9505 /* 9506 * hat_page_demote() could have decreased root's p_szc. 9507 * In this case pp's p_szc must also be smaller than pszc. 9508 * Retry. 9509 */ 9510 if (rszc < pszc) { 9511 szc = pp->p_szc; 9512 if (szc < pszc) { 9513 mutex_exit(mtx); 9514 pszc = szc; 9515 goto again; 9516 } 9517 /* 9518 * pp's p_szc increased after it was decreased. 9519 * page cannot be mapped. Return current lock. The caller 9520 * will drop it right away. 9521 */ 9522 return (mtx); 9523 } 9524 9525 /* 9526 * root's p_szc is greater than pp's p_szc. 9527 * hat_page_demote() is not done with all pages 9528 * yet. Wait for it to complete. 9529 */ 9530 mutex_exit(mtx); 9531 rootpp = PP_GROUPLEADER(rootpp, rszc); 9532 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9533 mutex_enter(mtx); 9534 mutex_exit(mtx); 9535 prev_rszc = rszc; 9536 goto again; 9537 } 9538 9539 static int 9540 sfmmu_mlspl_held(struct page *pp, int type) 9541 { 9542 kmutex_t *mtx; 9543 9544 ASSERT(pp != NULL); 9545 /* The lock lives in the root page */ 9546 pp = PP_PAGEROOT(pp); 9547 ASSERT(pp != NULL); 9548 9549 mtx = SFMMU_MLSPL_MTX(type, pp); 9550 return (MUTEX_HELD(mtx)); 9551 } 9552 9553 static uint_t 9554 sfmmu_get_free_hblk(struct hme_blk **hmeblkpp, uint_t critical) 9555 { 9556 struct hme_blk *hblkp; 9557 9558 if (freehblkp != NULL) { 9559 mutex_enter(&freehblkp_lock); 9560 if (freehblkp != NULL) { 9561 /* 9562 * If the current thread is owning hblk_reserve OR 9563 * critical request from sfmmu_hblk_steal() 9564 * let it succeed even if freehblkcnt is really low. 9565 */ 9566 if (freehblkcnt <= HBLK_RESERVE_MIN && !critical) { 9567 SFMMU_STAT(sf_get_free_throttle); 9568 mutex_exit(&freehblkp_lock); 9569 return (0); 9570 } 9571 freehblkcnt--; 9572 *hmeblkpp = freehblkp; 9573 hblkp = *hmeblkpp; 9574 freehblkp = hblkp->hblk_next; 9575 mutex_exit(&freehblkp_lock); 9576 hblkp->hblk_next = NULL; 9577 SFMMU_STAT(sf_get_free_success); 9578 return (1); 9579 } 9580 mutex_exit(&freehblkp_lock); 9581 } 9582 SFMMU_STAT(sf_get_free_fail); 9583 return (0); 9584 } 9585 9586 static uint_t 9587 sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical) 9588 { 9589 struct hme_blk *hblkp; 9590 9591 /* 9592 * If the current thread is mapping into kernel space, 9593 * let it succede even if freehblkcnt is max 9594 * so that it will avoid freeing it to kmem. 9595 * This will prevent stack overflow due to 9596 * possible recursion since kmem_cache_free() 9597 * might require creation of a slab which 9598 * in turn needs an hmeblk to map that slab; 9599 * let's break this vicious chain at the first 9600 * opportunity. 9601 */ 9602 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9603 mutex_enter(&freehblkp_lock); 9604 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9605 SFMMU_STAT(sf_put_free_success); 9606 freehblkcnt++; 9607 hmeblkp->hblk_next = freehblkp; 9608 freehblkp = hmeblkp; 9609 mutex_exit(&freehblkp_lock); 9610 return (1); 9611 } 9612 mutex_exit(&freehblkp_lock); 9613 } 9614 9615 /* 9616 * Bring down freehblkcnt to HBLK_RESERVE_CNT. We are here 9617 * only if freehblkcnt is at least HBLK_RESERVE_CNT *and* 9618 * we are not in the process of mapping into kernel space. 9619 */ 9620 ASSERT(!critical); 9621 while (freehblkcnt > HBLK_RESERVE_CNT) { 9622 mutex_enter(&freehblkp_lock); 9623 if (freehblkcnt > HBLK_RESERVE_CNT) { 9624 freehblkcnt--; 9625 hblkp = freehblkp; 9626 freehblkp = hblkp->hblk_next; 9627 mutex_exit(&freehblkp_lock); 9628 ASSERT(get_hblk_cache(hblkp) == sfmmu8_cache); 9629 kmem_cache_free(sfmmu8_cache, hblkp); 9630 continue; 9631 } 9632 mutex_exit(&freehblkp_lock); 9633 } 9634 SFMMU_STAT(sf_put_free_fail); 9635 return (0); 9636 } 9637 9638 static void 9639 sfmmu_hblk_swap(struct hme_blk *new) 9640 { 9641 struct hme_blk *old, *hblkp, *prev; 9642 uint64_t hblkpa, prevpa, newpa; 9643 caddr_t base, vaddr, endaddr; 9644 struct hmehash_bucket *hmebp; 9645 struct sf_hment *osfhme, *nsfhme; 9646 page_t *pp; 9647 kmutex_t *pml; 9648 tte_t tte; 9649 9650 #ifdef DEBUG 9651 hmeblk_tag hblktag; 9652 struct hme_blk *found; 9653 #endif 9654 old = HBLK_RESERVE; 9655 9656 /* 9657 * save pa before bcopy clobbers it 9658 */ 9659 newpa = new->hblk_nextpa; 9660 9661 base = (caddr_t)get_hblk_base(old); 9662 endaddr = base + get_hblk_span(old); 9663 9664 /* 9665 * acquire hash bucket lock. 9666 */ 9667 hmebp = sfmmu_tteload_acquire_hashbucket(ksfmmup, base, TTE8K); 9668 9669 /* 9670 * copy contents from old to new 9671 */ 9672 bcopy((void *)old, (void *)new, HME8BLK_SZ); 9673 9674 /* 9675 * add new to hash chain 9676 */ 9677 sfmmu_hblk_hash_add(hmebp, new, newpa); 9678 9679 /* 9680 * search hash chain for hblk_reserve; this needs to be performed 9681 * after adding new, otherwise prevpa and prev won't correspond 9682 * to the hblk which is prior to old in hash chain when we call 9683 * sfmmu_hblk_hash_rm to remove old later. 9684 */ 9685 for (prevpa = 0, prev = NULL, 9686 hblkpa = hmebp->hmeh_nextpa, hblkp = hmebp->hmeblkp; 9687 hblkp != NULL && hblkp != old; 9688 prevpa = hblkpa, prev = hblkp, 9689 hblkpa = hblkp->hblk_nextpa, hblkp = hblkp->hblk_next); 9690 9691 if (hblkp != old) 9692 panic("sfmmu_hblk_swap: hblk_reserve not found"); 9693 9694 /* 9695 * p_mapping list is still pointing to hments in hblk_reserve; 9696 * fix up p_mapping list so that they point to hments in new. 9697 * 9698 * Since all these mappings are created by hblk_reserve_thread 9699 * on the way and it's using at least one of the buffers from each of 9700 * the newly minted slabs, there is no danger of any of these 9701 * mappings getting unloaded by another thread. 9702 * 9703 * tsbmiss could only modify ref/mod bits of hments in old/new. 9704 * Since all of these hments hold mappings established by segkmem 9705 * and mappings in segkmem are setup with HAT_NOSYNC, ref/mod bits 9706 * have no meaning for the mappings in hblk_reserve. hments in 9707 * old and new are identical except for ref/mod bits. 9708 */ 9709 for (vaddr = base; vaddr < endaddr; vaddr += TTEBYTES(TTE8K)) { 9710 9711 HBLKTOHME(osfhme, old, vaddr); 9712 sfmmu_copytte(&osfhme->hme_tte, &tte); 9713 9714 if (TTE_IS_VALID(&tte)) { 9715 if ((pp = osfhme->hme_page) == NULL) 9716 panic("sfmmu_hblk_swap: page not mapped"); 9717 9718 pml = sfmmu_mlist_enter(pp); 9719 9720 if (pp != osfhme->hme_page) 9721 panic("sfmmu_hblk_swap: mapping changed"); 9722 9723 HBLKTOHME(nsfhme, new, vaddr); 9724 9725 HME_ADD(nsfhme, pp); 9726 HME_SUB(osfhme, pp); 9727 9728 sfmmu_mlist_exit(pml); 9729 } 9730 } 9731 9732 /* 9733 * remove old from hash chain 9734 */ 9735 sfmmu_hblk_hash_rm(hmebp, old, prevpa, prev); 9736 9737 #ifdef DEBUG 9738 9739 hblktag.htag_id = ksfmmup; 9740 hblktag.htag_bspage = HME_HASH_BSPAGE(base, HME_HASH_SHIFT(TTE8K)); 9741 hblktag.htag_rehash = HME_HASH_REHASH(TTE8K); 9742 HME_HASH_FAST_SEARCH(hmebp, hblktag, found); 9743 9744 if (found != new) 9745 panic("sfmmu_hblk_swap: new hblk not found"); 9746 #endif 9747 9748 SFMMU_HASH_UNLOCK(hmebp); 9749 9750 /* 9751 * Reset hblk_reserve 9752 */ 9753 bzero((void *)old, HME8BLK_SZ); 9754 old->hblk_nextpa = va_to_pa((caddr_t)old); 9755 } 9756 9757 /* 9758 * Grab the mlist mutex for both pages passed in. 9759 * 9760 * low and high will be returned as pointers to the mutexes for these pages. 9761 * low refers to the mutex residing in the lower bin of the mlist hash, while 9762 * high refers to the mutex residing in the higher bin of the mlist hash. This 9763 * is due to the locking order restrictions on the same thread grabbing 9764 * multiple mlist mutexes. The low lock must be acquired before the high lock. 9765 * 9766 * If both pages hash to the same mutex, only grab that single mutex, and 9767 * high will be returned as NULL 9768 * If the pages hash to different bins in the hash, grab the lower addressed 9769 * lock first and then the higher addressed lock in order to follow the locking 9770 * rules involved with the same thread grabbing multiple mlist mutexes. 9771 * low and high will both have non-NULL values. 9772 */ 9773 static void 9774 sfmmu_mlist_reloc_enter(struct page *targ, struct page *repl, 9775 kmutex_t **low, kmutex_t **high) 9776 { 9777 kmutex_t *mml_targ, *mml_repl; 9778 9779 /* 9780 * no need to do the dance around szc as in sfmmu_mlist_enter() 9781 * because this routine is only called by hat_page_relocate() and all 9782 * targ and repl pages are already locked EXCL so szc can't change. 9783 */ 9784 9785 mml_targ = MLIST_HASH(PP_PAGEROOT(targ)); 9786 mml_repl = MLIST_HASH(PP_PAGEROOT(repl)); 9787 9788 if (mml_targ == mml_repl) { 9789 *low = mml_targ; 9790 *high = NULL; 9791 } else { 9792 if (mml_targ < mml_repl) { 9793 *low = mml_targ; 9794 *high = mml_repl; 9795 } else { 9796 *low = mml_repl; 9797 *high = mml_targ; 9798 } 9799 } 9800 9801 mutex_enter(*low); 9802 if (*high) 9803 mutex_enter(*high); 9804 } 9805 9806 static void 9807 sfmmu_mlist_reloc_exit(kmutex_t *low, kmutex_t *high) 9808 { 9809 if (high) 9810 mutex_exit(high); 9811 mutex_exit(low); 9812 } 9813 9814 static hatlock_t * 9815 sfmmu_hat_enter(sfmmu_t *sfmmup) 9816 { 9817 hatlock_t *hatlockp; 9818 9819 if (sfmmup != ksfmmup) { 9820 hatlockp = TSB_HASH(sfmmup); 9821 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 9822 return (hatlockp); 9823 } 9824 return (NULL); 9825 } 9826 9827 static hatlock_t * 9828 sfmmu_hat_tryenter(sfmmu_t *sfmmup) 9829 { 9830 hatlock_t *hatlockp; 9831 9832 if (sfmmup != ksfmmup) { 9833 hatlockp = TSB_HASH(sfmmup); 9834 if (mutex_tryenter(HATLOCK_MUTEXP(hatlockp)) == 0) 9835 return (NULL); 9836 return (hatlockp); 9837 } 9838 return (NULL); 9839 } 9840 9841 static void 9842 sfmmu_hat_exit(hatlock_t *hatlockp) 9843 { 9844 if (hatlockp != NULL) 9845 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 9846 } 9847 9848 static void 9849 sfmmu_hat_lock_all(void) 9850 { 9851 int i; 9852 for (i = 0; i < SFMMU_NUM_LOCK; i++) 9853 mutex_enter(HATLOCK_MUTEXP(&hat_lock[i])); 9854 } 9855 9856 static void 9857 sfmmu_hat_unlock_all(void) 9858 { 9859 int i; 9860 for (i = SFMMU_NUM_LOCK - 1; i >= 0; i--) 9861 mutex_exit(HATLOCK_MUTEXP(&hat_lock[i])); 9862 } 9863 9864 int 9865 sfmmu_hat_lock_held(sfmmu_t *sfmmup) 9866 { 9867 ASSERT(sfmmup != ksfmmup); 9868 return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup)))); 9869 } 9870 9871 /* 9872 * Locking primitives to provide consistency between ISM unmap 9873 * and other operations. Since ISM unmap can take a long time, we 9874 * use HAT_ISMBUSY flag (protected by the hatlock) to avoid creating 9875 * contention on the hatlock buckets while ISM segments are being 9876 * unmapped. The tradeoff is that the flags don't prevent priority 9877 * inversion from occurring, so we must request kernel priority in 9878 * case we have to sleep to keep from getting buried while holding 9879 * the HAT_ISMBUSY flag set, which in turn could block other kernel 9880 * threads from running (for example, in sfmmu_uvatopfn()). 9881 */ 9882 static void 9883 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held) 9884 { 9885 hatlock_t *hatlockp; 9886 9887 THREAD_KPRI_REQUEST(); 9888 if (!hatlock_held) 9889 hatlockp = sfmmu_hat_enter(sfmmup); 9890 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) 9891 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 9892 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY); 9893 if (!hatlock_held) 9894 sfmmu_hat_exit(hatlockp); 9895 } 9896 9897 static void 9898 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held) 9899 { 9900 hatlock_t *hatlockp; 9901 9902 if (!hatlock_held) 9903 hatlockp = sfmmu_hat_enter(sfmmup); 9904 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 9905 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY); 9906 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 9907 if (!hatlock_held) 9908 sfmmu_hat_exit(hatlockp); 9909 THREAD_KPRI_RELEASE(); 9910 } 9911 9912 /* 9913 * 9914 * Algorithm: 9915 * 9916 * (1) if segkmem is not ready, allocate hblk from an array of pre-alloc'ed 9917 * hblks. 9918 * 9919 * (2) if we are allocating an hblk for mapping a slab in sfmmu_cache, 9920 * 9921 * (a) try to return an hblk from reserve pool of free hblks; 9922 * (b) if the reserve pool is empty, acquire hblk_reserve_lock 9923 * and return hblk_reserve. 9924 * 9925 * (3) call kmem_cache_alloc() to allocate hblk; 9926 * 9927 * (a) if hblk_reserve_lock is held by the current thread, 9928 * atomically replace hblk_reserve by the hblk that is 9929 * returned by kmem_cache_alloc; release hblk_reserve_lock 9930 * and call kmem_cache_alloc() again. 9931 * (b) if reserve pool is not full, add the hblk that is 9932 * returned by kmem_cache_alloc to reserve pool and 9933 * call kmem_cache_alloc again. 9934 * 9935 */ 9936 static struct hme_blk * 9937 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr, 9938 struct hmehash_bucket *hmebp, uint_t size, hmeblk_tag hblktag, 9939 uint_t flags) 9940 { 9941 struct hme_blk *hmeblkp = NULL; 9942 struct hme_blk *newhblkp; 9943 struct hme_blk *shw_hblkp = NULL; 9944 struct kmem_cache *sfmmu_cache = NULL; 9945 uint64_t hblkpa; 9946 ulong_t index; 9947 uint_t owner; /* set to 1 if using hblk_reserve */ 9948 uint_t forcefree; 9949 int sleep; 9950 9951 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 9952 9953 /* 9954 * If segkmem is not created yet, allocate from static hmeblks 9955 * created at the end of startup_modules(). See the block comment 9956 * in startup_modules() describing how we estimate the number of 9957 * static hmeblks that will be needed during re-map. 9958 */ 9959 if (!hblk_alloc_dynamic) { 9960 9961 if (size == TTE8K) { 9962 index = nucleus_hblk8.index; 9963 if (index >= nucleus_hblk8.len) { 9964 /* 9965 * If we panic here, see startup_modules() to 9966 * make sure that we are calculating the 9967 * number of hblk8's that we need correctly. 9968 */ 9969 prom_panic("no nucleus hblk8 to allocate"); 9970 } 9971 hmeblkp = 9972 (struct hme_blk *)&nucleus_hblk8.list[index]; 9973 nucleus_hblk8.index++; 9974 SFMMU_STAT(sf_hblk8_nalloc); 9975 } else { 9976 index = nucleus_hblk1.index; 9977 if (nucleus_hblk1.index >= nucleus_hblk1.len) { 9978 /* 9979 * If we panic here, see startup_modules(). 9980 * Most likely you need to update the 9981 * calculation of the number of hblk1 elements 9982 * that the kernel needs to boot. 9983 */ 9984 prom_panic("no nucleus hblk1 to allocate"); 9985 } 9986 hmeblkp = 9987 (struct hme_blk *)&nucleus_hblk1.list[index]; 9988 nucleus_hblk1.index++; 9989 SFMMU_STAT(sf_hblk1_nalloc); 9990 } 9991 9992 goto hblk_init; 9993 } 9994 9995 SFMMU_HASH_UNLOCK(hmebp); 9996 9997 if (sfmmup != KHATID) { 9998 if (mmu_page_sizes == max_mmu_page_sizes) { 9999 if (size < TTE256M) 10000 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 10001 size, flags); 10002 } else { 10003 if (size < TTE4M) 10004 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 10005 size, flags); 10006 } 10007 } 10008 10009 fill_hblk: 10010 owner = (hblk_reserve_thread == curthread) ? 1 : 0; 10011 10012 if (owner && size == TTE8K) { 10013 10014 /* 10015 * We are really in a tight spot. We already own 10016 * hblk_reserve and we need another hblk. In anticipation 10017 * of this kind of scenario, we specifically set aside 10018 * HBLK_RESERVE_MIN number of hblks to be used exclusively 10019 * by owner of hblk_reserve. 10020 */ 10021 SFMMU_STAT(sf_hblk_recurse_cnt); 10022 10023 if (!sfmmu_get_free_hblk(&hmeblkp, 1)) 10024 panic("sfmmu_hblk_alloc: reserve list is empty"); 10025 10026 goto hblk_verify; 10027 } 10028 10029 ASSERT(!owner); 10030 10031 if ((flags & HAT_NO_KALLOC) == 0) { 10032 10033 sfmmu_cache = ((size == TTE8K) ? sfmmu8_cache : sfmmu1_cache); 10034 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP); 10035 10036 if ((hmeblkp = kmem_cache_alloc(sfmmu_cache, sleep)) == NULL) { 10037 hmeblkp = sfmmu_hblk_steal(size); 10038 } else { 10039 /* 10040 * if we are the owner of hblk_reserve, 10041 * swap hblk_reserve with hmeblkp and 10042 * start a fresh life. Hope things go 10043 * better this time. 10044 */ 10045 if (hblk_reserve_thread == curthread) { 10046 ASSERT(sfmmu_cache == sfmmu8_cache); 10047 sfmmu_hblk_swap(hmeblkp); 10048 hblk_reserve_thread = NULL; 10049 mutex_exit(&hblk_reserve_lock); 10050 goto fill_hblk; 10051 } 10052 /* 10053 * let's donate this hblk to our reserve list if 10054 * we are not mapping kernel range 10055 */ 10056 if (size == TTE8K && sfmmup != KHATID) 10057 if (sfmmu_put_free_hblk(hmeblkp, 0)) 10058 goto fill_hblk; 10059 } 10060 } else { 10061 /* 10062 * We are here to map the slab in sfmmu8_cache; let's 10063 * check if we could tap our reserve list; if successful, 10064 * this will avoid the pain of going thru sfmmu_hblk_swap 10065 */ 10066 SFMMU_STAT(sf_hblk_slab_cnt); 10067 if (!sfmmu_get_free_hblk(&hmeblkp, 0)) { 10068 /* 10069 * let's start hblk_reserve dance 10070 */ 10071 SFMMU_STAT(sf_hblk_reserve_cnt); 10072 owner = 1; 10073 mutex_enter(&hblk_reserve_lock); 10074 hmeblkp = HBLK_RESERVE; 10075 hblk_reserve_thread = curthread; 10076 } 10077 } 10078 10079 hblk_verify: 10080 ASSERT(hmeblkp != NULL); 10081 set_hblk_sz(hmeblkp, size); 10082 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); 10083 SFMMU_HASH_LOCK(hmebp); 10084 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10085 if (newhblkp != NULL) { 10086 SFMMU_HASH_UNLOCK(hmebp); 10087 if (hmeblkp != HBLK_RESERVE) { 10088 /* 10089 * This is really tricky! 10090 * 10091 * vmem_alloc(vmem_seg_arena) 10092 * vmem_alloc(vmem_internal_arena) 10093 * segkmem_alloc(heap_arena) 10094 * vmem_alloc(heap_arena) 10095 * page_create() 10096 * hat_memload() 10097 * kmem_cache_free() 10098 * kmem_cache_alloc() 10099 * kmem_slab_create() 10100 * vmem_alloc(kmem_internal_arena) 10101 * segkmem_alloc(heap_arena) 10102 * vmem_alloc(heap_arena) 10103 * page_create() 10104 * hat_memload() 10105 * kmem_cache_free() 10106 * ... 10107 * 10108 * Thus, hat_memload() could call kmem_cache_free 10109 * for enough number of times that we could easily 10110 * hit the bottom of the stack or run out of reserve 10111 * list of vmem_seg structs. So, we must donate 10112 * this hblk to reserve list if it's allocated 10113 * from sfmmu8_cache *and* mapping kernel range. 10114 * We don't need to worry about freeing hmeblk1's 10115 * to kmem since they don't map any kmem slabs. 10116 * 10117 * Note: When segkmem supports largepages, we must 10118 * free hmeblk1's to reserve list as well. 10119 */ 10120 forcefree = (sfmmup == KHATID) ? 1 : 0; 10121 if (size == TTE8K && 10122 sfmmu_put_free_hblk(hmeblkp, forcefree)) { 10123 goto re_verify; 10124 } 10125 ASSERT(sfmmup != KHATID); 10126 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10127 } else { 10128 /* 10129 * Hey! we don't need hblk_reserve any more. 10130 */ 10131 ASSERT(owner); 10132 hblk_reserve_thread = NULL; 10133 mutex_exit(&hblk_reserve_lock); 10134 owner = 0; 10135 } 10136 re_verify: 10137 /* 10138 * let's check if the goodies are still present 10139 */ 10140 SFMMU_HASH_LOCK(hmebp); 10141 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10142 if (newhblkp != NULL) { 10143 /* 10144 * return newhblkp if it's not hblk_reserve; 10145 * if newhblkp is hblk_reserve, return it 10146 * _only if_ we are the owner of hblk_reserve. 10147 */ 10148 if (newhblkp != HBLK_RESERVE || owner) { 10149 return (newhblkp); 10150 } else { 10151 /* 10152 * we just hit hblk_reserve in the hash and 10153 * we are not the owner of that; 10154 * 10155 * block until hblk_reserve_thread completes 10156 * swapping hblk_reserve and try the dance 10157 * once again. 10158 */ 10159 SFMMU_HASH_UNLOCK(hmebp); 10160 mutex_enter(&hblk_reserve_lock); 10161 mutex_exit(&hblk_reserve_lock); 10162 SFMMU_STAT(sf_hblk_reserve_hit); 10163 goto fill_hblk; 10164 } 10165 } else { 10166 /* 10167 * it's no more! try the dance once again. 10168 */ 10169 SFMMU_HASH_UNLOCK(hmebp); 10170 goto fill_hblk; 10171 } 10172 } 10173 10174 hblk_init: 10175 set_hblk_sz(hmeblkp, size); 10176 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10177 hmeblkp->hblk_next = (struct hme_blk *)NULL; 10178 hmeblkp->hblk_tag = hblktag; 10179 hmeblkp->hblk_shadow = shw_hblkp; 10180 hblkpa = hmeblkp->hblk_nextpa; 10181 hmeblkp->hblk_nextpa = 0; 10182 10183 ASSERT(get_hblk_ttesz(hmeblkp) == size); 10184 ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size)); 10185 ASSERT(hmeblkp->hblk_hmecnt == 0); 10186 ASSERT(hmeblkp->hblk_vcnt == 0); 10187 ASSERT(hmeblkp->hblk_lckcnt == 0); 10188 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10189 sfmmu_hblk_hash_add(hmebp, hmeblkp, hblkpa); 10190 return (hmeblkp); 10191 } 10192 10193 /* 10194 * This function performs any cleanup required on the hme_blk 10195 * and returns it to the free list. 10196 */ 10197 /* ARGSUSED */ 10198 static void 10199 sfmmu_hblk_free(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10200 uint64_t hblkpa, struct hme_blk **listp) 10201 { 10202 int shw_size, vshift; 10203 struct hme_blk *shw_hblkp; 10204 uint_t shw_mask, newshw_mask; 10205 uintptr_t vaddr; 10206 int size; 10207 uint_t critical; 10208 10209 ASSERT(hmeblkp); 10210 ASSERT(!hmeblkp->hblk_hmecnt); 10211 ASSERT(!hmeblkp->hblk_vcnt); 10212 ASSERT(!hmeblkp->hblk_lckcnt); 10213 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10214 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 10215 10216 critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0; 10217 10218 size = get_hblk_ttesz(hmeblkp); 10219 shw_hblkp = hmeblkp->hblk_shadow; 10220 if (shw_hblkp) { 10221 ASSERT(hblktosfmmu(hmeblkp) != KHATID); 10222 if (mmu_page_sizes == max_mmu_page_sizes) { 10223 ASSERT(size < TTE256M); 10224 } else { 10225 ASSERT(size < TTE4M); 10226 } 10227 10228 shw_size = get_hblk_ttesz(shw_hblkp); 10229 vaddr = get_hblk_base(hmeblkp); 10230 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10231 ASSERT(vshift < 8); 10232 /* 10233 * Atomically clear shadow mask bit 10234 */ 10235 do { 10236 shw_mask = shw_hblkp->hblk_shw_mask; 10237 ASSERT(shw_mask & (1 << vshift)); 10238 newshw_mask = shw_mask & ~(1 << vshift); 10239 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10240 shw_mask, newshw_mask); 10241 } while (newshw_mask != shw_mask); 10242 hmeblkp->hblk_shadow = NULL; 10243 } 10244 hmeblkp->hblk_next = NULL; 10245 hmeblkp->hblk_nextpa = hblkpa; 10246 hmeblkp->hblk_shw_bit = 0; 10247 10248 if (hmeblkp->hblk_nuc_bit == 0) { 10249 10250 if (size == TTE8K && sfmmu_put_free_hblk(hmeblkp, critical)) 10251 return; 10252 10253 hmeblkp->hblk_next = *listp; 10254 *listp = hmeblkp; 10255 } 10256 } 10257 10258 static void 10259 sfmmu_hblks_list_purge(struct hme_blk **listp) 10260 { 10261 struct hme_blk *hmeblkp; 10262 10263 while ((hmeblkp = *listp) != NULL) { 10264 *listp = hmeblkp->hblk_next; 10265 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10266 } 10267 } 10268 10269 #define BUCKETS_TO_SEARCH_BEFORE_UNLOAD 30 10270 #define SFMMU_HBLK_STEAL_THRESHOLD 5 10271 10272 static uint_t sfmmu_hblk_steal_twice; 10273 static uint_t sfmmu_hblk_steal_count, sfmmu_hblk_steal_unload_count; 10274 10275 /* 10276 * Steal a hmeblk from user or kernel hme hash lists. 10277 * For 8K tte grab one from reserve pool (freehblkp) before proceeding to 10278 * steal and if we fail to steal after SFMMU_HBLK_STEAL_THRESHOLD attempts 10279 * tap into critical reserve of freehblkp. 10280 * Note: We remain looping in this routine until we find one. 10281 */ 10282 static struct hme_blk * 10283 sfmmu_hblk_steal(int size) 10284 { 10285 static struct hmehash_bucket *uhmehash_steal_hand = NULL; 10286 struct hmehash_bucket *hmebp; 10287 struct hme_blk *hmeblkp = NULL, *pr_hblk; 10288 uint64_t hblkpa, prevpa; 10289 int i; 10290 uint_t loop_cnt = 0, critical; 10291 10292 for (;;) { 10293 if (size == TTE8K) { 10294 critical = 10295 (++loop_cnt > SFMMU_HBLK_STEAL_THRESHOLD) ? 1 : 0; 10296 if (sfmmu_get_free_hblk(&hmeblkp, critical)) 10297 return (hmeblkp); 10298 } 10299 10300 hmebp = (uhmehash_steal_hand == NULL) ? uhme_hash : 10301 uhmehash_steal_hand; 10302 ASSERT(hmebp >= uhme_hash && hmebp <= &uhme_hash[UHMEHASH_SZ]); 10303 10304 for (i = 0; hmeblkp == NULL && i <= UHMEHASH_SZ + 10305 BUCKETS_TO_SEARCH_BEFORE_UNLOAD; i++) { 10306 SFMMU_HASH_LOCK(hmebp); 10307 hmeblkp = hmebp->hmeblkp; 10308 hblkpa = hmebp->hmeh_nextpa; 10309 prevpa = 0; 10310 pr_hblk = NULL; 10311 while (hmeblkp) { 10312 /* 10313 * check if it is a hmeblk that is not locked 10314 * and not shared. skip shadow hmeblks with 10315 * shadow_mask set i.e valid count non zero. 10316 */ 10317 if ((get_hblk_ttesz(hmeblkp) == size) && 10318 (hmeblkp->hblk_shw_bit == 0 || 10319 hmeblkp->hblk_vcnt == 0) && 10320 (hmeblkp->hblk_lckcnt == 0)) { 10321 /* 10322 * there is a high probability that we 10323 * will find a free one. search some 10324 * buckets for a free hmeblk initially 10325 * before unloading a valid hmeblk. 10326 */ 10327 if ((hmeblkp->hblk_vcnt == 0 && 10328 hmeblkp->hblk_hmecnt == 0) || (i >= 10329 BUCKETS_TO_SEARCH_BEFORE_UNLOAD)) { 10330 if (sfmmu_steal_this_hblk(hmebp, 10331 hmeblkp, hblkpa, prevpa, 10332 pr_hblk)) { 10333 /* 10334 * Hblk is unloaded 10335 * successfully 10336 */ 10337 break; 10338 } 10339 } 10340 } 10341 pr_hblk = hmeblkp; 10342 prevpa = hblkpa; 10343 hblkpa = hmeblkp->hblk_nextpa; 10344 hmeblkp = hmeblkp->hblk_next; 10345 } 10346 10347 SFMMU_HASH_UNLOCK(hmebp); 10348 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 10349 hmebp = uhme_hash; 10350 } 10351 uhmehash_steal_hand = hmebp; 10352 10353 if (hmeblkp != NULL) 10354 break; 10355 10356 /* 10357 * in the worst case, look for a free one in the kernel 10358 * hash table. 10359 */ 10360 for (i = 0, hmebp = khme_hash; i <= KHMEHASH_SZ; i++) { 10361 SFMMU_HASH_LOCK(hmebp); 10362 hmeblkp = hmebp->hmeblkp; 10363 hblkpa = hmebp->hmeh_nextpa; 10364 prevpa = 0; 10365 pr_hblk = NULL; 10366 while (hmeblkp) { 10367 /* 10368 * check if it is free hmeblk 10369 */ 10370 if ((get_hblk_ttesz(hmeblkp) == size) && 10371 (hmeblkp->hblk_lckcnt == 0) && 10372 (hmeblkp->hblk_vcnt == 0) && 10373 (hmeblkp->hblk_hmecnt == 0)) { 10374 if (sfmmu_steal_this_hblk(hmebp, 10375 hmeblkp, hblkpa, prevpa, pr_hblk)) { 10376 break; 10377 } else { 10378 /* 10379 * Cannot fail since we have 10380 * hash lock. 10381 */ 10382 panic("fail to steal?"); 10383 } 10384 } 10385 10386 pr_hblk = hmeblkp; 10387 prevpa = hblkpa; 10388 hblkpa = hmeblkp->hblk_nextpa; 10389 hmeblkp = hmeblkp->hblk_next; 10390 } 10391 10392 SFMMU_HASH_UNLOCK(hmebp); 10393 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 10394 hmebp = khme_hash; 10395 } 10396 10397 if (hmeblkp != NULL) 10398 break; 10399 sfmmu_hblk_steal_twice++; 10400 } 10401 return (hmeblkp); 10402 } 10403 10404 /* 10405 * This routine does real work to prepare a hblk to be "stolen" by 10406 * unloading the mappings, updating shadow counts .... 10407 * It returns 1 if the block is ready to be reused (stolen), or 0 10408 * means the block cannot be stolen yet- pageunload is still working 10409 * on this hblk. 10410 */ 10411 static int 10412 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10413 uint64_t hblkpa, uint64_t prevpa, struct hme_blk *pr_hblk) 10414 { 10415 int shw_size, vshift; 10416 struct hme_blk *shw_hblkp; 10417 uintptr_t vaddr; 10418 uint_t shw_mask, newshw_mask; 10419 10420 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10421 10422 /* 10423 * check if the hmeblk is free, unload if necessary 10424 */ 10425 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10426 sfmmu_t *sfmmup; 10427 demap_range_t dmr; 10428 10429 sfmmup = hblktosfmmu(hmeblkp); 10430 DEMAP_RANGE_INIT(sfmmup, &dmr); 10431 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 10432 (caddr_t)get_hblk_base(hmeblkp), 10433 get_hblk_endaddr(hmeblkp), &dmr, HAT_UNLOAD); 10434 DEMAP_RANGE_FLUSH(&dmr); 10435 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10436 /* 10437 * Pageunload is working on the same hblk. 10438 */ 10439 return (0); 10440 } 10441 10442 sfmmu_hblk_steal_unload_count++; 10443 } 10444 10445 ASSERT(hmeblkp->hblk_lckcnt == 0); 10446 ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0); 10447 10448 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 10449 hmeblkp->hblk_nextpa = hblkpa; 10450 10451 shw_hblkp = hmeblkp->hblk_shadow; 10452 if (shw_hblkp) { 10453 shw_size = get_hblk_ttesz(shw_hblkp); 10454 vaddr = get_hblk_base(hmeblkp); 10455 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10456 ASSERT(vshift < 8); 10457 /* 10458 * Atomically clear shadow mask bit 10459 */ 10460 do { 10461 shw_mask = shw_hblkp->hblk_shw_mask; 10462 ASSERT(shw_mask & (1 << vshift)); 10463 newshw_mask = shw_mask & ~(1 << vshift); 10464 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10465 shw_mask, newshw_mask); 10466 } while (newshw_mask != shw_mask); 10467 hmeblkp->hblk_shadow = NULL; 10468 } 10469 10470 /* 10471 * remove shadow bit if we are stealing an unused shadow hmeblk. 10472 * sfmmu_hblk_alloc needs it that way, will set shadow bit later if 10473 * we are indeed allocating a shadow hmeblk. 10474 */ 10475 hmeblkp->hblk_shw_bit = 0; 10476 10477 sfmmu_hblk_steal_count++; 10478 SFMMU_STAT(sf_steal_count); 10479 10480 return (1); 10481 } 10482 10483 struct hme_blk * 10484 sfmmu_hmetohblk(struct sf_hment *sfhme) 10485 { 10486 struct hme_blk *hmeblkp; 10487 struct sf_hment *sfhme0; 10488 struct hme_blk *hblk_dummy = 0; 10489 10490 /* 10491 * No dummy sf_hments, please. 10492 */ 10493 ASSERT(sfhme->hme_tte.ll != 0); 10494 10495 sfhme0 = sfhme - sfhme->hme_tte.tte_hmenum; 10496 hmeblkp = (struct hme_blk *)((uintptr_t)sfhme0 - 10497 (uintptr_t)&hblk_dummy->hblk_hme[0]); 10498 10499 return (hmeblkp); 10500 } 10501 10502 /* 10503 * On swapin, get appropriately sized TSB(s) and clear the HAT_SWAPPED flag. 10504 * If we can't get appropriately sized TSB(s), try for 8K TSB(s) using 10505 * KM_SLEEP allocation. 10506 * 10507 * Return 0 on success, -1 otherwise. 10508 */ 10509 static void 10510 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp) 10511 { 10512 struct tsb_info *tsbinfop, *next; 10513 tsb_replace_rc_t rc; 10514 boolean_t gotfirst = B_FALSE; 10515 10516 ASSERT(sfmmup != ksfmmup); 10517 ASSERT(sfmmu_hat_lock_held(sfmmup)); 10518 10519 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) { 10520 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 10521 } 10522 10523 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10524 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN); 10525 } else { 10526 return; 10527 } 10528 10529 ASSERT(sfmmup->sfmmu_tsb != NULL); 10530 10531 /* 10532 * Loop over all tsbinfo's replacing them with ones that actually have 10533 * a TSB. If any of the replacements ever fail, bail out of the loop. 10534 */ 10535 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) { 10536 ASSERT(tsbinfop->tsb_flags & TSB_SWAPPED); 10537 next = tsbinfop->tsb_next; 10538 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc, 10539 hatlockp, TSB_SWAPIN); 10540 if (rc != TSB_SUCCESS) { 10541 break; 10542 } 10543 gotfirst = B_TRUE; 10544 } 10545 10546 switch (rc) { 10547 case TSB_SUCCESS: 10548 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10549 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10550 return; 10551 case TSB_ALLOCFAIL: 10552 break; 10553 default: 10554 panic("sfmmu_replace_tsb returned unrecognized failure code " 10555 "%d", rc); 10556 } 10557 10558 /* 10559 * In this case, we failed to get one of our TSBs. If we failed to 10560 * get the first TSB, get one of minimum size (8KB). Walk the list 10561 * and throw away the tsbinfos, starting where the allocation failed; 10562 * we can get by with just one TSB as long as we don't leave the 10563 * SWAPPED tsbinfo structures lying around. 10564 */ 10565 tsbinfop = sfmmup->sfmmu_tsb; 10566 next = tsbinfop->tsb_next; 10567 tsbinfop->tsb_next = NULL; 10568 10569 sfmmu_hat_exit(hatlockp); 10570 for (tsbinfop = next; tsbinfop != NULL; tsbinfop = next) { 10571 next = tsbinfop->tsb_next; 10572 sfmmu_tsbinfo_free(tsbinfop); 10573 } 10574 hatlockp = sfmmu_hat_enter(sfmmup); 10575 10576 /* 10577 * If we don't have any TSBs, get a single 8K TSB for 8K, 64K and 512K 10578 * pages. 10579 */ 10580 if (!gotfirst) { 10581 tsbinfop = sfmmup->sfmmu_tsb; 10582 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE, 10583 hatlockp, TSB_SWAPIN | TSB_FORCEALLOC); 10584 ASSERT(rc == TSB_SUCCESS); 10585 } else { 10586 /* update machine specific tsbinfo */ 10587 sfmmu_setup_tsbinfo(sfmmup); 10588 } 10589 10590 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10591 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10592 } 10593 10594 /* 10595 * Handle exceptions for low level tsb_handler. 10596 * 10597 * There are many scenarios that could land us here: 10598 * 10599 * If the context is invalid we land here. The context can be invalid 10600 * for 3 reasons: 1) we couldn't allocate a new context and now need to 10601 * perform a wrap around operation in order to allocate a new context. 10602 * 2) Context was invalidated to change pagesize programming 3) ISMs or 10603 * TSBs configuration is changeing for this process and we are forced into 10604 * here to do a syncronization operation. If the context is valid we can 10605 * be here from window trap hanlder. In this case just call trap to handle 10606 * the fault. 10607 * 10608 * Note that the process will run in INVALID_CONTEXT before 10609 * faulting into here and subsequently loading the MMU registers 10610 * (including the TSB base register) associated with this process. 10611 * For this reason, the trap handlers must all test for 10612 * INVALID_CONTEXT before attempting to access any registers other 10613 * than the context registers. 10614 */ 10615 void 10616 sfmmu_tsbmiss_exception(struct regs *rp, uintptr_t tagaccess, uint_t traptype) 10617 { 10618 sfmmu_t *sfmmup; 10619 uint_t ctxtype; 10620 klwp_id_t lwp; 10621 char lwp_save_state; 10622 hatlock_t *hatlockp; 10623 struct tsb_info *tsbinfop; 10624 10625 SFMMU_STAT(sf_tsb_exceptions); 10626 SFMMU_MMU_STAT(mmu_tsb_exceptions); 10627 sfmmup = astosfmmu(curthread->t_procp->p_as); 10628 /* 10629 * note that in sun4u, tagacces register contains ctxnum 10630 * while sun4v passes ctxtype in the tagaccess register. 10631 */ 10632 ctxtype = tagaccess & TAGACC_CTX_MASK; 10633 10634 ASSERT(sfmmup != ksfmmup && ctxtype != KCONTEXT); 10635 ASSERT(sfmmup->sfmmu_ismhat == 0); 10636 /* 10637 * First, make sure we come out of here with a valid ctx, 10638 * since if we don't get one we'll simply loop on the 10639 * faulting instruction. 10640 * 10641 * If the ISM mappings are changing, the TSB is being relocated, or 10642 * the process is swapped out we serialize behind the controlling 10643 * thread with the sfmmu_flags and sfmmu_tsb_cv condition variable. 10644 * Otherwise we synchronize with the context stealer or the thread 10645 * that required us to change out our MMU registers (such 10646 * as a thread changing out our TSB while we were running) by 10647 * locking the HAT and grabbing the rwlock on the context as a 10648 * reader temporarily. 10649 */ 10650 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) || 10651 ctxtype == INVALID_CONTEXT); 10652 10653 if (ctxtype == INVALID_CONTEXT) { 10654 /* 10655 * Must set lwp state to LWP_SYS before 10656 * trying to acquire any adaptive lock 10657 */ 10658 lwp = ttolwp(curthread); 10659 ASSERT(lwp); 10660 lwp_save_state = lwp->lwp_state; 10661 lwp->lwp_state = LWP_SYS; 10662 10663 hatlockp = sfmmu_hat_enter(sfmmup); 10664 retry: 10665 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 10666 tsbinfop = tsbinfop->tsb_next) { 10667 if (tsbinfop->tsb_flags & TSB_RELOC_FLAG) { 10668 cv_wait(&sfmmup->sfmmu_tsb_cv, 10669 HATLOCK_MUTEXP(hatlockp)); 10670 goto retry; 10671 } 10672 } 10673 10674 /* 10675 * Wait for ISM maps to be updated. 10676 */ 10677 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) { 10678 cv_wait(&sfmmup->sfmmu_tsb_cv, 10679 HATLOCK_MUTEXP(hatlockp)); 10680 goto retry; 10681 } 10682 10683 /* 10684 * If we're swapping in, get TSB(s). Note that we must do 10685 * this before we get a ctx or load the MMU state. Once 10686 * we swap in we have to recheck to make sure the TSB(s) and 10687 * ISM mappings didn't change while we slept. 10688 */ 10689 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10690 sfmmu_tsb_swapin(sfmmup, hatlockp); 10691 goto retry; 10692 } 10693 10694 sfmmu_get_ctx(sfmmup); 10695 10696 sfmmu_hat_exit(hatlockp); 10697 /* 10698 * Must restore lwp_state if not calling 10699 * trap() for further processing. Restore 10700 * it anyway. 10701 */ 10702 lwp->lwp_state = lwp_save_state; 10703 if (sfmmup->sfmmu_ttecnt[TTE8K] != 0 || 10704 sfmmup->sfmmu_ttecnt[TTE64K] != 0 || 10705 sfmmup->sfmmu_ttecnt[TTE512K] != 0 || 10706 sfmmup->sfmmu_ttecnt[TTE4M] != 0 || 10707 sfmmup->sfmmu_ttecnt[TTE32M] != 0 || 10708 sfmmup->sfmmu_ttecnt[TTE256M] != 0) { 10709 return; 10710 } 10711 if (traptype == T_DATA_PROT) { 10712 traptype = T_DATA_MMU_MISS; 10713 } 10714 } 10715 trap(rp, (caddr_t)tagaccess, traptype, 0); 10716 } 10717 10718 /* 10719 * sfmmu_vatopfn_suspended is called from GET_TTE when TL=0 and 10720 * TTE_SUSPENDED bit set in tte we block on aquiring a page lock 10721 * rather than spinning to avoid send mondo timeouts with 10722 * interrupts enabled. When the lock is acquired it is immediately 10723 * released and we return back to sfmmu_vatopfn just after 10724 * the GET_TTE call. 10725 */ 10726 void 10727 sfmmu_vatopfn_suspended(caddr_t vaddr, sfmmu_t *sfmmu, tte_t *ttep) 10728 { 10729 struct page **pp; 10730 10731 (void) as_pagelock(sfmmu->sfmmu_as, &pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10732 as_pageunlock(sfmmu->sfmmu_as, pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10733 } 10734 10735 /* 10736 * sfmmu_tsbmiss_suspended is called from GET_TTE when TL>0 and 10737 * TTE_SUSPENDED bit set in tte. We do this so that we can handle 10738 * cross traps which cannot be handled while spinning in the 10739 * trap handlers. Simply enter and exit the kpr_suspendlock spin 10740 * mutex, which is held by the holder of the suspend bit, and then 10741 * retry the trapped instruction after unwinding. 10742 */ 10743 /*ARGSUSED*/ 10744 void 10745 sfmmu_tsbmiss_suspended(struct regs *rp, uintptr_t tagacc, uint_t traptype) 10746 { 10747 ASSERT(curthread != kreloc_thread); 10748 mutex_enter(&kpr_suspendlock); 10749 mutex_exit(&kpr_suspendlock); 10750 } 10751 10752 /* 10753 * Special routine to flush out ism mappings- TSBs, TLBs and D-caches. 10754 * This routine may be called with all cpu's captured. Therefore, the 10755 * caller is responsible for holding all locks and disabling kernel 10756 * preemption. 10757 */ 10758 /* ARGSUSED */ 10759 static void 10760 sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, 10761 struct hme_blk *hmeblkp, pfn_t pfnum, int cache_flush_flag) 10762 { 10763 cpuset_t cpuset; 10764 caddr_t va; 10765 ism_ment_t *ment; 10766 sfmmu_t *sfmmup; 10767 #ifdef VAC 10768 int vcolor; 10769 #endif 10770 int ttesz; 10771 10772 /* 10773 * Walk the ism_hat's mapping list and flush the page 10774 * from every hat sharing this ism_hat. This routine 10775 * may be called while all cpu's have been captured. 10776 * Therefore we can't attempt to grab any locks. For now 10777 * this means we will protect the ism mapping list under 10778 * a single lock which will be grabbed by the caller. 10779 * If hat_share/unshare scalibility becomes a performance 10780 * problem then we may need to re-think ism mapping list locking. 10781 */ 10782 ASSERT(ism_sfmmup->sfmmu_ismhat); 10783 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 10784 addr = addr - ISMID_STARTADDR; 10785 for (ment = ism_sfmmup->sfmmu_iment; ment; ment = ment->iment_next) { 10786 10787 sfmmup = ment->iment_hat; 10788 10789 va = ment->iment_base_va; 10790 va = (caddr_t)((uintptr_t)va + (uintptr_t)addr); 10791 10792 /* 10793 * Flush TSB of ISM mappings. 10794 */ 10795 ttesz = get_hblk_ttesz(hmeblkp); 10796 if (ttesz == TTE8K || ttesz == TTE4M) { 10797 sfmmu_unload_tsb(sfmmup, va, ttesz); 10798 } else { 10799 caddr_t sva = va; 10800 caddr_t eva; 10801 ASSERT(addr == (caddr_t)get_hblk_base(hmeblkp)); 10802 eva = sva + get_hblk_span(hmeblkp); 10803 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); 10804 } 10805 10806 cpuset = sfmmup->sfmmu_cpusran; 10807 CPUSET_AND(cpuset, cpu_ready_set); 10808 CPUSET_DEL(cpuset, CPU->cpu_id); 10809 10810 SFMMU_XCALL_STATS(sfmmup); 10811 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)va, 10812 (uint64_t)sfmmup); 10813 10814 vtag_flushpage(va, (uint64_t)sfmmup); 10815 10816 #ifdef VAC 10817 /* 10818 * Flush D$ 10819 * When flushing D$ we must flush all 10820 * cpu's. See sfmmu_cache_flush(). 10821 */ 10822 if (cache_flush_flag == CACHE_FLUSH) { 10823 cpuset = cpu_ready_set; 10824 CPUSET_DEL(cpuset, CPU->cpu_id); 10825 10826 SFMMU_XCALL_STATS(sfmmup); 10827 vcolor = addr_to_vcolor(va); 10828 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10829 vac_flushpage(pfnum, vcolor); 10830 } 10831 #endif /* VAC */ 10832 } 10833 } 10834 10835 /* 10836 * Demaps the TSB, CPU caches, and flushes all TLBs on all CPUs of 10837 * a particular virtual address and ctx. If noflush is set we do not 10838 * flush the TLB/TSB. This function may or may not be called with the 10839 * HAT lock held. 10840 */ 10841 static void 10842 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10843 pfn_t pfnum, int tlb_noflush, int cpu_flag, int cache_flush_flag, 10844 int hat_lock_held) 10845 { 10846 #ifdef VAC 10847 int vcolor; 10848 #endif 10849 cpuset_t cpuset; 10850 hatlock_t *hatlockp; 10851 10852 #if defined(lint) && !defined(VAC) 10853 pfnum = pfnum; 10854 cpu_flag = cpu_flag; 10855 cache_flush_flag = cache_flush_flag; 10856 #endif 10857 /* 10858 * There is no longer a need to protect against ctx being 10859 * stolen here since we don't store the ctx in the TSB anymore. 10860 */ 10861 #ifdef VAC 10862 vcolor = addr_to_vcolor(addr); 10863 #endif 10864 10865 /* 10866 * We must hold the hat lock during the flush of TLB, 10867 * to avoid a race with sfmmu_invalidate_ctx(), where 10868 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 10869 * causing TLB demap routine to skip flush on that MMU. 10870 * If the context on a MMU has already been set to 10871 * INVALID_CONTEXT, we just get an extra flush on 10872 * that MMU. 10873 */ 10874 if (!hat_lock_held && !tlb_noflush) 10875 hatlockp = sfmmu_hat_enter(sfmmup); 10876 10877 kpreempt_disable(); 10878 if (!tlb_noflush) { 10879 /* 10880 * Flush the TSB and TLB. 10881 */ 10882 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10883 10884 cpuset = sfmmup->sfmmu_cpusran; 10885 CPUSET_AND(cpuset, cpu_ready_set); 10886 CPUSET_DEL(cpuset, CPU->cpu_id); 10887 10888 SFMMU_XCALL_STATS(sfmmup); 10889 10890 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 10891 (uint64_t)sfmmup); 10892 10893 vtag_flushpage(addr, (uint64_t)sfmmup); 10894 } 10895 10896 if (!hat_lock_held && !tlb_noflush) 10897 sfmmu_hat_exit(hatlockp); 10898 10899 #ifdef VAC 10900 /* 10901 * Flush the D$ 10902 * 10903 * Even if the ctx is stolen, we need to flush the 10904 * cache. Our ctx stealer only flushes the TLBs. 10905 */ 10906 if (cache_flush_flag == CACHE_FLUSH) { 10907 if (cpu_flag & FLUSH_ALL_CPUS) { 10908 cpuset = cpu_ready_set; 10909 } else { 10910 cpuset = sfmmup->sfmmu_cpusran; 10911 CPUSET_AND(cpuset, cpu_ready_set); 10912 } 10913 CPUSET_DEL(cpuset, CPU->cpu_id); 10914 SFMMU_XCALL_STATS(sfmmup); 10915 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10916 vac_flushpage(pfnum, vcolor); 10917 } 10918 #endif /* VAC */ 10919 kpreempt_enable(); 10920 } 10921 10922 /* 10923 * Demaps the TSB and flushes all TLBs on all cpus for a particular virtual 10924 * address and ctx. If noflush is set we do not currently do anything. 10925 * This function may or may not be called with the HAT lock held. 10926 */ 10927 static void 10928 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10929 int tlb_noflush, int hat_lock_held) 10930 { 10931 cpuset_t cpuset; 10932 hatlock_t *hatlockp; 10933 10934 /* 10935 * If the process is exiting we have nothing to do. 10936 */ 10937 if (tlb_noflush) 10938 return; 10939 10940 /* 10941 * Flush TSB. 10942 */ 10943 if (!hat_lock_held) 10944 hatlockp = sfmmu_hat_enter(sfmmup); 10945 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10946 10947 kpreempt_disable(); 10948 10949 cpuset = sfmmup->sfmmu_cpusran; 10950 CPUSET_AND(cpuset, cpu_ready_set); 10951 CPUSET_DEL(cpuset, CPU->cpu_id); 10952 10953 SFMMU_XCALL_STATS(sfmmup); 10954 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup); 10955 10956 vtag_flushpage(addr, (uint64_t)sfmmup); 10957 10958 if (!hat_lock_held) 10959 sfmmu_hat_exit(hatlockp); 10960 10961 kpreempt_enable(); 10962 10963 } 10964 10965 /* 10966 * Special case of sfmmu_tlb_demap for MMU_PAGESIZE hblks. Use the xcall 10967 * call handler that can flush a range of pages to save on xcalls. 10968 */ 10969 static int sfmmu_xcall_save; 10970 10971 static void 10972 sfmmu_tlb_range_demap(demap_range_t *dmrp) 10973 { 10974 sfmmu_t *sfmmup = dmrp->dmr_sfmmup; 10975 hatlock_t *hatlockp; 10976 cpuset_t cpuset; 10977 uint64_t sfmmu_pgcnt; 10978 pgcnt_t pgcnt = 0; 10979 int pgunload = 0; 10980 int dirtypg = 0; 10981 caddr_t addr = dmrp->dmr_addr; 10982 caddr_t eaddr; 10983 uint64_t bitvec = dmrp->dmr_bitvec; 10984 10985 ASSERT(bitvec & 1); 10986 10987 /* 10988 * Flush TSB and calculate number of pages to flush. 10989 */ 10990 while (bitvec != 0) { 10991 dirtypg = 0; 10992 /* 10993 * Find the first page to flush and then count how many 10994 * pages there are after it that also need to be flushed. 10995 * This way the number of TSB flushes is minimized. 10996 */ 10997 while ((bitvec & 1) == 0) { 10998 pgcnt++; 10999 addr += MMU_PAGESIZE; 11000 bitvec >>= 1; 11001 } 11002 while (bitvec & 1) { 11003 dirtypg++; 11004 bitvec >>= 1; 11005 } 11006 eaddr = addr + ptob(dirtypg); 11007 hatlockp = sfmmu_hat_enter(sfmmup); 11008 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K); 11009 sfmmu_hat_exit(hatlockp); 11010 pgunload += dirtypg; 11011 addr = eaddr; 11012 pgcnt += dirtypg; 11013 } 11014 11015 ASSERT((pgcnt<<MMU_PAGESHIFT) <= dmrp->dmr_endaddr - dmrp->dmr_addr); 11016 if (sfmmup->sfmmu_free == 0) { 11017 addr = dmrp->dmr_addr; 11018 bitvec = dmrp->dmr_bitvec; 11019 11020 /* 11021 * make sure it has SFMMU_PGCNT_SHIFT bits only, 11022 * as it will be used to pack argument for xt_some 11023 */ 11024 ASSERT((pgcnt > 0) && 11025 (pgcnt <= (1 << SFMMU_PGCNT_SHIFT))); 11026 11027 /* 11028 * Encode pgcnt as (pgcnt -1 ), and pass (pgcnt - 1) in 11029 * the low 6 bits of sfmmup. This is doable since pgcnt 11030 * always >= 1. 11031 */ 11032 ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK)); 11033 sfmmu_pgcnt = (uint64_t)sfmmup | 11034 ((pgcnt - 1) & SFMMU_PGCNT_MASK); 11035 11036 /* 11037 * We must hold the hat lock during the flush of TLB, 11038 * to avoid a race with sfmmu_invalidate_ctx(), where 11039 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 11040 * causing TLB demap routine to skip flush on that MMU. 11041 * If the context on a MMU has already been set to 11042 * INVALID_CONTEXT, we just get an extra flush on 11043 * that MMU. 11044 */ 11045 hatlockp = sfmmu_hat_enter(sfmmup); 11046 kpreempt_disable(); 11047 11048 cpuset = sfmmup->sfmmu_cpusran; 11049 CPUSET_AND(cpuset, cpu_ready_set); 11050 CPUSET_DEL(cpuset, CPU->cpu_id); 11051 11052 SFMMU_XCALL_STATS(sfmmup); 11053 xt_some(cpuset, vtag_flush_pgcnt_tl1, (uint64_t)addr, 11054 sfmmu_pgcnt); 11055 11056 for (; bitvec != 0; bitvec >>= 1) { 11057 if (bitvec & 1) 11058 vtag_flushpage(addr, (uint64_t)sfmmup); 11059 addr += MMU_PAGESIZE; 11060 } 11061 kpreempt_enable(); 11062 sfmmu_hat_exit(hatlockp); 11063 11064 sfmmu_xcall_save += (pgunload-1); 11065 } 11066 dmrp->dmr_bitvec = 0; 11067 } 11068 11069 /* 11070 * In cases where we need to synchronize with TLB/TSB miss trap 11071 * handlers, _and_ need to flush the TLB, it's a lot easier to 11072 * throw away the context from the process than to do a 11073 * special song and dance to keep things consistent for the 11074 * handlers. 11075 * 11076 * Since the process suddenly ends up without a context and our caller 11077 * holds the hat lock, threads that fault after this function is called 11078 * will pile up on the lock. We can then do whatever we need to 11079 * atomically from the context of the caller. The first blocked thread 11080 * to resume executing will get the process a new context, and the 11081 * process will resume executing. 11082 * 11083 * One added advantage of this approach is that on MMUs that 11084 * support a "flush all" operation, we will delay the flush until 11085 * cnum wrap-around, and then flush the TLB one time. This 11086 * is rather rare, so it's a lot less expensive than making 8000 11087 * x-calls to flush the TLB 8000 times. 11088 * 11089 * A per-process (PP) lock is used to synchronize ctx allocations in 11090 * resume() and ctx invalidations here. 11091 */ 11092 static void 11093 sfmmu_invalidate_ctx(sfmmu_t *sfmmup) 11094 { 11095 cpuset_t cpuset; 11096 int cnum, currcnum; 11097 mmu_ctx_t *mmu_ctxp; 11098 int i; 11099 uint_t pstate_save; 11100 11101 SFMMU_STAT(sf_ctx_inv); 11102 11103 ASSERT(sfmmu_hat_lock_held(sfmmup)); 11104 ASSERT(sfmmup != ksfmmup); 11105 11106 kpreempt_disable(); 11107 11108 mmu_ctxp = CPU_MMU_CTXP(CPU); 11109 ASSERT(mmu_ctxp); 11110 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 11111 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 11112 11113 currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; 11114 11115 pstate_save = sfmmu_disable_intrs(); 11116 11117 lock_set(&sfmmup->sfmmu_ctx_lock); /* acquire PP lock */ 11118 /* set HAT cnum invalid across all context domains. */ 11119 for (i = 0; i < max_mmu_ctxdoms; i++) { 11120 11121 cnum = sfmmup->sfmmu_ctxs[i].cnum; 11122 if (cnum == INVALID_CONTEXT) { 11123 continue; 11124 } 11125 11126 sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT; 11127 } 11128 membar_enter(); /* make sure globally visible to all CPUs */ 11129 lock_clear(&sfmmup->sfmmu_ctx_lock); /* release PP lock */ 11130 11131 sfmmu_enable_intrs(pstate_save); 11132 11133 cpuset = sfmmup->sfmmu_cpusran; 11134 CPUSET_DEL(cpuset, CPU->cpu_id); 11135 CPUSET_AND(cpuset, cpu_ready_set); 11136 if (!CPUSET_ISNULL(cpuset)) { 11137 SFMMU_XCALL_STATS(sfmmup); 11138 xt_some(cpuset, sfmmu_raise_tsb_exception, 11139 (uint64_t)sfmmup, INVALID_CONTEXT); 11140 xt_sync(cpuset); 11141 SFMMU_STAT(sf_tsb_raise_exception); 11142 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 11143 } 11144 11145 /* 11146 * If the hat to-be-invalidated is the same as the current 11147 * process on local CPU we need to invalidate 11148 * this CPU context as well. 11149 */ 11150 if ((sfmmu_getctx_sec() == currcnum) && 11151 (currcnum != INVALID_CONTEXT)) { 11152 sfmmu_setctx_sec(INVALID_CONTEXT); 11153 sfmmu_clear_utsbinfo(); 11154 } 11155 11156 kpreempt_enable(); 11157 11158 /* 11159 * we hold the hat lock, so nobody should allocate a context 11160 * for us yet 11161 */ 11162 ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT); 11163 } 11164 11165 #ifdef VAC 11166 /* 11167 * We need to flush the cache in all cpus. It is possible that 11168 * a process referenced a page as cacheable but has sinced exited 11169 * and cleared the mapping list. We still to flush it but have no 11170 * state so all cpus is the only alternative. 11171 */ 11172 void 11173 sfmmu_cache_flush(pfn_t pfnum, int vcolor) 11174 { 11175 cpuset_t cpuset; 11176 11177 kpreempt_disable(); 11178 cpuset = cpu_ready_set; 11179 CPUSET_DEL(cpuset, CPU->cpu_id); 11180 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11181 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 11182 xt_sync(cpuset); 11183 vac_flushpage(pfnum, vcolor); 11184 kpreempt_enable(); 11185 } 11186 11187 void 11188 sfmmu_cache_flushcolor(int vcolor, pfn_t pfnum) 11189 { 11190 cpuset_t cpuset; 11191 11192 ASSERT(vcolor >= 0); 11193 11194 kpreempt_disable(); 11195 cpuset = cpu_ready_set; 11196 CPUSET_DEL(cpuset, CPU->cpu_id); 11197 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11198 xt_some(cpuset, vac_flushcolor_tl1, vcolor, pfnum); 11199 xt_sync(cpuset); 11200 vac_flushcolor(vcolor, pfnum); 11201 kpreempt_enable(); 11202 } 11203 #endif /* VAC */ 11204 11205 /* 11206 * We need to prevent processes from accessing the TSB using a cached physical 11207 * address. It's alright if they try to access the TSB via virtual address 11208 * since they will just fault on that virtual address once the mapping has 11209 * been suspended. 11210 */ 11211 #pragma weak sendmondo_in_recover 11212 11213 /* ARGSUSED */ 11214 static int 11215 sfmmu_tsb_pre_relocator(caddr_t va, uint_t tsbsz, uint_t flags, void *tsbinfo) 11216 { 11217 hatlock_t *hatlockp; 11218 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11219 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11220 extern uint32_t sendmondo_in_recover; 11221 11222 if (flags != HAT_PRESUSPEND) 11223 return (0); 11224 11225 hatlockp = sfmmu_hat_enter(sfmmup); 11226 11227 tsbinfop->tsb_flags |= TSB_RELOC_FLAG; 11228 11229 /* 11230 * For Cheetah+ Erratum 25: 11231 * Wait for any active recovery to finish. We can't risk 11232 * relocating the TSB of the thread running mondo_recover_proc() 11233 * since, if we did that, we would deadlock. The scenario we are 11234 * trying to avoid is as follows: 11235 * 11236 * THIS CPU RECOVER CPU 11237 * -------- ----------- 11238 * Begins recovery, walking through TSB 11239 * hat_pagesuspend() TSB TTE 11240 * TLB miss on TSB TTE, spins at TL1 11241 * xt_sync() 11242 * send_mondo_timeout() 11243 * mondo_recover_proc() 11244 * ((deadlocked)) 11245 * 11246 * The second half of the workaround is that mondo_recover_proc() 11247 * checks to see if the tsb_info has the RELOC flag set, and if it 11248 * does, it skips over that TSB without ever touching tsbinfop->tsb_va 11249 * and hence avoiding the TLB miss that could result in a deadlock. 11250 */ 11251 if (&sendmondo_in_recover) { 11252 membar_enter(); /* make sure RELOC flag visible */ 11253 while (sendmondo_in_recover) { 11254 drv_usecwait(1); 11255 membar_consumer(); 11256 } 11257 } 11258 11259 sfmmu_invalidate_ctx(sfmmup); 11260 sfmmu_hat_exit(hatlockp); 11261 11262 return (0); 11263 } 11264 11265 /* ARGSUSED */ 11266 static int 11267 sfmmu_tsb_post_relocator(caddr_t va, uint_t tsbsz, uint_t flags, 11268 void *tsbinfo, pfn_t newpfn) 11269 { 11270 hatlock_t *hatlockp; 11271 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11272 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11273 11274 if (flags != HAT_POSTUNSUSPEND) 11275 return (0); 11276 11277 hatlockp = sfmmu_hat_enter(sfmmup); 11278 11279 SFMMU_STAT(sf_tsb_reloc); 11280 11281 /* 11282 * The process may have swapped out while we were relocating one 11283 * of its TSBs. If so, don't bother doing the setup since the 11284 * process can't be using the memory anymore. 11285 */ 11286 if ((tsbinfop->tsb_flags & TSB_SWAPPED) == 0) { 11287 ASSERT(va == tsbinfop->tsb_va); 11288 sfmmu_tsbinfo_setup_phys(tsbinfop, newpfn); 11289 sfmmu_setup_tsbinfo(sfmmup); 11290 11291 if (tsbinfop->tsb_flags & TSB_FLUSH_NEEDED) { 11292 sfmmu_inv_tsb(tsbinfop->tsb_va, 11293 TSB_BYTES(tsbinfop->tsb_szc)); 11294 tsbinfop->tsb_flags &= ~TSB_FLUSH_NEEDED; 11295 } 11296 } 11297 11298 membar_exit(); 11299 tsbinfop->tsb_flags &= ~TSB_RELOC_FLAG; 11300 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 11301 11302 sfmmu_hat_exit(hatlockp); 11303 11304 return (0); 11305 } 11306 11307 /* 11308 * Allocate and initialize a tsb_info structure. Note that we may or may not 11309 * allocate a TSB here, depending on the flags passed in. 11310 */ 11311 static int 11312 sfmmu_tsbinfo_alloc(struct tsb_info **tsbinfopp, int tsb_szc, int tte_sz_mask, 11313 uint_t flags, sfmmu_t *sfmmup) 11314 { 11315 int err; 11316 11317 *tsbinfopp = (struct tsb_info *)kmem_cache_alloc( 11318 sfmmu_tsbinfo_cache, KM_SLEEP); 11319 11320 if ((err = sfmmu_init_tsbinfo(*tsbinfopp, tte_sz_mask, 11321 tsb_szc, flags, sfmmup)) != 0) { 11322 kmem_cache_free(sfmmu_tsbinfo_cache, *tsbinfopp); 11323 SFMMU_STAT(sf_tsb_allocfail); 11324 *tsbinfopp = NULL; 11325 return (err); 11326 } 11327 SFMMU_STAT(sf_tsb_alloc); 11328 11329 /* 11330 * Bump the TSB size counters for this TSB size. 11331 */ 11332 (*(((int *)&sfmmu_tsbsize_stat) + tsb_szc))++; 11333 return (0); 11334 } 11335 11336 static void 11337 sfmmu_tsb_free(struct tsb_info *tsbinfo) 11338 { 11339 caddr_t tsbva = tsbinfo->tsb_va; 11340 uint_t tsb_size = TSB_BYTES(tsbinfo->tsb_szc); 11341 struct kmem_cache *kmem_cachep = tsbinfo->tsb_cache; 11342 vmem_t *vmp = tsbinfo->tsb_vmp; 11343 11344 /* 11345 * If we allocated this TSB from relocatable kernel memory, then we 11346 * need to uninstall the callback handler. 11347 */ 11348 if (tsbinfo->tsb_cache != sfmmu_tsb8k_cache) { 11349 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11350 caddr_t slab_vaddr = (caddr_t)((uintptr_t)tsbva & slab_mask); 11351 page_t **ppl; 11352 int ret; 11353 11354 ret = as_pagelock(&kas, &ppl, slab_vaddr, PAGESIZE, S_WRITE); 11355 ASSERT(ret == 0); 11356 hat_delete_callback(tsbva, (uint_t)tsb_size, (void *)tsbinfo, 11357 0, NULL); 11358 as_pageunlock(&kas, ppl, slab_vaddr, PAGESIZE, S_WRITE); 11359 } 11360 11361 if (kmem_cachep != NULL) { 11362 kmem_cache_free(kmem_cachep, tsbva); 11363 } else { 11364 vmem_xfree(vmp, (void *)tsbva, tsb_size); 11365 } 11366 tsbinfo->tsb_va = (caddr_t)0xbad00bad; 11367 atomic_add_64(&tsb_alloc_bytes, -(int64_t)tsb_size); 11368 } 11369 11370 static void 11371 sfmmu_tsbinfo_free(struct tsb_info *tsbinfo) 11372 { 11373 if ((tsbinfo->tsb_flags & TSB_SWAPPED) == 0) { 11374 sfmmu_tsb_free(tsbinfo); 11375 } 11376 kmem_cache_free(sfmmu_tsbinfo_cache, tsbinfo); 11377 11378 } 11379 11380 /* 11381 * Setup all the references to physical memory for this tsbinfo. 11382 * The underlying page(s) must be locked. 11383 */ 11384 static void 11385 sfmmu_tsbinfo_setup_phys(struct tsb_info *tsbinfo, pfn_t pfn) 11386 { 11387 ASSERT(pfn != PFN_INVALID); 11388 ASSERT(pfn == va_to_pfn(tsbinfo->tsb_va)); 11389 11390 #ifndef sun4v 11391 if (tsbinfo->tsb_szc == 0) { 11392 sfmmu_memtte(&tsbinfo->tsb_tte, pfn, 11393 PROT_WRITE|PROT_READ, TTE8K); 11394 } else { 11395 /* 11396 * Round down PA and use a large mapping; the handlers will 11397 * compute the TSB pointer at the correct offset into the 11398 * big virtual page. NOTE: this assumes all TSBs larger 11399 * than 8K must come from physically contiguous slabs of 11400 * size tsb_slab_size. 11401 */ 11402 sfmmu_memtte(&tsbinfo->tsb_tte, pfn & ~tsb_slab_mask, 11403 PROT_WRITE|PROT_READ, tsb_slab_ttesz); 11404 } 11405 tsbinfo->tsb_pa = ptob(pfn); 11406 11407 TTE_SET_LOCKED(&tsbinfo->tsb_tte); /* lock the tte into dtlb */ 11408 TTE_SET_MOD(&tsbinfo->tsb_tte); /* enable writes */ 11409 11410 ASSERT(TTE_IS_PRIVILEGED(&tsbinfo->tsb_tte)); 11411 ASSERT(TTE_IS_LOCKED(&tsbinfo->tsb_tte)); 11412 #else /* sun4v */ 11413 tsbinfo->tsb_pa = ptob(pfn); 11414 #endif /* sun4v */ 11415 } 11416 11417 11418 /* 11419 * Returns zero on success, ENOMEM if over the high water mark, 11420 * or EAGAIN if the caller needs to retry with a smaller TSB 11421 * size (or specify TSB_FORCEALLOC if the allocation can't fail). 11422 * 11423 * This call cannot fail to allocate a TSB if TSB_FORCEALLOC 11424 * is specified and the TSB requested is PAGESIZE, though it 11425 * may sleep waiting for memory if sufficient memory is not 11426 * available. 11427 */ 11428 static int 11429 sfmmu_init_tsbinfo(struct tsb_info *tsbinfo, int tteszmask, 11430 int tsbcode, uint_t flags, sfmmu_t *sfmmup) 11431 { 11432 caddr_t vaddr = NULL; 11433 caddr_t slab_vaddr; 11434 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11435 int tsbbytes = TSB_BYTES(tsbcode); 11436 int lowmem = 0; 11437 struct kmem_cache *kmem_cachep = NULL; 11438 vmem_t *vmp = NULL; 11439 lgrp_id_t lgrpid = LGRP_NONE; 11440 pfn_t pfn; 11441 uint_t cbflags = HAC_SLEEP; 11442 page_t **pplist; 11443 int ret; 11444 11445 if (flags & (TSB_FORCEALLOC | TSB_SWAPIN | TSB_GROW | TSB_SHRINK)) 11446 flags |= TSB_ALLOC; 11447 11448 ASSERT((flags & TSB_FORCEALLOC) == 0 || tsbcode == TSB_MIN_SZCODE); 11449 11450 tsbinfo->tsb_sfmmu = sfmmup; 11451 11452 /* 11453 * If not allocating a TSB, set up the tsbinfo, set TSB_SWAPPED, and 11454 * return. 11455 */ 11456 if ((flags & TSB_ALLOC) == 0) { 11457 tsbinfo->tsb_szc = tsbcode; 11458 tsbinfo->tsb_ttesz_mask = tteszmask; 11459 tsbinfo->tsb_va = (caddr_t)0xbadbadbeef; 11460 tsbinfo->tsb_pa = -1; 11461 tsbinfo->tsb_tte.ll = 0; 11462 tsbinfo->tsb_next = NULL; 11463 tsbinfo->tsb_flags = TSB_SWAPPED; 11464 tsbinfo->tsb_cache = NULL; 11465 tsbinfo->tsb_vmp = NULL; 11466 return (0); 11467 } 11468 11469 #ifdef DEBUG 11470 /* 11471 * For debugging: 11472 * Randomly force allocation failures every tsb_alloc_mtbf 11473 * tries if TSB_FORCEALLOC is not specified. This will 11474 * return ENOMEM if tsb_alloc_mtbf is odd, or EAGAIN if 11475 * it is even, to allow testing of both failure paths... 11476 */ 11477 if (tsb_alloc_mtbf && ((flags & TSB_FORCEALLOC) == 0) && 11478 (tsb_alloc_count++ == tsb_alloc_mtbf)) { 11479 tsb_alloc_count = 0; 11480 tsb_alloc_fail_mtbf++; 11481 return ((tsb_alloc_mtbf & 1)? ENOMEM : EAGAIN); 11482 } 11483 #endif /* DEBUG */ 11484 11485 /* 11486 * Enforce high water mark if we are not doing a forced allocation 11487 * and are not shrinking a process' TSB. 11488 */ 11489 if ((flags & TSB_SHRINK) == 0 && 11490 (tsbbytes + tsb_alloc_bytes) > tsb_alloc_hiwater) { 11491 if ((flags & TSB_FORCEALLOC) == 0) 11492 return (ENOMEM); 11493 lowmem = 1; 11494 } 11495 11496 /* 11497 * Allocate from the correct location based upon the size of the TSB 11498 * compared to the base page size, and what memory conditions dictate. 11499 * Note we always do nonblocking allocations from the TSB arena since 11500 * we don't want memory fragmentation to cause processes to block 11501 * indefinitely waiting for memory; until the kernel algorithms that 11502 * coalesce large pages are improved this is our best option. 11503 * 11504 * Algorithm: 11505 * If allocating a "large" TSB (>8K), allocate from the 11506 * appropriate kmem_tsb_default_arena vmem arena 11507 * else if low on memory or the TSB_FORCEALLOC flag is set or 11508 * tsb_forceheap is set 11509 * Allocate from kernel heap via sfmmu_tsb8k_cache with 11510 * KM_SLEEP (never fails) 11511 * else 11512 * Allocate from appropriate sfmmu_tsb_cache with 11513 * KM_NOSLEEP 11514 * endif 11515 */ 11516 if (tsb_lgrp_affinity) 11517 lgrpid = lgrp_home_id(curthread); 11518 if (lgrpid == LGRP_NONE) 11519 lgrpid = 0; /* use lgrp of boot CPU */ 11520 11521 if (tsbbytes > MMU_PAGESIZE) { 11522 vmp = kmem_tsb_default_arena[lgrpid]; 11523 vaddr = (caddr_t)vmem_xalloc(vmp, tsbbytes, tsbbytes, 0, 0, 11524 NULL, NULL, VM_NOSLEEP); 11525 #ifdef DEBUG 11526 } else if (lowmem || (flags & TSB_FORCEALLOC) || tsb_forceheap) { 11527 #else /* !DEBUG */ 11528 } else if (lowmem || (flags & TSB_FORCEALLOC)) { 11529 #endif /* DEBUG */ 11530 kmem_cachep = sfmmu_tsb8k_cache; 11531 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_SLEEP); 11532 ASSERT(vaddr != NULL); 11533 } else { 11534 kmem_cachep = sfmmu_tsb_cache[lgrpid]; 11535 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_NOSLEEP); 11536 } 11537 11538 tsbinfo->tsb_cache = kmem_cachep; 11539 tsbinfo->tsb_vmp = vmp; 11540 11541 if (vaddr == NULL) { 11542 return (EAGAIN); 11543 } 11544 11545 atomic_add_64(&tsb_alloc_bytes, (int64_t)tsbbytes); 11546 kmem_cachep = tsbinfo->tsb_cache; 11547 11548 /* 11549 * If we are allocating from outside the cage, then we need to 11550 * register a relocation callback handler. Note that for now 11551 * since pseudo mappings always hang off of the slab's root page, 11552 * we need only lock the first 8K of the TSB slab. This is a bit 11553 * hacky but it is good for performance. 11554 */ 11555 if (kmem_cachep != sfmmu_tsb8k_cache) { 11556 slab_vaddr = (caddr_t)((uintptr_t)vaddr & slab_mask); 11557 ret = as_pagelock(&kas, &pplist, slab_vaddr, PAGESIZE, S_WRITE); 11558 ASSERT(ret == 0); 11559 ret = hat_add_callback(sfmmu_tsb_cb_id, vaddr, (uint_t)tsbbytes, 11560 cbflags, (void *)tsbinfo, &pfn, NULL); 11561 11562 /* 11563 * Need to free up resources if we could not successfully 11564 * add the callback function and return an error condition. 11565 */ 11566 if (ret != 0) { 11567 if (kmem_cachep) { 11568 kmem_cache_free(kmem_cachep, vaddr); 11569 } else { 11570 vmem_xfree(vmp, (void *)vaddr, tsbbytes); 11571 } 11572 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, 11573 S_WRITE); 11574 return (EAGAIN); 11575 } 11576 } else { 11577 /* 11578 * Since allocation of 8K TSBs from heap is rare and occurs 11579 * during memory pressure we allocate them from permanent 11580 * memory rather than using callbacks to get the PFN. 11581 */ 11582 pfn = hat_getpfnum(kas.a_hat, vaddr); 11583 } 11584 11585 tsbinfo->tsb_va = vaddr; 11586 tsbinfo->tsb_szc = tsbcode; 11587 tsbinfo->tsb_ttesz_mask = tteszmask; 11588 tsbinfo->tsb_next = NULL; 11589 tsbinfo->tsb_flags = 0; 11590 11591 sfmmu_tsbinfo_setup_phys(tsbinfo, pfn); 11592 11593 if (kmem_cachep != sfmmu_tsb8k_cache) { 11594 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, S_WRITE); 11595 } 11596 11597 sfmmu_inv_tsb(vaddr, tsbbytes); 11598 return (0); 11599 } 11600 11601 /* 11602 * Initialize per cpu tsb and per cpu tsbmiss_area 11603 */ 11604 void 11605 sfmmu_init_tsbs(void) 11606 { 11607 int i; 11608 struct tsbmiss *tsbmissp; 11609 struct kpmtsbm *kpmtsbmp; 11610 #ifndef sun4v 11611 extern int dcache_line_mask; 11612 #endif /* sun4v */ 11613 extern uint_t vac_colors; 11614 11615 /* 11616 * Init. tsb miss area. 11617 */ 11618 tsbmissp = tsbmiss_area; 11619 11620 for (i = 0; i < NCPU; tsbmissp++, i++) { 11621 /* 11622 * initialize the tsbmiss area. 11623 * Do this for all possible CPUs as some may be added 11624 * while the system is running. There is no cost to this. 11625 */ 11626 tsbmissp->ksfmmup = ksfmmup; 11627 #ifndef sun4v 11628 tsbmissp->dcache_line_mask = (uint16_t)dcache_line_mask; 11629 #endif /* sun4v */ 11630 tsbmissp->khashstart = 11631 (struct hmehash_bucket *)va_to_pa((caddr_t)khme_hash); 11632 tsbmissp->uhashstart = 11633 (struct hmehash_bucket *)va_to_pa((caddr_t)uhme_hash); 11634 tsbmissp->khashsz = khmehash_num; 11635 tsbmissp->uhashsz = uhmehash_num; 11636 } 11637 11638 sfmmu_tsb_cb_id = hat_register_callback('T'<<16 | 'S' << 8 | 'B', 11639 sfmmu_tsb_pre_relocator, sfmmu_tsb_post_relocator, NULL, 0); 11640 11641 if (kpm_enable == 0) 11642 return; 11643 11644 /* -- Begin KPM specific init -- */ 11645 11646 if (kpm_smallpages) { 11647 /* 11648 * If we're using base pagesize pages for seg_kpm 11649 * mappings, we use the kernel TSB since we can't afford 11650 * to allocate a second huge TSB for these mappings. 11651 */ 11652 kpm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11653 kpm_tsbsz = ktsb_szcode; 11654 kpmsm_tsbbase = kpm_tsbbase; 11655 kpmsm_tsbsz = kpm_tsbsz; 11656 } else { 11657 /* 11658 * In VAC conflict case, just put the entries in the 11659 * kernel 8K indexed TSB for now so we can find them. 11660 * This could really be changed in the future if we feel 11661 * the need... 11662 */ 11663 kpmsm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11664 kpmsm_tsbsz = ktsb_szcode; 11665 kpm_tsbbase = ktsb_phys? ktsb4m_pbase : (uint64_t)ktsb4m_base; 11666 kpm_tsbsz = ktsb4m_szcode; 11667 } 11668 11669 kpmtsbmp = kpmtsbm_area; 11670 for (i = 0; i < NCPU; kpmtsbmp++, i++) { 11671 /* 11672 * Initialize the kpmtsbm area. 11673 * Do this for all possible CPUs as some may be added 11674 * while the system is running. There is no cost to this. 11675 */ 11676 kpmtsbmp->vbase = kpm_vbase; 11677 kpmtsbmp->vend = kpm_vbase + kpm_size * vac_colors; 11678 kpmtsbmp->sz_shift = kpm_size_shift; 11679 kpmtsbmp->kpmp_shift = kpmp_shift; 11680 kpmtsbmp->kpmp2pshft = (uchar_t)kpmp2pshft; 11681 if (kpm_smallpages == 0) { 11682 kpmtsbmp->kpmp_table_sz = kpmp_table_sz; 11683 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_table); 11684 } else { 11685 kpmtsbmp->kpmp_table_sz = kpmp_stable_sz; 11686 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_stable); 11687 } 11688 kpmtsbmp->msegphashpa = va_to_pa(memseg_phash); 11689 kpmtsbmp->flags = KPMTSBM_ENABLE_FLAG; 11690 #ifdef DEBUG 11691 kpmtsbmp->flags |= (kpm_tsbmtl) ? KPMTSBM_TLTSBM_FLAG : 0; 11692 #endif /* DEBUG */ 11693 if (ktsb_phys) 11694 kpmtsbmp->flags |= KPMTSBM_TSBPHYS_FLAG; 11695 } 11696 11697 /* -- End KPM specific init -- */ 11698 } 11699 11700 /* Avoid using sfmmu_tsbinfo_alloc() to avoid kmem_alloc - no real reason */ 11701 struct tsb_info ktsb_info[2]; 11702 11703 /* 11704 * Called from hat_kern_setup() to setup the tsb_info for ksfmmup. 11705 */ 11706 void 11707 sfmmu_init_ktsbinfo() 11708 { 11709 ASSERT(ksfmmup != NULL); 11710 ASSERT(ksfmmup->sfmmu_tsb == NULL); 11711 /* 11712 * Allocate tsbinfos for kernel and copy in data 11713 * to make debug easier and sun4v setup easier. 11714 */ 11715 ktsb_info[0].tsb_sfmmu = ksfmmup; 11716 ktsb_info[0].tsb_szc = ktsb_szcode; 11717 ktsb_info[0].tsb_ttesz_mask = TSB8K|TSB64K|TSB512K; 11718 ktsb_info[0].tsb_va = ktsb_base; 11719 ktsb_info[0].tsb_pa = ktsb_pbase; 11720 ktsb_info[0].tsb_flags = 0; 11721 ktsb_info[0].tsb_tte.ll = 0; 11722 ktsb_info[0].tsb_cache = NULL; 11723 11724 ktsb_info[1].tsb_sfmmu = ksfmmup; 11725 ktsb_info[1].tsb_szc = ktsb4m_szcode; 11726 ktsb_info[1].tsb_ttesz_mask = TSB4M; 11727 ktsb_info[1].tsb_va = ktsb4m_base; 11728 ktsb_info[1].tsb_pa = ktsb4m_pbase; 11729 ktsb_info[1].tsb_flags = 0; 11730 ktsb_info[1].tsb_tte.ll = 0; 11731 ktsb_info[1].tsb_cache = NULL; 11732 11733 /* Link them into ksfmmup. */ 11734 ktsb_info[0].tsb_next = &ktsb_info[1]; 11735 ktsb_info[1].tsb_next = NULL; 11736 ksfmmup->sfmmu_tsb = &ktsb_info[0]; 11737 11738 sfmmu_setup_tsbinfo(ksfmmup); 11739 } 11740 11741 /* 11742 * Cache the last value returned from va_to_pa(). If the VA specified 11743 * in the current call to cached_va_to_pa() maps to the same Page (as the 11744 * previous call to cached_va_to_pa()), then compute the PA using 11745 * cached info, else call va_to_pa(). 11746 * 11747 * Note: this function is neither MT-safe nor consistent in the presence 11748 * of multiple, interleaved threads. This function was created to enable 11749 * an optimization used during boot (at a point when there's only one thread 11750 * executing on the "boot CPU", and before startup_vm() has been called). 11751 */ 11752 static uint64_t 11753 cached_va_to_pa(void *vaddr) 11754 { 11755 static uint64_t prev_vaddr_base = 0; 11756 static uint64_t prev_pfn = 0; 11757 11758 if ((((uint64_t)vaddr) & MMU_PAGEMASK) == prev_vaddr_base) { 11759 return (prev_pfn | ((uint64_t)vaddr & MMU_PAGEOFFSET)); 11760 } else { 11761 uint64_t pa = va_to_pa(vaddr); 11762 11763 if (pa != ((uint64_t)-1)) { 11764 /* 11765 * Computed physical address is valid. Cache its 11766 * related info for the next cached_va_to_pa() call. 11767 */ 11768 prev_pfn = pa & MMU_PAGEMASK; 11769 prev_vaddr_base = ((uint64_t)vaddr) & MMU_PAGEMASK; 11770 } 11771 11772 return (pa); 11773 } 11774 } 11775 11776 /* 11777 * Carve up our nucleus hblk region. We may allocate more hblks than 11778 * asked due to rounding errors but we are guaranteed to have at least 11779 * enough space to allocate the requested number of hblk8's and hblk1's. 11780 */ 11781 void 11782 sfmmu_init_nucleus_hblks(caddr_t addr, size_t size, int nhblk8, int nhblk1) 11783 { 11784 struct hme_blk *hmeblkp; 11785 size_t hme8blk_sz, hme1blk_sz; 11786 size_t i; 11787 size_t hblk8_bound; 11788 ulong_t j = 0, k = 0; 11789 11790 ASSERT(addr != NULL && size != 0); 11791 11792 /* Need to use proper structure alignment */ 11793 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 11794 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 11795 11796 nucleus_hblk8.list = (void *)addr; 11797 nucleus_hblk8.index = 0; 11798 11799 /* 11800 * Use as much memory as possible for hblk8's since we 11801 * expect all bop_alloc'ed memory to be allocated in 8k chunks. 11802 * We need to hold back enough space for the hblk1's which 11803 * we'll allocate next. 11804 */ 11805 hblk8_bound = size - (nhblk1 * hme1blk_sz) - hme8blk_sz; 11806 for (i = 0; i <= hblk8_bound; i += hme8blk_sz, j++) { 11807 hmeblkp = (struct hme_blk *)addr; 11808 addr += hme8blk_sz; 11809 hmeblkp->hblk_nuc_bit = 1; 11810 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11811 } 11812 nucleus_hblk8.len = j; 11813 ASSERT(j >= nhblk8); 11814 SFMMU_STAT_ADD(sf_hblk8_ncreate, j); 11815 11816 nucleus_hblk1.list = (void *)addr; 11817 nucleus_hblk1.index = 0; 11818 for (; i <= (size - hme1blk_sz); i += hme1blk_sz, k++) { 11819 hmeblkp = (struct hme_blk *)addr; 11820 addr += hme1blk_sz; 11821 hmeblkp->hblk_nuc_bit = 1; 11822 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11823 } 11824 ASSERT(k >= nhblk1); 11825 nucleus_hblk1.len = k; 11826 SFMMU_STAT_ADD(sf_hblk1_ncreate, k); 11827 } 11828 11829 /* 11830 * This function is currently not supported on this platform. For what 11831 * it's supposed to do, see hat.c and hat_srmmu.c 11832 */ 11833 /* ARGSUSED */ 11834 faultcode_t 11835 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp, 11836 uint_t flags) 11837 { 11838 ASSERT(hat->sfmmu_xhat_provider == NULL); 11839 return (FC_NOSUPPORT); 11840 } 11841 11842 /* 11843 * Searchs the mapping list of the page for a mapping of the same size. If not 11844 * found the corresponding bit is cleared in the p_index field. When large 11845 * pages are more prevalent in the system, we can maintain the mapping list 11846 * in order and we don't have to traverse the list each time. Just check the 11847 * next and prev entries, and if both are of different size, we clear the bit. 11848 */ 11849 static void 11850 sfmmu_rm_large_mappings(page_t *pp, int ttesz) 11851 { 11852 struct sf_hment *sfhmep; 11853 struct hme_blk *hmeblkp; 11854 int index; 11855 pgcnt_t npgs; 11856 11857 ASSERT(ttesz > TTE8K); 11858 11859 ASSERT(sfmmu_mlist_held(pp)); 11860 11861 ASSERT(PP_ISMAPPED_LARGE(pp)); 11862 11863 /* 11864 * Traverse mapping list looking for another mapping of same size. 11865 * since we only want to clear index field if all mappings of 11866 * that size are gone. 11867 */ 11868 11869 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 11870 hmeblkp = sfmmu_hmetohblk(sfhmep); 11871 if (hmeblkp->hblk_xhat_bit) 11872 continue; 11873 if (hme_size(sfhmep) == ttesz) { 11874 /* 11875 * another mapping of the same size. don't clear index. 11876 */ 11877 return; 11878 } 11879 } 11880 11881 /* 11882 * Clear the p_index bit for large page. 11883 */ 11884 index = PAGESZ_TO_INDEX(ttesz); 11885 npgs = TTEPAGES(ttesz); 11886 while (npgs-- > 0) { 11887 ASSERT(pp->p_index & index); 11888 pp->p_index &= ~index; 11889 pp = PP_PAGENEXT(pp); 11890 } 11891 } 11892 11893 /* 11894 * return supported features 11895 */ 11896 /* ARGSUSED */ 11897 int 11898 hat_supported(enum hat_features feature, void *arg) 11899 { 11900 switch (feature) { 11901 case HAT_SHARED_PT: 11902 case HAT_DYNAMIC_ISM_UNMAP: 11903 case HAT_VMODSORT: 11904 return (1); 11905 default: 11906 return (0); 11907 } 11908 } 11909 11910 void 11911 hat_enter(struct hat *hat) 11912 { 11913 hatlock_t *hatlockp; 11914 11915 if (hat != ksfmmup) { 11916 hatlockp = TSB_HASH(hat); 11917 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 11918 } 11919 } 11920 11921 void 11922 hat_exit(struct hat *hat) 11923 { 11924 hatlock_t *hatlockp; 11925 11926 if (hat != ksfmmup) { 11927 hatlockp = TSB_HASH(hat); 11928 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 11929 } 11930 } 11931 11932 /*ARGSUSED*/ 11933 void 11934 hat_reserve(struct as *as, caddr_t addr, size_t len) 11935 { 11936 } 11937 11938 static void 11939 hat_kstat_init(void) 11940 { 11941 kstat_t *ksp; 11942 11943 ksp = kstat_create("unix", 0, "sfmmu_global_stat", "hat", 11944 KSTAT_TYPE_RAW, sizeof (struct sfmmu_global_stat), 11945 KSTAT_FLAG_VIRTUAL); 11946 if (ksp) { 11947 ksp->ks_data = (void *) &sfmmu_global_stat; 11948 kstat_install(ksp); 11949 } 11950 ksp = kstat_create("unix", 0, "sfmmu_tsbsize_stat", "hat", 11951 KSTAT_TYPE_RAW, sizeof (struct sfmmu_tsbsize_stat), 11952 KSTAT_FLAG_VIRTUAL); 11953 if (ksp) { 11954 ksp->ks_data = (void *) &sfmmu_tsbsize_stat; 11955 kstat_install(ksp); 11956 } 11957 ksp = kstat_create("unix", 0, "sfmmu_percpu_stat", "hat", 11958 KSTAT_TYPE_RAW, sizeof (struct sfmmu_percpu_stat) * NCPU, 11959 KSTAT_FLAG_WRITABLE); 11960 if (ksp) { 11961 ksp->ks_update = sfmmu_kstat_percpu_update; 11962 kstat_install(ksp); 11963 } 11964 } 11965 11966 /* ARGSUSED */ 11967 static int 11968 sfmmu_kstat_percpu_update(kstat_t *ksp, int rw) 11969 { 11970 struct sfmmu_percpu_stat *cpu_kstat = ksp->ks_data; 11971 struct tsbmiss *tsbm = tsbmiss_area; 11972 struct kpmtsbm *kpmtsbm = kpmtsbm_area; 11973 int i; 11974 11975 ASSERT(cpu_kstat); 11976 if (rw == KSTAT_READ) { 11977 for (i = 0; i < NCPU; cpu_kstat++, tsbm++, kpmtsbm++, i++) { 11978 cpu_kstat->sf_itlb_misses = tsbm->itlb_misses; 11979 cpu_kstat->sf_dtlb_misses = tsbm->dtlb_misses; 11980 cpu_kstat->sf_utsb_misses = tsbm->utsb_misses - 11981 tsbm->uprot_traps; 11982 cpu_kstat->sf_ktsb_misses = tsbm->ktsb_misses + 11983 kpmtsbm->kpm_tsb_misses - tsbm->kprot_traps; 11984 11985 if (tsbm->itlb_misses > 0 && tsbm->dtlb_misses > 0) { 11986 cpu_kstat->sf_tsb_hits = 11987 (tsbm->itlb_misses + tsbm->dtlb_misses) - 11988 (tsbm->utsb_misses + tsbm->ktsb_misses + 11989 kpmtsbm->kpm_tsb_misses); 11990 } else { 11991 cpu_kstat->sf_tsb_hits = 0; 11992 } 11993 cpu_kstat->sf_umod_faults = tsbm->uprot_traps; 11994 cpu_kstat->sf_kmod_faults = tsbm->kprot_traps; 11995 } 11996 } else { 11997 /* KSTAT_WRITE is used to clear stats */ 11998 for (i = 0; i < NCPU; tsbm++, kpmtsbm++, i++) { 11999 tsbm->itlb_misses = 0; 12000 tsbm->dtlb_misses = 0; 12001 tsbm->utsb_misses = 0; 12002 tsbm->ktsb_misses = 0; 12003 tsbm->uprot_traps = 0; 12004 tsbm->kprot_traps = 0; 12005 kpmtsbm->kpm_dtlb_misses = 0; 12006 kpmtsbm->kpm_tsb_misses = 0; 12007 } 12008 } 12009 return (0); 12010 } 12011 12012 #ifdef DEBUG 12013 12014 tte_t *gorig[NCPU], *gcur[NCPU], *gnew[NCPU]; 12015 12016 /* 12017 * A tte checker. *orig_old is the value we read before cas. 12018 * *cur is the value returned by cas. 12019 * *new is the desired value when we do the cas. 12020 * 12021 * *hmeblkp is currently unused. 12022 */ 12023 12024 /* ARGSUSED */ 12025 void 12026 chk_tte(tte_t *orig_old, tte_t *cur, tte_t *new, struct hme_blk *hmeblkp) 12027 { 12028 pfn_t i, j, k; 12029 int cpuid = CPU->cpu_id; 12030 12031 gorig[cpuid] = orig_old; 12032 gcur[cpuid] = cur; 12033 gnew[cpuid] = new; 12034 12035 #ifdef lint 12036 hmeblkp = hmeblkp; 12037 #endif 12038 12039 if (TTE_IS_VALID(orig_old)) { 12040 if (TTE_IS_VALID(cur)) { 12041 i = TTE_TO_TTEPFN(orig_old); 12042 j = TTE_TO_TTEPFN(cur); 12043 k = TTE_TO_TTEPFN(new); 12044 if (i != j) { 12045 /* remap error? */ 12046 panic("chk_tte: bad pfn, 0x%lx, 0x%lx", i, j); 12047 } 12048 12049 if (i != k) { 12050 /* remap error? */ 12051 panic("chk_tte: bad pfn2, 0x%lx, 0x%lx", i, k); 12052 } 12053 } else { 12054 if (TTE_IS_VALID(new)) { 12055 panic("chk_tte: invalid cur? "); 12056 } 12057 12058 i = TTE_TO_TTEPFN(orig_old); 12059 k = TTE_TO_TTEPFN(new); 12060 if (i != k) { 12061 panic("chk_tte: bad pfn3, 0x%lx, 0x%lx", i, k); 12062 } 12063 } 12064 } else { 12065 if (TTE_IS_VALID(cur)) { 12066 j = TTE_TO_TTEPFN(cur); 12067 if (TTE_IS_VALID(new)) { 12068 k = TTE_TO_TTEPFN(new); 12069 if (j != k) { 12070 panic("chk_tte: bad pfn4, 0x%lx, 0x%lx", 12071 j, k); 12072 } 12073 } else { 12074 panic("chk_tte: why here?"); 12075 } 12076 } else { 12077 if (!TTE_IS_VALID(new)) { 12078 panic("chk_tte: why here2 ?"); 12079 } 12080 } 12081 } 12082 } 12083 12084 #endif /* DEBUG */ 12085 12086 extern void prefetch_tsbe_read(struct tsbe *); 12087 extern void prefetch_tsbe_write(struct tsbe *); 12088 12089 12090 /* 12091 * We want to prefetch 7 cache lines ahead for our read prefetch. This gives 12092 * us optimal performance on Cheetah+. You can only have 8 outstanding 12093 * prefetches at any one time, so we opted for 7 read prefetches and 1 write 12094 * prefetch to make the most utilization of the prefetch capability. 12095 */ 12096 #define TSBE_PREFETCH_STRIDE (7) 12097 12098 void 12099 sfmmu_copy_tsb(struct tsb_info *old_tsbinfo, struct tsb_info *new_tsbinfo) 12100 { 12101 int old_bytes = TSB_BYTES(old_tsbinfo->tsb_szc); 12102 int new_bytes = TSB_BYTES(new_tsbinfo->tsb_szc); 12103 int old_entries = TSB_ENTRIES(old_tsbinfo->tsb_szc); 12104 int new_entries = TSB_ENTRIES(new_tsbinfo->tsb_szc); 12105 struct tsbe *old; 12106 struct tsbe *new; 12107 struct tsbe *new_base = (struct tsbe *)new_tsbinfo->tsb_va; 12108 uint64_t va; 12109 int new_offset; 12110 int i; 12111 int vpshift; 12112 int last_prefetch; 12113 12114 if (old_bytes == new_bytes) { 12115 bcopy(old_tsbinfo->tsb_va, new_tsbinfo->tsb_va, new_bytes); 12116 } else { 12117 12118 /* 12119 * A TSBE is 16 bytes which means there are four TSBE's per 12120 * P$ line (64 bytes), thus every 4 TSBE's we prefetch. 12121 */ 12122 old = (struct tsbe *)old_tsbinfo->tsb_va; 12123 last_prefetch = old_entries - (4*(TSBE_PREFETCH_STRIDE+1)); 12124 for (i = 0; i < old_entries; i++, old++) { 12125 if (((i & (4-1)) == 0) && (i < last_prefetch)) 12126 prefetch_tsbe_read(old); 12127 if (!old->tte_tag.tag_invalid) { 12128 /* 12129 * We have a valid TTE to remap. Check the 12130 * size. We won't remap 64K or 512K TTEs 12131 * because they span more than one TSB entry 12132 * and are indexed using an 8K virt. page. 12133 * Ditto for 32M and 256M TTEs. 12134 */ 12135 if (TTE_CSZ(&old->tte_data) == TTE64K || 12136 TTE_CSZ(&old->tte_data) == TTE512K) 12137 continue; 12138 if (mmu_page_sizes == max_mmu_page_sizes) { 12139 if (TTE_CSZ(&old->tte_data) == TTE32M || 12140 TTE_CSZ(&old->tte_data) == TTE256M) 12141 continue; 12142 } 12143 12144 /* clear the lower 22 bits of the va */ 12145 va = *(uint64_t *)old << 22; 12146 /* turn va into a virtual pfn */ 12147 va >>= 22 - TSB_START_SIZE; 12148 /* 12149 * or in bits from the offset in the tsb 12150 * to get the real virtual pfn. These 12151 * correspond to bits [21:13] in the va 12152 */ 12153 vpshift = 12154 TTE_BSZS_SHIFT(TTE_CSZ(&old->tte_data)) & 12155 0x1ff; 12156 va |= (i << vpshift); 12157 va >>= vpshift; 12158 new_offset = va & (new_entries - 1); 12159 new = new_base + new_offset; 12160 prefetch_tsbe_write(new); 12161 *new = *old; 12162 } 12163 } 12164 } 12165 } 12166 12167 /* 12168 * unused in sfmmu 12169 */ 12170 void 12171 hat_dump(void) 12172 { 12173 } 12174 12175 /* 12176 * Called when a thread is exiting and we have switched to the kernel address 12177 * space. Perform the same VM initialization resume() uses when switching 12178 * processes. 12179 * 12180 * Note that sfmmu_load_mmustate() is currently a no-op for kernel threads, but 12181 * we call it anyway in case the semantics change in the future. 12182 */ 12183 /*ARGSUSED*/ 12184 void 12185 hat_thread_exit(kthread_t *thd) 12186 { 12187 uint64_t pgsz_cnum; 12188 uint_t pstate_save; 12189 12190 ASSERT(thd->t_procp->p_as == &kas); 12191 12192 pgsz_cnum = KCONTEXT; 12193 #ifdef sun4u 12194 pgsz_cnum |= (ksfmmup->sfmmu_cext << CTXREG_EXT_SHIFT); 12195 #endif 12196 /* 12197 * Note that sfmmu_load_mmustate() is currently a no-op for 12198 * kernel threads. We need to disable interrupts here, 12199 * simply because otherwise sfmmu_load_mmustate() would panic 12200 * if the caller does not disable interrupts. 12201 */ 12202 pstate_save = sfmmu_disable_intrs(); 12203 sfmmu_setctx_sec(pgsz_cnum); 12204 sfmmu_load_mmustate(ksfmmup); 12205 sfmmu_enable_intrs(pstate_save); 12206 } 12207