1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * VM - Hardware Address Translation management for Spitfire MMU. 30 * 31 * This file implements the machine specific hardware translation 32 * needed by the VM system. The machine independent interface is 33 * described in <vm/hat.h> while the machine dependent interface 34 * and data structures are described in <vm/hat_sfmmu.h>. 35 * 36 * The hat layer manages the address translation hardware as a cache 37 * driven by calls from the higher levels in the VM system. 38 */ 39 40 #include <sys/types.h> 41 #include <sys/kstat.h> 42 #include <vm/hat.h> 43 #include <vm/hat_sfmmu.h> 44 #include <vm/page.h> 45 #include <sys/pte.h> 46 #include <sys/systm.h> 47 #include <sys/mman.h> 48 #include <sys/sysmacros.h> 49 #include <sys/machparam.h> 50 #include <sys/vtrace.h> 51 #include <sys/kmem.h> 52 #include <sys/mmu.h> 53 #include <sys/cmn_err.h> 54 #include <sys/cpu.h> 55 #include <sys/cpuvar.h> 56 #include <sys/debug.h> 57 #include <sys/lgrp.h> 58 #include <sys/archsystm.h> 59 #include <sys/machsystm.h> 60 #include <sys/vmsystm.h> 61 #include <vm/as.h> 62 #include <vm/seg.h> 63 #include <vm/seg_kp.h> 64 #include <vm/seg_kmem.h> 65 #include <vm/seg_kpm.h> 66 #include <vm/rm.h> 67 #include <sys/t_lock.h> 68 #include <sys/obpdefs.h> 69 #include <sys/vm_machparam.h> 70 #include <sys/var.h> 71 #include <sys/trap.h> 72 #include <sys/machtrap.h> 73 #include <sys/scb.h> 74 #include <sys/bitmap.h> 75 #include <sys/machlock.h> 76 #include <sys/membar.h> 77 #include <sys/atomic.h> 78 #include <sys/cpu_module.h> 79 #include <sys/prom_debug.h> 80 #include <sys/ksynch.h> 81 #include <sys/mem_config.h> 82 #include <sys/mem_cage.h> 83 #include <sys/dtrace.h> 84 #include <vm/vm_dep.h> 85 #include <vm/xhat_sfmmu.h> 86 #include <sys/fpu/fpusystm.h> 87 #include <vm/mach_kpm.h> 88 89 #if defined(SF_ERRATA_57) 90 extern caddr_t errata57_limit; 91 #endif 92 93 #define HME8BLK_SZ_RND ((roundup(HME8BLK_SZ, sizeof (int64_t))) / \ 94 (sizeof (int64_t))) 95 #define HBLK_RESERVE ((struct hme_blk *)hblk_reserve) 96 97 #define HBLK_RESERVE_CNT 128 98 #define HBLK_RESERVE_MIN 20 99 100 static struct hme_blk *freehblkp; 101 static kmutex_t freehblkp_lock; 102 static int freehblkcnt; 103 104 static int64_t hblk_reserve[HME8BLK_SZ_RND]; 105 static kmutex_t hblk_reserve_lock; 106 static kthread_t *hblk_reserve_thread; 107 108 static nucleus_hblk8_info_t nucleus_hblk8; 109 static nucleus_hblk1_info_t nucleus_hblk1; 110 111 /* 112 * SFMMU specific hat functions 113 */ 114 void hat_pagecachectl(struct page *, int); 115 116 /* flags for hat_pagecachectl */ 117 #define HAT_CACHE 0x1 118 #define HAT_UNCACHE 0x2 119 #define HAT_TMPNC 0x4 120 121 /* 122 * Flag to allow the creation of non-cacheable translations 123 * to system memory. It is off by default. At the moment this 124 * flag is used by the ecache error injector. The error injector 125 * will turn it on when creating such a translation then shut it 126 * off when it's finished. 127 */ 128 129 int sfmmu_allow_nc_trans = 0; 130 131 /* 132 * Flag to disable large page support. 133 * value of 1 => disable all large pages. 134 * bits 1, 2, and 3 are to disable 64K, 512K and 4M pages respectively. 135 * 136 * For example, use the value 0x4 to disable 512K pages. 137 * 138 */ 139 #define LARGE_PAGES_OFF 0x1 140 141 /* 142 * The disable_large_pages and disable_ism_large_pages variables control 143 * hat_memload_array and the page sizes to be used by ISM and the kernel. 144 * 145 * The disable_auto_data_large_pages and disable_auto_text_large_pages variables 146 * are only used to control which OOB pages to use at upper VM segment creation 147 * time, and are set in hat_init_pagesizes and used in the map_pgsz* routines. 148 * Their values may come from platform or CPU specific code to disable page 149 * sizes that should not be used. 150 * 151 * WARNING: 512K pages are currently not supported for ISM/DISM. 152 */ 153 uint_t disable_large_pages = 0; 154 uint_t disable_ism_large_pages = (1 << TTE512K); 155 uint_t disable_auto_data_large_pages = 0; 156 uint_t disable_auto_text_large_pages = 0; 157 158 /* 159 * Private sfmmu data structures for hat management 160 */ 161 static struct kmem_cache *sfmmuid_cache; 162 static struct kmem_cache *mmuctxdom_cache; 163 164 /* 165 * Private sfmmu data structures for tsb management 166 */ 167 static struct kmem_cache *sfmmu_tsbinfo_cache; 168 static struct kmem_cache *sfmmu_tsb8k_cache; 169 static struct kmem_cache *sfmmu_tsb_cache[NLGRPS_MAX]; 170 static vmem_t *kmem_tsb_arena; 171 172 /* 173 * sfmmu static variables for hmeblk resource management. 174 */ 175 static vmem_t *hat_memload1_arena; /* HAT translation arena for sfmmu1_cache */ 176 static struct kmem_cache *sfmmu8_cache; 177 static struct kmem_cache *sfmmu1_cache; 178 static struct kmem_cache *pa_hment_cache; 179 180 static kmutex_t ism_mlist_lock; /* mutex for ism mapping list */ 181 /* 182 * private data for ism 183 */ 184 static struct kmem_cache *ism_blk_cache; 185 static struct kmem_cache *ism_ment_cache; 186 #define ISMID_STARTADDR NULL 187 188 /* 189 * Whether to delay TLB flushes and use Cheetah's flush-all support 190 * when removing contexts from the dirty list. 191 */ 192 int delay_tlb_flush; 193 int disable_delay_tlb_flush; 194 195 /* 196 * ``hat_lock'' is a hashed mutex lock for protecting sfmmu TSB lists, 197 * HAT flags, synchronizing TLB/TSB coherency, and context management. 198 * The lock is hashed on the sfmmup since the case where we need to lock 199 * all processes is rare but does occur (e.g. we need to unload a shared 200 * mapping from all processes using the mapping). We have a lot of buckets, 201 * and each slab of sfmmu_t's can use about a quarter of them, giving us 202 * a fairly good distribution without wasting too much space and overhead 203 * when we have to grab them all. 204 */ 205 #define SFMMU_NUM_LOCK 128 /* must be power of two */ 206 hatlock_t hat_lock[SFMMU_NUM_LOCK]; 207 208 /* 209 * Hash algorithm optimized for a small number of slabs. 210 * 7 is (highbit((sizeof sfmmu_t)) - 1) 211 * This hash algorithm is based upon the knowledge that sfmmu_t's come from a 212 * kmem_cache, and thus they will be sequential within that cache. In 213 * addition, each new slab will have a different "color" up to cache_maxcolor 214 * which will skew the hashing for each successive slab which is allocated. 215 * If the size of sfmmu_t changed to a larger size, this algorithm may need 216 * to be revisited. 217 */ 218 #define TSB_HASH_SHIFT_BITS (7) 219 #define PTR_HASH(x) ((uintptr_t)x >> TSB_HASH_SHIFT_BITS) 220 221 #ifdef DEBUG 222 int tsb_hash_debug = 0; 223 #define TSB_HASH(sfmmup) \ 224 (tsb_hash_debug ? &hat_lock[0] : \ 225 &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)]) 226 #else /* DEBUG */ 227 #define TSB_HASH(sfmmup) &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)] 228 #endif /* DEBUG */ 229 230 231 /* sfmmu_replace_tsb() return codes. */ 232 typedef enum tsb_replace_rc { 233 TSB_SUCCESS, 234 TSB_ALLOCFAIL, 235 TSB_LOSTRACE, 236 TSB_ALREADY_SWAPPED, 237 TSB_CANTGROW 238 } tsb_replace_rc_t; 239 240 /* 241 * Flags for TSB allocation routines. 242 */ 243 #define TSB_ALLOC 0x01 244 #define TSB_FORCEALLOC 0x02 245 #define TSB_GROW 0x04 246 #define TSB_SHRINK 0x08 247 #define TSB_SWAPIN 0x10 248 249 /* 250 * Support for HAT callbacks. 251 */ 252 #define SFMMU_MAX_RELOC_CALLBACKS 10 253 int sfmmu_max_cb_id = SFMMU_MAX_RELOC_CALLBACKS; 254 static id_t sfmmu_cb_nextid = 0; 255 static id_t sfmmu_tsb_cb_id; 256 struct sfmmu_callback *sfmmu_cb_table; 257 258 /* 259 * Kernel page relocation is enabled by default for non-caged 260 * kernel pages. This has little effect unless segkmem_reloc is 261 * set, since by default kernel memory comes from inside the 262 * kernel cage. 263 */ 264 int hat_kpr_enabled = 1; 265 266 kmutex_t kpr_mutex; 267 kmutex_t kpr_suspendlock; 268 kthread_t *kreloc_thread; 269 270 /* 271 * Enable VA->PA translation sanity checking on DEBUG kernels. 272 * Disabled by default. This is incompatible with some 273 * drivers (error injector, RSM) so if it breaks you get 274 * to keep both pieces. 275 */ 276 int hat_check_vtop = 0; 277 278 /* 279 * Private sfmmu routines (prototypes) 280 */ 281 static struct hme_blk *sfmmu_shadow_hcreate(sfmmu_t *, caddr_t, int, uint_t); 282 static struct hme_blk *sfmmu_hblk_alloc(sfmmu_t *, caddr_t, 283 struct hmehash_bucket *, uint_t, hmeblk_tag, uint_t); 284 static caddr_t sfmmu_hblk_unload(struct hat *, struct hme_blk *, caddr_t, 285 caddr_t, demap_range_t *, uint_t); 286 static caddr_t sfmmu_hblk_sync(struct hat *, struct hme_blk *, caddr_t, 287 caddr_t, int); 288 static void sfmmu_hblk_free(struct hmehash_bucket *, struct hme_blk *, 289 uint64_t, struct hme_blk **); 290 static void sfmmu_hblks_list_purge(struct hme_blk **); 291 static uint_t sfmmu_get_free_hblk(struct hme_blk **, uint_t); 292 static uint_t sfmmu_put_free_hblk(struct hme_blk *, uint_t); 293 static struct hme_blk *sfmmu_hblk_steal(int); 294 static int sfmmu_steal_this_hblk(struct hmehash_bucket *, 295 struct hme_blk *, uint64_t, uint64_t, 296 struct hme_blk *); 297 static caddr_t sfmmu_hblk_unlock(struct hme_blk *, caddr_t, caddr_t); 298 299 static void sfmmu_memload_batchsmall(struct hat *, caddr_t, page_t **, 300 uint_t, uint_t, pgcnt_t); 301 void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, 302 uint_t); 303 static int sfmmu_tteload_array(sfmmu_t *, tte_t *, caddr_t, page_t **, 304 uint_t); 305 static struct hmehash_bucket *sfmmu_tteload_acquire_hashbucket(sfmmu_t *, 306 caddr_t, int); 307 static struct hme_blk *sfmmu_tteload_find_hmeblk(sfmmu_t *, 308 struct hmehash_bucket *, caddr_t, uint_t, uint_t); 309 static int sfmmu_tteload_addentry(sfmmu_t *, struct hme_blk *, tte_t *, 310 caddr_t, page_t **, uint_t); 311 static void sfmmu_tteload_release_hashbucket(struct hmehash_bucket *); 312 313 static int sfmmu_pagearray_setup(caddr_t, page_t **, tte_t *, int); 314 pfn_t sfmmu_uvatopfn(caddr_t, sfmmu_t *); 315 void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 316 #ifdef VAC 317 static void sfmmu_vac_conflict(struct hat *, caddr_t, page_t *); 318 static int sfmmu_vacconflict_array(caddr_t, page_t *, int *); 319 int tst_tnc(page_t *pp, pgcnt_t); 320 void conv_tnc(page_t *pp, int); 321 #endif 322 323 static void sfmmu_get_ctx(sfmmu_t *); 324 static void sfmmu_free_sfmmu(sfmmu_t *); 325 326 static void sfmmu_gettte(struct hat *, caddr_t, tte_t *); 327 static void sfmmu_ttesync(struct hat *, caddr_t, tte_t *, page_t *); 328 static void sfmmu_chgattr(struct hat *, caddr_t, size_t, uint_t, int); 329 330 cpuset_t sfmmu_pageunload(page_t *, struct sf_hment *, int); 331 static void hat_pagereload(struct page *, struct page *); 332 static cpuset_t sfmmu_pagesync(page_t *, struct sf_hment *, uint_t); 333 #ifdef VAC 334 void sfmmu_page_cache_array(page_t *, int, int, pgcnt_t); 335 static void sfmmu_page_cache(page_t *, int, int, int); 336 #endif 337 338 static void sfmmu_tlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 339 pfn_t, int, int, int, int); 340 static void sfmmu_ismtlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 341 pfn_t, int); 342 static void sfmmu_tlb_demap(caddr_t, sfmmu_t *, struct hme_blk *, int, int); 343 static void sfmmu_tlb_range_demap(demap_range_t *); 344 static void sfmmu_invalidate_ctx(sfmmu_t *); 345 static void sfmmu_sync_mmustate(sfmmu_t *); 346 347 static void sfmmu_tsbinfo_setup_phys(struct tsb_info *, pfn_t); 348 static int sfmmu_tsbinfo_alloc(struct tsb_info **, int, int, uint_t, 349 sfmmu_t *); 350 static void sfmmu_tsb_free(struct tsb_info *); 351 static void sfmmu_tsbinfo_free(struct tsb_info *); 352 static int sfmmu_init_tsbinfo(struct tsb_info *, int, int, uint_t, 353 sfmmu_t *); 354 355 static void sfmmu_tsb_swapin(sfmmu_t *, hatlock_t *); 356 static int sfmmu_select_tsb_szc(pgcnt_t); 357 static void sfmmu_mod_tsb(sfmmu_t *, caddr_t, tte_t *, int); 358 #define sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \ 359 sfmmu_mod_tsb(sfmmup, vaddr, tte, szc) 360 #define sfmmu_unload_tsb(sfmmup, vaddr, szc) \ 361 sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc) 362 static void sfmmu_copy_tsb(struct tsb_info *, struct tsb_info *); 363 static tsb_replace_rc_t sfmmu_replace_tsb(sfmmu_t *, struct tsb_info *, uint_t, 364 hatlock_t *, uint_t); 365 static void sfmmu_size_tsb(sfmmu_t *, int, uint64_t, uint64_t, int); 366 367 #ifdef VAC 368 void sfmmu_cache_flush(pfn_t, int); 369 void sfmmu_cache_flushcolor(int, pfn_t); 370 #endif 371 static caddr_t sfmmu_hblk_chgattr(sfmmu_t *, struct hme_blk *, caddr_t, 372 caddr_t, demap_range_t *, uint_t, int); 373 374 static uint64_t sfmmu_vtop_attr(uint_t, int mode, tte_t *); 375 static uint_t sfmmu_ptov_attr(tte_t *); 376 static caddr_t sfmmu_hblk_chgprot(sfmmu_t *, struct hme_blk *, caddr_t, 377 caddr_t, demap_range_t *, uint_t); 378 static uint_t sfmmu_vtop_prot(uint_t, uint_t *); 379 static int sfmmu_idcache_constructor(void *, void *, int); 380 static void sfmmu_idcache_destructor(void *, void *); 381 static int sfmmu_hblkcache_constructor(void *, void *, int); 382 static void sfmmu_hblkcache_destructor(void *, void *); 383 static void sfmmu_hblkcache_reclaim(void *); 384 static void sfmmu_shadow_hcleanup(sfmmu_t *, struct hme_blk *, 385 struct hmehash_bucket *); 386 static void sfmmu_free_hblks(sfmmu_t *, caddr_t, caddr_t, int); 387 static void sfmmu_rm_large_mappings(page_t *, int); 388 389 static void hat_lock_init(void); 390 static void hat_kstat_init(void); 391 static int sfmmu_kstat_percpu_update(kstat_t *ksp, int rw); 392 static void sfmmu_check_page_sizes(sfmmu_t *, int); 393 int fnd_mapping_sz(page_t *); 394 static void iment_add(struct ism_ment *, struct hat *); 395 static void iment_sub(struct ism_ment *, struct hat *); 396 static pgcnt_t ism_tsb_entries(sfmmu_t *, int szc); 397 extern void sfmmu_setup_tsbinfo(sfmmu_t *); 398 #ifdef sun4v 399 extern void sfmmu_invalidate_tsbinfo(sfmmu_t *); 400 #endif /* sun4v */ 401 extern void sfmmu_clear_utsbinfo(void); 402 403 static void sfmmu_ctx_wrap_around(mmu_ctx_t *); 404 405 /* kpm globals */ 406 #ifdef DEBUG 407 /* 408 * Enable trap level tsbmiss handling 409 */ 410 int kpm_tsbmtl = 1; 411 412 /* 413 * Flush the TLB on kpm mapout. Note: Xcalls are used (again) for the 414 * required TLB shootdowns in this case, so handle w/ care. Off by default. 415 */ 416 int kpm_tlb_flush; 417 #endif /* DEBUG */ 418 419 static void *sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *, size_t, int); 420 421 #ifdef DEBUG 422 static void sfmmu_check_hblk_flist(); 423 #endif 424 425 /* 426 * Semi-private sfmmu data structures. Some of them are initialize in 427 * startup or in hat_init. Some of them are private but accessed by 428 * assembly code or mach_sfmmu.c 429 */ 430 struct hmehash_bucket *uhme_hash; /* user hmeblk hash table */ 431 struct hmehash_bucket *khme_hash; /* kernel hmeblk hash table */ 432 uint64_t uhme_hash_pa; /* PA of uhme_hash */ 433 uint64_t khme_hash_pa; /* PA of khme_hash */ 434 int uhmehash_num; /* # of buckets in user hash table */ 435 int khmehash_num; /* # of buckets in kernel hash table */ 436 437 uint_t max_mmu_ctxdoms = 0; /* max context domains in the system */ 438 mmu_ctx_t **mmu_ctxs_tbl; /* global array of context domains */ 439 uint64_t mmu_saved_gnum = 0; /* to init incoming MMUs' gnums */ 440 441 #define DEFAULT_NUM_CTXS_PER_MMU 8192 442 static uint_t nctxs = DEFAULT_NUM_CTXS_PER_MMU; 443 444 int cache; /* describes system cache */ 445 446 caddr_t ktsb_base; /* kernel 8k-indexed tsb base address */ 447 uint64_t ktsb_pbase; /* kernel 8k-indexed tsb phys address */ 448 int ktsb_szcode; /* kernel 8k-indexed tsb size code */ 449 int ktsb_sz; /* kernel 8k-indexed tsb size */ 450 451 caddr_t ktsb4m_base; /* kernel 4m-indexed tsb base address */ 452 uint64_t ktsb4m_pbase; /* kernel 4m-indexed tsb phys address */ 453 int ktsb4m_szcode; /* kernel 4m-indexed tsb size code */ 454 int ktsb4m_sz; /* kernel 4m-indexed tsb size */ 455 456 uint64_t kpm_tsbbase; /* kernel seg_kpm 4M TSB base address */ 457 int kpm_tsbsz; /* kernel seg_kpm 4M TSB size code */ 458 uint64_t kpmsm_tsbbase; /* kernel seg_kpm 8K TSB base address */ 459 int kpmsm_tsbsz; /* kernel seg_kpm 8K TSB size code */ 460 461 #ifndef sun4v 462 int utsb_dtlb_ttenum = -1; /* index in TLB for utsb locked TTE */ 463 int utsb4m_dtlb_ttenum = -1; /* index in TLB for 4M TSB TTE */ 464 int dtlb_resv_ttenum; /* index in TLB of first reserved TTE */ 465 caddr_t utsb_vabase; /* reserved kernel virtual memory */ 466 caddr_t utsb4m_vabase; /* for trap handler TSB accesses */ 467 #endif /* sun4v */ 468 uint64_t tsb_alloc_bytes = 0; /* bytes allocated to TSBs */ 469 vmem_t *kmem_tsb_default_arena[NLGRPS_MAX]; /* For dynamic TSBs */ 470 471 /* 472 * Size to use for TSB slabs. Future platforms that support page sizes 473 * larger than 4M may wish to change these values, and provide their own 474 * assembly macros for building and decoding the TSB base register contents. 475 * Note disable_large_pages will override the value set here. 476 */ 477 uint_t tsb_slab_ttesz = TTE4M; 478 uint_t tsb_slab_size; 479 uint_t tsb_slab_shift; 480 uint_t tsb_slab_mask; /* PFN mask for TTE */ 481 482 /* largest TSB size to grow to, will be smaller on smaller memory systems */ 483 int tsb_max_growsize = UTSB_MAX_SZCODE; 484 485 /* 486 * Tunable parameters dealing with TSB policies. 487 */ 488 489 /* 490 * This undocumented tunable forces all 8K TSBs to be allocated from 491 * the kernel heap rather than from the kmem_tsb_default_arena arenas. 492 */ 493 #ifdef DEBUG 494 int tsb_forceheap = 0; 495 #endif /* DEBUG */ 496 497 /* 498 * Decide whether to use per-lgroup arenas, or one global set of 499 * TSB arenas. The default is not to break up per-lgroup, since 500 * most platforms don't recognize any tangible benefit from it. 501 */ 502 int tsb_lgrp_affinity = 0; 503 504 /* 505 * Used for growing the TSB based on the process RSS. 506 * tsb_rss_factor is based on the smallest TSB, and is 507 * shifted by the TSB size to determine if we need to grow. 508 * The default will grow the TSB if the number of TTEs for 509 * this page size exceeds 75% of the number of TSB entries, 510 * which should _almost_ eliminate all conflict misses 511 * (at the expense of using up lots and lots of memory). 512 */ 513 #define TSB_RSS_FACTOR (TSB_ENTRIES(TSB_MIN_SZCODE) * 0.75) 514 #define SFMMU_RSS_TSBSIZE(tsbszc) (tsb_rss_factor << tsbszc) 515 #define SELECT_TSB_SIZECODE(pgcnt) ( \ 516 (enable_tsb_rss_sizing)? sfmmu_select_tsb_szc(pgcnt) : \ 517 default_tsb_size) 518 #define TSB_OK_SHRINK() \ 519 (tsb_alloc_bytes > tsb_alloc_hiwater || freemem < desfree) 520 #define TSB_OK_GROW() \ 521 (tsb_alloc_bytes < tsb_alloc_hiwater && freemem > desfree) 522 523 int enable_tsb_rss_sizing = 1; 524 int tsb_rss_factor = (int)TSB_RSS_FACTOR; 525 526 /* which TSB size code to use for new address spaces or if rss sizing off */ 527 int default_tsb_size = TSB_8K_SZCODE; 528 529 static uint64_t tsb_alloc_hiwater; /* limit TSB reserved memory */ 530 uint64_t tsb_alloc_hiwater_factor; /* tsb_alloc_hiwater = physmem / this */ 531 #define TSB_ALLOC_HIWATER_FACTOR_DEFAULT 32 532 533 #ifdef DEBUG 534 static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */ 535 static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */ 536 static int tsb_alloc_mtbf = 0; /* fail allocation every n attempts */ 537 static int tsb_alloc_fail_mtbf = 0; 538 static int tsb_alloc_count = 0; 539 #endif /* DEBUG */ 540 541 /* if set to 1, will remap valid TTEs when growing TSB. */ 542 int tsb_remap_ttes = 1; 543 544 /* 545 * If we have more than this many mappings, allocate a second TSB. 546 * This default is chosen because the I/D fully associative TLBs are 547 * assumed to have at least 8 available entries. Platforms with a 548 * larger fully-associative TLB could probably override the default. 549 */ 550 int tsb_sectsb_threshold = 8; 551 552 /* 553 * kstat data 554 */ 555 struct sfmmu_global_stat sfmmu_global_stat; 556 struct sfmmu_tsbsize_stat sfmmu_tsbsize_stat; 557 558 /* 559 * Global data 560 */ 561 sfmmu_t *ksfmmup; /* kernel's hat id */ 562 563 #ifdef DEBUG 564 static void chk_tte(tte_t *, tte_t *, tte_t *, struct hme_blk *); 565 #endif 566 567 /* sfmmu locking operations */ 568 static kmutex_t *sfmmu_mlspl_enter(struct page *, int); 569 static int sfmmu_mlspl_held(struct page *, int); 570 571 kmutex_t *sfmmu_page_enter(page_t *); 572 void sfmmu_page_exit(kmutex_t *); 573 int sfmmu_page_spl_held(struct page *); 574 575 /* sfmmu internal locking operations - accessed directly */ 576 static void sfmmu_mlist_reloc_enter(page_t *, page_t *, 577 kmutex_t **, kmutex_t **); 578 static void sfmmu_mlist_reloc_exit(kmutex_t *, kmutex_t *); 579 static hatlock_t * 580 sfmmu_hat_enter(sfmmu_t *); 581 static hatlock_t * 582 sfmmu_hat_tryenter(sfmmu_t *); 583 static void sfmmu_hat_exit(hatlock_t *); 584 static void sfmmu_hat_lock_all(void); 585 static void sfmmu_hat_unlock_all(void); 586 static void sfmmu_ismhat_enter(sfmmu_t *, int); 587 static void sfmmu_ismhat_exit(sfmmu_t *, int); 588 589 /* 590 * Array of mutexes protecting a page's mapping list and p_nrm field. 591 * 592 * The hash function looks complicated, but is made up so that: 593 * 594 * "pp" not shifted, so adjacent pp values will hash to different cache lines 595 * (8 byte alignment * 8 bytes/mutes == 64 byte coherency subblock) 596 * 597 * "pp" >> mml_shift, incorporates more source bits into the hash result 598 * 599 * "& (mml_table_size - 1), should be faster than using remainder "%" 600 * 601 * Hopefully, mml_table, mml_table_size and mml_shift are all in the same 602 * cacheline, since they get declared next to each other below. We'll trust 603 * ld not to do something random. 604 */ 605 #ifdef DEBUG 606 int mlist_hash_debug = 0; 607 #define MLIST_HASH(pp) (mlist_hash_debug ? &mml_table[0] : \ 608 &mml_table[((uintptr_t)(pp) + \ 609 ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)]) 610 #else /* !DEBUG */ 611 #define MLIST_HASH(pp) &mml_table[ \ 612 ((uintptr_t)(pp) + ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)] 613 #endif /* !DEBUG */ 614 615 kmutex_t *mml_table; 616 uint_t mml_table_sz; /* must be a power of 2 */ 617 uint_t mml_shift; /* log2(mml_table_sz) + 3 for align */ 618 619 kpm_hlk_t *kpmp_table; 620 uint_t kpmp_table_sz; /* must be a power of 2 */ 621 uchar_t kpmp_shift; 622 623 kpm_shlk_t *kpmp_stable; 624 uint_t kpmp_stable_sz; /* must be a power of 2 */ 625 626 /* 627 * SPL_HASH was improved to avoid false cache line sharing 628 */ 629 #define SPL_TABLE_SIZE 128 630 #define SPL_MASK (SPL_TABLE_SIZE - 1) 631 #define SPL_SHIFT 7 /* log2(SPL_TABLE_SIZE) */ 632 633 #define SPL_INDEX(pp) \ 634 ((((uintptr_t)(pp) >> SPL_SHIFT) ^ \ 635 ((uintptr_t)(pp) >> (SPL_SHIFT << 1))) & \ 636 (SPL_TABLE_SIZE - 1)) 637 638 #define SPL_HASH(pp) \ 639 (&sfmmu_page_lock[SPL_INDEX(pp) & SPL_MASK].pad_mutex) 640 641 static pad_mutex_t sfmmu_page_lock[SPL_TABLE_SIZE]; 642 643 644 /* 645 * hat_unload_callback() will group together callbacks in order 646 * to avoid xt_sync() calls. This is the maximum size of the group. 647 */ 648 #define MAX_CB_ADDR 32 649 650 tte_t hw_tte; 651 static ulong_t sfmmu_dmr_maxbit = DMR_MAXBIT; 652 653 static char *mmu_ctx_kstat_names[] = { 654 "mmu_ctx_tsb_exceptions", 655 "mmu_ctx_tsb_raise_exception", 656 "mmu_ctx_wrap_around", 657 }; 658 659 /* 660 * Wrapper for vmem_xalloc since vmem_create only allows limited 661 * parameters for vm_source_alloc functions. This function allows us 662 * to specify alignment consistent with the size of the object being 663 * allocated. 664 */ 665 static void * 666 sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *vmp, size_t size, int vmflag) 667 { 668 return (vmem_xalloc(vmp, size, size, 0, 0, NULL, NULL, vmflag)); 669 } 670 671 /* Common code for setting tsb_alloc_hiwater. */ 672 #define SFMMU_SET_TSB_ALLOC_HIWATER(pages) tsb_alloc_hiwater = \ 673 ptob(pages) / tsb_alloc_hiwater_factor 674 675 /* 676 * Set tsb_max_growsize to allow at most all of physical memory to be mapped by 677 * a single TSB. physmem is the number of physical pages so we need physmem 8K 678 * TTEs to represent all those physical pages. We round this up by using 679 * 1<<highbit(). To figure out which size code to use, remember that the size 680 * code is just an amount to shift the smallest TSB size to get the size of 681 * this TSB. So we subtract that size, TSB_START_SIZE, from highbit() (or 682 * highbit() - 1) to get the size code for the smallest TSB that can represent 683 * all of physical memory, while erring on the side of too much. 684 * 685 * If the computed size code is less than the current tsb_max_growsize, we set 686 * tsb_max_growsize to the computed size code. In the case where the computed 687 * size code is greater than tsb_max_growsize, we have these restrictions that 688 * apply to increasing tsb_max_growsize: 689 * 1) TSBs can't grow larger than the TSB slab size 690 * 2) TSBs can't grow larger than UTSB_MAX_SZCODE. 691 */ 692 #define SFMMU_SET_TSB_MAX_GROWSIZE(pages) { \ 693 int i, szc; \ 694 \ 695 i = highbit(pages); \ 696 if ((1 << (i - 1)) == (pages)) \ 697 i--; /* 2^n case, round down */ \ 698 szc = i - TSB_START_SIZE; \ 699 if (szc < tsb_max_growsize) \ 700 tsb_max_growsize = szc; \ 701 else if ((szc > tsb_max_growsize) && \ 702 (szc <= tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT))) \ 703 tsb_max_growsize = MIN(szc, UTSB_MAX_SZCODE); \ 704 } 705 706 /* 707 * Given a pointer to an sfmmu and a TTE size code, return a pointer to the 708 * tsb_info which handles that TTE size. 709 */ 710 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) \ 711 (tsbinfop) = (sfmmup)->sfmmu_tsb; \ 712 ASSERT(sfmmu_hat_lock_held(sfmmup)); \ 713 if ((tte_szc) >= TTE4M) \ 714 (tsbinfop) = (tsbinfop)->tsb_next; 715 716 /* 717 * Return the number of mappings present in the HAT 718 * for a particular process and page size. 719 */ 720 #define SFMMU_TTE_CNT(sfmmup, szc) \ 721 (sfmmup)->sfmmu_iblk? \ 722 (sfmmup)->sfmmu_ismttecnt[(szc)] + \ 723 (sfmmup)->sfmmu_ttecnt[(szc)] : \ 724 (sfmmup)->sfmmu_ttecnt[(szc)]; 725 726 /* 727 * Macro to use to unload entries from the TSB. 728 * It has knowledge of which page sizes get replicated in the TSB 729 * and will call the appropriate unload routine for the appropriate size. 730 */ 731 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp) \ 732 { \ 733 int ttesz = get_hblk_ttesz(hmeblkp); \ 734 if (ttesz == TTE8K || ttesz == TTE4M) { \ 735 sfmmu_unload_tsb(sfmmup, addr, ttesz); \ 736 } else { \ 737 caddr_t sva = (caddr_t)get_hblk_base(hmeblkp); \ 738 caddr_t eva = sva + get_hblk_span(hmeblkp); \ 739 ASSERT(addr >= sva && addr < eva); \ 740 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); \ 741 } \ 742 } 743 744 745 /* Update tsb_alloc_hiwater after memory is configured. */ 746 /*ARGSUSED*/ 747 static void 748 sfmmu_update_tsb_post_add(void *arg, pgcnt_t delta_pages) 749 { 750 /* Assumes physmem has already been updated. */ 751 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 752 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 753 } 754 755 /* 756 * Update tsb_alloc_hiwater before memory is deleted. We'll do nothing here 757 * and update tsb_alloc_hiwater and tsb_max_growsize after the memory is 758 * deleted. 759 */ 760 /*ARGSUSED*/ 761 static int 762 sfmmu_update_tsb_pre_del(void *arg, pgcnt_t delta_pages) 763 { 764 return (0); 765 } 766 767 /* Update tsb_alloc_hiwater after memory fails to be unconfigured. */ 768 /*ARGSUSED*/ 769 static void 770 sfmmu_update_tsb_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 771 { 772 /* 773 * Whether the delete was cancelled or not, just go ahead and update 774 * tsb_alloc_hiwater and tsb_max_growsize. 775 */ 776 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 777 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 778 } 779 780 static kphysm_setup_vector_t sfmmu_update_tsb_vec = { 781 KPHYSM_SETUP_VECTOR_VERSION, /* version */ 782 sfmmu_update_tsb_post_add, /* post_add */ 783 sfmmu_update_tsb_pre_del, /* pre_del */ 784 sfmmu_update_tsb_post_del /* post_del */ 785 }; 786 787 788 /* 789 * HME_BLK HASH PRIMITIVES 790 */ 791 792 /* 793 * Enter a hme on the mapping list for page pp. 794 * When large pages are more prevalent in the system we might want to 795 * keep the mapping list in ascending order by the hment size. For now, 796 * small pages are more frequent, so don't slow it down. 797 */ 798 #define HME_ADD(hme, pp) \ 799 { \ 800 ASSERT(sfmmu_mlist_held(pp)); \ 801 \ 802 hme->hme_prev = NULL; \ 803 hme->hme_next = pp->p_mapping; \ 804 hme->hme_page = pp; \ 805 if (pp->p_mapping) { \ 806 ((struct sf_hment *)(pp->p_mapping))->hme_prev = hme;\ 807 ASSERT(pp->p_share > 0); \ 808 } else { \ 809 /* EMPTY */ \ 810 ASSERT(pp->p_share == 0); \ 811 } \ 812 pp->p_mapping = hme; \ 813 pp->p_share++; \ 814 } 815 816 /* 817 * Enter a hme on the mapping list for page pp. 818 * If we are unmapping a large translation, we need to make sure that the 819 * change is reflect in the corresponding bit of the p_index field. 820 */ 821 #define HME_SUB(hme, pp) \ 822 { \ 823 ASSERT(sfmmu_mlist_held(pp)); \ 824 ASSERT(hme->hme_page == pp || IS_PAHME(hme)); \ 825 \ 826 if (pp->p_mapping == NULL) { \ 827 panic("hme_remove - no mappings"); \ 828 } \ 829 \ 830 membar_stst(); /* ensure previous stores finish */ \ 831 \ 832 ASSERT(pp->p_share > 0); \ 833 pp->p_share--; \ 834 \ 835 if (hme->hme_prev) { \ 836 ASSERT(pp->p_mapping != hme); \ 837 ASSERT(hme->hme_prev->hme_page == pp || \ 838 IS_PAHME(hme->hme_prev)); \ 839 hme->hme_prev->hme_next = hme->hme_next; \ 840 } else { \ 841 ASSERT(pp->p_mapping == hme); \ 842 pp->p_mapping = hme->hme_next; \ 843 ASSERT((pp->p_mapping == NULL) ? \ 844 (pp->p_share == 0) : 1); \ 845 } \ 846 \ 847 if (hme->hme_next) { \ 848 ASSERT(hme->hme_next->hme_page == pp || \ 849 IS_PAHME(hme->hme_next)); \ 850 hme->hme_next->hme_prev = hme->hme_prev; \ 851 } \ 852 \ 853 /* zero out the entry */ \ 854 hme->hme_next = NULL; \ 855 hme->hme_prev = NULL; \ 856 hme->hme_page = NULL; \ 857 \ 858 if (hme_size(hme) > TTE8K) { \ 859 /* remove mappings for remainder of large pg */ \ 860 sfmmu_rm_large_mappings(pp, hme_size(hme)); \ 861 } \ 862 } 863 864 /* 865 * This function returns the hment given the hme_blk and a vaddr. 866 * It assumes addr has already been checked to belong to hme_blk's 867 * range. 868 */ 869 #define HBLKTOHME(hment, hmeblkp, addr) \ 870 { \ 871 int index; \ 872 HBLKTOHME_IDX(hment, hmeblkp, addr, index) \ 873 } 874 875 /* 876 * Version of HBLKTOHME that also returns the index in hmeblkp 877 * of the hment. 878 */ 879 #define HBLKTOHME_IDX(hment, hmeblkp, addr, idx) \ 880 { \ 881 ASSERT(in_hblk_range((hmeblkp), (addr))); \ 882 \ 883 if (get_hblk_ttesz(hmeblkp) == TTE8K) { \ 884 idx = (((uintptr_t)(addr) >> MMU_PAGESHIFT) & (NHMENTS-1)); \ 885 } else \ 886 idx = 0; \ 887 \ 888 (hment) = &(hmeblkp)->hblk_hme[idx]; \ 889 } 890 891 /* 892 * Disable any page sizes not supported by the CPU 893 */ 894 void 895 hat_init_pagesizes() 896 { 897 int i; 898 899 mmu_exported_page_sizes = 0; 900 for (i = TTE8K; i < max_mmu_page_sizes; i++) { 901 902 szc_2_userszc[i] = (uint_t)-1; 903 userszc_2_szc[i] = (uint_t)-1; 904 905 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) { 906 disable_large_pages |= (1 << i); 907 } else { 908 szc_2_userszc[i] = mmu_exported_page_sizes; 909 userszc_2_szc[mmu_exported_page_sizes] = i; 910 mmu_exported_page_sizes++; 911 } 912 } 913 914 disable_ism_large_pages |= disable_large_pages; 915 disable_auto_data_large_pages = disable_large_pages; 916 disable_auto_text_large_pages = disable_large_pages; 917 918 /* 919 * Initialize mmu-specific large page sizes. 920 */ 921 if (&mmu_large_pages_disabled) { 922 disable_large_pages |= mmu_large_pages_disabled(HAT_LOAD); 923 disable_ism_large_pages |= 924 mmu_large_pages_disabled(HAT_LOAD_SHARE); 925 disable_auto_data_large_pages |= 926 mmu_large_pages_disabled(HAT_AUTO_DATA); 927 disable_auto_text_large_pages |= 928 mmu_large_pages_disabled(HAT_AUTO_TEXT); 929 } 930 } 931 932 /* 933 * Initialize the hardware address translation structures. 934 */ 935 void 936 hat_init(void) 937 { 938 int i; 939 uint_t sz; 940 uint_t maxtsb; 941 size_t size; 942 943 hat_lock_init(); 944 hat_kstat_init(); 945 946 /* 947 * Hardware-only bits in a TTE 948 */ 949 MAKE_TTE_MASK(&hw_tte); 950 951 hat_init_pagesizes(); 952 953 /* Initialize the hash locks */ 954 for (i = 0; i < khmehash_num; i++) { 955 mutex_init(&khme_hash[i].hmehash_mutex, NULL, 956 MUTEX_DEFAULT, NULL); 957 } 958 for (i = 0; i < uhmehash_num; i++) { 959 mutex_init(&uhme_hash[i].hmehash_mutex, NULL, 960 MUTEX_DEFAULT, NULL); 961 } 962 khmehash_num--; /* make sure counter starts from 0 */ 963 uhmehash_num--; /* make sure counter starts from 0 */ 964 965 /* 966 * Allocate context domain structures. 967 * 968 * A platform may choose to modify max_mmu_ctxdoms in 969 * set_platform_defaults(). If a platform does not define 970 * a set_platform_defaults() or does not choose to modify 971 * max_mmu_ctxdoms, it gets one MMU context domain for every CPU. 972 * 973 * For sun4v, there will be one global context domain, this is to 974 * avoid the ldom cpu substitution problem. 975 * 976 * For all platforms that have CPUs sharing MMUs, this 977 * value must be defined. 978 */ 979 if (max_mmu_ctxdoms == 0) { 980 #ifndef sun4v 981 max_mmu_ctxdoms = max_ncpus; 982 #else /* sun4v */ 983 max_mmu_ctxdoms = 1; 984 #endif /* sun4v */ 985 } 986 987 size = max_mmu_ctxdoms * sizeof (mmu_ctx_t *); 988 mmu_ctxs_tbl = kmem_zalloc(size, KM_SLEEP); 989 990 /* mmu_ctx_t is 64 bytes aligned */ 991 mmuctxdom_cache = kmem_cache_create("mmuctxdom_cache", 992 sizeof (mmu_ctx_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 993 /* 994 * MMU context domain initialization for the Boot CPU. 995 * This needs the context domains array allocated above. 996 */ 997 mutex_enter(&cpu_lock); 998 sfmmu_cpu_init(CPU); 999 mutex_exit(&cpu_lock); 1000 1001 /* 1002 * Intialize ism mapping list lock. 1003 */ 1004 1005 mutex_init(&ism_mlist_lock, NULL, MUTEX_DEFAULT, NULL); 1006 1007 /* 1008 * Each sfmmu structure carries an array of MMU context info 1009 * structures, one per context domain. The size of this array depends 1010 * on the maximum number of context domains. So, the size of the 1011 * sfmmu structure varies per platform. 1012 * 1013 * sfmmu is allocated from static arena, because trap 1014 * handler at TL > 0 is not allowed to touch kernel relocatable 1015 * memory. sfmmu's alignment is changed to 64 bytes from 1016 * default 8 bytes, as the lower 6 bits will be used to pass 1017 * pgcnt to vtag_flush_pgcnt_tl1. 1018 */ 1019 size = sizeof (sfmmu_t) + sizeof (sfmmu_ctx_t) * (max_mmu_ctxdoms - 1); 1020 1021 sfmmuid_cache = kmem_cache_create("sfmmuid_cache", size, 1022 64, sfmmu_idcache_constructor, sfmmu_idcache_destructor, 1023 NULL, NULL, static_arena, 0); 1024 1025 sfmmu_tsbinfo_cache = kmem_cache_create("sfmmu_tsbinfo_cache", 1026 sizeof (struct tsb_info), 0, NULL, NULL, NULL, NULL, NULL, 0); 1027 1028 /* 1029 * Since we only use the tsb8k cache to "borrow" pages for TSBs 1030 * from the heap when low on memory or when TSB_FORCEALLOC is 1031 * specified, don't use magazines to cache them--we want to return 1032 * them to the system as quickly as possible. 1033 */ 1034 sfmmu_tsb8k_cache = kmem_cache_create("sfmmu_tsb8k_cache", 1035 MMU_PAGESIZE, MMU_PAGESIZE, NULL, NULL, NULL, NULL, 1036 static_arena, KMC_NOMAGAZINE); 1037 1038 /* 1039 * Set tsb_alloc_hiwater to 1/tsb_alloc_hiwater_factor of physical 1040 * memory, which corresponds to the old static reserve for TSBs. 1041 * tsb_alloc_hiwater_factor defaults to 32. This caps the amount of 1042 * memory we'll allocate for TSB slabs; beyond this point TSB 1043 * allocations will be taken from the kernel heap (via 1044 * sfmmu_tsb8k_cache) and will be throttled as would any other kmem 1045 * consumer. 1046 */ 1047 if (tsb_alloc_hiwater_factor == 0) { 1048 tsb_alloc_hiwater_factor = TSB_ALLOC_HIWATER_FACTOR_DEFAULT; 1049 } 1050 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 1051 1052 /* Set tsb_max_growsize. */ 1053 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 1054 1055 /* 1056 * On smaller memory systems, allocate TSB memory in smaller chunks 1057 * than the default 4M slab size. We also honor disable_large_pages 1058 * here. 1059 * 1060 * The trap handlers need to be patched with the final slab shift, 1061 * since they need to be able to construct the TSB pointer at runtime. 1062 */ 1063 if (tsb_max_growsize <= TSB_512K_SZCODE) 1064 tsb_slab_ttesz = TTE512K; 1065 1066 for (sz = tsb_slab_ttesz; sz > 0; sz--) { 1067 if (!(disable_large_pages & (1 << sz))) 1068 break; 1069 } 1070 1071 tsb_slab_ttesz = sz; 1072 tsb_slab_shift = MMU_PAGESHIFT + (sz << 1) + sz; 1073 tsb_slab_size = 1 << tsb_slab_shift; 1074 tsb_slab_mask = (1 << (tsb_slab_shift - MMU_PAGESHIFT)) - 1; 1075 1076 maxtsb = tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT); 1077 if (tsb_max_growsize > maxtsb) 1078 tsb_max_growsize = maxtsb; 1079 1080 /* 1081 * Set up memory callback to update tsb_alloc_hiwater and 1082 * tsb_max_growsize. 1083 */ 1084 i = kphysm_setup_func_register(&sfmmu_update_tsb_vec, (void *) 0); 1085 ASSERT(i == 0); 1086 1087 /* 1088 * kmem_tsb_arena is the source from which large TSB slabs are 1089 * drawn. The quantum of this arena corresponds to the largest 1090 * TSB size we can dynamically allocate for user processes. 1091 * Currently it must also be a supported page size since we 1092 * use exactly one translation entry to map each slab page. 1093 * 1094 * The per-lgroup kmem_tsb_default_arena arenas are the arenas from 1095 * which most TSBs are allocated. Since most TSB allocations are 1096 * typically 8K we have a kmem cache we stack on top of each 1097 * kmem_tsb_default_arena to speed up those allocations. 1098 * 1099 * Note the two-level scheme of arenas is required only 1100 * because vmem_create doesn't allow us to specify alignment 1101 * requirements. If this ever changes the code could be 1102 * simplified to use only one level of arenas. 1103 */ 1104 kmem_tsb_arena = vmem_create("kmem_tsb", NULL, 0, tsb_slab_size, 1105 sfmmu_vmem_xalloc_aligned_wrapper, vmem_xfree, heap_arena, 1106 0, VM_SLEEP); 1107 1108 if (tsb_lgrp_affinity) { 1109 char s[50]; 1110 for (i = 0; i < NLGRPS_MAX; i++) { 1111 (void) sprintf(s, "kmem_tsb_lgrp%d", i); 1112 kmem_tsb_default_arena[i] = 1113 vmem_create(s, NULL, 0, PAGESIZE, 1114 sfmmu_tsb_segkmem_alloc, sfmmu_tsb_segkmem_free, 1115 kmem_tsb_arena, 0, VM_SLEEP | VM_BESTFIT); 1116 (void) sprintf(s, "sfmmu_tsb_lgrp%d_cache", i); 1117 sfmmu_tsb_cache[i] = kmem_cache_create(s, PAGESIZE, 1118 PAGESIZE, NULL, NULL, NULL, NULL, 1119 kmem_tsb_default_arena[i], 0); 1120 } 1121 } else { 1122 kmem_tsb_default_arena[0] = vmem_create("kmem_tsb_default", 1123 NULL, 0, PAGESIZE, sfmmu_tsb_segkmem_alloc, 1124 sfmmu_tsb_segkmem_free, kmem_tsb_arena, 0, 1125 VM_SLEEP | VM_BESTFIT); 1126 1127 sfmmu_tsb_cache[0] = kmem_cache_create("sfmmu_tsb_cache", 1128 PAGESIZE, PAGESIZE, NULL, NULL, NULL, NULL, 1129 kmem_tsb_default_arena[0], 0); 1130 } 1131 1132 sfmmu8_cache = kmem_cache_create("sfmmu8_cache", HME8BLK_SZ, 1133 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1134 sfmmu_hblkcache_destructor, 1135 sfmmu_hblkcache_reclaim, (void *)HME8BLK_SZ, 1136 hat_memload_arena, KMC_NOHASH); 1137 1138 hat_memload1_arena = vmem_create("hat_memload1", NULL, 0, PAGESIZE, 1139 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); 1140 1141 sfmmu1_cache = kmem_cache_create("sfmmu1_cache", HME1BLK_SZ, 1142 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1143 sfmmu_hblkcache_destructor, 1144 NULL, (void *)HME1BLK_SZ, 1145 hat_memload1_arena, KMC_NOHASH); 1146 1147 pa_hment_cache = kmem_cache_create("pa_hment_cache", PAHME_SZ, 1148 0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH); 1149 1150 ism_blk_cache = kmem_cache_create("ism_blk_cache", 1151 sizeof (ism_blk_t), ecache_alignsize, NULL, NULL, 1152 NULL, NULL, static_arena, KMC_NOHASH); 1153 1154 ism_ment_cache = kmem_cache_create("ism_ment_cache", 1155 sizeof (ism_ment_t), 0, NULL, NULL, 1156 NULL, NULL, NULL, 0); 1157 1158 /* 1159 * We grab the first hat for the kernel, 1160 */ 1161 AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER); 1162 kas.a_hat = hat_alloc(&kas); 1163 AS_LOCK_EXIT(&kas, &kas.a_lock); 1164 1165 /* 1166 * Initialize hblk_reserve. 1167 */ 1168 ((struct hme_blk *)hblk_reserve)->hblk_nextpa = 1169 va_to_pa((caddr_t)hblk_reserve); 1170 1171 #ifndef UTSB_PHYS 1172 /* 1173 * Reserve some kernel virtual address space for the locked TTEs 1174 * that allow us to probe the TSB from TL>0. 1175 */ 1176 utsb_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1177 0, 0, NULL, NULL, VM_SLEEP); 1178 utsb4m_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1179 0, 0, NULL, NULL, VM_SLEEP); 1180 #endif 1181 1182 #ifdef VAC 1183 /* 1184 * The big page VAC handling code assumes VAC 1185 * will not be bigger than the smallest big 1186 * page- which is 64K. 1187 */ 1188 if (TTEPAGES(TTE64K) < CACHE_NUM_COLOR) { 1189 cmn_err(CE_PANIC, "VAC too big!"); 1190 } 1191 #endif 1192 1193 (void) xhat_init(); 1194 1195 uhme_hash_pa = va_to_pa(uhme_hash); 1196 khme_hash_pa = va_to_pa(khme_hash); 1197 1198 /* 1199 * Initialize relocation locks. kpr_suspendlock is held 1200 * at PIL_MAX to prevent interrupts from pinning the holder 1201 * of a suspended TTE which may access it leading to a 1202 * deadlock condition. 1203 */ 1204 mutex_init(&kpr_mutex, NULL, MUTEX_DEFAULT, NULL); 1205 mutex_init(&kpr_suspendlock, NULL, MUTEX_SPIN, (void *)PIL_MAX); 1206 1207 /* 1208 * Pre-allocate hrm_hashtab before enabling the collection of 1209 * refmod statistics. Allocating on the fly would mean us 1210 * running the risk of suffering recursive mutex enters or 1211 * deadlocks. 1212 */ 1213 hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *), 1214 KM_SLEEP); 1215 } 1216 1217 /* 1218 * Initialize locking for the hat layer, called early during boot. 1219 */ 1220 static void 1221 hat_lock_init() 1222 { 1223 int i; 1224 1225 /* 1226 * initialize the array of mutexes protecting a page's mapping 1227 * list and p_nrm field. 1228 */ 1229 for (i = 0; i < mml_table_sz; i++) 1230 mutex_init(&mml_table[i], NULL, MUTEX_DEFAULT, NULL); 1231 1232 if (kpm_enable) { 1233 for (i = 0; i < kpmp_table_sz; i++) { 1234 mutex_init(&kpmp_table[i].khl_mutex, NULL, 1235 MUTEX_DEFAULT, NULL); 1236 } 1237 } 1238 1239 /* 1240 * Initialize array of mutex locks that protects sfmmu fields and 1241 * TSB lists. 1242 */ 1243 for (i = 0; i < SFMMU_NUM_LOCK; i++) 1244 mutex_init(HATLOCK_MUTEXP(&hat_lock[i]), NULL, MUTEX_DEFAULT, 1245 NULL); 1246 } 1247 1248 #define SFMMU_KERNEL_MAXVA \ 1249 (kmem64_base ? (uintptr_t)kmem64_end : (SYSLIMIT)) 1250 1251 /* 1252 * Allocate a hat structure. 1253 * Called when an address space first uses a hat. 1254 */ 1255 struct hat * 1256 hat_alloc(struct as *as) 1257 { 1258 sfmmu_t *sfmmup; 1259 int i; 1260 uint64_t cnum; 1261 extern uint_t get_color_start(struct as *); 1262 1263 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 1264 sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP); 1265 sfmmup->sfmmu_as = as; 1266 sfmmup->sfmmu_flags = 0; 1267 LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock); 1268 1269 if (as == &kas) { 1270 ksfmmup = sfmmup; 1271 sfmmup->sfmmu_cext = 0; 1272 cnum = KCONTEXT; 1273 1274 sfmmup->sfmmu_clrstart = 0; 1275 sfmmup->sfmmu_tsb = NULL; 1276 /* 1277 * hat_kern_setup() will call sfmmu_init_ktsbinfo() 1278 * to setup tsb_info for ksfmmup. 1279 */ 1280 } else { 1281 1282 /* 1283 * Just set to invalid ctx. When it faults, it will 1284 * get a valid ctx. This would avoid the situation 1285 * where we get a ctx, but it gets stolen and then 1286 * we fault when we try to run and so have to get 1287 * another ctx. 1288 */ 1289 sfmmup->sfmmu_cext = 0; 1290 cnum = INVALID_CONTEXT; 1291 1292 /* initialize original physical page coloring bin */ 1293 sfmmup->sfmmu_clrstart = get_color_start(as); 1294 #ifdef DEBUG 1295 if (tsb_random_size) { 1296 uint32_t randval = (uint32_t)gettick() >> 4; 1297 int size = randval % (tsb_max_growsize + 1); 1298 1299 /* chose a random tsb size for stress testing */ 1300 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size, 1301 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1302 } else 1303 #endif /* DEBUG */ 1304 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, 1305 default_tsb_size, 1306 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1307 sfmmup->sfmmu_flags = HAT_SWAPPED; 1308 ASSERT(sfmmup->sfmmu_tsb != NULL); 1309 } 1310 1311 ASSERT(max_mmu_ctxdoms > 0); 1312 for (i = 0; i < max_mmu_ctxdoms; i++) { 1313 sfmmup->sfmmu_ctxs[i].cnum = cnum; 1314 sfmmup->sfmmu_ctxs[i].gnum = 0; 1315 } 1316 1317 sfmmu_setup_tsbinfo(sfmmup); 1318 for (i = 0; i < max_mmu_page_sizes; i++) { 1319 sfmmup->sfmmu_ttecnt[i] = 0; 1320 sfmmup->sfmmu_ismttecnt[i] = 0; 1321 sfmmup->sfmmu_pgsz[i] = TTE8K; 1322 } 1323 1324 sfmmup->sfmmu_iblk = NULL; 1325 sfmmup->sfmmu_ismhat = 0; 1326 sfmmup->sfmmu_ismblkpa = (uint64_t)-1; 1327 if (sfmmup == ksfmmup) { 1328 CPUSET_ALL(sfmmup->sfmmu_cpusran); 1329 } else { 1330 CPUSET_ZERO(sfmmup->sfmmu_cpusran); 1331 } 1332 sfmmup->sfmmu_free = 0; 1333 sfmmup->sfmmu_rmstat = 0; 1334 sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart; 1335 sfmmup->sfmmu_xhat_provider = NULL; 1336 cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL); 1337 return (sfmmup); 1338 } 1339 1340 /* 1341 * Create per-MMU context domain kstats for a given MMU ctx. 1342 */ 1343 static void 1344 sfmmu_mmu_kstat_create(mmu_ctx_t *mmu_ctxp) 1345 { 1346 mmu_ctx_stat_t stat; 1347 kstat_t *mmu_kstat; 1348 1349 ASSERT(MUTEX_HELD(&cpu_lock)); 1350 ASSERT(mmu_ctxp->mmu_kstat == NULL); 1351 1352 mmu_kstat = kstat_create("unix", mmu_ctxp->mmu_idx, "mmu_ctx", 1353 "hat", KSTAT_TYPE_NAMED, MMU_CTX_NUM_STATS, KSTAT_FLAG_VIRTUAL); 1354 1355 if (mmu_kstat == NULL) { 1356 cmn_err(CE_WARN, "kstat_create for MMU %d failed", 1357 mmu_ctxp->mmu_idx); 1358 } else { 1359 mmu_kstat->ks_data = mmu_ctxp->mmu_kstat_data; 1360 for (stat = 0; stat < MMU_CTX_NUM_STATS; stat++) 1361 kstat_named_init(&mmu_ctxp->mmu_kstat_data[stat], 1362 mmu_ctx_kstat_names[stat], KSTAT_DATA_INT64); 1363 mmu_ctxp->mmu_kstat = mmu_kstat; 1364 kstat_install(mmu_kstat); 1365 } 1366 } 1367 1368 /* 1369 * plat_cpuid_to_mmu_ctx_info() is a platform interface that returns MMU 1370 * context domain information for a given CPU. If a platform does not 1371 * specify that interface, then the function below is used instead to return 1372 * default information. The defaults are as follows: 1373 * 1374 * - For sun4u systems there's one MMU context domain per CPU. 1375 * This default is used by all sun4u systems except OPL. OPL systems 1376 * provide platform specific interface to map CPU ids to MMU ids 1377 * because on OPL more than 1 CPU shares a single MMU. 1378 * Note that on sun4v, there is one global context domain for 1379 * the entire system. This is to avoid running into potential problem 1380 * with ldom physical cpu substitution feature. 1381 * - The number of MMU context IDs supported on any CPU in the 1382 * system is 8K. 1383 */ 1384 /*ARGSUSED*/ 1385 static void 1386 sfmmu_cpuid_to_mmu_ctx_info(processorid_t cpuid, mmu_ctx_info_t *infop) 1387 { 1388 infop->mmu_nctxs = nctxs; 1389 #ifndef sun4v 1390 infop->mmu_idx = cpu[cpuid]->cpu_seqid; 1391 #else /* sun4v */ 1392 infop->mmu_idx = 0; 1393 #endif /* sun4v */ 1394 } 1395 1396 /* 1397 * Called during CPU initialization to set the MMU context-related information 1398 * for a CPU. 1399 * 1400 * cpu_lock serializes accesses to mmu_ctxs and mmu_saved_gnum. 1401 */ 1402 void 1403 sfmmu_cpu_init(cpu_t *cp) 1404 { 1405 mmu_ctx_info_t info; 1406 mmu_ctx_t *mmu_ctxp; 1407 1408 ASSERT(MUTEX_HELD(&cpu_lock)); 1409 1410 if (&plat_cpuid_to_mmu_ctx_info == NULL) 1411 sfmmu_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1412 else 1413 plat_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1414 1415 ASSERT(info.mmu_idx < max_mmu_ctxdoms); 1416 1417 if ((mmu_ctxp = mmu_ctxs_tbl[info.mmu_idx]) == NULL) { 1418 /* Each mmu_ctx is cacheline aligned. */ 1419 mmu_ctxp = kmem_cache_alloc(mmuctxdom_cache, KM_SLEEP); 1420 bzero(mmu_ctxp, sizeof (mmu_ctx_t)); 1421 1422 mutex_init(&mmu_ctxp->mmu_lock, NULL, MUTEX_SPIN, 1423 (void *)ipltospl(DISP_LEVEL)); 1424 mmu_ctxp->mmu_idx = info.mmu_idx; 1425 mmu_ctxp->mmu_nctxs = info.mmu_nctxs; 1426 /* 1427 * Globally for lifetime of a system, 1428 * gnum must always increase. 1429 * mmu_saved_gnum is protected by the cpu_lock. 1430 */ 1431 mmu_ctxp->mmu_gnum = mmu_saved_gnum + 1; 1432 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 1433 1434 sfmmu_mmu_kstat_create(mmu_ctxp); 1435 1436 mmu_ctxs_tbl[info.mmu_idx] = mmu_ctxp; 1437 } else { 1438 ASSERT(mmu_ctxp->mmu_idx == info.mmu_idx); 1439 } 1440 1441 /* 1442 * The mmu_lock is acquired here to prevent races with 1443 * the wrap-around code. 1444 */ 1445 mutex_enter(&mmu_ctxp->mmu_lock); 1446 1447 1448 mmu_ctxp->mmu_ncpus++; 1449 CPUSET_ADD(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1450 CPU_MMU_IDX(cp) = info.mmu_idx; 1451 CPU_MMU_CTXP(cp) = mmu_ctxp; 1452 1453 mutex_exit(&mmu_ctxp->mmu_lock); 1454 } 1455 1456 /* 1457 * Called to perform MMU context-related cleanup for a CPU. 1458 */ 1459 void 1460 sfmmu_cpu_cleanup(cpu_t *cp) 1461 { 1462 mmu_ctx_t *mmu_ctxp; 1463 1464 ASSERT(MUTEX_HELD(&cpu_lock)); 1465 1466 mmu_ctxp = CPU_MMU_CTXP(cp); 1467 ASSERT(mmu_ctxp != NULL); 1468 1469 /* 1470 * The mmu_lock is acquired here to prevent races with 1471 * the wrap-around code. 1472 */ 1473 mutex_enter(&mmu_ctxp->mmu_lock); 1474 1475 CPU_MMU_CTXP(cp) = NULL; 1476 1477 CPUSET_DEL(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1478 if (--mmu_ctxp->mmu_ncpus == 0) { 1479 mmu_ctxs_tbl[mmu_ctxp->mmu_idx] = NULL; 1480 mutex_exit(&mmu_ctxp->mmu_lock); 1481 mutex_destroy(&mmu_ctxp->mmu_lock); 1482 1483 if (mmu_ctxp->mmu_kstat) 1484 kstat_delete(mmu_ctxp->mmu_kstat); 1485 1486 /* mmu_saved_gnum is protected by the cpu_lock. */ 1487 if (mmu_saved_gnum < mmu_ctxp->mmu_gnum) 1488 mmu_saved_gnum = mmu_ctxp->mmu_gnum; 1489 1490 kmem_cache_free(mmuctxdom_cache, mmu_ctxp); 1491 1492 return; 1493 } 1494 1495 mutex_exit(&mmu_ctxp->mmu_lock); 1496 } 1497 1498 /* 1499 * Hat_setup, makes an address space context the current active one. 1500 * In sfmmu this translates to setting the secondary context with the 1501 * corresponding context. 1502 */ 1503 void 1504 hat_setup(struct hat *sfmmup, int allocflag) 1505 { 1506 hatlock_t *hatlockp; 1507 1508 /* Init needs some special treatment. */ 1509 if (allocflag == HAT_INIT) { 1510 /* 1511 * Make sure that we have 1512 * 1. a TSB 1513 * 2. a valid ctx that doesn't get stolen after this point. 1514 */ 1515 hatlockp = sfmmu_hat_enter(sfmmup); 1516 1517 /* 1518 * Swap in the TSB. hat_init() allocates tsbinfos without 1519 * TSBs, but we need one for init, since the kernel does some 1520 * special things to set up its stack and needs the TSB to 1521 * resolve page faults. 1522 */ 1523 sfmmu_tsb_swapin(sfmmup, hatlockp); 1524 1525 sfmmu_get_ctx(sfmmup); 1526 1527 sfmmu_hat_exit(hatlockp); 1528 } else { 1529 ASSERT(allocflag == HAT_ALLOC); 1530 1531 hatlockp = sfmmu_hat_enter(sfmmup); 1532 kpreempt_disable(); 1533 1534 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); 1535 1536 /* 1537 * sfmmu_setctx_sec takes <pgsz|cnum> as a parameter, 1538 * pagesize bits don't matter in this case since we are passing 1539 * INVALID_CONTEXT to it. 1540 */ 1541 sfmmu_setctx_sec(INVALID_CONTEXT); 1542 sfmmu_clear_utsbinfo(); 1543 1544 kpreempt_enable(); 1545 sfmmu_hat_exit(hatlockp); 1546 } 1547 } 1548 1549 /* 1550 * Free all the translation resources for the specified address space. 1551 * Called from as_free when an address space is being destroyed. 1552 */ 1553 void 1554 hat_free_start(struct hat *sfmmup) 1555 { 1556 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 1557 ASSERT(sfmmup != ksfmmup); 1558 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1559 1560 sfmmup->sfmmu_free = 1; 1561 } 1562 1563 void 1564 hat_free_end(struct hat *sfmmup) 1565 { 1566 int i; 1567 1568 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1569 if (sfmmup->sfmmu_ismhat) { 1570 for (i = 0; i < mmu_page_sizes; i++) { 1571 sfmmup->sfmmu_ttecnt[i] = 0; 1572 sfmmup->sfmmu_ismttecnt[i] = 0; 1573 } 1574 } else { 1575 /* EMPTY */ 1576 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 1577 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 1578 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 1579 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 1580 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 1581 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 1582 } 1583 1584 if (sfmmup->sfmmu_rmstat) { 1585 hat_freestat(sfmmup->sfmmu_as, NULL); 1586 } 1587 1588 while (sfmmup->sfmmu_tsb != NULL) { 1589 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next; 1590 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb); 1591 sfmmup->sfmmu_tsb = next; 1592 } 1593 sfmmu_free_sfmmu(sfmmup); 1594 1595 kmem_cache_free(sfmmuid_cache, sfmmup); 1596 } 1597 1598 /* 1599 * Set up any translation structures, for the specified address space, 1600 * that are needed or preferred when the process is being swapped in. 1601 */ 1602 /* ARGSUSED */ 1603 void 1604 hat_swapin(struct hat *hat) 1605 { 1606 ASSERT(hat->sfmmu_xhat_provider == NULL); 1607 } 1608 1609 /* 1610 * Free all of the translation resources, for the specified address space, 1611 * that can be freed while the process is swapped out. Called from as_swapout. 1612 * Also, free up the ctx that this process was using. 1613 */ 1614 void 1615 hat_swapout(struct hat *sfmmup) 1616 { 1617 struct hmehash_bucket *hmebp; 1618 struct hme_blk *hmeblkp; 1619 struct hme_blk *pr_hblk = NULL; 1620 struct hme_blk *nx_hblk; 1621 int i; 1622 uint64_t hblkpa, prevpa, nx_pa; 1623 struct hme_blk *list = NULL; 1624 hatlock_t *hatlockp; 1625 struct tsb_info *tsbinfop; 1626 struct free_tsb { 1627 struct free_tsb *next; 1628 struct tsb_info *tsbinfop; 1629 }; /* free list of TSBs */ 1630 struct free_tsb *freelist, *last, *next; 1631 1632 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1633 SFMMU_STAT(sf_swapout); 1634 1635 /* 1636 * There is no way to go from an as to all its translations in sfmmu. 1637 * Here is one of the times when we take the big hit and traverse 1638 * the hash looking for hme_blks to free up. Not only do we free up 1639 * this as hme_blks but all those that are free. We are obviously 1640 * swapping because we need memory so let's free up as much 1641 * as we can. 1642 * 1643 * Note that we don't flush TLB/TSB here -- it's not necessary 1644 * because: 1645 * 1) we free the ctx we're using and throw away the TSB(s); 1646 * 2) processes aren't runnable while being swapped out. 1647 */ 1648 ASSERT(sfmmup != KHATID); 1649 for (i = 0; i <= UHMEHASH_SZ; i++) { 1650 hmebp = &uhme_hash[i]; 1651 SFMMU_HASH_LOCK(hmebp); 1652 hmeblkp = hmebp->hmeblkp; 1653 hblkpa = hmebp->hmeh_nextpa; 1654 prevpa = 0; 1655 pr_hblk = NULL; 1656 while (hmeblkp) { 1657 1658 ASSERT(!hmeblkp->hblk_xhat_bit); 1659 1660 if ((hmeblkp->hblk_tag.htag_id == sfmmup) && 1661 !hmeblkp->hblk_shw_bit && !hmeblkp->hblk_lckcnt) { 1662 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 1663 (caddr_t)get_hblk_base(hmeblkp), 1664 get_hblk_endaddr(hmeblkp), 1665 NULL, HAT_UNLOAD); 1666 } 1667 nx_hblk = hmeblkp->hblk_next; 1668 nx_pa = hmeblkp->hblk_nextpa; 1669 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 1670 ASSERT(!hmeblkp->hblk_lckcnt); 1671 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 1672 prevpa, pr_hblk); 1673 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 1674 } else { 1675 pr_hblk = hmeblkp; 1676 prevpa = hblkpa; 1677 } 1678 hmeblkp = nx_hblk; 1679 hblkpa = nx_pa; 1680 } 1681 SFMMU_HASH_UNLOCK(hmebp); 1682 } 1683 1684 sfmmu_hblks_list_purge(&list); 1685 1686 /* 1687 * Now free up the ctx so that others can reuse it. 1688 */ 1689 hatlockp = sfmmu_hat_enter(sfmmup); 1690 1691 sfmmu_invalidate_ctx(sfmmup); 1692 1693 /* 1694 * Free TSBs, but not tsbinfos, and set SWAPPED flag. 1695 * If TSBs were never swapped in, just return. 1696 * This implies that we don't support partial swapping 1697 * of TSBs -- either all are swapped out, or none are. 1698 * 1699 * We must hold the HAT lock here to prevent racing with another 1700 * thread trying to unmap TTEs from the TSB or running the post- 1701 * relocator after relocating the TSB's memory. Unfortunately, we 1702 * can't free memory while holding the HAT lock or we could 1703 * deadlock, so we build a list of TSBs to be freed after marking 1704 * the tsbinfos as swapped out and free them after dropping the 1705 * lock. 1706 */ 1707 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 1708 sfmmu_hat_exit(hatlockp); 1709 return; 1710 } 1711 1712 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED); 1713 last = freelist = NULL; 1714 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 1715 tsbinfop = tsbinfop->tsb_next) { 1716 ASSERT((tsbinfop->tsb_flags & TSB_SWAPPED) == 0); 1717 1718 /* 1719 * Cast the TSB into a struct free_tsb and put it on the free 1720 * list. 1721 */ 1722 if (freelist == NULL) { 1723 last = freelist = (struct free_tsb *)tsbinfop->tsb_va; 1724 } else { 1725 last->next = (struct free_tsb *)tsbinfop->tsb_va; 1726 last = last->next; 1727 } 1728 last->next = NULL; 1729 last->tsbinfop = tsbinfop; 1730 tsbinfop->tsb_flags |= TSB_SWAPPED; 1731 /* 1732 * Zero out the TTE to clear the valid bit. 1733 * Note we can't use a value like 0xbad because we want to 1734 * ensure diagnostic bits are NEVER set on TTEs that might 1735 * be loaded. The intent is to catch any invalid access 1736 * to the swapped TSB, such as a thread running with a valid 1737 * context without first calling sfmmu_tsb_swapin() to 1738 * allocate TSB memory. 1739 */ 1740 tsbinfop->tsb_tte.ll = 0; 1741 } 1742 1743 #ifdef sun4v 1744 if (freelist) 1745 sfmmu_invalidate_tsbinfo(sfmmup); 1746 #endif /* sun4v */ 1747 1748 /* Now we can drop the lock and free the TSB memory. */ 1749 sfmmu_hat_exit(hatlockp); 1750 for (; freelist != NULL; freelist = next) { 1751 next = freelist->next; 1752 sfmmu_tsb_free(freelist->tsbinfop); 1753 } 1754 } 1755 1756 /* 1757 * Duplicate the translations of an as into another newas 1758 */ 1759 /* ARGSUSED */ 1760 int 1761 hat_dup(struct hat *hat, struct hat *newhat, caddr_t addr, size_t len, 1762 uint_t flag) 1763 { 1764 extern uint_t get_color_start(struct as *); 1765 1766 ASSERT(hat->sfmmu_xhat_provider == NULL); 1767 ASSERT((flag == 0) || (flag == HAT_DUP_ALL) || (flag == HAT_DUP_COW)); 1768 1769 if (flag == HAT_DUP_COW) { 1770 panic("hat_dup: HAT_DUP_COW not supported"); 1771 } 1772 if (flag == HAT_DUP_ALL && consistent_coloring == 0 && 1773 update_proc_pgcolorbase_after_fork != 0) { 1774 hat->sfmmu_clrbin = get_color_start(hat->sfmmu_as); 1775 } 1776 return (0); 1777 } 1778 1779 /* 1780 * Set up addr to map to page pp with protection prot. 1781 * As an optimization we also load the TSB with the 1782 * corresponding tte but it is no big deal if the tte gets kicked out. 1783 */ 1784 void 1785 hat_memload(struct hat *hat, caddr_t addr, struct page *pp, 1786 uint_t attr, uint_t flags) 1787 { 1788 tte_t tte; 1789 1790 1791 ASSERT(hat != NULL); 1792 ASSERT(PAGE_LOCKED(pp)); 1793 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 1794 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1795 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1796 1797 if (PP_ISFREE(pp)) { 1798 panic("hat_memload: loading a mapping to free page %p", 1799 (void *)pp); 1800 } 1801 1802 if (hat->sfmmu_xhat_provider) { 1803 XHAT_MEMLOAD(hat, addr, pp, attr, flags); 1804 return; 1805 } 1806 1807 ASSERT((hat == ksfmmup) || 1808 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1809 1810 if (flags & ~SFMMU_LOAD_ALLFLAG) 1811 cmn_err(CE_NOTE, "hat_memload: unsupported flags %d", 1812 flags & ~SFMMU_LOAD_ALLFLAG); 1813 1814 if (hat->sfmmu_rmstat) 1815 hat_resvstat(MMU_PAGESIZE, hat->sfmmu_as, addr); 1816 1817 #if defined(SF_ERRATA_57) 1818 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1819 (addr < errata57_limit) && (attr & PROT_EXEC) && 1820 !(flags & HAT_LOAD_SHARE)) { 1821 cmn_err(CE_WARN, "hat_memload: illegal attempt to make user " 1822 " page executable"); 1823 attr &= ~PROT_EXEC; 1824 } 1825 #endif 1826 1827 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 1828 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, flags); 1829 1830 /* 1831 * Check TSB and TLB page sizes. 1832 */ 1833 if ((flags & HAT_LOAD_SHARE) == 0) { 1834 sfmmu_check_page_sizes(hat, 1); 1835 } 1836 } 1837 1838 /* 1839 * hat_devload can be called to map real memory (e.g. 1840 * /dev/kmem) and even though hat_devload will determine pf is 1841 * for memory, it will be unable to get a shared lock on the 1842 * page (because someone else has it exclusively) and will 1843 * pass dp = NULL. If tteload doesn't get a non-NULL 1844 * page pointer it can't cache memory. 1845 */ 1846 void 1847 hat_devload(struct hat *hat, caddr_t addr, size_t len, pfn_t pfn, 1848 uint_t attr, int flags) 1849 { 1850 tte_t tte; 1851 struct page *pp = NULL; 1852 int use_lgpg = 0; 1853 1854 ASSERT(hat != NULL); 1855 1856 if (hat->sfmmu_xhat_provider) { 1857 XHAT_DEVLOAD(hat, addr, len, pfn, attr, flags); 1858 return; 1859 } 1860 1861 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1862 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1863 ASSERT((hat == ksfmmup) || 1864 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1865 if (len == 0) 1866 panic("hat_devload: zero len"); 1867 if (flags & ~SFMMU_LOAD_ALLFLAG) 1868 cmn_err(CE_NOTE, "hat_devload: unsupported flags %d", 1869 flags & ~SFMMU_LOAD_ALLFLAG); 1870 1871 #if defined(SF_ERRATA_57) 1872 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1873 (addr < errata57_limit) && (attr & PROT_EXEC) && 1874 !(flags & HAT_LOAD_SHARE)) { 1875 cmn_err(CE_WARN, "hat_devload: illegal attempt to make user " 1876 " page executable"); 1877 attr &= ~PROT_EXEC; 1878 } 1879 #endif 1880 1881 /* 1882 * If it's a memory page find its pp 1883 */ 1884 if (!(flags & HAT_LOAD_NOCONSIST) && pf_is_memory(pfn)) { 1885 pp = page_numtopp_nolock(pfn); 1886 if (pp == NULL) { 1887 flags |= HAT_LOAD_NOCONSIST; 1888 } else { 1889 if (PP_ISFREE(pp)) { 1890 panic("hat_memload: loading " 1891 "a mapping to free page %p", 1892 (void *)pp); 1893 } 1894 if (!PAGE_LOCKED(pp) && !PP_ISNORELOC(pp)) { 1895 panic("hat_memload: loading a mapping " 1896 "to unlocked relocatable page %p", 1897 (void *)pp); 1898 } 1899 ASSERT(len == MMU_PAGESIZE); 1900 } 1901 } 1902 1903 if (hat->sfmmu_rmstat) 1904 hat_resvstat(len, hat->sfmmu_as, addr); 1905 1906 if (flags & HAT_LOAD_NOCONSIST) { 1907 attr |= SFMMU_UNCACHEVTTE; 1908 use_lgpg = 1; 1909 } 1910 if (!pf_is_memory(pfn)) { 1911 attr |= SFMMU_UNCACHEPTTE | HAT_NOSYNC; 1912 use_lgpg = 1; 1913 switch (attr & HAT_ORDER_MASK) { 1914 case HAT_STRICTORDER: 1915 case HAT_UNORDERED_OK: 1916 /* 1917 * we set the side effect bit for all non 1918 * memory mappings unless merging is ok 1919 */ 1920 attr |= SFMMU_SIDEFFECT; 1921 break; 1922 case HAT_MERGING_OK: 1923 case HAT_LOADCACHING_OK: 1924 case HAT_STORECACHING_OK: 1925 break; 1926 default: 1927 panic("hat_devload: bad attr"); 1928 break; 1929 } 1930 } 1931 while (len) { 1932 if (!use_lgpg) { 1933 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1934 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1935 flags); 1936 len -= MMU_PAGESIZE; 1937 addr += MMU_PAGESIZE; 1938 pfn++; 1939 continue; 1940 } 1941 /* 1942 * try to use large pages, check va/pa alignments 1943 * Note that 32M/256M page sizes are not (yet) supported. 1944 */ 1945 if ((len >= MMU_PAGESIZE4M) && 1946 !((uintptr_t)addr & MMU_PAGEOFFSET4M) && 1947 !(disable_large_pages & (1 << TTE4M)) && 1948 !(mmu_ptob(pfn) & MMU_PAGEOFFSET4M)) { 1949 sfmmu_memtte(&tte, pfn, attr, TTE4M); 1950 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1951 flags); 1952 len -= MMU_PAGESIZE4M; 1953 addr += MMU_PAGESIZE4M; 1954 pfn += MMU_PAGESIZE4M / MMU_PAGESIZE; 1955 } else if ((len >= MMU_PAGESIZE512K) && 1956 !((uintptr_t)addr & MMU_PAGEOFFSET512K) && 1957 !(disable_large_pages & (1 << TTE512K)) && 1958 !(mmu_ptob(pfn) & MMU_PAGEOFFSET512K)) { 1959 sfmmu_memtte(&tte, pfn, attr, TTE512K); 1960 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1961 flags); 1962 len -= MMU_PAGESIZE512K; 1963 addr += MMU_PAGESIZE512K; 1964 pfn += MMU_PAGESIZE512K / MMU_PAGESIZE; 1965 } else if ((len >= MMU_PAGESIZE64K) && 1966 !((uintptr_t)addr & MMU_PAGEOFFSET64K) && 1967 !(disable_large_pages & (1 << TTE64K)) && 1968 !(mmu_ptob(pfn) & MMU_PAGEOFFSET64K)) { 1969 sfmmu_memtte(&tte, pfn, attr, TTE64K); 1970 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1971 flags); 1972 len -= MMU_PAGESIZE64K; 1973 addr += MMU_PAGESIZE64K; 1974 pfn += MMU_PAGESIZE64K / MMU_PAGESIZE; 1975 } else { 1976 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1977 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1978 flags); 1979 len -= MMU_PAGESIZE; 1980 addr += MMU_PAGESIZE; 1981 pfn++; 1982 } 1983 } 1984 1985 /* 1986 * Check TSB and TLB page sizes. 1987 */ 1988 if ((flags & HAT_LOAD_SHARE) == 0) { 1989 sfmmu_check_page_sizes(hat, 1); 1990 } 1991 } 1992 1993 /* 1994 * Map the largest extend possible out of the page array. The array may NOT 1995 * be in order. The largest possible mapping a page can have 1996 * is specified in the p_szc field. The p_szc field 1997 * cannot change as long as there any mappings (large or small) 1998 * to any of the pages that make up the large page. (ie. any 1999 * promotion/demotion of page size is not up to the hat but up to 2000 * the page free list manager). The array 2001 * should consist of properly aligned contigous pages that are 2002 * part of a big page for a large mapping to be created. 2003 */ 2004 void 2005 hat_memload_array(struct hat *hat, caddr_t addr, size_t len, 2006 struct page **pps, uint_t attr, uint_t flags) 2007 { 2008 int ttesz; 2009 size_t mapsz; 2010 pgcnt_t numpg, npgs; 2011 tte_t tte; 2012 page_t *pp; 2013 uint_t large_pages_disable; 2014 2015 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 2016 2017 if (hat->sfmmu_xhat_provider) { 2018 XHAT_MEMLOAD_ARRAY(hat, addr, len, pps, attr, flags); 2019 return; 2020 } 2021 2022 if (hat->sfmmu_rmstat) 2023 hat_resvstat(len, hat->sfmmu_as, addr); 2024 2025 #if defined(SF_ERRATA_57) 2026 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 2027 (addr < errata57_limit) && (attr & PROT_EXEC) && 2028 !(flags & HAT_LOAD_SHARE)) { 2029 cmn_err(CE_WARN, "hat_memload_array: illegal attempt to make " 2030 "user page executable"); 2031 attr &= ~PROT_EXEC; 2032 } 2033 #endif 2034 2035 /* Get number of pages */ 2036 npgs = len >> MMU_PAGESHIFT; 2037 2038 if (flags & HAT_LOAD_SHARE) { 2039 large_pages_disable = disable_ism_large_pages; 2040 } else { 2041 large_pages_disable = disable_large_pages; 2042 } 2043 2044 if (npgs < NHMENTS || large_pages_disable == LARGE_PAGES_OFF) { 2045 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2046 return; 2047 } 2048 2049 while (npgs >= NHMENTS) { 2050 pp = *pps; 2051 for (ttesz = pp->p_szc; ttesz != TTE8K; ttesz--) { 2052 /* 2053 * Check if this page size is disabled. 2054 */ 2055 if (large_pages_disable & (1 << ttesz)) 2056 continue; 2057 2058 numpg = TTEPAGES(ttesz); 2059 mapsz = numpg << MMU_PAGESHIFT; 2060 if ((npgs >= numpg) && 2061 IS_P2ALIGNED(addr, mapsz) && 2062 IS_P2ALIGNED(pp->p_pagenum, numpg)) { 2063 /* 2064 * At this point we have enough pages and 2065 * we know the virtual address and the pfn 2066 * are properly aligned. We still need 2067 * to check for physical contiguity but since 2068 * it is very likely that this is the case 2069 * we will assume they are so and undo 2070 * the request if necessary. It would 2071 * be great if we could get a hint flag 2072 * like HAT_CONTIG which would tell us 2073 * the pages are contigous for sure. 2074 */ 2075 sfmmu_memtte(&tte, (*pps)->p_pagenum, 2076 attr, ttesz); 2077 if (!sfmmu_tteload_array(hat, &tte, addr, 2078 pps, flags)) { 2079 break; 2080 } 2081 } 2082 } 2083 if (ttesz == TTE8K) { 2084 /* 2085 * We were not able to map array using a large page 2086 * batch a hmeblk or fraction at a time. 2087 */ 2088 numpg = ((uintptr_t)addr >> MMU_PAGESHIFT) 2089 & (NHMENTS-1); 2090 numpg = NHMENTS - numpg; 2091 ASSERT(numpg <= npgs); 2092 mapsz = numpg * MMU_PAGESIZE; 2093 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, 2094 numpg); 2095 } 2096 addr += mapsz; 2097 npgs -= numpg; 2098 pps += numpg; 2099 } 2100 2101 if (npgs) { 2102 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2103 } 2104 2105 /* 2106 * Check TSB and TLB page sizes. 2107 */ 2108 if ((flags & HAT_LOAD_SHARE) == 0) { 2109 sfmmu_check_page_sizes(hat, 1); 2110 } 2111 } 2112 2113 /* 2114 * Function tries to batch 8K pages into the same hme blk. 2115 */ 2116 static void 2117 sfmmu_memload_batchsmall(struct hat *hat, caddr_t vaddr, page_t **pps, 2118 uint_t attr, uint_t flags, pgcnt_t npgs) 2119 { 2120 tte_t tte; 2121 page_t *pp; 2122 struct hmehash_bucket *hmebp; 2123 struct hme_blk *hmeblkp; 2124 int index; 2125 2126 while (npgs) { 2127 /* 2128 * Acquire the hash bucket. 2129 */ 2130 hmebp = sfmmu_tteload_acquire_hashbucket(hat, vaddr, TTE8K); 2131 ASSERT(hmebp); 2132 2133 /* 2134 * Find the hment block. 2135 */ 2136 hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr, 2137 TTE8K, flags); 2138 ASSERT(hmeblkp); 2139 2140 do { 2141 /* 2142 * Make the tte. 2143 */ 2144 pp = *pps; 2145 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 2146 2147 /* 2148 * Add the translation. 2149 */ 2150 (void) sfmmu_tteload_addentry(hat, hmeblkp, &tte, 2151 vaddr, pps, flags); 2152 2153 /* 2154 * Goto next page. 2155 */ 2156 pps++; 2157 npgs--; 2158 2159 /* 2160 * Goto next address. 2161 */ 2162 vaddr += MMU_PAGESIZE; 2163 2164 /* 2165 * Don't crossover into a different hmentblk. 2166 */ 2167 index = (int)(((uintptr_t)vaddr >> MMU_PAGESHIFT) & 2168 (NHMENTS-1)); 2169 2170 } while (index != 0 && npgs != 0); 2171 2172 /* 2173 * Release the hash bucket. 2174 */ 2175 2176 sfmmu_tteload_release_hashbucket(hmebp); 2177 } 2178 } 2179 2180 /* 2181 * Construct a tte for a page: 2182 * 2183 * tte_valid = 1 2184 * tte_size2 = size & TTE_SZ2_BITS (Panther and Olympus-C only) 2185 * tte_size = size 2186 * tte_nfo = attr & HAT_NOFAULT 2187 * tte_ie = attr & HAT_STRUCTURE_LE 2188 * tte_hmenum = hmenum 2189 * tte_pahi = pp->p_pagenum >> TTE_PASHIFT; 2190 * tte_palo = pp->p_pagenum & TTE_PALOMASK; 2191 * tte_ref = 1 (optimization) 2192 * tte_wr_perm = attr & PROT_WRITE; 2193 * tte_no_sync = attr & HAT_NOSYNC 2194 * tte_lock = attr & SFMMU_LOCKTTE 2195 * tte_cp = !(attr & SFMMU_UNCACHEPTTE) 2196 * tte_cv = !(attr & SFMMU_UNCACHEVTTE) 2197 * tte_e = attr & SFMMU_SIDEFFECT 2198 * tte_priv = !(attr & PROT_USER) 2199 * tte_hwwr = if nosync is set and it is writable we set the mod bit (opt) 2200 * tte_glb = 0 2201 */ 2202 void 2203 sfmmu_memtte(tte_t *ttep, pfn_t pfn, uint_t attr, int tte_sz) 2204 { 2205 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 2206 2207 ttep->tte_inthi = MAKE_TTE_INTHI(pfn, attr, tte_sz, 0 /* hmenum */); 2208 ttep->tte_intlo = MAKE_TTE_INTLO(pfn, attr, tte_sz, 0 /* hmenum */); 2209 2210 if (TTE_IS_NOSYNC(ttep)) { 2211 TTE_SET_REF(ttep); 2212 if (TTE_IS_WRITABLE(ttep)) { 2213 TTE_SET_MOD(ttep); 2214 } 2215 } 2216 if (TTE_IS_NFO(ttep) && TTE_IS_EXECUTABLE(ttep)) { 2217 panic("sfmmu_memtte: can't set both NFO and EXEC bits"); 2218 } 2219 } 2220 2221 /* 2222 * This function will add a translation to the hme_blk and allocate the 2223 * hme_blk if one does not exist. 2224 * If a page structure is specified then it will add the 2225 * corresponding hment to the mapping list. 2226 * It will also update the hmenum field for the tte. 2227 */ 2228 void 2229 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp, 2230 uint_t flags) 2231 { 2232 (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags); 2233 } 2234 2235 /* 2236 * Load (ttep != NULL) or unload (ttep == NULL) one entry in the TSB. 2237 * Assumes that a particular page size may only be resident in one TSB. 2238 */ 2239 static void 2240 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz) 2241 { 2242 struct tsb_info *tsbinfop = NULL; 2243 uint64_t tag; 2244 struct tsbe *tsbe_addr; 2245 uint64_t tsb_base; 2246 uint_t tsb_size; 2247 int vpshift = MMU_PAGESHIFT; 2248 int phys = 0; 2249 2250 if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */ 2251 phys = ktsb_phys; 2252 if (ttesz >= TTE4M) { 2253 #ifndef sun4v 2254 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2255 #endif 2256 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2257 tsb_size = ktsb4m_szcode; 2258 } else { 2259 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2260 tsb_size = ktsb_szcode; 2261 } 2262 } else { 2263 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2264 2265 /* 2266 * If there isn't a TSB for this page size, or the TSB is 2267 * swapped out, there is nothing to do. Note that the latter 2268 * case seems impossible but can occur if hat_pageunload() 2269 * is called on an ISM mapping while the process is swapped 2270 * out. 2271 */ 2272 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2273 return; 2274 2275 /* 2276 * If another thread is in the middle of relocating a TSB 2277 * we can't unload the entry so set a flag so that the 2278 * TSB will be flushed before it can be accessed by the 2279 * process. 2280 */ 2281 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2282 if (ttep == NULL) 2283 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2284 return; 2285 } 2286 #if defined(UTSB_PHYS) 2287 phys = 1; 2288 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2289 #else 2290 tsb_base = (uint64_t)tsbinfop->tsb_va; 2291 #endif 2292 tsb_size = tsbinfop->tsb_szc; 2293 } 2294 if (ttesz >= TTE4M) 2295 vpshift = MMU_PAGESHIFT4M; 2296 2297 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2298 tag = sfmmu_make_tsbtag(vaddr); 2299 2300 if (ttep == NULL) { 2301 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2302 } else { 2303 if (ttesz >= TTE4M) { 2304 SFMMU_STAT(sf_tsb_load4m); 2305 } else { 2306 SFMMU_STAT(sf_tsb_load8k); 2307 } 2308 2309 sfmmu_load_tsbe(tsbe_addr, tag, ttep, phys); 2310 } 2311 } 2312 2313 /* 2314 * Unmap all entries from [start, end) matching the given page size. 2315 * 2316 * This function is used primarily to unmap replicated 64K or 512K entries 2317 * from the TSB that are inserted using the base page size TSB pointer, but 2318 * it may also be called to unmap a range of addresses from the TSB. 2319 */ 2320 void 2321 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz) 2322 { 2323 struct tsb_info *tsbinfop; 2324 uint64_t tag; 2325 struct tsbe *tsbe_addr; 2326 caddr_t vaddr; 2327 uint64_t tsb_base; 2328 int vpshift, vpgsz; 2329 uint_t tsb_size; 2330 int phys = 0; 2331 2332 /* 2333 * Assumptions: 2334 * If ttesz == 8K, 64K or 512K, we walk through the range 8K 2335 * at a time shooting down any valid entries we encounter. 2336 * 2337 * If ttesz >= 4M we walk the range 4M at a time shooting 2338 * down any valid mappings we find. 2339 */ 2340 if (sfmmup == ksfmmup) { 2341 phys = ktsb_phys; 2342 if (ttesz >= TTE4M) { 2343 #ifndef sun4v 2344 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2345 #endif 2346 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2347 tsb_size = ktsb4m_szcode; 2348 } else { 2349 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2350 tsb_size = ktsb_szcode; 2351 } 2352 } else { 2353 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2354 2355 /* 2356 * If there isn't a TSB for this page size, or the TSB is 2357 * swapped out, there is nothing to do. Note that the latter 2358 * case seems impossible but can occur if hat_pageunload() 2359 * is called on an ISM mapping while the process is swapped 2360 * out. 2361 */ 2362 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2363 return; 2364 2365 /* 2366 * If another thread is in the middle of relocating a TSB 2367 * we can't unload the entry so set a flag so that the 2368 * TSB will be flushed before it can be accessed by the 2369 * process. 2370 */ 2371 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2372 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2373 return; 2374 } 2375 #if defined(UTSB_PHYS) 2376 phys = 1; 2377 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2378 #else 2379 tsb_base = (uint64_t)tsbinfop->tsb_va; 2380 #endif 2381 tsb_size = tsbinfop->tsb_szc; 2382 } 2383 if (ttesz >= TTE4M) { 2384 vpshift = MMU_PAGESHIFT4M; 2385 vpgsz = MMU_PAGESIZE4M; 2386 } else { 2387 vpshift = MMU_PAGESHIFT; 2388 vpgsz = MMU_PAGESIZE; 2389 } 2390 2391 for (vaddr = start; vaddr < end; vaddr += vpgsz) { 2392 tag = sfmmu_make_tsbtag(vaddr); 2393 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2394 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2395 } 2396 } 2397 2398 /* 2399 * Select the optimum TSB size given the number of mappings 2400 * that need to be cached. 2401 */ 2402 static int 2403 sfmmu_select_tsb_szc(pgcnt_t pgcnt) 2404 { 2405 int szc = 0; 2406 2407 #ifdef DEBUG 2408 if (tsb_grow_stress) { 2409 uint32_t randval = (uint32_t)gettick() >> 4; 2410 return (randval % (tsb_max_growsize + 1)); 2411 } 2412 #endif /* DEBUG */ 2413 2414 while ((szc < tsb_max_growsize) && (pgcnt > SFMMU_RSS_TSBSIZE(szc))) 2415 szc++; 2416 return (szc); 2417 } 2418 2419 /* 2420 * This function will add a translation to the hme_blk and allocate the 2421 * hme_blk if one does not exist. 2422 * If a page structure is specified then it will add the 2423 * corresponding hment to the mapping list. 2424 * It will also update the hmenum field for the tte. 2425 * Furthermore, it attempts to create a large page translation 2426 * for <addr,hat> at page array pps. It assumes addr and first 2427 * pp is correctly aligned. It returns 0 if successful and 1 otherwise. 2428 */ 2429 static int 2430 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr, 2431 page_t **pps, uint_t flags) 2432 { 2433 struct hmehash_bucket *hmebp; 2434 struct hme_blk *hmeblkp; 2435 int ret; 2436 uint_t size; 2437 2438 /* 2439 * Get mapping size. 2440 */ 2441 size = TTE_CSZ(ttep); 2442 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2443 2444 /* 2445 * Acquire the hash bucket. 2446 */ 2447 hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size); 2448 ASSERT(hmebp); 2449 2450 /* 2451 * Find the hment block. 2452 */ 2453 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags); 2454 ASSERT(hmeblkp); 2455 2456 /* 2457 * Add the translation. 2458 */ 2459 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags); 2460 2461 /* 2462 * Release the hash bucket. 2463 */ 2464 sfmmu_tteload_release_hashbucket(hmebp); 2465 2466 return (ret); 2467 } 2468 2469 /* 2470 * Function locks and returns a pointer to the hash bucket for vaddr and size. 2471 */ 2472 static struct hmehash_bucket * 2473 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size) 2474 { 2475 struct hmehash_bucket *hmebp; 2476 int hmeshift; 2477 2478 hmeshift = HME_HASH_SHIFT(size); 2479 2480 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 2481 2482 SFMMU_HASH_LOCK(hmebp); 2483 2484 return (hmebp); 2485 } 2486 2487 /* 2488 * Function returns a pointer to an hmeblk in the hash bucket, hmebp. If the 2489 * hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is 2490 * allocated. 2491 */ 2492 static struct hme_blk * 2493 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp, 2494 caddr_t vaddr, uint_t size, uint_t flags) 2495 { 2496 hmeblk_tag hblktag; 2497 int hmeshift; 2498 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 2499 uint64_t hblkpa, prevpa; 2500 struct kmem_cache *sfmmu_cache; 2501 uint_t forcefree; 2502 2503 hblktag.htag_id = sfmmup; 2504 hmeshift = HME_HASH_SHIFT(size); 2505 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 2506 hblktag.htag_rehash = HME_HASH_REHASH(size); 2507 2508 ttearray_realloc: 2509 2510 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, 2511 pr_hblk, prevpa, &list); 2512 2513 /* 2514 * We block until hblk_reserve_lock is released; it's held by 2515 * the thread, temporarily using hblk_reserve, until hblk_reserve is 2516 * replaced by a hblk from sfmmu8_cache. 2517 */ 2518 if (hmeblkp == (struct hme_blk *)hblk_reserve && 2519 hblk_reserve_thread != curthread) { 2520 SFMMU_HASH_UNLOCK(hmebp); 2521 mutex_enter(&hblk_reserve_lock); 2522 mutex_exit(&hblk_reserve_lock); 2523 SFMMU_STAT(sf_hblk_reserve_hit); 2524 SFMMU_HASH_LOCK(hmebp); 2525 goto ttearray_realloc; 2526 } 2527 2528 if (hmeblkp == NULL) { 2529 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 2530 hblktag, flags); 2531 } else { 2532 /* 2533 * It is possible for 8k and 64k hblks to collide since they 2534 * have the same rehash value. This is because we 2535 * lazily free hblks and 8K/64K blks could be lingering. 2536 * If we find size mismatch we free the block and & try again. 2537 */ 2538 if (get_hblk_ttesz(hmeblkp) != size) { 2539 ASSERT(!hmeblkp->hblk_vcnt); 2540 ASSERT(!hmeblkp->hblk_hmecnt); 2541 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 2542 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 2543 goto ttearray_realloc; 2544 } 2545 if (hmeblkp->hblk_shw_bit) { 2546 /* 2547 * if the hblk was previously used as a shadow hblk then 2548 * we will change it to a normal hblk 2549 */ 2550 if (hmeblkp->hblk_shw_mask) { 2551 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp); 2552 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2553 goto ttearray_realloc; 2554 } else { 2555 hmeblkp->hblk_shw_bit = 0; 2556 } 2557 } 2558 SFMMU_STAT(sf_hblk_hit); 2559 } 2560 2561 /* 2562 * hat_memload() should never call kmem_cache_free(); see block 2563 * comment showing the stacktrace in sfmmu_hblk_alloc(); 2564 * enqueue each hblk in the list to reserve list if it's created 2565 * from sfmmu8_cache *and* sfmmup == KHATID. 2566 */ 2567 forcefree = (sfmmup == KHATID) ? 1 : 0; 2568 while ((pr_hblk = list) != NULL) { 2569 list = pr_hblk->hblk_next; 2570 sfmmu_cache = get_hblk_cache(pr_hblk); 2571 if ((sfmmu_cache == sfmmu8_cache) && 2572 sfmmu_put_free_hblk(pr_hblk, forcefree)) 2573 continue; 2574 2575 ASSERT(sfmmup != KHATID); 2576 kmem_cache_free(sfmmu_cache, pr_hblk); 2577 } 2578 2579 ASSERT(get_hblk_ttesz(hmeblkp) == size); 2580 ASSERT(!hmeblkp->hblk_shw_bit); 2581 2582 return (hmeblkp); 2583 } 2584 2585 /* 2586 * Function adds a tte entry into the hmeblk. It returns 0 if successful and 1 2587 * otherwise. 2588 */ 2589 static int 2590 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep, 2591 caddr_t vaddr, page_t **pps, uint_t flags) 2592 { 2593 page_t *pp = *pps; 2594 int hmenum, size, remap; 2595 tte_t tteold, flush_tte; 2596 #ifdef DEBUG 2597 tte_t orig_old; 2598 #endif /* DEBUG */ 2599 struct sf_hment *sfhme; 2600 kmutex_t *pml, *pmtx; 2601 hatlock_t *hatlockp; 2602 2603 /* 2604 * remove this panic when we decide to let user virtual address 2605 * space be >= USERLIMIT. 2606 */ 2607 if (!TTE_IS_PRIVILEGED(ttep) && vaddr >= (caddr_t)USERLIMIT) 2608 panic("user addr %p in kernel space", vaddr); 2609 #if defined(TTE_IS_GLOBAL) 2610 if (TTE_IS_GLOBAL(ttep)) 2611 panic("sfmmu_tteload: creating global tte"); 2612 #endif 2613 2614 #ifdef DEBUG 2615 if (pf_is_memory(sfmmu_ttetopfn(ttep, vaddr)) && 2616 !TTE_IS_PCACHEABLE(ttep) && !sfmmu_allow_nc_trans) 2617 panic("sfmmu_tteload: non cacheable memory tte"); 2618 #endif /* DEBUG */ 2619 2620 if ((flags & HAT_LOAD_SHARE) || !TTE_IS_REF(ttep) || 2621 !TTE_IS_MOD(ttep)) { 2622 /* 2623 * Don't load TSB for dummy as in ISM. Also don't preload 2624 * the TSB if the TTE isn't writable since we're likely to 2625 * fault on it again -- preloading can be fairly expensive. 2626 */ 2627 flags |= SFMMU_NO_TSBLOAD; 2628 } 2629 2630 size = TTE_CSZ(ttep); 2631 switch (size) { 2632 case TTE8K: 2633 SFMMU_STAT(sf_tteload8k); 2634 break; 2635 case TTE64K: 2636 SFMMU_STAT(sf_tteload64k); 2637 break; 2638 case TTE512K: 2639 SFMMU_STAT(sf_tteload512k); 2640 break; 2641 case TTE4M: 2642 SFMMU_STAT(sf_tteload4m); 2643 break; 2644 case (TTE32M): 2645 SFMMU_STAT(sf_tteload32m); 2646 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2647 break; 2648 case (TTE256M): 2649 SFMMU_STAT(sf_tteload256m); 2650 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2651 break; 2652 } 2653 2654 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2655 2656 HBLKTOHME_IDX(sfhme, hmeblkp, vaddr, hmenum); 2657 2658 /* 2659 * Need to grab mlist lock here so that pageunload 2660 * will not change tte behind us. 2661 */ 2662 if (pp) { 2663 pml = sfmmu_mlist_enter(pp); 2664 } 2665 2666 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2667 /* 2668 * Look for corresponding hment and if valid verify 2669 * pfns are equal. 2670 */ 2671 remap = TTE_IS_VALID(&tteold); 2672 if (remap) { 2673 pfn_t new_pfn, old_pfn; 2674 2675 old_pfn = TTE_TO_PFN(vaddr, &tteold); 2676 new_pfn = TTE_TO_PFN(vaddr, ttep); 2677 2678 if (flags & HAT_LOAD_REMAP) { 2679 /* make sure we are remapping same type of pages */ 2680 if (pf_is_memory(old_pfn) != pf_is_memory(new_pfn)) { 2681 panic("sfmmu_tteload - tte remap io<->memory"); 2682 } 2683 if (old_pfn != new_pfn && 2684 (pp != NULL || sfhme->hme_page != NULL)) { 2685 panic("sfmmu_tteload - tte remap pp != NULL"); 2686 } 2687 } else if (old_pfn != new_pfn) { 2688 panic("sfmmu_tteload - tte remap, hmeblkp 0x%p", 2689 (void *)hmeblkp); 2690 } 2691 ASSERT(TTE_CSZ(&tteold) == TTE_CSZ(ttep)); 2692 } 2693 2694 if (pp) { 2695 if (size == TTE8K) { 2696 #ifdef VAC 2697 /* 2698 * Handle VAC consistency 2699 */ 2700 if (!remap && (cache & CACHE_VAC) && !PP_ISNC(pp)) { 2701 sfmmu_vac_conflict(sfmmup, vaddr, pp); 2702 } 2703 #endif 2704 2705 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2706 pmtx = sfmmu_page_enter(pp); 2707 PP_CLRRO(pp); 2708 sfmmu_page_exit(pmtx); 2709 } else if (!PP_ISMAPPED(pp) && 2710 (!TTE_IS_WRITABLE(ttep)) && !(PP_ISMOD(pp))) { 2711 pmtx = sfmmu_page_enter(pp); 2712 if (!(PP_ISMOD(pp))) { 2713 PP_SETRO(pp); 2714 } 2715 sfmmu_page_exit(pmtx); 2716 } 2717 2718 } else if (sfmmu_pagearray_setup(vaddr, pps, ttep, remap)) { 2719 /* 2720 * sfmmu_pagearray_setup failed so return 2721 */ 2722 sfmmu_mlist_exit(pml); 2723 return (1); 2724 } 2725 } 2726 2727 /* 2728 * Make sure hment is not on a mapping list. 2729 */ 2730 ASSERT(remap || (sfhme->hme_page == NULL)); 2731 2732 /* if it is not a remap then hme->next better be NULL */ 2733 ASSERT((!remap) ? sfhme->hme_next == NULL : 1); 2734 2735 if (flags & HAT_LOAD_LOCK) { 2736 if (((int)hmeblkp->hblk_lckcnt + 1) >= MAX_HBLK_LCKCNT) { 2737 panic("too high lckcnt-hmeblk %p", 2738 (void *)hmeblkp); 2739 } 2740 atomic_add_16(&hmeblkp->hblk_lckcnt, 1); 2741 2742 HBLK_STACK_TRACE(hmeblkp, HBLK_LOCK); 2743 } 2744 2745 #ifdef VAC 2746 if (pp && PP_ISNC(pp)) { 2747 /* 2748 * If the physical page is marked to be uncacheable, like 2749 * by a vac conflict, make sure the new mapping is also 2750 * uncacheable. 2751 */ 2752 TTE_CLR_VCACHEABLE(ttep); 2753 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 2754 } 2755 #endif 2756 ttep->tte_hmenum = hmenum; 2757 2758 #ifdef DEBUG 2759 orig_old = tteold; 2760 #endif /* DEBUG */ 2761 2762 while (sfmmu_modifytte_try(&tteold, ttep, &sfhme->hme_tte) < 0) { 2763 if ((sfmmup == KHATID) && 2764 (flags & (HAT_LOAD_LOCK | HAT_LOAD_REMAP))) { 2765 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2766 } 2767 #ifdef DEBUG 2768 chk_tte(&orig_old, &tteold, ttep, hmeblkp); 2769 #endif /* DEBUG */ 2770 } 2771 2772 if (!TTE_IS_VALID(&tteold)) { 2773 2774 atomic_add_16(&hmeblkp->hblk_vcnt, 1); 2775 atomic_add_long(&sfmmup->sfmmu_ttecnt[size], 1); 2776 2777 /* 2778 * HAT_RELOAD_SHARE has been deprecated with lpg DISM. 2779 */ 2780 2781 if (size > TTE8K && (flags & HAT_LOAD_SHARE) == 0 && 2782 sfmmup != ksfmmup) { 2783 /* 2784 * If this is the first large mapping for the process 2785 * we must force any CPUs running this process to TL=0 2786 * where they will reload the HAT flags from the 2787 * tsbmiss area. This is necessary to make the large 2788 * mappings we are about to load visible to those CPUs; 2789 * otherwise they'll loop forever calling pagefault() 2790 * since we don't search large hash chains by default. 2791 */ 2792 hatlockp = sfmmu_hat_enter(sfmmup); 2793 if (size == TTE512K && 2794 !SFMMU_FLAGS_ISSET(sfmmup, HAT_512K_FLAG)) { 2795 SFMMU_FLAGS_SET(sfmmup, HAT_512K_FLAG); 2796 sfmmu_sync_mmustate(sfmmup); 2797 } else if (size == TTE4M && 2798 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 2799 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 2800 sfmmu_sync_mmustate(sfmmup); 2801 } else if (size == TTE64K && 2802 !SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) { 2803 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 2804 /* no sync mmustate; 64K shares 8K hashes */ 2805 } else if (mmu_page_sizes == max_mmu_page_sizes) { 2806 if (size == TTE32M && 2807 !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 2808 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 2809 sfmmu_sync_mmustate(sfmmup); 2810 } else if (size == TTE256M && 2811 !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 2812 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 2813 sfmmu_sync_mmustate(sfmmup); 2814 } 2815 } 2816 if (size >= TTE4M && (flags & HAT_LOAD_TEXT) && 2817 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) { 2818 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG); 2819 } 2820 sfmmu_hat_exit(hatlockp); 2821 } 2822 } 2823 ASSERT(TTE_IS_VALID(&sfhme->hme_tte)); 2824 2825 flush_tte.tte_intlo = (tteold.tte_intlo ^ ttep->tte_intlo) & 2826 hw_tte.tte_intlo; 2827 flush_tte.tte_inthi = (tteold.tte_inthi ^ ttep->tte_inthi) & 2828 hw_tte.tte_inthi; 2829 2830 if (remap && (flush_tte.tte_inthi || flush_tte.tte_intlo)) { 2831 /* 2832 * If remap and new tte differs from old tte we need 2833 * to sync the mod bit and flush TLB/TSB. We don't 2834 * need to sync ref bit because we currently always set 2835 * ref bit in tteload. 2836 */ 2837 ASSERT(TTE_IS_REF(ttep)); 2838 if (TTE_IS_MOD(&tteold)) { 2839 sfmmu_ttesync(sfmmup, vaddr, &tteold, pp); 2840 } 2841 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0); 2842 xt_sync(sfmmup->sfmmu_cpusran); 2843 } 2844 2845 if ((flags & SFMMU_NO_TSBLOAD) == 0) { 2846 /* 2847 * We only preload 8K and 4M mappings into the TSB, since 2848 * 64K and 512K mappings are replicated and hence don't 2849 * have a single, unique TSB entry. Ditto for 32M/256M. 2850 */ 2851 if (size == TTE8K || size == TTE4M) { 2852 hatlockp = sfmmu_hat_enter(sfmmup); 2853 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte, size); 2854 sfmmu_hat_exit(hatlockp); 2855 } 2856 } 2857 if (pp) { 2858 if (!remap) { 2859 HME_ADD(sfhme, pp); 2860 atomic_add_16(&hmeblkp->hblk_hmecnt, 1); 2861 ASSERT(hmeblkp->hblk_hmecnt > 0); 2862 2863 /* 2864 * Cannot ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 2865 * see pageunload() for comment. 2866 */ 2867 } 2868 sfmmu_mlist_exit(pml); 2869 } 2870 2871 return (0); 2872 } 2873 /* 2874 * Function unlocks hash bucket. 2875 */ 2876 static void 2877 sfmmu_tteload_release_hashbucket(struct hmehash_bucket *hmebp) 2878 { 2879 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2880 SFMMU_HASH_UNLOCK(hmebp); 2881 } 2882 2883 /* 2884 * function which checks and sets up page array for a large 2885 * translation. Will set p_vcolor, p_index, p_ro fields. 2886 * Assumes addr and pfnum of first page are properly aligned. 2887 * Will check for physical contiguity. If check fails it return 2888 * non null. 2889 */ 2890 static int 2891 sfmmu_pagearray_setup(caddr_t addr, page_t **pps, tte_t *ttep, int remap) 2892 { 2893 int i, index, ttesz; 2894 pfn_t pfnum; 2895 pgcnt_t npgs; 2896 page_t *pp, *pp1; 2897 kmutex_t *pmtx; 2898 #ifdef VAC 2899 int osz; 2900 int cflags = 0; 2901 int vac_err = 0; 2902 #endif 2903 int newidx = 0; 2904 2905 ttesz = TTE_CSZ(ttep); 2906 2907 ASSERT(ttesz > TTE8K); 2908 2909 npgs = TTEPAGES(ttesz); 2910 index = PAGESZ_TO_INDEX(ttesz); 2911 2912 pfnum = (*pps)->p_pagenum; 2913 ASSERT(IS_P2ALIGNED(pfnum, npgs)); 2914 2915 /* 2916 * Save the first pp so we can do HAT_TMPNC at the end. 2917 */ 2918 pp1 = *pps; 2919 #ifdef VAC 2920 osz = fnd_mapping_sz(pp1); 2921 #endif 2922 2923 for (i = 0; i < npgs; i++, pps++) { 2924 pp = *pps; 2925 ASSERT(PAGE_LOCKED(pp)); 2926 ASSERT(pp->p_szc >= ttesz); 2927 ASSERT(pp->p_szc == pp1->p_szc); 2928 ASSERT(sfmmu_mlist_held(pp)); 2929 2930 /* 2931 * XXX is it possible to maintain P_RO on the root only? 2932 */ 2933 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2934 pmtx = sfmmu_page_enter(pp); 2935 PP_CLRRO(pp); 2936 sfmmu_page_exit(pmtx); 2937 } else if (!PP_ISMAPPED(pp) && !TTE_IS_WRITABLE(ttep) && 2938 !PP_ISMOD(pp)) { 2939 pmtx = sfmmu_page_enter(pp); 2940 if (!(PP_ISMOD(pp))) { 2941 PP_SETRO(pp); 2942 } 2943 sfmmu_page_exit(pmtx); 2944 } 2945 2946 /* 2947 * If this is a remap we skip vac & contiguity checks. 2948 */ 2949 if (remap) 2950 continue; 2951 2952 /* 2953 * set p_vcolor and detect any vac conflicts. 2954 */ 2955 #ifdef VAC 2956 if (vac_err == 0) { 2957 vac_err = sfmmu_vacconflict_array(addr, pp, &cflags); 2958 2959 } 2960 #endif 2961 2962 /* 2963 * Save current index in case we need to undo it. 2964 * Note: "PAGESZ_TO_INDEX(sz) (1 << (sz))" 2965 * "SFMMU_INDEX_SHIFT 6" 2966 * "SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1)" 2967 * "PP_MAPINDEX(p_index) (p_index & SFMMU_INDEX_MASK)" 2968 * 2969 * So: index = PAGESZ_TO_INDEX(ttesz); 2970 * if ttesz == 1 then index = 0x2 2971 * 2 then index = 0x4 2972 * 3 then index = 0x8 2973 * 4 then index = 0x10 2974 * 5 then index = 0x20 2975 * The code below checks if it's a new pagesize (ie, newidx) 2976 * in case we need to take it back out of p_index, 2977 * and then or's the new index into the existing index. 2978 */ 2979 if ((PP_MAPINDEX(pp) & index) == 0) 2980 newidx = 1; 2981 pp->p_index = (PP_MAPINDEX(pp) | index); 2982 2983 /* 2984 * contiguity check 2985 */ 2986 if (pp->p_pagenum != pfnum) { 2987 /* 2988 * If we fail the contiguity test then 2989 * the only thing we need to fix is the p_index field. 2990 * We might get a few extra flushes but since this 2991 * path is rare that is ok. The p_ro field will 2992 * get automatically fixed on the next tteload to 2993 * the page. NO TNC bit is set yet. 2994 */ 2995 while (i >= 0) { 2996 pp = *pps; 2997 if (newidx) 2998 pp->p_index = (PP_MAPINDEX(pp) & 2999 ~index); 3000 pps--; 3001 i--; 3002 } 3003 return (1); 3004 } 3005 pfnum++; 3006 addr += MMU_PAGESIZE; 3007 } 3008 3009 #ifdef VAC 3010 if (vac_err) { 3011 if (ttesz > osz) { 3012 /* 3013 * There are some smaller mappings that causes vac 3014 * conflicts. Convert all existing small mappings to 3015 * TNC. 3016 */ 3017 SFMMU_STAT_ADD(sf_uncache_conflict, npgs); 3018 sfmmu_page_cache_array(pp1, HAT_TMPNC, CACHE_FLUSH, 3019 npgs); 3020 } else { 3021 /* EMPTY */ 3022 /* 3023 * If there exists an big page mapping, 3024 * that means the whole existing big page 3025 * has TNC setting already. No need to covert to 3026 * TNC again. 3027 */ 3028 ASSERT(PP_ISTNC(pp1)); 3029 } 3030 } 3031 #endif /* VAC */ 3032 3033 return (0); 3034 } 3035 3036 #ifdef VAC 3037 /* 3038 * Routine that detects vac consistency for a large page. It also 3039 * sets virtual color for all pp's for this big mapping. 3040 */ 3041 static int 3042 sfmmu_vacconflict_array(caddr_t addr, page_t *pp, int *cflags) 3043 { 3044 int vcolor, ocolor; 3045 3046 ASSERT(sfmmu_mlist_held(pp)); 3047 3048 if (PP_ISNC(pp)) { 3049 return (HAT_TMPNC); 3050 } 3051 3052 vcolor = addr_to_vcolor(addr); 3053 if (PP_NEWPAGE(pp)) { 3054 PP_SET_VCOLOR(pp, vcolor); 3055 return (0); 3056 } 3057 3058 ocolor = PP_GET_VCOLOR(pp); 3059 if (ocolor == vcolor) { 3060 return (0); 3061 } 3062 3063 if (!PP_ISMAPPED(pp)) { 3064 /* 3065 * Previous user of page had a differnet color 3066 * but since there are no current users 3067 * we just flush the cache and change the color. 3068 * As an optimization for large pages we flush the 3069 * entire cache of that color and set a flag. 3070 */ 3071 SFMMU_STAT(sf_pgcolor_conflict); 3072 if (!CacheColor_IsFlushed(*cflags, ocolor)) { 3073 CacheColor_SetFlushed(*cflags, ocolor); 3074 sfmmu_cache_flushcolor(ocolor, pp->p_pagenum); 3075 } 3076 PP_SET_VCOLOR(pp, vcolor); 3077 return (0); 3078 } 3079 3080 /* 3081 * We got a real conflict with a current mapping. 3082 * set flags to start unencaching all mappings 3083 * and return failure so we restart looping 3084 * the pp array from the beginning. 3085 */ 3086 return (HAT_TMPNC); 3087 } 3088 #endif /* VAC */ 3089 3090 /* 3091 * creates a large page shadow hmeblk for a tte. 3092 * The purpose of this routine is to allow us to do quick unloads because 3093 * the vm layer can easily pass a very large but sparsely populated range. 3094 */ 3095 static struct hme_blk * 3096 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags) 3097 { 3098 struct hmehash_bucket *hmebp; 3099 hmeblk_tag hblktag; 3100 int hmeshift, size, vshift; 3101 uint_t shw_mask, newshw_mask; 3102 struct hme_blk *hmeblkp; 3103 3104 ASSERT(sfmmup != KHATID); 3105 if (mmu_page_sizes == max_mmu_page_sizes) { 3106 ASSERT(ttesz < TTE256M); 3107 } else { 3108 ASSERT(ttesz < TTE4M); 3109 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 3110 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 3111 } 3112 3113 if (ttesz == TTE8K) { 3114 size = TTE512K; 3115 } else { 3116 size = ++ttesz; 3117 } 3118 3119 hblktag.htag_id = sfmmup; 3120 hmeshift = HME_HASH_SHIFT(size); 3121 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 3122 hblktag.htag_rehash = HME_HASH_REHASH(size); 3123 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 3124 3125 SFMMU_HASH_LOCK(hmebp); 3126 3127 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3128 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 3129 if (hmeblkp == NULL) { 3130 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 3131 hblktag, flags); 3132 } 3133 ASSERT(hmeblkp); 3134 if (!hmeblkp->hblk_shw_mask) { 3135 /* 3136 * if this is a unused hblk it was just allocated or could 3137 * potentially be a previous large page hblk so we need to 3138 * set the shadow bit. 3139 */ 3140 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt); 3141 hmeblkp->hblk_shw_bit = 1; 3142 } else if (hmeblkp->hblk_shw_bit == 0) { 3143 panic("sfmmu_shadow_hcreate: shw bit not set in hmeblkp 0x%p", 3144 (void *)hmeblkp); 3145 } 3146 3147 vshift = vaddr_to_vshift(hblktag, vaddr, size); 3148 ASSERT(vshift < 8); 3149 /* 3150 * Atomically set shw mask bit 3151 */ 3152 do { 3153 shw_mask = hmeblkp->hblk_shw_mask; 3154 newshw_mask = shw_mask | (1 << vshift); 3155 newshw_mask = cas32(&hmeblkp->hblk_shw_mask, shw_mask, 3156 newshw_mask); 3157 } while (newshw_mask != shw_mask); 3158 3159 SFMMU_HASH_UNLOCK(hmebp); 3160 3161 return (hmeblkp); 3162 } 3163 3164 /* 3165 * This routine cleanup a previous shadow hmeblk and changes it to 3166 * a regular hblk. This happens rarely but it is possible 3167 * when a process wants to use large pages and there are hblks still 3168 * lying around from the previous as that used these hmeblks. 3169 * The alternative was to cleanup the shadow hblks at unload time 3170 * but since so few user processes actually use large pages, it is 3171 * better to be lazy and cleanup at this time. 3172 */ 3173 static void 3174 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 3175 struct hmehash_bucket *hmebp) 3176 { 3177 caddr_t addr, endaddr; 3178 int hashno, size; 3179 3180 ASSERT(hmeblkp->hblk_shw_bit); 3181 3182 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 3183 3184 if (!hmeblkp->hblk_shw_mask) { 3185 hmeblkp->hblk_shw_bit = 0; 3186 return; 3187 } 3188 addr = (caddr_t)get_hblk_base(hmeblkp); 3189 endaddr = get_hblk_endaddr(hmeblkp); 3190 size = get_hblk_ttesz(hmeblkp); 3191 hashno = size - 1; 3192 ASSERT(hashno > 0); 3193 SFMMU_HASH_UNLOCK(hmebp); 3194 3195 sfmmu_free_hblks(sfmmup, addr, endaddr, hashno); 3196 3197 SFMMU_HASH_LOCK(hmebp); 3198 } 3199 3200 static void 3201 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr, 3202 int hashno) 3203 { 3204 int hmeshift, shadow = 0; 3205 hmeblk_tag hblktag; 3206 struct hmehash_bucket *hmebp; 3207 struct hme_blk *hmeblkp; 3208 struct hme_blk *nx_hblk, *pr_hblk, *list = NULL; 3209 uint64_t hblkpa, prevpa, nx_pa; 3210 3211 ASSERT(hashno > 0); 3212 hblktag.htag_id = sfmmup; 3213 hblktag.htag_rehash = hashno; 3214 3215 hmeshift = HME_HASH_SHIFT(hashno); 3216 3217 while (addr < endaddr) { 3218 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3219 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3220 SFMMU_HASH_LOCK(hmebp); 3221 /* inline HME_HASH_SEARCH */ 3222 hmeblkp = hmebp->hmeblkp; 3223 hblkpa = hmebp->hmeh_nextpa; 3224 prevpa = 0; 3225 pr_hblk = NULL; 3226 while (hmeblkp) { 3227 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 3228 if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) { 3229 /* found hme_blk */ 3230 if (hmeblkp->hblk_shw_bit) { 3231 if (hmeblkp->hblk_shw_mask) { 3232 shadow = 1; 3233 sfmmu_shadow_hcleanup(sfmmup, 3234 hmeblkp, hmebp); 3235 break; 3236 } else { 3237 hmeblkp->hblk_shw_bit = 0; 3238 } 3239 } 3240 3241 /* 3242 * Hblk_hmecnt and hblk_vcnt could be non zero 3243 * since hblk_unload() does not gurantee that. 3244 * 3245 * XXX - this could cause tteload() to spin 3246 * where sfmmu_shadow_hcleanup() is called. 3247 */ 3248 } 3249 3250 nx_hblk = hmeblkp->hblk_next; 3251 nx_pa = hmeblkp->hblk_nextpa; 3252 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 3253 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 3254 pr_hblk); 3255 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 3256 } else { 3257 pr_hblk = hmeblkp; 3258 prevpa = hblkpa; 3259 } 3260 hmeblkp = nx_hblk; 3261 hblkpa = nx_pa; 3262 } 3263 3264 SFMMU_HASH_UNLOCK(hmebp); 3265 3266 if (shadow) { 3267 /* 3268 * We found another shadow hblk so cleaned its 3269 * children. We need to go back and cleanup 3270 * the original hblk so we don't change the 3271 * addr. 3272 */ 3273 shadow = 0; 3274 } else { 3275 addr = (caddr_t)roundup((uintptr_t)addr + 1, 3276 (1 << hmeshift)); 3277 } 3278 } 3279 sfmmu_hblks_list_purge(&list); 3280 } 3281 3282 /* 3283 * Release one hardware address translation lock on the given address range. 3284 */ 3285 void 3286 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len) 3287 { 3288 struct hmehash_bucket *hmebp; 3289 hmeblk_tag hblktag; 3290 int hmeshift, hashno = 1; 3291 struct hme_blk *hmeblkp, *list = NULL; 3292 caddr_t endaddr; 3293 3294 ASSERT(sfmmup != NULL); 3295 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3296 3297 ASSERT((sfmmup == ksfmmup) || 3298 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 3299 ASSERT((len & MMU_PAGEOFFSET) == 0); 3300 endaddr = addr + len; 3301 hblktag.htag_id = sfmmup; 3302 3303 /* 3304 * Spitfire supports 4 page sizes. 3305 * Most pages are expected to be of the smallest page size (8K) and 3306 * these will not need to be rehashed. 64K pages also don't need to be 3307 * rehashed because an hmeblk spans 64K of address space. 512K pages 3308 * might need 1 rehash and and 4M pages might need 2 rehashes. 3309 */ 3310 while (addr < endaddr) { 3311 hmeshift = HME_HASH_SHIFT(hashno); 3312 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3313 hblktag.htag_rehash = hashno; 3314 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3315 3316 SFMMU_HASH_LOCK(hmebp); 3317 3318 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 3319 if (hmeblkp != NULL) { 3320 /* 3321 * If we encounter a shadow hmeblk then 3322 * we know there are no valid hmeblks mapping 3323 * this address at this size or larger. 3324 * Just increment address by the smallest 3325 * page size. 3326 */ 3327 if (hmeblkp->hblk_shw_bit) { 3328 addr += MMU_PAGESIZE; 3329 } else { 3330 addr = sfmmu_hblk_unlock(hmeblkp, addr, 3331 endaddr); 3332 } 3333 SFMMU_HASH_UNLOCK(hmebp); 3334 hashno = 1; 3335 continue; 3336 } 3337 SFMMU_HASH_UNLOCK(hmebp); 3338 3339 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 3340 /* 3341 * We have traversed the whole list and rehashed 3342 * if necessary without finding the address to unlock 3343 * which should never happen. 3344 */ 3345 panic("sfmmu_unlock: addr not found. " 3346 "addr %p hat %p", (void *)addr, (void *)sfmmup); 3347 } else { 3348 hashno++; 3349 } 3350 } 3351 3352 sfmmu_hblks_list_purge(&list); 3353 } 3354 3355 /* 3356 * Function to unlock a range of addresses in an hmeblk. It returns the 3357 * next address that needs to be unlocked. 3358 * Should be called with the hash lock held. 3359 */ 3360 static caddr_t 3361 sfmmu_hblk_unlock(struct hme_blk *hmeblkp, caddr_t addr, caddr_t endaddr) 3362 { 3363 struct sf_hment *sfhme; 3364 tte_t tteold, ttemod; 3365 int ttesz, ret; 3366 3367 ASSERT(in_hblk_range(hmeblkp, addr)); 3368 ASSERT(hmeblkp->hblk_shw_bit == 0); 3369 3370 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 3371 ttesz = get_hblk_ttesz(hmeblkp); 3372 3373 HBLKTOHME(sfhme, hmeblkp, addr); 3374 while (addr < endaddr) { 3375 readtte: 3376 sfmmu_copytte(&sfhme->hme_tte, &tteold); 3377 if (TTE_IS_VALID(&tteold)) { 3378 3379 ttemod = tteold; 3380 3381 ret = sfmmu_modifytte_try(&tteold, &ttemod, 3382 &sfhme->hme_tte); 3383 3384 if (ret < 0) 3385 goto readtte; 3386 3387 if (hmeblkp->hblk_lckcnt == 0) 3388 panic("zero hblk lckcnt"); 3389 3390 if (((uintptr_t)addr + TTEBYTES(ttesz)) > 3391 (uintptr_t)endaddr) 3392 panic("can't unlock large tte"); 3393 3394 ASSERT(hmeblkp->hblk_lckcnt > 0); 3395 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 3396 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 3397 } else { 3398 panic("sfmmu_hblk_unlock: invalid tte"); 3399 } 3400 addr += TTEBYTES(ttesz); 3401 sfhme++; 3402 } 3403 return (addr); 3404 } 3405 3406 /* 3407 * Physical Address Mapping Framework 3408 * 3409 * General rules: 3410 * 3411 * (1) Applies only to seg_kmem memory pages. To make things easier, 3412 * seg_kpm addresses are also accepted by the routines, but nothing 3413 * is done with them since by definition their PA mappings are static. 3414 * (2) hat_add_callback() may only be called while holding the page lock 3415 * SE_SHARED or SE_EXCL of the underlying page (e.g., as_pagelock()), 3416 * or passing HAC_PAGELOCK flag. 3417 * (3) prehandler() and posthandler() may not call hat_add_callback() or 3418 * hat_delete_callback(), nor should they allocate memory. Post quiesce 3419 * callbacks may not sleep or acquire adaptive mutex locks. 3420 * (4) Either prehandler() or posthandler() (but not both) may be specified 3421 * as being NULL. Specifying an errhandler() is optional. 3422 * 3423 * Details of using the framework: 3424 * 3425 * registering a callback (hat_register_callback()) 3426 * 3427 * Pass prehandler, posthandler, errhandler addresses 3428 * as described below. If capture_cpus argument is nonzero, 3429 * suspend callback to the prehandler will occur with CPUs 3430 * captured and executing xc_loop() and CPUs will remain 3431 * captured until after the posthandler suspend callback 3432 * occurs. 3433 * 3434 * adding a callback (hat_add_callback()) 3435 * 3436 * as_pagelock(); 3437 * hat_add_callback(); 3438 * save returned pfn in private data structures or program registers; 3439 * as_pageunlock(); 3440 * 3441 * prehandler() 3442 * 3443 * Stop all accesses by physical address to this memory page. 3444 * Called twice: the first, PRESUSPEND, is a context safe to acquire 3445 * adaptive locks. The second, SUSPEND, is called at high PIL with 3446 * CPUs captured so adaptive locks may NOT be acquired (and all spin 3447 * locks must be XCALL_PIL or higher locks). 3448 * 3449 * May return the following errors: 3450 * EIO: A fatal error has occurred. This will result in panic. 3451 * EAGAIN: The page cannot be suspended. This will fail the 3452 * relocation. 3453 * 0: Success. 3454 * 3455 * posthandler() 3456 * 3457 * Save new pfn in private data structures or program registers; 3458 * not allowed to fail (non-zero return values will result in panic). 3459 * 3460 * errhandler() 3461 * 3462 * called when an error occurs related to the callback. Currently 3463 * the only such error is HAT_CB_ERR_LEAKED which indicates that 3464 * a page is being freed, but there are still outstanding callback(s) 3465 * registered on the page. 3466 * 3467 * removing a callback (hat_delete_callback(); e.g., prior to freeing memory) 3468 * 3469 * stop using physical address 3470 * hat_delete_callback(); 3471 * 3472 */ 3473 3474 /* 3475 * Register a callback class. Each subsystem should do this once and 3476 * cache the id_t returned for use in setting up and tearing down callbacks. 3477 * 3478 * There is no facility for removing callback IDs once they are created; 3479 * the "key" should be unique for each module, so in case a module is unloaded 3480 * and subsequently re-loaded, we can recycle the module's previous entry. 3481 */ 3482 id_t 3483 hat_register_callback(int key, 3484 int (*prehandler)(caddr_t, uint_t, uint_t, void *), 3485 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t), 3486 int (*errhandler)(caddr_t, uint_t, uint_t, void *), 3487 int capture_cpus) 3488 { 3489 id_t id; 3490 3491 /* 3492 * Search the table for a pre-existing callback associated with 3493 * the identifier "key". If one exists, we re-use that entry in 3494 * the table for this instance, otherwise we assign the next 3495 * available table slot. 3496 */ 3497 for (id = 0; id < sfmmu_max_cb_id; id++) { 3498 if (sfmmu_cb_table[id].key == key) 3499 break; 3500 } 3501 3502 if (id == sfmmu_max_cb_id) { 3503 id = sfmmu_cb_nextid++; 3504 if (id >= sfmmu_max_cb_id) 3505 panic("hat_register_callback: out of callback IDs"); 3506 } 3507 3508 ASSERT(prehandler != NULL || posthandler != NULL); 3509 3510 sfmmu_cb_table[id].key = key; 3511 sfmmu_cb_table[id].prehandler = prehandler; 3512 sfmmu_cb_table[id].posthandler = posthandler; 3513 sfmmu_cb_table[id].errhandler = errhandler; 3514 sfmmu_cb_table[id].capture_cpus = capture_cpus; 3515 3516 return (id); 3517 } 3518 3519 #define HAC_COOKIE_NONE (void *)-1 3520 3521 /* 3522 * Add relocation callbacks to the specified addr/len which will be called 3523 * when relocating the associated page. See the description of pre and 3524 * posthandler above for more details. 3525 * 3526 * If HAC_PAGELOCK is included in flags, the underlying memory page is 3527 * locked internally so the caller must be able to deal with the callback 3528 * running even before this function has returned. If HAC_PAGELOCK is not 3529 * set, it is assumed that the underlying memory pages are locked. 3530 * 3531 * Since the caller must track the individual page boundaries anyway, 3532 * we only allow a callback to be added to a single page (large 3533 * or small). Thus [addr, addr + len) MUST be contained within a single 3534 * page. 3535 * 3536 * Registering multiple callbacks on the same [addr, addr+len) is supported, 3537 * _provided_that_ a unique parameter is specified for each callback. 3538 * If multiple callbacks are registered on the same range the callback will 3539 * be invoked with each unique parameter. Registering the same callback with 3540 * the same argument more than once will result in corrupted kernel state. 3541 * 3542 * Returns the pfn of the underlying kernel page in *rpfn 3543 * on success, or PFN_INVALID on failure. 3544 * 3545 * cookiep (if passed) provides storage space for an opaque cookie 3546 * to return later to hat_delete_callback(). This cookie makes the callback 3547 * deletion significantly quicker by avoiding a potentially lengthy hash 3548 * search. 3549 * 3550 * Returns values: 3551 * 0: success 3552 * ENOMEM: memory allocation failure (e.g. flags was passed as HAC_NOSLEEP) 3553 * EINVAL: callback ID is not valid 3554 * ENXIO: ["vaddr", "vaddr" + len) is not mapped in the kernel's address 3555 * space 3556 * ERANGE: ["vaddr", "vaddr" + len) crosses a page boundary 3557 */ 3558 int 3559 hat_add_callback(id_t callback_id, caddr_t vaddr, uint_t len, uint_t flags, 3560 void *pvt, pfn_t *rpfn, void **cookiep) 3561 { 3562 struct hmehash_bucket *hmebp; 3563 hmeblk_tag hblktag; 3564 struct hme_blk *hmeblkp; 3565 int hmeshift, hashno; 3566 caddr_t saddr, eaddr, baseaddr; 3567 struct pa_hment *pahmep; 3568 struct sf_hment *sfhmep, *osfhmep; 3569 kmutex_t *pml; 3570 tte_t tte; 3571 page_t *pp; 3572 vnode_t *vp; 3573 u_offset_t off; 3574 pfn_t pfn; 3575 int kmflags = (flags & HAC_SLEEP)? KM_SLEEP : KM_NOSLEEP; 3576 int locked = 0; 3577 3578 /* 3579 * For KPM mappings, just return the physical address since we 3580 * don't need to register any callbacks. 3581 */ 3582 if (IS_KPM_ADDR(vaddr)) { 3583 uint64_t paddr; 3584 SFMMU_KPM_VTOP(vaddr, paddr); 3585 *rpfn = btop(paddr); 3586 if (cookiep != NULL) 3587 *cookiep = HAC_COOKIE_NONE; 3588 return (0); 3589 } 3590 3591 if (callback_id < (id_t)0 || callback_id >= sfmmu_cb_nextid) { 3592 *rpfn = PFN_INVALID; 3593 return (EINVAL); 3594 } 3595 3596 if ((pahmep = kmem_cache_alloc(pa_hment_cache, kmflags)) == NULL) { 3597 *rpfn = PFN_INVALID; 3598 return (ENOMEM); 3599 } 3600 3601 sfhmep = &pahmep->sfment; 3602 3603 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3604 eaddr = saddr + len; 3605 3606 rehash: 3607 /* Find the mapping(s) for this page */ 3608 for (hashno = TTE64K, hmeblkp = NULL; 3609 hmeblkp == NULL && hashno <= mmu_hashcnt; 3610 hashno++) { 3611 hmeshift = HME_HASH_SHIFT(hashno); 3612 hblktag.htag_id = ksfmmup; 3613 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3614 hblktag.htag_rehash = hashno; 3615 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3616 3617 SFMMU_HASH_LOCK(hmebp); 3618 3619 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3620 3621 if (hmeblkp == NULL) 3622 SFMMU_HASH_UNLOCK(hmebp); 3623 } 3624 3625 if (hmeblkp == NULL) { 3626 kmem_cache_free(pa_hment_cache, pahmep); 3627 *rpfn = PFN_INVALID; 3628 return (ENXIO); 3629 } 3630 3631 HBLKTOHME(osfhmep, hmeblkp, saddr); 3632 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3633 3634 if (!TTE_IS_VALID(&tte)) { 3635 SFMMU_HASH_UNLOCK(hmebp); 3636 kmem_cache_free(pa_hment_cache, pahmep); 3637 *rpfn = PFN_INVALID; 3638 return (ENXIO); 3639 } 3640 3641 /* 3642 * Make sure the boundaries for the callback fall within this 3643 * single mapping. 3644 */ 3645 baseaddr = (caddr_t)get_hblk_base(hmeblkp); 3646 ASSERT(saddr >= baseaddr); 3647 if (eaddr > saddr + TTEBYTES(TTE_CSZ(&tte))) { 3648 SFMMU_HASH_UNLOCK(hmebp); 3649 kmem_cache_free(pa_hment_cache, pahmep); 3650 *rpfn = PFN_INVALID; 3651 return (ERANGE); 3652 } 3653 3654 pfn = sfmmu_ttetopfn(&tte, vaddr); 3655 3656 /* 3657 * The pfn may not have a page_t underneath in which case we 3658 * just return it. This can happen if we are doing I/O to a 3659 * static portion of the kernel's address space, for instance. 3660 */ 3661 pp = osfhmep->hme_page; 3662 if (pp == NULL) { 3663 SFMMU_HASH_UNLOCK(hmebp); 3664 kmem_cache_free(pa_hment_cache, pahmep); 3665 *rpfn = pfn; 3666 if (cookiep) 3667 *cookiep = HAC_COOKIE_NONE; 3668 return (0); 3669 } 3670 ASSERT(pp == PP_PAGEROOT(pp)); 3671 3672 vp = pp->p_vnode; 3673 off = pp->p_offset; 3674 3675 pml = sfmmu_mlist_enter(pp); 3676 3677 if (flags & HAC_PAGELOCK) { 3678 if (!page_trylock(pp, SE_SHARED)) { 3679 /* 3680 * Somebody is holding SE_EXCL lock. Might 3681 * even be hat_page_relocate(). Drop all 3682 * our locks, lookup the page in &kvp, and 3683 * retry. If it doesn't exist in &kvp and &zvp, 3684 * then we must be dealing with a kernel mapped 3685 * page which doesn't actually belong to 3686 * segkmem so we punt. 3687 */ 3688 sfmmu_mlist_exit(pml); 3689 SFMMU_HASH_UNLOCK(hmebp); 3690 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3691 3692 /* check zvp before giving up */ 3693 if (pp == NULL) 3694 pp = page_lookup(&zvp, (u_offset_t)saddr, 3695 SE_SHARED); 3696 3697 /* Okay, we didn't find it, give up */ 3698 if (pp == NULL) { 3699 kmem_cache_free(pa_hment_cache, pahmep); 3700 *rpfn = pfn; 3701 if (cookiep) 3702 *cookiep = HAC_COOKIE_NONE; 3703 return (0); 3704 } 3705 page_unlock(pp); 3706 goto rehash; 3707 } 3708 locked = 1; 3709 } 3710 3711 if (!PAGE_LOCKED(pp) && !panicstr) 3712 panic("hat_add_callback: page 0x%p not locked", pp); 3713 3714 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3715 pp->p_offset != off) { 3716 /* 3717 * The page moved before we got our hands on it. Drop 3718 * all the locks and try again. 3719 */ 3720 ASSERT((flags & HAC_PAGELOCK) != 0); 3721 sfmmu_mlist_exit(pml); 3722 SFMMU_HASH_UNLOCK(hmebp); 3723 page_unlock(pp); 3724 locked = 0; 3725 goto rehash; 3726 } 3727 3728 if (!VN_ISKAS(vp)) { 3729 /* 3730 * This is not a segkmem page but another page which 3731 * has been kernel mapped. It had better have at least 3732 * a share lock on it. Return the pfn. 3733 */ 3734 sfmmu_mlist_exit(pml); 3735 SFMMU_HASH_UNLOCK(hmebp); 3736 if (locked) 3737 page_unlock(pp); 3738 kmem_cache_free(pa_hment_cache, pahmep); 3739 ASSERT(PAGE_LOCKED(pp)); 3740 *rpfn = pfn; 3741 if (cookiep) 3742 *cookiep = HAC_COOKIE_NONE; 3743 return (0); 3744 } 3745 3746 /* 3747 * Setup this pa_hment and link its embedded dummy sf_hment into 3748 * the mapping list. 3749 */ 3750 pp->p_share++; 3751 pahmep->cb_id = callback_id; 3752 pahmep->addr = vaddr; 3753 pahmep->len = len; 3754 pahmep->refcnt = 1; 3755 pahmep->flags = 0; 3756 pahmep->pvt = pvt; 3757 3758 sfhmep->hme_tte.ll = 0; 3759 sfhmep->hme_data = pahmep; 3760 sfhmep->hme_prev = osfhmep; 3761 sfhmep->hme_next = osfhmep->hme_next; 3762 3763 if (osfhmep->hme_next) 3764 osfhmep->hme_next->hme_prev = sfhmep; 3765 3766 osfhmep->hme_next = sfhmep; 3767 3768 sfmmu_mlist_exit(pml); 3769 SFMMU_HASH_UNLOCK(hmebp); 3770 3771 if (locked) 3772 page_unlock(pp); 3773 3774 *rpfn = pfn; 3775 if (cookiep) 3776 *cookiep = (void *)pahmep; 3777 3778 return (0); 3779 } 3780 3781 /* 3782 * Remove the relocation callbacks from the specified addr/len. 3783 */ 3784 void 3785 hat_delete_callback(caddr_t vaddr, uint_t len, void *pvt, uint_t flags, 3786 void *cookie) 3787 { 3788 struct hmehash_bucket *hmebp; 3789 hmeblk_tag hblktag; 3790 struct hme_blk *hmeblkp; 3791 int hmeshift, hashno; 3792 caddr_t saddr; 3793 struct pa_hment *pahmep; 3794 struct sf_hment *sfhmep, *osfhmep; 3795 kmutex_t *pml; 3796 tte_t tte; 3797 page_t *pp; 3798 vnode_t *vp; 3799 u_offset_t off; 3800 int locked = 0; 3801 3802 /* 3803 * If the cookie is HAC_COOKIE_NONE then there is no pa_hment to 3804 * remove so just return. 3805 */ 3806 if (cookie == HAC_COOKIE_NONE || IS_KPM_ADDR(vaddr)) 3807 return; 3808 3809 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3810 3811 rehash: 3812 /* Find the mapping(s) for this page */ 3813 for (hashno = TTE64K, hmeblkp = NULL; 3814 hmeblkp == NULL && hashno <= mmu_hashcnt; 3815 hashno++) { 3816 hmeshift = HME_HASH_SHIFT(hashno); 3817 hblktag.htag_id = ksfmmup; 3818 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3819 hblktag.htag_rehash = hashno; 3820 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3821 3822 SFMMU_HASH_LOCK(hmebp); 3823 3824 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3825 3826 if (hmeblkp == NULL) 3827 SFMMU_HASH_UNLOCK(hmebp); 3828 } 3829 3830 if (hmeblkp == NULL) 3831 return; 3832 3833 HBLKTOHME(osfhmep, hmeblkp, saddr); 3834 3835 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3836 if (!TTE_IS_VALID(&tte)) { 3837 SFMMU_HASH_UNLOCK(hmebp); 3838 return; 3839 } 3840 3841 pp = osfhmep->hme_page; 3842 if (pp == NULL) { 3843 SFMMU_HASH_UNLOCK(hmebp); 3844 ASSERT(cookie == NULL); 3845 return; 3846 } 3847 3848 vp = pp->p_vnode; 3849 off = pp->p_offset; 3850 3851 pml = sfmmu_mlist_enter(pp); 3852 3853 if (flags & HAC_PAGELOCK) { 3854 if (!page_trylock(pp, SE_SHARED)) { 3855 /* 3856 * Somebody is holding SE_EXCL lock. Might 3857 * even be hat_page_relocate(). Drop all 3858 * our locks, lookup the page in &kvp, and 3859 * retry. If it doesn't exist in &kvp and &zvp, 3860 * then we must be dealing with a kernel mapped 3861 * page which doesn't actually belong to 3862 * segkmem so we punt. 3863 */ 3864 sfmmu_mlist_exit(pml); 3865 SFMMU_HASH_UNLOCK(hmebp); 3866 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3867 /* check zvp before giving up */ 3868 if (pp == NULL) 3869 pp = page_lookup(&zvp, (u_offset_t)saddr, 3870 SE_SHARED); 3871 3872 if (pp == NULL) { 3873 ASSERT(cookie == NULL); 3874 return; 3875 } 3876 page_unlock(pp); 3877 goto rehash; 3878 } 3879 locked = 1; 3880 } 3881 3882 ASSERT(PAGE_LOCKED(pp)); 3883 3884 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3885 pp->p_offset != off) { 3886 /* 3887 * The page moved before we got our hands on it. Drop 3888 * all the locks and try again. 3889 */ 3890 ASSERT((flags & HAC_PAGELOCK) != 0); 3891 sfmmu_mlist_exit(pml); 3892 SFMMU_HASH_UNLOCK(hmebp); 3893 page_unlock(pp); 3894 locked = 0; 3895 goto rehash; 3896 } 3897 3898 if (!VN_ISKAS(vp)) { 3899 /* 3900 * This is not a segkmem page but another page which 3901 * has been kernel mapped. 3902 */ 3903 sfmmu_mlist_exit(pml); 3904 SFMMU_HASH_UNLOCK(hmebp); 3905 if (locked) 3906 page_unlock(pp); 3907 ASSERT(cookie == NULL); 3908 return; 3909 } 3910 3911 if (cookie != NULL) { 3912 pahmep = (struct pa_hment *)cookie; 3913 sfhmep = &pahmep->sfment; 3914 } else { 3915 for (sfhmep = pp->p_mapping; sfhmep != NULL; 3916 sfhmep = sfhmep->hme_next) { 3917 3918 /* 3919 * skip va<->pa mappings 3920 */ 3921 if (!IS_PAHME(sfhmep)) 3922 continue; 3923 3924 pahmep = sfhmep->hme_data; 3925 ASSERT(pahmep != NULL); 3926 3927 /* 3928 * if pa_hment matches, remove it 3929 */ 3930 if ((pahmep->pvt == pvt) && 3931 (pahmep->addr == vaddr) && 3932 (pahmep->len == len)) { 3933 break; 3934 } 3935 } 3936 } 3937 3938 if (sfhmep == NULL) { 3939 if (!panicstr) { 3940 panic("hat_delete_callback: pa_hment not found, pp %p", 3941 (void *)pp); 3942 } 3943 return; 3944 } 3945 3946 /* 3947 * Note: at this point a valid kernel mapping must still be 3948 * present on this page. 3949 */ 3950 pp->p_share--; 3951 if (pp->p_share <= 0) 3952 panic("hat_delete_callback: zero p_share"); 3953 3954 if (--pahmep->refcnt == 0) { 3955 if (pahmep->flags != 0) 3956 panic("hat_delete_callback: pa_hment is busy"); 3957 3958 /* 3959 * Remove sfhmep from the mapping list for the page. 3960 */ 3961 if (sfhmep->hme_prev) { 3962 sfhmep->hme_prev->hme_next = sfhmep->hme_next; 3963 } else { 3964 pp->p_mapping = sfhmep->hme_next; 3965 } 3966 3967 if (sfhmep->hme_next) 3968 sfhmep->hme_next->hme_prev = sfhmep->hme_prev; 3969 3970 sfmmu_mlist_exit(pml); 3971 SFMMU_HASH_UNLOCK(hmebp); 3972 3973 if (locked) 3974 page_unlock(pp); 3975 3976 kmem_cache_free(pa_hment_cache, pahmep); 3977 return; 3978 } 3979 3980 sfmmu_mlist_exit(pml); 3981 SFMMU_HASH_UNLOCK(hmebp); 3982 if (locked) 3983 page_unlock(pp); 3984 } 3985 3986 /* 3987 * hat_probe returns 1 if the translation for the address 'addr' is 3988 * loaded, zero otherwise. 3989 * 3990 * hat_probe should be used only for advisorary purposes because it may 3991 * occasionally return the wrong value. The implementation must guarantee that 3992 * returning the wrong value is a very rare event. hat_probe is used 3993 * to implement optimizations in the segment drivers. 3994 * 3995 */ 3996 int 3997 hat_probe(struct hat *sfmmup, caddr_t addr) 3998 { 3999 pfn_t pfn; 4000 tte_t tte; 4001 4002 ASSERT(sfmmup != NULL); 4003 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4004 4005 ASSERT((sfmmup == ksfmmup) || 4006 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4007 4008 if (sfmmup == ksfmmup) { 4009 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte)) 4010 == PFN_SUSPENDED) { 4011 sfmmu_vatopfn_suspended(addr, sfmmup, &tte); 4012 } 4013 } else { 4014 pfn = sfmmu_uvatopfn(addr, sfmmup); 4015 } 4016 4017 if (pfn != PFN_INVALID) 4018 return (1); 4019 else 4020 return (0); 4021 } 4022 4023 ssize_t 4024 hat_getpagesize(struct hat *sfmmup, caddr_t addr) 4025 { 4026 tte_t tte; 4027 4028 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4029 4030 sfmmu_gettte(sfmmup, addr, &tte); 4031 if (TTE_IS_VALID(&tte)) { 4032 return (TTEBYTES(TTE_CSZ(&tte))); 4033 } 4034 return (-1); 4035 } 4036 4037 static void 4038 sfmmu_gettte(struct hat *sfmmup, caddr_t addr, tte_t *ttep) 4039 { 4040 struct hmehash_bucket *hmebp; 4041 hmeblk_tag hblktag; 4042 int hmeshift, hashno = 1; 4043 struct hme_blk *hmeblkp, *list = NULL; 4044 struct sf_hment *sfhmep; 4045 4046 /* support for ISM */ 4047 ism_map_t *ism_map; 4048 ism_blk_t *ism_blkp; 4049 int i; 4050 sfmmu_t *ism_hatid = NULL; 4051 sfmmu_t *locked_hatid = NULL; 4052 4053 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 4054 4055 ism_blkp = sfmmup->sfmmu_iblk; 4056 if (ism_blkp) { 4057 sfmmu_ismhat_enter(sfmmup, 0); 4058 locked_hatid = sfmmup; 4059 } 4060 while (ism_blkp && ism_hatid == NULL) { 4061 ism_map = ism_blkp->iblk_maps; 4062 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 4063 if (addr >= ism_start(ism_map[i]) && 4064 addr < ism_end(ism_map[i])) { 4065 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 4066 addr = (caddr_t)(addr - 4067 ism_start(ism_map[i])); 4068 break; 4069 } 4070 } 4071 ism_blkp = ism_blkp->iblk_next; 4072 } 4073 if (locked_hatid) { 4074 sfmmu_ismhat_exit(locked_hatid, 0); 4075 } 4076 4077 hblktag.htag_id = sfmmup; 4078 ttep->ll = 0; 4079 4080 do { 4081 hmeshift = HME_HASH_SHIFT(hashno); 4082 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4083 hblktag.htag_rehash = hashno; 4084 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4085 4086 SFMMU_HASH_LOCK(hmebp); 4087 4088 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4089 if (hmeblkp != NULL) { 4090 HBLKTOHME(sfhmep, hmeblkp, addr); 4091 sfmmu_copytte(&sfhmep->hme_tte, ttep); 4092 SFMMU_HASH_UNLOCK(hmebp); 4093 break; 4094 } 4095 SFMMU_HASH_UNLOCK(hmebp); 4096 hashno++; 4097 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 4098 4099 sfmmu_hblks_list_purge(&list); 4100 } 4101 4102 uint_t 4103 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr) 4104 { 4105 tte_t tte; 4106 4107 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4108 4109 sfmmu_gettte(sfmmup, addr, &tte); 4110 if (TTE_IS_VALID(&tte)) { 4111 *attr = sfmmu_ptov_attr(&tte); 4112 return (0); 4113 } 4114 *attr = 0; 4115 return ((uint_t)0xffffffff); 4116 } 4117 4118 /* 4119 * Enables more attributes on specified address range (ie. logical OR) 4120 */ 4121 void 4122 hat_setattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4123 { 4124 if (hat->sfmmu_xhat_provider) { 4125 XHAT_SETATTR(hat, addr, len, attr); 4126 return; 4127 } else { 4128 /* 4129 * This must be a CPU HAT. If the address space has 4130 * XHATs attached, change attributes for all of them, 4131 * just in case 4132 */ 4133 ASSERT(hat->sfmmu_as != NULL); 4134 if (hat->sfmmu_as->a_xhat != NULL) 4135 xhat_setattr_all(hat->sfmmu_as, addr, len, attr); 4136 } 4137 4138 sfmmu_chgattr(hat, addr, len, attr, SFMMU_SETATTR); 4139 } 4140 4141 /* 4142 * Assigns attributes to the specified address range. All the attributes 4143 * are specified. 4144 */ 4145 void 4146 hat_chgattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4147 { 4148 if (hat->sfmmu_xhat_provider) { 4149 XHAT_CHGATTR(hat, addr, len, attr); 4150 return; 4151 } else { 4152 /* 4153 * This must be a CPU HAT. If the address space has 4154 * XHATs attached, change attributes for all of them, 4155 * just in case 4156 */ 4157 ASSERT(hat->sfmmu_as != NULL); 4158 if (hat->sfmmu_as->a_xhat != NULL) 4159 xhat_chgattr_all(hat->sfmmu_as, addr, len, attr); 4160 } 4161 4162 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CHGATTR); 4163 } 4164 4165 /* 4166 * Remove attributes on the specified address range (ie. loginal NAND) 4167 */ 4168 void 4169 hat_clrattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4170 { 4171 if (hat->sfmmu_xhat_provider) { 4172 XHAT_CLRATTR(hat, addr, len, attr); 4173 return; 4174 } else { 4175 /* 4176 * This must be a CPU HAT. If the address space has 4177 * XHATs attached, change attributes for all of them, 4178 * just in case 4179 */ 4180 ASSERT(hat->sfmmu_as != NULL); 4181 if (hat->sfmmu_as->a_xhat != NULL) 4182 xhat_clrattr_all(hat->sfmmu_as, addr, len, attr); 4183 } 4184 4185 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CLRATTR); 4186 } 4187 4188 /* 4189 * Change attributes on an address range to that specified by attr and mode. 4190 */ 4191 static void 4192 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr, 4193 int mode) 4194 { 4195 struct hmehash_bucket *hmebp; 4196 hmeblk_tag hblktag; 4197 int hmeshift, hashno = 1; 4198 struct hme_blk *hmeblkp, *list = NULL; 4199 caddr_t endaddr; 4200 cpuset_t cpuset; 4201 demap_range_t dmr; 4202 4203 CPUSET_ZERO(cpuset); 4204 4205 ASSERT((sfmmup == ksfmmup) || 4206 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4207 ASSERT((len & MMU_PAGEOFFSET) == 0); 4208 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4209 4210 if ((attr & PROT_USER) && (mode != SFMMU_CLRATTR) && 4211 ((addr + len) > (caddr_t)USERLIMIT)) { 4212 panic("user addr %p in kernel space", 4213 (void *)addr); 4214 } 4215 4216 endaddr = addr + len; 4217 hblktag.htag_id = sfmmup; 4218 DEMAP_RANGE_INIT(sfmmup, &dmr); 4219 4220 while (addr < endaddr) { 4221 hmeshift = HME_HASH_SHIFT(hashno); 4222 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4223 hblktag.htag_rehash = hashno; 4224 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4225 4226 SFMMU_HASH_LOCK(hmebp); 4227 4228 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4229 if (hmeblkp != NULL) { 4230 /* 4231 * We've encountered a shadow hmeblk so skip the range 4232 * of the next smaller mapping size. 4233 */ 4234 if (hmeblkp->hblk_shw_bit) { 4235 ASSERT(sfmmup != ksfmmup); 4236 ASSERT(hashno > 1); 4237 addr = (caddr_t)P2END((uintptr_t)addr, 4238 TTEBYTES(hashno - 1)); 4239 } else { 4240 addr = sfmmu_hblk_chgattr(sfmmup, 4241 hmeblkp, addr, endaddr, &dmr, attr, mode); 4242 } 4243 SFMMU_HASH_UNLOCK(hmebp); 4244 hashno = 1; 4245 continue; 4246 } 4247 SFMMU_HASH_UNLOCK(hmebp); 4248 4249 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4250 /* 4251 * We have traversed the whole list and rehashed 4252 * if necessary without finding the address to chgattr. 4253 * This is ok, so we increment the address by the 4254 * smallest hmeblk range for kernel mappings or for 4255 * user mappings with no large pages, and the largest 4256 * hmeblk range, to account for shadow hmeblks, for 4257 * user mappings with large pages and continue. 4258 */ 4259 if (sfmmup == ksfmmup) 4260 addr = (caddr_t)P2END((uintptr_t)addr, 4261 TTEBYTES(1)); 4262 else 4263 addr = (caddr_t)P2END((uintptr_t)addr, 4264 TTEBYTES(hashno)); 4265 hashno = 1; 4266 } else { 4267 hashno++; 4268 } 4269 } 4270 4271 sfmmu_hblks_list_purge(&list); 4272 DEMAP_RANGE_FLUSH(&dmr); 4273 cpuset = sfmmup->sfmmu_cpusran; 4274 xt_sync(cpuset); 4275 } 4276 4277 /* 4278 * This function chgattr on a range of addresses in an hmeblk. It returns the 4279 * next addres that needs to be chgattr. 4280 * It should be called with the hash lock held. 4281 * XXX It should be possible to optimize chgattr by not flushing every time but 4282 * on the other hand: 4283 * 1. do one flush crosscall. 4284 * 2. only flush if we are increasing permissions (make sure this will work) 4285 */ 4286 static caddr_t 4287 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4288 caddr_t endaddr, demap_range_t *dmrp, uint_t attr, int mode) 4289 { 4290 tte_t tte, tteattr, tteflags, ttemod; 4291 struct sf_hment *sfhmep; 4292 int ttesz; 4293 struct page *pp = NULL; 4294 kmutex_t *pml, *pmtx; 4295 int ret; 4296 int use_demap_range; 4297 #if defined(SF_ERRATA_57) 4298 int check_exec; 4299 #endif 4300 4301 ASSERT(in_hblk_range(hmeblkp, addr)); 4302 ASSERT(hmeblkp->hblk_shw_bit == 0); 4303 4304 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4305 ttesz = get_hblk_ttesz(hmeblkp); 4306 4307 /* 4308 * Flush the current demap region if addresses have been 4309 * skipped or the page size doesn't match. 4310 */ 4311 use_demap_range = (TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp)); 4312 if (use_demap_range) { 4313 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4314 } else { 4315 DEMAP_RANGE_FLUSH(dmrp); 4316 } 4317 4318 tteattr.ll = sfmmu_vtop_attr(attr, mode, &tteflags); 4319 #if defined(SF_ERRATA_57) 4320 check_exec = (sfmmup != ksfmmup) && 4321 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4322 TTE_IS_EXECUTABLE(&tteattr); 4323 #endif 4324 HBLKTOHME(sfhmep, hmeblkp, addr); 4325 while (addr < endaddr) { 4326 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4327 if (TTE_IS_VALID(&tte)) { 4328 if ((tte.ll & tteflags.ll) == tteattr.ll) { 4329 /* 4330 * if the new attr is the same as old 4331 * continue 4332 */ 4333 goto next_addr; 4334 } 4335 if (!TTE_IS_WRITABLE(&tteattr)) { 4336 /* 4337 * make sure we clear hw modify bit if we 4338 * removing write protections 4339 */ 4340 tteflags.tte_intlo |= TTE_HWWR_INT; 4341 } 4342 4343 pml = NULL; 4344 pp = sfhmep->hme_page; 4345 if (pp) { 4346 pml = sfmmu_mlist_enter(pp); 4347 } 4348 4349 if (pp != sfhmep->hme_page) { 4350 /* 4351 * tte must have been unloaded. 4352 */ 4353 ASSERT(pml); 4354 sfmmu_mlist_exit(pml); 4355 continue; 4356 } 4357 4358 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4359 4360 ttemod = tte; 4361 ttemod.ll = (ttemod.ll & ~tteflags.ll) | tteattr.ll; 4362 ASSERT(TTE_TO_TTEPFN(&ttemod) == TTE_TO_TTEPFN(&tte)); 4363 4364 #if defined(SF_ERRATA_57) 4365 if (check_exec && addr < errata57_limit) 4366 ttemod.tte_exec_perm = 0; 4367 #endif 4368 ret = sfmmu_modifytte_try(&tte, &ttemod, 4369 &sfhmep->hme_tte); 4370 4371 if (ret < 0) { 4372 /* tte changed underneath us */ 4373 if (pml) { 4374 sfmmu_mlist_exit(pml); 4375 } 4376 continue; 4377 } 4378 4379 if (tteflags.tte_intlo & TTE_HWWR_INT) { 4380 /* 4381 * need to sync if we are clearing modify bit. 4382 */ 4383 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4384 } 4385 4386 if (pp && PP_ISRO(pp)) { 4387 if (tteattr.tte_intlo & TTE_WRPRM_INT) { 4388 pmtx = sfmmu_page_enter(pp); 4389 PP_CLRRO(pp); 4390 sfmmu_page_exit(pmtx); 4391 } 4392 } 4393 4394 if (ret > 0 && use_demap_range) { 4395 DEMAP_RANGE_MARKPG(dmrp, addr); 4396 } else if (ret > 0) { 4397 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4398 } 4399 4400 if (pml) { 4401 sfmmu_mlist_exit(pml); 4402 } 4403 } 4404 next_addr: 4405 addr += TTEBYTES(ttesz); 4406 sfhmep++; 4407 DEMAP_RANGE_NEXTPG(dmrp); 4408 } 4409 return (addr); 4410 } 4411 4412 /* 4413 * This routine converts virtual attributes to physical ones. It will 4414 * update the tteflags field with the tte mask corresponding to the attributes 4415 * affected and it returns the new attributes. It will also clear the modify 4416 * bit if we are taking away write permission. This is necessary since the 4417 * modify bit is the hardware permission bit and we need to clear it in order 4418 * to detect write faults. 4419 */ 4420 static uint64_t 4421 sfmmu_vtop_attr(uint_t attr, int mode, tte_t *ttemaskp) 4422 { 4423 tte_t ttevalue; 4424 4425 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 4426 4427 switch (mode) { 4428 case SFMMU_CHGATTR: 4429 /* all attributes specified */ 4430 ttevalue.tte_inthi = MAKE_TTEATTR_INTHI(attr); 4431 ttevalue.tte_intlo = MAKE_TTEATTR_INTLO(attr); 4432 ttemaskp->tte_inthi = TTEINTHI_ATTR; 4433 ttemaskp->tte_intlo = TTEINTLO_ATTR; 4434 break; 4435 case SFMMU_SETATTR: 4436 ASSERT(!(attr & ~HAT_PROT_MASK)); 4437 ttemaskp->ll = 0; 4438 ttevalue.ll = 0; 4439 /* 4440 * a valid tte implies exec and read for sfmmu 4441 * so no need to do anything about them. 4442 * since priviledged access implies user access 4443 * PROT_USER doesn't make sense either. 4444 */ 4445 if (attr & PROT_WRITE) { 4446 ttemaskp->tte_intlo |= TTE_WRPRM_INT; 4447 ttevalue.tte_intlo |= TTE_WRPRM_INT; 4448 } 4449 break; 4450 case SFMMU_CLRATTR: 4451 /* attributes will be nand with current ones */ 4452 if (attr & ~(PROT_WRITE | PROT_USER)) { 4453 panic("sfmmu: attr %x not supported", attr); 4454 } 4455 ttemaskp->ll = 0; 4456 ttevalue.ll = 0; 4457 if (attr & PROT_WRITE) { 4458 /* clear both writable and modify bit */ 4459 ttemaskp->tte_intlo |= TTE_WRPRM_INT | TTE_HWWR_INT; 4460 } 4461 if (attr & PROT_USER) { 4462 ttemaskp->tte_intlo |= TTE_PRIV_INT; 4463 ttevalue.tte_intlo |= TTE_PRIV_INT; 4464 } 4465 break; 4466 default: 4467 panic("sfmmu_vtop_attr: bad mode %x", mode); 4468 } 4469 ASSERT(TTE_TO_TTEPFN(&ttevalue) == 0); 4470 return (ttevalue.ll); 4471 } 4472 4473 static uint_t 4474 sfmmu_ptov_attr(tte_t *ttep) 4475 { 4476 uint_t attr; 4477 4478 ASSERT(TTE_IS_VALID(ttep)); 4479 4480 attr = PROT_READ; 4481 4482 if (TTE_IS_WRITABLE(ttep)) { 4483 attr |= PROT_WRITE; 4484 } 4485 if (TTE_IS_EXECUTABLE(ttep)) { 4486 attr |= PROT_EXEC; 4487 } 4488 if (!TTE_IS_PRIVILEGED(ttep)) { 4489 attr |= PROT_USER; 4490 } 4491 if (TTE_IS_NFO(ttep)) { 4492 attr |= HAT_NOFAULT; 4493 } 4494 if (TTE_IS_NOSYNC(ttep)) { 4495 attr |= HAT_NOSYNC; 4496 } 4497 if (TTE_IS_SIDEFFECT(ttep)) { 4498 attr |= SFMMU_SIDEFFECT; 4499 } 4500 if (!TTE_IS_VCACHEABLE(ttep)) { 4501 attr |= SFMMU_UNCACHEVTTE; 4502 } 4503 if (!TTE_IS_PCACHEABLE(ttep)) { 4504 attr |= SFMMU_UNCACHEPTTE; 4505 } 4506 return (attr); 4507 } 4508 4509 /* 4510 * hat_chgprot is a deprecated hat call. New segment drivers 4511 * should store all attributes and use hat_*attr calls. 4512 * 4513 * Change the protections in the virtual address range 4514 * given to the specified virtual protection. If vprot is ~PROT_WRITE, 4515 * then remove write permission, leaving the other 4516 * permissions unchanged. If vprot is ~PROT_USER, remove user permissions. 4517 * 4518 */ 4519 void 4520 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot) 4521 { 4522 struct hmehash_bucket *hmebp; 4523 hmeblk_tag hblktag; 4524 int hmeshift, hashno = 1; 4525 struct hme_blk *hmeblkp, *list = NULL; 4526 caddr_t endaddr; 4527 cpuset_t cpuset; 4528 demap_range_t dmr; 4529 4530 ASSERT((len & MMU_PAGEOFFSET) == 0); 4531 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4532 4533 if (sfmmup->sfmmu_xhat_provider) { 4534 XHAT_CHGPROT(sfmmup, addr, len, vprot); 4535 return; 4536 } else { 4537 /* 4538 * This must be a CPU HAT. If the address space has 4539 * XHATs attached, change attributes for all of them, 4540 * just in case 4541 */ 4542 ASSERT(sfmmup->sfmmu_as != NULL); 4543 if (sfmmup->sfmmu_as->a_xhat != NULL) 4544 xhat_chgprot_all(sfmmup->sfmmu_as, addr, len, vprot); 4545 } 4546 4547 CPUSET_ZERO(cpuset); 4548 4549 if ((vprot != (uint_t)~PROT_WRITE) && (vprot & PROT_USER) && 4550 ((addr + len) > (caddr_t)USERLIMIT)) { 4551 panic("user addr %p vprot %x in kernel space", 4552 (void *)addr, vprot); 4553 } 4554 endaddr = addr + len; 4555 hblktag.htag_id = sfmmup; 4556 DEMAP_RANGE_INIT(sfmmup, &dmr); 4557 4558 while (addr < endaddr) { 4559 hmeshift = HME_HASH_SHIFT(hashno); 4560 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4561 hblktag.htag_rehash = hashno; 4562 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4563 4564 SFMMU_HASH_LOCK(hmebp); 4565 4566 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4567 if (hmeblkp != NULL) { 4568 /* 4569 * We've encountered a shadow hmeblk so skip the range 4570 * of the next smaller mapping size. 4571 */ 4572 if (hmeblkp->hblk_shw_bit) { 4573 ASSERT(sfmmup != ksfmmup); 4574 ASSERT(hashno > 1); 4575 addr = (caddr_t)P2END((uintptr_t)addr, 4576 TTEBYTES(hashno - 1)); 4577 } else { 4578 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp, 4579 addr, endaddr, &dmr, vprot); 4580 } 4581 SFMMU_HASH_UNLOCK(hmebp); 4582 hashno = 1; 4583 continue; 4584 } 4585 SFMMU_HASH_UNLOCK(hmebp); 4586 4587 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4588 /* 4589 * We have traversed the whole list and rehashed 4590 * if necessary without finding the address to chgprot. 4591 * This is ok so we increment the address by the 4592 * smallest hmeblk range for kernel mappings and the 4593 * largest hmeblk range, to account for shadow hmeblks, 4594 * for user mappings and continue. 4595 */ 4596 if (sfmmup == ksfmmup) 4597 addr = (caddr_t)P2END((uintptr_t)addr, 4598 TTEBYTES(1)); 4599 else 4600 addr = (caddr_t)P2END((uintptr_t)addr, 4601 TTEBYTES(hashno)); 4602 hashno = 1; 4603 } else { 4604 hashno++; 4605 } 4606 } 4607 4608 sfmmu_hblks_list_purge(&list); 4609 DEMAP_RANGE_FLUSH(&dmr); 4610 cpuset = sfmmup->sfmmu_cpusran; 4611 xt_sync(cpuset); 4612 } 4613 4614 /* 4615 * This function chgprots a range of addresses in an hmeblk. It returns the 4616 * next addres that needs to be chgprot. 4617 * It should be called with the hash lock held. 4618 * XXX It shold be possible to optimize chgprot by not flushing every time but 4619 * on the other hand: 4620 * 1. do one flush crosscall. 4621 * 2. only flush if we are increasing permissions (make sure this will work) 4622 */ 4623 static caddr_t 4624 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4625 caddr_t endaddr, demap_range_t *dmrp, uint_t vprot) 4626 { 4627 uint_t pprot; 4628 tte_t tte, ttemod; 4629 struct sf_hment *sfhmep; 4630 uint_t tteflags; 4631 int ttesz; 4632 struct page *pp = NULL; 4633 kmutex_t *pml, *pmtx; 4634 int ret; 4635 int use_demap_range; 4636 #if defined(SF_ERRATA_57) 4637 int check_exec; 4638 #endif 4639 4640 ASSERT(in_hblk_range(hmeblkp, addr)); 4641 ASSERT(hmeblkp->hblk_shw_bit == 0); 4642 4643 #ifdef DEBUG 4644 if (get_hblk_ttesz(hmeblkp) != TTE8K && 4645 (endaddr < get_hblk_endaddr(hmeblkp))) { 4646 panic("sfmmu_hblk_chgprot: partial chgprot of large page"); 4647 } 4648 #endif /* DEBUG */ 4649 4650 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4651 ttesz = get_hblk_ttesz(hmeblkp); 4652 4653 pprot = sfmmu_vtop_prot(vprot, &tteflags); 4654 #if defined(SF_ERRATA_57) 4655 check_exec = (sfmmup != ksfmmup) && 4656 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4657 ((vprot & PROT_EXEC) == PROT_EXEC); 4658 #endif 4659 HBLKTOHME(sfhmep, hmeblkp, addr); 4660 4661 /* 4662 * Flush the current demap region if addresses have been 4663 * skipped or the page size doesn't match. 4664 */ 4665 use_demap_range = (TTEBYTES(ttesz) == MMU_PAGESIZE); 4666 if (use_demap_range) { 4667 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4668 } else { 4669 DEMAP_RANGE_FLUSH(dmrp); 4670 } 4671 4672 while (addr < endaddr) { 4673 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4674 if (TTE_IS_VALID(&tte)) { 4675 if (TTE_GET_LOFLAGS(&tte, tteflags) == pprot) { 4676 /* 4677 * if the new protection is the same as old 4678 * continue 4679 */ 4680 goto next_addr; 4681 } 4682 pml = NULL; 4683 pp = sfhmep->hme_page; 4684 if (pp) { 4685 pml = sfmmu_mlist_enter(pp); 4686 } 4687 if (pp != sfhmep->hme_page) { 4688 /* 4689 * tte most have been unloaded 4690 * underneath us. Recheck 4691 */ 4692 ASSERT(pml); 4693 sfmmu_mlist_exit(pml); 4694 continue; 4695 } 4696 4697 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4698 4699 ttemod = tte; 4700 TTE_SET_LOFLAGS(&ttemod, tteflags, pprot); 4701 #if defined(SF_ERRATA_57) 4702 if (check_exec && addr < errata57_limit) 4703 ttemod.tte_exec_perm = 0; 4704 #endif 4705 ret = sfmmu_modifytte_try(&tte, &ttemod, 4706 &sfhmep->hme_tte); 4707 4708 if (ret < 0) { 4709 /* tte changed underneath us */ 4710 if (pml) { 4711 sfmmu_mlist_exit(pml); 4712 } 4713 continue; 4714 } 4715 4716 if (tteflags & TTE_HWWR_INT) { 4717 /* 4718 * need to sync if we are clearing modify bit. 4719 */ 4720 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4721 } 4722 4723 if (pp && PP_ISRO(pp)) { 4724 if (pprot & TTE_WRPRM_INT) { 4725 pmtx = sfmmu_page_enter(pp); 4726 PP_CLRRO(pp); 4727 sfmmu_page_exit(pmtx); 4728 } 4729 } 4730 4731 if (ret > 0 && use_demap_range) { 4732 DEMAP_RANGE_MARKPG(dmrp, addr); 4733 } else if (ret > 0) { 4734 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4735 } 4736 4737 if (pml) { 4738 sfmmu_mlist_exit(pml); 4739 } 4740 } 4741 next_addr: 4742 addr += TTEBYTES(ttesz); 4743 sfhmep++; 4744 DEMAP_RANGE_NEXTPG(dmrp); 4745 } 4746 return (addr); 4747 } 4748 4749 /* 4750 * This routine is deprecated and should only be used by hat_chgprot. 4751 * The correct routine is sfmmu_vtop_attr. 4752 * This routine converts virtual page protections to physical ones. It will 4753 * update the tteflags field with the tte mask corresponding to the protections 4754 * affected and it returns the new protections. It will also clear the modify 4755 * bit if we are taking away write permission. This is necessary since the 4756 * modify bit is the hardware permission bit and we need to clear it in order 4757 * to detect write faults. 4758 * It accepts the following special protections: 4759 * ~PROT_WRITE = remove write permissions. 4760 * ~PROT_USER = remove user permissions. 4761 */ 4762 static uint_t 4763 sfmmu_vtop_prot(uint_t vprot, uint_t *tteflagsp) 4764 { 4765 if (vprot == (uint_t)~PROT_WRITE) { 4766 *tteflagsp = TTE_WRPRM_INT | TTE_HWWR_INT; 4767 return (0); /* will cause wrprm to be cleared */ 4768 } 4769 if (vprot == (uint_t)~PROT_USER) { 4770 *tteflagsp = TTE_PRIV_INT; 4771 return (0); /* will cause privprm to be cleared */ 4772 } 4773 if ((vprot == 0) || (vprot == PROT_USER) || 4774 ((vprot & PROT_ALL) != vprot)) { 4775 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4776 } 4777 4778 switch (vprot) { 4779 case (PROT_READ): 4780 case (PROT_EXEC): 4781 case (PROT_EXEC | PROT_READ): 4782 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4783 return (TTE_PRIV_INT); /* set prv and clr wrt */ 4784 case (PROT_WRITE): 4785 case (PROT_WRITE | PROT_READ): 4786 case (PROT_EXEC | PROT_WRITE): 4787 case (PROT_EXEC | PROT_WRITE | PROT_READ): 4788 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4789 return (TTE_PRIV_INT | TTE_WRPRM_INT); /* set prv and wrt */ 4790 case (PROT_USER | PROT_READ): 4791 case (PROT_USER | PROT_EXEC): 4792 case (PROT_USER | PROT_EXEC | PROT_READ): 4793 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4794 return (0); /* clr prv and wrt */ 4795 case (PROT_USER | PROT_WRITE): 4796 case (PROT_USER | PROT_WRITE | PROT_READ): 4797 case (PROT_USER | PROT_EXEC | PROT_WRITE): 4798 case (PROT_USER | PROT_EXEC | PROT_WRITE | PROT_READ): 4799 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4800 return (TTE_WRPRM_INT); /* clr prv and set wrt */ 4801 default: 4802 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4803 } 4804 return (0); 4805 } 4806 4807 /* 4808 * Alternate unload for very large virtual ranges. With a true 64 bit VA, 4809 * the normal algorithm would take too long for a very large VA range with 4810 * few real mappings. This routine just walks thru all HMEs in the global 4811 * hash table to find and remove mappings. 4812 */ 4813 static void 4814 hat_unload_large_virtual( 4815 struct hat *sfmmup, 4816 caddr_t startaddr, 4817 size_t len, 4818 uint_t flags, 4819 hat_callback_t *callback) 4820 { 4821 struct hmehash_bucket *hmebp; 4822 struct hme_blk *hmeblkp; 4823 struct hme_blk *pr_hblk = NULL; 4824 struct hme_blk *nx_hblk; 4825 struct hme_blk *list = NULL; 4826 int i; 4827 uint64_t hblkpa, prevpa, nx_pa; 4828 demap_range_t dmr, *dmrp; 4829 cpuset_t cpuset; 4830 caddr_t endaddr = startaddr + len; 4831 caddr_t sa; 4832 caddr_t ea; 4833 caddr_t cb_sa[MAX_CB_ADDR]; 4834 caddr_t cb_ea[MAX_CB_ADDR]; 4835 int addr_cnt = 0; 4836 int a = 0; 4837 4838 if (sfmmup->sfmmu_free) { 4839 dmrp = NULL; 4840 } else { 4841 dmrp = &dmr; 4842 DEMAP_RANGE_INIT(sfmmup, dmrp); 4843 } 4844 4845 /* 4846 * Loop through all the hash buckets of HME blocks looking for matches. 4847 */ 4848 for (i = 0; i <= UHMEHASH_SZ; i++) { 4849 hmebp = &uhme_hash[i]; 4850 SFMMU_HASH_LOCK(hmebp); 4851 hmeblkp = hmebp->hmeblkp; 4852 hblkpa = hmebp->hmeh_nextpa; 4853 prevpa = 0; 4854 pr_hblk = NULL; 4855 while (hmeblkp) { 4856 nx_hblk = hmeblkp->hblk_next; 4857 nx_pa = hmeblkp->hblk_nextpa; 4858 4859 /* 4860 * skip if not this context, if a shadow block or 4861 * if the mapping is not in the requested range 4862 */ 4863 if (hmeblkp->hblk_tag.htag_id != sfmmup || 4864 hmeblkp->hblk_shw_bit || 4865 (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr || 4866 (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) { 4867 pr_hblk = hmeblkp; 4868 prevpa = hblkpa; 4869 goto next_block; 4870 } 4871 4872 /* 4873 * unload if there are any current valid mappings 4874 */ 4875 if (hmeblkp->hblk_vcnt != 0 || 4876 hmeblkp->hblk_hmecnt != 0) 4877 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 4878 sa, ea, dmrp, flags); 4879 4880 /* 4881 * on unmap we also release the HME block itself, once 4882 * all mappings are gone. 4883 */ 4884 if ((flags & HAT_UNLOAD_UNMAP) != 0 && 4885 !hmeblkp->hblk_vcnt && 4886 !hmeblkp->hblk_hmecnt) { 4887 ASSERT(!hmeblkp->hblk_lckcnt); 4888 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 4889 prevpa, pr_hblk); 4890 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 4891 } else { 4892 pr_hblk = hmeblkp; 4893 prevpa = hblkpa; 4894 } 4895 4896 if (callback == NULL) 4897 goto next_block; 4898 4899 /* 4900 * HME blocks may span more than one page, but we may be 4901 * unmapping only one page, so check for a smaller range 4902 * for the callback 4903 */ 4904 if (sa < startaddr) 4905 sa = startaddr; 4906 if (--ea > endaddr) 4907 ea = endaddr - 1; 4908 4909 cb_sa[addr_cnt] = sa; 4910 cb_ea[addr_cnt] = ea; 4911 if (++addr_cnt == MAX_CB_ADDR) { 4912 if (dmrp != NULL) { 4913 DEMAP_RANGE_FLUSH(dmrp); 4914 cpuset = sfmmup->sfmmu_cpusran; 4915 xt_sync(cpuset); 4916 } 4917 4918 for (a = 0; a < MAX_CB_ADDR; ++a) { 4919 callback->hcb_start_addr = cb_sa[a]; 4920 callback->hcb_end_addr = cb_ea[a]; 4921 callback->hcb_function(callback); 4922 } 4923 addr_cnt = 0; 4924 } 4925 4926 next_block: 4927 hmeblkp = nx_hblk; 4928 hblkpa = nx_pa; 4929 } 4930 SFMMU_HASH_UNLOCK(hmebp); 4931 } 4932 4933 sfmmu_hblks_list_purge(&list); 4934 if (dmrp != NULL) { 4935 DEMAP_RANGE_FLUSH(dmrp); 4936 cpuset = sfmmup->sfmmu_cpusran; 4937 xt_sync(cpuset); 4938 } 4939 4940 for (a = 0; a < addr_cnt; ++a) { 4941 callback->hcb_start_addr = cb_sa[a]; 4942 callback->hcb_end_addr = cb_ea[a]; 4943 callback->hcb_function(callback); 4944 } 4945 4946 /* 4947 * Check TSB and TLB page sizes if the process isn't exiting. 4948 */ 4949 if (!sfmmup->sfmmu_free) 4950 sfmmu_check_page_sizes(sfmmup, 0); 4951 } 4952 4953 /* 4954 * Unload all the mappings in the range [addr..addr+len). addr and len must 4955 * be MMU_PAGESIZE aligned. 4956 */ 4957 4958 extern struct seg *segkmap; 4959 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \ 4960 segkmap->s_base <= (addr) && (addr) < (segkmap->s_base + segkmap->s_size)) 4961 4962 4963 void 4964 hat_unload_callback( 4965 struct hat *sfmmup, 4966 caddr_t addr, 4967 size_t len, 4968 uint_t flags, 4969 hat_callback_t *callback) 4970 { 4971 struct hmehash_bucket *hmebp; 4972 hmeblk_tag hblktag; 4973 int hmeshift, hashno, iskernel; 4974 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 4975 caddr_t endaddr; 4976 cpuset_t cpuset; 4977 uint64_t hblkpa, prevpa; 4978 int addr_count = 0; 4979 int a; 4980 caddr_t cb_start_addr[MAX_CB_ADDR]; 4981 caddr_t cb_end_addr[MAX_CB_ADDR]; 4982 int issegkmap = ISSEGKMAP(sfmmup, addr); 4983 demap_range_t dmr, *dmrp; 4984 4985 if (sfmmup->sfmmu_xhat_provider) { 4986 XHAT_UNLOAD_CALLBACK(sfmmup, addr, len, flags, callback); 4987 return; 4988 } else { 4989 /* 4990 * This must be a CPU HAT. If the address space has 4991 * XHATs attached, unload the mappings for all of them, 4992 * just in case 4993 */ 4994 ASSERT(sfmmup->sfmmu_as != NULL); 4995 if (sfmmup->sfmmu_as->a_xhat != NULL) 4996 xhat_unload_callback_all(sfmmup->sfmmu_as, addr, 4997 len, flags, callback); 4998 } 4999 5000 ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \ 5001 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 5002 5003 ASSERT(sfmmup != NULL); 5004 ASSERT((len & MMU_PAGEOFFSET) == 0); 5005 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 5006 5007 /* 5008 * Probing through a large VA range (say 63 bits) will be slow, even 5009 * at 4 Meg steps between the probes. So, when the virtual address range 5010 * is very large, search the HME entries for what to unload. 5011 * 5012 * len >> TTE_PAGE_SHIFT(TTE4M) is the # of 4Meg probes we'd need 5013 * 5014 * UHMEHASH_SZ is number of hash buckets to examine 5015 * 5016 */ 5017 if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) { 5018 hat_unload_large_virtual(sfmmup, addr, len, flags, callback); 5019 return; 5020 } 5021 5022 CPUSET_ZERO(cpuset); 5023 5024 /* 5025 * If the process is exiting, we can save a lot of fuss since 5026 * we'll flush the TLB when we free the ctx anyway. 5027 */ 5028 if (sfmmup->sfmmu_free) 5029 dmrp = NULL; 5030 else 5031 dmrp = &dmr; 5032 5033 DEMAP_RANGE_INIT(sfmmup, dmrp); 5034 endaddr = addr + len; 5035 hblktag.htag_id = sfmmup; 5036 5037 /* 5038 * It is likely for the vm to call unload over a wide range of 5039 * addresses that are actually very sparsely populated by 5040 * translations. In order to speed this up the sfmmu hat supports 5041 * the concept of shadow hmeblks. Dummy large page hmeblks that 5042 * correspond to actual small translations are allocated at tteload 5043 * time and are referred to as shadow hmeblks. Now, during unload 5044 * time, we first check if we have a shadow hmeblk for that 5045 * translation. The absence of one means the corresponding address 5046 * range is empty and can be skipped. 5047 * 5048 * The kernel is an exception to above statement and that is why 5049 * we don't use shadow hmeblks and hash starting from the smallest 5050 * page size. 5051 */ 5052 if (sfmmup == KHATID) { 5053 iskernel = 1; 5054 hashno = TTE64K; 5055 } else { 5056 iskernel = 0; 5057 if (mmu_page_sizes == max_mmu_page_sizes) { 5058 hashno = TTE256M; 5059 } else { 5060 hashno = TTE4M; 5061 } 5062 } 5063 while (addr < endaddr) { 5064 hmeshift = HME_HASH_SHIFT(hashno); 5065 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5066 hblktag.htag_rehash = hashno; 5067 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5068 5069 SFMMU_HASH_LOCK(hmebp); 5070 5071 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, pr_hblk, 5072 prevpa, &list); 5073 if (hmeblkp == NULL) { 5074 /* 5075 * didn't find an hmeblk. skip the appropiate 5076 * address range. 5077 */ 5078 SFMMU_HASH_UNLOCK(hmebp); 5079 if (iskernel) { 5080 if (hashno < mmu_hashcnt) { 5081 hashno++; 5082 continue; 5083 } else { 5084 hashno = TTE64K; 5085 addr = (caddr_t)roundup((uintptr_t)addr 5086 + 1, MMU_PAGESIZE64K); 5087 continue; 5088 } 5089 } 5090 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5091 (1 << hmeshift)); 5092 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5093 ASSERT(hashno == TTE64K); 5094 continue; 5095 } 5096 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5097 hashno = TTE512K; 5098 continue; 5099 } 5100 if (mmu_page_sizes == max_mmu_page_sizes) { 5101 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5102 hashno = TTE4M; 5103 continue; 5104 } 5105 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5106 hashno = TTE32M; 5107 continue; 5108 } 5109 hashno = TTE256M; 5110 continue; 5111 } else { 5112 hashno = TTE4M; 5113 continue; 5114 } 5115 } 5116 ASSERT(hmeblkp); 5117 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5118 /* 5119 * If the valid count is zero we can skip the range 5120 * mapped by this hmeblk. 5121 * We free hblks in the case of HAT_UNMAP. HAT_UNMAP 5122 * is used by segment drivers as a hint 5123 * that the mapping resource won't be used any longer. 5124 * The best example of this is during exit(). 5125 */ 5126 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5127 get_hblk_span(hmeblkp)); 5128 if ((flags & HAT_UNLOAD_UNMAP) || 5129 (iskernel && !issegkmap)) { 5130 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5131 pr_hblk); 5132 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5133 } 5134 SFMMU_HASH_UNLOCK(hmebp); 5135 5136 if (iskernel) { 5137 hashno = TTE64K; 5138 continue; 5139 } 5140 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5141 ASSERT(hashno == TTE64K); 5142 continue; 5143 } 5144 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5145 hashno = TTE512K; 5146 continue; 5147 } 5148 if (mmu_page_sizes == max_mmu_page_sizes) { 5149 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5150 hashno = TTE4M; 5151 continue; 5152 } 5153 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5154 hashno = TTE32M; 5155 continue; 5156 } 5157 hashno = TTE256M; 5158 continue; 5159 } else { 5160 hashno = TTE4M; 5161 continue; 5162 } 5163 } 5164 if (hmeblkp->hblk_shw_bit) { 5165 /* 5166 * If we encounter a shadow hmeblk we know there is 5167 * smaller sized hmeblks mapping the same address space. 5168 * Decrement the hash size and rehash. 5169 */ 5170 ASSERT(sfmmup != KHATID); 5171 hashno--; 5172 SFMMU_HASH_UNLOCK(hmebp); 5173 continue; 5174 } 5175 5176 /* 5177 * track callback address ranges. 5178 * only start a new range when it's not contiguous 5179 */ 5180 if (callback != NULL) { 5181 if (addr_count > 0 && 5182 addr == cb_end_addr[addr_count - 1]) 5183 --addr_count; 5184 else 5185 cb_start_addr[addr_count] = addr; 5186 } 5187 5188 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr, 5189 dmrp, flags); 5190 5191 if (callback != NULL) 5192 cb_end_addr[addr_count++] = addr; 5193 5194 if (((flags & HAT_UNLOAD_UNMAP) || (iskernel && !issegkmap)) && 5195 !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5196 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5197 pr_hblk); 5198 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5199 } 5200 SFMMU_HASH_UNLOCK(hmebp); 5201 5202 /* 5203 * Notify our caller as to exactly which pages 5204 * have been unloaded. We do these in clumps, 5205 * to minimize the number of xt_sync()s that need to occur. 5206 */ 5207 if (callback != NULL && addr_count == MAX_CB_ADDR) { 5208 DEMAP_RANGE_FLUSH(dmrp); 5209 if (dmrp != NULL) { 5210 cpuset = sfmmup->sfmmu_cpusran; 5211 xt_sync(cpuset); 5212 } 5213 5214 for (a = 0; a < MAX_CB_ADDR; ++a) { 5215 callback->hcb_start_addr = cb_start_addr[a]; 5216 callback->hcb_end_addr = cb_end_addr[a]; 5217 callback->hcb_function(callback); 5218 } 5219 addr_count = 0; 5220 } 5221 if (iskernel) { 5222 hashno = TTE64K; 5223 continue; 5224 } 5225 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5226 ASSERT(hashno == TTE64K); 5227 continue; 5228 } 5229 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5230 hashno = TTE512K; 5231 continue; 5232 } 5233 if (mmu_page_sizes == max_mmu_page_sizes) { 5234 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5235 hashno = TTE4M; 5236 continue; 5237 } 5238 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5239 hashno = TTE32M; 5240 continue; 5241 } 5242 hashno = TTE256M; 5243 } else { 5244 hashno = TTE4M; 5245 } 5246 } 5247 5248 sfmmu_hblks_list_purge(&list); 5249 DEMAP_RANGE_FLUSH(dmrp); 5250 if (dmrp != NULL) { 5251 cpuset = sfmmup->sfmmu_cpusran; 5252 xt_sync(cpuset); 5253 } 5254 if (callback && addr_count != 0) { 5255 for (a = 0; a < addr_count; ++a) { 5256 callback->hcb_start_addr = cb_start_addr[a]; 5257 callback->hcb_end_addr = cb_end_addr[a]; 5258 callback->hcb_function(callback); 5259 } 5260 } 5261 5262 /* 5263 * Check TSB and TLB page sizes if the process isn't exiting. 5264 */ 5265 if (!sfmmup->sfmmu_free) 5266 sfmmu_check_page_sizes(sfmmup, 0); 5267 } 5268 5269 /* 5270 * Unload all the mappings in the range [addr..addr+len). addr and len must 5271 * be MMU_PAGESIZE aligned. 5272 */ 5273 void 5274 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags) 5275 { 5276 if (sfmmup->sfmmu_xhat_provider) { 5277 XHAT_UNLOAD(sfmmup, addr, len, flags); 5278 return; 5279 } 5280 hat_unload_callback(sfmmup, addr, len, flags, NULL); 5281 } 5282 5283 5284 /* 5285 * Find the largest mapping size for this page. 5286 */ 5287 int 5288 fnd_mapping_sz(page_t *pp) 5289 { 5290 int sz; 5291 int p_index; 5292 5293 p_index = PP_MAPINDEX(pp); 5294 5295 sz = 0; 5296 p_index >>= 1; /* don't care about 8K bit */ 5297 for (; p_index; p_index >>= 1) { 5298 sz++; 5299 } 5300 5301 return (sz); 5302 } 5303 5304 /* 5305 * This function unloads a range of addresses for an hmeblk. 5306 * It returns the next address to be unloaded. 5307 * It should be called with the hash lock held. 5308 */ 5309 static caddr_t 5310 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5311 caddr_t endaddr, demap_range_t *dmrp, uint_t flags) 5312 { 5313 tte_t tte, ttemod; 5314 struct sf_hment *sfhmep; 5315 int ttesz; 5316 long ttecnt; 5317 page_t *pp; 5318 kmutex_t *pml; 5319 int ret; 5320 int use_demap_range; 5321 5322 ASSERT(in_hblk_range(hmeblkp, addr)); 5323 ASSERT(!hmeblkp->hblk_shw_bit); 5324 #ifdef DEBUG 5325 if (get_hblk_ttesz(hmeblkp) != TTE8K && 5326 (endaddr < get_hblk_endaddr(hmeblkp))) { 5327 panic("sfmmu_hblk_unload: partial unload of large page"); 5328 } 5329 #endif /* DEBUG */ 5330 5331 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5332 ttesz = get_hblk_ttesz(hmeblkp); 5333 5334 use_demap_range = (do_virtual_coloring && 5335 ((dmrp == NULL) || TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp))); 5336 if (use_demap_range) { 5337 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 5338 } else { 5339 DEMAP_RANGE_FLUSH(dmrp); 5340 } 5341 ttecnt = 0; 5342 HBLKTOHME(sfhmep, hmeblkp, addr); 5343 5344 while (addr < endaddr) { 5345 pml = NULL; 5346 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5347 if (TTE_IS_VALID(&tte)) { 5348 pp = sfhmep->hme_page; 5349 if (pp != NULL) { 5350 pml = sfmmu_mlist_enter(pp); 5351 } 5352 5353 /* 5354 * Verify if hme still points to 'pp' now that 5355 * we have p_mapping lock. 5356 */ 5357 if (sfhmep->hme_page != pp) { 5358 if (pp != NULL && sfhmep->hme_page != NULL) { 5359 ASSERT(pml != NULL); 5360 sfmmu_mlist_exit(pml); 5361 /* Re-start this iteration. */ 5362 continue; 5363 } 5364 ASSERT((pp != NULL) && 5365 (sfhmep->hme_page == NULL)); 5366 goto tte_unloaded; 5367 } 5368 5369 /* 5370 * This point on we have both HASH and p_mapping 5371 * lock. 5372 */ 5373 ASSERT(pp == sfhmep->hme_page); 5374 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5375 5376 /* 5377 * We need to loop on modify tte because it is 5378 * possible for pagesync to come along and 5379 * change the software bits beneath us. 5380 * 5381 * Page_unload can also invalidate the tte after 5382 * we read tte outside of p_mapping lock. 5383 */ 5384 again: 5385 ttemod = tte; 5386 5387 TTE_SET_INVALID(&ttemod); 5388 ret = sfmmu_modifytte_try(&tte, &ttemod, 5389 &sfhmep->hme_tte); 5390 5391 if (ret <= 0) { 5392 if (TTE_IS_VALID(&tte)) { 5393 ASSERT(ret < 0); 5394 goto again; 5395 } 5396 if (pp != NULL) { 5397 panic("sfmmu_hblk_unload: pp = 0x%p " 5398 "tte became invalid under mlist" 5399 " lock = 0x%p", pp, pml); 5400 } 5401 continue; 5402 } 5403 5404 if (!(flags & HAT_UNLOAD_NOSYNC)) { 5405 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5406 } 5407 5408 /* 5409 * Ok- we invalidated the tte. Do the rest of the job. 5410 */ 5411 ttecnt++; 5412 5413 if (flags & HAT_UNLOAD_UNLOCK) { 5414 ASSERT(hmeblkp->hblk_lckcnt > 0); 5415 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 5416 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 5417 } 5418 5419 /* 5420 * Normally we would need to flush the page 5421 * from the virtual cache at this point in 5422 * order to prevent a potential cache alias 5423 * inconsistency. 5424 * The particular scenario we need to worry 5425 * about is: 5426 * Given: va1 and va2 are two virtual address 5427 * that alias and map the same physical 5428 * address. 5429 * 1. mapping exists from va1 to pa and data 5430 * has been read into the cache. 5431 * 2. unload va1. 5432 * 3. load va2 and modify data using va2. 5433 * 4 unload va2. 5434 * 5. load va1 and reference data. Unless we 5435 * flush the data cache when we unload we will 5436 * get stale data. 5437 * Fortunately, page coloring eliminates the 5438 * above scenario by remembering the color a 5439 * physical page was last or is currently 5440 * mapped to. Now, we delay the flush until 5441 * the loading of translations. Only when the 5442 * new translation is of a different color 5443 * are we forced to flush. 5444 */ 5445 if (use_demap_range) { 5446 /* 5447 * Mark this page as needing a demap. 5448 */ 5449 DEMAP_RANGE_MARKPG(dmrp, addr); 5450 } else { 5451 if (do_virtual_coloring) { 5452 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 5453 sfmmup->sfmmu_free, 0); 5454 } else { 5455 pfn_t pfnum; 5456 5457 pfnum = TTE_TO_PFN(addr, &tte); 5458 sfmmu_tlbcache_demap(addr, sfmmup, 5459 hmeblkp, pfnum, sfmmup->sfmmu_free, 5460 FLUSH_NECESSARY_CPUS, 5461 CACHE_FLUSH, 0); 5462 } 5463 } 5464 5465 if (pp) { 5466 /* 5467 * Remove the hment from the mapping list 5468 */ 5469 ASSERT(hmeblkp->hblk_hmecnt > 0); 5470 5471 /* 5472 * Again, we cannot 5473 * ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS); 5474 */ 5475 HME_SUB(sfhmep, pp); 5476 membar_stst(); 5477 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 5478 } 5479 5480 ASSERT(hmeblkp->hblk_vcnt > 0); 5481 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 5482 5483 ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 5484 !hmeblkp->hblk_lckcnt); 5485 5486 #ifdef VAC 5487 if (pp && (pp->p_nrm & (P_KPMC | P_KPMS | P_TNC))) { 5488 if (PP_ISTNC(pp)) { 5489 /* 5490 * If page was temporary 5491 * uncached, try to recache 5492 * it. Note that HME_SUB() was 5493 * called above so p_index and 5494 * mlist had been updated. 5495 */ 5496 conv_tnc(pp, ttesz); 5497 } else if (pp->p_mapping == NULL) { 5498 ASSERT(kpm_enable); 5499 /* 5500 * Page is marked to be in VAC conflict 5501 * to an existing kpm mapping and/or is 5502 * kpm mapped using only the regular 5503 * pagesize. 5504 */ 5505 sfmmu_kpm_hme_unload(pp); 5506 } 5507 } 5508 #endif /* VAC */ 5509 } else if ((pp = sfhmep->hme_page) != NULL) { 5510 /* 5511 * TTE is invalid but the hme 5512 * still exists. let pageunload 5513 * complete its job. 5514 */ 5515 ASSERT(pml == NULL); 5516 pml = sfmmu_mlist_enter(pp); 5517 if (sfhmep->hme_page != NULL) { 5518 sfmmu_mlist_exit(pml); 5519 continue; 5520 } 5521 ASSERT(sfhmep->hme_page == NULL); 5522 } else if (hmeblkp->hblk_hmecnt != 0) { 5523 /* 5524 * pageunload may have not finished decrementing 5525 * hblk_vcnt and hblk_hmecnt. Find page_t if any and 5526 * wait for pageunload to finish. Rely on pageunload 5527 * to decrement hblk_hmecnt after hblk_vcnt. 5528 */ 5529 pfn_t pfn = TTE_TO_TTEPFN(&tte); 5530 ASSERT(pml == NULL); 5531 if (pf_is_memory(pfn)) { 5532 pp = page_numtopp_nolock(pfn); 5533 if (pp != NULL) { 5534 pml = sfmmu_mlist_enter(pp); 5535 sfmmu_mlist_exit(pml); 5536 pml = NULL; 5537 } 5538 } 5539 } 5540 5541 tte_unloaded: 5542 /* 5543 * At this point, the tte we are looking at 5544 * should be unloaded, and hme has been unlinked 5545 * from page too. This is important because in 5546 * pageunload, it does ttesync() then HME_SUB. 5547 * We need to make sure HME_SUB has been completed 5548 * so we know ttesync() has been completed. Otherwise, 5549 * at exit time, after return from hat layer, VM will 5550 * release as structure which hat_setstat() (called 5551 * by ttesync()) needs. 5552 */ 5553 #ifdef DEBUG 5554 { 5555 tte_t dtte; 5556 5557 ASSERT(sfhmep->hme_page == NULL); 5558 5559 sfmmu_copytte(&sfhmep->hme_tte, &dtte); 5560 ASSERT(!TTE_IS_VALID(&dtte)); 5561 } 5562 #endif 5563 5564 if (pml) { 5565 sfmmu_mlist_exit(pml); 5566 } 5567 5568 addr += TTEBYTES(ttesz); 5569 sfhmep++; 5570 DEMAP_RANGE_NEXTPG(dmrp); 5571 } 5572 if (ttecnt > 0) 5573 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt); 5574 return (addr); 5575 } 5576 5577 /* 5578 * Synchronize all the mappings in the range [addr..addr+len). 5579 * Can be called with clearflag having two states: 5580 * HAT_SYNC_DONTZERO means just return the rm stats 5581 * HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats 5582 */ 5583 void 5584 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag) 5585 { 5586 struct hmehash_bucket *hmebp; 5587 hmeblk_tag hblktag; 5588 int hmeshift, hashno = 1; 5589 struct hme_blk *hmeblkp, *list = NULL; 5590 caddr_t endaddr; 5591 cpuset_t cpuset; 5592 5593 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 5594 ASSERT((sfmmup == ksfmmup) || 5595 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 5596 ASSERT((len & MMU_PAGEOFFSET) == 0); 5597 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 5598 (clearflag == HAT_SYNC_ZERORM)); 5599 5600 CPUSET_ZERO(cpuset); 5601 5602 endaddr = addr + len; 5603 hblktag.htag_id = sfmmup; 5604 /* 5605 * Spitfire supports 4 page sizes. 5606 * Most pages are expected to be of the smallest page 5607 * size (8K) and these will not need to be rehashed. 64K 5608 * pages also don't need to be rehashed because the an hmeblk 5609 * spans 64K of address space. 512K pages might need 1 rehash and 5610 * and 4M pages 2 rehashes. 5611 */ 5612 while (addr < endaddr) { 5613 hmeshift = HME_HASH_SHIFT(hashno); 5614 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5615 hblktag.htag_rehash = hashno; 5616 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5617 5618 SFMMU_HASH_LOCK(hmebp); 5619 5620 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 5621 if (hmeblkp != NULL) { 5622 /* 5623 * We've encountered a shadow hmeblk so skip the range 5624 * of the next smaller mapping size. 5625 */ 5626 if (hmeblkp->hblk_shw_bit) { 5627 ASSERT(sfmmup != ksfmmup); 5628 ASSERT(hashno > 1); 5629 addr = (caddr_t)P2END((uintptr_t)addr, 5630 TTEBYTES(hashno - 1)); 5631 } else { 5632 addr = sfmmu_hblk_sync(sfmmup, hmeblkp, 5633 addr, endaddr, clearflag); 5634 } 5635 SFMMU_HASH_UNLOCK(hmebp); 5636 hashno = 1; 5637 continue; 5638 } 5639 SFMMU_HASH_UNLOCK(hmebp); 5640 5641 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 5642 /* 5643 * We have traversed the whole list and rehashed 5644 * if necessary without finding the address to sync. 5645 * This is ok so we increment the address by the 5646 * smallest hmeblk range for kernel mappings and the 5647 * largest hmeblk range, to account for shadow hmeblks, 5648 * for user mappings and continue. 5649 */ 5650 if (sfmmup == ksfmmup) 5651 addr = (caddr_t)P2END((uintptr_t)addr, 5652 TTEBYTES(1)); 5653 else 5654 addr = (caddr_t)P2END((uintptr_t)addr, 5655 TTEBYTES(hashno)); 5656 hashno = 1; 5657 } else { 5658 hashno++; 5659 } 5660 } 5661 sfmmu_hblks_list_purge(&list); 5662 cpuset = sfmmup->sfmmu_cpusran; 5663 xt_sync(cpuset); 5664 } 5665 5666 static caddr_t 5667 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5668 caddr_t endaddr, int clearflag) 5669 { 5670 tte_t tte, ttemod; 5671 struct sf_hment *sfhmep; 5672 int ttesz; 5673 struct page *pp; 5674 kmutex_t *pml; 5675 int ret; 5676 5677 ASSERT(hmeblkp->hblk_shw_bit == 0); 5678 5679 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5680 5681 ttesz = get_hblk_ttesz(hmeblkp); 5682 HBLKTOHME(sfhmep, hmeblkp, addr); 5683 5684 while (addr < endaddr) { 5685 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5686 if (TTE_IS_VALID(&tte)) { 5687 pml = NULL; 5688 pp = sfhmep->hme_page; 5689 if (pp) { 5690 pml = sfmmu_mlist_enter(pp); 5691 } 5692 if (pp != sfhmep->hme_page) { 5693 /* 5694 * tte most have been unloaded 5695 * underneath us. Recheck 5696 */ 5697 ASSERT(pml); 5698 sfmmu_mlist_exit(pml); 5699 continue; 5700 } 5701 5702 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5703 5704 if (clearflag == HAT_SYNC_ZERORM) { 5705 ttemod = tte; 5706 TTE_CLR_RM(&ttemod); 5707 ret = sfmmu_modifytte_try(&tte, &ttemod, 5708 &sfhmep->hme_tte); 5709 if (ret < 0) { 5710 if (pml) { 5711 sfmmu_mlist_exit(pml); 5712 } 5713 continue; 5714 } 5715 5716 if (ret > 0) { 5717 sfmmu_tlb_demap(addr, sfmmup, 5718 hmeblkp, 0, 0); 5719 } 5720 } 5721 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5722 if (pml) { 5723 sfmmu_mlist_exit(pml); 5724 } 5725 } 5726 addr += TTEBYTES(ttesz); 5727 sfhmep++; 5728 } 5729 return (addr); 5730 } 5731 5732 /* 5733 * This function will sync a tte to the page struct and it will 5734 * update the hat stats. Currently it allows us to pass a NULL pp 5735 * and we will simply update the stats. We may want to change this 5736 * so we only keep stats for pages backed by pp's. 5737 */ 5738 static void 5739 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp) 5740 { 5741 uint_t rm = 0; 5742 int sz; 5743 pgcnt_t npgs; 5744 5745 ASSERT(TTE_IS_VALID(ttep)); 5746 5747 if (TTE_IS_NOSYNC(ttep)) { 5748 return; 5749 } 5750 5751 if (TTE_IS_REF(ttep)) { 5752 rm = P_REF; 5753 } 5754 if (TTE_IS_MOD(ttep)) { 5755 rm |= P_MOD; 5756 } 5757 5758 if (rm == 0) { 5759 return; 5760 } 5761 5762 sz = TTE_CSZ(ttep); 5763 if (sfmmup->sfmmu_rmstat) { 5764 int i; 5765 caddr_t vaddr = addr; 5766 5767 for (i = 0; i < TTEPAGES(sz); i++, vaddr += MMU_PAGESIZE) { 5768 hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm); 5769 } 5770 5771 } 5772 5773 /* 5774 * XXX I want to use cas to update nrm bits but they 5775 * currently belong in common/vm and not in hat where 5776 * they should be. 5777 * The nrm bits are protected by the same mutex as 5778 * the one that protects the page's mapping list. 5779 */ 5780 if (!pp) 5781 return; 5782 ASSERT(sfmmu_mlist_held(pp)); 5783 /* 5784 * If the tte is for a large page, we need to sync all the 5785 * pages covered by the tte. 5786 */ 5787 if (sz != TTE8K) { 5788 ASSERT(pp->p_szc != 0); 5789 pp = PP_GROUPLEADER(pp, sz); 5790 ASSERT(sfmmu_mlist_held(pp)); 5791 } 5792 5793 /* Get number of pages from tte size. */ 5794 npgs = TTEPAGES(sz); 5795 5796 do { 5797 ASSERT(pp); 5798 ASSERT(sfmmu_mlist_held(pp)); 5799 if (((rm & P_REF) != 0 && !PP_ISREF(pp)) || 5800 ((rm & P_MOD) != 0 && !PP_ISMOD(pp))) 5801 hat_page_setattr(pp, rm); 5802 5803 /* 5804 * Are we done? If not, we must have a large mapping. 5805 * For large mappings we need to sync the rest of the pages 5806 * covered by this tte; goto the next page. 5807 */ 5808 } while (--npgs > 0 && (pp = PP_PAGENEXT(pp))); 5809 } 5810 5811 /* 5812 * Execute pre-callback handler of each pa_hment linked to pp 5813 * 5814 * Inputs: 5815 * flag: either HAT_PRESUSPEND or HAT_SUSPEND. 5816 * capture_cpus: pointer to return value (below) 5817 * 5818 * Returns: 5819 * Propagates the subsystem callback return values back to the caller; 5820 * returns 0 on success. If capture_cpus is non-NULL, the value returned 5821 * is zero if all of the pa_hments are of a type that do not require 5822 * capturing CPUs prior to suspending the mapping, else it is 1. 5823 */ 5824 static int 5825 hat_pageprocess_precallbacks(struct page *pp, uint_t flag, int *capture_cpus) 5826 { 5827 struct sf_hment *sfhmep; 5828 struct pa_hment *pahmep; 5829 int (*f)(caddr_t, uint_t, uint_t, void *); 5830 int ret; 5831 id_t id; 5832 int locked = 0; 5833 kmutex_t *pml; 5834 5835 ASSERT(PAGE_EXCL(pp)); 5836 if (!sfmmu_mlist_held(pp)) { 5837 pml = sfmmu_mlist_enter(pp); 5838 locked = 1; 5839 } 5840 5841 if (capture_cpus) 5842 *capture_cpus = 0; 5843 5844 top: 5845 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5846 /* 5847 * skip sf_hments corresponding to VA<->PA mappings; 5848 * for pa_hment's, hme_tte.ll is zero 5849 */ 5850 if (!IS_PAHME(sfhmep)) 5851 continue; 5852 5853 pahmep = sfhmep->hme_data; 5854 ASSERT(pahmep != NULL); 5855 5856 /* 5857 * skip if pre-handler has been called earlier in this loop 5858 */ 5859 if (pahmep->flags & flag) 5860 continue; 5861 5862 id = pahmep->cb_id; 5863 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5864 if (capture_cpus && sfmmu_cb_table[id].capture_cpus != 0) 5865 *capture_cpus = 1; 5866 if ((f = sfmmu_cb_table[id].prehandler) == NULL) { 5867 pahmep->flags |= flag; 5868 continue; 5869 } 5870 5871 /* 5872 * Drop the mapping list lock to avoid locking order issues. 5873 */ 5874 if (locked) 5875 sfmmu_mlist_exit(pml); 5876 5877 ret = f(pahmep->addr, pahmep->len, flag, pahmep->pvt); 5878 if (ret != 0) 5879 return (ret); /* caller must do the cleanup */ 5880 5881 if (locked) { 5882 pml = sfmmu_mlist_enter(pp); 5883 pahmep->flags |= flag; 5884 goto top; 5885 } 5886 5887 pahmep->flags |= flag; 5888 } 5889 5890 if (locked) 5891 sfmmu_mlist_exit(pml); 5892 5893 return (0); 5894 } 5895 5896 /* 5897 * Execute post-callback handler of each pa_hment linked to pp 5898 * 5899 * Same overall assumptions and restrictions apply as for 5900 * hat_pageprocess_precallbacks(). 5901 */ 5902 static void 5903 hat_pageprocess_postcallbacks(struct page *pp, uint_t flag) 5904 { 5905 pfn_t pgpfn = pp->p_pagenum; 5906 pfn_t pgmask = btop(page_get_pagesize(pp->p_szc)) - 1; 5907 pfn_t newpfn; 5908 struct sf_hment *sfhmep; 5909 struct pa_hment *pahmep; 5910 int (*f)(caddr_t, uint_t, uint_t, void *, pfn_t); 5911 id_t id; 5912 int locked = 0; 5913 kmutex_t *pml; 5914 5915 ASSERT(PAGE_EXCL(pp)); 5916 if (!sfmmu_mlist_held(pp)) { 5917 pml = sfmmu_mlist_enter(pp); 5918 locked = 1; 5919 } 5920 5921 top: 5922 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5923 /* 5924 * skip sf_hments corresponding to VA<->PA mappings; 5925 * for pa_hment's, hme_tte.ll is zero 5926 */ 5927 if (!IS_PAHME(sfhmep)) 5928 continue; 5929 5930 pahmep = sfhmep->hme_data; 5931 ASSERT(pahmep != NULL); 5932 5933 if ((pahmep->flags & flag) == 0) 5934 continue; 5935 5936 pahmep->flags &= ~flag; 5937 5938 id = pahmep->cb_id; 5939 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5940 if ((f = sfmmu_cb_table[id].posthandler) == NULL) 5941 continue; 5942 5943 /* 5944 * Convert the base page PFN into the constituent PFN 5945 * which is needed by the callback handler. 5946 */ 5947 newpfn = pgpfn | (btop((uintptr_t)pahmep->addr) & pgmask); 5948 5949 /* 5950 * Drop the mapping list lock to avoid locking order issues. 5951 */ 5952 if (locked) 5953 sfmmu_mlist_exit(pml); 5954 5955 if (f(pahmep->addr, pahmep->len, flag, pahmep->pvt, newpfn) 5956 != 0) 5957 panic("sfmmu: posthandler failed"); 5958 5959 if (locked) { 5960 pml = sfmmu_mlist_enter(pp); 5961 goto top; 5962 } 5963 } 5964 5965 if (locked) 5966 sfmmu_mlist_exit(pml); 5967 } 5968 5969 /* 5970 * Suspend locked kernel mapping 5971 */ 5972 void 5973 hat_pagesuspend(struct page *pp) 5974 { 5975 struct sf_hment *sfhmep; 5976 sfmmu_t *sfmmup; 5977 tte_t tte, ttemod; 5978 struct hme_blk *hmeblkp; 5979 caddr_t addr; 5980 int index, cons; 5981 cpuset_t cpuset; 5982 5983 ASSERT(PAGE_EXCL(pp)); 5984 ASSERT(sfmmu_mlist_held(pp)); 5985 5986 mutex_enter(&kpr_suspendlock); 5987 5988 /* 5989 * Call into dtrace to tell it we're about to suspend a 5990 * kernel mapping. This prevents us from running into issues 5991 * with probe context trying to touch a suspended page 5992 * in the relocation codepath itself. 5993 */ 5994 if (dtrace_kreloc_init) 5995 (*dtrace_kreloc_init)(); 5996 5997 index = PP_MAPINDEX(pp); 5998 cons = TTE8K; 5999 6000 retry: 6001 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 6002 6003 if (IS_PAHME(sfhmep)) 6004 continue; 6005 6006 if (get_hblk_ttesz(sfmmu_hmetohblk(sfhmep)) != cons) 6007 continue; 6008 6009 /* 6010 * Loop until we successfully set the suspend bit in 6011 * the TTE. 6012 */ 6013 again: 6014 sfmmu_copytte(&sfhmep->hme_tte, &tte); 6015 ASSERT(TTE_IS_VALID(&tte)); 6016 6017 ttemod = tte; 6018 TTE_SET_SUSPEND(&ttemod); 6019 if (sfmmu_modifytte_try(&tte, &ttemod, 6020 &sfhmep->hme_tte) < 0) 6021 goto again; 6022 6023 /* 6024 * Invalidate TSB entry 6025 */ 6026 hmeblkp = sfmmu_hmetohblk(sfhmep); 6027 6028 sfmmup = hblktosfmmu(hmeblkp); 6029 ASSERT(sfmmup == ksfmmup); 6030 6031 addr = tte_to_vaddr(hmeblkp, tte); 6032 6033 /* 6034 * No need to make sure that the TSB for this sfmmu is 6035 * not being relocated since it is ksfmmup and thus it 6036 * will never be relocated. 6037 */ 6038 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 6039 6040 /* 6041 * Update xcall stats 6042 */ 6043 cpuset = cpu_ready_set; 6044 CPUSET_DEL(cpuset, CPU->cpu_id); 6045 6046 /* LINTED: constant in conditional context */ 6047 SFMMU_XCALL_STATS(ksfmmup); 6048 6049 /* 6050 * Flush TLB entry on remote CPU's 6051 */ 6052 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 6053 (uint64_t)ksfmmup); 6054 xt_sync(cpuset); 6055 6056 /* 6057 * Flush TLB entry on local CPU 6058 */ 6059 vtag_flushpage(addr, (uint64_t)ksfmmup); 6060 } 6061 6062 while (index != 0) { 6063 index = index >> 1; 6064 if (index != 0) 6065 cons++; 6066 if (index & 0x1) { 6067 pp = PP_GROUPLEADER(pp, cons); 6068 goto retry; 6069 } 6070 } 6071 } 6072 6073 #ifdef DEBUG 6074 6075 #define N_PRLE 1024 6076 struct prle { 6077 page_t *targ; 6078 page_t *repl; 6079 int status; 6080 int pausecpus; 6081 hrtime_t whence; 6082 }; 6083 6084 static struct prle page_relocate_log[N_PRLE]; 6085 static int prl_entry; 6086 static kmutex_t prl_mutex; 6087 6088 #define PAGE_RELOCATE_LOG(t, r, s, p) \ 6089 mutex_enter(&prl_mutex); \ 6090 page_relocate_log[prl_entry].targ = *(t); \ 6091 page_relocate_log[prl_entry].repl = *(r); \ 6092 page_relocate_log[prl_entry].status = (s); \ 6093 page_relocate_log[prl_entry].pausecpus = (p); \ 6094 page_relocate_log[prl_entry].whence = gethrtime(); \ 6095 prl_entry = (prl_entry == (N_PRLE - 1))? 0 : prl_entry + 1; \ 6096 mutex_exit(&prl_mutex); 6097 6098 #else /* !DEBUG */ 6099 #define PAGE_RELOCATE_LOG(t, r, s, p) 6100 #endif 6101 6102 /* 6103 * Core Kernel Page Relocation Algorithm 6104 * 6105 * Input: 6106 * 6107 * target : constituent pages are SE_EXCL locked. 6108 * replacement: constituent pages are SE_EXCL locked. 6109 * 6110 * Output: 6111 * 6112 * nrelocp: number of pages relocated 6113 */ 6114 int 6115 hat_page_relocate(page_t **target, page_t **replacement, spgcnt_t *nrelocp) 6116 { 6117 page_t *targ, *repl; 6118 page_t *tpp, *rpp; 6119 kmutex_t *low, *high; 6120 spgcnt_t npages, i; 6121 page_t *pl = NULL; 6122 int old_pil; 6123 cpuset_t cpuset; 6124 int cap_cpus; 6125 int ret; 6126 6127 if (hat_kpr_enabled == 0 || !kcage_on || PP_ISNORELOC(*target)) { 6128 PAGE_RELOCATE_LOG(target, replacement, EAGAIN, -1); 6129 return (EAGAIN); 6130 } 6131 6132 mutex_enter(&kpr_mutex); 6133 kreloc_thread = curthread; 6134 6135 targ = *target; 6136 repl = *replacement; 6137 ASSERT(repl != NULL); 6138 ASSERT(targ->p_szc == repl->p_szc); 6139 6140 npages = page_get_pagecnt(targ->p_szc); 6141 6142 /* 6143 * unload VA<->PA mappings that are not locked 6144 */ 6145 tpp = targ; 6146 for (i = 0; i < npages; i++) { 6147 (void) hat_pageunload(tpp, SFMMU_KERNEL_RELOC); 6148 tpp++; 6149 } 6150 6151 /* 6152 * Do "presuspend" callbacks, in a context from which we can still 6153 * block as needed. Note that we don't hold the mapping list lock 6154 * of "targ" at this point due to potential locking order issues; 6155 * we assume that between the hat_pageunload() above and holding 6156 * the SE_EXCL lock that the mapping list *cannot* change at this 6157 * point. 6158 */ 6159 ret = hat_pageprocess_precallbacks(targ, HAT_PRESUSPEND, &cap_cpus); 6160 if (ret != 0) { 6161 /* 6162 * EIO translates to fatal error, for all others cleanup 6163 * and return EAGAIN. 6164 */ 6165 ASSERT(ret != EIO); 6166 hat_pageprocess_postcallbacks(targ, HAT_POSTUNSUSPEND); 6167 PAGE_RELOCATE_LOG(target, replacement, ret, -1); 6168 kreloc_thread = NULL; 6169 mutex_exit(&kpr_mutex); 6170 return (EAGAIN); 6171 } 6172 6173 /* 6174 * acquire p_mapping list lock for both the target and replacement 6175 * root pages. 6176 * 6177 * low and high refer to the need to grab the mlist locks in a 6178 * specific order in order to prevent race conditions. Thus the 6179 * lower lock must be grabbed before the higher lock. 6180 * 6181 * This will block hat_unload's accessing p_mapping list. Since 6182 * we have SE_EXCL lock, hat_memload and hat_pageunload will be 6183 * blocked. Thus, no one else will be accessing the p_mapping list 6184 * while we suspend and reload the locked mapping below. 6185 */ 6186 tpp = targ; 6187 rpp = repl; 6188 sfmmu_mlist_reloc_enter(tpp, rpp, &low, &high); 6189 6190 kpreempt_disable(); 6191 6192 #ifdef VAC 6193 /* 6194 * If the replacement page is of a different virtual color 6195 * than the page it is replacing, we need to handle the VAC 6196 * consistency for it just as we would if we were setting up 6197 * a new mapping to a page. 6198 */ 6199 if ((tpp->p_szc == 0) && (PP_GET_VCOLOR(rpp) != NO_VCOLOR)) { 6200 if (tpp->p_vcolor != rpp->p_vcolor) { 6201 sfmmu_cache_flushcolor(PP_GET_VCOLOR(rpp), 6202 rpp->p_pagenum); 6203 } 6204 } 6205 #endif 6206 6207 /* 6208 * We raise our PIL to 13 so that we don't get captured by 6209 * another CPU or pinned by an interrupt thread. We can't go to 6210 * PIL 14 since the nexus driver(s) may need to interrupt at 6211 * that level in the case of IOMMU pseudo mappings. 6212 */ 6213 cpuset = cpu_ready_set; 6214 CPUSET_DEL(cpuset, CPU->cpu_id); 6215 if (!cap_cpus || CPUSET_ISNULL(cpuset)) { 6216 old_pil = splr(XCALL_PIL); 6217 } else { 6218 old_pil = -1; 6219 xc_attention(cpuset); 6220 } 6221 ASSERT(getpil() == XCALL_PIL); 6222 6223 /* 6224 * Now do suspend callbacks. In the case of an IOMMU mapping 6225 * this will suspend all DMA activity to the page while it is 6226 * being relocated. Since we are well above LOCK_LEVEL and CPUs 6227 * may be captured at this point we should have acquired any needed 6228 * locks in the presuspend callback. 6229 */ 6230 ret = hat_pageprocess_precallbacks(targ, HAT_SUSPEND, NULL); 6231 if (ret != 0) { 6232 repl = targ; 6233 goto suspend_fail; 6234 } 6235 6236 /* 6237 * Raise the PIL yet again, this time to block all high-level 6238 * interrupts on this CPU. This is necessary to prevent an 6239 * interrupt routine from pinning the thread which holds the 6240 * mapping suspended and then touching the suspended page. 6241 * 6242 * Once the page is suspended we also need to be careful to 6243 * avoid calling any functions which touch any seg_kmem memory 6244 * since that memory may be backed by the very page we are 6245 * relocating in here! 6246 */ 6247 hat_pagesuspend(targ); 6248 6249 /* 6250 * Now that we are confident everybody has stopped using this page, 6251 * copy the page contents. Note we use a physical copy to prevent 6252 * locking issues and to avoid fpRAS because we can't handle it in 6253 * this context. 6254 */ 6255 for (i = 0; i < npages; i++, tpp++, rpp++) { 6256 /* 6257 * Copy the contents of the page. 6258 */ 6259 ppcopy_kernel(tpp, rpp); 6260 } 6261 6262 tpp = targ; 6263 rpp = repl; 6264 for (i = 0; i < npages; i++, tpp++, rpp++) { 6265 /* 6266 * Copy attributes. VAC consistency was handled above, 6267 * if required. 6268 */ 6269 rpp->p_nrm = tpp->p_nrm; 6270 tpp->p_nrm = 0; 6271 rpp->p_index = tpp->p_index; 6272 tpp->p_index = 0; 6273 #ifdef VAC 6274 rpp->p_vcolor = tpp->p_vcolor; 6275 #endif 6276 } 6277 6278 /* 6279 * First, unsuspend the page, if we set the suspend bit, and transfer 6280 * the mapping list from the target page to the replacement page. 6281 * Next process postcallbacks; since pa_hment's are linked only to the 6282 * p_mapping list of root page, we don't iterate over the constituent 6283 * pages. 6284 */ 6285 hat_pagereload(targ, repl); 6286 6287 suspend_fail: 6288 hat_pageprocess_postcallbacks(repl, HAT_UNSUSPEND); 6289 6290 /* 6291 * Now lower our PIL and release any captured CPUs since we 6292 * are out of the "danger zone". After this it will again be 6293 * safe to acquire adaptive mutex locks, or to drop them... 6294 */ 6295 if (old_pil != -1) { 6296 splx(old_pil); 6297 } else { 6298 xc_dismissed(cpuset); 6299 } 6300 6301 kpreempt_enable(); 6302 6303 sfmmu_mlist_reloc_exit(low, high); 6304 6305 /* 6306 * Postsuspend callbacks should drop any locks held across 6307 * the suspend callbacks. As before, we don't hold the mapping 6308 * list lock at this point.. our assumption is that the mapping 6309 * list still can't change due to our holding SE_EXCL lock and 6310 * there being no unlocked mappings left. Hence the restriction 6311 * on calling context to hat_delete_callback() 6312 */ 6313 hat_pageprocess_postcallbacks(repl, HAT_POSTUNSUSPEND); 6314 if (ret != 0) { 6315 /* 6316 * The second presuspend call failed: we got here through 6317 * the suspend_fail label above. 6318 */ 6319 ASSERT(ret != EIO); 6320 PAGE_RELOCATE_LOG(target, replacement, ret, cap_cpus); 6321 kreloc_thread = NULL; 6322 mutex_exit(&kpr_mutex); 6323 return (EAGAIN); 6324 } 6325 6326 /* 6327 * Now that we're out of the performance critical section we can 6328 * take care of updating the hash table, since we still 6329 * hold all the pages locked SE_EXCL at this point we 6330 * needn't worry about things changing out from under us. 6331 */ 6332 tpp = targ; 6333 rpp = repl; 6334 for (i = 0; i < npages; i++, tpp++, rpp++) { 6335 6336 /* 6337 * replace targ with replacement in page_hash table 6338 */ 6339 targ = tpp; 6340 page_relocate_hash(rpp, targ); 6341 6342 /* 6343 * concatenate target; caller of platform_page_relocate() 6344 * expects target to be concatenated after returning. 6345 */ 6346 ASSERT(targ->p_next == targ); 6347 ASSERT(targ->p_prev == targ); 6348 page_list_concat(&pl, &targ); 6349 } 6350 6351 ASSERT(*target == pl); 6352 *nrelocp = npages; 6353 PAGE_RELOCATE_LOG(target, replacement, 0, cap_cpus); 6354 kreloc_thread = NULL; 6355 mutex_exit(&kpr_mutex); 6356 return (0); 6357 } 6358 6359 /* 6360 * Called when stray pa_hments are found attached to a page which is 6361 * being freed. Notify the subsystem which attached the pa_hment of 6362 * the error if it registered a suitable handler, else panic. 6363 */ 6364 static void 6365 sfmmu_pahment_leaked(struct pa_hment *pahmep) 6366 { 6367 id_t cb_id = pahmep->cb_id; 6368 6369 ASSERT(cb_id >= (id_t)0 && cb_id < sfmmu_cb_nextid); 6370 if (sfmmu_cb_table[cb_id].errhandler != NULL) { 6371 if (sfmmu_cb_table[cb_id].errhandler(pahmep->addr, pahmep->len, 6372 HAT_CB_ERR_LEAKED, pahmep->pvt) == 0) 6373 return; /* non-fatal */ 6374 } 6375 panic("pa_hment leaked: 0x%p", pahmep); 6376 } 6377 6378 /* 6379 * Remove all mappings to page 'pp'. 6380 */ 6381 int 6382 hat_pageunload(struct page *pp, uint_t forceflag) 6383 { 6384 struct page *origpp = pp; 6385 struct sf_hment *sfhme, *tmphme; 6386 struct hme_blk *hmeblkp; 6387 kmutex_t *pml; 6388 #ifdef VAC 6389 kmutex_t *pmtx; 6390 #endif 6391 cpuset_t cpuset, tset; 6392 int index, cons; 6393 int xhme_blks; 6394 int pa_hments; 6395 6396 ASSERT(PAGE_EXCL(pp)); 6397 6398 retry_xhat: 6399 tmphme = NULL; 6400 xhme_blks = 0; 6401 pa_hments = 0; 6402 CPUSET_ZERO(cpuset); 6403 6404 pml = sfmmu_mlist_enter(pp); 6405 6406 #ifdef VAC 6407 if (pp->p_kpmref) 6408 sfmmu_kpm_pageunload(pp); 6409 ASSERT(!PP_ISMAPPED_KPM(pp)); 6410 #endif 6411 6412 index = PP_MAPINDEX(pp); 6413 cons = TTE8K; 6414 retry: 6415 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6416 tmphme = sfhme->hme_next; 6417 6418 if (IS_PAHME(sfhme)) { 6419 ASSERT(sfhme->hme_data != NULL); 6420 pa_hments++; 6421 continue; 6422 } 6423 6424 hmeblkp = sfmmu_hmetohblk(sfhme); 6425 if (hmeblkp->hblk_xhat_bit) { 6426 struct xhat_hme_blk *xblk = 6427 (struct xhat_hme_blk *)hmeblkp; 6428 6429 (void) XHAT_PAGEUNLOAD(xblk->xhat_hme_blk_hat, 6430 pp, forceflag, XBLK2PROVBLK(xblk)); 6431 6432 xhme_blks = 1; 6433 continue; 6434 } 6435 6436 /* 6437 * If there are kernel mappings don't unload them, they will 6438 * be suspended. 6439 */ 6440 if (forceflag == SFMMU_KERNEL_RELOC && hmeblkp->hblk_lckcnt && 6441 hmeblkp->hblk_tag.htag_id == ksfmmup) 6442 continue; 6443 6444 tset = sfmmu_pageunload(pp, sfhme, cons); 6445 CPUSET_OR(cpuset, tset); 6446 } 6447 6448 while (index != 0) { 6449 index = index >> 1; 6450 if (index != 0) 6451 cons++; 6452 if (index & 0x1) { 6453 /* Go to leading page */ 6454 pp = PP_GROUPLEADER(pp, cons); 6455 ASSERT(sfmmu_mlist_held(pp)); 6456 goto retry; 6457 } 6458 } 6459 6460 /* 6461 * cpuset may be empty if the page was only mapped by segkpm, 6462 * in which case we won't actually cross-trap. 6463 */ 6464 xt_sync(cpuset); 6465 6466 /* 6467 * The page should have no mappings at this point, unless 6468 * we were called from hat_page_relocate() in which case we 6469 * leave the locked mappings which will be suspended later. 6470 */ 6471 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks || pa_hments || 6472 (forceflag == SFMMU_KERNEL_RELOC)); 6473 6474 #ifdef VAC 6475 if (PP_ISTNC(pp)) { 6476 if (cons == TTE8K) { 6477 pmtx = sfmmu_page_enter(pp); 6478 PP_CLRTNC(pp); 6479 sfmmu_page_exit(pmtx); 6480 } else { 6481 conv_tnc(pp, cons); 6482 } 6483 } 6484 #endif /* VAC */ 6485 6486 if (pa_hments && forceflag != SFMMU_KERNEL_RELOC) { 6487 /* 6488 * Unlink any pa_hments and free them, calling back 6489 * the responsible subsystem to notify it of the error. 6490 * This can occur in situations such as drivers leaking 6491 * DMA handles: naughty, but common enough that we'd like 6492 * to keep the system running rather than bringing it 6493 * down with an obscure error like "pa_hment leaked" 6494 * which doesn't aid the user in debugging their driver. 6495 */ 6496 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6497 tmphme = sfhme->hme_next; 6498 if (IS_PAHME(sfhme)) { 6499 struct pa_hment *pahmep = sfhme->hme_data; 6500 sfmmu_pahment_leaked(pahmep); 6501 HME_SUB(sfhme, pp); 6502 kmem_cache_free(pa_hment_cache, pahmep); 6503 } 6504 } 6505 6506 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks); 6507 } 6508 6509 sfmmu_mlist_exit(pml); 6510 6511 /* 6512 * XHAT may not have finished unloading pages 6513 * because some other thread was waiting for 6514 * mlist lock and XHAT_PAGEUNLOAD let it do 6515 * the job. 6516 */ 6517 if (xhme_blks) { 6518 pp = origpp; 6519 goto retry_xhat; 6520 } 6521 6522 return (0); 6523 } 6524 6525 cpuset_t 6526 sfmmu_pageunload(page_t *pp, struct sf_hment *sfhme, int cons) 6527 { 6528 struct hme_blk *hmeblkp; 6529 sfmmu_t *sfmmup; 6530 tte_t tte, ttemod; 6531 #ifdef DEBUG 6532 tte_t orig_old; 6533 #endif /* DEBUG */ 6534 caddr_t addr; 6535 int ttesz; 6536 int ret; 6537 cpuset_t cpuset; 6538 6539 ASSERT(pp != NULL); 6540 ASSERT(sfmmu_mlist_held(pp)); 6541 ASSERT(!PP_ISKAS(pp)); 6542 6543 CPUSET_ZERO(cpuset); 6544 6545 hmeblkp = sfmmu_hmetohblk(sfhme); 6546 6547 readtte: 6548 sfmmu_copytte(&sfhme->hme_tte, &tte); 6549 if (TTE_IS_VALID(&tte)) { 6550 sfmmup = hblktosfmmu(hmeblkp); 6551 ttesz = get_hblk_ttesz(hmeblkp); 6552 /* 6553 * Only unload mappings of 'cons' size. 6554 */ 6555 if (ttesz != cons) 6556 return (cpuset); 6557 6558 /* 6559 * Note that we have p_mapping lock, but no hash lock here. 6560 * hblk_unload() has to have both hash lock AND p_mapping 6561 * lock before it tries to modify tte. So, the tte could 6562 * not become invalid in the sfmmu_modifytte_try() below. 6563 */ 6564 ttemod = tte; 6565 #ifdef DEBUG 6566 orig_old = tte; 6567 #endif /* DEBUG */ 6568 6569 TTE_SET_INVALID(&ttemod); 6570 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6571 if (ret < 0) { 6572 #ifdef DEBUG 6573 /* only R/M bits can change. */ 6574 chk_tte(&orig_old, &tte, &ttemod, hmeblkp); 6575 #endif /* DEBUG */ 6576 goto readtte; 6577 } 6578 6579 if (ret == 0) { 6580 panic("pageunload: cas failed?"); 6581 } 6582 6583 addr = tte_to_vaddr(hmeblkp, tte); 6584 6585 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6586 6587 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -1); 6588 6589 /* 6590 * We need to flush the page from the virtual cache 6591 * in order to prevent a virtual cache alias 6592 * inconsistency. The particular scenario we need 6593 * to worry about is: 6594 * Given: va1 and va2 are two virtual address that 6595 * alias and will map the same physical address. 6596 * 1. mapping exists from va1 to pa and data has 6597 * been read into the cache. 6598 * 2. unload va1. 6599 * 3. load va2 and modify data using va2. 6600 * 4 unload va2. 6601 * 5. load va1 and reference data. Unless we flush 6602 * the data cache when we unload we will get 6603 * stale data. 6604 * This scenario is taken care of by using virtual 6605 * page coloring. 6606 */ 6607 if (sfmmup->sfmmu_ismhat) { 6608 /* 6609 * Flush TSBs, TLBs and caches 6610 * of every process 6611 * sharing this ism segment. 6612 */ 6613 sfmmu_hat_lock_all(); 6614 mutex_enter(&ism_mlist_lock); 6615 kpreempt_disable(); 6616 if (do_virtual_coloring) 6617 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6618 pp->p_pagenum, CACHE_NO_FLUSH); 6619 else 6620 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6621 pp->p_pagenum, CACHE_FLUSH); 6622 kpreempt_enable(); 6623 mutex_exit(&ism_mlist_lock); 6624 sfmmu_hat_unlock_all(); 6625 cpuset = cpu_ready_set; 6626 } else if (do_virtual_coloring) { 6627 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6628 cpuset = sfmmup->sfmmu_cpusran; 6629 } else { 6630 sfmmu_tlbcache_demap(addr, sfmmup, hmeblkp, 6631 pp->p_pagenum, 0, FLUSH_NECESSARY_CPUS, 6632 CACHE_FLUSH, 0); 6633 cpuset = sfmmup->sfmmu_cpusran; 6634 } 6635 6636 /* 6637 * Hme_sub has to run after ttesync() and a_rss update. 6638 * See hblk_unload(). 6639 */ 6640 HME_SUB(sfhme, pp); 6641 membar_stst(); 6642 6643 /* 6644 * We can not make ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 6645 * since pteload may have done a HME_ADD() right after 6646 * we did the HME_SUB() above. Hmecnt is now maintained 6647 * by cas only. no lock guranteed its value. The only 6648 * gurantee we have is the hmecnt should not be less than 6649 * what it should be so the hblk will not be taken away. 6650 * It's also important that we decremented the hmecnt after 6651 * we are done with hmeblkp so that this hmeblk won't be 6652 * stolen. 6653 */ 6654 ASSERT(hmeblkp->hblk_hmecnt > 0); 6655 ASSERT(hmeblkp->hblk_vcnt > 0); 6656 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 6657 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 6658 /* 6659 * This is bug 4063182. 6660 * XXX: fixme 6661 * ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 6662 * !hmeblkp->hblk_lckcnt); 6663 */ 6664 } else { 6665 panic("invalid tte? pp %p &tte %p", 6666 (void *)pp, (void *)&tte); 6667 } 6668 6669 return (cpuset); 6670 } 6671 6672 /* 6673 * While relocating a kernel page, this function will move the mappings 6674 * from tpp to dpp and modify any associated data with these mappings. 6675 * It also unsuspends the suspended kernel mapping. 6676 */ 6677 static void 6678 hat_pagereload(struct page *tpp, struct page *dpp) 6679 { 6680 struct sf_hment *sfhme; 6681 tte_t tte, ttemod; 6682 int index, cons; 6683 6684 ASSERT(getpil() == PIL_MAX); 6685 ASSERT(sfmmu_mlist_held(tpp)); 6686 ASSERT(sfmmu_mlist_held(dpp)); 6687 6688 index = PP_MAPINDEX(tpp); 6689 cons = TTE8K; 6690 6691 /* Update real mappings to the page */ 6692 retry: 6693 for (sfhme = tpp->p_mapping; sfhme != NULL; sfhme = sfhme->hme_next) { 6694 if (IS_PAHME(sfhme)) 6695 continue; 6696 sfmmu_copytte(&sfhme->hme_tte, &tte); 6697 ttemod = tte; 6698 6699 /* 6700 * replace old pfn with new pfn in TTE 6701 */ 6702 PFN_TO_TTE(ttemod, dpp->p_pagenum); 6703 6704 /* 6705 * clear suspend bit 6706 */ 6707 ASSERT(TTE_IS_SUSPEND(&ttemod)); 6708 TTE_CLR_SUSPEND(&ttemod); 6709 6710 if (sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte) < 0) 6711 panic("hat_pagereload(): sfmmu_modifytte_try() failed"); 6712 6713 /* 6714 * set hme_page point to new page 6715 */ 6716 sfhme->hme_page = dpp; 6717 } 6718 6719 /* 6720 * move p_mapping list from old page to new page 6721 */ 6722 dpp->p_mapping = tpp->p_mapping; 6723 tpp->p_mapping = NULL; 6724 dpp->p_share = tpp->p_share; 6725 tpp->p_share = 0; 6726 6727 while (index != 0) { 6728 index = index >> 1; 6729 if (index != 0) 6730 cons++; 6731 if (index & 0x1) { 6732 tpp = PP_GROUPLEADER(tpp, cons); 6733 dpp = PP_GROUPLEADER(dpp, cons); 6734 goto retry; 6735 } 6736 } 6737 6738 if (dtrace_kreloc_fini) 6739 (*dtrace_kreloc_fini)(); 6740 mutex_exit(&kpr_suspendlock); 6741 } 6742 6743 uint_t 6744 hat_pagesync(struct page *pp, uint_t clearflag) 6745 { 6746 struct sf_hment *sfhme, *tmphme = NULL; 6747 struct hme_blk *hmeblkp; 6748 kmutex_t *pml; 6749 cpuset_t cpuset, tset; 6750 int index, cons; 6751 extern ulong_t po_share; 6752 page_t *save_pp = pp; 6753 6754 CPUSET_ZERO(cpuset); 6755 6756 if (PP_ISRO(pp) && (clearflag & HAT_SYNC_STOPON_MOD)) { 6757 return (PP_GENERIC_ATTR(pp)); 6758 } 6759 6760 if ((clearflag == (HAT_SYNC_STOPON_REF | HAT_SYNC_DONTZERO)) && 6761 PP_ISREF(pp)) { 6762 return (PP_GENERIC_ATTR(pp)); 6763 } 6764 6765 if ((clearflag == (HAT_SYNC_STOPON_MOD | HAT_SYNC_DONTZERO)) && 6766 PP_ISMOD(pp)) { 6767 return (PP_GENERIC_ATTR(pp)); 6768 } 6769 6770 if ((clearflag & HAT_SYNC_STOPON_SHARED) != 0 && 6771 (pp->p_share > po_share) && 6772 !(clearflag & HAT_SYNC_ZERORM)) { 6773 if (PP_ISRO(pp)) 6774 hat_page_setattr(pp, P_REF); 6775 return (PP_GENERIC_ATTR(pp)); 6776 } 6777 6778 clearflag &= ~HAT_SYNC_STOPON_SHARED; 6779 pml = sfmmu_mlist_enter(pp); 6780 index = PP_MAPINDEX(pp); 6781 cons = TTE8K; 6782 retry: 6783 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6784 /* 6785 * We need to save the next hment on the list since 6786 * it is possible for pagesync to remove an invalid hment 6787 * from the list. 6788 */ 6789 tmphme = sfhme->hme_next; 6790 /* 6791 * If we are looking for large mappings and this hme doesn't 6792 * reach the range we are seeking, just ignore its. 6793 */ 6794 hmeblkp = sfmmu_hmetohblk(sfhme); 6795 if (hmeblkp->hblk_xhat_bit) 6796 continue; 6797 6798 if (hme_size(sfhme) < cons) 6799 continue; 6800 tset = sfmmu_pagesync(pp, sfhme, 6801 clearflag & ~HAT_SYNC_STOPON_RM); 6802 CPUSET_OR(cpuset, tset); 6803 /* 6804 * If clearflag is HAT_SYNC_DONTZERO, break out as soon 6805 * as the "ref" or "mod" is set. 6806 */ 6807 if ((clearflag & ~HAT_SYNC_STOPON_RM) == HAT_SYNC_DONTZERO && 6808 ((clearflag & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp)) || 6809 ((clearflag & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp))) { 6810 index = 0; 6811 break; 6812 } 6813 } 6814 6815 while (index) { 6816 index = index >> 1; 6817 cons++; 6818 if (index & 0x1) { 6819 /* Go to leading page */ 6820 pp = PP_GROUPLEADER(pp, cons); 6821 goto retry; 6822 } 6823 } 6824 6825 xt_sync(cpuset); 6826 sfmmu_mlist_exit(pml); 6827 return (PP_GENERIC_ATTR(save_pp)); 6828 } 6829 6830 /* 6831 * Get all the hardware dependent attributes for a page struct 6832 */ 6833 static cpuset_t 6834 sfmmu_pagesync(struct page *pp, struct sf_hment *sfhme, 6835 uint_t clearflag) 6836 { 6837 caddr_t addr; 6838 tte_t tte, ttemod; 6839 struct hme_blk *hmeblkp; 6840 int ret; 6841 sfmmu_t *sfmmup; 6842 cpuset_t cpuset; 6843 6844 ASSERT(pp != NULL); 6845 ASSERT(sfmmu_mlist_held(pp)); 6846 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 6847 (clearflag == HAT_SYNC_ZERORM)); 6848 6849 SFMMU_STAT(sf_pagesync); 6850 6851 CPUSET_ZERO(cpuset); 6852 6853 sfmmu_pagesync_retry: 6854 6855 sfmmu_copytte(&sfhme->hme_tte, &tte); 6856 if (TTE_IS_VALID(&tte)) { 6857 hmeblkp = sfmmu_hmetohblk(sfhme); 6858 sfmmup = hblktosfmmu(hmeblkp); 6859 addr = tte_to_vaddr(hmeblkp, tte); 6860 if (clearflag == HAT_SYNC_ZERORM) { 6861 ttemod = tte; 6862 TTE_CLR_RM(&ttemod); 6863 ret = sfmmu_modifytte_try(&tte, &ttemod, 6864 &sfhme->hme_tte); 6865 if (ret < 0) { 6866 /* 6867 * cas failed and the new value is not what 6868 * we want. 6869 */ 6870 goto sfmmu_pagesync_retry; 6871 } 6872 6873 if (ret > 0) { 6874 /* we win the cas */ 6875 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6876 cpuset = sfmmup->sfmmu_cpusran; 6877 } 6878 } 6879 6880 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6881 } 6882 return (cpuset); 6883 } 6884 6885 /* 6886 * Remove write permission from a mappings to a page, so that 6887 * we can detect the next modification of it. This requires modifying 6888 * the TTE then invalidating (demap) any TLB entry using that TTE. 6889 * This code is similar to sfmmu_pagesync(). 6890 */ 6891 static cpuset_t 6892 sfmmu_pageclrwrt(struct page *pp, struct sf_hment *sfhme) 6893 { 6894 caddr_t addr; 6895 tte_t tte; 6896 tte_t ttemod; 6897 struct hme_blk *hmeblkp; 6898 int ret; 6899 sfmmu_t *sfmmup; 6900 cpuset_t cpuset; 6901 6902 ASSERT(pp != NULL); 6903 ASSERT(sfmmu_mlist_held(pp)); 6904 6905 CPUSET_ZERO(cpuset); 6906 SFMMU_STAT(sf_clrwrt); 6907 6908 retry: 6909 6910 sfmmu_copytte(&sfhme->hme_tte, &tte); 6911 if (TTE_IS_VALID(&tte) && TTE_IS_WRITABLE(&tte)) { 6912 hmeblkp = sfmmu_hmetohblk(sfhme); 6913 6914 /* 6915 * xhat mappings should never be to a VMODSORT page. 6916 */ 6917 ASSERT(hmeblkp->hblk_xhat_bit == 0); 6918 6919 sfmmup = hblktosfmmu(hmeblkp); 6920 addr = tte_to_vaddr(hmeblkp, tte); 6921 6922 ttemod = tte; 6923 TTE_CLR_WRT(&ttemod); 6924 TTE_CLR_MOD(&ttemod); 6925 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6926 6927 /* 6928 * if cas failed and the new value is not what 6929 * we want retry 6930 */ 6931 if (ret < 0) 6932 goto retry; 6933 6934 /* we win the cas */ 6935 if (ret > 0) { 6936 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6937 cpuset = sfmmup->sfmmu_cpusran; 6938 } 6939 } 6940 6941 return (cpuset); 6942 } 6943 6944 /* 6945 * Walk all mappings of a page, removing write permission and clearing the 6946 * ref/mod bits. This code is similar to hat_pagesync() 6947 */ 6948 static void 6949 hat_page_clrwrt(page_t *pp) 6950 { 6951 struct sf_hment *sfhme; 6952 struct sf_hment *tmphme = NULL; 6953 kmutex_t *pml; 6954 cpuset_t cpuset; 6955 cpuset_t tset; 6956 int index; 6957 int cons; 6958 6959 CPUSET_ZERO(cpuset); 6960 6961 pml = sfmmu_mlist_enter(pp); 6962 index = PP_MAPINDEX(pp); 6963 cons = TTE8K; 6964 retry: 6965 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6966 tmphme = sfhme->hme_next; 6967 6968 /* 6969 * If we are looking for large mappings and this hme doesn't 6970 * reach the range we are seeking, just ignore its. 6971 */ 6972 6973 if (hme_size(sfhme) < cons) 6974 continue; 6975 6976 tset = sfmmu_pageclrwrt(pp, sfhme); 6977 CPUSET_OR(cpuset, tset); 6978 } 6979 6980 while (index) { 6981 index = index >> 1; 6982 cons++; 6983 if (index & 0x1) { 6984 /* Go to leading page */ 6985 pp = PP_GROUPLEADER(pp, cons); 6986 goto retry; 6987 } 6988 } 6989 6990 xt_sync(cpuset); 6991 sfmmu_mlist_exit(pml); 6992 } 6993 6994 /* 6995 * Set the given REF/MOD/RO bits for the given page. 6996 * For a vnode with a sorted v_pages list, we need to change 6997 * the attributes and the v_pages list together under page_vnode_mutex. 6998 */ 6999 void 7000 hat_page_setattr(page_t *pp, uint_t flag) 7001 { 7002 vnode_t *vp = pp->p_vnode; 7003 page_t **listp; 7004 kmutex_t *pmtx; 7005 kmutex_t *vphm = NULL; 7006 int noshuffle; 7007 7008 noshuffle = flag & P_NSH; 7009 flag &= ~P_NSH; 7010 7011 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7012 7013 /* 7014 * nothing to do if attribute already set 7015 */ 7016 if ((pp->p_nrm & flag) == flag) 7017 return; 7018 7019 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp) && 7020 !noshuffle) { 7021 vphm = page_vnode_mutex(vp); 7022 mutex_enter(vphm); 7023 } 7024 7025 pmtx = sfmmu_page_enter(pp); 7026 pp->p_nrm |= flag; 7027 sfmmu_page_exit(pmtx); 7028 7029 if (vphm != NULL) { 7030 /* 7031 * Some File Systems examine v_pages for NULL w/o 7032 * grabbing the vphm mutex. Must not let it become NULL when 7033 * pp is the only page on the list. 7034 */ 7035 if (pp->p_vpnext != pp) { 7036 page_vpsub(&vp->v_pages, pp); 7037 if (vp->v_pages != NULL) 7038 listp = &vp->v_pages->p_vpprev->p_vpnext; 7039 else 7040 listp = &vp->v_pages; 7041 page_vpadd(listp, pp); 7042 } 7043 mutex_exit(vphm); 7044 } 7045 } 7046 7047 void 7048 hat_page_clrattr(page_t *pp, uint_t flag) 7049 { 7050 vnode_t *vp = pp->p_vnode; 7051 kmutex_t *pmtx; 7052 7053 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7054 7055 pmtx = sfmmu_page_enter(pp); 7056 7057 /* 7058 * Caller is expected to hold page's io lock for VMODSORT to work 7059 * correctly with pvn_vplist_dirty() and pvn_getdirty() when mod 7060 * bit is cleared. 7061 * We don't have assert to avoid tripping some existing third party 7062 * code. The dirty page is moved back to top of the v_page list 7063 * after IO is done in pvn_write_done(). 7064 */ 7065 pp->p_nrm &= ~flag; 7066 sfmmu_page_exit(pmtx); 7067 7068 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 7069 7070 /* 7071 * VMODSORT works by removing write permissions and getting 7072 * a fault when a page is made dirty. At this point 7073 * we need to remove write permission from all mappings 7074 * to this page. 7075 */ 7076 hat_page_clrwrt(pp); 7077 } 7078 } 7079 7080 uint_t 7081 hat_page_getattr(page_t *pp, uint_t flag) 7082 { 7083 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7084 return ((uint_t)(pp->p_nrm & flag)); 7085 } 7086 7087 /* 7088 * DEBUG kernels: verify that a kernel va<->pa translation 7089 * is safe by checking the underlying page_t is in a page 7090 * relocation-safe state. 7091 */ 7092 #ifdef DEBUG 7093 void 7094 sfmmu_check_kpfn(pfn_t pfn) 7095 { 7096 page_t *pp; 7097 int index, cons; 7098 7099 if (hat_check_vtop == 0) 7100 return; 7101 7102 if (hat_kpr_enabled == 0 || kvseg.s_base == NULL || panicstr) 7103 return; 7104 7105 pp = page_numtopp_nolock(pfn); 7106 if (!pp) 7107 return; 7108 7109 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7110 return; 7111 7112 /* 7113 * Handed a large kernel page, we dig up the root page since we 7114 * know the root page might have the lock also. 7115 */ 7116 if (pp->p_szc != 0) { 7117 index = PP_MAPINDEX(pp); 7118 cons = TTE8K; 7119 again: 7120 while (index != 0) { 7121 index >>= 1; 7122 if (index != 0) 7123 cons++; 7124 if (index & 0x1) { 7125 pp = PP_GROUPLEADER(pp, cons); 7126 goto again; 7127 } 7128 } 7129 } 7130 7131 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7132 return; 7133 7134 /* 7135 * Pages need to be locked or allocated "permanent" (either from 7136 * static_arena arena or explicitly setting PG_NORELOC when calling 7137 * page_create_va()) for VA->PA translations to be valid. 7138 */ 7139 if (!PP_ISNORELOC(pp)) 7140 panic("Illegal VA->PA translation, pp 0x%p not permanent", pp); 7141 else 7142 panic("Illegal VA->PA translation, pp 0x%p not locked", pp); 7143 } 7144 #endif /* DEBUG */ 7145 7146 /* 7147 * Returns a page frame number for a given virtual address. 7148 * Returns PFN_INVALID to indicate an invalid mapping 7149 */ 7150 pfn_t 7151 hat_getpfnum(struct hat *hat, caddr_t addr) 7152 { 7153 pfn_t pfn; 7154 tte_t tte; 7155 7156 /* 7157 * We would like to 7158 * ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 7159 * but we can't because the iommu driver will call this 7160 * routine at interrupt time and it can't grab the as lock 7161 * or it will deadlock: A thread could have the as lock 7162 * and be waiting for io. The io can't complete 7163 * because the interrupt thread is blocked trying to grab 7164 * the as lock. 7165 */ 7166 7167 ASSERT(hat->sfmmu_xhat_provider == NULL); 7168 7169 if (hat == ksfmmup) { 7170 if (IS_KMEM_VA_LARGEPAGE(addr)) { 7171 ASSERT(segkmem_lpszc > 0); 7172 pfn = sfmmu_kvaszc2pfn(addr, segkmem_lpszc); 7173 if (pfn != PFN_INVALID) { 7174 sfmmu_check_kpfn(pfn); 7175 return (pfn); 7176 } 7177 } else if (segkpm && IS_KPM_ADDR(addr)) { 7178 return (sfmmu_kpm_vatopfn(addr)); 7179 } 7180 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7181 == PFN_SUSPENDED) { 7182 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7183 } 7184 sfmmu_check_kpfn(pfn); 7185 return (pfn); 7186 } else { 7187 return (sfmmu_uvatopfn(addr, hat)); 7188 } 7189 } 7190 7191 /* 7192 * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged. 7193 * Use hat_getpfnum(kas.a_hat, ...) instead. 7194 * 7195 * We'd like to return PFN_INVALID if the mappings have underlying page_t's 7196 * but can't right now due to the fact that some software has grown to use 7197 * this interface incorrectly. So for now when the interface is misused, 7198 * return a warning to the user that in the future it won't work in the 7199 * way they're abusing it, and carry on (after disabling page relocation). 7200 */ 7201 pfn_t 7202 hat_getkpfnum(caddr_t addr) 7203 { 7204 pfn_t pfn; 7205 tte_t tte; 7206 int badcaller = 0; 7207 extern int segkmem_reloc; 7208 7209 if (segkpm && IS_KPM_ADDR(addr)) { 7210 badcaller = 1; 7211 pfn = sfmmu_kpm_vatopfn(addr); 7212 } else { 7213 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7214 == PFN_SUSPENDED) { 7215 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7216 } 7217 badcaller = pf_is_memory(pfn); 7218 } 7219 7220 if (badcaller) { 7221 /* 7222 * We can't return PFN_INVALID or the caller may panic 7223 * or corrupt the system. The only alternative is to 7224 * disable page relocation at this point for all kernel 7225 * memory. This will impact any callers of page_relocate() 7226 * such as FMA or DR. 7227 * 7228 * RFE: Add junk here to spit out an ereport so the sysadmin 7229 * can be advised that he should upgrade his device driver 7230 * so that this doesn't happen. 7231 */ 7232 hat_getkpfnum_badcall(caller()); 7233 if (hat_kpr_enabled && segkmem_reloc) { 7234 hat_kpr_enabled = 0; 7235 segkmem_reloc = 0; 7236 cmn_err(CE_WARN, "Kernel Page Relocation is DISABLED"); 7237 } 7238 } 7239 return (pfn); 7240 } 7241 7242 pfn_t 7243 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup) 7244 { 7245 struct hmehash_bucket *hmebp; 7246 hmeblk_tag hblktag; 7247 int hmeshift, hashno = 1; 7248 struct hme_blk *hmeblkp = NULL; 7249 7250 struct sf_hment *sfhmep; 7251 tte_t tte; 7252 pfn_t pfn; 7253 7254 /* support for ISM */ 7255 ism_map_t *ism_map; 7256 ism_blk_t *ism_blkp; 7257 int i; 7258 sfmmu_t *ism_hatid = NULL; 7259 sfmmu_t *locked_hatid = NULL; 7260 7261 7262 ASSERT(sfmmup != ksfmmup); 7263 SFMMU_STAT(sf_user_vtop); 7264 /* 7265 * Set ism_hatid if vaddr falls in a ISM segment. 7266 */ 7267 ism_blkp = sfmmup->sfmmu_iblk; 7268 if (ism_blkp) { 7269 sfmmu_ismhat_enter(sfmmup, 0); 7270 locked_hatid = sfmmup; 7271 } 7272 while (ism_blkp && ism_hatid == NULL) { 7273 ism_map = ism_blkp->iblk_maps; 7274 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 7275 if (vaddr >= ism_start(ism_map[i]) && 7276 vaddr < ism_end(ism_map[i])) { 7277 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 7278 vaddr = (caddr_t)(vaddr - 7279 ism_start(ism_map[i])); 7280 break; 7281 } 7282 } 7283 ism_blkp = ism_blkp->iblk_next; 7284 } 7285 if (locked_hatid) { 7286 sfmmu_ismhat_exit(locked_hatid, 0); 7287 } 7288 7289 hblktag.htag_id = sfmmup; 7290 do { 7291 hmeshift = HME_HASH_SHIFT(hashno); 7292 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 7293 hblktag.htag_rehash = hashno; 7294 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 7295 7296 SFMMU_HASH_LOCK(hmebp); 7297 7298 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 7299 if (hmeblkp != NULL) { 7300 HBLKTOHME(sfhmep, hmeblkp, vaddr); 7301 sfmmu_copytte(&sfhmep->hme_tte, &tte); 7302 if (TTE_IS_VALID(&tte)) { 7303 pfn = TTE_TO_PFN(vaddr, &tte); 7304 } else { 7305 pfn = PFN_INVALID; 7306 } 7307 SFMMU_HASH_UNLOCK(hmebp); 7308 return (pfn); 7309 } 7310 SFMMU_HASH_UNLOCK(hmebp); 7311 hashno++; 7312 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 7313 return (PFN_INVALID); 7314 } 7315 7316 7317 /* 7318 * For compatability with AT&T and later optimizations 7319 */ 7320 /* ARGSUSED */ 7321 void 7322 hat_map(struct hat *hat, caddr_t addr, size_t len, uint_t flags) 7323 { 7324 ASSERT(hat != NULL); 7325 ASSERT(hat->sfmmu_xhat_provider == NULL); 7326 } 7327 7328 /* 7329 * Return the number of mappings to a particular page. 7330 * This number is an approximation of the number of 7331 * number of people sharing the page. 7332 */ 7333 ulong_t 7334 hat_page_getshare(page_t *pp) 7335 { 7336 page_t *spp = pp; /* start page */ 7337 kmutex_t *pml; 7338 ulong_t cnt; 7339 int index, sz = TTE64K; 7340 7341 /* 7342 * We need to grab the mlist lock to make sure any outstanding 7343 * load/unloads complete. Otherwise we could return zero 7344 * even though the unload(s) hasn't finished yet. 7345 */ 7346 pml = sfmmu_mlist_enter(spp); 7347 cnt = spp->p_share; 7348 7349 #ifdef VAC 7350 if (kpm_enable) 7351 cnt += spp->p_kpmref; 7352 #endif 7353 7354 /* 7355 * If we have any large mappings, we count the number of 7356 * mappings that this large page is part of. 7357 */ 7358 index = PP_MAPINDEX(spp); 7359 index >>= 1; 7360 while (index) { 7361 pp = PP_GROUPLEADER(spp, sz); 7362 if ((index & 0x1) && pp != spp) { 7363 cnt += pp->p_share; 7364 spp = pp; 7365 } 7366 index >>= 1; 7367 sz++; 7368 } 7369 sfmmu_mlist_exit(pml); 7370 return (cnt); 7371 } 7372 7373 /* 7374 * Unload all large mappings to the pp and reset the p_szc field of every 7375 * constituent page according to the remaining mappings. 7376 * 7377 * pp must be locked SE_EXCL. Even though no other constituent pages are 7378 * locked it's legal to unload the large mappings to the pp because all 7379 * constituent pages of large locked mappings have to be locked SE_SHARED. 7380 * This means if we have SE_EXCL lock on one of constituent pages none of the 7381 * large mappings to pp are locked. 7382 * 7383 * Decrease p_szc field starting from the last constituent page and ending 7384 * with the root page. This method is used because other threads rely on the 7385 * root's p_szc to find the lock to syncronize on. After a root page_t's p_szc 7386 * is demoted then other threads will succeed in sfmmu_mlspl_enter(). This 7387 * ensures that p_szc changes of the constituent pages appears atomic for all 7388 * threads that use sfmmu_mlspl_enter() to examine p_szc field. 7389 * 7390 * This mechanism is only used for file system pages where it's not always 7391 * possible to get SE_EXCL locks on all constituent pages to demote the size 7392 * code (as is done for anonymous or kernel large pages). 7393 * 7394 * See more comments in front of sfmmu_mlspl_enter(). 7395 */ 7396 void 7397 hat_page_demote(page_t *pp) 7398 { 7399 int index; 7400 int sz; 7401 cpuset_t cpuset; 7402 int sync = 0; 7403 page_t *rootpp; 7404 struct sf_hment *sfhme; 7405 struct sf_hment *tmphme = NULL; 7406 struct hme_blk *hmeblkp; 7407 uint_t pszc; 7408 page_t *lastpp; 7409 cpuset_t tset; 7410 pgcnt_t npgs; 7411 kmutex_t *pml; 7412 kmutex_t *pmtx = NULL; 7413 7414 ASSERT(PAGE_EXCL(pp)); 7415 ASSERT(!PP_ISFREE(pp)); 7416 ASSERT(page_szc_lock_assert(pp)); 7417 pml = sfmmu_mlist_enter(pp); 7418 7419 pszc = pp->p_szc; 7420 if (pszc == 0) { 7421 goto out; 7422 } 7423 7424 index = PP_MAPINDEX(pp) >> 1; 7425 7426 if (index) { 7427 CPUSET_ZERO(cpuset); 7428 sz = TTE64K; 7429 sync = 1; 7430 } 7431 7432 while (index) { 7433 if (!(index & 0x1)) { 7434 index >>= 1; 7435 sz++; 7436 continue; 7437 } 7438 ASSERT(sz <= pszc); 7439 rootpp = PP_GROUPLEADER(pp, sz); 7440 for (sfhme = rootpp->p_mapping; sfhme; sfhme = tmphme) { 7441 tmphme = sfhme->hme_next; 7442 hmeblkp = sfmmu_hmetohblk(sfhme); 7443 if (hme_size(sfhme) != sz) { 7444 continue; 7445 } 7446 if (hmeblkp->hblk_xhat_bit) { 7447 cmn_err(CE_PANIC, 7448 "hat_page_demote: xhat hmeblk"); 7449 } 7450 tset = sfmmu_pageunload(rootpp, sfhme, sz); 7451 CPUSET_OR(cpuset, tset); 7452 } 7453 if (index >>= 1) { 7454 sz++; 7455 } 7456 } 7457 7458 ASSERT(!PP_ISMAPPED_LARGE(pp)); 7459 7460 if (sync) { 7461 xt_sync(cpuset); 7462 #ifdef VAC 7463 if (PP_ISTNC(pp)) { 7464 conv_tnc(rootpp, sz); 7465 } 7466 #endif /* VAC */ 7467 } 7468 7469 pmtx = sfmmu_page_enter(pp); 7470 7471 ASSERT(pp->p_szc == pszc); 7472 rootpp = PP_PAGEROOT(pp); 7473 ASSERT(rootpp->p_szc == pszc); 7474 lastpp = PP_PAGENEXT_N(rootpp, TTEPAGES(pszc) - 1); 7475 7476 while (lastpp != rootpp) { 7477 sz = PP_MAPINDEX(lastpp) ? fnd_mapping_sz(lastpp) : 0; 7478 ASSERT(sz < pszc); 7479 npgs = (sz == 0) ? 1 : TTEPAGES(sz); 7480 ASSERT(P2PHASE(lastpp->p_pagenum, npgs) == npgs - 1); 7481 while (--npgs > 0) { 7482 lastpp->p_szc = (uchar_t)sz; 7483 lastpp = PP_PAGEPREV(lastpp); 7484 } 7485 if (sz) { 7486 /* 7487 * make sure before current root's pszc 7488 * is updated all updates to constituent pages pszc 7489 * fields are globally visible. 7490 */ 7491 membar_producer(); 7492 } 7493 lastpp->p_szc = sz; 7494 ASSERT(IS_P2ALIGNED(lastpp->p_pagenum, TTEPAGES(sz))); 7495 if (lastpp != rootpp) { 7496 lastpp = PP_PAGEPREV(lastpp); 7497 } 7498 } 7499 if (sz == 0) { 7500 /* the loop above doesn't cover this case */ 7501 rootpp->p_szc = 0; 7502 } 7503 out: 7504 ASSERT(pp->p_szc == 0); 7505 if (pmtx != NULL) { 7506 sfmmu_page_exit(pmtx); 7507 } 7508 sfmmu_mlist_exit(pml); 7509 } 7510 7511 /* 7512 * Refresh the HAT ismttecnt[] element for size szc. 7513 * Caller must have set ISM busy flag to prevent mapping 7514 * lists from changing while we're traversing them. 7515 */ 7516 pgcnt_t 7517 ism_tsb_entries(sfmmu_t *sfmmup, int szc) 7518 { 7519 ism_blk_t *ism_blkp = sfmmup->sfmmu_iblk; 7520 ism_map_t *ism_map; 7521 pgcnt_t npgs = 0; 7522 int j; 7523 7524 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 7525 for (; ism_blkp != NULL; ism_blkp = ism_blkp->iblk_next) { 7526 ism_map = ism_blkp->iblk_maps; 7527 for (j = 0; ism_map[j].imap_ismhat && j < ISM_MAP_SLOTS; j++) 7528 npgs += ism_map[j].imap_ismhat->sfmmu_ttecnt[szc]; 7529 } 7530 sfmmup->sfmmu_ismttecnt[szc] = npgs; 7531 return (npgs); 7532 } 7533 7534 /* 7535 * Yield the memory claim requirement for an address space. 7536 * 7537 * This is currently implemented as the number of bytes that have active 7538 * hardware translations that have page structures. Therefore, it can 7539 * underestimate the traditional resident set size, eg, if the 7540 * physical page is present and the hardware translation is missing; 7541 * and it can overestimate the rss, eg, if there are active 7542 * translations to a frame buffer with page structs. 7543 * Also, it does not take sharing into account. 7544 * 7545 * Note that we don't acquire locks here since this function is most often 7546 * called from the clock thread. 7547 */ 7548 size_t 7549 hat_get_mapped_size(struct hat *hat) 7550 { 7551 size_t assize = 0; 7552 int i; 7553 7554 if (hat == NULL) 7555 return (0); 7556 7557 ASSERT(hat->sfmmu_xhat_provider == NULL); 7558 7559 for (i = 0; i < mmu_page_sizes; i++) 7560 assize += (pgcnt_t)hat->sfmmu_ttecnt[i] * TTEBYTES(i); 7561 7562 if (hat->sfmmu_iblk == NULL) 7563 return (assize); 7564 7565 for (i = 0; i < mmu_page_sizes; i++) 7566 assize += (pgcnt_t)hat->sfmmu_ismttecnt[i] * TTEBYTES(i); 7567 7568 return (assize); 7569 } 7570 7571 int 7572 hat_stats_enable(struct hat *hat) 7573 { 7574 hatlock_t *hatlockp; 7575 7576 ASSERT(hat->sfmmu_xhat_provider == NULL); 7577 7578 hatlockp = sfmmu_hat_enter(hat); 7579 hat->sfmmu_rmstat++; 7580 sfmmu_hat_exit(hatlockp); 7581 return (1); 7582 } 7583 7584 void 7585 hat_stats_disable(struct hat *hat) 7586 { 7587 hatlock_t *hatlockp; 7588 7589 ASSERT(hat->sfmmu_xhat_provider == NULL); 7590 7591 hatlockp = sfmmu_hat_enter(hat); 7592 hat->sfmmu_rmstat--; 7593 sfmmu_hat_exit(hatlockp); 7594 } 7595 7596 /* 7597 * Routines for entering or removing ourselves from the 7598 * ism_hat's mapping list. 7599 */ 7600 static void 7601 iment_add(struct ism_ment *iment, struct hat *ism_hat) 7602 { 7603 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7604 7605 iment->iment_prev = NULL; 7606 iment->iment_next = ism_hat->sfmmu_iment; 7607 if (ism_hat->sfmmu_iment) { 7608 ism_hat->sfmmu_iment->iment_prev = iment; 7609 } 7610 ism_hat->sfmmu_iment = iment; 7611 } 7612 7613 static void 7614 iment_sub(struct ism_ment *iment, struct hat *ism_hat) 7615 { 7616 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7617 7618 if (ism_hat->sfmmu_iment == NULL) { 7619 panic("ism map entry remove - no entries"); 7620 } 7621 7622 if (iment->iment_prev) { 7623 ASSERT(ism_hat->sfmmu_iment != iment); 7624 iment->iment_prev->iment_next = iment->iment_next; 7625 } else { 7626 ASSERT(ism_hat->sfmmu_iment == iment); 7627 ism_hat->sfmmu_iment = iment->iment_next; 7628 } 7629 7630 if (iment->iment_next) { 7631 iment->iment_next->iment_prev = iment->iment_prev; 7632 } 7633 7634 /* 7635 * zero out the entry 7636 */ 7637 iment->iment_next = NULL; 7638 iment->iment_prev = NULL; 7639 iment->iment_hat = NULL; 7640 } 7641 7642 /* 7643 * Hat_share()/unshare() return an (non-zero) error 7644 * when saddr and daddr are not properly aligned. 7645 * 7646 * The top level mapping element determines the alignment 7647 * requirement for saddr and daddr, depending on different 7648 * architectures. 7649 * 7650 * When hat_share()/unshare() are not supported, 7651 * HATOP_SHARE()/UNSHARE() return 0 7652 */ 7653 int 7654 hat_share(struct hat *sfmmup, caddr_t addr, 7655 struct hat *ism_hatid, caddr_t sptaddr, size_t len, uint_t ismszc) 7656 { 7657 ism_blk_t *ism_blkp; 7658 ism_blk_t *new_iblk; 7659 ism_map_t *ism_map; 7660 ism_ment_t *ism_ment; 7661 int i, added; 7662 hatlock_t *hatlockp; 7663 int reload_mmu = 0; 7664 uint_t ismshift = page_get_shift(ismszc); 7665 size_t ismpgsz = page_get_pagesize(ismszc); 7666 uint_t ismmask = (uint_t)ismpgsz - 1; 7667 size_t sh_size = ISM_SHIFT(ismshift, len); 7668 ushort_t ismhatflag; 7669 7670 #ifdef DEBUG 7671 caddr_t eaddr = addr + len; 7672 #endif /* DEBUG */ 7673 7674 ASSERT(ism_hatid != NULL && sfmmup != NULL); 7675 ASSERT(sptaddr == ISMID_STARTADDR); 7676 /* 7677 * Check the alignment. 7678 */ 7679 if (!ISM_ALIGNED(ismshift, addr) || !ISM_ALIGNED(ismshift, sptaddr)) 7680 return (EINVAL); 7681 7682 /* 7683 * Check size alignment. 7684 */ 7685 if (!ISM_ALIGNED(ismshift, len)) 7686 return (EINVAL); 7687 7688 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 7689 7690 /* 7691 * Allocate ism_ment for the ism_hat's mapping list, and an 7692 * ism map blk in case we need one. We must do our 7693 * allocations before acquiring locks to prevent a deadlock 7694 * in the kmem allocator on the mapping list lock. 7695 */ 7696 new_iblk = kmem_cache_alloc(ism_blk_cache, KM_SLEEP); 7697 ism_ment = kmem_cache_alloc(ism_ment_cache, KM_SLEEP); 7698 7699 /* 7700 * Serialize ISM mappings with the ISM busy flag, and also the 7701 * trap handlers. 7702 */ 7703 sfmmu_ismhat_enter(sfmmup, 0); 7704 7705 /* 7706 * Allocate an ism map blk if necessary. 7707 */ 7708 if (sfmmup->sfmmu_iblk == NULL) { 7709 sfmmup->sfmmu_iblk = new_iblk; 7710 bzero(new_iblk, sizeof (*new_iblk)); 7711 new_iblk->iblk_nextpa = (uint64_t)-1; 7712 membar_stst(); /* make sure next ptr visible to all CPUs */ 7713 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk); 7714 reload_mmu = 1; 7715 new_iblk = NULL; 7716 } 7717 7718 #ifdef DEBUG 7719 /* 7720 * Make sure mapping does not already exist. 7721 */ 7722 ism_blkp = sfmmup->sfmmu_iblk; 7723 while (ism_blkp) { 7724 ism_map = ism_blkp->iblk_maps; 7725 for (i = 0; i < ISM_MAP_SLOTS && ism_map[i].imap_ismhat; i++) { 7726 if ((addr >= ism_start(ism_map[i]) && 7727 addr < ism_end(ism_map[i])) || 7728 eaddr > ism_start(ism_map[i]) && 7729 eaddr <= ism_end(ism_map[i])) { 7730 panic("sfmmu_share: Already mapped!"); 7731 } 7732 } 7733 ism_blkp = ism_blkp->iblk_next; 7734 } 7735 #endif /* DEBUG */ 7736 7737 ASSERT(ismszc >= TTE4M); 7738 if (ismszc == TTE4M) { 7739 ismhatflag = HAT_4M_FLAG; 7740 } else if (ismszc == TTE32M) { 7741 ismhatflag = HAT_32M_FLAG; 7742 } else if (ismszc == TTE256M) { 7743 ismhatflag = HAT_256M_FLAG; 7744 } 7745 /* 7746 * Add mapping to first available mapping slot. 7747 */ 7748 ism_blkp = sfmmup->sfmmu_iblk; 7749 added = 0; 7750 while (!added) { 7751 ism_map = ism_blkp->iblk_maps; 7752 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7753 if (ism_map[i].imap_ismhat == NULL) { 7754 7755 ism_map[i].imap_ismhat = ism_hatid; 7756 ism_map[i].imap_vb_shift = (ushort_t)ismshift; 7757 ism_map[i].imap_hatflags = ismhatflag; 7758 ism_map[i].imap_sz_mask = ismmask; 7759 /* 7760 * imap_seg is checked in ISM_CHECK to see if 7761 * non-NULL, then other info assumed valid. 7762 */ 7763 membar_stst(); 7764 ism_map[i].imap_seg = (uintptr_t)addr | sh_size; 7765 ism_map[i].imap_ment = ism_ment; 7766 7767 /* 7768 * Now add ourselves to the ism_hat's 7769 * mapping list. 7770 */ 7771 ism_ment->iment_hat = sfmmup; 7772 ism_ment->iment_base_va = addr; 7773 ism_hatid->sfmmu_ismhat = 1; 7774 ism_hatid->sfmmu_flags = 0; 7775 mutex_enter(&ism_mlist_lock); 7776 iment_add(ism_ment, ism_hatid); 7777 mutex_exit(&ism_mlist_lock); 7778 added = 1; 7779 break; 7780 } 7781 } 7782 if (!added && ism_blkp->iblk_next == NULL) { 7783 ism_blkp->iblk_next = new_iblk; 7784 new_iblk = NULL; 7785 bzero(ism_blkp->iblk_next, 7786 sizeof (*ism_blkp->iblk_next)); 7787 ism_blkp->iblk_next->iblk_nextpa = (uint64_t)-1; 7788 membar_stst(); 7789 ism_blkp->iblk_nextpa = 7790 va_to_pa((caddr_t)ism_blkp->iblk_next); 7791 } 7792 ism_blkp = ism_blkp->iblk_next; 7793 } 7794 7795 /* 7796 * Update our counters for this sfmmup's ism mappings. 7797 */ 7798 for (i = 0; i <= ismszc; i++) { 7799 if (!(disable_ism_large_pages & (1 << i))) 7800 (void) ism_tsb_entries(sfmmup, i); 7801 } 7802 7803 hatlockp = sfmmu_hat_enter(sfmmup); 7804 7805 /* 7806 * For ISM and DISM we do not support 512K pages, so we only 7807 * only search the 4M and 8K/64K hashes for 4 pagesize cpus, and search 7808 * the 256M or 32M, and 4M and 8K/64K hashes for 6 pagesize cpus. 7809 */ 7810 ASSERT((disable_ism_large_pages & (1 << TTE512K)) != 0); 7811 7812 if (ismszc > TTE4M && !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) 7813 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7814 7815 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) 7816 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 7817 7818 /* 7819 * If we updated the ismblkpa for this HAT or we need 7820 * to start searching the 256M or 32M or 4M hash, we must 7821 * make sure all CPUs running this process reload their 7822 * tsbmiss area. Otherwise they will fail to load the mappings 7823 * in the tsbmiss handler and will loop calling pagefault(). 7824 */ 7825 switch (ismszc) { 7826 case TTE256M: 7827 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 7828 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 7829 sfmmu_sync_mmustate(sfmmup); 7830 } 7831 break; 7832 case TTE32M: 7833 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 7834 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 7835 sfmmu_sync_mmustate(sfmmup); 7836 } 7837 break; 7838 case TTE4M: 7839 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 7840 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7841 sfmmu_sync_mmustate(sfmmup); 7842 } 7843 break; 7844 default: 7845 break; 7846 } 7847 7848 /* 7849 * Now we can drop the locks. 7850 */ 7851 sfmmu_ismhat_exit(sfmmup, 1); 7852 sfmmu_hat_exit(hatlockp); 7853 7854 /* 7855 * Free up ismblk if we didn't use it. 7856 */ 7857 if (new_iblk != NULL) 7858 kmem_cache_free(ism_blk_cache, new_iblk); 7859 7860 /* 7861 * Check TSB and TLB page sizes. 7862 */ 7863 sfmmu_check_page_sizes(sfmmup, 1); 7864 7865 return (0); 7866 } 7867 7868 /* 7869 * hat_unshare removes exactly one ism_map from 7870 * this process's as. It expects multiple calls 7871 * to hat_unshare for multiple shm segments. 7872 */ 7873 void 7874 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) 7875 { 7876 ism_map_t *ism_map; 7877 ism_ment_t *free_ment = NULL; 7878 ism_blk_t *ism_blkp; 7879 struct hat *ism_hatid; 7880 int found, i; 7881 hatlock_t *hatlockp; 7882 struct tsb_info *tsbinfo; 7883 uint_t ismshift = page_get_shift(ismszc); 7884 size_t sh_size = ISM_SHIFT(ismshift, len); 7885 7886 ASSERT(ISM_ALIGNED(ismshift, addr)); 7887 ASSERT(ISM_ALIGNED(ismshift, len)); 7888 ASSERT(sfmmup != NULL); 7889 ASSERT(sfmmup != ksfmmup); 7890 7891 if (sfmmup->sfmmu_xhat_provider) { 7892 XHAT_UNSHARE(sfmmup, addr, len); 7893 return; 7894 } else { 7895 /* 7896 * This must be a CPU HAT. If the address space has 7897 * XHATs attached, inform all XHATs that ISM segment 7898 * is going away 7899 */ 7900 ASSERT(sfmmup->sfmmu_as != NULL); 7901 if (sfmmup->sfmmu_as->a_xhat != NULL) 7902 xhat_unshare_all(sfmmup->sfmmu_as, addr, len); 7903 } 7904 7905 /* 7906 * Make sure that during the entire time ISM mappings are removed, 7907 * the trap handlers serialize behind us, and that no one else 7908 * can be mucking with ISM mappings. This also lets us get away 7909 * with not doing expensive cross calls to flush the TLB -- we 7910 * just discard the context, flush the entire TSB, and call it 7911 * a day. 7912 */ 7913 sfmmu_ismhat_enter(sfmmup, 0); 7914 7915 /* 7916 * Remove the mapping. 7917 * 7918 * We can't have any holes in the ism map. 7919 * The tsb miss code while searching the ism map will 7920 * stop on an empty map slot. So we must move 7921 * everyone past the hole up 1 if any. 7922 * 7923 * Also empty ism map blks are not freed until the 7924 * process exits. This is to prevent a MT race condition 7925 * between sfmmu_unshare() and sfmmu_tsbmiss_exception(). 7926 */ 7927 found = 0; 7928 ism_blkp = sfmmup->sfmmu_iblk; 7929 while (!found && ism_blkp) { 7930 ism_map = ism_blkp->iblk_maps; 7931 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7932 if (addr == ism_start(ism_map[i]) && 7933 sh_size == (size_t)(ism_size(ism_map[i]))) { 7934 found = 1; 7935 break; 7936 } 7937 } 7938 if (!found) 7939 ism_blkp = ism_blkp->iblk_next; 7940 } 7941 7942 if (found) { 7943 ism_hatid = ism_map[i].imap_ismhat; 7944 ASSERT(ism_hatid != NULL); 7945 ASSERT(ism_hatid->sfmmu_ismhat == 1); 7946 7947 /* 7948 * First remove ourselves from the ism mapping list. 7949 */ 7950 mutex_enter(&ism_mlist_lock); 7951 iment_sub(ism_map[i].imap_ment, ism_hatid); 7952 mutex_exit(&ism_mlist_lock); 7953 free_ment = ism_map[i].imap_ment; 7954 7955 /* 7956 * Now gurantee that any other cpu 7957 * that tries to process an ISM miss 7958 * will go to tl=0. 7959 */ 7960 hatlockp = sfmmu_hat_enter(sfmmup); 7961 7962 sfmmu_invalidate_ctx(sfmmup); 7963 7964 sfmmu_hat_exit(hatlockp); 7965 7966 /* 7967 * We delete the ism map by copying 7968 * the next map over the current one. 7969 * We will take the next one in the maps 7970 * array or from the next ism_blk. 7971 */ 7972 while (ism_blkp) { 7973 ism_map = ism_blkp->iblk_maps; 7974 while (i < (ISM_MAP_SLOTS - 1)) { 7975 ism_map[i] = ism_map[i + 1]; 7976 i++; 7977 } 7978 /* i == (ISM_MAP_SLOTS - 1) */ 7979 ism_blkp = ism_blkp->iblk_next; 7980 if (ism_blkp) { 7981 ism_map[i] = ism_blkp->iblk_maps[0]; 7982 i = 0; 7983 } else { 7984 ism_map[i].imap_seg = 0; 7985 ism_map[i].imap_vb_shift = 0; 7986 ism_map[i].imap_hatflags = 0; 7987 ism_map[i].imap_sz_mask = 0; 7988 ism_map[i].imap_ismhat = NULL; 7989 ism_map[i].imap_ment = NULL; 7990 } 7991 } 7992 7993 /* 7994 * Now flush entire TSB for the process, since 7995 * demapping page by page can be too expensive. 7996 * We don't have to flush the TLB here anymore 7997 * since we switch to a new TLB ctx instead. 7998 * Also, there is no need to flush if the process 7999 * is exiting since the TSB will be freed later. 8000 */ 8001 if (!sfmmup->sfmmu_free) { 8002 hatlockp = sfmmu_hat_enter(sfmmup); 8003 for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL; 8004 tsbinfo = tsbinfo->tsb_next) { 8005 if (tsbinfo->tsb_flags & TSB_SWAPPED) 8006 continue; 8007 sfmmu_inv_tsb(tsbinfo->tsb_va, 8008 TSB_BYTES(tsbinfo->tsb_szc)); 8009 } 8010 sfmmu_hat_exit(hatlockp); 8011 } 8012 } 8013 8014 /* 8015 * Update our counters for this sfmmup's ism mappings. 8016 */ 8017 for (i = 0; i <= ismszc; i++) { 8018 if (!(disable_ism_large_pages & (1 << i))) 8019 (void) ism_tsb_entries(sfmmup, i); 8020 } 8021 8022 sfmmu_ismhat_exit(sfmmup, 0); 8023 8024 /* 8025 * We must do our freeing here after dropping locks 8026 * to prevent a deadlock in the kmem allocator on the 8027 * mapping list lock. 8028 */ 8029 if (free_ment != NULL) 8030 kmem_cache_free(ism_ment_cache, free_ment); 8031 8032 /* 8033 * Check TSB and TLB page sizes if the process isn't exiting. 8034 */ 8035 if (!sfmmup->sfmmu_free) 8036 sfmmu_check_page_sizes(sfmmup, 0); 8037 } 8038 8039 /* ARGSUSED */ 8040 static int 8041 sfmmu_idcache_constructor(void *buf, void *cdrarg, int kmflags) 8042 { 8043 /* void *buf is sfmmu_t pointer */ 8044 return (0); 8045 } 8046 8047 /* ARGSUSED */ 8048 static void 8049 sfmmu_idcache_destructor(void *buf, void *cdrarg) 8050 { 8051 /* void *buf is sfmmu_t pointer */ 8052 } 8053 8054 /* 8055 * setup kmem hmeblks by bzeroing all members and initializing the nextpa 8056 * field to be the pa of this hmeblk 8057 */ 8058 /* ARGSUSED */ 8059 static int 8060 sfmmu_hblkcache_constructor(void *buf, void *cdrarg, int kmflags) 8061 { 8062 struct hme_blk *hmeblkp; 8063 8064 bzero(buf, (size_t)cdrarg); 8065 hmeblkp = (struct hme_blk *)buf; 8066 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); 8067 8068 #ifdef HBLK_TRACE 8069 mutex_init(&hmeblkp->hblk_audit_lock, NULL, MUTEX_DEFAULT, NULL); 8070 #endif /* HBLK_TRACE */ 8071 8072 return (0); 8073 } 8074 8075 /* ARGSUSED */ 8076 static void 8077 sfmmu_hblkcache_destructor(void *buf, void *cdrarg) 8078 { 8079 8080 #ifdef HBLK_TRACE 8081 8082 struct hme_blk *hmeblkp; 8083 8084 hmeblkp = (struct hme_blk *)buf; 8085 mutex_destroy(&hmeblkp->hblk_audit_lock); 8086 8087 #endif /* HBLK_TRACE */ 8088 } 8089 8090 #define SFMMU_CACHE_RECLAIM_SCAN_RATIO 8 8091 static int sfmmu_cache_reclaim_scan_ratio = SFMMU_CACHE_RECLAIM_SCAN_RATIO; 8092 /* 8093 * The kmem allocator will callback into our reclaim routine when the system 8094 * is running low in memory. We traverse the hash and free up all unused but 8095 * still cached hme_blks. We also traverse the free list and free them up 8096 * as well. 8097 */ 8098 /*ARGSUSED*/ 8099 static void 8100 sfmmu_hblkcache_reclaim(void *cdrarg) 8101 { 8102 int i; 8103 uint64_t hblkpa, prevpa, nx_pa; 8104 struct hmehash_bucket *hmebp; 8105 struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL; 8106 static struct hmehash_bucket *uhmehash_reclaim_hand; 8107 static struct hmehash_bucket *khmehash_reclaim_hand; 8108 struct hme_blk *list = NULL; 8109 8110 hmebp = uhmehash_reclaim_hand; 8111 if (hmebp == NULL || hmebp > &uhme_hash[UHMEHASH_SZ]) 8112 uhmehash_reclaim_hand = hmebp = uhme_hash; 8113 uhmehash_reclaim_hand += UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8114 8115 for (i = UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8116 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8117 hmeblkp = hmebp->hmeblkp; 8118 hblkpa = hmebp->hmeh_nextpa; 8119 prevpa = 0; 8120 pr_hblk = NULL; 8121 while (hmeblkp) { 8122 nx_hblk = hmeblkp->hblk_next; 8123 nx_pa = hmeblkp->hblk_nextpa; 8124 if (!hmeblkp->hblk_vcnt && 8125 !hmeblkp->hblk_hmecnt) { 8126 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8127 prevpa, pr_hblk); 8128 sfmmu_hblk_free(hmebp, hmeblkp, 8129 hblkpa, &list); 8130 } else { 8131 pr_hblk = hmeblkp; 8132 prevpa = hblkpa; 8133 } 8134 hmeblkp = nx_hblk; 8135 hblkpa = nx_pa; 8136 } 8137 SFMMU_HASH_UNLOCK(hmebp); 8138 } 8139 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 8140 hmebp = uhme_hash; 8141 } 8142 8143 hmebp = khmehash_reclaim_hand; 8144 if (hmebp == NULL || hmebp > &khme_hash[KHMEHASH_SZ]) 8145 khmehash_reclaim_hand = hmebp = khme_hash; 8146 khmehash_reclaim_hand += KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8147 8148 for (i = KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8149 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8150 hmeblkp = hmebp->hmeblkp; 8151 hblkpa = hmebp->hmeh_nextpa; 8152 prevpa = 0; 8153 pr_hblk = NULL; 8154 while (hmeblkp) { 8155 nx_hblk = hmeblkp->hblk_next; 8156 nx_pa = hmeblkp->hblk_nextpa; 8157 if (!hmeblkp->hblk_vcnt && 8158 !hmeblkp->hblk_hmecnt) { 8159 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8160 prevpa, pr_hblk); 8161 sfmmu_hblk_free(hmebp, hmeblkp, 8162 hblkpa, &list); 8163 } else { 8164 pr_hblk = hmeblkp; 8165 prevpa = hblkpa; 8166 } 8167 hmeblkp = nx_hblk; 8168 hblkpa = nx_pa; 8169 } 8170 SFMMU_HASH_UNLOCK(hmebp); 8171 } 8172 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 8173 hmebp = khme_hash; 8174 } 8175 sfmmu_hblks_list_purge(&list); 8176 } 8177 8178 /* 8179 * sfmmu_get_ppvcolor should become a vm_machdep or hatop interface. 8180 * same goes for sfmmu_get_addrvcolor(). 8181 * 8182 * This function will return the virtual color for the specified page. The 8183 * virtual color corresponds to this page current mapping or its last mapping. 8184 * It is used by memory allocators to choose addresses with the correct 8185 * alignment so vac consistency is automatically maintained. If the page 8186 * has no color it returns -1. 8187 */ 8188 /*ARGSUSED*/ 8189 int 8190 sfmmu_get_ppvcolor(struct page *pp) 8191 { 8192 #ifdef VAC 8193 int color; 8194 8195 if (!(cache & CACHE_VAC) || PP_NEWPAGE(pp)) { 8196 return (-1); 8197 } 8198 color = PP_GET_VCOLOR(pp); 8199 ASSERT(color < mmu_btop(shm_alignment)); 8200 return (color); 8201 #else 8202 return (-1); 8203 #endif /* VAC */ 8204 } 8205 8206 /* 8207 * This function will return the desired alignment for vac consistency 8208 * (vac color) given a virtual address. If no vac is present it returns -1. 8209 */ 8210 /*ARGSUSED*/ 8211 int 8212 sfmmu_get_addrvcolor(caddr_t vaddr) 8213 { 8214 #ifdef VAC 8215 if (cache & CACHE_VAC) { 8216 return (addr_to_vcolor(vaddr)); 8217 } else { 8218 return (-1); 8219 } 8220 #else 8221 return (-1); 8222 #endif /* VAC */ 8223 } 8224 8225 #ifdef VAC 8226 /* 8227 * Check for conflicts. 8228 * A conflict exists if the new and existent mappings do not match in 8229 * their "shm_alignment fields. If conflicts exist, the existant mappings 8230 * are flushed unless one of them is locked. If one of them is locked, then 8231 * the mappings are flushed and converted to non-cacheable mappings. 8232 */ 8233 static void 8234 sfmmu_vac_conflict(struct hat *hat, caddr_t addr, page_t *pp) 8235 { 8236 struct hat *tmphat; 8237 struct sf_hment *sfhmep, *tmphme = NULL; 8238 struct hme_blk *hmeblkp; 8239 int vcolor; 8240 tte_t tte; 8241 8242 ASSERT(sfmmu_mlist_held(pp)); 8243 ASSERT(!PP_ISNC(pp)); /* page better be cacheable */ 8244 8245 vcolor = addr_to_vcolor(addr); 8246 if (PP_NEWPAGE(pp)) { 8247 PP_SET_VCOLOR(pp, vcolor); 8248 return; 8249 } 8250 8251 if (PP_GET_VCOLOR(pp) == vcolor) { 8252 return; 8253 } 8254 8255 if (!PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp)) { 8256 /* 8257 * Previous user of page had a different color 8258 * but since there are no current users 8259 * we just flush the cache and change the color. 8260 */ 8261 SFMMU_STAT(sf_pgcolor_conflict); 8262 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8263 PP_SET_VCOLOR(pp, vcolor); 8264 return; 8265 } 8266 8267 /* 8268 * If we get here we have a vac conflict with a current 8269 * mapping. VAC conflict policy is as follows. 8270 * - The default is to unload the other mappings unless: 8271 * - If we have a large mapping we uncache the page. 8272 * We need to uncache the rest of the large page too. 8273 * - If any of the mappings are locked we uncache the page. 8274 * - If the requested mapping is inconsistent 8275 * with another mapping and that mapping 8276 * is in the same address space we have to 8277 * make it non-cached. The default thing 8278 * to do is unload the inconsistent mapping 8279 * but if they are in the same address space 8280 * we run the risk of unmapping the pc or the 8281 * stack which we will use as we return to the user, 8282 * in which case we can then fault on the thing 8283 * we just unloaded and get into an infinite loop. 8284 */ 8285 if (PP_ISMAPPED_LARGE(pp)) { 8286 int sz; 8287 8288 /* 8289 * Existing mapping is for big pages. We don't unload 8290 * existing big mappings to satisfy new mappings. 8291 * Always convert all mappings to TNC. 8292 */ 8293 sz = fnd_mapping_sz(pp); 8294 pp = PP_GROUPLEADER(pp, sz); 8295 SFMMU_STAT_ADD(sf_uncache_conflict, TTEPAGES(sz)); 8296 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 8297 TTEPAGES(sz)); 8298 8299 return; 8300 } 8301 8302 /* 8303 * check if any mapping is in same as or if it is locked 8304 * since in that case we need to uncache. 8305 */ 8306 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8307 tmphme = sfhmep->hme_next; 8308 hmeblkp = sfmmu_hmetohblk(sfhmep); 8309 if (hmeblkp->hblk_xhat_bit) 8310 continue; 8311 tmphat = hblktosfmmu(hmeblkp); 8312 sfmmu_copytte(&sfhmep->hme_tte, &tte); 8313 ASSERT(TTE_IS_VALID(&tte)); 8314 if ((tmphat == hat) || hmeblkp->hblk_lckcnt) { 8315 /* 8316 * We have an uncache conflict 8317 */ 8318 SFMMU_STAT(sf_uncache_conflict); 8319 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1); 8320 return; 8321 } 8322 } 8323 8324 /* 8325 * We have an unload conflict 8326 * We have already checked for LARGE mappings, therefore 8327 * the remaining mapping(s) must be TTE8K. 8328 */ 8329 SFMMU_STAT(sf_unload_conflict); 8330 8331 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8332 tmphme = sfhmep->hme_next; 8333 hmeblkp = sfmmu_hmetohblk(sfhmep); 8334 if (hmeblkp->hblk_xhat_bit) 8335 continue; 8336 (void) sfmmu_pageunload(pp, sfhmep, TTE8K); 8337 } 8338 8339 if (PP_ISMAPPED_KPM(pp)) 8340 sfmmu_kpm_vac_unload(pp, addr); 8341 8342 /* 8343 * Unloads only do TLB flushes so we need to flush the 8344 * cache here. 8345 */ 8346 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8347 PP_SET_VCOLOR(pp, vcolor); 8348 } 8349 8350 /* 8351 * Whenever a mapping is unloaded and the page is in TNC state, 8352 * we see if the page can be made cacheable again. 'pp' is 8353 * the page that we just unloaded a mapping from, the size 8354 * of mapping that was unloaded is 'ottesz'. 8355 * Remark: 8356 * The recache policy for mpss pages can leave a performance problem 8357 * under the following circumstances: 8358 * . A large page in uncached mode has just been unmapped. 8359 * . All constituent pages are TNC due to a conflicting small mapping. 8360 * . There are many other, non conflicting, small mappings around for 8361 * a lot of the constituent pages. 8362 * . We're called w/ the "old" groupleader page and the old ottesz, 8363 * but this is irrelevant, since we're no more "PP_ISMAPPED_LARGE", so 8364 * we end up w/ TTE8K or npages == 1. 8365 * . We call tst_tnc w/ the old groupleader only, and if there is no 8366 * conflict, we re-cache only this page. 8367 * . All other small mappings are not checked and will be left in TNC mode. 8368 * The problem is not very serious because: 8369 * . mpss is actually only defined for heap and stack, so the probability 8370 * is not very high that a large page mapping exists in parallel to a small 8371 * one (this is possible, but seems to be bad programming style in the 8372 * appl). 8373 * . The problem gets a little bit more serious, when those TNC pages 8374 * have to be mapped into kernel space, e.g. for networking. 8375 * . When VAC alias conflicts occur in applications, this is regarded 8376 * as an application bug. So if kstat's show them, the appl should 8377 * be changed anyway. 8378 */ 8379 void 8380 conv_tnc(page_t *pp, int ottesz) 8381 { 8382 int cursz, dosz; 8383 pgcnt_t curnpgs, dopgs; 8384 pgcnt_t pg64k; 8385 page_t *pp2; 8386 8387 /* 8388 * Determine how big a range we check for TNC and find 8389 * leader page. cursz is the size of the biggest 8390 * mapping that still exist on 'pp'. 8391 */ 8392 if (PP_ISMAPPED_LARGE(pp)) { 8393 cursz = fnd_mapping_sz(pp); 8394 } else { 8395 cursz = TTE8K; 8396 } 8397 8398 if (ottesz >= cursz) { 8399 dosz = ottesz; 8400 pp2 = pp; 8401 } else { 8402 dosz = cursz; 8403 pp2 = PP_GROUPLEADER(pp, dosz); 8404 } 8405 8406 pg64k = TTEPAGES(TTE64K); 8407 dopgs = TTEPAGES(dosz); 8408 8409 ASSERT(dopgs == 1 || ((dopgs & (pg64k - 1)) == 0)); 8410 8411 while (dopgs != 0) { 8412 curnpgs = TTEPAGES(cursz); 8413 if (tst_tnc(pp2, curnpgs)) { 8414 SFMMU_STAT_ADD(sf_recache, curnpgs); 8415 sfmmu_page_cache_array(pp2, HAT_CACHE, CACHE_NO_FLUSH, 8416 curnpgs); 8417 } 8418 8419 ASSERT(dopgs >= curnpgs); 8420 dopgs -= curnpgs; 8421 8422 if (dopgs == 0) { 8423 break; 8424 } 8425 8426 pp2 = PP_PAGENEXT_N(pp2, curnpgs); 8427 if (((dopgs & (pg64k - 1)) == 0) && PP_ISMAPPED_LARGE(pp2)) { 8428 cursz = fnd_mapping_sz(pp2); 8429 } else { 8430 cursz = TTE8K; 8431 } 8432 } 8433 } 8434 8435 /* 8436 * Returns 1 if page(s) can be converted from TNC to cacheable setting, 8437 * returns 0 otherwise. Note that oaddr argument is valid for only 8438 * 8k pages. 8439 */ 8440 int 8441 tst_tnc(page_t *pp, pgcnt_t npages) 8442 { 8443 struct sf_hment *sfhme; 8444 struct hme_blk *hmeblkp; 8445 tte_t tte; 8446 caddr_t vaddr; 8447 int clr_valid = 0; 8448 int color, color1, bcolor; 8449 int i, ncolors; 8450 8451 ASSERT(pp != NULL); 8452 ASSERT(!(cache & CACHE_WRITEBACK)); 8453 8454 if (npages > 1) { 8455 ncolors = CACHE_NUM_COLOR; 8456 } 8457 8458 for (i = 0; i < npages; i++) { 8459 ASSERT(sfmmu_mlist_held(pp)); 8460 ASSERT(PP_ISTNC(pp)); 8461 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 8462 8463 if (PP_ISPNC(pp)) { 8464 return (0); 8465 } 8466 8467 clr_valid = 0; 8468 if (PP_ISMAPPED_KPM(pp)) { 8469 caddr_t kpmvaddr; 8470 8471 ASSERT(kpm_enable); 8472 kpmvaddr = hat_kpm_page2va(pp, 1); 8473 ASSERT(!(npages > 1 && IS_KPM_ALIAS_RANGE(kpmvaddr))); 8474 color1 = addr_to_vcolor(kpmvaddr); 8475 clr_valid = 1; 8476 } 8477 8478 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8479 hmeblkp = sfmmu_hmetohblk(sfhme); 8480 if (hmeblkp->hblk_xhat_bit) 8481 continue; 8482 8483 sfmmu_copytte(&sfhme->hme_tte, &tte); 8484 ASSERT(TTE_IS_VALID(&tte)); 8485 8486 vaddr = tte_to_vaddr(hmeblkp, tte); 8487 color = addr_to_vcolor(vaddr); 8488 8489 if (npages > 1) { 8490 /* 8491 * If there is a big mapping, make sure 8492 * 8K mapping is consistent with the big 8493 * mapping. 8494 */ 8495 bcolor = i % ncolors; 8496 if (color != bcolor) { 8497 return (0); 8498 } 8499 } 8500 if (!clr_valid) { 8501 clr_valid = 1; 8502 color1 = color; 8503 } 8504 8505 if (color1 != color) { 8506 return (0); 8507 } 8508 } 8509 8510 pp = PP_PAGENEXT(pp); 8511 } 8512 8513 return (1); 8514 } 8515 8516 void 8517 sfmmu_page_cache_array(page_t *pp, int flags, int cache_flush_flag, 8518 pgcnt_t npages) 8519 { 8520 kmutex_t *pmtx; 8521 int i, ncolors, bcolor; 8522 kpm_hlk_t *kpmp; 8523 cpuset_t cpuset; 8524 8525 ASSERT(pp != NULL); 8526 ASSERT(!(cache & CACHE_WRITEBACK)); 8527 8528 kpmp = sfmmu_kpm_kpmp_enter(pp, npages); 8529 pmtx = sfmmu_page_enter(pp); 8530 8531 /* 8532 * Fast path caching single unmapped page 8533 */ 8534 if (npages == 1 && !PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp) && 8535 flags == HAT_CACHE) { 8536 PP_CLRTNC(pp); 8537 PP_CLRPNC(pp); 8538 sfmmu_page_exit(pmtx); 8539 sfmmu_kpm_kpmp_exit(kpmp); 8540 return; 8541 } 8542 8543 /* 8544 * We need to capture all cpus in order to change cacheability 8545 * because we can't allow one cpu to access the same physical 8546 * page using a cacheable and a non-cachebale mapping at the same 8547 * time. Since we may end up walking the ism mapping list 8548 * have to grab it's lock now since we can't after all the 8549 * cpus have been captured. 8550 */ 8551 sfmmu_hat_lock_all(); 8552 mutex_enter(&ism_mlist_lock); 8553 kpreempt_disable(); 8554 cpuset = cpu_ready_set; 8555 xc_attention(cpuset); 8556 8557 if (npages > 1) { 8558 /* 8559 * Make sure all colors are flushed since the 8560 * sfmmu_page_cache() only flushes one color- 8561 * it does not know big pages. 8562 */ 8563 ncolors = CACHE_NUM_COLOR; 8564 if (flags & HAT_TMPNC) { 8565 for (i = 0; i < ncolors; i++) { 8566 sfmmu_cache_flushcolor(i, pp->p_pagenum); 8567 } 8568 cache_flush_flag = CACHE_NO_FLUSH; 8569 } 8570 } 8571 8572 for (i = 0; i < npages; i++) { 8573 8574 ASSERT(sfmmu_mlist_held(pp)); 8575 8576 if (!(flags == HAT_TMPNC && PP_ISTNC(pp))) { 8577 8578 if (npages > 1) { 8579 bcolor = i % ncolors; 8580 } else { 8581 bcolor = NO_VCOLOR; 8582 } 8583 8584 sfmmu_page_cache(pp, flags, cache_flush_flag, 8585 bcolor); 8586 } 8587 8588 pp = PP_PAGENEXT(pp); 8589 } 8590 8591 xt_sync(cpuset); 8592 xc_dismissed(cpuset); 8593 mutex_exit(&ism_mlist_lock); 8594 sfmmu_hat_unlock_all(); 8595 sfmmu_page_exit(pmtx); 8596 sfmmu_kpm_kpmp_exit(kpmp); 8597 kpreempt_enable(); 8598 } 8599 8600 /* 8601 * This function changes the virtual cacheability of all mappings to a 8602 * particular page. When changing from uncache to cacheable the mappings will 8603 * only be changed if all of them have the same virtual color. 8604 * We need to flush the cache in all cpus. It is possible that 8605 * a process referenced a page as cacheable but has sinced exited 8606 * and cleared the mapping list. We still to flush it but have no 8607 * state so all cpus is the only alternative. 8608 */ 8609 static void 8610 sfmmu_page_cache(page_t *pp, int flags, int cache_flush_flag, int bcolor) 8611 { 8612 struct sf_hment *sfhme; 8613 struct hme_blk *hmeblkp; 8614 sfmmu_t *sfmmup; 8615 tte_t tte, ttemod; 8616 caddr_t vaddr; 8617 int ret, color; 8618 pfn_t pfn; 8619 8620 color = bcolor; 8621 pfn = pp->p_pagenum; 8622 8623 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8624 8625 hmeblkp = sfmmu_hmetohblk(sfhme); 8626 8627 if (hmeblkp->hblk_xhat_bit) 8628 continue; 8629 8630 sfmmu_copytte(&sfhme->hme_tte, &tte); 8631 ASSERT(TTE_IS_VALID(&tte)); 8632 vaddr = tte_to_vaddr(hmeblkp, tte); 8633 color = addr_to_vcolor(vaddr); 8634 8635 #ifdef DEBUG 8636 if ((flags & HAT_CACHE) && bcolor != NO_VCOLOR) { 8637 ASSERT(color == bcolor); 8638 } 8639 #endif 8640 8641 ASSERT(flags != HAT_TMPNC || color == PP_GET_VCOLOR(pp)); 8642 8643 ttemod = tte; 8644 if (flags & (HAT_UNCACHE | HAT_TMPNC)) { 8645 TTE_CLR_VCACHEABLE(&ttemod); 8646 } else { /* flags & HAT_CACHE */ 8647 TTE_SET_VCACHEABLE(&ttemod); 8648 } 8649 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 8650 if (ret < 0) { 8651 /* 8652 * Since all cpus are captured modifytte should not 8653 * fail. 8654 */ 8655 panic("sfmmu_page_cache: write to tte failed"); 8656 } 8657 8658 sfmmup = hblktosfmmu(hmeblkp); 8659 if (cache_flush_flag == CACHE_FLUSH) { 8660 /* 8661 * Flush TSBs, TLBs and caches 8662 */ 8663 if (sfmmup->sfmmu_ismhat) { 8664 if (flags & HAT_CACHE) { 8665 SFMMU_STAT(sf_ism_recache); 8666 } else { 8667 SFMMU_STAT(sf_ism_uncache); 8668 } 8669 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8670 pfn, CACHE_FLUSH); 8671 } else { 8672 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp, 8673 pfn, 0, FLUSH_ALL_CPUS, CACHE_FLUSH, 1); 8674 } 8675 8676 /* 8677 * all cache entries belonging to this pfn are 8678 * now flushed. 8679 */ 8680 cache_flush_flag = CACHE_NO_FLUSH; 8681 } else { 8682 8683 /* 8684 * Flush only TSBs and TLBs. 8685 */ 8686 if (sfmmup->sfmmu_ismhat) { 8687 if (flags & HAT_CACHE) { 8688 SFMMU_STAT(sf_ism_recache); 8689 } else { 8690 SFMMU_STAT(sf_ism_uncache); 8691 } 8692 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8693 pfn, CACHE_NO_FLUSH); 8694 } else { 8695 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1); 8696 } 8697 } 8698 } 8699 8700 if (PP_ISMAPPED_KPM(pp)) 8701 sfmmu_kpm_page_cache(pp, flags, cache_flush_flag); 8702 8703 switch (flags) { 8704 8705 default: 8706 panic("sfmmu_pagecache: unknown flags"); 8707 break; 8708 8709 case HAT_CACHE: 8710 PP_CLRTNC(pp); 8711 PP_CLRPNC(pp); 8712 PP_SET_VCOLOR(pp, color); 8713 break; 8714 8715 case HAT_TMPNC: 8716 PP_SETTNC(pp); 8717 PP_SET_VCOLOR(pp, NO_VCOLOR); 8718 break; 8719 8720 case HAT_UNCACHE: 8721 PP_SETPNC(pp); 8722 PP_CLRTNC(pp); 8723 PP_SET_VCOLOR(pp, NO_VCOLOR); 8724 break; 8725 } 8726 } 8727 #endif /* VAC */ 8728 8729 8730 /* 8731 * Wrapper routine used to return a context. 8732 * 8733 * It's the responsibility of the caller to guarantee that the 8734 * process serializes on calls here by taking the HAT lock for 8735 * the hat. 8736 * 8737 */ 8738 static void 8739 sfmmu_get_ctx(sfmmu_t *sfmmup) 8740 { 8741 mmu_ctx_t *mmu_ctxp; 8742 uint_t pstate_save; 8743 8744 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8745 ASSERT(sfmmup != ksfmmup); 8746 8747 kpreempt_disable(); 8748 8749 mmu_ctxp = CPU_MMU_CTXP(CPU); 8750 ASSERT(mmu_ctxp); 8751 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 8752 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 8753 8754 /* 8755 * Do a wrap-around if cnum reaches the max # cnum supported by a MMU. 8756 */ 8757 if (mmu_ctxp->mmu_cnum == mmu_ctxp->mmu_nctxs) 8758 sfmmu_ctx_wrap_around(mmu_ctxp); 8759 8760 /* 8761 * Let the MMU set up the page sizes to use for 8762 * this context in the TLB. Don't program 2nd dtlb for ism hat. 8763 */ 8764 if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) { 8765 mmu_set_ctx_page_sizes(sfmmup); 8766 } 8767 8768 /* 8769 * sfmmu_alloc_ctx and sfmmu_load_mmustate will be performed with 8770 * interrupts disabled to prevent race condition with wrap-around 8771 * ctx invalidatation. In sun4v, ctx invalidation also involves 8772 * a HV call to set the number of TSBs to 0. If interrupts are not 8773 * disabled until after sfmmu_load_mmustate is complete TSBs may 8774 * become assigned to INVALID_CONTEXT. This is not allowed. 8775 */ 8776 pstate_save = sfmmu_disable_intrs(); 8777 8778 sfmmu_alloc_ctx(sfmmup, 1, CPU); 8779 sfmmu_load_mmustate(sfmmup); 8780 8781 sfmmu_enable_intrs(pstate_save); 8782 8783 kpreempt_enable(); 8784 } 8785 8786 /* 8787 * When all cnums are used up in a MMU, cnum will wrap around to the 8788 * next generation and start from 2. 8789 */ 8790 static void 8791 sfmmu_ctx_wrap_around(mmu_ctx_t *mmu_ctxp) 8792 { 8793 8794 /* caller must have disabled the preemption */ 8795 ASSERT(curthread->t_preempt >= 1); 8796 ASSERT(mmu_ctxp != NULL); 8797 8798 /* acquire Per-MMU (PM) spin lock */ 8799 mutex_enter(&mmu_ctxp->mmu_lock); 8800 8801 /* re-check to see if wrap-around is needed */ 8802 if (mmu_ctxp->mmu_cnum < mmu_ctxp->mmu_nctxs) 8803 goto done; 8804 8805 SFMMU_MMU_STAT(mmu_wrap_around); 8806 8807 /* update gnum */ 8808 ASSERT(mmu_ctxp->mmu_gnum != 0); 8809 mmu_ctxp->mmu_gnum++; 8810 if (mmu_ctxp->mmu_gnum == 0 || 8811 mmu_ctxp->mmu_gnum > MAX_SFMMU_GNUM_VAL) { 8812 cmn_err(CE_PANIC, "mmu_gnum of mmu_ctx 0x%p is out of bound.", 8813 (void *)mmu_ctxp); 8814 } 8815 8816 if (mmu_ctxp->mmu_ncpus > 1) { 8817 cpuset_t cpuset; 8818 8819 membar_enter(); /* make sure updated gnum visible */ 8820 8821 SFMMU_XCALL_STATS(NULL); 8822 8823 /* xcall to others on the same MMU to invalidate ctx */ 8824 cpuset = mmu_ctxp->mmu_cpuset; 8825 ASSERT(CPU_IN_SET(cpuset, CPU->cpu_id)); 8826 CPUSET_DEL(cpuset, CPU->cpu_id); 8827 CPUSET_AND(cpuset, cpu_ready_set); 8828 8829 /* 8830 * Pass in INVALID_CONTEXT as the first parameter to 8831 * sfmmu_raise_tsb_exception, which invalidates the context 8832 * of any process running on the CPUs in the MMU. 8833 */ 8834 xt_some(cpuset, sfmmu_raise_tsb_exception, 8835 INVALID_CONTEXT, INVALID_CONTEXT); 8836 xt_sync(cpuset); 8837 8838 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 8839 } 8840 8841 if (sfmmu_getctx_sec() != INVALID_CONTEXT) { 8842 sfmmu_setctx_sec(INVALID_CONTEXT); 8843 sfmmu_clear_utsbinfo(); 8844 } 8845 8846 /* 8847 * No xcall is needed here. For sun4u systems all CPUs in context 8848 * domain share a single physical MMU therefore it's enough to flush 8849 * TLB on local CPU. On sun4v systems we use 1 global context 8850 * domain and flush all remote TLBs in sfmmu_raise_tsb_exception 8851 * handler. Note that vtag_flushall_uctxs() is called 8852 * for Ultra II machine, where the equivalent flushall functionality 8853 * is implemented in SW, and only user ctx TLB entries are flushed. 8854 */ 8855 if (&vtag_flushall_uctxs != NULL) { 8856 vtag_flushall_uctxs(); 8857 } else { 8858 vtag_flushall(); 8859 } 8860 8861 /* reset mmu cnum, skips cnum 0 and 1 */ 8862 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 8863 8864 done: 8865 mutex_exit(&mmu_ctxp->mmu_lock); 8866 } 8867 8868 8869 /* 8870 * For multi-threaded process, set the process context to INVALID_CONTEXT 8871 * so that it faults and reloads the MMU state from TL=0. For single-threaded 8872 * process, we can just load the MMU state directly without having to 8873 * set context invalid. Caller must hold the hat lock since we don't 8874 * acquire it here. 8875 */ 8876 static void 8877 sfmmu_sync_mmustate(sfmmu_t *sfmmup) 8878 { 8879 uint_t cnum; 8880 uint_t pstate_save; 8881 8882 ASSERT(sfmmup != ksfmmup); 8883 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8884 8885 kpreempt_disable(); 8886 8887 /* 8888 * We check whether the pass'ed-in sfmmup is the same as the 8889 * current running proc. This is to makes sure the current proc 8890 * stays single-threaded if it already is. 8891 */ 8892 if ((sfmmup == curthread->t_procp->p_as->a_hat) && 8893 (curthread->t_procp->p_lwpcnt == 1)) { 8894 /* single-thread */ 8895 cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum; 8896 if (cnum != INVALID_CONTEXT) { 8897 uint_t curcnum; 8898 /* 8899 * Disable interrupts to prevent race condition 8900 * with sfmmu_ctx_wrap_around ctx invalidation. 8901 * In sun4v, ctx invalidation involves setting 8902 * TSB to NULL, hence, interrupts should be disabled 8903 * untill after sfmmu_load_mmustate is completed. 8904 */ 8905 pstate_save = sfmmu_disable_intrs(); 8906 curcnum = sfmmu_getctx_sec(); 8907 if (curcnum == cnum) 8908 sfmmu_load_mmustate(sfmmup); 8909 sfmmu_enable_intrs(pstate_save); 8910 ASSERT(curcnum == cnum || curcnum == INVALID_CONTEXT); 8911 } 8912 } else { 8913 /* 8914 * multi-thread 8915 * or when sfmmup is not the same as the curproc. 8916 */ 8917 sfmmu_invalidate_ctx(sfmmup); 8918 } 8919 8920 kpreempt_enable(); 8921 } 8922 8923 8924 /* 8925 * Replace the specified TSB with a new TSB. This function gets called when 8926 * we grow, shrink or swapin a TSB. When swapping in a TSB (TSB_SWAPIN), the 8927 * TSB_FORCEALLOC flag may be used to force allocation of a minimum-sized TSB 8928 * (8K). 8929 * 8930 * Caller must hold the HAT lock, but should assume any tsb_info 8931 * pointers it has are no longer valid after calling this function. 8932 * 8933 * Return values: 8934 * TSB_ALLOCFAIL Failed to allocate a TSB, due to memory constraints 8935 * TSB_LOSTRACE HAT is busy, i.e. another thread is already doing 8936 * something to this tsbinfo/TSB 8937 * TSB_SUCCESS Operation succeeded 8938 */ 8939 static tsb_replace_rc_t 8940 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, 8941 hatlock_t *hatlockp, uint_t flags) 8942 { 8943 struct tsb_info *new_tsbinfo = NULL; 8944 struct tsb_info *curtsb, *prevtsb; 8945 uint_t tte_sz_mask; 8946 int i; 8947 8948 ASSERT(sfmmup != ksfmmup); 8949 ASSERT(sfmmup->sfmmu_ismhat == 0); 8950 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8951 ASSERT(szc <= tsb_max_growsize); 8952 8953 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY)) 8954 return (TSB_LOSTRACE); 8955 8956 /* 8957 * Find the tsb_info ahead of this one in the list, and 8958 * also make sure that the tsb_info passed in really 8959 * exists! 8960 */ 8961 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 8962 curtsb != old_tsbinfo && curtsb != NULL; 8963 prevtsb = curtsb, curtsb = curtsb->tsb_next); 8964 ASSERT(curtsb != NULL); 8965 8966 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8967 /* 8968 * The process is swapped out, so just set the new size 8969 * code. When it swaps back in, we'll allocate a new one 8970 * of the new chosen size. 8971 */ 8972 curtsb->tsb_szc = szc; 8973 return (TSB_SUCCESS); 8974 } 8975 SFMMU_FLAGS_SET(sfmmup, HAT_BUSY); 8976 8977 tte_sz_mask = old_tsbinfo->tsb_ttesz_mask; 8978 8979 /* 8980 * All initialization is done inside of sfmmu_tsbinfo_alloc(). 8981 * If we fail to allocate a TSB, exit. 8982 */ 8983 sfmmu_hat_exit(hatlockp); 8984 if (sfmmu_tsbinfo_alloc(&new_tsbinfo, szc, tte_sz_mask, 8985 flags, sfmmup)) { 8986 (void) sfmmu_hat_enter(sfmmup); 8987 if (!(flags & TSB_SWAPIN)) 8988 SFMMU_STAT(sf_tsb_resize_failures); 8989 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8990 return (TSB_ALLOCFAIL); 8991 } 8992 (void) sfmmu_hat_enter(sfmmup); 8993 8994 /* 8995 * Re-check to make sure somebody else didn't muck with us while we 8996 * didn't hold the HAT lock. If the process swapped out, fine, just 8997 * exit; this can happen if we try to shrink the TSB from the context 8998 * of another process (such as on an ISM unmap), though it is rare. 8999 */ 9000 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 9001 SFMMU_STAT(sf_tsb_resize_failures); 9002 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 9003 sfmmu_hat_exit(hatlockp); 9004 sfmmu_tsbinfo_free(new_tsbinfo); 9005 (void) sfmmu_hat_enter(sfmmup); 9006 return (TSB_LOSTRACE); 9007 } 9008 9009 #ifdef DEBUG 9010 /* Reverify that the tsb_info still exists.. for debugging only */ 9011 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 9012 curtsb != old_tsbinfo && curtsb != NULL; 9013 prevtsb = curtsb, curtsb = curtsb->tsb_next); 9014 ASSERT(curtsb != NULL); 9015 #endif /* DEBUG */ 9016 9017 /* 9018 * Quiesce any CPUs running this process on their next TLB miss 9019 * so they atomically see the new tsb_info. We temporarily set the 9020 * context to invalid context so new threads that come on processor 9021 * after we do the xcall to cpusran will also serialize behind the 9022 * HAT lock on TLB miss and will see the new TSB. Since this short 9023 * race with a new thread coming on processor is relatively rare, 9024 * this synchronization mechanism should be cheaper than always 9025 * pausing all CPUs for the duration of the setup, which is what 9026 * the old implementation did. This is particuarly true if we are 9027 * copying a huge chunk of memory around during that window. 9028 * 9029 * The memory barriers are to make sure things stay consistent 9030 * with resume() since it does not hold the HAT lock while 9031 * walking the list of tsb_info structures. 9032 */ 9033 if ((flags & TSB_SWAPIN) != TSB_SWAPIN) { 9034 /* The TSB is either growing or shrinking. */ 9035 sfmmu_invalidate_ctx(sfmmup); 9036 } else { 9037 /* 9038 * It is illegal to swap in TSBs from a process other 9039 * than a process being swapped in. This in turn 9040 * implies we do not have a valid MMU context here 9041 * since a process needs one to resolve translation 9042 * misses. 9043 */ 9044 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup); 9045 } 9046 9047 #ifdef DEBUG 9048 ASSERT(max_mmu_ctxdoms > 0); 9049 9050 /* 9051 * Process should have INVALID_CONTEXT on all MMUs 9052 */ 9053 for (i = 0; i < max_mmu_ctxdoms; i++) { 9054 9055 ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); 9056 } 9057 #endif 9058 9059 new_tsbinfo->tsb_next = old_tsbinfo->tsb_next; 9060 membar_stst(); /* strict ordering required */ 9061 if (prevtsb) 9062 prevtsb->tsb_next = new_tsbinfo; 9063 else 9064 sfmmup->sfmmu_tsb = new_tsbinfo; 9065 membar_enter(); /* make sure new TSB globally visible */ 9066 sfmmu_setup_tsbinfo(sfmmup); 9067 9068 /* 9069 * We need to migrate TSB entries from the old TSB to the new TSB 9070 * if tsb_remap_ttes is set and the TSB is growing. 9071 */ 9072 if (tsb_remap_ttes && ((flags & TSB_GROW) == TSB_GROW)) 9073 sfmmu_copy_tsb(old_tsbinfo, new_tsbinfo); 9074 9075 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 9076 9077 /* 9078 * Drop the HAT lock to free our old tsb_info. 9079 */ 9080 sfmmu_hat_exit(hatlockp); 9081 9082 if ((flags & TSB_GROW) == TSB_GROW) { 9083 SFMMU_STAT(sf_tsb_grow); 9084 } else if ((flags & TSB_SHRINK) == TSB_SHRINK) { 9085 SFMMU_STAT(sf_tsb_shrink); 9086 } 9087 9088 sfmmu_tsbinfo_free(old_tsbinfo); 9089 9090 (void) sfmmu_hat_enter(sfmmup); 9091 return (TSB_SUCCESS); 9092 } 9093 9094 /* 9095 * This function will re-program hat pgsz array, and invalidate the 9096 * process' context, forcing the process to switch to another 9097 * context on the next TLB miss, and therefore start using the 9098 * TLB that is reprogrammed for the new page sizes. 9099 */ 9100 void 9101 sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) 9102 { 9103 int i; 9104 hatlock_t *hatlockp = NULL; 9105 9106 hatlockp = sfmmu_hat_enter(sfmmup); 9107 /* USIII+-IV+ optimization, requires hat lock */ 9108 if (tmp_pgsz) { 9109 for (i = 0; i < mmu_page_sizes; i++) 9110 sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i]; 9111 } 9112 SFMMU_STAT(sf_tlb_reprog_pgsz); 9113 9114 sfmmu_invalidate_ctx(sfmmup); 9115 9116 sfmmu_hat_exit(hatlockp); 9117 } 9118 9119 /* 9120 * This function assumes that there are either four or six supported page 9121 * sizes and at most two programmable TLBs, so we need to decide which 9122 * page sizes are most important and then tell the MMU layer so it 9123 * can adjust the TLB page sizes accordingly (if supported). 9124 * 9125 * If these assumptions change, this function will need to be 9126 * updated to support whatever the new limits are. 9127 * 9128 * The growing flag is nonzero if we are growing the address space, 9129 * and zero if it is shrinking. This allows us to decide whether 9130 * to grow or shrink our TSB, depending upon available memory 9131 * conditions. 9132 */ 9133 static void 9134 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing) 9135 { 9136 uint64_t ttecnt[MMU_PAGE_SIZES]; 9137 uint64_t tte8k_cnt, tte4m_cnt; 9138 uint8_t i; 9139 int sectsb_thresh; 9140 9141 /* 9142 * Kernel threads, processes with small address spaces not using 9143 * large pages, and dummy ISM HATs need not apply. 9144 */ 9145 if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != NULL) 9146 return; 9147 9148 if ((sfmmup->sfmmu_flags & HAT_LGPG_FLAGS) == 0 && 9149 sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor) 9150 return; 9151 9152 for (i = 0; i < mmu_page_sizes; i++) { 9153 ttecnt[i] = SFMMU_TTE_CNT(sfmmup, i); 9154 } 9155 9156 /* Check pagesizes in use, and possibly reprogram DTLB. */ 9157 if (&mmu_check_page_sizes) 9158 mmu_check_page_sizes(sfmmup, ttecnt); 9159 9160 /* 9161 * Calculate the number of 8k ttes to represent the span of these 9162 * pages. 9163 */ 9164 tte8k_cnt = ttecnt[TTE8K] + 9165 (ttecnt[TTE64K] << (MMU_PAGESHIFT64K - MMU_PAGESHIFT)) + 9166 (ttecnt[TTE512K] << (MMU_PAGESHIFT512K - MMU_PAGESHIFT)); 9167 if (mmu_page_sizes == max_mmu_page_sizes) { 9168 tte4m_cnt = ttecnt[TTE4M] + 9169 (ttecnt[TTE32M] << (MMU_PAGESHIFT32M - MMU_PAGESHIFT4M)) + 9170 (ttecnt[TTE256M] << (MMU_PAGESHIFT256M - MMU_PAGESHIFT4M)); 9171 } else { 9172 tte4m_cnt = ttecnt[TTE4M]; 9173 } 9174 9175 /* 9176 * Inflate TSB sizes by a factor of 2 if this process 9177 * uses 4M text pages to minimize extra conflict misses 9178 * in the first TSB since without counting text pages 9179 * 8K TSB may become too small. 9180 * 9181 * Also double the size of the second TSB to minimize 9182 * extra conflict misses due to competition between 4M text pages 9183 * and data pages. 9184 * 9185 * We need to adjust the second TSB allocation threshold by the 9186 * inflation factor, since there is no point in creating a second 9187 * TSB when we know all the mappings can fit in the I/D TLBs. 9188 */ 9189 sectsb_thresh = tsb_sectsb_threshold; 9190 if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) { 9191 tte8k_cnt <<= 1; 9192 tte4m_cnt <<= 1; 9193 sectsb_thresh <<= 1; 9194 } 9195 9196 /* 9197 * Check to see if our TSB is the right size; we may need to 9198 * grow or shrink it. If the process is small, our work is 9199 * finished at this point. 9200 */ 9201 if (tte8k_cnt <= tsb_rss_factor && tte4m_cnt <= sectsb_thresh) { 9202 return; 9203 } 9204 sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh); 9205 } 9206 9207 static void 9208 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt, 9209 uint64_t tte4m_cnt, int sectsb_thresh) 9210 { 9211 int tsb_bits; 9212 uint_t tsb_szc; 9213 struct tsb_info *tsbinfop; 9214 hatlock_t *hatlockp = NULL; 9215 9216 hatlockp = sfmmu_hat_enter(sfmmup); 9217 ASSERT(hatlockp != NULL); 9218 tsbinfop = sfmmup->sfmmu_tsb; 9219 ASSERT(tsbinfop != NULL); 9220 9221 /* 9222 * If we're growing, select the size based on RSS. If we're 9223 * shrinking, leave some room so we don't have to turn around and 9224 * grow again immediately. 9225 */ 9226 if (growing) 9227 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt); 9228 else 9229 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt << 1); 9230 9231 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9232 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9233 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9234 hatlockp, TSB_SHRINK); 9235 } else if (growing && tsb_szc > tsbinfop->tsb_szc && TSB_OK_GROW()) { 9236 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9237 hatlockp, TSB_GROW); 9238 } 9239 tsbinfop = sfmmup->sfmmu_tsb; 9240 9241 /* 9242 * With the TLB and first TSB out of the way, we need to see if 9243 * we need a second TSB for 4M pages. If we managed to reprogram 9244 * the TLB page sizes above, the process will start using this new 9245 * TSB right away; otherwise, it will start using it on the next 9246 * context switch. Either way, it's no big deal so there's no 9247 * synchronization with the trap handlers here unless we grow the 9248 * TSB (in which case it's required to prevent using the old one 9249 * after it's freed). Note: second tsb is required for 32M/256M 9250 * page sizes. 9251 */ 9252 if (tte4m_cnt > sectsb_thresh) { 9253 /* 9254 * If we're growing, select the size based on RSS. If we're 9255 * shrinking, leave some room so we don't have to turn 9256 * around and grow again immediately. 9257 */ 9258 if (growing) 9259 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt); 9260 else 9261 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt << 1); 9262 if (tsbinfop->tsb_next == NULL) { 9263 struct tsb_info *newtsb; 9264 int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)? 9265 0 : TSB_ALLOC; 9266 9267 sfmmu_hat_exit(hatlockp); 9268 9269 /* 9270 * Try to allocate a TSB for 4[32|256]M pages. If we 9271 * can't get the size we want, retry w/a minimum sized 9272 * TSB. If that still didn't work, give up; we can 9273 * still run without one. 9274 */ 9275 tsb_bits = (mmu_page_sizes == max_mmu_page_sizes)? 9276 TSB4M|TSB32M|TSB256M:TSB4M; 9277 if ((sfmmu_tsbinfo_alloc(&newtsb, tsb_szc, tsb_bits, 9278 allocflags, sfmmup) != 0) && 9279 (sfmmu_tsbinfo_alloc(&newtsb, TSB_MIN_SZCODE, 9280 tsb_bits, allocflags, sfmmup) != 0)) { 9281 return; 9282 } 9283 9284 hatlockp = sfmmu_hat_enter(sfmmup); 9285 9286 if (sfmmup->sfmmu_tsb->tsb_next == NULL) { 9287 sfmmup->sfmmu_tsb->tsb_next = newtsb; 9288 SFMMU_STAT(sf_tsb_sectsb_create); 9289 sfmmu_setup_tsbinfo(sfmmup); 9290 sfmmu_hat_exit(hatlockp); 9291 return; 9292 } else { 9293 /* 9294 * It's annoying, but possible for us 9295 * to get here.. we dropped the HAT lock 9296 * because of locking order in the kmem 9297 * allocator, and while we were off getting 9298 * our memory, some other thread decided to 9299 * do us a favor and won the race to get a 9300 * second TSB for this process. Sigh. 9301 */ 9302 sfmmu_hat_exit(hatlockp); 9303 sfmmu_tsbinfo_free(newtsb); 9304 return; 9305 } 9306 } 9307 9308 /* 9309 * We have a second TSB, see if it's big enough. 9310 */ 9311 tsbinfop = tsbinfop->tsb_next; 9312 9313 /* 9314 * Check to see if our second TSB is the right size; 9315 * we may need to grow or shrink it. 9316 * To prevent thrashing (e.g. growing the TSB on a 9317 * subsequent map operation), only try to shrink if 9318 * the TSB reach exceeds twice the virtual address 9319 * space size. 9320 */ 9321 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9322 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9323 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9324 tsb_szc, hatlockp, TSB_SHRINK); 9325 } else if (growing && tsb_szc > tsbinfop->tsb_szc && 9326 TSB_OK_GROW()) { 9327 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9328 tsb_szc, hatlockp, TSB_GROW); 9329 } 9330 } 9331 9332 sfmmu_hat_exit(hatlockp); 9333 } 9334 9335 /* 9336 * Free up a sfmmu 9337 * Since the sfmmu is currently embedded in the hat struct we simply zero 9338 * out our fields and free up the ism map blk list if any. 9339 */ 9340 static void 9341 sfmmu_free_sfmmu(sfmmu_t *sfmmup) 9342 { 9343 ism_blk_t *blkp, *nx_blkp; 9344 #ifdef DEBUG 9345 ism_map_t *map; 9346 int i; 9347 #endif 9348 9349 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 9350 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 9351 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 9352 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 9353 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 9354 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 9355 9356 sfmmup->sfmmu_free = 0; 9357 sfmmup->sfmmu_ismhat = 0; 9358 9359 blkp = sfmmup->sfmmu_iblk; 9360 sfmmup->sfmmu_iblk = NULL; 9361 9362 while (blkp) { 9363 #ifdef DEBUG 9364 map = blkp->iblk_maps; 9365 for (i = 0; i < ISM_MAP_SLOTS; i++) { 9366 ASSERT(map[i].imap_seg == 0); 9367 ASSERT(map[i].imap_ismhat == NULL); 9368 ASSERT(map[i].imap_ment == NULL); 9369 } 9370 #endif 9371 nx_blkp = blkp->iblk_next; 9372 blkp->iblk_next = NULL; 9373 blkp->iblk_nextpa = (uint64_t)-1; 9374 kmem_cache_free(ism_blk_cache, blkp); 9375 blkp = nx_blkp; 9376 } 9377 } 9378 9379 /* 9380 * Locking primitves accessed by HATLOCK macros 9381 */ 9382 9383 #define SFMMU_SPL_MTX (0x0) 9384 #define SFMMU_ML_MTX (0x1) 9385 9386 #define SFMMU_MLSPL_MTX(type, pg) (((type) == SFMMU_SPL_MTX) ? \ 9387 SPL_HASH(pg) : MLIST_HASH(pg)) 9388 9389 kmutex_t * 9390 sfmmu_page_enter(struct page *pp) 9391 { 9392 return (sfmmu_mlspl_enter(pp, SFMMU_SPL_MTX)); 9393 } 9394 9395 void 9396 sfmmu_page_exit(kmutex_t *spl) 9397 { 9398 mutex_exit(spl); 9399 } 9400 9401 int 9402 sfmmu_page_spl_held(struct page *pp) 9403 { 9404 return (sfmmu_mlspl_held(pp, SFMMU_SPL_MTX)); 9405 } 9406 9407 kmutex_t * 9408 sfmmu_mlist_enter(struct page *pp) 9409 { 9410 return (sfmmu_mlspl_enter(pp, SFMMU_ML_MTX)); 9411 } 9412 9413 void 9414 sfmmu_mlist_exit(kmutex_t *mml) 9415 { 9416 mutex_exit(mml); 9417 } 9418 9419 int 9420 sfmmu_mlist_held(struct page *pp) 9421 { 9422 9423 return (sfmmu_mlspl_held(pp, SFMMU_ML_MTX)); 9424 } 9425 9426 /* 9427 * Common code for sfmmu_mlist_enter() and sfmmu_page_enter(). For 9428 * sfmmu_mlist_enter() case mml_table lock array is used and for 9429 * sfmmu_page_enter() sfmmu_page_lock lock array is used. 9430 * 9431 * The lock is taken on a root page so that it protects an operation on all 9432 * constituent pages of a large page pp belongs to. 9433 * 9434 * The routine takes a lock from the appropriate array. The lock is determined 9435 * by hashing the root page. After taking the lock this routine checks if the 9436 * root page has the same size code that was used to determine the root (i.e 9437 * that root hasn't changed). If root page has the expected p_szc field we 9438 * have the right lock and it's returned to the caller. If root's p_szc 9439 * decreased we release the lock and retry from the beginning. This case can 9440 * happen due to hat_page_demote() decreasing p_szc between our load of p_szc 9441 * value and taking the lock. The number of retries due to p_szc decrease is 9442 * limited by the maximum p_szc value. If p_szc is 0 we return the lock 9443 * determined by hashing pp itself. 9444 * 9445 * If our caller doesn't hold a SE_SHARED or SE_EXCL lock on pp it's also 9446 * possible that p_szc can increase. To increase p_szc a thread has to lock 9447 * all constituent pages EXCL and do hat_pageunload() on all of them. All the 9448 * callers that don't hold a page locked recheck if hmeblk through which pp 9449 * was found still maps this pp. If it doesn't map it anymore returned lock 9450 * is immediately dropped. Therefore if sfmmu_mlspl_enter() hits the case of 9451 * p_szc increase after taking the lock it returns this lock without further 9452 * retries because in this case the caller doesn't care about which lock was 9453 * taken. The caller will drop it right away. 9454 * 9455 * After the routine returns it's guaranteed that hat_page_demote() can't 9456 * change p_szc field of any of constituent pages of a large page pp belongs 9457 * to as long as pp was either locked at least SHARED prior to this call or 9458 * the caller finds that hment that pointed to this pp still references this 9459 * pp (this also assumes that the caller holds hme hash bucket lock so that 9460 * the same pp can't be remapped into the same hmeblk after it was unmapped by 9461 * hat_pageunload()). 9462 */ 9463 static kmutex_t * 9464 sfmmu_mlspl_enter(struct page *pp, int type) 9465 { 9466 kmutex_t *mtx; 9467 uint_t prev_rszc = UINT_MAX; 9468 page_t *rootpp; 9469 uint_t szc; 9470 uint_t rszc; 9471 uint_t pszc = pp->p_szc; 9472 9473 ASSERT(pp != NULL); 9474 9475 again: 9476 if (pszc == 0) { 9477 mtx = SFMMU_MLSPL_MTX(type, pp); 9478 mutex_enter(mtx); 9479 return (mtx); 9480 } 9481 9482 /* The lock lives in the root page */ 9483 rootpp = PP_GROUPLEADER(pp, pszc); 9484 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9485 mutex_enter(mtx); 9486 9487 /* 9488 * Return mml in the following 3 cases: 9489 * 9490 * 1) If pp itself is root since if its p_szc decreased before we took 9491 * the lock pp is still the root of smaller szc page. And if its p_szc 9492 * increased it doesn't matter what lock we return (see comment in 9493 * front of this routine). 9494 * 9495 * 2) If pp's not root but rootpp is the root of a rootpp->p_szc size 9496 * large page we have the right lock since any previous potential 9497 * hat_page_demote() is done demoting from greater than current root's 9498 * p_szc because hat_page_demote() changes root's p_szc last. No 9499 * further hat_page_demote() can start or be in progress since it 9500 * would need the same lock we currently hold. 9501 * 9502 * 3) If rootpp's p_szc increased since previous iteration it doesn't 9503 * matter what lock we return (see comment in front of this routine). 9504 */ 9505 if (pp == rootpp || (rszc = rootpp->p_szc) == pszc || 9506 rszc >= prev_rszc) { 9507 return (mtx); 9508 } 9509 9510 /* 9511 * hat_page_demote() could have decreased root's p_szc. 9512 * In this case pp's p_szc must also be smaller than pszc. 9513 * Retry. 9514 */ 9515 if (rszc < pszc) { 9516 szc = pp->p_szc; 9517 if (szc < pszc) { 9518 mutex_exit(mtx); 9519 pszc = szc; 9520 goto again; 9521 } 9522 /* 9523 * pp's p_szc increased after it was decreased. 9524 * page cannot be mapped. Return current lock. The caller 9525 * will drop it right away. 9526 */ 9527 return (mtx); 9528 } 9529 9530 /* 9531 * root's p_szc is greater than pp's p_szc. 9532 * hat_page_demote() is not done with all pages 9533 * yet. Wait for it to complete. 9534 */ 9535 mutex_exit(mtx); 9536 rootpp = PP_GROUPLEADER(rootpp, rszc); 9537 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9538 mutex_enter(mtx); 9539 mutex_exit(mtx); 9540 prev_rszc = rszc; 9541 goto again; 9542 } 9543 9544 static int 9545 sfmmu_mlspl_held(struct page *pp, int type) 9546 { 9547 kmutex_t *mtx; 9548 9549 ASSERT(pp != NULL); 9550 /* The lock lives in the root page */ 9551 pp = PP_PAGEROOT(pp); 9552 ASSERT(pp != NULL); 9553 9554 mtx = SFMMU_MLSPL_MTX(type, pp); 9555 return (MUTEX_HELD(mtx)); 9556 } 9557 9558 static uint_t 9559 sfmmu_get_free_hblk(struct hme_blk **hmeblkpp, uint_t critical) 9560 { 9561 struct hme_blk *hblkp; 9562 9563 if (freehblkp != NULL) { 9564 mutex_enter(&freehblkp_lock); 9565 if (freehblkp != NULL) { 9566 /* 9567 * If the current thread is owning hblk_reserve OR 9568 * critical request from sfmmu_hblk_steal() 9569 * let it succeed even if freehblkcnt is really low. 9570 */ 9571 if (freehblkcnt <= HBLK_RESERVE_MIN && !critical) { 9572 SFMMU_STAT(sf_get_free_throttle); 9573 mutex_exit(&freehblkp_lock); 9574 return (0); 9575 } 9576 freehblkcnt--; 9577 *hmeblkpp = freehblkp; 9578 hblkp = *hmeblkpp; 9579 freehblkp = hblkp->hblk_next; 9580 mutex_exit(&freehblkp_lock); 9581 hblkp->hblk_next = NULL; 9582 SFMMU_STAT(sf_get_free_success); 9583 return (1); 9584 } 9585 mutex_exit(&freehblkp_lock); 9586 } 9587 SFMMU_STAT(sf_get_free_fail); 9588 return (0); 9589 } 9590 9591 static uint_t 9592 sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical) 9593 { 9594 struct hme_blk *hblkp; 9595 9596 /* 9597 * If the current thread is mapping into kernel space, 9598 * let it succede even if freehblkcnt is max 9599 * so that it will avoid freeing it to kmem. 9600 * This will prevent stack overflow due to 9601 * possible recursion since kmem_cache_free() 9602 * might require creation of a slab which 9603 * in turn needs an hmeblk to map that slab; 9604 * let's break this vicious chain at the first 9605 * opportunity. 9606 */ 9607 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9608 mutex_enter(&freehblkp_lock); 9609 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9610 SFMMU_STAT(sf_put_free_success); 9611 freehblkcnt++; 9612 hmeblkp->hblk_next = freehblkp; 9613 freehblkp = hmeblkp; 9614 mutex_exit(&freehblkp_lock); 9615 return (1); 9616 } 9617 mutex_exit(&freehblkp_lock); 9618 } 9619 9620 /* 9621 * Bring down freehblkcnt to HBLK_RESERVE_CNT. We are here 9622 * only if freehblkcnt is at least HBLK_RESERVE_CNT *and* 9623 * we are not in the process of mapping into kernel space. 9624 */ 9625 ASSERT(!critical); 9626 while (freehblkcnt > HBLK_RESERVE_CNT) { 9627 mutex_enter(&freehblkp_lock); 9628 if (freehblkcnt > HBLK_RESERVE_CNT) { 9629 freehblkcnt--; 9630 hblkp = freehblkp; 9631 freehblkp = hblkp->hblk_next; 9632 mutex_exit(&freehblkp_lock); 9633 ASSERT(get_hblk_cache(hblkp) == sfmmu8_cache); 9634 kmem_cache_free(sfmmu8_cache, hblkp); 9635 continue; 9636 } 9637 mutex_exit(&freehblkp_lock); 9638 } 9639 SFMMU_STAT(sf_put_free_fail); 9640 return (0); 9641 } 9642 9643 static void 9644 sfmmu_hblk_swap(struct hme_blk *new) 9645 { 9646 struct hme_blk *old, *hblkp, *prev; 9647 uint64_t hblkpa, prevpa, newpa; 9648 caddr_t base, vaddr, endaddr; 9649 struct hmehash_bucket *hmebp; 9650 struct sf_hment *osfhme, *nsfhme; 9651 page_t *pp; 9652 kmutex_t *pml; 9653 tte_t tte; 9654 9655 #ifdef DEBUG 9656 hmeblk_tag hblktag; 9657 struct hme_blk *found; 9658 #endif 9659 old = HBLK_RESERVE; 9660 9661 /* 9662 * save pa before bcopy clobbers it 9663 */ 9664 newpa = new->hblk_nextpa; 9665 9666 base = (caddr_t)get_hblk_base(old); 9667 endaddr = base + get_hblk_span(old); 9668 9669 /* 9670 * acquire hash bucket lock. 9671 */ 9672 hmebp = sfmmu_tteload_acquire_hashbucket(ksfmmup, base, TTE8K); 9673 9674 /* 9675 * copy contents from old to new 9676 */ 9677 bcopy((void *)old, (void *)new, HME8BLK_SZ); 9678 9679 /* 9680 * add new to hash chain 9681 */ 9682 sfmmu_hblk_hash_add(hmebp, new, newpa); 9683 9684 /* 9685 * search hash chain for hblk_reserve; this needs to be performed 9686 * after adding new, otherwise prevpa and prev won't correspond 9687 * to the hblk which is prior to old in hash chain when we call 9688 * sfmmu_hblk_hash_rm to remove old later. 9689 */ 9690 for (prevpa = 0, prev = NULL, 9691 hblkpa = hmebp->hmeh_nextpa, hblkp = hmebp->hmeblkp; 9692 hblkp != NULL && hblkp != old; 9693 prevpa = hblkpa, prev = hblkp, 9694 hblkpa = hblkp->hblk_nextpa, hblkp = hblkp->hblk_next); 9695 9696 if (hblkp != old) 9697 panic("sfmmu_hblk_swap: hblk_reserve not found"); 9698 9699 /* 9700 * p_mapping list is still pointing to hments in hblk_reserve; 9701 * fix up p_mapping list so that they point to hments in new. 9702 * 9703 * Since all these mappings are created by hblk_reserve_thread 9704 * on the way and it's using at least one of the buffers from each of 9705 * the newly minted slabs, there is no danger of any of these 9706 * mappings getting unloaded by another thread. 9707 * 9708 * tsbmiss could only modify ref/mod bits of hments in old/new. 9709 * Since all of these hments hold mappings established by segkmem 9710 * and mappings in segkmem are setup with HAT_NOSYNC, ref/mod bits 9711 * have no meaning for the mappings in hblk_reserve. hments in 9712 * old and new are identical except for ref/mod bits. 9713 */ 9714 for (vaddr = base; vaddr < endaddr; vaddr += TTEBYTES(TTE8K)) { 9715 9716 HBLKTOHME(osfhme, old, vaddr); 9717 sfmmu_copytte(&osfhme->hme_tte, &tte); 9718 9719 if (TTE_IS_VALID(&tte)) { 9720 if ((pp = osfhme->hme_page) == NULL) 9721 panic("sfmmu_hblk_swap: page not mapped"); 9722 9723 pml = sfmmu_mlist_enter(pp); 9724 9725 if (pp != osfhme->hme_page) 9726 panic("sfmmu_hblk_swap: mapping changed"); 9727 9728 HBLKTOHME(nsfhme, new, vaddr); 9729 9730 HME_ADD(nsfhme, pp); 9731 HME_SUB(osfhme, pp); 9732 9733 sfmmu_mlist_exit(pml); 9734 } 9735 } 9736 9737 /* 9738 * remove old from hash chain 9739 */ 9740 sfmmu_hblk_hash_rm(hmebp, old, prevpa, prev); 9741 9742 #ifdef DEBUG 9743 9744 hblktag.htag_id = ksfmmup; 9745 hblktag.htag_bspage = HME_HASH_BSPAGE(base, HME_HASH_SHIFT(TTE8K)); 9746 hblktag.htag_rehash = HME_HASH_REHASH(TTE8K); 9747 HME_HASH_FAST_SEARCH(hmebp, hblktag, found); 9748 9749 if (found != new) 9750 panic("sfmmu_hblk_swap: new hblk not found"); 9751 #endif 9752 9753 SFMMU_HASH_UNLOCK(hmebp); 9754 9755 /* 9756 * Reset hblk_reserve 9757 */ 9758 bzero((void *)old, HME8BLK_SZ); 9759 old->hblk_nextpa = va_to_pa((caddr_t)old); 9760 } 9761 9762 /* 9763 * Grab the mlist mutex for both pages passed in. 9764 * 9765 * low and high will be returned as pointers to the mutexes for these pages. 9766 * low refers to the mutex residing in the lower bin of the mlist hash, while 9767 * high refers to the mutex residing in the higher bin of the mlist hash. This 9768 * is due to the locking order restrictions on the same thread grabbing 9769 * multiple mlist mutexes. The low lock must be acquired before the high lock. 9770 * 9771 * If both pages hash to the same mutex, only grab that single mutex, and 9772 * high will be returned as NULL 9773 * If the pages hash to different bins in the hash, grab the lower addressed 9774 * lock first and then the higher addressed lock in order to follow the locking 9775 * rules involved with the same thread grabbing multiple mlist mutexes. 9776 * low and high will both have non-NULL values. 9777 */ 9778 static void 9779 sfmmu_mlist_reloc_enter(struct page *targ, struct page *repl, 9780 kmutex_t **low, kmutex_t **high) 9781 { 9782 kmutex_t *mml_targ, *mml_repl; 9783 9784 /* 9785 * no need to do the dance around szc as in sfmmu_mlist_enter() 9786 * because this routine is only called by hat_page_relocate() and all 9787 * targ and repl pages are already locked EXCL so szc can't change. 9788 */ 9789 9790 mml_targ = MLIST_HASH(PP_PAGEROOT(targ)); 9791 mml_repl = MLIST_HASH(PP_PAGEROOT(repl)); 9792 9793 if (mml_targ == mml_repl) { 9794 *low = mml_targ; 9795 *high = NULL; 9796 } else { 9797 if (mml_targ < mml_repl) { 9798 *low = mml_targ; 9799 *high = mml_repl; 9800 } else { 9801 *low = mml_repl; 9802 *high = mml_targ; 9803 } 9804 } 9805 9806 mutex_enter(*low); 9807 if (*high) 9808 mutex_enter(*high); 9809 } 9810 9811 static void 9812 sfmmu_mlist_reloc_exit(kmutex_t *low, kmutex_t *high) 9813 { 9814 if (high) 9815 mutex_exit(high); 9816 mutex_exit(low); 9817 } 9818 9819 static hatlock_t * 9820 sfmmu_hat_enter(sfmmu_t *sfmmup) 9821 { 9822 hatlock_t *hatlockp; 9823 9824 if (sfmmup != ksfmmup) { 9825 hatlockp = TSB_HASH(sfmmup); 9826 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 9827 return (hatlockp); 9828 } 9829 return (NULL); 9830 } 9831 9832 static hatlock_t * 9833 sfmmu_hat_tryenter(sfmmu_t *sfmmup) 9834 { 9835 hatlock_t *hatlockp; 9836 9837 if (sfmmup != ksfmmup) { 9838 hatlockp = TSB_HASH(sfmmup); 9839 if (mutex_tryenter(HATLOCK_MUTEXP(hatlockp)) == 0) 9840 return (NULL); 9841 return (hatlockp); 9842 } 9843 return (NULL); 9844 } 9845 9846 static void 9847 sfmmu_hat_exit(hatlock_t *hatlockp) 9848 { 9849 if (hatlockp != NULL) 9850 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 9851 } 9852 9853 static void 9854 sfmmu_hat_lock_all(void) 9855 { 9856 int i; 9857 for (i = 0; i < SFMMU_NUM_LOCK; i++) 9858 mutex_enter(HATLOCK_MUTEXP(&hat_lock[i])); 9859 } 9860 9861 static void 9862 sfmmu_hat_unlock_all(void) 9863 { 9864 int i; 9865 for (i = SFMMU_NUM_LOCK - 1; i >= 0; i--) 9866 mutex_exit(HATLOCK_MUTEXP(&hat_lock[i])); 9867 } 9868 9869 int 9870 sfmmu_hat_lock_held(sfmmu_t *sfmmup) 9871 { 9872 ASSERT(sfmmup != ksfmmup); 9873 return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup)))); 9874 } 9875 9876 /* 9877 * Locking primitives to provide consistency between ISM unmap 9878 * and other operations. Since ISM unmap can take a long time, we 9879 * use HAT_ISMBUSY flag (protected by the hatlock) to avoid creating 9880 * contention on the hatlock buckets while ISM segments are being 9881 * unmapped. The tradeoff is that the flags don't prevent priority 9882 * inversion from occurring, so we must request kernel priority in 9883 * case we have to sleep to keep from getting buried while holding 9884 * the HAT_ISMBUSY flag set, which in turn could block other kernel 9885 * threads from running (for example, in sfmmu_uvatopfn()). 9886 */ 9887 static void 9888 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held) 9889 { 9890 hatlock_t *hatlockp; 9891 9892 THREAD_KPRI_REQUEST(); 9893 if (!hatlock_held) 9894 hatlockp = sfmmu_hat_enter(sfmmup); 9895 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) 9896 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 9897 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY); 9898 if (!hatlock_held) 9899 sfmmu_hat_exit(hatlockp); 9900 } 9901 9902 static void 9903 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held) 9904 { 9905 hatlock_t *hatlockp; 9906 9907 if (!hatlock_held) 9908 hatlockp = sfmmu_hat_enter(sfmmup); 9909 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 9910 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY); 9911 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 9912 if (!hatlock_held) 9913 sfmmu_hat_exit(hatlockp); 9914 THREAD_KPRI_RELEASE(); 9915 } 9916 9917 /* 9918 * 9919 * Algorithm: 9920 * 9921 * (1) if segkmem is not ready, allocate hblk from an array of pre-alloc'ed 9922 * hblks. 9923 * 9924 * (2) if we are allocating an hblk for mapping a slab in sfmmu_cache, 9925 * 9926 * (a) try to return an hblk from reserve pool of free hblks; 9927 * (b) if the reserve pool is empty, acquire hblk_reserve_lock 9928 * and return hblk_reserve. 9929 * 9930 * (3) call kmem_cache_alloc() to allocate hblk; 9931 * 9932 * (a) if hblk_reserve_lock is held by the current thread, 9933 * atomically replace hblk_reserve by the hblk that is 9934 * returned by kmem_cache_alloc; release hblk_reserve_lock 9935 * and call kmem_cache_alloc() again. 9936 * (b) if reserve pool is not full, add the hblk that is 9937 * returned by kmem_cache_alloc to reserve pool and 9938 * call kmem_cache_alloc again. 9939 * 9940 */ 9941 static struct hme_blk * 9942 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr, 9943 struct hmehash_bucket *hmebp, uint_t size, hmeblk_tag hblktag, 9944 uint_t flags) 9945 { 9946 struct hme_blk *hmeblkp = NULL; 9947 struct hme_blk *newhblkp; 9948 struct hme_blk *shw_hblkp = NULL; 9949 struct kmem_cache *sfmmu_cache = NULL; 9950 uint64_t hblkpa; 9951 ulong_t index; 9952 uint_t owner; /* set to 1 if using hblk_reserve */ 9953 uint_t forcefree; 9954 int sleep; 9955 9956 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 9957 9958 /* 9959 * If segkmem is not created yet, allocate from static hmeblks 9960 * created at the end of startup_modules(). See the block comment 9961 * in startup_modules() describing how we estimate the number of 9962 * static hmeblks that will be needed during re-map. 9963 */ 9964 if (!hblk_alloc_dynamic) { 9965 9966 if (size == TTE8K) { 9967 index = nucleus_hblk8.index; 9968 if (index >= nucleus_hblk8.len) { 9969 /* 9970 * If we panic here, see startup_modules() to 9971 * make sure that we are calculating the 9972 * number of hblk8's that we need correctly. 9973 */ 9974 prom_panic("no nucleus hblk8 to allocate"); 9975 } 9976 hmeblkp = 9977 (struct hme_blk *)&nucleus_hblk8.list[index]; 9978 nucleus_hblk8.index++; 9979 SFMMU_STAT(sf_hblk8_nalloc); 9980 } else { 9981 index = nucleus_hblk1.index; 9982 if (nucleus_hblk1.index >= nucleus_hblk1.len) { 9983 /* 9984 * If we panic here, see startup_modules(). 9985 * Most likely you need to update the 9986 * calculation of the number of hblk1 elements 9987 * that the kernel needs to boot. 9988 */ 9989 prom_panic("no nucleus hblk1 to allocate"); 9990 } 9991 hmeblkp = 9992 (struct hme_blk *)&nucleus_hblk1.list[index]; 9993 nucleus_hblk1.index++; 9994 SFMMU_STAT(sf_hblk1_nalloc); 9995 } 9996 9997 goto hblk_init; 9998 } 9999 10000 SFMMU_HASH_UNLOCK(hmebp); 10001 10002 if (sfmmup != KHATID) { 10003 if (mmu_page_sizes == max_mmu_page_sizes) { 10004 if (size < TTE256M) 10005 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 10006 size, flags); 10007 } else { 10008 if (size < TTE4M) 10009 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 10010 size, flags); 10011 } 10012 } 10013 10014 fill_hblk: 10015 owner = (hblk_reserve_thread == curthread) ? 1 : 0; 10016 10017 if (owner && size == TTE8K) { 10018 10019 /* 10020 * We are really in a tight spot. We already own 10021 * hblk_reserve and we need another hblk. In anticipation 10022 * of this kind of scenario, we specifically set aside 10023 * HBLK_RESERVE_MIN number of hblks to be used exclusively 10024 * by owner of hblk_reserve. 10025 */ 10026 SFMMU_STAT(sf_hblk_recurse_cnt); 10027 10028 if (!sfmmu_get_free_hblk(&hmeblkp, 1)) 10029 panic("sfmmu_hblk_alloc: reserve list is empty"); 10030 10031 goto hblk_verify; 10032 } 10033 10034 ASSERT(!owner); 10035 10036 if ((flags & HAT_NO_KALLOC) == 0) { 10037 10038 sfmmu_cache = ((size == TTE8K) ? sfmmu8_cache : sfmmu1_cache); 10039 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP); 10040 10041 if ((hmeblkp = kmem_cache_alloc(sfmmu_cache, sleep)) == NULL) { 10042 hmeblkp = sfmmu_hblk_steal(size); 10043 } else { 10044 /* 10045 * if we are the owner of hblk_reserve, 10046 * swap hblk_reserve with hmeblkp and 10047 * start a fresh life. Hope things go 10048 * better this time. 10049 */ 10050 if (hblk_reserve_thread == curthread) { 10051 ASSERT(sfmmu_cache == sfmmu8_cache); 10052 sfmmu_hblk_swap(hmeblkp); 10053 hblk_reserve_thread = NULL; 10054 mutex_exit(&hblk_reserve_lock); 10055 goto fill_hblk; 10056 } 10057 /* 10058 * let's donate this hblk to our reserve list if 10059 * we are not mapping kernel range 10060 */ 10061 if (size == TTE8K && sfmmup != KHATID) 10062 if (sfmmu_put_free_hblk(hmeblkp, 0)) 10063 goto fill_hblk; 10064 } 10065 } else { 10066 /* 10067 * We are here to map the slab in sfmmu8_cache; let's 10068 * check if we could tap our reserve list; if successful, 10069 * this will avoid the pain of going thru sfmmu_hblk_swap 10070 */ 10071 SFMMU_STAT(sf_hblk_slab_cnt); 10072 if (!sfmmu_get_free_hblk(&hmeblkp, 0)) { 10073 /* 10074 * let's start hblk_reserve dance 10075 */ 10076 SFMMU_STAT(sf_hblk_reserve_cnt); 10077 owner = 1; 10078 mutex_enter(&hblk_reserve_lock); 10079 hmeblkp = HBLK_RESERVE; 10080 hblk_reserve_thread = curthread; 10081 } 10082 } 10083 10084 hblk_verify: 10085 ASSERT(hmeblkp != NULL); 10086 set_hblk_sz(hmeblkp, size); 10087 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); 10088 SFMMU_HASH_LOCK(hmebp); 10089 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10090 if (newhblkp != NULL) { 10091 SFMMU_HASH_UNLOCK(hmebp); 10092 if (hmeblkp != HBLK_RESERVE) { 10093 /* 10094 * This is really tricky! 10095 * 10096 * vmem_alloc(vmem_seg_arena) 10097 * vmem_alloc(vmem_internal_arena) 10098 * segkmem_alloc(heap_arena) 10099 * vmem_alloc(heap_arena) 10100 * page_create() 10101 * hat_memload() 10102 * kmem_cache_free() 10103 * kmem_cache_alloc() 10104 * kmem_slab_create() 10105 * vmem_alloc(kmem_internal_arena) 10106 * segkmem_alloc(heap_arena) 10107 * vmem_alloc(heap_arena) 10108 * page_create() 10109 * hat_memload() 10110 * kmem_cache_free() 10111 * ... 10112 * 10113 * Thus, hat_memload() could call kmem_cache_free 10114 * for enough number of times that we could easily 10115 * hit the bottom of the stack or run out of reserve 10116 * list of vmem_seg structs. So, we must donate 10117 * this hblk to reserve list if it's allocated 10118 * from sfmmu8_cache *and* mapping kernel range. 10119 * We don't need to worry about freeing hmeblk1's 10120 * to kmem since they don't map any kmem slabs. 10121 * 10122 * Note: When segkmem supports largepages, we must 10123 * free hmeblk1's to reserve list as well. 10124 */ 10125 forcefree = (sfmmup == KHATID) ? 1 : 0; 10126 if (size == TTE8K && 10127 sfmmu_put_free_hblk(hmeblkp, forcefree)) { 10128 goto re_verify; 10129 } 10130 ASSERT(sfmmup != KHATID); 10131 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10132 } else { 10133 /* 10134 * Hey! we don't need hblk_reserve any more. 10135 */ 10136 ASSERT(owner); 10137 hblk_reserve_thread = NULL; 10138 mutex_exit(&hblk_reserve_lock); 10139 owner = 0; 10140 } 10141 re_verify: 10142 /* 10143 * let's check if the goodies are still present 10144 */ 10145 SFMMU_HASH_LOCK(hmebp); 10146 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10147 if (newhblkp != NULL) { 10148 /* 10149 * return newhblkp if it's not hblk_reserve; 10150 * if newhblkp is hblk_reserve, return it 10151 * _only if_ we are the owner of hblk_reserve. 10152 */ 10153 if (newhblkp != HBLK_RESERVE || owner) { 10154 return (newhblkp); 10155 } else { 10156 /* 10157 * we just hit hblk_reserve in the hash and 10158 * we are not the owner of that; 10159 * 10160 * block until hblk_reserve_thread completes 10161 * swapping hblk_reserve and try the dance 10162 * once again. 10163 */ 10164 SFMMU_HASH_UNLOCK(hmebp); 10165 mutex_enter(&hblk_reserve_lock); 10166 mutex_exit(&hblk_reserve_lock); 10167 SFMMU_STAT(sf_hblk_reserve_hit); 10168 goto fill_hblk; 10169 } 10170 } else { 10171 /* 10172 * it's no more! try the dance once again. 10173 */ 10174 SFMMU_HASH_UNLOCK(hmebp); 10175 goto fill_hblk; 10176 } 10177 } 10178 10179 hblk_init: 10180 set_hblk_sz(hmeblkp, size); 10181 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10182 hmeblkp->hblk_next = (struct hme_blk *)NULL; 10183 hmeblkp->hblk_tag = hblktag; 10184 hmeblkp->hblk_shadow = shw_hblkp; 10185 hblkpa = hmeblkp->hblk_nextpa; 10186 hmeblkp->hblk_nextpa = 0; 10187 10188 ASSERT(get_hblk_ttesz(hmeblkp) == size); 10189 ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size)); 10190 ASSERT(hmeblkp->hblk_hmecnt == 0); 10191 ASSERT(hmeblkp->hblk_vcnt == 0); 10192 ASSERT(hmeblkp->hblk_lckcnt == 0); 10193 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10194 sfmmu_hblk_hash_add(hmebp, hmeblkp, hblkpa); 10195 return (hmeblkp); 10196 } 10197 10198 /* 10199 * This function performs any cleanup required on the hme_blk 10200 * and returns it to the free list. 10201 */ 10202 /* ARGSUSED */ 10203 static void 10204 sfmmu_hblk_free(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10205 uint64_t hblkpa, struct hme_blk **listp) 10206 { 10207 int shw_size, vshift; 10208 struct hme_blk *shw_hblkp; 10209 uint_t shw_mask, newshw_mask; 10210 uintptr_t vaddr; 10211 int size; 10212 uint_t critical; 10213 10214 ASSERT(hmeblkp); 10215 ASSERT(!hmeblkp->hblk_hmecnt); 10216 ASSERT(!hmeblkp->hblk_vcnt); 10217 ASSERT(!hmeblkp->hblk_lckcnt); 10218 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10219 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 10220 10221 critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0; 10222 10223 size = get_hblk_ttesz(hmeblkp); 10224 shw_hblkp = hmeblkp->hblk_shadow; 10225 if (shw_hblkp) { 10226 ASSERT(hblktosfmmu(hmeblkp) != KHATID); 10227 if (mmu_page_sizes == max_mmu_page_sizes) { 10228 ASSERT(size < TTE256M); 10229 } else { 10230 ASSERT(size < TTE4M); 10231 } 10232 10233 shw_size = get_hblk_ttesz(shw_hblkp); 10234 vaddr = get_hblk_base(hmeblkp); 10235 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10236 ASSERT(vshift < 8); 10237 /* 10238 * Atomically clear shadow mask bit 10239 */ 10240 do { 10241 shw_mask = shw_hblkp->hblk_shw_mask; 10242 ASSERT(shw_mask & (1 << vshift)); 10243 newshw_mask = shw_mask & ~(1 << vshift); 10244 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10245 shw_mask, newshw_mask); 10246 } while (newshw_mask != shw_mask); 10247 hmeblkp->hblk_shadow = NULL; 10248 } 10249 hmeblkp->hblk_next = NULL; 10250 hmeblkp->hblk_nextpa = hblkpa; 10251 hmeblkp->hblk_shw_bit = 0; 10252 10253 if (hmeblkp->hblk_nuc_bit == 0) { 10254 10255 if (size == TTE8K && sfmmu_put_free_hblk(hmeblkp, critical)) 10256 return; 10257 10258 hmeblkp->hblk_next = *listp; 10259 *listp = hmeblkp; 10260 } 10261 } 10262 10263 static void 10264 sfmmu_hblks_list_purge(struct hme_blk **listp) 10265 { 10266 struct hme_blk *hmeblkp; 10267 10268 while ((hmeblkp = *listp) != NULL) { 10269 *listp = hmeblkp->hblk_next; 10270 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10271 } 10272 } 10273 10274 #define BUCKETS_TO_SEARCH_BEFORE_UNLOAD 30 10275 #define SFMMU_HBLK_STEAL_THRESHOLD 5 10276 10277 static uint_t sfmmu_hblk_steal_twice; 10278 static uint_t sfmmu_hblk_steal_count, sfmmu_hblk_steal_unload_count; 10279 10280 /* 10281 * Steal a hmeblk from user or kernel hme hash lists. 10282 * For 8K tte grab one from reserve pool (freehblkp) before proceeding to 10283 * steal and if we fail to steal after SFMMU_HBLK_STEAL_THRESHOLD attempts 10284 * tap into critical reserve of freehblkp. 10285 * Note: We remain looping in this routine until we find one. 10286 */ 10287 static struct hme_blk * 10288 sfmmu_hblk_steal(int size) 10289 { 10290 static struct hmehash_bucket *uhmehash_steal_hand = NULL; 10291 struct hmehash_bucket *hmebp; 10292 struct hme_blk *hmeblkp = NULL, *pr_hblk; 10293 uint64_t hblkpa, prevpa; 10294 int i; 10295 uint_t loop_cnt = 0, critical; 10296 10297 for (;;) { 10298 if (size == TTE8K) { 10299 critical = 10300 (++loop_cnt > SFMMU_HBLK_STEAL_THRESHOLD) ? 1 : 0; 10301 if (sfmmu_get_free_hblk(&hmeblkp, critical)) 10302 return (hmeblkp); 10303 } 10304 10305 hmebp = (uhmehash_steal_hand == NULL) ? uhme_hash : 10306 uhmehash_steal_hand; 10307 ASSERT(hmebp >= uhme_hash && hmebp <= &uhme_hash[UHMEHASH_SZ]); 10308 10309 for (i = 0; hmeblkp == NULL && i <= UHMEHASH_SZ + 10310 BUCKETS_TO_SEARCH_BEFORE_UNLOAD; i++) { 10311 SFMMU_HASH_LOCK(hmebp); 10312 hmeblkp = hmebp->hmeblkp; 10313 hblkpa = hmebp->hmeh_nextpa; 10314 prevpa = 0; 10315 pr_hblk = NULL; 10316 while (hmeblkp) { 10317 /* 10318 * check if it is a hmeblk that is not locked 10319 * and not shared. skip shadow hmeblks with 10320 * shadow_mask set i.e valid count non zero. 10321 */ 10322 if ((get_hblk_ttesz(hmeblkp) == size) && 10323 (hmeblkp->hblk_shw_bit == 0 || 10324 hmeblkp->hblk_vcnt == 0) && 10325 (hmeblkp->hblk_lckcnt == 0)) { 10326 /* 10327 * there is a high probability that we 10328 * will find a free one. search some 10329 * buckets for a free hmeblk initially 10330 * before unloading a valid hmeblk. 10331 */ 10332 if ((hmeblkp->hblk_vcnt == 0 && 10333 hmeblkp->hblk_hmecnt == 0) || (i >= 10334 BUCKETS_TO_SEARCH_BEFORE_UNLOAD)) { 10335 if (sfmmu_steal_this_hblk(hmebp, 10336 hmeblkp, hblkpa, prevpa, 10337 pr_hblk)) { 10338 /* 10339 * Hblk is unloaded 10340 * successfully 10341 */ 10342 break; 10343 } 10344 } 10345 } 10346 pr_hblk = hmeblkp; 10347 prevpa = hblkpa; 10348 hblkpa = hmeblkp->hblk_nextpa; 10349 hmeblkp = hmeblkp->hblk_next; 10350 } 10351 10352 SFMMU_HASH_UNLOCK(hmebp); 10353 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 10354 hmebp = uhme_hash; 10355 } 10356 uhmehash_steal_hand = hmebp; 10357 10358 if (hmeblkp != NULL) 10359 break; 10360 10361 /* 10362 * in the worst case, look for a free one in the kernel 10363 * hash table. 10364 */ 10365 for (i = 0, hmebp = khme_hash; i <= KHMEHASH_SZ; i++) { 10366 SFMMU_HASH_LOCK(hmebp); 10367 hmeblkp = hmebp->hmeblkp; 10368 hblkpa = hmebp->hmeh_nextpa; 10369 prevpa = 0; 10370 pr_hblk = NULL; 10371 while (hmeblkp) { 10372 /* 10373 * check if it is free hmeblk 10374 */ 10375 if ((get_hblk_ttesz(hmeblkp) == size) && 10376 (hmeblkp->hblk_lckcnt == 0) && 10377 (hmeblkp->hblk_vcnt == 0) && 10378 (hmeblkp->hblk_hmecnt == 0)) { 10379 if (sfmmu_steal_this_hblk(hmebp, 10380 hmeblkp, hblkpa, prevpa, pr_hblk)) { 10381 break; 10382 } else { 10383 /* 10384 * Cannot fail since we have 10385 * hash lock. 10386 */ 10387 panic("fail to steal?"); 10388 } 10389 } 10390 10391 pr_hblk = hmeblkp; 10392 prevpa = hblkpa; 10393 hblkpa = hmeblkp->hblk_nextpa; 10394 hmeblkp = hmeblkp->hblk_next; 10395 } 10396 10397 SFMMU_HASH_UNLOCK(hmebp); 10398 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 10399 hmebp = khme_hash; 10400 } 10401 10402 if (hmeblkp != NULL) 10403 break; 10404 sfmmu_hblk_steal_twice++; 10405 } 10406 return (hmeblkp); 10407 } 10408 10409 /* 10410 * This routine does real work to prepare a hblk to be "stolen" by 10411 * unloading the mappings, updating shadow counts .... 10412 * It returns 1 if the block is ready to be reused (stolen), or 0 10413 * means the block cannot be stolen yet- pageunload is still working 10414 * on this hblk. 10415 */ 10416 static int 10417 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10418 uint64_t hblkpa, uint64_t prevpa, struct hme_blk *pr_hblk) 10419 { 10420 int shw_size, vshift; 10421 struct hme_blk *shw_hblkp; 10422 uintptr_t vaddr; 10423 uint_t shw_mask, newshw_mask; 10424 10425 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10426 10427 /* 10428 * check if the hmeblk is free, unload if necessary 10429 */ 10430 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10431 sfmmu_t *sfmmup; 10432 demap_range_t dmr; 10433 10434 sfmmup = hblktosfmmu(hmeblkp); 10435 DEMAP_RANGE_INIT(sfmmup, &dmr); 10436 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 10437 (caddr_t)get_hblk_base(hmeblkp), 10438 get_hblk_endaddr(hmeblkp), &dmr, HAT_UNLOAD); 10439 DEMAP_RANGE_FLUSH(&dmr); 10440 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10441 /* 10442 * Pageunload is working on the same hblk. 10443 */ 10444 return (0); 10445 } 10446 10447 sfmmu_hblk_steal_unload_count++; 10448 } 10449 10450 ASSERT(hmeblkp->hblk_lckcnt == 0); 10451 ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0); 10452 10453 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 10454 hmeblkp->hblk_nextpa = hblkpa; 10455 10456 shw_hblkp = hmeblkp->hblk_shadow; 10457 if (shw_hblkp) { 10458 shw_size = get_hblk_ttesz(shw_hblkp); 10459 vaddr = get_hblk_base(hmeblkp); 10460 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10461 ASSERT(vshift < 8); 10462 /* 10463 * Atomically clear shadow mask bit 10464 */ 10465 do { 10466 shw_mask = shw_hblkp->hblk_shw_mask; 10467 ASSERT(shw_mask & (1 << vshift)); 10468 newshw_mask = shw_mask & ~(1 << vshift); 10469 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10470 shw_mask, newshw_mask); 10471 } while (newshw_mask != shw_mask); 10472 hmeblkp->hblk_shadow = NULL; 10473 } 10474 10475 /* 10476 * remove shadow bit if we are stealing an unused shadow hmeblk. 10477 * sfmmu_hblk_alloc needs it that way, will set shadow bit later if 10478 * we are indeed allocating a shadow hmeblk. 10479 */ 10480 hmeblkp->hblk_shw_bit = 0; 10481 10482 sfmmu_hblk_steal_count++; 10483 SFMMU_STAT(sf_steal_count); 10484 10485 return (1); 10486 } 10487 10488 struct hme_blk * 10489 sfmmu_hmetohblk(struct sf_hment *sfhme) 10490 { 10491 struct hme_blk *hmeblkp; 10492 struct sf_hment *sfhme0; 10493 struct hme_blk *hblk_dummy = 0; 10494 10495 /* 10496 * No dummy sf_hments, please. 10497 */ 10498 ASSERT(sfhme->hme_tte.ll != 0); 10499 10500 sfhme0 = sfhme - sfhme->hme_tte.tte_hmenum; 10501 hmeblkp = (struct hme_blk *)((uintptr_t)sfhme0 - 10502 (uintptr_t)&hblk_dummy->hblk_hme[0]); 10503 10504 return (hmeblkp); 10505 } 10506 10507 /* 10508 * On swapin, get appropriately sized TSB(s) and clear the HAT_SWAPPED flag. 10509 * If we can't get appropriately sized TSB(s), try for 8K TSB(s) using 10510 * KM_SLEEP allocation. 10511 * 10512 * Return 0 on success, -1 otherwise. 10513 */ 10514 static void 10515 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp) 10516 { 10517 struct tsb_info *tsbinfop, *next; 10518 tsb_replace_rc_t rc; 10519 boolean_t gotfirst = B_FALSE; 10520 10521 ASSERT(sfmmup != ksfmmup); 10522 ASSERT(sfmmu_hat_lock_held(sfmmup)); 10523 10524 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) { 10525 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 10526 } 10527 10528 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10529 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN); 10530 } else { 10531 return; 10532 } 10533 10534 ASSERT(sfmmup->sfmmu_tsb != NULL); 10535 10536 /* 10537 * Loop over all tsbinfo's replacing them with ones that actually have 10538 * a TSB. If any of the replacements ever fail, bail out of the loop. 10539 */ 10540 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) { 10541 ASSERT(tsbinfop->tsb_flags & TSB_SWAPPED); 10542 next = tsbinfop->tsb_next; 10543 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc, 10544 hatlockp, TSB_SWAPIN); 10545 if (rc != TSB_SUCCESS) { 10546 break; 10547 } 10548 gotfirst = B_TRUE; 10549 } 10550 10551 switch (rc) { 10552 case TSB_SUCCESS: 10553 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10554 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10555 return; 10556 case TSB_ALLOCFAIL: 10557 break; 10558 default: 10559 panic("sfmmu_replace_tsb returned unrecognized failure code " 10560 "%d", rc); 10561 } 10562 10563 /* 10564 * In this case, we failed to get one of our TSBs. If we failed to 10565 * get the first TSB, get one of minimum size (8KB). Walk the list 10566 * and throw away the tsbinfos, starting where the allocation failed; 10567 * we can get by with just one TSB as long as we don't leave the 10568 * SWAPPED tsbinfo structures lying around. 10569 */ 10570 tsbinfop = sfmmup->sfmmu_tsb; 10571 next = tsbinfop->tsb_next; 10572 tsbinfop->tsb_next = NULL; 10573 10574 sfmmu_hat_exit(hatlockp); 10575 for (tsbinfop = next; tsbinfop != NULL; tsbinfop = next) { 10576 next = tsbinfop->tsb_next; 10577 sfmmu_tsbinfo_free(tsbinfop); 10578 } 10579 hatlockp = sfmmu_hat_enter(sfmmup); 10580 10581 /* 10582 * If we don't have any TSBs, get a single 8K TSB for 8K, 64K and 512K 10583 * pages. 10584 */ 10585 if (!gotfirst) { 10586 tsbinfop = sfmmup->sfmmu_tsb; 10587 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE, 10588 hatlockp, TSB_SWAPIN | TSB_FORCEALLOC); 10589 ASSERT(rc == TSB_SUCCESS); 10590 } else { 10591 /* update machine specific tsbinfo */ 10592 sfmmu_setup_tsbinfo(sfmmup); 10593 } 10594 10595 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10596 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10597 } 10598 10599 /* 10600 * Handle exceptions for low level tsb_handler. 10601 * 10602 * There are many scenarios that could land us here: 10603 * 10604 * If the context is invalid we land here. The context can be invalid 10605 * for 3 reasons: 1) we couldn't allocate a new context and now need to 10606 * perform a wrap around operation in order to allocate a new context. 10607 * 2) Context was invalidated to change pagesize programming 3) ISMs or 10608 * TSBs configuration is changeing for this process and we are forced into 10609 * here to do a syncronization operation. If the context is valid we can 10610 * be here from window trap hanlder. In this case just call trap to handle 10611 * the fault. 10612 * 10613 * Note that the process will run in INVALID_CONTEXT before 10614 * faulting into here and subsequently loading the MMU registers 10615 * (including the TSB base register) associated with this process. 10616 * For this reason, the trap handlers must all test for 10617 * INVALID_CONTEXT before attempting to access any registers other 10618 * than the context registers. 10619 */ 10620 void 10621 sfmmu_tsbmiss_exception(struct regs *rp, uintptr_t tagaccess, uint_t traptype) 10622 { 10623 sfmmu_t *sfmmup; 10624 uint_t ctxtype; 10625 klwp_id_t lwp; 10626 char lwp_save_state; 10627 hatlock_t *hatlockp; 10628 struct tsb_info *tsbinfop; 10629 10630 SFMMU_STAT(sf_tsb_exceptions); 10631 SFMMU_MMU_STAT(mmu_tsb_exceptions); 10632 sfmmup = astosfmmu(curthread->t_procp->p_as); 10633 /* 10634 * note that in sun4u, tagacces register contains ctxnum 10635 * while sun4v passes ctxtype in the tagaccess register. 10636 */ 10637 ctxtype = tagaccess & TAGACC_CTX_MASK; 10638 10639 ASSERT(sfmmup != ksfmmup && ctxtype != KCONTEXT); 10640 ASSERT(sfmmup->sfmmu_ismhat == 0); 10641 /* 10642 * First, make sure we come out of here with a valid ctx, 10643 * since if we don't get one we'll simply loop on the 10644 * faulting instruction. 10645 * 10646 * If the ISM mappings are changing, the TSB is being relocated, or 10647 * the process is swapped out we serialize behind the controlling 10648 * thread with the sfmmu_flags and sfmmu_tsb_cv condition variable. 10649 * Otherwise we synchronize with the context stealer or the thread 10650 * that required us to change out our MMU registers (such 10651 * as a thread changing out our TSB while we were running) by 10652 * locking the HAT and grabbing the rwlock on the context as a 10653 * reader temporarily. 10654 */ 10655 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) || 10656 ctxtype == INVALID_CONTEXT); 10657 10658 if (ctxtype == INVALID_CONTEXT) { 10659 /* 10660 * Must set lwp state to LWP_SYS before 10661 * trying to acquire any adaptive lock 10662 */ 10663 lwp = ttolwp(curthread); 10664 ASSERT(lwp); 10665 lwp_save_state = lwp->lwp_state; 10666 lwp->lwp_state = LWP_SYS; 10667 10668 hatlockp = sfmmu_hat_enter(sfmmup); 10669 retry: 10670 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 10671 tsbinfop = tsbinfop->tsb_next) { 10672 if (tsbinfop->tsb_flags & TSB_RELOC_FLAG) { 10673 cv_wait(&sfmmup->sfmmu_tsb_cv, 10674 HATLOCK_MUTEXP(hatlockp)); 10675 goto retry; 10676 } 10677 } 10678 10679 /* 10680 * Wait for ISM maps to be updated. 10681 */ 10682 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) { 10683 cv_wait(&sfmmup->sfmmu_tsb_cv, 10684 HATLOCK_MUTEXP(hatlockp)); 10685 goto retry; 10686 } 10687 10688 /* 10689 * If we're swapping in, get TSB(s). Note that we must do 10690 * this before we get a ctx or load the MMU state. Once 10691 * we swap in we have to recheck to make sure the TSB(s) and 10692 * ISM mappings didn't change while we slept. 10693 */ 10694 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10695 sfmmu_tsb_swapin(sfmmup, hatlockp); 10696 goto retry; 10697 } 10698 10699 sfmmu_get_ctx(sfmmup); 10700 10701 sfmmu_hat_exit(hatlockp); 10702 /* 10703 * Must restore lwp_state if not calling 10704 * trap() for further processing. Restore 10705 * it anyway. 10706 */ 10707 lwp->lwp_state = lwp_save_state; 10708 if (sfmmup->sfmmu_ttecnt[TTE8K] != 0 || 10709 sfmmup->sfmmu_ttecnt[TTE64K] != 0 || 10710 sfmmup->sfmmu_ttecnt[TTE512K] != 0 || 10711 sfmmup->sfmmu_ttecnt[TTE4M] != 0 || 10712 sfmmup->sfmmu_ttecnt[TTE32M] != 0 || 10713 sfmmup->sfmmu_ttecnt[TTE256M] != 0) { 10714 return; 10715 } 10716 if (traptype == T_DATA_PROT) { 10717 traptype = T_DATA_MMU_MISS; 10718 } 10719 } 10720 trap(rp, (caddr_t)tagaccess, traptype, 0); 10721 } 10722 10723 /* 10724 * sfmmu_vatopfn_suspended is called from GET_TTE when TL=0 and 10725 * TTE_SUSPENDED bit set in tte we block on aquiring a page lock 10726 * rather than spinning to avoid send mondo timeouts with 10727 * interrupts enabled. When the lock is acquired it is immediately 10728 * released and we return back to sfmmu_vatopfn just after 10729 * the GET_TTE call. 10730 */ 10731 void 10732 sfmmu_vatopfn_suspended(caddr_t vaddr, sfmmu_t *sfmmu, tte_t *ttep) 10733 { 10734 struct page **pp; 10735 10736 (void) as_pagelock(sfmmu->sfmmu_as, &pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10737 as_pageunlock(sfmmu->sfmmu_as, pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10738 } 10739 10740 /* 10741 * sfmmu_tsbmiss_suspended is called from GET_TTE when TL>0 and 10742 * TTE_SUSPENDED bit set in tte. We do this so that we can handle 10743 * cross traps which cannot be handled while spinning in the 10744 * trap handlers. Simply enter and exit the kpr_suspendlock spin 10745 * mutex, which is held by the holder of the suspend bit, and then 10746 * retry the trapped instruction after unwinding. 10747 */ 10748 /*ARGSUSED*/ 10749 void 10750 sfmmu_tsbmiss_suspended(struct regs *rp, uintptr_t tagacc, uint_t traptype) 10751 { 10752 ASSERT(curthread != kreloc_thread); 10753 mutex_enter(&kpr_suspendlock); 10754 mutex_exit(&kpr_suspendlock); 10755 } 10756 10757 /* 10758 * Special routine to flush out ism mappings- TSBs, TLBs and D-caches. 10759 * This routine may be called with all cpu's captured. Therefore, the 10760 * caller is responsible for holding all locks and disabling kernel 10761 * preemption. 10762 */ 10763 /* ARGSUSED */ 10764 static void 10765 sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, 10766 struct hme_blk *hmeblkp, pfn_t pfnum, int cache_flush_flag) 10767 { 10768 cpuset_t cpuset; 10769 caddr_t va; 10770 ism_ment_t *ment; 10771 sfmmu_t *sfmmup; 10772 #ifdef VAC 10773 int vcolor; 10774 #endif 10775 int ttesz; 10776 10777 /* 10778 * Walk the ism_hat's mapping list and flush the page 10779 * from every hat sharing this ism_hat. This routine 10780 * may be called while all cpu's have been captured. 10781 * Therefore we can't attempt to grab any locks. For now 10782 * this means we will protect the ism mapping list under 10783 * a single lock which will be grabbed by the caller. 10784 * If hat_share/unshare scalibility becomes a performance 10785 * problem then we may need to re-think ism mapping list locking. 10786 */ 10787 ASSERT(ism_sfmmup->sfmmu_ismhat); 10788 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 10789 addr = addr - ISMID_STARTADDR; 10790 for (ment = ism_sfmmup->sfmmu_iment; ment; ment = ment->iment_next) { 10791 10792 sfmmup = ment->iment_hat; 10793 10794 va = ment->iment_base_va; 10795 va = (caddr_t)((uintptr_t)va + (uintptr_t)addr); 10796 10797 /* 10798 * Flush TSB of ISM mappings. 10799 */ 10800 ttesz = get_hblk_ttesz(hmeblkp); 10801 if (ttesz == TTE8K || ttesz == TTE4M) { 10802 sfmmu_unload_tsb(sfmmup, va, ttesz); 10803 } else { 10804 caddr_t sva = va; 10805 caddr_t eva; 10806 ASSERT(addr == (caddr_t)get_hblk_base(hmeblkp)); 10807 eva = sva + get_hblk_span(hmeblkp); 10808 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); 10809 } 10810 10811 cpuset = sfmmup->sfmmu_cpusran; 10812 CPUSET_AND(cpuset, cpu_ready_set); 10813 CPUSET_DEL(cpuset, CPU->cpu_id); 10814 10815 SFMMU_XCALL_STATS(sfmmup); 10816 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)va, 10817 (uint64_t)sfmmup); 10818 10819 vtag_flushpage(va, (uint64_t)sfmmup); 10820 10821 #ifdef VAC 10822 /* 10823 * Flush D$ 10824 * When flushing D$ we must flush all 10825 * cpu's. See sfmmu_cache_flush(). 10826 */ 10827 if (cache_flush_flag == CACHE_FLUSH) { 10828 cpuset = cpu_ready_set; 10829 CPUSET_DEL(cpuset, CPU->cpu_id); 10830 10831 SFMMU_XCALL_STATS(sfmmup); 10832 vcolor = addr_to_vcolor(va); 10833 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10834 vac_flushpage(pfnum, vcolor); 10835 } 10836 #endif /* VAC */ 10837 } 10838 } 10839 10840 /* 10841 * Demaps the TSB, CPU caches, and flushes all TLBs on all CPUs of 10842 * a particular virtual address and ctx. If noflush is set we do not 10843 * flush the TLB/TSB. This function may or may not be called with the 10844 * HAT lock held. 10845 */ 10846 static void 10847 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10848 pfn_t pfnum, int tlb_noflush, int cpu_flag, int cache_flush_flag, 10849 int hat_lock_held) 10850 { 10851 #ifdef VAC 10852 int vcolor; 10853 #endif 10854 cpuset_t cpuset; 10855 hatlock_t *hatlockp; 10856 10857 #if defined(lint) && !defined(VAC) 10858 pfnum = pfnum; 10859 cpu_flag = cpu_flag; 10860 cache_flush_flag = cache_flush_flag; 10861 #endif 10862 /* 10863 * There is no longer a need to protect against ctx being 10864 * stolen here since we don't store the ctx in the TSB anymore. 10865 */ 10866 #ifdef VAC 10867 vcolor = addr_to_vcolor(addr); 10868 #endif 10869 10870 /* 10871 * We must hold the hat lock during the flush of TLB, 10872 * to avoid a race with sfmmu_invalidate_ctx(), where 10873 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 10874 * causing TLB demap routine to skip flush on that MMU. 10875 * If the context on a MMU has already been set to 10876 * INVALID_CONTEXT, we just get an extra flush on 10877 * that MMU. 10878 */ 10879 if (!hat_lock_held && !tlb_noflush) 10880 hatlockp = sfmmu_hat_enter(sfmmup); 10881 10882 kpreempt_disable(); 10883 if (!tlb_noflush) { 10884 /* 10885 * Flush the TSB and TLB. 10886 */ 10887 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10888 10889 cpuset = sfmmup->sfmmu_cpusran; 10890 CPUSET_AND(cpuset, cpu_ready_set); 10891 CPUSET_DEL(cpuset, CPU->cpu_id); 10892 10893 SFMMU_XCALL_STATS(sfmmup); 10894 10895 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 10896 (uint64_t)sfmmup); 10897 10898 vtag_flushpage(addr, (uint64_t)sfmmup); 10899 } 10900 10901 if (!hat_lock_held && !tlb_noflush) 10902 sfmmu_hat_exit(hatlockp); 10903 10904 #ifdef VAC 10905 /* 10906 * Flush the D$ 10907 * 10908 * Even if the ctx is stolen, we need to flush the 10909 * cache. Our ctx stealer only flushes the TLBs. 10910 */ 10911 if (cache_flush_flag == CACHE_FLUSH) { 10912 if (cpu_flag & FLUSH_ALL_CPUS) { 10913 cpuset = cpu_ready_set; 10914 } else { 10915 cpuset = sfmmup->sfmmu_cpusran; 10916 CPUSET_AND(cpuset, cpu_ready_set); 10917 } 10918 CPUSET_DEL(cpuset, CPU->cpu_id); 10919 SFMMU_XCALL_STATS(sfmmup); 10920 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10921 vac_flushpage(pfnum, vcolor); 10922 } 10923 #endif /* VAC */ 10924 kpreempt_enable(); 10925 } 10926 10927 /* 10928 * Demaps the TSB and flushes all TLBs on all cpus for a particular virtual 10929 * address and ctx. If noflush is set we do not currently do anything. 10930 * This function may or may not be called with the HAT lock held. 10931 */ 10932 static void 10933 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10934 int tlb_noflush, int hat_lock_held) 10935 { 10936 cpuset_t cpuset; 10937 hatlock_t *hatlockp; 10938 10939 /* 10940 * If the process is exiting we have nothing to do. 10941 */ 10942 if (tlb_noflush) 10943 return; 10944 10945 /* 10946 * Flush TSB. 10947 */ 10948 if (!hat_lock_held) 10949 hatlockp = sfmmu_hat_enter(sfmmup); 10950 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10951 10952 kpreempt_disable(); 10953 10954 cpuset = sfmmup->sfmmu_cpusran; 10955 CPUSET_AND(cpuset, cpu_ready_set); 10956 CPUSET_DEL(cpuset, CPU->cpu_id); 10957 10958 SFMMU_XCALL_STATS(sfmmup); 10959 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup); 10960 10961 vtag_flushpage(addr, (uint64_t)sfmmup); 10962 10963 if (!hat_lock_held) 10964 sfmmu_hat_exit(hatlockp); 10965 10966 kpreempt_enable(); 10967 10968 } 10969 10970 /* 10971 * Special case of sfmmu_tlb_demap for MMU_PAGESIZE hblks. Use the xcall 10972 * call handler that can flush a range of pages to save on xcalls. 10973 */ 10974 static int sfmmu_xcall_save; 10975 10976 static void 10977 sfmmu_tlb_range_demap(demap_range_t *dmrp) 10978 { 10979 sfmmu_t *sfmmup = dmrp->dmr_sfmmup; 10980 hatlock_t *hatlockp; 10981 cpuset_t cpuset; 10982 uint64_t sfmmu_pgcnt; 10983 pgcnt_t pgcnt = 0; 10984 int pgunload = 0; 10985 int dirtypg = 0; 10986 caddr_t addr = dmrp->dmr_addr; 10987 caddr_t eaddr; 10988 uint64_t bitvec = dmrp->dmr_bitvec; 10989 10990 ASSERT(bitvec & 1); 10991 10992 /* 10993 * Flush TSB and calculate number of pages to flush. 10994 */ 10995 while (bitvec != 0) { 10996 dirtypg = 0; 10997 /* 10998 * Find the first page to flush and then count how many 10999 * pages there are after it that also need to be flushed. 11000 * This way the number of TSB flushes is minimized. 11001 */ 11002 while ((bitvec & 1) == 0) { 11003 pgcnt++; 11004 addr += MMU_PAGESIZE; 11005 bitvec >>= 1; 11006 } 11007 while (bitvec & 1) { 11008 dirtypg++; 11009 bitvec >>= 1; 11010 } 11011 eaddr = addr + ptob(dirtypg); 11012 hatlockp = sfmmu_hat_enter(sfmmup); 11013 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K); 11014 sfmmu_hat_exit(hatlockp); 11015 pgunload += dirtypg; 11016 addr = eaddr; 11017 pgcnt += dirtypg; 11018 } 11019 11020 ASSERT((pgcnt<<MMU_PAGESHIFT) <= dmrp->dmr_endaddr - dmrp->dmr_addr); 11021 if (sfmmup->sfmmu_free == 0) { 11022 addr = dmrp->dmr_addr; 11023 bitvec = dmrp->dmr_bitvec; 11024 11025 /* 11026 * make sure it has SFMMU_PGCNT_SHIFT bits only, 11027 * as it will be used to pack argument for xt_some 11028 */ 11029 ASSERT((pgcnt > 0) && 11030 (pgcnt <= (1 << SFMMU_PGCNT_SHIFT))); 11031 11032 /* 11033 * Encode pgcnt as (pgcnt -1 ), and pass (pgcnt - 1) in 11034 * the low 6 bits of sfmmup. This is doable since pgcnt 11035 * always >= 1. 11036 */ 11037 ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK)); 11038 sfmmu_pgcnt = (uint64_t)sfmmup | 11039 ((pgcnt - 1) & SFMMU_PGCNT_MASK); 11040 11041 /* 11042 * We must hold the hat lock during the flush of TLB, 11043 * to avoid a race with sfmmu_invalidate_ctx(), where 11044 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 11045 * causing TLB demap routine to skip flush on that MMU. 11046 * If the context on a MMU has already been set to 11047 * INVALID_CONTEXT, we just get an extra flush on 11048 * that MMU. 11049 */ 11050 hatlockp = sfmmu_hat_enter(sfmmup); 11051 kpreempt_disable(); 11052 11053 cpuset = sfmmup->sfmmu_cpusran; 11054 CPUSET_AND(cpuset, cpu_ready_set); 11055 CPUSET_DEL(cpuset, CPU->cpu_id); 11056 11057 SFMMU_XCALL_STATS(sfmmup); 11058 xt_some(cpuset, vtag_flush_pgcnt_tl1, (uint64_t)addr, 11059 sfmmu_pgcnt); 11060 11061 for (; bitvec != 0; bitvec >>= 1) { 11062 if (bitvec & 1) 11063 vtag_flushpage(addr, (uint64_t)sfmmup); 11064 addr += MMU_PAGESIZE; 11065 } 11066 kpreempt_enable(); 11067 sfmmu_hat_exit(hatlockp); 11068 11069 sfmmu_xcall_save += (pgunload-1); 11070 } 11071 dmrp->dmr_bitvec = 0; 11072 } 11073 11074 /* 11075 * In cases where we need to synchronize with TLB/TSB miss trap 11076 * handlers, _and_ need to flush the TLB, it's a lot easier to 11077 * throw away the context from the process than to do a 11078 * special song and dance to keep things consistent for the 11079 * handlers. 11080 * 11081 * Since the process suddenly ends up without a context and our caller 11082 * holds the hat lock, threads that fault after this function is called 11083 * will pile up on the lock. We can then do whatever we need to 11084 * atomically from the context of the caller. The first blocked thread 11085 * to resume executing will get the process a new context, and the 11086 * process will resume executing. 11087 * 11088 * One added advantage of this approach is that on MMUs that 11089 * support a "flush all" operation, we will delay the flush until 11090 * cnum wrap-around, and then flush the TLB one time. This 11091 * is rather rare, so it's a lot less expensive than making 8000 11092 * x-calls to flush the TLB 8000 times. 11093 * 11094 * A per-process (PP) lock is used to synchronize ctx allocations in 11095 * resume() and ctx invalidations here. 11096 */ 11097 static void 11098 sfmmu_invalidate_ctx(sfmmu_t *sfmmup) 11099 { 11100 cpuset_t cpuset; 11101 int cnum, currcnum; 11102 mmu_ctx_t *mmu_ctxp; 11103 int i; 11104 uint_t pstate_save; 11105 11106 SFMMU_STAT(sf_ctx_inv); 11107 11108 ASSERT(sfmmu_hat_lock_held(sfmmup)); 11109 ASSERT(sfmmup != ksfmmup); 11110 11111 kpreempt_disable(); 11112 11113 mmu_ctxp = CPU_MMU_CTXP(CPU); 11114 ASSERT(mmu_ctxp); 11115 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 11116 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 11117 11118 currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; 11119 11120 pstate_save = sfmmu_disable_intrs(); 11121 11122 lock_set(&sfmmup->sfmmu_ctx_lock); /* acquire PP lock */ 11123 /* set HAT cnum invalid across all context domains. */ 11124 for (i = 0; i < max_mmu_ctxdoms; i++) { 11125 11126 cnum = sfmmup->sfmmu_ctxs[i].cnum; 11127 if (cnum == INVALID_CONTEXT) { 11128 continue; 11129 } 11130 11131 sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT; 11132 } 11133 membar_enter(); /* make sure globally visible to all CPUs */ 11134 lock_clear(&sfmmup->sfmmu_ctx_lock); /* release PP lock */ 11135 11136 sfmmu_enable_intrs(pstate_save); 11137 11138 cpuset = sfmmup->sfmmu_cpusran; 11139 CPUSET_DEL(cpuset, CPU->cpu_id); 11140 CPUSET_AND(cpuset, cpu_ready_set); 11141 if (!CPUSET_ISNULL(cpuset)) { 11142 SFMMU_XCALL_STATS(sfmmup); 11143 xt_some(cpuset, sfmmu_raise_tsb_exception, 11144 (uint64_t)sfmmup, INVALID_CONTEXT); 11145 xt_sync(cpuset); 11146 SFMMU_STAT(sf_tsb_raise_exception); 11147 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 11148 } 11149 11150 /* 11151 * If the hat to-be-invalidated is the same as the current 11152 * process on local CPU we need to invalidate 11153 * this CPU context as well. 11154 */ 11155 if ((sfmmu_getctx_sec() == currcnum) && 11156 (currcnum != INVALID_CONTEXT)) { 11157 sfmmu_setctx_sec(INVALID_CONTEXT); 11158 sfmmu_clear_utsbinfo(); 11159 } 11160 11161 kpreempt_enable(); 11162 11163 /* 11164 * we hold the hat lock, so nobody should allocate a context 11165 * for us yet 11166 */ 11167 ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT); 11168 } 11169 11170 #ifdef VAC 11171 /* 11172 * We need to flush the cache in all cpus. It is possible that 11173 * a process referenced a page as cacheable but has sinced exited 11174 * and cleared the mapping list. We still to flush it but have no 11175 * state so all cpus is the only alternative. 11176 */ 11177 void 11178 sfmmu_cache_flush(pfn_t pfnum, int vcolor) 11179 { 11180 cpuset_t cpuset; 11181 11182 kpreempt_disable(); 11183 cpuset = cpu_ready_set; 11184 CPUSET_DEL(cpuset, CPU->cpu_id); 11185 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11186 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 11187 xt_sync(cpuset); 11188 vac_flushpage(pfnum, vcolor); 11189 kpreempt_enable(); 11190 } 11191 11192 void 11193 sfmmu_cache_flushcolor(int vcolor, pfn_t pfnum) 11194 { 11195 cpuset_t cpuset; 11196 11197 ASSERT(vcolor >= 0); 11198 11199 kpreempt_disable(); 11200 cpuset = cpu_ready_set; 11201 CPUSET_DEL(cpuset, CPU->cpu_id); 11202 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11203 xt_some(cpuset, vac_flushcolor_tl1, vcolor, pfnum); 11204 xt_sync(cpuset); 11205 vac_flushcolor(vcolor, pfnum); 11206 kpreempt_enable(); 11207 } 11208 #endif /* VAC */ 11209 11210 /* 11211 * We need to prevent processes from accessing the TSB using a cached physical 11212 * address. It's alright if they try to access the TSB via virtual address 11213 * since they will just fault on that virtual address once the mapping has 11214 * been suspended. 11215 */ 11216 #pragma weak sendmondo_in_recover 11217 11218 /* ARGSUSED */ 11219 static int 11220 sfmmu_tsb_pre_relocator(caddr_t va, uint_t tsbsz, uint_t flags, void *tsbinfo) 11221 { 11222 hatlock_t *hatlockp; 11223 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11224 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11225 extern uint32_t sendmondo_in_recover; 11226 11227 if (flags != HAT_PRESUSPEND) 11228 return (0); 11229 11230 hatlockp = sfmmu_hat_enter(sfmmup); 11231 11232 tsbinfop->tsb_flags |= TSB_RELOC_FLAG; 11233 11234 /* 11235 * For Cheetah+ Erratum 25: 11236 * Wait for any active recovery to finish. We can't risk 11237 * relocating the TSB of the thread running mondo_recover_proc() 11238 * since, if we did that, we would deadlock. The scenario we are 11239 * trying to avoid is as follows: 11240 * 11241 * THIS CPU RECOVER CPU 11242 * -------- ----------- 11243 * Begins recovery, walking through TSB 11244 * hat_pagesuspend() TSB TTE 11245 * TLB miss on TSB TTE, spins at TL1 11246 * xt_sync() 11247 * send_mondo_timeout() 11248 * mondo_recover_proc() 11249 * ((deadlocked)) 11250 * 11251 * The second half of the workaround is that mondo_recover_proc() 11252 * checks to see if the tsb_info has the RELOC flag set, and if it 11253 * does, it skips over that TSB without ever touching tsbinfop->tsb_va 11254 * and hence avoiding the TLB miss that could result in a deadlock. 11255 */ 11256 if (&sendmondo_in_recover) { 11257 membar_enter(); /* make sure RELOC flag visible */ 11258 while (sendmondo_in_recover) { 11259 drv_usecwait(1); 11260 membar_consumer(); 11261 } 11262 } 11263 11264 sfmmu_invalidate_ctx(sfmmup); 11265 sfmmu_hat_exit(hatlockp); 11266 11267 return (0); 11268 } 11269 11270 /* ARGSUSED */ 11271 static int 11272 sfmmu_tsb_post_relocator(caddr_t va, uint_t tsbsz, uint_t flags, 11273 void *tsbinfo, pfn_t newpfn) 11274 { 11275 hatlock_t *hatlockp; 11276 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11277 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11278 11279 if (flags != HAT_POSTUNSUSPEND) 11280 return (0); 11281 11282 hatlockp = sfmmu_hat_enter(sfmmup); 11283 11284 SFMMU_STAT(sf_tsb_reloc); 11285 11286 /* 11287 * The process may have swapped out while we were relocating one 11288 * of its TSBs. If so, don't bother doing the setup since the 11289 * process can't be using the memory anymore. 11290 */ 11291 if ((tsbinfop->tsb_flags & TSB_SWAPPED) == 0) { 11292 ASSERT(va == tsbinfop->tsb_va); 11293 sfmmu_tsbinfo_setup_phys(tsbinfop, newpfn); 11294 sfmmu_setup_tsbinfo(sfmmup); 11295 11296 if (tsbinfop->tsb_flags & TSB_FLUSH_NEEDED) { 11297 sfmmu_inv_tsb(tsbinfop->tsb_va, 11298 TSB_BYTES(tsbinfop->tsb_szc)); 11299 tsbinfop->tsb_flags &= ~TSB_FLUSH_NEEDED; 11300 } 11301 } 11302 11303 membar_exit(); 11304 tsbinfop->tsb_flags &= ~TSB_RELOC_FLAG; 11305 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 11306 11307 sfmmu_hat_exit(hatlockp); 11308 11309 return (0); 11310 } 11311 11312 /* 11313 * Allocate and initialize a tsb_info structure. Note that we may or may not 11314 * allocate a TSB here, depending on the flags passed in. 11315 */ 11316 static int 11317 sfmmu_tsbinfo_alloc(struct tsb_info **tsbinfopp, int tsb_szc, int tte_sz_mask, 11318 uint_t flags, sfmmu_t *sfmmup) 11319 { 11320 int err; 11321 11322 *tsbinfopp = (struct tsb_info *)kmem_cache_alloc( 11323 sfmmu_tsbinfo_cache, KM_SLEEP); 11324 11325 if ((err = sfmmu_init_tsbinfo(*tsbinfopp, tte_sz_mask, 11326 tsb_szc, flags, sfmmup)) != 0) { 11327 kmem_cache_free(sfmmu_tsbinfo_cache, *tsbinfopp); 11328 SFMMU_STAT(sf_tsb_allocfail); 11329 *tsbinfopp = NULL; 11330 return (err); 11331 } 11332 SFMMU_STAT(sf_tsb_alloc); 11333 11334 /* 11335 * Bump the TSB size counters for this TSB size. 11336 */ 11337 (*(((int *)&sfmmu_tsbsize_stat) + tsb_szc))++; 11338 return (0); 11339 } 11340 11341 static void 11342 sfmmu_tsb_free(struct tsb_info *tsbinfo) 11343 { 11344 caddr_t tsbva = tsbinfo->tsb_va; 11345 uint_t tsb_size = TSB_BYTES(tsbinfo->tsb_szc); 11346 struct kmem_cache *kmem_cachep = tsbinfo->tsb_cache; 11347 vmem_t *vmp = tsbinfo->tsb_vmp; 11348 11349 /* 11350 * If we allocated this TSB from relocatable kernel memory, then we 11351 * need to uninstall the callback handler. 11352 */ 11353 if (tsbinfo->tsb_cache != sfmmu_tsb8k_cache) { 11354 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11355 caddr_t slab_vaddr = (caddr_t)((uintptr_t)tsbva & slab_mask); 11356 page_t **ppl; 11357 int ret; 11358 11359 ret = as_pagelock(&kas, &ppl, slab_vaddr, PAGESIZE, S_WRITE); 11360 ASSERT(ret == 0); 11361 hat_delete_callback(tsbva, (uint_t)tsb_size, (void *)tsbinfo, 11362 0, NULL); 11363 as_pageunlock(&kas, ppl, slab_vaddr, PAGESIZE, S_WRITE); 11364 } 11365 11366 if (kmem_cachep != NULL) { 11367 kmem_cache_free(kmem_cachep, tsbva); 11368 } else { 11369 vmem_xfree(vmp, (void *)tsbva, tsb_size); 11370 } 11371 tsbinfo->tsb_va = (caddr_t)0xbad00bad; 11372 atomic_add_64(&tsb_alloc_bytes, -(int64_t)tsb_size); 11373 } 11374 11375 static void 11376 sfmmu_tsbinfo_free(struct tsb_info *tsbinfo) 11377 { 11378 if ((tsbinfo->tsb_flags & TSB_SWAPPED) == 0) { 11379 sfmmu_tsb_free(tsbinfo); 11380 } 11381 kmem_cache_free(sfmmu_tsbinfo_cache, tsbinfo); 11382 11383 } 11384 11385 /* 11386 * Setup all the references to physical memory for this tsbinfo. 11387 * The underlying page(s) must be locked. 11388 */ 11389 static void 11390 sfmmu_tsbinfo_setup_phys(struct tsb_info *tsbinfo, pfn_t pfn) 11391 { 11392 ASSERT(pfn != PFN_INVALID); 11393 ASSERT(pfn == va_to_pfn(tsbinfo->tsb_va)); 11394 11395 #ifndef sun4v 11396 if (tsbinfo->tsb_szc == 0) { 11397 sfmmu_memtte(&tsbinfo->tsb_tte, pfn, 11398 PROT_WRITE|PROT_READ, TTE8K); 11399 } else { 11400 /* 11401 * Round down PA and use a large mapping; the handlers will 11402 * compute the TSB pointer at the correct offset into the 11403 * big virtual page. NOTE: this assumes all TSBs larger 11404 * than 8K must come from physically contiguous slabs of 11405 * size tsb_slab_size. 11406 */ 11407 sfmmu_memtte(&tsbinfo->tsb_tte, pfn & ~tsb_slab_mask, 11408 PROT_WRITE|PROT_READ, tsb_slab_ttesz); 11409 } 11410 tsbinfo->tsb_pa = ptob(pfn); 11411 11412 TTE_SET_LOCKED(&tsbinfo->tsb_tte); /* lock the tte into dtlb */ 11413 TTE_SET_MOD(&tsbinfo->tsb_tte); /* enable writes */ 11414 11415 ASSERT(TTE_IS_PRIVILEGED(&tsbinfo->tsb_tte)); 11416 ASSERT(TTE_IS_LOCKED(&tsbinfo->tsb_tte)); 11417 #else /* sun4v */ 11418 tsbinfo->tsb_pa = ptob(pfn); 11419 #endif /* sun4v */ 11420 } 11421 11422 11423 /* 11424 * Returns zero on success, ENOMEM if over the high water mark, 11425 * or EAGAIN if the caller needs to retry with a smaller TSB 11426 * size (or specify TSB_FORCEALLOC if the allocation can't fail). 11427 * 11428 * This call cannot fail to allocate a TSB if TSB_FORCEALLOC 11429 * is specified and the TSB requested is PAGESIZE, though it 11430 * may sleep waiting for memory if sufficient memory is not 11431 * available. 11432 */ 11433 static int 11434 sfmmu_init_tsbinfo(struct tsb_info *tsbinfo, int tteszmask, 11435 int tsbcode, uint_t flags, sfmmu_t *sfmmup) 11436 { 11437 caddr_t vaddr = NULL; 11438 caddr_t slab_vaddr; 11439 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11440 int tsbbytes = TSB_BYTES(tsbcode); 11441 int lowmem = 0; 11442 struct kmem_cache *kmem_cachep = NULL; 11443 vmem_t *vmp = NULL; 11444 lgrp_id_t lgrpid = LGRP_NONE; 11445 pfn_t pfn; 11446 uint_t cbflags = HAC_SLEEP; 11447 page_t **pplist; 11448 int ret; 11449 11450 if (flags & (TSB_FORCEALLOC | TSB_SWAPIN | TSB_GROW | TSB_SHRINK)) 11451 flags |= TSB_ALLOC; 11452 11453 ASSERT((flags & TSB_FORCEALLOC) == 0 || tsbcode == TSB_MIN_SZCODE); 11454 11455 tsbinfo->tsb_sfmmu = sfmmup; 11456 11457 /* 11458 * If not allocating a TSB, set up the tsbinfo, set TSB_SWAPPED, and 11459 * return. 11460 */ 11461 if ((flags & TSB_ALLOC) == 0) { 11462 tsbinfo->tsb_szc = tsbcode; 11463 tsbinfo->tsb_ttesz_mask = tteszmask; 11464 tsbinfo->tsb_va = (caddr_t)0xbadbadbeef; 11465 tsbinfo->tsb_pa = -1; 11466 tsbinfo->tsb_tte.ll = 0; 11467 tsbinfo->tsb_next = NULL; 11468 tsbinfo->tsb_flags = TSB_SWAPPED; 11469 tsbinfo->tsb_cache = NULL; 11470 tsbinfo->tsb_vmp = NULL; 11471 return (0); 11472 } 11473 11474 #ifdef DEBUG 11475 /* 11476 * For debugging: 11477 * Randomly force allocation failures every tsb_alloc_mtbf 11478 * tries if TSB_FORCEALLOC is not specified. This will 11479 * return ENOMEM if tsb_alloc_mtbf is odd, or EAGAIN if 11480 * it is even, to allow testing of both failure paths... 11481 */ 11482 if (tsb_alloc_mtbf && ((flags & TSB_FORCEALLOC) == 0) && 11483 (tsb_alloc_count++ == tsb_alloc_mtbf)) { 11484 tsb_alloc_count = 0; 11485 tsb_alloc_fail_mtbf++; 11486 return ((tsb_alloc_mtbf & 1)? ENOMEM : EAGAIN); 11487 } 11488 #endif /* DEBUG */ 11489 11490 /* 11491 * Enforce high water mark if we are not doing a forced allocation 11492 * and are not shrinking a process' TSB. 11493 */ 11494 if ((flags & TSB_SHRINK) == 0 && 11495 (tsbbytes + tsb_alloc_bytes) > tsb_alloc_hiwater) { 11496 if ((flags & TSB_FORCEALLOC) == 0) 11497 return (ENOMEM); 11498 lowmem = 1; 11499 } 11500 11501 /* 11502 * Allocate from the correct location based upon the size of the TSB 11503 * compared to the base page size, and what memory conditions dictate. 11504 * Note we always do nonblocking allocations from the TSB arena since 11505 * we don't want memory fragmentation to cause processes to block 11506 * indefinitely waiting for memory; until the kernel algorithms that 11507 * coalesce large pages are improved this is our best option. 11508 * 11509 * Algorithm: 11510 * If allocating a "large" TSB (>8K), allocate from the 11511 * appropriate kmem_tsb_default_arena vmem arena 11512 * else if low on memory or the TSB_FORCEALLOC flag is set or 11513 * tsb_forceheap is set 11514 * Allocate from kernel heap via sfmmu_tsb8k_cache with 11515 * KM_SLEEP (never fails) 11516 * else 11517 * Allocate from appropriate sfmmu_tsb_cache with 11518 * KM_NOSLEEP 11519 * endif 11520 */ 11521 if (tsb_lgrp_affinity) 11522 lgrpid = lgrp_home_id(curthread); 11523 if (lgrpid == LGRP_NONE) 11524 lgrpid = 0; /* use lgrp of boot CPU */ 11525 11526 if (tsbbytes > MMU_PAGESIZE) { 11527 vmp = kmem_tsb_default_arena[lgrpid]; 11528 vaddr = (caddr_t)vmem_xalloc(vmp, tsbbytes, tsbbytes, 0, 0, 11529 NULL, NULL, VM_NOSLEEP); 11530 #ifdef DEBUG 11531 } else if (lowmem || (flags & TSB_FORCEALLOC) || tsb_forceheap) { 11532 #else /* !DEBUG */ 11533 } else if (lowmem || (flags & TSB_FORCEALLOC)) { 11534 #endif /* DEBUG */ 11535 kmem_cachep = sfmmu_tsb8k_cache; 11536 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_SLEEP); 11537 ASSERT(vaddr != NULL); 11538 } else { 11539 kmem_cachep = sfmmu_tsb_cache[lgrpid]; 11540 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_NOSLEEP); 11541 } 11542 11543 tsbinfo->tsb_cache = kmem_cachep; 11544 tsbinfo->tsb_vmp = vmp; 11545 11546 if (vaddr == NULL) { 11547 return (EAGAIN); 11548 } 11549 11550 atomic_add_64(&tsb_alloc_bytes, (int64_t)tsbbytes); 11551 kmem_cachep = tsbinfo->tsb_cache; 11552 11553 /* 11554 * If we are allocating from outside the cage, then we need to 11555 * register a relocation callback handler. Note that for now 11556 * since pseudo mappings always hang off of the slab's root page, 11557 * we need only lock the first 8K of the TSB slab. This is a bit 11558 * hacky but it is good for performance. 11559 */ 11560 if (kmem_cachep != sfmmu_tsb8k_cache) { 11561 slab_vaddr = (caddr_t)((uintptr_t)vaddr & slab_mask); 11562 ret = as_pagelock(&kas, &pplist, slab_vaddr, PAGESIZE, S_WRITE); 11563 ASSERT(ret == 0); 11564 ret = hat_add_callback(sfmmu_tsb_cb_id, vaddr, (uint_t)tsbbytes, 11565 cbflags, (void *)tsbinfo, &pfn, NULL); 11566 11567 /* 11568 * Need to free up resources if we could not successfully 11569 * add the callback function and return an error condition. 11570 */ 11571 if (ret != 0) { 11572 if (kmem_cachep) { 11573 kmem_cache_free(kmem_cachep, vaddr); 11574 } else { 11575 vmem_xfree(vmp, (void *)vaddr, tsbbytes); 11576 } 11577 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, 11578 S_WRITE); 11579 return (EAGAIN); 11580 } 11581 } else { 11582 /* 11583 * Since allocation of 8K TSBs from heap is rare and occurs 11584 * during memory pressure we allocate them from permanent 11585 * memory rather than using callbacks to get the PFN. 11586 */ 11587 pfn = hat_getpfnum(kas.a_hat, vaddr); 11588 } 11589 11590 tsbinfo->tsb_va = vaddr; 11591 tsbinfo->tsb_szc = tsbcode; 11592 tsbinfo->tsb_ttesz_mask = tteszmask; 11593 tsbinfo->tsb_next = NULL; 11594 tsbinfo->tsb_flags = 0; 11595 11596 sfmmu_tsbinfo_setup_phys(tsbinfo, pfn); 11597 11598 if (kmem_cachep != sfmmu_tsb8k_cache) { 11599 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, S_WRITE); 11600 } 11601 11602 sfmmu_inv_tsb(vaddr, tsbbytes); 11603 return (0); 11604 } 11605 11606 /* 11607 * Initialize per cpu tsb and per cpu tsbmiss_area 11608 */ 11609 void 11610 sfmmu_init_tsbs(void) 11611 { 11612 int i; 11613 struct tsbmiss *tsbmissp; 11614 struct kpmtsbm *kpmtsbmp; 11615 #ifndef sun4v 11616 extern int dcache_line_mask; 11617 #endif /* sun4v */ 11618 extern uint_t vac_colors; 11619 11620 /* 11621 * Init. tsb miss area. 11622 */ 11623 tsbmissp = tsbmiss_area; 11624 11625 for (i = 0; i < NCPU; tsbmissp++, i++) { 11626 /* 11627 * initialize the tsbmiss area. 11628 * Do this for all possible CPUs as some may be added 11629 * while the system is running. There is no cost to this. 11630 */ 11631 tsbmissp->ksfmmup = ksfmmup; 11632 #ifndef sun4v 11633 tsbmissp->dcache_line_mask = (uint16_t)dcache_line_mask; 11634 #endif /* sun4v */ 11635 tsbmissp->khashstart = 11636 (struct hmehash_bucket *)va_to_pa((caddr_t)khme_hash); 11637 tsbmissp->uhashstart = 11638 (struct hmehash_bucket *)va_to_pa((caddr_t)uhme_hash); 11639 tsbmissp->khashsz = khmehash_num; 11640 tsbmissp->uhashsz = uhmehash_num; 11641 } 11642 11643 sfmmu_tsb_cb_id = hat_register_callback('T'<<16 | 'S' << 8 | 'B', 11644 sfmmu_tsb_pre_relocator, sfmmu_tsb_post_relocator, NULL, 0); 11645 11646 if (kpm_enable == 0) 11647 return; 11648 11649 /* -- Begin KPM specific init -- */ 11650 11651 if (kpm_smallpages) { 11652 /* 11653 * If we're using base pagesize pages for seg_kpm 11654 * mappings, we use the kernel TSB since we can't afford 11655 * to allocate a second huge TSB for these mappings. 11656 */ 11657 kpm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11658 kpm_tsbsz = ktsb_szcode; 11659 kpmsm_tsbbase = kpm_tsbbase; 11660 kpmsm_tsbsz = kpm_tsbsz; 11661 } else { 11662 /* 11663 * In VAC conflict case, just put the entries in the 11664 * kernel 8K indexed TSB for now so we can find them. 11665 * This could really be changed in the future if we feel 11666 * the need... 11667 */ 11668 kpmsm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11669 kpmsm_tsbsz = ktsb_szcode; 11670 kpm_tsbbase = ktsb_phys? ktsb4m_pbase : (uint64_t)ktsb4m_base; 11671 kpm_tsbsz = ktsb4m_szcode; 11672 } 11673 11674 kpmtsbmp = kpmtsbm_area; 11675 for (i = 0; i < NCPU; kpmtsbmp++, i++) { 11676 /* 11677 * Initialize the kpmtsbm area. 11678 * Do this for all possible CPUs as some may be added 11679 * while the system is running. There is no cost to this. 11680 */ 11681 kpmtsbmp->vbase = kpm_vbase; 11682 kpmtsbmp->vend = kpm_vbase + kpm_size * vac_colors; 11683 kpmtsbmp->sz_shift = kpm_size_shift; 11684 kpmtsbmp->kpmp_shift = kpmp_shift; 11685 kpmtsbmp->kpmp2pshft = (uchar_t)kpmp2pshft; 11686 if (kpm_smallpages == 0) { 11687 kpmtsbmp->kpmp_table_sz = kpmp_table_sz; 11688 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_table); 11689 } else { 11690 kpmtsbmp->kpmp_table_sz = kpmp_stable_sz; 11691 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_stable); 11692 } 11693 kpmtsbmp->msegphashpa = va_to_pa(memseg_phash); 11694 kpmtsbmp->flags = KPMTSBM_ENABLE_FLAG; 11695 #ifdef DEBUG 11696 kpmtsbmp->flags |= (kpm_tsbmtl) ? KPMTSBM_TLTSBM_FLAG : 0; 11697 #endif /* DEBUG */ 11698 if (ktsb_phys) 11699 kpmtsbmp->flags |= KPMTSBM_TSBPHYS_FLAG; 11700 } 11701 11702 /* -- End KPM specific init -- */ 11703 } 11704 11705 /* Avoid using sfmmu_tsbinfo_alloc() to avoid kmem_alloc - no real reason */ 11706 struct tsb_info ktsb_info[2]; 11707 11708 /* 11709 * Called from hat_kern_setup() to setup the tsb_info for ksfmmup. 11710 */ 11711 void 11712 sfmmu_init_ktsbinfo() 11713 { 11714 ASSERT(ksfmmup != NULL); 11715 ASSERT(ksfmmup->sfmmu_tsb == NULL); 11716 /* 11717 * Allocate tsbinfos for kernel and copy in data 11718 * to make debug easier and sun4v setup easier. 11719 */ 11720 ktsb_info[0].tsb_sfmmu = ksfmmup; 11721 ktsb_info[0].tsb_szc = ktsb_szcode; 11722 ktsb_info[0].tsb_ttesz_mask = TSB8K|TSB64K|TSB512K; 11723 ktsb_info[0].tsb_va = ktsb_base; 11724 ktsb_info[0].tsb_pa = ktsb_pbase; 11725 ktsb_info[0].tsb_flags = 0; 11726 ktsb_info[0].tsb_tte.ll = 0; 11727 ktsb_info[0].tsb_cache = NULL; 11728 11729 ktsb_info[1].tsb_sfmmu = ksfmmup; 11730 ktsb_info[1].tsb_szc = ktsb4m_szcode; 11731 ktsb_info[1].tsb_ttesz_mask = TSB4M; 11732 ktsb_info[1].tsb_va = ktsb4m_base; 11733 ktsb_info[1].tsb_pa = ktsb4m_pbase; 11734 ktsb_info[1].tsb_flags = 0; 11735 ktsb_info[1].tsb_tte.ll = 0; 11736 ktsb_info[1].tsb_cache = NULL; 11737 11738 /* Link them into ksfmmup. */ 11739 ktsb_info[0].tsb_next = &ktsb_info[1]; 11740 ktsb_info[1].tsb_next = NULL; 11741 ksfmmup->sfmmu_tsb = &ktsb_info[0]; 11742 11743 sfmmu_setup_tsbinfo(ksfmmup); 11744 } 11745 11746 /* 11747 * Cache the last value returned from va_to_pa(). If the VA specified 11748 * in the current call to cached_va_to_pa() maps to the same Page (as the 11749 * previous call to cached_va_to_pa()), then compute the PA using 11750 * cached info, else call va_to_pa(). 11751 * 11752 * Note: this function is neither MT-safe nor consistent in the presence 11753 * of multiple, interleaved threads. This function was created to enable 11754 * an optimization used during boot (at a point when there's only one thread 11755 * executing on the "boot CPU", and before startup_vm() has been called). 11756 */ 11757 static uint64_t 11758 cached_va_to_pa(void *vaddr) 11759 { 11760 static uint64_t prev_vaddr_base = 0; 11761 static uint64_t prev_pfn = 0; 11762 11763 if ((((uint64_t)vaddr) & MMU_PAGEMASK) == prev_vaddr_base) { 11764 return (prev_pfn | ((uint64_t)vaddr & MMU_PAGEOFFSET)); 11765 } else { 11766 uint64_t pa = va_to_pa(vaddr); 11767 11768 if (pa != ((uint64_t)-1)) { 11769 /* 11770 * Computed physical address is valid. Cache its 11771 * related info for the next cached_va_to_pa() call. 11772 */ 11773 prev_pfn = pa & MMU_PAGEMASK; 11774 prev_vaddr_base = ((uint64_t)vaddr) & MMU_PAGEMASK; 11775 } 11776 11777 return (pa); 11778 } 11779 } 11780 11781 /* 11782 * Carve up our nucleus hblk region. We may allocate more hblks than 11783 * asked due to rounding errors but we are guaranteed to have at least 11784 * enough space to allocate the requested number of hblk8's and hblk1's. 11785 */ 11786 void 11787 sfmmu_init_nucleus_hblks(caddr_t addr, size_t size, int nhblk8, int nhblk1) 11788 { 11789 struct hme_blk *hmeblkp; 11790 size_t hme8blk_sz, hme1blk_sz; 11791 size_t i; 11792 size_t hblk8_bound; 11793 ulong_t j = 0, k = 0; 11794 11795 ASSERT(addr != NULL && size != 0); 11796 11797 /* Need to use proper structure alignment */ 11798 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 11799 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 11800 11801 nucleus_hblk8.list = (void *)addr; 11802 nucleus_hblk8.index = 0; 11803 11804 /* 11805 * Use as much memory as possible for hblk8's since we 11806 * expect all bop_alloc'ed memory to be allocated in 8k chunks. 11807 * We need to hold back enough space for the hblk1's which 11808 * we'll allocate next. 11809 */ 11810 hblk8_bound = size - (nhblk1 * hme1blk_sz) - hme8blk_sz; 11811 for (i = 0; i <= hblk8_bound; i += hme8blk_sz, j++) { 11812 hmeblkp = (struct hme_blk *)addr; 11813 addr += hme8blk_sz; 11814 hmeblkp->hblk_nuc_bit = 1; 11815 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11816 } 11817 nucleus_hblk8.len = j; 11818 ASSERT(j >= nhblk8); 11819 SFMMU_STAT_ADD(sf_hblk8_ncreate, j); 11820 11821 nucleus_hblk1.list = (void *)addr; 11822 nucleus_hblk1.index = 0; 11823 for (; i <= (size - hme1blk_sz); i += hme1blk_sz, k++) { 11824 hmeblkp = (struct hme_blk *)addr; 11825 addr += hme1blk_sz; 11826 hmeblkp->hblk_nuc_bit = 1; 11827 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11828 } 11829 ASSERT(k >= nhblk1); 11830 nucleus_hblk1.len = k; 11831 SFMMU_STAT_ADD(sf_hblk1_ncreate, k); 11832 } 11833 11834 /* 11835 * This function is currently not supported on this platform. For what 11836 * it's supposed to do, see hat.c and hat_srmmu.c 11837 */ 11838 /* ARGSUSED */ 11839 faultcode_t 11840 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp, 11841 uint_t flags) 11842 { 11843 ASSERT(hat->sfmmu_xhat_provider == NULL); 11844 return (FC_NOSUPPORT); 11845 } 11846 11847 /* 11848 * Searchs the mapping list of the page for a mapping of the same size. If not 11849 * found the corresponding bit is cleared in the p_index field. When large 11850 * pages are more prevalent in the system, we can maintain the mapping list 11851 * in order and we don't have to traverse the list each time. Just check the 11852 * next and prev entries, and if both are of different size, we clear the bit. 11853 */ 11854 static void 11855 sfmmu_rm_large_mappings(page_t *pp, int ttesz) 11856 { 11857 struct sf_hment *sfhmep; 11858 struct hme_blk *hmeblkp; 11859 int index; 11860 pgcnt_t npgs; 11861 11862 ASSERT(ttesz > TTE8K); 11863 11864 ASSERT(sfmmu_mlist_held(pp)); 11865 11866 ASSERT(PP_ISMAPPED_LARGE(pp)); 11867 11868 /* 11869 * Traverse mapping list looking for another mapping of same size. 11870 * since we only want to clear index field if all mappings of 11871 * that size are gone. 11872 */ 11873 11874 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 11875 hmeblkp = sfmmu_hmetohblk(sfhmep); 11876 if (hmeblkp->hblk_xhat_bit) 11877 continue; 11878 if (hme_size(sfhmep) == ttesz) { 11879 /* 11880 * another mapping of the same size. don't clear index. 11881 */ 11882 return; 11883 } 11884 } 11885 11886 /* 11887 * Clear the p_index bit for large page. 11888 */ 11889 index = PAGESZ_TO_INDEX(ttesz); 11890 npgs = TTEPAGES(ttesz); 11891 while (npgs-- > 0) { 11892 ASSERT(pp->p_index & index); 11893 pp->p_index &= ~index; 11894 pp = PP_PAGENEXT(pp); 11895 } 11896 } 11897 11898 /* 11899 * return supported features 11900 */ 11901 /* ARGSUSED */ 11902 int 11903 hat_supported(enum hat_features feature, void *arg) 11904 { 11905 switch (feature) { 11906 case HAT_SHARED_PT: 11907 case HAT_DYNAMIC_ISM_UNMAP: 11908 case HAT_VMODSORT: 11909 return (1); 11910 default: 11911 return (0); 11912 } 11913 } 11914 11915 void 11916 hat_enter(struct hat *hat) 11917 { 11918 hatlock_t *hatlockp; 11919 11920 if (hat != ksfmmup) { 11921 hatlockp = TSB_HASH(hat); 11922 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 11923 } 11924 } 11925 11926 void 11927 hat_exit(struct hat *hat) 11928 { 11929 hatlock_t *hatlockp; 11930 11931 if (hat != ksfmmup) { 11932 hatlockp = TSB_HASH(hat); 11933 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 11934 } 11935 } 11936 11937 /*ARGSUSED*/ 11938 void 11939 hat_reserve(struct as *as, caddr_t addr, size_t len) 11940 { 11941 } 11942 11943 static void 11944 hat_kstat_init(void) 11945 { 11946 kstat_t *ksp; 11947 11948 ksp = kstat_create("unix", 0, "sfmmu_global_stat", "hat", 11949 KSTAT_TYPE_RAW, sizeof (struct sfmmu_global_stat), 11950 KSTAT_FLAG_VIRTUAL); 11951 if (ksp) { 11952 ksp->ks_data = (void *) &sfmmu_global_stat; 11953 kstat_install(ksp); 11954 } 11955 ksp = kstat_create("unix", 0, "sfmmu_tsbsize_stat", "hat", 11956 KSTAT_TYPE_RAW, sizeof (struct sfmmu_tsbsize_stat), 11957 KSTAT_FLAG_VIRTUAL); 11958 if (ksp) { 11959 ksp->ks_data = (void *) &sfmmu_tsbsize_stat; 11960 kstat_install(ksp); 11961 } 11962 ksp = kstat_create("unix", 0, "sfmmu_percpu_stat", "hat", 11963 KSTAT_TYPE_RAW, sizeof (struct sfmmu_percpu_stat) * NCPU, 11964 KSTAT_FLAG_WRITABLE); 11965 if (ksp) { 11966 ksp->ks_update = sfmmu_kstat_percpu_update; 11967 kstat_install(ksp); 11968 } 11969 } 11970 11971 /* ARGSUSED */ 11972 static int 11973 sfmmu_kstat_percpu_update(kstat_t *ksp, int rw) 11974 { 11975 struct sfmmu_percpu_stat *cpu_kstat = ksp->ks_data; 11976 struct tsbmiss *tsbm = tsbmiss_area; 11977 struct kpmtsbm *kpmtsbm = kpmtsbm_area; 11978 int i; 11979 11980 ASSERT(cpu_kstat); 11981 if (rw == KSTAT_READ) { 11982 for (i = 0; i < NCPU; cpu_kstat++, tsbm++, kpmtsbm++, i++) { 11983 cpu_kstat->sf_itlb_misses = tsbm->itlb_misses; 11984 cpu_kstat->sf_dtlb_misses = tsbm->dtlb_misses; 11985 cpu_kstat->sf_utsb_misses = tsbm->utsb_misses - 11986 tsbm->uprot_traps; 11987 cpu_kstat->sf_ktsb_misses = tsbm->ktsb_misses + 11988 kpmtsbm->kpm_tsb_misses - tsbm->kprot_traps; 11989 11990 if (tsbm->itlb_misses > 0 && tsbm->dtlb_misses > 0) { 11991 cpu_kstat->sf_tsb_hits = 11992 (tsbm->itlb_misses + tsbm->dtlb_misses) - 11993 (tsbm->utsb_misses + tsbm->ktsb_misses + 11994 kpmtsbm->kpm_tsb_misses); 11995 } else { 11996 cpu_kstat->sf_tsb_hits = 0; 11997 } 11998 cpu_kstat->sf_umod_faults = tsbm->uprot_traps; 11999 cpu_kstat->sf_kmod_faults = tsbm->kprot_traps; 12000 } 12001 } else { 12002 /* KSTAT_WRITE is used to clear stats */ 12003 for (i = 0; i < NCPU; tsbm++, kpmtsbm++, i++) { 12004 tsbm->itlb_misses = 0; 12005 tsbm->dtlb_misses = 0; 12006 tsbm->utsb_misses = 0; 12007 tsbm->ktsb_misses = 0; 12008 tsbm->uprot_traps = 0; 12009 tsbm->kprot_traps = 0; 12010 kpmtsbm->kpm_dtlb_misses = 0; 12011 kpmtsbm->kpm_tsb_misses = 0; 12012 } 12013 } 12014 return (0); 12015 } 12016 12017 #ifdef DEBUG 12018 12019 tte_t *gorig[NCPU], *gcur[NCPU], *gnew[NCPU]; 12020 12021 /* 12022 * A tte checker. *orig_old is the value we read before cas. 12023 * *cur is the value returned by cas. 12024 * *new is the desired value when we do the cas. 12025 * 12026 * *hmeblkp is currently unused. 12027 */ 12028 12029 /* ARGSUSED */ 12030 void 12031 chk_tte(tte_t *orig_old, tte_t *cur, tte_t *new, struct hme_blk *hmeblkp) 12032 { 12033 pfn_t i, j, k; 12034 int cpuid = CPU->cpu_id; 12035 12036 gorig[cpuid] = orig_old; 12037 gcur[cpuid] = cur; 12038 gnew[cpuid] = new; 12039 12040 #ifdef lint 12041 hmeblkp = hmeblkp; 12042 #endif 12043 12044 if (TTE_IS_VALID(orig_old)) { 12045 if (TTE_IS_VALID(cur)) { 12046 i = TTE_TO_TTEPFN(orig_old); 12047 j = TTE_TO_TTEPFN(cur); 12048 k = TTE_TO_TTEPFN(new); 12049 if (i != j) { 12050 /* remap error? */ 12051 panic("chk_tte: bad pfn, 0x%lx, 0x%lx", i, j); 12052 } 12053 12054 if (i != k) { 12055 /* remap error? */ 12056 panic("chk_tte: bad pfn2, 0x%lx, 0x%lx", i, k); 12057 } 12058 } else { 12059 if (TTE_IS_VALID(new)) { 12060 panic("chk_tte: invalid cur? "); 12061 } 12062 12063 i = TTE_TO_TTEPFN(orig_old); 12064 k = TTE_TO_TTEPFN(new); 12065 if (i != k) { 12066 panic("chk_tte: bad pfn3, 0x%lx, 0x%lx", i, k); 12067 } 12068 } 12069 } else { 12070 if (TTE_IS_VALID(cur)) { 12071 j = TTE_TO_TTEPFN(cur); 12072 if (TTE_IS_VALID(new)) { 12073 k = TTE_TO_TTEPFN(new); 12074 if (j != k) { 12075 panic("chk_tte: bad pfn4, 0x%lx, 0x%lx", 12076 j, k); 12077 } 12078 } else { 12079 panic("chk_tte: why here?"); 12080 } 12081 } else { 12082 if (!TTE_IS_VALID(new)) { 12083 panic("chk_tte: why here2 ?"); 12084 } 12085 } 12086 } 12087 } 12088 12089 #endif /* DEBUG */ 12090 12091 extern void prefetch_tsbe_read(struct tsbe *); 12092 extern void prefetch_tsbe_write(struct tsbe *); 12093 12094 12095 /* 12096 * We want to prefetch 7 cache lines ahead for our read prefetch. This gives 12097 * us optimal performance on Cheetah+. You can only have 8 outstanding 12098 * prefetches at any one time, so we opted for 7 read prefetches and 1 write 12099 * prefetch to make the most utilization of the prefetch capability. 12100 */ 12101 #define TSBE_PREFETCH_STRIDE (7) 12102 12103 void 12104 sfmmu_copy_tsb(struct tsb_info *old_tsbinfo, struct tsb_info *new_tsbinfo) 12105 { 12106 int old_bytes = TSB_BYTES(old_tsbinfo->tsb_szc); 12107 int new_bytes = TSB_BYTES(new_tsbinfo->tsb_szc); 12108 int old_entries = TSB_ENTRIES(old_tsbinfo->tsb_szc); 12109 int new_entries = TSB_ENTRIES(new_tsbinfo->tsb_szc); 12110 struct tsbe *old; 12111 struct tsbe *new; 12112 struct tsbe *new_base = (struct tsbe *)new_tsbinfo->tsb_va; 12113 uint64_t va; 12114 int new_offset; 12115 int i; 12116 int vpshift; 12117 int last_prefetch; 12118 12119 if (old_bytes == new_bytes) { 12120 bcopy(old_tsbinfo->tsb_va, new_tsbinfo->tsb_va, new_bytes); 12121 } else { 12122 12123 /* 12124 * A TSBE is 16 bytes which means there are four TSBE's per 12125 * P$ line (64 bytes), thus every 4 TSBE's we prefetch. 12126 */ 12127 old = (struct tsbe *)old_tsbinfo->tsb_va; 12128 last_prefetch = old_entries - (4*(TSBE_PREFETCH_STRIDE+1)); 12129 for (i = 0; i < old_entries; i++, old++) { 12130 if (((i & (4-1)) == 0) && (i < last_prefetch)) 12131 prefetch_tsbe_read(old); 12132 if (!old->tte_tag.tag_invalid) { 12133 /* 12134 * We have a valid TTE to remap. Check the 12135 * size. We won't remap 64K or 512K TTEs 12136 * because they span more than one TSB entry 12137 * and are indexed using an 8K virt. page. 12138 * Ditto for 32M and 256M TTEs. 12139 */ 12140 if (TTE_CSZ(&old->tte_data) == TTE64K || 12141 TTE_CSZ(&old->tte_data) == TTE512K) 12142 continue; 12143 if (mmu_page_sizes == max_mmu_page_sizes) { 12144 if (TTE_CSZ(&old->tte_data) == TTE32M || 12145 TTE_CSZ(&old->tte_data) == TTE256M) 12146 continue; 12147 } 12148 12149 /* clear the lower 22 bits of the va */ 12150 va = *(uint64_t *)old << 22; 12151 /* turn va into a virtual pfn */ 12152 va >>= 22 - TSB_START_SIZE; 12153 /* 12154 * or in bits from the offset in the tsb 12155 * to get the real virtual pfn. These 12156 * correspond to bits [21:13] in the va 12157 */ 12158 vpshift = 12159 TTE_BSZS_SHIFT(TTE_CSZ(&old->tte_data)) & 12160 0x1ff; 12161 va |= (i << vpshift); 12162 va >>= vpshift; 12163 new_offset = va & (new_entries - 1); 12164 new = new_base + new_offset; 12165 prefetch_tsbe_write(new); 12166 *new = *old; 12167 } 12168 } 12169 } 12170 } 12171 12172 /* 12173 * unused in sfmmu 12174 */ 12175 void 12176 hat_dump(void) 12177 { 12178 } 12179 12180 /* 12181 * Called when a thread is exiting and we have switched to the kernel address 12182 * space. Perform the same VM initialization resume() uses when switching 12183 * processes. 12184 * 12185 * Note that sfmmu_load_mmustate() is currently a no-op for kernel threads, but 12186 * we call it anyway in case the semantics change in the future. 12187 */ 12188 /*ARGSUSED*/ 12189 void 12190 hat_thread_exit(kthread_t *thd) 12191 { 12192 uint64_t pgsz_cnum; 12193 uint_t pstate_save; 12194 12195 ASSERT(thd->t_procp->p_as == &kas); 12196 12197 pgsz_cnum = KCONTEXT; 12198 #ifdef sun4u 12199 pgsz_cnum |= (ksfmmup->sfmmu_cext << CTXREG_EXT_SHIFT); 12200 #endif 12201 /* 12202 * Note that sfmmu_load_mmustate() is currently a no-op for 12203 * kernel threads. We need to disable interrupts here, 12204 * simply because otherwise sfmmu_load_mmustate() would panic 12205 * if the caller does not disable interrupts. 12206 */ 12207 pstate_save = sfmmu_disable_intrs(); 12208 sfmmu_setctx_sec(pgsz_cnum); 12209 sfmmu_load_mmustate(ksfmmup); 12210 sfmmu_enable_intrs(pstate_save); 12211 } 12212