1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * VM - Hardware Address Translation management for Spitfire MMU. 30 * 31 * This file implements the machine specific hardware translation 32 * needed by the VM system. The machine independent interface is 33 * described in <vm/hat.h> while the machine dependent interface 34 * and data structures are described in <vm/hat_sfmmu.h>. 35 * 36 * The hat layer manages the address translation hardware as a cache 37 * driven by calls from the higher levels in the VM system. 38 */ 39 40 #include <sys/types.h> 41 #include <sys/kstat.h> 42 #include <vm/hat.h> 43 #include <vm/hat_sfmmu.h> 44 #include <vm/page.h> 45 #include <sys/pte.h> 46 #include <sys/systm.h> 47 #include <sys/mman.h> 48 #include <sys/sysmacros.h> 49 #include <sys/machparam.h> 50 #include <sys/vtrace.h> 51 #include <sys/kmem.h> 52 #include <sys/mmu.h> 53 #include <sys/cmn_err.h> 54 #include <sys/cpu.h> 55 #include <sys/cpuvar.h> 56 #include <sys/debug.h> 57 #include <sys/lgrp.h> 58 #include <sys/archsystm.h> 59 #include <sys/machsystm.h> 60 #include <sys/vmsystm.h> 61 #include <vm/as.h> 62 #include <vm/seg.h> 63 #include <vm/seg_kp.h> 64 #include <vm/seg_kmem.h> 65 #include <vm/seg_kpm.h> 66 #include <vm/rm.h> 67 #include <sys/t_lock.h> 68 #include <sys/obpdefs.h> 69 #include <sys/vm_machparam.h> 70 #include <sys/var.h> 71 #include <sys/trap.h> 72 #include <sys/machtrap.h> 73 #include <sys/scb.h> 74 #include <sys/bitmap.h> 75 #include <sys/machlock.h> 76 #include <sys/membar.h> 77 #include <sys/atomic.h> 78 #include <sys/cpu_module.h> 79 #include <sys/prom_debug.h> 80 #include <sys/ksynch.h> 81 #include <sys/mem_config.h> 82 #include <sys/mem_cage.h> 83 #include <sys/dtrace.h> 84 #include <vm/vm_dep.h> 85 #include <vm/xhat_sfmmu.h> 86 #include <sys/fpu/fpusystm.h> 87 #include <vm/mach_kpm.h> 88 89 #if defined(SF_ERRATA_57) 90 extern caddr_t errata57_limit; 91 #endif 92 93 #define HME8BLK_SZ_RND ((roundup(HME8BLK_SZ, sizeof (int64_t))) / \ 94 (sizeof (int64_t))) 95 #define HBLK_RESERVE ((struct hme_blk *)hblk_reserve) 96 97 #define HBLK_RESERVE_CNT 128 98 #define HBLK_RESERVE_MIN 20 99 100 static struct hme_blk *freehblkp; 101 static kmutex_t freehblkp_lock; 102 static int freehblkcnt; 103 104 static int64_t hblk_reserve[HME8BLK_SZ_RND]; 105 static kmutex_t hblk_reserve_lock; 106 static kthread_t *hblk_reserve_thread; 107 108 static nucleus_hblk8_info_t nucleus_hblk8; 109 static nucleus_hblk1_info_t nucleus_hblk1; 110 111 /* 112 * SFMMU specific hat functions 113 */ 114 void hat_pagecachectl(struct page *, int); 115 116 /* flags for hat_pagecachectl */ 117 #define HAT_CACHE 0x1 118 #define HAT_UNCACHE 0x2 119 #define HAT_TMPNC 0x4 120 121 /* 122 * Flag to allow the creation of non-cacheable translations 123 * to system memory. It is off by default. At the moment this 124 * flag is used by the ecache error injector. The error injector 125 * will turn it on when creating such a translation then shut it 126 * off when it's finished. 127 */ 128 129 int sfmmu_allow_nc_trans = 0; 130 131 /* 132 * Flag to disable large page support. 133 * value of 1 => disable all large pages. 134 * bits 1, 2, and 3 are to disable 64K, 512K and 4M pages respectively. 135 * 136 * For example, use the value 0x4 to disable 512K pages. 137 * 138 */ 139 #define LARGE_PAGES_OFF 0x1 140 141 /* 142 * The disable_large_pages and disable_ism_large_pages variables control 143 * hat_memload_array and the page sizes to be used by ISM and the kernel. 144 * 145 * The disable_auto_data_large_pages and disable_auto_text_large_pages variables 146 * are only used to control which OOB pages to use at upper VM segment creation 147 * time, and are set in hat_init_pagesizes and used in the map_pgsz* routines. 148 * Their values may come from platform or CPU specific code to disable page 149 * sizes that should not be used. 150 * 151 * WARNING: 512K pages are currently not supported for ISM/DISM. 152 */ 153 uint_t disable_large_pages = 0; 154 uint_t disable_ism_large_pages = (1 << TTE512K); 155 uint_t disable_auto_data_large_pages = 0; 156 uint_t disable_auto_text_large_pages = 0; 157 158 /* 159 * Private sfmmu data structures for hat management 160 */ 161 static struct kmem_cache *sfmmuid_cache; 162 static struct kmem_cache *mmuctxdom_cache; 163 164 /* 165 * Private sfmmu data structures for tsb management 166 */ 167 static struct kmem_cache *sfmmu_tsbinfo_cache; 168 static struct kmem_cache *sfmmu_tsb8k_cache; 169 static struct kmem_cache *sfmmu_tsb_cache[NLGRPS_MAX]; 170 static vmem_t *kmem_tsb_arena; 171 172 /* 173 * sfmmu static variables for hmeblk resource management. 174 */ 175 static vmem_t *hat_memload1_arena; /* HAT translation arena for sfmmu1_cache */ 176 static struct kmem_cache *sfmmu8_cache; 177 static struct kmem_cache *sfmmu1_cache; 178 static struct kmem_cache *pa_hment_cache; 179 180 static kmutex_t ism_mlist_lock; /* mutex for ism mapping list */ 181 /* 182 * private data for ism 183 */ 184 static struct kmem_cache *ism_blk_cache; 185 static struct kmem_cache *ism_ment_cache; 186 #define ISMID_STARTADDR NULL 187 188 /* 189 * Whether to delay TLB flushes and use Cheetah's flush-all support 190 * when removing contexts from the dirty list. 191 */ 192 int delay_tlb_flush; 193 int disable_delay_tlb_flush; 194 195 /* 196 * ``hat_lock'' is a hashed mutex lock for protecting sfmmu TSB lists, 197 * HAT flags, synchronizing TLB/TSB coherency, and context management. 198 * The lock is hashed on the sfmmup since the case where we need to lock 199 * all processes is rare but does occur (e.g. we need to unload a shared 200 * mapping from all processes using the mapping). We have a lot of buckets, 201 * and each slab of sfmmu_t's can use about a quarter of them, giving us 202 * a fairly good distribution without wasting too much space and overhead 203 * when we have to grab them all. 204 */ 205 #define SFMMU_NUM_LOCK 128 /* must be power of two */ 206 hatlock_t hat_lock[SFMMU_NUM_LOCK]; 207 208 /* 209 * Hash algorithm optimized for a small number of slabs. 210 * 7 is (highbit((sizeof sfmmu_t)) - 1) 211 * This hash algorithm is based upon the knowledge that sfmmu_t's come from a 212 * kmem_cache, and thus they will be sequential within that cache. In 213 * addition, each new slab will have a different "color" up to cache_maxcolor 214 * which will skew the hashing for each successive slab which is allocated. 215 * If the size of sfmmu_t changed to a larger size, this algorithm may need 216 * to be revisited. 217 */ 218 #define TSB_HASH_SHIFT_BITS (7) 219 #define PTR_HASH(x) ((uintptr_t)x >> TSB_HASH_SHIFT_BITS) 220 221 #ifdef DEBUG 222 int tsb_hash_debug = 0; 223 #define TSB_HASH(sfmmup) \ 224 (tsb_hash_debug ? &hat_lock[0] : \ 225 &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)]) 226 #else /* DEBUG */ 227 #define TSB_HASH(sfmmup) &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)] 228 #endif /* DEBUG */ 229 230 231 /* sfmmu_replace_tsb() return codes. */ 232 typedef enum tsb_replace_rc { 233 TSB_SUCCESS, 234 TSB_ALLOCFAIL, 235 TSB_LOSTRACE, 236 TSB_ALREADY_SWAPPED, 237 TSB_CANTGROW 238 } tsb_replace_rc_t; 239 240 /* 241 * Flags for TSB allocation routines. 242 */ 243 #define TSB_ALLOC 0x01 244 #define TSB_FORCEALLOC 0x02 245 #define TSB_GROW 0x04 246 #define TSB_SHRINK 0x08 247 #define TSB_SWAPIN 0x10 248 249 /* 250 * Support for HAT callbacks. 251 */ 252 #define SFMMU_MAX_RELOC_CALLBACKS 10 253 int sfmmu_max_cb_id = SFMMU_MAX_RELOC_CALLBACKS; 254 static id_t sfmmu_cb_nextid = 0; 255 static id_t sfmmu_tsb_cb_id; 256 struct sfmmu_callback *sfmmu_cb_table; 257 258 /* 259 * Kernel page relocation is enabled by default for non-caged 260 * kernel pages. This has little effect unless segkmem_reloc is 261 * set, since by default kernel memory comes from inside the 262 * kernel cage. 263 */ 264 int hat_kpr_enabled = 1; 265 266 kmutex_t kpr_mutex; 267 kmutex_t kpr_suspendlock; 268 kthread_t *kreloc_thread; 269 270 /* 271 * Enable VA->PA translation sanity checking on DEBUG kernels. 272 * Disabled by default. This is incompatible with some 273 * drivers (error injector, RSM) so if it breaks you get 274 * to keep both pieces. 275 */ 276 int hat_check_vtop = 0; 277 278 /* 279 * Private sfmmu routines (prototypes) 280 */ 281 static struct hme_blk *sfmmu_shadow_hcreate(sfmmu_t *, caddr_t, int, uint_t); 282 static struct hme_blk *sfmmu_hblk_alloc(sfmmu_t *, caddr_t, 283 struct hmehash_bucket *, uint_t, hmeblk_tag, uint_t); 284 static caddr_t sfmmu_hblk_unload(struct hat *, struct hme_blk *, caddr_t, 285 caddr_t, demap_range_t *, uint_t); 286 static caddr_t sfmmu_hblk_sync(struct hat *, struct hme_blk *, caddr_t, 287 caddr_t, int); 288 static void sfmmu_hblk_free(struct hmehash_bucket *, struct hme_blk *, 289 uint64_t, struct hme_blk **); 290 static void sfmmu_hblks_list_purge(struct hme_blk **); 291 static uint_t sfmmu_get_free_hblk(struct hme_blk **, uint_t); 292 static uint_t sfmmu_put_free_hblk(struct hme_blk *, uint_t); 293 static struct hme_blk *sfmmu_hblk_steal(int); 294 static int sfmmu_steal_this_hblk(struct hmehash_bucket *, 295 struct hme_blk *, uint64_t, uint64_t, 296 struct hme_blk *); 297 static caddr_t sfmmu_hblk_unlock(struct hme_blk *, caddr_t, caddr_t); 298 299 static void sfmmu_memload_batchsmall(struct hat *, caddr_t, page_t **, 300 uint_t, uint_t, pgcnt_t); 301 void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, 302 uint_t); 303 static int sfmmu_tteload_array(sfmmu_t *, tte_t *, caddr_t, page_t **, 304 uint_t); 305 static struct hmehash_bucket *sfmmu_tteload_acquire_hashbucket(sfmmu_t *, 306 caddr_t, int); 307 static struct hme_blk *sfmmu_tteload_find_hmeblk(sfmmu_t *, 308 struct hmehash_bucket *, caddr_t, uint_t, uint_t); 309 static int sfmmu_tteload_addentry(sfmmu_t *, struct hme_blk *, tte_t *, 310 caddr_t, page_t **, uint_t); 311 static void sfmmu_tteload_release_hashbucket(struct hmehash_bucket *); 312 313 static int sfmmu_pagearray_setup(caddr_t, page_t **, tte_t *, int); 314 pfn_t sfmmu_uvatopfn(caddr_t, sfmmu_t *); 315 void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 316 #ifdef VAC 317 static void sfmmu_vac_conflict(struct hat *, caddr_t, page_t *); 318 static int sfmmu_vacconflict_array(caddr_t, page_t *, int *); 319 int tst_tnc(page_t *pp, pgcnt_t); 320 void conv_tnc(page_t *pp, int); 321 #endif 322 323 static void sfmmu_get_ctx(sfmmu_t *); 324 static void sfmmu_free_sfmmu(sfmmu_t *); 325 326 static void sfmmu_gettte(struct hat *, caddr_t, tte_t *); 327 static void sfmmu_ttesync(struct hat *, caddr_t, tte_t *, page_t *); 328 static void sfmmu_chgattr(struct hat *, caddr_t, size_t, uint_t, int); 329 330 cpuset_t sfmmu_pageunload(page_t *, struct sf_hment *, int); 331 static void hat_pagereload(struct page *, struct page *); 332 static cpuset_t sfmmu_pagesync(page_t *, struct sf_hment *, uint_t); 333 #ifdef VAC 334 void sfmmu_page_cache_array(page_t *, int, int, pgcnt_t); 335 static void sfmmu_page_cache(page_t *, int, int, int); 336 #endif 337 338 static void sfmmu_tlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 339 pfn_t, int, int, int, int); 340 static void sfmmu_ismtlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 341 pfn_t, int); 342 static void sfmmu_tlb_demap(caddr_t, sfmmu_t *, struct hme_blk *, int, int); 343 static void sfmmu_tlb_range_demap(demap_range_t *); 344 static void sfmmu_invalidate_ctx(sfmmu_t *); 345 static void sfmmu_sync_mmustate(sfmmu_t *); 346 347 static void sfmmu_tsbinfo_setup_phys(struct tsb_info *, pfn_t); 348 static int sfmmu_tsbinfo_alloc(struct tsb_info **, int, int, uint_t, 349 sfmmu_t *); 350 static void sfmmu_tsb_free(struct tsb_info *); 351 static void sfmmu_tsbinfo_free(struct tsb_info *); 352 static int sfmmu_init_tsbinfo(struct tsb_info *, int, int, uint_t, 353 sfmmu_t *); 354 355 static void sfmmu_tsb_swapin(sfmmu_t *, hatlock_t *); 356 static int sfmmu_select_tsb_szc(pgcnt_t); 357 static void sfmmu_mod_tsb(sfmmu_t *, caddr_t, tte_t *, int); 358 #define sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \ 359 sfmmu_mod_tsb(sfmmup, vaddr, tte, szc) 360 #define sfmmu_unload_tsb(sfmmup, vaddr, szc) \ 361 sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc) 362 static void sfmmu_copy_tsb(struct tsb_info *, struct tsb_info *); 363 static tsb_replace_rc_t sfmmu_replace_tsb(sfmmu_t *, struct tsb_info *, uint_t, 364 hatlock_t *, uint_t); 365 static void sfmmu_size_tsb(sfmmu_t *, int, uint64_t, uint64_t, int); 366 367 #ifdef VAC 368 void sfmmu_cache_flush(pfn_t, int); 369 void sfmmu_cache_flushcolor(int, pfn_t); 370 #endif 371 static caddr_t sfmmu_hblk_chgattr(sfmmu_t *, struct hme_blk *, caddr_t, 372 caddr_t, demap_range_t *, uint_t, int); 373 374 static uint64_t sfmmu_vtop_attr(uint_t, int mode, tte_t *); 375 static uint_t sfmmu_ptov_attr(tte_t *); 376 static caddr_t sfmmu_hblk_chgprot(sfmmu_t *, struct hme_blk *, caddr_t, 377 caddr_t, demap_range_t *, uint_t); 378 static uint_t sfmmu_vtop_prot(uint_t, uint_t *); 379 static int sfmmu_idcache_constructor(void *, void *, int); 380 static void sfmmu_idcache_destructor(void *, void *); 381 static int sfmmu_hblkcache_constructor(void *, void *, int); 382 static void sfmmu_hblkcache_destructor(void *, void *); 383 static void sfmmu_hblkcache_reclaim(void *); 384 static void sfmmu_shadow_hcleanup(sfmmu_t *, struct hme_blk *, 385 struct hmehash_bucket *); 386 static void sfmmu_free_hblks(sfmmu_t *, caddr_t, caddr_t, int); 387 static void sfmmu_rm_large_mappings(page_t *, int); 388 389 static void hat_lock_init(void); 390 static void hat_kstat_init(void); 391 static int sfmmu_kstat_percpu_update(kstat_t *ksp, int rw); 392 static void sfmmu_check_page_sizes(sfmmu_t *, int); 393 int fnd_mapping_sz(page_t *); 394 static void iment_add(struct ism_ment *, struct hat *); 395 static void iment_sub(struct ism_ment *, struct hat *); 396 static pgcnt_t ism_tsb_entries(sfmmu_t *, int szc); 397 extern void sfmmu_setup_tsbinfo(sfmmu_t *); 398 #ifdef sun4v 399 extern void sfmmu_invalidate_tsbinfo(sfmmu_t *); 400 #endif /* sun4v */ 401 extern void sfmmu_clear_utsbinfo(void); 402 403 static void sfmmu_ctx_wrap_around(mmu_ctx_t *); 404 405 /* kpm globals */ 406 #ifdef DEBUG 407 /* 408 * Enable trap level tsbmiss handling 409 */ 410 int kpm_tsbmtl = 1; 411 412 /* 413 * Flush the TLB on kpm mapout. Note: Xcalls are used (again) for the 414 * required TLB shootdowns in this case, so handle w/ care. Off by default. 415 */ 416 int kpm_tlb_flush; 417 #endif /* DEBUG */ 418 419 static void *sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *, size_t, int); 420 421 #ifdef DEBUG 422 static void sfmmu_check_hblk_flist(); 423 #endif 424 425 /* 426 * Semi-private sfmmu data structures. Some of them are initialize in 427 * startup or in hat_init. Some of them are private but accessed by 428 * assembly code or mach_sfmmu.c 429 */ 430 struct hmehash_bucket *uhme_hash; /* user hmeblk hash table */ 431 struct hmehash_bucket *khme_hash; /* kernel hmeblk hash table */ 432 uint64_t uhme_hash_pa; /* PA of uhme_hash */ 433 uint64_t khme_hash_pa; /* PA of khme_hash */ 434 int uhmehash_num; /* # of buckets in user hash table */ 435 int khmehash_num; /* # of buckets in kernel hash table */ 436 437 uint_t max_mmu_ctxdoms = 0; /* max context domains in the system */ 438 mmu_ctx_t **mmu_ctxs_tbl; /* global array of context domains */ 439 uint64_t mmu_saved_gnum = 0; /* to init incoming MMUs' gnums */ 440 441 #define DEFAULT_NUM_CTXS_PER_MMU 8192 442 static uint_t nctxs = DEFAULT_NUM_CTXS_PER_MMU; 443 444 int cache; /* describes system cache */ 445 446 caddr_t ktsb_base; /* kernel 8k-indexed tsb base address */ 447 uint64_t ktsb_pbase; /* kernel 8k-indexed tsb phys address */ 448 int ktsb_szcode; /* kernel 8k-indexed tsb size code */ 449 int ktsb_sz; /* kernel 8k-indexed tsb size */ 450 451 caddr_t ktsb4m_base; /* kernel 4m-indexed tsb base address */ 452 uint64_t ktsb4m_pbase; /* kernel 4m-indexed tsb phys address */ 453 int ktsb4m_szcode; /* kernel 4m-indexed tsb size code */ 454 int ktsb4m_sz; /* kernel 4m-indexed tsb size */ 455 456 uint64_t kpm_tsbbase; /* kernel seg_kpm 4M TSB base address */ 457 int kpm_tsbsz; /* kernel seg_kpm 4M TSB size code */ 458 uint64_t kpmsm_tsbbase; /* kernel seg_kpm 8K TSB base address */ 459 int kpmsm_tsbsz; /* kernel seg_kpm 8K TSB size code */ 460 461 #ifndef sun4v 462 int utsb_dtlb_ttenum = -1; /* index in TLB for utsb locked TTE */ 463 int utsb4m_dtlb_ttenum = -1; /* index in TLB for 4M TSB TTE */ 464 int dtlb_resv_ttenum; /* index in TLB of first reserved TTE */ 465 caddr_t utsb_vabase; /* reserved kernel virtual memory */ 466 caddr_t utsb4m_vabase; /* for trap handler TSB accesses */ 467 #endif /* sun4v */ 468 uint64_t tsb_alloc_bytes = 0; /* bytes allocated to TSBs */ 469 vmem_t *kmem_tsb_default_arena[NLGRPS_MAX]; /* For dynamic TSBs */ 470 471 /* 472 * Size to use for TSB slabs. Future platforms that support page sizes 473 * larger than 4M may wish to change these values, and provide their own 474 * assembly macros for building and decoding the TSB base register contents. 475 * Note disable_large_pages will override the value set here. 476 */ 477 uint_t tsb_slab_ttesz = TTE4M; 478 uint_t tsb_slab_size; 479 uint_t tsb_slab_shift; 480 uint_t tsb_slab_mask; /* PFN mask for TTE */ 481 482 /* largest TSB size to grow to, will be smaller on smaller memory systems */ 483 int tsb_max_growsize = UTSB_MAX_SZCODE; 484 485 /* 486 * Tunable parameters dealing with TSB policies. 487 */ 488 489 /* 490 * This undocumented tunable forces all 8K TSBs to be allocated from 491 * the kernel heap rather than from the kmem_tsb_default_arena arenas. 492 */ 493 #ifdef DEBUG 494 int tsb_forceheap = 0; 495 #endif /* DEBUG */ 496 497 /* 498 * Decide whether to use per-lgroup arenas, or one global set of 499 * TSB arenas. The default is not to break up per-lgroup, since 500 * most platforms don't recognize any tangible benefit from it. 501 */ 502 int tsb_lgrp_affinity = 0; 503 504 /* 505 * Used for growing the TSB based on the process RSS. 506 * tsb_rss_factor is based on the smallest TSB, and is 507 * shifted by the TSB size to determine if we need to grow. 508 * The default will grow the TSB if the number of TTEs for 509 * this page size exceeds 75% of the number of TSB entries, 510 * which should _almost_ eliminate all conflict misses 511 * (at the expense of using up lots and lots of memory). 512 */ 513 #define TSB_RSS_FACTOR (TSB_ENTRIES(TSB_MIN_SZCODE) * 0.75) 514 #define SFMMU_RSS_TSBSIZE(tsbszc) (tsb_rss_factor << tsbszc) 515 #define SELECT_TSB_SIZECODE(pgcnt) ( \ 516 (enable_tsb_rss_sizing)? sfmmu_select_tsb_szc(pgcnt) : \ 517 default_tsb_size) 518 #define TSB_OK_SHRINK() \ 519 (tsb_alloc_bytes > tsb_alloc_hiwater || freemem < desfree) 520 #define TSB_OK_GROW() \ 521 (tsb_alloc_bytes < tsb_alloc_hiwater && freemem > desfree) 522 523 int enable_tsb_rss_sizing = 1; 524 int tsb_rss_factor = (int)TSB_RSS_FACTOR; 525 526 /* which TSB size code to use for new address spaces or if rss sizing off */ 527 int default_tsb_size = TSB_8K_SZCODE; 528 529 static uint64_t tsb_alloc_hiwater; /* limit TSB reserved memory */ 530 uint64_t tsb_alloc_hiwater_factor; /* tsb_alloc_hiwater = physmem / this */ 531 #define TSB_ALLOC_HIWATER_FACTOR_DEFAULT 32 532 533 #ifdef DEBUG 534 static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */ 535 static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */ 536 static int tsb_alloc_mtbf = 0; /* fail allocation every n attempts */ 537 static int tsb_alloc_fail_mtbf = 0; 538 static int tsb_alloc_count = 0; 539 #endif /* DEBUG */ 540 541 /* if set to 1, will remap valid TTEs when growing TSB. */ 542 int tsb_remap_ttes = 1; 543 544 /* 545 * If we have more than this many mappings, allocate a second TSB. 546 * This default is chosen because the I/D fully associative TLBs are 547 * assumed to have at least 8 available entries. Platforms with a 548 * larger fully-associative TLB could probably override the default. 549 */ 550 int tsb_sectsb_threshold = 8; 551 552 /* 553 * kstat data 554 */ 555 struct sfmmu_global_stat sfmmu_global_stat; 556 struct sfmmu_tsbsize_stat sfmmu_tsbsize_stat; 557 558 /* 559 * Global data 560 */ 561 sfmmu_t *ksfmmup; /* kernel's hat id */ 562 563 #ifdef DEBUG 564 static void chk_tte(tte_t *, tte_t *, tte_t *, struct hme_blk *); 565 #endif 566 567 /* sfmmu locking operations */ 568 static kmutex_t *sfmmu_mlspl_enter(struct page *, int); 569 static int sfmmu_mlspl_held(struct page *, int); 570 571 kmutex_t *sfmmu_page_enter(page_t *); 572 void sfmmu_page_exit(kmutex_t *); 573 int sfmmu_page_spl_held(struct page *); 574 575 /* sfmmu internal locking operations - accessed directly */ 576 static void sfmmu_mlist_reloc_enter(page_t *, page_t *, 577 kmutex_t **, kmutex_t **); 578 static void sfmmu_mlist_reloc_exit(kmutex_t *, kmutex_t *); 579 static hatlock_t * 580 sfmmu_hat_enter(sfmmu_t *); 581 static hatlock_t * 582 sfmmu_hat_tryenter(sfmmu_t *); 583 static void sfmmu_hat_exit(hatlock_t *); 584 static void sfmmu_hat_lock_all(void); 585 static void sfmmu_hat_unlock_all(void); 586 static void sfmmu_ismhat_enter(sfmmu_t *, int); 587 static void sfmmu_ismhat_exit(sfmmu_t *, int); 588 589 /* 590 * Array of mutexes protecting a page's mapping list and p_nrm field. 591 * 592 * The hash function looks complicated, but is made up so that: 593 * 594 * "pp" not shifted, so adjacent pp values will hash to different cache lines 595 * (8 byte alignment * 8 bytes/mutes == 64 byte coherency subblock) 596 * 597 * "pp" >> mml_shift, incorporates more source bits into the hash result 598 * 599 * "& (mml_table_size - 1), should be faster than using remainder "%" 600 * 601 * Hopefully, mml_table, mml_table_size and mml_shift are all in the same 602 * cacheline, since they get declared next to each other below. We'll trust 603 * ld not to do something random. 604 */ 605 #ifdef DEBUG 606 int mlist_hash_debug = 0; 607 #define MLIST_HASH(pp) (mlist_hash_debug ? &mml_table[0] : \ 608 &mml_table[((uintptr_t)(pp) + \ 609 ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)]) 610 #else /* !DEBUG */ 611 #define MLIST_HASH(pp) &mml_table[ \ 612 ((uintptr_t)(pp) + ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)] 613 #endif /* !DEBUG */ 614 615 kmutex_t *mml_table; 616 uint_t mml_table_sz; /* must be a power of 2 */ 617 uint_t mml_shift; /* log2(mml_table_sz) + 3 for align */ 618 619 kpm_hlk_t *kpmp_table; 620 uint_t kpmp_table_sz; /* must be a power of 2 */ 621 uchar_t kpmp_shift; 622 623 kpm_shlk_t *kpmp_stable; 624 uint_t kpmp_stable_sz; /* must be a power of 2 */ 625 626 /* 627 * SPL_HASH was improved to avoid false cache line sharing 628 */ 629 #define SPL_TABLE_SIZE 128 630 #define SPL_MASK (SPL_TABLE_SIZE - 1) 631 #define SPL_SHIFT 7 /* log2(SPL_TABLE_SIZE) */ 632 633 #define SPL_INDEX(pp) \ 634 ((((uintptr_t)(pp) >> SPL_SHIFT) ^ \ 635 ((uintptr_t)(pp) >> (SPL_SHIFT << 1))) & \ 636 (SPL_TABLE_SIZE - 1)) 637 638 #define SPL_HASH(pp) \ 639 (&sfmmu_page_lock[SPL_INDEX(pp) & SPL_MASK].pad_mutex) 640 641 static pad_mutex_t sfmmu_page_lock[SPL_TABLE_SIZE]; 642 643 644 /* 645 * hat_unload_callback() will group together callbacks in order 646 * to avoid xt_sync() calls. This is the maximum size of the group. 647 */ 648 #define MAX_CB_ADDR 32 649 650 tte_t hw_tte; 651 static ulong_t sfmmu_dmr_maxbit = DMR_MAXBIT; 652 653 static char *mmu_ctx_kstat_names[] = { 654 "mmu_ctx_tsb_exceptions", 655 "mmu_ctx_tsb_raise_exception", 656 "mmu_ctx_wrap_around", 657 }; 658 659 /* 660 * Wrapper for vmem_xalloc since vmem_create only allows limited 661 * parameters for vm_source_alloc functions. This function allows us 662 * to specify alignment consistent with the size of the object being 663 * allocated. 664 */ 665 static void * 666 sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *vmp, size_t size, int vmflag) 667 { 668 return (vmem_xalloc(vmp, size, size, 0, 0, NULL, NULL, vmflag)); 669 } 670 671 /* Common code for setting tsb_alloc_hiwater. */ 672 #define SFMMU_SET_TSB_ALLOC_HIWATER(pages) tsb_alloc_hiwater = \ 673 ptob(pages) / tsb_alloc_hiwater_factor 674 675 /* 676 * Set tsb_max_growsize to allow at most all of physical memory to be mapped by 677 * a single TSB. physmem is the number of physical pages so we need physmem 8K 678 * TTEs to represent all those physical pages. We round this up by using 679 * 1<<highbit(). To figure out which size code to use, remember that the size 680 * code is just an amount to shift the smallest TSB size to get the size of 681 * this TSB. So we subtract that size, TSB_START_SIZE, from highbit() (or 682 * highbit() - 1) to get the size code for the smallest TSB that can represent 683 * all of physical memory, while erring on the side of too much. 684 * 685 * If the computed size code is less than the current tsb_max_growsize, we set 686 * tsb_max_growsize to the computed size code. In the case where the computed 687 * size code is greater than tsb_max_growsize, we have these restrictions that 688 * apply to increasing tsb_max_growsize: 689 * 1) TSBs can't grow larger than the TSB slab size 690 * 2) TSBs can't grow larger than UTSB_MAX_SZCODE. 691 */ 692 #define SFMMU_SET_TSB_MAX_GROWSIZE(pages) { \ 693 int i, szc; \ 694 \ 695 i = highbit(pages); \ 696 if ((1 << (i - 1)) == (pages)) \ 697 i--; /* 2^n case, round down */ \ 698 szc = i - TSB_START_SIZE; \ 699 if (szc < tsb_max_growsize) \ 700 tsb_max_growsize = szc; \ 701 else if ((szc > tsb_max_growsize) && \ 702 (szc <= tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT))) \ 703 tsb_max_growsize = MIN(szc, UTSB_MAX_SZCODE); \ 704 } 705 706 /* 707 * Given a pointer to an sfmmu and a TTE size code, return a pointer to the 708 * tsb_info which handles that TTE size. 709 */ 710 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) \ 711 (tsbinfop) = (sfmmup)->sfmmu_tsb; \ 712 ASSERT(sfmmu_hat_lock_held(sfmmup)); \ 713 if ((tte_szc) >= TTE4M) \ 714 (tsbinfop) = (tsbinfop)->tsb_next; 715 716 /* 717 * Return the number of mappings present in the HAT 718 * for a particular process and page size. 719 */ 720 #define SFMMU_TTE_CNT(sfmmup, szc) \ 721 (sfmmup)->sfmmu_iblk? \ 722 (sfmmup)->sfmmu_ismttecnt[(szc)] + \ 723 (sfmmup)->sfmmu_ttecnt[(szc)] : \ 724 (sfmmup)->sfmmu_ttecnt[(szc)]; 725 726 /* 727 * Macro to use to unload entries from the TSB. 728 * It has knowledge of which page sizes get replicated in the TSB 729 * and will call the appropriate unload routine for the appropriate size. 730 */ 731 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp) \ 732 { \ 733 int ttesz = get_hblk_ttesz(hmeblkp); \ 734 if (ttesz == TTE8K || ttesz == TTE4M) { \ 735 sfmmu_unload_tsb(sfmmup, addr, ttesz); \ 736 } else { \ 737 caddr_t sva = (caddr_t)get_hblk_base(hmeblkp); \ 738 caddr_t eva = sva + get_hblk_span(hmeblkp); \ 739 ASSERT(addr >= sva && addr < eva); \ 740 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); \ 741 } \ 742 } 743 744 745 /* Update tsb_alloc_hiwater after memory is configured. */ 746 /*ARGSUSED*/ 747 static void 748 sfmmu_update_tsb_post_add(void *arg, pgcnt_t delta_pages) 749 { 750 /* Assumes physmem has already been updated. */ 751 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 752 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 753 } 754 755 /* 756 * Update tsb_alloc_hiwater before memory is deleted. We'll do nothing here 757 * and update tsb_alloc_hiwater and tsb_max_growsize after the memory is 758 * deleted. 759 */ 760 /*ARGSUSED*/ 761 static int 762 sfmmu_update_tsb_pre_del(void *arg, pgcnt_t delta_pages) 763 { 764 return (0); 765 } 766 767 /* Update tsb_alloc_hiwater after memory fails to be unconfigured. */ 768 /*ARGSUSED*/ 769 static void 770 sfmmu_update_tsb_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 771 { 772 /* 773 * Whether the delete was cancelled or not, just go ahead and update 774 * tsb_alloc_hiwater and tsb_max_growsize. 775 */ 776 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 777 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 778 } 779 780 static kphysm_setup_vector_t sfmmu_update_tsb_vec = { 781 KPHYSM_SETUP_VECTOR_VERSION, /* version */ 782 sfmmu_update_tsb_post_add, /* post_add */ 783 sfmmu_update_tsb_pre_del, /* pre_del */ 784 sfmmu_update_tsb_post_del /* post_del */ 785 }; 786 787 788 /* 789 * HME_BLK HASH PRIMITIVES 790 */ 791 792 /* 793 * Enter a hme on the mapping list for page pp. 794 * When large pages are more prevalent in the system we might want to 795 * keep the mapping list in ascending order by the hment size. For now, 796 * small pages are more frequent, so don't slow it down. 797 */ 798 #define HME_ADD(hme, pp) \ 799 { \ 800 ASSERT(sfmmu_mlist_held(pp)); \ 801 \ 802 hme->hme_prev = NULL; \ 803 hme->hme_next = pp->p_mapping; \ 804 hme->hme_page = pp; \ 805 if (pp->p_mapping) { \ 806 ((struct sf_hment *)(pp->p_mapping))->hme_prev = hme;\ 807 ASSERT(pp->p_share > 0); \ 808 } else { \ 809 /* EMPTY */ \ 810 ASSERT(pp->p_share == 0); \ 811 } \ 812 pp->p_mapping = hme; \ 813 pp->p_share++; \ 814 } 815 816 /* 817 * Enter a hme on the mapping list for page pp. 818 * If we are unmapping a large translation, we need to make sure that the 819 * change is reflect in the corresponding bit of the p_index field. 820 */ 821 #define HME_SUB(hme, pp) \ 822 { \ 823 ASSERT(sfmmu_mlist_held(pp)); \ 824 ASSERT(hme->hme_page == pp || IS_PAHME(hme)); \ 825 \ 826 if (pp->p_mapping == NULL) { \ 827 panic("hme_remove - no mappings"); \ 828 } \ 829 \ 830 membar_stst(); /* ensure previous stores finish */ \ 831 \ 832 ASSERT(pp->p_share > 0); \ 833 pp->p_share--; \ 834 \ 835 if (hme->hme_prev) { \ 836 ASSERT(pp->p_mapping != hme); \ 837 ASSERT(hme->hme_prev->hme_page == pp || \ 838 IS_PAHME(hme->hme_prev)); \ 839 hme->hme_prev->hme_next = hme->hme_next; \ 840 } else { \ 841 ASSERT(pp->p_mapping == hme); \ 842 pp->p_mapping = hme->hme_next; \ 843 ASSERT((pp->p_mapping == NULL) ? \ 844 (pp->p_share == 0) : 1); \ 845 } \ 846 \ 847 if (hme->hme_next) { \ 848 ASSERT(hme->hme_next->hme_page == pp || \ 849 IS_PAHME(hme->hme_next)); \ 850 hme->hme_next->hme_prev = hme->hme_prev; \ 851 } \ 852 \ 853 /* zero out the entry */ \ 854 hme->hme_next = NULL; \ 855 hme->hme_prev = NULL; \ 856 hme->hme_page = NULL; \ 857 \ 858 if (hme_size(hme) > TTE8K) { \ 859 /* remove mappings for remainder of large pg */ \ 860 sfmmu_rm_large_mappings(pp, hme_size(hme)); \ 861 } \ 862 } 863 864 /* 865 * This function returns the hment given the hme_blk and a vaddr. 866 * It assumes addr has already been checked to belong to hme_blk's 867 * range. 868 */ 869 #define HBLKTOHME(hment, hmeblkp, addr) \ 870 { \ 871 int index; \ 872 HBLKTOHME_IDX(hment, hmeblkp, addr, index) \ 873 } 874 875 /* 876 * Version of HBLKTOHME that also returns the index in hmeblkp 877 * of the hment. 878 */ 879 #define HBLKTOHME_IDX(hment, hmeblkp, addr, idx) \ 880 { \ 881 ASSERT(in_hblk_range((hmeblkp), (addr))); \ 882 \ 883 if (get_hblk_ttesz(hmeblkp) == TTE8K) { \ 884 idx = (((uintptr_t)(addr) >> MMU_PAGESHIFT) & (NHMENTS-1)); \ 885 } else \ 886 idx = 0; \ 887 \ 888 (hment) = &(hmeblkp)->hblk_hme[idx]; \ 889 } 890 891 /* 892 * Disable any page sizes not supported by the CPU 893 */ 894 void 895 hat_init_pagesizes() 896 { 897 int i; 898 899 mmu_exported_page_sizes = 0; 900 for (i = TTE8K; i < max_mmu_page_sizes; i++) { 901 902 szc_2_userszc[i] = (uint_t)-1; 903 userszc_2_szc[i] = (uint_t)-1; 904 905 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) { 906 disable_large_pages |= (1 << i); 907 } else { 908 szc_2_userszc[i] = mmu_exported_page_sizes; 909 userszc_2_szc[mmu_exported_page_sizes] = i; 910 mmu_exported_page_sizes++; 911 } 912 } 913 914 disable_ism_large_pages |= disable_large_pages; 915 disable_auto_data_large_pages = disable_large_pages; 916 disable_auto_text_large_pages = disable_large_pages; 917 918 /* 919 * Initialize mmu-specific large page sizes. 920 */ 921 if (&mmu_large_pages_disabled) { 922 disable_large_pages |= mmu_large_pages_disabled(HAT_LOAD); 923 disable_ism_large_pages |= 924 mmu_large_pages_disabled(HAT_LOAD_SHARE); 925 disable_auto_data_large_pages |= 926 mmu_large_pages_disabled(HAT_AUTO_DATA); 927 disable_auto_text_large_pages |= 928 mmu_large_pages_disabled(HAT_AUTO_TEXT); 929 } 930 } 931 932 /* 933 * Initialize the hardware address translation structures. 934 */ 935 void 936 hat_init(void) 937 { 938 int i; 939 uint_t sz; 940 uint_t maxtsb; 941 size_t size; 942 943 hat_lock_init(); 944 hat_kstat_init(); 945 946 /* 947 * Hardware-only bits in a TTE 948 */ 949 MAKE_TTE_MASK(&hw_tte); 950 951 hat_init_pagesizes(); 952 953 /* Initialize the hash locks */ 954 for (i = 0; i < khmehash_num; i++) { 955 mutex_init(&khme_hash[i].hmehash_mutex, NULL, 956 MUTEX_DEFAULT, NULL); 957 } 958 for (i = 0; i < uhmehash_num; i++) { 959 mutex_init(&uhme_hash[i].hmehash_mutex, NULL, 960 MUTEX_DEFAULT, NULL); 961 } 962 khmehash_num--; /* make sure counter starts from 0 */ 963 uhmehash_num--; /* make sure counter starts from 0 */ 964 965 /* 966 * Allocate context domain structures. 967 * 968 * A platform may choose to modify max_mmu_ctxdoms in 969 * set_platform_defaults(). If a platform does not define 970 * a set_platform_defaults() or does not choose to modify 971 * max_mmu_ctxdoms, it gets one MMU context domain for every CPU. 972 * 973 * For sun4v, there will be one global context domain, this is to 974 * avoid the ldom cpu substitution problem. 975 * 976 * For all platforms that have CPUs sharing MMUs, this 977 * value must be defined. 978 */ 979 if (max_mmu_ctxdoms == 0) { 980 #ifndef sun4v 981 max_mmu_ctxdoms = max_ncpus; 982 #else /* sun4v */ 983 max_mmu_ctxdoms = 1; 984 #endif /* sun4v */ 985 } 986 987 size = max_mmu_ctxdoms * sizeof (mmu_ctx_t *); 988 mmu_ctxs_tbl = kmem_zalloc(size, KM_SLEEP); 989 990 /* mmu_ctx_t is 64 bytes aligned */ 991 mmuctxdom_cache = kmem_cache_create("mmuctxdom_cache", 992 sizeof (mmu_ctx_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 993 /* 994 * MMU context domain initialization for the Boot CPU. 995 * This needs the context domains array allocated above. 996 */ 997 mutex_enter(&cpu_lock); 998 sfmmu_cpu_init(CPU); 999 mutex_exit(&cpu_lock); 1000 1001 /* 1002 * Intialize ism mapping list lock. 1003 */ 1004 1005 mutex_init(&ism_mlist_lock, NULL, MUTEX_DEFAULT, NULL); 1006 1007 /* 1008 * Each sfmmu structure carries an array of MMU context info 1009 * structures, one per context domain. The size of this array depends 1010 * on the maximum number of context domains. So, the size of the 1011 * sfmmu structure varies per platform. 1012 * 1013 * sfmmu is allocated from static arena, because trap 1014 * handler at TL > 0 is not allowed to touch kernel relocatable 1015 * memory. sfmmu's alignment is changed to 64 bytes from 1016 * default 8 bytes, as the lower 6 bits will be used to pass 1017 * pgcnt to vtag_flush_pgcnt_tl1. 1018 */ 1019 size = sizeof (sfmmu_t) + sizeof (sfmmu_ctx_t) * (max_mmu_ctxdoms - 1); 1020 1021 sfmmuid_cache = kmem_cache_create("sfmmuid_cache", size, 1022 64, sfmmu_idcache_constructor, sfmmu_idcache_destructor, 1023 NULL, NULL, static_arena, 0); 1024 1025 sfmmu_tsbinfo_cache = kmem_cache_create("sfmmu_tsbinfo_cache", 1026 sizeof (struct tsb_info), 0, NULL, NULL, NULL, NULL, NULL, 0); 1027 1028 /* 1029 * Since we only use the tsb8k cache to "borrow" pages for TSBs 1030 * from the heap when low on memory or when TSB_FORCEALLOC is 1031 * specified, don't use magazines to cache them--we want to return 1032 * them to the system as quickly as possible. 1033 */ 1034 sfmmu_tsb8k_cache = kmem_cache_create("sfmmu_tsb8k_cache", 1035 MMU_PAGESIZE, MMU_PAGESIZE, NULL, NULL, NULL, NULL, 1036 static_arena, KMC_NOMAGAZINE); 1037 1038 /* 1039 * Set tsb_alloc_hiwater to 1/tsb_alloc_hiwater_factor of physical 1040 * memory, which corresponds to the old static reserve for TSBs. 1041 * tsb_alloc_hiwater_factor defaults to 32. This caps the amount of 1042 * memory we'll allocate for TSB slabs; beyond this point TSB 1043 * allocations will be taken from the kernel heap (via 1044 * sfmmu_tsb8k_cache) and will be throttled as would any other kmem 1045 * consumer. 1046 */ 1047 if (tsb_alloc_hiwater_factor == 0) { 1048 tsb_alloc_hiwater_factor = TSB_ALLOC_HIWATER_FACTOR_DEFAULT; 1049 } 1050 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 1051 1052 /* Set tsb_max_growsize. */ 1053 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 1054 1055 /* 1056 * On smaller memory systems, allocate TSB memory in smaller chunks 1057 * than the default 4M slab size. We also honor disable_large_pages 1058 * here. 1059 * 1060 * The trap handlers need to be patched with the final slab shift, 1061 * since they need to be able to construct the TSB pointer at runtime. 1062 */ 1063 if (tsb_max_growsize <= TSB_512K_SZCODE) 1064 tsb_slab_ttesz = TTE512K; 1065 1066 for (sz = tsb_slab_ttesz; sz > 0; sz--) { 1067 if (!(disable_large_pages & (1 << sz))) 1068 break; 1069 } 1070 1071 tsb_slab_ttesz = sz; 1072 tsb_slab_shift = MMU_PAGESHIFT + (sz << 1) + sz; 1073 tsb_slab_size = 1 << tsb_slab_shift; 1074 tsb_slab_mask = (1 << (tsb_slab_shift - MMU_PAGESHIFT)) - 1; 1075 1076 maxtsb = tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT); 1077 if (tsb_max_growsize > maxtsb) 1078 tsb_max_growsize = maxtsb; 1079 1080 /* 1081 * Set up memory callback to update tsb_alloc_hiwater and 1082 * tsb_max_growsize. 1083 */ 1084 i = kphysm_setup_func_register(&sfmmu_update_tsb_vec, (void *) 0); 1085 ASSERT(i == 0); 1086 1087 /* 1088 * kmem_tsb_arena is the source from which large TSB slabs are 1089 * drawn. The quantum of this arena corresponds to the largest 1090 * TSB size we can dynamically allocate for user processes. 1091 * Currently it must also be a supported page size since we 1092 * use exactly one translation entry to map each slab page. 1093 * 1094 * The per-lgroup kmem_tsb_default_arena arenas are the arenas from 1095 * which most TSBs are allocated. Since most TSB allocations are 1096 * typically 8K we have a kmem cache we stack on top of each 1097 * kmem_tsb_default_arena to speed up those allocations. 1098 * 1099 * Note the two-level scheme of arenas is required only 1100 * because vmem_create doesn't allow us to specify alignment 1101 * requirements. If this ever changes the code could be 1102 * simplified to use only one level of arenas. 1103 */ 1104 kmem_tsb_arena = vmem_create("kmem_tsb", NULL, 0, tsb_slab_size, 1105 sfmmu_vmem_xalloc_aligned_wrapper, vmem_xfree, heap_arena, 1106 0, VM_SLEEP); 1107 1108 if (tsb_lgrp_affinity) { 1109 char s[50]; 1110 for (i = 0; i < NLGRPS_MAX; i++) { 1111 (void) sprintf(s, "kmem_tsb_lgrp%d", i); 1112 kmem_tsb_default_arena[i] = 1113 vmem_create(s, NULL, 0, PAGESIZE, 1114 sfmmu_tsb_segkmem_alloc, sfmmu_tsb_segkmem_free, 1115 kmem_tsb_arena, 0, VM_SLEEP | VM_BESTFIT); 1116 (void) sprintf(s, "sfmmu_tsb_lgrp%d_cache", i); 1117 sfmmu_tsb_cache[i] = kmem_cache_create(s, PAGESIZE, 1118 PAGESIZE, NULL, NULL, NULL, NULL, 1119 kmem_tsb_default_arena[i], 0); 1120 } 1121 } else { 1122 kmem_tsb_default_arena[0] = vmem_create("kmem_tsb_default", 1123 NULL, 0, PAGESIZE, sfmmu_tsb_segkmem_alloc, 1124 sfmmu_tsb_segkmem_free, kmem_tsb_arena, 0, 1125 VM_SLEEP | VM_BESTFIT); 1126 1127 sfmmu_tsb_cache[0] = kmem_cache_create("sfmmu_tsb_cache", 1128 PAGESIZE, PAGESIZE, NULL, NULL, NULL, NULL, 1129 kmem_tsb_default_arena[0], 0); 1130 } 1131 1132 sfmmu8_cache = kmem_cache_create("sfmmu8_cache", HME8BLK_SZ, 1133 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1134 sfmmu_hblkcache_destructor, 1135 sfmmu_hblkcache_reclaim, (void *)HME8BLK_SZ, 1136 hat_memload_arena, KMC_NOHASH); 1137 1138 hat_memload1_arena = vmem_create("hat_memload1", NULL, 0, PAGESIZE, 1139 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); 1140 1141 sfmmu1_cache = kmem_cache_create("sfmmu1_cache", HME1BLK_SZ, 1142 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1143 sfmmu_hblkcache_destructor, 1144 NULL, (void *)HME1BLK_SZ, 1145 hat_memload1_arena, KMC_NOHASH); 1146 1147 pa_hment_cache = kmem_cache_create("pa_hment_cache", PAHME_SZ, 1148 0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH); 1149 1150 ism_blk_cache = kmem_cache_create("ism_blk_cache", 1151 sizeof (ism_blk_t), ecache_alignsize, NULL, NULL, 1152 NULL, NULL, static_arena, KMC_NOHASH); 1153 1154 ism_ment_cache = kmem_cache_create("ism_ment_cache", 1155 sizeof (ism_ment_t), 0, NULL, NULL, 1156 NULL, NULL, NULL, 0); 1157 1158 /* 1159 * We grab the first hat for the kernel, 1160 */ 1161 AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER); 1162 kas.a_hat = hat_alloc(&kas); 1163 AS_LOCK_EXIT(&kas, &kas.a_lock); 1164 1165 /* 1166 * Initialize hblk_reserve. 1167 */ 1168 ((struct hme_blk *)hblk_reserve)->hblk_nextpa = 1169 va_to_pa((caddr_t)hblk_reserve); 1170 1171 #ifndef UTSB_PHYS 1172 /* 1173 * Reserve some kernel virtual address space for the locked TTEs 1174 * that allow us to probe the TSB from TL>0. 1175 */ 1176 utsb_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1177 0, 0, NULL, NULL, VM_SLEEP); 1178 utsb4m_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1179 0, 0, NULL, NULL, VM_SLEEP); 1180 #endif 1181 1182 #ifdef VAC 1183 /* 1184 * The big page VAC handling code assumes VAC 1185 * will not be bigger than the smallest big 1186 * page- which is 64K. 1187 */ 1188 if (TTEPAGES(TTE64K) < CACHE_NUM_COLOR) { 1189 cmn_err(CE_PANIC, "VAC too big!"); 1190 } 1191 #endif 1192 1193 (void) xhat_init(); 1194 1195 uhme_hash_pa = va_to_pa(uhme_hash); 1196 khme_hash_pa = va_to_pa(khme_hash); 1197 1198 /* 1199 * Initialize relocation locks. kpr_suspendlock is held 1200 * at PIL_MAX to prevent interrupts from pinning the holder 1201 * of a suspended TTE which may access it leading to a 1202 * deadlock condition. 1203 */ 1204 mutex_init(&kpr_mutex, NULL, MUTEX_DEFAULT, NULL); 1205 mutex_init(&kpr_suspendlock, NULL, MUTEX_SPIN, (void *)PIL_MAX); 1206 1207 /* 1208 * Pre-allocate hrm_hashtab before enabling the collection of 1209 * refmod statistics. Allocating on the fly would mean us 1210 * running the risk of suffering recursive mutex enters or 1211 * deadlocks. 1212 */ 1213 hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *), 1214 KM_SLEEP); 1215 } 1216 1217 /* 1218 * Initialize locking for the hat layer, called early during boot. 1219 */ 1220 static void 1221 hat_lock_init() 1222 { 1223 int i; 1224 1225 /* 1226 * initialize the array of mutexes protecting a page's mapping 1227 * list and p_nrm field. 1228 */ 1229 for (i = 0; i < mml_table_sz; i++) 1230 mutex_init(&mml_table[i], NULL, MUTEX_DEFAULT, NULL); 1231 1232 if (kpm_enable) { 1233 for (i = 0; i < kpmp_table_sz; i++) { 1234 mutex_init(&kpmp_table[i].khl_mutex, NULL, 1235 MUTEX_DEFAULT, NULL); 1236 } 1237 } 1238 1239 /* 1240 * Initialize array of mutex locks that protects sfmmu fields and 1241 * TSB lists. 1242 */ 1243 for (i = 0; i < SFMMU_NUM_LOCK; i++) 1244 mutex_init(HATLOCK_MUTEXP(&hat_lock[i]), NULL, MUTEX_DEFAULT, 1245 NULL); 1246 } 1247 1248 #define SFMMU_KERNEL_MAXVA \ 1249 (kmem64_base ? (uintptr_t)kmem64_end : (SYSLIMIT)) 1250 1251 /* 1252 * Allocate a hat structure. 1253 * Called when an address space first uses a hat. 1254 */ 1255 struct hat * 1256 hat_alloc(struct as *as) 1257 { 1258 sfmmu_t *sfmmup; 1259 int i; 1260 uint64_t cnum; 1261 extern uint_t get_color_start(struct as *); 1262 1263 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 1264 sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP); 1265 sfmmup->sfmmu_as = as; 1266 sfmmup->sfmmu_flags = 0; 1267 LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock); 1268 1269 if (as == &kas) { 1270 ksfmmup = sfmmup; 1271 sfmmup->sfmmu_cext = 0; 1272 cnum = KCONTEXT; 1273 1274 sfmmup->sfmmu_clrstart = 0; 1275 sfmmup->sfmmu_tsb = NULL; 1276 /* 1277 * hat_kern_setup() will call sfmmu_init_ktsbinfo() 1278 * to setup tsb_info for ksfmmup. 1279 */ 1280 } else { 1281 1282 /* 1283 * Just set to invalid ctx. When it faults, it will 1284 * get a valid ctx. This would avoid the situation 1285 * where we get a ctx, but it gets stolen and then 1286 * we fault when we try to run and so have to get 1287 * another ctx. 1288 */ 1289 sfmmup->sfmmu_cext = 0; 1290 cnum = INVALID_CONTEXT; 1291 1292 /* initialize original physical page coloring bin */ 1293 sfmmup->sfmmu_clrstart = get_color_start(as); 1294 #ifdef DEBUG 1295 if (tsb_random_size) { 1296 uint32_t randval = (uint32_t)gettick() >> 4; 1297 int size = randval % (tsb_max_growsize + 1); 1298 1299 /* chose a random tsb size for stress testing */ 1300 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size, 1301 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1302 } else 1303 #endif /* DEBUG */ 1304 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, 1305 default_tsb_size, 1306 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1307 sfmmup->sfmmu_flags = HAT_SWAPPED; 1308 ASSERT(sfmmup->sfmmu_tsb != NULL); 1309 } 1310 1311 ASSERT(max_mmu_ctxdoms > 0); 1312 for (i = 0; i < max_mmu_ctxdoms; i++) { 1313 sfmmup->sfmmu_ctxs[i].cnum = cnum; 1314 sfmmup->sfmmu_ctxs[i].gnum = 0; 1315 } 1316 1317 sfmmu_setup_tsbinfo(sfmmup); 1318 for (i = 0; i < max_mmu_page_sizes; i++) { 1319 sfmmup->sfmmu_ttecnt[i] = 0; 1320 sfmmup->sfmmu_ismttecnt[i] = 0; 1321 sfmmup->sfmmu_pgsz[i] = TTE8K; 1322 } 1323 1324 sfmmup->sfmmu_iblk = NULL; 1325 sfmmup->sfmmu_ismhat = 0; 1326 sfmmup->sfmmu_ismblkpa = (uint64_t)-1; 1327 if (sfmmup == ksfmmup) { 1328 CPUSET_ALL(sfmmup->sfmmu_cpusran); 1329 } else { 1330 CPUSET_ZERO(sfmmup->sfmmu_cpusran); 1331 } 1332 sfmmup->sfmmu_free = 0; 1333 sfmmup->sfmmu_rmstat = 0; 1334 sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart; 1335 sfmmup->sfmmu_xhat_provider = NULL; 1336 cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL); 1337 return (sfmmup); 1338 } 1339 1340 /* 1341 * Create per-MMU context domain kstats for a given MMU ctx. 1342 */ 1343 static void 1344 sfmmu_mmu_kstat_create(mmu_ctx_t *mmu_ctxp) 1345 { 1346 mmu_ctx_stat_t stat; 1347 kstat_t *mmu_kstat; 1348 1349 ASSERT(MUTEX_HELD(&cpu_lock)); 1350 ASSERT(mmu_ctxp->mmu_kstat == NULL); 1351 1352 mmu_kstat = kstat_create("unix", mmu_ctxp->mmu_idx, "mmu_ctx", 1353 "hat", KSTAT_TYPE_NAMED, MMU_CTX_NUM_STATS, KSTAT_FLAG_VIRTUAL); 1354 1355 if (mmu_kstat == NULL) { 1356 cmn_err(CE_WARN, "kstat_create for MMU %d failed", 1357 mmu_ctxp->mmu_idx); 1358 } else { 1359 mmu_kstat->ks_data = mmu_ctxp->mmu_kstat_data; 1360 for (stat = 0; stat < MMU_CTX_NUM_STATS; stat++) 1361 kstat_named_init(&mmu_ctxp->mmu_kstat_data[stat], 1362 mmu_ctx_kstat_names[stat], KSTAT_DATA_INT64); 1363 mmu_ctxp->mmu_kstat = mmu_kstat; 1364 kstat_install(mmu_kstat); 1365 } 1366 } 1367 1368 /* 1369 * plat_cpuid_to_mmu_ctx_info() is a platform interface that returns MMU 1370 * context domain information for a given CPU. If a platform does not 1371 * specify that interface, then the function below is used instead to return 1372 * default information. The defaults are as follows: 1373 * 1374 * - For sun4u systems there's one MMU context domain per CPU. 1375 * This default is used by all sun4u systems except OPL. OPL systems 1376 * provide platform specific interface to map CPU ids to MMU ids 1377 * because on OPL more than 1 CPU shares a single MMU. 1378 * Note that on sun4v, there is one global context domain for 1379 * the entire system. This is to avoid running into potential problem 1380 * with ldom physical cpu substitution feature. 1381 * - The number of MMU context IDs supported on any CPU in the 1382 * system is 8K. 1383 */ 1384 /*ARGSUSED*/ 1385 static void 1386 sfmmu_cpuid_to_mmu_ctx_info(processorid_t cpuid, mmu_ctx_info_t *infop) 1387 { 1388 infop->mmu_nctxs = nctxs; 1389 #ifndef sun4v 1390 infop->mmu_idx = cpu[cpuid]->cpu_seqid; 1391 #else /* sun4v */ 1392 infop->mmu_idx = 0; 1393 #endif /* sun4v */ 1394 } 1395 1396 /* 1397 * Called during CPU initialization to set the MMU context-related information 1398 * for a CPU. 1399 * 1400 * cpu_lock serializes accesses to mmu_ctxs and mmu_saved_gnum. 1401 */ 1402 void 1403 sfmmu_cpu_init(cpu_t *cp) 1404 { 1405 mmu_ctx_info_t info; 1406 mmu_ctx_t *mmu_ctxp; 1407 1408 ASSERT(MUTEX_HELD(&cpu_lock)); 1409 1410 if (&plat_cpuid_to_mmu_ctx_info == NULL) 1411 sfmmu_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1412 else 1413 plat_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1414 1415 ASSERT(info.mmu_idx < max_mmu_ctxdoms); 1416 1417 if ((mmu_ctxp = mmu_ctxs_tbl[info.mmu_idx]) == NULL) { 1418 /* Each mmu_ctx is cacheline aligned. */ 1419 mmu_ctxp = kmem_cache_alloc(mmuctxdom_cache, KM_SLEEP); 1420 bzero(mmu_ctxp, sizeof (mmu_ctx_t)); 1421 1422 mutex_init(&mmu_ctxp->mmu_lock, NULL, MUTEX_SPIN, 1423 (void *)ipltospl(DISP_LEVEL)); 1424 mmu_ctxp->mmu_idx = info.mmu_idx; 1425 mmu_ctxp->mmu_nctxs = info.mmu_nctxs; 1426 /* 1427 * Globally for lifetime of a system, 1428 * gnum must always increase. 1429 * mmu_saved_gnum is protected by the cpu_lock. 1430 */ 1431 mmu_ctxp->mmu_gnum = mmu_saved_gnum + 1; 1432 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 1433 1434 sfmmu_mmu_kstat_create(mmu_ctxp); 1435 1436 mmu_ctxs_tbl[info.mmu_idx] = mmu_ctxp; 1437 } else { 1438 ASSERT(mmu_ctxp->mmu_idx == info.mmu_idx); 1439 } 1440 1441 /* 1442 * The mmu_lock is acquired here to prevent races with 1443 * the wrap-around code. 1444 */ 1445 mutex_enter(&mmu_ctxp->mmu_lock); 1446 1447 1448 mmu_ctxp->mmu_ncpus++; 1449 CPUSET_ADD(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1450 CPU_MMU_IDX(cp) = info.mmu_idx; 1451 CPU_MMU_CTXP(cp) = mmu_ctxp; 1452 1453 mutex_exit(&mmu_ctxp->mmu_lock); 1454 } 1455 1456 /* 1457 * Called to perform MMU context-related cleanup for a CPU. 1458 */ 1459 void 1460 sfmmu_cpu_cleanup(cpu_t *cp) 1461 { 1462 mmu_ctx_t *mmu_ctxp; 1463 1464 ASSERT(MUTEX_HELD(&cpu_lock)); 1465 1466 mmu_ctxp = CPU_MMU_CTXP(cp); 1467 ASSERT(mmu_ctxp != NULL); 1468 1469 /* 1470 * The mmu_lock is acquired here to prevent races with 1471 * the wrap-around code. 1472 */ 1473 mutex_enter(&mmu_ctxp->mmu_lock); 1474 1475 CPU_MMU_CTXP(cp) = NULL; 1476 1477 CPUSET_DEL(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1478 if (--mmu_ctxp->mmu_ncpus == 0) { 1479 mmu_ctxs_tbl[mmu_ctxp->mmu_idx] = NULL; 1480 mutex_exit(&mmu_ctxp->mmu_lock); 1481 mutex_destroy(&mmu_ctxp->mmu_lock); 1482 1483 if (mmu_ctxp->mmu_kstat) 1484 kstat_delete(mmu_ctxp->mmu_kstat); 1485 1486 /* mmu_saved_gnum is protected by the cpu_lock. */ 1487 if (mmu_saved_gnum < mmu_ctxp->mmu_gnum) 1488 mmu_saved_gnum = mmu_ctxp->mmu_gnum; 1489 1490 kmem_cache_free(mmuctxdom_cache, mmu_ctxp); 1491 1492 return; 1493 } 1494 1495 mutex_exit(&mmu_ctxp->mmu_lock); 1496 } 1497 1498 /* 1499 * Hat_setup, makes an address space context the current active one. 1500 * In sfmmu this translates to setting the secondary context with the 1501 * corresponding context. 1502 */ 1503 void 1504 hat_setup(struct hat *sfmmup, int allocflag) 1505 { 1506 hatlock_t *hatlockp; 1507 1508 /* Init needs some special treatment. */ 1509 if (allocflag == HAT_INIT) { 1510 /* 1511 * Make sure that we have 1512 * 1. a TSB 1513 * 2. a valid ctx that doesn't get stolen after this point. 1514 */ 1515 hatlockp = sfmmu_hat_enter(sfmmup); 1516 1517 /* 1518 * Swap in the TSB. hat_init() allocates tsbinfos without 1519 * TSBs, but we need one for init, since the kernel does some 1520 * special things to set up its stack and needs the TSB to 1521 * resolve page faults. 1522 */ 1523 sfmmu_tsb_swapin(sfmmup, hatlockp); 1524 1525 sfmmu_get_ctx(sfmmup); 1526 1527 sfmmu_hat_exit(hatlockp); 1528 } else { 1529 ASSERT(allocflag == HAT_ALLOC); 1530 1531 hatlockp = sfmmu_hat_enter(sfmmup); 1532 kpreempt_disable(); 1533 1534 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); 1535 1536 /* 1537 * sfmmu_setctx_sec takes <pgsz|cnum> as a parameter, 1538 * pagesize bits don't matter in this case since we are passing 1539 * INVALID_CONTEXT to it. 1540 */ 1541 sfmmu_setctx_sec(INVALID_CONTEXT); 1542 sfmmu_clear_utsbinfo(); 1543 1544 kpreempt_enable(); 1545 sfmmu_hat_exit(hatlockp); 1546 } 1547 } 1548 1549 /* 1550 * Free all the translation resources for the specified address space. 1551 * Called from as_free when an address space is being destroyed. 1552 */ 1553 void 1554 hat_free_start(struct hat *sfmmup) 1555 { 1556 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 1557 ASSERT(sfmmup != ksfmmup); 1558 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1559 1560 sfmmup->sfmmu_free = 1; 1561 } 1562 1563 void 1564 hat_free_end(struct hat *sfmmup) 1565 { 1566 int i; 1567 1568 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1569 if (sfmmup->sfmmu_ismhat) { 1570 for (i = 0; i < mmu_page_sizes; i++) { 1571 sfmmup->sfmmu_ttecnt[i] = 0; 1572 sfmmup->sfmmu_ismttecnt[i] = 0; 1573 } 1574 } else { 1575 /* EMPTY */ 1576 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 1577 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 1578 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 1579 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 1580 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 1581 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 1582 } 1583 1584 if (sfmmup->sfmmu_rmstat) { 1585 hat_freestat(sfmmup->sfmmu_as, NULL); 1586 } 1587 1588 while (sfmmup->sfmmu_tsb != NULL) { 1589 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next; 1590 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb); 1591 sfmmup->sfmmu_tsb = next; 1592 } 1593 sfmmu_free_sfmmu(sfmmup); 1594 1595 kmem_cache_free(sfmmuid_cache, sfmmup); 1596 } 1597 1598 /* 1599 * Set up any translation structures, for the specified address space, 1600 * that are needed or preferred when the process is being swapped in. 1601 */ 1602 /* ARGSUSED */ 1603 void 1604 hat_swapin(struct hat *hat) 1605 { 1606 ASSERT(hat->sfmmu_xhat_provider == NULL); 1607 } 1608 1609 /* 1610 * Free all of the translation resources, for the specified address space, 1611 * that can be freed while the process is swapped out. Called from as_swapout. 1612 * Also, free up the ctx that this process was using. 1613 */ 1614 void 1615 hat_swapout(struct hat *sfmmup) 1616 { 1617 struct hmehash_bucket *hmebp; 1618 struct hme_blk *hmeblkp; 1619 struct hme_blk *pr_hblk = NULL; 1620 struct hme_blk *nx_hblk; 1621 int i; 1622 uint64_t hblkpa, prevpa, nx_pa; 1623 struct hme_blk *list = NULL; 1624 hatlock_t *hatlockp; 1625 struct tsb_info *tsbinfop; 1626 struct free_tsb { 1627 struct free_tsb *next; 1628 struct tsb_info *tsbinfop; 1629 }; /* free list of TSBs */ 1630 struct free_tsb *freelist, *last, *next; 1631 1632 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1633 SFMMU_STAT(sf_swapout); 1634 1635 /* 1636 * There is no way to go from an as to all its translations in sfmmu. 1637 * Here is one of the times when we take the big hit and traverse 1638 * the hash looking for hme_blks to free up. Not only do we free up 1639 * this as hme_blks but all those that are free. We are obviously 1640 * swapping because we need memory so let's free up as much 1641 * as we can. 1642 * 1643 * Note that we don't flush TLB/TSB here -- it's not necessary 1644 * because: 1645 * 1) we free the ctx we're using and throw away the TSB(s); 1646 * 2) processes aren't runnable while being swapped out. 1647 */ 1648 ASSERT(sfmmup != KHATID); 1649 for (i = 0; i <= UHMEHASH_SZ; i++) { 1650 hmebp = &uhme_hash[i]; 1651 SFMMU_HASH_LOCK(hmebp); 1652 hmeblkp = hmebp->hmeblkp; 1653 hblkpa = hmebp->hmeh_nextpa; 1654 prevpa = 0; 1655 pr_hblk = NULL; 1656 while (hmeblkp) { 1657 1658 ASSERT(!hmeblkp->hblk_xhat_bit); 1659 1660 if ((hmeblkp->hblk_tag.htag_id == sfmmup) && 1661 !hmeblkp->hblk_shw_bit && !hmeblkp->hblk_lckcnt) { 1662 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 1663 (caddr_t)get_hblk_base(hmeblkp), 1664 get_hblk_endaddr(hmeblkp), 1665 NULL, HAT_UNLOAD); 1666 } 1667 nx_hblk = hmeblkp->hblk_next; 1668 nx_pa = hmeblkp->hblk_nextpa; 1669 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 1670 ASSERT(!hmeblkp->hblk_lckcnt); 1671 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 1672 prevpa, pr_hblk); 1673 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 1674 } else { 1675 pr_hblk = hmeblkp; 1676 prevpa = hblkpa; 1677 } 1678 hmeblkp = nx_hblk; 1679 hblkpa = nx_pa; 1680 } 1681 SFMMU_HASH_UNLOCK(hmebp); 1682 } 1683 1684 sfmmu_hblks_list_purge(&list); 1685 1686 /* 1687 * Now free up the ctx so that others can reuse it. 1688 */ 1689 hatlockp = sfmmu_hat_enter(sfmmup); 1690 1691 sfmmu_invalidate_ctx(sfmmup); 1692 1693 /* 1694 * Free TSBs, but not tsbinfos, and set SWAPPED flag. 1695 * If TSBs were never swapped in, just return. 1696 * This implies that we don't support partial swapping 1697 * of TSBs -- either all are swapped out, or none are. 1698 * 1699 * We must hold the HAT lock here to prevent racing with another 1700 * thread trying to unmap TTEs from the TSB or running the post- 1701 * relocator after relocating the TSB's memory. Unfortunately, we 1702 * can't free memory while holding the HAT lock or we could 1703 * deadlock, so we build a list of TSBs to be freed after marking 1704 * the tsbinfos as swapped out and free them after dropping the 1705 * lock. 1706 */ 1707 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 1708 sfmmu_hat_exit(hatlockp); 1709 return; 1710 } 1711 1712 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED); 1713 last = freelist = NULL; 1714 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 1715 tsbinfop = tsbinfop->tsb_next) { 1716 ASSERT((tsbinfop->tsb_flags & TSB_SWAPPED) == 0); 1717 1718 /* 1719 * Cast the TSB into a struct free_tsb and put it on the free 1720 * list. 1721 */ 1722 if (freelist == NULL) { 1723 last = freelist = (struct free_tsb *)tsbinfop->tsb_va; 1724 } else { 1725 last->next = (struct free_tsb *)tsbinfop->tsb_va; 1726 last = last->next; 1727 } 1728 last->next = NULL; 1729 last->tsbinfop = tsbinfop; 1730 tsbinfop->tsb_flags |= TSB_SWAPPED; 1731 /* 1732 * Zero out the TTE to clear the valid bit. 1733 * Note we can't use a value like 0xbad because we want to 1734 * ensure diagnostic bits are NEVER set on TTEs that might 1735 * be loaded. The intent is to catch any invalid access 1736 * to the swapped TSB, such as a thread running with a valid 1737 * context without first calling sfmmu_tsb_swapin() to 1738 * allocate TSB memory. 1739 */ 1740 tsbinfop->tsb_tte.ll = 0; 1741 } 1742 1743 #ifdef sun4v 1744 if (freelist) 1745 sfmmu_invalidate_tsbinfo(sfmmup); 1746 #endif /* sun4v */ 1747 1748 /* Now we can drop the lock and free the TSB memory. */ 1749 sfmmu_hat_exit(hatlockp); 1750 for (; freelist != NULL; freelist = next) { 1751 next = freelist->next; 1752 sfmmu_tsb_free(freelist->tsbinfop); 1753 } 1754 } 1755 1756 /* 1757 * Duplicate the translations of an as into another newas 1758 */ 1759 /* ARGSUSED */ 1760 int 1761 hat_dup(struct hat *hat, struct hat *newhat, caddr_t addr, size_t len, 1762 uint_t flag) 1763 { 1764 ASSERT(hat->sfmmu_xhat_provider == NULL); 1765 ASSERT((flag == 0) || (flag == HAT_DUP_ALL) || (flag == HAT_DUP_COW)); 1766 1767 if (flag == HAT_DUP_COW) { 1768 panic("hat_dup: HAT_DUP_COW not supported"); 1769 } 1770 return (0); 1771 } 1772 1773 /* 1774 * Set up addr to map to page pp with protection prot. 1775 * As an optimization we also load the TSB with the 1776 * corresponding tte but it is no big deal if the tte gets kicked out. 1777 */ 1778 void 1779 hat_memload(struct hat *hat, caddr_t addr, struct page *pp, 1780 uint_t attr, uint_t flags) 1781 { 1782 tte_t tte; 1783 1784 1785 ASSERT(hat != NULL); 1786 ASSERT(PAGE_LOCKED(pp)); 1787 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 1788 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1789 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1790 1791 if (PP_ISFREE(pp)) { 1792 panic("hat_memload: loading a mapping to free page %p", 1793 (void *)pp); 1794 } 1795 1796 if (hat->sfmmu_xhat_provider) { 1797 XHAT_MEMLOAD(hat, addr, pp, attr, flags); 1798 return; 1799 } 1800 1801 ASSERT((hat == ksfmmup) || 1802 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1803 1804 if (flags & ~SFMMU_LOAD_ALLFLAG) 1805 cmn_err(CE_NOTE, "hat_memload: unsupported flags %d", 1806 flags & ~SFMMU_LOAD_ALLFLAG); 1807 1808 if (hat->sfmmu_rmstat) 1809 hat_resvstat(MMU_PAGESIZE, hat->sfmmu_as, addr); 1810 1811 #if defined(SF_ERRATA_57) 1812 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1813 (addr < errata57_limit) && (attr & PROT_EXEC) && 1814 !(flags & HAT_LOAD_SHARE)) { 1815 cmn_err(CE_WARN, "hat_memload: illegal attempt to make user " 1816 " page executable"); 1817 attr &= ~PROT_EXEC; 1818 } 1819 #endif 1820 1821 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 1822 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, flags); 1823 1824 /* 1825 * Check TSB and TLB page sizes. 1826 */ 1827 if ((flags & HAT_LOAD_SHARE) == 0) { 1828 sfmmu_check_page_sizes(hat, 1); 1829 } 1830 } 1831 1832 /* 1833 * hat_devload can be called to map real memory (e.g. 1834 * /dev/kmem) and even though hat_devload will determine pf is 1835 * for memory, it will be unable to get a shared lock on the 1836 * page (because someone else has it exclusively) and will 1837 * pass dp = NULL. If tteload doesn't get a non-NULL 1838 * page pointer it can't cache memory. 1839 */ 1840 void 1841 hat_devload(struct hat *hat, caddr_t addr, size_t len, pfn_t pfn, 1842 uint_t attr, int flags) 1843 { 1844 tte_t tte; 1845 struct page *pp = NULL; 1846 int use_lgpg = 0; 1847 1848 ASSERT(hat != NULL); 1849 1850 if (hat->sfmmu_xhat_provider) { 1851 XHAT_DEVLOAD(hat, addr, len, pfn, attr, flags); 1852 return; 1853 } 1854 1855 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1856 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1857 ASSERT((hat == ksfmmup) || 1858 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1859 if (len == 0) 1860 panic("hat_devload: zero len"); 1861 if (flags & ~SFMMU_LOAD_ALLFLAG) 1862 cmn_err(CE_NOTE, "hat_devload: unsupported flags %d", 1863 flags & ~SFMMU_LOAD_ALLFLAG); 1864 1865 #if defined(SF_ERRATA_57) 1866 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1867 (addr < errata57_limit) && (attr & PROT_EXEC) && 1868 !(flags & HAT_LOAD_SHARE)) { 1869 cmn_err(CE_WARN, "hat_devload: illegal attempt to make user " 1870 " page executable"); 1871 attr &= ~PROT_EXEC; 1872 } 1873 #endif 1874 1875 /* 1876 * If it's a memory page find its pp 1877 */ 1878 if (!(flags & HAT_LOAD_NOCONSIST) && pf_is_memory(pfn)) { 1879 pp = page_numtopp_nolock(pfn); 1880 if (pp == NULL) { 1881 flags |= HAT_LOAD_NOCONSIST; 1882 } else { 1883 if (PP_ISFREE(pp)) { 1884 panic("hat_memload: loading " 1885 "a mapping to free page %p", 1886 (void *)pp); 1887 } 1888 if (!PAGE_LOCKED(pp) && !PP_ISNORELOC(pp)) { 1889 panic("hat_memload: loading a mapping " 1890 "to unlocked relocatable page %p", 1891 (void *)pp); 1892 } 1893 ASSERT(len == MMU_PAGESIZE); 1894 } 1895 } 1896 1897 if (hat->sfmmu_rmstat) 1898 hat_resvstat(len, hat->sfmmu_as, addr); 1899 1900 if (flags & HAT_LOAD_NOCONSIST) { 1901 attr |= SFMMU_UNCACHEVTTE; 1902 use_lgpg = 1; 1903 } 1904 if (!pf_is_memory(pfn)) { 1905 attr |= SFMMU_UNCACHEPTTE | HAT_NOSYNC; 1906 use_lgpg = 1; 1907 switch (attr & HAT_ORDER_MASK) { 1908 case HAT_STRICTORDER: 1909 case HAT_UNORDERED_OK: 1910 /* 1911 * we set the side effect bit for all non 1912 * memory mappings unless merging is ok 1913 */ 1914 attr |= SFMMU_SIDEFFECT; 1915 break; 1916 case HAT_MERGING_OK: 1917 case HAT_LOADCACHING_OK: 1918 case HAT_STORECACHING_OK: 1919 break; 1920 default: 1921 panic("hat_devload: bad attr"); 1922 break; 1923 } 1924 } 1925 while (len) { 1926 if (!use_lgpg) { 1927 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1928 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1929 flags); 1930 len -= MMU_PAGESIZE; 1931 addr += MMU_PAGESIZE; 1932 pfn++; 1933 continue; 1934 } 1935 /* 1936 * try to use large pages, check va/pa alignments 1937 * Note that 32M/256M page sizes are not (yet) supported. 1938 */ 1939 if ((len >= MMU_PAGESIZE4M) && 1940 !((uintptr_t)addr & MMU_PAGEOFFSET4M) && 1941 !(disable_large_pages & (1 << TTE4M)) && 1942 !(mmu_ptob(pfn) & MMU_PAGEOFFSET4M)) { 1943 sfmmu_memtte(&tte, pfn, attr, TTE4M); 1944 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1945 flags); 1946 len -= MMU_PAGESIZE4M; 1947 addr += MMU_PAGESIZE4M; 1948 pfn += MMU_PAGESIZE4M / MMU_PAGESIZE; 1949 } else if ((len >= MMU_PAGESIZE512K) && 1950 !((uintptr_t)addr & MMU_PAGEOFFSET512K) && 1951 !(disable_large_pages & (1 << TTE512K)) && 1952 !(mmu_ptob(pfn) & MMU_PAGEOFFSET512K)) { 1953 sfmmu_memtte(&tte, pfn, attr, TTE512K); 1954 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1955 flags); 1956 len -= MMU_PAGESIZE512K; 1957 addr += MMU_PAGESIZE512K; 1958 pfn += MMU_PAGESIZE512K / MMU_PAGESIZE; 1959 } else if ((len >= MMU_PAGESIZE64K) && 1960 !((uintptr_t)addr & MMU_PAGEOFFSET64K) && 1961 !(disable_large_pages & (1 << TTE64K)) && 1962 !(mmu_ptob(pfn) & MMU_PAGEOFFSET64K)) { 1963 sfmmu_memtte(&tte, pfn, attr, TTE64K); 1964 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1965 flags); 1966 len -= MMU_PAGESIZE64K; 1967 addr += MMU_PAGESIZE64K; 1968 pfn += MMU_PAGESIZE64K / MMU_PAGESIZE; 1969 } else { 1970 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1971 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1972 flags); 1973 len -= MMU_PAGESIZE; 1974 addr += MMU_PAGESIZE; 1975 pfn++; 1976 } 1977 } 1978 1979 /* 1980 * Check TSB and TLB page sizes. 1981 */ 1982 if ((flags & HAT_LOAD_SHARE) == 0) { 1983 sfmmu_check_page_sizes(hat, 1); 1984 } 1985 } 1986 1987 /* 1988 * Map the largest extend possible out of the page array. The array may NOT 1989 * be in order. The largest possible mapping a page can have 1990 * is specified in the p_szc field. The p_szc field 1991 * cannot change as long as there any mappings (large or small) 1992 * to any of the pages that make up the large page. (ie. any 1993 * promotion/demotion of page size is not up to the hat but up to 1994 * the page free list manager). The array 1995 * should consist of properly aligned contigous pages that are 1996 * part of a big page for a large mapping to be created. 1997 */ 1998 void 1999 hat_memload_array(struct hat *hat, caddr_t addr, size_t len, 2000 struct page **pps, uint_t attr, uint_t flags) 2001 { 2002 int ttesz; 2003 size_t mapsz; 2004 pgcnt_t numpg, npgs; 2005 tte_t tte; 2006 page_t *pp; 2007 uint_t large_pages_disable; 2008 2009 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 2010 2011 if (hat->sfmmu_xhat_provider) { 2012 XHAT_MEMLOAD_ARRAY(hat, addr, len, pps, attr, flags); 2013 return; 2014 } 2015 2016 if (hat->sfmmu_rmstat) 2017 hat_resvstat(len, hat->sfmmu_as, addr); 2018 2019 #if defined(SF_ERRATA_57) 2020 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 2021 (addr < errata57_limit) && (attr & PROT_EXEC) && 2022 !(flags & HAT_LOAD_SHARE)) { 2023 cmn_err(CE_WARN, "hat_memload_array: illegal attempt to make " 2024 "user page executable"); 2025 attr &= ~PROT_EXEC; 2026 } 2027 #endif 2028 2029 /* Get number of pages */ 2030 npgs = len >> MMU_PAGESHIFT; 2031 2032 if (flags & HAT_LOAD_SHARE) { 2033 large_pages_disable = disable_ism_large_pages; 2034 } else { 2035 large_pages_disable = disable_large_pages; 2036 } 2037 2038 if (npgs < NHMENTS || large_pages_disable == LARGE_PAGES_OFF) { 2039 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2040 return; 2041 } 2042 2043 while (npgs >= NHMENTS) { 2044 pp = *pps; 2045 for (ttesz = pp->p_szc; ttesz != TTE8K; ttesz--) { 2046 /* 2047 * Check if this page size is disabled. 2048 */ 2049 if (large_pages_disable & (1 << ttesz)) 2050 continue; 2051 2052 numpg = TTEPAGES(ttesz); 2053 mapsz = numpg << MMU_PAGESHIFT; 2054 if ((npgs >= numpg) && 2055 IS_P2ALIGNED(addr, mapsz) && 2056 IS_P2ALIGNED(pp->p_pagenum, numpg)) { 2057 /* 2058 * At this point we have enough pages and 2059 * we know the virtual address and the pfn 2060 * are properly aligned. We still need 2061 * to check for physical contiguity but since 2062 * it is very likely that this is the case 2063 * we will assume they are so and undo 2064 * the request if necessary. It would 2065 * be great if we could get a hint flag 2066 * like HAT_CONTIG which would tell us 2067 * the pages are contigous for sure. 2068 */ 2069 sfmmu_memtte(&tte, (*pps)->p_pagenum, 2070 attr, ttesz); 2071 if (!sfmmu_tteload_array(hat, &tte, addr, 2072 pps, flags)) { 2073 break; 2074 } 2075 } 2076 } 2077 if (ttesz == TTE8K) { 2078 /* 2079 * We were not able to map array using a large page 2080 * batch a hmeblk or fraction at a time. 2081 */ 2082 numpg = ((uintptr_t)addr >> MMU_PAGESHIFT) 2083 & (NHMENTS-1); 2084 numpg = NHMENTS - numpg; 2085 ASSERT(numpg <= npgs); 2086 mapsz = numpg * MMU_PAGESIZE; 2087 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, 2088 numpg); 2089 } 2090 addr += mapsz; 2091 npgs -= numpg; 2092 pps += numpg; 2093 } 2094 2095 if (npgs) { 2096 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2097 } 2098 2099 /* 2100 * Check TSB and TLB page sizes. 2101 */ 2102 if ((flags & HAT_LOAD_SHARE) == 0) { 2103 sfmmu_check_page_sizes(hat, 1); 2104 } 2105 } 2106 2107 /* 2108 * Function tries to batch 8K pages into the same hme blk. 2109 */ 2110 static void 2111 sfmmu_memload_batchsmall(struct hat *hat, caddr_t vaddr, page_t **pps, 2112 uint_t attr, uint_t flags, pgcnt_t npgs) 2113 { 2114 tte_t tte; 2115 page_t *pp; 2116 struct hmehash_bucket *hmebp; 2117 struct hme_blk *hmeblkp; 2118 int index; 2119 2120 while (npgs) { 2121 /* 2122 * Acquire the hash bucket. 2123 */ 2124 hmebp = sfmmu_tteload_acquire_hashbucket(hat, vaddr, TTE8K); 2125 ASSERT(hmebp); 2126 2127 /* 2128 * Find the hment block. 2129 */ 2130 hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr, 2131 TTE8K, flags); 2132 ASSERT(hmeblkp); 2133 2134 do { 2135 /* 2136 * Make the tte. 2137 */ 2138 pp = *pps; 2139 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 2140 2141 /* 2142 * Add the translation. 2143 */ 2144 (void) sfmmu_tteload_addentry(hat, hmeblkp, &tte, 2145 vaddr, pps, flags); 2146 2147 /* 2148 * Goto next page. 2149 */ 2150 pps++; 2151 npgs--; 2152 2153 /* 2154 * Goto next address. 2155 */ 2156 vaddr += MMU_PAGESIZE; 2157 2158 /* 2159 * Don't crossover into a different hmentblk. 2160 */ 2161 index = (int)(((uintptr_t)vaddr >> MMU_PAGESHIFT) & 2162 (NHMENTS-1)); 2163 2164 } while (index != 0 && npgs != 0); 2165 2166 /* 2167 * Release the hash bucket. 2168 */ 2169 2170 sfmmu_tteload_release_hashbucket(hmebp); 2171 } 2172 } 2173 2174 /* 2175 * Construct a tte for a page: 2176 * 2177 * tte_valid = 1 2178 * tte_size2 = size & TTE_SZ2_BITS (Panther and Olympus-C only) 2179 * tte_size = size 2180 * tte_nfo = attr & HAT_NOFAULT 2181 * tte_ie = attr & HAT_STRUCTURE_LE 2182 * tte_hmenum = hmenum 2183 * tte_pahi = pp->p_pagenum >> TTE_PASHIFT; 2184 * tte_palo = pp->p_pagenum & TTE_PALOMASK; 2185 * tte_ref = 1 (optimization) 2186 * tte_wr_perm = attr & PROT_WRITE; 2187 * tte_no_sync = attr & HAT_NOSYNC 2188 * tte_lock = attr & SFMMU_LOCKTTE 2189 * tte_cp = !(attr & SFMMU_UNCACHEPTTE) 2190 * tte_cv = !(attr & SFMMU_UNCACHEVTTE) 2191 * tte_e = attr & SFMMU_SIDEFFECT 2192 * tte_priv = !(attr & PROT_USER) 2193 * tte_hwwr = if nosync is set and it is writable we set the mod bit (opt) 2194 * tte_glb = 0 2195 */ 2196 void 2197 sfmmu_memtte(tte_t *ttep, pfn_t pfn, uint_t attr, int tte_sz) 2198 { 2199 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 2200 2201 ttep->tte_inthi = MAKE_TTE_INTHI(pfn, attr, tte_sz, 0 /* hmenum */); 2202 ttep->tte_intlo = MAKE_TTE_INTLO(pfn, attr, tte_sz, 0 /* hmenum */); 2203 2204 if (TTE_IS_NOSYNC(ttep)) { 2205 TTE_SET_REF(ttep); 2206 if (TTE_IS_WRITABLE(ttep)) { 2207 TTE_SET_MOD(ttep); 2208 } 2209 } 2210 if (TTE_IS_NFO(ttep) && TTE_IS_EXECUTABLE(ttep)) { 2211 panic("sfmmu_memtte: can't set both NFO and EXEC bits"); 2212 } 2213 } 2214 2215 /* 2216 * This function will add a translation to the hme_blk and allocate the 2217 * hme_blk if one does not exist. 2218 * If a page structure is specified then it will add the 2219 * corresponding hment to the mapping list. 2220 * It will also update the hmenum field for the tte. 2221 */ 2222 void 2223 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp, 2224 uint_t flags) 2225 { 2226 (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags); 2227 } 2228 2229 /* 2230 * Load (ttep != NULL) or unload (ttep == NULL) one entry in the TSB. 2231 * Assumes that a particular page size may only be resident in one TSB. 2232 */ 2233 static void 2234 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz) 2235 { 2236 struct tsb_info *tsbinfop = NULL; 2237 uint64_t tag; 2238 struct tsbe *tsbe_addr; 2239 uint64_t tsb_base; 2240 uint_t tsb_size; 2241 int vpshift = MMU_PAGESHIFT; 2242 int phys = 0; 2243 2244 if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */ 2245 phys = ktsb_phys; 2246 if (ttesz >= TTE4M) { 2247 #ifndef sun4v 2248 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2249 #endif 2250 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2251 tsb_size = ktsb4m_szcode; 2252 } else { 2253 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2254 tsb_size = ktsb_szcode; 2255 } 2256 } else { 2257 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2258 2259 /* 2260 * If there isn't a TSB for this page size, or the TSB is 2261 * swapped out, there is nothing to do. Note that the latter 2262 * case seems impossible but can occur if hat_pageunload() 2263 * is called on an ISM mapping while the process is swapped 2264 * out. 2265 */ 2266 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2267 return; 2268 2269 /* 2270 * If another thread is in the middle of relocating a TSB 2271 * we can't unload the entry so set a flag so that the 2272 * TSB will be flushed before it can be accessed by the 2273 * process. 2274 */ 2275 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2276 if (ttep == NULL) 2277 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2278 return; 2279 } 2280 #if defined(UTSB_PHYS) 2281 phys = 1; 2282 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2283 #else 2284 tsb_base = (uint64_t)tsbinfop->tsb_va; 2285 #endif 2286 tsb_size = tsbinfop->tsb_szc; 2287 } 2288 if (ttesz >= TTE4M) 2289 vpshift = MMU_PAGESHIFT4M; 2290 2291 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2292 tag = sfmmu_make_tsbtag(vaddr); 2293 2294 if (ttep == NULL) { 2295 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2296 } else { 2297 if (ttesz >= TTE4M) { 2298 SFMMU_STAT(sf_tsb_load4m); 2299 } else { 2300 SFMMU_STAT(sf_tsb_load8k); 2301 } 2302 2303 sfmmu_load_tsbe(tsbe_addr, tag, ttep, phys); 2304 } 2305 } 2306 2307 /* 2308 * Unmap all entries from [start, end) matching the given page size. 2309 * 2310 * This function is used primarily to unmap replicated 64K or 512K entries 2311 * from the TSB that are inserted using the base page size TSB pointer, but 2312 * it may also be called to unmap a range of addresses from the TSB. 2313 */ 2314 void 2315 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz) 2316 { 2317 struct tsb_info *tsbinfop; 2318 uint64_t tag; 2319 struct tsbe *tsbe_addr; 2320 caddr_t vaddr; 2321 uint64_t tsb_base; 2322 int vpshift, vpgsz; 2323 uint_t tsb_size; 2324 int phys = 0; 2325 2326 /* 2327 * Assumptions: 2328 * If ttesz == 8K, 64K or 512K, we walk through the range 8K 2329 * at a time shooting down any valid entries we encounter. 2330 * 2331 * If ttesz >= 4M we walk the range 4M at a time shooting 2332 * down any valid mappings we find. 2333 */ 2334 if (sfmmup == ksfmmup) { 2335 phys = ktsb_phys; 2336 if (ttesz >= TTE4M) { 2337 #ifndef sun4v 2338 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2339 #endif 2340 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2341 tsb_size = ktsb4m_szcode; 2342 } else { 2343 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2344 tsb_size = ktsb_szcode; 2345 } 2346 } else { 2347 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2348 2349 /* 2350 * If there isn't a TSB for this page size, or the TSB is 2351 * swapped out, there is nothing to do. Note that the latter 2352 * case seems impossible but can occur if hat_pageunload() 2353 * is called on an ISM mapping while the process is swapped 2354 * out. 2355 */ 2356 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2357 return; 2358 2359 /* 2360 * If another thread is in the middle of relocating a TSB 2361 * we can't unload the entry so set a flag so that the 2362 * TSB will be flushed before it can be accessed by the 2363 * process. 2364 */ 2365 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2366 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2367 return; 2368 } 2369 #if defined(UTSB_PHYS) 2370 phys = 1; 2371 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2372 #else 2373 tsb_base = (uint64_t)tsbinfop->tsb_va; 2374 #endif 2375 tsb_size = tsbinfop->tsb_szc; 2376 } 2377 if (ttesz >= TTE4M) { 2378 vpshift = MMU_PAGESHIFT4M; 2379 vpgsz = MMU_PAGESIZE4M; 2380 } else { 2381 vpshift = MMU_PAGESHIFT; 2382 vpgsz = MMU_PAGESIZE; 2383 } 2384 2385 for (vaddr = start; vaddr < end; vaddr += vpgsz) { 2386 tag = sfmmu_make_tsbtag(vaddr); 2387 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2388 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2389 } 2390 } 2391 2392 /* 2393 * Select the optimum TSB size given the number of mappings 2394 * that need to be cached. 2395 */ 2396 static int 2397 sfmmu_select_tsb_szc(pgcnt_t pgcnt) 2398 { 2399 int szc = 0; 2400 2401 #ifdef DEBUG 2402 if (tsb_grow_stress) { 2403 uint32_t randval = (uint32_t)gettick() >> 4; 2404 return (randval % (tsb_max_growsize + 1)); 2405 } 2406 #endif /* DEBUG */ 2407 2408 while ((szc < tsb_max_growsize) && (pgcnt > SFMMU_RSS_TSBSIZE(szc))) 2409 szc++; 2410 return (szc); 2411 } 2412 2413 /* 2414 * This function will add a translation to the hme_blk and allocate the 2415 * hme_blk if one does not exist. 2416 * If a page structure is specified then it will add the 2417 * corresponding hment to the mapping list. 2418 * It will also update the hmenum field for the tte. 2419 * Furthermore, it attempts to create a large page translation 2420 * for <addr,hat> at page array pps. It assumes addr and first 2421 * pp is correctly aligned. It returns 0 if successful and 1 otherwise. 2422 */ 2423 static int 2424 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr, 2425 page_t **pps, uint_t flags) 2426 { 2427 struct hmehash_bucket *hmebp; 2428 struct hme_blk *hmeblkp; 2429 int ret; 2430 uint_t size; 2431 2432 /* 2433 * Get mapping size. 2434 */ 2435 size = TTE_CSZ(ttep); 2436 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2437 2438 /* 2439 * Acquire the hash bucket. 2440 */ 2441 hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size); 2442 ASSERT(hmebp); 2443 2444 /* 2445 * Find the hment block. 2446 */ 2447 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags); 2448 ASSERT(hmeblkp); 2449 2450 /* 2451 * Add the translation. 2452 */ 2453 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags); 2454 2455 /* 2456 * Release the hash bucket. 2457 */ 2458 sfmmu_tteload_release_hashbucket(hmebp); 2459 2460 return (ret); 2461 } 2462 2463 /* 2464 * Function locks and returns a pointer to the hash bucket for vaddr and size. 2465 */ 2466 static struct hmehash_bucket * 2467 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size) 2468 { 2469 struct hmehash_bucket *hmebp; 2470 int hmeshift; 2471 2472 hmeshift = HME_HASH_SHIFT(size); 2473 2474 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 2475 2476 SFMMU_HASH_LOCK(hmebp); 2477 2478 return (hmebp); 2479 } 2480 2481 /* 2482 * Function returns a pointer to an hmeblk in the hash bucket, hmebp. If the 2483 * hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is 2484 * allocated. 2485 */ 2486 static struct hme_blk * 2487 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp, 2488 caddr_t vaddr, uint_t size, uint_t flags) 2489 { 2490 hmeblk_tag hblktag; 2491 int hmeshift; 2492 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 2493 uint64_t hblkpa, prevpa; 2494 struct kmem_cache *sfmmu_cache; 2495 uint_t forcefree; 2496 2497 hblktag.htag_id = sfmmup; 2498 hmeshift = HME_HASH_SHIFT(size); 2499 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 2500 hblktag.htag_rehash = HME_HASH_REHASH(size); 2501 2502 ttearray_realloc: 2503 2504 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, 2505 pr_hblk, prevpa, &list); 2506 2507 /* 2508 * We block until hblk_reserve_lock is released; it's held by 2509 * the thread, temporarily using hblk_reserve, until hblk_reserve is 2510 * replaced by a hblk from sfmmu8_cache. 2511 */ 2512 if (hmeblkp == (struct hme_blk *)hblk_reserve && 2513 hblk_reserve_thread != curthread) { 2514 SFMMU_HASH_UNLOCK(hmebp); 2515 mutex_enter(&hblk_reserve_lock); 2516 mutex_exit(&hblk_reserve_lock); 2517 SFMMU_STAT(sf_hblk_reserve_hit); 2518 SFMMU_HASH_LOCK(hmebp); 2519 goto ttearray_realloc; 2520 } 2521 2522 if (hmeblkp == NULL) { 2523 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 2524 hblktag, flags); 2525 } else { 2526 /* 2527 * It is possible for 8k and 64k hblks to collide since they 2528 * have the same rehash value. This is because we 2529 * lazily free hblks and 8K/64K blks could be lingering. 2530 * If we find size mismatch we free the block and & try again. 2531 */ 2532 if (get_hblk_ttesz(hmeblkp) != size) { 2533 ASSERT(!hmeblkp->hblk_vcnt); 2534 ASSERT(!hmeblkp->hblk_hmecnt); 2535 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 2536 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 2537 goto ttearray_realloc; 2538 } 2539 if (hmeblkp->hblk_shw_bit) { 2540 /* 2541 * if the hblk was previously used as a shadow hblk then 2542 * we will change it to a normal hblk 2543 */ 2544 if (hmeblkp->hblk_shw_mask) { 2545 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp); 2546 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2547 goto ttearray_realloc; 2548 } else { 2549 hmeblkp->hblk_shw_bit = 0; 2550 } 2551 } 2552 SFMMU_STAT(sf_hblk_hit); 2553 } 2554 2555 /* 2556 * hat_memload() should never call kmem_cache_free(); see block 2557 * comment showing the stacktrace in sfmmu_hblk_alloc(); 2558 * enqueue each hblk in the list to reserve list if it's created 2559 * from sfmmu8_cache *and* sfmmup == KHATID. 2560 */ 2561 forcefree = (sfmmup == KHATID) ? 1 : 0; 2562 while ((pr_hblk = list) != NULL) { 2563 list = pr_hblk->hblk_next; 2564 sfmmu_cache = get_hblk_cache(pr_hblk); 2565 if ((sfmmu_cache == sfmmu8_cache) && 2566 sfmmu_put_free_hblk(pr_hblk, forcefree)) 2567 continue; 2568 2569 ASSERT(sfmmup != KHATID); 2570 kmem_cache_free(sfmmu_cache, pr_hblk); 2571 } 2572 2573 ASSERT(get_hblk_ttesz(hmeblkp) == size); 2574 ASSERT(!hmeblkp->hblk_shw_bit); 2575 2576 return (hmeblkp); 2577 } 2578 2579 /* 2580 * Function adds a tte entry into the hmeblk. It returns 0 if successful and 1 2581 * otherwise. 2582 */ 2583 static int 2584 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep, 2585 caddr_t vaddr, page_t **pps, uint_t flags) 2586 { 2587 page_t *pp = *pps; 2588 int hmenum, size, remap; 2589 tte_t tteold, flush_tte; 2590 #ifdef DEBUG 2591 tte_t orig_old; 2592 #endif /* DEBUG */ 2593 struct sf_hment *sfhme; 2594 kmutex_t *pml, *pmtx; 2595 hatlock_t *hatlockp; 2596 2597 /* 2598 * remove this panic when we decide to let user virtual address 2599 * space be >= USERLIMIT. 2600 */ 2601 if (!TTE_IS_PRIVILEGED(ttep) && vaddr >= (caddr_t)USERLIMIT) 2602 panic("user addr %p in kernel space", vaddr); 2603 #if defined(TTE_IS_GLOBAL) 2604 if (TTE_IS_GLOBAL(ttep)) 2605 panic("sfmmu_tteload: creating global tte"); 2606 #endif 2607 2608 #ifdef DEBUG 2609 if (pf_is_memory(sfmmu_ttetopfn(ttep, vaddr)) && 2610 !TTE_IS_PCACHEABLE(ttep) && !sfmmu_allow_nc_trans) 2611 panic("sfmmu_tteload: non cacheable memory tte"); 2612 #endif /* DEBUG */ 2613 2614 if ((flags & HAT_LOAD_SHARE) || !TTE_IS_REF(ttep) || 2615 !TTE_IS_MOD(ttep)) { 2616 /* 2617 * Don't load TSB for dummy as in ISM. Also don't preload 2618 * the TSB if the TTE isn't writable since we're likely to 2619 * fault on it again -- preloading can be fairly expensive. 2620 */ 2621 flags |= SFMMU_NO_TSBLOAD; 2622 } 2623 2624 size = TTE_CSZ(ttep); 2625 switch (size) { 2626 case TTE8K: 2627 SFMMU_STAT(sf_tteload8k); 2628 break; 2629 case TTE64K: 2630 SFMMU_STAT(sf_tteload64k); 2631 break; 2632 case TTE512K: 2633 SFMMU_STAT(sf_tteload512k); 2634 break; 2635 case TTE4M: 2636 SFMMU_STAT(sf_tteload4m); 2637 break; 2638 case (TTE32M): 2639 SFMMU_STAT(sf_tteload32m); 2640 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2641 break; 2642 case (TTE256M): 2643 SFMMU_STAT(sf_tteload256m); 2644 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2645 break; 2646 } 2647 2648 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2649 2650 HBLKTOHME_IDX(sfhme, hmeblkp, vaddr, hmenum); 2651 2652 /* 2653 * Need to grab mlist lock here so that pageunload 2654 * will not change tte behind us. 2655 */ 2656 if (pp) { 2657 pml = sfmmu_mlist_enter(pp); 2658 } 2659 2660 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2661 /* 2662 * Look for corresponding hment and if valid verify 2663 * pfns are equal. 2664 */ 2665 remap = TTE_IS_VALID(&tteold); 2666 if (remap) { 2667 pfn_t new_pfn, old_pfn; 2668 2669 old_pfn = TTE_TO_PFN(vaddr, &tteold); 2670 new_pfn = TTE_TO_PFN(vaddr, ttep); 2671 2672 if (flags & HAT_LOAD_REMAP) { 2673 /* make sure we are remapping same type of pages */ 2674 if (pf_is_memory(old_pfn) != pf_is_memory(new_pfn)) { 2675 panic("sfmmu_tteload - tte remap io<->memory"); 2676 } 2677 if (old_pfn != new_pfn && 2678 (pp != NULL || sfhme->hme_page != NULL)) { 2679 panic("sfmmu_tteload - tte remap pp != NULL"); 2680 } 2681 } else if (old_pfn != new_pfn) { 2682 panic("sfmmu_tteload - tte remap, hmeblkp 0x%p", 2683 (void *)hmeblkp); 2684 } 2685 ASSERT(TTE_CSZ(&tteold) == TTE_CSZ(ttep)); 2686 } 2687 2688 if (pp) { 2689 if (size == TTE8K) { 2690 #ifdef VAC 2691 /* 2692 * Handle VAC consistency 2693 */ 2694 if (!remap && (cache & CACHE_VAC) && !PP_ISNC(pp)) { 2695 sfmmu_vac_conflict(sfmmup, vaddr, pp); 2696 } 2697 #endif 2698 2699 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2700 pmtx = sfmmu_page_enter(pp); 2701 PP_CLRRO(pp); 2702 sfmmu_page_exit(pmtx); 2703 } else if (!PP_ISMAPPED(pp) && 2704 (!TTE_IS_WRITABLE(ttep)) && !(PP_ISMOD(pp))) { 2705 pmtx = sfmmu_page_enter(pp); 2706 if (!(PP_ISMOD(pp))) { 2707 PP_SETRO(pp); 2708 } 2709 sfmmu_page_exit(pmtx); 2710 } 2711 2712 } else if (sfmmu_pagearray_setup(vaddr, pps, ttep, remap)) { 2713 /* 2714 * sfmmu_pagearray_setup failed so return 2715 */ 2716 sfmmu_mlist_exit(pml); 2717 return (1); 2718 } 2719 } 2720 2721 /* 2722 * Make sure hment is not on a mapping list. 2723 */ 2724 ASSERT(remap || (sfhme->hme_page == NULL)); 2725 2726 /* if it is not a remap then hme->next better be NULL */ 2727 ASSERT((!remap) ? sfhme->hme_next == NULL : 1); 2728 2729 if (flags & HAT_LOAD_LOCK) { 2730 if (((int)hmeblkp->hblk_lckcnt + 1) >= MAX_HBLK_LCKCNT) { 2731 panic("too high lckcnt-hmeblk %p", 2732 (void *)hmeblkp); 2733 } 2734 atomic_add_16(&hmeblkp->hblk_lckcnt, 1); 2735 2736 HBLK_STACK_TRACE(hmeblkp, HBLK_LOCK); 2737 } 2738 2739 #ifdef VAC 2740 if (pp && PP_ISNC(pp)) { 2741 /* 2742 * If the physical page is marked to be uncacheable, like 2743 * by a vac conflict, make sure the new mapping is also 2744 * uncacheable. 2745 */ 2746 TTE_CLR_VCACHEABLE(ttep); 2747 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 2748 } 2749 #endif 2750 ttep->tte_hmenum = hmenum; 2751 2752 #ifdef DEBUG 2753 orig_old = tteold; 2754 #endif /* DEBUG */ 2755 2756 while (sfmmu_modifytte_try(&tteold, ttep, &sfhme->hme_tte) < 0) { 2757 if ((sfmmup == KHATID) && 2758 (flags & (HAT_LOAD_LOCK | HAT_LOAD_REMAP))) { 2759 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2760 } 2761 #ifdef DEBUG 2762 chk_tte(&orig_old, &tteold, ttep, hmeblkp); 2763 #endif /* DEBUG */ 2764 } 2765 2766 if (!TTE_IS_VALID(&tteold)) { 2767 2768 atomic_add_16(&hmeblkp->hblk_vcnt, 1); 2769 atomic_add_long(&sfmmup->sfmmu_ttecnt[size], 1); 2770 2771 /* 2772 * HAT_RELOAD_SHARE has been deprecated with lpg DISM. 2773 */ 2774 2775 if (size > TTE8K && (flags & HAT_LOAD_SHARE) == 0 && 2776 sfmmup != ksfmmup) { 2777 /* 2778 * If this is the first large mapping for the process 2779 * we must force any CPUs running this process to TL=0 2780 * where they will reload the HAT flags from the 2781 * tsbmiss area. This is necessary to make the large 2782 * mappings we are about to load visible to those CPUs; 2783 * otherwise they'll loop forever calling pagefault() 2784 * since we don't search large hash chains by default. 2785 */ 2786 hatlockp = sfmmu_hat_enter(sfmmup); 2787 if (size == TTE512K && 2788 !SFMMU_FLAGS_ISSET(sfmmup, HAT_512K_FLAG)) { 2789 SFMMU_FLAGS_SET(sfmmup, HAT_512K_FLAG); 2790 sfmmu_sync_mmustate(sfmmup); 2791 } else if (size == TTE4M && 2792 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 2793 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 2794 sfmmu_sync_mmustate(sfmmup); 2795 } else if (size == TTE64K && 2796 !SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) { 2797 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 2798 /* no sync mmustate; 64K shares 8K hashes */ 2799 } else if (mmu_page_sizes == max_mmu_page_sizes) { 2800 if (size == TTE32M && 2801 !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 2802 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 2803 sfmmu_sync_mmustate(sfmmup); 2804 } else if (size == TTE256M && 2805 !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 2806 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 2807 sfmmu_sync_mmustate(sfmmup); 2808 } 2809 } 2810 if (size >= TTE4M && (flags & HAT_LOAD_TEXT) && 2811 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) { 2812 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG); 2813 } 2814 sfmmu_hat_exit(hatlockp); 2815 } 2816 } 2817 ASSERT(TTE_IS_VALID(&sfhme->hme_tte)); 2818 2819 flush_tte.tte_intlo = (tteold.tte_intlo ^ ttep->tte_intlo) & 2820 hw_tte.tte_intlo; 2821 flush_tte.tte_inthi = (tteold.tte_inthi ^ ttep->tte_inthi) & 2822 hw_tte.tte_inthi; 2823 2824 if (remap && (flush_tte.tte_inthi || flush_tte.tte_intlo)) { 2825 /* 2826 * If remap and new tte differs from old tte we need 2827 * to sync the mod bit and flush TLB/TSB. We don't 2828 * need to sync ref bit because we currently always set 2829 * ref bit in tteload. 2830 */ 2831 ASSERT(TTE_IS_REF(ttep)); 2832 if (TTE_IS_MOD(&tteold)) { 2833 sfmmu_ttesync(sfmmup, vaddr, &tteold, pp); 2834 } 2835 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0); 2836 xt_sync(sfmmup->sfmmu_cpusran); 2837 } 2838 2839 if ((flags & SFMMU_NO_TSBLOAD) == 0) { 2840 /* 2841 * We only preload 8K and 4M mappings into the TSB, since 2842 * 64K and 512K mappings are replicated and hence don't 2843 * have a single, unique TSB entry. Ditto for 32M/256M. 2844 */ 2845 if (size == TTE8K || size == TTE4M) { 2846 hatlockp = sfmmu_hat_enter(sfmmup); 2847 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte, size); 2848 sfmmu_hat_exit(hatlockp); 2849 } 2850 } 2851 if (pp) { 2852 if (!remap) { 2853 HME_ADD(sfhme, pp); 2854 atomic_add_16(&hmeblkp->hblk_hmecnt, 1); 2855 ASSERT(hmeblkp->hblk_hmecnt > 0); 2856 2857 /* 2858 * Cannot ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 2859 * see pageunload() for comment. 2860 */ 2861 } 2862 sfmmu_mlist_exit(pml); 2863 } 2864 2865 return (0); 2866 } 2867 /* 2868 * Function unlocks hash bucket. 2869 */ 2870 static void 2871 sfmmu_tteload_release_hashbucket(struct hmehash_bucket *hmebp) 2872 { 2873 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2874 SFMMU_HASH_UNLOCK(hmebp); 2875 } 2876 2877 /* 2878 * function which checks and sets up page array for a large 2879 * translation. Will set p_vcolor, p_index, p_ro fields. 2880 * Assumes addr and pfnum of first page are properly aligned. 2881 * Will check for physical contiguity. If check fails it return 2882 * non null. 2883 */ 2884 static int 2885 sfmmu_pagearray_setup(caddr_t addr, page_t **pps, tte_t *ttep, int remap) 2886 { 2887 int i, index, ttesz; 2888 pfn_t pfnum; 2889 pgcnt_t npgs; 2890 page_t *pp, *pp1; 2891 kmutex_t *pmtx; 2892 #ifdef VAC 2893 int osz; 2894 int cflags = 0; 2895 int vac_err = 0; 2896 #endif 2897 int newidx = 0; 2898 2899 ttesz = TTE_CSZ(ttep); 2900 2901 ASSERT(ttesz > TTE8K); 2902 2903 npgs = TTEPAGES(ttesz); 2904 index = PAGESZ_TO_INDEX(ttesz); 2905 2906 pfnum = (*pps)->p_pagenum; 2907 ASSERT(IS_P2ALIGNED(pfnum, npgs)); 2908 2909 /* 2910 * Save the first pp so we can do HAT_TMPNC at the end. 2911 */ 2912 pp1 = *pps; 2913 #ifdef VAC 2914 osz = fnd_mapping_sz(pp1); 2915 #endif 2916 2917 for (i = 0; i < npgs; i++, pps++) { 2918 pp = *pps; 2919 ASSERT(PAGE_LOCKED(pp)); 2920 ASSERT(pp->p_szc >= ttesz); 2921 ASSERT(pp->p_szc == pp1->p_szc); 2922 ASSERT(sfmmu_mlist_held(pp)); 2923 2924 /* 2925 * XXX is it possible to maintain P_RO on the root only? 2926 */ 2927 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2928 pmtx = sfmmu_page_enter(pp); 2929 PP_CLRRO(pp); 2930 sfmmu_page_exit(pmtx); 2931 } else if (!PP_ISMAPPED(pp) && !TTE_IS_WRITABLE(ttep) && 2932 !PP_ISMOD(pp)) { 2933 pmtx = sfmmu_page_enter(pp); 2934 if (!(PP_ISMOD(pp))) { 2935 PP_SETRO(pp); 2936 } 2937 sfmmu_page_exit(pmtx); 2938 } 2939 2940 /* 2941 * If this is a remap we skip vac & contiguity checks. 2942 */ 2943 if (remap) 2944 continue; 2945 2946 /* 2947 * set p_vcolor and detect any vac conflicts. 2948 */ 2949 #ifdef VAC 2950 if (vac_err == 0) { 2951 vac_err = sfmmu_vacconflict_array(addr, pp, &cflags); 2952 2953 } 2954 #endif 2955 2956 /* 2957 * Save current index in case we need to undo it. 2958 * Note: "PAGESZ_TO_INDEX(sz) (1 << (sz))" 2959 * "SFMMU_INDEX_SHIFT 6" 2960 * "SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1)" 2961 * "PP_MAPINDEX(p_index) (p_index & SFMMU_INDEX_MASK)" 2962 * 2963 * So: index = PAGESZ_TO_INDEX(ttesz); 2964 * if ttesz == 1 then index = 0x2 2965 * 2 then index = 0x4 2966 * 3 then index = 0x8 2967 * 4 then index = 0x10 2968 * 5 then index = 0x20 2969 * The code below checks if it's a new pagesize (ie, newidx) 2970 * in case we need to take it back out of p_index, 2971 * and then or's the new index into the existing index. 2972 */ 2973 if ((PP_MAPINDEX(pp) & index) == 0) 2974 newidx = 1; 2975 pp->p_index = (PP_MAPINDEX(pp) | index); 2976 2977 /* 2978 * contiguity check 2979 */ 2980 if (pp->p_pagenum != pfnum) { 2981 /* 2982 * If we fail the contiguity test then 2983 * the only thing we need to fix is the p_index field. 2984 * We might get a few extra flushes but since this 2985 * path is rare that is ok. The p_ro field will 2986 * get automatically fixed on the next tteload to 2987 * the page. NO TNC bit is set yet. 2988 */ 2989 while (i >= 0) { 2990 pp = *pps; 2991 if (newidx) 2992 pp->p_index = (PP_MAPINDEX(pp) & 2993 ~index); 2994 pps--; 2995 i--; 2996 } 2997 return (1); 2998 } 2999 pfnum++; 3000 addr += MMU_PAGESIZE; 3001 } 3002 3003 #ifdef VAC 3004 if (vac_err) { 3005 if (ttesz > osz) { 3006 /* 3007 * There are some smaller mappings that causes vac 3008 * conflicts. Convert all existing small mappings to 3009 * TNC. 3010 */ 3011 SFMMU_STAT_ADD(sf_uncache_conflict, npgs); 3012 sfmmu_page_cache_array(pp1, HAT_TMPNC, CACHE_FLUSH, 3013 npgs); 3014 } else { 3015 /* EMPTY */ 3016 /* 3017 * If there exists an big page mapping, 3018 * that means the whole existing big page 3019 * has TNC setting already. No need to covert to 3020 * TNC again. 3021 */ 3022 ASSERT(PP_ISTNC(pp1)); 3023 } 3024 } 3025 #endif /* VAC */ 3026 3027 return (0); 3028 } 3029 3030 #ifdef VAC 3031 /* 3032 * Routine that detects vac consistency for a large page. It also 3033 * sets virtual color for all pp's for this big mapping. 3034 */ 3035 static int 3036 sfmmu_vacconflict_array(caddr_t addr, page_t *pp, int *cflags) 3037 { 3038 int vcolor, ocolor; 3039 3040 ASSERT(sfmmu_mlist_held(pp)); 3041 3042 if (PP_ISNC(pp)) { 3043 return (HAT_TMPNC); 3044 } 3045 3046 vcolor = addr_to_vcolor(addr); 3047 if (PP_NEWPAGE(pp)) { 3048 PP_SET_VCOLOR(pp, vcolor); 3049 return (0); 3050 } 3051 3052 ocolor = PP_GET_VCOLOR(pp); 3053 if (ocolor == vcolor) { 3054 return (0); 3055 } 3056 3057 if (!PP_ISMAPPED(pp)) { 3058 /* 3059 * Previous user of page had a differnet color 3060 * but since there are no current users 3061 * we just flush the cache and change the color. 3062 * As an optimization for large pages we flush the 3063 * entire cache of that color and set a flag. 3064 */ 3065 SFMMU_STAT(sf_pgcolor_conflict); 3066 if (!CacheColor_IsFlushed(*cflags, ocolor)) { 3067 CacheColor_SetFlushed(*cflags, ocolor); 3068 sfmmu_cache_flushcolor(ocolor, pp->p_pagenum); 3069 } 3070 PP_SET_VCOLOR(pp, vcolor); 3071 return (0); 3072 } 3073 3074 /* 3075 * We got a real conflict with a current mapping. 3076 * set flags to start unencaching all mappings 3077 * and return failure so we restart looping 3078 * the pp array from the beginning. 3079 */ 3080 return (HAT_TMPNC); 3081 } 3082 #endif /* VAC */ 3083 3084 /* 3085 * creates a large page shadow hmeblk for a tte. 3086 * The purpose of this routine is to allow us to do quick unloads because 3087 * the vm layer can easily pass a very large but sparsely populated range. 3088 */ 3089 static struct hme_blk * 3090 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags) 3091 { 3092 struct hmehash_bucket *hmebp; 3093 hmeblk_tag hblktag; 3094 int hmeshift, size, vshift; 3095 uint_t shw_mask, newshw_mask; 3096 struct hme_blk *hmeblkp; 3097 3098 ASSERT(sfmmup != KHATID); 3099 if (mmu_page_sizes == max_mmu_page_sizes) { 3100 ASSERT(ttesz < TTE256M); 3101 } else { 3102 ASSERT(ttesz < TTE4M); 3103 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 3104 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 3105 } 3106 3107 if (ttesz == TTE8K) { 3108 size = TTE512K; 3109 } else { 3110 size = ++ttesz; 3111 } 3112 3113 hblktag.htag_id = sfmmup; 3114 hmeshift = HME_HASH_SHIFT(size); 3115 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 3116 hblktag.htag_rehash = HME_HASH_REHASH(size); 3117 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 3118 3119 SFMMU_HASH_LOCK(hmebp); 3120 3121 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3122 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 3123 if (hmeblkp == NULL) { 3124 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 3125 hblktag, flags); 3126 } 3127 ASSERT(hmeblkp); 3128 if (!hmeblkp->hblk_shw_mask) { 3129 /* 3130 * if this is a unused hblk it was just allocated or could 3131 * potentially be a previous large page hblk so we need to 3132 * set the shadow bit. 3133 */ 3134 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt); 3135 hmeblkp->hblk_shw_bit = 1; 3136 } else if (hmeblkp->hblk_shw_bit == 0) { 3137 panic("sfmmu_shadow_hcreate: shw bit not set in hmeblkp 0x%p", 3138 (void *)hmeblkp); 3139 } 3140 3141 vshift = vaddr_to_vshift(hblktag, vaddr, size); 3142 ASSERT(vshift < 8); 3143 /* 3144 * Atomically set shw mask bit 3145 */ 3146 do { 3147 shw_mask = hmeblkp->hblk_shw_mask; 3148 newshw_mask = shw_mask | (1 << vshift); 3149 newshw_mask = cas32(&hmeblkp->hblk_shw_mask, shw_mask, 3150 newshw_mask); 3151 } while (newshw_mask != shw_mask); 3152 3153 SFMMU_HASH_UNLOCK(hmebp); 3154 3155 return (hmeblkp); 3156 } 3157 3158 /* 3159 * This routine cleanup a previous shadow hmeblk and changes it to 3160 * a regular hblk. This happens rarely but it is possible 3161 * when a process wants to use large pages and there are hblks still 3162 * lying around from the previous as that used these hmeblks. 3163 * The alternative was to cleanup the shadow hblks at unload time 3164 * but since so few user processes actually use large pages, it is 3165 * better to be lazy and cleanup at this time. 3166 */ 3167 static void 3168 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 3169 struct hmehash_bucket *hmebp) 3170 { 3171 caddr_t addr, endaddr; 3172 int hashno, size; 3173 3174 ASSERT(hmeblkp->hblk_shw_bit); 3175 3176 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 3177 3178 if (!hmeblkp->hblk_shw_mask) { 3179 hmeblkp->hblk_shw_bit = 0; 3180 return; 3181 } 3182 addr = (caddr_t)get_hblk_base(hmeblkp); 3183 endaddr = get_hblk_endaddr(hmeblkp); 3184 size = get_hblk_ttesz(hmeblkp); 3185 hashno = size - 1; 3186 ASSERT(hashno > 0); 3187 SFMMU_HASH_UNLOCK(hmebp); 3188 3189 sfmmu_free_hblks(sfmmup, addr, endaddr, hashno); 3190 3191 SFMMU_HASH_LOCK(hmebp); 3192 } 3193 3194 static void 3195 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr, 3196 int hashno) 3197 { 3198 int hmeshift, shadow = 0; 3199 hmeblk_tag hblktag; 3200 struct hmehash_bucket *hmebp; 3201 struct hme_blk *hmeblkp; 3202 struct hme_blk *nx_hblk, *pr_hblk, *list = NULL; 3203 uint64_t hblkpa, prevpa, nx_pa; 3204 3205 ASSERT(hashno > 0); 3206 hblktag.htag_id = sfmmup; 3207 hblktag.htag_rehash = hashno; 3208 3209 hmeshift = HME_HASH_SHIFT(hashno); 3210 3211 while (addr < endaddr) { 3212 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3213 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3214 SFMMU_HASH_LOCK(hmebp); 3215 /* inline HME_HASH_SEARCH */ 3216 hmeblkp = hmebp->hmeblkp; 3217 hblkpa = hmebp->hmeh_nextpa; 3218 prevpa = 0; 3219 pr_hblk = NULL; 3220 while (hmeblkp) { 3221 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 3222 if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) { 3223 /* found hme_blk */ 3224 if (hmeblkp->hblk_shw_bit) { 3225 if (hmeblkp->hblk_shw_mask) { 3226 shadow = 1; 3227 sfmmu_shadow_hcleanup(sfmmup, 3228 hmeblkp, hmebp); 3229 break; 3230 } else { 3231 hmeblkp->hblk_shw_bit = 0; 3232 } 3233 } 3234 3235 /* 3236 * Hblk_hmecnt and hblk_vcnt could be non zero 3237 * since hblk_unload() does not gurantee that. 3238 * 3239 * XXX - this could cause tteload() to spin 3240 * where sfmmu_shadow_hcleanup() is called. 3241 */ 3242 } 3243 3244 nx_hblk = hmeblkp->hblk_next; 3245 nx_pa = hmeblkp->hblk_nextpa; 3246 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 3247 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 3248 pr_hblk); 3249 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 3250 } else { 3251 pr_hblk = hmeblkp; 3252 prevpa = hblkpa; 3253 } 3254 hmeblkp = nx_hblk; 3255 hblkpa = nx_pa; 3256 } 3257 3258 SFMMU_HASH_UNLOCK(hmebp); 3259 3260 if (shadow) { 3261 /* 3262 * We found another shadow hblk so cleaned its 3263 * children. We need to go back and cleanup 3264 * the original hblk so we don't change the 3265 * addr. 3266 */ 3267 shadow = 0; 3268 } else { 3269 addr = (caddr_t)roundup((uintptr_t)addr + 1, 3270 (1 << hmeshift)); 3271 } 3272 } 3273 sfmmu_hblks_list_purge(&list); 3274 } 3275 3276 /* 3277 * Release one hardware address translation lock on the given address range. 3278 */ 3279 void 3280 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len) 3281 { 3282 struct hmehash_bucket *hmebp; 3283 hmeblk_tag hblktag; 3284 int hmeshift, hashno = 1; 3285 struct hme_blk *hmeblkp, *list = NULL; 3286 caddr_t endaddr; 3287 3288 ASSERT(sfmmup != NULL); 3289 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3290 3291 ASSERT((sfmmup == ksfmmup) || 3292 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 3293 ASSERT((len & MMU_PAGEOFFSET) == 0); 3294 endaddr = addr + len; 3295 hblktag.htag_id = sfmmup; 3296 3297 /* 3298 * Spitfire supports 4 page sizes. 3299 * Most pages are expected to be of the smallest page size (8K) and 3300 * these will not need to be rehashed. 64K pages also don't need to be 3301 * rehashed because an hmeblk spans 64K of address space. 512K pages 3302 * might need 1 rehash and and 4M pages might need 2 rehashes. 3303 */ 3304 while (addr < endaddr) { 3305 hmeshift = HME_HASH_SHIFT(hashno); 3306 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3307 hblktag.htag_rehash = hashno; 3308 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3309 3310 SFMMU_HASH_LOCK(hmebp); 3311 3312 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 3313 if (hmeblkp != NULL) { 3314 /* 3315 * If we encounter a shadow hmeblk then 3316 * we know there are no valid hmeblks mapping 3317 * this address at this size or larger. 3318 * Just increment address by the smallest 3319 * page size. 3320 */ 3321 if (hmeblkp->hblk_shw_bit) { 3322 addr += MMU_PAGESIZE; 3323 } else { 3324 addr = sfmmu_hblk_unlock(hmeblkp, addr, 3325 endaddr); 3326 } 3327 SFMMU_HASH_UNLOCK(hmebp); 3328 hashno = 1; 3329 continue; 3330 } 3331 SFMMU_HASH_UNLOCK(hmebp); 3332 3333 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 3334 /* 3335 * We have traversed the whole list and rehashed 3336 * if necessary without finding the address to unlock 3337 * which should never happen. 3338 */ 3339 panic("sfmmu_unlock: addr not found. " 3340 "addr %p hat %p", (void *)addr, (void *)sfmmup); 3341 } else { 3342 hashno++; 3343 } 3344 } 3345 3346 sfmmu_hblks_list_purge(&list); 3347 } 3348 3349 /* 3350 * Function to unlock a range of addresses in an hmeblk. It returns the 3351 * next address that needs to be unlocked. 3352 * Should be called with the hash lock held. 3353 */ 3354 static caddr_t 3355 sfmmu_hblk_unlock(struct hme_blk *hmeblkp, caddr_t addr, caddr_t endaddr) 3356 { 3357 struct sf_hment *sfhme; 3358 tte_t tteold, ttemod; 3359 int ttesz, ret; 3360 3361 ASSERT(in_hblk_range(hmeblkp, addr)); 3362 ASSERT(hmeblkp->hblk_shw_bit == 0); 3363 3364 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 3365 ttesz = get_hblk_ttesz(hmeblkp); 3366 3367 HBLKTOHME(sfhme, hmeblkp, addr); 3368 while (addr < endaddr) { 3369 readtte: 3370 sfmmu_copytte(&sfhme->hme_tte, &tteold); 3371 if (TTE_IS_VALID(&tteold)) { 3372 3373 ttemod = tteold; 3374 3375 ret = sfmmu_modifytte_try(&tteold, &ttemod, 3376 &sfhme->hme_tte); 3377 3378 if (ret < 0) 3379 goto readtte; 3380 3381 if (hmeblkp->hblk_lckcnt == 0) 3382 panic("zero hblk lckcnt"); 3383 3384 if (((uintptr_t)addr + TTEBYTES(ttesz)) > 3385 (uintptr_t)endaddr) 3386 panic("can't unlock large tte"); 3387 3388 ASSERT(hmeblkp->hblk_lckcnt > 0); 3389 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 3390 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 3391 } else { 3392 panic("sfmmu_hblk_unlock: invalid tte"); 3393 } 3394 addr += TTEBYTES(ttesz); 3395 sfhme++; 3396 } 3397 return (addr); 3398 } 3399 3400 /* 3401 * Physical Address Mapping Framework 3402 * 3403 * General rules: 3404 * 3405 * (1) Applies only to seg_kmem memory pages. To make things easier, 3406 * seg_kpm addresses are also accepted by the routines, but nothing 3407 * is done with them since by definition their PA mappings are static. 3408 * (2) hat_add_callback() may only be called while holding the page lock 3409 * SE_SHARED or SE_EXCL of the underlying page (e.g., as_pagelock()), 3410 * or passing HAC_PAGELOCK flag. 3411 * (3) prehandler() and posthandler() may not call hat_add_callback() or 3412 * hat_delete_callback(), nor should they allocate memory. Post quiesce 3413 * callbacks may not sleep or acquire adaptive mutex locks. 3414 * (4) Either prehandler() or posthandler() (but not both) may be specified 3415 * as being NULL. Specifying an errhandler() is optional. 3416 * 3417 * Details of using the framework: 3418 * 3419 * registering a callback (hat_register_callback()) 3420 * 3421 * Pass prehandler, posthandler, errhandler addresses 3422 * as described below. If capture_cpus argument is nonzero, 3423 * suspend callback to the prehandler will occur with CPUs 3424 * captured and executing xc_loop() and CPUs will remain 3425 * captured until after the posthandler suspend callback 3426 * occurs. 3427 * 3428 * adding a callback (hat_add_callback()) 3429 * 3430 * as_pagelock(); 3431 * hat_add_callback(); 3432 * save returned pfn in private data structures or program registers; 3433 * as_pageunlock(); 3434 * 3435 * prehandler() 3436 * 3437 * Stop all accesses by physical address to this memory page. 3438 * Called twice: the first, PRESUSPEND, is a context safe to acquire 3439 * adaptive locks. The second, SUSPEND, is called at high PIL with 3440 * CPUs captured so adaptive locks may NOT be acquired (and all spin 3441 * locks must be XCALL_PIL or higher locks). 3442 * 3443 * May return the following errors: 3444 * EIO: A fatal error has occurred. This will result in panic. 3445 * EAGAIN: The page cannot be suspended. This will fail the 3446 * relocation. 3447 * 0: Success. 3448 * 3449 * posthandler() 3450 * 3451 * Save new pfn in private data structures or program registers; 3452 * not allowed to fail (non-zero return values will result in panic). 3453 * 3454 * errhandler() 3455 * 3456 * called when an error occurs related to the callback. Currently 3457 * the only such error is HAT_CB_ERR_LEAKED which indicates that 3458 * a page is being freed, but there are still outstanding callback(s) 3459 * registered on the page. 3460 * 3461 * removing a callback (hat_delete_callback(); e.g., prior to freeing memory) 3462 * 3463 * stop using physical address 3464 * hat_delete_callback(); 3465 * 3466 */ 3467 3468 /* 3469 * Register a callback class. Each subsystem should do this once and 3470 * cache the id_t returned for use in setting up and tearing down callbacks. 3471 * 3472 * There is no facility for removing callback IDs once they are created; 3473 * the "key" should be unique for each module, so in case a module is unloaded 3474 * and subsequently re-loaded, we can recycle the module's previous entry. 3475 */ 3476 id_t 3477 hat_register_callback(int key, 3478 int (*prehandler)(caddr_t, uint_t, uint_t, void *), 3479 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t), 3480 int (*errhandler)(caddr_t, uint_t, uint_t, void *), 3481 int capture_cpus) 3482 { 3483 id_t id; 3484 3485 /* 3486 * Search the table for a pre-existing callback associated with 3487 * the identifier "key". If one exists, we re-use that entry in 3488 * the table for this instance, otherwise we assign the next 3489 * available table slot. 3490 */ 3491 for (id = 0; id < sfmmu_max_cb_id; id++) { 3492 if (sfmmu_cb_table[id].key == key) 3493 break; 3494 } 3495 3496 if (id == sfmmu_max_cb_id) { 3497 id = sfmmu_cb_nextid++; 3498 if (id >= sfmmu_max_cb_id) 3499 panic("hat_register_callback: out of callback IDs"); 3500 } 3501 3502 ASSERT(prehandler != NULL || posthandler != NULL); 3503 3504 sfmmu_cb_table[id].key = key; 3505 sfmmu_cb_table[id].prehandler = prehandler; 3506 sfmmu_cb_table[id].posthandler = posthandler; 3507 sfmmu_cb_table[id].errhandler = errhandler; 3508 sfmmu_cb_table[id].capture_cpus = capture_cpus; 3509 3510 return (id); 3511 } 3512 3513 #define HAC_COOKIE_NONE (void *)-1 3514 3515 /* 3516 * Add relocation callbacks to the specified addr/len which will be called 3517 * when relocating the associated page. See the description of pre and 3518 * posthandler above for more details. 3519 * 3520 * If HAC_PAGELOCK is included in flags, the underlying memory page is 3521 * locked internally so the caller must be able to deal with the callback 3522 * running even before this function has returned. If HAC_PAGELOCK is not 3523 * set, it is assumed that the underlying memory pages are locked. 3524 * 3525 * Since the caller must track the individual page boundaries anyway, 3526 * we only allow a callback to be added to a single page (large 3527 * or small). Thus [addr, addr + len) MUST be contained within a single 3528 * page. 3529 * 3530 * Registering multiple callbacks on the same [addr, addr+len) is supported, 3531 * _provided_that_ a unique parameter is specified for each callback. 3532 * If multiple callbacks are registered on the same range the callback will 3533 * be invoked with each unique parameter. Registering the same callback with 3534 * the same argument more than once will result in corrupted kernel state. 3535 * 3536 * Returns the pfn of the underlying kernel page in *rpfn 3537 * on success, or PFN_INVALID on failure. 3538 * 3539 * cookiep (if passed) provides storage space for an opaque cookie 3540 * to return later to hat_delete_callback(). This cookie makes the callback 3541 * deletion significantly quicker by avoiding a potentially lengthy hash 3542 * search. 3543 * 3544 * Returns values: 3545 * 0: success 3546 * ENOMEM: memory allocation failure (e.g. flags was passed as HAC_NOSLEEP) 3547 * EINVAL: callback ID is not valid 3548 * ENXIO: ["vaddr", "vaddr" + len) is not mapped in the kernel's address 3549 * space 3550 * ERANGE: ["vaddr", "vaddr" + len) crosses a page boundary 3551 */ 3552 int 3553 hat_add_callback(id_t callback_id, caddr_t vaddr, uint_t len, uint_t flags, 3554 void *pvt, pfn_t *rpfn, void **cookiep) 3555 { 3556 struct hmehash_bucket *hmebp; 3557 hmeblk_tag hblktag; 3558 struct hme_blk *hmeblkp; 3559 int hmeshift, hashno; 3560 caddr_t saddr, eaddr, baseaddr; 3561 struct pa_hment *pahmep; 3562 struct sf_hment *sfhmep, *osfhmep; 3563 kmutex_t *pml; 3564 tte_t tte; 3565 page_t *pp; 3566 vnode_t *vp; 3567 u_offset_t off; 3568 pfn_t pfn; 3569 int kmflags = (flags & HAC_SLEEP)? KM_SLEEP : KM_NOSLEEP; 3570 int locked = 0; 3571 3572 /* 3573 * For KPM mappings, just return the physical address since we 3574 * don't need to register any callbacks. 3575 */ 3576 if (IS_KPM_ADDR(vaddr)) { 3577 uint64_t paddr; 3578 SFMMU_KPM_VTOP(vaddr, paddr); 3579 *rpfn = btop(paddr); 3580 if (cookiep != NULL) 3581 *cookiep = HAC_COOKIE_NONE; 3582 return (0); 3583 } 3584 3585 if (callback_id < (id_t)0 || callback_id >= sfmmu_cb_nextid) { 3586 *rpfn = PFN_INVALID; 3587 return (EINVAL); 3588 } 3589 3590 if ((pahmep = kmem_cache_alloc(pa_hment_cache, kmflags)) == NULL) { 3591 *rpfn = PFN_INVALID; 3592 return (ENOMEM); 3593 } 3594 3595 sfhmep = &pahmep->sfment; 3596 3597 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3598 eaddr = saddr + len; 3599 3600 rehash: 3601 /* Find the mapping(s) for this page */ 3602 for (hashno = TTE64K, hmeblkp = NULL; 3603 hmeblkp == NULL && hashno <= mmu_hashcnt; 3604 hashno++) { 3605 hmeshift = HME_HASH_SHIFT(hashno); 3606 hblktag.htag_id = ksfmmup; 3607 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3608 hblktag.htag_rehash = hashno; 3609 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3610 3611 SFMMU_HASH_LOCK(hmebp); 3612 3613 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3614 3615 if (hmeblkp == NULL) 3616 SFMMU_HASH_UNLOCK(hmebp); 3617 } 3618 3619 if (hmeblkp == NULL) { 3620 kmem_cache_free(pa_hment_cache, pahmep); 3621 *rpfn = PFN_INVALID; 3622 return (ENXIO); 3623 } 3624 3625 HBLKTOHME(osfhmep, hmeblkp, saddr); 3626 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3627 3628 if (!TTE_IS_VALID(&tte)) { 3629 SFMMU_HASH_UNLOCK(hmebp); 3630 kmem_cache_free(pa_hment_cache, pahmep); 3631 *rpfn = PFN_INVALID; 3632 return (ENXIO); 3633 } 3634 3635 /* 3636 * Make sure the boundaries for the callback fall within this 3637 * single mapping. 3638 */ 3639 baseaddr = (caddr_t)get_hblk_base(hmeblkp); 3640 ASSERT(saddr >= baseaddr); 3641 if (eaddr > saddr + TTEBYTES(TTE_CSZ(&tte))) { 3642 SFMMU_HASH_UNLOCK(hmebp); 3643 kmem_cache_free(pa_hment_cache, pahmep); 3644 *rpfn = PFN_INVALID; 3645 return (ERANGE); 3646 } 3647 3648 pfn = sfmmu_ttetopfn(&tte, vaddr); 3649 3650 /* 3651 * The pfn may not have a page_t underneath in which case we 3652 * just return it. This can happen if we are doing I/O to a 3653 * static portion of the kernel's address space, for instance. 3654 */ 3655 pp = osfhmep->hme_page; 3656 if (pp == NULL) { 3657 SFMMU_HASH_UNLOCK(hmebp); 3658 kmem_cache_free(pa_hment_cache, pahmep); 3659 *rpfn = pfn; 3660 if (cookiep) 3661 *cookiep = HAC_COOKIE_NONE; 3662 return (0); 3663 } 3664 ASSERT(pp == PP_PAGEROOT(pp)); 3665 3666 vp = pp->p_vnode; 3667 off = pp->p_offset; 3668 3669 pml = sfmmu_mlist_enter(pp); 3670 3671 if (flags & HAC_PAGELOCK) { 3672 if (!page_trylock(pp, SE_SHARED)) { 3673 /* 3674 * Somebody is holding SE_EXCL lock. Might 3675 * even be hat_page_relocate(). Drop all 3676 * our locks, lookup the page in &kvp, and 3677 * retry. If it doesn't exist in &kvp and &zvp, 3678 * then we must be dealing with a kernel mapped 3679 * page which doesn't actually belong to 3680 * segkmem so we punt. 3681 */ 3682 sfmmu_mlist_exit(pml); 3683 SFMMU_HASH_UNLOCK(hmebp); 3684 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3685 3686 /* check zvp before giving up */ 3687 if (pp == NULL) 3688 pp = page_lookup(&zvp, (u_offset_t)saddr, 3689 SE_SHARED); 3690 3691 /* Okay, we didn't find it, give up */ 3692 if (pp == NULL) { 3693 kmem_cache_free(pa_hment_cache, pahmep); 3694 *rpfn = pfn; 3695 if (cookiep) 3696 *cookiep = HAC_COOKIE_NONE; 3697 return (0); 3698 } 3699 page_unlock(pp); 3700 goto rehash; 3701 } 3702 locked = 1; 3703 } 3704 3705 if (!PAGE_LOCKED(pp) && !panicstr) 3706 panic("hat_add_callback: page 0x%p not locked", pp); 3707 3708 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3709 pp->p_offset != off) { 3710 /* 3711 * The page moved before we got our hands on it. Drop 3712 * all the locks and try again. 3713 */ 3714 ASSERT((flags & HAC_PAGELOCK) != 0); 3715 sfmmu_mlist_exit(pml); 3716 SFMMU_HASH_UNLOCK(hmebp); 3717 page_unlock(pp); 3718 locked = 0; 3719 goto rehash; 3720 } 3721 3722 if (!VN_ISKAS(vp)) { 3723 /* 3724 * This is not a segkmem page but another page which 3725 * has been kernel mapped. It had better have at least 3726 * a share lock on it. Return the pfn. 3727 */ 3728 sfmmu_mlist_exit(pml); 3729 SFMMU_HASH_UNLOCK(hmebp); 3730 if (locked) 3731 page_unlock(pp); 3732 kmem_cache_free(pa_hment_cache, pahmep); 3733 ASSERT(PAGE_LOCKED(pp)); 3734 *rpfn = pfn; 3735 if (cookiep) 3736 *cookiep = HAC_COOKIE_NONE; 3737 return (0); 3738 } 3739 3740 /* 3741 * Setup this pa_hment and link its embedded dummy sf_hment into 3742 * the mapping list. 3743 */ 3744 pp->p_share++; 3745 pahmep->cb_id = callback_id; 3746 pahmep->addr = vaddr; 3747 pahmep->len = len; 3748 pahmep->refcnt = 1; 3749 pahmep->flags = 0; 3750 pahmep->pvt = pvt; 3751 3752 sfhmep->hme_tte.ll = 0; 3753 sfhmep->hme_data = pahmep; 3754 sfhmep->hme_prev = osfhmep; 3755 sfhmep->hme_next = osfhmep->hme_next; 3756 3757 if (osfhmep->hme_next) 3758 osfhmep->hme_next->hme_prev = sfhmep; 3759 3760 osfhmep->hme_next = sfhmep; 3761 3762 sfmmu_mlist_exit(pml); 3763 SFMMU_HASH_UNLOCK(hmebp); 3764 3765 if (locked) 3766 page_unlock(pp); 3767 3768 *rpfn = pfn; 3769 if (cookiep) 3770 *cookiep = (void *)pahmep; 3771 3772 return (0); 3773 } 3774 3775 /* 3776 * Remove the relocation callbacks from the specified addr/len. 3777 */ 3778 void 3779 hat_delete_callback(caddr_t vaddr, uint_t len, void *pvt, uint_t flags, 3780 void *cookie) 3781 { 3782 struct hmehash_bucket *hmebp; 3783 hmeblk_tag hblktag; 3784 struct hme_blk *hmeblkp; 3785 int hmeshift, hashno; 3786 caddr_t saddr; 3787 struct pa_hment *pahmep; 3788 struct sf_hment *sfhmep, *osfhmep; 3789 kmutex_t *pml; 3790 tte_t tte; 3791 page_t *pp; 3792 vnode_t *vp; 3793 u_offset_t off; 3794 int locked = 0; 3795 3796 /* 3797 * If the cookie is HAC_COOKIE_NONE then there is no pa_hment to 3798 * remove so just return. 3799 */ 3800 if (cookie == HAC_COOKIE_NONE || IS_KPM_ADDR(vaddr)) 3801 return; 3802 3803 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3804 3805 rehash: 3806 /* Find the mapping(s) for this page */ 3807 for (hashno = TTE64K, hmeblkp = NULL; 3808 hmeblkp == NULL && hashno <= mmu_hashcnt; 3809 hashno++) { 3810 hmeshift = HME_HASH_SHIFT(hashno); 3811 hblktag.htag_id = ksfmmup; 3812 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3813 hblktag.htag_rehash = hashno; 3814 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3815 3816 SFMMU_HASH_LOCK(hmebp); 3817 3818 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3819 3820 if (hmeblkp == NULL) 3821 SFMMU_HASH_UNLOCK(hmebp); 3822 } 3823 3824 if (hmeblkp == NULL) 3825 return; 3826 3827 HBLKTOHME(osfhmep, hmeblkp, saddr); 3828 3829 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3830 if (!TTE_IS_VALID(&tte)) { 3831 SFMMU_HASH_UNLOCK(hmebp); 3832 return; 3833 } 3834 3835 pp = osfhmep->hme_page; 3836 if (pp == NULL) { 3837 SFMMU_HASH_UNLOCK(hmebp); 3838 ASSERT(cookie == NULL); 3839 return; 3840 } 3841 3842 vp = pp->p_vnode; 3843 off = pp->p_offset; 3844 3845 pml = sfmmu_mlist_enter(pp); 3846 3847 if (flags & HAC_PAGELOCK) { 3848 if (!page_trylock(pp, SE_SHARED)) { 3849 /* 3850 * Somebody is holding SE_EXCL lock. Might 3851 * even be hat_page_relocate(). Drop all 3852 * our locks, lookup the page in &kvp, and 3853 * retry. If it doesn't exist in &kvp and &zvp, 3854 * then we must be dealing with a kernel mapped 3855 * page which doesn't actually belong to 3856 * segkmem so we punt. 3857 */ 3858 sfmmu_mlist_exit(pml); 3859 SFMMU_HASH_UNLOCK(hmebp); 3860 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3861 /* check zvp before giving up */ 3862 if (pp == NULL) 3863 pp = page_lookup(&zvp, (u_offset_t)saddr, 3864 SE_SHARED); 3865 3866 if (pp == NULL) { 3867 ASSERT(cookie == NULL); 3868 return; 3869 } 3870 page_unlock(pp); 3871 goto rehash; 3872 } 3873 locked = 1; 3874 } 3875 3876 ASSERT(PAGE_LOCKED(pp)); 3877 3878 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3879 pp->p_offset != off) { 3880 /* 3881 * The page moved before we got our hands on it. Drop 3882 * all the locks and try again. 3883 */ 3884 ASSERT((flags & HAC_PAGELOCK) != 0); 3885 sfmmu_mlist_exit(pml); 3886 SFMMU_HASH_UNLOCK(hmebp); 3887 page_unlock(pp); 3888 locked = 0; 3889 goto rehash; 3890 } 3891 3892 if (!VN_ISKAS(vp)) { 3893 /* 3894 * This is not a segkmem page but another page which 3895 * has been kernel mapped. 3896 */ 3897 sfmmu_mlist_exit(pml); 3898 SFMMU_HASH_UNLOCK(hmebp); 3899 if (locked) 3900 page_unlock(pp); 3901 ASSERT(cookie == NULL); 3902 return; 3903 } 3904 3905 if (cookie != NULL) { 3906 pahmep = (struct pa_hment *)cookie; 3907 sfhmep = &pahmep->sfment; 3908 } else { 3909 for (sfhmep = pp->p_mapping; sfhmep != NULL; 3910 sfhmep = sfhmep->hme_next) { 3911 3912 /* 3913 * skip va<->pa mappings 3914 */ 3915 if (!IS_PAHME(sfhmep)) 3916 continue; 3917 3918 pahmep = sfhmep->hme_data; 3919 ASSERT(pahmep != NULL); 3920 3921 /* 3922 * if pa_hment matches, remove it 3923 */ 3924 if ((pahmep->pvt == pvt) && 3925 (pahmep->addr == vaddr) && 3926 (pahmep->len == len)) { 3927 break; 3928 } 3929 } 3930 } 3931 3932 if (sfhmep == NULL) { 3933 if (!panicstr) { 3934 panic("hat_delete_callback: pa_hment not found, pp %p", 3935 (void *)pp); 3936 } 3937 return; 3938 } 3939 3940 /* 3941 * Note: at this point a valid kernel mapping must still be 3942 * present on this page. 3943 */ 3944 pp->p_share--; 3945 if (pp->p_share <= 0) 3946 panic("hat_delete_callback: zero p_share"); 3947 3948 if (--pahmep->refcnt == 0) { 3949 if (pahmep->flags != 0) 3950 panic("hat_delete_callback: pa_hment is busy"); 3951 3952 /* 3953 * Remove sfhmep from the mapping list for the page. 3954 */ 3955 if (sfhmep->hme_prev) { 3956 sfhmep->hme_prev->hme_next = sfhmep->hme_next; 3957 } else { 3958 pp->p_mapping = sfhmep->hme_next; 3959 } 3960 3961 if (sfhmep->hme_next) 3962 sfhmep->hme_next->hme_prev = sfhmep->hme_prev; 3963 3964 sfmmu_mlist_exit(pml); 3965 SFMMU_HASH_UNLOCK(hmebp); 3966 3967 if (locked) 3968 page_unlock(pp); 3969 3970 kmem_cache_free(pa_hment_cache, pahmep); 3971 return; 3972 } 3973 3974 sfmmu_mlist_exit(pml); 3975 SFMMU_HASH_UNLOCK(hmebp); 3976 if (locked) 3977 page_unlock(pp); 3978 } 3979 3980 /* 3981 * hat_probe returns 1 if the translation for the address 'addr' is 3982 * loaded, zero otherwise. 3983 * 3984 * hat_probe should be used only for advisorary purposes because it may 3985 * occasionally return the wrong value. The implementation must guarantee that 3986 * returning the wrong value is a very rare event. hat_probe is used 3987 * to implement optimizations in the segment drivers. 3988 * 3989 */ 3990 int 3991 hat_probe(struct hat *sfmmup, caddr_t addr) 3992 { 3993 pfn_t pfn; 3994 tte_t tte; 3995 3996 ASSERT(sfmmup != NULL); 3997 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3998 3999 ASSERT((sfmmup == ksfmmup) || 4000 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4001 4002 if (sfmmup == ksfmmup) { 4003 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte)) 4004 == PFN_SUSPENDED) { 4005 sfmmu_vatopfn_suspended(addr, sfmmup, &tte); 4006 } 4007 } else { 4008 pfn = sfmmu_uvatopfn(addr, sfmmup); 4009 } 4010 4011 if (pfn != PFN_INVALID) 4012 return (1); 4013 else 4014 return (0); 4015 } 4016 4017 ssize_t 4018 hat_getpagesize(struct hat *sfmmup, caddr_t addr) 4019 { 4020 tte_t tte; 4021 4022 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4023 4024 sfmmu_gettte(sfmmup, addr, &tte); 4025 if (TTE_IS_VALID(&tte)) { 4026 return (TTEBYTES(TTE_CSZ(&tte))); 4027 } 4028 return (-1); 4029 } 4030 4031 static void 4032 sfmmu_gettte(struct hat *sfmmup, caddr_t addr, tte_t *ttep) 4033 { 4034 struct hmehash_bucket *hmebp; 4035 hmeblk_tag hblktag; 4036 int hmeshift, hashno = 1; 4037 struct hme_blk *hmeblkp, *list = NULL; 4038 struct sf_hment *sfhmep; 4039 4040 /* support for ISM */ 4041 ism_map_t *ism_map; 4042 ism_blk_t *ism_blkp; 4043 int i; 4044 sfmmu_t *ism_hatid = NULL; 4045 sfmmu_t *locked_hatid = NULL; 4046 4047 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 4048 4049 ism_blkp = sfmmup->sfmmu_iblk; 4050 if (ism_blkp) { 4051 sfmmu_ismhat_enter(sfmmup, 0); 4052 locked_hatid = sfmmup; 4053 } 4054 while (ism_blkp && ism_hatid == NULL) { 4055 ism_map = ism_blkp->iblk_maps; 4056 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 4057 if (addr >= ism_start(ism_map[i]) && 4058 addr < ism_end(ism_map[i])) { 4059 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 4060 addr = (caddr_t)(addr - 4061 ism_start(ism_map[i])); 4062 break; 4063 } 4064 } 4065 ism_blkp = ism_blkp->iblk_next; 4066 } 4067 if (locked_hatid) { 4068 sfmmu_ismhat_exit(locked_hatid, 0); 4069 } 4070 4071 hblktag.htag_id = sfmmup; 4072 ttep->ll = 0; 4073 4074 do { 4075 hmeshift = HME_HASH_SHIFT(hashno); 4076 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4077 hblktag.htag_rehash = hashno; 4078 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4079 4080 SFMMU_HASH_LOCK(hmebp); 4081 4082 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4083 if (hmeblkp != NULL) { 4084 HBLKTOHME(sfhmep, hmeblkp, addr); 4085 sfmmu_copytte(&sfhmep->hme_tte, ttep); 4086 SFMMU_HASH_UNLOCK(hmebp); 4087 break; 4088 } 4089 SFMMU_HASH_UNLOCK(hmebp); 4090 hashno++; 4091 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 4092 4093 sfmmu_hblks_list_purge(&list); 4094 } 4095 4096 uint_t 4097 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr) 4098 { 4099 tte_t tte; 4100 4101 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4102 4103 sfmmu_gettte(sfmmup, addr, &tte); 4104 if (TTE_IS_VALID(&tte)) { 4105 *attr = sfmmu_ptov_attr(&tte); 4106 return (0); 4107 } 4108 *attr = 0; 4109 return ((uint_t)0xffffffff); 4110 } 4111 4112 /* 4113 * Enables more attributes on specified address range (ie. logical OR) 4114 */ 4115 void 4116 hat_setattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4117 { 4118 if (hat->sfmmu_xhat_provider) { 4119 XHAT_SETATTR(hat, addr, len, attr); 4120 return; 4121 } else { 4122 /* 4123 * This must be a CPU HAT. If the address space has 4124 * XHATs attached, change attributes for all of them, 4125 * just in case 4126 */ 4127 ASSERT(hat->sfmmu_as != NULL); 4128 if (hat->sfmmu_as->a_xhat != NULL) 4129 xhat_setattr_all(hat->sfmmu_as, addr, len, attr); 4130 } 4131 4132 sfmmu_chgattr(hat, addr, len, attr, SFMMU_SETATTR); 4133 } 4134 4135 /* 4136 * Assigns attributes to the specified address range. All the attributes 4137 * are specified. 4138 */ 4139 void 4140 hat_chgattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4141 { 4142 if (hat->sfmmu_xhat_provider) { 4143 XHAT_CHGATTR(hat, addr, len, attr); 4144 return; 4145 } else { 4146 /* 4147 * This must be a CPU HAT. If the address space has 4148 * XHATs attached, change attributes for all of them, 4149 * just in case 4150 */ 4151 ASSERT(hat->sfmmu_as != NULL); 4152 if (hat->sfmmu_as->a_xhat != NULL) 4153 xhat_chgattr_all(hat->sfmmu_as, addr, len, attr); 4154 } 4155 4156 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CHGATTR); 4157 } 4158 4159 /* 4160 * Remove attributes on the specified address range (ie. loginal NAND) 4161 */ 4162 void 4163 hat_clrattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4164 { 4165 if (hat->sfmmu_xhat_provider) { 4166 XHAT_CLRATTR(hat, addr, len, attr); 4167 return; 4168 } else { 4169 /* 4170 * This must be a CPU HAT. If the address space has 4171 * XHATs attached, change attributes for all of them, 4172 * just in case 4173 */ 4174 ASSERT(hat->sfmmu_as != NULL); 4175 if (hat->sfmmu_as->a_xhat != NULL) 4176 xhat_clrattr_all(hat->sfmmu_as, addr, len, attr); 4177 } 4178 4179 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CLRATTR); 4180 } 4181 4182 /* 4183 * Change attributes on an address range to that specified by attr and mode. 4184 */ 4185 static void 4186 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr, 4187 int mode) 4188 { 4189 struct hmehash_bucket *hmebp; 4190 hmeblk_tag hblktag; 4191 int hmeshift, hashno = 1; 4192 struct hme_blk *hmeblkp, *list = NULL; 4193 caddr_t endaddr; 4194 cpuset_t cpuset; 4195 demap_range_t dmr; 4196 4197 CPUSET_ZERO(cpuset); 4198 4199 ASSERT((sfmmup == ksfmmup) || 4200 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4201 ASSERT((len & MMU_PAGEOFFSET) == 0); 4202 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4203 4204 if ((attr & PROT_USER) && (mode != SFMMU_CLRATTR) && 4205 ((addr + len) > (caddr_t)USERLIMIT)) { 4206 panic("user addr %p in kernel space", 4207 (void *)addr); 4208 } 4209 4210 endaddr = addr + len; 4211 hblktag.htag_id = sfmmup; 4212 DEMAP_RANGE_INIT(sfmmup, &dmr); 4213 4214 while (addr < endaddr) { 4215 hmeshift = HME_HASH_SHIFT(hashno); 4216 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4217 hblktag.htag_rehash = hashno; 4218 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4219 4220 SFMMU_HASH_LOCK(hmebp); 4221 4222 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4223 if (hmeblkp != NULL) { 4224 /* 4225 * We've encountered a shadow hmeblk so skip the range 4226 * of the next smaller mapping size. 4227 */ 4228 if (hmeblkp->hblk_shw_bit) { 4229 ASSERT(sfmmup != ksfmmup); 4230 ASSERT(hashno > 1); 4231 addr = (caddr_t)P2END((uintptr_t)addr, 4232 TTEBYTES(hashno - 1)); 4233 } else { 4234 addr = sfmmu_hblk_chgattr(sfmmup, 4235 hmeblkp, addr, endaddr, &dmr, attr, mode); 4236 } 4237 SFMMU_HASH_UNLOCK(hmebp); 4238 hashno = 1; 4239 continue; 4240 } 4241 SFMMU_HASH_UNLOCK(hmebp); 4242 4243 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4244 /* 4245 * We have traversed the whole list and rehashed 4246 * if necessary without finding the address to chgattr. 4247 * This is ok, so we increment the address by the 4248 * smallest hmeblk range for kernel mappings or for 4249 * user mappings with no large pages, and the largest 4250 * hmeblk range, to account for shadow hmeblks, for 4251 * user mappings with large pages and continue. 4252 */ 4253 if (sfmmup == ksfmmup) 4254 addr = (caddr_t)P2END((uintptr_t)addr, 4255 TTEBYTES(1)); 4256 else 4257 addr = (caddr_t)P2END((uintptr_t)addr, 4258 TTEBYTES(hashno)); 4259 hashno = 1; 4260 } else { 4261 hashno++; 4262 } 4263 } 4264 4265 sfmmu_hblks_list_purge(&list); 4266 DEMAP_RANGE_FLUSH(&dmr); 4267 cpuset = sfmmup->sfmmu_cpusran; 4268 xt_sync(cpuset); 4269 } 4270 4271 /* 4272 * This function chgattr on a range of addresses in an hmeblk. It returns the 4273 * next addres that needs to be chgattr. 4274 * It should be called with the hash lock held. 4275 * XXX It should be possible to optimize chgattr by not flushing every time but 4276 * on the other hand: 4277 * 1. do one flush crosscall. 4278 * 2. only flush if we are increasing permissions (make sure this will work) 4279 */ 4280 static caddr_t 4281 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4282 caddr_t endaddr, demap_range_t *dmrp, uint_t attr, int mode) 4283 { 4284 tte_t tte, tteattr, tteflags, ttemod; 4285 struct sf_hment *sfhmep; 4286 int ttesz; 4287 struct page *pp = NULL; 4288 kmutex_t *pml, *pmtx; 4289 int ret; 4290 int use_demap_range; 4291 #if defined(SF_ERRATA_57) 4292 int check_exec; 4293 #endif 4294 4295 ASSERT(in_hblk_range(hmeblkp, addr)); 4296 ASSERT(hmeblkp->hblk_shw_bit == 0); 4297 4298 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4299 ttesz = get_hblk_ttesz(hmeblkp); 4300 4301 /* 4302 * Flush the current demap region if addresses have been 4303 * skipped or the page size doesn't match. 4304 */ 4305 use_demap_range = (TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp)); 4306 if (use_demap_range) { 4307 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4308 } else { 4309 DEMAP_RANGE_FLUSH(dmrp); 4310 } 4311 4312 tteattr.ll = sfmmu_vtop_attr(attr, mode, &tteflags); 4313 #if defined(SF_ERRATA_57) 4314 check_exec = (sfmmup != ksfmmup) && 4315 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4316 TTE_IS_EXECUTABLE(&tteattr); 4317 #endif 4318 HBLKTOHME(sfhmep, hmeblkp, addr); 4319 while (addr < endaddr) { 4320 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4321 if (TTE_IS_VALID(&tte)) { 4322 if ((tte.ll & tteflags.ll) == tteattr.ll) { 4323 /* 4324 * if the new attr is the same as old 4325 * continue 4326 */ 4327 goto next_addr; 4328 } 4329 if (!TTE_IS_WRITABLE(&tteattr)) { 4330 /* 4331 * make sure we clear hw modify bit if we 4332 * removing write protections 4333 */ 4334 tteflags.tte_intlo |= TTE_HWWR_INT; 4335 } 4336 4337 pml = NULL; 4338 pp = sfhmep->hme_page; 4339 if (pp) { 4340 pml = sfmmu_mlist_enter(pp); 4341 } 4342 4343 if (pp != sfhmep->hme_page) { 4344 /* 4345 * tte must have been unloaded. 4346 */ 4347 ASSERT(pml); 4348 sfmmu_mlist_exit(pml); 4349 continue; 4350 } 4351 4352 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4353 4354 ttemod = tte; 4355 ttemod.ll = (ttemod.ll & ~tteflags.ll) | tteattr.ll; 4356 ASSERT(TTE_TO_TTEPFN(&ttemod) == TTE_TO_TTEPFN(&tte)); 4357 4358 #if defined(SF_ERRATA_57) 4359 if (check_exec && addr < errata57_limit) 4360 ttemod.tte_exec_perm = 0; 4361 #endif 4362 ret = sfmmu_modifytte_try(&tte, &ttemod, 4363 &sfhmep->hme_tte); 4364 4365 if (ret < 0) { 4366 /* tte changed underneath us */ 4367 if (pml) { 4368 sfmmu_mlist_exit(pml); 4369 } 4370 continue; 4371 } 4372 4373 if (tteflags.tte_intlo & TTE_HWWR_INT) { 4374 /* 4375 * need to sync if we are clearing modify bit. 4376 */ 4377 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4378 } 4379 4380 if (pp && PP_ISRO(pp)) { 4381 if (tteattr.tte_intlo & TTE_WRPRM_INT) { 4382 pmtx = sfmmu_page_enter(pp); 4383 PP_CLRRO(pp); 4384 sfmmu_page_exit(pmtx); 4385 } 4386 } 4387 4388 if (ret > 0 && use_demap_range) { 4389 DEMAP_RANGE_MARKPG(dmrp, addr); 4390 } else if (ret > 0) { 4391 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4392 } 4393 4394 if (pml) { 4395 sfmmu_mlist_exit(pml); 4396 } 4397 } 4398 next_addr: 4399 addr += TTEBYTES(ttesz); 4400 sfhmep++; 4401 DEMAP_RANGE_NEXTPG(dmrp); 4402 } 4403 return (addr); 4404 } 4405 4406 /* 4407 * This routine converts virtual attributes to physical ones. It will 4408 * update the tteflags field with the tte mask corresponding to the attributes 4409 * affected and it returns the new attributes. It will also clear the modify 4410 * bit if we are taking away write permission. This is necessary since the 4411 * modify bit is the hardware permission bit and we need to clear it in order 4412 * to detect write faults. 4413 */ 4414 static uint64_t 4415 sfmmu_vtop_attr(uint_t attr, int mode, tte_t *ttemaskp) 4416 { 4417 tte_t ttevalue; 4418 4419 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 4420 4421 switch (mode) { 4422 case SFMMU_CHGATTR: 4423 /* all attributes specified */ 4424 ttevalue.tte_inthi = MAKE_TTEATTR_INTHI(attr); 4425 ttevalue.tte_intlo = MAKE_TTEATTR_INTLO(attr); 4426 ttemaskp->tte_inthi = TTEINTHI_ATTR; 4427 ttemaskp->tte_intlo = TTEINTLO_ATTR; 4428 break; 4429 case SFMMU_SETATTR: 4430 ASSERT(!(attr & ~HAT_PROT_MASK)); 4431 ttemaskp->ll = 0; 4432 ttevalue.ll = 0; 4433 /* 4434 * a valid tte implies exec and read for sfmmu 4435 * so no need to do anything about them. 4436 * since priviledged access implies user access 4437 * PROT_USER doesn't make sense either. 4438 */ 4439 if (attr & PROT_WRITE) { 4440 ttemaskp->tte_intlo |= TTE_WRPRM_INT; 4441 ttevalue.tte_intlo |= TTE_WRPRM_INT; 4442 } 4443 break; 4444 case SFMMU_CLRATTR: 4445 /* attributes will be nand with current ones */ 4446 if (attr & ~(PROT_WRITE | PROT_USER)) { 4447 panic("sfmmu: attr %x not supported", attr); 4448 } 4449 ttemaskp->ll = 0; 4450 ttevalue.ll = 0; 4451 if (attr & PROT_WRITE) { 4452 /* clear both writable and modify bit */ 4453 ttemaskp->tte_intlo |= TTE_WRPRM_INT | TTE_HWWR_INT; 4454 } 4455 if (attr & PROT_USER) { 4456 ttemaskp->tte_intlo |= TTE_PRIV_INT; 4457 ttevalue.tte_intlo |= TTE_PRIV_INT; 4458 } 4459 break; 4460 default: 4461 panic("sfmmu_vtop_attr: bad mode %x", mode); 4462 } 4463 ASSERT(TTE_TO_TTEPFN(&ttevalue) == 0); 4464 return (ttevalue.ll); 4465 } 4466 4467 static uint_t 4468 sfmmu_ptov_attr(tte_t *ttep) 4469 { 4470 uint_t attr; 4471 4472 ASSERT(TTE_IS_VALID(ttep)); 4473 4474 attr = PROT_READ; 4475 4476 if (TTE_IS_WRITABLE(ttep)) { 4477 attr |= PROT_WRITE; 4478 } 4479 if (TTE_IS_EXECUTABLE(ttep)) { 4480 attr |= PROT_EXEC; 4481 } 4482 if (!TTE_IS_PRIVILEGED(ttep)) { 4483 attr |= PROT_USER; 4484 } 4485 if (TTE_IS_NFO(ttep)) { 4486 attr |= HAT_NOFAULT; 4487 } 4488 if (TTE_IS_NOSYNC(ttep)) { 4489 attr |= HAT_NOSYNC; 4490 } 4491 if (TTE_IS_SIDEFFECT(ttep)) { 4492 attr |= SFMMU_SIDEFFECT; 4493 } 4494 if (!TTE_IS_VCACHEABLE(ttep)) { 4495 attr |= SFMMU_UNCACHEVTTE; 4496 } 4497 if (!TTE_IS_PCACHEABLE(ttep)) { 4498 attr |= SFMMU_UNCACHEPTTE; 4499 } 4500 return (attr); 4501 } 4502 4503 /* 4504 * hat_chgprot is a deprecated hat call. New segment drivers 4505 * should store all attributes and use hat_*attr calls. 4506 * 4507 * Change the protections in the virtual address range 4508 * given to the specified virtual protection. If vprot is ~PROT_WRITE, 4509 * then remove write permission, leaving the other 4510 * permissions unchanged. If vprot is ~PROT_USER, remove user permissions. 4511 * 4512 */ 4513 void 4514 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot) 4515 { 4516 struct hmehash_bucket *hmebp; 4517 hmeblk_tag hblktag; 4518 int hmeshift, hashno = 1; 4519 struct hme_blk *hmeblkp, *list = NULL; 4520 caddr_t endaddr; 4521 cpuset_t cpuset; 4522 demap_range_t dmr; 4523 4524 ASSERT((len & MMU_PAGEOFFSET) == 0); 4525 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4526 4527 if (sfmmup->sfmmu_xhat_provider) { 4528 XHAT_CHGPROT(sfmmup, addr, len, vprot); 4529 return; 4530 } else { 4531 /* 4532 * This must be a CPU HAT. If the address space has 4533 * XHATs attached, change attributes for all of them, 4534 * just in case 4535 */ 4536 ASSERT(sfmmup->sfmmu_as != NULL); 4537 if (sfmmup->sfmmu_as->a_xhat != NULL) 4538 xhat_chgprot_all(sfmmup->sfmmu_as, addr, len, vprot); 4539 } 4540 4541 CPUSET_ZERO(cpuset); 4542 4543 if ((vprot != (uint_t)~PROT_WRITE) && (vprot & PROT_USER) && 4544 ((addr + len) > (caddr_t)USERLIMIT)) { 4545 panic("user addr %p vprot %x in kernel space", 4546 (void *)addr, vprot); 4547 } 4548 endaddr = addr + len; 4549 hblktag.htag_id = sfmmup; 4550 DEMAP_RANGE_INIT(sfmmup, &dmr); 4551 4552 while (addr < endaddr) { 4553 hmeshift = HME_HASH_SHIFT(hashno); 4554 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4555 hblktag.htag_rehash = hashno; 4556 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4557 4558 SFMMU_HASH_LOCK(hmebp); 4559 4560 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4561 if (hmeblkp != NULL) { 4562 /* 4563 * We've encountered a shadow hmeblk so skip the range 4564 * of the next smaller mapping size. 4565 */ 4566 if (hmeblkp->hblk_shw_bit) { 4567 ASSERT(sfmmup != ksfmmup); 4568 ASSERT(hashno > 1); 4569 addr = (caddr_t)P2END((uintptr_t)addr, 4570 TTEBYTES(hashno - 1)); 4571 } else { 4572 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp, 4573 addr, endaddr, &dmr, vprot); 4574 } 4575 SFMMU_HASH_UNLOCK(hmebp); 4576 hashno = 1; 4577 continue; 4578 } 4579 SFMMU_HASH_UNLOCK(hmebp); 4580 4581 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4582 /* 4583 * We have traversed the whole list and rehashed 4584 * if necessary without finding the address to chgprot. 4585 * This is ok so we increment the address by the 4586 * smallest hmeblk range for kernel mappings and the 4587 * largest hmeblk range, to account for shadow hmeblks, 4588 * for user mappings and continue. 4589 */ 4590 if (sfmmup == ksfmmup) 4591 addr = (caddr_t)P2END((uintptr_t)addr, 4592 TTEBYTES(1)); 4593 else 4594 addr = (caddr_t)P2END((uintptr_t)addr, 4595 TTEBYTES(hashno)); 4596 hashno = 1; 4597 } else { 4598 hashno++; 4599 } 4600 } 4601 4602 sfmmu_hblks_list_purge(&list); 4603 DEMAP_RANGE_FLUSH(&dmr); 4604 cpuset = sfmmup->sfmmu_cpusran; 4605 xt_sync(cpuset); 4606 } 4607 4608 /* 4609 * This function chgprots a range of addresses in an hmeblk. It returns the 4610 * next addres that needs to be chgprot. 4611 * It should be called with the hash lock held. 4612 * XXX It shold be possible to optimize chgprot by not flushing every time but 4613 * on the other hand: 4614 * 1. do one flush crosscall. 4615 * 2. only flush if we are increasing permissions (make sure this will work) 4616 */ 4617 static caddr_t 4618 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4619 caddr_t endaddr, demap_range_t *dmrp, uint_t vprot) 4620 { 4621 uint_t pprot; 4622 tte_t tte, ttemod; 4623 struct sf_hment *sfhmep; 4624 uint_t tteflags; 4625 int ttesz; 4626 struct page *pp = NULL; 4627 kmutex_t *pml, *pmtx; 4628 int ret; 4629 int use_demap_range; 4630 #if defined(SF_ERRATA_57) 4631 int check_exec; 4632 #endif 4633 4634 ASSERT(in_hblk_range(hmeblkp, addr)); 4635 ASSERT(hmeblkp->hblk_shw_bit == 0); 4636 4637 #ifdef DEBUG 4638 if (get_hblk_ttesz(hmeblkp) != TTE8K && 4639 (endaddr < get_hblk_endaddr(hmeblkp))) { 4640 panic("sfmmu_hblk_chgprot: partial chgprot of large page"); 4641 } 4642 #endif /* DEBUG */ 4643 4644 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4645 ttesz = get_hblk_ttesz(hmeblkp); 4646 4647 pprot = sfmmu_vtop_prot(vprot, &tteflags); 4648 #if defined(SF_ERRATA_57) 4649 check_exec = (sfmmup != ksfmmup) && 4650 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4651 ((vprot & PROT_EXEC) == PROT_EXEC); 4652 #endif 4653 HBLKTOHME(sfhmep, hmeblkp, addr); 4654 4655 /* 4656 * Flush the current demap region if addresses have been 4657 * skipped or the page size doesn't match. 4658 */ 4659 use_demap_range = (TTEBYTES(ttesz) == MMU_PAGESIZE); 4660 if (use_demap_range) { 4661 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4662 } else { 4663 DEMAP_RANGE_FLUSH(dmrp); 4664 } 4665 4666 while (addr < endaddr) { 4667 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4668 if (TTE_IS_VALID(&tte)) { 4669 if (TTE_GET_LOFLAGS(&tte, tteflags) == pprot) { 4670 /* 4671 * if the new protection is the same as old 4672 * continue 4673 */ 4674 goto next_addr; 4675 } 4676 pml = NULL; 4677 pp = sfhmep->hme_page; 4678 if (pp) { 4679 pml = sfmmu_mlist_enter(pp); 4680 } 4681 if (pp != sfhmep->hme_page) { 4682 /* 4683 * tte most have been unloaded 4684 * underneath us. Recheck 4685 */ 4686 ASSERT(pml); 4687 sfmmu_mlist_exit(pml); 4688 continue; 4689 } 4690 4691 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4692 4693 ttemod = tte; 4694 TTE_SET_LOFLAGS(&ttemod, tteflags, pprot); 4695 #if defined(SF_ERRATA_57) 4696 if (check_exec && addr < errata57_limit) 4697 ttemod.tte_exec_perm = 0; 4698 #endif 4699 ret = sfmmu_modifytte_try(&tte, &ttemod, 4700 &sfhmep->hme_tte); 4701 4702 if (ret < 0) { 4703 /* tte changed underneath us */ 4704 if (pml) { 4705 sfmmu_mlist_exit(pml); 4706 } 4707 continue; 4708 } 4709 4710 if (tteflags & TTE_HWWR_INT) { 4711 /* 4712 * need to sync if we are clearing modify bit. 4713 */ 4714 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4715 } 4716 4717 if (pp && PP_ISRO(pp)) { 4718 if (pprot & TTE_WRPRM_INT) { 4719 pmtx = sfmmu_page_enter(pp); 4720 PP_CLRRO(pp); 4721 sfmmu_page_exit(pmtx); 4722 } 4723 } 4724 4725 if (ret > 0 && use_demap_range) { 4726 DEMAP_RANGE_MARKPG(dmrp, addr); 4727 } else if (ret > 0) { 4728 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4729 } 4730 4731 if (pml) { 4732 sfmmu_mlist_exit(pml); 4733 } 4734 } 4735 next_addr: 4736 addr += TTEBYTES(ttesz); 4737 sfhmep++; 4738 DEMAP_RANGE_NEXTPG(dmrp); 4739 } 4740 return (addr); 4741 } 4742 4743 /* 4744 * This routine is deprecated and should only be used by hat_chgprot. 4745 * The correct routine is sfmmu_vtop_attr. 4746 * This routine converts virtual page protections to physical ones. It will 4747 * update the tteflags field with the tte mask corresponding to the protections 4748 * affected and it returns the new protections. It will also clear the modify 4749 * bit if we are taking away write permission. This is necessary since the 4750 * modify bit is the hardware permission bit and we need to clear it in order 4751 * to detect write faults. 4752 * It accepts the following special protections: 4753 * ~PROT_WRITE = remove write permissions. 4754 * ~PROT_USER = remove user permissions. 4755 */ 4756 static uint_t 4757 sfmmu_vtop_prot(uint_t vprot, uint_t *tteflagsp) 4758 { 4759 if (vprot == (uint_t)~PROT_WRITE) { 4760 *tteflagsp = TTE_WRPRM_INT | TTE_HWWR_INT; 4761 return (0); /* will cause wrprm to be cleared */ 4762 } 4763 if (vprot == (uint_t)~PROT_USER) { 4764 *tteflagsp = TTE_PRIV_INT; 4765 return (0); /* will cause privprm to be cleared */ 4766 } 4767 if ((vprot == 0) || (vprot == PROT_USER) || 4768 ((vprot & PROT_ALL) != vprot)) { 4769 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4770 } 4771 4772 switch (vprot) { 4773 case (PROT_READ): 4774 case (PROT_EXEC): 4775 case (PROT_EXEC | PROT_READ): 4776 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4777 return (TTE_PRIV_INT); /* set prv and clr wrt */ 4778 case (PROT_WRITE): 4779 case (PROT_WRITE | PROT_READ): 4780 case (PROT_EXEC | PROT_WRITE): 4781 case (PROT_EXEC | PROT_WRITE | PROT_READ): 4782 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4783 return (TTE_PRIV_INT | TTE_WRPRM_INT); /* set prv and wrt */ 4784 case (PROT_USER | PROT_READ): 4785 case (PROT_USER | PROT_EXEC): 4786 case (PROT_USER | PROT_EXEC | PROT_READ): 4787 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4788 return (0); /* clr prv and wrt */ 4789 case (PROT_USER | PROT_WRITE): 4790 case (PROT_USER | PROT_WRITE | PROT_READ): 4791 case (PROT_USER | PROT_EXEC | PROT_WRITE): 4792 case (PROT_USER | PROT_EXEC | PROT_WRITE | PROT_READ): 4793 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4794 return (TTE_WRPRM_INT); /* clr prv and set wrt */ 4795 default: 4796 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4797 } 4798 return (0); 4799 } 4800 4801 /* 4802 * Alternate unload for very large virtual ranges. With a true 64 bit VA, 4803 * the normal algorithm would take too long for a very large VA range with 4804 * few real mappings. This routine just walks thru all HMEs in the global 4805 * hash table to find and remove mappings. 4806 */ 4807 static void 4808 hat_unload_large_virtual( 4809 struct hat *sfmmup, 4810 caddr_t startaddr, 4811 size_t len, 4812 uint_t flags, 4813 hat_callback_t *callback) 4814 { 4815 struct hmehash_bucket *hmebp; 4816 struct hme_blk *hmeblkp; 4817 struct hme_blk *pr_hblk = NULL; 4818 struct hme_blk *nx_hblk; 4819 struct hme_blk *list = NULL; 4820 int i; 4821 uint64_t hblkpa, prevpa, nx_pa; 4822 demap_range_t dmr, *dmrp; 4823 cpuset_t cpuset; 4824 caddr_t endaddr = startaddr + len; 4825 caddr_t sa; 4826 caddr_t ea; 4827 caddr_t cb_sa[MAX_CB_ADDR]; 4828 caddr_t cb_ea[MAX_CB_ADDR]; 4829 int addr_cnt = 0; 4830 int a = 0; 4831 4832 if (sfmmup->sfmmu_free) { 4833 dmrp = NULL; 4834 } else { 4835 dmrp = &dmr; 4836 DEMAP_RANGE_INIT(sfmmup, dmrp); 4837 } 4838 4839 /* 4840 * Loop through all the hash buckets of HME blocks looking for matches. 4841 */ 4842 for (i = 0; i <= UHMEHASH_SZ; i++) { 4843 hmebp = &uhme_hash[i]; 4844 SFMMU_HASH_LOCK(hmebp); 4845 hmeblkp = hmebp->hmeblkp; 4846 hblkpa = hmebp->hmeh_nextpa; 4847 prevpa = 0; 4848 pr_hblk = NULL; 4849 while (hmeblkp) { 4850 nx_hblk = hmeblkp->hblk_next; 4851 nx_pa = hmeblkp->hblk_nextpa; 4852 4853 /* 4854 * skip if not this context, if a shadow block or 4855 * if the mapping is not in the requested range 4856 */ 4857 if (hmeblkp->hblk_tag.htag_id != sfmmup || 4858 hmeblkp->hblk_shw_bit || 4859 (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr || 4860 (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) { 4861 pr_hblk = hmeblkp; 4862 prevpa = hblkpa; 4863 goto next_block; 4864 } 4865 4866 /* 4867 * unload if there are any current valid mappings 4868 */ 4869 if (hmeblkp->hblk_vcnt != 0 || 4870 hmeblkp->hblk_hmecnt != 0) 4871 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 4872 sa, ea, dmrp, flags); 4873 4874 /* 4875 * on unmap we also release the HME block itself, once 4876 * all mappings are gone. 4877 */ 4878 if ((flags & HAT_UNLOAD_UNMAP) != 0 && 4879 !hmeblkp->hblk_vcnt && 4880 !hmeblkp->hblk_hmecnt) { 4881 ASSERT(!hmeblkp->hblk_lckcnt); 4882 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 4883 prevpa, pr_hblk); 4884 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 4885 } else { 4886 pr_hblk = hmeblkp; 4887 prevpa = hblkpa; 4888 } 4889 4890 if (callback == NULL) 4891 goto next_block; 4892 4893 /* 4894 * HME blocks may span more than one page, but we may be 4895 * unmapping only one page, so check for a smaller range 4896 * for the callback 4897 */ 4898 if (sa < startaddr) 4899 sa = startaddr; 4900 if (--ea > endaddr) 4901 ea = endaddr - 1; 4902 4903 cb_sa[addr_cnt] = sa; 4904 cb_ea[addr_cnt] = ea; 4905 if (++addr_cnt == MAX_CB_ADDR) { 4906 if (dmrp != NULL) { 4907 DEMAP_RANGE_FLUSH(dmrp); 4908 cpuset = sfmmup->sfmmu_cpusran; 4909 xt_sync(cpuset); 4910 } 4911 4912 for (a = 0; a < MAX_CB_ADDR; ++a) { 4913 callback->hcb_start_addr = cb_sa[a]; 4914 callback->hcb_end_addr = cb_ea[a]; 4915 callback->hcb_function(callback); 4916 } 4917 addr_cnt = 0; 4918 } 4919 4920 next_block: 4921 hmeblkp = nx_hblk; 4922 hblkpa = nx_pa; 4923 } 4924 SFMMU_HASH_UNLOCK(hmebp); 4925 } 4926 4927 sfmmu_hblks_list_purge(&list); 4928 if (dmrp != NULL) { 4929 DEMAP_RANGE_FLUSH(dmrp); 4930 cpuset = sfmmup->sfmmu_cpusran; 4931 xt_sync(cpuset); 4932 } 4933 4934 for (a = 0; a < addr_cnt; ++a) { 4935 callback->hcb_start_addr = cb_sa[a]; 4936 callback->hcb_end_addr = cb_ea[a]; 4937 callback->hcb_function(callback); 4938 } 4939 4940 /* 4941 * Check TSB and TLB page sizes if the process isn't exiting. 4942 */ 4943 if (!sfmmup->sfmmu_free) 4944 sfmmu_check_page_sizes(sfmmup, 0); 4945 } 4946 4947 /* 4948 * Unload all the mappings in the range [addr..addr+len). addr and len must 4949 * be MMU_PAGESIZE aligned. 4950 */ 4951 4952 extern struct seg *segkmap; 4953 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \ 4954 segkmap->s_base <= (addr) && (addr) < (segkmap->s_base + segkmap->s_size)) 4955 4956 4957 void 4958 hat_unload_callback( 4959 struct hat *sfmmup, 4960 caddr_t addr, 4961 size_t len, 4962 uint_t flags, 4963 hat_callback_t *callback) 4964 { 4965 struct hmehash_bucket *hmebp; 4966 hmeblk_tag hblktag; 4967 int hmeshift, hashno, iskernel; 4968 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 4969 caddr_t endaddr; 4970 cpuset_t cpuset; 4971 uint64_t hblkpa, prevpa; 4972 int addr_count = 0; 4973 int a; 4974 caddr_t cb_start_addr[MAX_CB_ADDR]; 4975 caddr_t cb_end_addr[MAX_CB_ADDR]; 4976 int issegkmap = ISSEGKMAP(sfmmup, addr); 4977 demap_range_t dmr, *dmrp; 4978 4979 if (sfmmup->sfmmu_xhat_provider) { 4980 XHAT_UNLOAD_CALLBACK(sfmmup, addr, len, flags, callback); 4981 return; 4982 } else { 4983 /* 4984 * This must be a CPU HAT. If the address space has 4985 * XHATs attached, unload the mappings for all of them, 4986 * just in case 4987 */ 4988 ASSERT(sfmmup->sfmmu_as != NULL); 4989 if (sfmmup->sfmmu_as->a_xhat != NULL) 4990 xhat_unload_callback_all(sfmmup->sfmmu_as, addr, 4991 len, flags, callback); 4992 } 4993 4994 ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \ 4995 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4996 4997 ASSERT(sfmmup != NULL); 4998 ASSERT((len & MMU_PAGEOFFSET) == 0); 4999 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 5000 5001 /* 5002 * Probing through a large VA range (say 63 bits) will be slow, even 5003 * at 4 Meg steps between the probes. So, when the virtual address range 5004 * is very large, search the HME entries for what to unload. 5005 * 5006 * len >> TTE_PAGE_SHIFT(TTE4M) is the # of 4Meg probes we'd need 5007 * 5008 * UHMEHASH_SZ is number of hash buckets to examine 5009 * 5010 */ 5011 if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) { 5012 hat_unload_large_virtual(sfmmup, addr, len, flags, callback); 5013 return; 5014 } 5015 5016 CPUSET_ZERO(cpuset); 5017 5018 /* 5019 * If the process is exiting, we can save a lot of fuss since 5020 * we'll flush the TLB when we free the ctx anyway. 5021 */ 5022 if (sfmmup->sfmmu_free) 5023 dmrp = NULL; 5024 else 5025 dmrp = &dmr; 5026 5027 DEMAP_RANGE_INIT(sfmmup, dmrp); 5028 endaddr = addr + len; 5029 hblktag.htag_id = sfmmup; 5030 5031 /* 5032 * It is likely for the vm to call unload over a wide range of 5033 * addresses that are actually very sparsely populated by 5034 * translations. In order to speed this up the sfmmu hat supports 5035 * the concept of shadow hmeblks. Dummy large page hmeblks that 5036 * correspond to actual small translations are allocated at tteload 5037 * time and are referred to as shadow hmeblks. Now, during unload 5038 * time, we first check if we have a shadow hmeblk for that 5039 * translation. The absence of one means the corresponding address 5040 * range is empty and can be skipped. 5041 * 5042 * The kernel is an exception to above statement and that is why 5043 * we don't use shadow hmeblks and hash starting from the smallest 5044 * page size. 5045 */ 5046 if (sfmmup == KHATID) { 5047 iskernel = 1; 5048 hashno = TTE64K; 5049 } else { 5050 iskernel = 0; 5051 if (mmu_page_sizes == max_mmu_page_sizes) { 5052 hashno = TTE256M; 5053 } else { 5054 hashno = TTE4M; 5055 } 5056 } 5057 while (addr < endaddr) { 5058 hmeshift = HME_HASH_SHIFT(hashno); 5059 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5060 hblktag.htag_rehash = hashno; 5061 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5062 5063 SFMMU_HASH_LOCK(hmebp); 5064 5065 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, pr_hblk, 5066 prevpa, &list); 5067 if (hmeblkp == NULL) { 5068 /* 5069 * didn't find an hmeblk. skip the appropiate 5070 * address range. 5071 */ 5072 SFMMU_HASH_UNLOCK(hmebp); 5073 if (iskernel) { 5074 if (hashno < mmu_hashcnt) { 5075 hashno++; 5076 continue; 5077 } else { 5078 hashno = TTE64K; 5079 addr = (caddr_t)roundup((uintptr_t)addr 5080 + 1, MMU_PAGESIZE64K); 5081 continue; 5082 } 5083 } 5084 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5085 (1 << hmeshift)); 5086 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5087 ASSERT(hashno == TTE64K); 5088 continue; 5089 } 5090 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5091 hashno = TTE512K; 5092 continue; 5093 } 5094 if (mmu_page_sizes == max_mmu_page_sizes) { 5095 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5096 hashno = TTE4M; 5097 continue; 5098 } 5099 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5100 hashno = TTE32M; 5101 continue; 5102 } 5103 hashno = TTE256M; 5104 continue; 5105 } else { 5106 hashno = TTE4M; 5107 continue; 5108 } 5109 } 5110 ASSERT(hmeblkp); 5111 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5112 /* 5113 * If the valid count is zero we can skip the range 5114 * mapped by this hmeblk. 5115 * We free hblks in the case of HAT_UNMAP. HAT_UNMAP 5116 * is used by segment drivers as a hint 5117 * that the mapping resource won't be used any longer. 5118 * The best example of this is during exit(). 5119 */ 5120 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5121 get_hblk_span(hmeblkp)); 5122 if ((flags & HAT_UNLOAD_UNMAP) || 5123 (iskernel && !issegkmap)) { 5124 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5125 pr_hblk); 5126 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5127 } 5128 SFMMU_HASH_UNLOCK(hmebp); 5129 5130 if (iskernel) { 5131 hashno = TTE64K; 5132 continue; 5133 } 5134 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5135 ASSERT(hashno == TTE64K); 5136 continue; 5137 } 5138 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5139 hashno = TTE512K; 5140 continue; 5141 } 5142 if (mmu_page_sizes == max_mmu_page_sizes) { 5143 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5144 hashno = TTE4M; 5145 continue; 5146 } 5147 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5148 hashno = TTE32M; 5149 continue; 5150 } 5151 hashno = TTE256M; 5152 continue; 5153 } else { 5154 hashno = TTE4M; 5155 continue; 5156 } 5157 } 5158 if (hmeblkp->hblk_shw_bit) { 5159 /* 5160 * If we encounter a shadow hmeblk we know there is 5161 * smaller sized hmeblks mapping the same address space. 5162 * Decrement the hash size and rehash. 5163 */ 5164 ASSERT(sfmmup != KHATID); 5165 hashno--; 5166 SFMMU_HASH_UNLOCK(hmebp); 5167 continue; 5168 } 5169 5170 /* 5171 * track callback address ranges. 5172 * only start a new range when it's not contiguous 5173 */ 5174 if (callback != NULL) { 5175 if (addr_count > 0 && 5176 addr == cb_end_addr[addr_count - 1]) 5177 --addr_count; 5178 else 5179 cb_start_addr[addr_count] = addr; 5180 } 5181 5182 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr, 5183 dmrp, flags); 5184 5185 if (callback != NULL) 5186 cb_end_addr[addr_count++] = addr; 5187 5188 if (((flags & HAT_UNLOAD_UNMAP) || (iskernel && !issegkmap)) && 5189 !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5190 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5191 pr_hblk); 5192 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5193 } 5194 SFMMU_HASH_UNLOCK(hmebp); 5195 5196 /* 5197 * Notify our caller as to exactly which pages 5198 * have been unloaded. We do these in clumps, 5199 * to minimize the number of xt_sync()s that need to occur. 5200 */ 5201 if (callback != NULL && addr_count == MAX_CB_ADDR) { 5202 DEMAP_RANGE_FLUSH(dmrp); 5203 if (dmrp != NULL) { 5204 cpuset = sfmmup->sfmmu_cpusran; 5205 xt_sync(cpuset); 5206 } 5207 5208 for (a = 0; a < MAX_CB_ADDR; ++a) { 5209 callback->hcb_start_addr = cb_start_addr[a]; 5210 callback->hcb_end_addr = cb_end_addr[a]; 5211 callback->hcb_function(callback); 5212 } 5213 addr_count = 0; 5214 } 5215 if (iskernel) { 5216 hashno = TTE64K; 5217 continue; 5218 } 5219 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5220 ASSERT(hashno == TTE64K); 5221 continue; 5222 } 5223 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5224 hashno = TTE512K; 5225 continue; 5226 } 5227 if (mmu_page_sizes == max_mmu_page_sizes) { 5228 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5229 hashno = TTE4M; 5230 continue; 5231 } 5232 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5233 hashno = TTE32M; 5234 continue; 5235 } 5236 hashno = TTE256M; 5237 } else { 5238 hashno = TTE4M; 5239 } 5240 } 5241 5242 sfmmu_hblks_list_purge(&list); 5243 DEMAP_RANGE_FLUSH(dmrp); 5244 if (dmrp != NULL) { 5245 cpuset = sfmmup->sfmmu_cpusran; 5246 xt_sync(cpuset); 5247 } 5248 if (callback && addr_count != 0) { 5249 for (a = 0; a < addr_count; ++a) { 5250 callback->hcb_start_addr = cb_start_addr[a]; 5251 callback->hcb_end_addr = cb_end_addr[a]; 5252 callback->hcb_function(callback); 5253 } 5254 } 5255 5256 /* 5257 * Check TSB and TLB page sizes if the process isn't exiting. 5258 */ 5259 if (!sfmmup->sfmmu_free) 5260 sfmmu_check_page_sizes(sfmmup, 0); 5261 } 5262 5263 /* 5264 * Unload all the mappings in the range [addr..addr+len). addr and len must 5265 * be MMU_PAGESIZE aligned. 5266 */ 5267 void 5268 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags) 5269 { 5270 if (sfmmup->sfmmu_xhat_provider) { 5271 XHAT_UNLOAD(sfmmup, addr, len, flags); 5272 return; 5273 } 5274 hat_unload_callback(sfmmup, addr, len, flags, NULL); 5275 } 5276 5277 5278 /* 5279 * Find the largest mapping size for this page. 5280 */ 5281 int 5282 fnd_mapping_sz(page_t *pp) 5283 { 5284 int sz; 5285 int p_index; 5286 5287 p_index = PP_MAPINDEX(pp); 5288 5289 sz = 0; 5290 p_index >>= 1; /* don't care about 8K bit */ 5291 for (; p_index; p_index >>= 1) { 5292 sz++; 5293 } 5294 5295 return (sz); 5296 } 5297 5298 /* 5299 * This function unloads a range of addresses for an hmeblk. 5300 * It returns the next address to be unloaded. 5301 * It should be called with the hash lock held. 5302 */ 5303 static caddr_t 5304 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5305 caddr_t endaddr, demap_range_t *dmrp, uint_t flags) 5306 { 5307 tte_t tte, ttemod; 5308 struct sf_hment *sfhmep; 5309 int ttesz; 5310 long ttecnt; 5311 page_t *pp; 5312 kmutex_t *pml; 5313 int ret; 5314 int use_demap_range; 5315 5316 ASSERT(in_hblk_range(hmeblkp, addr)); 5317 ASSERT(!hmeblkp->hblk_shw_bit); 5318 #ifdef DEBUG 5319 if (get_hblk_ttesz(hmeblkp) != TTE8K && 5320 (endaddr < get_hblk_endaddr(hmeblkp))) { 5321 panic("sfmmu_hblk_unload: partial unload of large page"); 5322 } 5323 #endif /* DEBUG */ 5324 5325 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5326 ttesz = get_hblk_ttesz(hmeblkp); 5327 5328 use_demap_range = (do_virtual_coloring && 5329 ((dmrp == NULL) || TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp))); 5330 if (use_demap_range) { 5331 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 5332 } else { 5333 DEMAP_RANGE_FLUSH(dmrp); 5334 } 5335 ttecnt = 0; 5336 HBLKTOHME(sfhmep, hmeblkp, addr); 5337 5338 while (addr < endaddr) { 5339 pml = NULL; 5340 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5341 if (TTE_IS_VALID(&tte)) { 5342 pp = sfhmep->hme_page; 5343 if (pp != NULL) { 5344 pml = sfmmu_mlist_enter(pp); 5345 } 5346 5347 /* 5348 * Verify if hme still points to 'pp' now that 5349 * we have p_mapping lock. 5350 */ 5351 if (sfhmep->hme_page != pp) { 5352 if (pp != NULL && sfhmep->hme_page != NULL) { 5353 ASSERT(pml != NULL); 5354 sfmmu_mlist_exit(pml); 5355 /* Re-start this iteration. */ 5356 continue; 5357 } 5358 ASSERT((pp != NULL) && 5359 (sfhmep->hme_page == NULL)); 5360 goto tte_unloaded; 5361 } 5362 5363 /* 5364 * This point on we have both HASH and p_mapping 5365 * lock. 5366 */ 5367 ASSERT(pp == sfhmep->hme_page); 5368 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5369 5370 /* 5371 * We need to loop on modify tte because it is 5372 * possible for pagesync to come along and 5373 * change the software bits beneath us. 5374 * 5375 * Page_unload can also invalidate the tte after 5376 * we read tte outside of p_mapping lock. 5377 */ 5378 again: 5379 ttemod = tte; 5380 5381 TTE_SET_INVALID(&ttemod); 5382 ret = sfmmu_modifytte_try(&tte, &ttemod, 5383 &sfhmep->hme_tte); 5384 5385 if (ret <= 0) { 5386 if (TTE_IS_VALID(&tte)) { 5387 ASSERT(ret < 0); 5388 goto again; 5389 } 5390 if (pp != NULL) { 5391 panic("sfmmu_hblk_unload: pp = 0x%p " 5392 "tte became invalid under mlist" 5393 " lock = 0x%p", pp, pml); 5394 } 5395 continue; 5396 } 5397 5398 if (!(flags & HAT_UNLOAD_NOSYNC)) { 5399 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5400 } 5401 5402 /* 5403 * Ok- we invalidated the tte. Do the rest of the job. 5404 */ 5405 ttecnt++; 5406 5407 if (flags & HAT_UNLOAD_UNLOCK) { 5408 ASSERT(hmeblkp->hblk_lckcnt > 0); 5409 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 5410 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 5411 } 5412 5413 /* 5414 * Normally we would need to flush the page 5415 * from the virtual cache at this point in 5416 * order to prevent a potential cache alias 5417 * inconsistency. 5418 * The particular scenario we need to worry 5419 * about is: 5420 * Given: va1 and va2 are two virtual address 5421 * that alias and map the same physical 5422 * address. 5423 * 1. mapping exists from va1 to pa and data 5424 * has been read into the cache. 5425 * 2. unload va1. 5426 * 3. load va2 and modify data using va2. 5427 * 4 unload va2. 5428 * 5. load va1 and reference data. Unless we 5429 * flush the data cache when we unload we will 5430 * get stale data. 5431 * Fortunately, page coloring eliminates the 5432 * above scenario by remembering the color a 5433 * physical page was last or is currently 5434 * mapped to. Now, we delay the flush until 5435 * the loading of translations. Only when the 5436 * new translation is of a different color 5437 * are we forced to flush. 5438 */ 5439 if (use_demap_range) { 5440 /* 5441 * Mark this page as needing a demap. 5442 */ 5443 DEMAP_RANGE_MARKPG(dmrp, addr); 5444 } else { 5445 if (do_virtual_coloring) { 5446 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 5447 sfmmup->sfmmu_free, 0); 5448 } else { 5449 pfn_t pfnum; 5450 5451 pfnum = TTE_TO_PFN(addr, &tte); 5452 sfmmu_tlbcache_demap(addr, sfmmup, 5453 hmeblkp, pfnum, sfmmup->sfmmu_free, 5454 FLUSH_NECESSARY_CPUS, 5455 CACHE_FLUSH, 0); 5456 } 5457 } 5458 5459 if (pp) { 5460 /* 5461 * Remove the hment from the mapping list 5462 */ 5463 ASSERT(hmeblkp->hblk_hmecnt > 0); 5464 5465 /* 5466 * Again, we cannot 5467 * ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS); 5468 */ 5469 HME_SUB(sfhmep, pp); 5470 membar_stst(); 5471 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 5472 } 5473 5474 ASSERT(hmeblkp->hblk_vcnt > 0); 5475 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 5476 5477 ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 5478 !hmeblkp->hblk_lckcnt); 5479 5480 #ifdef VAC 5481 if (pp && (pp->p_nrm & (P_KPMC | P_KPMS | P_TNC))) { 5482 if (PP_ISTNC(pp)) { 5483 /* 5484 * If page was temporary 5485 * uncached, try to recache 5486 * it. Note that HME_SUB() was 5487 * called above so p_index and 5488 * mlist had been updated. 5489 */ 5490 conv_tnc(pp, ttesz); 5491 } else if (pp->p_mapping == NULL) { 5492 ASSERT(kpm_enable); 5493 /* 5494 * Page is marked to be in VAC conflict 5495 * to an existing kpm mapping and/or is 5496 * kpm mapped using only the regular 5497 * pagesize. 5498 */ 5499 sfmmu_kpm_hme_unload(pp); 5500 } 5501 } 5502 #endif /* VAC */ 5503 } else if ((pp = sfhmep->hme_page) != NULL) { 5504 /* 5505 * TTE is invalid but the hme 5506 * still exists. let pageunload 5507 * complete its job. 5508 */ 5509 ASSERT(pml == NULL); 5510 pml = sfmmu_mlist_enter(pp); 5511 if (sfhmep->hme_page != NULL) { 5512 sfmmu_mlist_exit(pml); 5513 continue; 5514 } 5515 ASSERT(sfhmep->hme_page == NULL); 5516 } else if (hmeblkp->hblk_hmecnt != 0) { 5517 /* 5518 * pageunload may have not finished decrementing 5519 * hblk_vcnt and hblk_hmecnt. Find page_t if any and 5520 * wait for pageunload to finish. Rely on pageunload 5521 * to decrement hblk_hmecnt after hblk_vcnt. 5522 */ 5523 pfn_t pfn = TTE_TO_TTEPFN(&tte); 5524 ASSERT(pml == NULL); 5525 if (pf_is_memory(pfn)) { 5526 pp = page_numtopp_nolock(pfn); 5527 if (pp != NULL) { 5528 pml = sfmmu_mlist_enter(pp); 5529 sfmmu_mlist_exit(pml); 5530 pml = NULL; 5531 } 5532 } 5533 } 5534 5535 tte_unloaded: 5536 /* 5537 * At this point, the tte we are looking at 5538 * should be unloaded, and hme has been unlinked 5539 * from page too. This is important because in 5540 * pageunload, it does ttesync() then HME_SUB. 5541 * We need to make sure HME_SUB has been completed 5542 * so we know ttesync() has been completed. Otherwise, 5543 * at exit time, after return from hat layer, VM will 5544 * release as structure which hat_setstat() (called 5545 * by ttesync()) needs. 5546 */ 5547 #ifdef DEBUG 5548 { 5549 tte_t dtte; 5550 5551 ASSERT(sfhmep->hme_page == NULL); 5552 5553 sfmmu_copytte(&sfhmep->hme_tte, &dtte); 5554 ASSERT(!TTE_IS_VALID(&dtte)); 5555 } 5556 #endif 5557 5558 if (pml) { 5559 sfmmu_mlist_exit(pml); 5560 } 5561 5562 addr += TTEBYTES(ttesz); 5563 sfhmep++; 5564 DEMAP_RANGE_NEXTPG(dmrp); 5565 } 5566 if (ttecnt > 0) 5567 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt); 5568 return (addr); 5569 } 5570 5571 /* 5572 * Synchronize all the mappings in the range [addr..addr+len). 5573 * Can be called with clearflag having two states: 5574 * HAT_SYNC_DONTZERO means just return the rm stats 5575 * HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats 5576 */ 5577 void 5578 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag) 5579 { 5580 struct hmehash_bucket *hmebp; 5581 hmeblk_tag hblktag; 5582 int hmeshift, hashno = 1; 5583 struct hme_blk *hmeblkp, *list = NULL; 5584 caddr_t endaddr; 5585 cpuset_t cpuset; 5586 5587 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 5588 ASSERT((sfmmup == ksfmmup) || 5589 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 5590 ASSERT((len & MMU_PAGEOFFSET) == 0); 5591 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 5592 (clearflag == HAT_SYNC_ZERORM)); 5593 5594 CPUSET_ZERO(cpuset); 5595 5596 endaddr = addr + len; 5597 hblktag.htag_id = sfmmup; 5598 /* 5599 * Spitfire supports 4 page sizes. 5600 * Most pages are expected to be of the smallest page 5601 * size (8K) and these will not need to be rehashed. 64K 5602 * pages also don't need to be rehashed because the an hmeblk 5603 * spans 64K of address space. 512K pages might need 1 rehash and 5604 * and 4M pages 2 rehashes. 5605 */ 5606 while (addr < endaddr) { 5607 hmeshift = HME_HASH_SHIFT(hashno); 5608 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5609 hblktag.htag_rehash = hashno; 5610 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5611 5612 SFMMU_HASH_LOCK(hmebp); 5613 5614 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 5615 if (hmeblkp != NULL) { 5616 /* 5617 * We've encountered a shadow hmeblk so skip the range 5618 * of the next smaller mapping size. 5619 */ 5620 if (hmeblkp->hblk_shw_bit) { 5621 ASSERT(sfmmup != ksfmmup); 5622 ASSERT(hashno > 1); 5623 addr = (caddr_t)P2END((uintptr_t)addr, 5624 TTEBYTES(hashno - 1)); 5625 } else { 5626 addr = sfmmu_hblk_sync(sfmmup, hmeblkp, 5627 addr, endaddr, clearflag); 5628 } 5629 SFMMU_HASH_UNLOCK(hmebp); 5630 hashno = 1; 5631 continue; 5632 } 5633 SFMMU_HASH_UNLOCK(hmebp); 5634 5635 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 5636 /* 5637 * We have traversed the whole list and rehashed 5638 * if necessary without finding the address to sync. 5639 * This is ok so we increment the address by the 5640 * smallest hmeblk range for kernel mappings and the 5641 * largest hmeblk range, to account for shadow hmeblks, 5642 * for user mappings and continue. 5643 */ 5644 if (sfmmup == ksfmmup) 5645 addr = (caddr_t)P2END((uintptr_t)addr, 5646 TTEBYTES(1)); 5647 else 5648 addr = (caddr_t)P2END((uintptr_t)addr, 5649 TTEBYTES(hashno)); 5650 hashno = 1; 5651 } else { 5652 hashno++; 5653 } 5654 } 5655 sfmmu_hblks_list_purge(&list); 5656 cpuset = sfmmup->sfmmu_cpusran; 5657 xt_sync(cpuset); 5658 } 5659 5660 static caddr_t 5661 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5662 caddr_t endaddr, int clearflag) 5663 { 5664 tte_t tte, ttemod; 5665 struct sf_hment *sfhmep; 5666 int ttesz; 5667 struct page *pp; 5668 kmutex_t *pml; 5669 int ret; 5670 5671 ASSERT(hmeblkp->hblk_shw_bit == 0); 5672 5673 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5674 5675 ttesz = get_hblk_ttesz(hmeblkp); 5676 HBLKTOHME(sfhmep, hmeblkp, addr); 5677 5678 while (addr < endaddr) { 5679 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5680 if (TTE_IS_VALID(&tte)) { 5681 pml = NULL; 5682 pp = sfhmep->hme_page; 5683 if (pp) { 5684 pml = sfmmu_mlist_enter(pp); 5685 } 5686 if (pp != sfhmep->hme_page) { 5687 /* 5688 * tte most have been unloaded 5689 * underneath us. Recheck 5690 */ 5691 ASSERT(pml); 5692 sfmmu_mlist_exit(pml); 5693 continue; 5694 } 5695 5696 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5697 5698 if (clearflag == HAT_SYNC_ZERORM) { 5699 ttemod = tte; 5700 TTE_CLR_RM(&ttemod); 5701 ret = sfmmu_modifytte_try(&tte, &ttemod, 5702 &sfhmep->hme_tte); 5703 if (ret < 0) { 5704 if (pml) { 5705 sfmmu_mlist_exit(pml); 5706 } 5707 continue; 5708 } 5709 5710 if (ret > 0) { 5711 sfmmu_tlb_demap(addr, sfmmup, 5712 hmeblkp, 0, 0); 5713 } 5714 } 5715 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5716 if (pml) { 5717 sfmmu_mlist_exit(pml); 5718 } 5719 } 5720 addr += TTEBYTES(ttesz); 5721 sfhmep++; 5722 } 5723 return (addr); 5724 } 5725 5726 /* 5727 * This function will sync a tte to the page struct and it will 5728 * update the hat stats. Currently it allows us to pass a NULL pp 5729 * and we will simply update the stats. We may want to change this 5730 * so we only keep stats for pages backed by pp's. 5731 */ 5732 static void 5733 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp) 5734 { 5735 uint_t rm = 0; 5736 int sz; 5737 pgcnt_t npgs; 5738 5739 ASSERT(TTE_IS_VALID(ttep)); 5740 5741 if (TTE_IS_NOSYNC(ttep)) { 5742 return; 5743 } 5744 5745 if (TTE_IS_REF(ttep)) { 5746 rm = P_REF; 5747 } 5748 if (TTE_IS_MOD(ttep)) { 5749 rm |= P_MOD; 5750 } 5751 5752 if (rm == 0) { 5753 return; 5754 } 5755 5756 sz = TTE_CSZ(ttep); 5757 if (sfmmup->sfmmu_rmstat) { 5758 int i; 5759 caddr_t vaddr = addr; 5760 5761 for (i = 0; i < TTEPAGES(sz); i++, vaddr += MMU_PAGESIZE) { 5762 hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm); 5763 } 5764 5765 } 5766 5767 /* 5768 * XXX I want to use cas to update nrm bits but they 5769 * currently belong in common/vm and not in hat where 5770 * they should be. 5771 * The nrm bits are protected by the same mutex as 5772 * the one that protects the page's mapping list. 5773 */ 5774 if (!pp) 5775 return; 5776 ASSERT(sfmmu_mlist_held(pp)); 5777 /* 5778 * If the tte is for a large page, we need to sync all the 5779 * pages covered by the tte. 5780 */ 5781 if (sz != TTE8K) { 5782 ASSERT(pp->p_szc != 0); 5783 pp = PP_GROUPLEADER(pp, sz); 5784 ASSERT(sfmmu_mlist_held(pp)); 5785 } 5786 5787 /* Get number of pages from tte size. */ 5788 npgs = TTEPAGES(sz); 5789 5790 do { 5791 ASSERT(pp); 5792 ASSERT(sfmmu_mlist_held(pp)); 5793 if (((rm & P_REF) != 0 && !PP_ISREF(pp)) || 5794 ((rm & P_MOD) != 0 && !PP_ISMOD(pp))) 5795 hat_page_setattr(pp, rm); 5796 5797 /* 5798 * Are we done? If not, we must have a large mapping. 5799 * For large mappings we need to sync the rest of the pages 5800 * covered by this tte; goto the next page. 5801 */ 5802 } while (--npgs > 0 && (pp = PP_PAGENEXT(pp))); 5803 } 5804 5805 /* 5806 * Execute pre-callback handler of each pa_hment linked to pp 5807 * 5808 * Inputs: 5809 * flag: either HAT_PRESUSPEND or HAT_SUSPEND. 5810 * capture_cpus: pointer to return value (below) 5811 * 5812 * Returns: 5813 * Propagates the subsystem callback return values back to the caller; 5814 * returns 0 on success. If capture_cpus is non-NULL, the value returned 5815 * is zero if all of the pa_hments are of a type that do not require 5816 * capturing CPUs prior to suspending the mapping, else it is 1. 5817 */ 5818 static int 5819 hat_pageprocess_precallbacks(struct page *pp, uint_t flag, int *capture_cpus) 5820 { 5821 struct sf_hment *sfhmep; 5822 struct pa_hment *pahmep; 5823 int (*f)(caddr_t, uint_t, uint_t, void *); 5824 int ret; 5825 id_t id; 5826 int locked = 0; 5827 kmutex_t *pml; 5828 5829 ASSERT(PAGE_EXCL(pp)); 5830 if (!sfmmu_mlist_held(pp)) { 5831 pml = sfmmu_mlist_enter(pp); 5832 locked = 1; 5833 } 5834 5835 if (capture_cpus) 5836 *capture_cpus = 0; 5837 5838 top: 5839 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5840 /* 5841 * skip sf_hments corresponding to VA<->PA mappings; 5842 * for pa_hment's, hme_tte.ll is zero 5843 */ 5844 if (!IS_PAHME(sfhmep)) 5845 continue; 5846 5847 pahmep = sfhmep->hme_data; 5848 ASSERT(pahmep != NULL); 5849 5850 /* 5851 * skip if pre-handler has been called earlier in this loop 5852 */ 5853 if (pahmep->flags & flag) 5854 continue; 5855 5856 id = pahmep->cb_id; 5857 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5858 if (capture_cpus && sfmmu_cb_table[id].capture_cpus != 0) 5859 *capture_cpus = 1; 5860 if ((f = sfmmu_cb_table[id].prehandler) == NULL) { 5861 pahmep->flags |= flag; 5862 continue; 5863 } 5864 5865 /* 5866 * Drop the mapping list lock to avoid locking order issues. 5867 */ 5868 if (locked) 5869 sfmmu_mlist_exit(pml); 5870 5871 ret = f(pahmep->addr, pahmep->len, flag, pahmep->pvt); 5872 if (ret != 0) 5873 return (ret); /* caller must do the cleanup */ 5874 5875 if (locked) { 5876 pml = sfmmu_mlist_enter(pp); 5877 pahmep->flags |= flag; 5878 goto top; 5879 } 5880 5881 pahmep->flags |= flag; 5882 } 5883 5884 if (locked) 5885 sfmmu_mlist_exit(pml); 5886 5887 return (0); 5888 } 5889 5890 /* 5891 * Execute post-callback handler of each pa_hment linked to pp 5892 * 5893 * Same overall assumptions and restrictions apply as for 5894 * hat_pageprocess_precallbacks(). 5895 */ 5896 static void 5897 hat_pageprocess_postcallbacks(struct page *pp, uint_t flag) 5898 { 5899 pfn_t pgpfn = pp->p_pagenum; 5900 pfn_t pgmask = btop(page_get_pagesize(pp->p_szc)) - 1; 5901 pfn_t newpfn; 5902 struct sf_hment *sfhmep; 5903 struct pa_hment *pahmep; 5904 int (*f)(caddr_t, uint_t, uint_t, void *, pfn_t); 5905 id_t id; 5906 int locked = 0; 5907 kmutex_t *pml; 5908 5909 ASSERT(PAGE_EXCL(pp)); 5910 if (!sfmmu_mlist_held(pp)) { 5911 pml = sfmmu_mlist_enter(pp); 5912 locked = 1; 5913 } 5914 5915 top: 5916 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5917 /* 5918 * skip sf_hments corresponding to VA<->PA mappings; 5919 * for pa_hment's, hme_tte.ll is zero 5920 */ 5921 if (!IS_PAHME(sfhmep)) 5922 continue; 5923 5924 pahmep = sfhmep->hme_data; 5925 ASSERT(pahmep != NULL); 5926 5927 if ((pahmep->flags & flag) == 0) 5928 continue; 5929 5930 pahmep->flags &= ~flag; 5931 5932 id = pahmep->cb_id; 5933 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5934 if ((f = sfmmu_cb_table[id].posthandler) == NULL) 5935 continue; 5936 5937 /* 5938 * Convert the base page PFN into the constituent PFN 5939 * which is needed by the callback handler. 5940 */ 5941 newpfn = pgpfn | (btop((uintptr_t)pahmep->addr) & pgmask); 5942 5943 /* 5944 * Drop the mapping list lock to avoid locking order issues. 5945 */ 5946 if (locked) 5947 sfmmu_mlist_exit(pml); 5948 5949 if (f(pahmep->addr, pahmep->len, flag, pahmep->pvt, newpfn) 5950 != 0) 5951 panic("sfmmu: posthandler failed"); 5952 5953 if (locked) { 5954 pml = sfmmu_mlist_enter(pp); 5955 goto top; 5956 } 5957 } 5958 5959 if (locked) 5960 sfmmu_mlist_exit(pml); 5961 } 5962 5963 /* 5964 * Suspend locked kernel mapping 5965 */ 5966 void 5967 hat_pagesuspend(struct page *pp) 5968 { 5969 struct sf_hment *sfhmep; 5970 sfmmu_t *sfmmup; 5971 tte_t tte, ttemod; 5972 struct hme_blk *hmeblkp; 5973 caddr_t addr; 5974 int index, cons; 5975 cpuset_t cpuset; 5976 5977 ASSERT(PAGE_EXCL(pp)); 5978 ASSERT(sfmmu_mlist_held(pp)); 5979 5980 mutex_enter(&kpr_suspendlock); 5981 5982 /* 5983 * Call into dtrace to tell it we're about to suspend a 5984 * kernel mapping. This prevents us from running into issues 5985 * with probe context trying to touch a suspended page 5986 * in the relocation codepath itself. 5987 */ 5988 if (dtrace_kreloc_init) 5989 (*dtrace_kreloc_init)(); 5990 5991 index = PP_MAPINDEX(pp); 5992 cons = TTE8K; 5993 5994 retry: 5995 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5996 5997 if (IS_PAHME(sfhmep)) 5998 continue; 5999 6000 if (get_hblk_ttesz(sfmmu_hmetohblk(sfhmep)) != cons) 6001 continue; 6002 6003 /* 6004 * Loop until we successfully set the suspend bit in 6005 * the TTE. 6006 */ 6007 again: 6008 sfmmu_copytte(&sfhmep->hme_tte, &tte); 6009 ASSERT(TTE_IS_VALID(&tte)); 6010 6011 ttemod = tte; 6012 TTE_SET_SUSPEND(&ttemod); 6013 if (sfmmu_modifytte_try(&tte, &ttemod, 6014 &sfhmep->hme_tte) < 0) 6015 goto again; 6016 6017 /* 6018 * Invalidate TSB entry 6019 */ 6020 hmeblkp = sfmmu_hmetohblk(sfhmep); 6021 6022 sfmmup = hblktosfmmu(hmeblkp); 6023 ASSERT(sfmmup == ksfmmup); 6024 6025 addr = tte_to_vaddr(hmeblkp, tte); 6026 6027 /* 6028 * No need to make sure that the TSB for this sfmmu is 6029 * not being relocated since it is ksfmmup and thus it 6030 * will never be relocated. 6031 */ 6032 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 6033 6034 /* 6035 * Update xcall stats 6036 */ 6037 cpuset = cpu_ready_set; 6038 CPUSET_DEL(cpuset, CPU->cpu_id); 6039 6040 /* LINTED: constant in conditional context */ 6041 SFMMU_XCALL_STATS(ksfmmup); 6042 6043 /* 6044 * Flush TLB entry on remote CPU's 6045 */ 6046 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 6047 (uint64_t)ksfmmup); 6048 xt_sync(cpuset); 6049 6050 /* 6051 * Flush TLB entry on local CPU 6052 */ 6053 vtag_flushpage(addr, (uint64_t)ksfmmup); 6054 } 6055 6056 while (index != 0) { 6057 index = index >> 1; 6058 if (index != 0) 6059 cons++; 6060 if (index & 0x1) { 6061 pp = PP_GROUPLEADER(pp, cons); 6062 goto retry; 6063 } 6064 } 6065 } 6066 6067 #ifdef DEBUG 6068 6069 #define N_PRLE 1024 6070 struct prle { 6071 page_t *targ; 6072 page_t *repl; 6073 int status; 6074 int pausecpus; 6075 hrtime_t whence; 6076 }; 6077 6078 static struct prle page_relocate_log[N_PRLE]; 6079 static int prl_entry; 6080 static kmutex_t prl_mutex; 6081 6082 #define PAGE_RELOCATE_LOG(t, r, s, p) \ 6083 mutex_enter(&prl_mutex); \ 6084 page_relocate_log[prl_entry].targ = *(t); \ 6085 page_relocate_log[prl_entry].repl = *(r); \ 6086 page_relocate_log[prl_entry].status = (s); \ 6087 page_relocate_log[prl_entry].pausecpus = (p); \ 6088 page_relocate_log[prl_entry].whence = gethrtime(); \ 6089 prl_entry = (prl_entry == (N_PRLE - 1))? 0 : prl_entry + 1; \ 6090 mutex_exit(&prl_mutex); 6091 6092 #else /* !DEBUG */ 6093 #define PAGE_RELOCATE_LOG(t, r, s, p) 6094 #endif 6095 6096 /* 6097 * Core Kernel Page Relocation Algorithm 6098 * 6099 * Input: 6100 * 6101 * target : constituent pages are SE_EXCL locked. 6102 * replacement: constituent pages are SE_EXCL locked. 6103 * 6104 * Output: 6105 * 6106 * nrelocp: number of pages relocated 6107 */ 6108 int 6109 hat_page_relocate(page_t **target, page_t **replacement, spgcnt_t *nrelocp) 6110 { 6111 page_t *targ, *repl; 6112 page_t *tpp, *rpp; 6113 kmutex_t *low, *high; 6114 spgcnt_t npages, i; 6115 page_t *pl = NULL; 6116 int old_pil; 6117 cpuset_t cpuset; 6118 int cap_cpus; 6119 int ret; 6120 6121 if (hat_kpr_enabled == 0 || !kcage_on || PP_ISNORELOC(*target)) { 6122 PAGE_RELOCATE_LOG(target, replacement, EAGAIN, -1); 6123 return (EAGAIN); 6124 } 6125 6126 mutex_enter(&kpr_mutex); 6127 kreloc_thread = curthread; 6128 6129 targ = *target; 6130 repl = *replacement; 6131 ASSERT(repl != NULL); 6132 ASSERT(targ->p_szc == repl->p_szc); 6133 6134 npages = page_get_pagecnt(targ->p_szc); 6135 6136 /* 6137 * unload VA<->PA mappings that are not locked 6138 */ 6139 tpp = targ; 6140 for (i = 0; i < npages; i++) { 6141 (void) hat_pageunload(tpp, SFMMU_KERNEL_RELOC); 6142 tpp++; 6143 } 6144 6145 /* 6146 * Do "presuspend" callbacks, in a context from which we can still 6147 * block as needed. Note that we don't hold the mapping list lock 6148 * of "targ" at this point due to potential locking order issues; 6149 * we assume that between the hat_pageunload() above and holding 6150 * the SE_EXCL lock that the mapping list *cannot* change at this 6151 * point. 6152 */ 6153 ret = hat_pageprocess_precallbacks(targ, HAT_PRESUSPEND, &cap_cpus); 6154 if (ret != 0) { 6155 /* 6156 * EIO translates to fatal error, for all others cleanup 6157 * and return EAGAIN. 6158 */ 6159 ASSERT(ret != EIO); 6160 hat_pageprocess_postcallbacks(targ, HAT_POSTUNSUSPEND); 6161 PAGE_RELOCATE_LOG(target, replacement, ret, -1); 6162 kreloc_thread = NULL; 6163 mutex_exit(&kpr_mutex); 6164 return (EAGAIN); 6165 } 6166 6167 /* 6168 * acquire p_mapping list lock for both the target and replacement 6169 * root pages. 6170 * 6171 * low and high refer to the need to grab the mlist locks in a 6172 * specific order in order to prevent race conditions. Thus the 6173 * lower lock must be grabbed before the higher lock. 6174 * 6175 * This will block hat_unload's accessing p_mapping list. Since 6176 * we have SE_EXCL lock, hat_memload and hat_pageunload will be 6177 * blocked. Thus, no one else will be accessing the p_mapping list 6178 * while we suspend and reload the locked mapping below. 6179 */ 6180 tpp = targ; 6181 rpp = repl; 6182 sfmmu_mlist_reloc_enter(tpp, rpp, &low, &high); 6183 6184 kpreempt_disable(); 6185 6186 #ifdef VAC 6187 /* 6188 * If the replacement page is of a different virtual color 6189 * than the page it is replacing, we need to handle the VAC 6190 * consistency for it just as we would if we were setting up 6191 * a new mapping to a page. 6192 */ 6193 if ((tpp->p_szc == 0) && (PP_GET_VCOLOR(rpp) != NO_VCOLOR)) { 6194 if (tpp->p_vcolor != rpp->p_vcolor) { 6195 sfmmu_cache_flushcolor(PP_GET_VCOLOR(rpp), 6196 rpp->p_pagenum); 6197 } 6198 } 6199 #endif 6200 6201 /* 6202 * We raise our PIL to 13 so that we don't get captured by 6203 * another CPU or pinned by an interrupt thread. We can't go to 6204 * PIL 14 since the nexus driver(s) may need to interrupt at 6205 * that level in the case of IOMMU pseudo mappings. 6206 */ 6207 cpuset = cpu_ready_set; 6208 CPUSET_DEL(cpuset, CPU->cpu_id); 6209 if (!cap_cpus || CPUSET_ISNULL(cpuset)) { 6210 old_pil = splr(XCALL_PIL); 6211 } else { 6212 old_pil = -1; 6213 xc_attention(cpuset); 6214 } 6215 ASSERT(getpil() == XCALL_PIL); 6216 6217 /* 6218 * Now do suspend callbacks. In the case of an IOMMU mapping 6219 * this will suspend all DMA activity to the page while it is 6220 * being relocated. Since we are well above LOCK_LEVEL and CPUs 6221 * may be captured at this point we should have acquired any needed 6222 * locks in the presuspend callback. 6223 */ 6224 ret = hat_pageprocess_precallbacks(targ, HAT_SUSPEND, NULL); 6225 if (ret != 0) { 6226 repl = targ; 6227 goto suspend_fail; 6228 } 6229 6230 /* 6231 * Raise the PIL yet again, this time to block all high-level 6232 * interrupts on this CPU. This is necessary to prevent an 6233 * interrupt routine from pinning the thread which holds the 6234 * mapping suspended and then touching the suspended page. 6235 * 6236 * Once the page is suspended we also need to be careful to 6237 * avoid calling any functions which touch any seg_kmem memory 6238 * since that memory may be backed by the very page we are 6239 * relocating in here! 6240 */ 6241 hat_pagesuspend(targ); 6242 6243 /* 6244 * Now that we are confident everybody has stopped using this page, 6245 * copy the page contents. Note we use a physical copy to prevent 6246 * locking issues and to avoid fpRAS because we can't handle it in 6247 * this context. 6248 */ 6249 for (i = 0; i < npages; i++, tpp++, rpp++) { 6250 /* 6251 * Copy the contents of the page. 6252 */ 6253 ppcopy_kernel(tpp, rpp); 6254 } 6255 6256 tpp = targ; 6257 rpp = repl; 6258 for (i = 0; i < npages; i++, tpp++, rpp++) { 6259 /* 6260 * Copy attributes. VAC consistency was handled above, 6261 * if required. 6262 */ 6263 rpp->p_nrm = tpp->p_nrm; 6264 tpp->p_nrm = 0; 6265 rpp->p_index = tpp->p_index; 6266 tpp->p_index = 0; 6267 #ifdef VAC 6268 rpp->p_vcolor = tpp->p_vcolor; 6269 #endif 6270 } 6271 6272 /* 6273 * First, unsuspend the page, if we set the suspend bit, and transfer 6274 * the mapping list from the target page to the replacement page. 6275 * Next process postcallbacks; since pa_hment's are linked only to the 6276 * p_mapping list of root page, we don't iterate over the constituent 6277 * pages. 6278 */ 6279 hat_pagereload(targ, repl); 6280 6281 suspend_fail: 6282 hat_pageprocess_postcallbacks(repl, HAT_UNSUSPEND); 6283 6284 /* 6285 * Now lower our PIL and release any captured CPUs since we 6286 * are out of the "danger zone". After this it will again be 6287 * safe to acquire adaptive mutex locks, or to drop them... 6288 */ 6289 if (old_pil != -1) { 6290 splx(old_pil); 6291 } else { 6292 xc_dismissed(cpuset); 6293 } 6294 6295 kpreempt_enable(); 6296 6297 sfmmu_mlist_reloc_exit(low, high); 6298 6299 /* 6300 * Postsuspend callbacks should drop any locks held across 6301 * the suspend callbacks. As before, we don't hold the mapping 6302 * list lock at this point.. our assumption is that the mapping 6303 * list still can't change due to our holding SE_EXCL lock and 6304 * there being no unlocked mappings left. Hence the restriction 6305 * on calling context to hat_delete_callback() 6306 */ 6307 hat_pageprocess_postcallbacks(repl, HAT_POSTUNSUSPEND); 6308 if (ret != 0) { 6309 /* 6310 * The second presuspend call failed: we got here through 6311 * the suspend_fail label above. 6312 */ 6313 ASSERT(ret != EIO); 6314 PAGE_RELOCATE_LOG(target, replacement, ret, cap_cpus); 6315 kreloc_thread = NULL; 6316 mutex_exit(&kpr_mutex); 6317 return (EAGAIN); 6318 } 6319 6320 /* 6321 * Now that we're out of the performance critical section we can 6322 * take care of updating the hash table, since we still 6323 * hold all the pages locked SE_EXCL at this point we 6324 * needn't worry about things changing out from under us. 6325 */ 6326 tpp = targ; 6327 rpp = repl; 6328 for (i = 0; i < npages; i++, tpp++, rpp++) { 6329 6330 /* 6331 * replace targ with replacement in page_hash table 6332 */ 6333 targ = tpp; 6334 page_relocate_hash(rpp, targ); 6335 6336 /* 6337 * concatenate target; caller of platform_page_relocate() 6338 * expects target to be concatenated after returning. 6339 */ 6340 ASSERT(targ->p_next == targ); 6341 ASSERT(targ->p_prev == targ); 6342 page_list_concat(&pl, &targ); 6343 } 6344 6345 ASSERT(*target == pl); 6346 *nrelocp = npages; 6347 PAGE_RELOCATE_LOG(target, replacement, 0, cap_cpus); 6348 kreloc_thread = NULL; 6349 mutex_exit(&kpr_mutex); 6350 return (0); 6351 } 6352 6353 /* 6354 * Called when stray pa_hments are found attached to a page which is 6355 * being freed. Notify the subsystem which attached the pa_hment of 6356 * the error if it registered a suitable handler, else panic. 6357 */ 6358 static void 6359 sfmmu_pahment_leaked(struct pa_hment *pahmep) 6360 { 6361 id_t cb_id = pahmep->cb_id; 6362 6363 ASSERT(cb_id >= (id_t)0 && cb_id < sfmmu_cb_nextid); 6364 if (sfmmu_cb_table[cb_id].errhandler != NULL) { 6365 if (sfmmu_cb_table[cb_id].errhandler(pahmep->addr, pahmep->len, 6366 HAT_CB_ERR_LEAKED, pahmep->pvt) == 0) 6367 return; /* non-fatal */ 6368 } 6369 panic("pa_hment leaked: 0x%p", pahmep); 6370 } 6371 6372 /* 6373 * Remove all mappings to page 'pp'. 6374 */ 6375 int 6376 hat_pageunload(struct page *pp, uint_t forceflag) 6377 { 6378 struct page *origpp = pp; 6379 struct sf_hment *sfhme, *tmphme; 6380 struct hme_blk *hmeblkp; 6381 kmutex_t *pml; 6382 #ifdef VAC 6383 kmutex_t *pmtx; 6384 #endif 6385 cpuset_t cpuset, tset; 6386 int index, cons; 6387 int xhme_blks; 6388 int pa_hments; 6389 6390 ASSERT(PAGE_EXCL(pp)); 6391 6392 retry_xhat: 6393 tmphme = NULL; 6394 xhme_blks = 0; 6395 pa_hments = 0; 6396 CPUSET_ZERO(cpuset); 6397 6398 pml = sfmmu_mlist_enter(pp); 6399 6400 #ifdef VAC 6401 if (pp->p_kpmref) 6402 sfmmu_kpm_pageunload(pp); 6403 ASSERT(!PP_ISMAPPED_KPM(pp)); 6404 #endif 6405 6406 index = PP_MAPINDEX(pp); 6407 cons = TTE8K; 6408 retry: 6409 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6410 tmphme = sfhme->hme_next; 6411 6412 if (IS_PAHME(sfhme)) { 6413 ASSERT(sfhme->hme_data != NULL); 6414 pa_hments++; 6415 continue; 6416 } 6417 6418 hmeblkp = sfmmu_hmetohblk(sfhme); 6419 if (hmeblkp->hblk_xhat_bit) { 6420 struct xhat_hme_blk *xblk = 6421 (struct xhat_hme_blk *)hmeblkp; 6422 6423 (void) XHAT_PAGEUNLOAD(xblk->xhat_hme_blk_hat, 6424 pp, forceflag, XBLK2PROVBLK(xblk)); 6425 6426 xhme_blks = 1; 6427 continue; 6428 } 6429 6430 /* 6431 * If there are kernel mappings don't unload them, they will 6432 * be suspended. 6433 */ 6434 if (forceflag == SFMMU_KERNEL_RELOC && hmeblkp->hblk_lckcnt && 6435 hmeblkp->hblk_tag.htag_id == ksfmmup) 6436 continue; 6437 6438 tset = sfmmu_pageunload(pp, sfhme, cons); 6439 CPUSET_OR(cpuset, tset); 6440 } 6441 6442 while (index != 0) { 6443 index = index >> 1; 6444 if (index != 0) 6445 cons++; 6446 if (index & 0x1) { 6447 /* Go to leading page */ 6448 pp = PP_GROUPLEADER(pp, cons); 6449 ASSERT(sfmmu_mlist_held(pp)); 6450 goto retry; 6451 } 6452 } 6453 6454 /* 6455 * cpuset may be empty if the page was only mapped by segkpm, 6456 * in which case we won't actually cross-trap. 6457 */ 6458 xt_sync(cpuset); 6459 6460 /* 6461 * The page should have no mappings at this point, unless 6462 * we were called from hat_page_relocate() in which case we 6463 * leave the locked mappings which will be suspended later. 6464 */ 6465 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks || pa_hments || 6466 (forceflag == SFMMU_KERNEL_RELOC)); 6467 6468 #ifdef VAC 6469 if (PP_ISTNC(pp)) { 6470 if (cons == TTE8K) { 6471 pmtx = sfmmu_page_enter(pp); 6472 PP_CLRTNC(pp); 6473 sfmmu_page_exit(pmtx); 6474 } else { 6475 conv_tnc(pp, cons); 6476 } 6477 } 6478 #endif /* VAC */ 6479 6480 if (pa_hments && forceflag != SFMMU_KERNEL_RELOC) { 6481 /* 6482 * Unlink any pa_hments and free them, calling back 6483 * the responsible subsystem to notify it of the error. 6484 * This can occur in situations such as drivers leaking 6485 * DMA handles: naughty, but common enough that we'd like 6486 * to keep the system running rather than bringing it 6487 * down with an obscure error like "pa_hment leaked" 6488 * which doesn't aid the user in debugging their driver. 6489 */ 6490 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6491 tmphme = sfhme->hme_next; 6492 if (IS_PAHME(sfhme)) { 6493 struct pa_hment *pahmep = sfhme->hme_data; 6494 sfmmu_pahment_leaked(pahmep); 6495 HME_SUB(sfhme, pp); 6496 kmem_cache_free(pa_hment_cache, pahmep); 6497 } 6498 } 6499 6500 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks); 6501 } 6502 6503 sfmmu_mlist_exit(pml); 6504 6505 /* 6506 * XHAT may not have finished unloading pages 6507 * because some other thread was waiting for 6508 * mlist lock and XHAT_PAGEUNLOAD let it do 6509 * the job. 6510 */ 6511 if (xhme_blks) { 6512 pp = origpp; 6513 goto retry_xhat; 6514 } 6515 6516 return (0); 6517 } 6518 6519 cpuset_t 6520 sfmmu_pageunload(page_t *pp, struct sf_hment *sfhme, int cons) 6521 { 6522 struct hme_blk *hmeblkp; 6523 sfmmu_t *sfmmup; 6524 tte_t tte, ttemod; 6525 #ifdef DEBUG 6526 tte_t orig_old; 6527 #endif /* DEBUG */ 6528 caddr_t addr; 6529 int ttesz; 6530 int ret; 6531 cpuset_t cpuset; 6532 6533 ASSERT(pp != NULL); 6534 ASSERT(sfmmu_mlist_held(pp)); 6535 ASSERT(!PP_ISKAS(pp)); 6536 6537 CPUSET_ZERO(cpuset); 6538 6539 hmeblkp = sfmmu_hmetohblk(sfhme); 6540 6541 readtte: 6542 sfmmu_copytte(&sfhme->hme_tte, &tte); 6543 if (TTE_IS_VALID(&tte)) { 6544 sfmmup = hblktosfmmu(hmeblkp); 6545 ttesz = get_hblk_ttesz(hmeblkp); 6546 /* 6547 * Only unload mappings of 'cons' size. 6548 */ 6549 if (ttesz != cons) 6550 return (cpuset); 6551 6552 /* 6553 * Note that we have p_mapping lock, but no hash lock here. 6554 * hblk_unload() has to have both hash lock AND p_mapping 6555 * lock before it tries to modify tte. So, the tte could 6556 * not become invalid in the sfmmu_modifytte_try() below. 6557 */ 6558 ttemod = tte; 6559 #ifdef DEBUG 6560 orig_old = tte; 6561 #endif /* DEBUG */ 6562 6563 TTE_SET_INVALID(&ttemod); 6564 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6565 if (ret < 0) { 6566 #ifdef DEBUG 6567 /* only R/M bits can change. */ 6568 chk_tte(&orig_old, &tte, &ttemod, hmeblkp); 6569 #endif /* DEBUG */ 6570 goto readtte; 6571 } 6572 6573 if (ret == 0) { 6574 panic("pageunload: cas failed?"); 6575 } 6576 6577 addr = tte_to_vaddr(hmeblkp, tte); 6578 6579 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6580 6581 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -1); 6582 6583 /* 6584 * We need to flush the page from the virtual cache 6585 * in order to prevent a virtual cache alias 6586 * inconsistency. The particular scenario we need 6587 * to worry about is: 6588 * Given: va1 and va2 are two virtual address that 6589 * alias and will map the same physical address. 6590 * 1. mapping exists from va1 to pa and data has 6591 * been read into the cache. 6592 * 2. unload va1. 6593 * 3. load va2 and modify data using va2. 6594 * 4 unload va2. 6595 * 5. load va1 and reference data. Unless we flush 6596 * the data cache when we unload we will get 6597 * stale data. 6598 * This scenario is taken care of by using virtual 6599 * page coloring. 6600 */ 6601 if (sfmmup->sfmmu_ismhat) { 6602 /* 6603 * Flush TSBs, TLBs and caches 6604 * of every process 6605 * sharing this ism segment. 6606 */ 6607 sfmmu_hat_lock_all(); 6608 mutex_enter(&ism_mlist_lock); 6609 kpreempt_disable(); 6610 if (do_virtual_coloring) 6611 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6612 pp->p_pagenum, CACHE_NO_FLUSH); 6613 else 6614 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6615 pp->p_pagenum, CACHE_FLUSH); 6616 kpreempt_enable(); 6617 mutex_exit(&ism_mlist_lock); 6618 sfmmu_hat_unlock_all(); 6619 cpuset = cpu_ready_set; 6620 } else if (do_virtual_coloring) { 6621 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6622 cpuset = sfmmup->sfmmu_cpusran; 6623 } else { 6624 sfmmu_tlbcache_demap(addr, sfmmup, hmeblkp, 6625 pp->p_pagenum, 0, FLUSH_NECESSARY_CPUS, 6626 CACHE_FLUSH, 0); 6627 cpuset = sfmmup->sfmmu_cpusran; 6628 } 6629 6630 /* 6631 * Hme_sub has to run after ttesync() and a_rss update. 6632 * See hblk_unload(). 6633 */ 6634 HME_SUB(sfhme, pp); 6635 membar_stst(); 6636 6637 /* 6638 * We can not make ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 6639 * since pteload may have done a HME_ADD() right after 6640 * we did the HME_SUB() above. Hmecnt is now maintained 6641 * by cas only. no lock guranteed its value. The only 6642 * gurantee we have is the hmecnt should not be less than 6643 * what it should be so the hblk will not be taken away. 6644 * It's also important that we decremented the hmecnt after 6645 * we are done with hmeblkp so that this hmeblk won't be 6646 * stolen. 6647 */ 6648 ASSERT(hmeblkp->hblk_hmecnt > 0); 6649 ASSERT(hmeblkp->hblk_vcnt > 0); 6650 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 6651 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 6652 /* 6653 * This is bug 4063182. 6654 * XXX: fixme 6655 * ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 6656 * !hmeblkp->hblk_lckcnt); 6657 */ 6658 } else { 6659 panic("invalid tte? pp %p &tte %p", 6660 (void *)pp, (void *)&tte); 6661 } 6662 6663 return (cpuset); 6664 } 6665 6666 /* 6667 * While relocating a kernel page, this function will move the mappings 6668 * from tpp to dpp and modify any associated data with these mappings. 6669 * It also unsuspends the suspended kernel mapping. 6670 */ 6671 static void 6672 hat_pagereload(struct page *tpp, struct page *dpp) 6673 { 6674 struct sf_hment *sfhme; 6675 tte_t tte, ttemod; 6676 int index, cons; 6677 6678 ASSERT(getpil() == PIL_MAX); 6679 ASSERT(sfmmu_mlist_held(tpp)); 6680 ASSERT(sfmmu_mlist_held(dpp)); 6681 6682 index = PP_MAPINDEX(tpp); 6683 cons = TTE8K; 6684 6685 /* Update real mappings to the page */ 6686 retry: 6687 for (sfhme = tpp->p_mapping; sfhme != NULL; sfhme = sfhme->hme_next) { 6688 if (IS_PAHME(sfhme)) 6689 continue; 6690 sfmmu_copytte(&sfhme->hme_tte, &tte); 6691 ttemod = tte; 6692 6693 /* 6694 * replace old pfn with new pfn in TTE 6695 */ 6696 PFN_TO_TTE(ttemod, dpp->p_pagenum); 6697 6698 /* 6699 * clear suspend bit 6700 */ 6701 ASSERT(TTE_IS_SUSPEND(&ttemod)); 6702 TTE_CLR_SUSPEND(&ttemod); 6703 6704 if (sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte) < 0) 6705 panic("hat_pagereload(): sfmmu_modifytte_try() failed"); 6706 6707 /* 6708 * set hme_page point to new page 6709 */ 6710 sfhme->hme_page = dpp; 6711 } 6712 6713 /* 6714 * move p_mapping list from old page to new page 6715 */ 6716 dpp->p_mapping = tpp->p_mapping; 6717 tpp->p_mapping = NULL; 6718 dpp->p_share = tpp->p_share; 6719 tpp->p_share = 0; 6720 6721 while (index != 0) { 6722 index = index >> 1; 6723 if (index != 0) 6724 cons++; 6725 if (index & 0x1) { 6726 tpp = PP_GROUPLEADER(tpp, cons); 6727 dpp = PP_GROUPLEADER(dpp, cons); 6728 goto retry; 6729 } 6730 } 6731 6732 if (dtrace_kreloc_fini) 6733 (*dtrace_kreloc_fini)(); 6734 mutex_exit(&kpr_suspendlock); 6735 } 6736 6737 uint_t 6738 hat_pagesync(struct page *pp, uint_t clearflag) 6739 { 6740 struct sf_hment *sfhme, *tmphme = NULL; 6741 struct hme_blk *hmeblkp; 6742 kmutex_t *pml; 6743 cpuset_t cpuset, tset; 6744 int index, cons; 6745 extern ulong_t po_share; 6746 page_t *save_pp = pp; 6747 6748 CPUSET_ZERO(cpuset); 6749 6750 if (PP_ISRO(pp) && (clearflag & HAT_SYNC_STOPON_MOD)) { 6751 return (PP_GENERIC_ATTR(pp)); 6752 } 6753 6754 if ((clearflag == (HAT_SYNC_STOPON_REF | HAT_SYNC_DONTZERO)) && 6755 PP_ISREF(pp)) { 6756 return (PP_GENERIC_ATTR(pp)); 6757 } 6758 6759 if ((clearflag == (HAT_SYNC_STOPON_MOD | HAT_SYNC_DONTZERO)) && 6760 PP_ISMOD(pp)) { 6761 return (PP_GENERIC_ATTR(pp)); 6762 } 6763 6764 if ((clearflag & HAT_SYNC_STOPON_SHARED) != 0 && 6765 (pp->p_share > po_share) && 6766 !(clearflag & HAT_SYNC_ZERORM)) { 6767 if (PP_ISRO(pp)) 6768 hat_page_setattr(pp, P_REF); 6769 return (PP_GENERIC_ATTR(pp)); 6770 } 6771 6772 clearflag &= ~HAT_SYNC_STOPON_SHARED; 6773 pml = sfmmu_mlist_enter(pp); 6774 index = PP_MAPINDEX(pp); 6775 cons = TTE8K; 6776 retry: 6777 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6778 /* 6779 * We need to save the next hment on the list since 6780 * it is possible for pagesync to remove an invalid hment 6781 * from the list. 6782 */ 6783 tmphme = sfhme->hme_next; 6784 /* 6785 * If we are looking for large mappings and this hme doesn't 6786 * reach the range we are seeking, just ignore its. 6787 */ 6788 hmeblkp = sfmmu_hmetohblk(sfhme); 6789 if (hmeblkp->hblk_xhat_bit) 6790 continue; 6791 6792 if (hme_size(sfhme) < cons) 6793 continue; 6794 tset = sfmmu_pagesync(pp, sfhme, 6795 clearflag & ~HAT_SYNC_STOPON_RM); 6796 CPUSET_OR(cpuset, tset); 6797 /* 6798 * If clearflag is HAT_SYNC_DONTZERO, break out as soon 6799 * as the "ref" or "mod" is set. 6800 */ 6801 if ((clearflag & ~HAT_SYNC_STOPON_RM) == HAT_SYNC_DONTZERO && 6802 ((clearflag & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp)) || 6803 ((clearflag & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp))) { 6804 index = 0; 6805 break; 6806 } 6807 } 6808 6809 while (index) { 6810 index = index >> 1; 6811 cons++; 6812 if (index & 0x1) { 6813 /* Go to leading page */ 6814 pp = PP_GROUPLEADER(pp, cons); 6815 goto retry; 6816 } 6817 } 6818 6819 xt_sync(cpuset); 6820 sfmmu_mlist_exit(pml); 6821 return (PP_GENERIC_ATTR(save_pp)); 6822 } 6823 6824 /* 6825 * Get all the hardware dependent attributes for a page struct 6826 */ 6827 static cpuset_t 6828 sfmmu_pagesync(struct page *pp, struct sf_hment *sfhme, 6829 uint_t clearflag) 6830 { 6831 caddr_t addr; 6832 tte_t tte, ttemod; 6833 struct hme_blk *hmeblkp; 6834 int ret; 6835 sfmmu_t *sfmmup; 6836 cpuset_t cpuset; 6837 6838 ASSERT(pp != NULL); 6839 ASSERT(sfmmu_mlist_held(pp)); 6840 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 6841 (clearflag == HAT_SYNC_ZERORM)); 6842 6843 SFMMU_STAT(sf_pagesync); 6844 6845 CPUSET_ZERO(cpuset); 6846 6847 sfmmu_pagesync_retry: 6848 6849 sfmmu_copytte(&sfhme->hme_tte, &tte); 6850 if (TTE_IS_VALID(&tte)) { 6851 hmeblkp = sfmmu_hmetohblk(sfhme); 6852 sfmmup = hblktosfmmu(hmeblkp); 6853 addr = tte_to_vaddr(hmeblkp, tte); 6854 if (clearflag == HAT_SYNC_ZERORM) { 6855 ttemod = tte; 6856 TTE_CLR_RM(&ttemod); 6857 ret = sfmmu_modifytte_try(&tte, &ttemod, 6858 &sfhme->hme_tte); 6859 if (ret < 0) { 6860 /* 6861 * cas failed and the new value is not what 6862 * we want. 6863 */ 6864 goto sfmmu_pagesync_retry; 6865 } 6866 6867 if (ret > 0) { 6868 /* we win the cas */ 6869 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6870 cpuset = sfmmup->sfmmu_cpusran; 6871 } 6872 } 6873 6874 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6875 } 6876 return (cpuset); 6877 } 6878 6879 /* 6880 * Remove write permission from a mappings to a page, so that 6881 * we can detect the next modification of it. This requires modifying 6882 * the TTE then invalidating (demap) any TLB entry using that TTE. 6883 * This code is similar to sfmmu_pagesync(). 6884 */ 6885 static cpuset_t 6886 sfmmu_pageclrwrt(struct page *pp, struct sf_hment *sfhme) 6887 { 6888 caddr_t addr; 6889 tte_t tte; 6890 tte_t ttemod; 6891 struct hme_blk *hmeblkp; 6892 int ret; 6893 sfmmu_t *sfmmup; 6894 cpuset_t cpuset; 6895 6896 ASSERT(pp != NULL); 6897 ASSERT(sfmmu_mlist_held(pp)); 6898 6899 CPUSET_ZERO(cpuset); 6900 SFMMU_STAT(sf_clrwrt); 6901 6902 retry: 6903 6904 sfmmu_copytte(&sfhme->hme_tte, &tte); 6905 if (TTE_IS_VALID(&tte) && TTE_IS_WRITABLE(&tte)) { 6906 hmeblkp = sfmmu_hmetohblk(sfhme); 6907 6908 /* 6909 * xhat mappings should never be to a VMODSORT page. 6910 */ 6911 ASSERT(hmeblkp->hblk_xhat_bit == 0); 6912 6913 sfmmup = hblktosfmmu(hmeblkp); 6914 addr = tte_to_vaddr(hmeblkp, tte); 6915 6916 ttemod = tte; 6917 TTE_CLR_WRT(&ttemod); 6918 TTE_CLR_MOD(&ttemod); 6919 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6920 6921 /* 6922 * if cas failed and the new value is not what 6923 * we want retry 6924 */ 6925 if (ret < 0) 6926 goto retry; 6927 6928 /* we win the cas */ 6929 if (ret > 0) { 6930 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6931 cpuset = sfmmup->sfmmu_cpusran; 6932 } 6933 } 6934 6935 return (cpuset); 6936 } 6937 6938 /* 6939 * Walk all mappings of a page, removing write permission and clearing the 6940 * ref/mod bits. This code is similar to hat_pagesync() 6941 */ 6942 static void 6943 hat_page_clrwrt(page_t *pp) 6944 { 6945 struct sf_hment *sfhme; 6946 struct sf_hment *tmphme = NULL; 6947 kmutex_t *pml; 6948 cpuset_t cpuset; 6949 cpuset_t tset; 6950 int index; 6951 int cons; 6952 6953 CPUSET_ZERO(cpuset); 6954 6955 pml = sfmmu_mlist_enter(pp); 6956 index = PP_MAPINDEX(pp); 6957 cons = TTE8K; 6958 retry: 6959 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6960 tmphme = sfhme->hme_next; 6961 6962 /* 6963 * If we are looking for large mappings and this hme doesn't 6964 * reach the range we are seeking, just ignore its. 6965 */ 6966 6967 if (hme_size(sfhme) < cons) 6968 continue; 6969 6970 tset = sfmmu_pageclrwrt(pp, sfhme); 6971 CPUSET_OR(cpuset, tset); 6972 } 6973 6974 while (index) { 6975 index = index >> 1; 6976 cons++; 6977 if (index & 0x1) { 6978 /* Go to leading page */ 6979 pp = PP_GROUPLEADER(pp, cons); 6980 goto retry; 6981 } 6982 } 6983 6984 xt_sync(cpuset); 6985 sfmmu_mlist_exit(pml); 6986 } 6987 6988 /* 6989 * Set the given REF/MOD/RO bits for the given page. 6990 * For a vnode with a sorted v_pages list, we need to change 6991 * the attributes and the v_pages list together under page_vnode_mutex. 6992 */ 6993 void 6994 hat_page_setattr(page_t *pp, uint_t flag) 6995 { 6996 vnode_t *vp = pp->p_vnode; 6997 page_t **listp; 6998 kmutex_t *pmtx; 6999 kmutex_t *vphm = NULL; 7000 7001 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7002 7003 /* 7004 * nothing to do if attribute already set 7005 */ 7006 if ((pp->p_nrm & flag) == flag) 7007 return; 7008 7009 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 7010 vphm = page_vnode_mutex(vp); 7011 mutex_enter(vphm); 7012 } 7013 7014 pmtx = sfmmu_page_enter(pp); 7015 pp->p_nrm |= flag; 7016 sfmmu_page_exit(pmtx); 7017 7018 if (vphm != NULL) { 7019 /* 7020 * Some File Systems examine v_pages for NULL w/o 7021 * grabbing the vphm mutex. Must not let it become NULL when 7022 * pp is the only page on the list. 7023 */ 7024 if (pp->p_vpnext != pp) { 7025 page_vpsub(&vp->v_pages, pp); 7026 if (vp->v_pages != NULL) 7027 listp = &vp->v_pages->p_vpprev->p_vpnext; 7028 else 7029 listp = &vp->v_pages; 7030 page_vpadd(listp, pp); 7031 } 7032 mutex_exit(vphm); 7033 } 7034 } 7035 7036 void 7037 hat_page_clrattr(page_t *pp, uint_t flag) 7038 { 7039 vnode_t *vp = pp->p_vnode; 7040 kmutex_t *pmtx; 7041 7042 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7043 7044 pmtx = sfmmu_page_enter(pp); 7045 7046 /* 7047 * Caller is expected to hold page's io lock for VMODSORT to work 7048 * correctly with pvn_vplist_dirty() and pvn_getdirty() when mod 7049 * bit is cleared. 7050 * We don't have assert to avoid tripping some existing third party 7051 * code. The dirty page is moved back to top of the v_page list 7052 * after IO is done in pvn_write_done(). 7053 */ 7054 pp->p_nrm &= ~flag; 7055 sfmmu_page_exit(pmtx); 7056 7057 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 7058 7059 /* 7060 * VMODSORT works by removing write permissions and getting 7061 * a fault when a page is made dirty. At this point 7062 * we need to remove write permission from all mappings 7063 * to this page. 7064 */ 7065 hat_page_clrwrt(pp); 7066 } 7067 } 7068 7069 uint_t 7070 hat_page_getattr(page_t *pp, uint_t flag) 7071 { 7072 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7073 return ((uint_t)(pp->p_nrm & flag)); 7074 } 7075 7076 /* 7077 * DEBUG kernels: verify that a kernel va<->pa translation 7078 * is safe by checking the underlying page_t is in a page 7079 * relocation-safe state. 7080 */ 7081 #ifdef DEBUG 7082 void 7083 sfmmu_check_kpfn(pfn_t pfn) 7084 { 7085 page_t *pp; 7086 int index, cons; 7087 7088 if (hat_check_vtop == 0) 7089 return; 7090 7091 if (hat_kpr_enabled == 0 || kvseg.s_base == NULL || panicstr) 7092 return; 7093 7094 pp = page_numtopp_nolock(pfn); 7095 if (!pp) 7096 return; 7097 7098 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7099 return; 7100 7101 /* 7102 * Handed a large kernel page, we dig up the root page since we 7103 * know the root page might have the lock also. 7104 */ 7105 if (pp->p_szc != 0) { 7106 index = PP_MAPINDEX(pp); 7107 cons = TTE8K; 7108 again: 7109 while (index != 0) { 7110 index >>= 1; 7111 if (index != 0) 7112 cons++; 7113 if (index & 0x1) { 7114 pp = PP_GROUPLEADER(pp, cons); 7115 goto again; 7116 } 7117 } 7118 } 7119 7120 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7121 return; 7122 7123 /* 7124 * Pages need to be locked or allocated "permanent" (either from 7125 * static_arena arena or explicitly setting PG_NORELOC when calling 7126 * page_create_va()) for VA->PA translations to be valid. 7127 */ 7128 if (!PP_ISNORELOC(pp)) 7129 panic("Illegal VA->PA translation, pp 0x%p not permanent", pp); 7130 else 7131 panic("Illegal VA->PA translation, pp 0x%p not locked", pp); 7132 } 7133 #endif /* DEBUG */ 7134 7135 /* 7136 * Returns a page frame number for a given virtual address. 7137 * Returns PFN_INVALID to indicate an invalid mapping 7138 */ 7139 pfn_t 7140 hat_getpfnum(struct hat *hat, caddr_t addr) 7141 { 7142 pfn_t pfn; 7143 tte_t tte; 7144 7145 /* 7146 * We would like to 7147 * ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 7148 * but we can't because the iommu driver will call this 7149 * routine at interrupt time and it can't grab the as lock 7150 * or it will deadlock: A thread could have the as lock 7151 * and be waiting for io. The io can't complete 7152 * because the interrupt thread is blocked trying to grab 7153 * the as lock. 7154 */ 7155 7156 ASSERT(hat->sfmmu_xhat_provider == NULL); 7157 7158 if (hat == ksfmmup) { 7159 if (IS_KMEM_VA_LARGEPAGE(addr)) { 7160 ASSERT(segkmem_lpszc > 0); 7161 pfn = sfmmu_kvaszc2pfn(addr, segkmem_lpszc); 7162 if (pfn != PFN_INVALID) { 7163 sfmmu_check_kpfn(pfn); 7164 return (pfn); 7165 } 7166 } else if (segkpm && IS_KPM_ADDR(addr)) { 7167 return (sfmmu_kpm_vatopfn(addr)); 7168 } 7169 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7170 == PFN_SUSPENDED) { 7171 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7172 } 7173 sfmmu_check_kpfn(pfn); 7174 return (pfn); 7175 } else { 7176 return (sfmmu_uvatopfn(addr, hat)); 7177 } 7178 } 7179 7180 /* 7181 * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged. 7182 * Use hat_getpfnum(kas.a_hat, ...) instead. 7183 * 7184 * We'd like to return PFN_INVALID if the mappings have underlying page_t's 7185 * but can't right now due to the fact that some software has grown to use 7186 * this interface incorrectly. So for now when the interface is misused, 7187 * return a warning to the user that in the future it won't work in the 7188 * way they're abusing it, and carry on (after disabling page relocation). 7189 */ 7190 pfn_t 7191 hat_getkpfnum(caddr_t addr) 7192 { 7193 pfn_t pfn; 7194 tte_t tte; 7195 int badcaller = 0; 7196 extern int segkmem_reloc; 7197 7198 if (segkpm && IS_KPM_ADDR(addr)) { 7199 badcaller = 1; 7200 pfn = sfmmu_kpm_vatopfn(addr); 7201 } else { 7202 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7203 == PFN_SUSPENDED) { 7204 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7205 } 7206 badcaller = pf_is_memory(pfn); 7207 } 7208 7209 if (badcaller) { 7210 /* 7211 * We can't return PFN_INVALID or the caller may panic 7212 * or corrupt the system. The only alternative is to 7213 * disable page relocation at this point for all kernel 7214 * memory. This will impact any callers of page_relocate() 7215 * such as FMA or DR. 7216 * 7217 * RFE: Add junk here to spit out an ereport so the sysadmin 7218 * can be advised that he should upgrade his device driver 7219 * so that this doesn't happen. 7220 */ 7221 hat_getkpfnum_badcall(caller()); 7222 if (hat_kpr_enabled && segkmem_reloc) { 7223 hat_kpr_enabled = 0; 7224 segkmem_reloc = 0; 7225 cmn_err(CE_WARN, "Kernel Page Relocation is DISABLED"); 7226 } 7227 } 7228 return (pfn); 7229 } 7230 7231 pfn_t 7232 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup) 7233 { 7234 struct hmehash_bucket *hmebp; 7235 hmeblk_tag hblktag; 7236 int hmeshift, hashno = 1; 7237 struct hme_blk *hmeblkp = NULL; 7238 7239 struct sf_hment *sfhmep; 7240 tte_t tte; 7241 pfn_t pfn; 7242 7243 /* support for ISM */ 7244 ism_map_t *ism_map; 7245 ism_blk_t *ism_blkp; 7246 int i; 7247 sfmmu_t *ism_hatid = NULL; 7248 sfmmu_t *locked_hatid = NULL; 7249 7250 7251 ASSERT(sfmmup != ksfmmup); 7252 SFMMU_STAT(sf_user_vtop); 7253 /* 7254 * Set ism_hatid if vaddr falls in a ISM segment. 7255 */ 7256 ism_blkp = sfmmup->sfmmu_iblk; 7257 if (ism_blkp) { 7258 sfmmu_ismhat_enter(sfmmup, 0); 7259 locked_hatid = sfmmup; 7260 } 7261 while (ism_blkp && ism_hatid == NULL) { 7262 ism_map = ism_blkp->iblk_maps; 7263 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 7264 if (vaddr >= ism_start(ism_map[i]) && 7265 vaddr < ism_end(ism_map[i])) { 7266 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 7267 vaddr = (caddr_t)(vaddr - 7268 ism_start(ism_map[i])); 7269 break; 7270 } 7271 } 7272 ism_blkp = ism_blkp->iblk_next; 7273 } 7274 if (locked_hatid) { 7275 sfmmu_ismhat_exit(locked_hatid, 0); 7276 } 7277 7278 hblktag.htag_id = sfmmup; 7279 do { 7280 hmeshift = HME_HASH_SHIFT(hashno); 7281 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 7282 hblktag.htag_rehash = hashno; 7283 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 7284 7285 SFMMU_HASH_LOCK(hmebp); 7286 7287 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 7288 if (hmeblkp != NULL) { 7289 HBLKTOHME(sfhmep, hmeblkp, vaddr); 7290 sfmmu_copytte(&sfhmep->hme_tte, &tte); 7291 if (TTE_IS_VALID(&tte)) { 7292 pfn = TTE_TO_PFN(vaddr, &tte); 7293 } else { 7294 pfn = PFN_INVALID; 7295 } 7296 SFMMU_HASH_UNLOCK(hmebp); 7297 return (pfn); 7298 } 7299 SFMMU_HASH_UNLOCK(hmebp); 7300 hashno++; 7301 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 7302 return (PFN_INVALID); 7303 } 7304 7305 7306 /* 7307 * For compatability with AT&T and later optimizations 7308 */ 7309 /* ARGSUSED */ 7310 void 7311 hat_map(struct hat *hat, caddr_t addr, size_t len, uint_t flags) 7312 { 7313 ASSERT(hat != NULL); 7314 ASSERT(hat->sfmmu_xhat_provider == NULL); 7315 } 7316 7317 /* 7318 * Return the number of mappings to a particular page. 7319 * This number is an approximation of the number of 7320 * number of people sharing the page. 7321 */ 7322 ulong_t 7323 hat_page_getshare(page_t *pp) 7324 { 7325 page_t *spp = pp; /* start page */ 7326 kmutex_t *pml; 7327 ulong_t cnt; 7328 int index, sz = TTE64K; 7329 7330 /* 7331 * We need to grab the mlist lock to make sure any outstanding 7332 * load/unloads complete. Otherwise we could return zero 7333 * even though the unload(s) hasn't finished yet. 7334 */ 7335 pml = sfmmu_mlist_enter(spp); 7336 cnt = spp->p_share; 7337 7338 #ifdef VAC 7339 if (kpm_enable) 7340 cnt += spp->p_kpmref; 7341 #endif 7342 7343 /* 7344 * If we have any large mappings, we count the number of 7345 * mappings that this large page is part of. 7346 */ 7347 index = PP_MAPINDEX(spp); 7348 index >>= 1; 7349 while (index) { 7350 pp = PP_GROUPLEADER(spp, sz); 7351 if ((index & 0x1) && pp != spp) { 7352 cnt += pp->p_share; 7353 spp = pp; 7354 } 7355 index >>= 1; 7356 sz++; 7357 } 7358 sfmmu_mlist_exit(pml); 7359 return (cnt); 7360 } 7361 7362 /* 7363 * Unload all large mappings to the pp and reset the p_szc field of every 7364 * constituent page according to the remaining mappings. 7365 * 7366 * pp must be locked SE_EXCL. Even though no other constituent pages are 7367 * locked it's legal to unload the large mappings to the pp because all 7368 * constituent pages of large locked mappings have to be locked SE_SHARED. 7369 * This means if we have SE_EXCL lock on one of constituent pages none of the 7370 * large mappings to pp are locked. 7371 * 7372 * Decrease p_szc field starting from the last constituent page and ending 7373 * with the root page. This method is used because other threads rely on the 7374 * root's p_szc to find the lock to syncronize on. After a root page_t's p_szc 7375 * is demoted then other threads will succeed in sfmmu_mlspl_enter(). This 7376 * ensures that p_szc changes of the constituent pages appears atomic for all 7377 * threads that use sfmmu_mlspl_enter() to examine p_szc field. 7378 * 7379 * This mechanism is only used for file system pages where it's not always 7380 * possible to get SE_EXCL locks on all constituent pages to demote the size 7381 * code (as is done for anonymous or kernel large pages). 7382 * 7383 * See more comments in front of sfmmu_mlspl_enter(). 7384 */ 7385 void 7386 hat_page_demote(page_t *pp) 7387 { 7388 int index; 7389 int sz; 7390 cpuset_t cpuset; 7391 int sync = 0; 7392 page_t *rootpp; 7393 struct sf_hment *sfhme; 7394 struct sf_hment *tmphme = NULL; 7395 struct hme_blk *hmeblkp; 7396 uint_t pszc; 7397 page_t *lastpp; 7398 cpuset_t tset; 7399 pgcnt_t npgs; 7400 kmutex_t *pml; 7401 kmutex_t *pmtx = NULL; 7402 7403 ASSERT(PAGE_EXCL(pp)); 7404 ASSERT(!PP_ISFREE(pp)); 7405 ASSERT(page_szc_lock_assert(pp)); 7406 pml = sfmmu_mlist_enter(pp); 7407 7408 pszc = pp->p_szc; 7409 if (pszc == 0) { 7410 goto out; 7411 } 7412 7413 index = PP_MAPINDEX(pp) >> 1; 7414 7415 if (index) { 7416 CPUSET_ZERO(cpuset); 7417 sz = TTE64K; 7418 sync = 1; 7419 } 7420 7421 while (index) { 7422 if (!(index & 0x1)) { 7423 index >>= 1; 7424 sz++; 7425 continue; 7426 } 7427 ASSERT(sz <= pszc); 7428 rootpp = PP_GROUPLEADER(pp, sz); 7429 for (sfhme = rootpp->p_mapping; sfhme; sfhme = tmphme) { 7430 tmphme = sfhme->hme_next; 7431 hmeblkp = sfmmu_hmetohblk(sfhme); 7432 if (hme_size(sfhme) != sz) { 7433 continue; 7434 } 7435 if (hmeblkp->hblk_xhat_bit) { 7436 cmn_err(CE_PANIC, 7437 "hat_page_demote: xhat hmeblk"); 7438 } 7439 tset = sfmmu_pageunload(rootpp, sfhme, sz); 7440 CPUSET_OR(cpuset, tset); 7441 } 7442 if (index >>= 1) { 7443 sz++; 7444 } 7445 } 7446 7447 ASSERT(!PP_ISMAPPED_LARGE(pp)); 7448 7449 if (sync) { 7450 xt_sync(cpuset); 7451 #ifdef VAC 7452 if (PP_ISTNC(pp)) { 7453 conv_tnc(rootpp, sz); 7454 } 7455 #endif /* VAC */ 7456 } 7457 7458 pmtx = sfmmu_page_enter(pp); 7459 7460 ASSERT(pp->p_szc == pszc); 7461 rootpp = PP_PAGEROOT(pp); 7462 ASSERT(rootpp->p_szc == pszc); 7463 lastpp = PP_PAGENEXT_N(rootpp, TTEPAGES(pszc) - 1); 7464 7465 while (lastpp != rootpp) { 7466 sz = PP_MAPINDEX(lastpp) ? fnd_mapping_sz(lastpp) : 0; 7467 ASSERT(sz < pszc); 7468 npgs = (sz == 0) ? 1 : TTEPAGES(sz); 7469 ASSERT(P2PHASE(lastpp->p_pagenum, npgs) == npgs - 1); 7470 while (--npgs > 0) { 7471 lastpp->p_szc = (uchar_t)sz; 7472 lastpp = PP_PAGEPREV(lastpp); 7473 } 7474 if (sz) { 7475 /* 7476 * make sure before current root's pszc 7477 * is updated all updates to constituent pages pszc 7478 * fields are globally visible. 7479 */ 7480 membar_producer(); 7481 } 7482 lastpp->p_szc = sz; 7483 ASSERT(IS_P2ALIGNED(lastpp->p_pagenum, TTEPAGES(sz))); 7484 if (lastpp != rootpp) { 7485 lastpp = PP_PAGEPREV(lastpp); 7486 } 7487 } 7488 if (sz == 0) { 7489 /* the loop above doesn't cover this case */ 7490 rootpp->p_szc = 0; 7491 } 7492 out: 7493 ASSERT(pp->p_szc == 0); 7494 if (pmtx != NULL) { 7495 sfmmu_page_exit(pmtx); 7496 } 7497 sfmmu_mlist_exit(pml); 7498 } 7499 7500 /* 7501 * Refresh the HAT ismttecnt[] element for size szc. 7502 * Caller must have set ISM busy flag to prevent mapping 7503 * lists from changing while we're traversing them. 7504 */ 7505 pgcnt_t 7506 ism_tsb_entries(sfmmu_t *sfmmup, int szc) 7507 { 7508 ism_blk_t *ism_blkp = sfmmup->sfmmu_iblk; 7509 ism_map_t *ism_map; 7510 pgcnt_t npgs = 0; 7511 int j; 7512 7513 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 7514 for (; ism_blkp != NULL; ism_blkp = ism_blkp->iblk_next) { 7515 ism_map = ism_blkp->iblk_maps; 7516 for (j = 0; ism_map[j].imap_ismhat && j < ISM_MAP_SLOTS; j++) 7517 npgs += ism_map[j].imap_ismhat->sfmmu_ttecnt[szc]; 7518 } 7519 sfmmup->sfmmu_ismttecnt[szc] = npgs; 7520 return (npgs); 7521 } 7522 7523 /* 7524 * Yield the memory claim requirement for an address space. 7525 * 7526 * This is currently implemented as the number of bytes that have active 7527 * hardware translations that have page structures. Therefore, it can 7528 * underestimate the traditional resident set size, eg, if the 7529 * physical page is present and the hardware translation is missing; 7530 * and it can overestimate the rss, eg, if there are active 7531 * translations to a frame buffer with page structs. 7532 * Also, it does not take sharing into account. 7533 * 7534 * Note that we don't acquire locks here since this function is most often 7535 * called from the clock thread. 7536 */ 7537 size_t 7538 hat_get_mapped_size(struct hat *hat) 7539 { 7540 size_t assize = 0; 7541 int i; 7542 7543 if (hat == NULL) 7544 return (0); 7545 7546 ASSERT(hat->sfmmu_xhat_provider == NULL); 7547 7548 for (i = 0; i < mmu_page_sizes; i++) 7549 assize += (pgcnt_t)hat->sfmmu_ttecnt[i] * TTEBYTES(i); 7550 7551 if (hat->sfmmu_iblk == NULL) 7552 return (assize); 7553 7554 for (i = 0; i < mmu_page_sizes; i++) 7555 assize += (pgcnt_t)hat->sfmmu_ismttecnt[i] * TTEBYTES(i); 7556 7557 return (assize); 7558 } 7559 7560 int 7561 hat_stats_enable(struct hat *hat) 7562 { 7563 hatlock_t *hatlockp; 7564 7565 ASSERT(hat->sfmmu_xhat_provider == NULL); 7566 7567 hatlockp = sfmmu_hat_enter(hat); 7568 hat->sfmmu_rmstat++; 7569 sfmmu_hat_exit(hatlockp); 7570 return (1); 7571 } 7572 7573 void 7574 hat_stats_disable(struct hat *hat) 7575 { 7576 hatlock_t *hatlockp; 7577 7578 ASSERT(hat->sfmmu_xhat_provider == NULL); 7579 7580 hatlockp = sfmmu_hat_enter(hat); 7581 hat->sfmmu_rmstat--; 7582 sfmmu_hat_exit(hatlockp); 7583 } 7584 7585 /* 7586 * Routines for entering or removing ourselves from the 7587 * ism_hat's mapping list. 7588 */ 7589 static void 7590 iment_add(struct ism_ment *iment, struct hat *ism_hat) 7591 { 7592 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7593 7594 iment->iment_prev = NULL; 7595 iment->iment_next = ism_hat->sfmmu_iment; 7596 if (ism_hat->sfmmu_iment) { 7597 ism_hat->sfmmu_iment->iment_prev = iment; 7598 } 7599 ism_hat->sfmmu_iment = iment; 7600 } 7601 7602 static void 7603 iment_sub(struct ism_ment *iment, struct hat *ism_hat) 7604 { 7605 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7606 7607 if (ism_hat->sfmmu_iment == NULL) { 7608 panic("ism map entry remove - no entries"); 7609 } 7610 7611 if (iment->iment_prev) { 7612 ASSERT(ism_hat->sfmmu_iment != iment); 7613 iment->iment_prev->iment_next = iment->iment_next; 7614 } else { 7615 ASSERT(ism_hat->sfmmu_iment == iment); 7616 ism_hat->sfmmu_iment = iment->iment_next; 7617 } 7618 7619 if (iment->iment_next) { 7620 iment->iment_next->iment_prev = iment->iment_prev; 7621 } 7622 7623 /* 7624 * zero out the entry 7625 */ 7626 iment->iment_next = NULL; 7627 iment->iment_prev = NULL; 7628 iment->iment_hat = NULL; 7629 } 7630 7631 /* 7632 * Hat_share()/unshare() return an (non-zero) error 7633 * when saddr and daddr are not properly aligned. 7634 * 7635 * The top level mapping element determines the alignment 7636 * requirement for saddr and daddr, depending on different 7637 * architectures. 7638 * 7639 * When hat_share()/unshare() are not supported, 7640 * HATOP_SHARE()/UNSHARE() return 0 7641 */ 7642 int 7643 hat_share(struct hat *sfmmup, caddr_t addr, 7644 struct hat *ism_hatid, caddr_t sptaddr, size_t len, uint_t ismszc) 7645 { 7646 ism_blk_t *ism_blkp; 7647 ism_blk_t *new_iblk; 7648 ism_map_t *ism_map; 7649 ism_ment_t *ism_ment; 7650 int i, added; 7651 hatlock_t *hatlockp; 7652 int reload_mmu = 0; 7653 uint_t ismshift = page_get_shift(ismszc); 7654 size_t ismpgsz = page_get_pagesize(ismszc); 7655 uint_t ismmask = (uint_t)ismpgsz - 1; 7656 size_t sh_size = ISM_SHIFT(ismshift, len); 7657 ushort_t ismhatflag; 7658 7659 #ifdef DEBUG 7660 caddr_t eaddr = addr + len; 7661 #endif /* DEBUG */ 7662 7663 ASSERT(ism_hatid != NULL && sfmmup != NULL); 7664 ASSERT(sptaddr == ISMID_STARTADDR); 7665 /* 7666 * Check the alignment. 7667 */ 7668 if (!ISM_ALIGNED(ismshift, addr) || !ISM_ALIGNED(ismshift, sptaddr)) 7669 return (EINVAL); 7670 7671 /* 7672 * Check size alignment. 7673 */ 7674 if (!ISM_ALIGNED(ismshift, len)) 7675 return (EINVAL); 7676 7677 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 7678 7679 /* 7680 * Allocate ism_ment for the ism_hat's mapping list, and an 7681 * ism map blk in case we need one. We must do our 7682 * allocations before acquiring locks to prevent a deadlock 7683 * in the kmem allocator on the mapping list lock. 7684 */ 7685 new_iblk = kmem_cache_alloc(ism_blk_cache, KM_SLEEP); 7686 ism_ment = kmem_cache_alloc(ism_ment_cache, KM_SLEEP); 7687 7688 /* 7689 * Serialize ISM mappings with the ISM busy flag, and also the 7690 * trap handlers. 7691 */ 7692 sfmmu_ismhat_enter(sfmmup, 0); 7693 7694 /* 7695 * Allocate an ism map blk if necessary. 7696 */ 7697 if (sfmmup->sfmmu_iblk == NULL) { 7698 sfmmup->sfmmu_iblk = new_iblk; 7699 bzero(new_iblk, sizeof (*new_iblk)); 7700 new_iblk->iblk_nextpa = (uint64_t)-1; 7701 membar_stst(); /* make sure next ptr visible to all CPUs */ 7702 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk); 7703 reload_mmu = 1; 7704 new_iblk = NULL; 7705 } 7706 7707 #ifdef DEBUG 7708 /* 7709 * Make sure mapping does not already exist. 7710 */ 7711 ism_blkp = sfmmup->sfmmu_iblk; 7712 while (ism_blkp) { 7713 ism_map = ism_blkp->iblk_maps; 7714 for (i = 0; i < ISM_MAP_SLOTS && ism_map[i].imap_ismhat; i++) { 7715 if ((addr >= ism_start(ism_map[i]) && 7716 addr < ism_end(ism_map[i])) || 7717 eaddr > ism_start(ism_map[i]) && 7718 eaddr <= ism_end(ism_map[i])) { 7719 panic("sfmmu_share: Already mapped!"); 7720 } 7721 } 7722 ism_blkp = ism_blkp->iblk_next; 7723 } 7724 #endif /* DEBUG */ 7725 7726 ASSERT(ismszc >= TTE4M); 7727 if (ismszc == TTE4M) { 7728 ismhatflag = HAT_4M_FLAG; 7729 } else if (ismszc == TTE32M) { 7730 ismhatflag = HAT_32M_FLAG; 7731 } else if (ismszc == TTE256M) { 7732 ismhatflag = HAT_256M_FLAG; 7733 } 7734 /* 7735 * Add mapping to first available mapping slot. 7736 */ 7737 ism_blkp = sfmmup->sfmmu_iblk; 7738 added = 0; 7739 while (!added) { 7740 ism_map = ism_blkp->iblk_maps; 7741 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7742 if (ism_map[i].imap_ismhat == NULL) { 7743 7744 ism_map[i].imap_ismhat = ism_hatid; 7745 ism_map[i].imap_vb_shift = (ushort_t)ismshift; 7746 ism_map[i].imap_hatflags = ismhatflag; 7747 ism_map[i].imap_sz_mask = ismmask; 7748 /* 7749 * imap_seg is checked in ISM_CHECK to see if 7750 * non-NULL, then other info assumed valid. 7751 */ 7752 membar_stst(); 7753 ism_map[i].imap_seg = (uintptr_t)addr | sh_size; 7754 ism_map[i].imap_ment = ism_ment; 7755 7756 /* 7757 * Now add ourselves to the ism_hat's 7758 * mapping list. 7759 */ 7760 ism_ment->iment_hat = sfmmup; 7761 ism_ment->iment_base_va = addr; 7762 ism_hatid->sfmmu_ismhat = 1; 7763 ism_hatid->sfmmu_flags = 0; 7764 mutex_enter(&ism_mlist_lock); 7765 iment_add(ism_ment, ism_hatid); 7766 mutex_exit(&ism_mlist_lock); 7767 added = 1; 7768 break; 7769 } 7770 } 7771 if (!added && ism_blkp->iblk_next == NULL) { 7772 ism_blkp->iblk_next = new_iblk; 7773 new_iblk = NULL; 7774 bzero(ism_blkp->iblk_next, 7775 sizeof (*ism_blkp->iblk_next)); 7776 ism_blkp->iblk_next->iblk_nextpa = (uint64_t)-1; 7777 membar_stst(); 7778 ism_blkp->iblk_nextpa = 7779 va_to_pa((caddr_t)ism_blkp->iblk_next); 7780 } 7781 ism_blkp = ism_blkp->iblk_next; 7782 } 7783 7784 /* 7785 * Update our counters for this sfmmup's ism mappings. 7786 */ 7787 for (i = 0; i <= ismszc; i++) { 7788 if (!(disable_ism_large_pages & (1 << i))) 7789 (void) ism_tsb_entries(sfmmup, i); 7790 } 7791 7792 hatlockp = sfmmu_hat_enter(sfmmup); 7793 7794 /* 7795 * For ISM and DISM we do not support 512K pages, so we only 7796 * only search the 4M and 8K/64K hashes for 4 pagesize cpus, and search 7797 * the 256M or 32M, and 4M and 8K/64K hashes for 6 pagesize cpus. 7798 */ 7799 ASSERT((disable_ism_large_pages & (1 << TTE512K)) != 0); 7800 7801 if (ismszc > TTE4M && !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) 7802 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7803 7804 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) 7805 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 7806 7807 /* 7808 * If we updated the ismblkpa for this HAT or we need 7809 * to start searching the 256M or 32M or 4M hash, we must 7810 * make sure all CPUs running this process reload their 7811 * tsbmiss area. Otherwise they will fail to load the mappings 7812 * in the tsbmiss handler and will loop calling pagefault(). 7813 */ 7814 switch (ismszc) { 7815 case TTE256M: 7816 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 7817 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 7818 sfmmu_sync_mmustate(sfmmup); 7819 } 7820 break; 7821 case TTE32M: 7822 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 7823 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 7824 sfmmu_sync_mmustate(sfmmup); 7825 } 7826 break; 7827 case TTE4M: 7828 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 7829 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7830 sfmmu_sync_mmustate(sfmmup); 7831 } 7832 break; 7833 default: 7834 break; 7835 } 7836 7837 /* 7838 * Now we can drop the locks. 7839 */ 7840 sfmmu_ismhat_exit(sfmmup, 1); 7841 sfmmu_hat_exit(hatlockp); 7842 7843 /* 7844 * Free up ismblk if we didn't use it. 7845 */ 7846 if (new_iblk != NULL) 7847 kmem_cache_free(ism_blk_cache, new_iblk); 7848 7849 /* 7850 * Check TSB and TLB page sizes. 7851 */ 7852 sfmmu_check_page_sizes(sfmmup, 1); 7853 7854 return (0); 7855 } 7856 7857 /* 7858 * hat_unshare removes exactly one ism_map from 7859 * this process's as. It expects multiple calls 7860 * to hat_unshare for multiple shm segments. 7861 */ 7862 void 7863 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) 7864 { 7865 ism_map_t *ism_map; 7866 ism_ment_t *free_ment = NULL; 7867 ism_blk_t *ism_blkp; 7868 struct hat *ism_hatid; 7869 int found, i; 7870 hatlock_t *hatlockp; 7871 struct tsb_info *tsbinfo; 7872 uint_t ismshift = page_get_shift(ismszc); 7873 size_t sh_size = ISM_SHIFT(ismshift, len); 7874 7875 ASSERT(ISM_ALIGNED(ismshift, addr)); 7876 ASSERT(ISM_ALIGNED(ismshift, len)); 7877 ASSERT(sfmmup != NULL); 7878 ASSERT(sfmmup != ksfmmup); 7879 7880 if (sfmmup->sfmmu_xhat_provider) { 7881 XHAT_UNSHARE(sfmmup, addr, len); 7882 return; 7883 } else { 7884 /* 7885 * This must be a CPU HAT. If the address space has 7886 * XHATs attached, inform all XHATs that ISM segment 7887 * is going away 7888 */ 7889 ASSERT(sfmmup->sfmmu_as != NULL); 7890 if (sfmmup->sfmmu_as->a_xhat != NULL) 7891 xhat_unshare_all(sfmmup->sfmmu_as, addr, len); 7892 } 7893 7894 /* 7895 * Make sure that during the entire time ISM mappings are removed, 7896 * the trap handlers serialize behind us, and that no one else 7897 * can be mucking with ISM mappings. This also lets us get away 7898 * with not doing expensive cross calls to flush the TLB -- we 7899 * just discard the context, flush the entire TSB, and call it 7900 * a day. 7901 */ 7902 sfmmu_ismhat_enter(sfmmup, 0); 7903 7904 /* 7905 * Remove the mapping. 7906 * 7907 * We can't have any holes in the ism map. 7908 * The tsb miss code while searching the ism map will 7909 * stop on an empty map slot. So we must move 7910 * everyone past the hole up 1 if any. 7911 * 7912 * Also empty ism map blks are not freed until the 7913 * process exits. This is to prevent a MT race condition 7914 * between sfmmu_unshare() and sfmmu_tsbmiss_exception(). 7915 */ 7916 found = 0; 7917 ism_blkp = sfmmup->sfmmu_iblk; 7918 while (!found && ism_blkp) { 7919 ism_map = ism_blkp->iblk_maps; 7920 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7921 if (addr == ism_start(ism_map[i]) && 7922 sh_size == (size_t)(ism_size(ism_map[i]))) { 7923 found = 1; 7924 break; 7925 } 7926 } 7927 if (!found) 7928 ism_blkp = ism_blkp->iblk_next; 7929 } 7930 7931 if (found) { 7932 ism_hatid = ism_map[i].imap_ismhat; 7933 ASSERT(ism_hatid != NULL); 7934 ASSERT(ism_hatid->sfmmu_ismhat == 1); 7935 7936 /* 7937 * First remove ourselves from the ism mapping list. 7938 */ 7939 mutex_enter(&ism_mlist_lock); 7940 iment_sub(ism_map[i].imap_ment, ism_hatid); 7941 mutex_exit(&ism_mlist_lock); 7942 free_ment = ism_map[i].imap_ment; 7943 7944 /* 7945 * Now gurantee that any other cpu 7946 * that tries to process an ISM miss 7947 * will go to tl=0. 7948 */ 7949 hatlockp = sfmmu_hat_enter(sfmmup); 7950 7951 sfmmu_invalidate_ctx(sfmmup); 7952 7953 sfmmu_hat_exit(hatlockp); 7954 7955 /* 7956 * We delete the ism map by copying 7957 * the next map over the current one. 7958 * We will take the next one in the maps 7959 * array or from the next ism_blk. 7960 */ 7961 while (ism_blkp) { 7962 ism_map = ism_blkp->iblk_maps; 7963 while (i < (ISM_MAP_SLOTS - 1)) { 7964 ism_map[i] = ism_map[i + 1]; 7965 i++; 7966 } 7967 /* i == (ISM_MAP_SLOTS - 1) */ 7968 ism_blkp = ism_blkp->iblk_next; 7969 if (ism_blkp) { 7970 ism_map[i] = ism_blkp->iblk_maps[0]; 7971 i = 0; 7972 } else { 7973 ism_map[i].imap_seg = 0; 7974 ism_map[i].imap_vb_shift = 0; 7975 ism_map[i].imap_hatflags = 0; 7976 ism_map[i].imap_sz_mask = 0; 7977 ism_map[i].imap_ismhat = NULL; 7978 ism_map[i].imap_ment = NULL; 7979 } 7980 } 7981 7982 /* 7983 * Now flush entire TSB for the process, since 7984 * demapping page by page can be too expensive. 7985 * We don't have to flush the TLB here anymore 7986 * since we switch to a new TLB ctx instead. 7987 * Also, there is no need to flush if the process 7988 * is exiting since the TSB will be freed later. 7989 */ 7990 if (!sfmmup->sfmmu_free) { 7991 hatlockp = sfmmu_hat_enter(sfmmup); 7992 for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL; 7993 tsbinfo = tsbinfo->tsb_next) { 7994 if (tsbinfo->tsb_flags & TSB_SWAPPED) 7995 continue; 7996 sfmmu_inv_tsb(tsbinfo->tsb_va, 7997 TSB_BYTES(tsbinfo->tsb_szc)); 7998 } 7999 sfmmu_hat_exit(hatlockp); 8000 } 8001 } 8002 8003 /* 8004 * Update our counters for this sfmmup's ism mappings. 8005 */ 8006 for (i = 0; i <= ismszc; i++) { 8007 if (!(disable_ism_large_pages & (1 << i))) 8008 (void) ism_tsb_entries(sfmmup, i); 8009 } 8010 8011 sfmmu_ismhat_exit(sfmmup, 0); 8012 8013 /* 8014 * We must do our freeing here after dropping locks 8015 * to prevent a deadlock in the kmem allocator on the 8016 * mapping list lock. 8017 */ 8018 if (free_ment != NULL) 8019 kmem_cache_free(ism_ment_cache, free_ment); 8020 8021 /* 8022 * Check TSB and TLB page sizes if the process isn't exiting. 8023 */ 8024 if (!sfmmup->sfmmu_free) 8025 sfmmu_check_page_sizes(sfmmup, 0); 8026 } 8027 8028 /* ARGSUSED */ 8029 static int 8030 sfmmu_idcache_constructor(void *buf, void *cdrarg, int kmflags) 8031 { 8032 /* void *buf is sfmmu_t pointer */ 8033 return (0); 8034 } 8035 8036 /* ARGSUSED */ 8037 static void 8038 sfmmu_idcache_destructor(void *buf, void *cdrarg) 8039 { 8040 /* void *buf is sfmmu_t pointer */ 8041 } 8042 8043 /* 8044 * setup kmem hmeblks by bzeroing all members and initializing the nextpa 8045 * field to be the pa of this hmeblk 8046 */ 8047 /* ARGSUSED */ 8048 static int 8049 sfmmu_hblkcache_constructor(void *buf, void *cdrarg, int kmflags) 8050 { 8051 struct hme_blk *hmeblkp; 8052 8053 bzero(buf, (size_t)cdrarg); 8054 hmeblkp = (struct hme_blk *)buf; 8055 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); 8056 8057 #ifdef HBLK_TRACE 8058 mutex_init(&hmeblkp->hblk_audit_lock, NULL, MUTEX_DEFAULT, NULL); 8059 #endif /* HBLK_TRACE */ 8060 8061 return (0); 8062 } 8063 8064 /* ARGSUSED */ 8065 static void 8066 sfmmu_hblkcache_destructor(void *buf, void *cdrarg) 8067 { 8068 8069 #ifdef HBLK_TRACE 8070 8071 struct hme_blk *hmeblkp; 8072 8073 hmeblkp = (struct hme_blk *)buf; 8074 mutex_destroy(&hmeblkp->hblk_audit_lock); 8075 8076 #endif /* HBLK_TRACE */ 8077 } 8078 8079 #define SFMMU_CACHE_RECLAIM_SCAN_RATIO 8 8080 static int sfmmu_cache_reclaim_scan_ratio = SFMMU_CACHE_RECLAIM_SCAN_RATIO; 8081 /* 8082 * The kmem allocator will callback into our reclaim routine when the system 8083 * is running low in memory. We traverse the hash and free up all unused but 8084 * still cached hme_blks. We also traverse the free list and free them up 8085 * as well. 8086 */ 8087 /*ARGSUSED*/ 8088 static void 8089 sfmmu_hblkcache_reclaim(void *cdrarg) 8090 { 8091 int i; 8092 uint64_t hblkpa, prevpa, nx_pa; 8093 struct hmehash_bucket *hmebp; 8094 struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL; 8095 static struct hmehash_bucket *uhmehash_reclaim_hand; 8096 static struct hmehash_bucket *khmehash_reclaim_hand; 8097 struct hme_blk *list = NULL; 8098 8099 hmebp = uhmehash_reclaim_hand; 8100 if (hmebp == NULL || hmebp > &uhme_hash[UHMEHASH_SZ]) 8101 uhmehash_reclaim_hand = hmebp = uhme_hash; 8102 uhmehash_reclaim_hand += UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8103 8104 for (i = UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8105 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8106 hmeblkp = hmebp->hmeblkp; 8107 hblkpa = hmebp->hmeh_nextpa; 8108 prevpa = 0; 8109 pr_hblk = NULL; 8110 while (hmeblkp) { 8111 nx_hblk = hmeblkp->hblk_next; 8112 nx_pa = hmeblkp->hblk_nextpa; 8113 if (!hmeblkp->hblk_vcnt && 8114 !hmeblkp->hblk_hmecnt) { 8115 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8116 prevpa, pr_hblk); 8117 sfmmu_hblk_free(hmebp, hmeblkp, 8118 hblkpa, &list); 8119 } else { 8120 pr_hblk = hmeblkp; 8121 prevpa = hblkpa; 8122 } 8123 hmeblkp = nx_hblk; 8124 hblkpa = nx_pa; 8125 } 8126 SFMMU_HASH_UNLOCK(hmebp); 8127 } 8128 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 8129 hmebp = uhme_hash; 8130 } 8131 8132 hmebp = khmehash_reclaim_hand; 8133 if (hmebp == NULL || hmebp > &khme_hash[KHMEHASH_SZ]) 8134 khmehash_reclaim_hand = hmebp = khme_hash; 8135 khmehash_reclaim_hand += KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8136 8137 for (i = KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8138 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8139 hmeblkp = hmebp->hmeblkp; 8140 hblkpa = hmebp->hmeh_nextpa; 8141 prevpa = 0; 8142 pr_hblk = NULL; 8143 while (hmeblkp) { 8144 nx_hblk = hmeblkp->hblk_next; 8145 nx_pa = hmeblkp->hblk_nextpa; 8146 if (!hmeblkp->hblk_vcnt && 8147 !hmeblkp->hblk_hmecnt) { 8148 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8149 prevpa, pr_hblk); 8150 sfmmu_hblk_free(hmebp, hmeblkp, 8151 hblkpa, &list); 8152 } else { 8153 pr_hblk = hmeblkp; 8154 prevpa = hblkpa; 8155 } 8156 hmeblkp = nx_hblk; 8157 hblkpa = nx_pa; 8158 } 8159 SFMMU_HASH_UNLOCK(hmebp); 8160 } 8161 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 8162 hmebp = khme_hash; 8163 } 8164 sfmmu_hblks_list_purge(&list); 8165 } 8166 8167 /* 8168 * sfmmu_get_ppvcolor should become a vm_machdep or hatop interface. 8169 * same goes for sfmmu_get_addrvcolor(). 8170 * 8171 * This function will return the virtual color for the specified page. The 8172 * virtual color corresponds to this page current mapping or its last mapping. 8173 * It is used by memory allocators to choose addresses with the correct 8174 * alignment so vac consistency is automatically maintained. If the page 8175 * has no color it returns -1. 8176 */ 8177 /*ARGSUSED*/ 8178 int 8179 sfmmu_get_ppvcolor(struct page *pp) 8180 { 8181 #ifdef VAC 8182 int color; 8183 8184 if (!(cache & CACHE_VAC) || PP_NEWPAGE(pp)) { 8185 return (-1); 8186 } 8187 color = PP_GET_VCOLOR(pp); 8188 ASSERT(color < mmu_btop(shm_alignment)); 8189 return (color); 8190 #else 8191 return (-1); 8192 #endif /* VAC */ 8193 } 8194 8195 /* 8196 * This function will return the desired alignment for vac consistency 8197 * (vac color) given a virtual address. If no vac is present it returns -1. 8198 */ 8199 /*ARGSUSED*/ 8200 int 8201 sfmmu_get_addrvcolor(caddr_t vaddr) 8202 { 8203 #ifdef VAC 8204 if (cache & CACHE_VAC) { 8205 return (addr_to_vcolor(vaddr)); 8206 } else { 8207 return (-1); 8208 } 8209 #else 8210 return (-1); 8211 #endif /* VAC */ 8212 } 8213 8214 #ifdef VAC 8215 /* 8216 * Check for conflicts. 8217 * A conflict exists if the new and existent mappings do not match in 8218 * their "shm_alignment fields. If conflicts exist, the existant mappings 8219 * are flushed unless one of them is locked. If one of them is locked, then 8220 * the mappings are flushed and converted to non-cacheable mappings. 8221 */ 8222 static void 8223 sfmmu_vac_conflict(struct hat *hat, caddr_t addr, page_t *pp) 8224 { 8225 struct hat *tmphat; 8226 struct sf_hment *sfhmep, *tmphme = NULL; 8227 struct hme_blk *hmeblkp; 8228 int vcolor; 8229 tte_t tte; 8230 8231 ASSERT(sfmmu_mlist_held(pp)); 8232 ASSERT(!PP_ISNC(pp)); /* page better be cacheable */ 8233 8234 vcolor = addr_to_vcolor(addr); 8235 if (PP_NEWPAGE(pp)) { 8236 PP_SET_VCOLOR(pp, vcolor); 8237 return; 8238 } 8239 8240 if (PP_GET_VCOLOR(pp) == vcolor) { 8241 return; 8242 } 8243 8244 if (!PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp)) { 8245 /* 8246 * Previous user of page had a different color 8247 * but since there are no current users 8248 * we just flush the cache and change the color. 8249 */ 8250 SFMMU_STAT(sf_pgcolor_conflict); 8251 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8252 PP_SET_VCOLOR(pp, vcolor); 8253 return; 8254 } 8255 8256 /* 8257 * If we get here we have a vac conflict with a current 8258 * mapping. VAC conflict policy is as follows. 8259 * - The default is to unload the other mappings unless: 8260 * - If we have a large mapping we uncache the page. 8261 * We need to uncache the rest of the large page too. 8262 * - If any of the mappings are locked we uncache the page. 8263 * - If the requested mapping is inconsistent 8264 * with another mapping and that mapping 8265 * is in the same address space we have to 8266 * make it non-cached. The default thing 8267 * to do is unload the inconsistent mapping 8268 * but if they are in the same address space 8269 * we run the risk of unmapping the pc or the 8270 * stack which we will use as we return to the user, 8271 * in which case we can then fault on the thing 8272 * we just unloaded and get into an infinite loop. 8273 */ 8274 if (PP_ISMAPPED_LARGE(pp)) { 8275 int sz; 8276 8277 /* 8278 * Existing mapping is for big pages. We don't unload 8279 * existing big mappings to satisfy new mappings. 8280 * Always convert all mappings to TNC. 8281 */ 8282 sz = fnd_mapping_sz(pp); 8283 pp = PP_GROUPLEADER(pp, sz); 8284 SFMMU_STAT_ADD(sf_uncache_conflict, TTEPAGES(sz)); 8285 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 8286 TTEPAGES(sz)); 8287 8288 return; 8289 } 8290 8291 /* 8292 * check if any mapping is in same as or if it is locked 8293 * since in that case we need to uncache. 8294 */ 8295 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8296 tmphme = sfhmep->hme_next; 8297 hmeblkp = sfmmu_hmetohblk(sfhmep); 8298 if (hmeblkp->hblk_xhat_bit) 8299 continue; 8300 tmphat = hblktosfmmu(hmeblkp); 8301 sfmmu_copytte(&sfhmep->hme_tte, &tte); 8302 ASSERT(TTE_IS_VALID(&tte)); 8303 if ((tmphat == hat) || hmeblkp->hblk_lckcnt) { 8304 /* 8305 * We have an uncache conflict 8306 */ 8307 SFMMU_STAT(sf_uncache_conflict); 8308 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1); 8309 return; 8310 } 8311 } 8312 8313 /* 8314 * We have an unload conflict 8315 * We have already checked for LARGE mappings, therefore 8316 * the remaining mapping(s) must be TTE8K. 8317 */ 8318 SFMMU_STAT(sf_unload_conflict); 8319 8320 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8321 tmphme = sfhmep->hme_next; 8322 hmeblkp = sfmmu_hmetohblk(sfhmep); 8323 if (hmeblkp->hblk_xhat_bit) 8324 continue; 8325 (void) sfmmu_pageunload(pp, sfhmep, TTE8K); 8326 } 8327 8328 if (PP_ISMAPPED_KPM(pp)) 8329 sfmmu_kpm_vac_unload(pp, addr); 8330 8331 /* 8332 * Unloads only do TLB flushes so we need to flush the 8333 * cache here. 8334 */ 8335 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8336 PP_SET_VCOLOR(pp, vcolor); 8337 } 8338 8339 /* 8340 * Whenever a mapping is unloaded and the page is in TNC state, 8341 * we see if the page can be made cacheable again. 'pp' is 8342 * the page that we just unloaded a mapping from, the size 8343 * of mapping that was unloaded is 'ottesz'. 8344 * Remark: 8345 * The recache policy for mpss pages can leave a performance problem 8346 * under the following circumstances: 8347 * . A large page in uncached mode has just been unmapped. 8348 * . All constituent pages are TNC due to a conflicting small mapping. 8349 * . There are many other, non conflicting, small mappings around for 8350 * a lot of the constituent pages. 8351 * . We're called w/ the "old" groupleader page and the old ottesz, 8352 * but this is irrelevant, since we're no more "PP_ISMAPPED_LARGE", so 8353 * we end up w/ TTE8K or npages == 1. 8354 * . We call tst_tnc w/ the old groupleader only, and if there is no 8355 * conflict, we re-cache only this page. 8356 * . All other small mappings are not checked and will be left in TNC mode. 8357 * The problem is not very serious because: 8358 * . mpss is actually only defined for heap and stack, so the probability 8359 * is not very high that a large page mapping exists in parallel to a small 8360 * one (this is possible, but seems to be bad programming style in the 8361 * appl). 8362 * . The problem gets a little bit more serious, when those TNC pages 8363 * have to be mapped into kernel space, e.g. for networking. 8364 * . When VAC alias conflicts occur in applications, this is regarded 8365 * as an application bug. So if kstat's show them, the appl should 8366 * be changed anyway. 8367 */ 8368 void 8369 conv_tnc(page_t *pp, int ottesz) 8370 { 8371 int cursz, dosz; 8372 pgcnt_t curnpgs, dopgs; 8373 pgcnt_t pg64k; 8374 page_t *pp2; 8375 8376 /* 8377 * Determine how big a range we check for TNC and find 8378 * leader page. cursz is the size of the biggest 8379 * mapping that still exist on 'pp'. 8380 */ 8381 if (PP_ISMAPPED_LARGE(pp)) { 8382 cursz = fnd_mapping_sz(pp); 8383 } else { 8384 cursz = TTE8K; 8385 } 8386 8387 if (ottesz >= cursz) { 8388 dosz = ottesz; 8389 pp2 = pp; 8390 } else { 8391 dosz = cursz; 8392 pp2 = PP_GROUPLEADER(pp, dosz); 8393 } 8394 8395 pg64k = TTEPAGES(TTE64K); 8396 dopgs = TTEPAGES(dosz); 8397 8398 ASSERT(dopgs == 1 || ((dopgs & (pg64k - 1)) == 0)); 8399 8400 while (dopgs != 0) { 8401 curnpgs = TTEPAGES(cursz); 8402 if (tst_tnc(pp2, curnpgs)) { 8403 SFMMU_STAT_ADD(sf_recache, curnpgs); 8404 sfmmu_page_cache_array(pp2, HAT_CACHE, CACHE_NO_FLUSH, 8405 curnpgs); 8406 } 8407 8408 ASSERT(dopgs >= curnpgs); 8409 dopgs -= curnpgs; 8410 8411 if (dopgs == 0) { 8412 break; 8413 } 8414 8415 pp2 = PP_PAGENEXT_N(pp2, curnpgs); 8416 if (((dopgs & (pg64k - 1)) == 0) && PP_ISMAPPED_LARGE(pp2)) { 8417 cursz = fnd_mapping_sz(pp2); 8418 } else { 8419 cursz = TTE8K; 8420 } 8421 } 8422 } 8423 8424 /* 8425 * Returns 1 if page(s) can be converted from TNC to cacheable setting, 8426 * returns 0 otherwise. Note that oaddr argument is valid for only 8427 * 8k pages. 8428 */ 8429 int 8430 tst_tnc(page_t *pp, pgcnt_t npages) 8431 { 8432 struct sf_hment *sfhme; 8433 struct hme_blk *hmeblkp; 8434 tte_t tte; 8435 caddr_t vaddr; 8436 int clr_valid = 0; 8437 int color, color1, bcolor; 8438 int i, ncolors; 8439 8440 ASSERT(pp != NULL); 8441 ASSERT(!(cache & CACHE_WRITEBACK)); 8442 8443 if (npages > 1) { 8444 ncolors = CACHE_NUM_COLOR; 8445 } 8446 8447 for (i = 0; i < npages; i++) { 8448 ASSERT(sfmmu_mlist_held(pp)); 8449 ASSERT(PP_ISTNC(pp)); 8450 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 8451 8452 if (PP_ISPNC(pp)) { 8453 return (0); 8454 } 8455 8456 clr_valid = 0; 8457 if (PP_ISMAPPED_KPM(pp)) { 8458 caddr_t kpmvaddr; 8459 8460 ASSERT(kpm_enable); 8461 kpmvaddr = hat_kpm_page2va(pp, 1); 8462 ASSERT(!(npages > 1 && IS_KPM_ALIAS_RANGE(kpmvaddr))); 8463 color1 = addr_to_vcolor(kpmvaddr); 8464 clr_valid = 1; 8465 } 8466 8467 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8468 hmeblkp = sfmmu_hmetohblk(sfhme); 8469 if (hmeblkp->hblk_xhat_bit) 8470 continue; 8471 8472 sfmmu_copytte(&sfhme->hme_tte, &tte); 8473 ASSERT(TTE_IS_VALID(&tte)); 8474 8475 vaddr = tte_to_vaddr(hmeblkp, tte); 8476 color = addr_to_vcolor(vaddr); 8477 8478 if (npages > 1) { 8479 /* 8480 * If there is a big mapping, make sure 8481 * 8K mapping is consistent with the big 8482 * mapping. 8483 */ 8484 bcolor = i % ncolors; 8485 if (color != bcolor) { 8486 return (0); 8487 } 8488 } 8489 if (!clr_valid) { 8490 clr_valid = 1; 8491 color1 = color; 8492 } 8493 8494 if (color1 != color) { 8495 return (0); 8496 } 8497 } 8498 8499 pp = PP_PAGENEXT(pp); 8500 } 8501 8502 return (1); 8503 } 8504 8505 void 8506 sfmmu_page_cache_array(page_t *pp, int flags, int cache_flush_flag, 8507 pgcnt_t npages) 8508 { 8509 kmutex_t *pmtx; 8510 int i, ncolors, bcolor; 8511 kpm_hlk_t *kpmp; 8512 cpuset_t cpuset; 8513 8514 ASSERT(pp != NULL); 8515 ASSERT(!(cache & CACHE_WRITEBACK)); 8516 8517 kpmp = sfmmu_kpm_kpmp_enter(pp, npages); 8518 pmtx = sfmmu_page_enter(pp); 8519 8520 /* 8521 * Fast path caching single unmapped page 8522 */ 8523 if (npages == 1 && !PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp) && 8524 flags == HAT_CACHE) { 8525 PP_CLRTNC(pp); 8526 PP_CLRPNC(pp); 8527 sfmmu_page_exit(pmtx); 8528 sfmmu_kpm_kpmp_exit(kpmp); 8529 return; 8530 } 8531 8532 /* 8533 * We need to capture all cpus in order to change cacheability 8534 * because we can't allow one cpu to access the same physical 8535 * page using a cacheable and a non-cachebale mapping at the same 8536 * time. Since we may end up walking the ism mapping list 8537 * have to grab it's lock now since we can't after all the 8538 * cpus have been captured. 8539 */ 8540 sfmmu_hat_lock_all(); 8541 mutex_enter(&ism_mlist_lock); 8542 kpreempt_disable(); 8543 cpuset = cpu_ready_set; 8544 xc_attention(cpuset); 8545 8546 if (npages > 1) { 8547 /* 8548 * Make sure all colors are flushed since the 8549 * sfmmu_page_cache() only flushes one color- 8550 * it does not know big pages. 8551 */ 8552 ncolors = CACHE_NUM_COLOR; 8553 if (flags & HAT_TMPNC) { 8554 for (i = 0; i < ncolors; i++) { 8555 sfmmu_cache_flushcolor(i, pp->p_pagenum); 8556 } 8557 cache_flush_flag = CACHE_NO_FLUSH; 8558 } 8559 } 8560 8561 for (i = 0; i < npages; i++) { 8562 8563 ASSERT(sfmmu_mlist_held(pp)); 8564 8565 if (!(flags == HAT_TMPNC && PP_ISTNC(pp))) { 8566 8567 if (npages > 1) { 8568 bcolor = i % ncolors; 8569 } else { 8570 bcolor = NO_VCOLOR; 8571 } 8572 8573 sfmmu_page_cache(pp, flags, cache_flush_flag, 8574 bcolor); 8575 } 8576 8577 pp = PP_PAGENEXT(pp); 8578 } 8579 8580 xt_sync(cpuset); 8581 xc_dismissed(cpuset); 8582 mutex_exit(&ism_mlist_lock); 8583 sfmmu_hat_unlock_all(); 8584 sfmmu_page_exit(pmtx); 8585 sfmmu_kpm_kpmp_exit(kpmp); 8586 kpreempt_enable(); 8587 } 8588 8589 /* 8590 * This function changes the virtual cacheability of all mappings to a 8591 * particular page. When changing from uncache to cacheable the mappings will 8592 * only be changed if all of them have the same virtual color. 8593 * We need to flush the cache in all cpus. It is possible that 8594 * a process referenced a page as cacheable but has sinced exited 8595 * and cleared the mapping list. We still to flush it but have no 8596 * state so all cpus is the only alternative. 8597 */ 8598 static void 8599 sfmmu_page_cache(page_t *pp, int flags, int cache_flush_flag, int bcolor) 8600 { 8601 struct sf_hment *sfhme; 8602 struct hme_blk *hmeblkp; 8603 sfmmu_t *sfmmup; 8604 tte_t tte, ttemod; 8605 caddr_t vaddr; 8606 int ret, color; 8607 pfn_t pfn; 8608 8609 color = bcolor; 8610 pfn = pp->p_pagenum; 8611 8612 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8613 8614 hmeblkp = sfmmu_hmetohblk(sfhme); 8615 8616 if (hmeblkp->hblk_xhat_bit) 8617 continue; 8618 8619 sfmmu_copytte(&sfhme->hme_tte, &tte); 8620 ASSERT(TTE_IS_VALID(&tte)); 8621 vaddr = tte_to_vaddr(hmeblkp, tte); 8622 color = addr_to_vcolor(vaddr); 8623 8624 #ifdef DEBUG 8625 if ((flags & HAT_CACHE) && bcolor != NO_VCOLOR) { 8626 ASSERT(color == bcolor); 8627 } 8628 #endif 8629 8630 ASSERT(flags != HAT_TMPNC || color == PP_GET_VCOLOR(pp)); 8631 8632 ttemod = tte; 8633 if (flags & (HAT_UNCACHE | HAT_TMPNC)) { 8634 TTE_CLR_VCACHEABLE(&ttemod); 8635 } else { /* flags & HAT_CACHE */ 8636 TTE_SET_VCACHEABLE(&ttemod); 8637 } 8638 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 8639 if (ret < 0) { 8640 /* 8641 * Since all cpus are captured modifytte should not 8642 * fail. 8643 */ 8644 panic("sfmmu_page_cache: write to tte failed"); 8645 } 8646 8647 sfmmup = hblktosfmmu(hmeblkp); 8648 if (cache_flush_flag == CACHE_FLUSH) { 8649 /* 8650 * Flush TSBs, TLBs and caches 8651 */ 8652 if (sfmmup->sfmmu_ismhat) { 8653 if (flags & HAT_CACHE) { 8654 SFMMU_STAT(sf_ism_recache); 8655 } else { 8656 SFMMU_STAT(sf_ism_uncache); 8657 } 8658 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8659 pfn, CACHE_FLUSH); 8660 } else { 8661 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp, 8662 pfn, 0, FLUSH_ALL_CPUS, CACHE_FLUSH, 1); 8663 } 8664 8665 /* 8666 * all cache entries belonging to this pfn are 8667 * now flushed. 8668 */ 8669 cache_flush_flag = CACHE_NO_FLUSH; 8670 } else { 8671 8672 /* 8673 * Flush only TSBs and TLBs. 8674 */ 8675 if (sfmmup->sfmmu_ismhat) { 8676 if (flags & HAT_CACHE) { 8677 SFMMU_STAT(sf_ism_recache); 8678 } else { 8679 SFMMU_STAT(sf_ism_uncache); 8680 } 8681 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8682 pfn, CACHE_NO_FLUSH); 8683 } else { 8684 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1); 8685 } 8686 } 8687 } 8688 8689 if (PP_ISMAPPED_KPM(pp)) 8690 sfmmu_kpm_page_cache(pp, flags, cache_flush_flag); 8691 8692 switch (flags) { 8693 8694 default: 8695 panic("sfmmu_pagecache: unknown flags"); 8696 break; 8697 8698 case HAT_CACHE: 8699 PP_CLRTNC(pp); 8700 PP_CLRPNC(pp); 8701 PP_SET_VCOLOR(pp, color); 8702 break; 8703 8704 case HAT_TMPNC: 8705 PP_SETTNC(pp); 8706 PP_SET_VCOLOR(pp, NO_VCOLOR); 8707 break; 8708 8709 case HAT_UNCACHE: 8710 PP_SETPNC(pp); 8711 PP_CLRTNC(pp); 8712 PP_SET_VCOLOR(pp, NO_VCOLOR); 8713 break; 8714 } 8715 } 8716 #endif /* VAC */ 8717 8718 8719 /* 8720 * Wrapper routine used to return a context. 8721 * 8722 * It's the responsibility of the caller to guarantee that the 8723 * process serializes on calls here by taking the HAT lock for 8724 * the hat. 8725 * 8726 */ 8727 static void 8728 sfmmu_get_ctx(sfmmu_t *sfmmup) 8729 { 8730 mmu_ctx_t *mmu_ctxp; 8731 uint_t pstate_save; 8732 8733 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8734 ASSERT(sfmmup != ksfmmup); 8735 8736 kpreempt_disable(); 8737 8738 mmu_ctxp = CPU_MMU_CTXP(CPU); 8739 ASSERT(mmu_ctxp); 8740 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 8741 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 8742 8743 /* 8744 * Do a wrap-around if cnum reaches the max # cnum supported by a MMU. 8745 */ 8746 if (mmu_ctxp->mmu_cnum == mmu_ctxp->mmu_nctxs) 8747 sfmmu_ctx_wrap_around(mmu_ctxp); 8748 8749 /* 8750 * Let the MMU set up the page sizes to use for 8751 * this context in the TLB. Don't program 2nd dtlb for ism hat. 8752 */ 8753 if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) { 8754 mmu_set_ctx_page_sizes(sfmmup); 8755 } 8756 8757 /* 8758 * sfmmu_alloc_ctx and sfmmu_load_mmustate will be performed with 8759 * interrupts disabled to prevent race condition with wrap-around 8760 * ctx invalidatation. In sun4v, ctx invalidation also involves 8761 * a HV call to set the number of TSBs to 0. If interrupts are not 8762 * disabled until after sfmmu_load_mmustate is complete TSBs may 8763 * become assigned to INVALID_CONTEXT. This is not allowed. 8764 */ 8765 pstate_save = sfmmu_disable_intrs(); 8766 8767 sfmmu_alloc_ctx(sfmmup, 1, CPU); 8768 sfmmu_load_mmustate(sfmmup); 8769 8770 sfmmu_enable_intrs(pstate_save); 8771 8772 kpreempt_enable(); 8773 } 8774 8775 /* 8776 * When all cnums are used up in a MMU, cnum will wrap around to the 8777 * next generation and start from 2. 8778 */ 8779 static void 8780 sfmmu_ctx_wrap_around(mmu_ctx_t *mmu_ctxp) 8781 { 8782 8783 /* caller must have disabled the preemption */ 8784 ASSERT(curthread->t_preempt >= 1); 8785 ASSERT(mmu_ctxp != NULL); 8786 8787 /* acquire Per-MMU (PM) spin lock */ 8788 mutex_enter(&mmu_ctxp->mmu_lock); 8789 8790 /* re-check to see if wrap-around is needed */ 8791 if (mmu_ctxp->mmu_cnum < mmu_ctxp->mmu_nctxs) 8792 goto done; 8793 8794 SFMMU_MMU_STAT(mmu_wrap_around); 8795 8796 /* update gnum */ 8797 ASSERT(mmu_ctxp->mmu_gnum != 0); 8798 mmu_ctxp->mmu_gnum++; 8799 if (mmu_ctxp->mmu_gnum == 0 || 8800 mmu_ctxp->mmu_gnum > MAX_SFMMU_GNUM_VAL) { 8801 cmn_err(CE_PANIC, "mmu_gnum of mmu_ctx 0x%p is out of bound.", 8802 (void *)mmu_ctxp); 8803 } 8804 8805 if (mmu_ctxp->mmu_ncpus > 1) { 8806 cpuset_t cpuset; 8807 8808 membar_enter(); /* make sure updated gnum visible */ 8809 8810 SFMMU_XCALL_STATS(NULL); 8811 8812 /* xcall to others on the same MMU to invalidate ctx */ 8813 cpuset = mmu_ctxp->mmu_cpuset; 8814 ASSERT(CPU_IN_SET(cpuset, CPU->cpu_id)); 8815 CPUSET_DEL(cpuset, CPU->cpu_id); 8816 CPUSET_AND(cpuset, cpu_ready_set); 8817 8818 /* 8819 * Pass in INVALID_CONTEXT as the first parameter to 8820 * sfmmu_raise_tsb_exception, which invalidates the context 8821 * of any process running on the CPUs in the MMU. 8822 */ 8823 xt_some(cpuset, sfmmu_raise_tsb_exception, 8824 INVALID_CONTEXT, INVALID_CONTEXT); 8825 xt_sync(cpuset); 8826 8827 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 8828 } 8829 8830 if (sfmmu_getctx_sec() != INVALID_CONTEXT) { 8831 sfmmu_setctx_sec(INVALID_CONTEXT); 8832 sfmmu_clear_utsbinfo(); 8833 } 8834 8835 /* 8836 * No xcall is needed here. For sun4u systems all CPUs in context 8837 * domain share a single physical MMU therefore it's enough to flush 8838 * TLB on local CPU. On sun4v systems we use 1 global context 8839 * domain and flush all remote TLBs in sfmmu_raise_tsb_exception 8840 * handler. Note that vtag_flushall_uctxs() is called 8841 * for Ultra II machine, where the equivalent flushall functionality 8842 * is implemented in SW, and only user ctx TLB entries are flushed. 8843 */ 8844 if (&vtag_flushall_uctxs != NULL) { 8845 vtag_flushall_uctxs(); 8846 } else { 8847 vtag_flushall(); 8848 } 8849 8850 /* reset mmu cnum, skips cnum 0 and 1 */ 8851 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 8852 8853 done: 8854 mutex_exit(&mmu_ctxp->mmu_lock); 8855 } 8856 8857 8858 /* 8859 * For multi-threaded process, set the process context to INVALID_CONTEXT 8860 * so that it faults and reloads the MMU state from TL=0. For single-threaded 8861 * process, we can just load the MMU state directly without having to 8862 * set context invalid. Caller must hold the hat lock since we don't 8863 * acquire it here. 8864 */ 8865 static void 8866 sfmmu_sync_mmustate(sfmmu_t *sfmmup) 8867 { 8868 uint_t cnum; 8869 uint_t pstate_save; 8870 8871 ASSERT(sfmmup != ksfmmup); 8872 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8873 8874 kpreempt_disable(); 8875 8876 /* 8877 * We check whether the pass'ed-in sfmmup is the same as the 8878 * current running proc. This is to makes sure the current proc 8879 * stays single-threaded if it already is. 8880 */ 8881 if ((sfmmup == curthread->t_procp->p_as->a_hat) && 8882 (curthread->t_procp->p_lwpcnt == 1)) { 8883 /* single-thread */ 8884 cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum; 8885 if (cnum != INVALID_CONTEXT) { 8886 uint_t curcnum; 8887 /* 8888 * Disable interrupts to prevent race condition 8889 * with sfmmu_ctx_wrap_around ctx invalidation. 8890 * In sun4v, ctx invalidation involves setting 8891 * TSB to NULL, hence, interrupts should be disabled 8892 * untill after sfmmu_load_mmustate is completed. 8893 */ 8894 pstate_save = sfmmu_disable_intrs(); 8895 curcnum = sfmmu_getctx_sec(); 8896 if (curcnum == cnum) 8897 sfmmu_load_mmustate(sfmmup); 8898 sfmmu_enable_intrs(pstate_save); 8899 ASSERT(curcnum == cnum || curcnum == INVALID_CONTEXT); 8900 } 8901 } else { 8902 /* 8903 * multi-thread 8904 * or when sfmmup is not the same as the curproc. 8905 */ 8906 sfmmu_invalidate_ctx(sfmmup); 8907 } 8908 8909 kpreempt_enable(); 8910 } 8911 8912 8913 /* 8914 * Replace the specified TSB with a new TSB. This function gets called when 8915 * we grow, shrink or swapin a TSB. When swapping in a TSB (TSB_SWAPIN), the 8916 * TSB_FORCEALLOC flag may be used to force allocation of a minimum-sized TSB 8917 * (8K). 8918 * 8919 * Caller must hold the HAT lock, but should assume any tsb_info 8920 * pointers it has are no longer valid after calling this function. 8921 * 8922 * Return values: 8923 * TSB_ALLOCFAIL Failed to allocate a TSB, due to memory constraints 8924 * TSB_LOSTRACE HAT is busy, i.e. another thread is already doing 8925 * something to this tsbinfo/TSB 8926 * TSB_SUCCESS Operation succeeded 8927 */ 8928 static tsb_replace_rc_t 8929 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, 8930 hatlock_t *hatlockp, uint_t flags) 8931 { 8932 struct tsb_info *new_tsbinfo = NULL; 8933 struct tsb_info *curtsb, *prevtsb; 8934 uint_t tte_sz_mask; 8935 int i; 8936 8937 ASSERT(sfmmup != ksfmmup); 8938 ASSERT(sfmmup->sfmmu_ismhat == 0); 8939 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8940 ASSERT(szc <= tsb_max_growsize); 8941 8942 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY)) 8943 return (TSB_LOSTRACE); 8944 8945 /* 8946 * Find the tsb_info ahead of this one in the list, and 8947 * also make sure that the tsb_info passed in really 8948 * exists! 8949 */ 8950 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 8951 curtsb != old_tsbinfo && curtsb != NULL; 8952 prevtsb = curtsb, curtsb = curtsb->tsb_next); 8953 ASSERT(curtsb != NULL); 8954 8955 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8956 /* 8957 * The process is swapped out, so just set the new size 8958 * code. When it swaps back in, we'll allocate a new one 8959 * of the new chosen size. 8960 */ 8961 curtsb->tsb_szc = szc; 8962 return (TSB_SUCCESS); 8963 } 8964 SFMMU_FLAGS_SET(sfmmup, HAT_BUSY); 8965 8966 tte_sz_mask = old_tsbinfo->tsb_ttesz_mask; 8967 8968 /* 8969 * All initialization is done inside of sfmmu_tsbinfo_alloc(). 8970 * If we fail to allocate a TSB, exit. 8971 */ 8972 sfmmu_hat_exit(hatlockp); 8973 if (sfmmu_tsbinfo_alloc(&new_tsbinfo, szc, tte_sz_mask, 8974 flags, sfmmup)) { 8975 (void) sfmmu_hat_enter(sfmmup); 8976 if (!(flags & TSB_SWAPIN)) 8977 SFMMU_STAT(sf_tsb_resize_failures); 8978 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8979 return (TSB_ALLOCFAIL); 8980 } 8981 (void) sfmmu_hat_enter(sfmmup); 8982 8983 /* 8984 * Re-check to make sure somebody else didn't muck with us while we 8985 * didn't hold the HAT lock. If the process swapped out, fine, just 8986 * exit; this can happen if we try to shrink the TSB from the context 8987 * of another process (such as on an ISM unmap), though it is rare. 8988 */ 8989 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8990 SFMMU_STAT(sf_tsb_resize_failures); 8991 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8992 sfmmu_hat_exit(hatlockp); 8993 sfmmu_tsbinfo_free(new_tsbinfo); 8994 (void) sfmmu_hat_enter(sfmmup); 8995 return (TSB_LOSTRACE); 8996 } 8997 8998 #ifdef DEBUG 8999 /* Reverify that the tsb_info still exists.. for debugging only */ 9000 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 9001 curtsb != old_tsbinfo && curtsb != NULL; 9002 prevtsb = curtsb, curtsb = curtsb->tsb_next); 9003 ASSERT(curtsb != NULL); 9004 #endif /* DEBUG */ 9005 9006 /* 9007 * Quiesce any CPUs running this process on their next TLB miss 9008 * so they atomically see the new tsb_info. We temporarily set the 9009 * context to invalid context so new threads that come on processor 9010 * after we do the xcall to cpusran will also serialize behind the 9011 * HAT lock on TLB miss and will see the new TSB. Since this short 9012 * race with a new thread coming on processor is relatively rare, 9013 * this synchronization mechanism should be cheaper than always 9014 * pausing all CPUs for the duration of the setup, which is what 9015 * the old implementation did. This is particuarly true if we are 9016 * copying a huge chunk of memory around during that window. 9017 * 9018 * The memory barriers are to make sure things stay consistent 9019 * with resume() since it does not hold the HAT lock while 9020 * walking the list of tsb_info structures. 9021 */ 9022 if ((flags & TSB_SWAPIN) != TSB_SWAPIN) { 9023 /* The TSB is either growing or shrinking. */ 9024 sfmmu_invalidate_ctx(sfmmup); 9025 } else { 9026 /* 9027 * It is illegal to swap in TSBs from a process other 9028 * than a process being swapped in. This in turn 9029 * implies we do not have a valid MMU context here 9030 * since a process needs one to resolve translation 9031 * misses. 9032 */ 9033 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup); 9034 } 9035 9036 #ifdef DEBUG 9037 ASSERT(max_mmu_ctxdoms > 0); 9038 9039 /* 9040 * Process should have INVALID_CONTEXT on all MMUs 9041 */ 9042 for (i = 0; i < max_mmu_ctxdoms; i++) { 9043 9044 ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); 9045 } 9046 #endif 9047 9048 new_tsbinfo->tsb_next = old_tsbinfo->tsb_next; 9049 membar_stst(); /* strict ordering required */ 9050 if (prevtsb) 9051 prevtsb->tsb_next = new_tsbinfo; 9052 else 9053 sfmmup->sfmmu_tsb = new_tsbinfo; 9054 membar_enter(); /* make sure new TSB globally visible */ 9055 sfmmu_setup_tsbinfo(sfmmup); 9056 9057 /* 9058 * We need to migrate TSB entries from the old TSB to the new TSB 9059 * if tsb_remap_ttes is set and the TSB is growing. 9060 */ 9061 if (tsb_remap_ttes && ((flags & TSB_GROW) == TSB_GROW)) 9062 sfmmu_copy_tsb(old_tsbinfo, new_tsbinfo); 9063 9064 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 9065 9066 /* 9067 * Drop the HAT lock to free our old tsb_info. 9068 */ 9069 sfmmu_hat_exit(hatlockp); 9070 9071 if ((flags & TSB_GROW) == TSB_GROW) { 9072 SFMMU_STAT(sf_tsb_grow); 9073 } else if ((flags & TSB_SHRINK) == TSB_SHRINK) { 9074 SFMMU_STAT(sf_tsb_shrink); 9075 } 9076 9077 sfmmu_tsbinfo_free(old_tsbinfo); 9078 9079 (void) sfmmu_hat_enter(sfmmup); 9080 return (TSB_SUCCESS); 9081 } 9082 9083 /* 9084 * This function will re-program hat pgsz array, and invalidate the 9085 * process' context, forcing the process to switch to another 9086 * context on the next TLB miss, and therefore start using the 9087 * TLB that is reprogrammed for the new page sizes. 9088 */ 9089 void 9090 sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) 9091 { 9092 int i; 9093 hatlock_t *hatlockp = NULL; 9094 9095 hatlockp = sfmmu_hat_enter(sfmmup); 9096 /* USIII+-IV+ optimization, requires hat lock */ 9097 if (tmp_pgsz) { 9098 for (i = 0; i < mmu_page_sizes; i++) 9099 sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i]; 9100 } 9101 SFMMU_STAT(sf_tlb_reprog_pgsz); 9102 9103 sfmmu_invalidate_ctx(sfmmup); 9104 9105 sfmmu_hat_exit(hatlockp); 9106 } 9107 9108 /* 9109 * This function assumes that there are either four or six supported page 9110 * sizes and at most two programmable TLBs, so we need to decide which 9111 * page sizes are most important and then tell the MMU layer so it 9112 * can adjust the TLB page sizes accordingly (if supported). 9113 * 9114 * If these assumptions change, this function will need to be 9115 * updated to support whatever the new limits are. 9116 * 9117 * The growing flag is nonzero if we are growing the address space, 9118 * and zero if it is shrinking. This allows us to decide whether 9119 * to grow or shrink our TSB, depending upon available memory 9120 * conditions. 9121 */ 9122 static void 9123 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing) 9124 { 9125 uint64_t ttecnt[MMU_PAGE_SIZES]; 9126 uint64_t tte8k_cnt, tte4m_cnt; 9127 uint8_t i; 9128 int sectsb_thresh; 9129 9130 /* 9131 * Kernel threads, processes with small address spaces not using 9132 * large pages, and dummy ISM HATs need not apply. 9133 */ 9134 if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != NULL) 9135 return; 9136 9137 if ((sfmmup->sfmmu_flags & HAT_LGPG_FLAGS) == 0 && 9138 sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor) 9139 return; 9140 9141 for (i = 0; i < mmu_page_sizes; i++) { 9142 ttecnt[i] = SFMMU_TTE_CNT(sfmmup, i); 9143 } 9144 9145 /* Check pagesizes in use, and possibly reprogram DTLB. */ 9146 if (&mmu_check_page_sizes) 9147 mmu_check_page_sizes(sfmmup, ttecnt); 9148 9149 /* 9150 * Calculate the number of 8k ttes to represent the span of these 9151 * pages. 9152 */ 9153 tte8k_cnt = ttecnt[TTE8K] + 9154 (ttecnt[TTE64K] << (MMU_PAGESHIFT64K - MMU_PAGESHIFT)) + 9155 (ttecnt[TTE512K] << (MMU_PAGESHIFT512K - MMU_PAGESHIFT)); 9156 if (mmu_page_sizes == max_mmu_page_sizes) { 9157 tte4m_cnt = ttecnt[TTE4M] + 9158 (ttecnt[TTE32M] << (MMU_PAGESHIFT32M - MMU_PAGESHIFT4M)) + 9159 (ttecnt[TTE256M] << (MMU_PAGESHIFT256M - MMU_PAGESHIFT4M)); 9160 } else { 9161 tte4m_cnt = ttecnt[TTE4M]; 9162 } 9163 9164 /* 9165 * Inflate TSB sizes by a factor of 2 if this process 9166 * uses 4M text pages to minimize extra conflict misses 9167 * in the first TSB since without counting text pages 9168 * 8K TSB may become too small. 9169 * 9170 * Also double the size of the second TSB to minimize 9171 * extra conflict misses due to competition between 4M text pages 9172 * and data pages. 9173 * 9174 * We need to adjust the second TSB allocation threshold by the 9175 * inflation factor, since there is no point in creating a second 9176 * TSB when we know all the mappings can fit in the I/D TLBs. 9177 */ 9178 sectsb_thresh = tsb_sectsb_threshold; 9179 if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) { 9180 tte8k_cnt <<= 1; 9181 tte4m_cnt <<= 1; 9182 sectsb_thresh <<= 1; 9183 } 9184 9185 /* 9186 * Check to see if our TSB is the right size; we may need to 9187 * grow or shrink it. If the process is small, our work is 9188 * finished at this point. 9189 */ 9190 if (tte8k_cnt <= tsb_rss_factor && tte4m_cnt <= sectsb_thresh) { 9191 return; 9192 } 9193 sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh); 9194 } 9195 9196 static void 9197 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt, 9198 uint64_t tte4m_cnt, int sectsb_thresh) 9199 { 9200 int tsb_bits; 9201 uint_t tsb_szc; 9202 struct tsb_info *tsbinfop; 9203 hatlock_t *hatlockp = NULL; 9204 9205 hatlockp = sfmmu_hat_enter(sfmmup); 9206 ASSERT(hatlockp != NULL); 9207 tsbinfop = sfmmup->sfmmu_tsb; 9208 ASSERT(tsbinfop != NULL); 9209 9210 /* 9211 * If we're growing, select the size based on RSS. If we're 9212 * shrinking, leave some room so we don't have to turn around and 9213 * grow again immediately. 9214 */ 9215 if (growing) 9216 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt); 9217 else 9218 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt << 1); 9219 9220 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9221 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9222 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9223 hatlockp, TSB_SHRINK); 9224 } else if (growing && tsb_szc > tsbinfop->tsb_szc && TSB_OK_GROW()) { 9225 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9226 hatlockp, TSB_GROW); 9227 } 9228 tsbinfop = sfmmup->sfmmu_tsb; 9229 9230 /* 9231 * With the TLB and first TSB out of the way, we need to see if 9232 * we need a second TSB for 4M pages. If we managed to reprogram 9233 * the TLB page sizes above, the process will start using this new 9234 * TSB right away; otherwise, it will start using it on the next 9235 * context switch. Either way, it's no big deal so there's no 9236 * synchronization with the trap handlers here unless we grow the 9237 * TSB (in which case it's required to prevent using the old one 9238 * after it's freed). Note: second tsb is required for 32M/256M 9239 * page sizes. 9240 */ 9241 if (tte4m_cnt > sectsb_thresh) { 9242 /* 9243 * If we're growing, select the size based on RSS. If we're 9244 * shrinking, leave some room so we don't have to turn 9245 * around and grow again immediately. 9246 */ 9247 if (growing) 9248 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt); 9249 else 9250 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt << 1); 9251 if (tsbinfop->tsb_next == NULL) { 9252 struct tsb_info *newtsb; 9253 int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)? 9254 0 : TSB_ALLOC; 9255 9256 sfmmu_hat_exit(hatlockp); 9257 9258 /* 9259 * Try to allocate a TSB for 4[32|256]M pages. If we 9260 * can't get the size we want, retry w/a minimum sized 9261 * TSB. If that still didn't work, give up; we can 9262 * still run without one. 9263 */ 9264 tsb_bits = (mmu_page_sizes == max_mmu_page_sizes)? 9265 TSB4M|TSB32M|TSB256M:TSB4M; 9266 if ((sfmmu_tsbinfo_alloc(&newtsb, tsb_szc, tsb_bits, 9267 allocflags, sfmmup) != 0) && 9268 (sfmmu_tsbinfo_alloc(&newtsb, TSB_MIN_SZCODE, 9269 tsb_bits, allocflags, sfmmup) != 0)) { 9270 return; 9271 } 9272 9273 hatlockp = sfmmu_hat_enter(sfmmup); 9274 9275 if (sfmmup->sfmmu_tsb->tsb_next == NULL) { 9276 sfmmup->sfmmu_tsb->tsb_next = newtsb; 9277 SFMMU_STAT(sf_tsb_sectsb_create); 9278 sfmmu_setup_tsbinfo(sfmmup); 9279 sfmmu_hat_exit(hatlockp); 9280 return; 9281 } else { 9282 /* 9283 * It's annoying, but possible for us 9284 * to get here.. we dropped the HAT lock 9285 * because of locking order in the kmem 9286 * allocator, and while we were off getting 9287 * our memory, some other thread decided to 9288 * do us a favor and won the race to get a 9289 * second TSB for this process. Sigh. 9290 */ 9291 sfmmu_hat_exit(hatlockp); 9292 sfmmu_tsbinfo_free(newtsb); 9293 return; 9294 } 9295 } 9296 9297 /* 9298 * We have a second TSB, see if it's big enough. 9299 */ 9300 tsbinfop = tsbinfop->tsb_next; 9301 9302 /* 9303 * Check to see if our second TSB is the right size; 9304 * we may need to grow or shrink it. 9305 * To prevent thrashing (e.g. growing the TSB on a 9306 * subsequent map operation), only try to shrink if 9307 * the TSB reach exceeds twice the virtual address 9308 * space size. 9309 */ 9310 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9311 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9312 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9313 tsb_szc, hatlockp, TSB_SHRINK); 9314 } else if (growing && tsb_szc > tsbinfop->tsb_szc && 9315 TSB_OK_GROW()) { 9316 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9317 tsb_szc, hatlockp, TSB_GROW); 9318 } 9319 } 9320 9321 sfmmu_hat_exit(hatlockp); 9322 } 9323 9324 /* 9325 * Free up a sfmmu 9326 * Since the sfmmu is currently embedded in the hat struct we simply zero 9327 * out our fields and free up the ism map blk list if any. 9328 */ 9329 static void 9330 sfmmu_free_sfmmu(sfmmu_t *sfmmup) 9331 { 9332 ism_blk_t *blkp, *nx_blkp; 9333 #ifdef DEBUG 9334 ism_map_t *map; 9335 int i; 9336 #endif 9337 9338 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 9339 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 9340 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 9341 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 9342 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 9343 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 9344 9345 sfmmup->sfmmu_free = 0; 9346 sfmmup->sfmmu_ismhat = 0; 9347 9348 blkp = sfmmup->sfmmu_iblk; 9349 sfmmup->sfmmu_iblk = NULL; 9350 9351 while (blkp) { 9352 #ifdef DEBUG 9353 map = blkp->iblk_maps; 9354 for (i = 0; i < ISM_MAP_SLOTS; i++) { 9355 ASSERT(map[i].imap_seg == 0); 9356 ASSERT(map[i].imap_ismhat == NULL); 9357 ASSERT(map[i].imap_ment == NULL); 9358 } 9359 #endif 9360 nx_blkp = blkp->iblk_next; 9361 blkp->iblk_next = NULL; 9362 blkp->iblk_nextpa = (uint64_t)-1; 9363 kmem_cache_free(ism_blk_cache, blkp); 9364 blkp = nx_blkp; 9365 } 9366 } 9367 9368 /* 9369 * Locking primitves accessed by HATLOCK macros 9370 */ 9371 9372 #define SFMMU_SPL_MTX (0x0) 9373 #define SFMMU_ML_MTX (0x1) 9374 9375 #define SFMMU_MLSPL_MTX(type, pg) (((type) == SFMMU_SPL_MTX) ? \ 9376 SPL_HASH(pg) : MLIST_HASH(pg)) 9377 9378 kmutex_t * 9379 sfmmu_page_enter(struct page *pp) 9380 { 9381 return (sfmmu_mlspl_enter(pp, SFMMU_SPL_MTX)); 9382 } 9383 9384 void 9385 sfmmu_page_exit(kmutex_t *spl) 9386 { 9387 mutex_exit(spl); 9388 } 9389 9390 int 9391 sfmmu_page_spl_held(struct page *pp) 9392 { 9393 return (sfmmu_mlspl_held(pp, SFMMU_SPL_MTX)); 9394 } 9395 9396 kmutex_t * 9397 sfmmu_mlist_enter(struct page *pp) 9398 { 9399 return (sfmmu_mlspl_enter(pp, SFMMU_ML_MTX)); 9400 } 9401 9402 void 9403 sfmmu_mlist_exit(kmutex_t *mml) 9404 { 9405 mutex_exit(mml); 9406 } 9407 9408 int 9409 sfmmu_mlist_held(struct page *pp) 9410 { 9411 9412 return (sfmmu_mlspl_held(pp, SFMMU_ML_MTX)); 9413 } 9414 9415 /* 9416 * Common code for sfmmu_mlist_enter() and sfmmu_page_enter(). For 9417 * sfmmu_mlist_enter() case mml_table lock array is used and for 9418 * sfmmu_page_enter() sfmmu_page_lock lock array is used. 9419 * 9420 * The lock is taken on a root page so that it protects an operation on all 9421 * constituent pages of a large page pp belongs to. 9422 * 9423 * The routine takes a lock from the appropriate array. The lock is determined 9424 * by hashing the root page. After taking the lock this routine checks if the 9425 * root page has the same size code that was used to determine the root (i.e 9426 * that root hasn't changed). If root page has the expected p_szc field we 9427 * have the right lock and it's returned to the caller. If root's p_szc 9428 * decreased we release the lock and retry from the beginning. This case can 9429 * happen due to hat_page_demote() decreasing p_szc between our load of p_szc 9430 * value and taking the lock. The number of retries due to p_szc decrease is 9431 * limited by the maximum p_szc value. If p_szc is 0 we return the lock 9432 * determined by hashing pp itself. 9433 * 9434 * If our caller doesn't hold a SE_SHARED or SE_EXCL lock on pp it's also 9435 * possible that p_szc can increase. To increase p_szc a thread has to lock 9436 * all constituent pages EXCL and do hat_pageunload() on all of them. All the 9437 * callers that don't hold a page locked recheck if hmeblk through which pp 9438 * was found still maps this pp. If it doesn't map it anymore returned lock 9439 * is immediately dropped. Therefore if sfmmu_mlspl_enter() hits the case of 9440 * p_szc increase after taking the lock it returns this lock without further 9441 * retries because in this case the caller doesn't care about which lock was 9442 * taken. The caller will drop it right away. 9443 * 9444 * After the routine returns it's guaranteed that hat_page_demote() can't 9445 * change p_szc field of any of constituent pages of a large page pp belongs 9446 * to as long as pp was either locked at least SHARED prior to this call or 9447 * the caller finds that hment that pointed to this pp still references this 9448 * pp (this also assumes that the caller holds hme hash bucket lock so that 9449 * the same pp can't be remapped into the same hmeblk after it was unmapped by 9450 * hat_pageunload()). 9451 */ 9452 static kmutex_t * 9453 sfmmu_mlspl_enter(struct page *pp, int type) 9454 { 9455 kmutex_t *mtx; 9456 uint_t prev_rszc = UINT_MAX; 9457 page_t *rootpp; 9458 uint_t szc; 9459 uint_t rszc; 9460 uint_t pszc = pp->p_szc; 9461 9462 ASSERT(pp != NULL); 9463 9464 again: 9465 if (pszc == 0) { 9466 mtx = SFMMU_MLSPL_MTX(type, pp); 9467 mutex_enter(mtx); 9468 return (mtx); 9469 } 9470 9471 /* The lock lives in the root page */ 9472 rootpp = PP_GROUPLEADER(pp, pszc); 9473 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9474 mutex_enter(mtx); 9475 9476 /* 9477 * Return mml in the following 3 cases: 9478 * 9479 * 1) If pp itself is root since if its p_szc decreased before we took 9480 * the lock pp is still the root of smaller szc page. And if its p_szc 9481 * increased it doesn't matter what lock we return (see comment in 9482 * front of this routine). 9483 * 9484 * 2) If pp's not root but rootpp is the root of a rootpp->p_szc size 9485 * large page we have the right lock since any previous potential 9486 * hat_page_demote() is done demoting from greater than current root's 9487 * p_szc because hat_page_demote() changes root's p_szc last. No 9488 * further hat_page_demote() can start or be in progress since it 9489 * would need the same lock we currently hold. 9490 * 9491 * 3) If rootpp's p_szc increased since previous iteration it doesn't 9492 * matter what lock we return (see comment in front of this routine). 9493 */ 9494 if (pp == rootpp || (rszc = rootpp->p_szc) == pszc || 9495 rszc >= prev_rszc) { 9496 return (mtx); 9497 } 9498 9499 /* 9500 * hat_page_demote() could have decreased root's p_szc. 9501 * In this case pp's p_szc must also be smaller than pszc. 9502 * Retry. 9503 */ 9504 if (rszc < pszc) { 9505 szc = pp->p_szc; 9506 if (szc < pszc) { 9507 mutex_exit(mtx); 9508 pszc = szc; 9509 goto again; 9510 } 9511 /* 9512 * pp's p_szc increased after it was decreased. 9513 * page cannot be mapped. Return current lock. The caller 9514 * will drop it right away. 9515 */ 9516 return (mtx); 9517 } 9518 9519 /* 9520 * root's p_szc is greater than pp's p_szc. 9521 * hat_page_demote() is not done with all pages 9522 * yet. Wait for it to complete. 9523 */ 9524 mutex_exit(mtx); 9525 rootpp = PP_GROUPLEADER(rootpp, rszc); 9526 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9527 mutex_enter(mtx); 9528 mutex_exit(mtx); 9529 prev_rszc = rszc; 9530 goto again; 9531 } 9532 9533 static int 9534 sfmmu_mlspl_held(struct page *pp, int type) 9535 { 9536 kmutex_t *mtx; 9537 9538 ASSERT(pp != NULL); 9539 /* The lock lives in the root page */ 9540 pp = PP_PAGEROOT(pp); 9541 ASSERT(pp != NULL); 9542 9543 mtx = SFMMU_MLSPL_MTX(type, pp); 9544 return (MUTEX_HELD(mtx)); 9545 } 9546 9547 static uint_t 9548 sfmmu_get_free_hblk(struct hme_blk **hmeblkpp, uint_t critical) 9549 { 9550 struct hme_blk *hblkp; 9551 9552 if (freehblkp != NULL) { 9553 mutex_enter(&freehblkp_lock); 9554 if (freehblkp != NULL) { 9555 /* 9556 * If the current thread is owning hblk_reserve OR 9557 * critical request from sfmmu_hblk_steal() 9558 * let it succeed even if freehblkcnt is really low. 9559 */ 9560 if (freehblkcnt <= HBLK_RESERVE_MIN && !critical) { 9561 SFMMU_STAT(sf_get_free_throttle); 9562 mutex_exit(&freehblkp_lock); 9563 return (0); 9564 } 9565 freehblkcnt--; 9566 *hmeblkpp = freehblkp; 9567 hblkp = *hmeblkpp; 9568 freehblkp = hblkp->hblk_next; 9569 mutex_exit(&freehblkp_lock); 9570 hblkp->hblk_next = NULL; 9571 SFMMU_STAT(sf_get_free_success); 9572 return (1); 9573 } 9574 mutex_exit(&freehblkp_lock); 9575 } 9576 SFMMU_STAT(sf_get_free_fail); 9577 return (0); 9578 } 9579 9580 static uint_t 9581 sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical) 9582 { 9583 struct hme_blk *hblkp; 9584 9585 /* 9586 * If the current thread is mapping into kernel space, 9587 * let it succede even if freehblkcnt is max 9588 * so that it will avoid freeing it to kmem. 9589 * This will prevent stack overflow due to 9590 * possible recursion since kmem_cache_free() 9591 * might require creation of a slab which 9592 * in turn needs an hmeblk to map that slab; 9593 * let's break this vicious chain at the first 9594 * opportunity. 9595 */ 9596 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9597 mutex_enter(&freehblkp_lock); 9598 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9599 SFMMU_STAT(sf_put_free_success); 9600 freehblkcnt++; 9601 hmeblkp->hblk_next = freehblkp; 9602 freehblkp = hmeblkp; 9603 mutex_exit(&freehblkp_lock); 9604 return (1); 9605 } 9606 mutex_exit(&freehblkp_lock); 9607 } 9608 9609 /* 9610 * Bring down freehblkcnt to HBLK_RESERVE_CNT. We are here 9611 * only if freehblkcnt is at least HBLK_RESERVE_CNT *and* 9612 * we are not in the process of mapping into kernel space. 9613 */ 9614 ASSERT(!critical); 9615 while (freehblkcnt > HBLK_RESERVE_CNT) { 9616 mutex_enter(&freehblkp_lock); 9617 if (freehblkcnt > HBLK_RESERVE_CNT) { 9618 freehblkcnt--; 9619 hblkp = freehblkp; 9620 freehblkp = hblkp->hblk_next; 9621 mutex_exit(&freehblkp_lock); 9622 ASSERT(get_hblk_cache(hblkp) == sfmmu8_cache); 9623 kmem_cache_free(sfmmu8_cache, hblkp); 9624 continue; 9625 } 9626 mutex_exit(&freehblkp_lock); 9627 } 9628 SFMMU_STAT(sf_put_free_fail); 9629 return (0); 9630 } 9631 9632 static void 9633 sfmmu_hblk_swap(struct hme_blk *new) 9634 { 9635 struct hme_blk *old, *hblkp, *prev; 9636 uint64_t hblkpa, prevpa, newpa; 9637 caddr_t base, vaddr, endaddr; 9638 struct hmehash_bucket *hmebp; 9639 struct sf_hment *osfhme, *nsfhme; 9640 page_t *pp; 9641 kmutex_t *pml; 9642 tte_t tte; 9643 9644 #ifdef DEBUG 9645 hmeblk_tag hblktag; 9646 struct hme_blk *found; 9647 #endif 9648 old = HBLK_RESERVE; 9649 9650 /* 9651 * save pa before bcopy clobbers it 9652 */ 9653 newpa = new->hblk_nextpa; 9654 9655 base = (caddr_t)get_hblk_base(old); 9656 endaddr = base + get_hblk_span(old); 9657 9658 /* 9659 * acquire hash bucket lock. 9660 */ 9661 hmebp = sfmmu_tteload_acquire_hashbucket(ksfmmup, base, TTE8K); 9662 9663 /* 9664 * copy contents from old to new 9665 */ 9666 bcopy((void *)old, (void *)new, HME8BLK_SZ); 9667 9668 /* 9669 * add new to hash chain 9670 */ 9671 sfmmu_hblk_hash_add(hmebp, new, newpa); 9672 9673 /* 9674 * search hash chain for hblk_reserve; this needs to be performed 9675 * after adding new, otherwise prevpa and prev won't correspond 9676 * to the hblk which is prior to old in hash chain when we call 9677 * sfmmu_hblk_hash_rm to remove old later. 9678 */ 9679 for (prevpa = 0, prev = NULL, 9680 hblkpa = hmebp->hmeh_nextpa, hblkp = hmebp->hmeblkp; 9681 hblkp != NULL && hblkp != old; 9682 prevpa = hblkpa, prev = hblkp, 9683 hblkpa = hblkp->hblk_nextpa, hblkp = hblkp->hblk_next); 9684 9685 if (hblkp != old) 9686 panic("sfmmu_hblk_swap: hblk_reserve not found"); 9687 9688 /* 9689 * p_mapping list is still pointing to hments in hblk_reserve; 9690 * fix up p_mapping list so that they point to hments in new. 9691 * 9692 * Since all these mappings are created by hblk_reserve_thread 9693 * on the way and it's using at least one of the buffers from each of 9694 * the newly minted slabs, there is no danger of any of these 9695 * mappings getting unloaded by another thread. 9696 * 9697 * tsbmiss could only modify ref/mod bits of hments in old/new. 9698 * Since all of these hments hold mappings established by segkmem 9699 * and mappings in segkmem are setup with HAT_NOSYNC, ref/mod bits 9700 * have no meaning for the mappings in hblk_reserve. hments in 9701 * old and new are identical except for ref/mod bits. 9702 */ 9703 for (vaddr = base; vaddr < endaddr; vaddr += TTEBYTES(TTE8K)) { 9704 9705 HBLKTOHME(osfhme, old, vaddr); 9706 sfmmu_copytte(&osfhme->hme_tte, &tte); 9707 9708 if (TTE_IS_VALID(&tte)) { 9709 if ((pp = osfhme->hme_page) == NULL) 9710 panic("sfmmu_hblk_swap: page not mapped"); 9711 9712 pml = sfmmu_mlist_enter(pp); 9713 9714 if (pp != osfhme->hme_page) 9715 panic("sfmmu_hblk_swap: mapping changed"); 9716 9717 HBLKTOHME(nsfhme, new, vaddr); 9718 9719 HME_ADD(nsfhme, pp); 9720 HME_SUB(osfhme, pp); 9721 9722 sfmmu_mlist_exit(pml); 9723 } 9724 } 9725 9726 /* 9727 * remove old from hash chain 9728 */ 9729 sfmmu_hblk_hash_rm(hmebp, old, prevpa, prev); 9730 9731 #ifdef DEBUG 9732 9733 hblktag.htag_id = ksfmmup; 9734 hblktag.htag_bspage = HME_HASH_BSPAGE(base, HME_HASH_SHIFT(TTE8K)); 9735 hblktag.htag_rehash = HME_HASH_REHASH(TTE8K); 9736 HME_HASH_FAST_SEARCH(hmebp, hblktag, found); 9737 9738 if (found != new) 9739 panic("sfmmu_hblk_swap: new hblk not found"); 9740 #endif 9741 9742 SFMMU_HASH_UNLOCK(hmebp); 9743 9744 /* 9745 * Reset hblk_reserve 9746 */ 9747 bzero((void *)old, HME8BLK_SZ); 9748 old->hblk_nextpa = va_to_pa((caddr_t)old); 9749 } 9750 9751 /* 9752 * Grab the mlist mutex for both pages passed in. 9753 * 9754 * low and high will be returned as pointers to the mutexes for these pages. 9755 * low refers to the mutex residing in the lower bin of the mlist hash, while 9756 * high refers to the mutex residing in the higher bin of the mlist hash. This 9757 * is due to the locking order restrictions on the same thread grabbing 9758 * multiple mlist mutexes. The low lock must be acquired before the high lock. 9759 * 9760 * If both pages hash to the same mutex, only grab that single mutex, and 9761 * high will be returned as NULL 9762 * If the pages hash to different bins in the hash, grab the lower addressed 9763 * lock first and then the higher addressed lock in order to follow the locking 9764 * rules involved with the same thread grabbing multiple mlist mutexes. 9765 * low and high will both have non-NULL values. 9766 */ 9767 static void 9768 sfmmu_mlist_reloc_enter(struct page *targ, struct page *repl, 9769 kmutex_t **low, kmutex_t **high) 9770 { 9771 kmutex_t *mml_targ, *mml_repl; 9772 9773 /* 9774 * no need to do the dance around szc as in sfmmu_mlist_enter() 9775 * because this routine is only called by hat_page_relocate() and all 9776 * targ and repl pages are already locked EXCL so szc can't change. 9777 */ 9778 9779 mml_targ = MLIST_HASH(PP_PAGEROOT(targ)); 9780 mml_repl = MLIST_HASH(PP_PAGEROOT(repl)); 9781 9782 if (mml_targ == mml_repl) { 9783 *low = mml_targ; 9784 *high = NULL; 9785 } else { 9786 if (mml_targ < mml_repl) { 9787 *low = mml_targ; 9788 *high = mml_repl; 9789 } else { 9790 *low = mml_repl; 9791 *high = mml_targ; 9792 } 9793 } 9794 9795 mutex_enter(*low); 9796 if (*high) 9797 mutex_enter(*high); 9798 } 9799 9800 static void 9801 sfmmu_mlist_reloc_exit(kmutex_t *low, kmutex_t *high) 9802 { 9803 if (high) 9804 mutex_exit(high); 9805 mutex_exit(low); 9806 } 9807 9808 static hatlock_t * 9809 sfmmu_hat_enter(sfmmu_t *sfmmup) 9810 { 9811 hatlock_t *hatlockp; 9812 9813 if (sfmmup != ksfmmup) { 9814 hatlockp = TSB_HASH(sfmmup); 9815 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 9816 return (hatlockp); 9817 } 9818 return (NULL); 9819 } 9820 9821 static hatlock_t * 9822 sfmmu_hat_tryenter(sfmmu_t *sfmmup) 9823 { 9824 hatlock_t *hatlockp; 9825 9826 if (sfmmup != ksfmmup) { 9827 hatlockp = TSB_HASH(sfmmup); 9828 if (mutex_tryenter(HATLOCK_MUTEXP(hatlockp)) == 0) 9829 return (NULL); 9830 return (hatlockp); 9831 } 9832 return (NULL); 9833 } 9834 9835 static void 9836 sfmmu_hat_exit(hatlock_t *hatlockp) 9837 { 9838 if (hatlockp != NULL) 9839 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 9840 } 9841 9842 static void 9843 sfmmu_hat_lock_all(void) 9844 { 9845 int i; 9846 for (i = 0; i < SFMMU_NUM_LOCK; i++) 9847 mutex_enter(HATLOCK_MUTEXP(&hat_lock[i])); 9848 } 9849 9850 static void 9851 sfmmu_hat_unlock_all(void) 9852 { 9853 int i; 9854 for (i = SFMMU_NUM_LOCK - 1; i >= 0; i--) 9855 mutex_exit(HATLOCK_MUTEXP(&hat_lock[i])); 9856 } 9857 9858 int 9859 sfmmu_hat_lock_held(sfmmu_t *sfmmup) 9860 { 9861 ASSERT(sfmmup != ksfmmup); 9862 return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup)))); 9863 } 9864 9865 /* 9866 * Locking primitives to provide consistency between ISM unmap 9867 * and other operations. Since ISM unmap can take a long time, we 9868 * use HAT_ISMBUSY flag (protected by the hatlock) to avoid creating 9869 * contention on the hatlock buckets while ISM segments are being 9870 * unmapped. The tradeoff is that the flags don't prevent priority 9871 * inversion from occurring, so we must request kernel priority in 9872 * case we have to sleep to keep from getting buried while holding 9873 * the HAT_ISMBUSY flag set, which in turn could block other kernel 9874 * threads from running (for example, in sfmmu_uvatopfn()). 9875 */ 9876 static void 9877 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held) 9878 { 9879 hatlock_t *hatlockp; 9880 9881 THREAD_KPRI_REQUEST(); 9882 if (!hatlock_held) 9883 hatlockp = sfmmu_hat_enter(sfmmup); 9884 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) 9885 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 9886 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY); 9887 if (!hatlock_held) 9888 sfmmu_hat_exit(hatlockp); 9889 } 9890 9891 static void 9892 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held) 9893 { 9894 hatlock_t *hatlockp; 9895 9896 if (!hatlock_held) 9897 hatlockp = sfmmu_hat_enter(sfmmup); 9898 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 9899 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY); 9900 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 9901 if (!hatlock_held) 9902 sfmmu_hat_exit(hatlockp); 9903 THREAD_KPRI_RELEASE(); 9904 } 9905 9906 /* 9907 * 9908 * Algorithm: 9909 * 9910 * (1) if segkmem is not ready, allocate hblk from an array of pre-alloc'ed 9911 * hblks. 9912 * 9913 * (2) if we are allocating an hblk for mapping a slab in sfmmu_cache, 9914 * 9915 * (a) try to return an hblk from reserve pool of free hblks; 9916 * (b) if the reserve pool is empty, acquire hblk_reserve_lock 9917 * and return hblk_reserve. 9918 * 9919 * (3) call kmem_cache_alloc() to allocate hblk; 9920 * 9921 * (a) if hblk_reserve_lock is held by the current thread, 9922 * atomically replace hblk_reserve by the hblk that is 9923 * returned by kmem_cache_alloc; release hblk_reserve_lock 9924 * and call kmem_cache_alloc() again. 9925 * (b) if reserve pool is not full, add the hblk that is 9926 * returned by kmem_cache_alloc to reserve pool and 9927 * call kmem_cache_alloc again. 9928 * 9929 */ 9930 static struct hme_blk * 9931 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr, 9932 struct hmehash_bucket *hmebp, uint_t size, hmeblk_tag hblktag, 9933 uint_t flags) 9934 { 9935 struct hme_blk *hmeblkp = NULL; 9936 struct hme_blk *newhblkp; 9937 struct hme_blk *shw_hblkp = NULL; 9938 struct kmem_cache *sfmmu_cache = NULL; 9939 uint64_t hblkpa; 9940 ulong_t index; 9941 uint_t owner; /* set to 1 if using hblk_reserve */ 9942 uint_t forcefree; 9943 int sleep; 9944 9945 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 9946 9947 /* 9948 * If segkmem is not created yet, allocate from static hmeblks 9949 * created at the end of startup_modules(). See the block comment 9950 * in startup_modules() describing how we estimate the number of 9951 * static hmeblks that will be needed during re-map. 9952 */ 9953 if (!hblk_alloc_dynamic) { 9954 9955 if (size == TTE8K) { 9956 index = nucleus_hblk8.index; 9957 if (index >= nucleus_hblk8.len) { 9958 /* 9959 * If we panic here, see startup_modules() to 9960 * make sure that we are calculating the 9961 * number of hblk8's that we need correctly. 9962 */ 9963 prom_panic("no nucleus hblk8 to allocate"); 9964 } 9965 hmeblkp = 9966 (struct hme_blk *)&nucleus_hblk8.list[index]; 9967 nucleus_hblk8.index++; 9968 SFMMU_STAT(sf_hblk8_nalloc); 9969 } else { 9970 index = nucleus_hblk1.index; 9971 if (nucleus_hblk1.index >= nucleus_hblk1.len) { 9972 /* 9973 * If we panic here, see startup_modules(). 9974 * Most likely you need to update the 9975 * calculation of the number of hblk1 elements 9976 * that the kernel needs to boot. 9977 */ 9978 prom_panic("no nucleus hblk1 to allocate"); 9979 } 9980 hmeblkp = 9981 (struct hme_blk *)&nucleus_hblk1.list[index]; 9982 nucleus_hblk1.index++; 9983 SFMMU_STAT(sf_hblk1_nalloc); 9984 } 9985 9986 goto hblk_init; 9987 } 9988 9989 SFMMU_HASH_UNLOCK(hmebp); 9990 9991 if (sfmmup != KHATID) { 9992 if (mmu_page_sizes == max_mmu_page_sizes) { 9993 if (size < TTE256M) 9994 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 9995 size, flags); 9996 } else { 9997 if (size < TTE4M) 9998 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 9999 size, flags); 10000 } 10001 } 10002 10003 fill_hblk: 10004 owner = (hblk_reserve_thread == curthread) ? 1 : 0; 10005 10006 if (owner && size == TTE8K) { 10007 10008 /* 10009 * We are really in a tight spot. We already own 10010 * hblk_reserve and we need another hblk. In anticipation 10011 * of this kind of scenario, we specifically set aside 10012 * HBLK_RESERVE_MIN number of hblks to be used exclusively 10013 * by owner of hblk_reserve. 10014 */ 10015 SFMMU_STAT(sf_hblk_recurse_cnt); 10016 10017 if (!sfmmu_get_free_hblk(&hmeblkp, 1)) 10018 panic("sfmmu_hblk_alloc: reserve list is empty"); 10019 10020 goto hblk_verify; 10021 } 10022 10023 ASSERT(!owner); 10024 10025 if ((flags & HAT_NO_KALLOC) == 0) { 10026 10027 sfmmu_cache = ((size == TTE8K) ? sfmmu8_cache : sfmmu1_cache); 10028 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP); 10029 10030 if ((hmeblkp = kmem_cache_alloc(sfmmu_cache, sleep)) == NULL) { 10031 hmeblkp = sfmmu_hblk_steal(size); 10032 } else { 10033 /* 10034 * if we are the owner of hblk_reserve, 10035 * swap hblk_reserve with hmeblkp and 10036 * start a fresh life. Hope things go 10037 * better this time. 10038 */ 10039 if (hblk_reserve_thread == curthread) { 10040 ASSERT(sfmmu_cache == sfmmu8_cache); 10041 sfmmu_hblk_swap(hmeblkp); 10042 hblk_reserve_thread = NULL; 10043 mutex_exit(&hblk_reserve_lock); 10044 goto fill_hblk; 10045 } 10046 /* 10047 * let's donate this hblk to our reserve list if 10048 * we are not mapping kernel range 10049 */ 10050 if (size == TTE8K && sfmmup != KHATID) 10051 if (sfmmu_put_free_hblk(hmeblkp, 0)) 10052 goto fill_hblk; 10053 } 10054 } else { 10055 /* 10056 * We are here to map the slab in sfmmu8_cache; let's 10057 * check if we could tap our reserve list; if successful, 10058 * this will avoid the pain of going thru sfmmu_hblk_swap 10059 */ 10060 SFMMU_STAT(sf_hblk_slab_cnt); 10061 if (!sfmmu_get_free_hblk(&hmeblkp, 0)) { 10062 /* 10063 * let's start hblk_reserve dance 10064 */ 10065 SFMMU_STAT(sf_hblk_reserve_cnt); 10066 owner = 1; 10067 mutex_enter(&hblk_reserve_lock); 10068 hmeblkp = HBLK_RESERVE; 10069 hblk_reserve_thread = curthread; 10070 } 10071 } 10072 10073 hblk_verify: 10074 ASSERT(hmeblkp != NULL); 10075 set_hblk_sz(hmeblkp, size); 10076 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); 10077 SFMMU_HASH_LOCK(hmebp); 10078 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10079 if (newhblkp != NULL) { 10080 SFMMU_HASH_UNLOCK(hmebp); 10081 if (hmeblkp != HBLK_RESERVE) { 10082 /* 10083 * This is really tricky! 10084 * 10085 * vmem_alloc(vmem_seg_arena) 10086 * vmem_alloc(vmem_internal_arena) 10087 * segkmem_alloc(heap_arena) 10088 * vmem_alloc(heap_arena) 10089 * page_create() 10090 * hat_memload() 10091 * kmem_cache_free() 10092 * kmem_cache_alloc() 10093 * kmem_slab_create() 10094 * vmem_alloc(kmem_internal_arena) 10095 * segkmem_alloc(heap_arena) 10096 * vmem_alloc(heap_arena) 10097 * page_create() 10098 * hat_memload() 10099 * kmem_cache_free() 10100 * ... 10101 * 10102 * Thus, hat_memload() could call kmem_cache_free 10103 * for enough number of times that we could easily 10104 * hit the bottom of the stack or run out of reserve 10105 * list of vmem_seg structs. So, we must donate 10106 * this hblk to reserve list if it's allocated 10107 * from sfmmu8_cache *and* mapping kernel range. 10108 * We don't need to worry about freeing hmeblk1's 10109 * to kmem since they don't map any kmem slabs. 10110 * 10111 * Note: When segkmem supports largepages, we must 10112 * free hmeblk1's to reserve list as well. 10113 */ 10114 forcefree = (sfmmup == KHATID) ? 1 : 0; 10115 if (size == TTE8K && 10116 sfmmu_put_free_hblk(hmeblkp, forcefree)) { 10117 goto re_verify; 10118 } 10119 ASSERT(sfmmup != KHATID); 10120 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10121 } else { 10122 /* 10123 * Hey! we don't need hblk_reserve any more. 10124 */ 10125 ASSERT(owner); 10126 hblk_reserve_thread = NULL; 10127 mutex_exit(&hblk_reserve_lock); 10128 owner = 0; 10129 } 10130 re_verify: 10131 /* 10132 * let's check if the goodies are still present 10133 */ 10134 SFMMU_HASH_LOCK(hmebp); 10135 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10136 if (newhblkp != NULL) { 10137 /* 10138 * return newhblkp if it's not hblk_reserve; 10139 * if newhblkp is hblk_reserve, return it 10140 * _only if_ we are the owner of hblk_reserve. 10141 */ 10142 if (newhblkp != HBLK_RESERVE || owner) { 10143 return (newhblkp); 10144 } else { 10145 /* 10146 * we just hit hblk_reserve in the hash and 10147 * we are not the owner of that; 10148 * 10149 * block until hblk_reserve_thread completes 10150 * swapping hblk_reserve and try the dance 10151 * once again. 10152 */ 10153 SFMMU_HASH_UNLOCK(hmebp); 10154 mutex_enter(&hblk_reserve_lock); 10155 mutex_exit(&hblk_reserve_lock); 10156 SFMMU_STAT(sf_hblk_reserve_hit); 10157 goto fill_hblk; 10158 } 10159 } else { 10160 /* 10161 * it's no more! try the dance once again. 10162 */ 10163 SFMMU_HASH_UNLOCK(hmebp); 10164 goto fill_hblk; 10165 } 10166 } 10167 10168 hblk_init: 10169 set_hblk_sz(hmeblkp, size); 10170 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10171 hmeblkp->hblk_next = (struct hme_blk *)NULL; 10172 hmeblkp->hblk_tag = hblktag; 10173 hmeblkp->hblk_shadow = shw_hblkp; 10174 hblkpa = hmeblkp->hblk_nextpa; 10175 hmeblkp->hblk_nextpa = 0; 10176 10177 ASSERT(get_hblk_ttesz(hmeblkp) == size); 10178 ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size)); 10179 ASSERT(hmeblkp->hblk_hmecnt == 0); 10180 ASSERT(hmeblkp->hblk_vcnt == 0); 10181 ASSERT(hmeblkp->hblk_lckcnt == 0); 10182 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10183 sfmmu_hblk_hash_add(hmebp, hmeblkp, hblkpa); 10184 return (hmeblkp); 10185 } 10186 10187 /* 10188 * This function performs any cleanup required on the hme_blk 10189 * and returns it to the free list. 10190 */ 10191 /* ARGSUSED */ 10192 static void 10193 sfmmu_hblk_free(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10194 uint64_t hblkpa, struct hme_blk **listp) 10195 { 10196 int shw_size, vshift; 10197 struct hme_blk *shw_hblkp; 10198 uint_t shw_mask, newshw_mask; 10199 uintptr_t vaddr; 10200 int size; 10201 uint_t critical; 10202 10203 ASSERT(hmeblkp); 10204 ASSERT(!hmeblkp->hblk_hmecnt); 10205 ASSERT(!hmeblkp->hblk_vcnt); 10206 ASSERT(!hmeblkp->hblk_lckcnt); 10207 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10208 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 10209 10210 critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0; 10211 10212 size = get_hblk_ttesz(hmeblkp); 10213 shw_hblkp = hmeblkp->hblk_shadow; 10214 if (shw_hblkp) { 10215 ASSERT(hblktosfmmu(hmeblkp) != KHATID); 10216 if (mmu_page_sizes == max_mmu_page_sizes) { 10217 ASSERT(size < TTE256M); 10218 } else { 10219 ASSERT(size < TTE4M); 10220 } 10221 10222 shw_size = get_hblk_ttesz(shw_hblkp); 10223 vaddr = get_hblk_base(hmeblkp); 10224 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10225 ASSERT(vshift < 8); 10226 /* 10227 * Atomically clear shadow mask bit 10228 */ 10229 do { 10230 shw_mask = shw_hblkp->hblk_shw_mask; 10231 ASSERT(shw_mask & (1 << vshift)); 10232 newshw_mask = shw_mask & ~(1 << vshift); 10233 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10234 shw_mask, newshw_mask); 10235 } while (newshw_mask != shw_mask); 10236 hmeblkp->hblk_shadow = NULL; 10237 } 10238 hmeblkp->hblk_next = NULL; 10239 hmeblkp->hblk_nextpa = hblkpa; 10240 hmeblkp->hblk_shw_bit = 0; 10241 10242 if (hmeblkp->hblk_nuc_bit == 0) { 10243 10244 if (size == TTE8K && sfmmu_put_free_hblk(hmeblkp, critical)) 10245 return; 10246 10247 hmeblkp->hblk_next = *listp; 10248 *listp = hmeblkp; 10249 } 10250 } 10251 10252 static void 10253 sfmmu_hblks_list_purge(struct hme_blk **listp) 10254 { 10255 struct hme_blk *hmeblkp; 10256 10257 while ((hmeblkp = *listp) != NULL) { 10258 *listp = hmeblkp->hblk_next; 10259 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10260 } 10261 } 10262 10263 #define BUCKETS_TO_SEARCH_BEFORE_UNLOAD 30 10264 #define SFMMU_HBLK_STEAL_THRESHOLD 5 10265 10266 static uint_t sfmmu_hblk_steal_twice; 10267 static uint_t sfmmu_hblk_steal_count, sfmmu_hblk_steal_unload_count; 10268 10269 /* 10270 * Steal a hmeblk from user or kernel hme hash lists. 10271 * For 8K tte grab one from reserve pool (freehblkp) before proceeding to 10272 * steal and if we fail to steal after SFMMU_HBLK_STEAL_THRESHOLD attempts 10273 * tap into critical reserve of freehblkp. 10274 * Note: We remain looping in this routine until we find one. 10275 */ 10276 static struct hme_blk * 10277 sfmmu_hblk_steal(int size) 10278 { 10279 static struct hmehash_bucket *uhmehash_steal_hand = NULL; 10280 struct hmehash_bucket *hmebp; 10281 struct hme_blk *hmeblkp = NULL, *pr_hblk; 10282 uint64_t hblkpa, prevpa; 10283 int i; 10284 uint_t loop_cnt = 0, critical; 10285 10286 for (;;) { 10287 if (size == TTE8K) { 10288 critical = 10289 (++loop_cnt > SFMMU_HBLK_STEAL_THRESHOLD) ? 1 : 0; 10290 if (sfmmu_get_free_hblk(&hmeblkp, critical)) 10291 return (hmeblkp); 10292 } 10293 10294 hmebp = (uhmehash_steal_hand == NULL) ? uhme_hash : 10295 uhmehash_steal_hand; 10296 ASSERT(hmebp >= uhme_hash && hmebp <= &uhme_hash[UHMEHASH_SZ]); 10297 10298 for (i = 0; hmeblkp == NULL && i <= UHMEHASH_SZ + 10299 BUCKETS_TO_SEARCH_BEFORE_UNLOAD; i++) { 10300 SFMMU_HASH_LOCK(hmebp); 10301 hmeblkp = hmebp->hmeblkp; 10302 hblkpa = hmebp->hmeh_nextpa; 10303 prevpa = 0; 10304 pr_hblk = NULL; 10305 while (hmeblkp) { 10306 /* 10307 * check if it is a hmeblk that is not locked 10308 * and not shared. skip shadow hmeblks with 10309 * shadow_mask set i.e valid count non zero. 10310 */ 10311 if ((get_hblk_ttesz(hmeblkp) == size) && 10312 (hmeblkp->hblk_shw_bit == 0 || 10313 hmeblkp->hblk_vcnt == 0) && 10314 (hmeblkp->hblk_lckcnt == 0)) { 10315 /* 10316 * there is a high probability that we 10317 * will find a free one. search some 10318 * buckets for a free hmeblk initially 10319 * before unloading a valid hmeblk. 10320 */ 10321 if ((hmeblkp->hblk_vcnt == 0 && 10322 hmeblkp->hblk_hmecnt == 0) || (i >= 10323 BUCKETS_TO_SEARCH_BEFORE_UNLOAD)) { 10324 if (sfmmu_steal_this_hblk(hmebp, 10325 hmeblkp, hblkpa, prevpa, 10326 pr_hblk)) { 10327 /* 10328 * Hblk is unloaded 10329 * successfully 10330 */ 10331 break; 10332 } 10333 } 10334 } 10335 pr_hblk = hmeblkp; 10336 prevpa = hblkpa; 10337 hblkpa = hmeblkp->hblk_nextpa; 10338 hmeblkp = hmeblkp->hblk_next; 10339 } 10340 10341 SFMMU_HASH_UNLOCK(hmebp); 10342 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 10343 hmebp = uhme_hash; 10344 } 10345 uhmehash_steal_hand = hmebp; 10346 10347 if (hmeblkp != NULL) 10348 break; 10349 10350 /* 10351 * in the worst case, look for a free one in the kernel 10352 * hash table. 10353 */ 10354 for (i = 0, hmebp = khme_hash; i <= KHMEHASH_SZ; i++) { 10355 SFMMU_HASH_LOCK(hmebp); 10356 hmeblkp = hmebp->hmeblkp; 10357 hblkpa = hmebp->hmeh_nextpa; 10358 prevpa = 0; 10359 pr_hblk = NULL; 10360 while (hmeblkp) { 10361 /* 10362 * check if it is free hmeblk 10363 */ 10364 if ((get_hblk_ttesz(hmeblkp) == size) && 10365 (hmeblkp->hblk_lckcnt == 0) && 10366 (hmeblkp->hblk_vcnt == 0) && 10367 (hmeblkp->hblk_hmecnt == 0)) { 10368 if (sfmmu_steal_this_hblk(hmebp, 10369 hmeblkp, hblkpa, prevpa, pr_hblk)) { 10370 break; 10371 } else { 10372 /* 10373 * Cannot fail since we have 10374 * hash lock. 10375 */ 10376 panic("fail to steal?"); 10377 } 10378 } 10379 10380 pr_hblk = hmeblkp; 10381 prevpa = hblkpa; 10382 hblkpa = hmeblkp->hblk_nextpa; 10383 hmeblkp = hmeblkp->hblk_next; 10384 } 10385 10386 SFMMU_HASH_UNLOCK(hmebp); 10387 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 10388 hmebp = khme_hash; 10389 } 10390 10391 if (hmeblkp != NULL) 10392 break; 10393 sfmmu_hblk_steal_twice++; 10394 } 10395 return (hmeblkp); 10396 } 10397 10398 /* 10399 * This routine does real work to prepare a hblk to be "stolen" by 10400 * unloading the mappings, updating shadow counts .... 10401 * It returns 1 if the block is ready to be reused (stolen), or 0 10402 * means the block cannot be stolen yet- pageunload is still working 10403 * on this hblk. 10404 */ 10405 static int 10406 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10407 uint64_t hblkpa, uint64_t prevpa, struct hme_blk *pr_hblk) 10408 { 10409 int shw_size, vshift; 10410 struct hme_blk *shw_hblkp; 10411 uintptr_t vaddr; 10412 uint_t shw_mask, newshw_mask; 10413 10414 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10415 10416 /* 10417 * check if the hmeblk is free, unload if necessary 10418 */ 10419 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10420 sfmmu_t *sfmmup; 10421 demap_range_t dmr; 10422 10423 sfmmup = hblktosfmmu(hmeblkp); 10424 DEMAP_RANGE_INIT(sfmmup, &dmr); 10425 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 10426 (caddr_t)get_hblk_base(hmeblkp), 10427 get_hblk_endaddr(hmeblkp), &dmr, HAT_UNLOAD); 10428 DEMAP_RANGE_FLUSH(&dmr); 10429 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10430 /* 10431 * Pageunload is working on the same hblk. 10432 */ 10433 return (0); 10434 } 10435 10436 sfmmu_hblk_steal_unload_count++; 10437 } 10438 10439 ASSERT(hmeblkp->hblk_lckcnt == 0); 10440 ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0); 10441 10442 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 10443 hmeblkp->hblk_nextpa = hblkpa; 10444 10445 shw_hblkp = hmeblkp->hblk_shadow; 10446 if (shw_hblkp) { 10447 shw_size = get_hblk_ttesz(shw_hblkp); 10448 vaddr = get_hblk_base(hmeblkp); 10449 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10450 ASSERT(vshift < 8); 10451 /* 10452 * Atomically clear shadow mask bit 10453 */ 10454 do { 10455 shw_mask = shw_hblkp->hblk_shw_mask; 10456 ASSERT(shw_mask & (1 << vshift)); 10457 newshw_mask = shw_mask & ~(1 << vshift); 10458 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10459 shw_mask, newshw_mask); 10460 } while (newshw_mask != shw_mask); 10461 hmeblkp->hblk_shadow = NULL; 10462 } 10463 10464 /* 10465 * remove shadow bit if we are stealing an unused shadow hmeblk. 10466 * sfmmu_hblk_alloc needs it that way, will set shadow bit later if 10467 * we are indeed allocating a shadow hmeblk. 10468 */ 10469 hmeblkp->hblk_shw_bit = 0; 10470 10471 sfmmu_hblk_steal_count++; 10472 SFMMU_STAT(sf_steal_count); 10473 10474 return (1); 10475 } 10476 10477 struct hme_blk * 10478 sfmmu_hmetohblk(struct sf_hment *sfhme) 10479 { 10480 struct hme_blk *hmeblkp; 10481 struct sf_hment *sfhme0; 10482 struct hme_blk *hblk_dummy = 0; 10483 10484 /* 10485 * No dummy sf_hments, please. 10486 */ 10487 ASSERT(sfhme->hme_tte.ll != 0); 10488 10489 sfhme0 = sfhme - sfhme->hme_tte.tte_hmenum; 10490 hmeblkp = (struct hme_blk *)((uintptr_t)sfhme0 - 10491 (uintptr_t)&hblk_dummy->hblk_hme[0]); 10492 10493 return (hmeblkp); 10494 } 10495 10496 /* 10497 * On swapin, get appropriately sized TSB(s) and clear the HAT_SWAPPED flag. 10498 * If we can't get appropriately sized TSB(s), try for 8K TSB(s) using 10499 * KM_SLEEP allocation. 10500 * 10501 * Return 0 on success, -1 otherwise. 10502 */ 10503 static void 10504 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp) 10505 { 10506 struct tsb_info *tsbinfop, *next; 10507 tsb_replace_rc_t rc; 10508 boolean_t gotfirst = B_FALSE; 10509 10510 ASSERT(sfmmup != ksfmmup); 10511 ASSERT(sfmmu_hat_lock_held(sfmmup)); 10512 10513 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) { 10514 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 10515 } 10516 10517 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10518 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN); 10519 } else { 10520 return; 10521 } 10522 10523 ASSERT(sfmmup->sfmmu_tsb != NULL); 10524 10525 /* 10526 * Loop over all tsbinfo's replacing them with ones that actually have 10527 * a TSB. If any of the replacements ever fail, bail out of the loop. 10528 */ 10529 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) { 10530 ASSERT(tsbinfop->tsb_flags & TSB_SWAPPED); 10531 next = tsbinfop->tsb_next; 10532 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc, 10533 hatlockp, TSB_SWAPIN); 10534 if (rc != TSB_SUCCESS) { 10535 break; 10536 } 10537 gotfirst = B_TRUE; 10538 } 10539 10540 switch (rc) { 10541 case TSB_SUCCESS: 10542 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10543 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10544 return; 10545 case TSB_ALLOCFAIL: 10546 break; 10547 default: 10548 panic("sfmmu_replace_tsb returned unrecognized failure code " 10549 "%d", rc); 10550 } 10551 10552 /* 10553 * In this case, we failed to get one of our TSBs. If we failed to 10554 * get the first TSB, get one of minimum size (8KB). Walk the list 10555 * and throw away the tsbinfos, starting where the allocation failed; 10556 * we can get by with just one TSB as long as we don't leave the 10557 * SWAPPED tsbinfo structures lying around. 10558 */ 10559 tsbinfop = sfmmup->sfmmu_tsb; 10560 next = tsbinfop->tsb_next; 10561 tsbinfop->tsb_next = NULL; 10562 10563 sfmmu_hat_exit(hatlockp); 10564 for (tsbinfop = next; tsbinfop != NULL; tsbinfop = next) { 10565 next = tsbinfop->tsb_next; 10566 sfmmu_tsbinfo_free(tsbinfop); 10567 } 10568 hatlockp = sfmmu_hat_enter(sfmmup); 10569 10570 /* 10571 * If we don't have any TSBs, get a single 8K TSB for 8K, 64K and 512K 10572 * pages. 10573 */ 10574 if (!gotfirst) { 10575 tsbinfop = sfmmup->sfmmu_tsb; 10576 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE, 10577 hatlockp, TSB_SWAPIN | TSB_FORCEALLOC); 10578 ASSERT(rc == TSB_SUCCESS); 10579 } else { 10580 /* update machine specific tsbinfo */ 10581 sfmmu_setup_tsbinfo(sfmmup); 10582 } 10583 10584 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10585 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10586 } 10587 10588 /* 10589 * Handle exceptions for low level tsb_handler. 10590 * 10591 * There are many scenarios that could land us here: 10592 * 10593 * If the context is invalid we land here. The context can be invalid 10594 * for 3 reasons: 1) we couldn't allocate a new context and now need to 10595 * perform a wrap around operation in order to allocate a new context. 10596 * 2) Context was invalidated to change pagesize programming 3) ISMs or 10597 * TSBs configuration is changeing for this process and we are forced into 10598 * here to do a syncronization operation. If the context is valid we can 10599 * be here from window trap hanlder. In this case just call trap to handle 10600 * the fault. 10601 * 10602 * Note that the process will run in INVALID_CONTEXT before 10603 * faulting into here and subsequently loading the MMU registers 10604 * (including the TSB base register) associated with this process. 10605 * For this reason, the trap handlers must all test for 10606 * INVALID_CONTEXT before attempting to access any registers other 10607 * than the context registers. 10608 */ 10609 void 10610 sfmmu_tsbmiss_exception(struct regs *rp, uintptr_t tagaccess, uint_t traptype) 10611 { 10612 sfmmu_t *sfmmup; 10613 uint_t ctxtype; 10614 klwp_id_t lwp; 10615 char lwp_save_state; 10616 hatlock_t *hatlockp; 10617 struct tsb_info *tsbinfop; 10618 10619 SFMMU_STAT(sf_tsb_exceptions); 10620 SFMMU_MMU_STAT(mmu_tsb_exceptions); 10621 sfmmup = astosfmmu(curthread->t_procp->p_as); 10622 /* 10623 * note that in sun4u, tagacces register contains ctxnum 10624 * while sun4v passes ctxtype in the tagaccess register. 10625 */ 10626 ctxtype = tagaccess & TAGACC_CTX_MASK; 10627 10628 ASSERT(sfmmup != ksfmmup && ctxtype != KCONTEXT); 10629 ASSERT(sfmmup->sfmmu_ismhat == 0); 10630 /* 10631 * First, make sure we come out of here with a valid ctx, 10632 * since if we don't get one we'll simply loop on the 10633 * faulting instruction. 10634 * 10635 * If the ISM mappings are changing, the TSB is being relocated, or 10636 * the process is swapped out we serialize behind the controlling 10637 * thread with the sfmmu_flags and sfmmu_tsb_cv condition variable. 10638 * Otherwise we synchronize with the context stealer or the thread 10639 * that required us to change out our MMU registers (such 10640 * as a thread changing out our TSB while we were running) by 10641 * locking the HAT and grabbing the rwlock on the context as a 10642 * reader temporarily. 10643 */ 10644 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) || 10645 ctxtype == INVALID_CONTEXT); 10646 10647 if (ctxtype == INVALID_CONTEXT) { 10648 /* 10649 * Must set lwp state to LWP_SYS before 10650 * trying to acquire any adaptive lock 10651 */ 10652 lwp = ttolwp(curthread); 10653 ASSERT(lwp); 10654 lwp_save_state = lwp->lwp_state; 10655 lwp->lwp_state = LWP_SYS; 10656 10657 hatlockp = sfmmu_hat_enter(sfmmup); 10658 retry: 10659 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 10660 tsbinfop = tsbinfop->tsb_next) { 10661 if (tsbinfop->tsb_flags & TSB_RELOC_FLAG) { 10662 cv_wait(&sfmmup->sfmmu_tsb_cv, 10663 HATLOCK_MUTEXP(hatlockp)); 10664 goto retry; 10665 } 10666 } 10667 10668 /* 10669 * Wait for ISM maps to be updated. 10670 */ 10671 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) { 10672 cv_wait(&sfmmup->sfmmu_tsb_cv, 10673 HATLOCK_MUTEXP(hatlockp)); 10674 goto retry; 10675 } 10676 10677 /* 10678 * If we're swapping in, get TSB(s). Note that we must do 10679 * this before we get a ctx or load the MMU state. Once 10680 * we swap in we have to recheck to make sure the TSB(s) and 10681 * ISM mappings didn't change while we slept. 10682 */ 10683 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10684 sfmmu_tsb_swapin(sfmmup, hatlockp); 10685 goto retry; 10686 } 10687 10688 sfmmu_get_ctx(sfmmup); 10689 10690 sfmmu_hat_exit(hatlockp); 10691 /* 10692 * Must restore lwp_state if not calling 10693 * trap() for further processing. Restore 10694 * it anyway. 10695 */ 10696 lwp->lwp_state = lwp_save_state; 10697 if (sfmmup->sfmmu_ttecnt[TTE8K] != 0 || 10698 sfmmup->sfmmu_ttecnt[TTE64K] != 0 || 10699 sfmmup->sfmmu_ttecnt[TTE512K] != 0 || 10700 sfmmup->sfmmu_ttecnt[TTE4M] != 0 || 10701 sfmmup->sfmmu_ttecnt[TTE32M] != 0 || 10702 sfmmup->sfmmu_ttecnt[TTE256M] != 0) { 10703 return; 10704 } 10705 if (traptype == T_DATA_PROT) { 10706 traptype = T_DATA_MMU_MISS; 10707 } 10708 } 10709 trap(rp, (caddr_t)tagaccess, traptype, 0); 10710 } 10711 10712 /* 10713 * sfmmu_vatopfn_suspended is called from GET_TTE when TL=0 and 10714 * TTE_SUSPENDED bit set in tte we block on aquiring a page lock 10715 * rather than spinning to avoid send mondo timeouts with 10716 * interrupts enabled. When the lock is acquired it is immediately 10717 * released and we return back to sfmmu_vatopfn just after 10718 * the GET_TTE call. 10719 */ 10720 void 10721 sfmmu_vatopfn_suspended(caddr_t vaddr, sfmmu_t *sfmmu, tte_t *ttep) 10722 { 10723 struct page **pp; 10724 10725 (void) as_pagelock(sfmmu->sfmmu_as, &pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10726 as_pageunlock(sfmmu->sfmmu_as, pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10727 } 10728 10729 /* 10730 * sfmmu_tsbmiss_suspended is called from GET_TTE when TL>0 and 10731 * TTE_SUSPENDED bit set in tte. We do this so that we can handle 10732 * cross traps which cannot be handled while spinning in the 10733 * trap handlers. Simply enter and exit the kpr_suspendlock spin 10734 * mutex, which is held by the holder of the suspend bit, and then 10735 * retry the trapped instruction after unwinding. 10736 */ 10737 /*ARGSUSED*/ 10738 void 10739 sfmmu_tsbmiss_suspended(struct regs *rp, uintptr_t tagacc, uint_t traptype) 10740 { 10741 ASSERT(curthread != kreloc_thread); 10742 mutex_enter(&kpr_suspendlock); 10743 mutex_exit(&kpr_suspendlock); 10744 } 10745 10746 /* 10747 * Special routine to flush out ism mappings- TSBs, TLBs and D-caches. 10748 * This routine may be called with all cpu's captured. Therefore, the 10749 * caller is responsible for holding all locks and disabling kernel 10750 * preemption. 10751 */ 10752 /* ARGSUSED */ 10753 static void 10754 sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, 10755 struct hme_blk *hmeblkp, pfn_t pfnum, int cache_flush_flag) 10756 { 10757 cpuset_t cpuset; 10758 caddr_t va; 10759 ism_ment_t *ment; 10760 sfmmu_t *sfmmup; 10761 #ifdef VAC 10762 int vcolor; 10763 #endif 10764 int ttesz; 10765 10766 /* 10767 * Walk the ism_hat's mapping list and flush the page 10768 * from every hat sharing this ism_hat. This routine 10769 * may be called while all cpu's have been captured. 10770 * Therefore we can't attempt to grab any locks. For now 10771 * this means we will protect the ism mapping list under 10772 * a single lock which will be grabbed by the caller. 10773 * If hat_share/unshare scalibility becomes a performance 10774 * problem then we may need to re-think ism mapping list locking. 10775 */ 10776 ASSERT(ism_sfmmup->sfmmu_ismhat); 10777 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 10778 addr = addr - ISMID_STARTADDR; 10779 for (ment = ism_sfmmup->sfmmu_iment; ment; ment = ment->iment_next) { 10780 10781 sfmmup = ment->iment_hat; 10782 10783 va = ment->iment_base_va; 10784 va = (caddr_t)((uintptr_t)va + (uintptr_t)addr); 10785 10786 /* 10787 * Flush TSB of ISM mappings. 10788 */ 10789 ttesz = get_hblk_ttesz(hmeblkp); 10790 if (ttesz == TTE8K || ttesz == TTE4M) { 10791 sfmmu_unload_tsb(sfmmup, va, ttesz); 10792 } else { 10793 caddr_t sva = va; 10794 caddr_t eva; 10795 ASSERT(addr == (caddr_t)get_hblk_base(hmeblkp)); 10796 eva = sva + get_hblk_span(hmeblkp); 10797 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); 10798 } 10799 10800 cpuset = sfmmup->sfmmu_cpusran; 10801 CPUSET_AND(cpuset, cpu_ready_set); 10802 CPUSET_DEL(cpuset, CPU->cpu_id); 10803 10804 SFMMU_XCALL_STATS(sfmmup); 10805 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)va, 10806 (uint64_t)sfmmup); 10807 10808 vtag_flushpage(va, (uint64_t)sfmmup); 10809 10810 #ifdef VAC 10811 /* 10812 * Flush D$ 10813 * When flushing D$ we must flush all 10814 * cpu's. See sfmmu_cache_flush(). 10815 */ 10816 if (cache_flush_flag == CACHE_FLUSH) { 10817 cpuset = cpu_ready_set; 10818 CPUSET_DEL(cpuset, CPU->cpu_id); 10819 10820 SFMMU_XCALL_STATS(sfmmup); 10821 vcolor = addr_to_vcolor(va); 10822 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10823 vac_flushpage(pfnum, vcolor); 10824 } 10825 #endif /* VAC */ 10826 } 10827 } 10828 10829 /* 10830 * Demaps the TSB, CPU caches, and flushes all TLBs on all CPUs of 10831 * a particular virtual address and ctx. If noflush is set we do not 10832 * flush the TLB/TSB. This function may or may not be called with the 10833 * HAT lock held. 10834 */ 10835 static void 10836 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10837 pfn_t pfnum, int tlb_noflush, int cpu_flag, int cache_flush_flag, 10838 int hat_lock_held) 10839 { 10840 #ifdef VAC 10841 int vcolor; 10842 #endif 10843 cpuset_t cpuset; 10844 hatlock_t *hatlockp; 10845 10846 #if defined(lint) && !defined(VAC) 10847 pfnum = pfnum; 10848 cpu_flag = cpu_flag; 10849 cache_flush_flag = cache_flush_flag; 10850 #endif 10851 /* 10852 * There is no longer a need to protect against ctx being 10853 * stolen here since we don't store the ctx in the TSB anymore. 10854 */ 10855 #ifdef VAC 10856 vcolor = addr_to_vcolor(addr); 10857 #endif 10858 10859 /* 10860 * We must hold the hat lock during the flush of TLB, 10861 * to avoid a race with sfmmu_invalidate_ctx(), where 10862 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 10863 * causing TLB demap routine to skip flush on that MMU. 10864 * If the context on a MMU has already been set to 10865 * INVALID_CONTEXT, we just get an extra flush on 10866 * that MMU. 10867 */ 10868 if (!hat_lock_held && !tlb_noflush) 10869 hatlockp = sfmmu_hat_enter(sfmmup); 10870 10871 kpreempt_disable(); 10872 if (!tlb_noflush) { 10873 /* 10874 * Flush the TSB and TLB. 10875 */ 10876 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10877 10878 cpuset = sfmmup->sfmmu_cpusran; 10879 CPUSET_AND(cpuset, cpu_ready_set); 10880 CPUSET_DEL(cpuset, CPU->cpu_id); 10881 10882 SFMMU_XCALL_STATS(sfmmup); 10883 10884 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 10885 (uint64_t)sfmmup); 10886 10887 vtag_flushpage(addr, (uint64_t)sfmmup); 10888 } 10889 10890 if (!hat_lock_held && !tlb_noflush) 10891 sfmmu_hat_exit(hatlockp); 10892 10893 #ifdef VAC 10894 /* 10895 * Flush the D$ 10896 * 10897 * Even if the ctx is stolen, we need to flush the 10898 * cache. Our ctx stealer only flushes the TLBs. 10899 */ 10900 if (cache_flush_flag == CACHE_FLUSH) { 10901 if (cpu_flag & FLUSH_ALL_CPUS) { 10902 cpuset = cpu_ready_set; 10903 } else { 10904 cpuset = sfmmup->sfmmu_cpusran; 10905 CPUSET_AND(cpuset, cpu_ready_set); 10906 } 10907 CPUSET_DEL(cpuset, CPU->cpu_id); 10908 SFMMU_XCALL_STATS(sfmmup); 10909 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10910 vac_flushpage(pfnum, vcolor); 10911 } 10912 #endif /* VAC */ 10913 kpreempt_enable(); 10914 } 10915 10916 /* 10917 * Demaps the TSB and flushes all TLBs on all cpus for a particular virtual 10918 * address and ctx. If noflush is set we do not currently do anything. 10919 * This function may or may not be called with the HAT lock held. 10920 */ 10921 static void 10922 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10923 int tlb_noflush, int hat_lock_held) 10924 { 10925 cpuset_t cpuset; 10926 hatlock_t *hatlockp; 10927 10928 /* 10929 * If the process is exiting we have nothing to do. 10930 */ 10931 if (tlb_noflush) 10932 return; 10933 10934 /* 10935 * Flush TSB. 10936 */ 10937 if (!hat_lock_held) 10938 hatlockp = sfmmu_hat_enter(sfmmup); 10939 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10940 10941 kpreempt_disable(); 10942 10943 cpuset = sfmmup->sfmmu_cpusran; 10944 CPUSET_AND(cpuset, cpu_ready_set); 10945 CPUSET_DEL(cpuset, CPU->cpu_id); 10946 10947 SFMMU_XCALL_STATS(sfmmup); 10948 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup); 10949 10950 vtag_flushpage(addr, (uint64_t)sfmmup); 10951 10952 if (!hat_lock_held) 10953 sfmmu_hat_exit(hatlockp); 10954 10955 kpreempt_enable(); 10956 10957 } 10958 10959 /* 10960 * Special case of sfmmu_tlb_demap for MMU_PAGESIZE hblks. Use the xcall 10961 * call handler that can flush a range of pages to save on xcalls. 10962 */ 10963 static int sfmmu_xcall_save; 10964 10965 static void 10966 sfmmu_tlb_range_demap(demap_range_t *dmrp) 10967 { 10968 sfmmu_t *sfmmup = dmrp->dmr_sfmmup; 10969 hatlock_t *hatlockp; 10970 cpuset_t cpuset; 10971 uint64_t sfmmu_pgcnt; 10972 pgcnt_t pgcnt = 0; 10973 int pgunload = 0; 10974 int dirtypg = 0; 10975 caddr_t addr = dmrp->dmr_addr; 10976 caddr_t eaddr; 10977 uint64_t bitvec = dmrp->dmr_bitvec; 10978 10979 ASSERT(bitvec & 1); 10980 10981 /* 10982 * Flush TSB and calculate number of pages to flush. 10983 */ 10984 while (bitvec != 0) { 10985 dirtypg = 0; 10986 /* 10987 * Find the first page to flush and then count how many 10988 * pages there are after it that also need to be flushed. 10989 * This way the number of TSB flushes is minimized. 10990 */ 10991 while ((bitvec & 1) == 0) { 10992 pgcnt++; 10993 addr += MMU_PAGESIZE; 10994 bitvec >>= 1; 10995 } 10996 while (bitvec & 1) { 10997 dirtypg++; 10998 bitvec >>= 1; 10999 } 11000 eaddr = addr + ptob(dirtypg); 11001 hatlockp = sfmmu_hat_enter(sfmmup); 11002 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K); 11003 sfmmu_hat_exit(hatlockp); 11004 pgunload += dirtypg; 11005 addr = eaddr; 11006 pgcnt += dirtypg; 11007 } 11008 11009 ASSERT((pgcnt<<MMU_PAGESHIFT) <= dmrp->dmr_endaddr - dmrp->dmr_addr); 11010 if (sfmmup->sfmmu_free == 0) { 11011 addr = dmrp->dmr_addr; 11012 bitvec = dmrp->dmr_bitvec; 11013 11014 /* 11015 * make sure it has SFMMU_PGCNT_SHIFT bits only, 11016 * as it will be used to pack argument for xt_some 11017 */ 11018 ASSERT((pgcnt > 0) && 11019 (pgcnt <= (1 << SFMMU_PGCNT_SHIFT))); 11020 11021 /* 11022 * Encode pgcnt as (pgcnt -1 ), and pass (pgcnt - 1) in 11023 * the low 6 bits of sfmmup. This is doable since pgcnt 11024 * always >= 1. 11025 */ 11026 ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK)); 11027 sfmmu_pgcnt = (uint64_t)sfmmup | 11028 ((pgcnt - 1) & SFMMU_PGCNT_MASK); 11029 11030 /* 11031 * We must hold the hat lock during the flush of TLB, 11032 * to avoid a race with sfmmu_invalidate_ctx(), where 11033 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 11034 * causing TLB demap routine to skip flush on that MMU. 11035 * If the context on a MMU has already been set to 11036 * INVALID_CONTEXT, we just get an extra flush on 11037 * that MMU. 11038 */ 11039 hatlockp = sfmmu_hat_enter(sfmmup); 11040 kpreempt_disable(); 11041 11042 cpuset = sfmmup->sfmmu_cpusran; 11043 CPUSET_AND(cpuset, cpu_ready_set); 11044 CPUSET_DEL(cpuset, CPU->cpu_id); 11045 11046 SFMMU_XCALL_STATS(sfmmup); 11047 xt_some(cpuset, vtag_flush_pgcnt_tl1, (uint64_t)addr, 11048 sfmmu_pgcnt); 11049 11050 for (; bitvec != 0; bitvec >>= 1) { 11051 if (bitvec & 1) 11052 vtag_flushpage(addr, (uint64_t)sfmmup); 11053 addr += MMU_PAGESIZE; 11054 } 11055 kpreempt_enable(); 11056 sfmmu_hat_exit(hatlockp); 11057 11058 sfmmu_xcall_save += (pgunload-1); 11059 } 11060 dmrp->dmr_bitvec = 0; 11061 } 11062 11063 /* 11064 * In cases where we need to synchronize with TLB/TSB miss trap 11065 * handlers, _and_ need to flush the TLB, it's a lot easier to 11066 * throw away the context from the process than to do a 11067 * special song and dance to keep things consistent for the 11068 * handlers. 11069 * 11070 * Since the process suddenly ends up without a context and our caller 11071 * holds the hat lock, threads that fault after this function is called 11072 * will pile up on the lock. We can then do whatever we need to 11073 * atomically from the context of the caller. The first blocked thread 11074 * to resume executing will get the process a new context, and the 11075 * process will resume executing. 11076 * 11077 * One added advantage of this approach is that on MMUs that 11078 * support a "flush all" operation, we will delay the flush until 11079 * cnum wrap-around, and then flush the TLB one time. This 11080 * is rather rare, so it's a lot less expensive than making 8000 11081 * x-calls to flush the TLB 8000 times. 11082 * 11083 * A per-process (PP) lock is used to synchronize ctx allocations in 11084 * resume() and ctx invalidations here. 11085 */ 11086 static void 11087 sfmmu_invalidate_ctx(sfmmu_t *sfmmup) 11088 { 11089 cpuset_t cpuset; 11090 int cnum, currcnum; 11091 mmu_ctx_t *mmu_ctxp; 11092 int i; 11093 uint_t pstate_save; 11094 11095 SFMMU_STAT(sf_ctx_inv); 11096 11097 ASSERT(sfmmu_hat_lock_held(sfmmup)); 11098 ASSERT(sfmmup != ksfmmup); 11099 11100 kpreempt_disable(); 11101 11102 mmu_ctxp = CPU_MMU_CTXP(CPU); 11103 ASSERT(mmu_ctxp); 11104 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 11105 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 11106 11107 currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; 11108 11109 pstate_save = sfmmu_disable_intrs(); 11110 11111 lock_set(&sfmmup->sfmmu_ctx_lock); /* acquire PP lock */ 11112 /* set HAT cnum invalid across all context domains. */ 11113 for (i = 0; i < max_mmu_ctxdoms; i++) { 11114 11115 cnum = sfmmup->sfmmu_ctxs[i].cnum; 11116 if (cnum == INVALID_CONTEXT) { 11117 continue; 11118 } 11119 11120 sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT; 11121 } 11122 membar_enter(); /* make sure globally visible to all CPUs */ 11123 lock_clear(&sfmmup->sfmmu_ctx_lock); /* release PP lock */ 11124 11125 sfmmu_enable_intrs(pstate_save); 11126 11127 cpuset = sfmmup->sfmmu_cpusran; 11128 CPUSET_DEL(cpuset, CPU->cpu_id); 11129 CPUSET_AND(cpuset, cpu_ready_set); 11130 if (!CPUSET_ISNULL(cpuset)) { 11131 SFMMU_XCALL_STATS(sfmmup); 11132 xt_some(cpuset, sfmmu_raise_tsb_exception, 11133 (uint64_t)sfmmup, INVALID_CONTEXT); 11134 xt_sync(cpuset); 11135 SFMMU_STAT(sf_tsb_raise_exception); 11136 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 11137 } 11138 11139 /* 11140 * If the hat to-be-invalidated is the same as the current 11141 * process on local CPU we need to invalidate 11142 * this CPU context as well. 11143 */ 11144 if ((sfmmu_getctx_sec() == currcnum) && 11145 (currcnum != INVALID_CONTEXT)) { 11146 sfmmu_setctx_sec(INVALID_CONTEXT); 11147 sfmmu_clear_utsbinfo(); 11148 } 11149 11150 kpreempt_enable(); 11151 11152 /* 11153 * we hold the hat lock, so nobody should allocate a context 11154 * for us yet 11155 */ 11156 ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT); 11157 } 11158 11159 #ifdef VAC 11160 /* 11161 * We need to flush the cache in all cpus. It is possible that 11162 * a process referenced a page as cacheable but has sinced exited 11163 * and cleared the mapping list. We still to flush it but have no 11164 * state so all cpus is the only alternative. 11165 */ 11166 void 11167 sfmmu_cache_flush(pfn_t pfnum, int vcolor) 11168 { 11169 cpuset_t cpuset; 11170 11171 kpreempt_disable(); 11172 cpuset = cpu_ready_set; 11173 CPUSET_DEL(cpuset, CPU->cpu_id); 11174 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11175 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 11176 xt_sync(cpuset); 11177 vac_flushpage(pfnum, vcolor); 11178 kpreempt_enable(); 11179 } 11180 11181 void 11182 sfmmu_cache_flushcolor(int vcolor, pfn_t pfnum) 11183 { 11184 cpuset_t cpuset; 11185 11186 ASSERT(vcolor >= 0); 11187 11188 kpreempt_disable(); 11189 cpuset = cpu_ready_set; 11190 CPUSET_DEL(cpuset, CPU->cpu_id); 11191 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11192 xt_some(cpuset, vac_flushcolor_tl1, vcolor, pfnum); 11193 xt_sync(cpuset); 11194 vac_flushcolor(vcolor, pfnum); 11195 kpreempt_enable(); 11196 } 11197 #endif /* VAC */ 11198 11199 /* 11200 * We need to prevent processes from accessing the TSB using a cached physical 11201 * address. It's alright if they try to access the TSB via virtual address 11202 * since they will just fault on that virtual address once the mapping has 11203 * been suspended. 11204 */ 11205 #pragma weak sendmondo_in_recover 11206 11207 /* ARGSUSED */ 11208 static int 11209 sfmmu_tsb_pre_relocator(caddr_t va, uint_t tsbsz, uint_t flags, void *tsbinfo) 11210 { 11211 hatlock_t *hatlockp; 11212 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11213 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11214 extern uint32_t sendmondo_in_recover; 11215 11216 if (flags != HAT_PRESUSPEND) 11217 return (0); 11218 11219 hatlockp = sfmmu_hat_enter(sfmmup); 11220 11221 tsbinfop->tsb_flags |= TSB_RELOC_FLAG; 11222 11223 /* 11224 * For Cheetah+ Erratum 25: 11225 * Wait for any active recovery to finish. We can't risk 11226 * relocating the TSB of the thread running mondo_recover_proc() 11227 * since, if we did that, we would deadlock. The scenario we are 11228 * trying to avoid is as follows: 11229 * 11230 * THIS CPU RECOVER CPU 11231 * -------- ----------- 11232 * Begins recovery, walking through TSB 11233 * hat_pagesuspend() TSB TTE 11234 * TLB miss on TSB TTE, spins at TL1 11235 * xt_sync() 11236 * send_mondo_timeout() 11237 * mondo_recover_proc() 11238 * ((deadlocked)) 11239 * 11240 * The second half of the workaround is that mondo_recover_proc() 11241 * checks to see if the tsb_info has the RELOC flag set, and if it 11242 * does, it skips over that TSB without ever touching tsbinfop->tsb_va 11243 * and hence avoiding the TLB miss that could result in a deadlock. 11244 */ 11245 if (&sendmondo_in_recover) { 11246 membar_enter(); /* make sure RELOC flag visible */ 11247 while (sendmondo_in_recover) { 11248 drv_usecwait(1); 11249 membar_consumer(); 11250 } 11251 } 11252 11253 sfmmu_invalidate_ctx(sfmmup); 11254 sfmmu_hat_exit(hatlockp); 11255 11256 return (0); 11257 } 11258 11259 /* ARGSUSED */ 11260 static int 11261 sfmmu_tsb_post_relocator(caddr_t va, uint_t tsbsz, uint_t flags, 11262 void *tsbinfo, pfn_t newpfn) 11263 { 11264 hatlock_t *hatlockp; 11265 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11266 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11267 11268 if (flags != HAT_POSTUNSUSPEND) 11269 return (0); 11270 11271 hatlockp = sfmmu_hat_enter(sfmmup); 11272 11273 SFMMU_STAT(sf_tsb_reloc); 11274 11275 /* 11276 * The process may have swapped out while we were relocating one 11277 * of its TSBs. If so, don't bother doing the setup since the 11278 * process can't be using the memory anymore. 11279 */ 11280 if ((tsbinfop->tsb_flags & TSB_SWAPPED) == 0) { 11281 ASSERT(va == tsbinfop->tsb_va); 11282 sfmmu_tsbinfo_setup_phys(tsbinfop, newpfn); 11283 sfmmu_setup_tsbinfo(sfmmup); 11284 11285 if (tsbinfop->tsb_flags & TSB_FLUSH_NEEDED) { 11286 sfmmu_inv_tsb(tsbinfop->tsb_va, 11287 TSB_BYTES(tsbinfop->tsb_szc)); 11288 tsbinfop->tsb_flags &= ~TSB_FLUSH_NEEDED; 11289 } 11290 } 11291 11292 membar_exit(); 11293 tsbinfop->tsb_flags &= ~TSB_RELOC_FLAG; 11294 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 11295 11296 sfmmu_hat_exit(hatlockp); 11297 11298 return (0); 11299 } 11300 11301 /* 11302 * Allocate and initialize a tsb_info structure. Note that we may or may not 11303 * allocate a TSB here, depending on the flags passed in. 11304 */ 11305 static int 11306 sfmmu_tsbinfo_alloc(struct tsb_info **tsbinfopp, int tsb_szc, int tte_sz_mask, 11307 uint_t flags, sfmmu_t *sfmmup) 11308 { 11309 int err; 11310 11311 *tsbinfopp = (struct tsb_info *)kmem_cache_alloc( 11312 sfmmu_tsbinfo_cache, KM_SLEEP); 11313 11314 if ((err = sfmmu_init_tsbinfo(*tsbinfopp, tte_sz_mask, 11315 tsb_szc, flags, sfmmup)) != 0) { 11316 kmem_cache_free(sfmmu_tsbinfo_cache, *tsbinfopp); 11317 SFMMU_STAT(sf_tsb_allocfail); 11318 *tsbinfopp = NULL; 11319 return (err); 11320 } 11321 SFMMU_STAT(sf_tsb_alloc); 11322 11323 /* 11324 * Bump the TSB size counters for this TSB size. 11325 */ 11326 (*(((int *)&sfmmu_tsbsize_stat) + tsb_szc))++; 11327 return (0); 11328 } 11329 11330 static void 11331 sfmmu_tsb_free(struct tsb_info *tsbinfo) 11332 { 11333 caddr_t tsbva = tsbinfo->tsb_va; 11334 uint_t tsb_size = TSB_BYTES(tsbinfo->tsb_szc); 11335 struct kmem_cache *kmem_cachep = tsbinfo->tsb_cache; 11336 vmem_t *vmp = tsbinfo->tsb_vmp; 11337 11338 /* 11339 * If we allocated this TSB from relocatable kernel memory, then we 11340 * need to uninstall the callback handler. 11341 */ 11342 if (tsbinfo->tsb_cache != sfmmu_tsb8k_cache) { 11343 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11344 caddr_t slab_vaddr = (caddr_t)((uintptr_t)tsbva & slab_mask); 11345 page_t **ppl; 11346 int ret; 11347 11348 ret = as_pagelock(&kas, &ppl, slab_vaddr, PAGESIZE, S_WRITE); 11349 ASSERT(ret == 0); 11350 hat_delete_callback(tsbva, (uint_t)tsb_size, (void *)tsbinfo, 11351 0, NULL); 11352 as_pageunlock(&kas, ppl, slab_vaddr, PAGESIZE, S_WRITE); 11353 } 11354 11355 if (kmem_cachep != NULL) { 11356 kmem_cache_free(kmem_cachep, tsbva); 11357 } else { 11358 vmem_xfree(vmp, (void *)tsbva, tsb_size); 11359 } 11360 tsbinfo->tsb_va = (caddr_t)0xbad00bad; 11361 atomic_add_64(&tsb_alloc_bytes, -(int64_t)tsb_size); 11362 } 11363 11364 static void 11365 sfmmu_tsbinfo_free(struct tsb_info *tsbinfo) 11366 { 11367 if ((tsbinfo->tsb_flags & TSB_SWAPPED) == 0) { 11368 sfmmu_tsb_free(tsbinfo); 11369 } 11370 kmem_cache_free(sfmmu_tsbinfo_cache, tsbinfo); 11371 11372 } 11373 11374 /* 11375 * Setup all the references to physical memory for this tsbinfo. 11376 * The underlying page(s) must be locked. 11377 */ 11378 static void 11379 sfmmu_tsbinfo_setup_phys(struct tsb_info *tsbinfo, pfn_t pfn) 11380 { 11381 ASSERT(pfn != PFN_INVALID); 11382 ASSERT(pfn == va_to_pfn(tsbinfo->tsb_va)); 11383 11384 #ifndef sun4v 11385 if (tsbinfo->tsb_szc == 0) { 11386 sfmmu_memtte(&tsbinfo->tsb_tte, pfn, 11387 PROT_WRITE|PROT_READ, TTE8K); 11388 } else { 11389 /* 11390 * Round down PA and use a large mapping; the handlers will 11391 * compute the TSB pointer at the correct offset into the 11392 * big virtual page. NOTE: this assumes all TSBs larger 11393 * than 8K must come from physically contiguous slabs of 11394 * size tsb_slab_size. 11395 */ 11396 sfmmu_memtte(&tsbinfo->tsb_tte, pfn & ~tsb_slab_mask, 11397 PROT_WRITE|PROT_READ, tsb_slab_ttesz); 11398 } 11399 tsbinfo->tsb_pa = ptob(pfn); 11400 11401 TTE_SET_LOCKED(&tsbinfo->tsb_tte); /* lock the tte into dtlb */ 11402 TTE_SET_MOD(&tsbinfo->tsb_tte); /* enable writes */ 11403 11404 ASSERT(TTE_IS_PRIVILEGED(&tsbinfo->tsb_tte)); 11405 ASSERT(TTE_IS_LOCKED(&tsbinfo->tsb_tte)); 11406 #else /* sun4v */ 11407 tsbinfo->tsb_pa = ptob(pfn); 11408 #endif /* sun4v */ 11409 } 11410 11411 11412 /* 11413 * Returns zero on success, ENOMEM if over the high water mark, 11414 * or EAGAIN if the caller needs to retry with a smaller TSB 11415 * size (or specify TSB_FORCEALLOC if the allocation can't fail). 11416 * 11417 * This call cannot fail to allocate a TSB if TSB_FORCEALLOC 11418 * is specified and the TSB requested is PAGESIZE, though it 11419 * may sleep waiting for memory if sufficient memory is not 11420 * available. 11421 */ 11422 static int 11423 sfmmu_init_tsbinfo(struct tsb_info *tsbinfo, int tteszmask, 11424 int tsbcode, uint_t flags, sfmmu_t *sfmmup) 11425 { 11426 caddr_t vaddr = NULL; 11427 caddr_t slab_vaddr; 11428 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11429 int tsbbytes = TSB_BYTES(tsbcode); 11430 int lowmem = 0; 11431 struct kmem_cache *kmem_cachep = NULL; 11432 vmem_t *vmp = NULL; 11433 lgrp_id_t lgrpid = LGRP_NONE; 11434 pfn_t pfn; 11435 uint_t cbflags = HAC_SLEEP; 11436 page_t **pplist; 11437 int ret; 11438 11439 if (flags & (TSB_FORCEALLOC | TSB_SWAPIN | TSB_GROW | TSB_SHRINK)) 11440 flags |= TSB_ALLOC; 11441 11442 ASSERT((flags & TSB_FORCEALLOC) == 0 || tsbcode == TSB_MIN_SZCODE); 11443 11444 tsbinfo->tsb_sfmmu = sfmmup; 11445 11446 /* 11447 * If not allocating a TSB, set up the tsbinfo, set TSB_SWAPPED, and 11448 * return. 11449 */ 11450 if ((flags & TSB_ALLOC) == 0) { 11451 tsbinfo->tsb_szc = tsbcode; 11452 tsbinfo->tsb_ttesz_mask = tteszmask; 11453 tsbinfo->tsb_va = (caddr_t)0xbadbadbeef; 11454 tsbinfo->tsb_pa = -1; 11455 tsbinfo->tsb_tte.ll = 0; 11456 tsbinfo->tsb_next = NULL; 11457 tsbinfo->tsb_flags = TSB_SWAPPED; 11458 tsbinfo->tsb_cache = NULL; 11459 tsbinfo->tsb_vmp = NULL; 11460 return (0); 11461 } 11462 11463 #ifdef DEBUG 11464 /* 11465 * For debugging: 11466 * Randomly force allocation failures every tsb_alloc_mtbf 11467 * tries if TSB_FORCEALLOC is not specified. This will 11468 * return ENOMEM if tsb_alloc_mtbf is odd, or EAGAIN if 11469 * it is even, to allow testing of both failure paths... 11470 */ 11471 if (tsb_alloc_mtbf && ((flags & TSB_FORCEALLOC) == 0) && 11472 (tsb_alloc_count++ == tsb_alloc_mtbf)) { 11473 tsb_alloc_count = 0; 11474 tsb_alloc_fail_mtbf++; 11475 return ((tsb_alloc_mtbf & 1)? ENOMEM : EAGAIN); 11476 } 11477 #endif /* DEBUG */ 11478 11479 /* 11480 * Enforce high water mark if we are not doing a forced allocation 11481 * and are not shrinking a process' TSB. 11482 */ 11483 if ((flags & TSB_SHRINK) == 0 && 11484 (tsbbytes + tsb_alloc_bytes) > tsb_alloc_hiwater) { 11485 if ((flags & TSB_FORCEALLOC) == 0) 11486 return (ENOMEM); 11487 lowmem = 1; 11488 } 11489 11490 /* 11491 * Allocate from the correct location based upon the size of the TSB 11492 * compared to the base page size, and what memory conditions dictate. 11493 * Note we always do nonblocking allocations from the TSB arena since 11494 * we don't want memory fragmentation to cause processes to block 11495 * indefinitely waiting for memory; until the kernel algorithms that 11496 * coalesce large pages are improved this is our best option. 11497 * 11498 * Algorithm: 11499 * If allocating a "large" TSB (>8K), allocate from the 11500 * appropriate kmem_tsb_default_arena vmem arena 11501 * else if low on memory or the TSB_FORCEALLOC flag is set or 11502 * tsb_forceheap is set 11503 * Allocate from kernel heap via sfmmu_tsb8k_cache with 11504 * KM_SLEEP (never fails) 11505 * else 11506 * Allocate from appropriate sfmmu_tsb_cache with 11507 * KM_NOSLEEP 11508 * endif 11509 */ 11510 if (tsb_lgrp_affinity) 11511 lgrpid = lgrp_home_id(curthread); 11512 if (lgrpid == LGRP_NONE) 11513 lgrpid = 0; /* use lgrp of boot CPU */ 11514 11515 if (tsbbytes > MMU_PAGESIZE) { 11516 vmp = kmem_tsb_default_arena[lgrpid]; 11517 vaddr = (caddr_t)vmem_xalloc(vmp, tsbbytes, tsbbytes, 0, 0, 11518 NULL, NULL, VM_NOSLEEP); 11519 #ifdef DEBUG 11520 } else if (lowmem || (flags & TSB_FORCEALLOC) || tsb_forceheap) { 11521 #else /* !DEBUG */ 11522 } else if (lowmem || (flags & TSB_FORCEALLOC)) { 11523 #endif /* DEBUG */ 11524 kmem_cachep = sfmmu_tsb8k_cache; 11525 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_SLEEP); 11526 ASSERT(vaddr != NULL); 11527 } else { 11528 kmem_cachep = sfmmu_tsb_cache[lgrpid]; 11529 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_NOSLEEP); 11530 } 11531 11532 tsbinfo->tsb_cache = kmem_cachep; 11533 tsbinfo->tsb_vmp = vmp; 11534 11535 if (vaddr == NULL) { 11536 return (EAGAIN); 11537 } 11538 11539 atomic_add_64(&tsb_alloc_bytes, (int64_t)tsbbytes); 11540 kmem_cachep = tsbinfo->tsb_cache; 11541 11542 /* 11543 * If we are allocating from outside the cage, then we need to 11544 * register a relocation callback handler. Note that for now 11545 * since pseudo mappings always hang off of the slab's root page, 11546 * we need only lock the first 8K of the TSB slab. This is a bit 11547 * hacky but it is good for performance. 11548 */ 11549 if (kmem_cachep != sfmmu_tsb8k_cache) { 11550 slab_vaddr = (caddr_t)((uintptr_t)vaddr & slab_mask); 11551 ret = as_pagelock(&kas, &pplist, slab_vaddr, PAGESIZE, S_WRITE); 11552 ASSERT(ret == 0); 11553 ret = hat_add_callback(sfmmu_tsb_cb_id, vaddr, (uint_t)tsbbytes, 11554 cbflags, (void *)tsbinfo, &pfn, NULL); 11555 11556 /* 11557 * Need to free up resources if we could not successfully 11558 * add the callback function and return an error condition. 11559 */ 11560 if (ret != 0) { 11561 if (kmem_cachep) { 11562 kmem_cache_free(kmem_cachep, vaddr); 11563 } else { 11564 vmem_xfree(vmp, (void *)vaddr, tsbbytes); 11565 } 11566 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, 11567 S_WRITE); 11568 return (EAGAIN); 11569 } 11570 } else { 11571 /* 11572 * Since allocation of 8K TSBs from heap is rare and occurs 11573 * during memory pressure we allocate them from permanent 11574 * memory rather than using callbacks to get the PFN. 11575 */ 11576 pfn = hat_getpfnum(kas.a_hat, vaddr); 11577 } 11578 11579 tsbinfo->tsb_va = vaddr; 11580 tsbinfo->tsb_szc = tsbcode; 11581 tsbinfo->tsb_ttesz_mask = tteszmask; 11582 tsbinfo->tsb_next = NULL; 11583 tsbinfo->tsb_flags = 0; 11584 11585 sfmmu_tsbinfo_setup_phys(tsbinfo, pfn); 11586 11587 if (kmem_cachep != sfmmu_tsb8k_cache) { 11588 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, S_WRITE); 11589 } 11590 11591 sfmmu_inv_tsb(vaddr, tsbbytes); 11592 return (0); 11593 } 11594 11595 /* 11596 * Initialize per cpu tsb and per cpu tsbmiss_area 11597 */ 11598 void 11599 sfmmu_init_tsbs(void) 11600 { 11601 int i; 11602 struct tsbmiss *tsbmissp; 11603 struct kpmtsbm *kpmtsbmp; 11604 #ifndef sun4v 11605 extern int dcache_line_mask; 11606 #endif /* sun4v */ 11607 extern uint_t vac_colors; 11608 11609 /* 11610 * Init. tsb miss area. 11611 */ 11612 tsbmissp = tsbmiss_area; 11613 11614 for (i = 0; i < NCPU; tsbmissp++, i++) { 11615 /* 11616 * initialize the tsbmiss area. 11617 * Do this for all possible CPUs as some may be added 11618 * while the system is running. There is no cost to this. 11619 */ 11620 tsbmissp->ksfmmup = ksfmmup; 11621 #ifndef sun4v 11622 tsbmissp->dcache_line_mask = (uint16_t)dcache_line_mask; 11623 #endif /* sun4v */ 11624 tsbmissp->khashstart = 11625 (struct hmehash_bucket *)va_to_pa((caddr_t)khme_hash); 11626 tsbmissp->uhashstart = 11627 (struct hmehash_bucket *)va_to_pa((caddr_t)uhme_hash); 11628 tsbmissp->khashsz = khmehash_num; 11629 tsbmissp->uhashsz = uhmehash_num; 11630 } 11631 11632 sfmmu_tsb_cb_id = hat_register_callback('T'<<16 | 'S' << 8 | 'B', 11633 sfmmu_tsb_pre_relocator, sfmmu_tsb_post_relocator, NULL, 0); 11634 11635 if (kpm_enable == 0) 11636 return; 11637 11638 /* -- Begin KPM specific init -- */ 11639 11640 if (kpm_smallpages) { 11641 /* 11642 * If we're using base pagesize pages for seg_kpm 11643 * mappings, we use the kernel TSB since we can't afford 11644 * to allocate a second huge TSB for these mappings. 11645 */ 11646 kpm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11647 kpm_tsbsz = ktsb_szcode; 11648 kpmsm_tsbbase = kpm_tsbbase; 11649 kpmsm_tsbsz = kpm_tsbsz; 11650 } else { 11651 /* 11652 * In VAC conflict case, just put the entries in the 11653 * kernel 8K indexed TSB for now so we can find them. 11654 * This could really be changed in the future if we feel 11655 * the need... 11656 */ 11657 kpmsm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11658 kpmsm_tsbsz = ktsb_szcode; 11659 kpm_tsbbase = ktsb_phys? ktsb4m_pbase : (uint64_t)ktsb4m_base; 11660 kpm_tsbsz = ktsb4m_szcode; 11661 } 11662 11663 kpmtsbmp = kpmtsbm_area; 11664 for (i = 0; i < NCPU; kpmtsbmp++, i++) { 11665 /* 11666 * Initialize the kpmtsbm area. 11667 * Do this for all possible CPUs as some may be added 11668 * while the system is running. There is no cost to this. 11669 */ 11670 kpmtsbmp->vbase = kpm_vbase; 11671 kpmtsbmp->vend = kpm_vbase + kpm_size * vac_colors; 11672 kpmtsbmp->sz_shift = kpm_size_shift; 11673 kpmtsbmp->kpmp_shift = kpmp_shift; 11674 kpmtsbmp->kpmp2pshft = (uchar_t)kpmp2pshft; 11675 if (kpm_smallpages == 0) { 11676 kpmtsbmp->kpmp_table_sz = kpmp_table_sz; 11677 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_table); 11678 } else { 11679 kpmtsbmp->kpmp_table_sz = kpmp_stable_sz; 11680 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_stable); 11681 } 11682 kpmtsbmp->msegphashpa = va_to_pa(memseg_phash); 11683 kpmtsbmp->flags = KPMTSBM_ENABLE_FLAG; 11684 #ifdef DEBUG 11685 kpmtsbmp->flags |= (kpm_tsbmtl) ? KPMTSBM_TLTSBM_FLAG : 0; 11686 #endif /* DEBUG */ 11687 if (ktsb_phys) 11688 kpmtsbmp->flags |= KPMTSBM_TSBPHYS_FLAG; 11689 } 11690 11691 /* -- End KPM specific init -- */ 11692 } 11693 11694 /* Avoid using sfmmu_tsbinfo_alloc() to avoid kmem_alloc - no real reason */ 11695 struct tsb_info ktsb_info[2]; 11696 11697 /* 11698 * Called from hat_kern_setup() to setup the tsb_info for ksfmmup. 11699 */ 11700 void 11701 sfmmu_init_ktsbinfo() 11702 { 11703 ASSERT(ksfmmup != NULL); 11704 ASSERT(ksfmmup->sfmmu_tsb == NULL); 11705 /* 11706 * Allocate tsbinfos for kernel and copy in data 11707 * to make debug easier and sun4v setup easier. 11708 */ 11709 ktsb_info[0].tsb_sfmmu = ksfmmup; 11710 ktsb_info[0].tsb_szc = ktsb_szcode; 11711 ktsb_info[0].tsb_ttesz_mask = TSB8K|TSB64K|TSB512K; 11712 ktsb_info[0].tsb_va = ktsb_base; 11713 ktsb_info[0].tsb_pa = ktsb_pbase; 11714 ktsb_info[0].tsb_flags = 0; 11715 ktsb_info[0].tsb_tte.ll = 0; 11716 ktsb_info[0].tsb_cache = NULL; 11717 11718 ktsb_info[1].tsb_sfmmu = ksfmmup; 11719 ktsb_info[1].tsb_szc = ktsb4m_szcode; 11720 ktsb_info[1].tsb_ttesz_mask = TSB4M; 11721 ktsb_info[1].tsb_va = ktsb4m_base; 11722 ktsb_info[1].tsb_pa = ktsb4m_pbase; 11723 ktsb_info[1].tsb_flags = 0; 11724 ktsb_info[1].tsb_tte.ll = 0; 11725 ktsb_info[1].tsb_cache = NULL; 11726 11727 /* Link them into ksfmmup. */ 11728 ktsb_info[0].tsb_next = &ktsb_info[1]; 11729 ktsb_info[1].tsb_next = NULL; 11730 ksfmmup->sfmmu_tsb = &ktsb_info[0]; 11731 11732 sfmmu_setup_tsbinfo(ksfmmup); 11733 } 11734 11735 /* 11736 * Cache the last value returned from va_to_pa(). If the VA specified 11737 * in the current call to cached_va_to_pa() maps to the same Page (as the 11738 * previous call to cached_va_to_pa()), then compute the PA using 11739 * cached info, else call va_to_pa(). 11740 * 11741 * Note: this function is neither MT-safe nor consistent in the presence 11742 * of multiple, interleaved threads. This function was created to enable 11743 * an optimization used during boot (at a point when there's only one thread 11744 * executing on the "boot CPU", and before startup_vm() has been called). 11745 */ 11746 static uint64_t 11747 cached_va_to_pa(void *vaddr) 11748 { 11749 static uint64_t prev_vaddr_base = 0; 11750 static uint64_t prev_pfn = 0; 11751 11752 if ((((uint64_t)vaddr) & MMU_PAGEMASK) == prev_vaddr_base) { 11753 return (prev_pfn | ((uint64_t)vaddr & MMU_PAGEOFFSET)); 11754 } else { 11755 uint64_t pa = va_to_pa(vaddr); 11756 11757 if (pa != ((uint64_t)-1)) { 11758 /* 11759 * Computed physical address is valid. Cache its 11760 * related info for the next cached_va_to_pa() call. 11761 */ 11762 prev_pfn = pa & MMU_PAGEMASK; 11763 prev_vaddr_base = ((uint64_t)vaddr) & MMU_PAGEMASK; 11764 } 11765 11766 return (pa); 11767 } 11768 } 11769 11770 /* 11771 * Carve up our nucleus hblk region. We may allocate more hblks than 11772 * asked due to rounding errors but we are guaranteed to have at least 11773 * enough space to allocate the requested number of hblk8's and hblk1's. 11774 */ 11775 void 11776 sfmmu_init_nucleus_hblks(caddr_t addr, size_t size, int nhblk8, int nhblk1) 11777 { 11778 struct hme_blk *hmeblkp; 11779 size_t hme8blk_sz, hme1blk_sz; 11780 size_t i; 11781 size_t hblk8_bound; 11782 ulong_t j = 0, k = 0; 11783 11784 ASSERT(addr != NULL && size != 0); 11785 11786 /* Need to use proper structure alignment */ 11787 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 11788 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 11789 11790 nucleus_hblk8.list = (void *)addr; 11791 nucleus_hblk8.index = 0; 11792 11793 /* 11794 * Use as much memory as possible for hblk8's since we 11795 * expect all bop_alloc'ed memory to be allocated in 8k chunks. 11796 * We need to hold back enough space for the hblk1's which 11797 * we'll allocate next. 11798 */ 11799 hblk8_bound = size - (nhblk1 * hme1blk_sz) - hme8blk_sz; 11800 for (i = 0; i <= hblk8_bound; i += hme8blk_sz, j++) { 11801 hmeblkp = (struct hme_blk *)addr; 11802 addr += hme8blk_sz; 11803 hmeblkp->hblk_nuc_bit = 1; 11804 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11805 } 11806 nucleus_hblk8.len = j; 11807 ASSERT(j >= nhblk8); 11808 SFMMU_STAT_ADD(sf_hblk8_ncreate, j); 11809 11810 nucleus_hblk1.list = (void *)addr; 11811 nucleus_hblk1.index = 0; 11812 for (; i <= (size - hme1blk_sz); i += hme1blk_sz, k++) { 11813 hmeblkp = (struct hme_blk *)addr; 11814 addr += hme1blk_sz; 11815 hmeblkp->hblk_nuc_bit = 1; 11816 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11817 } 11818 ASSERT(k >= nhblk1); 11819 nucleus_hblk1.len = k; 11820 SFMMU_STAT_ADD(sf_hblk1_ncreate, k); 11821 } 11822 11823 /* 11824 * This function is currently not supported on this platform. For what 11825 * it's supposed to do, see hat.c and hat_srmmu.c 11826 */ 11827 /* ARGSUSED */ 11828 faultcode_t 11829 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp, 11830 uint_t flags) 11831 { 11832 ASSERT(hat->sfmmu_xhat_provider == NULL); 11833 return (FC_NOSUPPORT); 11834 } 11835 11836 /* 11837 * Searchs the mapping list of the page for a mapping of the same size. If not 11838 * found the corresponding bit is cleared in the p_index field. When large 11839 * pages are more prevalent in the system, we can maintain the mapping list 11840 * in order and we don't have to traverse the list each time. Just check the 11841 * next and prev entries, and if both are of different size, we clear the bit. 11842 */ 11843 static void 11844 sfmmu_rm_large_mappings(page_t *pp, int ttesz) 11845 { 11846 struct sf_hment *sfhmep; 11847 struct hme_blk *hmeblkp; 11848 int index; 11849 pgcnt_t npgs; 11850 11851 ASSERT(ttesz > TTE8K); 11852 11853 ASSERT(sfmmu_mlist_held(pp)); 11854 11855 ASSERT(PP_ISMAPPED_LARGE(pp)); 11856 11857 /* 11858 * Traverse mapping list looking for another mapping of same size. 11859 * since we only want to clear index field if all mappings of 11860 * that size are gone. 11861 */ 11862 11863 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 11864 hmeblkp = sfmmu_hmetohblk(sfhmep); 11865 if (hmeblkp->hblk_xhat_bit) 11866 continue; 11867 if (hme_size(sfhmep) == ttesz) { 11868 /* 11869 * another mapping of the same size. don't clear index. 11870 */ 11871 return; 11872 } 11873 } 11874 11875 /* 11876 * Clear the p_index bit for large page. 11877 */ 11878 index = PAGESZ_TO_INDEX(ttesz); 11879 npgs = TTEPAGES(ttesz); 11880 while (npgs-- > 0) { 11881 ASSERT(pp->p_index & index); 11882 pp->p_index &= ~index; 11883 pp = PP_PAGENEXT(pp); 11884 } 11885 } 11886 11887 /* 11888 * return supported features 11889 */ 11890 /* ARGSUSED */ 11891 int 11892 hat_supported(enum hat_features feature, void *arg) 11893 { 11894 switch (feature) { 11895 case HAT_SHARED_PT: 11896 case HAT_DYNAMIC_ISM_UNMAP: 11897 case HAT_VMODSORT: 11898 return (1); 11899 default: 11900 return (0); 11901 } 11902 } 11903 11904 void 11905 hat_enter(struct hat *hat) 11906 { 11907 hatlock_t *hatlockp; 11908 11909 if (hat != ksfmmup) { 11910 hatlockp = TSB_HASH(hat); 11911 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 11912 } 11913 } 11914 11915 void 11916 hat_exit(struct hat *hat) 11917 { 11918 hatlock_t *hatlockp; 11919 11920 if (hat != ksfmmup) { 11921 hatlockp = TSB_HASH(hat); 11922 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 11923 } 11924 } 11925 11926 /*ARGSUSED*/ 11927 void 11928 hat_reserve(struct as *as, caddr_t addr, size_t len) 11929 { 11930 } 11931 11932 static void 11933 hat_kstat_init(void) 11934 { 11935 kstat_t *ksp; 11936 11937 ksp = kstat_create("unix", 0, "sfmmu_global_stat", "hat", 11938 KSTAT_TYPE_RAW, sizeof (struct sfmmu_global_stat), 11939 KSTAT_FLAG_VIRTUAL); 11940 if (ksp) { 11941 ksp->ks_data = (void *) &sfmmu_global_stat; 11942 kstat_install(ksp); 11943 } 11944 ksp = kstat_create("unix", 0, "sfmmu_tsbsize_stat", "hat", 11945 KSTAT_TYPE_RAW, sizeof (struct sfmmu_tsbsize_stat), 11946 KSTAT_FLAG_VIRTUAL); 11947 if (ksp) { 11948 ksp->ks_data = (void *) &sfmmu_tsbsize_stat; 11949 kstat_install(ksp); 11950 } 11951 ksp = kstat_create("unix", 0, "sfmmu_percpu_stat", "hat", 11952 KSTAT_TYPE_RAW, sizeof (struct sfmmu_percpu_stat) * NCPU, 11953 KSTAT_FLAG_WRITABLE); 11954 if (ksp) { 11955 ksp->ks_update = sfmmu_kstat_percpu_update; 11956 kstat_install(ksp); 11957 } 11958 } 11959 11960 /* ARGSUSED */ 11961 static int 11962 sfmmu_kstat_percpu_update(kstat_t *ksp, int rw) 11963 { 11964 struct sfmmu_percpu_stat *cpu_kstat = ksp->ks_data; 11965 struct tsbmiss *tsbm = tsbmiss_area; 11966 struct kpmtsbm *kpmtsbm = kpmtsbm_area; 11967 int i; 11968 11969 ASSERT(cpu_kstat); 11970 if (rw == KSTAT_READ) { 11971 for (i = 0; i < NCPU; cpu_kstat++, tsbm++, kpmtsbm++, i++) { 11972 cpu_kstat->sf_itlb_misses = tsbm->itlb_misses; 11973 cpu_kstat->sf_dtlb_misses = tsbm->dtlb_misses; 11974 cpu_kstat->sf_utsb_misses = tsbm->utsb_misses - 11975 tsbm->uprot_traps; 11976 cpu_kstat->sf_ktsb_misses = tsbm->ktsb_misses + 11977 kpmtsbm->kpm_tsb_misses - tsbm->kprot_traps; 11978 11979 if (tsbm->itlb_misses > 0 && tsbm->dtlb_misses > 0) { 11980 cpu_kstat->sf_tsb_hits = 11981 (tsbm->itlb_misses + tsbm->dtlb_misses) - 11982 (tsbm->utsb_misses + tsbm->ktsb_misses + 11983 kpmtsbm->kpm_tsb_misses); 11984 } else { 11985 cpu_kstat->sf_tsb_hits = 0; 11986 } 11987 cpu_kstat->sf_umod_faults = tsbm->uprot_traps; 11988 cpu_kstat->sf_kmod_faults = tsbm->kprot_traps; 11989 } 11990 } else { 11991 /* KSTAT_WRITE is used to clear stats */ 11992 for (i = 0; i < NCPU; tsbm++, kpmtsbm++, i++) { 11993 tsbm->itlb_misses = 0; 11994 tsbm->dtlb_misses = 0; 11995 tsbm->utsb_misses = 0; 11996 tsbm->ktsb_misses = 0; 11997 tsbm->uprot_traps = 0; 11998 tsbm->kprot_traps = 0; 11999 kpmtsbm->kpm_dtlb_misses = 0; 12000 kpmtsbm->kpm_tsb_misses = 0; 12001 } 12002 } 12003 return (0); 12004 } 12005 12006 #ifdef DEBUG 12007 12008 tte_t *gorig[NCPU], *gcur[NCPU], *gnew[NCPU]; 12009 12010 /* 12011 * A tte checker. *orig_old is the value we read before cas. 12012 * *cur is the value returned by cas. 12013 * *new is the desired value when we do the cas. 12014 * 12015 * *hmeblkp is currently unused. 12016 */ 12017 12018 /* ARGSUSED */ 12019 void 12020 chk_tte(tte_t *orig_old, tte_t *cur, tte_t *new, struct hme_blk *hmeblkp) 12021 { 12022 pfn_t i, j, k; 12023 int cpuid = CPU->cpu_id; 12024 12025 gorig[cpuid] = orig_old; 12026 gcur[cpuid] = cur; 12027 gnew[cpuid] = new; 12028 12029 #ifdef lint 12030 hmeblkp = hmeblkp; 12031 #endif 12032 12033 if (TTE_IS_VALID(orig_old)) { 12034 if (TTE_IS_VALID(cur)) { 12035 i = TTE_TO_TTEPFN(orig_old); 12036 j = TTE_TO_TTEPFN(cur); 12037 k = TTE_TO_TTEPFN(new); 12038 if (i != j) { 12039 /* remap error? */ 12040 panic("chk_tte: bad pfn, 0x%lx, 0x%lx", i, j); 12041 } 12042 12043 if (i != k) { 12044 /* remap error? */ 12045 panic("chk_tte: bad pfn2, 0x%lx, 0x%lx", i, k); 12046 } 12047 } else { 12048 if (TTE_IS_VALID(new)) { 12049 panic("chk_tte: invalid cur? "); 12050 } 12051 12052 i = TTE_TO_TTEPFN(orig_old); 12053 k = TTE_TO_TTEPFN(new); 12054 if (i != k) { 12055 panic("chk_tte: bad pfn3, 0x%lx, 0x%lx", i, k); 12056 } 12057 } 12058 } else { 12059 if (TTE_IS_VALID(cur)) { 12060 j = TTE_TO_TTEPFN(cur); 12061 if (TTE_IS_VALID(new)) { 12062 k = TTE_TO_TTEPFN(new); 12063 if (j != k) { 12064 panic("chk_tte: bad pfn4, 0x%lx, 0x%lx", 12065 j, k); 12066 } 12067 } else { 12068 panic("chk_tte: why here?"); 12069 } 12070 } else { 12071 if (!TTE_IS_VALID(new)) { 12072 panic("chk_tte: why here2 ?"); 12073 } 12074 } 12075 } 12076 } 12077 12078 #endif /* DEBUG */ 12079 12080 extern void prefetch_tsbe_read(struct tsbe *); 12081 extern void prefetch_tsbe_write(struct tsbe *); 12082 12083 12084 /* 12085 * We want to prefetch 7 cache lines ahead for our read prefetch. This gives 12086 * us optimal performance on Cheetah+. You can only have 8 outstanding 12087 * prefetches at any one time, so we opted for 7 read prefetches and 1 write 12088 * prefetch to make the most utilization of the prefetch capability. 12089 */ 12090 #define TSBE_PREFETCH_STRIDE (7) 12091 12092 void 12093 sfmmu_copy_tsb(struct tsb_info *old_tsbinfo, struct tsb_info *new_tsbinfo) 12094 { 12095 int old_bytes = TSB_BYTES(old_tsbinfo->tsb_szc); 12096 int new_bytes = TSB_BYTES(new_tsbinfo->tsb_szc); 12097 int old_entries = TSB_ENTRIES(old_tsbinfo->tsb_szc); 12098 int new_entries = TSB_ENTRIES(new_tsbinfo->tsb_szc); 12099 struct tsbe *old; 12100 struct tsbe *new; 12101 struct tsbe *new_base = (struct tsbe *)new_tsbinfo->tsb_va; 12102 uint64_t va; 12103 int new_offset; 12104 int i; 12105 int vpshift; 12106 int last_prefetch; 12107 12108 if (old_bytes == new_bytes) { 12109 bcopy(old_tsbinfo->tsb_va, new_tsbinfo->tsb_va, new_bytes); 12110 } else { 12111 12112 /* 12113 * A TSBE is 16 bytes which means there are four TSBE's per 12114 * P$ line (64 bytes), thus every 4 TSBE's we prefetch. 12115 */ 12116 old = (struct tsbe *)old_tsbinfo->tsb_va; 12117 last_prefetch = old_entries - (4*(TSBE_PREFETCH_STRIDE+1)); 12118 for (i = 0; i < old_entries; i++, old++) { 12119 if (((i & (4-1)) == 0) && (i < last_prefetch)) 12120 prefetch_tsbe_read(old); 12121 if (!old->tte_tag.tag_invalid) { 12122 /* 12123 * We have a valid TTE to remap. Check the 12124 * size. We won't remap 64K or 512K TTEs 12125 * because they span more than one TSB entry 12126 * and are indexed using an 8K virt. page. 12127 * Ditto for 32M and 256M TTEs. 12128 */ 12129 if (TTE_CSZ(&old->tte_data) == TTE64K || 12130 TTE_CSZ(&old->tte_data) == TTE512K) 12131 continue; 12132 if (mmu_page_sizes == max_mmu_page_sizes) { 12133 if (TTE_CSZ(&old->tte_data) == TTE32M || 12134 TTE_CSZ(&old->tte_data) == TTE256M) 12135 continue; 12136 } 12137 12138 /* clear the lower 22 bits of the va */ 12139 va = *(uint64_t *)old << 22; 12140 /* turn va into a virtual pfn */ 12141 va >>= 22 - TSB_START_SIZE; 12142 /* 12143 * or in bits from the offset in the tsb 12144 * to get the real virtual pfn. These 12145 * correspond to bits [21:13] in the va 12146 */ 12147 vpshift = 12148 TTE_BSZS_SHIFT(TTE_CSZ(&old->tte_data)) & 12149 0x1ff; 12150 va |= (i << vpshift); 12151 va >>= vpshift; 12152 new_offset = va & (new_entries - 1); 12153 new = new_base + new_offset; 12154 prefetch_tsbe_write(new); 12155 *new = *old; 12156 } 12157 } 12158 } 12159 } 12160 12161 /* 12162 * unused in sfmmu 12163 */ 12164 void 12165 hat_dump(void) 12166 { 12167 } 12168 12169 /* 12170 * Called when a thread is exiting and we have switched to the kernel address 12171 * space. Perform the same VM initialization resume() uses when switching 12172 * processes. 12173 * 12174 * Note that sfmmu_load_mmustate() is currently a no-op for kernel threads, but 12175 * we call it anyway in case the semantics change in the future. 12176 */ 12177 /*ARGSUSED*/ 12178 void 12179 hat_thread_exit(kthread_t *thd) 12180 { 12181 uint64_t pgsz_cnum; 12182 uint_t pstate_save; 12183 12184 ASSERT(thd->t_procp->p_as == &kas); 12185 12186 pgsz_cnum = KCONTEXT; 12187 #ifdef sun4u 12188 pgsz_cnum |= (ksfmmup->sfmmu_cext << CTXREG_EXT_SHIFT); 12189 #endif 12190 /* 12191 * Note that sfmmu_load_mmustate() is currently a no-op for 12192 * kernel threads. We need to disable interrupts here, 12193 * simply because otherwise sfmmu_load_mmustate() would panic 12194 * if the caller does not disable interrupts. 12195 */ 12196 pstate_save = sfmmu_disable_intrs(); 12197 sfmmu_setctx_sec(pgsz_cnum); 12198 sfmmu_load_mmustate(ksfmmup); 12199 sfmmu_enable_intrs(pstate_save); 12200 } 12201