1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * VM - Hardware Address Translation management for Spitfire MMU. 31 * 32 * This file implements the machine specific hardware translation 33 * needed by the VM system. The machine independent interface is 34 * described in <vm/hat.h> while the machine dependent interface 35 * and data structures are described in <vm/hat_sfmmu.h>. 36 * 37 * The hat layer manages the address translation hardware as a cache 38 * driven by calls from the higher levels in the VM system. 39 */ 40 41 #include <sys/types.h> 42 #include <vm/hat.h> 43 #include <vm/hat_sfmmu.h> 44 #include <vm/page.h> 45 #include <sys/pte.h> 46 #include <sys/systm.h> 47 #include <sys/mman.h> 48 #include <sys/sysmacros.h> 49 #include <sys/machparam.h> 50 #include <sys/vtrace.h> 51 #include <sys/kmem.h> 52 #include <sys/mmu.h> 53 #include <sys/cmn_err.h> 54 #include <sys/cpu.h> 55 #include <sys/cpuvar.h> 56 #include <sys/debug.h> 57 #include <sys/lgrp.h> 58 #include <sys/archsystm.h> 59 #include <sys/machsystm.h> 60 #include <sys/vmsystm.h> 61 #include <vm/as.h> 62 #include <vm/seg.h> 63 #include <vm/seg_kp.h> 64 #include <vm/seg_kmem.h> 65 #include <vm/seg_kpm.h> 66 #include <vm/rm.h> 67 #include <sys/t_lock.h> 68 #include <sys/obpdefs.h> 69 #include <sys/vm_machparam.h> 70 #include <sys/var.h> 71 #include <sys/trap.h> 72 #include <sys/machtrap.h> 73 #include <sys/scb.h> 74 #include <sys/bitmap.h> 75 #include <sys/machlock.h> 76 #include <sys/membar.h> 77 #include <sys/atomic.h> 78 #include <sys/cpu_module.h> 79 #include <sys/prom_debug.h> 80 #include <sys/ksynch.h> 81 #include <sys/mem_config.h> 82 #include <sys/mem_cage.h> 83 #include <sys/dtrace.h> 84 #include <vm/vm_dep.h> 85 #include <vm/xhat_sfmmu.h> 86 #include <sys/fpu/fpusystm.h> 87 88 #if defined(SF_ERRATA_57) 89 extern caddr_t errata57_limit; 90 #endif 91 92 #define HME8BLK_SZ_RND ((roundup(HME8BLK_SZ, sizeof (int64_t))) / \ 93 (sizeof (int64_t))) 94 #define HBLK_RESERVE ((struct hme_blk *)hblk_reserve) 95 96 #define HBLK_RESERVE_CNT 128 97 #define HBLK_RESERVE_MIN 20 98 99 static struct hme_blk *freehblkp; 100 static kmutex_t freehblkp_lock; 101 static int freehblkcnt; 102 103 static int64_t hblk_reserve[HME8BLK_SZ_RND]; 104 static kmutex_t hblk_reserve_lock; 105 static kthread_t *hblk_reserve_thread; 106 107 static nucleus_hblk8_info_t nucleus_hblk8; 108 static nucleus_hblk1_info_t nucleus_hblk1; 109 110 /* 111 * SFMMU specific hat functions 112 */ 113 void hat_pagecachectl(struct page *, int); 114 115 /* flags for hat_pagecachectl */ 116 #define HAT_CACHE 0x1 117 #define HAT_UNCACHE 0x2 118 #define HAT_TMPNC 0x4 119 120 /* 121 * Flag to allow the creation of non-cacheable translations 122 * to system memory. It is off by default. At the moment this 123 * flag is used by the ecache error injector. The error injector 124 * will turn it on when creating such a translation then shut it 125 * off when it's finished. 126 */ 127 128 int sfmmu_allow_nc_trans = 0; 129 130 /* 131 * Flag to disable large page support. 132 * value of 1 => disable all large pages. 133 * bits 1, 2, and 3 are to disable 64K, 512K and 4M pages respectively. 134 * 135 * For example, use the value 0x4 to disable 512K pages. 136 * 137 */ 138 #define LARGE_PAGES_OFF 0x1 139 140 /* 141 * WARNING: 512K pages MUST be disabled for ISM/DISM. If not 142 * a process would page fault indefinitely if it tried to 143 * access a 512K page. 144 */ 145 int disable_ism_large_pages = (1 << TTE512K); 146 int disable_large_pages = 0; 147 int disable_auto_large_pages = 0; 148 149 /* 150 * Private sfmmu data structures for hat management 151 */ 152 static struct kmem_cache *sfmmuid_cache; 153 154 /* 155 * Private sfmmu data structures for ctx management 156 */ 157 static struct ctx *ctxhand; /* hand used while stealing ctxs */ 158 static struct ctx *ctxfree; /* head of free ctx list */ 159 static struct ctx *ctxdirty; /* head of dirty ctx list */ 160 161 /* 162 * Private sfmmu data structures for tsb management 163 */ 164 static struct kmem_cache *sfmmu_tsbinfo_cache; 165 static struct kmem_cache *sfmmu_tsb8k_cache; 166 static struct kmem_cache *sfmmu_tsb_cache[NLGRPS_MAX]; 167 static vmem_t *kmem_tsb_arena; 168 169 /* 170 * sfmmu static variables for hmeblk resource management. 171 */ 172 static vmem_t *hat_memload1_arena; /* HAT translation arena for sfmmu1_cache */ 173 static struct kmem_cache *sfmmu8_cache; 174 static struct kmem_cache *sfmmu1_cache; 175 static struct kmem_cache *pa_hment_cache; 176 177 static kmutex_t ctx_list_lock; /* mutex for ctx free/dirty lists */ 178 static kmutex_t ism_mlist_lock; /* mutex for ism mapping list */ 179 /* 180 * private data for ism 181 */ 182 static struct kmem_cache *ism_blk_cache; 183 static struct kmem_cache *ism_ment_cache; 184 #define ISMID_STARTADDR NULL 185 186 /* 187 * Whether to delay TLB flushes and use Cheetah's flush-all support 188 * when removing contexts from the dirty list. 189 */ 190 int delay_tlb_flush; 191 int disable_delay_tlb_flush; 192 193 /* 194 * ``hat_lock'' is a hashed mutex lock for protecting sfmmu TSB lists, 195 * HAT flags, synchronizing TLB/TSB coherency, and context management. 196 * The lock is hashed on the sfmmup since the case where we need to lock 197 * all processes is rare but does occur (e.g. we need to unload a shared 198 * mapping from all processes using the mapping). We have a lot of buckets, 199 * and each slab of sfmmu_t's can use about a quarter of them, giving us 200 * a fairly good distribution without wasting too much space and overhead 201 * when we have to grab them all. 202 */ 203 #define SFMMU_NUM_LOCK 128 /* must be power of two */ 204 hatlock_t hat_lock[SFMMU_NUM_LOCK]; 205 206 /* 207 * Hash algorithm optimized for a small number of slabs. 208 * 7 is (highbit((sizeof sfmmu_t)) - 1) 209 * This hash algorithm is based upon the knowledge that sfmmu_t's come from a 210 * kmem_cache, and thus they will be sequential within that cache. In 211 * addition, each new slab will have a different "color" up to cache_maxcolor 212 * which will skew the hashing for each successive slab which is allocated. 213 * If the size of sfmmu_t changed to a larger size, this algorithm may need 214 * to be revisited. 215 */ 216 #define TSB_HASH_SHIFT_BITS (7) 217 #define PTR_HASH(x) ((uintptr_t)x >> TSB_HASH_SHIFT_BITS) 218 219 #ifdef DEBUG 220 int tsb_hash_debug = 0; 221 #define TSB_HASH(sfmmup) \ 222 (tsb_hash_debug ? &hat_lock[0] : \ 223 &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)]) 224 #else /* DEBUG */ 225 #define TSB_HASH(sfmmup) &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)] 226 #endif /* DEBUG */ 227 228 229 /* sfmmu_replace_tsb() return codes. */ 230 typedef enum tsb_replace_rc { 231 TSB_SUCCESS, 232 TSB_ALLOCFAIL, 233 TSB_LOSTRACE, 234 TSB_ALREADY_SWAPPED, 235 TSB_CANTGROW 236 } tsb_replace_rc_t; 237 238 /* 239 * Flags for TSB allocation routines. 240 */ 241 #define TSB_ALLOC 0x01 242 #define TSB_FORCEALLOC 0x02 243 #define TSB_GROW 0x04 244 #define TSB_SHRINK 0x08 245 #define TSB_SWAPIN 0x10 246 247 /* 248 * Support for HAT callbacks. 249 */ 250 #define SFMMU_MAX_RELOC_CALLBACKS 10 251 int sfmmu_max_cb_id = SFMMU_MAX_RELOC_CALLBACKS; 252 static id_t sfmmu_cb_nextid = 0; 253 static id_t sfmmu_tsb_cb_id; 254 struct sfmmu_callback *sfmmu_cb_table; 255 256 /* 257 * Kernel page relocation is enabled by default for non-caged 258 * kernel pages. This has little effect unless segkmem_reloc is 259 * set, since by default kernel memory comes from inside the 260 * kernel cage. 261 */ 262 int hat_kpr_enabled = 1; 263 264 kmutex_t kpr_mutex; 265 kmutex_t kpr_suspendlock; 266 kthread_t *kreloc_thread; 267 268 /* 269 * Enable VA->PA translation sanity checking on DEBUG kernels. 270 * Disabled by default. This is incompatible with some 271 * drivers (error injector, RSM) so if it breaks you get 272 * to keep both pieces. 273 */ 274 int hat_check_vtop = 0; 275 276 /* 277 * Private sfmmu routines (prototypes) 278 */ 279 static struct hme_blk *sfmmu_shadow_hcreate(sfmmu_t *, caddr_t, int, uint_t); 280 static struct hme_blk *sfmmu_hblk_alloc(sfmmu_t *, caddr_t, 281 struct hmehash_bucket *, uint_t, hmeblk_tag, uint_t); 282 static caddr_t sfmmu_hblk_unload(struct hat *, struct hme_blk *, caddr_t, 283 caddr_t, demap_range_t *, uint_t); 284 static caddr_t sfmmu_hblk_sync(struct hat *, struct hme_blk *, caddr_t, 285 caddr_t, int); 286 static void sfmmu_hblk_free(struct hmehash_bucket *, struct hme_blk *, 287 uint64_t, struct hme_blk **); 288 static void sfmmu_hblks_list_purge(struct hme_blk **); 289 static uint_t sfmmu_get_free_hblk(struct hme_blk **, uint_t); 290 static uint_t sfmmu_put_free_hblk(struct hme_blk *, uint_t); 291 static struct hme_blk *sfmmu_hblk_steal(int); 292 static int sfmmu_steal_this_hblk(struct hmehash_bucket *, 293 struct hme_blk *, uint64_t, uint64_t, 294 struct hme_blk *); 295 static caddr_t sfmmu_hblk_unlock(struct hme_blk *, caddr_t, caddr_t); 296 297 static void sfmmu_memload_batchsmall(struct hat *, caddr_t, page_t **, 298 uint_t, uint_t, pgcnt_t); 299 void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, 300 uint_t); 301 static int sfmmu_tteload_array(sfmmu_t *, tte_t *, caddr_t, page_t **, 302 uint_t); 303 static struct hmehash_bucket *sfmmu_tteload_acquire_hashbucket(sfmmu_t *, 304 caddr_t, int); 305 static struct hme_blk *sfmmu_tteload_find_hmeblk(sfmmu_t *, 306 struct hmehash_bucket *, caddr_t, uint_t, uint_t); 307 static int sfmmu_tteload_addentry(sfmmu_t *, struct hme_blk *, tte_t *, 308 caddr_t, page_t **, uint_t); 309 static void sfmmu_tteload_release_hashbucket(struct hmehash_bucket *); 310 311 static int sfmmu_pagearray_setup(caddr_t, page_t **, tte_t *, int); 312 pfn_t sfmmu_uvatopfn(caddr_t, sfmmu_t *); 313 void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 314 static void sfmmu_vac_conflict(struct hat *, caddr_t, page_t *); 315 static int sfmmu_vacconflict_array(caddr_t, page_t *, int *); 316 static int tst_tnc(page_t *pp, pgcnt_t); 317 static void conv_tnc(page_t *pp, int); 318 319 static struct ctx *sfmmu_get_ctx(sfmmu_t *); 320 static void sfmmu_free_ctx(sfmmu_t *, struct ctx *); 321 static void sfmmu_free_sfmmu(sfmmu_t *); 322 323 static void sfmmu_gettte(struct hat *, caddr_t, tte_t *); 324 static void sfmmu_ttesync(struct hat *, caddr_t, tte_t *, page_t *); 325 static void sfmmu_chgattr(struct hat *, caddr_t, size_t, uint_t, int); 326 327 static cpuset_t sfmmu_pageunload(page_t *, struct sf_hment *, int); 328 static void hat_pagereload(struct page *, struct page *); 329 static cpuset_t sfmmu_pagesync(page_t *, struct sf_hment *, uint_t); 330 static void sfmmu_page_cache_array(page_t *, int, int, pgcnt_t); 331 static void sfmmu_page_cache(page_t *, int, int, int); 332 333 static void sfmmu_tlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 334 pfn_t, int, int, int, int); 335 static void sfmmu_ismtlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 336 pfn_t, int); 337 static void sfmmu_tlb_demap(caddr_t, sfmmu_t *, struct hme_blk *, int, int); 338 static void sfmmu_tlb_range_demap(demap_range_t *); 339 static void sfmmu_tlb_ctx_demap(sfmmu_t *); 340 static void sfmmu_tlb_all_demap(void); 341 static void sfmmu_tlb_swap_ctx(sfmmu_t *, struct ctx *); 342 static void sfmmu_sync_mmustate(sfmmu_t *); 343 344 static void sfmmu_tsbinfo_setup_phys(struct tsb_info *, pfn_t); 345 static int sfmmu_tsbinfo_alloc(struct tsb_info **, int, int, uint_t, 346 sfmmu_t *); 347 static void sfmmu_tsb_free(struct tsb_info *); 348 static void sfmmu_tsbinfo_free(struct tsb_info *); 349 static int sfmmu_init_tsbinfo(struct tsb_info *, int, int, uint_t, 350 sfmmu_t *); 351 352 static void sfmmu_tsb_swapin(sfmmu_t *, hatlock_t *); 353 static int sfmmu_select_tsb_szc(pgcnt_t); 354 static void sfmmu_mod_tsb(sfmmu_t *, caddr_t, tte_t *, int); 355 #define sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \ 356 sfmmu_mod_tsb(sfmmup, vaddr, tte, szc) 357 #define sfmmu_unload_tsb(sfmmup, vaddr, szc) \ 358 sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc) 359 static void sfmmu_copy_tsb(struct tsb_info *, struct tsb_info *); 360 static tsb_replace_rc_t sfmmu_replace_tsb(sfmmu_t *, struct tsb_info *, uint_t, 361 hatlock_t *, uint_t); 362 static void sfmmu_size_tsb(sfmmu_t *, int, uint64_t, uint64_t, int); 363 364 static void sfmmu_cache_flush(pfn_t, int); 365 void sfmmu_cache_flushcolor(int, pfn_t); 366 static caddr_t sfmmu_hblk_chgattr(sfmmu_t *, struct hme_blk *, caddr_t, 367 caddr_t, demap_range_t *, uint_t, int); 368 369 static uint64_t sfmmu_vtop_attr(uint_t, int mode, tte_t *); 370 static uint_t sfmmu_ptov_attr(tte_t *); 371 static caddr_t sfmmu_hblk_chgprot(sfmmu_t *, struct hme_blk *, caddr_t, 372 caddr_t, demap_range_t *, uint_t); 373 static uint_t sfmmu_vtop_prot(uint_t, uint_t *); 374 static int sfmmu_idcache_constructor(void *, void *, int); 375 static void sfmmu_idcache_destructor(void *, void *); 376 static int sfmmu_hblkcache_constructor(void *, void *, int); 377 static void sfmmu_hblkcache_destructor(void *, void *); 378 static void sfmmu_hblkcache_reclaim(void *); 379 static void sfmmu_shadow_hcleanup(sfmmu_t *, struct hme_blk *, 380 struct hmehash_bucket *); 381 static void sfmmu_free_hblks(sfmmu_t *, caddr_t, caddr_t, int); 382 383 static void sfmmu_reuse_ctx(struct ctx *, sfmmu_t *); 384 static void sfmmu_disallow_ctx_steal(sfmmu_t *); 385 static void sfmmu_allow_ctx_steal(sfmmu_t *); 386 387 static void sfmmu_rm_large_mappings(page_t *, int); 388 389 static void hat_lock_init(void); 390 static void hat_kstat_init(void); 391 static int sfmmu_kstat_percpu_update(kstat_t *ksp, int rw); 392 static void sfmmu_check_page_sizes(sfmmu_t *, int); 393 static int fnd_mapping_sz(page_t *); 394 static void iment_add(struct ism_ment *, struct hat *); 395 static void iment_sub(struct ism_ment *, struct hat *); 396 static pgcnt_t ism_tsb_entries(sfmmu_t *, int szc); 397 extern void sfmmu_setup_tsbinfo(sfmmu_t *); 398 extern void sfmmu_clear_utsbinfo(void); 399 400 /* kpm prototypes */ 401 static caddr_t sfmmu_kpm_mapin(page_t *); 402 static void sfmmu_kpm_mapout(page_t *, caddr_t); 403 static int sfmmu_kpme_lookup(struct kpme *, page_t *); 404 static void sfmmu_kpme_add(struct kpme *, page_t *); 405 static void sfmmu_kpme_sub(struct kpme *, page_t *); 406 static caddr_t sfmmu_kpm_getvaddr(page_t *, int *); 407 static int sfmmu_kpm_fault(caddr_t, struct memseg *, page_t *); 408 static int sfmmu_kpm_fault_small(caddr_t, struct memseg *, page_t *); 409 static void sfmmu_kpm_vac_conflict(page_t *, caddr_t); 410 static void sfmmu_kpm_pageunload(page_t *); 411 static void sfmmu_kpm_vac_unload(page_t *, caddr_t); 412 static void sfmmu_kpm_demap_large(caddr_t); 413 static void sfmmu_kpm_demap_small(caddr_t); 414 static void sfmmu_kpm_demap_tlbs(caddr_t, int); 415 static void sfmmu_kpm_hme_unload(page_t *); 416 static kpm_hlk_t *sfmmu_kpm_kpmp_enter(page_t *, pgcnt_t); 417 static void sfmmu_kpm_kpmp_exit(kpm_hlk_t *kpmp); 418 static void sfmmu_kpm_page_cache(page_t *, int, int); 419 420 /* kpm globals */ 421 #ifdef DEBUG 422 /* 423 * Enable trap level tsbmiss handling 424 */ 425 int kpm_tsbmtl = 1; 426 427 /* 428 * Flush the TLB on kpm mapout. Note: Xcalls are used (again) for the 429 * required TLB shootdowns in this case, so handle w/ care. Off by default. 430 */ 431 int kpm_tlb_flush; 432 #endif /* DEBUG */ 433 434 static void *sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *, size_t, int); 435 436 #ifdef DEBUG 437 static void sfmmu_check_hblk_flist(); 438 #endif 439 440 /* 441 * Semi-private sfmmu data structures. Some of them are initialize in 442 * startup or in hat_init. Some of them are private but accessed by 443 * assembly code or mach_sfmmu.c 444 */ 445 struct hmehash_bucket *uhme_hash; /* user hmeblk hash table */ 446 struct hmehash_bucket *khme_hash; /* kernel hmeblk hash table */ 447 uint64_t uhme_hash_pa; /* PA of uhme_hash */ 448 uint64_t khme_hash_pa; /* PA of khme_hash */ 449 int uhmehash_num; /* # of buckets in user hash table */ 450 int khmehash_num; /* # of buckets in kernel hash table */ 451 struct ctx *ctxs; /* used by <machine/mmu.c> */ 452 uint_t nctxs; /* total number of contexts */ 453 454 int cache; /* describes system cache */ 455 456 caddr_t ktsb_base; /* kernel 8k-indexed tsb base address */ 457 uint64_t ktsb_pbase; /* kernel 8k-indexed tsb phys address */ 458 int ktsb_szcode; /* kernel 8k-indexed tsb size code */ 459 int ktsb_sz; /* kernel 8k-indexed tsb size */ 460 461 caddr_t ktsb4m_base; /* kernel 4m-indexed tsb base address */ 462 uint64_t ktsb4m_pbase; /* kernel 4m-indexed tsb phys address */ 463 int ktsb4m_szcode; /* kernel 4m-indexed tsb size code */ 464 int ktsb4m_sz; /* kernel 4m-indexed tsb size */ 465 466 uint64_t kpm_tsbbase; /* kernel seg_kpm 4M TSB base address */ 467 int kpm_tsbsz; /* kernel seg_kpm 4M TSB size code */ 468 uint64_t kpmsm_tsbbase; /* kernel seg_kpm 8K TSB base address */ 469 int kpmsm_tsbsz; /* kernel seg_kpm 8K TSB size code */ 470 471 #ifndef sun4v 472 int utsb_dtlb_ttenum = -1; /* index in TLB for utsb locked TTE */ 473 int utsb4m_dtlb_ttenum = -1; /* index in TLB for 4M TSB TTE */ 474 int dtlb_resv_ttenum; /* index in TLB of first reserved TTE */ 475 caddr_t utsb_vabase; /* reserved kernel virtual memory */ 476 caddr_t utsb4m_vabase; /* for trap handler TSB accesses */ 477 #endif /* sun4v */ 478 uint64_t tsb_alloc_bytes = 0; /* bytes allocated to TSBs */ 479 vmem_t *kmem_tsb_default_arena[NLGRPS_MAX]; /* For dynamic TSBs */ 480 481 /* 482 * Size to use for TSB slabs. Future platforms that support page sizes 483 * larger than 4M may wish to change these values, and provide their own 484 * assembly macros for building and decoding the TSB base register contents. 485 */ 486 uint_t tsb_slab_size = MMU_PAGESIZE4M; 487 uint_t tsb_slab_shift = MMU_PAGESHIFT4M; 488 uint_t tsb_slab_ttesz = TTE4M; 489 uint_t tsb_slab_mask = 0x1ff; /* 4M page alignment for 8K pfn */ 490 491 /* largest TSB size to grow to, will be smaller on smaller memory systems */ 492 int tsb_max_growsize = UTSB_MAX_SZCODE; 493 494 /* 495 * Tunable parameters dealing with TSB policies. 496 */ 497 498 /* 499 * This undocumented tunable forces all 8K TSBs to be allocated from 500 * the kernel heap rather than from the kmem_tsb_default_arena arenas. 501 */ 502 #ifdef DEBUG 503 int tsb_forceheap = 0; 504 #endif /* DEBUG */ 505 506 /* 507 * Decide whether to use per-lgroup arenas, or one global set of 508 * TSB arenas. The default is not to break up per-lgroup, since 509 * most platforms don't recognize any tangible benefit from it. 510 */ 511 int tsb_lgrp_affinity = 0; 512 513 /* 514 * Used for growing the TSB based on the process RSS. 515 * tsb_rss_factor is based on the smallest TSB, and is 516 * shifted by the TSB size to determine if we need to grow. 517 * The default will grow the TSB if the number of TTEs for 518 * this page size exceeds 75% of the number of TSB entries, 519 * which should _almost_ eliminate all conflict misses 520 * (at the expense of using up lots and lots of memory). 521 */ 522 #define TSB_RSS_FACTOR (TSB_ENTRIES(TSB_MIN_SZCODE) * 0.75) 523 #define SFMMU_RSS_TSBSIZE(tsbszc) (tsb_rss_factor << tsbszc) 524 #define SELECT_TSB_SIZECODE(pgcnt) ( \ 525 (enable_tsb_rss_sizing)? sfmmu_select_tsb_szc(pgcnt) : \ 526 default_tsb_size) 527 #define TSB_OK_SHRINK() \ 528 (tsb_alloc_bytes > tsb_alloc_hiwater || freemem < desfree) 529 #define TSB_OK_GROW() \ 530 (tsb_alloc_bytes < tsb_alloc_hiwater && freemem > desfree) 531 532 int enable_tsb_rss_sizing = 1; 533 int tsb_rss_factor = (int)TSB_RSS_FACTOR; 534 535 /* which TSB size code to use for new address spaces or if rss sizing off */ 536 int default_tsb_size = TSB_8K_SZCODE; 537 538 static uint64_t tsb_alloc_hiwater; /* limit TSB reserved memory */ 539 uint64_t tsb_alloc_hiwater_factor; /* tsb_alloc_hiwater = physmem / this */ 540 #define TSB_ALLOC_HIWATER_FACTOR_DEFAULT 32 541 542 #ifdef DEBUG 543 static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */ 544 static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */ 545 static int tsb_alloc_mtbf = 0; /* fail allocation every n attempts */ 546 static int tsb_alloc_fail_mtbf = 0; 547 static int tsb_alloc_count = 0; 548 #endif /* DEBUG */ 549 550 /* if set to 1, will remap valid TTEs when growing TSB. */ 551 int tsb_remap_ttes = 1; 552 553 /* 554 * If we have more than this many mappings, allocate a second TSB. 555 * This default is chosen because the I/D fully associative TLBs are 556 * assumed to have at least 8 available entries. Platforms with a 557 * larger fully-associative TLB could probably override the default. 558 */ 559 int tsb_sectsb_threshold = 8; 560 561 /* 562 * kstat data 563 */ 564 struct sfmmu_global_stat sfmmu_global_stat; 565 struct sfmmu_tsbsize_stat sfmmu_tsbsize_stat; 566 567 /* 568 * Global data 569 */ 570 sfmmu_t *ksfmmup; /* kernel's hat id */ 571 struct ctx *kctx; /* kernel's context */ 572 573 #ifdef DEBUG 574 static void chk_tte(tte_t *, tte_t *, tte_t *, struct hme_blk *); 575 #endif 576 577 /* sfmmu locking operations */ 578 static kmutex_t *sfmmu_mlspl_enter(struct page *, int); 579 static int sfmmu_mlspl_held(struct page *, int); 580 581 static kmutex_t *sfmmu_page_enter(page_t *); 582 static void sfmmu_page_exit(kmutex_t *); 583 static int sfmmu_page_spl_held(struct page *); 584 585 /* sfmmu internal locking operations - accessed directly */ 586 static void sfmmu_mlist_reloc_enter(page_t *, page_t *, 587 kmutex_t **, kmutex_t **); 588 static void sfmmu_mlist_reloc_exit(kmutex_t *, kmutex_t *); 589 static hatlock_t * 590 sfmmu_hat_enter(sfmmu_t *); 591 static hatlock_t * 592 sfmmu_hat_tryenter(sfmmu_t *); 593 static void sfmmu_hat_exit(hatlock_t *); 594 static void sfmmu_hat_lock_all(void); 595 static void sfmmu_hat_unlock_all(void); 596 static void sfmmu_ismhat_enter(sfmmu_t *, int); 597 static void sfmmu_ismhat_exit(sfmmu_t *, int); 598 599 /* 600 * Array of mutexes protecting a page's mapping list and p_nrm field. 601 * 602 * The hash function looks complicated, but is made up so that: 603 * 604 * "pp" not shifted, so adjacent pp values will hash to different cache lines 605 * (8 byte alignment * 8 bytes/mutes == 64 byte coherency subblock) 606 * 607 * "pp" >> mml_shift, incorporates more source bits into the hash result 608 * 609 * "& (mml_table_size - 1), should be faster than using remainder "%" 610 * 611 * Hopefully, mml_table, mml_table_size and mml_shift are all in the same 612 * cacheline, since they get declared next to each other below. We'll trust 613 * ld not to do something random. 614 */ 615 #ifdef DEBUG 616 int mlist_hash_debug = 0; 617 #define MLIST_HASH(pp) (mlist_hash_debug ? &mml_table[0] : \ 618 &mml_table[((uintptr_t)(pp) + \ 619 ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)]) 620 #else /* !DEBUG */ 621 #define MLIST_HASH(pp) &mml_table[ \ 622 ((uintptr_t)(pp) + ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)] 623 #endif /* !DEBUG */ 624 625 kmutex_t *mml_table; 626 uint_t mml_table_sz; /* must be a power of 2 */ 627 uint_t mml_shift; /* log2(mml_table_sz) + 3 for align */ 628 629 /* 630 * kpm_page lock hash. 631 * All slots should be used equally and 2 adjacent kpm_page_t's 632 * shouldn't have their mutexes in the same cache line. 633 */ 634 #ifdef DEBUG 635 int kpmp_hash_debug = 0; 636 #define KPMP_HASH(kpp) (kpmp_hash_debug ? &kpmp_table[0] : &kpmp_table[ \ 637 ((uintptr_t)(kpp) + ((uintptr_t)(kpp) >> kpmp_shift)) \ 638 & (kpmp_table_sz - 1)]) 639 #else /* !DEBUG */ 640 #define KPMP_HASH(kpp) &kpmp_table[ \ 641 ((uintptr_t)(kpp) + ((uintptr_t)(kpp) >> kpmp_shift)) \ 642 & (kpmp_table_sz - 1)] 643 #endif /* DEBUG */ 644 645 kpm_hlk_t *kpmp_table; 646 uint_t kpmp_table_sz; /* must be a power of 2 */ 647 uchar_t kpmp_shift; 648 649 #ifdef DEBUG 650 #define KPMP_SHASH(kpp) (kpmp_hash_debug ? &kpmp_stable[0] : &kpmp_stable[ \ 651 (((uintptr_t)(kpp) << kpmp_shift) + (uintptr_t)(kpp)) \ 652 & (kpmp_stable_sz - 1)]) 653 #else /* !DEBUG */ 654 #define KPMP_SHASH(kpp) &kpmp_stable[ \ 655 (((uintptr_t)(kpp) << kpmp_shift) + (uintptr_t)(kpp)) \ 656 & (kpmp_stable_sz - 1)] 657 #endif /* DEBUG */ 658 659 kpm_shlk_t *kpmp_stable; 660 uint_t kpmp_stable_sz; /* must be a power of 2 */ 661 662 /* 663 * SPL_HASH was improved to avoid false cache line sharing 664 */ 665 #define SPL_TABLE_SIZE 128 666 #define SPL_MASK (SPL_TABLE_SIZE - 1) 667 #define SPL_SHIFT 7 /* log2(SPL_TABLE_SIZE) */ 668 669 #define SPL_INDEX(pp) \ 670 ((((uintptr_t)(pp) >> SPL_SHIFT) ^ \ 671 ((uintptr_t)(pp) >> (SPL_SHIFT << 1))) & \ 672 (SPL_TABLE_SIZE - 1)) 673 674 #define SPL_HASH(pp) \ 675 (&sfmmu_page_lock[SPL_INDEX(pp) & SPL_MASK].pad_mutex) 676 677 static pad_mutex_t sfmmu_page_lock[SPL_TABLE_SIZE]; 678 679 680 /* 681 * hat_unload_callback() will group together callbacks in order 682 * to avoid xt_sync() calls. This is the maximum size of the group. 683 */ 684 #define MAX_CB_ADDR 32 685 686 #ifdef DEBUG 687 688 /* 689 * Debugging trace ring buffer for stolen and freed ctxs. The 690 * stolen_ctxs[] array is protected by the ctx_trace_mutex. 691 */ 692 struct ctx_trace stolen_ctxs[TRSIZE]; 693 struct ctx_trace *ctx_trace_first = &stolen_ctxs[0]; 694 struct ctx_trace *ctx_trace_last = &stolen_ctxs[TRSIZE-1]; 695 struct ctx_trace *ctx_trace_ptr = &stolen_ctxs[0]; 696 kmutex_t ctx_trace_mutex; 697 uint_t num_ctx_stolen = 0; 698 699 int ism_debug = 0; 700 701 #endif /* DEBUG */ 702 703 tte_t hw_tte; 704 static ulong_t sfmmu_dmr_maxbit = DMR_MAXBIT; 705 706 /* 707 * kpm virtual address to physical address 708 */ 709 #define SFMMU_KPM_VTOP(vaddr, paddr) { \ 710 uintptr_t r, v; \ 711 \ 712 r = ((vaddr) - kpm_vbase) >> (uintptr_t)kpm_size_shift; \ 713 (paddr) = (vaddr) - kpm_vbase; \ 714 if (r != 0) { \ 715 v = ((uintptr_t)(vaddr) >> MMU_PAGESHIFT) & \ 716 vac_colors_mask; \ 717 (paddr) -= r << kpm_size_shift; \ 718 if (r > v) \ 719 (paddr) += (r - v) << MMU_PAGESHIFT; \ 720 else \ 721 (paddr) -= r << MMU_PAGESHIFT; \ 722 } \ 723 } 724 725 /* 726 * Wrapper for vmem_xalloc since vmem_create only allows limited 727 * parameters for vm_source_alloc functions. This function allows us 728 * to specify alignment consistent with the size of the object being 729 * allocated. 730 */ 731 static void * 732 sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *vmp, size_t size, int vmflag) 733 { 734 return (vmem_xalloc(vmp, size, size, 0, 0, NULL, NULL, vmflag)); 735 } 736 737 /* Common code for setting tsb_alloc_hiwater. */ 738 #define SFMMU_SET_TSB_ALLOC_HIWATER(pages) tsb_alloc_hiwater = \ 739 ptob(pages) / tsb_alloc_hiwater_factor 740 741 /* 742 * Set tsb_max_growsize to allow at most all of physical memory to be mapped by 743 * a single TSB. physmem is the number of physical pages so we need physmem 8K 744 * TTEs to represent all those physical pages. We round this up by using 745 * 1<<highbit(). To figure out which size code to use, remember that the size 746 * code is just an amount to shift the smallest TSB size to get the size of 747 * this TSB. So we subtract that size, TSB_START_SIZE, from highbit() (or 748 * highbit() - 1) to get the size code for the smallest TSB that can represent 749 * all of physical memory, while erring on the side of too much. 750 * 751 * If the computed size code is less than the current tsb_max_growsize, we set 752 * tsb_max_growsize to the computed size code. In the case where the computed 753 * size code is greater than tsb_max_growsize, we have these restrictions that 754 * apply to increasing tsb_max_growsize: 755 * 1) TSBs can't grow larger than the TSB slab size 756 * 2) TSBs can't grow larger than UTSB_MAX_SZCODE. 757 */ 758 #define SFMMU_SET_TSB_MAX_GROWSIZE(pages) { \ 759 int i, szc; \ 760 \ 761 i = highbit(pages); \ 762 if ((1 << (i - 1)) == (pages)) \ 763 i--; /* 2^n case, round down */ \ 764 szc = i - TSB_START_SIZE; \ 765 if (szc < tsb_max_growsize) \ 766 tsb_max_growsize = szc; \ 767 else if ((szc > tsb_max_growsize) && \ 768 (szc <= tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT))) \ 769 tsb_max_growsize = MIN(szc, UTSB_MAX_SZCODE); \ 770 } 771 772 /* 773 * Given a pointer to an sfmmu and a TTE size code, return a pointer to the 774 * tsb_info which handles that TTE size. 775 */ 776 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) \ 777 (tsbinfop) = (sfmmup)->sfmmu_tsb; \ 778 ASSERT(sfmmu_hat_lock_held(sfmmup)); \ 779 if ((tte_szc) >= TTE4M) \ 780 (tsbinfop) = (tsbinfop)->tsb_next; 781 782 /* 783 * Return the number of mappings present in the HAT 784 * for a particular process and page size. 785 */ 786 #define SFMMU_TTE_CNT(sfmmup, szc) \ 787 (sfmmup)->sfmmu_iblk? \ 788 (sfmmup)->sfmmu_ismttecnt[(szc)] + \ 789 (sfmmup)->sfmmu_ttecnt[(szc)] : \ 790 (sfmmup)->sfmmu_ttecnt[(szc)]; 791 792 /* 793 * Macro to use to unload entries from the TSB. 794 * It has knowledge of which page sizes get replicated in the TSB 795 * and will call the appropriate unload routine for the appropriate size. 796 */ 797 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp) \ 798 { \ 799 int ttesz = get_hblk_ttesz(hmeblkp); \ 800 if (ttesz == TTE8K || ttesz == TTE4M) { \ 801 sfmmu_unload_tsb(sfmmup, addr, ttesz); \ 802 } else { \ 803 caddr_t sva = (caddr_t)get_hblk_base(hmeblkp); \ 804 caddr_t eva = sva + get_hblk_span(hmeblkp); \ 805 ASSERT(addr >= sva && addr < eva); \ 806 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); \ 807 } \ 808 } 809 810 811 /* Update tsb_alloc_hiwater after memory is configured. */ 812 /*ARGSUSED*/ 813 static void 814 sfmmu_update_tsb_post_add(void *arg, pgcnt_t delta_pages) 815 { 816 /* Assumes physmem has already been updated. */ 817 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 818 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 819 } 820 821 /* 822 * Update tsb_alloc_hiwater before memory is deleted. We'll do nothing here 823 * and update tsb_alloc_hiwater and tsb_max_growsize after the memory is 824 * deleted. 825 */ 826 /*ARGSUSED*/ 827 static int 828 sfmmu_update_tsb_pre_del(void *arg, pgcnt_t delta_pages) 829 { 830 return (0); 831 } 832 833 /* Update tsb_alloc_hiwater after memory fails to be unconfigured. */ 834 /*ARGSUSED*/ 835 static void 836 sfmmu_update_tsb_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 837 { 838 /* 839 * Whether the delete was cancelled or not, just go ahead and update 840 * tsb_alloc_hiwater and tsb_max_growsize. 841 */ 842 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 843 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 844 } 845 846 static kphysm_setup_vector_t sfmmu_update_tsb_vec = { 847 KPHYSM_SETUP_VECTOR_VERSION, /* version */ 848 sfmmu_update_tsb_post_add, /* post_add */ 849 sfmmu_update_tsb_pre_del, /* pre_del */ 850 sfmmu_update_tsb_post_del /* post_del */ 851 }; 852 853 854 /* 855 * HME_BLK HASH PRIMITIVES 856 */ 857 858 /* 859 * Enter a hme on the mapping list for page pp. 860 * When large pages are more prevalent in the system we might want to 861 * keep the mapping list in ascending order by the hment size. For now, 862 * small pages are more frequent, so don't slow it down. 863 */ 864 #define HME_ADD(hme, pp) \ 865 { \ 866 ASSERT(sfmmu_mlist_held(pp)); \ 867 \ 868 hme->hme_prev = NULL; \ 869 hme->hme_next = pp->p_mapping; \ 870 hme->hme_page = pp; \ 871 if (pp->p_mapping) { \ 872 ((struct sf_hment *)(pp->p_mapping))->hme_prev = hme;\ 873 ASSERT(pp->p_share > 0); \ 874 } else { \ 875 /* EMPTY */ \ 876 ASSERT(pp->p_share == 0); \ 877 } \ 878 pp->p_mapping = hme; \ 879 pp->p_share++; \ 880 } 881 882 /* 883 * Enter a hme on the mapping list for page pp. 884 * If we are unmapping a large translation, we need to make sure that the 885 * change is reflect in the corresponding bit of the p_index field. 886 */ 887 #define HME_SUB(hme, pp) \ 888 { \ 889 ASSERT(sfmmu_mlist_held(pp)); \ 890 ASSERT(hme->hme_page == pp || IS_PAHME(hme)); \ 891 \ 892 if (pp->p_mapping == NULL) { \ 893 panic("hme_remove - no mappings"); \ 894 } \ 895 \ 896 membar_stst(); /* ensure previous stores finish */ \ 897 \ 898 ASSERT(pp->p_share > 0); \ 899 pp->p_share--; \ 900 \ 901 if (hme->hme_prev) { \ 902 ASSERT(pp->p_mapping != hme); \ 903 ASSERT(hme->hme_prev->hme_page == pp || \ 904 IS_PAHME(hme->hme_prev)); \ 905 hme->hme_prev->hme_next = hme->hme_next; \ 906 } else { \ 907 ASSERT(pp->p_mapping == hme); \ 908 pp->p_mapping = hme->hme_next; \ 909 ASSERT((pp->p_mapping == NULL) ? \ 910 (pp->p_share == 0) : 1); \ 911 } \ 912 \ 913 if (hme->hme_next) { \ 914 ASSERT(hme->hme_next->hme_page == pp || \ 915 IS_PAHME(hme->hme_next)); \ 916 hme->hme_next->hme_prev = hme->hme_prev; \ 917 } \ 918 \ 919 /* zero out the entry */ \ 920 hme->hme_next = NULL; \ 921 hme->hme_prev = NULL; \ 922 hme->hme_page = NULL; \ 923 \ 924 if (hme_size(hme) > TTE8K) { \ 925 /* remove mappings for remainder of large pg */ \ 926 sfmmu_rm_large_mappings(pp, hme_size(hme)); \ 927 } \ 928 } 929 930 /* 931 * This function returns the hment given the hme_blk and a vaddr. 932 * It assumes addr has already been checked to belong to hme_blk's 933 * range. 934 */ 935 #define HBLKTOHME(hment, hmeblkp, addr) \ 936 { \ 937 int index; \ 938 HBLKTOHME_IDX(hment, hmeblkp, addr, index) \ 939 } 940 941 /* 942 * Version of HBLKTOHME that also returns the index in hmeblkp 943 * of the hment. 944 */ 945 #define HBLKTOHME_IDX(hment, hmeblkp, addr, idx) \ 946 { \ 947 ASSERT(in_hblk_range((hmeblkp), (addr))); \ 948 \ 949 if (get_hblk_ttesz(hmeblkp) == TTE8K) { \ 950 idx = (((uintptr_t)(addr) >> MMU_PAGESHIFT) & (NHMENTS-1)); \ 951 } else \ 952 idx = 0; \ 953 \ 954 (hment) = &(hmeblkp)->hblk_hme[idx]; \ 955 } 956 957 /* 958 * Disable any page sizes not supported by the CPU 959 */ 960 void 961 hat_init_pagesizes() 962 { 963 int i; 964 965 mmu_exported_page_sizes = 0; 966 for (i = TTE8K; i < max_mmu_page_sizes; i++) { 967 extern int disable_text_largepages; 968 extern int disable_initdata_largepages; 969 970 szc_2_userszc[i] = (uint_t)-1; 971 userszc_2_szc[i] = (uint_t)-1; 972 973 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) { 974 disable_large_pages |= (1 << i); 975 disable_ism_large_pages |= (1 << i); 976 disable_text_largepages |= (1 << i); 977 disable_initdata_largepages |= (1 << i); 978 } else { 979 szc_2_userszc[i] = mmu_exported_page_sizes; 980 userszc_2_szc[mmu_exported_page_sizes] = i; 981 mmu_exported_page_sizes++; 982 } 983 } 984 985 disable_auto_large_pages = disable_large_pages; 986 987 /* 988 * Initialize mmu-specific large page sizes. 989 */ 990 if ((mmu_page_sizes == max_mmu_page_sizes) && 991 (&mmu_large_pages_disabled)) { 992 disable_large_pages |= mmu_large_pages_disabled(HAT_LOAD); 993 disable_ism_large_pages |= 994 mmu_large_pages_disabled(HAT_LOAD_SHARE); 995 disable_auto_large_pages |= 996 mmu_large_pages_disabled(HAT_LOAD_AUTOLPG); 997 } 998 999 } 1000 1001 /* 1002 * Initialize the hardware address translation structures. 1003 */ 1004 void 1005 hat_init(void) 1006 { 1007 struct ctx *ctx; 1008 struct ctx *cur_ctx = NULL; 1009 int i; 1010 1011 hat_lock_init(); 1012 hat_kstat_init(); 1013 1014 /* 1015 * Hardware-only bits in a TTE 1016 */ 1017 MAKE_TTE_MASK(&hw_tte); 1018 1019 hat_init_pagesizes(); 1020 1021 /* Initialize the hash locks */ 1022 for (i = 0; i < khmehash_num; i++) { 1023 mutex_init(&khme_hash[i].hmehash_mutex, NULL, 1024 MUTEX_DEFAULT, NULL); 1025 } 1026 for (i = 0; i < uhmehash_num; i++) { 1027 mutex_init(&uhme_hash[i].hmehash_mutex, NULL, 1028 MUTEX_DEFAULT, NULL); 1029 } 1030 khmehash_num--; /* make sure counter starts from 0 */ 1031 uhmehash_num--; /* make sure counter starts from 0 */ 1032 1033 /* 1034 * Initialize ctx structures and list lock. 1035 * We keep two lists of ctxs. The "free" list contains contexts 1036 * ready to use. The "dirty" list contains contexts that are OK 1037 * to use after flushing the TLBs of any stale mappings. 1038 */ 1039 mutex_init(&ctx_list_lock, NULL, MUTEX_DEFAULT, NULL); 1040 kctx = &ctxs[KCONTEXT]; 1041 ctx = &ctxs[NUM_LOCKED_CTXS]; 1042 ctxhand = ctxfree = ctx; /* head of free list */ 1043 ctxdirty = NULL; 1044 for (i = NUM_LOCKED_CTXS; i < nctxs; i++) { 1045 cur_ctx = &ctxs[i]; 1046 cur_ctx->ctx_flags = CTX_FREE_FLAG; 1047 cur_ctx->ctx_free = &ctxs[i + 1]; 1048 } 1049 cur_ctx->ctx_free = NULL; /* tail of free list */ 1050 1051 /* 1052 * Intialize ism mapping list lock. 1053 */ 1054 mutex_init(&ism_mlist_lock, NULL, MUTEX_DEFAULT, NULL); 1055 1056 sfmmuid_cache = kmem_cache_create("sfmmuid_cache", sizeof (sfmmu_t), 1057 0, sfmmu_idcache_constructor, sfmmu_idcache_destructor, 1058 NULL, NULL, NULL, 0); 1059 1060 sfmmu_tsbinfo_cache = kmem_cache_create("sfmmu_tsbinfo_cache", 1061 sizeof (struct tsb_info), 0, NULL, NULL, NULL, NULL, NULL, 0); 1062 1063 /* 1064 * Since we only use the tsb8k cache to "borrow" pages for TSBs 1065 * from the heap when low on memory or when TSB_FORCEALLOC is 1066 * specified, don't use magazines to cache them--we want to return 1067 * them to the system as quickly as possible. 1068 */ 1069 sfmmu_tsb8k_cache = kmem_cache_create("sfmmu_tsb8k_cache", 1070 MMU_PAGESIZE, MMU_PAGESIZE, NULL, NULL, NULL, NULL, 1071 static_arena, KMC_NOMAGAZINE); 1072 1073 /* 1074 * Set tsb_alloc_hiwater to 1/tsb_alloc_hiwater_factor of physical 1075 * memory, which corresponds to the old static reserve for TSBs. 1076 * tsb_alloc_hiwater_factor defaults to 32. This caps the amount of 1077 * memory we'll allocate for TSB slabs; beyond this point TSB 1078 * allocations will be taken from the kernel heap (via 1079 * sfmmu_tsb8k_cache) and will be throttled as would any other kmem 1080 * consumer. 1081 */ 1082 if (tsb_alloc_hiwater_factor == 0) { 1083 tsb_alloc_hiwater_factor = TSB_ALLOC_HIWATER_FACTOR_DEFAULT; 1084 } 1085 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 1086 1087 /* Set tsb_max_growsize. */ 1088 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 1089 1090 /* 1091 * On smaller memory systems, allocate TSB memory in 512K chunks 1092 * instead of the default 4M slab size. The trap handlers need to 1093 * be patched with the final slab shift since they need to be able 1094 * to construct the TSB pointer at runtime. 1095 */ 1096 if ((tsb_max_growsize <= TSB_512K_SZCODE) && 1097 !(disable_large_pages & (1 << TTE512K))) { 1098 tsb_slab_size = MMU_PAGESIZE512K; 1099 tsb_slab_shift = MMU_PAGESHIFT512K; 1100 tsb_slab_ttesz = TTE512K; 1101 tsb_slab_mask = 0x3f; /* 512K page alignment for 8K pfn */ 1102 } 1103 1104 /* 1105 * Set up memory callback to update tsb_alloc_hiwater and 1106 * tsb_max_growsize. 1107 */ 1108 i = kphysm_setup_func_register(&sfmmu_update_tsb_vec, (void *) 0); 1109 ASSERT(i == 0); 1110 1111 /* 1112 * kmem_tsb_arena is the source from which large TSB slabs are 1113 * drawn. The quantum of this arena corresponds to the largest 1114 * TSB size we can dynamically allocate for user processes. 1115 * Currently it must also be a supported page size since we 1116 * use exactly one translation entry to map each slab page. 1117 * 1118 * The per-lgroup kmem_tsb_default_arena arenas are the arenas from 1119 * which most TSBs are allocated. Since most TSB allocations are 1120 * typically 8K we have a kmem cache we stack on top of each 1121 * kmem_tsb_default_arena to speed up those allocations. 1122 * 1123 * Note the two-level scheme of arenas is required only 1124 * because vmem_create doesn't allow us to specify alignment 1125 * requirements. If this ever changes the code could be 1126 * simplified to use only one level of arenas. 1127 */ 1128 kmem_tsb_arena = vmem_create("kmem_tsb", NULL, 0, tsb_slab_size, 1129 sfmmu_vmem_xalloc_aligned_wrapper, vmem_xfree, heap_arena, 1130 0, VM_SLEEP); 1131 1132 if (tsb_lgrp_affinity) { 1133 char s[50]; 1134 for (i = 0; i < NLGRPS_MAX; i++) { 1135 (void) sprintf(s, "kmem_tsb_lgrp%d", i); 1136 kmem_tsb_default_arena[i] = 1137 vmem_create(s, NULL, 0, PAGESIZE, 1138 sfmmu_tsb_segkmem_alloc, sfmmu_tsb_segkmem_free, 1139 kmem_tsb_arena, 0, VM_SLEEP | VM_BESTFIT); 1140 (void) sprintf(s, "sfmmu_tsb_lgrp%d_cache", i); 1141 sfmmu_tsb_cache[i] = kmem_cache_create(s, PAGESIZE, 1142 PAGESIZE, NULL, NULL, NULL, NULL, 1143 kmem_tsb_default_arena[i], 0); 1144 } 1145 } else { 1146 kmem_tsb_default_arena[0] = vmem_create("kmem_tsb_default", 1147 NULL, 0, PAGESIZE, sfmmu_tsb_segkmem_alloc, 1148 sfmmu_tsb_segkmem_free, kmem_tsb_arena, 0, 1149 VM_SLEEP | VM_BESTFIT); 1150 1151 sfmmu_tsb_cache[0] = kmem_cache_create("sfmmu_tsb_cache", 1152 PAGESIZE, PAGESIZE, NULL, NULL, NULL, NULL, 1153 kmem_tsb_default_arena[0], 0); 1154 } 1155 1156 sfmmu8_cache = kmem_cache_create("sfmmu8_cache", HME8BLK_SZ, 1157 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1158 sfmmu_hblkcache_destructor, 1159 sfmmu_hblkcache_reclaim, (void *)HME8BLK_SZ, 1160 hat_memload_arena, KMC_NOHASH); 1161 1162 hat_memload1_arena = vmem_create("hat_memload1", NULL, 0, PAGESIZE, 1163 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); 1164 1165 sfmmu1_cache = kmem_cache_create("sfmmu1_cache", HME1BLK_SZ, 1166 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1167 sfmmu_hblkcache_destructor, 1168 NULL, (void *)HME1BLK_SZ, 1169 hat_memload1_arena, KMC_NOHASH); 1170 1171 pa_hment_cache = kmem_cache_create("pa_hment_cache", PAHME_SZ, 1172 0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH); 1173 1174 ism_blk_cache = kmem_cache_create("ism_blk_cache", 1175 sizeof (ism_blk_t), ecache_alignsize, NULL, NULL, 1176 NULL, NULL, static_arena, KMC_NOHASH); 1177 1178 ism_ment_cache = kmem_cache_create("ism_ment_cache", 1179 sizeof (ism_ment_t), 0, NULL, NULL, 1180 NULL, NULL, NULL, 0); 1181 1182 /* 1183 * We grab the first hat for the kernel, 1184 */ 1185 AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER); 1186 kas.a_hat = hat_alloc(&kas); 1187 AS_LOCK_EXIT(&kas, &kas.a_lock); 1188 1189 /* 1190 * Initialize hblk_reserve. 1191 */ 1192 ((struct hme_blk *)hblk_reserve)->hblk_nextpa = 1193 va_to_pa((caddr_t)hblk_reserve); 1194 1195 #ifndef sun4v 1196 /* 1197 * Reserve some kernel virtual address space for the locked TTEs 1198 * that allow us to probe the TSB from TL>0. 1199 */ 1200 utsb_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1201 0, 0, NULL, NULL, VM_SLEEP); 1202 utsb4m_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1203 0, 0, NULL, NULL, VM_SLEEP); 1204 #endif 1205 1206 /* 1207 * The big page VAC handling code assumes VAC 1208 * will not be bigger than the smallest big 1209 * page- which is 64K. 1210 */ 1211 if (TTEPAGES(TTE64K) < CACHE_NUM_COLOR) { 1212 cmn_err(CE_PANIC, "VAC too big!"); 1213 } 1214 1215 (void) xhat_init(); 1216 1217 uhme_hash_pa = va_to_pa(uhme_hash); 1218 khme_hash_pa = va_to_pa(khme_hash); 1219 1220 /* 1221 * Initialize relocation locks. kpr_suspendlock is held 1222 * at PIL_MAX to prevent interrupts from pinning the holder 1223 * of a suspended TTE which may access it leading to a 1224 * deadlock condition. 1225 */ 1226 mutex_init(&kpr_mutex, NULL, MUTEX_DEFAULT, NULL); 1227 mutex_init(&kpr_suspendlock, NULL, MUTEX_SPIN, (void *)PIL_MAX); 1228 } 1229 1230 /* 1231 * Initialize locking for the hat layer, called early during boot. 1232 */ 1233 static void 1234 hat_lock_init() 1235 { 1236 int i; 1237 struct ctx *ctx; 1238 1239 /* 1240 * initialize the array of mutexes protecting a page's mapping 1241 * list and p_nrm field. 1242 */ 1243 for (i = 0; i < mml_table_sz; i++) 1244 mutex_init(&mml_table[i], NULL, MUTEX_DEFAULT, NULL); 1245 1246 if (kpm_enable) { 1247 for (i = 0; i < kpmp_table_sz; i++) { 1248 mutex_init(&kpmp_table[i].khl_mutex, NULL, 1249 MUTEX_DEFAULT, NULL); 1250 } 1251 } 1252 1253 /* 1254 * Initialize array of mutex locks that protects sfmmu fields and 1255 * TSB lists. 1256 */ 1257 for (i = 0; i < SFMMU_NUM_LOCK; i++) 1258 mutex_init(HATLOCK_MUTEXP(&hat_lock[i]), NULL, MUTEX_DEFAULT, 1259 NULL); 1260 1261 #ifdef DEBUG 1262 mutex_init(&ctx_trace_mutex, NULL, MUTEX_DEFAULT, NULL); 1263 #endif /* DEBUG */ 1264 1265 for (ctx = ctxs, i = 0; i < nctxs; i++, ctx++) { 1266 rw_init(&ctx->ctx_rwlock, NULL, RW_DEFAULT, NULL); 1267 } 1268 } 1269 1270 extern caddr_t kmem64_base, kmem64_end; 1271 1272 #define SFMMU_KERNEL_MAXVA \ 1273 (kmem64_base ? (uintptr_t)kmem64_end : (SYSLIMIT)) 1274 1275 /* 1276 * Allocate a hat structure. 1277 * Called when an address space first uses a hat. 1278 */ 1279 struct hat * 1280 hat_alloc(struct as *as) 1281 { 1282 sfmmu_t *sfmmup; 1283 struct ctx *ctx; 1284 int i; 1285 extern uint_t get_color_start(struct as *); 1286 1287 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 1288 sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP); 1289 sfmmup->sfmmu_as = as; 1290 sfmmup->sfmmu_flags = 0; 1291 1292 if (as == &kas) { 1293 ctx = kctx; 1294 ksfmmup = sfmmup; 1295 sfmmup->sfmmu_cnum = ctxtoctxnum(ctx); 1296 ASSERT(sfmmup->sfmmu_cnum == KCONTEXT); 1297 sfmmup->sfmmu_cext = 0; 1298 ctx->ctx_sfmmu = sfmmup; 1299 ctx->ctx_flags = 0; 1300 sfmmup->sfmmu_clrstart = 0; 1301 sfmmup->sfmmu_tsb = NULL; 1302 /* 1303 * hat_kern_setup() will call sfmmu_init_ktsbinfo() 1304 * to setup tsb_info for ksfmmup. 1305 */ 1306 } else { 1307 1308 /* 1309 * Just set to invalid ctx. When it faults, it will 1310 * get a valid ctx. This would avoid the situation 1311 * where we get a ctx, but it gets stolen and then 1312 * we fault when we try to run and so have to get 1313 * another ctx. 1314 */ 1315 sfmmup->sfmmu_cnum = INVALID_CONTEXT; 1316 sfmmup->sfmmu_cext = 0; 1317 /* initialize original physical page coloring bin */ 1318 sfmmup->sfmmu_clrstart = get_color_start(as); 1319 #ifdef DEBUG 1320 if (tsb_random_size) { 1321 uint32_t randval = (uint32_t)gettick() >> 4; 1322 int size = randval % (tsb_max_growsize + 1); 1323 1324 /* chose a random tsb size for stress testing */ 1325 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size, 1326 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1327 } else 1328 #endif /* DEBUG */ 1329 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, 1330 default_tsb_size, 1331 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1332 sfmmup->sfmmu_flags = HAT_SWAPPED; 1333 ASSERT(sfmmup->sfmmu_tsb != NULL); 1334 } 1335 sfmmu_setup_tsbinfo(sfmmup); 1336 for (i = 0; i < max_mmu_page_sizes; i++) { 1337 sfmmup->sfmmu_ttecnt[i] = 0; 1338 sfmmup->sfmmu_ismttecnt[i] = 0; 1339 sfmmup->sfmmu_pgsz[i] = TTE8K; 1340 } 1341 1342 sfmmup->sfmmu_iblk = NULL; 1343 sfmmup->sfmmu_ismhat = 0; 1344 sfmmup->sfmmu_ismblkpa = (uint64_t)-1; 1345 if (sfmmup == ksfmmup) { 1346 CPUSET_ALL(sfmmup->sfmmu_cpusran); 1347 } else { 1348 CPUSET_ZERO(sfmmup->sfmmu_cpusran); 1349 } 1350 sfmmup->sfmmu_free = 0; 1351 sfmmup->sfmmu_rmstat = 0; 1352 sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart; 1353 sfmmup->sfmmu_xhat_provider = NULL; 1354 cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL); 1355 return (sfmmup); 1356 } 1357 1358 /* 1359 * Hat_setup, makes an address space context the current active one. 1360 * In sfmmu this translates to setting the secondary context with the 1361 * corresponding context. 1362 */ 1363 void 1364 hat_setup(struct hat *sfmmup, int allocflag) 1365 { 1366 struct ctx *ctx; 1367 uint_t ctx_num; 1368 hatlock_t *hatlockp; 1369 1370 /* Init needs some special treatment. */ 1371 if (allocflag == HAT_INIT) { 1372 /* 1373 * Make sure that we have 1374 * 1. a TSB 1375 * 2. a valid ctx that doesn't get stolen after this point. 1376 */ 1377 hatlockp = sfmmu_hat_enter(sfmmup); 1378 1379 /* 1380 * Swap in the TSB. hat_init() allocates tsbinfos without 1381 * TSBs, but we need one for init, since the kernel does some 1382 * special things to set up its stack and needs the TSB to 1383 * resolve page faults. 1384 */ 1385 sfmmu_tsb_swapin(sfmmup, hatlockp); 1386 1387 sfmmu_disallow_ctx_steal(sfmmup); 1388 1389 kpreempt_disable(); 1390 1391 ctx = sfmmutoctx(sfmmup); 1392 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); 1393 ctx_num = ctxtoctxnum(ctx); 1394 ASSERT(sfmmup == ctx->ctx_sfmmu); 1395 ASSERT(ctx_num >= NUM_LOCKED_CTXS); 1396 sfmmu_setctx_sec(ctx_num); 1397 sfmmu_load_mmustate(sfmmup); 1398 1399 kpreempt_enable(); 1400 1401 /* 1402 * Allow ctx to be stolen. 1403 */ 1404 sfmmu_allow_ctx_steal(sfmmup); 1405 sfmmu_hat_exit(hatlockp); 1406 } else { 1407 ASSERT(allocflag == HAT_ALLOC); 1408 1409 hatlockp = sfmmu_hat_enter(sfmmup); 1410 kpreempt_disable(); 1411 1412 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); 1413 sfmmu_setctx_sec(INVALID_CONTEXT); 1414 sfmmu_clear_utsbinfo(); 1415 1416 kpreempt_enable(); 1417 sfmmu_hat_exit(hatlockp); 1418 } 1419 } 1420 1421 /* 1422 * Free all the translation resources for the specified address space. 1423 * Called from as_free when an address space is being destroyed. 1424 */ 1425 void 1426 hat_free_start(struct hat *sfmmup) 1427 { 1428 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 1429 ASSERT(sfmmup != ksfmmup); 1430 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1431 1432 sfmmup->sfmmu_free = 1; 1433 } 1434 1435 void 1436 hat_free_end(struct hat *sfmmup) 1437 { 1438 int i; 1439 1440 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1441 if (sfmmup->sfmmu_ismhat) { 1442 for (i = 0; i < mmu_page_sizes; i++) { 1443 sfmmup->sfmmu_ttecnt[i] = 0; 1444 sfmmup->sfmmu_ismttecnt[i] = 0; 1445 } 1446 } else { 1447 /* EMPTY */ 1448 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 1449 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 1450 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 1451 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 1452 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 1453 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 1454 } 1455 1456 if (sfmmup->sfmmu_rmstat) { 1457 hat_freestat(sfmmup->sfmmu_as, NULL); 1458 } 1459 if (!delay_tlb_flush) { 1460 sfmmu_tlb_ctx_demap(sfmmup); 1461 xt_sync(sfmmup->sfmmu_cpusran); 1462 } else { 1463 SFMMU_STAT(sf_tlbflush_deferred); 1464 } 1465 sfmmu_free_ctx(sfmmup, sfmmutoctx(sfmmup)); 1466 while (sfmmup->sfmmu_tsb != NULL) { 1467 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next; 1468 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb); 1469 sfmmup->sfmmu_tsb = next; 1470 } 1471 sfmmu_free_sfmmu(sfmmup); 1472 1473 kmem_cache_free(sfmmuid_cache, sfmmup); 1474 } 1475 1476 /* 1477 * Set up any translation structures, for the specified address space, 1478 * that are needed or preferred when the process is being swapped in. 1479 */ 1480 /* ARGSUSED */ 1481 void 1482 hat_swapin(struct hat *hat) 1483 { 1484 ASSERT(hat->sfmmu_xhat_provider == NULL); 1485 } 1486 1487 /* 1488 * Free all of the translation resources, for the specified address space, 1489 * that can be freed while the process is swapped out. Called from as_swapout. 1490 * Also, free up the ctx that this process was using. 1491 */ 1492 void 1493 hat_swapout(struct hat *sfmmup) 1494 { 1495 struct hmehash_bucket *hmebp; 1496 struct hme_blk *hmeblkp; 1497 struct hme_blk *pr_hblk = NULL; 1498 struct hme_blk *nx_hblk; 1499 struct ctx *ctx; 1500 int cnum; 1501 int i; 1502 uint64_t hblkpa, prevpa, nx_pa; 1503 struct hme_blk *list = NULL; 1504 hatlock_t *hatlockp; 1505 struct tsb_info *tsbinfop; 1506 struct free_tsb { 1507 struct free_tsb *next; 1508 struct tsb_info *tsbinfop; 1509 }; /* free list of TSBs */ 1510 struct free_tsb *freelist, *last, *next; 1511 1512 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1513 SFMMU_STAT(sf_swapout); 1514 1515 /* 1516 * There is no way to go from an as to all its translations in sfmmu. 1517 * Here is one of the times when we take the big hit and traverse 1518 * the hash looking for hme_blks to free up. Not only do we free up 1519 * this as hme_blks but all those that are free. We are obviously 1520 * swapping because we need memory so let's free up as much 1521 * as we can. 1522 * 1523 * Note that we don't flush TLB/TSB here -- it's not necessary 1524 * because: 1525 * 1) we free the ctx we're using and throw away the TSB(s); 1526 * 2) processes aren't runnable while being swapped out. 1527 */ 1528 ASSERT(sfmmup != KHATID); 1529 for (i = 0; i <= UHMEHASH_SZ; i++) { 1530 hmebp = &uhme_hash[i]; 1531 SFMMU_HASH_LOCK(hmebp); 1532 hmeblkp = hmebp->hmeblkp; 1533 hblkpa = hmebp->hmeh_nextpa; 1534 prevpa = 0; 1535 pr_hblk = NULL; 1536 while (hmeblkp) { 1537 1538 ASSERT(!hmeblkp->hblk_xhat_bit); 1539 1540 if ((hmeblkp->hblk_tag.htag_id == sfmmup) && 1541 !hmeblkp->hblk_shw_bit && !hmeblkp->hblk_lckcnt) { 1542 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 1543 (caddr_t)get_hblk_base(hmeblkp), 1544 get_hblk_endaddr(hmeblkp), 1545 NULL, HAT_UNLOAD); 1546 } 1547 nx_hblk = hmeblkp->hblk_next; 1548 nx_pa = hmeblkp->hblk_nextpa; 1549 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 1550 ASSERT(!hmeblkp->hblk_lckcnt); 1551 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 1552 prevpa, pr_hblk); 1553 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 1554 } else { 1555 pr_hblk = hmeblkp; 1556 prevpa = hblkpa; 1557 } 1558 hmeblkp = nx_hblk; 1559 hblkpa = nx_pa; 1560 } 1561 SFMMU_HASH_UNLOCK(hmebp); 1562 } 1563 1564 sfmmu_hblks_list_purge(&list); 1565 1566 /* 1567 * Now free up the ctx so that others can reuse it. 1568 */ 1569 hatlockp = sfmmu_hat_enter(sfmmup); 1570 ctx = sfmmutoctx(sfmmup); 1571 cnum = ctxtoctxnum(ctx); 1572 1573 if (cnum != INVALID_CONTEXT) { 1574 rw_enter(&ctx->ctx_rwlock, RW_WRITER); 1575 if (sfmmup->sfmmu_cnum == cnum) { 1576 sfmmu_reuse_ctx(ctx, sfmmup); 1577 /* 1578 * Put ctx back to the free list. 1579 */ 1580 mutex_enter(&ctx_list_lock); 1581 CTX_SET_FLAGS(ctx, CTX_FREE_FLAG); 1582 ctx->ctx_free = ctxfree; 1583 ctxfree = ctx; 1584 mutex_exit(&ctx_list_lock); 1585 } 1586 rw_exit(&ctx->ctx_rwlock); 1587 } 1588 1589 /* 1590 * Free TSBs, but not tsbinfos, and set SWAPPED flag. 1591 * If TSBs were never swapped in, just return. 1592 * This implies that we don't support partial swapping 1593 * of TSBs -- either all are swapped out, or none are. 1594 * 1595 * We must hold the HAT lock here to prevent racing with another 1596 * thread trying to unmap TTEs from the TSB or running the post- 1597 * relocator after relocating the TSB's memory. Unfortunately, we 1598 * can't free memory while holding the HAT lock or we could 1599 * deadlock, so we build a list of TSBs to be freed after marking 1600 * the tsbinfos as swapped out and free them after dropping the 1601 * lock. 1602 */ 1603 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 1604 sfmmu_hat_exit(hatlockp); 1605 return; 1606 } 1607 1608 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED); 1609 last = freelist = NULL; 1610 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 1611 tsbinfop = tsbinfop->tsb_next) { 1612 ASSERT((tsbinfop->tsb_flags & TSB_SWAPPED) == 0); 1613 1614 /* 1615 * Cast the TSB into a struct free_tsb and put it on the free 1616 * list. 1617 */ 1618 if (freelist == NULL) { 1619 last = freelist = (struct free_tsb *)tsbinfop->tsb_va; 1620 } else { 1621 last->next = (struct free_tsb *)tsbinfop->tsb_va; 1622 last = last->next; 1623 } 1624 last->next = NULL; 1625 last->tsbinfop = tsbinfop; 1626 tsbinfop->tsb_flags |= TSB_SWAPPED; 1627 /* 1628 * Zero out the TTE to clear the valid bit. 1629 * Note we can't use a value like 0xbad because we want to 1630 * ensure diagnostic bits are NEVER set on TTEs that might 1631 * be loaded. The intent is to catch any invalid access 1632 * to the swapped TSB, such as a thread running with a valid 1633 * context without first calling sfmmu_tsb_swapin() to 1634 * allocate TSB memory. 1635 */ 1636 tsbinfop->tsb_tte.ll = 0; 1637 } 1638 1639 /* Now we can drop the lock and free the TSB memory. */ 1640 sfmmu_hat_exit(hatlockp); 1641 for (; freelist != NULL; freelist = next) { 1642 next = freelist->next; 1643 sfmmu_tsb_free(freelist->tsbinfop); 1644 } 1645 } 1646 1647 /* 1648 * Duplicate the translations of an as into another newas 1649 */ 1650 /* ARGSUSED */ 1651 int 1652 hat_dup(struct hat *hat, struct hat *newhat, caddr_t addr, size_t len, 1653 uint_t flag) 1654 { 1655 ASSERT(hat->sfmmu_xhat_provider == NULL); 1656 ASSERT((flag == 0) || (flag == HAT_DUP_ALL) || (flag == HAT_DUP_COW)); 1657 1658 if (flag == HAT_DUP_COW) { 1659 panic("hat_dup: HAT_DUP_COW not supported"); 1660 } 1661 return (0); 1662 } 1663 1664 /* 1665 * Set up addr to map to page pp with protection prot. 1666 * As an optimization we also load the TSB with the 1667 * corresponding tte but it is no big deal if the tte gets kicked out. 1668 */ 1669 void 1670 hat_memload(struct hat *hat, caddr_t addr, struct page *pp, 1671 uint_t attr, uint_t flags) 1672 { 1673 tte_t tte; 1674 1675 1676 ASSERT(hat != NULL); 1677 ASSERT(PAGE_LOCKED(pp)); 1678 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 1679 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1680 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1681 1682 if (PP_ISFREE(pp)) { 1683 panic("hat_memload: loading a mapping to free page %p", 1684 (void *)pp); 1685 } 1686 1687 if (hat->sfmmu_xhat_provider) { 1688 XHAT_MEMLOAD(hat, addr, pp, attr, flags); 1689 return; 1690 } 1691 1692 ASSERT((hat == ksfmmup) || 1693 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1694 1695 if (flags & ~SFMMU_LOAD_ALLFLAG) 1696 cmn_err(CE_NOTE, "hat_memload: unsupported flags %d", 1697 flags & ~SFMMU_LOAD_ALLFLAG); 1698 1699 if (hat->sfmmu_rmstat) 1700 hat_resvstat(MMU_PAGESIZE, hat->sfmmu_as, addr); 1701 1702 #if defined(SF_ERRATA_57) 1703 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1704 (addr < errata57_limit) && (attr & PROT_EXEC) && 1705 !(flags & HAT_LOAD_SHARE)) { 1706 cmn_err(CE_WARN, "hat_memload: illegal attempt to make user " 1707 " page executable"); 1708 attr &= ~PROT_EXEC; 1709 } 1710 #endif 1711 1712 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 1713 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, flags); 1714 1715 /* 1716 * Check TSB and TLB page sizes. 1717 */ 1718 if ((flags & HAT_LOAD_SHARE) == 0) { 1719 sfmmu_check_page_sizes(hat, 1); 1720 } 1721 } 1722 1723 /* 1724 * hat_devload can be called to map real memory (e.g. 1725 * /dev/kmem) and even though hat_devload will determine pf is 1726 * for memory, it will be unable to get a shared lock on the 1727 * page (because someone else has it exclusively) and will 1728 * pass dp = NULL. If tteload doesn't get a non-NULL 1729 * page pointer it can't cache memory. 1730 */ 1731 void 1732 hat_devload(struct hat *hat, caddr_t addr, size_t len, pfn_t pfn, 1733 uint_t attr, int flags) 1734 { 1735 tte_t tte; 1736 struct page *pp = NULL; 1737 int use_lgpg = 0; 1738 1739 ASSERT(hat != NULL); 1740 1741 if (hat->sfmmu_xhat_provider) { 1742 XHAT_DEVLOAD(hat, addr, len, pfn, attr, flags); 1743 return; 1744 } 1745 1746 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1747 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1748 ASSERT((hat == ksfmmup) || 1749 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1750 if (len == 0) 1751 panic("hat_devload: zero len"); 1752 if (flags & ~SFMMU_LOAD_ALLFLAG) 1753 cmn_err(CE_NOTE, "hat_devload: unsupported flags %d", 1754 flags & ~SFMMU_LOAD_ALLFLAG); 1755 1756 #if defined(SF_ERRATA_57) 1757 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1758 (addr < errata57_limit) && (attr & PROT_EXEC) && 1759 !(flags & HAT_LOAD_SHARE)) { 1760 cmn_err(CE_WARN, "hat_devload: illegal attempt to make user " 1761 " page executable"); 1762 attr &= ~PROT_EXEC; 1763 } 1764 #endif 1765 1766 /* 1767 * If it's a memory page find its pp 1768 */ 1769 if (!(flags & HAT_LOAD_NOCONSIST) && pf_is_memory(pfn)) { 1770 pp = page_numtopp_nolock(pfn); 1771 if (pp == NULL) { 1772 flags |= HAT_LOAD_NOCONSIST; 1773 } else { 1774 if (PP_ISFREE(pp)) { 1775 panic("hat_memload: loading " 1776 "a mapping to free page %p", 1777 (void *)pp); 1778 } 1779 if (!PAGE_LOCKED(pp) && !PP_ISNORELOC(pp)) { 1780 panic("hat_memload: loading a mapping " 1781 "to unlocked relocatable page %p", 1782 (void *)pp); 1783 } 1784 ASSERT(len == MMU_PAGESIZE); 1785 } 1786 } 1787 1788 if (hat->sfmmu_rmstat) 1789 hat_resvstat(len, hat->sfmmu_as, addr); 1790 1791 if (flags & HAT_LOAD_NOCONSIST) { 1792 attr |= SFMMU_UNCACHEVTTE; 1793 use_lgpg = 1; 1794 } 1795 if (!pf_is_memory(pfn)) { 1796 attr |= SFMMU_UNCACHEPTTE | HAT_NOSYNC; 1797 use_lgpg = 1; 1798 switch (attr & HAT_ORDER_MASK) { 1799 case HAT_STRICTORDER: 1800 case HAT_UNORDERED_OK: 1801 /* 1802 * we set the side effect bit for all non 1803 * memory mappings unless merging is ok 1804 */ 1805 attr |= SFMMU_SIDEFFECT; 1806 break; 1807 case HAT_MERGING_OK: 1808 case HAT_LOADCACHING_OK: 1809 case HAT_STORECACHING_OK: 1810 break; 1811 default: 1812 panic("hat_devload: bad attr"); 1813 break; 1814 } 1815 } 1816 while (len) { 1817 if (!use_lgpg) { 1818 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1819 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1820 flags); 1821 len -= MMU_PAGESIZE; 1822 addr += MMU_PAGESIZE; 1823 pfn++; 1824 continue; 1825 } 1826 /* 1827 * try to use large pages, check va/pa alignments 1828 * Note that 32M/256M page sizes are not (yet) supported. 1829 */ 1830 if ((len >= MMU_PAGESIZE4M) && 1831 !((uintptr_t)addr & MMU_PAGEOFFSET4M) && 1832 !(disable_large_pages & (1 << TTE4M)) && 1833 !(mmu_ptob(pfn) & MMU_PAGEOFFSET4M)) { 1834 sfmmu_memtte(&tte, pfn, attr, TTE4M); 1835 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1836 flags); 1837 len -= MMU_PAGESIZE4M; 1838 addr += MMU_PAGESIZE4M; 1839 pfn += MMU_PAGESIZE4M / MMU_PAGESIZE; 1840 } else if ((len >= MMU_PAGESIZE512K) && 1841 !((uintptr_t)addr & MMU_PAGEOFFSET512K) && 1842 !(disable_large_pages & (1 << TTE512K)) && 1843 !(mmu_ptob(pfn) & MMU_PAGEOFFSET512K)) { 1844 sfmmu_memtte(&tte, pfn, attr, TTE512K); 1845 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1846 flags); 1847 len -= MMU_PAGESIZE512K; 1848 addr += MMU_PAGESIZE512K; 1849 pfn += MMU_PAGESIZE512K / MMU_PAGESIZE; 1850 } else if ((len >= MMU_PAGESIZE64K) && 1851 !((uintptr_t)addr & MMU_PAGEOFFSET64K) && 1852 !(disable_large_pages & (1 << TTE64K)) && 1853 !(mmu_ptob(pfn) & MMU_PAGEOFFSET64K)) { 1854 sfmmu_memtte(&tte, pfn, attr, TTE64K); 1855 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1856 flags); 1857 len -= MMU_PAGESIZE64K; 1858 addr += MMU_PAGESIZE64K; 1859 pfn += MMU_PAGESIZE64K / MMU_PAGESIZE; 1860 } else { 1861 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1862 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1863 flags); 1864 len -= MMU_PAGESIZE; 1865 addr += MMU_PAGESIZE; 1866 pfn++; 1867 } 1868 } 1869 1870 /* 1871 * Check TSB and TLB page sizes. 1872 */ 1873 if ((flags & HAT_LOAD_SHARE) == 0) { 1874 sfmmu_check_page_sizes(hat, 1); 1875 } 1876 } 1877 1878 /* 1879 * Map the largest extend possible out of the page array. The array may NOT 1880 * be in order. The largest possible mapping a page can have 1881 * is specified in the p_szc field. The p_szc field 1882 * cannot change as long as there any mappings (large or small) 1883 * to any of the pages that make up the large page. (ie. any 1884 * promotion/demotion of page size is not up to the hat but up to 1885 * the page free list manager). The array 1886 * should consist of properly aligned contigous pages that are 1887 * part of a big page for a large mapping to be created. 1888 */ 1889 void 1890 hat_memload_array(struct hat *hat, caddr_t addr, size_t len, 1891 struct page **pps, uint_t attr, uint_t flags) 1892 { 1893 int ttesz; 1894 size_t mapsz; 1895 pgcnt_t numpg, npgs; 1896 tte_t tte; 1897 page_t *pp; 1898 int large_pages_disable; 1899 1900 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 1901 1902 if (hat->sfmmu_xhat_provider) { 1903 XHAT_MEMLOAD_ARRAY(hat, addr, len, pps, attr, flags); 1904 return; 1905 } 1906 1907 if (hat->sfmmu_rmstat) 1908 hat_resvstat(len, hat->sfmmu_as, addr); 1909 1910 #if defined(SF_ERRATA_57) 1911 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1912 (addr < errata57_limit) && (attr & PROT_EXEC) && 1913 !(flags & HAT_LOAD_SHARE)) { 1914 cmn_err(CE_WARN, "hat_memload_array: illegal attempt to make " 1915 "user page executable"); 1916 attr &= ~PROT_EXEC; 1917 } 1918 #endif 1919 1920 /* Get number of pages */ 1921 npgs = len >> MMU_PAGESHIFT; 1922 1923 if (flags & HAT_LOAD_SHARE) { 1924 large_pages_disable = disable_ism_large_pages; 1925 } else { 1926 large_pages_disable = disable_large_pages; 1927 } 1928 1929 if (npgs < NHMENTS || large_pages_disable == LARGE_PAGES_OFF) { 1930 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 1931 return; 1932 } 1933 1934 while (npgs >= NHMENTS) { 1935 pp = *pps; 1936 for (ttesz = pp->p_szc; ttesz != TTE8K; ttesz--) { 1937 /* 1938 * Check if this page size is disabled. 1939 */ 1940 if (large_pages_disable & (1 << ttesz)) 1941 continue; 1942 1943 numpg = TTEPAGES(ttesz); 1944 mapsz = numpg << MMU_PAGESHIFT; 1945 if ((npgs >= numpg) && 1946 IS_P2ALIGNED(addr, mapsz) && 1947 IS_P2ALIGNED(pp->p_pagenum, numpg)) { 1948 /* 1949 * At this point we have enough pages and 1950 * we know the virtual address and the pfn 1951 * are properly aligned. We still need 1952 * to check for physical contiguity but since 1953 * it is very likely that this is the case 1954 * we will assume they are so and undo 1955 * the request if necessary. It would 1956 * be great if we could get a hint flag 1957 * like HAT_CONTIG which would tell us 1958 * the pages are contigous for sure. 1959 */ 1960 sfmmu_memtte(&tte, (*pps)->p_pagenum, 1961 attr, ttesz); 1962 if (!sfmmu_tteload_array(hat, &tte, addr, 1963 pps, flags)) { 1964 break; 1965 } 1966 } 1967 } 1968 if (ttesz == TTE8K) { 1969 /* 1970 * We were not able to map array using a large page 1971 * batch a hmeblk or fraction at a time. 1972 */ 1973 numpg = ((uintptr_t)addr >> MMU_PAGESHIFT) 1974 & (NHMENTS-1); 1975 numpg = NHMENTS - numpg; 1976 ASSERT(numpg <= npgs); 1977 mapsz = numpg * MMU_PAGESIZE; 1978 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, 1979 numpg); 1980 } 1981 addr += mapsz; 1982 npgs -= numpg; 1983 pps += numpg; 1984 } 1985 1986 if (npgs) { 1987 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 1988 } 1989 1990 /* 1991 * Check TSB and TLB page sizes. 1992 */ 1993 if ((flags & HAT_LOAD_SHARE) == 0) { 1994 sfmmu_check_page_sizes(hat, 1); 1995 } 1996 } 1997 1998 /* 1999 * Function tries to batch 8K pages into the same hme blk. 2000 */ 2001 static void 2002 sfmmu_memload_batchsmall(struct hat *hat, caddr_t vaddr, page_t **pps, 2003 uint_t attr, uint_t flags, pgcnt_t npgs) 2004 { 2005 tte_t tte; 2006 page_t *pp; 2007 struct hmehash_bucket *hmebp; 2008 struct hme_blk *hmeblkp; 2009 int index; 2010 2011 while (npgs) { 2012 /* 2013 * Acquire the hash bucket. 2014 */ 2015 hmebp = sfmmu_tteload_acquire_hashbucket(hat, vaddr, TTE8K); 2016 ASSERT(hmebp); 2017 2018 /* 2019 * Find the hment block. 2020 */ 2021 hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr, 2022 TTE8K, flags); 2023 ASSERT(hmeblkp); 2024 2025 do { 2026 /* 2027 * Make the tte. 2028 */ 2029 pp = *pps; 2030 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 2031 2032 /* 2033 * Add the translation. 2034 */ 2035 (void) sfmmu_tteload_addentry(hat, hmeblkp, &tte, 2036 vaddr, pps, flags); 2037 2038 /* 2039 * Goto next page. 2040 */ 2041 pps++; 2042 npgs--; 2043 2044 /* 2045 * Goto next address. 2046 */ 2047 vaddr += MMU_PAGESIZE; 2048 2049 /* 2050 * Don't crossover into a different hmentblk. 2051 */ 2052 index = (int)(((uintptr_t)vaddr >> MMU_PAGESHIFT) & 2053 (NHMENTS-1)); 2054 2055 } while (index != 0 && npgs != 0); 2056 2057 /* 2058 * Release the hash bucket. 2059 */ 2060 2061 sfmmu_tteload_release_hashbucket(hmebp); 2062 } 2063 } 2064 2065 /* 2066 * Construct a tte for a page: 2067 * 2068 * tte_valid = 1 2069 * tte_size2 = size & TTE_SZ2_BITS (Panther-only) 2070 * tte_size = size 2071 * tte_nfo = attr & HAT_NOFAULT 2072 * tte_ie = attr & HAT_STRUCTURE_LE 2073 * tte_hmenum = hmenum 2074 * tte_pahi = pp->p_pagenum >> TTE_PASHIFT; 2075 * tte_palo = pp->p_pagenum & TTE_PALOMASK; 2076 * tte_ref = 1 (optimization) 2077 * tte_wr_perm = attr & PROT_WRITE; 2078 * tte_no_sync = attr & HAT_NOSYNC 2079 * tte_lock = attr & SFMMU_LOCKTTE 2080 * tte_cp = !(attr & SFMMU_UNCACHEPTTE) 2081 * tte_cv = !(attr & SFMMU_UNCACHEVTTE) 2082 * tte_e = attr & SFMMU_SIDEFFECT 2083 * tte_priv = !(attr & PROT_USER) 2084 * tte_hwwr = if nosync is set and it is writable we set the mod bit (opt) 2085 * tte_glb = 0 2086 */ 2087 void 2088 sfmmu_memtte(tte_t *ttep, pfn_t pfn, uint_t attr, int tte_sz) 2089 { 2090 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 2091 2092 ttep->tte_inthi = MAKE_TTE_INTHI(pfn, attr, tte_sz, 0 /* hmenum */); 2093 ttep->tte_intlo = MAKE_TTE_INTLO(pfn, attr, tte_sz, 0 /* hmenum */); 2094 2095 if (TTE_IS_NOSYNC(ttep)) { 2096 TTE_SET_REF(ttep); 2097 if (TTE_IS_WRITABLE(ttep)) { 2098 TTE_SET_MOD(ttep); 2099 } 2100 } 2101 if (TTE_IS_NFO(ttep) && TTE_IS_EXECUTABLE(ttep)) { 2102 panic("sfmmu_memtte: can't set both NFO and EXEC bits"); 2103 } 2104 } 2105 2106 /* 2107 * This function will add a translation to the hme_blk and allocate the 2108 * hme_blk if one does not exist. 2109 * If a page structure is specified then it will add the 2110 * corresponding hment to the mapping list. 2111 * It will also update the hmenum field for the tte. 2112 */ 2113 void 2114 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp, 2115 uint_t flags) 2116 { 2117 (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags); 2118 } 2119 2120 /* 2121 * Load (ttep != NULL) or unload (ttep == NULL) one entry in the TSB. 2122 * Assumes that a particular page size may only be resident in one TSB. 2123 */ 2124 static void 2125 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz) 2126 { 2127 struct tsb_info *tsbinfop = NULL; 2128 uint64_t tag; 2129 struct tsbe *tsbe_addr; 2130 uint64_t tsb_base; 2131 uint_t tsb_size; 2132 int vpshift = MMU_PAGESHIFT; 2133 int phys = 0; 2134 2135 if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */ 2136 phys = ktsb_phys; 2137 if (ttesz >= TTE4M) { 2138 #ifndef sun4v 2139 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2140 #endif 2141 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2142 tsb_size = ktsb4m_szcode; 2143 } else { 2144 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2145 tsb_size = ktsb_szcode; 2146 } 2147 } else { 2148 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2149 2150 /* 2151 * If there isn't a TSB for this page size, or the TSB is 2152 * swapped out, there is nothing to do. Note that the latter 2153 * case seems impossible but can occur if hat_pageunload() 2154 * is called on an ISM mapping while the process is swapped 2155 * out. 2156 */ 2157 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2158 return; 2159 2160 /* 2161 * If another thread is in the middle of relocating a TSB 2162 * we can't unload the entry so set a flag so that the 2163 * TSB will be flushed before it can be accessed by the 2164 * process. 2165 */ 2166 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2167 if (ttep == NULL) 2168 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2169 return; 2170 } 2171 #if defined(UTSB_PHYS) 2172 phys = 1; 2173 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2174 #else 2175 tsb_base = (uint64_t)tsbinfop->tsb_va; 2176 #endif 2177 tsb_size = tsbinfop->tsb_szc; 2178 } 2179 if (ttesz >= TTE4M) 2180 vpshift = MMU_PAGESHIFT4M; 2181 2182 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2183 tag = sfmmu_make_tsbtag(vaddr); 2184 2185 if (ttep == NULL) { 2186 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2187 } else { 2188 if (ttesz >= TTE4M) { 2189 SFMMU_STAT(sf_tsb_load4m); 2190 } else { 2191 SFMMU_STAT(sf_tsb_load8k); 2192 } 2193 2194 sfmmu_load_tsbe(tsbe_addr, tag, ttep, phys); 2195 } 2196 } 2197 2198 /* 2199 * Unmap all entries from [start, end) matching the given page size. 2200 * 2201 * This function is used primarily to unmap replicated 64K or 512K entries 2202 * from the TSB that are inserted using the base page size TSB pointer, but 2203 * it may also be called to unmap a range of addresses from the TSB. 2204 */ 2205 void 2206 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz) 2207 { 2208 struct tsb_info *tsbinfop; 2209 uint64_t tag; 2210 struct tsbe *tsbe_addr; 2211 caddr_t vaddr; 2212 uint64_t tsb_base; 2213 int vpshift, vpgsz; 2214 uint_t tsb_size; 2215 int phys = 0; 2216 2217 /* 2218 * Assumptions: 2219 * If ttesz == 8K, 64K or 512K, we walk through the range 8K 2220 * at a time shooting down any valid entries we encounter. 2221 * 2222 * If ttesz >= 4M we walk the range 4M at a time shooting 2223 * down any valid mappings we find. 2224 */ 2225 if (sfmmup == ksfmmup) { 2226 phys = ktsb_phys; 2227 if (ttesz >= TTE4M) { 2228 #ifndef sun4v 2229 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2230 #endif 2231 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2232 tsb_size = ktsb4m_szcode; 2233 } else { 2234 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2235 tsb_size = ktsb_szcode; 2236 } 2237 } else { 2238 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2239 2240 /* 2241 * If there isn't a TSB for this page size, or the TSB is 2242 * swapped out, there is nothing to do. Note that the latter 2243 * case seems impossible but can occur if hat_pageunload() 2244 * is called on an ISM mapping while the process is swapped 2245 * out. 2246 */ 2247 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2248 return; 2249 2250 /* 2251 * If another thread is in the middle of relocating a TSB 2252 * we can't unload the entry so set a flag so that the 2253 * TSB will be flushed before it can be accessed by the 2254 * process. 2255 */ 2256 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2257 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2258 return; 2259 } 2260 #if defined(UTSB_PHYS) 2261 phys = 1; 2262 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2263 #else 2264 tsb_base = (uint64_t)tsbinfop->tsb_va; 2265 #endif 2266 tsb_size = tsbinfop->tsb_szc; 2267 } 2268 if (ttesz >= TTE4M) { 2269 vpshift = MMU_PAGESHIFT4M; 2270 vpgsz = MMU_PAGESIZE4M; 2271 } else { 2272 vpshift = MMU_PAGESHIFT; 2273 vpgsz = MMU_PAGESIZE; 2274 } 2275 2276 for (vaddr = start; vaddr < end; vaddr += vpgsz) { 2277 tag = sfmmu_make_tsbtag(vaddr); 2278 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2279 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2280 } 2281 } 2282 2283 /* 2284 * Select the optimum TSB size given the number of mappings 2285 * that need to be cached. 2286 */ 2287 static int 2288 sfmmu_select_tsb_szc(pgcnt_t pgcnt) 2289 { 2290 int szc = 0; 2291 2292 #ifdef DEBUG 2293 if (tsb_grow_stress) { 2294 uint32_t randval = (uint32_t)gettick() >> 4; 2295 return (randval % (tsb_max_growsize + 1)); 2296 } 2297 #endif /* DEBUG */ 2298 2299 while ((szc < tsb_max_growsize) && (pgcnt > SFMMU_RSS_TSBSIZE(szc))) 2300 szc++; 2301 return (szc); 2302 } 2303 2304 /* 2305 * This function will add a translation to the hme_blk and allocate the 2306 * hme_blk if one does not exist. 2307 * If a page structure is specified then it will add the 2308 * corresponding hment to the mapping list. 2309 * It will also update the hmenum field for the tte. 2310 * Furthermore, it attempts to create a large page translation 2311 * for <addr,hat> at page array pps. It assumes addr and first 2312 * pp is correctly aligned. It returns 0 if successful and 1 otherwise. 2313 */ 2314 static int 2315 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr, 2316 page_t **pps, uint_t flags) 2317 { 2318 struct hmehash_bucket *hmebp; 2319 struct hme_blk *hmeblkp; 2320 int ret; 2321 uint_t size; 2322 2323 /* 2324 * Get mapping size. 2325 */ 2326 size = TTE_CSZ(ttep); 2327 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2328 2329 /* 2330 * Acquire the hash bucket. 2331 */ 2332 hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size); 2333 ASSERT(hmebp); 2334 2335 /* 2336 * Find the hment block. 2337 */ 2338 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags); 2339 ASSERT(hmeblkp); 2340 2341 /* 2342 * Add the translation. 2343 */ 2344 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags); 2345 2346 /* 2347 * Release the hash bucket. 2348 */ 2349 sfmmu_tteload_release_hashbucket(hmebp); 2350 2351 return (ret); 2352 } 2353 2354 /* 2355 * Function locks and returns a pointer to the hash bucket for vaddr and size. 2356 */ 2357 static struct hmehash_bucket * 2358 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size) 2359 { 2360 struct hmehash_bucket *hmebp; 2361 int hmeshift; 2362 2363 hmeshift = HME_HASH_SHIFT(size); 2364 2365 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 2366 2367 SFMMU_HASH_LOCK(hmebp); 2368 2369 return (hmebp); 2370 } 2371 2372 /* 2373 * Function returns a pointer to an hmeblk in the hash bucket, hmebp. If the 2374 * hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is 2375 * allocated. 2376 */ 2377 static struct hme_blk * 2378 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp, 2379 caddr_t vaddr, uint_t size, uint_t flags) 2380 { 2381 hmeblk_tag hblktag; 2382 int hmeshift; 2383 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 2384 uint64_t hblkpa, prevpa; 2385 struct kmem_cache *sfmmu_cache; 2386 uint_t forcefree; 2387 2388 hblktag.htag_id = sfmmup; 2389 hmeshift = HME_HASH_SHIFT(size); 2390 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 2391 hblktag.htag_rehash = HME_HASH_REHASH(size); 2392 2393 ttearray_realloc: 2394 2395 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, 2396 pr_hblk, prevpa, &list); 2397 2398 /* 2399 * We block until hblk_reserve_lock is released; it's held by 2400 * the thread, temporarily using hblk_reserve, until hblk_reserve is 2401 * replaced by a hblk from sfmmu8_cache. 2402 */ 2403 if (hmeblkp == (struct hme_blk *)hblk_reserve && 2404 hblk_reserve_thread != curthread) { 2405 SFMMU_HASH_UNLOCK(hmebp); 2406 mutex_enter(&hblk_reserve_lock); 2407 mutex_exit(&hblk_reserve_lock); 2408 SFMMU_STAT(sf_hblk_reserve_hit); 2409 SFMMU_HASH_LOCK(hmebp); 2410 goto ttearray_realloc; 2411 } 2412 2413 if (hmeblkp == NULL) { 2414 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 2415 hblktag, flags); 2416 } else { 2417 /* 2418 * It is possible for 8k and 64k hblks to collide since they 2419 * have the same rehash value. This is because we 2420 * lazily free hblks and 8K/64K blks could be lingering. 2421 * If we find size mismatch we free the block and & try again. 2422 */ 2423 if (get_hblk_ttesz(hmeblkp) != size) { 2424 ASSERT(!hmeblkp->hblk_vcnt); 2425 ASSERT(!hmeblkp->hblk_hmecnt); 2426 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 2427 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 2428 goto ttearray_realloc; 2429 } 2430 if (hmeblkp->hblk_shw_bit) { 2431 /* 2432 * if the hblk was previously used as a shadow hblk then 2433 * we will change it to a normal hblk 2434 */ 2435 if (hmeblkp->hblk_shw_mask) { 2436 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp); 2437 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2438 goto ttearray_realloc; 2439 } else { 2440 hmeblkp->hblk_shw_bit = 0; 2441 } 2442 } 2443 SFMMU_STAT(sf_hblk_hit); 2444 } 2445 2446 /* 2447 * hat_memload() should never call kmem_cache_free(); see block 2448 * comment showing the stacktrace in sfmmu_hblk_alloc(); 2449 * enqueue each hblk in the list to reserve list if it's created 2450 * from sfmmu8_cache *and* sfmmup == KHATID. 2451 */ 2452 forcefree = (sfmmup == KHATID) ? 1 : 0; 2453 while ((pr_hblk = list) != NULL) { 2454 list = pr_hblk->hblk_next; 2455 sfmmu_cache = get_hblk_cache(pr_hblk); 2456 if ((sfmmu_cache == sfmmu8_cache) && 2457 sfmmu_put_free_hblk(pr_hblk, forcefree)) 2458 continue; 2459 2460 ASSERT(sfmmup != KHATID); 2461 kmem_cache_free(sfmmu_cache, pr_hblk); 2462 } 2463 2464 ASSERT(get_hblk_ttesz(hmeblkp) == size); 2465 ASSERT(!hmeblkp->hblk_shw_bit); 2466 2467 return (hmeblkp); 2468 } 2469 2470 /* 2471 * Function adds a tte entry into the hmeblk. It returns 0 if successful and 1 2472 * otherwise. 2473 */ 2474 static int 2475 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep, 2476 caddr_t vaddr, page_t **pps, uint_t flags) 2477 { 2478 page_t *pp = *pps; 2479 int hmenum, size, remap; 2480 tte_t tteold, flush_tte; 2481 #ifdef DEBUG 2482 tte_t orig_old; 2483 #endif /* DEBUG */ 2484 struct sf_hment *sfhme; 2485 kmutex_t *pml, *pmtx; 2486 hatlock_t *hatlockp; 2487 2488 /* 2489 * remove this panic when we decide to let user virtual address 2490 * space be >= USERLIMIT. 2491 */ 2492 if (!TTE_IS_PRIVILEGED(ttep) && vaddr >= (caddr_t)USERLIMIT) 2493 panic("user addr %p in kernel space", vaddr); 2494 #if defined(TTE_IS_GLOBAL) 2495 if (TTE_IS_GLOBAL(ttep)) 2496 panic("sfmmu_tteload: creating global tte"); 2497 #endif 2498 2499 #ifdef DEBUG 2500 if (pf_is_memory(sfmmu_ttetopfn(ttep, vaddr)) && 2501 !TTE_IS_PCACHEABLE(ttep) && !sfmmu_allow_nc_trans) 2502 panic("sfmmu_tteload: non cacheable memory tte"); 2503 #endif /* DEBUG */ 2504 2505 if ((flags & HAT_LOAD_SHARE) || !TTE_IS_REF(ttep) || 2506 !TTE_IS_MOD(ttep)) { 2507 /* 2508 * Don't load TSB for dummy as in ISM. Also don't preload 2509 * the TSB if the TTE isn't writable since we're likely to 2510 * fault on it again -- preloading can be fairly expensive. 2511 */ 2512 flags |= SFMMU_NO_TSBLOAD; 2513 } 2514 2515 size = TTE_CSZ(ttep); 2516 switch (size) { 2517 case TTE8K: 2518 SFMMU_STAT(sf_tteload8k); 2519 break; 2520 case TTE64K: 2521 SFMMU_STAT(sf_tteload64k); 2522 break; 2523 case TTE512K: 2524 SFMMU_STAT(sf_tteload512k); 2525 break; 2526 case TTE4M: 2527 SFMMU_STAT(sf_tteload4m); 2528 break; 2529 case (TTE32M): 2530 SFMMU_STAT(sf_tteload32m); 2531 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2532 break; 2533 case (TTE256M): 2534 SFMMU_STAT(sf_tteload256m); 2535 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2536 break; 2537 } 2538 2539 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2540 2541 HBLKTOHME_IDX(sfhme, hmeblkp, vaddr, hmenum); 2542 2543 /* 2544 * Need to grab mlist lock here so that pageunload 2545 * will not change tte behind us. 2546 */ 2547 if (pp) { 2548 pml = sfmmu_mlist_enter(pp); 2549 } 2550 2551 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2552 /* 2553 * Look for corresponding hment and if valid verify 2554 * pfns are equal. 2555 */ 2556 remap = TTE_IS_VALID(&tteold); 2557 if (remap) { 2558 pfn_t new_pfn, old_pfn; 2559 2560 old_pfn = TTE_TO_PFN(vaddr, &tteold); 2561 new_pfn = TTE_TO_PFN(vaddr, ttep); 2562 2563 if (flags & HAT_LOAD_REMAP) { 2564 /* make sure we are remapping same type of pages */ 2565 if (pf_is_memory(old_pfn) != pf_is_memory(new_pfn)) { 2566 panic("sfmmu_tteload - tte remap io<->memory"); 2567 } 2568 if (old_pfn != new_pfn && 2569 (pp != NULL || sfhme->hme_page != NULL)) { 2570 panic("sfmmu_tteload - tte remap pp != NULL"); 2571 } 2572 } else if (old_pfn != new_pfn) { 2573 panic("sfmmu_tteload - tte remap, hmeblkp 0x%p", 2574 (void *)hmeblkp); 2575 } 2576 ASSERT(TTE_CSZ(&tteold) == TTE_CSZ(ttep)); 2577 } 2578 2579 if (pp) { 2580 if (size == TTE8K) { 2581 /* 2582 * Handle VAC consistency 2583 */ 2584 if (!remap && (cache & CACHE_VAC) && !PP_ISNC(pp)) { 2585 sfmmu_vac_conflict(sfmmup, vaddr, pp); 2586 } 2587 2588 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2589 pmtx = sfmmu_page_enter(pp); 2590 PP_CLRRO(pp); 2591 sfmmu_page_exit(pmtx); 2592 } else if (!PP_ISMAPPED(pp) && 2593 (!TTE_IS_WRITABLE(ttep)) && !(PP_ISMOD(pp))) { 2594 pmtx = sfmmu_page_enter(pp); 2595 if (!(PP_ISMOD(pp))) { 2596 PP_SETRO(pp); 2597 } 2598 sfmmu_page_exit(pmtx); 2599 } 2600 2601 } else if (sfmmu_pagearray_setup(vaddr, pps, ttep, remap)) { 2602 /* 2603 * sfmmu_pagearray_setup failed so return 2604 */ 2605 sfmmu_mlist_exit(pml); 2606 return (1); 2607 } 2608 } 2609 2610 /* 2611 * Make sure hment is not on a mapping list. 2612 */ 2613 ASSERT(remap || (sfhme->hme_page == NULL)); 2614 2615 /* if it is not a remap then hme->next better be NULL */ 2616 ASSERT((!remap) ? sfhme->hme_next == NULL : 1); 2617 2618 if (flags & HAT_LOAD_LOCK) { 2619 if (((int)hmeblkp->hblk_lckcnt + 1) >= MAX_HBLK_LCKCNT) { 2620 panic("too high lckcnt-hmeblk %p", 2621 (void *)hmeblkp); 2622 } 2623 atomic_add_16(&hmeblkp->hblk_lckcnt, 1); 2624 2625 HBLK_STACK_TRACE(hmeblkp, HBLK_LOCK); 2626 } 2627 2628 if (pp && PP_ISNC(pp)) { 2629 /* 2630 * If the physical page is marked to be uncacheable, like 2631 * by a vac conflict, make sure the new mapping is also 2632 * uncacheable. 2633 */ 2634 TTE_CLR_VCACHEABLE(ttep); 2635 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 2636 } 2637 ttep->tte_hmenum = hmenum; 2638 2639 #ifdef DEBUG 2640 orig_old = tteold; 2641 #endif /* DEBUG */ 2642 2643 while (sfmmu_modifytte_try(&tteold, ttep, &sfhme->hme_tte) < 0) { 2644 if ((sfmmup == KHATID) && 2645 (flags & (HAT_LOAD_LOCK | HAT_LOAD_REMAP))) { 2646 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2647 } 2648 #ifdef DEBUG 2649 chk_tte(&orig_old, &tteold, ttep, hmeblkp); 2650 #endif /* DEBUG */ 2651 } 2652 2653 if (!TTE_IS_VALID(&tteold)) { 2654 2655 atomic_add_16(&hmeblkp->hblk_vcnt, 1); 2656 atomic_add_long(&sfmmup->sfmmu_ttecnt[size], 1); 2657 2658 /* 2659 * HAT_RELOAD_SHARE has been deprecated with lpg DISM. 2660 */ 2661 2662 if (size > TTE8K && (flags & HAT_LOAD_SHARE) == 0 && 2663 sfmmup != ksfmmup) { 2664 /* 2665 * If this is the first large mapping for the process 2666 * we must force any CPUs running this process to TL=0 2667 * where they will reload the HAT flags from the 2668 * tsbmiss area. This is necessary to make the large 2669 * mappings we are about to load visible to those CPUs; 2670 * otherwise they'll loop forever calling pagefault() 2671 * since we don't search large hash chains by default. 2672 */ 2673 hatlockp = sfmmu_hat_enter(sfmmup); 2674 if (size == TTE512K && 2675 !SFMMU_FLAGS_ISSET(sfmmup, HAT_512K_FLAG)) { 2676 SFMMU_FLAGS_SET(sfmmup, HAT_512K_FLAG); 2677 sfmmu_sync_mmustate(sfmmup); 2678 } else if (size == TTE4M && 2679 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 2680 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 2681 sfmmu_sync_mmustate(sfmmup); 2682 } else if (size == TTE64K && 2683 !SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) { 2684 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 2685 /* no sync mmustate; 64K shares 8K hashes */ 2686 } else if (mmu_page_sizes == max_mmu_page_sizes) { 2687 if (size == TTE32M && 2688 !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 2689 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 2690 sfmmu_sync_mmustate(sfmmup); 2691 } else if (size == TTE256M && 2692 !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 2693 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 2694 sfmmu_sync_mmustate(sfmmup); 2695 } 2696 } 2697 if (size >= TTE4M && (flags & HAT_LOAD_TEXT) && 2698 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) { 2699 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG); 2700 } 2701 sfmmu_hat_exit(hatlockp); 2702 } 2703 } 2704 ASSERT(TTE_IS_VALID(&sfhme->hme_tte)); 2705 2706 flush_tte.tte_intlo = (tteold.tte_intlo ^ ttep->tte_intlo) & 2707 hw_tte.tte_intlo; 2708 flush_tte.tte_inthi = (tteold.tte_inthi ^ ttep->tte_inthi) & 2709 hw_tte.tte_inthi; 2710 2711 if (remap && (flush_tte.tte_inthi || flush_tte.tte_intlo)) { 2712 /* 2713 * If remap and new tte differs from old tte we need 2714 * to sync the mod bit and flush TLB/TSB. We don't 2715 * need to sync ref bit because we currently always set 2716 * ref bit in tteload. 2717 */ 2718 ASSERT(TTE_IS_REF(ttep)); 2719 if (TTE_IS_MOD(&tteold)) { 2720 sfmmu_ttesync(sfmmup, vaddr, &tteold, pp); 2721 } 2722 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0); 2723 xt_sync(sfmmup->sfmmu_cpusran); 2724 } 2725 2726 if ((flags & SFMMU_NO_TSBLOAD) == 0) { 2727 /* 2728 * We only preload 8K and 4M mappings into the TSB, since 2729 * 64K and 512K mappings are replicated and hence don't 2730 * have a single, unique TSB entry. Ditto for 32M/256M. 2731 */ 2732 if (size == TTE8K || size == TTE4M) { 2733 hatlockp = sfmmu_hat_enter(sfmmup); 2734 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte, size); 2735 sfmmu_hat_exit(hatlockp); 2736 } 2737 } 2738 if (pp) { 2739 if (!remap) { 2740 HME_ADD(sfhme, pp); 2741 atomic_add_16(&hmeblkp->hblk_hmecnt, 1); 2742 ASSERT(hmeblkp->hblk_hmecnt > 0); 2743 2744 /* 2745 * Cannot ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 2746 * see pageunload() for comment. 2747 */ 2748 } 2749 sfmmu_mlist_exit(pml); 2750 } 2751 2752 return (0); 2753 } 2754 /* 2755 * Function unlocks hash bucket. 2756 */ 2757 static void 2758 sfmmu_tteload_release_hashbucket(struct hmehash_bucket *hmebp) 2759 { 2760 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2761 SFMMU_HASH_UNLOCK(hmebp); 2762 } 2763 2764 /* 2765 * function which checks and sets up page array for a large 2766 * translation. Will set p_vcolor, p_index, p_ro fields. 2767 * Assumes addr and pfnum of first page are properly aligned. 2768 * Will check for physical contiguity. If check fails it return 2769 * non null. 2770 */ 2771 static int 2772 sfmmu_pagearray_setup(caddr_t addr, page_t **pps, tte_t *ttep, int remap) 2773 { 2774 int i, index, ttesz, osz; 2775 pfn_t pfnum; 2776 pgcnt_t npgs; 2777 int cflags = 0; 2778 page_t *pp, *pp1; 2779 kmutex_t *pmtx; 2780 int vac_err = 0; 2781 int newidx = 0; 2782 2783 ttesz = TTE_CSZ(ttep); 2784 2785 ASSERT(ttesz > TTE8K); 2786 2787 npgs = TTEPAGES(ttesz); 2788 index = PAGESZ_TO_INDEX(ttesz); 2789 2790 pfnum = (*pps)->p_pagenum; 2791 ASSERT(IS_P2ALIGNED(pfnum, npgs)); 2792 2793 /* 2794 * Save the first pp so we can do HAT_TMPNC at the end. 2795 */ 2796 pp1 = *pps; 2797 osz = fnd_mapping_sz(pp1); 2798 2799 for (i = 0; i < npgs; i++, pps++) { 2800 pp = *pps; 2801 ASSERT(PAGE_LOCKED(pp)); 2802 ASSERT(pp->p_szc >= ttesz); 2803 ASSERT(pp->p_szc == pp1->p_szc); 2804 ASSERT(sfmmu_mlist_held(pp)); 2805 2806 /* 2807 * XXX is it possible to maintain P_RO on the root only? 2808 */ 2809 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2810 pmtx = sfmmu_page_enter(pp); 2811 PP_CLRRO(pp); 2812 sfmmu_page_exit(pmtx); 2813 } else if (!PP_ISMAPPED(pp) && !TTE_IS_WRITABLE(ttep) && 2814 !PP_ISMOD(pp)) { 2815 pmtx = sfmmu_page_enter(pp); 2816 if (!(PP_ISMOD(pp))) { 2817 PP_SETRO(pp); 2818 } 2819 sfmmu_page_exit(pmtx); 2820 } 2821 2822 /* 2823 * If this is a remap we skip vac & contiguity checks. 2824 */ 2825 if (remap) 2826 continue; 2827 2828 /* 2829 * set p_vcolor and detect any vac conflicts. 2830 */ 2831 if (vac_err == 0) { 2832 vac_err = sfmmu_vacconflict_array(addr, pp, &cflags); 2833 2834 } 2835 2836 /* 2837 * Save current index in case we need to undo it. 2838 * Note: "PAGESZ_TO_INDEX(sz) (1 << (sz))" 2839 * "SFMMU_INDEX_SHIFT 6" 2840 * "SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1)" 2841 * "PP_MAPINDEX(p_index) (p_index & SFMMU_INDEX_MASK)" 2842 * 2843 * So: index = PAGESZ_TO_INDEX(ttesz); 2844 * if ttesz == 1 then index = 0x2 2845 * 2 then index = 0x4 2846 * 3 then index = 0x8 2847 * 4 then index = 0x10 2848 * 5 then index = 0x20 2849 * The code below checks if it's a new pagesize (ie, newidx) 2850 * in case we need to take it back out of p_index, 2851 * and then or's the new index into the existing index. 2852 */ 2853 if ((PP_MAPINDEX(pp) & index) == 0) 2854 newidx = 1; 2855 pp->p_index = (PP_MAPINDEX(pp) | index); 2856 2857 /* 2858 * contiguity check 2859 */ 2860 if (pp->p_pagenum != pfnum) { 2861 /* 2862 * If we fail the contiguity test then 2863 * the only thing we need to fix is the p_index field. 2864 * We might get a few extra flushes but since this 2865 * path is rare that is ok. The p_ro field will 2866 * get automatically fixed on the next tteload to 2867 * the page. NO TNC bit is set yet. 2868 */ 2869 while (i >= 0) { 2870 pp = *pps; 2871 if (newidx) 2872 pp->p_index = (PP_MAPINDEX(pp) & 2873 ~index); 2874 pps--; 2875 i--; 2876 } 2877 return (1); 2878 } 2879 pfnum++; 2880 addr += MMU_PAGESIZE; 2881 } 2882 2883 if (vac_err) { 2884 if (ttesz > osz) { 2885 /* 2886 * There are some smaller mappings that causes vac 2887 * conflicts. Convert all existing small mappings to 2888 * TNC. 2889 */ 2890 SFMMU_STAT_ADD(sf_uncache_conflict, npgs); 2891 sfmmu_page_cache_array(pp1, HAT_TMPNC, CACHE_FLUSH, 2892 npgs); 2893 } else { 2894 /* EMPTY */ 2895 /* 2896 * If there exists an big page mapping, 2897 * that means the whole existing big page 2898 * has TNC setting already. No need to covert to 2899 * TNC again. 2900 */ 2901 ASSERT(PP_ISTNC(pp1)); 2902 } 2903 } 2904 2905 return (0); 2906 } 2907 2908 /* 2909 * Routine that detects vac consistency for a large page. It also 2910 * sets virtual color for all pp's for this big mapping. 2911 */ 2912 static int 2913 sfmmu_vacconflict_array(caddr_t addr, page_t *pp, int *cflags) 2914 { 2915 int vcolor, ocolor; 2916 2917 ASSERT(sfmmu_mlist_held(pp)); 2918 2919 if (PP_ISNC(pp)) { 2920 return (HAT_TMPNC); 2921 } 2922 2923 vcolor = addr_to_vcolor(addr); 2924 if (PP_NEWPAGE(pp)) { 2925 PP_SET_VCOLOR(pp, vcolor); 2926 return (0); 2927 } 2928 2929 ocolor = PP_GET_VCOLOR(pp); 2930 if (ocolor == vcolor) { 2931 return (0); 2932 } 2933 2934 if (!PP_ISMAPPED(pp)) { 2935 /* 2936 * Previous user of page had a differnet color 2937 * but since there are no current users 2938 * we just flush the cache and change the color. 2939 * As an optimization for large pages we flush the 2940 * entire cache of that color and set a flag. 2941 */ 2942 SFMMU_STAT(sf_pgcolor_conflict); 2943 if (!CacheColor_IsFlushed(*cflags, ocolor)) { 2944 CacheColor_SetFlushed(*cflags, ocolor); 2945 sfmmu_cache_flushcolor(ocolor, pp->p_pagenum); 2946 } 2947 PP_SET_VCOLOR(pp, vcolor); 2948 return (0); 2949 } 2950 2951 /* 2952 * We got a real conflict with a current mapping. 2953 * set flags to start unencaching all mappings 2954 * and return failure so we restart looping 2955 * the pp array from the beginning. 2956 */ 2957 return (HAT_TMPNC); 2958 } 2959 2960 /* 2961 * creates a large page shadow hmeblk for a tte. 2962 * The purpose of this routine is to allow us to do quick unloads because 2963 * the vm layer can easily pass a very large but sparsely populated range. 2964 */ 2965 static struct hme_blk * 2966 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags) 2967 { 2968 struct hmehash_bucket *hmebp; 2969 hmeblk_tag hblktag; 2970 int hmeshift, size, vshift; 2971 uint_t shw_mask, newshw_mask; 2972 struct hme_blk *hmeblkp; 2973 2974 ASSERT(sfmmup != KHATID); 2975 if (mmu_page_sizes == max_mmu_page_sizes) { 2976 ASSERT(ttesz < TTE256M); 2977 } else { 2978 ASSERT(ttesz < TTE4M); 2979 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 2980 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 2981 } 2982 2983 if (ttesz == TTE8K) { 2984 size = TTE512K; 2985 } else { 2986 size = ++ttesz; 2987 } 2988 2989 hblktag.htag_id = sfmmup; 2990 hmeshift = HME_HASH_SHIFT(size); 2991 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 2992 hblktag.htag_rehash = HME_HASH_REHASH(size); 2993 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 2994 2995 SFMMU_HASH_LOCK(hmebp); 2996 2997 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 2998 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 2999 if (hmeblkp == NULL) { 3000 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 3001 hblktag, flags); 3002 } 3003 ASSERT(hmeblkp); 3004 if (!hmeblkp->hblk_shw_mask) { 3005 /* 3006 * if this is a unused hblk it was just allocated or could 3007 * potentially be a previous large page hblk so we need to 3008 * set the shadow bit. 3009 */ 3010 hmeblkp->hblk_shw_bit = 1; 3011 } 3012 ASSERT(hmeblkp->hblk_shw_bit == 1); 3013 vshift = vaddr_to_vshift(hblktag, vaddr, size); 3014 ASSERT(vshift < 8); 3015 /* 3016 * Atomically set shw mask bit 3017 */ 3018 do { 3019 shw_mask = hmeblkp->hblk_shw_mask; 3020 newshw_mask = shw_mask | (1 << vshift); 3021 newshw_mask = cas32(&hmeblkp->hblk_shw_mask, shw_mask, 3022 newshw_mask); 3023 } while (newshw_mask != shw_mask); 3024 3025 SFMMU_HASH_UNLOCK(hmebp); 3026 3027 return (hmeblkp); 3028 } 3029 3030 /* 3031 * This routine cleanup a previous shadow hmeblk and changes it to 3032 * a regular hblk. This happens rarely but it is possible 3033 * when a process wants to use large pages and there are hblks still 3034 * lying around from the previous as that used these hmeblks. 3035 * The alternative was to cleanup the shadow hblks at unload time 3036 * but since so few user processes actually use large pages, it is 3037 * better to be lazy and cleanup at this time. 3038 */ 3039 static void 3040 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 3041 struct hmehash_bucket *hmebp) 3042 { 3043 caddr_t addr, endaddr; 3044 int hashno, size; 3045 3046 ASSERT(hmeblkp->hblk_shw_bit); 3047 3048 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 3049 3050 if (!hmeblkp->hblk_shw_mask) { 3051 hmeblkp->hblk_shw_bit = 0; 3052 return; 3053 } 3054 addr = (caddr_t)get_hblk_base(hmeblkp); 3055 endaddr = get_hblk_endaddr(hmeblkp); 3056 size = get_hblk_ttesz(hmeblkp); 3057 hashno = size - 1; 3058 ASSERT(hashno > 0); 3059 SFMMU_HASH_UNLOCK(hmebp); 3060 3061 sfmmu_free_hblks(sfmmup, addr, endaddr, hashno); 3062 3063 SFMMU_HASH_LOCK(hmebp); 3064 } 3065 3066 static void 3067 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr, 3068 int hashno) 3069 { 3070 int hmeshift, shadow = 0; 3071 hmeblk_tag hblktag; 3072 struct hmehash_bucket *hmebp; 3073 struct hme_blk *hmeblkp; 3074 struct hme_blk *nx_hblk, *pr_hblk, *list = NULL; 3075 uint64_t hblkpa, prevpa, nx_pa; 3076 3077 ASSERT(hashno > 0); 3078 hblktag.htag_id = sfmmup; 3079 hblktag.htag_rehash = hashno; 3080 3081 hmeshift = HME_HASH_SHIFT(hashno); 3082 3083 while (addr < endaddr) { 3084 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3085 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3086 SFMMU_HASH_LOCK(hmebp); 3087 /* inline HME_HASH_SEARCH */ 3088 hmeblkp = hmebp->hmeblkp; 3089 hblkpa = hmebp->hmeh_nextpa; 3090 prevpa = 0; 3091 pr_hblk = NULL; 3092 while (hmeblkp) { 3093 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 3094 if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) { 3095 /* found hme_blk */ 3096 if (hmeblkp->hblk_shw_bit) { 3097 if (hmeblkp->hblk_shw_mask) { 3098 shadow = 1; 3099 sfmmu_shadow_hcleanup(sfmmup, 3100 hmeblkp, hmebp); 3101 break; 3102 } else { 3103 hmeblkp->hblk_shw_bit = 0; 3104 } 3105 } 3106 3107 /* 3108 * Hblk_hmecnt and hblk_vcnt could be non zero 3109 * since hblk_unload() does not gurantee that. 3110 * 3111 * XXX - this could cause tteload() to spin 3112 * where sfmmu_shadow_hcleanup() is called. 3113 */ 3114 } 3115 3116 nx_hblk = hmeblkp->hblk_next; 3117 nx_pa = hmeblkp->hblk_nextpa; 3118 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 3119 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 3120 pr_hblk); 3121 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 3122 } else { 3123 pr_hblk = hmeblkp; 3124 prevpa = hblkpa; 3125 } 3126 hmeblkp = nx_hblk; 3127 hblkpa = nx_pa; 3128 } 3129 3130 SFMMU_HASH_UNLOCK(hmebp); 3131 3132 if (shadow) { 3133 /* 3134 * We found another shadow hblk so cleaned its 3135 * children. We need to go back and cleanup 3136 * the original hblk so we don't change the 3137 * addr. 3138 */ 3139 shadow = 0; 3140 } else { 3141 addr = (caddr_t)roundup((uintptr_t)addr + 1, 3142 (1 << hmeshift)); 3143 } 3144 } 3145 sfmmu_hblks_list_purge(&list); 3146 } 3147 3148 /* 3149 * Release one hardware address translation lock on the given address range. 3150 */ 3151 void 3152 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len) 3153 { 3154 struct hmehash_bucket *hmebp; 3155 hmeblk_tag hblktag; 3156 int hmeshift, hashno = 1; 3157 struct hme_blk *hmeblkp, *list = NULL; 3158 caddr_t endaddr; 3159 3160 ASSERT(sfmmup != NULL); 3161 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3162 3163 ASSERT((sfmmup == ksfmmup) || 3164 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 3165 ASSERT((len & MMU_PAGEOFFSET) == 0); 3166 endaddr = addr + len; 3167 hblktag.htag_id = sfmmup; 3168 3169 /* 3170 * Spitfire supports 4 page sizes. 3171 * Most pages are expected to be of the smallest page size (8K) and 3172 * these will not need to be rehashed. 64K pages also don't need to be 3173 * rehashed because an hmeblk spans 64K of address space. 512K pages 3174 * might need 1 rehash and and 4M pages might need 2 rehashes. 3175 */ 3176 while (addr < endaddr) { 3177 hmeshift = HME_HASH_SHIFT(hashno); 3178 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3179 hblktag.htag_rehash = hashno; 3180 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3181 3182 SFMMU_HASH_LOCK(hmebp); 3183 3184 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 3185 if (hmeblkp != NULL) { 3186 /* 3187 * If we encounter a shadow hmeblk then 3188 * we know there are no valid hmeblks mapping 3189 * this address at this size or larger. 3190 * Just increment address by the smallest 3191 * page size. 3192 */ 3193 if (hmeblkp->hblk_shw_bit) { 3194 addr += MMU_PAGESIZE; 3195 } else { 3196 addr = sfmmu_hblk_unlock(hmeblkp, addr, 3197 endaddr); 3198 } 3199 SFMMU_HASH_UNLOCK(hmebp); 3200 hashno = 1; 3201 continue; 3202 } 3203 SFMMU_HASH_UNLOCK(hmebp); 3204 3205 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 3206 /* 3207 * We have traversed the whole list and rehashed 3208 * if necessary without finding the address to unlock 3209 * which should never happen. 3210 */ 3211 panic("sfmmu_unlock: addr not found. " 3212 "addr %p hat %p", (void *)addr, (void *)sfmmup); 3213 } else { 3214 hashno++; 3215 } 3216 } 3217 3218 sfmmu_hblks_list_purge(&list); 3219 } 3220 3221 /* 3222 * Function to unlock a range of addresses in an hmeblk. It returns the 3223 * next address that needs to be unlocked. 3224 * Should be called with the hash lock held. 3225 */ 3226 static caddr_t 3227 sfmmu_hblk_unlock(struct hme_blk *hmeblkp, caddr_t addr, caddr_t endaddr) 3228 { 3229 struct sf_hment *sfhme; 3230 tte_t tteold, ttemod; 3231 int ttesz, ret; 3232 3233 ASSERT(in_hblk_range(hmeblkp, addr)); 3234 ASSERT(hmeblkp->hblk_shw_bit == 0); 3235 3236 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 3237 ttesz = get_hblk_ttesz(hmeblkp); 3238 3239 HBLKTOHME(sfhme, hmeblkp, addr); 3240 while (addr < endaddr) { 3241 readtte: 3242 sfmmu_copytte(&sfhme->hme_tte, &tteold); 3243 if (TTE_IS_VALID(&tteold)) { 3244 3245 ttemod = tteold; 3246 3247 ret = sfmmu_modifytte_try(&tteold, &ttemod, 3248 &sfhme->hme_tte); 3249 3250 if (ret < 0) 3251 goto readtte; 3252 3253 if (hmeblkp->hblk_lckcnt == 0) 3254 panic("zero hblk lckcnt"); 3255 3256 if (((uintptr_t)addr + TTEBYTES(ttesz)) > 3257 (uintptr_t)endaddr) 3258 panic("can't unlock large tte"); 3259 3260 ASSERT(hmeblkp->hblk_lckcnt > 0); 3261 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 3262 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 3263 } else { 3264 panic("sfmmu_hblk_unlock: invalid tte"); 3265 } 3266 addr += TTEBYTES(ttesz); 3267 sfhme++; 3268 } 3269 return (addr); 3270 } 3271 3272 /* 3273 * Physical Address Mapping Framework 3274 * 3275 * General rules: 3276 * 3277 * (1) Applies only to seg_kmem memory pages. To make things easier, 3278 * seg_kpm addresses are also accepted by the routines, but nothing 3279 * is done with them since by definition their PA mappings are static. 3280 * (2) hat_add_callback() may only be called while holding the page lock 3281 * SE_SHARED or SE_EXCL of the underlying page (e.g., as_pagelock()). 3282 * (3) prehandler() and posthandler() may not call hat_add_callback() or 3283 * hat_delete_callback(), nor should they allocate memory. Post quiesce 3284 * callbacks may not sleep or acquire adaptive mutex locks. 3285 * (4) Either prehandler() or posthandler() (but not both) may be specified 3286 * as being NULL. Specifying an errhandler() is optional. 3287 * 3288 * Details of using the framework: 3289 * 3290 * registering a callback (hat_register_callback()) 3291 * 3292 * Pass prehandler, posthandler, errhandler addresses 3293 * as described below. If capture_cpus argument is nonzero, 3294 * suspend callback to the prehandler will occur with CPUs 3295 * captured and executing xc_loop() and CPUs will remain 3296 * captured until after the posthandler suspend callback 3297 * occurs. 3298 * 3299 * adding a callback (hat_add_callback()) 3300 * 3301 * as_pagelock(); 3302 * hat_add_callback(); 3303 * save returned pfn in private data structures or program registers; 3304 * as_pageunlock(); 3305 * 3306 * prehandler() 3307 * 3308 * Stop all accesses by physical address to this memory page. 3309 * Called twice: the first, PRESUSPEND, is a context safe to acquire 3310 * adaptive locks. The second, SUSPEND, is called at high PIL with 3311 * CPUs captured so adaptive locks may NOT be acquired (and all spin 3312 * locks must be XCALL_PIL or higher locks). 3313 * 3314 * May return the following errors: 3315 * EIO: A fatal error has occurred. This will result in panic. 3316 * EAGAIN: The page cannot be suspended. This will fail the 3317 * relocation. 3318 * 0: Success. 3319 * 3320 * posthandler() 3321 * 3322 * Save new pfn in private data structures or program registers; 3323 * not allowed to fail (non-zero return values will result in panic). 3324 * 3325 * errhandler() 3326 * 3327 * called when an error occurs related to the callback. Currently 3328 * the only such error is HAT_CB_ERR_LEAKED which indicates that 3329 * a page is being freed, but there are still outstanding callback(s) 3330 * registered on the page. 3331 * 3332 * removing a callback (hat_delete_callback(); e.g., prior to freeing memory) 3333 * 3334 * stop using physical address 3335 * hat_delete_callback(); 3336 * 3337 */ 3338 3339 /* 3340 * Register a callback class. Each subsystem should do this once and 3341 * cache the id_t returned for use in setting up and tearing down callbacks. 3342 * 3343 * There is no facility for removing callback IDs once they are created; 3344 * the "key" should be unique for each module, so in case a module is unloaded 3345 * and subsequently re-loaded, we can recycle the module's previous entry. 3346 */ 3347 id_t 3348 hat_register_callback(int key, 3349 int (*prehandler)(caddr_t, uint_t, uint_t, void *), 3350 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t), 3351 int (*errhandler)(caddr_t, uint_t, uint_t, void *), 3352 int capture_cpus) 3353 { 3354 id_t id; 3355 3356 /* 3357 * Search the table for a pre-existing callback associated with 3358 * the identifier "key". If one exists, we re-use that entry in 3359 * the table for this instance, otherwise we assign the next 3360 * available table slot. 3361 */ 3362 for (id = 0; id < sfmmu_max_cb_id; id++) { 3363 if (sfmmu_cb_table[id].key == key) 3364 break; 3365 } 3366 3367 if (id == sfmmu_max_cb_id) { 3368 id = sfmmu_cb_nextid++; 3369 if (id >= sfmmu_max_cb_id) 3370 panic("hat_register_callback: out of callback IDs"); 3371 } 3372 3373 ASSERT(prehandler != NULL || posthandler != NULL); 3374 3375 sfmmu_cb_table[id].key = key; 3376 sfmmu_cb_table[id].prehandler = prehandler; 3377 sfmmu_cb_table[id].posthandler = posthandler; 3378 sfmmu_cb_table[id].errhandler = errhandler; 3379 sfmmu_cb_table[id].capture_cpus = capture_cpus; 3380 3381 return (id); 3382 } 3383 3384 /* 3385 * Add relocation callbacks to the specified addr/len which will be called 3386 * when relocating the associated page. See the description of pre and 3387 * posthandler above for more details. IMPT: this operation is only valid 3388 * on seg_kmem pages!! 3389 * 3390 * If HAC_PAGELOCK is included in flags, the underlying memory page is 3391 * locked internally so the caller must be able to deal with the callback 3392 * running even before this function has returned. If HAC_PAGELOCK is not 3393 * set, it is assumed that the underlying memory pages are locked. 3394 * 3395 * Since the caller must track the individual page boundaries anyway, 3396 * we only allow a callback to be added to a single page (large 3397 * or small). Thus [addr, addr + len) MUST be contained within a single 3398 * page. 3399 * 3400 * Registering multiple callbacks on the same [addr, addr+len) is supported, 3401 * in which case the corresponding callback will be called once with each 3402 * unique parameter specified. The number of subsequent deletes must match 3403 * since reference counts are held. If a callback is desired for each 3404 * virtual object with the same parameter specified for multiple callbacks, 3405 * a different virtual address should be specified at the time of 3406 * callback registration. 3407 * 3408 * Returns the pfn of the underlying kernel page in *rpfn 3409 * on success, or PFN_INVALID on failure. 3410 * 3411 * Returns values: 3412 * 0: success 3413 * ENOMEM: memory allocation failure (e.g. flags was passed as HAC_NOSLEEP) 3414 * EINVAL: callback ID is not valid 3415 * ENXIO: ["vaddr", "vaddr" + len) is not mapped in the kernel's address 3416 * space, or crosses a page boundary 3417 */ 3418 int 3419 hat_add_callback(id_t callback_id, caddr_t vaddr, uint_t len, uint_t flags, 3420 void *pvt, pfn_t *rpfn) 3421 { 3422 struct hmehash_bucket *hmebp; 3423 hmeblk_tag hblktag; 3424 struct hme_blk *hmeblkp; 3425 int hmeshift, hashno; 3426 caddr_t saddr, eaddr, baseaddr; 3427 struct pa_hment *pahmep, *tpahmep; 3428 struct sf_hment *sfhmep, *osfhmep, *tsfhmep; 3429 kmutex_t *pml; 3430 tte_t tte; 3431 page_t *pp, *rpp; 3432 pfn_t pfn; 3433 int kmflags = (flags & HAC_SLEEP)? KM_SLEEP : KM_NOSLEEP; 3434 int locked = 0; 3435 3436 /* 3437 * For KPM mappings, just return the physical address since we 3438 * don't need to register any callbacks. 3439 */ 3440 if (IS_KPM_ADDR(vaddr)) { 3441 uint64_t paddr; 3442 SFMMU_KPM_VTOP(vaddr, paddr); 3443 *rpfn = btop(paddr); 3444 return (0); 3445 } 3446 3447 if (callback_id < (id_t)0 || callback_id >= sfmmu_cb_nextid) { 3448 *rpfn = PFN_INVALID; 3449 return (EINVAL); 3450 } 3451 3452 if ((pahmep = kmem_cache_alloc(pa_hment_cache, kmflags)) == NULL) { 3453 *rpfn = PFN_INVALID; 3454 return (ENOMEM); 3455 } 3456 3457 sfhmep = &pahmep->sfment; 3458 3459 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3460 eaddr = saddr + len; 3461 3462 rehash: 3463 /* Find the mapping(s) for this page */ 3464 for (hashno = TTE64K, hmeblkp = NULL; 3465 hmeblkp == NULL && hashno <= mmu_hashcnt; 3466 hashno++) { 3467 hmeshift = HME_HASH_SHIFT(hashno); 3468 hblktag.htag_id = ksfmmup; 3469 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3470 hblktag.htag_rehash = hashno; 3471 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3472 3473 SFMMU_HASH_LOCK(hmebp); 3474 3475 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3476 3477 if (hmeblkp == NULL) 3478 SFMMU_HASH_UNLOCK(hmebp); 3479 } 3480 3481 if (hmeblkp == NULL) { 3482 kmem_cache_free(pa_hment_cache, pahmep); 3483 *rpfn = PFN_INVALID; 3484 return (ENXIO); 3485 } 3486 3487 /* 3488 * Make sure the boundaries for the callback fall within this 3489 * single mapping. 3490 */ 3491 baseaddr = (caddr_t)get_hblk_base(hmeblkp); 3492 ASSERT(saddr >= baseaddr); 3493 if (eaddr > (caddr_t)get_hblk_endaddr(hmeblkp)) { 3494 SFMMU_HASH_UNLOCK(hmebp); 3495 kmem_cache_free(pa_hment_cache, pahmep); 3496 *rpfn = PFN_INVALID; 3497 return (ENXIO); 3498 } 3499 3500 HBLKTOHME(osfhmep, hmeblkp, saddr); 3501 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3502 3503 ASSERT(TTE_IS_VALID(&tte)); 3504 pfn = sfmmu_ttetopfn(&tte, vaddr); 3505 3506 /* 3507 * The pfn may not have a page_t underneath in which case we 3508 * just return it. This can happen if we are doing I/O to a 3509 * static portion of the kernel's address space, for instance. 3510 */ 3511 pp = osfhmep->hme_page; 3512 if (pp == NULL || pp->p_vnode != &kvp) { 3513 SFMMU_HASH_UNLOCK(hmebp); 3514 kmem_cache_free(pa_hment_cache, pahmep); 3515 *rpfn = pfn; 3516 return (0); 3517 } 3518 3519 pml = sfmmu_mlist_enter(pp); 3520 3521 if ((flags & HAC_PAGELOCK) && !locked) { 3522 if (!page_trylock(pp, SE_SHARED)) { 3523 page_t *tpp; 3524 3525 /* 3526 * Somebody is holding SE_EXCL lock. Drop all 3527 * our locks, lookup the page in &kvp, and 3528 * retry. If it doesn't exist in &kvp, then we 3529 * die here; we should have caught it above, 3530 * meaning the page must have changed identity 3531 * (e.g. the caller didn't hold onto the page 3532 * lock after establishing the kernel mapping) 3533 */ 3534 sfmmu_mlist_exit(pml); 3535 SFMMU_HASH_UNLOCK(hmebp); 3536 tpp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3537 if (tpp == NULL) { 3538 panic("hat_add_callback: page not found: 0x%p", 3539 pp); 3540 } 3541 pp = tpp; 3542 rpp = PP_PAGEROOT(pp); 3543 if (rpp != pp) { 3544 page_unlock(pp); 3545 (void) page_lock(rpp, SE_SHARED, NULL, 3546 P_NO_RECLAIM); 3547 } 3548 locked = 1; 3549 goto rehash; 3550 } 3551 locked = 1; 3552 } 3553 3554 if (!PAGE_LOCKED(pp) && !panicstr) 3555 panic("hat_add_callback: page 0x%p not locked", pp); 3556 3557 if (osfhmep->hme_page != pp || pp->p_vnode != &kvp || 3558 pp->p_offset < (u_offset_t)baseaddr || 3559 pp->p_offset > (u_offset_t)eaddr) { 3560 /* 3561 * The page moved before we got our hands on it. Drop 3562 * all the locks and try again. 3563 */ 3564 ASSERT((flags & HAC_PAGELOCK) != 0); 3565 sfmmu_mlist_exit(pml); 3566 SFMMU_HASH_UNLOCK(hmebp); 3567 page_unlock(pp); 3568 locked = 0; 3569 goto rehash; 3570 } 3571 3572 ASSERT(osfhmep->hme_page == pp); 3573 3574 for (tsfhmep = pp->p_mapping; tsfhmep != NULL; 3575 tsfhmep = tsfhmep->hme_next) { 3576 3577 /* 3578 * skip va to pa mappings 3579 */ 3580 if (!IS_PAHME(tsfhmep)) 3581 continue; 3582 3583 tpahmep = tsfhmep->hme_data; 3584 ASSERT(tpahmep != NULL); 3585 3586 /* 3587 * See if the pahment already exists. 3588 */ 3589 if ((tpahmep->pvt == pvt) && 3590 (tpahmep->addr == vaddr) && 3591 (tpahmep->len == len)) { 3592 ASSERT(tpahmep->cb_id == callback_id); 3593 tpahmep->refcnt++; 3594 pp->p_share++; 3595 3596 sfmmu_mlist_exit(pml); 3597 SFMMU_HASH_UNLOCK(hmebp); 3598 3599 if (locked) 3600 page_unlock(pp); 3601 3602 kmem_cache_free(pa_hment_cache, pahmep); 3603 3604 *rpfn = pfn; 3605 return (0); 3606 } 3607 } 3608 3609 /* 3610 * setup this shiny new pa_hment .. 3611 */ 3612 pp->p_share++; 3613 pahmep->cb_id = callback_id; 3614 pahmep->addr = vaddr; 3615 pahmep->len = len; 3616 pahmep->refcnt = 1; 3617 pahmep->flags = 0; 3618 pahmep->pvt = pvt; 3619 3620 /* 3621 * .. and also set up the sf_hment and link to p_mapping list. 3622 */ 3623 sfhmep->hme_tte.ll = 0; 3624 sfhmep->hme_data = pahmep; 3625 sfhmep->hme_prev = osfhmep; 3626 sfhmep->hme_next = osfhmep->hme_next; 3627 3628 if (osfhmep->hme_next) 3629 osfhmep->hme_next->hme_prev = sfhmep; 3630 3631 osfhmep->hme_next = sfhmep; 3632 3633 sfmmu_mlist_exit(pml); 3634 SFMMU_HASH_UNLOCK(hmebp); 3635 3636 *rpfn = pfn; 3637 if (locked) 3638 page_unlock(pp); 3639 3640 return (0); 3641 } 3642 3643 /* 3644 * Remove the relocation callbacks from the specified addr/len. 3645 */ 3646 void 3647 hat_delete_callback(caddr_t vaddr, uint_t len, void *pvt, uint_t flags) 3648 { 3649 struct hmehash_bucket *hmebp; 3650 hmeblk_tag hblktag; 3651 struct hme_blk *hmeblkp; 3652 int hmeshift, hashno; 3653 caddr_t saddr, eaddr, baseaddr; 3654 struct pa_hment *pahmep; 3655 struct sf_hment *sfhmep, *osfhmep; 3656 kmutex_t *pml; 3657 tte_t tte; 3658 page_t *pp, *rpp; 3659 int locked = 0; 3660 3661 if (IS_KPM_ADDR(vaddr)) 3662 return; 3663 3664 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3665 eaddr = saddr + len; 3666 3667 rehash: 3668 /* Find the mapping(s) for this page */ 3669 for (hashno = TTE64K, hmeblkp = NULL; 3670 hmeblkp == NULL && hashno <= mmu_hashcnt; 3671 hashno++) { 3672 hmeshift = HME_HASH_SHIFT(hashno); 3673 hblktag.htag_id = ksfmmup; 3674 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3675 hblktag.htag_rehash = hashno; 3676 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3677 3678 SFMMU_HASH_LOCK(hmebp); 3679 3680 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3681 3682 if (hmeblkp == NULL) 3683 SFMMU_HASH_UNLOCK(hmebp); 3684 } 3685 3686 if (hmeblkp == NULL) { 3687 if (!panicstr) { 3688 panic("hat_delete_callback: addr 0x%p not found", 3689 saddr); 3690 } 3691 return; 3692 } 3693 3694 baseaddr = (caddr_t)get_hblk_base(hmeblkp); 3695 HBLKTOHME(osfhmep, hmeblkp, saddr); 3696 3697 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3698 ASSERT(TTE_IS_VALID(&tte)); 3699 3700 pp = osfhmep->hme_page; 3701 if (pp == NULL || pp->p_vnode != &kvp) { 3702 SFMMU_HASH_UNLOCK(hmebp); 3703 return; 3704 } 3705 3706 pml = sfmmu_mlist_enter(pp); 3707 3708 if ((flags & HAC_PAGELOCK) && !locked) { 3709 if (!page_trylock(pp, SE_SHARED)) { 3710 /* 3711 * Somebody is holding SE_EXCL lock. Drop all 3712 * our locks, lookup the page in &kvp, and 3713 * retry. 3714 */ 3715 sfmmu_mlist_exit(pml); 3716 SFMMU_HASH_UNLOCK(hmebp); 3717 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3718 ASSERT(pp != NULL); 3719 rpp = PP_PAGEROOT(pp); 3720 if (rpp != pp) { 3721 page_unlock(pp); 3722 (void) page_lock(rpp, SE_SHARED, NULL, 3723 P_NO_RECLAIM); 3724 } 3725 locked = 1; 3726 goto rehash; 3727 } 3728 locked = 1; 3729 } 3730 3731 ASSERT(PAGE_LOCKED(pp)); 3732 3733 if (osfhmep->hme_page != pp || pp->p_vnode != &kvp || 3734 pp->p_offset < (u_offset_t)baseaddr || 3735 pp->p_offset > (u_offset_t)eaddr) { 3736 /* 3737 * The page moved before we got our hands on it. Drop 3738 * all the locks and try again. 3739 */ 3740 ASSERT((flags & HAC_PAGELOCK) != 0); 3741 sfmmu_mlist_exit(pml); 3742 SFMMU_HASH_UNLOCK(hmebp); 3743 page_unlock(pp); 3744 locked = 0; 3745 goto rehash; 3746 } 3747 3748 ASSERT(osfhmep->hme_page == pp); 3749 3750 for (sfhmep = pp->p_mapping; sfhmep != NULL; 3751 sfhmep = sfhmep->hme_next) { 3752 3753 /* 3754 * skip va<->pa mappings 3755 */ 3756 if (!IS_PAHME(sfhmep)) 3757 continue; 3758 3759 pahmep = sfhmep->hme_data; 3760 ASSERT(pahmep != NULL); 3761 3762 /* 3763 * if pa_hment matches, remove it 3764 */ 3765 if ((pahmep->pvt == pvt) && 3766 (pahmep->addr == vaddr) && 3767 (pahmep->len == len)) { 3768 break; 3769 } 3770 } 3771 3772 if (sfhmep == NULL) { 3773 if (!panicstr) { 3774 panic("hat_delete_callback: pa_hment not found, pp %p", 3775 (void *)pp); 3776 } 3777 return; 3778 } 3779 3780 /* 3781 * Note: at this point a valid kernel mapping must still be 3782 * present on this page. 3783 */ 3784 pp->p_share--; 3785 if (pp->p_share <= 0) 3786 panic("hat_delete_callback: zero p_share"); 3787 3788 if (--pahmep->refcnt == 0) { 3789 if (pahmep->flags != 0) 3790 panic("hat_delete_callback: pa_hment is busy"); 3791 3792 /* 3793 * Remove sfhmep from the mapping list for the page. 3794 */ 3795 if (sfhmep->hme_prev) { 3796 sfhmep->hme_prev->hme_next = sfhmep->hme_next; 3797 } else { 3798 pp->p_mapping = sfhmep->hme_next; 3799 } 3800 3801 if (sfhmep->hme_next) 3802 sfhmep->hme_next->hme_prev = sfhmep->hme_prev; 3803 3804 sfmmu_mlist_exit(pml); 3805 SFMMU_HASH_UNLOCK(hmebp); 3806 3807 if (locked) 3808 page_unlock(pp); 3809 3810 kmem_cache_free(pa_hment_cache, pahmep); 3811 return; 3812 } 3813 3814 sfmmu_mlist_exit(pml); 3815 SFMMU_HASH_UNLOCK(hmebp); 3816 if (locked) 3817 page_unlock(pp); 3818 } 3819 3820 /* 3821 * hat_probe returns 1 if the translation for the address 'addr' is 3822 * loaded, zero otherwise. 3823 * 3824 * hat_probe should be used only for advisorary purposes because it may 3825 * occasionally return the wrong value. The implementation must guarantee that 3826 * returning the wrong value is a very rare event. hat_probe is used 3827 * to implement optimizations in the segment drivers. 3828 * 3829 */ 3830 int 3831 hat_probe(struct hat *sfmmup, caddr_t addr) 3832 { 3833 pfn_t pfn; 3834 tte_t tte; 3835 3836 ASSERT(sfmmup != NULL); 3837 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3838 3839 ASSERT((sfmmup == ksfmmup) || 3840 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 3841 3842 if (sfmmup == ksfmmup) { 3843 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte)) 3844 == PFN_SUSPENDED) { 3845 sfmmu_vatopfn_suspended(addr, sfmmup, &tte); 3846 } 3847 } else { 3848 pfn = sfmmu_uvatopfn(addr, sfmmup); 3849 } 3850 3851 if (pfn != PFN_INVALID) 3852 return (1); 3853 else 3854 return (0); 3855 } 3856 3857 ssize_t 3858 hat_getpagesize(struct hat *sfmmup, caddr_t addr) 3859 { 3860 tte_t tte; 3861 3862 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3863 3864 sfmmu_gettte(sfmmup, addr, &tte); 3865 if (TTE_IS_VALID(&tte)) { 3866 return (TTEBYTES(TTE_CSZ(&tte))); 3867 } 3868 return (-1); 3869 } 3870 3871 static void 3872 sfmmu_gettte(struct hat *sfmmup, caddr_t addr, tte_t *ttep) 3873 { 3874 struct hmehash_bucket *hmebp; 3875 hmeblk_tag hblktag; 3876 int hmeshift, hashno = 1; 3877 struct hme_blk *hmeblkp, *list = NULL; 3878 struct sf_hment *sfhmep; 3879 3880 /* support for ISM */ 3881 ism_map_t *ism_map; 3882 ism_blk_t *ism_blkp; 3883 int i; 3884 sfmmu_t *ism_hatid = NULL; 3885 sfmmu_t *locked_hatid = NULL; 3886 3887 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 3888 3889 ism_blkp = sfmmup->sfmmu_iblk; 3890 if (ism_blkp) { 3891 sfmmu_ismhat_enter(sfmmup, 0); 3892 locked_hatid = sfmmup; 3893 } 3894 while (ism_blkp && ism_hatid == NULL) { 3895 ism_map = ism_blkp->iblk_maps; 3896 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 3897 if (addr >= ism_start(ism_map[i]) && 3898 addr < ism_end(ism_map[i])) { 3899 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 3900 addr = (caddr_t)(addr - 3901 ism_start(ism_map[i])); 3902 break; 3903 } 3904 } 3905 ism_blkp = ism_blkp->iblk_next; 3906 } 3907 if (locked_hatid) { 3908 sfmmu_ismhat_exit(locked_hatid, 0); 3909 } 3910 3911 hblktag.htag_id = sfmmup; 3912 ttep->ll = 0; 3913 3914 do { 3915 hmeshift = HME_HASH_SHIFT(hashno); 3916 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3917 hblktag.htag_rehash = hashno; 3918 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3919 3920 SFMMU_HASH_LOCK(hmebp); 3921 3922 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 3923 if (hmeblkp != NULL) { 3924 HBLKTOHME(sfhmep, hmeblkp, addr); 3925 sfmmu_copytte(&sfhmep->hme_tte, ttep); 3926 SFMMU_HASH_UNLOCK(hmebp); 3927 break; 3928 } 3929 SFMMU_HASH_UNLOCK(hmebp); 3930 hashno++; 3931 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 3932 3933 sfmmu_hblks_list_purge(&list); 3934 } 3935 3936 uint_t 3937 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr) 3938 { 3939 tte_t tte; 3940 3941 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3942 3943 sfmmu_gettte(sfmmup, addr, &tte); 3944 if (TTE_IS_VALID(&tte)) { 3945 *attr = sfmmu_ptov_attr(&tte); 3946 return (0); 3947 } 3948 *attr = 0; 3949 return ((uint_t)0xffffffff); 3950 } 3951 3952 /* 3953 * Enables more attributes on specified address range (ie. logical OR) 3954 */ 3955 void 3956 hat_setattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 3957 { 3958 if (hat->sfmmu_xhat_provider) { 3959 XHAT_SETATTR(hat, addr, len, attr); 3960 return; 3961 } else { 3962 /* 3963 * This must be a CPU HAT. If the address space has 3964 * XHATs attached, change attributes for all of them, 3965 * just in case 3966 */ 3967 ASSERT(hat->sfmmu_as != NULL); 3968 if (hat->sfmmu_as->a_xhat != NULL) 3969 xhat_setattr_all(hat->sfmmu_as, addr, len, attr); 3970 } 3971 3972 sfmmu_chgattr(hat, addr, len, attr, SFMMU_SETATTR); 3973 } 3974 3975 /* 3976 * Assigns attributes to the specified address range. All the attributes 3977 * are specified. 3978 */ 3979 void 3980 hat_chgattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 3981 { 3982 if (hat->sfmmu_xhat_provider) { 3983 XHAT_CHGATTR(hat, addr, len, attr); 3984 return; 3985 } else { 3986 /* 3987 * This must be a CPU HAT. If the address space has 3988 * XHATs attached, change attributes for all of them, 3989 * just in case 3990 */ 3991 ASSERT(hat->sfmmu_as != NULL); 3992 if (hat->sfmmu_as->a_xhat != NULL) 3993 xhat_chgattr_all(hat->sfmmu_as, addr, len, attr); 3994 } 3995 3996 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CHGATTR); 3997 } 3998 3999 /* 4000 * Remove attributes on the specified address range (ie. loginal NAND) 4001 */ 4002 void 4003 hat_clrattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4004 { 4005 if (hat->sfmmu_xhat_provider) { 4006 XHAT_CLRATTR(hat, addr, len, attr); 4007 return; 4008 } else { 4009 /* 4010 * This must be a CPU HAT. If the address space has 4011 * XHATs attached, change attributes for all of them, 4012 * just in case 4013 */ 4014 ASSERT(hat->sfmmu_as != NULL); 4015 if (hat->sfmmu_as->a_xhat != NULL) 4016 xhat_clrattr_all(hat->sfmmu_as, addr, len, attr); 4017 } 4018 4019 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CLRATTR); 4020 } 4021 4022 /* 4023 * Change attributes on an address range to that specified by attr and mode. 4024 */ 4025 static void 4026 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr, 4027 int mode) 4028 { 4029 struct hmehash_bucket *hmebp; 4030 hmeblk_tag hblktag; 4031 int hmeshift, hashno = 1; 4032 struct hme_blk *hmeblkp, *list = NULL; 4033 caddr_t endaddr; 4034 cpuset_t cpuset; 4035 demap_range_t dmr; 4036 4037 CPUSET_ZERO(cpuset); 4038 4039 ASSERT((sfmmup == ksfmmup) || 4040 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4041 ASSERT((len & MMU_PAGEOFFSET) == 0); 4042 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4043 4044 if ((attr & PROT_USER) && (mode != SFMMU_CLRATTR) && 4045 ((addr + len) > (caddr_t)USERLIMIT)) { 4046 panic("user addr %p in kernel space", 4047 (void *)addr); 4048 } 4049 4050 endaddr = addr + len; 4051 hblktag.htag_id = sfmmup; 4052 DEMAP_RANGE_INIT(sfmmup, &dmr); 4053 4054 while (addr < endaddr) { 4055 hmeshift = HME_HASH_SHIFT(hashno); 4056 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4057 hblktag.htag_rehash = hashno; 4058 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4059 4060 SFMMU_HASH_LOCK(hmebp); 4061 4062 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4063 if (hmeblkp != NULL) { 4064 /* 4065 * We've encountered a shadow hmeblk so skip the range 4066 * of the next smaller mapping size. 4067 */ 4068 if (hmeblkp->hblk_shw_bit) { 4069 ASSERT(sfmmup != ksfmmup); 4070 ASSERT(hashno > 1); 4071 addr = (caddr_t)P2END((uintptr_t)addr, 4072 TTEBYTES(hashno - 1)); 4073 } else { 4074 addr = sfmmu_hblk_chgattr(sfmmup, 4075 hmeblkp, addr, endaddr, &dmr, attr, mode); 4076 } 4077 SFMMU_HASH_UNLOCK(hmebp); 4078 hashno = 1; 4079 continue; 4080 } 4081 SFMMU_HASH_UNLOCK(hmebp); 4082 4083 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4084 /* 4085 * We have traversed the whole list and rehashed 4086 * if necessary without finding the address to chgattr. 4087 * This is ok, so we increment the address by the 4088 * smallest hmeblk range for kernel mappings or for 4089 * user mappings with no large pages, and the largest 4090 * hmeblk range, to account for shadow hmeblks, for 4091 * user mappings with large pages and continue. 4092 */ 4093 if (sfmmup == ksfmmup) 4094 addr = (caddr_t)P2END((uintptr_t)addr, 4095 TTEBYTES(1)); 4096 else 4097 addr = (caddr_t)P2END((uintptr_t)addr, 4098 TTEBYTES(hashno)); 4099 hashno = 1; 4100 } else { 4101 hashno++; 4102 } 4103 } 4104 4105 sfmmu_hblks_list_purge(&list); 4106 DEMAP_RANGE_FLUSH(&dmr); 4107 cpuset = sfmmup->sfmmu_cpusran; 4108 xt_sync(cpuset); 4109 } 4110 4111 /* 4112 * This function chgattr on a range of addresses in an hmeblk. It returns the 4113 * next addres that needs to be chgattr. 4114 * It should be called with the hash lock held. 4115 * XXX It should be possible to optimize chgattr by not flushing every time but 4116 * on the other hand: 4117 * 1. do one flush crosscall. 4118 * 2. only flush if we are increasing permissions (make sure this will work) 4119 */ 4120 static caddr_t 4121 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4122 caddr_t endaddr, demap_range_t *dmrp, uint_t attr, int mode) 4123 { 4124 tte_t tte, tteattr, tteflags, ttemod; 4125 struct sf_hment *sfhmep; 4126 int ttesz; 4127 struct page *pp = NULL; 4128 kmutex_t *pml, *pmtx; 4129 int ret; 4130 int use_demap_range; 4131 #if defined(SF_ERRATA_57) 4132 int check_exec; 4133 #endif 4134 4135 ASSERT(in_hblk_range(hmeblkp, addr)); 4136 ASSERT(hmeblkp->hblk_shw_bit == 0); 4137 4138 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4139 ttesz = get_hblk_ttesz(hmeblkp); 4140 4141 /* 4142 * Flush the current demap region if addresses have been 4143 * skipped or the page size doesn't match. 4144 */ 4145 use_demap_range = (TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp)); 4146 if (use_demap_range) { 4147 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4148 } else { 4149 DEMAP_RANGE_FLUSH(dmrp); 4150 } 4151 4152 tteattr.ll = sfmmu_vtop_attr(attr, mode, &tteflags); 4153 #if defined(SF_ERRATA_57) 4154 check_exec = (sfmmup != ksfmmup) && 4155 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4156 TTE_IS_EXECUTABLE(&tteattr); 4157 #endif 4158 HBLKTOHME(sfhmep, hmeblkp, addr); 4159 while (addr < endaddr) { 4160 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4161 if (TTE_IS_VALID(&tte)) { 4162 if ((tte.ll & tteflags.ll) == tteattr.ll) { 4163 /* 4164 * if the new attr is the same as old 4165 * continue 4166 */ 4167 goto next_addr; 4168 } 4169 if (!TTE_IS_WRITABLE(&tteattr)) { 4170 /* 4171 * make sure we clear hw modify bit if we 4172 * removing write protections 4173 */ 4174 tteflags.tte_intlo |= TTE_HWWR_INT; 4175 } 4176 4177 pml = NULL; 4178 pp = sfhmep->hme_page; 4179 if (pp) { 4180 pml = sfmmu_mlist_enter(pp); 4181 } 4182 4183 if (pp != sfhmep->hme_page) { 4184 /* 4185 * tte must have been unloaded. 4186 */ 4187 ASSERT(pml); 4188 sfmmu_mlist_exit(pml); 4189 continue; 4190 } 4191 4192 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4193 4194 ttemod = tte; 4195 ttemod.ll = (ttemod.ll & ~tteflags.ll) | tteattr.ll; 4196 ASSERT(TTE_TO_TTEPFN(&ttemod) == TTE_TO_TTEPFN(&tte)); 4197 4198 #if defined(SF_ERRATA_57) 4199 if (check_exec && addr < errata57_limit) 4200 ttemod.tte_exec_perm = 0; 4201 #endif 4202 ret = sfmmu_modifytte_try(&tte, &ttemod, 4203 &sfhmep->hme_tte); 4204 4205 if (ret < 0) { 4206 /* tte changed underneath us */ 4207 if (pml) { 4208 sfmmu_mlist_exit(pml); 4209 } 4210 continue; 4211 } 4212 4213 if (tteflags.tte_intlo & TTE_HWWR_INT) { 4214 /* 4215 * need to sync if we are clearing modify bit. 4216 */ 4217 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4218 } 4219 4220 if (pp && PP_ISRO(pp)) { 4221 if (tteattr.tte_intlo & TTE_WRPRM_INT) { 4222 pmtx = sfmmu_page_enter(pp); 4223 PP_CLRRO(pp); 4224 sfmmu_page_exit(pmtx); 4225 } 4226 } 4227 4228 if (ret > 0 && use_demap_range) { 4229 DEMAP_RANGE_MARKPG(dmrp, addr); 4230 } else if (ret > 0) { 4231 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4232 } 4233 4234 if (pml) { 4235 sfmmu_mlist_exit(pml); 4236 } 4237 } 4238 next_addr: 4239 addr += TTEBYTES(ttesz); 4240 sfhmep++; 4241 DEMAP_RANGE_NEXTPG(dmrp); 4242 } 4243 return (addr); 4244 } 4245 4246 /* 4247 * This routine converts virtual attributes to physical ones. It will 4248 * update the tteflags field with the tte mask corresponding to the attributes 4249 * affected and it returns the new attributes. It will also clear the modify 4250 * bit if we are taking away write permission. This is necessary since the 4251 * modify bit is the hardware permission bit and we need to clear it in order 4252 * to detect write faults. 4253 */ 4254 static uint64_t 4255 sfmmu_vtop_attr(uint_t attr, int mode, tte_t *ttemaskp) 4256 { 4257 tte_t ttevalue; 4258 4259 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 4260 4261 switch (mode) { 4262 case SFMMU_CHGATTR: 4263 /* all attributes specified */ 4264 ttevalue.tte_inthi = MAKE_TTEATTR_INTHI(attr); 4265 ttevalue.tte_intlo = MAKE_TTEATTR_INTLO(attr); 4266 ttemaskp->tte_inthi = TTEINTHI_ATTR; 4267 ttemaskp->tte_intlo = TTEINTLO_ATTR; 4268 break; 4269 case SFMMU_SETATTR: 4270 ASSERT(!(attr & ~HAT_PROT_MASK)); 4271 ttemaskp->ll = 0; 4272 ttevalue.ll = 0; 4273 /* 4274 * a valid tte implies exec and read for sfmmu 4275 * so no need to do anything about them. 4276 * since priviledged access implies user access 4277 * PROT_USER doesn't make sense either. 4278 */ 4279 if (attr & PROT_WRITE) { 4280 ttemaskp->tte_intlo |= TTE_WRPRM_INT; 4281 ttevalue.tte_intlo |= TTE_WRPRM_INT; 4282 } 4283 break; 4284 case SFMMU_CLRATTR: 4285 /* attributes will be nand with current ones */ 4286 if (attr & ~(PROT_WRITE | PROT_USER)) { 4287 panic("sfmmu: attr %x not supported", attr); 4288 } 4289 ttemaskp->ll = 0; 4290 ttevalue.ll = 0; 4291 if (attr & PROT_WRITE) { 4292 /* clear both writable and modify bit */ 4293 ttemaskp->tte_intlo |= TTE_WRPRM_INT | TTE_HWWR_INT; 4294 } 4295 if (attr & PROT_USER) { 4296 ttemaskp->tte_intlo |= TTE_PRIV_INT; 4297 ttevalue.tte_intlo |= TTE_PRIV_INT; 4298 } 4299 break; 4300 default: 4301 panic("sfmmu_vtop_attr: bad mode %x", mode); 4302 } 4303 ASSERT(TTE_TO_TTEPFN(&ttevalue) == 0); 4304 return (ttevalue.ll); 4305 } 4306 4307 static uint_t 4308 sfmmu_ptov_attr(tte_t *ttep) 4309 { 4310 uint_t attr; 4311 4312 ASSERT(TTE_IS_VALID(ttep)); 4313 4314 attr = PROT_READ; 4315 4316 if (TTE_IS_WRITABLE(ttep)) { 4317 attr |= PROT_WRITE; 4318 } 4319 if (TTE_IS_EXECUTABLE(ttep)) { 4320 attr |= PROT_EXEC; 4321 } 4322 if (!TTE_IS_PRIVILEGED(ttep)) { 4323 attr |= PROT_USER; 4324 } 4325 if (TTE_IS_NFO(ttep)) { 4326 attr |= HAT_NOFAULT; 4327 } 4328 if (TTE_IS_NOSYNC(ttep)) { 4329 attr |= HAT_NOSYNC; 4330 } 4331 if (TTE_IS_SIDEFFECT(ttep)) { 4332 attr |= SFMMU_SIDEFFECT; 4333 } 4334 if (!TTE_IS_VCACHEABLE(ttep)) { 4335 attr |= SFMMU_UNCACHEVTTE; 4336 } 4337 if (!TTE_IS_PCACHEABLE(ttep)) { 4338 attr |= SFMMU_UNCACHEPTTE; 4339 } 4340 return (attr); 4341 } 4342 4343 /* 4344 * hat_chgprot is a deprecated hat call. New segment drivers 4345 * should store all attributes and use hat_*attr calls. 4346 * 4347 * Change the protections in the virtual address range 4348 * given to the specified virtual protection. If vprot is ~PROT_WRITE, 4349 * then remove write permission, leaving the other 4350 * permissions unchanged. If vprot is ~PROT_USER, remove user permissions. 4351 * 4352 */ 4353 void 4354 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot) 4355 { 4356 struct hmehash_bucket *hmebp; 4357 hmeblk_tag hblktag; 4358 int hmeshift, hashno = 1; 4359 struct hme_blk *hmeblkp, *list = NULL; 4360 caddr_t endaddr; 4361 cpuset_t cpuset; 4362 demap_range_t dmr; 4363 4364 ASSERT((len & MMU_PAGEOFFSET) == 0); 4365 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4366 4367 if (sfmmup->sfmmu_xhat_provider) { 4368 XHAT_CHGPROT(sfmmup, addr, len, vprot); 4369 return; 4370 } else { 4371 /* 4372 * This must be a CPU HAT. If the address space has 4373 * XHATs attached, change attributes for all of them, 4374 * just in case 4375 */ 4376 ASSERT(sfmmup->sfmmu_as != NULL); 4377 if (sfmmup->sfmmu_as->a_xhat != NULL) 4378 xhat_chgprot_all(sfmmup->sfmmu_as, addr, len, vprot); 4379 } 4380 4381 CPUSET_ZERO(cpuset); 4382 4383 if ((vprot != (uint_t)~PROT_WRITE) && (vprot & PROT_USER) && 4384 ((addr + len) > (caddr_t)USERLIMIT)) { 4385 panic("user addr %p vprot %x in kernel space", 4386 (void *)addr, vprot); 4387 } 4388 endaddr = addr + len; 4389 hblktag.htag_id = sfmmup; 4390 DEMAP_RANGE_INIT(sfmmup, &dmr); 4391 4392 while (addr < endaddr) { 4393 hmeshift = HME_HASH_SHIFT(hashno); 4394 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4395 hblktag.htag_rehash = hashno; 4396 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4397 4398 SFMMU_HASH_LOCK(hmebp); 4399 4400 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4401 if (hmeblkp != NULL) { 4402 /* 4403 * We've encountered a shadow hmeblk so skip the range 4404 * of the next smaller mapping size. 4405 */ 4406 if (hmeblkp->hblk_shw_bit) { 4407 ASSERT(sfmmup != ksfmmup); 4408 ASSERT(hashno > 1); 4409 addr = (caddr_t)P2END((uintptr_t)addr, 4410 TTEBYTES(hashno - 1)); 4411 } else { 4412 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp, 4413 addr, endaddr, &dmr, vprot); 4414 } 4415 SFMMU_HASH_UNLOCK(hmebp); 4416 hashno = 1; 4417 continue; 4418 } 4419 SFMMU_HASH_UNLOCK(hmebp); 4420 4421 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4422 /* 4423 * We have traversed the whole list and rehashed 4424 * if necessary without finding the address to chgprot. 4425 * This is ok so we increment the address by the 4426 * smallest hmeblk range for kernel mappings and the 4427 * largest hmeblk range, to account for shadow hmeblks, 4428 * for user mappings and continue. 4429 */ 4430 if (sfmmup == ksfmmup) 4431 addr = (caddr_t)P2END((uintptr_t)addr, 4432 TTEBYTES(1)); 4433 else 4434 addr = (caddr_t)P2END((uintptr_t)addr, 4435 TTEBYTES(hashno)); 4436 hashno = 1; 4437 } else { 4438 hashno++; 4439 } 4440 } 4441 4442 sfmmu_hblks_list_purge(&list); 4443 DEMAP_RANGE_FLUSH(&dmr); 4444 cpuset = sfmmup->sfmmu_cpusran; 4445 xt_sync(cpuset); 4446 } 4447 4448 /* 4449 * This function chgprots a range of addresses in an hmeblk. It returns the 4450 * next addres that needs to be chgprot. 4451 * It should be called with the hash lock held. 4452 * XXX It shold be possible to optimize chgprot by not flushing every time but 4453 * on the other hand: 4454 * 1. do one flush crosscall. 4455 * 2. only flush if we are increasing permissions (make sure this will work) 4456 */ 4457 static caddr_t 4458 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4459 caddr_t endaddr, demap_range_t *dmrp, uint_t vprot) 4460 { 4461 uint_t pprot; 4462 tte_t tte, ttemod; 4463 struct sf_hment *sfhmep; 4464 uint_t tteflags; 4465 int ttesz; 4466 struct page *pp = NULL; 4467 kmutex_t *pml, *pmtx; 4468 int ret; 4469 int use_demap_range; 4470 #if defined(SF_ERRATA_57) 4471 int check_exec; 4472 #endif 4473 4474 ASSERT(in_hblk_range(hmeblkp, addr)); 4475 ASSERT(hmeblkp->hblk_shw_bit == 0); 4476 4477 #ifdef DEBUG 4478 if (get_hblk_ttesz(hmeblkp) != TTE8K && 4479 (endaddr < get_hblk_endaddr(hmeblkp))) { 4480 panic("sfmmu_hblk_chgprot: partial chgprot of large page"); 4481 } 4482 #endif /* DEBUG */ 4483 4484 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4485 ttesz = get_hblk_ttesz(hmeblkp); 4486 4487 pprot = sfmmu_vtop_prot(vprot, &tteflags); 4488 #if defined(SF_ERRATA_57) 4489 check_exec = (sfmmup != ksfmmup) && 4490 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4491 ((vprot & PROT_EXEC) == PROT_EXEC); 4492 #endif 4493 HBLKTOHME(sfhmep, hmeblkp, addr); 4494 4495 /* 4496 * Flush the current demap region if addresses have been 4497 * skipped or the page size doesn't match. 4498 */ 4499 use_demap_range = (TTEBYTES(ttesz) == MMU_PAGESIZE); 4500 if (use_demap_range) { 4501 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4502 } else { 4503 DEMAP_RANGE_FLUSH(dmrp); 4504 } 4505 4506 while (addr < endaddr) { 4507 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4508 if (TTE_IS_VALID(&tte)) { 4509 if (TTE_GET_LOFLAGS(&tte, tteflags) == pprot) { 4510 /* 4511 * if the new protection is the same as old 4512 * continue 4513 */ 4514 goto next_addr; 4515 } 4516 pml = NULL; 4517 pp = sfhmep->hme_page; 4518 if (pp) { 4519 pml = sfmmu_mlist_enter(pp); 4520 } 4521 if (pp != sfhmep->hme_page) { 4522 /* 4523 * tte most have been unloaded 4524 * underneath us. Recheck 4525 */ 4526 ASSERT(pml); 4527 sfmmu_mlist_exit(pml); 4528 continue; 4529 } 4530 4531 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4532 4533 ttemod = tte; 4534 TTE_SET_LOFLAGS(&ttemod, tteflags, pprot); 4535 #if defined(SF_ERRATA_57) 4536 if (check_exec && addr < errata57_limit) 4537 ttemod.tte_exec_perm = 0; 4538 #endif 4539 ret = sfmmu_modifytte_try(&tte, &ttemod, 4540 &sfhmep->hme_tte); 4541 4542 if (ret < 0) { 4543 /* tte changed underneath us */ 4544 if (pml) { 4545 sfmmu_mlist_exit(pml); 4546 } 4547 continue; 4548 } 4549 4550 if (tteflags & TTE_HWWR_INT) { 4551 /* 4552 * need to sync if we are clearing modify bit. 4553 */ 4554 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4555 } 4556 4557 if (pp && PP_ISRO(pp)) { 4558 if (pprot & TTE_WRPRM_INT) { 4559 pmtx = sfmmu_page_enter(pp); 4560 PP_CLRRO(pp); 4561 sfmmu_page_exit(pmtx); 4562 } 4563 } 4564 4565 if (ret > 0 && use_demap_range) { 4566 DEMAP_RANGE_MARKPG(dmrp, addr); 4567 } else if (ret > 0) { 4568 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4569 } 4570 4571 if (pml) { 4572 sfmmu_mlist_exit(pml); 4573 } 4574 } 4575 next_addr: 4576 addr += TTEBYTES(ttesz); 4577 sfhmep++; 4578 DEMAP_RANGE_NEXTPG(dmrp); 4579 } 4580 return (addr); 4581 } 4582 4583 /* 4584 * This routine is deprecated and should only be used by hat_chgprot. 4585 * The correct routine is sfmmu_vtop_attr. 4586 * This routine converts virtual page protections to physical ones. It will 4587 * update the tteflags field with the tte mask corresponding to the protections 4588 * affected and it returns the new protections. It will also clear the modify 4589 * bit if we are taking away write permission. This is necessary since the 4590 * modify bit is the hardware permission bit and we need to clear it in order 4591 * to detect write faults. 4592 * It accepts the following special protections: 4593 * ~PROT_WRITE = remove write permissions. 4594 * ~PROT_USER = remove user permissions. 4595 */ 4596 static uint_t 4597 sfmmu_vtop_prot(uint_t vprot, uint_t *tteflagsp) 4598 { 4599 if (vprot == (uint_t)~PROT_WRITE) { 4600 *tteflagsp = TTE_WRPRM_INT | TTE_HWWR_INT; 4601 return (0); /* will cause wrprm to be cleared */ 4602 } 4603 if (vprot == (uint_t)~PROT_USER) { 4604 *tteflagsp = TTE_PRIV_INT; 4605 return (0); /* will cause privprm to be cleared */ 4606 } 4607 if ((vprot == 0) || (vprot == PROT_USER) || 4608 ((vprot & PROT_ALL) != vprot)) { 4609 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4610 } 4611 4612 switch (vprot) { 4613 case (PROT_READ): 4614 case (PROT_EXEC): 4615 case (PROT_EXEC | PROT_READ): 4616 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4617 return (TTE_PRIV_INT); /* set prv and clr wrt */ 4618 case (PROT_WRITE): 4619 case (PROT_WRITE | PROT_READ): 4620 case (PROT_EXEC | PROT_WRITE): 4621 case (PROT_EXEC | PROT_WRITE | PROT_READ): 4622 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4623 return (TTE_PRIV_INT | TTE_WRPRM_INT); /* set prv and wrt */ 4624 case (PROT_USER | PROT_READ): 4625 case (PROT_USER | PROT_EXEC): 4626 case (PROT_USER | PROT_EXEC | PROT_READ): 4627 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4628 return (0); /* clr prv and wrt */ 4629 case (PROT_USER | PROT_WRITE): 4630 case (PROT_USER | PROT_WRITE | PROT_READ): 4631 case (PROT_USER | PROT_EXEC | PROT_WRITE): 4632 case (PROT_USER | PROT_EXEC | PROT_WRITE | PROT_READ): 4633 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4634 return (TTE_WRPRM_INT); /* clr prv and set wrt */ 4635 default: 4636 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4637 } 4638 return (0); 4639 } 4640 4641 /* 4642 * Alternate unload for very large virtual ranges. With a true 64 bit VA, 4643 * the normal algorithm would take too long for a very large VA range with 4644 * few real mappings. This routine just walks thru all HMEs in the global 4645 * hash table to find and remove mappings. 4646 */ 4647 static void 4648 hat_unload_large_virtual( 4649 struct hat *sfmmup, 4650 caddr_t startaddr, 4651 size_t len, 4652 uint_t flags, 4653 hat_callback_t *callback) 4654 { 4655 struct hmehash_bucket *hmebp; 4656 struct hme_blk *hmeblkp; 4657 struct hme_blk *pr_hblk = NULL; 4658 struct hme_blk *nx_hblk; 4659 struct hme_blk *list = NULL; 4660 int i; 4661 uint64_t hblkpa, prevpa, nx_pa; 4662 hatlock_t *hatlockp; 4663 struct tsb_info *tsbinfop; 4664 struct ctx *ctx; 4665 caddr_t endaddr = startaddr + len; 4666 caddr_t sa; 4667 caddr_t ea; 4668 caddr_t cb_sa[MAX_CB_ADDR]; 4669 caddr_t cb_ea[MAX_CB_ADDR]; 4670 int addr_cnt = 0; 4671 int a = 0; 4672 int cnum; 4673 4674 hatlockp = sfmmu_hat_enter(sfmmup); 4675 4676 /* 4677 * Since we know we're unmapping a huge range of addresses, 4678 * just throw away the context and switch to another. It's 4679 * cheaper than trying to unmap all of the TTEs we may find 4680 * from the TLB individually, which is too expensive in terms 4681 * of xcalls. Better yet, if we're exiting, no need to flush 4682 * anything at all! 4683 */ 4684 if (!sfmmup->sfmmu_free) { 4685 ctx = sfmmutoctx(sfmmup); 4686 rw_enter(&ctx->ctx_rwlock, RW_WRITER); 4687 cnum = sfmmutoctxnum(sfmmup); 4688 if (cnum != INVALID_CONTEXT) { 4689 sfmmu_tlb_swap_ctx(sfmmup, ctx); 4690 } 4691 rw_exit(&ctx->ctx_rwlock); 4692 4693 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 4694 tsbinfop = tsbinfop->tsb_next) { 4695 if (tsbinfop->tsb_flags & TSB_SWAPPED) 4696 continue; 4697 sfmmu_inv_tsb(tsbinfop->tsb_va, 4698 TSB_BYTES(tsbinfop->tsb_szc)); 4699 } 4700 } 4701 4702 /* 4703 * Loop through all the hash buckets of HME blocks looking for matches. 4704 */ 4705 for (i = 0; i <= UHMEHASH_SZ; i++) { 4706 hmebp = &uhme_hash[i]; 4707 SFMMU_HASH_LOCK(hmebp); 4708 hmeblkp = hmebp->hmeblkp; 4709 hblkpa = hmebp->hmeh_nextpa; 4710 prevpa = 0; 4711 pr_hblk = NULL; 4712 while (hmeblkp) { 4713 nx_hblk = hmeblkp->hblk_next; 4714 nx_pa = hmeblkp->hblk_nextpa; 4715 4716 /* 4717 * skip if not this context, if a shadow block or 4718 * if the mapping is not in the requested range 4719 */ 4720 if (hmeblkp->hblk_tag.htag_id != sfmmup || 4721 hmeblkp->hblk_shw_bit || 4722 (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr || 4723 (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) { 4724 pr_hblk = hmeblkp; 4725 prevpa = hblkpa; 4726 goto next_block; 4727 } 4728 4729 /* 4730 * unload if there are any current valid mappings 4731 */ 4732 if (hmeblkp->hblk_vcnt != 0 || 4733 hmeblkp->hblk_hmecnt != 0) 4734 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 4735 sa, ea, NULL, flags); 4736 4737 /* 4738 * on unmap we also release the HME block itself, once 4739 * all mappings are gone. 4740 */ 4741 if ((flags & HAT_UNLOAD_UNMAP) != 0 && 4742 !hmeblkp->hblk_vcnt && 4743 !hmeblkp->hblk_hmecnt) { 4744 ASSERT(!hmeblkp->hblk_lckcnt); 4745 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 4746 prevpa, pr_hblk); 4747 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 4748 } else { 4749 pr_hblk = hmeblkp; 4750 prevpa = hblkpa; 4751 } 4752 4753 if (callback == NULL) 4754 goto next_block; 4755 4756 /* 4757 * HME blocks may span more than one page, but we may be 4758 * unmapping only one page, so check for a smaller range 4759 * for the callback 4760 */ 4761 if (sa < startaddr) 4762 sa = startaddr; 4763 if (--ea > endaddr) 4764 ea = endaddr - 1; 4765 4766 cb_sa[addr_cnt] = sa; 4767 cb_ea[addr_cnt] = ea; 4768 if (++addr_cnt == MAX_CB_ADDR) { 4769 for (a = 0; a < MAX_CB_ADDR; ++a) { 4770 callback->hcb_start_addr = cb_sa[a]; 4771 callback->hcb_end_addr = cb_ea[a]; 4772 callback->hcb_function(callback); 4773 } 4774 addr_cnt = 0; 4775 } 4776 4777 next_block: 4778 hmeblkp = nx_hblk; 4779 hblkpa = nx_pa; 4780 } 4781 SFMMU_HASH_UNLOCK(hmebp); 4782 } 4783 4784 sfmmu_hblks_list_purge(&list); 4785 4786 for (a = 0; a < addr_cnt; ++a) { 4787 callback->hcb_start_addr = cb_sa[a]; 4788 callback->hcb_end_addr = cb_ea[a]; 4789 callback->hcb_function(callback); 4790 } 4791 4792 sfmmu_hat_exit(hatlockp); 4793 4794 /* 4795 * Check TSB and TLB page sizes if the process isn't exiting. 4796 */ 4797 if (!sfmmup->sfmmu_free) 4798 sfmmu_check_page_sizes(sfmmup, 0); 4799 } 4800 4801 4802 /* 4803 * Unload all the mappings in the range [addr..addr+len). addr and len must 4804 * be MMU_PAGESIZE aligned. 4805 */ 4806 4807 extern struct seg *segkmap; 4808 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \ 4809 segkmap->s_base <= (addr) && (addr) < (segkmap->s_base + segkmap->s_size)) 4810 4811 4812 void 4813 hat_unload_callback( 4814 struct hat *sfmmup, 4815 caddr_t addr, 4816 size_t len, 4817 uint_t flags, 4818 hat_callback_t *callback) 4819 { 4820 struct hmehash_bucket *hmebp; 4821 hmeblk_tag hblktag; 4822 int hmeshift, hashno, iskernel; 4823 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 4824 caddr_t endaddr; 4825 cpuset_t cpuset; 4826 uint64_t hblkpa, prevpa; 4827 int addr_count = 0; 4828 int a; 4829 caddr_t cb_start_addr[MAX_CB_ADDR]; 4830 caddr_t cb_end_addr[MAX_CB_ADDR]; 4831 int issegkmap = ISSEGKMAP(sfmmup, addr); 4832 demap_range_t dmr, *dmrp; 4833 4834 if (sfmmup->sfmmu_xhat_provider) { 4835 XHAT_UNLOAD_CALLBACK(sfmmup, addr, len, flags, callback); 4836 return; 4837 } else { 4838 /* 4839 * This must be a CPU HAT. If the address space has 4840 * XHATs attached, unload the mappings for all of them, 4841 * just in case 4842 */ 4843 ASSERT(sfmmup->sfmmu_as != NULL); 4844 if (sfmmup->sfmmu_as->a_xhat != NULL) 4845 xhat_unload_callback_all(sfmmup->sfmmu_as, addr, 4846 len, flags, callback); 4847 } 4848 4849 ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \ 4850 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4851 4852 ASSERT(sfmmup != NULL); 4853 ASSERT((len & MMU_PAGEOFFSET) == 0); 4854 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 4855 4856 /* 4857 * Probing through a large VA range (say 63 bits) will be slow, even 4858 * at 4 Meg steps between the probes. So, when the virtual address range 4859 * is very large, search the HME entries for what to unload. 4860 * 4861 * len >> TTE_PAGE_SHIFT(TTE4M) is the # of 4Meg probes we'd need 4862 * 4863 * UHMEHASH_SZ is number of hash buckets to examine 4864 * 4865 */ 4866 if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) { 4867 hat_unload_large_virtual(sfmmup, addr, len, flags, callback); 4868 return; 4869 } 4870 4871 CPUSET_ZERO(cpuset); 4872 4873 /* 4874 * If the process is exiting, we can save a lot of fuss since 4875 * we'll flush the TLB when we free the ctx anyway. 4876 */ 4877 if (sfmmup->sfmmu_free) 4878 dmrp = NULL; 4879 else 4880 dmrp = &dmr; 4881 4882 DEMAP_RANGE_INIT(sfmmup, dmrp); 4883 endaddr = addr + len; 4884 hblktag.htag_id = sfmmup; 4885 4886 /* 4887 * It is likely for the vm to call unload over a wide range of 4888 * addresses that are actually very sparsely populated by 4889 * translations. In order to speed this up the sfmmu hat supports 4890 * the concept of shadow hmeblks. Dummy large page hmeblks that 4891 * correspond to actual small translations are allocated at tteload 4892 * time and are referred to as shadow hmeblks. Now, during unload 4893 * time, we first check if we have a shadow hmeblk for that 4894 * translation. The absence of one means the corresponding address 4895 * range is empty and can be skipped. 4896 * 4897 * The kernel is an exception to above statement and that is why 4898 * we don't use shadow hmeblks and hash starting from the smallest 4899 * page size. 4900 */ 4901 if (sfmmup == KHATID) { 4902 iskernel = 1; 4903 hashno = TTE64K; 4904 } else { 4905 iskernel = 0; 4906 if (mmu_page_sizes == max_mmu_page_sizes) { 4907 hashno = TTE256M; 4908 } else { 4909 hashno = TTE4M; 4910 } 4911 } 4912 while (addr < endaddr) { 4913 hmeshift = HME_HASH_SHIFT(hashno); 4914 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4915 hblktag.htag_rehash = hashno; 4916 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4917 4918 SFMMU_HASH_LOCK(hmebp); 4919 4920 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, pr_hblk, 4921 prevpa, &list); 4922 if (hmeblkp == NULL) { 4923 /* 4924 * didn't find an hmeblk. skip the appropiate 4925 * address range. 4926 */ 4927 SFMMU_HASH_UNLOCK(hmebp); 4928 if (iskernel) { 4929 if (hashno < mmu_hashcnt) { 4930 hashno++; 4931 continue; 4932 } else { 4933 hashno = TTE64K; 4934 addr = (caddr_t)roundup((uintptr_t)addr 4935 + 1, MMU_PAGESIZE64K); 4936 continue; 4937 } 4938 } 4939 addr = (caddr_t)roundup((uintptr_t)addr + 1, 4940 (1 << hmeshift)); 4941 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 4942 ASSERT(hashno == TTE64K); 4943 continue; 4944 } 4945 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 4946 hashno = TTE512K; 4947 continue; 4948 } 4949 if (mmu_page_sizes == max_mmu_page_sizes) { 4950 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 4951 hashno = TTE4M; 4952 continue; 4953 } 4954 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 4955 hashno = TTE32M; 4956 continue; 4957 } 4958 hashno = TTE256M; 4959 continue; 4960 } else { 4961 hashno = TTE4M; 4962 continue; 4963 } 4964 } 4965 ASSERT(hmeblkp); 4966 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 4967 /* 4968 * If the valid count is zero we can skip the range 4969 * mapped by this hmeblk. 4970 * We free hblks in the case of HAT_UNMAP. HAT_UNMAP 4971 * is used by segment drivers as a hint 4972 * that the mapping resource won't be used any longer. 4973 * The best example of this is during exit(). 4974 */ 4975 addr = (caddr_t)roundup((uintptr_t)addr + 1, 4976 get_hblk_span(hmeblkp)); 4977 if ((flags & HAT_UNLOAD_UNMAP) || 4978 (iskernel && !issegkmap)) { 4979 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 4980 pr_hblk); 4981 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 4982 } 4983 SFMMU_HASH_UNLOCK(hmebp); 4984 4985 if (iskernel) { 4986 hashno = TTE64K; 4987 continue; 4988 } 4989 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 4990 ASSERT(hashno == TTE64K); 4991 continue; 4992 } 4993 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 4994 hashno = TTE512K; 4995 continue; 4996 } 4997 if (mmu_page_sizes == max_mmu_page_sizes) { 4998 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 4999 hashno = TTE4M; 5000 continue; 5001 } 5002 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5003 hashno = TTE32M; 5004 continue; 5005 } 5006 hashno = TTE256M; 5007 continue; 5008 } else { 5009 hashno = TTE4M; 5010 continue; 5011 } 5012 } 5013 if (hmeblkp->hblk_shw_bit) { 5014 /* 5015 * If we encounter a shadow hmeblk we know there is 5016 * smaller sized hmeblks mapping the same address space. 5017 * Decrement the hash size and rehash. 5018 */ 5019 ASSERT(sfmmup != KHATID); 5020 hashno--; 5021 SFMMU_HASH_UNLOCK(hmebp); 5022 continue; 5023 } 5024 5025 /* 5026 * track callback address ranges. 5027 * only start a new range when it's not contiguous 5028 */ 5029 if (callback != NULL) { 5030 if (addr_count > 0 && 5031 addr == cb_end_addr[addr_count - 1]) 5032 --addr_count; 5033 else 5034 cb_start_addr[addr_count] = addr; 5035 } 5036 5037 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr, 5038 dmrp, flags); 5039 5040 if (callback != NULL) 5041 cb_end_addr[addr_count++] = addr; 5042 5043 if (((flags & HAT_UNLOAD_UNMAP) || (iskernel && !issegkmap)) && 5044 !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5045 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5046 pr_hblk); 5047 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5048 } 5049 SFMMU_HASH_UNLOCK(hmebp); 5050 5051 /* 5052 * Notify our caller as to exactly which pages 5053 * have been unloaded. We do these in clumps, 5054 * to minimize the number of xt_sync()s that need to occur. 5055 */ 5056 if (callback != NULL && addr_count == MAX_CB_ADDR) { 5057 DEMAP_RANGE_FLUSH(dmrp); 5058 if (dmrp != NULL) { 5059 cpuset = sfmmup->sfmmu_cpusran; 5060 xt_sync(cpuset); 5061 } 5062 5063 for (a = 0; a < MAX_CB_ADDR; ++a) { 5064 callback->hcb_start_addr = cb_start_addr[a]; 5065 callback->hcb_end_addr = cb_end_addr[a]; 5066 callback->hcb_function(callback); 5067 } 5068 addr_count = 0; 5069 } 5070 if (iskernel) { 5071 hashno = TTE64K; 5072 continue; 5073 } 5074 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5075 ASSERT(hashno == TTE64K); 5076 continue; 5077 } 5078 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5079 hashno = TTE512K; 5080 continue; 5081 } 5082 if (mmu_page_sizes == max_mmu_page_sizes) { 5083 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5084 hashno = TTE4M; 5085 continue; 5086 } 5087 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5088 hashno = TTE32M; 5089 continue; 5090 } 5091 hashno = TTE256M; 5092 } else { 5093 hashno = TTE4M; 5094 } 5095 } 5096 5097 sfmmu_hblks_list_purge(&list); 5098 DEMAP_RANGE_FLUSH(dmrp); 5099 if (dmrp != NULL) { 5100 cpuset = sfmmup->sfmmu_cpusran; 5101 xt_sync(cpuset); 5102 } 5103 if (callback && addr_count != 0) { 5104 for (a = 0; a < addr_count; ++a) { 5105 callback->hcb_start_addr = cb_start_addr[a]; 5106 callback->hcb_end_addr = cb_end_addr[a]; 5107 callback->hcb_function(callback); 5108 } 5109 } 5110 5111 /* 5112 * Check TSB and TLB page sizes if the process isn't exiting. 5113 */ 5114 if (!sfmmup->sfmmu_free) 5115 sfmmu_check_page_sizes(sfmmup, 0); 5116 } 5117 5118 /* 5119 * Unload all the mappings in the range [addr..addr+len). addr and len must 5120 * be MMU_PAGESIZE aligned. 5121 */ 5122 void 5123 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags) 5124 { 5125 if (sfmmup->sfmmu_xhat_provider) { 5126 XHAT_UNLOAD(sfmmup, addr, len, flags); 5127 return; 5128 } 5129 hat_unload_callback(sfmmup, addr, len, flags, NULL); 5130 } 5131 5132 5133 /* 5134 * Find the largest mapping size for this page. 5135 */ 5136 static int 5137 fnd_mapping_sz(page_t *pp) 5138 { 5139 int sz; 5140 int p_index; 5141 5142 p_index = PP_MAPINDEX(pp); 5143 5144 sz = 0; 5145 p_index >>= 1; /* don't care about 8K bit */ 5146 for (; p_index; p_index >>= 1) { 5147 sz++; 5148 } 5149 5150 return (sz); 5151 } 5152 5153 /* 5154 * This function unloads a range of addresses for an hmeblk. 5155 * It returns the next address to be unloaded. 5156 * It should be called with the hash lock held. 5157 */ 5158 static caddr_t 5159 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5160 caddr_t endaddr, demap_range_t *dmrp, uint_t flags) 5161 { 5162 tte_t tte, ttemod; 5163 struct sf_hment *sfhmep; 5164 int ttesz; 5165 long ttecnt; 5166 page_t *pp; 5167 kmutex_t *pml; 5168 int ret; 5169 int use_demap_range; 5170 5171 ASSERT(in_hblk_range(hmeblkp, addr)); 5172 ASSERT(!hmeblkp->hblk_shw_bit); 5173 #ifdef DEBUG 5174 if (get_hblk_ttesz(hmeblkp) != TTE8K && 5175 (endaddr < get_hblk_endaddr(hmeblkp))) { 5176 panic("sfmmu_hblk_unload: partial unload of large page"); 5177 } 5178 #endif /* DEBUG */ 5179 5180 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5181 ttesz = get_hblk_ttesz(hmeblkp); 5182 5183 use_demap_range = (do_virtual_coloring && 5184 TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp)); 5185 if (use_demap_range) { 5186 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 5187 } else { 5188 DEMAP_RANGE_FLUSH(dmrp); 5189 } 5190 ttecnt = 0; 5191 HBLKTOHME(sfhmep, hmeblkp, addr); 5192 5193 while (addr < endaddr) { 5194 pml = NULL; 5195 again: 5196 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5197 if (TTE_IS_VALID(&tte)) { 5198 pp = sfhmep->hme_page; 5199 if (pp && pml == NULL) { 5200 pml = sfmmu_mlist_enter(pp); 5201 } 5202 5203 /* 5204 * Verify if hme still points to 'pp' now that 5205 * we have p_mapping lock. 5206 */ 5207 if (sfhmep->hme_page != pp) { 5208 if (pp != NULL && sfhmep->hme_page != NULL) { 5209 if (pml) { 5210 sfmmu_mlist_exit(pml); 5211 } 5212 /* Re-start this iteration. */ 5213 continue; 5214 } 5215 ASSERT((pp != NULL) && 5216 (sfhmep->hme_page == NULL)); 5217 goto tte_unloaded; 5218 } 5219 5220 /* 5221 * This point on we have both HASH and p_mapping 5222 * lock. 5223 */ 5224 ASSERT(pp == sfhmep->hme_page); 5225 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5226 5227 /* 5228 * We need to loop on modify tte because it is 5229 * possible for pagesync to come along and 5230 * change the software bits beneath us. 5231 * 5232 * Page_unload can also invalidate the tte after 5233 * we read tte outside of p_mapping lock. 5234 */ 5235 ttemod = tte; 5236 5237 TTE_SET_INVALID(&ttemod); 5238 ret = sfmmu_modifytte_try(&tte, &ttemod, 5239 &sfhmep->hme_tte); 5240 5241 if (ret <= 0) { 5242 if (TTE_IS_VALID(&tte)) { 5243 goto again; 5244 } else { 5245 /* 5246 * We read in a valid pte, but it 5247 * is unloaded by page_unload. 5248 * hme_page has become NULL and 5249 * we hold no p_mapping lock. 5250 */ 5251 ASSERT(pp == NULL && pml == NULL); 5252 goto tte_unloaded; 5253 } 5254 } 5255 5256 if (!(flags & HAT_UNLOAD_NOSYNC)) { 5257 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5258 } 5259 5260 /* 5261 * Ok- we invalidated the tte. Do the rest of the job. 5262 */ 5263 ttecnt++; 5264 5265 if (flags & HAT_UNLOAD_UNLOCK) { 5266 ASSERT(hmeblkp->hblk_lckcnt > 0); 5267 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 5268 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 5269 } 5270 5271 /* 5272 * Normally we would need to flush the page 5273 * from the virtual cache at this point in 5274 * order to prevent a potential cache alias 5275 * inconsistency. 5276 * The particular scenario we need to worry 5277 * about is: 5278 * Given: va1 and va2 are two virtual address 5279 * that alias and map the same physical 5280 * address. 5281 * 1. mapping exists from va1 to pa and data 5282 * has been read into the cache. 5283 * 2. unload va1. 5284 * 3. load va2 and modify data using va2. 5285 * 4 unload va2. 5286 * 5. load va1 and reference data. Unless we 5287 * flush the data cache when we unload we will 5288 * get stale data. 5289 * Fortunately, page coloring eliminates the 5290 * above scenario by remembering the color a 5291 * physical page was last or is currently 5292 * mapped to. Now, we delay the flush until 5293 * the loading of translations. Only when the 5294 * new translation is of a different color 5295 * are we forced to flush. 5296 */ 5297 if (use_demap_range) { 5298 /* 5299 * Mark this page as needing a demap. 5300 */ 5301 DEMAP_RANGE_MARKPG(dmrp, addr); 5302 } else { 5303 if (do_virtual_coloring) { 5304 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 5305 sfmmup->sfmmu_free, 0); 5306 } else { 5307 pfn_t pfnum; 5308 5309 pfnum = TTE_TO_PFN(addr, &tte); 5310 sfmmu_tlbcache_demap(addr, sfmmup, 5311 hmeblkp, pfnum, sfmmup->sfmmu_free, 5312 FLUSH_NECESSARY_CPUS, 5313 CACHE_FLUSH, 0); 5314 } 5315 } 5316 5317 if (pp) { 5318 /* 5319 * Remove the hment from the mapping list 5320 */ 5321 ASSERT(hmeblkp->hblk_hmecnt > 0); 5322 5323 /* 5324 * Again, we cannot 5325 * ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS); 5326 */ 5327 HME_SUB(sfhmep, pp); 5328 membar_stst(); 5329 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 5330 } 5331 5332 ASSERT(hmeblkp->hblk_vcnt > 0); 5333 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 5334 5335 ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 5336 !hmeblkp->hblk_lckcnt); 5337 5338 if (pp && (pp->p_nrm & (P_KPMC | P_KPMS | P_TNC))) { 5339 if (PP_ISTNC(pp)) { 5340 /* 5341 * If page was temporary 5342 * uncached, try to recache 5343 * it. Note that HME_SUB() was 5344 * called above so p_index and 5345 * mlist had been updated. 5346 */ 5347 conv_tnc(pp, ttesz); 5348 } else if (pp->p_mapping == NULL) { 5349 ASSERT(kpm_enable); 5350 /* 5351 * Page is marked to be in VAC conflict 5352 * to an existing kpm mapping and/or is 5353 * kpm mapped using only the regular 5354 * pagesize. 5355 */ 5356 sfmmu_kpm_hme_unload(pp); 5357 } 5358 } 5359 } else if ((pp = sfhmep->hme_page) != NULL) { 5360 /* 5361 * TTE is invalid but the hme 5362 * still exists. let pageunload 5363 * complete its job. 5364 */ 5365 ASSERT(pml == NULL); 5366 pml = sfmmu_mlist_enter(pp); 5367 if (sfhmep->hme_page != NULL) { 5368 sfmmu_mlist_exit(pml); 5369 pml = NULL; 5370 goto again; 5371 } 5372 ASSERT(sfhmep->hme_page == NULL); 5373 } else if (hmeblkp->hblk_hmecnt != 0) { 5374 /* 5375 * pageunload may have not finished decrementing 5376 * hblk_vcnt and hblk_hmecnt. Find page_t if any and 5377 * wait for pageunload to finish. Rely on pageunload 5378 * to decrement hblk_hmecnt after hblk_vcnt. 5379 */ 5380 pfn_t pfn = TTE_TO_TTEPFN(&tte); 5381 ASSERT(pml == NULL); 5382 if (pf_is_memory(pfn)) { 5383 pp = page_numtopp_nolock(pfn); 5384 if (pp != NULL) { 5385 pml = sfmmu_mlist_enter(pp); 5386 sfmmu_mlist_exit(pml); 5387 pml = NULL; 5388 } 5389 } 5390 } 5391 5392 tte_unloaded: 5393 /* 5394 * At this point, the tte we are looking at 5395 * should be unloaded, and hme has been unlinked 5396 * from page too. This is important because in 5397 * pageunload, it does ttesync() then HME_SUB. 5398 * We need to make sure HME_SUB has been completed 5399 * so we know ttesync() has been completed. Otherwise, 5400 * at exit time, after return from hat layer, VM will 5401 * release as structure which hat_setstat() (called 5402 * by ttesync()) needs. 5403 */ 5404 #ifdef DEBUG 5405 { 5406 tte_t dtte; 5407 5408 ASSERT(sfhmep->hme_page == NULL); 5409 5410 sfmmu_copytte(&sfhmep->hme_tte, &dtte); 5411 ASSERT(!TTE_IS_VALID(&dtte)); 5412 } 5413 #endif 5414 5415 if (pml) { 5416 sfmmu_mlist_exit(pml); 5417 } 5418 5419 addr += TTEBYTES(ttesz); 5420 sfhmep++; 5421 DEMAP_RANGE_NEXTPG(dmrp); 5422 } 5423 if (ttecnt > 0) 5424 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt); 5425 return (addr); 5426 } 5427 5428 /* 5429 * Synchronize all the mappings in the range [addr..addr+len). 5430 * Can be called with clearflag having two states: 5431 * HAT_SYNC_DONTZERO means just return the rm stats 5432 * HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats 5433 */ 5434 void 5435 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag) 5436 { 5437 struct hmehash_bucket *hmebp; 5438 hmeblk_tag hblktag; 5439 int hmeshift, hashno = 1; 5440 struct hme_blk *hmeblkp, *list = NULL; 5441 caddr_t endaddr; 5442 cpuset_t cpuset; 5443 5444 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 5445 ASSERT((sfmmup == ksfmmup) || 5446 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 5447 ASSERT((len & MMU_PAGEOFFSET) == 0); 5448 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 5449 (clearflag == HAT_SYNC_ZERORM)); 5450 5451 CPUSET_ZERO(cpuset); 5452 5453 endaddr = addr + len; 5454 hblktag.htag_id = sfmmup; 5455 /* 5456 * Spitfire supports 4 page sizes. 5457 * Most pages are expected to be of the smallest page 5458 * size (8K) and these will not need to be rehashed. 64K 5459 * pages also don't need to be rehashed because the an hmeblk 5460 * spans 64K of address space. 512K pages might need 1 rehash and 5461 * and 4M pages 2 rehashes. 5462 */ 5463 while (addr < endaddr) { 5464 hmeshift = HME_HASH_SHIFT(hashno); 5465 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5466 hblktag.htag_rehash = hashno; 5467 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5468 5469 SFMMU_HASH_LOCK(hmebp); 5470 5471 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 5472 if (hmeblkp != NULL) { 5473 /* 5474 * We've encountered a shadow hmeblk so skip the range 5475 * of the next smaller mapping size. 5476 */ 5477 if (hmeblkp->hblk_shw_bit) { 5478 ASSERT(sfmmup != ksfmmup); 5479 ASSERT(hashno > 1); 5480 addr = (caddr_t)P2END((uintptr_t)addr, 5481 TTEBYTES(hashno - 1)); 5482 } else { 5483 addr = sfmmu_hblk_sync(sfmmup, hmeblkp, 5484 addr, endaddr, clearflag); 5485 } 5486 SFMMU_HASH_UNLOCK(hmebp); 5487 hashno = 1; 5488 continue; 5489 } 5490 SFMMU_HASH_UNLOCK(hmebp); 5491 5492 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 5493 /* 5494 * We have traversed the whole list and rehashed 5495 * if necessary without finding the address to sync. 5496 * This is ok so we increment the address by the 5497 * smallest hmeblk range for kernel mappings and the 5498 * largest hmeblk range, to account for shadow hmeblks, 5499 * for user mappings and continue. 5500 */ 5501 if (sfmmup == ksfmmup) 5502 addr = (caddr_t)P2END((uintptr_t)addr, 5503 TTEBYTES(1)); 5504 else 5505 addr = (caddr_t)P2END((uintptr_t)addr, 5506 TTEBYTES(hashno)); 5507 hashno = 1; 5508 } else { 5509 hashno++; 5510 } 5511 } 5512 sfmmu_hblks_list_purge(&list); 5513 cpuset = sfmmup->sfmmu_cpusran; 5514 xt_sync(cpuset); 5515 } 5516 5517 static caddr_t 5518 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5519 caddr_t endaddr, int clearflag) 5520 { 5521 tte_t tte, ttemod; 5522 struct sf_hment *sfhmep; 5523 int ttesz; 5524 struct page *pp; 5525 kmutex_t *pml; 5526 int ret; 5527 5528 ASSERT(hmeblkp->hblk_shw_bit == 0); 5529 5530 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5531 5532 ttesz = get_hblk_ttesz(hmeblkp); 5533 HBLKTOHME(sfhmep, hmeblkp, addr); 5534 5535 while (addr < endaddr) { 5536 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5537 if (TTE_IS_VALID(&tte)) { 5538 pml = NULL; 5539 pp = sfhmep->hme_page; 5540 if (pp) { 5541 pml = sfmmu_mlist_enter(pp); 5542 } 5543 if (pp != sfhmep->hme_page) { 5544 /* 5545 * tte most have been unloaded 5546 * underneath us. Recheck 5547 */ 5548 ASSERT(pml); 5549 sfmmu_mlist_exit(pml); 5550 continue; 5551 } 5552 5553 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5554 5555 if (clearflag == HAT_SYNC_ZERORM) { 5556 ttemod = tte; 5557 TTE_CLR_RM(&ttemod); 5558 ret = sfmmu_modifytte_try(&tte, &ttemod, 5559 &sfhmep->hme_tte); 5560 if (ret < 0) { 5561 if (pml) { 5562 sfmmu_mlist_exit(pml); 5563 } 5564 continue; 5565 } 5566 5567 if (ret > 0) { 5568 sfmmu_tlb_demap(addr, sfmmup, 5569 hmeblkp, 0, 0); 5570 } 5571 } 5572 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5573 if (pml) { 5574 sfmmu_mlist_exit(pml); 5575 } 5576 } 5577 addr += TTEBYTES(ttesz); 5578 sfhmep++; 5579 } 5580 return (addr); 5581 } 5582 5583 /* 5584 * This function will sync a tte to the page struct and it will 5585 * update the hat stats. Currently it allows us to pass a NULL pp 5586 * and we will simply update the stats. We may want to change this 5587 * so we only keep stats for pages backed by pp's. 5588 */ 5589 static void 5590 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp) 5591 { 5592 uint_t rm = 0; 5593 int sz; 5594 pgcnt_t npgs; 5595 5596 ASSERT(TTE_IS_VALID(ttep)); 5597 5598 if (TTE_IS_NOSYNC(ttep)) { 5599 return; 5600 } 5601 5602 if (TTE_IS_REF(ttep)) { 5603 rm = P_REF; 5604 } 5605 if (TTE_IS_MOD(ttep)) { 5606 rm |= P_MOD; 5607 } 5608 5609 if (rm == 0) { 5610 return; 5611 } 5612 5613 sz = TTE_CSZ(ttep); 5614 if (sfmmup->sfmmu_rmstat) { 5615 int i; 5616 caddr_t vaddr = addr; 5617 5618 for (i = 0; i < TTEPAGES(sz); i++, vaddr += MMU_PAGESIZE) { 5619 hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm); 5620 } 5621 5622 } 5623 5624 /* 5625 * XXX I want to use cas to update nrm bits but they 5626 * currently belong in common/vm and not in hat where 5627 * they should be. 5628 * The nrm bits are protected by the same mutex as 5629 * the one that protects the page's mapping list. 5630 */ 5631 if (!pp) 5632 return; 5633 ASSERT(sfmmu_mlist_held(pp)); 5634 /* 5635 * If the tte is for a large page, we need to sync all the 5636 * pages covered by the tte. 5637 */ 5638 if (sz != TTE8K) { 5639 ASSERT(pp->p_szc != 0); 5640 pp = PP_GROUPLEADER(pp, sz); 5641 ASSERT(sfmmu_mlist_held(pp)); 5642 } 5643 5644 /* Get number of pages from tte size. */ 5645 npgs = TTEPAGES(sz); 5646 5647 do { 5648 ASSERT(pp); 5649 ASSERT(sfmmu_mlist_held(pp)); 5650 if (((rm & P_REF) != 0 && !PP_ISREF(pp)) || 5651 ((rm & P_MOD) != 0 && !PP_ISMOD(pp))) 5652 hat_page_setattr(pp, rm); 5653 5654 /* 5655 * Are we done? If not, we must have a large mapping. 5656 * For large mappings we need to sync the rest of the pages 5657 * covered by this tte; goto the next page. 5658 */ 5659 } while (--npgs > 0 && (pp = PP_PAGENEXT(pp))); 5660 } 5661 5662 /* 5663 * Execute pre-callback handler of each pa_hment linked to pp 5664 * 5665 * Inputs: 5666 * flag: either HAT_PRESUSPEND or HAT_SUSPEND. 5667 * capture_cpus: pointer to return value (below) 5668 * 5669 * Returns: 5670 * Propagates the subsystem callback return values back to the caller; 5671 * returns 0 on success. If capture_cpus is non-NULL, the value returned 5672 * is zero if all of the pa_hments are of a type that do not require 5673 * capturing CPUs prior to suspending the mapping, else it is 1. 5674 */ 5675 static int 5676 hat_pageprocess_precallbacks(struct page *pp, uint_t flag, int *capture_cpus) 5677 { 5678 struct sf_hment *sfhmep; 5679 struct pa_hment *pahmep; 5680 int (*f)(caddr_t, uint_t, uint_t, void *); 5681 int ret; 5682 id_t id; 5683 int locked = 0; 5684 kmutex_t *pml; 5685 5686 ASSERT(PAGE_EXCL(pp)); 5687 if (!sfmmu_mlist_held(pp)) { 5688 pml = sfmmu_mlist_enter(pp); 5689 locked = 1; 5690 } 5691 5692 if (capture_cpus) 5693 *capture_cpus = 0; 5694 5695 top: 5696 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5697 /* 5698 * skip sf_hments corresponding to VA<->PA mappings; 5699 * for pa_hment's, hme_tte.ll is zero 5700 */ 5701 if (!IS_PAHME(sfhmep)) 5702 continue; 5703 5704 pahmep = sfhmep->hme_data; 5705 ASSERT(pahmep != NULL); 5706 5707 /* 5708 * skip if pre-handler has been called earlier in this loop 5709 */ 5710 if (pahmep->flags & flag) 5711 continue; 5712 5713 id = pahmep->cb_id; 5714 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5715 if (capture_cpus && sfmmu_cb_table[id].capture_cpus != 0) 5716 *capture_cpus = 1; 5717 if ((f = sfmmu_cb_table[id].prehandler) == NULL) { 5718 pahmep->flags |= flag; 5719 continue; 5720 } 5721 5722 /* 5723 * Drop the mapping list lock to avoid locking order issues. 5724 */ 5725 if (locked) 5726 sfmmu_mlist_exit(pml); 5727 5728 ret = f(pahmep->addr, pahmep->len, flag, pahmep->pvt); 5729 if (ret != 0) 5730 return (ret); /* caller must do the cleanup */ 5731 5732 if (locked) { 5733 pml = sfmmu_mlist_enter(pp); 5734 pahmep->flags |= flag; 5735 goto top; 5736 } 5737 5738 pahmep->flags |= flag; 5739 } 5740 5741 if (locked) 5742 sfmmu_mlist_exit(pml); 5743 5744 return (0); 5745 } 5746 5747 /* 5748 * Execute post-callback handler of each pa_hment linked to pp 5749 * 5750 * Same overall assumptions and restrictions apply as for 5751 * hat_pageprocess_precallbacks(). 5752 */ 5753 static void 5754 hat_pageprocess_postcallbacks(struct page *pp, uint_t flag) 5755 { 5756 pfn_t pgpfn = pp->p_pagenum; 5757 pfn_t pgmask = btop(page_get_pagesize(pp->p_szc)) - 1; 5758 pfn_t newpfn; 5759 struct sf_hment *sfhmep; 5760 struct pa_hment *pahmep; 5761 int (*f)(caddr_t, uint_t, uint_t, void *, pfn_t); 5762 id_t id; 5763 int locked = 0; 5764 kmutex_t *pml; 5765 5766 ASSERT(PAGE_EXCL(pp)); 5767 if (!sfmmu_mlist_held(pp)) { 5768 pml = sfmmu_mlist_enter(pp); 5769 locked = 1; 5770 } 5771 5772 top: 5773 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5774 /* 5775 * skip sf_hments corresponding to VA<->PA mappings; 5776 * for pa_hment's, hme_tte.ll is zero 5777 */ 5778 if (!IS_PAHME(sfhmep)) 5779 continue; 5780 5781 pahmep = sfhmep->hme_data; 5782 ASSERT(pahmep != NULL); 5783 5784 if ((pahmep->flags & flag) == 0) 5785 continue; 5786 5787 pahmep->flags &= ~flag; 5788 5789 id = pahmep->cb_id; 5790 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5791 if ((f = sfmmu_cb_table[id].posthandler) == NULL) 5792 continue; 5793 5794 /* 5795 * Convert the base page PFN into the constituent PFN 5796 * which is needed by the callback handler. 5797 */ 5798 newpfn = pgpfn | (btop((uintptr_t)pahmep->addr) & pgmask); 5799 5800 /* 5801 * Drop the mapping list lock to avoid locking order issues. 5802 */ 5803 if (locked) 5804 sfmmu_mlist_exit(pml); 5805 5806 if (f(pahmep->addr, pahmep->len, flag, pahmep->pvt, newpfn) 5807 != 0) 5808 panic("sfmmu: posthandler failed"); 5809 5810 if (locked) { 5811 pml = sfmmu_mlist_enter(pp); 5812 goto top; 5813 } 5814 } 5815 5816 if (locked) 5817 sfmmu_mlist_exit(pml); 5818 } 5819 5820 /* 5821 * Suspend locked kernel mapping 5822 */ 5823 void 5824 hat_pagesuspend(struct page *pp) 5825 { 5826 struct sf_hment *sfhmep; 5827 sfmmu_t *sfmmup; 5828 tte_t tte, ttemod; 5829 struct hme_blk *hmeblkp; 5830 caddr_t addr; 5831 int index, cons; 5832 cpuset_t cpuset; 5833 5834 ASSERT(PAGE_EXCL(pp)); 5835 ASSERT(sfmmu_mlist_held(pp)); 5836 5837 mutex_enter(&kpr_suspendlock); 5838 5839 /* 5840 * Call into dtrace to tell it we're about to suspend a 5841 * kernel mapping. This prevents us from running into issues 5842 * with probe context trying to touch a suspended page 5843 * in the relocation codepath itself. 5844 */ 5845 if (dtrace_kreloc_init) 5846 (*dtrace_kreloc_init)(); 5847 5848 index = PP_MAPINDEX(pp); 5849 cons = TTE8K; 5850 5851 retry: 5852 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5853 5854 if (IS_PAHME(sfhmep)) 5855 continue; 5856 5857 if (get_hblk_ttesz(sfmmu_hmetohblk(sfhmep)) != cons) 5858 continue; 5859 5860 /* 5861 * Loop until we successfully set the suspend bit in 5862 * the TTE. 5863 */ 5864 again: 5865 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5866 ASSERT(TTE_IS_VALID(&tte)); 5867 5868 ttemod = tte; 5869 TTE_SET_SUSPEND(&ttemod); 5870 if (sfmmu_modifytte_try(&tte, &ttemod, 5871 &sfhmep->hme_tte) < 0) 5872 goto again; 5873 5874 /* 5875 * Invalidate TSB entry 5876 */ 5877 hmeblkp = sfmmu_hmetohblk(sfhmep); 5878 5879 sfmmup = hblktosfmmu(hmeblkp); 5880 ASSERT(sfmmup == ksfmmup); 5881 5882 addr = tte_to_vaddr(hmeblkp, tte); 5883 5884 /* 5885 * No need to make sure that the TSB for this sfmmu is 5886 * not being relocated since it is ksfmmup and thus it 5887 * will never be relocated. 5888 */ 5889 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 5890 5891 /* 5892 * Update xcall stats 5893 */ 5894 cpuset = cpu_ready_set; 5895 CPUSET_DEL(cpuset, CPU->cpu_id); 5896 5897 /* LINTED: constant in conditional context */ 5898 SFMMU_XCALL_STATS(KCONTEXT); 5899 5900 /* 5901 * Flush TLB entry on remote CPU's 5902 */ 5903 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, KCONTEXT); 5904 xt_sync(cpuset); 5905 5906 /* 5907 * Flush TLB entry on local CPU 5908 */ 5909 vtag_flushpage(addr, KCONTEXT); 5910 } 5911 5912 while (index != 0) { 5913 index = index >> 1; 5914 if (index != 0) 5915 cons++; 5916 if (index & 0x1) { 5917 pp = PP_GROUPLEADER(pp, cons); 5918 goto retry; 5919 } 5920 } 5921 } 5922 5923 #ifdef DEBUG 5924 5925 #define N_PRLE 1024 5926 struct prle { 5927 page_t *targ; 5928 page_t *repl; 5929 int status; 5930 int pausecpus; 5931 hrtime_t whence; 5932 }; 5933 5934 static struct prle page_relocate_log[N_PRLE]; 5935 static int prl_entry; 5936 static kmutex_t prl_mutex; 5937 5938 #define PAGE_RELOCATE_LOG(t, r, s, p) \ 5939 mutex_enter(&prl_mutex); \ 5940 page_relocate_log[prl_entry].targ = *(t); \ 5941 page_relocate_log[prl_entry].repl = *(r); \ 5942 page_relocate_log[prl_entry].status = (s); \ 5943 page_relocate_log[prl_entry].pausecpus = (p); \ 5944 page_relocate_log[prl_entry].whence = gethrtime(); \ 5945 prl_entry = (prl_entry == (N_PRLE - 1))? 0 : prl_entry + 1; \ 5946 mutex_exit(&prl_mutex); 5947 5948 #else /* !DEBUG */ 5949 #define PAGE_RELOCATE_LOG(t, r, s, p) 5950 #endif 5951 5952 /* 5953 * Core Kernel Page Relocation Algorithm 5954 * 5955 * Input: 5956 * 5957 * target : constituent pages are SE_EXCL locked. 5958 * replacement: constituent pages are SE_EXCL locked. 5959 * 5960 * Output: 5961 * 5962 * nrelocp: number of pages relocated 5963 */ 5964 int 5965 hat_page_relocate(page_t **target, page_t **replacement, spgcnt_t *nrelocp) 5966 { 5967 page_t *targ, *repl; 5968 page_t *tpp, *rpp; 5969 kmutex_t *low, *high; 5970 spgcnt_t npages, i; 5971 page_t *pl = NULL; 5972 int old_pil; 5973 cpuset_t cpuset; 5974 int cap_cpus; 5975 int ret; 5976 5977 if (hat_kpr_enabled == 0 || !kcage_on || PP_ISNORELOC(*target)) { 5978 PAGE_RELOCATE_LOG(target, replacement, EAGAIN, -1); 5979 return (EAGAIN); 5980 } 5981 5982 mutex_enter(&kpr_mutex); 5983 kreloc_thread = curthread; 5984 5985 targ = *target; 5986 repl = *replacement; 5987 ASSERT(repl != NULL); 5988 ASSERT(targ->p_szc == repl->p_szc); 5989 5990 npages = page_get_pagecnt(targ->p_szc); 5991 5992 /* 5993 * unload VA<->PA mappings that are not locked 5994 */ 5995 tpp = targ; 5996 for (i = 0; i < npages; i++) { 5997 (void) hat_pageunload(tpp, SFMMU_KERNEL_RELOC); 5998 tpp++; 5999 } 6000 6001 /* 6002 * Do "presuspend" callbacks, in a context from which we can still 6003 * block as needed. Note that we don't hold the mapping list lock 6004 * of "targ" at this point due to potential locking order issues; 6005 * we assume that between the hat_pageunload() above and holding 6006 * the SE_EXCL lock that the mapping list *cannot* change at this 6007 * point. 6008 */ 6009 ret = hat_pageprocess_precallbacks(targ, HAT_PRESUSPEND, &cap_cpus); 6010 if (ret != 0) { 6011 /* 6012 * EIO translates to fatal error, for all others cleanup 6013 * and return EAGAIN. 6014 */ 6015 ASSERT(ret != EIO); 6016 hat_pageprocess_postcallbacks(targ, HAT_POSTUNSUSPEND); 6017 PAGE_RELOCATE_LOG(target, replacement, ret, -1); 6018 kreloc_thread = NULL; 6019 mutex_exit(&kpr_mutex); 6020 return (EAGAIN); 6021 } 6022 6023 /* 6024 * acquire p_mapping list lock for both the target and replacement 6025 * root pages. 6026 * 6027 * low and high refer to the need to grab the mlist locks in a 6028 * specific order in order to prevent race conditions. Thus the 6029 * lower lock must be grabbed before the higher lock. 6030 * 6031 * This will block hat_unload's accessing p_mapping list. Since 6032 * we have SE_EXCL lock, hat_memload and hat_pageunload will be 6033 * blocked. Thus, no one else will be accessing the p_mapping list 6034 * while we suspend and reload the locked mapping below. 6035 */ 6036 tpp = targ; 6037 rpp = repl; 6038 sfmmu_mlist_reloc_enter(tpp, rpp, &low, &high); 6039 6040 kpreempt_disable(); 6041 6042 /* 6043 * If the replacement page is of a different virtual color 6044 * than the page it is replacing, we need to handle the VAC 6045 * consistency for it just as we would if we were setting up 6046 * a new mapping to a page. 6047 */ 6048 if ((tpp->p_szc == 0) && (PP_GET_VCOLOR(rpp) != NO_VCOLOR)) { 6049 if (tpp->p_vcolor != rpp->p_vcolor) { 6050 sfmmu_cache_flushcolor(PP_GET_VCOLOR(rpp), 6051 rpp->p_pagenum); 6052 } 6053 } 6054 6055 /* 6056 * We raise our PIL to 13 so that we don't get captured by 6057 * another CPU or pinned by an interrupt thread. We can't go to 6058 * PIL 14 since the nexus driver(s) may need to interrupt at 6059 * that level in the case of IOMMU pseudo mappings. 6060 */ 6061 cpuset = cpu_ready_set; 6062 CPUSET_DEL(cpuset, CPU->cpu_id); 6063 if (!cap_cpus || CPUSET_ISNULL(cpuset)) { 6064 old_pil = splr(XCALL_PIL); 6065 } else { 6066 old_pil = -1; 6067 xc_attention(cpuset); 6068 } 6069 ASSERT(getpil() == XCALL_PIL); 6070 6071 /* 6072 * Now do suspend callbacks. In the case of an IOMMU mapping 6073 * this will suspend all DMA activity to the page while it is 6074 * being relocated. Since we are well above LOCK_LEVEL and CPUs 6075 * may be captured at this point we should have acquired any needed 6076 * locks in the presuspend callback. 6077 */ 6078 ret = hat_pageprocess_precallbacks(targ, HAT_SUSPEND, NULL); 6079 if (ret != 0) { 6080 repl = targ; 6081 goto suspend_fail; 6082 } 6083 6084 /* 6085 * Raise the PIL yet again, this time to block all high-level 6086 * interrupts on this CPU. This is necessary to prevent an 6087 * interrupt routine from pinning the thread which holds the 6088 * mapping suspended and then touching the suspended page. 6089 * 6090 * Once the page is suspended we also need to be careful to 6091 * avoid calling any functions which touch any seg_kmem memory 6092 * since that memory may be backed by the very page we are 6093 * relocating in here! 6094 */ 6095 hat_pagesuspend(targ); 6096 6097 /* 6098 * Now that we are confident everybody has stopped using this page, 6099 * copy the page contents. Note we use a physical copy to prevent 6100 * locking issues and to avoid fpRAS because we can't handle it in 6101 * this context. 6102 */ 6103 for (i = 0; i < npages; i++, tpp++, rpp++) { 6104 /* 6105 * Copy the contents of the page. 6106 */ 6107 ppcopy_kernel(tpp, rpp); 6108 } 6109 6110 tpp = targ; 6111 rpp = repl; 6112 for (i = 0; i < npages; i++, tpp++, rpp++) { 6113 /* 6114 * Copy attributes. VAC consistency was handled above, 6115 * if required. 6116 */ 6117 rpp->p_nrm = tpp->p_nrm; 6118 tpp->p_nrm = 0; 6119 rpp->p_index = tpp->p_index; 6120 tpp->p_index = 0; 6121 rpp->p_vcolor = tpp->p_vcolor; 6122 } 6123 6124 /* 6125 * First, unsuspend the page, if we set the suspend bit, and transfer 6126 * the mapping list from the target page to the replacement page. 6127 * Next process postcallbacks; since pa_hment's are linked only to the 6128 * p_mapping list of root page, we don't iterate over the constituent 6129 * pages. 6130 */ 6131 hat_pagereload(targ, repl); 6132 6133 suspend_fail: 6134 hat_pageprocess_postcallbacks(repl, HAT_UNSUSPEND); 6135 6136 /* 6137 * Now lower our PIL and release any captured CPUs since we 6138 * are out of the "danger zone". After this it will again be 6139 * safe to acquire adaptive mutex locks, or to drop them... 6140 */ 6141 if (old_pil != -1) { 6142 splx(old_pil); 6143 } else { 6144 xc_dismissed(cpuset); 6145 } 6146 6147 kpreempt_enable(); 6148 6149 sfmmu_mlist_reloc_exit(low, high); 6150 6151 /* 6152 * Postsuspend callbacks should drop any locks held across 6153 * the suspend callbacks. As before, we don't hold the mapping 6154 * list lock at this point.. our assumption is that the mapping 6155 * list still can't change due to our holding SE_EXCL lock and 6156 * there being no unlocked mappings left. Hence the restriction 6157 * on calling context to hat_delete_callback() 6158 */ 6159 hat_pageprocess_postcallbacks(repl, HAT_POSTUNSUSPEND); 6160 if (ret != 0) { 6161 /* 6162 * The second presuspend call failed: we got here through 6163 * the suspend_fail label above. 6164 */ 6165 ASSERT(ret != EIO); 6166 PAGE_RELOCATE_LOG(target, replacement, ret, cap_cpus); 6167 kreloc_thread = NULL; 6168 mutex_exit(&kpr_mutex); 6169 return (EAGAIN); 6170 } 6171 6172 /* 6173 * Now that we're out of the performance critical section we can 6174 * take care of updating the hash table, since we still 6175 * hold all the pages locked SE_EXCL at this point we 6176 * needn't worry about things changing out from under us. 6177 */ 6178 tpp = targ; 6179 rpp = repl; 6180 for (i = 0; i < npages; i++, tpp++, rpp++) { 6181 6182 /* 6183 * replace targ with replacement in page_hash table 6184 */ 6185 targ = tpp; 6186 page_relocate_hash(rpp, targ); 6187 6188 /* 6189 * concatenate target; caller of platform_page_relocate() 6190 * expects target to be concatenated after returning. 6191 */ 6192 ASSERT(targ->p_next == targ); 6193 ASSERT(targ->p_prev == targ); 6194 page_list_concat(&pl, &targ); 6195 } 6196 6197 ASSERT(*target == pl); 6198 *nrelocp = npages; 6199 PAGE_RELOCATE_LOG(target, replacement, 0, cap_cpus); 6200 kreloc_thread = NULL; 6201 mutex_exit(&kpr_mutex); 6202 return (0); 6203 } 6204 6205 /* 6206 * Called when stray pa_hments are found attached to a page which is 6207 * being freed. Notify the subsystem which attached the pa_hment of 6208 * the error if it registered a suitable handler, else panic. 6209 */ 6210 static void 6211 sfmmu_pahment_leaked(struct pa_hment *pahmep) 6212 { 6213 id_t cb_id = pahmep->cb_id; 6214 6215 ASSERT(cb_id >= (id_t)0 && cb_id < sfmmu_cb_nextid); 6216 if (sfmmu_cb_table[cb_id].errhandler != NULL) { 6217 if (sfmmu_cb_table[cb_id].errhandler(pahmep->addr, pahmep->len, 6218 HAT_CB_ERR_LEAKED, pahmep->pvt) == 0) 6219 return; /* non-fatal */ 6220 } 6221 panic("pa_hment leaked: 0x%p", pahmep); 6222 } 6223 6224 /* 6225 * Remove all mappings to page 'pp'. 6226 */ 6227 int 6228 hat_pageunload(struct page *pp, uint_t forceflag) 6229 { 6230 struct page *origpp = pp; 6231 struct sf_hment *sfhme, *tmphme; 6232 struct hme_blk *hmeblkp; 6233 kmutex_t *pml, *pmtx; 6234 cpuset_t cpuset, tset; 6235 int index, cons; 6236 int xhme_blks; 6237 int pa_hments; 6238 6239 ASSERT(PAGE_EXCL(pp)); 6240 6241 retry_xhat: 6242 tmphme = NULL; 6243 xhme_blks = 0; 6244 pa_hments = 0; 6245 CPUSET_ZERO(cpuset); 6246 6247 pml = sfmmu_mlist_enter(pp); 6248 6249 if (pp->p_kpmref) 6250 sfmmu_kpm_pageunload(pp); 6251 ASSERT(!PP_ISMAPPED_KPM(pp)); 6252 6253 index = PP_MAPINDEX(pp); 6254 cons = TTE8K; 6255 retry: 6256 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6257 tmphme = sfhme->hme_next; 6258 6259 if (IS_PAHME(sfhme)) { 6260 ASSERT(sfhme->hme_data != NULL); 6261 pa_hments++; 6262 continue; 6263 } 6264 6265 hmeblkp = sfmmu_hmetohblk(sfhme); 6266 if (hmeblkp->hblk_xhat_bit) { 6267 struct xhat_hme_blk *xblk = 6268 (struct xhat_hme_blk *)hmeblkp; 6269 6270 (void) XHAT_PAGEUNLOAD(xblk->xhat_hme_blk_hat, 6271 pp, forceflag, XBLK2PROVBLK(xblk)); 6272 6273 xhme_blks = 1; 6274 continue; 6275 } 6276 6277 /* 6278 * If there are kernel mappings don't unload them, they will 6279 * be suspended. 6280 */ 6281 if (forceflag == SFMMU_KERNEL_RELOC && hmeblkp->hblk_lckcnt && 6282 hmeblkp->hblk_tag.htag_id == ksfmmup) 6283 continue; 6284 6285 tset = sfmmu_pageunload(pp, sfhme, cons); 6286 CPUSET_OR(cpuset, tset); 6287 } 6288 6289 while (index != 0) { 6290 index = index >> 1; 6291 if (index != 0) 6292 cons++; 6293 if (index & 0x1) { 6294 /* Go to leading page */ 6295 pp = PP_GROUPLEADER(pp, cons); 6296 ASSERT(sfmmu_mlist_held(pp)); 6297 goto retry; 6298 } 6299 } 6300 6301 /* 6302 * cpuset may be empty if the page was only mapped by segkpm, 6303 * in which case we won't actually cross-trap. 6304 */ 6305 xt_sync(cpuset); 6306 6307 /* 6308 * The page should have no mappings at this point, unless 6309 * we were called from hat_page_relocate() in which case we 6310 * leave the locked mappings which will be suspended later. 6311 */ 6312 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks || pa_hments || 6313 (forceflag == SFMMU_KERNEL_RELOC)); 6314 6315 if (PP_ISTNC(pp)) { 6316 if (cons == TTE8K) { 6317 pmtx = sfmmu_page_enter(pp); 6318 PP_CLRTNC(pp); 6319 sfmmu_page_exit(pmtx); 6320 } else { 6321 conv_tnc(pp, cons); 6322 } 6323 } 6324 6325 if (pa_hments && forceflag != SFMMU_KERNEL_RELOC) { 6326 /* 6327 * Unlink any pa_hments and free them, calling back 6328 * the responsible subsystem to notify it of the error. 6329 * This can occur in situations such as drivers leaking 6330 * DMA handles: naughty, but common enough that we'd like 6331 * to keep the system running rather than bringing it 6332 * down with an obscure error like "pa_hment leaked" 6333 * which doesn't aid the user in debugging their driver. 6334 */ 6335 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6336 tmphme = sfhme->hme_next; 6337 if (IS_PAHME(sfhme)) { 6338 struct pa_hment *pahmep = sfhme->hme_data; 6339 sfmmu_pahment_leaked(pahmep); 6340 HME_SUB(sfhme, pp); 6341 kmem_cache_free(pa_hment_cache, pahmep); 6342 } 6343 } 6344 6345 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks); 6346 } 6347 6348 sfmmu_mlist_exit(pml); 6349 6350 /* 6351 * XHAT may not have finished unloading pages 6352 * because some other thread was waiting for 6353 * mlist lock and XHAT_PAGEUNLOAD let it do 6354 * the job. 6355 */ 6356 if (xhme_blks) { 6357 pp = origpp; 6358 goto retry_xhat; 6359 } 6360 6361 return (0); 6362 } 6363 6364 static cpuset_t 6365 sfmmu_pageunload(page_t *pp, struct sf_hment *sfhme, int cons) 6366 { 6367 struct hme_blk *hmeblkp; 6368 sfmmu_t *sfmmup; 6369 tte_t tte, ttemod; 6370 #ifdef DEBUG 6371 tte_t orig_old; 6372 #endif /* DEBUG */ 6373 caddr_t addr; 6374 int ttesz; 6375 int ret; 6376 cpuset_t cpuset; 6377 6378 ASSERT(pp != NULL); 6379 ASSERT(sfmmu_mlist_held(pp)); 6380 ASSERT(pp->p_vnode != &kvp); 6381 6382 CPUSET_ZERO(cpuset); 6383 6384 hmeblkp = sfmmu_hmetohblk(sfhme); 6385 6386 readtte: 6387 sfmmu_copytte(&sfhme->hme_tte, &tte); 6388 if (TTE_IS_VALID(&tte)) { 6389 sfmmup = hblktosfmmu(hmeblkp); 6390 ttesz = get_hblk_ttesz(hmeblkp); 6391 /* 6392 * Only unload mappings of 'cons' size. 6393 */ 6394 if (ttesz != cons) 6395 return (cpuset); 6396 6397 /* 6398 * Note that we have p_mapping lock, but no hash lock here. 6399 * hblk_unload() has to have both hash lock AND p_mapping 6400 * lock before it tries to modify tte. So, the tte could 6401 * not become invalid in the sfmmu_modifytte_try() below. 6402 */ 6403 ttemod = tte; 6404 #ifdef DEBUG 6405 orig_old = tte; 6406 #endif /* DEBUG */ 6407 6408 TTE_SET_INVALID(&ttemod); 6409 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6410 if (ret < 0) { 6411 #ifdef DEBUG 6412 /* only R/M bits can change. */ 6413 chk_tte(&orig_old, &tte, &ttemod, hmeblkp); 6414 #endif /* DEBUG */ 6415 goto readtte; 6416 } 6417 6418 if (ret == 0) { 6419 panic("pageunload: cas failed?"); 6420 } 6421 6422 addr = tte_to_vaddr(hmeblkp, tte); 6423 6424 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6425 6426 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -1); 6427 6428 /* 6429 * We need to flush the page from the virtual cache 6430 * in order to prevent a virtual cache alias 6431 * inconsistency. The particular scenario we need 6432 * to worry about is: 6433 * Given: va1 and va2 are two virtual address that 6434 * alias and will map the same physical address. 6435 * 1. mapping exists from va1 to pa and data has 6436 * been read into the cache. 6437 * 2. unload va1. 6438 * 3. load va2 and modify data using va2. 6439 * 4 unload va2. 6440 * 5. load va1 and reference data. Unless we flush 6441 * the data cache when we unload we will get 6442 * stale data. 6443 * This scenario is taken care of by using virtual 6444 * page coloring. 6445 */ 6446 if (sfmmup->sfmmu_ismhat) { 6447 /* 6448 * Flush TSBs, TLBs and caches 6449 * of every process 6450 * sharing this ism segment. 6451 */ 6452 sfmmu_hat_lock_all(); 6453 mutex_enter(&ism_mlist_lock); 6454 kpreempt_disable(); 6455 if (do_virtual_coloring) 6456 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6457 pp->p_pagenum, CACHE_NO_FLUSH); 6458 else 6459 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6460 pp->p_pagenum, CACHE_FLUSH); 6461 kpreempt_enable(); 6462 mutex_exit(&ism_mlist_lock); 6463 sfmmu_hat_unlock_all(); 6464 cpuset = cpu_ready_set; 6465 } else if (do_virtual_coloring) { 6466 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6467 cpuset = sfmmup->sfmmu_cpusran; 6468 } else { 6469 sfmmu_tlbcache_demap(addr, sfmmup, hmeblkp, 6470 pp->p_pagenum, 0, FLUSH_NECESSARY_CPUS, 6471 CACHE_FLUSH, 0); 6472 cpuset = sfmmup->sfmmu_cpusran; 6473 } 6474 6475 /* 6476 * Hme_sub has to run after ttesync() and a_rss update. 6477 * See hblk_unload(). 6478 */ 6479 HME_SUB(sfhme, pp); 6480 membar_stst(); 6481 6482 /* 6483 * We can not make ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 6484 * since pteload may have done a HME_ADD() right after 6485 * we did the HME_SUB() above. Hmecnt is now maintained 6486 * by cas only. no lock guranteed its value. The only 6487 * gurantee we have is the hmecnt should not be less than 6488 * what it should be so the hblk will not be taken away. 6489 * It's also important that we decremented the hmecnt after 6490 * we are done with hmeblkp so that this hmeblk won't be 6491 * stolen. 6492 */ 6493 ASSERT(hmeblkp->hblk_hmecnt > 0); 6494 ASSERT(hmeblkp->hblk_vcnt > 0); 6495 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 6496 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 6497 /* 6498 * This is bug 4063182. 6499 * XXX: fixme 6500 * ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 6501 * !hmeblkp->hblk_lckcnt); 6502 */ 6503 } else { 6504 panic("invalid tte? pp %p &tte %p", 6505 (void *)pp, (void *)&tte); 6506 } 6507 6508 return (cpuset); 6509 } 6510 6511 /* 6512 * While relocating a kernel page, this function will move the mappings 6513 * from tpp to dpp and modify any associated data with these mappings. 6514 * It also unsuspends the suspended kernel mapping. 6515 */ 6516 static void 6517 hat_pagereload(struct page *tpp, struct page *dpp) 6518 { 6519 struct sf_hment *sfhme; 6520 tte_t tte, ttemod; 6521 int index, cons; 6522 6523 ASSERT(getpil() == PIL_MAX); 6524 ASSERT(sfmmu_mlist_held(tpp)); 6525 ASSERT(sfmmu_mlist_held(dpp)); 6526 6527 index = PP_MAPINDEX(tpp); 6528 cons = TTE8K; 6529 6530 /* Update real mappings to the page */ 6531 retry: 6532 for (sfhme = tpp->p_mapping; sfhme != NULL; sfhme = sfhme->hme_next) { 6533 if (IS_PAHME(sfhme)) 6534 continue; 6535 sfmmu_copytte(&sfhme->hme_tte, &tte); 6536 ttemod = tte; 6537 6538 /* 6539 * replace old pfn with new pfn in TTE 6540 */ 6541 PFN_TO_TTE(ttemod, dpp->p_pagenum); 6542 6543 /* 6544 * clear suspend bit 6545 */ 6546 ASSERT(TTE_IS_SUSPEND(&ttemod)); 6547 TTE_CLR_SUSPEND(&ttemod); 6548 6549 if (sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte) < 0) 6550 panic("hat_pagereload(): sfmmu_modifytte_try() failed"); 6551 6552 /* 6553 * set hme_page point to new page 6554 */ 6555 sfhme->hme_page = dpp; 6556 } 6557 6558 /* 6559 * move p_mapping list from old page to new page 6560 */ 6561 dpp->p_mapping = tpp->p_mapping; 6562 tpp->p_mapping = NULL; 6563 dpp->p_share = tpp->p_share; 6564 tpp->p_share = 0; 6565 6566 while (index != 0) { 6567 index = index >> 1; 6568 if (index != 0) 6569 cons++; 6570 if (index & 0x1) { 6571 tpp = PP_GROUPLEADER(tpp, cons); 6572 dpp = PP_GROUPLEADER(dpp, cons); 6573 goto retry; 6574 } 6575 } 6576 6577 if (dtrace_kreloc_fini) 6578 (*dtrace_kreloc_fini)(); 6579 mutex_exit(&kpr_suspendlock); 6580 } 6581 6582 uint_t 6583 hat_pagesync(struct page *pp, uint_t clearflag) 6584 { 6585 struct sf_hment *sfhme, *tmphme = NULL; 6586 struct hme_blk *hmeblkp; 6587 kmutex_t *pml; 6588 cpuset_t cpuset, tset; 6589 int index, cons; 6590 extern ulong_t po_share; 6591 page_t *save_pp = pp; 6592 6593 CPUSET_ZERO(cpuset); 6594 6595 if (PP_ISRO(pp) && (clearflag & HAT_SYNC_STOPON_MOD)) { 6596 return (PP_GENERIC_ATTR(pp)); 6597 } 6598 6599 if ((clearflag == (HAT_SYNC_STOPON_REF | HAT_SYNC_DONTZERO)) && 6600 PP_ISREF(pp)) { 6601 return (PP_GENERIC_ATTR(pp)); 6602 } 6603 6604 if ((clearflag == (HAT_SYNC_STOPON_MOD | HAT_SYNC_DONTZERO)) && 6605 PP_ISMOD(pp)) { 6606 return (PP_GENERIC_ATTR(pp)); 6607 } 6608 6609 if ((clearflag & HAT_SYNC_STOPON_SHARED) != 0 && 6610 (pp->p_share > po_share) && 6611 !(clearflag & HAT_SYNC_ZERORM)) { 6612 if (PP_ISRO(pp)) 6613 hat_page_setattr(pp, P_REF); 6614 return (PP_GENERIC_ATTR(pp)); 6615 } 6616 6617 clearflag &= ~HAT_SYNC_STOPON_SHARED; 6618 pml = sfmmu_mlist_enter(pp); 6619 index = PP_MAPINDEX(pp); 6620 cons = TTE8K; 6621 retry: 6622 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6623 /* 6624 * We need to save the next hment on the list since 6625 * it is possible for pagesync to remove an invalid hment 6626 * from the list. 6627 */ 6628 tmphme = sfhme->hme_next; 6629 /* 6630 * If we are looking for large mappings and this hme doesn't 6631 * reach the range we are seeking, just ignore its. 6632 */ 6633 hmeblkp = sfmmu_hmetohblk(sfhme); 6634 if (hmeblkp->hblk_xhat_bit) 6635 continue; 6636 6637 if (hme_size(sfhme) < cons) 6638 continue; 6639 tset = sfmmu_pagesync(pp, sfhme, 6640 clearflag & ~HAT_SYNC_STOPON_RM); 6641 CPUSET_OR(cpuset, tset); 6642 /* 6643 * If clearflag is HAT_SYNC_DONTZERO, break out as soon 6644 * as the "ref" or "mod" is set. 6645 */ 6646 if ((clearflag & ~HAT_SYNC_STOPON_RM) == HAT_SYNC_DONTZERO && 6647 ((clearflag & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp)) || 6648 ((clearflag & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp))) { 6649 index = 0; 6650 break; 6651 } 6652 } 6653 6654 while (index) { 6655 index = index >> 1; 6656 cons++; 6657 if (index & 0x1) { 6658 /* Go to leading page */ 6659 pp = PP_GROUPLEADER(pp, cons); 6660 goto retry; 6661 } 6662 } 6663 6664 xt_sync(cpuset); 6665 sfmmu_mlist_exit(pml); 6666 return (PP_GENERIC_ATTR(save_pp)); 6667 } 6668 6669 /* 6670 * Get all the hardware dependent attributes for a page struct 6671 */ 6672 static cpuset_t 6673 sfmmu_pagesync(struct page *pp, struct sf_hment *sfhme, 6674 uint_t clearflag) 6675 { 6676 caddr_t addr; 6677 tte_t tte, ttemod; 6678 struct hme_blk *hmeblkp; 6679 int ret; 6680 sfmmu_t *sfmmup; 6681 cpuset_t cpuset; 6682 6683 ASSERT(pp != NULL); 6684 ASSERT(sfmmu_mlist_held(pp)); 6685 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 6686 (clearflag == HAT_SYNC_ZERORM)); 6687 6688 SFMMU_STAT(sf_pagesync); 6689 6690 CPUSET_ZERO(cpuset); 6691 6692 sfmmu_pagesync_retry: 6693 6694 sfmmu_copytte(&sfhme->hme_tte, &tte); 6695 if (TTE_IS_VALID(&tte)) { 6696 hmeblkp = sfmmu_hmetohblk(sfhme); 6697 sfmmup = hblktosfmmu(hmeblkp); 6698 addr = tte_to_vaddr(hmeblkp, tte); 6699 if (clearflag == HAT_SYNC_ZERORM) { 6700 ttemod = tte; 6701 TTE_CLR_RM(&ttemod); 6702 ret = sfmmu_modifytte_try(&tte, &ttemod, 6703 &sfhme->hme_tte); 6704 if (ret < 0) { 6705 /* 6706 * cas failed and the new value is not what 6707 * we want. 6708 */ 6709 goto sfmmu_pagesync_retry; 6710 } 6711 6712 if (ret > 0) { 6713 /* we win the cas */ 6714 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6715 cpuset = sfmmup->sfmmu_cpusran; 6716 } 6717 } 6718 6719 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6720 } 6721 return (cpuset); 6722 } 6723 6724 /* 6725 * Remove write permission from a mappings to a page, so that 6726 * we can detect the next modification of it. This requires modifying 6727 * the TTE then invalidating (demap) any TLB entry using that TTE. 6728 * This code is similar to sfmmu_pagesync(). 6729 */ 6730 static cpuset_t 6731 sfmmu_pageclrwrt(struct page *pp, struct sf_hment *sfhme) 6732 { 6733 caddr_t addr; 6734 tte_t tte; 6735 tte_t ttemod; 6736 struct hme_blk *hmeblkp; 6737 int ret; 6738 sfmmu_t *sfmmup; 6739 cpuset_t cpuset; 6740 6741 ASSERT(pp != NULL); 6742 ASSERT(sfmmu_mlist_held(pp)); 6743 6744 CPUSET_ZERO(cpuset); 6745 SFMMU_STAT(sf_clrwrt); 6746 6747 retry: 6748 6749 sfmmu_copytte(&sfhme->hme_tte, &tte); 6750 if (TTE_IS_VALID(&tte) && TTE_IS_WRITABLE(&tte)) { 6751 hmeblkp = sfmmu_hmetohblk(sfhme); 6752 6753 /* 6754 * xhat mappings should never be to a VMODSORT page. 6755 */ 6756 ASSERT(hmeblkp->hblk_xhat_bit == 0); 6757 6758 sfmmup = hblktosfmmu(hmeblkp); 6759 addr = tte_to_vaddr(hmeblkp, tte); 6760 6761 ttemod = tte; 6762 TTE_CLR_WRT(&ttemod); 6763 TTE_CLR_MOD(&ttemod); 6764 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6765 6766 /* 6767 * if cas failed and the new value is not what 6768 * we want retry 6769 */ 6770 if (ret < 0) 6771 goto retry; 6772 6773 /* we win the cas */ 6774 if (ret > 0) { 6775 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6776 cpuset = sfmmup->sfmmu_cpusran; 6777 } 6778 } 6779 6780 return (cpuset); 6781 } 6782 6783 /* 6784 * Walk all mappings of a page, removing write permission and clearing the 6785 * ref/mod bits. This code is similar to hat_pagesync() 6786 */ 6787 static void 6788 hat_page_clrwrt(page_t *pp) 6789 { 6790 struct sf_hment *sfhme; 6791 struct sf_hment *tmphme = NULL; 6792 kmutex_t *pml; 6793 cpuset_t cpuset; 6794 cpuset_t tset; 6795 int index; 6796 int cons; 6797 6798 CPUSET_ZERO(cpuset); 6799 6800 pml = sfmmu_mlist_enter(pp); 6801 index = PP_MAPINDEX(pp); 6802 cons = TTE8K; 6803 retry: 6804 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6805 tmphme = sfhme->hme_next; 6806 6807 /* 6808 * If we are looking for large mappings and this hme doesn't 6809 * reach the range we are seeking, just ignore its. 6810 */ 6811 6812 if (hme_size(sfhme) < cons) 6813 continue; 6814 6815 tset = sfmmu_pageclrwrt(pp, sfhme); 6816 CPUSET_OR(cpuset, tset); 6817 } 6818 6819 while (index) { 6820 index = index >> 1; 6821 cons++; 6822 if (index & 0x1) { 6823 /* Go to leading page */ 6824 pp = PP_GROUPLEADER(pp, cons); 6825 goto retry; 6826 } 6827 } 6828 6829 xt_sync(cpuset); 6830 sfmmu_mlist_exit(pml); 6831 } 6832 6833 /* 6834 * Set the given REF/MOD/RO bits for the given page. 6835 * For a vnode with a sorted v_pages list, we need to change 6836 * the attributes and the v_pages list together under page_vnode_mutex. 6837 */ 6838 void 6839 hat_page_setattr(page_t *pp, uint_t flag) 6840 { 6841 vnode_t *vp = pp->p_vnode; 6842 page_t **listp; 6843 kmutex_t *pmtx; 6844 kmutex_t *vphm = NULL; 6845 6846 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 6847 6848 /* 6849 * nothing to do if attribute already set 6850 */ 6851 if ((pp->p_nrm & flag) == flag) 6852 return; 6853 6854 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 6855 vphm = page_vnode_mutex(vp); 6856 mutex_enter(vphm); 6857 } 6858 6859 pmtx = sfmmu_page_enter(pp); 6860 pp->p_nrm |= flag; 6861 sfmmu_page_exit(pmtx); 6862 6863 if (vphm != NULL) { 6864 /* 6865 * Some File Systems examine v_pages for NULL w/o 6866 * grabbing the vphm mutex. Must not let it become NULL when 6867 * pp is the only page on the list. 6868 */ 6869 if (pp->p_vpnext != pp) { 6870 page_vpsub(&vp->v_pages, pp); 6871 if (vp->v_pages != NULL) 6872 listp = &vp->v_pages->p_vpprev->p_vpnext; 6873 else 6874 listp = &vp->v_pages; 6875 page_vpadd(listp, pp); 6876 } 6877 mutex_exit(vphm); 6878 } 6879 } 6880 6881 void 6882 hat_page_clrattr(page_t *pp, uint_t flag) 6883 { 6884 vnode_t *vp = pp->p_vnode; 6885 kmutex_t *vphm = NULL; 6886 kmutex_t *pmtx; 6887 6888 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 6889 6890 /* 6891 * For vnode with a sorted v_pages list, we need to change 6892 * the attributes and the v_pages list together under page_vnode_mutex. 6893 */ 6894 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 6895 vphm = page_vnode_mutex(vp); 6896 mutex_enter(vphm); 6897 } 6898 6899 pmtx = sfmmu_page_enter(pp); 6900 pp->p_nrm &= ~flag; 6901 sfmmu_page_exit(pmtx); 6902 6903 if (vphm != NULL) { 6904 /* 6905 * Some File Systems examine v_pages for NULL w/o 6906 * grabbing the vphm mutex. Must not let it become NULL when 6907 * pp is the only page on the list. 6908 */ 6909 if (pp->p_vpnext != pp) { 6910 page_vpsub(&vp->v_pages, pp); 6911 page_vpadd(&vp->v_pages, pp); 6912 } 6913 mutex_exit(vphm); 6914 6915 /* 6916 * VMODSORT works by removing write permissions and getting 6917 * a fault when a page is made dirty. At this point 6918 * we need to remove write permission from all mappings 6919 * to this page. 6920 */ 6921 hat_page_clrwrt(pp); 6922 } 6923 } 6924 6925 6926 uint_t 6927 hat_page_getattr(page_t *pp, uint_t flag) 6928 { 6929 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 6930 return ((uint_t)(pp->p_nrm & flag)); 6931 } 6932 6933 /* 6934 * DEBUG kernels: verify that a kernel va<->pa translation 6935 * is safe by checking the underlying page_t is in a page 6936 * relocation-safe state. 6937 */ 6938 #ifdef DEBUG 6939 void 6940 sfmmu_check_kpfn(pfn_t pfn) 6941 { 6942 page_t *pp; 6943 int index, cons; 6944 6945 if (hat_check_vtop == 0) 6946 return; 6947 6948 if (hat_kpr_enabled == 0 || kvseg.s_base == NULL || panicstr) 6949 return; 6950 6951 pp = page_numtopp_nolock(pfn); 6952 if (!pp) 6953 return; 6954 6955 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 6956 return; 6957 6958 /* 6959 * Handed a large kernel page, we dig up the root page since we 6960 * know the root page might have the lock also. 6961 */ 6962 if (pp->p_szc != 0) { 6963 index = PP_MAPINDEX(pp); 6964 cons = TTE8K; 6965 again: 6966 while (index != 0) { 6967 index >>= 1; 6968 if (index != 0) 6969 cons++; 6970 if (index & 0x1) { 6971 pp = PP_GROUPLEADER(pp, cons); 6972 goto again; 6973 } 6974 } 6975 } 6976 6977 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 6978 return; 6979 6980 /* 6981 * Pages need to be locked or allocated "permanent" (either from 6982 * static_arena arena or explicitly setting PG_NORELOC when calling 6983 * page_create_va()) for VA->PA translations to be valid. 6984 */ 6985 if (!PP_ISNORELOC(pp)) 6986 panic("Illegal VA->PA translation, pp 0x%p not permanent", pp); 6987 else 6988 panic("Illegal VA->PA translation, pp 0x%p not locked", pp); 6989 } 6990 #endif /* DEBUG */ 6991 6992 /* 6993 * Returns a page frame number for a given virtual address. 6994 * Returns PFN_INVALID to indicate an invalid mapping 6995 */ 6996 pfn_t 6997 hat_getpfnum(struct hat *hat, caddr_t addr) 6998 { 6999 pfn_t pfn; 7000 tte_t tte; 7001 7002 /* 7003 * We would like to 7004 * ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 7005 * but we can't because the iommu driver will call this 7006 * routine at interrupt time and it can't grab the as lock 7007 * or it will deadlock: A thread could have the as lock 7008 * and be waiting for io. The io can't complete 7009 * because the interrupt thread is blocked trying to grab 7010 * the as lock. 7011 */ 7012 7013 ASSERT(hat->sfmmu_xhat_provider == NULL); 7014 7015 if (hat == ksfmmup) { 7016 if (segkpm && IS_KPM_ADDR(addr)) 7017 return (sfmmu_kpm_vatopfn(addr)); 7018 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7019 == PFN_SUSPENDED) { 7020 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7021 } 7022 sfmmu_check_kpfn(pfn); 7023 return (pfn); 7024 } else { 7025 return (sfmmu_uvatopfn(addr, hat)); 7026 } 7027 } 7028 7029 /* 7030 * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged. 7031 * Use hat_getpfnum(kas.a_hat, ...) instead. 7032 * 7033 * We'd like to return PFN_INVALID if the mappings have underlying page_t's 7034 * but can't right now due to the fact that some software has grown to use 7035 * this interface incorrectly. So for now when the interface is misused, 7036 * return a warning to the user that in the future it won't work in the 7037 * way they're abusing it, and carry on (after disabling page relocation). 7038 */ 7039 pfn_t 7040 hat_getkpfnum(caddr_t addr) 7041 { 7042 pfn_t pfn; 7043 tte_t tte; 7044 int badcaller = 0; 7045 extern int segkmem_reloc; 7046 7047 if (segkpm && IS_KPM_ADDR(addr)) { 7048 badcaller = 1; 7049 pfn = sfmmu_kpm_vatopfn(addr); 7050 } else { 7051 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7052 == PFN_SUSPENDED) { 7053 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7054 } 7055 badcaller = pf_is_memory(pfn); 7056 } 7057 7058 if (badcaller) { 7059 /* 7060 * We can't return PFN_INVALID or the caller may panic 7061 * or corrupt the system. The only alternative is to 7062 * disable page relocation at this point for all kernel 7063 * memory. This will impact any callers of page_relocate() 7064 * such as FMA or DR. 7065 * 7066 * RFE: Add junk here to spit out an ereport so the sysadmin 7067 * can be advised that he should upgrade his device driver 7068 * so that this doesn't happen. 7069 */ 7070 hat_getkpfnum_badcall(caller()); 7071 if (hat_kpr_enabled && segkmem_reloc) { 7072 hat_kpr_enabled = 0; 7073 segkmem_reloc = 0; 7074 cmn_err(CE_WARN, "Kernel Page Relocation is DISABLED"); 7075 } 7076 } 7077 return (pfn); 7078 } 7079 7080 pfn_t 7081 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup) 7082 { 7083 struct hmehash_bucket *hmebp; 7084 hmeblk_tag hblktag; 7085 int hmeshift, hashno = 1; 7086 struct hme_blk *hmeblkp = NULL; 7087 7088 struct sf_hment *sfhmep; 7089 tte_t tte; 7090 pfn_t pfn; 7091 7092 /* support for ISM */ 7093 ism_map_t *ism_map; 7094 ism_blk_t *ism_blkp; 7095 int i; 7096 sfmmu_t *ism_hatid = NULL; 7097 sfmmu_t *locked_hatid = NULL; 7098 7099 7100 ASSERT(sfmmup != ksfmmup); 7101 SFMMU_STAT(sf_user_vtop); 7102 /* 7103 * Set ism_hatid if vaddr falls in a ISM segment. 7104 */ 7105 ism_blkp = sfmmup->sfmmu_iblk; 7106 if (ism_blkp) { 7107 sfmmu_ismhat_enter(sfmmup, 0); 7108 locked_hatid = sfmmup; 7109 } 7110 while (ism_blkp && ism_hatid == NULL) { 7111 ism_map = ism_blkp->iblk_maps; 7112 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 7113 if (vaddr >= ism_start(ism_map[i]) && 7114 vaddr < ism_end(ism_map[i])) { 7115 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 7116 vaddr = (caddr_t)(vaddr - 7117 ism_start(ism_map[i])); 7118 break; 7119 } 7120 } 7121 ism_blkp = ism_blkp->iblk_next; 7122 } 7123 if (locked_hatid) { 7124 sfmmu_ismhat_exit(locked_hatid, 0); 7125 } 7126 7127 hblktag.htag_id = sfmmup; 7128 do { 7129 hmeshift = HME_HASH_SHIFT(hashno); 7130 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 7131 hblktag.htag_rehash = hashno; 7132 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 7133 7134 SFMMU_HASH_LOCK(hmebp); 7135 7136 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 7137 if (hmeblkp != NULL) { 7138 HBLKTOHME(sfhmep, hmeblkp, vaddr); 7139 sfmmu_copytte(&sfhmep->hme_tte, &tte); 7140 if (TTE_IS_VALID(&tte)) { 7141 pfn = TTE_TO_PFN(vaddr, &tte); 7142 } else { 7143 pfn = PFN_INVALID; 7144 } 7145 SFMMU_HASH_UNLOCK(hmebp); 7146 return (pfn); 7147 } 7148 SFMMU_HASH_UNLOCK(hmebp); 7149 hashno++; 7150 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 7151 return (PFN_INVALID); 7152 } 7153 7154 7155 /* 7156 * For compatability with AT&T and later optimizations 7157 */ 7158 /* ARGSUSED */ 7159 void 7160 hat_map(struct hat *hat, caddr_t addr, size_t len, uint_t flags) 7161 { 7162 ASSERT(hat != NULL); 7163 ASSERT(hat->sfmmu_xhat_provider == NULL); 7164 } 7165 7166 /* 7167 * Return the number of mappings to a particular page. 7168 * This number is an approximation of the number of 7169 * number of people sharing the page. 7170 */ 7171 ulong_t 7172 hat_page_getshare(page_t *pp) 7173 { 7174 page_t *spp = pp; /* start page */ 7175 kmutex_t *pml; 7176 ulong_t cnt; 7177 int index, sz = TTE64K; 7178 7179 /* 7180 * We need to grab the mlist lock to make sure any outstanding 7181 * load/unloads complete. Otherwise we could return zero 7182 * even though the unload(s) hasn't finished yet. 7183 */ 7184 pml = sfmmu_mlist_enter(spp); 7185 cnt = spp->p_share; 7186 7187 if (kpm_enable) 7188 cnt += spp->p_kpmref; 7189 7190 /* 7191 * If we have any large mappings, we count the number of 7192 * mappings that this large page is part of. 7193 */ 7194 index = PP_MAPINDEX(spp); 7195 index >>= 1; 7196 while (index) { 7197 pp = PP_GROUPLEADER(spp, sz); 7198 if ((index & 0x1) && pp != spp) { 7199 cnt += pp->p_share; 7200 spp = pp; 7201 } 7202 index >>= 1; 7203 sz++; 7204 } 7205 sfmmu_mlist_exit(pml); 7206 return (cnt); 7207 } 7208 7209 /* 7210 * Unload all large mappings to the pp and reset the p_szc field of every 7211 * constituent page according to the remaining mappings. 7212 * 7213 * pp must be locked SE_EXCL. Even though no other constituent pages are 7214 * locked it's legal to unload the large mappings to the pp because all 7215 * constituent pages of large locked mappings have to be locked SE_SHARED. 7216 * This means if we have SE_EXCL lock on one of constituent pages none of the 7217 * large mappings to pp are locked. 7218 * 7219 * Decrease p_szc field starting from the last constituent page and ending 7220 * with the root page. This method is used because other threads rely on the 7221 * root's p_szc to find the lock to syncronize on. After a root page_t's p_szc 7222 * is demoted then other threads will succeed in sfmmu_mlspl_enter(). This 7223 * ensures that p_szc changes of the constituent pages appears atomic for all 7224 * threads that use sfmmu_mlspl_enter() to examine p_szc field. 7225 * 7226 * This mechanism is only used for file system pages where it's not always 7227 * possible to get SE_EXCL locks on all constituent pages to demote the size 7228 * code (as is done for anonymous or kernel large pages). 7229 * 7230 * See more comments in front of sfmmu_mlspl_enter(). 7231 */ 7232 void 7233 hat_page_demote(page_t *pp) 7234 { 7235 int index; 7236 int sz; 7237 cpuset_t cpuset; 7238 int sync = 0; 7239 page_t *rootpp; 7240 struct sf_hment *sfhme; 7241 struct sf_hment *tmphme = NULL; 7242 struct hme_blk *hmeblkp; 7243 uint_t pszc; 7244 page_t *lastpp; 7245 cpuset_t tset; 7246 pgcnt_t npgs; 7247 kmutex_t *pml; 7248 kmutex_t *pmtx = NULL; 7249 7250 ASSERT(PAGE_EXCL(pp)); 7251 ASSERT(!PP_ISFREE(pp)); 7252 ASSERT(page_szc_lock_assert(pp)); 7253 pml = sfmmu_mlist_enter(pp); 7254 7255 pszc = pp->p_szc; 7256 if (pszc == 0) { 7257 goto out; 7258 } 7259 7260 index = PP_MAPINDEX(pp) >> 1; 7261 7262 if (index) { 7263 CPUSET_ZERO(cpuset); 7264 sz = TTE64K; 7265 sync = 1; 7266 } 7267 7268 while (index) { 7269 if (!(index & 0x1)) { 7270 index >>= 1; 7271 sz++; 7272 continue; 7273 } 7274 ASSERT(sz <= pszc); 7275 rootpp = PP_GROUPLEADER(pp, sz); 7276 for (sfhme = rootpp->p_mapping; sfhme; sfhme = tmphme) { 7277 tmphme = sfhme->hme_next; 7278 hmeblkp = sfmmu_hmetohblk(sfhme); 7279 if (hme_size(sfhme) != sz) { 7280 continue; 7281 } 7282 if (hmeblkp->hblk_xhat_bit) { 7283 cmn_err(CE_PANIC, 7284 "hat_page_demote: xhat hmeblk"); 7285 } 7286 tset = sfmmu_pageunload(rootpp, sfhme, sz); 7287 CPUSET_OR(cpuset, tset); 7288 } 7289 if (index >>= 1) { 7290 sz++; 7291 } 7292 } 7293 7294 ASSERT(!PP_ISMAPPED_LARGE(pp)); 7295 7296 if (sync) { 7297 xt_sync(cpuset); 7298 if (PP_ISTNC(pp)) { 7299 conv_tnc(rootpp, sz); 7300 } 7301 } 7302 7303 pmtx = sfmmu_page_enter(pp); 7304 7305 ASSERT(pp->p_szc == pszc); 7306 rootpp = PP_PAGEROOT(pp); 7307 ASSERT(rootpp->p_szc == pszc); 7308 lastpp = PP_PAGENEXT_N(rootpp, TTEPAGES(pszc) - 1); 7309 7310 while (lastpp != rootpp) { 7311 sz = PP_MAPINDEX(lastpp) ? fnd_mapping_sz(lastpp) : 0; 7312 ASSERT(sz < pszc); 7313 npgs = (sz == 0) ? 1 : TTEPAGES(sz); 7314 ASSERT(P2PHASE(lastpp->p_pagenum, npgs) == npgs - 1); 7315 while (--npgs > 0) { 7316 lastpp->p_szc = (uchar_t)sz; 7317 lastpp = PP_PAGEPREV(lastpp); 7318 } 7319 if (sz) { 7320 /* 7321 * make sure before current root's pszc 7322 * is updated all updates to constituent pages pszc 7323 * fields are globally visible. 7324 */ 7325 membar_producer(); 7326 } 7327 lastpp->p_szc = sz; 7328 ASSERT(IS_P2ALIGNED(lastpp->p_pagenum, TTEPAGES(sz))); 7329 if (lastpp != rootpp) { 7330 lastpp = PP_PAGEPREV(lastpp); 7331 } 7332 } 7333 if (sz == 0) { 7334 /* the loop above doesn't cover this case */ 7335 rootpp->p_szc = 0; 7336 } 7337 out: 7338 ASSERT(pp->p_szc == 0); 7339 if (pmtx != NULL) { 7340 sfmmu_page_exit(pmtx); 7341 } 7342 sfmmu_mlist_exit(pml); 7343 } 7344 7345 /* 7346 * Refresh the HAT ismttecnt[] element for size szc. 7347 * Caller must have set ISM busy flag to prevent mapping 7348 * lists from changing while we're traversing them. 7349 */ 7350 pgcnt_t 7351 ism_tsb_entries(sfmmu_t *sfmmup, int szc) 7352 { 7353 ism_blk_t *ism_blkp = sfmmup->sfmmu_iblk; 7354 ism_map_t *ism_map; 7355 pgcnt_t npgs = 0; 7356 int j; 7357 7358 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 7359 for (; ism_blkp != NULL; ism_blkp = ism_blkp->iblk_next) { 7360 ism_map = ism_blkp->iblk_maps; 7361 for (j = 0; ism_map[j].imap_ismhat && j < ISM_MAP_SLOTS; j++) 7362 npgs += ism_map[j].imap_ismhat->sfmmu_ttecnt[szc]; 7363 } 7364 sfmmup->sfmmu_ismttecnt[szc] = npgs; 7365 return (npgs); 7366 } 7367 7368 /* 7369 * Yield the memory claim requirement for an address space. 7370 * 7371 * This is currently implemented as the number of bytes that have active 7372 * hardware translations that have page structures. Therefore, it can 7373 * underestimate the traditional resident set size, eg, if the 7374 * physical page is present and the hardware translation is missing; 7375 * and it can overestimate the rss, eg, if there are active 7376 * translations to a frame buffer with page structs. 7377 * Also, it does not take sharing into account. 7378 * 7379 * Note that we don't acquire locks here since this function is most often 7380 * called from the clock thread. 7381 */ 7382 size_t 7383 hat_get_mapped_size(struct hat *hat) 7384 { 7385 size_t assize = 0; 7386 int i; 7387 7388 if (hat == NULL) 7389 return (0); 7390 7391 ASSERT(hat->sfmmu_xhat_provider == NULL); 7392 7393 for (i = 0; i < mmu_page_sizes; i++) 7394 assize += (pgcnt_t)hat->sfmmu_ttecnt[i] * TTEBYTES(i); 7395 7396 if (hat->sfmmu_iblk == NULL) 7397 return (assize); 7398 7399 for (i = 0; i < mmu_page_sizes; i++) 7400 assize += (pgcnt_t)hat->sfmmu_ismttecnt[i] * TTEBYTES(i); 7401 7402 return (assize); 7403 } 7404 7405 int 7406 hat_stats_enable(struct hat *hat) 7407 { 7408 hatlock_t *hatlockp; 7409 7410 ASSERT(hat->sfmmu_xhat_provider == NULL); 7411 7412 hatlockp = sfmmu_hat_enter(hat); 7413 hat->sfmmu_rmstat++; 7414 sfmmu_hat_exit(hatlockp); 7415 return (1); 7416 } 7417 7418 void 7419 hat_stats_disable(struct hat *hat) 7420 { 7421 hatlock_t *hatlockp; 7422 7423 ASSERT(hat->sfmmu_xhat_provider == NULL); 7424 7425 hatlockp = sfmmu_hat_enter(hat); 7426 hat->sfmmu_rmstat--; 7427 sfmmu_hat_exit(hatlockp); 7428 } 7429 7430 /* 7431 * Routines for entering or removing ourselves from the 7432 * ism_hat's mapping list. 7433 */ 7434 static void 7435 iment_add(struct ism_ment *iment, struct hat *ism_hat) 7436 { 7437 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7438 7439 iment->iment_prev = NULL; 7440 iment->iment_next = ism_hat->sfmmu_iment; 7441 if (ism_hat->sfmmu_iment) { 7442 ism_hat->sfmmu_iment->iment_prev = iment; 7443 } 7444 ism_hat->sfmmu_iment = iment; 7445 } 7446 7447 static void 7448 iment_sub(struct ism_ment *iment, struct hat *ism_hat) 7449 { 7450 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7451 7452 if (ism_hat->sfmmu_iment == NULL) { 7453 panic("ism map entry remove - no entries"); 7454 } 7455 7456 if (iment->iment_prev) { 7457 ASSERT(ism_hat->sfmmu_iment != iment); 7458 iment->iment_prev->iment_next = iment->iment_next; 7459 } else { 7460 ASSERT(ism_hat->sfmmu_iment == iment); 7461 ism_hat->sfmmu_iment = iment->iment_next; 7462 } 7463 7464 if (iment->iment_next) { 7465 iment->iment_next->iment_prev = iment->iment_prev; 7466 } 7467 7468 /* 7469 * zero out the entry 7470 */ 7471 iment->iment_next = NULL; 7472 iment->iment_prev = NULL; 7473 iment->iment_hat = NULL; 7474 } 7475 7476 /* 7477 * Hat_share()/unshare() return an (non-zero) error 7478 * when saddr and daddr are not properly aligned. 7479 * 7480 * The top level mapping element determines the alignment 7481 * requirement for saddr and daddr, depending on different 7482 * architectures. 7483 * 7484 * When hat_share()/unshare() are not supported, 7485 * HATOP_SHARE()/UNSHARE() return 0 7486 */ 7487 int 7488 hat_share(struct hat *sfmmup, caddr_t addr, 7489 struct hat *ism_hatid, caddr_t sptaddr, size_t len, uint_t ismszc) 7490 { 7491 ism_blk_t *ism_blkp; 7492 ism_blk_t *new_iblk; 7493 ism_map_t *ism_map; 7494 ism_ment_t *ism_ment; 7495 int i, added; 7496 hatlock_t *hatlockp; 7497 int reload_mmu = 0; 7498 uint_t ismshift = page_get_shift(ismszc); 7499 size_t ismpgsz = page_get_pagesize(ismszc); 7500 uint_t ismmask = (uint_t)ismpgsz - 1; 7501 size_t sh_size = ISM_SHIFT(ismshift, len); 7502 ushort_t ismhatflag; 7503 7504 #ifdef DEBUG 7505 caddr_t eaddr = addr + len; 7506 #endif /* DEBUG */ 7507 7508 ASSERT(ism_hatid != NULL && sfmmup != NULL); 7509 ASSERT(sptaddr == ISMID_STARTADDR); 7510 /* 7511 * Check the alignment. 7512 */ 7513 if (!ISM_ALIGNED(ismshift, addr) || !ISM_ALIGNED(ismshift, sptaddr)) 7514 return (EINVAL); 7515 7516 /* 7517 * Check size alignment. 7518 */ 7519 if (!ISM_ALIGNED(ismshift, len)) 7520 return (EINVAL); 7521 7522 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 7523 7524 /* 7525 * Allocate ism_ment for the ism_hat's mapping list, and an 7526 * ism map blk in case we need one. We must do our 7527 * allocations before acquiring locks to prevent a deadlock 7528 * in the kmem allocator on the mapping list lock. 7529 */ 7530 new_iblk = kmem_cache_alloc(ism_blk_cache, KM_SLEEP); 7531 ism_ment = kmem_cache_alloc(ism_ment_cache, KM_SLEEP); 7532 7533 /* 7534 * Serialize ISM mappings with the ISM busy flag, and also the 7535 * trap handlers. 7536 */ 7537 sfmmu_ismhat_enter(sfmmup, 0); 7538 7539 /* 7540 * Allocate an ism map blk if necessary. 7541 */ 7542 if (sfmmup->sfmmu_iblk == NULL) { 7543 sfmmup->sfmmu_iblk = new_iblk; 7544 bzero(new_iblk, sizeof (*new_iblk)); 7545 new_iblk->iblk_nextpa = (uint64_t)-1; 7546 membar_stst(); /* make sure next ptr visible to all CPUs */ 7547 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk); 7548 reload_mmu = 1; 7549 new_iblk = NULL; 7550 } 7551 7552 #ifdef DEBUG 7553 /* 7554 * Make sure mapping does not already exist. 7555 */ 7556 ism_blkp = sfmmup->sfmmu_iblk; 7557 while (ism_blkp) { 7558 ism_map = ism_blkp->iblk_maps; 7559 for (i = 0; i < ISM_MAP_SLOTS && ism_map[i].imap_ismhat; i++) { 7560 if ((addr >= ism_start(ism_map[i]) && 7561 addr < ism_end(ism_map[i])) || 7562 eaddr > ism_start(ism_map[i]) && 7563 eaddr <= ism_end(ism_map[i])) { 7564 panic("sfmmu_share: Already mapped!"); 7565 } 7566 } 7567 ism_blkp = ism_blkp->iblk_next; 7568 } 7569 #endif /* DEBUG */ 7570 7571 ASSERT(ismszc >= TTE4M); 7572 if (ismszc == TTE4M) { 7573 ismhatflag = HAT_4M_FLAG; 7574 } else if (ismszc == TTE32M) { 7575 ismhatflag = HAT_32M_FLAG; 7576 } else if (ismszc == TTE256M) { 7577 ismhatflag = HAT_256M_FLAG; 7578 } 7579 /* 7580 * Add mapping to first available mapping slot. 7581 */ 7582 ism_blkp = sfmmup->sfmmu_iblk; 7583 added = 0; 7584 while (!added) { 7585 ism_map = ism_blkp->iblk_maps; 7586 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7587 if (ism_map[i].imap_ismhat == NULL) { 7588 7589 ism_map[i].imap_ismhat = ism_hatid; 7590 ism_map[i].imap_vb_shift = (ushort_t)ismshift; 7591 ism_map[i].imap_hatflags = ismhatflag; 7592 ism_map[i].imap_sz_mask = ismmask; 7593 /* 7594 * imap_seg is checked in ISM_CHECK to see if 7595 * non-NULL, then other info assumed valid. 7596 */ 7597 membar_stst(); 7598 ism_map[i].imap_seg = (uintptr_t)addr | sh_size; 7599 ism_map[i].imap_ment = ism_ment; 7600 7601 /* 7602 * Now add ourselves to the ism_hat's 7603 * mapping list. 7604 */ 7605 ism_ment->iment_hat = sfmmup; 7606 ism_ment->iment_base_va = addr; 7607 ism_hatid->sfmmu_ismhat = 1; 7608 ism_hatid->sfmmu_flags = 0; 7609 mutex_enter(&ism_mlist_lock); 7610 iment_add(ism_ment, ism_hatid); 7611 mutex_exit(&ism_mlist_lock); 7612 added = 1; 7613 break; 7614 } 7615 } 7616 if (!added && ism_blkp->iblk_next == NULL) { 7617 ism_blkp->iblk_next = new_iblk; 7618 new_iblk = NULL; 7619 bzero(ism_blkp->iblk_next, 7620 sizeof (*ism_blkp->iblk_next)); 7621 ism_blkp->iblk_next->iblk_nextpa = (uint64_t)-1; 7622 membar_stst(); 7623 ism_blkp->iblk_nextpa = 7624 va_to_pa((caddr_t)ism_blkp->iblk_next); 7625 } 7626 ism_blkp = ism_blkp->iblk_next; 7627 } 7628 7629 /* 7630 * Update our counters for this sfmmup's ism mappings. 7631 */ 7632 for (i = 0; i <= ismszc; i++) { 7633 if (!(disable_ism_large_pages & (1 << i))) 7634 (void) ism_tsb_entries(sfmmup, i); 7635 } 7636 7637 hatlockp = sfmmu_hat_enter(sfmmup); 7638 7639 /* 7640 * For ISM and DISM we do not support 512K pages, so we only 7641 * only search the 4M and 8K/64K hashes for 4 pagesize cpus, and search 7642 * the 256M or 32M, and 4M and 8K/64K hashes for 6 pagesize cpus. 7643 */ 7644 ASSERT((disable_ism_large_pages & (1 << TTE512K)) != 0); 7645 7646 if (ismszc > TTE4M && !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) 7647 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7648 7649 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) 7650 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 7651 7652 /* 7653 * If we updated the ismblkpa for this HAT or we need 7654 * to start searching the 256M or 32M or 4M hash, we must 7655 * make sure all CPUs running this process reload their 7656 * tsbmiss area. Otherwise they will fail to load the mappings 7657 * in the tsbmiss handler and will loop calling pagefault(). 7658 */ 7659 switch (ismszc) { 7660 case TTE256M: 7661 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 7662 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 7663 sfmmu_sync_mmustate(sfmmup); 7664 } 7665 break; 7666 case TTE32M: 7667 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 7668 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 7669 sfmmu_sync_mmustate(sfmmup); 7670 } 7671 break; 7672 case TTE4M: 7673 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 7674 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7675 sfmmu_sync_mmustate(sfmmup); 7676 } 7677 break; 7678 default: 7679 break; 7680 } 7681 7682 /* 7683 * Now we can drop the locks. 7684 */ 7685 sfmmu_ismhat_exit(sfmmup, 1); 7686 sfmmu_hat_exit(hatlockp); 7687 7688 /* 7689 * Free up ismblk if we didn't use it. 7690 */ 7691 if (new_iblk != NULL) 7692 kmem_cache_free(ism_blk_cache, new_iblk); 7693 7694 /* 7695 * Check TSB and TLB page sizes. 7696 */ 7697 sfmmu_check_page_sizes(sfmmup, 1); 7698 7699 return (0); 7700 } 7701 7702 /* 7703 * hat_unshare removes exactly one ism_map from 7704 * this process's as. It expects multiple calls 7705 * to hat_unshare for multiple shm segments. 7706 */ 7707 void 7708 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) 7709 { 7710 ism_map_t *ism_map; 7711 ism_ment_t *free_ment = NULL; 7712 ism_blk_t *ism_blkp; 7713 struct hat *ism_hatid; 7714 struct ctx *ctx; 7715 int cnum, found, i; 7716 hatlock_t *hatlockp; 7717 struct tsb_info *tsbinfo; 7718 uint_t ismshift = page_get_shift(ismszc); 7719 size_t sh_size = ISM_SHIFT(ismshift, len); 7720 7721 ASSERT(ISM_ALIGNED(ismshift, addr)); 7722 ASSERT(ISM_ALIGNED(ismshift, len)); 7723 ASSERT(sfmmup != NULL); 7724 ASSERT(sfmmup != ksfmmup); 7725 7726 if (sfmmup->sfmmu_xhat_provider) { 7727 XHAT_UNSHARE(sfmmup, addr, len); 7728 return; 7729 } else { 7730 /* 7731 * This must be a CPU HAT. If the address space has 7732 * XHATs attached, inform all XHATs that ISM segment 7733 * is going away 7734 */ 7735 ASSERT(sfmmup->sfmmu_as != NULL); 7736 if (sfmmup->sfmmu_as->a_xhat != NULL) 7737 xhat_unshare_all(sfmmup->sfmmu_as, addr, len); 7738 } 7739 7740 /* 7741 * Make sure that during the entire time ISM mappings are removed, 7742 * the trap handlers serialize behind us, and that no one else 7743 * can be mucking with ISM mappings. This also lets us get away 7744 * with not doing expensive cross calls to flush the TLB -- we 7745 * just discard the context, flush the entire TSB, and call it 7746 * a day. 7747 */ 7748 sfmmu_ismhat_enter(sfmmup, 0); 7749 7750 /* 7751 * Remove the mapping. 7752 * 7753 * We can't have any holes in the ism map. 7754 * The tsb miss code while searching the ism map will 7755 * stop on an empty map slot. So we must move 7756 * everyone past the hole up 1 if any. 7757 * 7758 * Also empty ism map blks are not freed until the 7759 * process exits. This is to prevent a MT race condition 7760 * between sfmmu_unshare() and sfmmu_tsbmiss_exception(). 7761 */ 7762 found = 0; 7763 ism_blkp = sfmmup->sfmmu_iblk; 7764 while (!found && ism_blkp) { 7765 ism_map = ism_blkp->iblk_maps; 7766 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7767 if (addr == ism_start(ism_map[i]) && 7768 sh_size == (size_t)(ism_size(ism_map[i]))) { 7769 found = 1; 7770 break; 7771 } 7772 } 7773 if (!found) 7774 ism_blkp = ism_blkp->iblk_next; 7775 } 7776 7777 if (found) { 7778 ism_hatid = ism_map[i].imap_ismhat; 7779 ASSERT(ism_hatid != NULL); 7780 ASSERT(ism_hatid->sfmmu_ismhat == 1); 7781 ASSERT(ism_hatid->sfmmu_cnum == INVALID_CONTEXT); 7782 7783 /* 7784 * First remove ourselves from the ism mapping list. 7785 */ 7786 mutex_enter(&ism_mlist_lock); 7787 iment_sub(ism_map[i].imap_ment, ism_hatid); 7788 mutex_exit(&ism_mlist_lock); 7789 free_ment = ism_map[i].imap_ment; 7790 7791 /* 7792 * Now gurantee that any other cpu 7793 * that tries to process an ISM miss 7794 * will go to tl=0. 7795 */ 7796 hatlockp = sfmmu_hat_enter(sfmmup); 7797 ctx = sfmmutoctx(sfmmup); 7798 rw_enter(&ctx->ctx_rwlock, RW_WRITER); 7799 cnum = sfmmutoctxnum(sfmmup); 7800 7801 if (cnum != INVALID_CONTEXT) { 7802 sfmmu_tlb_swap_ctx(sfmmup, ctx); 7803 } 7804 rw_exit(&ctx->ctx_rwlock); 7805 sfmmu_hat_exit(hatlockp); 7806 7807 /* 7808 * We delete the ism map by copying 7809 * the next map over the current one. 7810 * We will take the next one in the maps 7811 * array or from the next ism_blk. 7812 */ 7813 while (ism_blkp) { 7814 ism_map = ism_blkp->iblk_maps; 7815 while (i < (ISM_MAP_SLOTS - 1)) { 7816 ism_map[i] = ism_map[i + 1]; 7817 i++; 7818 } 7819 /* i == (ISM_MAP_SLOTS - 1) */ 7820 ism_blkp = ism_blkp->iblk_next; 7821 if (ism_blkp) { 7822 ism_map[i] = ism_blkp->iblk_maps[0]; 7823 i = 0; 7824 } else { 7825 ism_map[i].imap_seg = 0; 7826 ism_map[i].imap_vb_shift = 0; 7827 ism_map[i].imap_hatflags = 0; 7828 ism_map[i].imap_sz_mask = 0; 7829 ism_map[i].imap_ismhat = NULL; 7830 ism_map[i].imap_ment = NULL; 7831 } 7832 } 7833 7834 /* 7835 * Now flush entire TSB for the process, since 7836 * demapping page by page can be too expensive. 7837 * We don't have to flush the TLB here anymore 7838 * since we switch to a new TLB ctx instead. 7839 * Also, there is no need to flush if the process 7840 * is exiting since the TSB will be freed later. 7841 */ 7842 if (!sfmmup->sfmmu_free) { 7843 hatlockp = sfmmu_hat_enter(sfmmup); 7844 for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL; 7845 tsbinfo = tsbinfo->tsb_next) { 7846 if (tsbinfo->tsb_flags & TSB_SWAPPED) 7847 continue; 7848 sfmmu_inv_tsb(tsbinfo->tsb_va, 7849 TSB_BYTES(tsbinfo->tsb_szc)); 7850 } 7851 sfmmu_hat_exit(hatlockp); 7852 } 7853 } 7854 7855 /* 7856 * Update our counters for this sfmmup's ism mappings. 7857 */ 7858 for (i = 0; i <= ismszc; i++) { 7859 if (!(disable_ism_large_pages & (1 << i))) 7860 (void) ism_tsb_entries(sfmmup, i); 7861 } 7862 7863 sfmmu_ismhat_exit(sfmmup, 0); 7864 7865 /* 7866 * We must do our freeing here after dropping locks 7867 * to prevent a deadlock in the kmem allocator on the 7868 * mapping list lock. 7869 */ 7870 if (free_ment != NULL) 7871 kmem_cache_free(ism_ment_cache, free_ment); 7872 7873 /* 7874 * Check TSB and TLB page sizes if the process isn't exiting. 7875 */ 7876 if (!sfmmup->sfmmu_free) 7877 sfmmu_check_page_sizes(sfmmup, 0); 7878 } 7879 7880 /* ARGSUSED */ 7881 static int 7882 sfmmu_idcache_constructor(void *buf, void *cdrarg, int kmflags) 7883 { 7884 /* void *buf is sfmmu_t pointer */ 7885 return (0); 7886 } 7887 7888 /* ARGSUSED */ 7889 static void 7890 sfmmu_idcache_destructor(void *buf, void *cdrarg) 7891 { 7892 /* void *buf is sfmmu_t pointer */ 7893 } 7894 7895 /* 7896 * setup kmem hmeblks by bzeroing all members and initializing the nextpa 7897 * field to be the pa of this hmeblk 7898 */ 7899 /* ARGSUSED */ 7900 static int 7901 sfmmu_hblkcache_constructor(void *buf, void *cdrarg, int kmflags) 7902 { 7903 struct hme_blk *hmeblkp; 7904 7905 bzero(buf, (size_t)cdrarg); 7906 hmeblkp = (struct hme_blk *)buf; 7907 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); 7908 7909 #ifdef HBLK_TRACE 7910 mutex_init(&hmeblkp->hblk_audit_lock, NULL, MUTEX_DEFAULT, NULL); 7911 #endif /* HBLK_TRACE */ 7912 7913 return (0); 7914 } 7915 7916 /* ARGSUSED */ 7917 static void 7918 sfmmu_hblkcache_destructor(void *buf, void *cdrarg) 7919 { 7920 7921 #ifdef HBLK_TRACE 7922 7923 struct hme_blk *hmeblkp; 7924 7925 hmeblkp = (struct hme_blk *)buf; 7926 mutex_destroy(&hmeblkp->hblk_audit_lock); 7927 7928 #endif /* HBLK_TRACE */ 7929 } 7930 7931 #define SFMMU_CACHE_RECLAIM_SCAN_RATIO 8 7932 static int sfmmu_cache_reclaim_scan_ratio = SFMMU_CACHE_RECLAIM_SCAN_RATIO; 7933 /* 7934 * The kmem allocator will callback into our reclaim routine when the system 7935 * is running low in memory. We traverse the hash and free up all unused but 7936 * still cached hme_blks. We also traverse the free list and free them up 7937 * as well. 7938 */ 7939 /*ARGSUSED*/ 7940 static void 7941 sfmmu_hblkcache_reclaim(void *cdrarg) 7942 { 7943 int i; 7944 uint64_t hblkpa, prevpa, nx_pa; 7945 struct hmehash_bucket *hmebp; 7946 struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL; 7947 static struct hmehash_bucket *uhmehash_reclaim_hand; 7948 static struct hmehash_bucket *khmehash_reclaim_hand; 7949 struct hme_blk *list = NULL; 7950 7951 hmebp = uhmehash_reclaim_hand; 7952 if (hmebp == NULL || hmebp > &uhme_hash[UHMEHASH_SZ]) 7953 uhmehash_reclaim_hand = hmebp = uhme_hash; 7954 uhmehash_reclaim_hand += UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 7955 7956 for (i = UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 7957 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 7958 hmeblkp = hmebp->hmeblkp; 7959 hblkpa = hmebp->hmeh_nextpa; 7960 prevpa = 0; 7961 pr_hblk = NULL; 7962 while (hmeblkp) { 7963 nx_hblk = hmeblkp->hblk_next; 7964 nx_pa = hmeblkp->hblk_nextpa; 7965 if (!hmeblkp->hblk_vcnt && 7966 !hmeblkp->hblk_hmecnt) { 7967 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 7968 prevpa, pr_hblk); 7969 sfmmu_hblk_free(hmebp, hmeblkp, 7970 hblkpa, &list); 7971 } else { 7972 pr_hblk = hmeblkp; 7973 prevpa = hblkpa; 7974 } 7975 hmeblkp = nx_hblk; 7976 hblkpa = nx_pa; 7977 } 7978 SFMMU_HASH_UNLOCK(hmebp); 7979 } 7980 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 7981 hmebp = uhme_hash; 7982 } 7983 7984 hmebp = khmehash_reclaim_hand; 7985 if (hmebp == NULL || hmebp > &khme_hash[KHMEHASH_SZ]) 7986 khmehash_reclaim_hand = hmebp = khme_hash; 7987 khmehash_reclaim_hand += KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 7988 7989 for (i = KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 7990 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 7991 hmeblkp = hmebp->hmeblkp; 7992 hblkpa = hmebp->hmeh_nextpa; 7993 prevpa = 0; 7994 pr_hblk = NULL; 7995 while (hmeblkp) { 7996 nx_hblk = hmeblkp->hblk_next; 7997 nx_pa = hmeblkp->hblk_nextpa; 7998 if (!hmeblkp->hblk_vcnt && 7999 !hmeblkp->hblk_hmecnt) { 8000 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8001 prevpa, pr_hblk); 8002 sfmmu_hblk_free(hmebp, hmeblkp, 8003 hblkpa, &list); 8004 } else { 8005 pr_hblk = hmeblkp; 8006 prevpa = hblkpa; 8007 } 8008 hmeblkp = nx_hblk; 8009 hblkpa = nx_pa; 8010 } 8011 SFMMU_HASH_UNLOCK(hmebp); 8012 } 8013 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 8014 hmebp = khme_hash; 8015 } 8016 sfmmu_hblks_list_purge(&list); 8017 } 8018 8019 /* 8020 * sfmmu_get_ppvcolor should become a vm_machdep or hatop interface. 8021 * same goes for sfmmu_get_addrvcolor(). 8022 * 8023 * This function will return the virtual color for the specified page. The 8024 * virtual color corresponds to this page current mapping or its last mapping. 8025 * It is used by memory allocators to choose addresses with the correct 8026 * alignment so vac consistency is automatically maintained. If the page 8027 * has no color it returns -1. 8028 */ 8029 int 8030 sfmmu_get_ppvcolor(struct page *pp) 8031 { 8032 int color; 8033 8034 if (!(cache & CACHE_VAC) || PP_NEWPAGE(pp)) { 8035 return (-1); 8036 } 8037 color = PP_GET_VCOLOR(pp); 8038 ASSERT(color < mmu_btop(shm_alignment)); 8039 return (color); 8040 } 8041 8042 /* 8043 * This function will return the desired alignment for vac consistency 8044 * (vac color) given a virtual address. If no vac is present it returns -1. 8045 */ 8046 int 8047 sfmmu_get_addrvcolor(caddr_t vaddr) 8048 { 8049 if (cache & CACHE_VAC) { 8050 return (addr_to_vcolor(vaddr)); 8051 } else { 8052 return (-1); 8053 } 8054 8055 } 8056 8057 /* 8058 * Check for conflicts. 8059 * A conflict exists if the new and existent mappings do not match in 8060 * their "shm_alignment fields. If conflicts exist, the existant mappings 8061 * are flushed unless one of them is locked. If one of them is locked, then 8062 * the mappings are flushed and converted to non-cacheable mappings. 8063 */ 8064 static void 8065 sfmmu_vac_conflict(struct hat *hat, caddr_t addr, page_t *pp) 8066 { 8067 struct hat *tmphat; 8068 struct sf_hment *sfhmep, *tmphme = NULL; 8069 struct hme_blk *hmeblkp; 8070 int vcolor; 8071 tte_t tte; 8072 8073 ASSERT(sfmmu_mlist_held(pp)); 8074 ASSERT(!PP_ISNC(pp)); /* page better be cacheable */ 8075 8076 vcolor = addr_to_vcolor(addr); 8077 if (PP_NEWPAGE(pp)) { 8078 PP_SET_VCOLOR(pp, vcolor); 8079 return; 8080 } 8081 8082 if (PP_GET_VCOLOR(pp) == vcolor) { 8083 return; 8084 } 8085 8086 if (!PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp)) { 8087 /* 8088 * Previous user of page had a different color 8089 * but since there are no current users 8090 * we just flush the cache and change the color. 8091 */ 8092 SFMMU_STAT(sf_pgcolor_conflict); 8093 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8094 PP_SET_VCOLOR(pp, vcolor); 8095 return; 8096 } 8097 8098 /* 8099 * If we get here we have a vac conflict with a current 8100 * mapping. VAC conflict policy is as follows. 8101 * - The default is to unload the other mappings unless: 8102 * - If we have a large mapping we uncache the page. 8103 * We need to uncache the rest of the large page too. 8104 * - If any of the mappings are locked we uncache the page. 8105 * - If the requested mapping is inconsistent 8106 * with another mapping and that mapping 8107 * is in the same address space we have to 8108 * make it non-cached. The default thing 8109 * to do is unload the inconsistent mapping 8110 * but if they are in the same address space 8111 * we run the risk of unmapping the pc or the 8112 * stack which we will use as we return to the user, 8113 * in which case we can then fault on the thing 8114 * we just unloaded and get into an infinite loop. 8115 */ 8116 if (PP_ISMAPPED_LARGE(pp)) { 8117 int sz; 8118 8119 /* 8120 * Existing mapping is for big pages. We don't unload 8121 * existing big mappings to satisfy new mappings. 8122 * Always convert all mappings to TNC. 8123 */ 8124 sz = fnd_mapping_sz(pp); 8125 pp = PP_GROUPLEADER(pp, sz); 8126 SFMMU_STAT_ADD(sf_uncache_conflict, TTEPAGES(sz)); 8127 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 8128 TTEPAGES(sz)); 8129 8130 return; 8131 } 8132 8133 /* 8134 * check if any mapping is in same as or if it is locked 8135 * since in that case we need to uncache. 8136 */ 8137 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8138 tmphme = sfhmep->hme_next; 8139 hmeblkp = sfmmu_hmetohblk(sfhmep); 8140 if (hmeblkp->hblk_xhat_bit) 8141 continue; 8142 tmphat = hblktosfmmu(hmeblkp); 8143 sfmmu_copytte(&sfhmep->hme_tte, &tte); 8144 ASSERT(TTE_IS_VALID(&tte)); 8145 if ((tmphat == hat) || hmeblkp->hblk_lckcnt) { 8146 /* 8147 * We have an uncache conflict 8148 */ 8149 SFMMU_STAT(sf_uncache_conflict); 8150 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1); 8151 return; 8152 } 8153 } 8154 8155 /* 8156 * We have an unload conflict 8157 * We have already checked for LARGE mappings, therefore 8158 * the remaining mapping(s) must be TTE8K. 8159 */ 8160 SFMMU_STAT(sf_unload_conflict); 8161 8162 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8163 tmphme = sfhmep->hme_next; 8164 hmeblkp = sfmmu_hmetohblk(sfhmep); 8165 if (hmeblkp->hblk_xhat_bit) 8166 continue; 8167 (void) sfmmu_pageunload(pp, sfhmep, TTE8K); 8168 } 8169 8170 if (PP_ISMAPPED_KPM(pp)) 8171 sfmmu_kpm_vac_unload(pp, addr); 8172 8173 /* 8174 * Unloads only do TLB flushes so we need to flush the 8175 * cache here. 8176 */ 8177 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8178 PP_SET_VCOLOR(pp, vcolor); 8179 } 8180 8181 /* 8182 * Whenever a mapping is unloaded and the page is in TNC state, 8183 * we see if the page can be made cacheable again. 'pp' is 8184 * the page that we just unloaded a mapping from, the size 8185 * of mapping that was unloaded is 'ottesz'. 8186 * Remark: 8187 * The recache policy for mpss pages can leave a performance problem 8188 * under the following circumstances: 8189 * . A large page in uncached mode has just been unmapped. 8190 * . All constituent pages are TNC due to a conflicting small mapping. 8191 * . There are many other, non conflicting, small mappings around for 8192 * a lot of the constituent pages. 8193 * . We're called w/ the "old" groupleader page and the old ottesz, 8194 * but this is irrelevant, since we're no more "PP_ISMAPPED_LARGE", so 8195 * we end up w/ TTE8K or npages == 1. 8196 * . We call tst_tnc w/ the old groupleader only, and if there is no 8197 * conflict, we re-cache only this page. 8198 * . All other small mappings are not checked and will be left in TNC mode. 8199 * The problem is not very serious because: 8200 * . mpss is actually only defined for heap and stack, so the probability 8201 * is not very high that a large page mapping exists in parallel to a small 8202 * one (this is possible, but seems to be bad programming style in the 8203 * appl). 8204 * . The problem gets a little bit more serious, when those TNC pages 8205 * have to be mapped into kernel space, e.g. for networking. 8206 * . When VAC alias conflicts occur in applications, this is regarded 8207 * as an application bug. So if kstat's show them, the appl should 8208 * be changed anyway. 8209 */ 8210 static void 8211 conv_tnc(page_t *pp, int ottesz) 8212 { 8213 int cursz, dosz; 8214 pgcnt_t curnpgs, dopgs; 8215 pgcnt_t pg64k; 8216 page_t *pp2; 8217 8218 /* 8219 * Determine how big a range we check for TNC and find 8220 * leader page. cursz is the size of the biggest 8221 * mapping that still exist on 'pp'. 8222 */ 8223 if (PP_ISMAPPED_LARGE(pp)) { 8224 cursz = fnd_mapping_sz(pp); 8225 } else { 8226 cursz = TTE8K; 8227 } 8228 8229 if (ottesz >= cursz) { 8230 dosz = ottesz; 8231 pp2 = pp; 8232 } else { 8233 dosz = cursz; 8234 pp2 = PP_GROUPLEADER(pp, dosz); 8235 } 8236 8237 pg64k = TTEPAGES(TTE64K); 8238 dopgs = TTEPAGES(dosz); 8239 8240 ASSERT(dopgs == 1 || ((dopgs & (pg64k - 1)) == 0)); 8241 8242 while (dopgs != 0) { 8243 curnpgs = TTEPAGES(cursz); 8244 if (tst_tnc(pp2, curnpgs)) { 8245 SFMMU_STAT_ADD(sf_recache, curnpgs); 8246 sfmmu_page_cache_array(pp2, HAT_CACHE, CACHE_NO_FLUSH, 8247 curnpgs); 8248 } 8249 8250 ASSERT(dopgs >= curnpgs); 8251 dopgs -= curnpgs; 8252 8253 if (dopgs == 0) { 8254 break; 8255 } 8256 8257 pp2 = PP_PAGENEXT_N(pp2, curnpgs); 8258 if (((dopgs & (pg64k - 1)) == 0) && PP_ISMAPPED_LARGE(pp2)) { 8259 cursz = fnd_mapping_sz(pp2); 8260 } else { 8261 cursz = TTE8K; 8262 } 8263 } 8264 } 8265 8266 /* 8267 * Returns 1 if page(s) can be converted from TNC to cacheable setting, 8268 * returns 0 otherwise. Note that oaddr argument is valid for only 8269 * 8k pages. 8270 */ 8271 static int 8272 tst_tnc(page_t *pp, pgcnt_t npages) 8273 { 8274 struct sf_hment *sfhme; 8275 struct hme_blk *hmeblkp; 8276 tte_t tte; 8277 caddr_t vaddr; 8278 int clr_valid = 0; 8279 int color, color1, bcolor; 8280 int i, ncolors; 8281 8282 ASSERT(pp != NULL); 8283 ASSERT(!(cache & CACHE_WRITEBACK)); 8284 8285 if (npages > 1) { 8286 ncolors = CACHE_NUM_COLOR; 8287 } 8288 8289 for (i = 0; i < npages; i++) { 8290 ASSERT(sfmmu_mlist_held(pp)); 8291 ASSERT(PP_ISTNC(pp)); 8292 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 8293 8294 if (PP_ISPNC(pp)) { 8295 return (0); 8296 } 8297 8298 clr_valid = 0; 8299 if (PP_ISMAPPED_KPM(pp)) { 8300 caddr_t kpmvaddr; 8301 8302 ASSERT(kpm_enable); 8303 kpmvaddr = hat_kpm_page2va(pp, 1); 8304 ASSERT(!(npages > 1 && IS_KPM_ALIAS_RANGE(kpmvaddr))); 8305 color1 = addr_to_vcolor(kpmvaddr); 8306 clr_valid = 1; 8307 } 8308 8309 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8310 hmeblkp = sfmmu_hmetohblk(sfhme); 8311 if (hmeblkp->hblk_xhat_bit) 8312 continue; 8313 8314 sfmmu_copytte(&sfhme->hme_tte, &tte); 8315 ASSERT(TTE_IS_VALID(&tte)); 8316 8317 vaddr = tte_to_vaddr(hmeblkp, tte); 8318 color = addr_to_vcolor(vaddr); 8319 8320 if (npages > 1) { 8321 /* 8322 * If there is a big mapping, make sure 8323 * 8K mapping is consistent with the big 8324 * mapping. 8325 */ 8326 bcolor = i % ncolors; 8327 if (color != bcolor) { 8328 return (0); 8329 } 8330 } 8331 if (!clr_valid) { 8332 clr_valid = 1; 8333 color1 = color; 8334 } 8335 8336 if (color1 != color) { 8337 return (0); 8338 } 8339 } 8340 8341 pp = PP_PAGENEXT(pp); 8342 } 8343 8344 return (1); 8345 } 8346 8347 static void 8348 sfmmu_page_cache_array(page_t *pp, int flags, int cache_flush_flag, 8349 pgcnt_t npages) 8350 { 8351 kmutex_t *pmtx; 8352 int i, ncolors, bcolor; 8353 kpm_hlk_t *kpmp; 8354 cpuset_t cpuset; 8355 8356 ASSERT(pp != NULL); 8357 ASSERT(!(cache & CACHE_WRITEBACK)); 8358 8359 kpmp = sfmmu_kpm_kpmp_enter(pp, npages); 8360 pmtx = sfmmu_page_enter(pp); 8361 8362 /* 8363 * Fast path caching single unmapped page 8364 */ 8365 if (npages == 1 && !PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp) && 8366 flags == HAT_CACHE) { 8367 PP_CLRTNC(pp); 8368 PP_CLRPNC(pp); 8369 sfmmu_page_exit(pmtx); 8370 sfmmu_kpm_kpmp_exit(kpmp); 8371 return; 8372 } 8373 8374 /* 8375 * We need to capture all cpus in order to change cacheability 8376 * because we can't allow one cpu to access the same physical 8377 * page using a cacheable and a non-cachebale mapping at the same 8378 * time. Since we may end up walking the ism mapping list 8379 * have to grab it's lock now since we can't after all the 8380 * cpus have been captured. 8381 */ 8382 sfmmu_hat_lock_all(); 8383 mutex_enter(&ism_mlist_lock); 8384 kpreempt_disable(); 8385 cpuset = cpu_ready_set; 8386 xc_attention(cpuset); 8387 8388 if (npages > 1) { 8389 /* 8390 * Make sure all colors are flushed since the 8391 * sfmmu_page_cache() only flushes one color- 8392 * it does not know big pages. 8393 */ 8394 ncolors = CACHE_NUM_COLOR; 8395 if (flags & HAT_TMPNC) { 8396 for (i = 0; i < ncolors; i++) { 8397 sfmmu_cache_flushcolor(i, pp->p_pagenum); 8398 } 8399 cache_flush_flag = CACHE_NO_FLUSH; 8400 } 8401 } 8402 8403 for (i = 0; i < npages; i++) { 8404 8405 ASSERT(sfmmu_mlist_held(pp)); 8406 8407 if (!(flags == HAT_TMPNC && PP_ISTNC(pp))) { 8408 8409 if (npages > 1) { 8410 bcolor = i % ncolors; 8411 } else { 8412 bcolor = NO_VCOLOR; 8413 } 8414 8415 sfmmu_page_cache(pp, flags, cache_flush_flag, 8416 bcolor); 8417 } 8418 8419 pp = PP_PAGENEXT(pp); 8420 } 8421 8422 xt_sync(cpuset); 8423 xc_dismissed(cpuset); 8424 mutex_exit(&ism_mlist_lock); 8425 sfmmu_hat_unlock_all(); 8426 sfmmu_page_exit(pmtx); 8427 sfmmu_kpm_kpmp_exit(kpmp); 8428 kpreempt_enable(); 8429 } 8430 8431 /* 8432 * This function changes the virtual cacheability of all mappings to a 8433 * particular page. When changing from uncache to cacheable the mappings will 8434 * only be changed if all of them have the same virtual color. 8435 * We need to flush the cache in all cpus. It is possible that 8436 * a process referenced a page as cacheable but has sinced exited 8437 * and cleared the mapping list. We still to flush it but have no 8438 * state so all cpus is the only alternative. 8439 */ 8440 static void 8441 sfmmu_page_cache(page_t *pp, int flags, int cache_flush_flag, int bcolor) 8442 { 8443 struct sf_hment *sfhme; 8444 struct hme_blk *hmeblkp; 8445 sfmmu_t *sfmmup; 8446 tte_t tte, ttemod; 8447 caddr_t vaddr; 8448 int ret, color; 8449 pfn_t pfn; 8450 8451 color = bcolor; 8452 pfn = pp->p_pagenum; 8453 8454 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8455 8456 hmeblkp = sfmmu_hmetohblk(sfhme); 8457 8458 if (hmeblkp->hblk_xhat_bit) 8459 continue; 8460 8461 sfmmu_copytte(&sfhme->hme_tte, &tte); 8462 ASSERT(TTE_IS_VALID(&tte)); 8463 vaddr = tte_to_vaddr(hmeblkp, tte); 8464 color = addr_to_vcolor(vaddr); 8465 8466 #ifdef DEBUG 8467 if ((flags & HAT_CACHE) && bcolor != NO_VCOLOR) { 8468 ASSERT(color == bcolor); 8469 } 8470 #endif 8471 8472 ASSERT(flags != HAT_TMPNC || color == PP_GET_VCOLOR(pp)); 8473 8474 ttemod = tte; 8475 if (flags & (HAT_UNCACHE | HAT_TMPNC)) { 8476 TTE_CLR_VCACHEABLE(&ttemod); 8477 } else { /* flags & HAT_CACHE */ 8478 TTE_SET_VCACHEABLE(&ttemod); 8479 } 8480 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 8481 if (ret < 0) { 8482 /* 8483 * Since all cpus are captured modifytte should not 8484 * fail. 8485 */ 8486 panic("sfmmu_page_cache: write to tte failed"); 8487 } 8488 8489 sfmmup = hblktosfmmu(hmeblkp); 8490 if (cache_flush_flag == CACHE_FLUSH) { 8491 /* 8492 * Flush TSBs, TLBs and caches 8493 */ 8494 if (sfmmup->sfmmu_ismhat) { 8495 if (flags & HAT_CACHE) { 8496 SFMMU_STAT(sf_ism_recache); 8497 } else { 8498 SFMMU_STAT(sf_ism_uncache); 8499 } 8500 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8501 pfn, CACHE_FLUSH); 8502 } else { 8503 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp, 8504 pfn, 0, FLUSH_ALL_CPUS, CACHE_FLUSH, 1); 8505 } 8506 8507 /* 8508 * all cache entries belonging to this pfn are 8509 * now flushed. 8510 */ 8511 cache_flush_flag = CACHE_NO_FLUSH; 8512 } else { 8513 8514 /* 8515 * Flush only TSBs and TLBs. 8516 */ 8517 if (sfmmup->sfmmu_ismhat) { 8518 if (flags & HAT_CACHE) { 8519 SFMMU_STAT(sf_ism_recache); 8520 } else { 8521 SFMMU_STAT(sf_ism_uncache); 8522 } 8523 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8524 pfn, CACHE_NO_FLUSH); 8525 } else { 8526 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1); 8527 } 8528 } 8529 } 8530 8531 if (PP_ISMAPPED_KPM(pp)) 8532 sfmmu_kpm_page_cache(pp, flags, cache_flush_flag); 8533 8534 switch (flags) { 8535 8536 default: 8537 panic("sfmmu_pagecache: unknown flags"); 8538 break; 8539 8540 case HAT_CACHE: 8541 PP_CLRTNC(pp); 8542 PP_CLRPNC(pp); 8543 PP_SET_VCOLOR(pp, color); 8544 break; 8545 8546 case HAT_TMPNC: 8547 PP_SETTNC(pp); 8548 PP_SET_VCOLOR(pp, NO_VCOLOR); 8549 break; 8550 8551 case HAT_UNCACHE: 8552 PP_SETPNC(pp); 8553 PP_CLRTNC(pp); 8554 PP_SET_VCOLOR(pp, NO_VCOLOR); 8555 break; 8556 } 8557 } 8558 8559 /* 8560 * This routine gets called when the system has run out of free contexts. 8561 * This will simply choose context passed to it to be stolen and reused. 8562 */ 8563 /* ARGSUSED */ 8564 static void 8565 sfmmu_reuse_ctx(struct ctx *ctx, sfmmu_t *sfmmup) 8566 { 8567 sfmmu_t *stolen_sfmmup; 8568 cpuset_t cpuset; 8569 ushort_t cnum = ctxtoctxnum(ctx); 8570 8571 ASSERT(cnum != KCONTEXT); 8572 ASSERT(rw_read_locked(&ctx->ctx_rwlock) == 0); /* write locked */ 8573 8574 /* 8575 * simply steal and reuse the ctx passed to us. 8576 */ 8577 stolen_sfmmup = ctx->ctx_sfmmu; 8578 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8579 ASSERT(stolen_sfmmup->sfmmu_cnum == cnum); 8580 ASSERT(stolen_sfmmup != ksfmmup); 8581 8582 TRACE_CTXS(&ctx_trace_mutex, ctx_trace_ptr, cnum, stolen_sfmmup, 8583 sfmmup, CTX_TRC_STEAL); 8584 SFMMU_STAT(sf_ctxsteal); 8585 8586 /* 8587 * Update sfmmu and ctx structs. After this point all threads 8588 * belonging to this hat/proc will fault and not use the ctx 8589 * being stolen. 8590 */ 8591 kpreempt_disable(); 8592 /* 8593 * Enforce reverse order of assignments from sfmmu_get_ctx(). This 8594 * is done to prevent a race where a thread faults with the context 8595 * but the TSB has changed. 8596 */ 8597 stolen_sfmmup->sfmmu_cnum = INVALID_CONTEXT; 8598 membar_enter(); 8599 ctx->ctx_sfmmu = NULL; 8600 8601 /* 8602 * 1. flush TLB in all CPUs that ran the process whose ctx 8603 * we are stealing. 8604 * 2. change context for all other CPUs to INVALID_CONTEXT, 8605 * if they are running in the context that we are going to steal. 8606 */ 8607 cpuset = stolen_sfmmup->sfmmu_cpusran; 8608 CPUSET_DEL(cpuset, CPU->cpu_id); 8609 CPUSET_AND(cpuset, cpu_ready_set); 8610 SFMMU_XCALL_STATS(cnum); 8611 xt_some(cpuset, sfmmu_ctx_steal_tl1, cnum, INVALID_CONTEXT); 8612 xt_sync(cpuset); 8613 8614 /* 8615 * flush TLB of local processor 8616 */ 8617 vtag_flushctx(cnum); 8618 8619 /* 8620 * If we just stole the ctx from the current process 8621 * on local cpu then we also invalidate his context 8622 * here. 8623 */ 8624 if (sfmmu_getctx_sec() == cnum) { 8625 sfmmu_setctx_sec(INVALID_CONTEXT); 8626 sfmmu_clear_utsbinfo(); 8627 } 8628 8629 kpreempt_enable(); 8630 SFMMU_STAT(sf_tlbflush_ctx); 8631 } 8632 8633 /* 8634 * Returns a context with the reader lock held. 8635 * 8636 * We maintain 2 different list of contexts. The first list 8637 * is the free list and it is headed by ctxfree. These contexts 8638 * are ready to use. The second list is the dirty list and is 8639 * headed by ctxdirty. These contexts have been freed but haven't 8640 * been flushed from the TLB. 8641 * 8642 * It's the responsibility of the caller to guarantee that the 8643 * process serializes on calls here by taking the HAT lock for 8644 * the hat. 8645 * 8646 * Changing the page size is a rather complicated process, so 8647 * rather than jump through lots of hoops to special case it, 8648 * the easiest way to go about it is to tell the MMU we want 8649 * to change page sizes and then switch to using a different 8650 * context. When we program the context registers for the 8651 * process, we can take care of setting up the (new) page size 8652 * for that context at that point. 8653 */ 8654 8655 static struct ctx * 8656 sfmmu_get_ctx(sfmmu_t *sfmmup) 8657 { 8658 struct ctx *ctx; 8659 ushort_t cnum; 8660 struct ctx *lastctx = &ctxs[nctxs-1]; 8661 struct ctx *firstctx = &ctxs[NUM_LOCKED_CTXS]; 8662 uint_t found_stealable_ctx; 8663 uint_t retry_count = 0; 8664 8665 #define NEXT_CTX(ctx) (((ctx) >= lastctx) ? firstctx : ((ctx) + 1)) 8666 8667 retry: 8668 8669 ASSERT(sfmmup->sfmmu_cnum != KCONTEXT); 8670 /* 8671 * Check to see if this process has already got a ctx. 8672 * In that case just set the sec-ctx, grab a readers lock, and 8673 * return. 8674 * 8675 * We have to double check after we get the readers lock on the 8676 * context, since it could be stolen in this short window. 8677 */ 8678 if (sfmmup->sfmmu_cnum >= NUM_LOCKED_CTXS) { 8679 ctx = sfmmutoctx(sfmmup); 8680 rw_enter(&ctx->ctx_rwlock, RW_READER); 8681 if (ctx->ctx_sfmmu == sfmmup) { 8682 return (ctx); 8683 } else { 8684 rw_exit(&ctx->ctx_rwlock); 8685 } 8686 } 8687 8688 found_stealable_ctx = 0; 8689 mutex_enter(&ctx_list_lock); 8690 if ((ctx = ctxfree) != NULL) { 8691 /* 8692 * Found a ctx in free list. Delete it from the list and 8693 * use it. There's a short window where the stealer can 8694 * look at the context before we grab the lock on the 8695 * context, so we have to handle that with the free flag. 8696 */ 8697 SFMMU_STAT(sf_ctxfree); 8698 ctxfree = ctx->ctx_free; 8699 ctx->ctx_sfmmu = NULL; 8700 mutex_exit(&ctx_list_lock); 8701 rw_enter(&ctx->ctx_rwlock, RW_WRITER); 8702 ASSERT(ctx->ctx_sfmmu == NULL); 8703 ASSERT((ctx->ctx_flags & CTX_FREE_FLAG) != 0); 8704 } else if ((ctx = ctxdirty) != NULL) { 8705 /* 8706 * No free contexts. If we have at least one dirty ctx 8707 * then flush the TLBs on all cpus if necessary and move 8708 * the dirty list to the free list. 8709 */ 8710 SFMMU_STAT(sf_ctxdirty); 8711 ctxdirty = NULL; 8712 if (delay_tlb_flush) 8713 sfmmu_tlb_all_demap(); 8714 ctxfree = ctx->ctx_free; 8715 ctx->ctx_sfmmu = NULL; 8716 mutex_exit(&ctx_list_lock); 8717 rw_enter(&ctx->ctx_rwlock, RW_WRITER); 8718 ASSERT(ctx->ctx_sfmmu == NULL); 8719 ASSERT((ctx->ctx_flags & CTX_FREE_FLAG) != 0); 8720 } else { 8721 /* 8722 * No free context available, so steal one. 8723 * 8724 * The policy to choose the appropriate context is simple; 8725 * just sweep all the ctxs using ctxhand. This will steal 8726 * the LRU ctx. 8727 * 8728 * We however only steal a non-free context that can be 8729 * write locked. Keep searching till we find a stealable 8730 * ctx. 8731 */ 8732 mutex_exit(&ctx_list_lock); 8733 ctx = ctxhand; 8734 do { 8735 /* 8736 * If you get the writers lock, and the ctx isn't 8737 * a free ctx, THEN you can steal this ctx. 8738 */ 8739 if ((ctx->ctx_flags & CTX_FREE_FLAG) == 0 && 8740 rw_tryenter(&ctx->ctx_rwlock, RW_WRITER) != 0) { 8741 if (ctx->ctx_flags & CTX_FREE_FLAG) { 8742 /* let the first guy have it */ 8743 rw_exit(&ctx->ctx_rwlock); 8744 } else { 8745 found_stealable_ctx = 1; 8746 break; 8747 } 8748 } 8749 ctx = NEXT_CTX(ctx); 8750 } while (ctx != ctxhand); 8751 8752 if (found_stealable_ctx) { 8753 /* 8754 * Try and reuse the ctx. 8755 */ 8756 sfmmu_reuse_ctx(ctx, sfmmup); 8757 8758 } else if (retry_count++ < GET_CTX_RETRY_CNT) { 8759 goto retry; 8760 8761 } else { 8762 panic("Can't find any stealable context"); 8763 } 8764 } 8765 8766 ASSERT(rw_read_locked(&ctx->ctx_rwlock) == 0); /* write locked */ 8767 ctx->ctx_sfmmu = sfmmup; 8768 8769 /* 8770 * Clear the ctx_flags field. 8771 */ 8772 ctx->ctx_flags = 0; 8773 8774 cnum = ctxtoctxnum(ctx); 8775 membar_exit(); 8776 sfmmup->sfmmu_cnum = cnum; 8777 8778 /* 8779 * Let the MMU set up the page sizes to use for 8780 * this context in the TLB. Don't program 2nd dtlb for ism hat. 8781 */ 8782 if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) 8783 mmu_set_ctx_page_sizes(sfmmup); 8784 8785 /* 8786 * Downgrade to reader's lock. 8787 */ 8788 rw_downgrade(&ctx->ctx_rwlock); 8789 8790 /* 8791 * If this value doesn't get set to what we want 8792 * it won't matter, so don't worry about locking. 8793 */ 8794 ctxhand = NEXT_CTX(ctx); 8795 8796 /* 8797 * Better not have been stolen while we held the ctx' 8798 * lock or we're hosed. 8799 */ 8800 ASSERT(sfmmup == sfmmutoctx(sfmmup)->ctx_sfmmu); 8801 8802 return (ctx); 8803 8804 #undef NEXT_CTX 8805 } 8806 8807 8808 /* 8809 * Set the process context to INVALID_CONTEXT (but 8810 * without stealing the ctx) so that it faults and 8811 * reloads the MMU state from TL=0. Caller must 8812 * hold the hat lock since we don't acquire it here. 8813 */ 8814 static void 8815 sfmmu_sync_mmustate(sfmmu_t *sfmmup) 8816 { 8817 int cnum; 8818 cpuset_t cpuset; 8819 8820 ASSERT(sfmmup != ksfmmup); 8821 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8822 8823 kpreempt_disable(); 8824 8825 cnum = sfmmutoctxnum(sfmmup); 8826 if (cnum != INVALID_CONTEXT) { 8827 cpuset = sfmmup->sfmmu_cpusran; 8828 CPUSET_DEL(cpuset, CPU->cpu_id); 8829 CPUSET_AND(cpuset, cpu_ready_set); 8830 SFMMU_XCALL_STATS(cnum); 8831 8832 xt_some(cpuset, sfmmu_raise_tsb_exception, 8833 cnum, INVALID_CONTEXT); 8834 xt_sync(cpuset); 8835 8836 /* 8837 * If the process is running on the local CPU 8838 * we need to update the MMU state here as well. 8839 */ 8840 if (sfmmu_getctx_sec() == cnum) 8841 sfmmu_load_mmustate(sfmmup); 8842 8843 SFMMU_STAT(sf_tsb_raise_exception); 8844 } 8845 8846 kpreempt_enable(); 8847 } 8848 8849 8850 /* 8851 * Replace the specified TSB with a new TSB. This function gets called when 8852 * we grow, shrink or swapin a TSB. When swapping in a TSB (TSB_SWAPIN), the 8853 * TSB_FORCEALLOC flag may be used to force allocation of a minimum-sized TSB 8854 * (8K). 8855 * 8856 * Caller must hold the HAT lock, but should assume any tsb_info 8857 * pointers it has are no longer valid after calling this function. 8858 * 8859 * Return values: 8860 * TSB_ALLOCFAIL Failed to allocate a TSB, due to memory constraints 8861 * TSB_LOSTRACE HAT is busy, i.e. another thread is already doing 8862 * something to this tsbinfo/TSB 8863 * TSB_SUCCESS Operation succeeded 8864 */ 8865 static tsb_replace_rc_t 8866 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, 8867 hatlock_t *hatlockp, uint_t flags) 8868 { 8869 struct tsb_info *new_tsbinfo = NULL; 8870 struct tsb_info *curtsb, *prevtsb; 8871 uint_t tte_sz_mask; 8872 cpuset_t cpuset; 8873 struct ctx *ctx = NULL; 8874 int ctxnum; 8875 8876 ASSERT(sfmmup != ksfmmup); 8877 ASSERT(sfmmup->sfmmu_ismhat == 0); 8878 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8879 ASSERT(szc <= tsb_max_growsize); 8880 8881 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY)) 8882 return (TSB_LOSTRACE); 8883 8884 /* 8885 * Find the tsb_info ahead of this one in the list, and 8886 * also make sure that the tsb_info passed in really 8887 * exists! 8888 */ 8889 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 8890 curtsb != old_tsbinfo && curtsb != NULL; 8891 prevtsb = curtsb, curtsb = curtsb->tsb_next); 8892 ASSERT(curtsb != NULL); 8893 8894 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8895 /* 8896 * The process is swapped out, so just set the new size 8897 * code. When it swaps back in, we'll allocate a new one 8898 * of the new chosen size. 8899 */ 8900 curtsb->tsb_szc = szc; 8901 return (TSB_SUCCESS); 8902 } 8903 SFMMU_FLAGS_SET(sfmmup, HAT_BUSY); 8904 8905 tte_sz_mask = old_tsbinfo->tsb_ttesz_mask; 8906 8907 /* 8908 * All initialization is done inside of sfmmu_tsbinfo_alloc(). 8909 * If we fail to allocate a TSB, exit. 8910 */ 8911 sfmmu_hat_exit(hatlockp); 8912 if (sfmmu_tsbinfo_alloc(&new_tsbinfo, szc, tte_sz_mask, 8913 flags, sfmmup)) { 8914 (void) sfmmu_hat_enter(sfmmup); 8915 if (!(flags & TSB_SWAPIN)) 8916 SFMMU_STAT(sf_tsb_resize_failures); 8917 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8918 return (TSB_ALLOCFAIL); 8919 } 8920 (void) sfmmu_hat_enter(sfmmup); 8921 8922 /* 8923 * Re-check to make sure somebody else didn't muck with us while we 8924 * didn't hold the HAT lock. If the process swapped out, fine, just 8925 * exit; this can happen if we try to shrink the TSB from the context 8926 * of another process (such as on an ISM unmap), though it is rare. 8927 */ 8928 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8929 SFMMU_STAT(sf_tsb_resize_failures); 8930 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8931 sfmmu_hat_exit(hatlockp); 8932 sfmmu_tsbinfo_free(new_tsbinfo); 8933 (void) sfmmu_hat_enter(sfmmup); 8934 return (TSB_LOSTRACE); 8935 } 8936 8937 #ifdef DEBUG 8938 /* Reverify that the tsb_info still exists.. for debugging only */ 8939 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 8940 curtsb != old_tsbinfo && curtsb != NULL; 8941 prevtsb = curtsb, curtsb = curtsb->tsb_next); 8942 ASSERT(curtsb != NULL); 8943 #endif /* DEBUG */ 8944 8945 /* 8946 * Quiesce any CPUs running this process on their next TLB miss 8947 * so they atomically see the new tsb_info. We temporarily set the 8948 * context to invalid context so new threads that come on processor 8949 * after we do the xcall to cpusran will also serialize behind the 8950 * HAT lock on TLB miss and will see the new TSB. Since this short 8951 * race with a new thread coming on processor is relatively rare, 8952 * this synchronization mechanism should be cheaper than always 8953 * pausing all CPUs for the duration of the setup, which is what 8954 * the old implementation did. This is particuarly true if we are 8955 * copying a huge chunk of memory around during that window. 8956 * 8957 * The memory barriers are to make sure things stay consistent 8958 * with resume() since it does not hold the HAT lock while 8959 * walking the list of tsb_info structures. 8960 */ 8961 if ((flags & TSB_SWAPIN) != TSB_SWAPIN) { 8962 /* The TSB is either growing or shrinking. */ 8963 ctx = sfmmutoctx(sfmmup); 8964 rw_enter(&ctx->ctx_rwlock, RW_WRITER); 8965 8966 ctxnum = sfmmutoctxnum(sfmmup); 8967 sfmmup->sfmmu_cnum = INVALID_CONTEXT; 8968 membar_enter(); /* make sure visible on all CPUs */ 8969 8970 kpreempt_disable(); 8971 if (ctxnum != INVALID_CONTEXT) { 8972 cpuset = sfmmup->sfmmu_cpusran; 8973 CPUSET_DEL(cpuset, CPU->cpu_id); 8974 CPUSET_AND(cpuset, cpu_ready_set); 8975 SFMMU_XCALL_STATS(ctxnum); 8976 8977 xt_some(cpuset, sfmmu_raise_tsb_exception, 8978 ctxnum, INVALID_CONTEXT); 8979 xt_sync(cpuset); 8980 8981 SFMMU_STAT(sf_tsb_raise_exception); 8982 } 8983 kpreempt_enable(); 8984 } else { 8985 /* 8986 * It is illegal to swap in TSBs from a process other 8987 * than a process being swapped in. This in turn 8988 * implies we do not have a valid MMU context here 8989 * since a process needs one to resolve translation 8990 * misses. 8991 */ 8992 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup); 8993 ASSERT(sfmmutoctxnum(sfmmup) == INVALID_CONTEXT); 8994 } 8995 8996 new_tsbinfo->tsb_next = old_tsbinfo->tsb_next; 8997 membar_stst(); /* strict ordering required */ 8998 if (prevtsb) 8999 prevtsb->tsb_next = new_tsbinfo; 9000 else 9001 sfmmup->sfmmu_tsb = new_tsbinfo; 9002 membar_enter(); /* make sure new TSB globally visible */ 9003 sfmmu_setup_tsbinfo(sfmmup); 9004 9005 /* 9006 * We need to migrate TSB entries from the old TSB to the new TSB 9007 * if tsb_remap_ttes is set and the TSB is growing. 9008 */ 9009 if (tsb_remap_ttes && ((flags & TSB_GROW) == TSB_GROW)) 9010 sfmmu_copy_tsb(old_tsbinfo, new_tsbinfo); 9011 9012 if ((flags & TSB_SWAPIN) != TSB_SWAPIN) { 9013 kpreempt_disable(); 9014 membar_exit(); 9015 sfmmup->sfmmu_cnum = ctxnum; 9016 if (ctxnum != INVALID_CONTEXT && 9017 sfmmu_getctx_sec() == ctxnum) { 9018 sfmmu_load_mmustate(sfmmup); 9019 } 9020 kpreempt_enable(); 9021 rw_exit(&ctx->ctx_rwlock); 9022 } 9023 9024 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 9025 9026 /* 9027 * Drop the HAT lock to free our old tsb_info. 9028 */ 9029 sfmmu_hat_exit(hatlockp); 9030 9031 if ((flags & TSB_GROW) == TSB_GROW) { 9032 SFMMU_STAT(sf_tsb_grow); 9033 } else if ((flags & TSB_SHRINK) == TSB_SHRINK) { 9034 SFMMU_STAT(sf_tsb_shrink); 9035 } 9036 9037 sfmmu_tsbinfo_free(old_tsbinfo); 9038 9039 (void) sfmmu_hat_enter(sfmmup); 9040 return (TSB_SUCCESS); 9041 } 9042 9043 /* 9044 * Steal context from process, forcing the process to switch to another 9045 * context on the next TLB miss, and therefore start using the TLB that 9046 * is reprogrammed for the new page sizes. 9047 */ 9048 void 9049 sfmmu_steal_context(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) 9050 { 9051 struct ctx *ctx; 9052 int i, cnum; 9053 hatlock_t *hatlockp = NULL; 9054 9055 hatlockp = sfmmu_hat_enter(sfmmup); 9056 /* USIII+-IV+ optimization, requires hat lock */ 9057 if (tmp_pgsz) { 9058 for (i = 0; i < mmu_page_sizes; i++) 9059 sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i]; 9060 } 9061 SFMMU_STAT(sf_tlb_reprog_pgsz); 9062 ctx = sfmmutoctx(sfmmup); 9063 rw_enter(&ctx->ctx_rwlock, RW_WRITER); 9064 cnum = sfmmutoctxnum(sfmmup); 9065 9066 if (cnum != INVALID_CONTEXT) { 9067 sfmmu_tlb_swap_ctx(sfmmup, ctx); 9068 } 9069 rw_exit(&ctx->ctx_rwlock); 9070 sfmmu_hat_exit(hatlockp); 9071 } 9072 9073 /* 9074 * This function assumes that there are either four or six supported page 9075 * sizes and at most two programmable TLBs, so we need to decide which 9076 * page sizes are most important and then tell the MMU layer so it 9077 * can adjust the TLB page sizes accordingly (if supported). 9078 * 9079 * If these assumptions change, this function will need to be 9080 * updated to support whatever the new limits are. 9081 * 9082 * The growing flag is nonzero if we are growing the address space, 9083 * and zero if it is shrinking. This allows us to decide whether 9084 * to grow or shrink our TSB, depending upon available memory 9085 * conditions. 9086 */ 9087 static void 9088 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing) 9089 { 9090 uint64_t ttecnt[MMU_PAGE_SIZES]; 9091 uint64_t tte8k_cnt, tte4m_cnt; 9092 uint8_t i; 9093 int sectsb_thresh; 9094 9095 /* 9096 * Kernel threads, processes with small address spaces not using 9097 * large pages, and dummy ISM HATs need not apply. 9098 */ 9099 if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != NULL) 9100 return; 9101 9102 if ((sfmmup->sfmmu_flags & HAT_LGPG_FLAGS) == 0 && 9103 sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor) 9104 return; 9105 9106 for (i = 0; i < mmu_page_sizes; i++) { 9107 ttecnt[i] = SFMMU_TTE_CNT(sfmmup, i); 9108 } 9109 9110 /* Check pagesizes in use, and possibly reprogram DTLB. */ 9111 if (&mmu_check_page_sizes) 9112 mmu_check_page_sizes(sfmmup, ttecnt); 9113 9114 /* 9115 * Calculate the number of 8k ttes to represent the span of these 9116 * pages. 9117 */ 9118 tte8k_cnt = ttecnt[TTE8K] + 9119 (ttecnt[TTE64K] << (MMU_PAGESHIFT64K - MMU_PAGESHIFT)) + 9120 (ttecnt[TTE512K] << (MMU_PAGESHIFT512K - MMU_PAGESHIFT)); 9121 if (mmu_page_sizes == max_mmu_page_sizes) { 9122 tte4m_cnt = ttecnt[TTE4M] + 9123 (ttecnt[TTE32M] << (MMU_PAGESHIFT32M - MMU_PAGESHIFT4M)) + 9124 (ttecnt[TTE256M] << (MMU_PAGESHIFT256M - MMU_PAGESHIFT4M)); 9125 } else { 9126 tte4m_cnt = ttecnt[TTE4M]; 9127 } 9128 9129 /* 9130 * Inflate TSB sizes by a factor of 2 if this process 9131 * uses 4M text pages to minimize extra conflict misses 9132 * in the first TSB since without counting text pages 9133 * 8K TSB may become too small. 9134 * 9135 * Also double the size of the second TSB to minimize 9136 * extra conflict misses due to competition between 4M text pages 9137 * and data pages. 9138 * 9139 * We need to adjust the second TSB allocation threshold by the 9140 * inflation factor, since there is no point in creating a second 9141 * TSB when we know all the mappings can fit in the I/D TLBs. 9142 */ 9143 sectsb_thresh = tsb_sectsb_threshold; 9144 if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) { 9145 tte8k_cnt <<= 1; 9146 tte4m_cnt <<= 1; 9147 sectsb_thresh <<= 1; 9148 } 9149 9150 /* 9151 * Check to see if our TSB is the right size; we may need to 9152 * grow or shrink it. If the process is small, our work is 9153 * finished at this point. 9154 */ 9155 if (tte8k_cnt <= tsb_rss_factor && tte4m_cnt <= sectsb_thresh) { 9156 return; 9157 } 9158 sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh); 9159 } 9160 9161 static void 9162 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt, 9163 uint64_t tte4m_cnt, int sectsb_thresh) 9164 { 9165 int tsb_bits; 9166 uint_t tsb_szc; 9167 struct tsb_info *tsbinfop; 9168 hatlock_t *hatlockp = NULL; 9169 9170 hatlockp = sfmmu_hat_enter(sfmmup); 9171 ASSERT(hatlockp != NULL); 9172 tsbinfop = sfmmup->sfmmu_tsb; 9173 ASSERT(tsbinfop != NULL); 9174 9175 /* 9176 * If we're growing, select the size based on RSS. If we're 9177 * shrinking, leave some room so we don't have to turn around and 9178 * grow again immediately. 9179 */ 9180 if (growing) 9181 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt); 9182 else 9183 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt << 1); 9184 9185 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9186 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9187 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9188 hatlockp, TSB_SHRINK); 9189 } else if (growing && tsb_szc > tsbinfop->tsb_szc && TSB_OK_GROW()) { 9190 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9191 hatlockp, TSB_GROW); 9192 } 9193 tsbinfop = sfmmup->sfmmu_tsb; 9194 9195 /* 9196 * With the TLB and first TSB out of the way, we need to see if 9197 * we need a second TSB for 4M pages. If we managed to reprogram 9198 * the TLB page sizes above, the process will start using this new 9199 * TSB right away; otherwise, it will start using it on the next 9200 * context switch. Either way, it's no big deal so there's no 9201 * synchronization with the trap handlers here unless we grow the 9202 * TSB (in which case it's required to prevent using the old one 9203 * after it's freed). Note: second tsb is required for 32M/256M 9204 * page sizes. 9205 */ 9206 if (tte4m_cnt > sectsb_thresh) { 9207 /* 9208 * If we're growing, select the size based on RSS. If we're 9209 * shrinking, leave some room so we don't have to turn 9210 * around and grow again immediately. 9211 */ 9212 if (growing) 9213 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt); 9214 else 9215 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt << 1); 9216 if (tsbinfop->tsb_next == NULL) { 9217 struct tsb_info *newtsb; 9218 int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)? 9219 0 : TSB_ALLOC; 9220 9221 sfmmu_hat_exit(hatlockp); 9222 9223 /* 9224 * Try to allocate a TSB for 4[32|256]M pages. If we 9225 * can't get the size we want, retry w/a minimum sized 9226 * TSB. If that still didn't work, give up; we can 9227 * still run without one. 9228 */ 9229 tsb_bits = (mmu_page_sizes == max_mmu_page_sizes)? 9230 TSB4M|TSB32M|TSB256M:TSB4M; 9231 if ((sfmmu_tsbinfo_alloc(&newtsb, tsb_szc, tsb_bits, 9232 allocflags, sfmmup) != 0) && 9233 (sfmmu_tsbinfo_alloc(&newtsb, TSB_MIN_SZCODE, 9234 tsb_bits, allocflags, sfmmup) != 0)) { 9235 return; 9236 } 9237 9238 hatlockp = sfmmu_hat_enter(sfmmup); 9239 9240 if (sfmmup->sfmmu_tsb->tsb_next == NULL) { 9241 sfmmup->sfmmu_tsb->tsb_next = newtsb; 9242 SFMMU_STAT(sf_tsb_sectsb_create); 9243 sfmmu_setup_tsbinfo(sfmmup); 9244 sfmmu_hat_exit(hatlockp); 9245 return; 9246 } else { 9247 /* 9248 * It's annoying, but possible for us 9249 * to get here.. we dropped the HAT lock 9250 * because of locking order in the kmem 9251 * allocator, and while we were off getting 9252 * our memory, some other thread decided to 9253 * do us a favor and won the race to get a 9254 * second TSB for this process. Sigh. 9255 */ 9256 sfmmu_hat_exit(hatlockp); 9257 sfmmu_tsbinfo_free(newtsb); 9258 return; 9259 } 9260 } 9261 9262 /* 9263 * We have a second TSB, see if it's big enough. 9264 */ 9265 tsbinfop = tsbinfop->tsb_next; 9266 9267 /* 9268 * Check to see if our second TSB is the right size; 9269 * we may need to grow or shrink it. 9270 * To prevent thrashing (e.g. growing the TSB on a 9271 * subsequent map operation), only try to shrink if 9272 * the TSB reach exceeds twice the virtual address 9273 * space size. 9274 */ 9275 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9276 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9277 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9278 tsb_szc, hatlockp, TSB_SHRINK); 9279 } else if (growing && tsb_szc > tsbinfop->tsb_szc && 9280 TSB_OK_GROW()) { 9281 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9282 tsb_szc, hatlockp, TSB_GROW); 9283 } 9284 } 9285 9286 sfmmu_hat_exit(hatlockp); 9287 } 9288 9289 /* 9290 * Get the preferred page size code for a hat. 9291 * This is only advice, so locking is not done; 9292 * this transitory information could change 9293 * following the call anyway. This interface is 9294 * sun4 private. 9295 */ 9296 /*ARGSUSED*/ 9297 uint_t 9298 hat_preferred_pgsz(struct hat *hat, caddr_t vaddr, size_t maplen, int maptype) 9299 { 9300 sfmmu_t *sfmmup = (sfmmu_t *)hat; 9301 uint_t szc, maxszc = mmu_page_sizes - 1; 9302 size_t pgsz; 9303 9304 if (maptype == MAPPGSZ_ISM) { 9305 for (szc = maxszc; szc >= TTE4M; szc--) { 9306 if (disable_ism_large_pages & (1 << szc)) 9307 continue; 9308 9309 pgsz = hw_page_array[szc].hp_size; 9310 if ((maplen >= pgsz) && IS_P2ALIGNED(vaddr, pgsz)) 9311 return (szc); 9312 } 9313 return (TTE4M); 9314 } else if (&mmu_preferred_pgsz) { /* USIII+-USIV+ */ 9315 return (mmu_preferred_pgsz(sfmmup, vaddr, maplen)); 9316 } else { /* USIII, USII, Niagara */ 9317 for (szc = maxszc; szc > TTE8K; szc--) { 9318 if (disable_large_pages & (1 << szc)) 9319 continue; 9320 9321 pgsz = hw_page_array[szc].hp_size; 9322 if ((maplen >= pgsz) && IS_P2ALIGNED(vaddr, pgsz)) 9323 return (szc); 9324 } 9325 return (TTE8K); 9326 } 9327 } 9328 9329 /* 9330 * Free up a ctx 9331 */ 9332 static void 9333 sfmmu_free_ctx(sfmmu_t *sfmmup, struct ctx *ctx) 9334 { 9335 int ctxnum; 9336 9337 rw_enter(&ctx->ctx_rwlock, RW_WRITER); 9338 9339 TRACE_CTXS(&ctx_trace_mutex, ctx_trace_ptr, sfmmup->sfmmu_cnum, 9340 sfmmup, 0, CTX_TRC_FREE); 9341 9342 if (sfmmup->sfmmu_cnum == INVALID_CONTEXT) { 9343 CPUSET_ZERO(sfmmup->sfmmu_cpusran); 9344 rw_exit(&ctx->ctx_rwlock); 9345 return; 9346 } 9347 9348 ASSERT(sfmmup == ctx->ctx_sfmmu); 9349 9350 ctx->ctx_sfmmu = NULL; 9351 ctx->ctx_flags = 0; 9352 sfmmup->sfmmu_cnum = INVALID_CONTEXT; 9353 membar_enter(); 9354 CPUSET_ZERO(sfmmup->sfmmu_cpusran); 9355 ctxnum = sfmmu_getctx_sec(); 9356 if (ctxnum == ctxtoctxnum(ctx)) { 9357 sfmmu_setctx_sec(INVALID_CONTEXT); 9358 sfmmu_clear_utsbinfo(); 9359 } 9360 9361 /* 9362 * Put the freed ctx on the dirty list 9363 */ 9364 mutex_enter(&ctx_list_lock); 9365 CTX_SET_FLAGS(ctx, CTX_FREE_FLAG); 9366 ctx->ctx_free = ctxdirty; 9367 ctxdirty = ctx; 9368 mutex_exit(&ctx_list_lock); 9369 9370 rw_exit(&ctx->ctx_rwlock); 9371 } 9372 9373 /* 9374 * Free up a sfmmu 9375 * Since the sfmmu is currently embedded in the hat struct we simply zero 9376 * out our fields and free up the ism map blk list if any. 9377 */ 9378 static void 9379 sfmmu_free_sfmmu(sfmmu_t *sfmmup) 9380 { 9381 ism_blk_t *blkp, *nx_blkp; 9382 #ifdef DEBUG 9383 ism_map_t *map; 9384 int i; 9385 #endif 9386 9387 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 9388 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 9389 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 9390 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 9391 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 9392 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 9393 ASSERT(sfmmup->sfmmu_cnum == INVALID_CONTEXT); 9394 sfmmup->sfmmu_free = 0; 9395 sfmmup->sfmmu_ismhat = 0; 9396 9397 blkp = sfmmup->sfmmu_iblk; 9398 sfmmup->sfmmu_iblk = NULL; 9399 9400 while (blkp) { 9401 #ifdef DEBUG 9402 map = blkp->iblk_maps; 9403 for (i = 0; i < ISM_MAP_SLOTS; i++) { 9404 ASSERT(map[i].imap_seg == 0); 9405 ASSERT(map[i].imap_ismhat == NULL); 9406 ASSERT(map[i].imap_ment == NULL); 9407 } 9408 #endif 9409 nx_blkp = blkp->iblk_next; 9410 blkp->iblk_next = NULL; 9411 blkp->iblk_nextpa = (uint64_t)-1; 9412 kmem_cache_free(ism_blk_cache, blkp); 9413 blkp = nx_blkp; 9414 } 9415 } 9416 9417 /* 9418 * Locking primitves accessed by HATLOCK macros 9419 */ 9420 9421 #define SFMMU_SPL_MTX (0x0) 9422 #define SFMMU_ML_MTX (0x1) 9423 9424 #define SFMMU_MLSPL_MTX(type, pg) (((type) == SFMMU_SPL_MTX) ? \ 9425 SPL_HASH(pg) : MLIST_HASH(pg)) 9426 9427 kmutex_t * 9428 sfmmu_page_enter(struct page *pp) 9429 { 9430 return (sfmmu_mlspl_enter(pp, SFMMU_SPL_MTX)); 9431 } 9432 9433 static void 9434 sfmmu_page_exit(kmutex_t *spl) 9435 { 9436 mutex_exit(spl); 9437 } 9438 9439 static int 9440 sfmmu_page_spl_held(struct page *pp) 9441 { 9442 return (sfmmu_mlspl_held(pp, SFMMU_SPL_MTX)); 9443 } 9444 9445 kmutex_t * 9446 sfmmu_mlist_enter(struct page *pp) 9447 { 9448 return (sfmmu_mlspl_enter(pp, SFMMU_ML_MTX)); 9449 } 9450 9451 void 9452 sfmmu_mlist_exit(kmutex_t *mml) 9453 { 9454 mutex_exit(mml); 9455 } 9456 9457 int 9458 sfmmu_mlist_held(struct page *pp) 9459 { 9460 9461 return (sfmmu_mlspl_held(pp, SFMMU_ML_MTX)); 9462 } 9463 9464 /* 9465 * Common code for sfmmu_mlist_enter() and sfmmu_page_enter(). For 9466 * sfmmu_mlist_enter() case mml_table lock array is used and for 9467 * sfmmu_page_enter() sfmmu_page_lock lock array is used. 9468 * 9469 * The lock is taken on a root page so that it protects an operation on all 9470 * constituent pages of a large page pp belongs to. 9471 * 9472 * The routine takes a lock from the appropriate array. The lock is determined 9473 * by hashing the root page. After taking the lock this routine checks if the 9474 * root page has the same size code that was used to determine the root (i.e 9475 * that root hasn't changed). If root page has the expected p_szc field we 9476 * have the right lock and it's returned to the caller. If root's p_szc 9477 * decreased we release the lock and retry from the beginning. This case can 9478 * happen due to hat_page_demote() decreasing p_szc between our load of p_szc 9479 * value and taking the lock. The number of retries due to p_szc decrease is 9480 * limited by the maximum p_szc value. If p_szc is 0 we return the lock 9481 * determined by hashing pp itself. 9482 * 9483 * If our caller doesn't hold a SE_SHARED or SE_EXCL lock on pp it's also 9484 * possible that p_szc can increase. To increase p_szc a thread has to lock 9485 * all constituent pages EXCL and do hat_pageunload() on all of them. All the 9486 * callers that don't hold a page locked recheck if hmeblk through which pp 9487 * was found still maps this pp. If it doesn't map it anymore returned lock 9488 * is immediately dropped. Therefore if sfmmu_mlspl_enter() hits the case of 9489 * p_szc increase after taking the lock it returns this lock without further 9490 * retries because in this case the caller doesn't care about which lock was 9491 * taken. The caller will drop it right away. 9492 * 9493 * After the routine returns it's guaranteed that hat_page_demote() can't 9494 * change p_szc field of any of constituent pages of a large page pp belongs 9495 * to as long as pp was either locked at least SHARED prior to this call or 9496 * the caller finds that hment that pointed to this pp still references this 9497 * pp (this also assumes that the caller holds hme hash bucket lock so that 9498 * the same pp can't be remapped into the same hmeblk after it was unmapped by 9499 * hat_pageunload()). 9500 */ 9501 static kmutex_t * 9502 sfmmu_mlspl_enter(struct page *pp, int type) 9503 { 9504 kmutex_t *mtx; 9505 uint_t prev_rszc = UINT_MAX; 9506 page_t *rootpp; 9507 uint_t szc; 9508 uint_t rszc; 9509 uint_t pszc = pp->p_szc; 9510 9511 ASSERT(pp != NULL); 9512 9513 again: 9514 if (pszc == 0) { 9515 mtx = SFMMU_MLSPL_MTX(type, pp); 9516 mutex_enter(mtx); 9517 return (mtx); 9518 } 9519 9520 /* The lock lives in the root page */ 9521 rootpp = PP_GROUPLEADER(pp, pszc); 9522 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9523 mutex_enter(mtx); 9524 9525 /* 9526 * Return mml in the following 3 cases: 9527 * 9528 * 1) If pp itself is root since if its p_szc decreased before we took 9529 * the lock pp is still the root of smaller szc page. And if its p_szc 9530 * increased it doesn't matter what lock we return (see comment in 9531 * front of this routine). 9532 * 9533 * 2) If pp's not root but rootpp is the root of a rootpp->p_szc size 9534 * large page we have the right lock since any previous potential 9535 * hat_page_demote() is done demoting from greater than current root's 9536 * p_szc because hat_page_demote() changes root's p_szc last. No 9537 * further hat_page_demote() can start or be in progress since it 9538 * would need the same lock we currently hold. 9539 * 9540 * 3) If rootpp's p_szc increased since previous iteration it doesn't 9541 * matter what lock we return (see comment in front of this routine). 9542 */ 9543 if (pp == rootpp || (rszc = rootpp->p_szc) == pszc || 9544 rszc >= prev_rszc) { 9545 return (mtx); 9546 } 9547 9548 /* 9549 * hat_page_demote() could have decreased root's p_szc. 9550 * In this case pp's p_szc must also be smaller than pszc. 9551 * Retry. 9552 */ 9553 if (rszc < pszc) { 9554 szc = pp->p_szc; 9555 if (szc < pszc) { 9556 mutex_exit(mtx); 9557 pszc = szc; 9558 goto again; 9559 } 9560 /* 9561 * pp's p_szc increased after it was decreased. 9562 * page cannot be mapped. Return current lock. The caller 9563 * will drop it right away. 9564 */ 9565 return (mtx); 9566 } 9567 9568 /* 9569 * root's p_szc is greater than pp's p_szc. 9570 * hat_page_demote() is not done with all pages 9571 * yet. Wait for it to complete. 9572 */ 9573 mutex_exit(mtx); 9574 rootpp = PP_GROUPLEADER(rootpp, rszc); 9575 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9576 mutex_enter(mtx); 9577 mutex_exit(mtx); 9578 prev_rszc = rszc; 9579 goto again; 9580 } 9581 9582 static int 9583 sfmmu_mlspl_held(struct page *pp, int type) 9584 { 9585 kmutex_t *mtx; 9586 9587 ASSERT(pp != NULL); 9588 /* The lock lives in the root page */ 9589 pp = PP_PAGEROOT(pp); 9590 ASSERT(pp != NULL); 9591 9592 mtx = SFMMU_MLSPL_MTX(type, pp); 9593 return (MUTEX_HELD(mtx)); 9594 } 9595 9596 static uint_t 9597 sfmmu_get_free_hblk(struct hme_blk **hmeblkpp, uint_t critical) 9598 { 9599 struct hme_blk *hblkp; 9600 9601 if (freehblkp != NULL) { 9602 mutex_enter(&freehblkp_lock); 9603 if (freehblkp != NULL) { 9604 /* 9605 * If the current thread is owning hblk_reserve, 9606 * let it succede even if freehblkcnt is really low. 9607 */ 9608 if (freehblkcnt <= HBLK_RESERVE_MIN && !critical) { 9609 SFMMU_STAT(sf_get_free_throttle); 9610 mutex_exit(&freehblkp_lock); 9611 return (0); 9612 } 9613 freehblkcnt--; 9614 *hmeblkpp = freehblkp; 9615 hblkp = *hmeblkpp; 9616 freehblkp = hblkp->hblk_next; 9617 mutex_exit(&freehblkp_lock); 9618 hblkp->hblk_next = NULL; 9619 SFMMU_STAT(sf_get_free_success); 9620 return (1); 9621 } 9622 mutex_exit(&freehblkp_lock); 9623 } 9624 SFMMU_STAT(sf_get_free_fail); 9625 return (0); 9626 } 9627 9628 static uint_t 9629 sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical) 9630 { 9631 struct hme_blk *hblkp; 9632 9633 /* 9634 * If the current thread is mapping into kernel space, 9635 * let it succede even if freehblkcnt is max 9636 * so that it will avoid freeing it to kmem. 9637 * This will prevent stack overflow due to 9638 * possible recursion since kmem_cache_free() 9639 * might require creation of a slab which 9640 * in turn needs an hmeblk to map that slab; 9641 * let's break this vicious chain at the first 9642 * opportunity. 9643 */ 9644 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9645 mutex_enter(&freehblkp_lock); 9646 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9647 SFMMU_STAT(sf_put_free_success); 9648 freehblkcnt++; 9649 hmeblkp->hblk_next = freehblkp; 9650 freehblkp = hmeblkp; 9651 mutex_exit(&freehblkp_lock); 9652 return (1); 9653 } 9654 mutex_exit(&freehblkp_lock); 9655 } 9656 9657 /* 9658 * Bring down freehblkcnt to HBLK_RESERVE_CNT. We are here 9659 * only if freehblkcnt is at least HBLK_RESERVE_CNT *and* 9660 * we are not in the process of mapping into kernel space. 9661 */ 9662 ASSERT(!critical); 9663 while (freehblkcnt > HBLK_RESERVE_CNT) { 9664 mutex_enter(&freehblkp_lock); 9665 if (freehblkcnt > HBLK_RESERVE_CNT) { 9666 freehblkcnt--; 9667 hblkp = freehblkp; 9668 freehblkp = hblkp->hblk_next; 9669 mutex_exit(&freehblkp_lock); 9670 ASSERT(get_hblk_cache(hblkp) == sfmmu8_cache); 9671 kmem_cache_free(sfmmu8_cache, hblkp); 9672 continue; 9673 } 9674 mutex_exit(&freehblkp_lock); 9675 } 9676 SFMMU_STAT(sf_put_free_fail); 9677 return (0); 9678 } 9679 9680 static void 9681 sfmmu_hblk_swap(struct hme_blk *new) 9682 { 9683 struct hme_blk *old, *hblkp, *prev; 9684 uint64_t hblkpa, prevpa, newpa; 9685 caddr_t base, vaddr, endaddr; 9686 struct hmehash_bucket *hmebp; 9687 struct sf_hment *osfhme, *nsfhme; 9688 page_t *pp; 9689 kmutex_t *pml; 9690 tte_t tte; 9691 9692 #ifdef DEBUG 9693 hmeblk_tag hblktag; 9694 struct hme_blk *found; 9695 #endif 9696 old = HBLK_RESERVE; 9697 9698 /* 9699 * save pa before bcopy clobbers it 9700 */ 9701 newpa = new->hblk_nextpa; 9702 9703 base = (caddr_t)get_hblk_base(old); 9704 endaddr = base + get_hblk_span(old); 9705 9706 /* 9707 * acquire hash bucket lock. 9708 */ 9709 hmebp = sfmmu_tteload_acquire_hashbucket(ksfmmup, base, TTE8K); 9710 9711 /* 9712 * copy contents from old to new 9713 */ 9714 bcopy((void *)old, (void *)new, HME8BLK_SZ); 9715 9716 /* 9717 * add new to hash chain 9718 */ 9719 sfmmu_hblk_hash_add(hmebp, new, newpa); 9720 9721 /* 9722 * search hash chain for hblk_reserve; this needs to be performed 9723 * after adding new, otherwise prevpa and prev won't correspond 9724 * to the hblk which is prior to old in hash chain when we call 9725 * sfmmu_hblk_hash_rm to remove old later. 9726 */ 9727 for (prevpa = 0, prev = NULL, 9728 hblkpa = hmebp->hmeh_nextpa, hblkp = hmebp->hmeblkp; 9729 hblkp != NULL && hblkp != old; 9730 prevpa = hblkpa, prev = hblkp, 9731 hblkpa = hblkp->hblk_nextpa, hblkp = hblkp->hblk_next); 9732 9733 if (hblkp != old) 9734 panic("sfmmu_hblk_swap: hblk_reserve not found"); 9735 9736 /* 9737 * p_mapping list is still pointing to hments in hblk_reserve; 9738 * fix up p_mapping list so that they point to hments in new. 9739 * 9740 * Since all these mappings are created by hblk_reserve_thread 9741 * on the way and it's using at least one of the buffers from each of 9742 * the newly minted slabs, there is no danger of any of these 9743 * mappings getting unloaded by another thread. 9744 * 9745 * tsbmiss could only modify ref/mod bits of hments in old/new. 9746 * Since all of these hments hold mappings established by segkmem 9747 * and mappings in segkmem are setup with HAT_NOSYNC, ref/mod bits 9748 * have no meaning for the mappings in hblk_reserve. hments in 9749 * old and new are identical except for ref/mod bits. 9750 */ 9751 for (vaddr = base; vaddr < endaddr; vaddr += TTEBYTES(TTE8K)) { 9752 9753 HBLKTOHME(osfhme, old, vaddr); 9754 sfmmu_copytte(&osfhme->hme_tte, &tte); 9755 9756 if (TTE_IS_VALID(&tte)) { 9757 if ((pp = osfhme->hme_page) == NULL) 9758 panic("sfmmu_hblk_swap: page not mapped"); 9759 9760 pml = sfmmu_mlist_enter(pp); 9761 9762 if (pp != osfhme->hme_page) 9763 panic("sfmmu_hblk_swap: mapping changed"); 9764 9765 HBLKTOHME(nsfhme, new, vaddr); 9766 9767 HME_ADD(nsfhme, pp); 9768 HME_SUB(osfhme, pp); 9769 9770 sfmmu_mlist_exit(pml); 9771 } 9772 } 9773 9774 /* 9775 * remove old from hash chain 9776 */ 9777 sfmmu_hblk_hash_rm(hmebp, old, prevpa, prev); 9778 9779 #ifdef DEBUG 9780 9781 hblktag.htag_id = ksfmmup; 9782 hblktag.htag_bspage = HME_HASH_BSPAGE(base, HME_HASH_SHIFT(TTE8K)); 9783 hblktag.htag_rehash = HME_HASH_REHASH(TTE8K); 9784 HME_HASH_FAST_SEARCH(hmebp, hblktag, found); 9785 9786 if (found != new) 9787 panic("sfmmu_hblk_swap: new hblk not found"); 9788 #endif 9789 9790 SFMMU_HASH_UNLOCK(hmebp); 9791 9792 /* 9793 * Reset hblk_reserve 9794 */ 9795 bzero((void *)old, HME8BLK_SZ); 9796 old->hblk_nextpa = va_to_pa((caddr_t)old); 9797 } 9798 9799 /* 9800 * Grab the mlist mutex for both pages passed in. 9801 * 9802 * low and high will be returned as pointers to the mutexes for these pages. 9803 * low refers to the mutex residing in the lower bin of the mlist hash, while 9804 * high refers to the mutex residing in the higher bin of the mlist hash. This 9805 * is due to the locking order restrictions on the same thread grabbing 9806 * multiple mlist mutexes. The low lock must be acquired before the high lock. 9807 * 9808 * If both pages hash to the same mutex, only grab that single mutex, and 9809 * high will be returned as NULL 9810 * If the pages hash to different bins in the hash, grab the lower addressed 9811 * lock first and then the higher addressed lock in order to follow the locking 9812 * rules involved with the same thread grabbing multiple mlist mutexes. 9813 * low and high will both have non-NULL values. 9814 */ 9815 static void 9816 sfmmu_mlist_reloc_enter(struct page *targ, struct page *repl, 9817 kmutex_t **low, kmutex_t **high) 9818 { 9819 kmutex_t *mml_targ, *mml_repl; 9820 9821 /* 9822 * no need to do the dance around szc as in sfmmu_mlist_enter() 9823 * because this routine is only called by hat_page_relocate() and all 9824 * targ and repl pages are already locked EXCL so szc can't change. 9825 */ 9826 9827 mml_targ = MLIST_HASH(PP_PAGEROOT(targ)); 9828 mml_repl = MLIST_HASH(PP_PAGEROOT(repl)); 9829 9830 if (mml_targ == mml_repl) { 9831 *low = mml_targ; 9832 *high = NULL; 9833 } else { 9834 if (mml_targ < mml_repl) { 9835 *low = mml_targ; 9836 *high = mml_repl; 9837 } else { 9838 *low = mml_repl; 9839 *high = mml_targ; 9840 } 9841 } 9842 9843 mutex_enter(*low); 9844 if (*high) 9845 mutex_enter(*high); 9846 } 9847 9848 static void 9849 sfmmu_mlist_reloc_exit(kmutex_t *low, kmutex_t *high) 9850 { 9851 if (high) 9852 mutex_exit(high); 9853 mutex_exit(low); 9854 } 9855 9856 static hatlock_t * 9857 sfmmu_hat_enter(sfmmu_t *sfmmup) 9858 { 9859 hatlock_t *hatlockp; 9860 9861 if (sfmmup != ksfmmup) { 9862 hatlockp = TSB_HASH(sfmmup); 9863 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 9864 return (hatlockp); 9865 } 9866 return (NULL); 9867 } 9868 9869 static hatlock_t * 9870 sfmmu_hat_tryenter(sfmmu_t *sfmmup) 9871 { 9872 hatlock_t *hatlockp; 9873 9874 if (sfmmup != ksfmmup) { 9875 hatlockp = TSB_HASH(sfmmup); 9876 if (mutex_tryenter(HATLOCK_MUTEXP(hatlockp)) == 0) 9877 return (NULL); 9878 return (hatlockp); 9879 } 9880 return (NULL); 9881 } 9882 9883 static void 9884 sfmmu_hat_exit(hatlock_t *hatlockp) 9885 { 9886 if (hatlockp != NULL) 9887 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 9888 } 9889 9890 static void 9891 sfmmu_hat_lock_all(void) 9892 { 9893 int i; 9894 for (i = 0; i < SFMMU_NUM_LOCK; i++) 9895 mutex_enter(HATLOCK_MUTEXP(&hat_lock[i])); 9896 } 9897 9898 static void 9899 sfmmu_hat_unlock_all(void) 9900 { 9901 int i; 9902 for (i = SFMMU_NUM_LOCK - 1; i >= 0; i--) 9903 mutex_exit(HATLOCK_MUTEXP(&hat_lock[i])); 9904 } 9905 9906 int 9907 sfmmu_hat_lock_held(sfmmu_t *sfmmup) 9908 { 9909 ASSERT(sfmmup != ksfmmup); 9910 return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup)))); 9911 } 9912 9913 /* 9914 * Locking primitives to provide consistency between ISM unmap 9915 * and other operations. Since ISM unmap can take a long time, we 9916 * use HAT_ISMBUSY flag (protected by the hatlock) to avoid creating 9917 * contention on the hatlock buckets while ISM segments are being 9918 * unmapped. The tradeoff is that the flags don't prevent priority 9919 * inversion from occurring, so we must request kernel priority in 9920 * case we have to sleep to keep from getting buried while holding 9921 * the HAT_ISMBUSY flag set, which in turn could block other kernel 9922 * threads from running (for example, in sfmmu_uvatopfn()). 9923 */ 9924 static void 9925 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held) 9926 { 9927 hatlock_t *hatlockp; 9928 9929 THREAD_KPRI_REQUEST(); 9930 if (!hatlock_held) 9931 hatlockp = sfmmu_hat_enter(sfmmup); 9932 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) 9933 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 9934 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY); 9935 if (!hatlock_held) 9936 sfmmu_hat_exit(hatlockp); 9937 } 9938 9939 static void 9940 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held) 9941 { 9942 hatlock_t *hatlockp; 9943 9944 if (!hatlock_held) 9945 hatlockp = sfmmu_hat_enter(sfmmup); 9946 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 9947 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY); 9948 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 9949 if (!hatlock_held) 9950 sfmmu_hat_exit(hatlockp); 9951 THREAD_KPRI_RELEASE(); 9952 } 9953 9954 /* 9955 * 9956 * Algorithm: 9957 * 9958 * (1) if segkmem is not ready, allocate hblk from an array of pre-alloc'ed 9959 * hblks. 9960 * 9961 * (2) if we are allocating an hblk for mapping a slab in sfmmu_cache, 9962 * 9963 * (a) try to return an hblk from reserve pool of free hblks; 9964 * (b) if the reserve pool is empty, acquire hblk_reserve_lock 9965 * and return hblk_reserve. 9966 * 9967 * (3) call kmem_cache_alloc() to allocate hblk; 9968 * 9969 * (a) if hblk_reserve_lock is held by the current thread, 9970 * atomically replace hblk_reserve by the hblk that is 9971 * returned by kmem_cache_alloc; release hblk_reserve_lock 9972 * and call kmem_cache_alloc() again. 9973 * (b) if reserve pool is not full, add the hblk that is 9974 * returned by kmem_cache_alloc to reserve pool and 9975 * call kmem_cache_alloc again. 9976 * 9977 */ 9978 static struct hme_blk * 9979 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr, 9980 struct hmehash_bucket *hmebp, uint_t size, hmeblk_tag hblktag, 9981 uint_t flags) 9982 { 9983 struct hme_blk *hmeblkp = NULL; 9984 struct hme_blk *newhblkp; 9985 struct hme_blk *shw_hblkp = NULL; 9986 struct kmem_cache *sfmmu_cache = NULL; 9987 uint64_t hblkpa; 9988 ulong_t index; 9989 uint_t owner; /* set to 1 if using hblk_reserve */ 9990 uint_t forcefree; 9991 int sleep; 9992 9993 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 9994 9995 /* 9996 * If segkmem is not created yet, allocate from static hmeblks 9997 * created at the end of startup_modules(). See the block comment 9998 * in startup_modules() describing how we estimate the number of 9999 * static hmeblks that will be needed during re-map. 10000 */ 10001 if (!hblk_alloc_dynamic) { 10002 10003 if (size == TTE8K) { 10004 index = nucleus_hblk8.index; 10005 if (index >= nucleus_hblk8.len) { 10006 /* 10007 * If we panic here, see startup_modules() to 10008 * make sure that we are calculating the 10009 * number of hblk8's that we need correctly. 10010 */ 10011 panic("no nucleus hblk8 to allocate"); 10012 } 10013 hmeblkp = 10014 (struct hme_blk *)&nucleus_hblk8.list[index]; 10015 nucleus_hblk8.index++; 10016 SFMMU_STAT(sf_hblk8_nalloc); 10017 } else { 10018 index = nucleus_hblk1.index; 10019 if (nucleus_hblk1.index >= nucleus_hblk1.len) { 10020 /* 10021 * If we panic here, see startup_modules() 10022 * and H8TOH1; most likely you need to 10023 * update the calculation of the number 10024 * of hblk1's the kernel needs to boot. 10025 */ 10026 panic("no nucleus hblk1 to allocate"); 10027 } 10028 hmeblkp = 10029 (struct hme_blk *)&nucleus_hblk1.list[index]; 10030 nucleus_hblk1.index++; 10031 SFMMU_STAT(sf_hblk1_nalloc); 10032 } 10033 10034 goto hblk_init; 10035 } 10036 10037 SFMMU_HASH_UNLOCK(hmebp); 10038 10039 if (sfmmup != KHATID) { 10040 if (mmu_page_sizes == max_mmu_page_sizes) { 10041 if (size < TTE256M) 10042 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 10043 size, flags); 10044 } else { 10045 if (size < TTE4M) 10046 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 10047 size, flags); 10048 } 10049 } 10050 10051 fill_hblk: 10052 owner = (hblk_reserve_thread == curthread) ? 1 : 0; 10053 10054 if (owner && size == TTE8K) { 10055 10056 /* 10057 * We are really in a tight spot. We already own 10058 * hblk_reserve and we need another hblk. In anticipation 10059 * of this kind of scenario, we specifically set aside 10060 * HBLK_RESERVE_MIN number of hblks to be used exclusively 10061 * by owner of hblk_reserve. 10062 */ 10063 SFMMU_STAT(sf_hblk_recurse_cnt); 10064 10065 if (!sfmmu_get_free_hblk(&hmeblkp, 1)) 10066 panic("sfmmu_hblk_alloc: reserve list is empty"); 10067 10068 goto hblk_verify; 10069 } 10070 10071 ASSERT(!owner); 10072 10073 if ((flags & HAT_NO_KALLOC) == 0) { 10074 10075 sfmmu_cache = ((size == TTE8K) ? sfmmu8_cache : sfmmu1_cache); 10076 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP); 10077 10078 if ((hmeblkp = kmem_cache_alloc(sfmmu_cache, sleep)) == NULL) { 10079 hmeblkp = sfmmu_hblk_steal(size); 10080 } else { 10081 /* 10082 * if we are the owner of hblk_reserve, 10083 * swap hblk_reserve with hmeblkp and 10084 * start a fresh life. Hope things go 10085 * better this time. 10086 */ 10087 if (hblk_reserve_thread == curthread) { 10088 ASSERT(sfmmu_cache == sfmmu8_cache); 10089 sfmmu_hblk_swap(hmeblkp); 10090 hblk_reserve_thread = NULL; 10091 mutex_exit(&hblk_reserve_lock); 10092 goto fill_hblk; 10093 } 10094 /* 10095 * let's donate this hblk to our reserve list if 10096 * we are not mapping kernel range 10097 */ 10098 if (size == TTE8K && sfmmup != KHATID) 10099 if (sfmmu_put_free_hblk(hmeblkp, 0)) 10100 goto fill_hblk; 10101 } 10102 } else { 10103 /* 10104 * We are here to map the slab in sfmmu8_cache; let's 10105 * check if we could tap our reserve list; if successful, 10106 * this will avoid the pain of going thru sfmmu_hblk_swap 10107 */ 10108 SFMMU_STAT(sf_hblk_slab_cnt); 10109 if (!sfmmu_get_free_hblk(&hmeblkp, 0)) { 10110 /* 10111 * let's start hblk_reserve dance 10112 */ 10113 SFMMU_STAT(sf_hblk_reserve_cnt); 10114 owner = 1; 10115 mutex_enter(&hblk_reserve_lock); 10116 hmeblkp = HBLK_RESERVE; 10117 hblk_reserve_thread = curthread; 10118 } 10119 } 10120 10121 hblk_verify: 10122 ASSERT(hmeblkp != NULL); 10123 set_hblk_sz(hmeblkp, size); 10124 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); 10125 SFMMU_HASH_LOCK(hmebp); 10126 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10127 if (newhblkp != NULL) { 10128 SFMMU_HASH_UNLOCK(hmebp); 10129 if (hmeblkp != HBLK_RESERVE) { 10130 /* 10131 * This is really tricky! 10132 * 10133 * vmem_alloc(vmem_seg_arena) 10134 * vmem_alloc(vmem_internal_arena) 10135 * segkmem_alloc(heap_arena) 10136 * vmem_alloc(heap_arena) 10137 * page_create() 10138 * hat_memload() 10139 * kmem_cache_free() 10140 * kmem_cache_alloc() 10141 * kmem_slab_create() 10142 * vmem_alloc(kmem_internal_arena) 10143 * segkmem_alloc(heap_arena) 10144 * vmem_alloc(heap_arena) 10145 * page_create() 10146 * hat_memload() 10147 * kmem_cache_free() 10148 * ... 10149 * 10150 * Thus, hat_memload() could call kmem_cache_free 10151 * for enough number of times that we could easily 10152 * hit the bottom of the stack or run out of reserve 10153 * list of vmem_seg structs. So, we must donate 10154 * this hblk to reserve list if it's allocated 10155 * from sfmmu8_cache *and* mapping kernel range. 10156 * We don't need to worry about freeing hmeblk1's 10157 * to kmem since they don't map any kmem slabs. 10158 * 10159 * Note: When segkmem supports largepages, we must 10160 * free hmeblk1's to reserve list as well. 10161 */ 10162 forcefree = (sfmmup == KHATID) ? 1 : 0; 10163 if (size == TTE8K && 10164 sfmmu_put_free_hblk(hmeblkp, forcefree)) { 10165 goto re_verify; 10166 } 10167 ASSERT(sfmmup != KHATID); 10168 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10169 } else { 10170 /* 10171 * Hey! we don't need hblk_reserve any more. 10172 */ 10173 ASSERT(owner); 10174 hblk_reserve_thread = NULL; 10175 mutex_exit(&hblk_reserve_lock); 10176 owner = 0; 10177 } 10178 re_verify: 10179 /* 10180 * let's check if the goodies are still present 10181 */ 10182 SFMMU_HASH_LOCK(hmebp); 10183 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10184 if (newhblkp != NULL) { 10185 /* 10186 * return newhblkp if it's not hblk_reserve; 10187 * if newhblkp is hblk_reserve, return it 10188 * _only if_ we are the owner of hblk_reserve. 10189 */ 10190 if (newhblkp != HBLK_RESERVE || owner) { 10191 return (newhblkp); 10192 } else { 10193 /* 10194 * we just hit hblk_reserve in the hash and 10195 * we are not the owner of that; 10196 * 10197 * block until hblk_reserve_thread completes 10198 * swapping hblk_reserve and try the dance 10199 * once again. 10200 */ 10201 SFMMU_HASH_UNLOCK(hmebp); 10202 mutex_enter(&hblk_reserve_lock); 10203 mutex_exit(&hblk_reserve_lock); 10204 SFMMU_STAT(sf_hblk_reserve_hit); 10205 goto fill_hblk; 10206 } 10207 } else { 10208 /* 10209 * it's no more! try the dance once again. 10210 */ 10211 SFMMU_HASH_UNLOCK(hmebp); 10212 goto fill_hblk; 10213 } 10214 } 10215 10216 hblk_init: 10217 set_hblk_sz(hmeblkp, size); 10218 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10219 hmeblkp->hblk_next = (struct hme_blk *)NULL; 10220 hmeblkp->hblk_tag = hblktag; 10221 hmeblkp->hblk_shadow = shw_hblkp; 10222 hblkpa = hmeblkp->hblk_nextpa; 10223 hmeblkp->hblk_nextpa = 0; 10224 10225 ASSERT(get_hblk_ttesz(hmeblkp) == size); 10226 ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size)); 10227 ASSERT(hmeblkp->hblk_hmecnt == 0); 10228 ASSERT(hmeblkp->hblk_vcnt == 0); 10229 ASSERT(hmeblkp->hblk_lckcnt == 0); 10230 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10231 sfmmu_hblk_hash_add(hmebp, hmeblkp, hblkpa); 10232 return (hmeblkp); 10233 } 10234 10235 /* 10236 * This function performs any cleanup required on the hme_blk 10237 * and returns it to the free list. 10238 */ 10239 /* ARGSUSED */ 10240 static void 10241 sfmmu_hblk_free(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10242 uint64_t hblkpa, struct hme_blk **listp) 10243 { 10244 int shw_size, vshift; 10245 struct hme_blk *shw_hblkp; 10246 uint_t shw_mask, newshw_mask; 10247 uintptr_t vaddr; 10248 int size; 10249 uint_t critical; 10250 10251 ASSERT(hmeblkp); 10252 ASSERT(!hmeblkp->hblk_hmecnt); 10253 ASSERT(!hmeblkp->hblk_vcnt); 10254 ASSERT(!hmeblkp->hblk_lckcnt); 10255 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10256 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 10257 10258 critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0; 10259 10260 size = get_hblk_ttesz(hmeblkp); 10261 shw_hblkp = hmeblkp->hblk_shadow; 10262 if (shw_hblkp) { 10263 ASSERT(hblktosfmmu(hmeblkp) != KHATID); 10264 if (mmu_page_sizes == max_mmu_page_sizes) { 10265 ASSERT(size < TTE256M); 10266 } else { 10267 ASSERT(size < TTE4M); 10268 } 10269 10270 shw_size = get_hblk_ttesz(shw_hblkp); 10271 vaddr = get_hblk_base(hmeblkp); 10272 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10273 ASSERT(vshift < 8); 10274 /* 10275 * Atomically clear shadow mask bit 10276 */ 10277 do { 10278 shw_mask = shw_hblkp->hblk_shw_mask; 10279 ASSERT(shw_mask & (1 << vshift)); 10280 newshw_mask = shw_mask & ~(1 << vshift); 10281 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10282 shw_mask, newshw_mask); 10283 } while (newshw_mask != shw_mask); 10284 hmeblkp->hblk_shadow = NULL; 10285 } 10286 hmeblkp->hblk_next = NULL; 10287 hmeblkp->hblk_nextpa = hblkpa; 10288 hmeblkp->hblk_shw_bit = 0; 10289 10290 if (hmeblkp->hblk_nuc_bit == 0) { 10291 10292 if (size == TTE8K && sfmmu_put_free_hblk(hmeblkp, critical)) 10293 return; 10294 10295 hmeblkp->hblk_next = *listp; 10296 *listp = hmeblkp; 10297 } 10298 } 10299 10300 static void 10301 sfmmu_hblks_list_purge(struct hme_blk **listp) 10302 { 10303 struct hme_blk *hmeblkp; 10304 10305 while ((hmeblkp = *listp) != NULL) { 10306 *listp = hmeblkp->hblk_next; 10307 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10308 } 10309 } 10310 10311 #define BUCKETS_TO_SEARCH_BEFORE_UNLOAD 30 10312 10313 static uint_t sfmmu_hblk_steal_twice; 10314 static uint_t sfmmu_hblk_steal_count, sfmmu_hblk_steal_unload_count; 10315 10316 /* 10317 * Steal a hmeblk 10318 * Enough hmeblks were allocated at startup (nucleus hmeblks) and also 10319 * hmeblks were added dynamically. We should never ever not be able to 10320 * find one. Look for an unused/unlocked hmeblk in user hash table. 10321 */ 10322 static struct hme_blk * 10323 sfmmu_hblk_steal(int size) 10324 { 10325 static struct hmehash_bucket *uhmehash_steal_hand = NULL; 10326 struct hmehash_bucket *hmebp; 10327 struct hme_blk *hmeblkp = NULL, *pr_hblk; 10328 uint64_t hblkpa, prevpa; 10329 int i; 10330 10331 for (;;) { 10332 hmebp = (uhmehash_steal_hand == NULL) ? uhme_hash : 10333 uhmehash_steal_hand; 10334 ASSERT(hmebp >= uhme_hash && hmebp <= &uhme_hash[UHMEHASH_SZ]); 10335 10336 for (i = 0; hmeblkp == NULL && i <= UHMEHASH_SZ + 10337 BUCKETS_TO_SEARCH_BEFORE_UNLOAD; i++) { 10338 SFMMU_HASH_LOCK(hmebp); 10339 hmeblkp = hmebp->hmeblkp; 10340 hblkpa = hmebp->hmeh_nextpa; 10341 prevpa = 0; 10342 pr_hblk = NULL; 10343 while (hmeblkp) { 10344 /* 10345 * check if it is a hmeblk that is not locked 10346 * and not shared. skip shadow hmeblks with 10347 * shadow_mask set i.e valid count non zero. 10348 */ 10349 if ((get_hblk_ttesz(hmeblkp) == size) && 10350 (hmeblkp->hblk_shw_bit == 0 || 10351 hmeblkp->hblk_vcnt == 0) && 10352 (hmeblkp->hblk_lckcnt == 0)) { 10353 /* 10354 * there is a high probability that we 10355 * will find a free one. search some 10356 * buckets for a free hmeblk initially 10357 * before unloading a valid hmeblk. 10358 */ 10359 if ((hmeblkp->hblk_vcnt == 0 && 10360 hmeblkp->hblk_hmecnt == 0) || (i >= 10361 BUCKETS_TO_SEARCH_BEFORE_UNLOAD)) { 10362 if (sfmmu_steal_this_hblk(hmebp, 10363 hmeblkp, hblkpa, prevpa, 10364 pr_hblk)) { 10365 /* 10366 * Hblk is unloaded 10367 * successfully 10368 */ 10369 break; 10370 } 10371 } 10372 } 10373 pr_hblk = hmeblkp; 10374 prevpa = hblkpa; 10375 hblkpa = hmeblkp->hblk_nextpa; 10376 hmeblkp = hmeblkp->hblk_next; 10377 } 10378 10379 SFMMU_HASH_UNLOCK(hmebp); 10380 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 10381 hmebp = uhme_hash; 10382 } 10383 uhmehash_steal_hand = hmebp; 10384 10385 if (hmeblkp != NULL) 10386 break; 10387 10388 /* 10389 * in the worst case, look for a free one in the kernel 10390 * hash table. 10391 */ 10392 for (i = 0, hmebp = khme_hash; i <= KHMEHASH_SZ; i++) { 10393 SFMMU_HASH_LOCK(hmebp); 10394 hmeblkp = hmebp->hmeblkp; 10395 hblkpa = hmebp->hmeh_nextpa; 10396 prevpa = 0; 10397 pr_hblk = NULL; 10398 while (hmeblkp) { 10399 /* 10400 * check if it is free hmeblk 10401 */ 10402 if ((get_hblk_ttesz(hmeblkp) == size) && 10403 (hmeblkp->hblk_lckcnt == 0) && 10404 (hmeblkp->hblk_vcnt == 0) && 10405 (hmeblkp->hblk_hmecnt == 0)) { 10406 if (sfmmu_steal_this_hblk(hmebp, 10407 hmeblkp, hblkpa, prevpa, pr_hblk)) { 10408 break; 10409 } else { 10410 /* 10411 * Cannot fail since we have 10412 * hash lock. 10413 */ 10414 panic("fail to steal?"); 10415 } 10416 } 10417 10418 pr_hblk = hmeblkp; 10419 prevpa = hblkpa; 10420 hblkpa = hmeblkp->hblk_nextpa; 10421 hmeblkp = hmeblkp->hblk_next; 10422 } 10423 10424 SFMMU_HASH_UNLOCK(hmebp); 10425 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 10426 hmebp = khme_hash; 10427 } 10428 10429 if (hmeblkp != NULL) 10430 break; 10431 sfmmu_hblk_steal_twice++; 10432 } 10433 return (hmeblkp); 10434 } 10435 10436 /* 10437 * This routine does real work to prepare a hblk to be "stolen" by 10438 * unloading the mappings, updating shadow counts .... 10439 * It returns 1 if the block is ready to be reused (stolen), or 0 10440 * means the block cannot be stolen yet- pageunload is still working 10441 * on this hblk. 10442 */ 10443 static int 10444 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10445 uint64_t hblkpa, uint64_t prevpa, struct hme_blk *pr_hblk) 10446 { 10447 int shw_size, vshift; 10448 struct hme_blk *shw_hblkp; 10449 uintptr_t vaddr; 10450 uint_t shw_mask, newshw_mask; 10451 10452 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10453 10454 /* 10455 * check if the hmeblk is free, unload if necessary 10456 */ 10457 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10458 sfmmu_t *sfmmup; 10459 demap_range_t dmr; 10460 10461 sfmmup = hblktosfmmu(hmeblkp); 10462 DEMAP_RANGE_INIT(sfmmup, &dmr); 10463 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 10464 (caddr_t)get_hblk_base(hmeblkp), 10465 get_hblk_endaddr(hmeblkp), &dmr, HAT_UNLOAD); 10466 DEMAP_RANGE_FLUSH(&dmr); 10467 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10468 /* 10469 * Pageunload is working on the same hblk. 10470 */ 10471 return (0); 10472 } 10473 10474 sfmmu_hblk_steal_unload_count++; 10475 } 10476 10477 ASSERT(hmeblkp->hblk_lckcnt == 0); 10478 ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0); 10479 10480 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 10481 hmeblkp->hblk_nextpa = hblkpa; 10482 10483 shw_hblkp = hmeblkp->hblk_shadow; 10484 if (shw_hblkp) { 10485 shw_size = get_hblk_ttesz(shw_hblkp); 10486 vaddr = get_hblk_base(hmeblkp); 10487 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10488 ASSERT(vshift < 8); 10489 /* 10490 * Atomically clear shadow mask bit 10491 */ 10492 do { 10493 shw_mask = shw_hblkp->hblk_shw_mask; 10494 ASSERT(shw_mask & (1 << vshift)); 10495 newshw_mask = shw_mask & ~(1 << vshift); 10496 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10497 shw_mask, newshw_mask); 10498 } while (newshw_mask != shw_mask); 10499 hmeblkp->hblk_shadow = NULL; 10500 } 10501 10502 /* 10503 * remove shadow bit if we are stealing an unused shadow hmeblk. 10504 * sfmmu_hblk_alloc needs it that way, will set shadow bit later if 10505 * we are indeed allocating a shadow hmeblk. 10506 */ 10507 hmeblkp->hblk_shw_bit = 0; 10508 10509 sfmmu_hblk_steal_count++; 10510 SFMMU_STAT(sf_steal_count); 10511 10512 return (1); 10513 } 10514 10515 struct hme_blk * 10516 sfmmu_hmetohblk(struct sf_hment *sfhme) 10517 { 10518 struct hme_blk *hmeblkp; 10519 struct sf_hment *sfhme0; 10520 struct hme_blk *hblk_dummy = 0; 10521 10522 /* 10523 * No dummy sf_hments, please. 10524 */ 10525 ASSERT(sfhme->hme_tte.ll != 0); 10526 10527 sfhme0 = sfhme - sfhme->hme_tte.tte_hmenum; 10528 hmeblkp = (struct hme_blk *)((uintptr_t)sfhme0 - 10529 (uintptr_t)&hblk_dummy->hblk_hme[0]); 10530 10531 return (hmeblkp); 10532 } 10533 10534 /* 10535 * Make sure that there is a valid ctx, if not get a ctx. 10536 * Also, get a readers lock on the ctx, so that the ctx cannot 10537 * be stolen underneath us. 10538 */ 10539 static void 10540 sfmmu_disallow_ctx_steal(sfmmu_t *sfmmup) 10541 { 10542 struct ctx *ctx; 10543 10544 ASSERT(sfmmup != ksfmmup); 10545 ASSERT(sfmmup->sfmmu_ismhat == 0); 10546 10547 /* 10548 * If ctx has been stolen, get a ctx. 10549 */ 10550 if (sfmmup->sfmmu_cnum == INVALID_CONTEXT) { 10551 /* 10552 * Our ctx was stolen. Get a ctx with rlock. 10553 */ 10554 ctx = sfmmu_get_ctx(sfmmup); 10555 return; 10556 } else { 10557 ctx = sfmmutoctx(sfmmup); 10558 } 10559 10560 /* 10561 * Get the reader lock. 10562 */ 10563 rw_enter(&ctx->ctx_rwlock, RW_READER); 10564 if (ctx->ctx_sfmmu != sfmmup) { 10565 /* 10566 * The ctx got stolen, so spin again. 10567 */ 10568 rw_exit(&ctx->ctx_rwlock); 10569 ctx = sfmmu_get_ctx(sfmmup); 10570 } 10571 10572 ASSERT(sfmmup->sfmmu_cnum >= NUM_LOCKED_CTXS); 10573 } 10574 10575 /* 10576 * Decrement reference count for our ctx. If the reference count 10577 * becomes 0, our ctx can be stolen by someone. 10578 */ 10579 static void 10580 sfmmu_allow_ctx_steal(sfmmu_t *sfmmup) 10581 { 10582 struct ctx *ctx; 10583 10584 ASSERT(sfmmup != ksfmmup); 10585 ASSERT(sfmmup->sfmmu_ismhat == 0); 10586 ctx = sfmmutoctx(sfmmup); 10587 10588 ASSERT(sfmmup == ctx->ctx_sfmmu); 10589 ASSERT(sfmmup->sfmmu_cnum != INVALID_CONTEXT); 10590 rw_exit(&ctx->ctx_rwlock); 10591 } 10592 10593 /* 10594 * On swapin, get appropriately sized TSB(s) and clear the HAT_SWAPPED flag. 10595 * If we can't get appropriately sized TSB(s), try for 8K TSB(s) using 10596 * KM_SLEEP allocation. 10597 * 10598 * Return 0 on success, -1 otherwise. 10599 */ 10600 static void 10601 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp) 10602 { 10603 struct tsb_info *tsbinfop, *next; 10604 tsb_replace_rc_t rc; 10605 boolean_t gotfirst = B_FALSE; 10606 10607 ASSERT(sfmmup != ksfmmup); 10608 ASSERT(sfmmu_hat_lock_held(sfmmup)); 10609 10610 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) { 10611 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 10612 } 10613 10614 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10615 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN); 10616 } else { 10617 return; 10618 } 10619 10620 ASSERT(sfmmup->sfmmu_tsb != NULL); 10621 10622 /* 10623 * Loop over all tsbinfo's replacing them with ones that actually have 10624 * a TSB. If any of the replacements ever fail, bail out of the loop. 10625 */ 10626 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) { 10627 ASSERT(tsbinfop->tsb_flags & TSB_SWAPPED); 10628 next = tsbinfop->tsb_next; 10629 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc, 10630 hatlockp, TSB_SWAPIN); 10631 if (rc != TSB_SUCCESS) { 10632 break; 10633 } 10634 gotfirst = B_TRUE; 10635 } 10636 10637 switch (rc) { 10638 case TSB_SUCCESS: 10639 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10640 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10641 return; 10642 case TSB_ALLOCFAIL: 10643 break; 10644 default: 10645 panic("sfmmu_replace_tsb returned unrecognized failure code " 10646 "%d", rc); 10647 } 10648 10649 /* 10650 * In this case, we failed to get one of our TSBs. If we failed to 10651 * get the first TSB, get one of minimum size (8KB). Walk the list 10652 * and throw away the tsbinfos, starting where the allocation failed; 10653 * we can get by with just one TSB as long as we don't leave the 10654 * SWAPPED tsbinfo structures lying around. 10655 */ 10656 tsbinfop = sfmmup->sfmmu_tsb; 10657 next = tsbinfop->tsb_next; 10658 tsbinfop->tsb_next = NULL; 10659 10660 sfmmu_hat_exit(hatlockp); 10661 for (tsbinfop = next; tsbinfop != NULL; tsbinfop = next) { 10662 next = tsbinfop->tsb_next; 10663 sfmmu_tsbinfo_free(tsbinfop); 10664 } 10665 hatlockp = sfmmu_hat_enter(sfmmup); 10666 10667 /* 10668 * If we don't have any TSBs, get a single 8K TSB for 8K, 64K and 512K 10669 * pages. 10670 */ 10671 if (!gotfirst) { 10672 tsbinfop = sfmmup->sfmmu_tsb; 10673 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE, 10674 hatlockp, TSB_SWAPIN | TSB_FORCEALLOC); 10675 ASSERT(rc == TSB_SUCCESS); 10676 } 10677 10678 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10679 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10680 } 10681 10682 /* 10683 * Handle exceptions for low level tsb_handler. 10684 * 10685 * There are many scenarios that could land us here: 10686 * 10687 * 1) Process has no context. In this case, ctx is 10688 * INVALID_CONTEXT and sfmmup->sfmmu_cnum == 1 so 10689 * we will acquire a context before returning. 10690 * 2) Need to re-load our MMU state. In this case, 10691 * ctx is INVALID_CONTEXT and sfmmup->sfmmu_cnum != 1. 10692 * 3) ISM mappings are being updated. This is handled 10693 * just like case #2. 10694 * 4) We wish to program a new page size into the TLB. 10695 * This is handled just like case #1, since changing 10696 * TLB page size requires us to flush the TLB. 10697 * 5) Window fault and no valid translation found. 10698 * 10699 * Cases 1-4, ctx is INVALID_CONTEXT so we handle it and then 10700 * exit which will retry the trapped instruction. Case #5 we 10701 * punt to trap() which will raise us a trap level and handle 10702 * the fault before unwinding. 10703 * 10704 * Note that the process will run in INVALID_CONTEXT before 10705 * faulting into here and subsequently loading the MMU registers 10706 * (including the TSB base register) associated with this process. 10707 * For this reason, the trap handlers must all test for 10708 * INVALID_CONTEXT before attempting to access any registers other 10709 * than the context registers. 10710 */ 10711 void 10712 sfmmu_tsbmiss_exception(struct regs *rp, uintptr_t tagaccess, uint_t traptype) 10713 { 10714 sfmmu_t *sfmmup; 10715 uint_t ctxnum; 10716 klwp_id_t lwp; 10717 char lwp_save_state; 10718 hatlock_t *hatlockp; 10719 struct tsb_info *tsbinfop; 10720 10721 SFMMU_STAT(sf_tsb_exceptions); 10722 sfmmup = astosfmmu(curthread->t_procp->p_as); 10723 ctxnum = tagaccess & TAGACC_CTX_MASK; 10724 10725 ASSERT(sfmmup != ksfmmup && ctxnum != KCONTEXT); 10726 ASSERT(sfmmup->sfmmu_ismhat == 0); 10727 /* 10728 * First, make sure we come out of here with a valid ctx, 10729 * since if we don't get one we'll simply loop on the 10730 * faulting instruction. 10731 * 10732 * If the ISM mappings are changing, the TSB is being relocated, or 10733 * the process is swapped out we serialize behind the controlling 10734 * thread with the sfmmu_flags and sfmmu_tsb_cv condition variable. 10735 * Otherwise we synchronize with the context stealer or the thread 10736 * that required us to change out our MMU registers (such 10737 * as a thread changing out our TSB while we were running) by 10738 * locking the HAT and grabbing the rwlock on the context as a 10739 * reader temporarily. 10740 */ 10741 if (ctxnum == INVALID_CONTEXT || 10742 SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10743 /* 10744 * Must set lwp state to LWP_SYS before 10745 * trying to acquire any adaptive lock 10746 */ 10747 lwp = ttolwp(curthread); 10748 ASSERT(lwp); 10749 lwp_save_state = lwp->lwp_state; 10750 lwp->lwp_state = LWP_SYS; 10751 10752 hatlockp = sfmmu_hat_enter(sfmmup); 10753 retry: 10754 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 10755 tsbinfop = tsbinfop->tsb_next) { 10756 if (tsbinfop->tsb_flags & TSB_RELOC_FLAG) { 10757 cv_wait(&sfmmup->sfmmu_tsb_cv, 10758 HATLOCK_MUTEXP(hatlockp)); 10759 goto retry; 10760 } 10761 } 10762 10763 /* 10764 * Wait for ISM maps to be updated. 10765 */ 10766 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) { 10767 cv_wait(&sfmmup->sfmmu_tsb_cv, 10768 HATLOCK_MUTEXP(hatlockp)); 10769 goto retry; 10770 } 10771 10772 /* 10773 * If we're swapping in, get TSB(s). Note that we must do 10774 * this before we get a ctx or load the MMU state. Once 10775 * we swap in we have to recheck to make sure the TSB(s) and 10776 * ISM mappings didn't change while we slept. 10777 */ 10778 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10779 sfmmu_tsb_swapin(sfmmup, hatlockp); 10780 goto retry; 10781 } 10782 10783 sfmmu_disallow_ctx_steal(sfmmup); 10784 ctxnum = sfmmup->sfmmu_cnum; 10785 kpreempt_disable(); 10786 sfmmu_setctx_sec(ctxnum); 10787 sfmmu_load_mmustate(sfmmup); 10788 kpreempt_enable(); 10789 sfmmu_allow_ctx_steal(sfmmup); 10790 sfmmu_hat_exit(hatlockp); 10791 /* 10792 * Must restore lwp_state if not calling 10793 * trap() for further processing. Restore 10794 * it anyway. 10795 */ 10796 lwp->lwp_state = lwp_save_state; 10797 if (sfmmup->sfmmu_ttecnt[TTE8K] != 0 || 10798 sfmmup->sfmmu_ttecnt[TTE64K] != 0 || 10799 sfmmup->sfmmu_ttecnt[TTE512K] != 0 || 10800 sfmmup->sfmmu_ttecnt[TTE4M] != 0 || 10801 sfmmup->sfmmu_ttecnt[TTE32M] != 0 || 10802 sfmmup->sfmmu_ttecnt[TTE256M] != 0) { 10803 return; 10804 } 10805 if (traptype == T_DATA_PROT) { 10806 traptype = T_DATA_MMU_MISS; 10807 } 10808 } 10809 trap(rp, (caddr_t)tagaccess, traptype, 0); 10810 } 10811 10812 /* 10813 * sfmmu_vatopfn_suspended is called from GET_TTE when TL=0 and 10814 * TTE_SUSPENDED bit set in tte we block on aquiring a page lock 10815 * rather than spinning to avoid send mondo timeouts with 10816 * interrupts enabled. When the lock is acquired it is immediately 10817 * released and we return back to sfmmu_vatopfn just after 10818 * the GET_TTE call. 10819 */ 10820 void 10821 sfmmu_vatopfn_suspended(caddr_t vaddr, sfmmu_t *sfmmu, tte_t *ttep) 10822 { 10823 struct page **pp; 10824 10825 (void) as_pagelock(sfmmu->sfmmu_as, &pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10826 as_pageunlock(sfmmu->sfmmu_as, pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10827 } 10828 10829 /* 10830 * sfmmu_tsbmiss_suspended is called from GET_TTE when TL>0 and 10831 * TTE_SUSPENDED bit set in tte. We do this so that we can handle 10832 * cross traps which cannot be handled while spinning in the 10833 * trap handlers. Simply enter and exit the kpr_suspendlock spin 10834 * mutex, which is held by the holder of the suspend bit, and then 10835 * retry the trapped instruction after unwinding. 10836 */ 10837 /*ARGSUSED*/ 10838 void 10839 sfmmu_tsbmiss_suspended(struct regs *rp, uintptr_t tagacc, uint_t traptype) 10840 { 10841 ASSERT(curthread != kreloc_thread); 10842 mutex_enter(&kpr_suspendlock); 10843 mutex_exit(&kpr_suspendlock); 10844 } 10845 10846 /* 10847 * Special routine to flush out ism mappings- TSBs, TLBs and D-caches. 10848 * This routine may be called with all cpu's captured. Therefore, the 10849 * caller is responsible for holding all locks and disabling kernel 10850 * preemption. 10851 */ 10852 /* ARGSUSED */ 10853 static void 10854 sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, 10855 struct hme_blk *hmeblkp, pfn_t pfnum, int cache_flush_flag) 10856 { 10857 cpuset_t cpuset; 10858 caddr_t va; 10859 ism_ment_t *ment; 10860 sfmmu_t *sfmmup; 10861 int ctxnum; 10862 int vcolor; 10863 int ttesz; 10864 10865 /* 10866 * Walk the ism_hat's mapping list and flush the page 10867 * from every hat sharing this ism_hat. This routine 10868 * may be called while all cpu's have been captured. 10869 * Therefore we can't attempt to grab any locks. For now 10870 * this means we will protect the ism mapping list under 10871 * a single lock which will be grabbed by the caller. 10872 * If hat_share/unshare scalibility becomes a performance 10873 * problem then we may need to re-think ism mapping list locking. 10874 */ 10875 ASSERT(ism_sfmmup->sfmmu_ismhat); 10876 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 10877 addr = addr - ISMID_STARTADDR; 10878 for (ment = ism_sfmmup->sfmmu_iment; ment; ment = ment->iment_next) { 10879 10880 sfmmup = ment->iment_hat; 10881 ctxnum = sfmmup->sfmmu_cnum; 10882 va = ment->iment_base_va; 10883 va = (caddr_t)((uintptr_t)va + (uintptr_t)addr); 10884 10885 /* 10886 * Flush TSB of ISM mappings. 10887 */ 10888 ttesz = get_hblk_ttesz(hmeblkp); 10889 if (ttesz == TTE8K || ttesz == TTE4M) { 10890 sfmmu_unload_tsb(sfmmup, va, ttesz); 10891 } else { 10892 caddr_t sva = va; 10893 caddr_t eva; 10894 ASSERT(addr == (caddr_t)get_hblk_base(hmeblkp)); 10895 eva = sva + get_hblk_span(hmeblkp); 10896 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); 10897 } 10898 10899 if (ctxnum != INVALID_CONTEXT) { 10900 /* 10901 * Flush TLBs. We don't need to do this for 10902 * invalid context since the flushing is already 10903 * done as part of context stealing. 10904 */ 10905 cpuset = sfmmup->sfmmu_cpusran; 10906 CPUSET_AND(cpuset, cpu_ready_set); 10907 CPUSET_DEL(cpuset, CPU->cpu_id); 10908 SFMMU_XCALL_STATS(ctxnum); 10909 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)va, 10910 ctxnum); 10911 vtag_flushpage(va, ctxnum); 10912 } 10913 10914 /* 10915 * Flush D$ 10916 * When flushing D$ we must flush all 10917 * cpu's. See sfmmu_cache_flush(). 10918 */ 10919 if (cache_flush_flag == CACHE_FLUSH) { 10920 cpuset = cpu_ready_set; 10921 CPUSET_DEL(cpuset, CPU->cpu_id); 10922 SFMMU_XCALL_STATS(ctxnum); 10923 vcolor = addr_to_vcolor(va); 10924 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10925 vac_flushpage(pfnum, vcolor); 10926 } 10927 } 10928 } 10929 10930 /* 10931 * Demaps the TSB, CPU caches, and flushes all TLBs on all CPUs of 10932 * a particular virtual address and ctx. If noflush is set we do not 10933 * flush the TLB/TSB. This function may or may not be called with the 10934 * HAT lock held. 10935 */ 10936 static void 10937 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10938 pfn_t pfnum, int tlb_noflush, int cpu_flag, int cache_flush_flag, 10939 int hat_lock_held) 10940 { 10941 int ctxnum, vcolor; 10942 cpuset_t cpuset; 10943 hatlock_t *hatlockp; 10944 10945 /* 10946 * There is no longer a need to protect against ctx being 10947 * stolen here since we don't store the ctx in the TSB anymore. 10948 */ 10949 vcolor = addr_to_vcolor(addr); 10950 10951 kpreempt_disable(); 10952 if (!tlb_noflush) { 10953 /* 10954 * Flush the TSB. 10955 */ 10956 if (!hat_lock_held) 10957 hatlockp = sfmmu_hat_enter(sfmmup); 10958 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10959 ctxnum = (int)sfmmutoctxnum(sfmmup); 10960 if (!hat_lock_held) 10961 sfmmu_hat_exit(hatlockp); 10962 10963 if (ctxnum != INVALID_CONTEXT) { 10964 /* 10965 * Flush TLBs. We don't need to do this if our 10966 * context is invalid context. Since we hold the 10967 * HAT lock the context must have been stolen and 10968 * hence will be flushed before re-use. 10969 */ 10970 cpuset = sfmmup->sfmmu_cpusran; 10971 CPUSET_AND(cpuset, cpu_ready_set); 10972 CPUSET_DEL(cpuset, CPU->cpu_id); 10973 SFMMU_XCALL_STATS(ctxnum); 10974 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 10975 ctxnum); 10976 vtag_flushpage(addr, ctxnum); 10977 } 10978 } 10979 10980 /* 10981 * Flush the D$ 10982 * 10983 * Even if the ctx is stolen, we need to flush the 10984 * cache. Our ctx stealer only flushes the TLBs. 10985 */ 10986 if (cache_flush_flag == CACHE_FLUSH) { 10987 if (cpu_flag & FLUSH_ALL_CPUS) { 10988 cpuset = cpu_ready_set; 10989 } else { 10990 cpuset = sfmmup->sfmmu_cpusran; 10991 CPUSET_AND(cpuset, cpu_ready_set); 10992 } 10993 CPUSET_DEL(cpuset, CPU->cpu_id); 10994 SFMMU_XCALL_STATS(sfmmutoctxnum(sfmmup)); 10995 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10996 vac_flushpage(pfnum, vcolor); 10997 } 10998 kpreempt_enable(); 10999 } 11000 11001 /* 11002 * Demaps the TSB and flushes all TLBs on all cpus for a particular virtual 11003 * address and ctx. If noflush is set we do not currently do anything. 11004 * This function may or may not be called with the HAT lock held. 11005 */ 11006 static void 11007 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 11008 int tlb_noflush, int hat_lock_held) 11009 { 11010 int ctxnum; 11011 cpuset_t cpuset; 11012 hatlock_t *hatlockp; 11013 11014 /* 11015 * If the process is exiting we have nothing to do. 11016 */ 11017 if (tlb_noflush) 11018 return; 11019 11020 /* 11021 * Flush TSB. 11022 */ 11023 if (!hat_lock_held) 11024 hatlockp = sfmmu_hat_enter(sfmmup); 11025 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 11026 ctxnum = sfmmutoctxnum(sfmmup); 11027 if (!hat_lock_held) 11028 sfmmu_hat_exit(hatlockp); 11029 11030 /* 11031 * Flush TLBs. We don't need to do this if our context is invalid 11032 * context. Since we hold the HAT lock the context must have been 11033 * stolen and hence will be flushed before re-use. 11034 */ 11035 if (ctxnum != INVALID_CONTEXT) { 11036 /* 11037 * There is no need to protect against ctx being stolen. 11038 * If the ctx is stolen we will simply get an extra flush. 11039 */ 11040 kpreempt_disable(); 11041 cpuset = sfmmup->sfmmu_cpusran; 11042 CPUSET_AND(cpuset, cpu_ready_set); 11043 CPUSET_DEL(cpuset, CPU->cpu_id); 11044 SFMMU_XCALL_STATS(ctxnum); 11045 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, ctxnum); 11046 vtag_flushpage(addr, ctxnum); 11047 kpreempt_enable(); 11048 } 11049 } 11050 11051 /* 11052 * Special case of sfmmu_tlb_demap for MMU_PAGESIZE hblks. Use the xcall 11053 * call handler that can flush a range of pages to save on xcalls. 11054 */ 11055 static int sfmmu_xcall_save; 11056 11057 static void 11058 sfmmu_tlb_range_demap(demap_range_t *dmrp) 11059 { 11060 sfmmu_t *sfmmup = dmrp->dmr_sfmmup; 11061 int ctxnum; 11062 hatlock_t *hatlockp; 11063 cpuset_t cpuset; 11064 uint64_t ctx_pgcnt; 11065 pgcnt_t pgcnt = 0; 11066 int pgunload = 0; 11067 int dirtypg = 0; 11068 caddr_t addr = dmrp->dmr_addr; 11069 caddr_t eaddr; 11070 uint64_t bitvec = dmrp->dmr_bitvec; 11071 11072 ASSERT(bitvec & 1); 11073 11074 /* 11075 * Flush TSB and calculate number of pages to flush. 11076 */ 11077 while (bitvec != 0) { 11078 dirtypg = 0; 11079 /* 11080 * Find the first page to flush and then count how many 11081 * pages there are after it that also need to be flushed. 11082 * This way the number of TSB flushes is minimized. 11083 */ 11084 while ((bitvec & 1) == 0) { 11085 pgcnt++; 11086 addr += MMU_PAGESIZE; 11087 bitvec >>= 1; 11088 } 11089 while (bitvec & 1) { 11090 dirtypg++; 11091 bitvec >>= 1; 11092 } 11093 eaddr = addr + ptob(dirtypg); 11094 hatlockp = sfmmu_hat_enter(sfmmup); 11095 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K); 11096 sfmmu_hat_exit(hatlockp); 11097 pgunload += dirtypg; 11098 addr = eaddr; 11099 pgcnt += dirtypg; 11100 } 11101 11102 /* 11103 * In the case where context is invalid context, bail. 11104 * We hold the hat lock while checking the ctx to prevent 11105 * a race with sfmmu_replace_tsb() which temporarily sets 11106 * the ctx to INVALID_CONTEXT to force processes to enter 11107 * sfmmu_tsbmiss_exception(). 11108 */ 11109 hatlockp = sfmmu_hat_enter(sfmmup); 11110 ctxnum = sfmmutoctxnum(sfmmup); 11111 sfmmu_hat_exit(hatlockp); 11112 if (ctxnum == INVALID_CONTEXT) { 11113 dmrp->dmr_bitvec = 0; 11114 return; 11115 } 11116 11117 ASSERT((pgcnt<<MMU_PAGESHIFT) <= dmrp->dmr_endaddr - dmrp->dmr_addr); 11118 if (sfmmup->sfmmu_free == 0) { 11119 addr = dmrp->dmr_addr; 11120 bitvec = dmrp->dmr_bitvec; 11121 ctx_pgcnt = (uint64_t)((ctxnum << 16) | pgcnt); 11122 kpreempt_disable(); 11123 cpuset = sfmmup->sfmmu_cpusran; 11124 CPUSET_AND(cpuset, cpu_ready_set); 11125 CPUSET_DEL(cpuset, CPU->cpu_id); 11126 SFMMU_XCALL_STATS(ctxnum); 11127 xt_some(cpuset, vtag_flush_pgcnt_tl1, (uint64_t)addr, 11128 ctx_pgcnt); 11129 for (; bitvec != 0; bitvec >>= 1) { 11130 if (bitvec & 1) 11131 vtag_flushpage(addr, ctxnum); 11132 addr += MMU_PAGESIZE; 11133 } 11134 kpreempt_enable(); 11135 sfmmu_xcall_save += (pgunload-1); 11136 } 11137 dmrp->dmr_bitvec = 0; 11138 } 11139 11140 /* 11141 * Flushes only TLB. 11142 */ 11143 static void 11144 sfmmu_tlb_ctx_demap(sfmmu_t *sfmmup) 11145 { 11146 int ctxnum; 11147 cpuset_t cpuset; 11148 11149 ctxnum = (int)sfmmutoctxnum(sfmmup); 11150 if (ctxnum == INVALID_CONTEXT) { 11151 /* 11152 * if ctx was stolen then simply return 11153 * whoever stole ctx is responsible for flush. 11154 */ 11155 return; 11156 } 11157 ASSERT(ctxnum != KCONTEXT); 11158 /* 11159 * There is no need to protect against ctx being stolen. If the 11160 * ctx is stolen we will simply get an extra flush. 11161 */ 11162 kpreempt_disable(); 11163 11164 cpuset = sfmmup->sfmmu_cpusran; 11165 CPUSET_DEL(cpuset, CPU->cpu_id); 11166 CPUSET_AND(cpuset, cpu_ready_set); 11167 SFMMU_XCALL_STATS(ctxnum); 11168 11169 /* 11170 * Flush TLB. 11171 * RFE: it might be worth delaying the TLB flush as well. In that 11172 * case each cpu would have to traverse the dirty list and flush 11173 * each one of those ctx from the TLB. 11174 */ 11175 vtag_flushctx(ctxnum); 11176 xt_some(cpuset, vtag_flushctx_tl1, ctxnum, 0); 11177 11178 kpreempt_enable(); 11179 SFMMU_STAT(sf_tlbflush_ctx); 11180 } 11181 11182 /* 11183 * Flushes all TLBs. 11184 */ 11185 static void 11186 sfmmu_tlb_all_demap(void) 11187 { 11188 cpuset_t cpuset; 11189 11190 /* 11191 * There is no need to protect against ctx being stolen. If the 11192 * ctx is stolen we will simply get an extra flush. 11193 */ 11194 kpreempt_disable(); 11195 11196 cpuset = cpu_ready_set; 11197 CPUSET_DEL(cpuset, CPU->cpu_id); 11198 /* LINTED: constant in conditional context */ 11199 SFMMU_XCALL_STATS(INVALID_CONTEXT); 11200 11201 vtag_flushall(); 11202 xt_some(cpuset, vtag_flushall_tl1, 0, 0); 11203 xt_sync(cpuset); 11204 11205 kpreempt_enable(); 11206 SFMMU_STAT(sf_tlbflush_all); 11207 } 11208 11209 /* 11210 * In cases where we need to synchronize with TLB/TSB miss trap 11211 * handlers, _and_ need to flush the TLB, it's a lot easier to 11212 * steal the context from the process and free it than to do a 11213 * special song and dance to keep things consistent for the 11214 * handlers. 11215 * 11216 * Since the process suddenly ends up without a context and our caller 11217 * holds the hat lock, threads that fault after this function is called 11218 * will pile up on the lock. We can then do whatever we need to 11219 * atomically from the context of the caller. The first blocked thread 11220 * to resume executing will get the process a new context, and the 11221 * process will resume executing. 11222 * 11223 * One added advantage of this approach is that on MMUs that 11224 * support a "flush all" operation, we will delay the flush until 11225 * we run out of contexts, and then flush the TLB one time. This 11226 * is rather rare, so it's a lot less expensive than making 8000 11227 * x-calls to flush the TLB 8000 times. Another is that we can do 11228 * all of this without pausing CPUs, due to some knowledge of how 11229 * resume() loads processes onto the processor; it sets the thread 11230 * into cpusran, and _then_ looks at cnum. Because we do things in 11231 * the reverse order here, we guarantee exactly one of the following 11232 * statements is always true: 11233 * 11234 * 1) Nobody is in resume() so we have nothing to worry about anyway. 11235 * 2) The thread in resume() isn't in cpusran when we do the xcall, 11236 * so we know when it does set itself it'll see cnum is 11237 * INVALID_CONTEXT. 11238 * 3) The thread in resume() is in cpusran, and already might have 11239 * looked at the old cnum. That's OK, because we'll xcall it 11240 * and, if necessary, flush the TLB along with the rest of the 11241 * crowd. 11242 */ 11243 static void 11244 sfmmu_tlb_swap_ctx(sfmmu_t *sfmmup, struct ctx *ctx) 11245 { 11246 cpuset_t cpuset; 11247 int cnum; 11248 11249 if (sfmmup->sfmmu_cnum == INVALID_CONTEXT) 11250 return; 11251 11252 SFMMU_STAT(sf_ctx_swap); 11253 11254 kpreempt_disable(); 11255 11256 ASSERT(rw_read_locked(&ctx->ctx_rwlock) == 0); 11257 ASSERT(ctx->ctx_sfmmu == sfmmup); 11258 11259 cnum = ctxtoctxnum(ctx); 11260 ASSERT(sfmmup->sfmmu_cnum == cnum); 11261 ASSERT(cnum >= NUM_LOCKED_CTXS); 11262 11263 sfmmup->sfmmu_cnum = INVALID_CONTEXT; 11264 membar_enter(); /* make sure visible on all CPUs */ 11265 ctx->ctx_sfmmu = NULL; 11266 11267 cpuset = sfmmup->sfmmu_cpusran; 11268 CPUSET_DEL(cpuset, CPU->cpu_id); 11269 CPUSET_AND(cpuset, cpu_ready_set); 11270 SFMMU_XCALL_STATS(cnum); 11271 11272 /* 11273 * Force anybody running this process on CPU 11274 * to enter sfmmu_tsbmiss_exception() on the 11275 * next TLB miss, synchronize behind us on 11276 * the HAT lock, and grab a new context. At 11277 * that point the new page size will become 11278 * active in the TLB for the new context. 11279 * See sfmmu_get_ctx() for details. 11280 */ 11281 if (delay_tlb_flush) { 11282 xt_some(cpuset, sfmmu_raise_tsb_exception, 11283 cnum, INVALID_CONTEXT); 11284 SFMMU_STAT(sf_tlbflush_deferred); 11285 } else { 11286 xt_some(cpuset, sfmmu_ctx_steal_tl1, cnum, INVALID_CONTEXT); 11287 vtag_flushctx(cnum); 11288 SFMMU_STAT(sf_tlbflush_ctx); 11289 } 11290 xt_sync(cpuset); 11291 11292 /* 11293 * If we just stole the ctx from the current 11294 * process on local CPU we need to invalidate 11295 * this CPU context as well. 11296 */ 11297 if (sfmmu_getctx_sec() == cnum) { 11298 sfmmu_setctx_sec(INVALID_CONTEXT); 11299 sfmmu_clear_utsbinfo(); 11300 } 11301 11302 kpreempt_enable(); 11303 11304 /* 11305 * Now put old ctx on the dirty list since we may not 11306 * have flushed the context out of the TLB. We'll let 11307 * the next guy who uses this ctx flush it instead. 11308 */ 11309 mutex_enter(&ctx_list_lock); 11310 CTX_SET_FLAGS(ctx, CTX_FREE_FLAG); 11311 ctx->ctx_free = ctxdirty; 11312 ctxdirty = ctx; 11313 mutex_exit(&ctx_list_lock); 11314 } 11315 11316 /* 11317 * We need to flush the cache in all cpus. It is possible that 11318 * a process referenced a page as cacheable but has sinced exited 11319 * and cleared the mapping list. We still to flush it but have no 11320 * state so all cpus is the only alternative. 11321 */ 11322 void 11323 sfmmu_cache_flush(pfn_t pfnum, int vcolor) 11324 { 11325 cpuset_t cpuset; 11326 int ctxnum = INVALID_CONTEXT; 11327 11328 kpreempt_disable(); 11329 cpuset = cpu_ready_set; 11330 CPUSET_DEL(cpuset, CPU->cpu_id); 11331 SFMMU_XCALL_STATS(ctxnum); /* account to any ctx */ 11332 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 11333 xt_sync(cpuset); 11334 vac_flushpage(pfnum, vcolor); 11335 kpreempt_enable(); 11336 } 11337 11338 void 11339 sfmmu_cache_flushcolor(int vcolor, pfn_t pfnum) 11340 { 11341 cpuset_t cpuset; 11342 int ctxnum = INVALID_CONTEXT; 11343 11344 ASSERT(vcolor >= 0); 11345 11346 kpreempt_disable(); 11347 cpuset = cpu_ready_set; 11348 CPUSET_DEL(cpuset, CPU->cpu_id); 11349 SFMMU_XCALL_STATS(ctxnum); /* account to any ctx */ 11350 xt_some(cpuset, vac_flushcolor_tl1, vcolor, pfnum); 11351 xt_sync(cpuset); 11352 vac_flushcolor(vcolor, pfnum); 11353 kpreempt_enable(); 11354 } 11355 11356 /* 11357 * We need to prevent processes from accessing the TSB using a cached physical 11358 * address. It's alright if they try to access the TSB via virtual address 11359 * since they will just fault on that virtual address once the mapping has 11360 * been suspended. 11361 */ 11362 #pragma weak sendmondo_in_recover 11363 11364 /* ARGSUSED */ 11365 static int 11366 sfmmu_tsb_pre_relocator(caddr_t va, uint_t tsbsz, uint_t flags, void *tsbinfo) 11367 { 11368 hatlock_t *hatlockp; 11369 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11370 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11371 struct ctx *ctx; 11372 int cnum; 11373 extern uint32_t sendmondo_in_recover; 11374 11375 if (flags != HAT_PRESUSPEND) 11376 return (0); 11377 11378 hatlockp = sfmmu_hat_enter(sfmmup); 11379 11380 tsbinfop->tsb_flags |= TSB_RELOC_FLAG; 11381 11382 /* 11383 * For Cheetah+ Erratum 25: 11384 * Wait for any active recovery to finish. We can't risk 11385 * relocating the TSB of the thread running mondo_recover_proc() 11386 * since, if we did that, we would deadlock. The scenario we are 11387 * trying to avoid is as follows: 11388 * 11389 * THIS CPU RECOVER CPU 11390 * -------- ----------- 11391 * Begins recovery, walking through TSB 11392 * hat_pagesuspend() TSB TTE 11393 * TLB miss on TSB TTE, spins at TL1 11394 * xt_sync() 11395 * send_mondo_timeout() 11396 * mondo_recover_proc() 11397 * ((deadlocked)) 11398 * 11399 * The second half of the workaround is that mondo_recover_proc() 11400 * checks to see if the tsb_info has the RELOC flag set, and if it 11401 * does, it skips over that TSB without ever touching tsbinfop->tsb_va 11402 * and hence avoiding the TLB miss that could result in a deadlock. 11403 */ 11404 if (&sendmondo_in_recover) { 11405 membar_enter(); /* make sure RELOC flag visible */ 11406 while (sendmondo_in_recover) { 11407 drv_usecwait(1); 11408 membar_consumer(); 11409 } 11410 } 11411 11412 ctx = sfmmutoctx(sfmmup); 11413 rw_enter(&ctx->ctx_rwlock, RW_WRITER); 11414 cnum = sfmmutoctxnum(sfmmup); 11415 11416 if (cnum != INVALID_CONTEXT) { 11417 /* 11418 * Force all threads for this sfmmu to sfmmu_tsbmiss_exception 11419 * on their next TLB miss. 11420 */ 11421 sfmmu_tlb_swap_ctx(sfmmup, ctx); 11422 } 11423 11424 rw_exit(&ctx->ctx_rwlock); 11425 11426 sfmmu_hat_exit(hatlockp); 11427 11428 return (0); 11429 } 11430 11431 /* ARGSUSED */ 11432 static int 11433 sfmmu_tsb_post_relocator(caddr_t va, uint_t tsbsz, uint_t flags, 11434 void *tsbinfo, pfn_t newpfn) 11435 { 11436 hatlock_t *hatlockp; 11437 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11438 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11439 11440 if (flags != HAT_POSTUNSUSPEND) 11441 return (0); 11442 11443 hatlockp = sfmmu_hat_enter(sfmmup); 11444 11445 SFMMU_STAT(sf_tsb_reloc); 11446 11447 /* 11448 * The process may have swapped out while we were relocating one 11449 * of its TSBs. If so, don't bother doing the setup since the 11450 * process can't be using the memory anymore. 11451 */ 11452 if ((tsbinfop->tsb_flags & TSB_SWAPPED) == 0) { 11453 ASSERT(va == tsbinfop->tsb_va); 11454 sfmmu_tsbinfo_setup_phys(tsbinfop, newpfn); 11455 sfmmu_setup_tsbinfo(sfmmup); 11456 11457 if (tsbinfop->tsb_flags & TSB_FLUSH_NEEDED) { 11458 sfmmu_inv_tsb(tsbinfop->tsb_va, 11459 TSB_BYTES(tsbinfop->tsb_szc)); 11460 tsbinfop->tsb_flags &= ~TSB_FLUSH_NEEDED; 11461 } 11462 } 11463 11464 membar_exit(); 11465 tsbinfop->tsb_flags &= ~TSB_RELOC_FLAG; 11466 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 11467 11468 sfmmu_hat_exit(hatlockp); 11469 11470 return (0); 11471 } 11472 11473 /* 11474 * Allocate and initialize a tsb_info structure. Note that we may or may not 11475 * allocate a TSB here, depending on the flags passed in. 11476 */ 11477 static int 11478 sfmmu_tsbinfo_alloc(struct tsb_info **tsbinfopp, int tsb_szc, int tte_sz_mask, 11479 uint_t flags, sfmmu_t *sfmmup) 11480 { 11481 int err; 11482 11483 *tsbinfopp = (struct tsb_info *)kmem_cache_alloc( 11484 sfmmu_tsbinfo_cache, KM_SLEEP); 11485 11486 if ((err = sfmmu_init_tsbinfo(*tsbinfopp, tte_sz_mask, 11487 tsb_szc, flags, sfmmup)) != 0) { 11488 kmem_cache_free(sfmmu_tsbinfo_cache, *tsbinfopp); 11489 SFMMU_STAT(sf_tsb_allocfail); 11490 *tsbinfopp = NULL; 11491 return (err); 11492 } 11493 SFMMU_STAT(sf_tsb_alloc); 11494 11495 /* 11496 * Bump the TSB size counters for this TSB size. 11497 */ 11498 (*(((int *)&sfmmu_tsbsize_stat) + tsb_szc))++; 11499 return (0); 11500 } 11501 11502 static void 11503 sfmmu_tsb_free(struct tsb_info *tsbinfo) 11504 { 11505 caddr_t tsbva = tsbinfo->tsb_va; 11506 uint_t tsb_size = TSB_BYTES(tsbinfo->tsb_szc); 11507 struct kmem_cache *kmem_cachep = tsbinfo->tsb_cache; 11508 vmem_t *vmp = tsbinfo->tsb_vmp; 11509 11510 /* 11511 * If we allocated this TSB from relocatable kernel memory, then we 11512 * need to uninstall the callback handler. 11513 */ 11514 if (tsbinfo->tsb_cache != sfmmu_tsb8k_cache) { 11515 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11516 caddr_t slab_vaddr = (caddr_t)((uintptr_t)tsbva & slab_mask); 11517 page_t **ppl; 11518 int ret; 11519 11520 ret = as_pagelock(&kas, &ppl, slab_vaddr, PAGESIZE, S_WRITE); 11521 ASSERT(ret == 0); 11522 hat_delete_callback(tsbva, (uint_t)tsb_size, (void *)tsbinfo, 11523 0); 11524 as_pageunlock(&kas, ppl, slab_vaddr, PAGESIZE, S_WRITE); 11525 } 11526 11527 if (kmem_cachep != NULL) { 11528 kmem_cache_free(kmem_cachep, tsbva); 11529 } else { 11530 vmem_xfree(vmp, (void *)tsbva, tsb_size); 11531 } 11532 tsbinfo->tsb_va = (caddr_t)0xbad00bad; 11533 atomic_add_64(&tsb_alloc_bytes, -(int64_t)tsb_size); 11534 } 11535 11536 static void 11537 sfmmu_tsbinfo_free(struct tsb_info *tsbinfo) 11538 { 11539 if ((tsbinfo->tsb_flags & TSB_SWAPPED) == 0) { 11540 sfmmu_tsb_free(tsbinfo); 11541 } 11542 kmem_cache_free(sfmmu_tsbinfo_cache, tsbinfo); 11543 11544 } 11545 11546 /* 11547 * Setup all the references to physical memory for this tsbinfo. 11548 * The underlying page(s) must be locked. 11549 */ 11550 static void 11551 sfmmu_tsbinfo_setup_phys(struct tsb_info *tsbinfo, pfn_t pfn) 11552 { 11553 ASSERT(pfn != PFN_INVALID); 11554 ASSERT(pfn == va_to_pfn(tsbinfo->tsb_va)); 11555 11556 #ifndef sun4v 11557 if (tsbinfo->tsb_szc == 0) { 11558 sfmmu_memtte(&tsbinfo->tsb_tte, pfn, 11559 PROT_WRITE|PROT_READ, TTE8K); 11560 } else { 11561 /* 11562 * Round down PA and use a large mapping; the handlers will 11563 * compute the TSB pointer at the correct offset into the 11564 * big virtual page. NOTE: this assumes all TSBs larger 11565 * than 8K must come from physically contiguous slabs of 11566 * size tsb_slab_size. 11567 */ 11568 sfmmu_memtte(&tsbinfo->tsb_tte, pfn & ~tsb_slab_mask, 11569 PROT_WRITE|PROT_READ, tsb_slab_ttesz); 11570 } 11571 tsbinfo->tsb_pa = ptob(pfn); 11572 11573 TTE_SET_LOCKED(&tsbinfo->tsb_tte); /* lock the tte into dtlb */ 11574 TTE_SET_MOD(&tsbinfo->tsb_tte); /* enable writes */ 11575 11576 ASSERT(TTE_IS_PRIVILEGED(&tsbinfo->tsb_tte)); 11577 ASSERT(TTE_IS_LOCKED(&tsbinfo->tsb_tte)); 11578 #else /* sun4v */ 11579 tsbinfo->tsb_pa = ptob(pfn); 11580 #endif /* sun4v */ 11581 } 11582 11583 11584 /* 11585 * Returns zero on success, ENOMEM if over the high water mark, 11586 * or EAGAIN if the caller needs to retry with a smaller TSB 11587 * size (or specify TSB_FORCEALLOC if the allocation can't fail). 11588 * 11589 * This call cannot fail to allocate a TSB if TSB_FORCEALLOC 11590 * is specified and the TSB requested is PAGESIZE, though it 11591 * may sleep waiting for memory if sufficient memory is not 11592 * available. 11593 */ 11594 static int 11595 sfmmu_init_tsbinfo(struct tsb_info *tsbinfo, int tteszmask, 11596 int tsbcode, uint_t flags, sfmmu_t *sfmmup) 11597 { 11598 caddr_t vaddr = NULL; 11599 caddr_t slab_vaddr; 11600 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11601 int tsbbytes = TSB_BYTES(tsbcode); 11602 int lowmem = 0; 11603 struct kmem_cache *kmem_cachep = NULL; 11604 vmem_t *vmp = NULL; 11605 lgrp_id_t lgrpid = LGRP_NONE; 11606 pfn_t pfn; 11607 uint_t cbflags = HAC_SLEEP; 11608 page_t **pplist; 11609 int ret; 11610 11611 if (flags & (TSB_FORCEALLOC | TSB_SWAPIN | TSB_GROW | TSB_SHRINK)) 11612 flags |= TSB_ALLOC; 11613 11614 ASSERT((flags & TSB_FORCEALLOC) == 0 || tsbcode == TSB_MIN_SZCODE); 11615 11616 tsbinfo->tsb_sfmmu = sfmmup; 11617 11618 /* 11619 * If not allocating a TSB, set up the tsbinfo, set TSB_SWAPPED, and 11620 * return. 11621 */ 11622 if ((flags & TSB_ALLOC) == 0) { 11623 tsbinfo->tsb_szc = tsbcode; 11624 tsbinfo->tsb_ttesz_mask = tteszmask; 11625 tsbinfo->tsb_va = (caddr_t)0xbadbadbeef; 11626 tsbinfo->tsb_pa = -1; 11627 tsbinfo->tsb_tte.ll = 0; 11628 tsbinfo->tsb_next = NULL; 11629 tsbinfo->tsb_flags = TSB_SWAPPED; 11630 tsbinfo->tsb_cache = NULL; 11631 tsbinfo->tsb_vmp = NULL; 11632 return (0); 11633 } 11634 11635 #ifdef DEBUG 11636 /* 11637 * For debugging: 11638 * Randomly force allocation failures every tsb_alloc_mtbf 11639 * tries if TSB_FORCEALLOC is not specified. This will 11640 * return ENOMEM if tsb_alloc_mtbf is odd, or EAGAIN if 11641 * it is even, to allow testing of both failure paths... 11642 */ 11643 if (tsb_alloc_mtbf && ((flags & TSB_FORCEALLOC) == 0) && 11644 (tsb_alloc_count++ == tsb_alloc_mtbf)) { 11645 tsb_alloc_count = 0; 11646 tsb_alloc_fail_mtbf++; 11647 return ((tsb_alloc_mtbf & 1)? ENOMEM : EAGAIN); 11648 } 11649 #endif /* DEBUG */ 11650 11651 /* 11652 * Enforce high water mark if we are not doing a forced allocation 11653 * and are not shrinking a process' TSB. 11654 */ 11655 if ((flags & TSB_SHRINK) == 0 && 11656 (tsbbytes + tsb_alloc_bytes) > tsb_alloc_hiwater) { 11657 if ((flags & TSB_FORCEALLOC) == 0) 11658 return (ENOMEM); 11659 lowmem = 1; 11660 } 11661 11662 /* 11663 * Allocate from the correct location based upon the size of the TSB 11664 * compared to the base page size, and what memory conditions dictate. 11665 * Note we always do nonblocking allocations from the TSB arena since 11666 * we don't want memory fragmentation to cause processes to block 11667 * indefinitely waiting for memory; until the kernel algorithms that 11668 * coalesce large pages are improved this is our best option. 11669 * 11670 * Algorithm: 11671 * If allocating a "large" TSB (>8K), allocate from the 11672 * appropriate kmem_tsb_default_arena vmem arena 11673 * else if low on memory or the TSB_FORCEALLOC flag is set or 11674 * tsb_forceheap is set 11675 * Allocate from kernel heap via sfmmu_tsb8k_cache with 11676 * KM_SLEEP (never fails) 11677 * else 11678 * Allocate from appropriate sfmmu_tsb_cache with 11679 * KM_NOSLEEP 11680 * endif 11681 */ 11682 if (tsb_lgrp_affinity) 11683 lgrpid = lgrp_home_id(curthread); 11684 if (lgrpid == LGRP_NONE) 11685 lgrpid = 0; /* use lgrp of boot CPU */ 11686 11687 if (tsbbytes > MMU_PAGESIZE) { 11688 vmp = kmem_tsb_default_arena[lgrpid]; 11689 vaddr = (caddr_t)vmem_xalloc(vmp, tsbbytes, tsbbytes, 0, 0, 11690 NULL, NULL, VM_NOSLEEP); 11691 #ifdef DEBUG 11692 } else if (lowmem || (flags & TSB_FORCEALLOC) || tsb_forceheap) { 11693 #else /* !DEBUG */ 11694 } else if (lowmem || (flags & TSB_FORCEALLOC)) { 11695 #endif /* DEBUG */ 11696 kmem_cachep = sfmmu_tsb8k_cache; 11697 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_SLEEP); 11698 ASSERT(vaddr != NULL); 11699 } else { 11700 kmem_cachep = sfmmu_tsb_cache[lgrpid]; 11701 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_NOSLEEP); 11702 } 11703 11704 tsbinfo->tsb_cache = kmem_cachep; 11705 tsbinfo->tsb_vmp = vmp; 11706 11707 if (vaddr == NULL) { 11708 return (EAGAIN); 11709 } 11710 11711 atomic_add_64(&tsb_alloc_bytes, (int64_t)tsbbytes); 11712 kmem_cachep = tsbinfo->tsb_cache; 11713 11714 /* 11715 * If we are allocating from outside the cage, then we need to 11716 * register a relocation callback handler. Note that for now 11717 * since pseudo mappings always hang off of the slab's root page, 11718 * we need only lock the first 8K of the TSB slab. This is a bit 11719 * hacky but it is good for performance. 11720 */ 11721 if (kmem_cachep != sfmmu_tsb8k_cache) { 11722 slab_vaddr = (caddr_t)((uintptr_t)vaddr & slab_mask); 11723 ret = as_pagelock(&kas, &pplist, slab_vaddr, PAGESIZE, S_WRITE); 11724 ASSERT(ret == 0); 11725 ret = hat_add_callback(sfmmu_tsb_cb_id, vaddr, (uint_t)tsbbytes, 11726 cbflags, (void *)tsbinfo, &pfn); 11727 11728 /* 11729 * Need to free up resources if we could not successfully 11730 * add the callback function and return an error condition. 11731 */ 11732 if (ret != 0) { 11733 if (kmem_cachep) { 11734 kmem_cache_free(kmem_cachep, vaddr); 11735 } else { 11736 vmem_xfree(vmp, (void *)vaddr, tsbbytes); 11737 } 11738 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, 11739 S_WRITE); 11740 return (EAGAIN); 11741 } 11742 } else { 11743 /* 11744 * Since allocation of 8K TSBs from heap is rare and occurs 11745 * during memory pressure we allocate them from permanent 11746 * memory rather than using callbacks to get the PFN. 11747 */ 11748 pfn = hat_getpfnum(kas.a_hat, vaddr); 11749 } 11750 11751 tsbinfo->tsb_va = vaddr; 11752 tsbinfo->tsb_szc = tsbcode; 11753 tsbinfo->tsb_ttesz_mask = tteszmask; 11754 tsbinfo->tsb_next = NULL; 11755 tsbinfo->tsb_flags = 0; 11756 11757 sfmmu_tsbinfo_setup_phys(tsbinfo, pfn); 11758 11759 if (kmem_cachep != sfmmu_tsb8k_cache) { 11760 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, S_WRITE); 11761 } 11762 11763 sfmmu_inv_tsb(vaddr, tsbbytes); 11764 return (0); 11765 } 11766 11767 /* 11768 * Initialize per cpu tsb and per cpu tsbmiss_area 11769 */ 11770 void 11771 sfmmu_init_tsbs(void) 11772 { 11773 int i; 11774 struct tsbmiss *tsbmissp; 11775 struct kpmtsbm *kpmtsbmp; 11776 #ifndef sun4v 11777 extern int dcache_line_mask; 11778 #endif /* sun4v */ 11779 extern uint_t vac_colors; 11780 11781 /* 11782 * Init. tsb miss area. 11783 */ 11784 tsbmissp = tsbmiss_area; 11785 11786 for (i = 0; i < NCPU; tsbmissp++, i++) { 11787 /* 11788 * initialize the tsbmiss area. 11789 * Do this for all possible CPUs as some may be added 11790 * while the system is running. There is no cost to this. 11791 */ 11792 tsbmissp->ksfmmup = ksfmmup; 11793 #ifndef sun4v 11794 tsbmissp->dcache_line_mask = (uint16_t)dcache_line_mask; 11795 #endif /* sun4v */ 11796 tsbmissp->khashstart = 11797 (struct hmehash_bucket *)va_to_pa((caddr_t)khme_hash); 11798 tsbmissp->uhashstart = 11799 (struct hmehash_bucket *)va_to_pa((caddr_t)uhme_hash); 11800 tsbmissp->khashsz = khmehash_num; 11801 tsbmissp->uhashsz = uhmehash_num; 11802 } 11803 11804 if (kpm_enable == 0) 11805 return; 11806 11807 if (kpm_smallpages) { 11808 /* 11809 * If we're using base pagesize pages for seg_kpm 11810 * mappings, we use the kernel TSB since we can't afford 11811 * to allocate a second huge TSB for these mappings. 11812 */ 11813 kpm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11814 kpm_tsbsz = ktsb_szcode; 11815 kpmsm_tsbbase = kpm_tsbbase; 11816 kpmsm_tsbsz = kpm_tsbsz; 11817 } else { 11818 /* 11819 * In VAC conflict case, just put the entries in the 11820 * kernel 8K indexed TSB for now so we can find them. 11821 * This could really be changed in the future if we feel 11822 * the need... 11823 */ 11824 kpmsm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11825 kpmsm_tsbsz = ktsb_szcode; 11826 kpm_tsbbase = ktsb_phys? ktsb4m_pbase : (uint64_t)ktsb4m_base; 11827 kpm_tsbsz = ktsb4m_szcode; 11828 } 11829 11830 kpmtsbmp = kpmtsbm_area; 11831 for (i = 0; i < NCPU; kpmtsbmp++, i++) { 11832 /* 11833 * Initialize the kpmtsbm area. 11834 * Do this for all possible CPUs as some may be added 11835 * while the system is running. There is no cost to this. 11836 */ 11837 kpmtsbmp->vbase = kpm_vbase; 11838 kpmtsbmp->vend = kpm_vbase + kpm_size * vac_colors; 11839 kpmtsbmp->sz_shift = kpm_size_shift; 11840 kpmtsbmp->kpmp_shift = kpmp_shift; 11841 kpmtsbmp->kpmp2pshft = (uchar_t)kpmp2pshft; 11842 if (kpm_smallpages == 0) { 11843 kpmtsbmp->kpmp_table_sz = kpmp_table_sz; 11844 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_table); 11845 } else { 11846 kpmtsbmp->kpmp_table_sz = kpmp_stable_sz; 11847 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_stable); 11848 } 11849 kpmtsbmp->msegphashpa = va_to_pa(memseg_phash); 11850 kpmtsbmp->flags = KPMTSBM_ENABLE_FLAG; 11851 #ifdef DEBUG 11852 kpmtsbmp->flags |= (kpm_tsbmtl) ? KPMTSBM_TLTSBM_FLAG : 0; 11853 #endif /* DEBUG */ 11854 if (ktsb_phys) 11855 kpmtsbmp->flags |= KPMTSBM_TSBPHYS_FLAG; 11856 } 11857 11858 sfmmu_tsb_cb_id = hat_register_callback('T'<<16 | 'S' << 8 | 'B', 11859 sfmmu_tsb_pre_relocator, sfmmu_tsb_post_relocator, NULL, 0); 11860 } 11861 11862 /* Avoid using sfmmu_tsbinfo_alloc() to avoid kmem_alloc - no real reason */ 11863 struct tsb_info ktsb_info[2]; 11864 11865 /* 11866 * Called from hat_kern_setup() to setup the tsb_info for ksfmmup. 11867 */ 11868 void 11869 sfmmu_init_ktsbinfo() 11870 { 11871 ASSERT(ksfmmup != NULL); 11872 ASSERT(ksfmmup->sfmmu_tsb == NULL); 11873 /* 11874 * Allocate tsbinfos for kernel and copy in data 11875 * to make debug easier and sun4v setup easier. 11876 */ 11877 ktsb_info[0].tsb_sfmmu = ksfmmup; 11878 ktsb_info[0].tsb_szc = ktsb_szcode; 11879 ktsb_info[0].tsb_ttesz_mask = TSB8K|TSB64K|TSB512K; 11880 ktsb_info[0].tsb_va = ktsb_base; 11881 ktsb_info[0].tsb_pa = ktsb_pbase; 11882 ktsb_info[0].tsb_flags = 0; 11883 ktsb_info[0].tsb_tte.ll = 0; 11884 ktsb_info[0].tsb_cache = NULL; 11885 11886 ktsb_info[1].tsb_sfmmu = ksfmmup; 11887 ktsb_info[1].tsb_szc = ktsb4m_szcode; 11888 ktsb_info[1].tsb_ttesz_mask = TSB4M; 11889 ktsb_info[1].tsb_va = ktsb4m_base; 11890 ktsb_info[1].tsb_pa = ktsb4m_pbase; 11891 ktsb_info[1].tsb_flags = 0; 11892 ktsb_info[1].tsb_tte.ll = 0; 11893 ktsb_info[1].tsb_cache = NULL; 11894 11895 /* Link them into ksfmmup. */ 11896 ktsb_info[0].tsb_next = &ktsb_info[1]; 11897 ktsb_info[1].tsb_next = NULL; 11898 ksfmmup->sfmmu_tsb = &ktsb_info[0]; 11899 11900 sfmmu_setup_tsbinfo(ksfmmup); 11901 } 11902 11903 /* 11904 * Cache the last value returned from va_to_pa(). If the VA specified 11905 * in the current call to cached_va_to_pa() maps to the same Page (as the 11906 * previous call to cached_va_to_pa()), then compute the PA using 11907 * cached info, else call va_to_pa(). 11908 * 11909 * Note: this function is neither MT-safe nor consistent in the presence 11910 * of multiple, interleaved threads. This function was created to enable 11911 * an optimization used during boot (at a point when there's only one thread 11912 * executing on the "boot CPU", and before startup_vm() has been called). 11913 */ 11914 static uint64_t 11915 cached_va_to_pa(void *vaddr) 11916 { 11917 static uint64_t prev_vaddr_base = 0; 11918 static uint64_t prev_pfn = 0; 11919 11920 if ((((uint64_t)vaddr) & MMU_PAGEMASK) == prev_vaddr_base) { 11921 return (prev_pfn | ((uint64_t)vaddr & MMU_PAGEOFFSET)); 11922 } else { 11923 uint64_t pa = va_to_pa(vaddr); 11924 11925 if (pa != ((uint64_t)-1)) { 11926 /* 11927 * Computed physical address is valid. Cache its 11928 * related info for the next cached_va_to_pa() call. 11929 */ 11930 prev_pfn = pa & MMU_PAGEMASK; 11931 prev_vaddr_base = ((uint64_t)vaddr) & MMU_PAGEMASK; 11932 } 11933 11934 return (pa); 11935 } 11936 } 11937 11938 /* 11939 * Carve up our nucleus hblk region. We may allocate more hblks than 11940 * asked due to rounding errors but we are guaranteed to have at least 11941 * enough space to allocate the requested number of hblk8's and hblk1's. 11942 */ 11943 void 11944 sfmmu_init_nucleus_hblks(caddr_t addr, size_t size, int nhblk8, int nhblk1) 11945 { 11946 struct hme_blk *hmeblkp; 11947 size_t hme8blk_sz, hme1blk_sz; 11948 size_t i; 11949 size_t hblk8_bound; 11950 ulong_t j = 0, k = 0; 11951 11952 ASSERT(addr != NULL && size != 0); 11953 11954 /* Need to use proper structure alignment */ 11955 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 11956 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 11957 11958 nucleus_hblk8.list = (void *)addr; 11959 nucleus_hblk8.index = 0; 11960 11961 /* 11962 * Use as much memory as possible for hblk8's since we 11963 * expect all bop_alloc'ed memory to be allocated in 8k chunks. 11964 * We need to hold back enough space for the hblk1's which 11965 * we'll allocate next. 11966 */ 11967 hblk8_bound = size - (nhblk1 * hme1blk_sz) - hme8blk_sz; 11968 for (i = 0; i <= hblk8_bound; i += hme8blk_sz, j++) { 11969 hmeblkp = (struct hme_blk *)addr; 11970 addr += hme8blk_sz; 11971 hmeblkp->hblk_nuc_bit = 1; 11972 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11973 } 11974 nucleus_hblk8.len = j; 11975 ASSERT(j >= nhblk8); 11976 SFMMU_STAT_ADD(sf_hblk8_ncreate, j); 11977 11978 nucleus_hblk1.list = (void *)addr; 11979 nucleus_hblk1.index = 0; 11980 for (; i <= (size - hme1blk_sz); i += hme1blk_sz, k++) { 11981 hmeblkp = (struct hme_blk *)addr; 11982 addr += hme1blk_sz; 11983 hmeblkp->hblk_nuc_bit = 1; 11984 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11985 } 11986 ASSERT(k >= nhblk1); 11987 nucleus_hblk1.len = k; 11988 SFMMU_STAT_ADD(sf_hblk1_ncreate, k); 11989 } 11990 11991 /* 11992 * This function is currently not supported on this platform. For what 11993 * it's supposed to do, see hat.c and hat_srmmu.c 11994 */ 11995 /* ARGSUSED */ 11996 faultcode_t 11997 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp, 11998 uint_t flags) 11999 { 12000 ASSERT(hat->sfmmu_xhat_provider == NULL); 12001 return (FC_NOSUPPORT); 12002 } 12003 12004 /* 12005 * Searchs the mapping list of the page for a mapping of the same size. If not 12006 * found the corresponding bit is cleared in the p_index field. When large 12007 * pages are more prevalent in the system, we can maintain the mapping list 12008 * in order and we don't have to traverse the list each time. Just check the 12009 * next and prev entries, and if both are of different size, we clear the bit. 12010 */ 12011 static void 12012 sfmmu_rm_large_mappings(page_t *pp, int ttesz) 12013 { 12014 struct sf_hment *sfhmep; 12015 struct hme_blk *hmeblkp; 12016 int index; 12017 pgcnt_t npgs; 12018 12019 ASSERT(ttesz > TTE8K); 12020 12021 ASSERT(sfmmu_mlist_held(pp)); 12022 12023 ASSERT(PP_ISMAPPED_LARGE(pp)); 12024 12025 /* 12026 * Traverse mapping list looking for another mapping of same size. 12027 * since we only want to clear index field if all mappings of 12028 * that size are gone. 12029 */ 12030 12031 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 12032 hmeblkp = sfmmu_hmetohblk(sfhmep); 12033 if (hmeblkp->hblk_xhat_bit) 12034 continue; 12035 if (hme_size(sfhmep) == ttesz) { 12036 /* 12037 * another mapping of the same size. don't clear index. 12038 */ 12039 return; 12040 } 12041 } 12042 12043 /* 12044 * Clear the p_index bit for large page. 12045 */ 12046 index = PAGESZ_TO_INDEX(ttesz); 12047 npgs = TTEPAGES(ttesz); 12048 while (npgs-- > 0) { 12049 ASSERT(pp->p_index & index); 12050 pp->p_index &= ~index; 12051 pp = PP_PAGENEXT(pp); 12052 } 12053 } 12054 12055 /* 12056 * return supported features 12057 */ 12058 /* ARGSUSED */ 12059 int 12060 hat_supported(enum hat_features feature, void *arg) 12061 { 12062 switch (feature) { 12063 case HAT_SHARED_PT: 12064 case HAT_DYNAMIC_ISM_UNMAP: 12065 case HAT_VMODSORT: 12066 return (1); 12067 default: 12068 return (0); 12069 } 12070 } 12071 12072 void 12073 hat_enter(struct hat *hat) 12074 { 12075 hatlock_t *hatlockp; 12076 12077 if (hat != ksfmmup) { 12078 hatlockp = TSB_HASH(hat); 12079 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 12080 } 12081 } 12082 12083 void 12084 hat_exit(struct hat *hat) 12085 { 12086 hatlock_t *hatlockp; 12087 12088 if (hat != ksfmmup) { 12089 hatlockp = TSB_HASH(hat); 12090 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 12091 } 12092 } 12093 12094 /*ARGSUSED*/ 12095 void 12096 hat_reserve(struct as *as, caddr_t addr, size_t len) 12097 { 12098 } 12099 12100 static void 12101 hat_kstat_init(void) 12102 { 12103 kstat_t *ksp; 12104 12105 ksp = kstat_create("unix", 0, "sfmmu_global_stat", "hat", 12106 KSTAT_TYPE_RAW, sizeof (struct sfmmu_global_stat), 12107 KSTAT_FLAG_VIRTUAL); 12108 if (ksp) { 12109 ksp->ks_data = (void *) &sfmmu_global_stat; 12110 kstat_install(ksp); 12111 } 12112 ksp = kstat_create("unix", 0, "sfmmu_tsbsize_stat", "hat", 12113 KSTAT_TYPE_RAW, sizeof (struct sfmmu_tsbsize_stat), 12114 KSTAT_FLAG_VIRTUAL); 12115 if (ksp) { 12116 ksp->ks_data = (void *) &sfmmu_tsbsize_stat; 12117 kstat_install(ksp); 12118 } 12119 ksp = kstat_create("unix", 0, "sfmmu_percpu_stat", "hat", 12120 KSTAT_TYPE_RAW, sizeof (struct sfmmu_percpu_stat) * NCPU, 12121 KSTAT_FLAG_WRITABLE); 12122 if (ksp) { 12123 ksp->ks_update = sfmmu_kstat_percpu_update; 12124 kstat_install(ksp); 12125 } 12126 } 12127 12128 /* ARGSUSED */ 12129 static int 12130 sfmmu_kstat_percpu_update(kstat_t *ksp, int rw) 12131 { 12132 struct sfmmu_percpu_stat *cpu_kstat = ksp->ks_data; 12133 struct tsbmiss *tsbm = tsbmiss_area; 12134 struct kpmtsbm *kpmtsbm = kpmtsbm_area; 12135 int i; 12136 12137 ASSERT(cpu_kstat); 12138 if (rw == KSTAT_READ) { 12139 for (i = 0; i < NCPU; cpu_kstat++, tsbm++, kpmtsbm++, i++) { 12140 cpu_kstat->sf_itlb_misses = tsbm->itlb_misses; 12141 cpu_kstat->sf_dtlb_misses = tsbm->dtlb_misses; 12142 cpu_kstat->sf_utsb_misses = tsbm->utsb_misses - 12143 tsbm->uprot_traps; 12144 cpu_kstat->sf_ktsb_misses = tsbm->ktsb_misses + 12145 kpmtsbm->kpm_tsb_misses - tsbm->kprot_traps; 12146 12147 if (tsbm->itlb_misses > 0 && tsbm->dtlb_misses > 0) { 12148 cpu_kstat->sf_tsb_hits = 12149 (tsbm->itlb_misses + tsbm->dtlb_misses) - 12150 (tsbm->utsb_misses + tsbm->ktsb_misses + 12151 kpmtsbm->kpm_tsb_misses); 12152 } else { 12153 cpu_kstat->sf_tsb_hits = 0; 12154 } 12155 cpu_kstat->sf_umod_faults = tsbm->uprot_traps; 12156 cpu_kstat->sf_kmod_faults = tsbm->kprot_traps; 12157 } 12158 } else { 12159 /* KSTAT_WRITE is used to clear stats */ 12160 for (i = 0; i < NCPU; tsbm++, kpmtsbm++, i++) { 12161 tsbm->itlb_misses = 0; 12162 tsbm->dtlb_misses = 0; 12163 tsbm->utsb_misses = 0; 12164 tsbm->ktsb_misses = 0; 12165 tsbm->uprot_traps = 0; 12166 tsbm->kprot_traps = 0; 12167 kpmtsbm->kpm_dtlb_misses = 0; 12168 kpmtsbm->kpm_tsb_misses = 0; 12169 } 12170 } 12171 return (0); 12172 } 12173 12174 #ifdef DEBUG 12175 12176 tte_t *gorig[NCPU], *gcur[NCPU], *gnew[NCPU]; 12177 12178 /* 12179 * A tte checker. *orig_old is the value we read before cas. 12180 * *cur is the value returned by cas. 12181 * *new is the desired value when we do the cas. 12182 * 12183 * *hmeblkp is currently unused. 12184 */ 12185 12186 /* ARGSUSED */ 12187 void 12188 chk_tte(tte_t *orig_old, tte_t *cur, tte_t *new, struct hme_blk *hmeblkp) 12189 { 12190 uint_t i, j, k; 12191 int cpuid = CPU->cpu_id; 12192 12193 gorig[cpuid] = orig_old; 12194 gcur[cpuid] = cur; 12195 gnew[cpuid] = new; 12196 12197 #ifdef lint 12198 hmeblkp = hmeblkp; 12199 #endif 12200 12201 if (TTE_IS_VALID(orig_old)) { 12202 if (TTE_IS_VALID(cur)) { 12203 i = TTE_TO_TTEPFN(orig_old); 12204 j = TTE_TO_TTEPFN(cur); 12205 k = TTE_TO_TTEPFN(new); 12206 if (i != j) { 12207 /* remap error? */ 12208 panic("chk_tte: bad pfn, 0x%x, 0x%x", 12209 i, j); 12210 } 12211 12212 if (i != k) { 12213 /* remap error? */ 12214 panic("chk_tte: bad pfn2, 0x%x, 0x%x", 12215 i, k); 12216 } 12217 } else { 12218 if (TTE_IS_VALID(new)) { 12219 panic("chk_tte: invalid cur? "); 12220 } 12221 12222 i = TTE_TO_TTEPFN(orig_old); 12223 k = TTE_TO_TTEPFN(new); 12224 if (i != k) { 12225 panic("chk_tte: bad pfn3, 0x%x, 0x%x", 12226 i, k); 12227 } 12228 } 12229 } else { 12230 if (TTE_IS_VALID(cur)) { 12231 j = TTE_TO_TTEPFN(cur); 12232 if (TTE_IS_VALID(new)) { 12233 k = TTE_TO_TTEPFN(new); 12234 if (j != k) { 12235 panic("chk_tte: bad pfn4, 0x%x, 0x%x", 12236 j, k); 12237 } 12238 } else { 12239 panic("chk_tte: why here?"); 12240 } 12241 } else { 12242 if (!TTE_IS_VALID(new)) { 12243 panic("chk_tte: why here2 ?"); 12244 } 12245 } 12246 } 12247 } 12248 12249 #endif /* DEBUG */ 12250 12251 extern void prefetch_tsbe_read(struct tsbe *); 12252 extern void prefetch_tsbe_write(struct tsbe *); 12253 12254 12255 /* 12256 * We want to prefetch 7 cache lines ahead for our read prefetch. This gives 12257 * us optimal performance on Cheetah+. You can only have 8 outstanding 12258 * prefetches at any one time, so we opted for 7 read prefetches and 1 write 12259 * prefetch to make the most utilization of the prefetch capability. 12260 */ 12261 #define TSBE_PREFETCH_STRIDE (7) 12262 12263 void 12264 sfmmu_copy_tsb(struct tsb_info *old_tsbinfo, struct tsb_info *new_tsbinfo) 12265 { 12266 int old_bytes = TSB_BYTES(old_tsbinfo->tsb_szc); 12267 int new_bytes = TSB_BYTES(new_tsbinfo->tsb_szc); 12268 int old_entries = TSB_ENTRIES(old_tsbinfo->tsb_szc); 12269 int new_entries = TSB_ENTRIES(new_tsbinfo->tsb_szc); 12270 struct tsbe *old; 12271 struct tsbe *new; 12272 struct tsbe *new_base = (struct tsbe *)new_tsbinfo->tsb_va; 12273 uint64_t va; 12274 int new_offset; 12275 int i; 12276 int vpshift; 12277 int last_prefetch; 12278 12279 if (old_bytes == new_bytes) { 12280 bcopy(old_tsbinfo->tsb_va, new_tsbinfo->tsb_va, new_bytes); 12281 } else { 12282 12283 /* 12284 * A TSBE is 16 bytes which means there are four TSBE's per 12285 * P$ line (64 bytes), thus every 4 TSBE's we prefetch. 12286 */ 12287 old = (struct tsbe *)old_tsbinfo->tsb_va; 12288 last_prefetch = old_entries - (4*(TSBE_PREFETCH_STRIDE+1)); 12289 for (i = 0; i < old_entries; i++, old++) { 12290 if (((i & (4-1)) == 0) && (i < last_prefetch)) 12291 prefetch_tsbe_read(old); 12292 if (!old->tte_tag.tag_invalid) { 12293 /* 12294 * We have a valid TTE to remap. Check the 12295 * size. We won't remap 64K or 512K TTEs 12296 * because they span more than one TSB entry 12297 * and are indexed using an 8K virt. page. 12298 * Ditto for 32M and 256M TTEs. 12299 */ 12300 if (TTE_CSZ(&old->tte_data) == TTE64K || 12301 TTE_CSZ(&old->tte_data) == TTE512K) 12302 continue; 12303 if (mmu_page_sizes == max_mmu_page_sizes) { 12304 if (TTE_CSZ(&old->tte_data) == TTE32M || 12305 TTE_CSZ(&old->tte_data) == TTE256M) 12306 continue; 12307 } 12308 12309 /* clear the lower 22 bits of the va */ 12310 va = *(uint64_t *)old << 22; 12311 /* turn va into a virtual pfn */ 12312 va >>= 22 - TSB_START_SIZE; 12313 /* 12314 * or in bits from the offset in the tsb 12315 * to get the real virtual pfn. These 12316 * correspond to bits [21:13] in the va 12317 */ 12318 vpshift = 12319 TTE_BSZS_SHIFT(TTE_CSZ(&old->tte_data)) & 12320 0x1ff; 12321 va |= (i << vpshift); 12322 va >>= vpshift; 12323 new_offset = va & (new_entries - 1); 12324 new = new_base + new_offset; 12325 prefetch_tsbe_write(new); 12326 *new = *old; 12327 } 12328 } 12329 } 12330 } 12331 12332 /* 12333 * Kernel Physical Mapping (kpm) facility 12334 */ 12335 12336 /* -- hat_kpm interface section -- */ 12337 12338 /* 12339 * Mapin a locked page and return the vaddr. 12340 * When a kpme is provided by the caller it is added to 12341 * the page p_kpmelist. The page to be mapped in must 12342 * be at least read locked (p_selock). 12343 */ 12344 caddr_t 12345 hat_kpm_mapin(struct page *pp, struct kpme *kpme) 12346 { 12347 kmutex_t *pml; 12348 caddr_t vaddr; 12349 12350 if (kpm_enable == 0) { 12351 cmn_err(CE_WARN, "hat_kpm_mapin: kpm_enable not set"); 12352 return ((caddr_t)NULL); 12353 } 12354 12355 if (pp == NULL || PAGE_LOCKED(pp) == 0) { 12356 cmn_err(CE_WARN, "hat_kpm_mapin: pp zero or not locked"); 12357 return ((caddr_t)NULL); 12358 } 12359 12360 pml = sfmmu_mlist_enter(pp); 12361 ASSERT(pp->p_kpmref >= 0); 12362 12363 vaddr = (pp->p_kpmref == 0) ? 12364 sfmmu_kpm_mapin(pp) : hat_kpm_page2va(pp, 1); 12365 12366 if (kpme != NULL) { 12367 /* 12368 * Tolerate multiple mapins for the same kpme to avoid 12369 * the need for an extra serialization. 12370 */ 12371 if ((sfmmu_kpme_lookup(kpme, pp)) == 0) 12372 sfmmu_kpme_add(kpme, pp); 12373 12374 ASSERT(pp->p_kpmref > 0); 12375 12376 } else { 12377 pp->p_kpmref++; 12378 } 12379 12380 sfmmu_mlist_exit(pml); 12381 return (vaddr); 12382 } 12383 12384 /* 12385 * Mapout a locked page. 12386 * When a kpme is provided by the caller it is removed from 12387 * the page p_kpmelist. The page to be mapped out must be at 12388 * least read locked (p_selock). 12389 * Note: The seg_kpm layer provides a mapout interface for the 12390 * case that a kpme is used and the underlying page is unlocked. 12391 * This can be used instead of calling this function directly. 12392 */ 12393 void 12394 hat_kpm_mapout(struct page *pp, struct kpme *kpme, caddr_t vaddr) 12395 { 12396 kmutex_t *pml; 12397 12398 if (kpm_enable == 0) { 12399 cmn_err(CE_WARN, "hat_kpm_mapout: kpm_enable not set"); 12400 return; 12401 } 12402 12403 if (IS_KPM_ADDR(vaddr) == 0) { 12404 cmn_err(CE_WARN, "hat_kpm_mapout: no kpm address"); 12405 return; 12406 } 12407 12408 if (pp == NULL || PAGE_LOCKED(pp) == 0) { 12409 cmn_err(CE_WARN, "hat_kpm_mapout: page zero or not locked"); 12410 return; 12411 } 12412 12413 if (kpme != NULL) { 12414 ASSERT(pp == kpme->kpe_page); 12415 pp = kpme->kpe_page; 12416 pml = sfmmu_mlist_enter(pp); 12417 12418 if (sfmmu_kpme_lookup(kpme, pp) == 0) 12419 panic("hat_kpm_mapout: kpme not found pp=%p", 12420 (void *)pp); 12421 12422 ASSERT(pp->p_kpmref > 0); 12423 sfmmu_kpme_sub(kpme, pp); 12424 12425 } else { 12426 pml = sfmmu_mlist_enter(pp); 12427 pp->p_kpmref--; 12428 } 12429 12430 ASSERT(pp->p_kpmref >= 0); 12431 if (pp->p_kpmref == 0) 12432 sfmmu_kpm_mapout(pp, vaddr); 12433 12434 sfmmu_mlist_exit(pml); 12435 } 12436 12437 /* 12438 * Return the kpm virtual address for the page at pp. 12439 * If checkswap is non zero and the page is backed by a 12440 * swap vnode the physical address is used rather than 12441 * p_offset to determine the kpm region. 12442 * Note: The function has to be used w/ extreme care. The 12443 * stability of the page identity is in the responsibility 12444 * of the caller. 12445 */ 12446 caddr_t 12447 hat_kpm_page2va(struct page *pp, int checkswap) 12448 { 12449 int vcolor, vcolor_pa; 12450 uintptr_t paddr, vaddr; 12451 12452 ASSERT(kpm_enable); 12453 12454 paddr = ptob(pp->p_pagenum); 12455 vcolor_pa = addr_to_vcolor(paddr); 12456 12457 if (checkswap && pp->p_vnode && IS_SWAPFSVP(pp->p_vnode)) 12458 vcolor = (PP_ISNC(pp)) ? vcolor_pa : PP_GET_VCOLOR(pp); 12459 else 12460 vcolor = addr_to_vcolor(pp->p_offset); 12461 12462 vaddr = (uintptr_t)kpm_vbase + paddr; 12463 12464 if (vcolor_pa != vcolor) { 12465 vaddr += ((uintptr_t)(vcolor - vcolor_pa) << MMU_PAGESHIFT); 12466 vaddr += (vcolor_pa > vcolor) ? 12467 ((uintptr_t)vcolor_pa << kpm_size_shift) : 12468 ((uintptr_t)(vcolor - vcolor_pa) << kpm_size_shift); 12469 } 12470 12471 return ((caddr_t)vaddr); 12472 } 12473 12474 /* 12475 * Return the page for the kpm virtual address vaddr. 12476 * Caller is responsible for the kpm mapping and lock 12477 * state of the page. 12478 */ 12479 page_t * 12480 hat_kpm_vaddr2page(caddr_t vaddr) 12481 { 12482 uintptr_t paddr; 12483 pfn_t pfn; 12484 12485 ASSERT(IS_KPM_ADDR(vaddr)); 12486 12487 SFMMU_KPM_VTOP(vaddr, paddr); 12488 pfn = (pfn_t)btop(paddr); 12489 12490 return (page_numtopp_nolock(pfn)); 12491 } 12492 12493 /* page to kpm_page */ 12494 #define PP2KPMPG(pp, kp) { \ 12495 struct memseg *mseg; \ 12496 pgcnt_t inx; \ 12497 pfn_t pfn; \ 12498 \ 12499 pfn = pp->p_pagenum; \ 12500 mseg = page_numtomemseg_nolock(pfn); \ 12501 ASSERT(mseg); \ 12502 inx = ptokpmp(kpmptop(ptokpmp(pfn)) - mseg->kpm_pbase); \ 12503 ASSERT(inx < mseg->kpm_nkpmpgs); \ 12504 kp = &mseg->kpm_pages[inx]; \ 12505 } 12506 12507 /* page to kpm_spage */ 12508 #define PP2KPMSPG(pp, ksp) { \ 12509 struct memseg *mseg; \ 12510 pgcnt_t inx; \ 12511 pfn_t pfn; \ 12512 \ 12513 pfn = pp->p_pagenum; \ 12514 mseg = page_numtomemseg_nolock(pfn); \ 12515 ASSERT(mseg); \ 12516 inx = pfn - mseg->kpm_pbase; \ 12517 ksp = &mseg->kpm_spages[inx]; \ 12518 } 12519 12520 /* 12521 * hat_kpm_fault is called from segkpm_fault when a kpm tsbmiss occurred 12522 * which could not be resolved by the trap level tsbmiss handler for the 12523 * following reasons: 12524 * . The vaddr is in VAC alias range (always PAGESIZE mapping size). 12525 * . The kpm (s)page range of vaddr is in a VAC alias prevention state. 12526 * . tsbmiss handling at trap level is not desired (DEBUG kernel only, 12527 * kpm_tsbmtl == 0). 12528 */ 12529 int 12530 hat_kpm_fault(struct hat *hat, caddr_t vaddr) 12531 { 12532 int error; 12533 uintptr_t paddr; 12534 pfn_t pfn; 12535 struct memseg *mseg; 12536 page_t *pp; 12537 12538 if (kpm_enable == 0) { 12539 cmn_err(CE_WARN, "hat_kpm_fault: kpm_enable not set"); 12540 return (ENOTSUP); 12541 } 12542 12543 ASSERT(hat == ksfmmup); 12544 ASSERT(IS_KPM_ADDR(vaddr)); 12545 12546 SFMMU_KPM_VTOP(vaddr, paddr); 12547 pfn = (pfn_t)btop(paddr); 12548 mseg = page_numtomemseg_nolock(pfn); 12549 if (mseg == NULL) 12550 return (EFAULT); 12551 12552 pp = &mseg->pages[(pgcnt_t)(pfn - mseg->pages_base)]; 12553 ASSERT((pfn_t)pp->p_pagenum == pfn); 12554 12555 if (!PAGE_LOCKED(pp)) 12556 return (EFAULT); 12557 12558 if (kpm_smallpages == 0) 12559 error = sfmmu_kpm_fault(vaddr, mseg, pp); 12560 else 12561 error = sfmmu_kpm_fault_small(vaddr, mseg, pp); 12562 12563 return (error); 12564 } 12565 12566 extern krwlock_t memsegslock; 12567 12568 /* 12569 * memseg_hash[] was cleared, need to clear memseg_phash[] too. 12570 */ 12571 void 12572 hat_kpm_mseghash_clear(int nentries) 12573 { 12574 pgcnt_t i; 12575 12576 if (kpm_enable == 0) 12577 return; 12578 12579 for (i = 0; i < nentries; i++) 12580 memseg_phash[i] = MSEG_NULLPTR_PA; 12581 } 12582 12583 /* 12584 * Update memseg_phash[inx] when memseg_hash[inx] was changed. 12585 */ 12586 void 12587 hat_kpm_mseghash_update(pgcnt_t inx, struct memseg *msp) 12588 { 12589 if (kpm_enable == 0) 12590 return; 12591 12592 memseg_phash[inx] = (msp) ? va_to_pa(msp) : MSEG_NULLPTR_PA; 12593 } 12594 12595 /* 12596 * Update kpm memseg members from basic memseg info. 12597 */ 12598 void 12599 hat_kpm_addmem_mseg_update(struct memseg *msp, pgcnt_t nkpmpgs, 12600 offset_t kpm_pages_off) 12601 { 12602 if (kpm_enable == 0) 12603 return; 12604 12605 msp->kpm_pages = (kpm_page_t *)((caddr_t)msp->pages + kpm_pages_off); 12606 msp->kpm_nkpmpgs = nkpmpgs; 12607 msp->kpm_pbase = kpmptop(ptokpmp(msp->pages_base)); 12608 msp->pagespa = va_to_pa(msp->pages); 12609 msp->epagespa = va_to_pa(msp->epages); 12610 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 12611 } 12612 12613 /* 12614 * Setup nextpa when a memseg is inserted. 12615 * Assumes that the memsegslock is already held. 12616 */ 12617 void 12618 hat_kpm_addmem_mseg_insert(struct memseg *msp) 12619 { 12620 if (kpm_enable == 0) 12621 return; 12622 12623 ASSERT(RW_LOCK_HELD(&memsegslock)); 12624 msp->nextpa = (memsegs) ? va_to_pa(memsegs) : MSEG_NULLPTR_PA; 12625 } 12626 12627 /* 12628 * Setup memsegspa when a memseg is (head) inserted. 12629 * Called before memsegs is updated to complete a 12630 * memseg insert operation. 12631 * Assumes that the memsegslock is already held. 12632 */ 12633 void 12634 hat_kpm_addmem_memsegs_update(struct memseg *msp) 12635 { 12636 if (kpm_enable == 0) 12637 return; 12638 12639 ASSERT(RW_LOCK_HELD(&memsegslock)); 12640 ASSERT(memsegs); 12641 memsegspa = va_to_pa(msp); 12642 } 12643 12644 /* 12645 * Return end of metadata for an already setup memseg. 12646 * 12647 * Note: kpm_pages and kpm_spages are aliases and the underlying 12648 * member of struct memseg is a union, therefore they always have 12649 * the same address within a memseg. They must be differentiated 12650 * when pointer arithmetic is used with them. 12651 */ 12652 caddr_t 12653 hat_kpm_mseg_reuse(struct memseg *msp) 12654 { 12655 caddr_t end; 12656 12657 if (kpm_smallpages == 0) 12658 end = (caddr_t)(msp->kpm_pages + msp->kpm_nkpmpgs); 12659 else 12660 end = (caddr_t)(msp->kpm_spages + msp->kpm_nkpmpgs); 12661 12662 return (end); 12663 } 12664 12665 /* 12666 * Update memsegspa (when first memseg in list 12667 * is deleted) or nextpa when a memseg deleted. 12668 * Assumes that the memsegslock is already held. 12669 */ 12670 void 12671 hat_kpm_delmem_mseg_update(struct memseg *msp, struct memseg **mspp) 12672 { 12673 struct memseg *lmsp; 12674 12675 if (kpm_enable == 0) 12676 return; 12677 12678 ASSERT(RW_LOCK_HELD(&memsegslock)); 12679 12680 if (mspp == &memsegs) { 12681 memsegspa = (msp->next) ? 12682 va_to_pa(msp->next) : MSEG_NULLPTR_PA; 12683 } else { 12684 lmsp = (struct memseg *) 12685 ((uint64_t)mspp - offsetof(struct memseg, next)); 12686 lmsp->nextpa = (msp->next) ? 12687 va_to_pa(msp->next) : MSEG_NULLPTR_PA; 12688 } 12689 } 12690 12691 /* 12692 * Update kpm members for all memseg's involved in a split operation 12693 * and do the atomic update of the physical memseg chain. 12694 * 12695 * Note: kpm_pages and kpm_spages are aliases and the underlying member 12696 * of struct memseg is a union, therefore they always have the same 12697 * address within a memseg. With that the direct assignments and 12698 * va_to_pa conversions below don't have to be distinguished wrt. to 12699 * kpm_smallpages. They must be differentiated when pointer arithmetic 12700 * is used with them. 12701 * 12702 * Assumes that the memsegslock is already held. 12703 */ 12704 void 12705 hat_kpm_split_mseg_update(struct memseg *msp, struct memseg **mspp, 12706 struct memseg *lo, struct memseg *mid, struct memseg *hi) 12707 { 12708 pgcnt_t start, end, kbase, kstart, num; 12709 struct memseg *lmsp; 12710 12711 if (kpm_enable == 0) 12712 return; 12713 12714 ASSERT(RW_LOCK_HELD(&memsegslock)); 12715 ASSERT(msp && mid && msp->kpm_pages); 12716 12717 kbase = ptokpmp(msp->kpm_pbase); 12718 12719 if (lo) { 12720 num = lo->pages_end - lo->pages_base; 12721 start = kpmptop(ptokpmp(lo->pages_base)); 12722 /* align end to kpm page size granularity */ 12723 end = kpmptop(ptokpmp(start + num - 1)) + kpmpnpgs; 12724 lo->kpm_pbase = start; 12725 lo->kpm_nkpmpgs = ptokpmp(end - start); 12726 lo->kpm_pages = msp->kpm_pages; 12727 lo->kpm_pagespa = va_to_pa(lo->kpm_pages); 12728 lo->pagespa = va_to_pa(lo->pages); 12729 lo->epagespa = va_to_pa(lo->epages); 12730 lo->nextpa = va_to_pa(lo->next); 12731 } 12732 12733 /* mid */ 12734 num = mid->pages_end - mid->pages_base; 12735 kstart = ptokpmp(mid->pages_base); 12736 start = kpmptop(kstart); 12737 /* align end to kpm page size granularity */ 12738 end = kpmptop(ptokpmp(start + num - 1)) + kpmpnpgs; 12739 mid->kpm_pbase = start; 12740 mid->kpm_nkpmpgs = ptokpmp(end - start); 12741 if (kpm_smallpages == 0) { 12742 mid->kpm_pages = msp->kpm_pages + (kstart - kbase); 12743 } else { 12744 mid->kpm_spages = msp->kpm_spages + (kstart - kbase); 12745 } 12746 mid->kpm_pagespa = va_to_pa(mid->kpm_pages); 12747 mid->pagespa = va_to_pa(mid->pages); 12748 mid->epagespa = va_to_pa(mid->epages); 12749 mid->nextpa = (mid->next) ? va_to_pa(mid->next) : MSEG_NULLPTR_PA; 12750 12751 if (hi) { 12752 num = hi->pages_end - hi->pages_base; 12753 kstart = ptokpmp(hi->pages_base); 12754 start = kpmptop(kstart); 12755 /* align end to kpm page size granularity */ 12756 end = kpmptop(ptokpmp(start + num - 1)) + kpmpnpgs; 12757 hi->kpm_pbase = start; 12758 hi->kpm_nkpmpgs = ptokpmp(end - start); 12759 if (kpm_smallpages == 0) { 12760 hi->kpm_pages = msp->kpm_pages + (kstart - kbase); 12761 } else { 12762 hi->kpm_spages = msp->kpm_spages + (kstart - kbase); 12763 } 12764 hi->kpm_pagespa = va_to_pa(hi->kpm_pages); 12765 hi->pagespa = va_to_pa(hi->pages); 12766 hi->epagespa = va_to_pa(hi->epages); 12767 hi->nextpa = (hi->next) ? va_to_pa(hi->next) : MSEG_NULLPTR_PA; 12768 } 12769 12770 /* 12771 * Atomic update of the physical memseg chain 12772 */ 12773 if (mspp == &memsegs) { 12774 memsegspa = (lo) ? va_to_pa(lo) : va_to_pa(mid); 12775 } else { 12776 lmsp = (struct memseg *) 12777 ((uint64_t)mspp - offsetof(struct memseg, next)); 12778 lmsp->nextpa = (lo) ? va_to_pa(lo) : va_to_pa(mid); 12779 } 12780 } 12781 12782 /* 12783 * Walk the memsegs chain, applying func to each memseg span and vcolor. 12784 */ 12785 void 12786 hat_kpm_walk(void (*func)(void *, void *, size_t), void *arg) 12787 { 12788 pfn_t pbase, pend; 12789 int vcolor; 12790 void *base; 12791 size_t size; 12792 struct memseg *msp; 12793 extern uint_t vac_colors; 12794 12795 for (msp = memsegs; msp; msp = msp->next) { 12796 pbase = msp->pages_base; 12797 pend = msp->pages_end; 12798 for (vcolor = 0; vcolor < vac_colors; vcolor++) { 12799 base = ptob(pbase) + kpm_vbase + kpm_size * vcolor; 12800 size = ptob(pend - pbase); 12801 func(arg, base, size); 12802 } 12803 } 12804 } 12805 12806 12807 /* -- sfmmu_kpm internal section -- */ 12808 12809 /* 12810 * Return the page frame number if a valid segkpm mapping exists 12811 * for vaddr, otherwise return PFN_INVALID. No locks are grabbed. 12812 * Should only be used by other sfmmu routines. 12813 */ 12814 pfn_t 12815 sfmmu_kpm_vatopfn(caddr_t vaddr) 12816 { 12817 uintptr_t paddr; 12818 pfn_t pfn; 12819 page_t *pp; 12820 12821 ASSERT(kpm_enable && IS_KPM_ADDR(vaddr)); 12822 12823 SFMMU_KPM_VTOP(vaddr, paddr); 12824 pfn = (pfn_t)btop(paddr); 12825 pp = page_numtopp_nolock(pfn); 12826 if (pp && pp->p_kpmref) 12827 return (pfn); 12828 else 12829 return ((pfn_t)PFN_INVALID); 12830 } 12831 12832 /* 12833 * Lookup a kpme in the p_kpmelist. 12834 */ 12835 static int 12836 sfmmu_kpme_lookup(struct kpme *kpme, page_t *pp) 12837 { 12838 struct kpme *p; 12839 12840 for (p = pp->p_kpmelist; p; p = p->kpe_next) { 12841 if (p == kpme) 12842 return (1); 12843 } 12844 return (0); 12845 } 12846 12847 /* 12848 * Insert a kpme into the p_kpmelist and increment 12849 * the per page kpm reference count. 12850 */ 12851 static void 12852 sfmmu_kpme_add(struct kpme *kpme, page_t *pp) 12853 { 12854 ASSERT(pp->p_kpmref >= 0); 12855 12856 /* head insert */ 12857 kpme->kpe_prev = NULL; 12858 kpme->kpe_next = pp->p_kpmelist; 12859 12860 if (pp->p_kpmelist) 12861 pp->p_kpmelist->kpe_prev = kpme; 12862 12863 pp->p_kpmelist = kpme; 12864 kpme->kpe_page = pp; 12865 pp->p_kpmref++; 12866 } 12867 12868 /* 12869 * Remove a kpme from the p_kpmelist and decrement 12870 * the per page kpm reference count. 12871 */ 12872 static void 12873 sfmmu_kpme_sub(struct kpme *kpme, page_t *pp) 12874 { 12875 ASSERT(pp->p_kpmref > 0); 12876 12877 if (kpme->kpe_prev) { 12878 ASSERT(pp->p_kpmelist != kpme); 12879 ASSERT(kpme->kpe_prev->kpe_page == pp); 12880 kpme->kpe_prev->kpe_next = kpme->kpe_next; 12881 } else { 12882 ASSERT(pp->p_kpmelist == kpme); 12883 pp->p_kpmelist = kpme->kpe_next; 12884 } 12885 12886 if (kpme->kpe_next) { 12887 ASSERT(kpme->kpe_next->kpe_page == pp); 12888 kpme->kpe_next->kpe_prev = kpme->kpe_prev; 12889 } 12890 12891 kpme->kpe_next = kpme->kpe_prev = NULL; 12892 kpme->kpe_page = NULL; 12893 pp->p_kpmref--; 12894 } 12895 12896 /* 12897 * Mapin a single page, it is called every time a page changes it's state 12898 * from kpm-unmapped to kpm-mapped. It may not be called, when only a new 12899 * kpm instance does a mapin and wants to share the mapping. 12900 * Assumes that the mlist mutex is already grabbed. 12901 */ 12902 static caddr_t 12903 sfmmu_kpm_mapin(page_t *pp) 12904 { 12905 kpm_page_t *kp; 12906 kpm_hlk_t *kpmp; 12907 caddr_t vaddr; 12908 int kpm_vac_range; 12909 pfn_t pfn; 12910 tte_t tte; 12911 kmutex_t *pmtx; 12912 int uncached; 12913 kpm_spage_t *ksp; 12914 kpm_shlk_t *kpmsp; 12915 int oldval; 12916 12917 ASSERT(sfmmu_mlist_held(pp)); 12918 ASSERT(pp->p_kpmref == 0); 12919 12920 vaddr = sfmmu_kpm_getvaddr(pp, &kpm_vac_range); 12921 12922 ASSERT(IS_KPM_ADDR(vaddr)); 12923 uncached = PP_ISNC(pp); 12924 pfn = pp->p_pagenum; 12925 12926 if (kpm_smallpages) 12927 goto smallpages_mapin; 12928 12929 PP2KPMPG(pp, kp); 12930 12931 kpmp = KPMP_HASH(kp); 12932 mutex_enter(&kpmp->khl_mutex); 12933 12934 ASSERT(PP_ISKPMC(pp) == 0); 12935 ASSERT(PP_ISKPMS(pp) == 0); 12936 12937 if (uncached) { 12938 /* ASSERT(pp->p_share); XXX use hat_page_getshare */ 12939 if (kpm_vac_range == 0) { 12940 if (kp->kp_refcnts == 0) { 12941 /* 12942 * Must remove large page mapping if it exists. 12943 * Pages in uncached state can only be mapped 12944 * small (PAGESIZE) within the regular kpm 12945 * range. 12946 */ 12947 if (kp->kp_refcntc == -1) { 12948 /* remove go indication */ 12949 sfmmu_kpm_tsbmtl(&kp->kp_refcntc, 12950 &kpmp->khl_lock, KPMTSBM_STOP); 12951 } 12952 if (kp->kp_refcnt > 0 && kp->kp_refcntc == 0) 12953 sfmmu_kpm_demap_large(vaddr); 12954 } 12955 ASSERT(kp->kp_refcntc >= 0); 12956 kp->kp_refcntc++; 12957 } 12958 pmtx = sfmmu_page_enter(pp); 12959 PP_SETKPMC(pp); 12960 sfmmu_page_exit(pmtx); 12961 } 12962 12963 if ((kp->kp_refcntc > 0 || kp->kp_refcnts > 0) && kpm_vac_range == 0) { 12964 /* 12965 * Have to do a small (PAGESIZE) mapin within this kpm_page 12966 * range since it is marked to be in VAC conflict mode or 12967 * when there are still other small mappings around. 12968 */ 12969 12970 /* tte assembly */ 12971 if (uncached == 0) 12972 KPM_TTE_VCACHED(tte.ll, pfn, TTE8K); 12973 else 12974 KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K); 12975 12976 /* tsb dropin */ 12977 sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT); 12978 12979 pmtx = sfmmu_page_enter(pp); 12980 PP_SETKPMS(pp); 12981 sfmmu_page_exit(pmtx); 12982 12983 kp->kp_refcnts++; 12984 ASSERT(kp->kp_refcnts > 0); 12985 goto exit; 12986 } 12987 12988 if (kpm_vac_range == 0) { 12989 /* 12990 * Fast path / regular case, no VAC conflict handling 12991 * in progress within this kpm_page range. 12992 */ 12993 if (kp->kp_refcnt == 0) { 12994 12995 /* tte assembly */ 12996 KPM_TTE_VCACHED(tte.ll, pfn, TTE4M); 12997 12998 /* tsb dropin */ 12999 sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT4M); 13000 13001 /* Set go flag for TL tsbmiss handler */ 13002 if (kp->kp_refcntc == 0) 13003 sfmmu_kpm_tsbmtl(&kp->kp_refcntc, 13004 &kpmp->khl_lock, KPMTSBM_START); 13005 13006 ASSERT(kp->kp_refcntc == -1); 13007 } 13008 kp->kp_refcnt++; 13009 ASSERT(kp->kp_refcnt); 13010 13011 } else { 13012 /* 13013 * The page is not setup according to the common VAC 13014 * prevention rules for the regular and kpm mapping layer 13015 * E.g. the page layer was not able to deliver a right 13016 * vcolor'ed page for a given vaddr corresponding to 13017 * the wanted p_offset. It has to be mapped in small in 13018 * within the corresponding kpm vac range in order to 13019 * prevent VAC alias conflicts. 13020 */ 13021 13022 /* tte assembly */ 13023 if (uncached == 0) { 13024 KPM_TTE_VCACHED(tte.ll, pfn, TTE8K); 13025 } else { 13026 KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K); 13027 } 13028 13029 /* tsb dropin */ 13030 sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT); 13031 13032 kp->kp_refcnta++; 13033 if (kp->kp_refcntc == -1) { 13034 ASSERT(kp->kp_refcnt > 0); 13035 13036 /* remove go indication */ 13037 sfmmu_kpm_tsbmtl(&kp->kp_refcntc, &kpmp->khl_lock, 13038 KPMTSBM_STOP); 13039 } 13040 ASSERT(kp->kp_refcntc >= 0); 13041 } 13042 exit: 13043 mutex_exit(&kpmp->khl_mutex); 13044 return (vaddr); 13045 13046 smallpages_mapin: 13047 if (uncached == 0) { 13048 /* tte assembly */ 13049 KPM_TTE_VCACHED(tte.ll, pfn, TTE8K); 13050 } else { 13051 /* ASSERT(pp->p_share); XXX use hat_page_getshare */ 13052 pmtx = sfmmu_page_enter(pp); 13053 PP_SETKPMC(pp); 13054 sfmmu_page_exit(pmtx); 13055 /* tte assembly */ 13056 KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K); 13057 } 13058 13059 /* tsb dropin */ 13060 sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT); 13061 13062 PP2KPMSPG(pp, ksp); 13063 kpmsp = KPMP_SHASH(ksp); 13064 13065 oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped, &kpmsp->kshl_lock, 13066 (uncached) ? KPM_MAPPEDSC : KPM_MAPPEDS); 13067 13068 if (oldval != 0) 13069 panic("sfmmu_kpm_mapin: stale smallpages mapping"); 13070 13071 return (vaddr); 13072 } 13073 13074 /* 13075 * Mapout a single page, it is called every time a page changes it's state 13076 * from kpm-mapped to kpm-unmapped. It may not be called, when only a kpm 13077 * instance calls mapout and there are still other instances mapping the 13078 * page. Assumes that the mlist mutex is already grabbed. 13079 * 13080 * Note: In normal mode (no VAC conflict prevention pending) TLB's are 13081 * not flushed. This is the core segkpm behavior to avoid xcalls. It is 13082 * no problem because a translation from a segkpm virtual address to a 13083 * physical address is always the same. The only downside is a slighty 13084 * increased window of vulnerability for misbehaving _kernel_ modules. 13085 */ 13086 static void 13087 sfmmu_kpm_mapout(page_t *pp, caddr_t vaddr) 13088 { 13089 kpm_page_t *kp; 13090 kpm_hlk_t *kpmp; 13091 int alias_range; 13092 kmutex_t *pmtx; 13093 kpm_spage_t *ksp; 13094 kpm_shlk_t *kpmsp; 13095 int oldval; 13096 13097 ASSERT(sfmmu_mlist_held(pp)); 13098 ASSERT(pp->p_kpmref == 0); 13099 13100 alias_range = IS_KPM_ALIAS_RANGE(vaddr); 13101 13102 if (kpm_smallpages) 13103 goto smallpages_mapout; 13104 13105 PP2KPMPG(pp, kp); 13106 kpmp = KPMP_HASH(kp); 13107 mutex_enter(&kpmp->khl_mutex); 13108 13109 if (alias_range) { 13110 ASSERT(PP_ISKPMS(pp) == 0); 13111 if (kp->kp_refcnta <= 0) { 13112 panic("sfmmu_kpm_mapout: bad refcnta kp=%p", 13113 (void *)kp); 13114 } 13115 13116 if (PP_ISTNC(pp)) { 13117 if (PP_ISKPMC(pp) == 0) { 13118 /* 13119 * Uncached kpm mappings must always have 13120 * forced "small page" mode. 13121 */ 13122 panic("sfmmu_kpm_mapout: uncached page not " 13123 "kpm marked"); 13124 } 13125 sfmmu_kpm_demap_small(vaddr); 13126 13127 pmtx = sfmmu_page_enter(pp); 13128 PP_CLRKPMC(pp); 13129 sfmmu_page_exit(pmtx); 13130 13131 /* 13132 * Check if we can resume cached mode. This might 13133 * be the case if the kpm mapping was the only 13134 * mapping in conflict with other non rule 13135 * compliant mappings. The page is no more marked 13136 * as kpm mapped, so the conv_tnc path will not 13137 * change kpm state. 13138 */ 13139 conv_tnc(pp, TTE8K); 13140 13141 } else if (PP_ISKPMC(pp) == 0) { 13142 /* remove TSB entry only */ 13143 sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT); 13144 13145 } else { 13146 /* already demapped */ 13147 pmtx = sfmmu_page_enter(pp); 13148 PP_CLRKPMC(pp); 13149 sfmmu_page_exit(pmtx); 13150 } 13151 kp->kp_refcnta--; 13152 goto exit; 13153 } 13154 13155 if (kp->kp_refcntc <= 0 && kp->kp_refcnts == 0) { 13156 /* 13157 * Fast path / regular case. 13158 */ 13159 ASSERT(kp->kp_refcntc >= -1); 13160 ASSERT(!(pp->p_nrm & (P_KPMC | P_KPMS | P_TNC | P_PNC))); 13161 13162 if (kp->kp_refcnt <= 0) 13163 panic("sfmmu_kpm_mapout: bad refcnt kp=%p", (void *)kp); 13164 13165 if (--kp->kp_refcnt == 0) { 13166 /* remove go indication */ 13167 if (kp->kp_refcntc == -1) { 13168 sfmmu_kpm_tsbmtl(&kp->kp_refcntc, 13169 &kpmp->khl_lock, KPMTSBM_STOP); 13170 } 13171 ASSERT(kp->kp_refcntc == 0); 13172 13173 /* remove TSB entry */ 13174 sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT4M); 13175 #ifdef DEBUG 13176 if (kpm_tlb_flush) 13177 sfmmu_kpm_demap_tlbs(vaddr, KCONTEXT); 13178 #endif 13179 } 13180 13181 } else { 13182 /* 13183 * The VAC alias path. 13184 * We come here if the kpm vaddr is not in any alias_range 13185 * and we are unmapping a page within the regular kpm_page 13186 * range. The kpm_page either holds conflict pages and/or 13187 * is in "small page" mode. If the page is not marked 13188 * P_KPMS it couldn't have a valid PAGESIZE sized TSB 13189 * entry. Dcache flushing is done lazy and follows the 13190 * rules of the regular virtual page coloring scheme. 13191 * 13192 * Per page states and required actions: 13193 * P_KPMC: remove a kpm mapping that is conflicting. 13194 * P_KPMS: remove a small kpm mapping within a kpm_page. 13195 * P_TNC: check if we can re-cache the page. 13196 * P_PNC: we cannot re-cache, sorry. 13197 * Per kpm_page: 13198 * kp_refcntc > 0: page is part of a kpm_page with conflicts. 13199 * kp_refcnts > 0: rm a small mapped page within a kpm_page. 13200 */ 13201 13202 if (PP_ISKPMS(pp)) { 13203 if (kp->kp_refcnts < 1) { 13204 panic("sfmmu_kpm_mapout: bad refcnts kp=%p", 13205 (void *)kp); 13206 } 13207 sfmmu_kpm_demap_small(vaddr); 13208 13209 /* 13210 * Check if we can resume cached mode. This might 13211 * be the case if the kpm mapping was the only 13212 * mapping in conflict with other non rule 13213 * compliant mappings. The page is no more marked 13214 * as kpm mapped, so the conv_tnc path will not 13215 * change kpm state. 13216 */ 13217 if (PP_ISTNC(pp)) { 13218 if (!PP_ISKPMC(pp)) { 13219 /* 13220 * Uncached kpm mappings must always 13221 * have forced "small page" mode. 13222 */ 13223 panic("sfmmu_kpm_mapout: uncached " 13224 "page not kpm marked"); 13225 } 13226 conv_tnc(pp, TTE8K); 13227 } 13228 kp->kp_refcnts--; 13229 kp->kp_refcnt++; 13230 pmtx = sfmmu_page_enter(pp); 13231 PP_CLRKPMS(pp); 13232 sfmmu_page_exit(pmtx); 13233 } 13234 13235 if (PP_ISKPMC(pp)) { 13236 if (kp->kp_refcntc < 1) { 13237 panic("sfmmu_kpm_mapout: bad refcntc kp=%p", 13238 (void *)kp); 13239 } 13240 pmtx = sfmmu_page_enter(pp); 13241 PP_CLRKPMC(pp); 13242 sfmmu_page_exit(pmtx); 13243 kp->kp_refcntc--; 13244 } 13245 13246 if (kp->kp_refcnt-- < 1) 13247 panic("sfmmu_kpm_mapout: bad refcnt kp=%p", (void *)kp); 13248 } 13249 exit: 13250 mutex_exit(&kpmp->khl_mutex); 13251 return; 13252 13253 smallpages_mapout: 13254 PP2KPMSPG(pp, ksp); 13255 kpmsp = KPMP_SHASH(ksp); 13256 13257 if (PP_ISKPMC(pp) == 0) { 13258 oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped, 13259 &kpmsp->kshl_lock, 0); 13260 13261 if (oldval != KPM_MAPPEDS) { 13262 /* 13263 * When we're called after sfmmu_kpm_hme_unload, 13264 * KPM_MAPPEDSC is valid too. 13265 */ 13266 if (oldval != KPM_MAPPEDSC) 13267 panic("sfmmu_kpm_mapout: incorrect mapping"); 13268 } 13269 13270 /* remove TSB entry */ 13271 sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT); 13272 #ifdef DEBUG 13273 if (kpm_tlb_flush) 13274 sfmmu_kpm_demap_tlbs(vaddr, KCONTEXT); 13275 #endif 13276 13277 } else if (PP_ISTNC(pp)) { 13278 oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped, 13279 &kpmsp->kshl_lock, 0); 13280 13281 if (oldval != KPM_MAPPEDSC || PP_ISKPMC(pp) == 0) 13282 panic("sfmmu_kpm_mapout: inconsistent TNC mapping"); 13283 13284 sfmmu_kpm_demap_small(vaddr); 13285 13286 pmtx = sfmmu_page_enter(pp); 13287 PP_CLRKPMC(pp); 13288 sfmmu_page_exit(pmtx); 13289 13290 /* 13291 * Check if we can resume cached mode. This might be 13292 * the case if the kpm mapping was the only mapping 13293 * in conflict with other non rule compliant mappings. 13294 * The page is no more marked as kpm mapped, so the 13295 * conv_tnc path will not change the kpm state. 13296 */ 13297 conv_tnc(pp, TTE8K); 13298 13299 } else { 13300 oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped, 13301 &kpmsp->kshl_lock, 0); 13302 13303 if (oldval != KPM_MAPPEDSC) 13304 panic("sfmmu_kpm_mapout: inconsistent mapping"); 13305 13306 pmtx = sfmmu_page_enter(pp); 13307 PP_CLRKPMC(pp); 13308 sfmmu_page_exit(pmtx); 13309 } 13310 } 13311 13312 #define abs(x) ((x) < 0 ? -(x) : (x)) 13313 13314 /* 13315 * Determine appropriate kpm mapping address and handle any kpm/hme 13316 * conflicts. Page mapping list and its vcolor parts must be protected. 13317 */ 13318 static caddr_t 13319 sfmmu_kpm_getvaddr(page_t *pp, int *kpm_vac_rangep) 13320 { 13321 int vcolor, vcolor_pa; 13322 caddr_t vaddr; 13323 uintptr_t paddr; 13324 13325 13326 ASSERT(sfmmu_mlist_held(pp)); 13327 13328 paddr = ptob(pp->p_pagenum); 13329 vcolor_pa = addr_to_vcolor(paddr); 13330 13331 if (IS_SWAPFSVP(pp->p_vnode)) { 13332 vcolor = (PP_NEWPAGE(pp) || PP_ISNC(pp)) ? 13333 vcolor_pa : PP_GET_VCOLOR(pp); 13334 } else { 13335 vcolor = addr_to_vcolor(pp->p_offset); 13336 } 13337 13338 vaddr = kpm_vbase + paddr; 13339 *kpm_vac_rangep = 0; 13340 13341 if (vcolor_pa != vcolor) { 13342 *kpm_vac_rangep = abs(vcolor - vcolor_pa); 13343 vaddr += ((uintptr_t)(vcolor - vcolor_pa) << MMU_PAGESHIFT); 13344 vaddr += (vcolor_pa > vcolor) ? 13345 ((uintptr_t)vcolor_pa << kpm_size_shift) : 13346 ((uintptr_t)(vcolor - vcolor_pa) << kpm_size_shift); 13347 13348 ASSERT(!PP_ISMAPPED_LARGE(pp)); 13349 } 13350 13351 if (PP_ISNC(pp)) 13352 return (vaddr); 13353 13354 if (PP_NEWPAGE(pp)) { 13355 PP_SET_VCOLOR(pp, vcolor); 13356 return (vaddr); 13357 } 13358 13359 if (PP_GET_VCOLOR(pp) == vcolor) 13360 return (vaddr); 13361 13362 ASSERT(!PP_ISMAPPED_KPM(pp)); 13363 sfmmu_kpm_vac_conflict(pp, vaddr); 13364 13365 return (vaddr); 13366 } 13367 13368 /* 13369 * VAC conflict state bit values. 13370 * The following defines are used to make the handling of the 13371 * various input states more concise. For that the kpm states 13372 * per kpm_page and per page are combined in a summary state. 13373 * Each single state has a corresponding bit value in the 13374 * summary state. These defines only apply for kpm large page 13375 * mappings. Within comments the abbreviations "kc, c, ks, s" 13376 * are used as short form of the actual state, e.g. "kc" for 13377 * "kp_refcntc > 0", etc. 13378 */ 13379 #define KPM_KC 0x00000008 /* kpm_page: kp_refcntc > 0 */ 13380 #define KPM_C 0x00000004 /* page: P_KPMC set */ 13381 #define KPM_KS 0x00000002 /* kpm_page: kp_refcnts > 0 */ 13382 #define KPM_S 0x00000001 /* page: P_KPMS set */ 13383 13384 /* 13385 * Summary states used in sfmmu_kpm_fault (KPM_TSBM_*). 13386 * See also more detailed comments within in the sfmmu_kpm_fault switch. 13387 * Abbreviations used: 13388 * CONFL: VAC conflict(s) within a kpm_page. 13389 * MAPS: Mapped small: Page mapped in using a regular page size kpm mapping. 13390 * RASM: Re-assembling of a large page mapping possible. 13391 * RPLS: Replace: TSB miss due to TSB replacement only. 13392 * BRKO: Breakup Other: A large kpm mapping has to be broken because another 13393 * page within the kpm_page is already involved in a VAC conflict. 13394 * BRKT: Breakup This: A large kpm mapping has to be broken, this page is 13395 * is involved in a VAC conflict. 13396 */ 13397 #define KPM_TSBM_CONFL_GONE (0) 13398 #define KPM_TSBM_MAPS_RASM (KPM_KS) 13399 #define KPM_TSBM_RPLS_RASM (KPM_KS | KPM_S) 13400 #define KPM_TSBM_MAPS_BRKO (KPM_KC) 13401 #define KPM_TSBM_MAPS (KPM_KC | KPM_KS) 13402 #define KPM_TSBM_RPLS (KPM_KC | KPM_KS | KPM_S) 13403 #define KPM_TSBM_MAPS_BRKT (KPM_KC | KPM_C) 13404 #define KPM_TSBM_MAPS_CONFL (KPM_KC | KPM_C | KPM_KS) 13405 #define KPM_TSBM_RPLS_CONFL (KPM_KC | KPM_C | KPM_KS | KPM_S) 13406 13407 /* 13408 * kpm fault handler for mappings with large page size. 13409 */ 13410 int 13411 sfmmu_kpm_fault(caddr_t vaddr, struct memseg *mseg, page_t *pp) 13412 { 13413 int error; 13414 pgcnt_t inx; 13415 kpm_page_t *kp; 13416 tte_t tte; 13417 pfn_t pfn = pp->p_pagenum; 13418 kpm_hlk_t *kpmp; 13419 kmutex_t *pml; 13420 int alias_range; 13421 int uncached = 0; 13422 kmutex_t *pmtx; 13423 int badstate; 13424 uint_t tsbmcase; 13425 13426 alias_range = IS_KPM_ALIAS_RANGE(vaddr); 13427 13428 inx = ptokpmp(kpmptop(ptokpmp(pfn)) - mseg->kpm_pbase); 13429 if (inx >= mseg->kpm_nkpmpgs) { 13430 cmn_err(CE_PANIC, "sfmmu_kpm_fault: kpm overflow in memseg " 13431 "0x%p pp 0x%p", (void *)mseg, (void *)pp); 13432 } 13433 13434 kp = &mseg->kpm_pages[inx]; 13435 kpmp = KPMP_HASH(kp); 13436 13437 pml = sfmmu_mlist_enter(pp); 13438 13439 if (!PP_ISMAPPED_KPM(pp)) { 13440 sfmmu_mlist_exit(pml); 13441 return (EFAULT); 13442 } 13443 13444 mutex_enter(&kpmp->khl_mutex); 13445 13446 if (alias_range) { 13447 ASSERT(!PP_ISMAPPED_LARGE(pp)); 13448 if (kp->kp_refcnta > 0) { 13449 if (PP_ISKPMC(pp)) { 13450 pmtx = sfmmu_page_enter(pp); 13451 PP_CLRKPMC(pp); 13452 sfmmu_page_exit(pmtx); 13453 } 13454 /* 13455 * Check for vcolor conflicts. Return here 13456 * w/ either no conflict (fast path), removed hme 13457 * mapping chains (unload conflict) or uncached 13458 * (uncache conflict). VACaches are cleaned and 13459 * p_vcolor and PP_TNC are set accordingly for the 13460 * conflict cases. Drop kpmp for uncache conflict 13461 * cases since it will be grabbed within 13462 * sfmmu_kpm_page_cache in case of an uncache 13463 * conflict. 13464 */ 13465 mutex_exit(&kpmp->khl_mutex); 13466 sfmmu_kpm_vac_conflict(pp, vaddr); 13467 mutex_enter(&kpmp->khl_mutex); 13468 13469 if (PP_ISNC(pp)) { 13470 uncached = 1; 13471 pmtx = sfmmu_page_enter(pp); 13472 PP_SETKPMC(pp); 13473 sfmmu_page_exit(pmtx); 13474 } 13475 goto smallexit; 13476 13477 } else { 13478 /* 13479 * We got a tsbmiss on a not active kpm_page range. 13480 * Let segkpm_fault decide how to panic. 13481 */ 13482 error = EFAULT; 13483 } 13484 goto exit; 13485 } 13486 13487 badstate = (kp->kp_refcnt < 0 || kp->kp_refcnts < 0); 13488 if (kp->kp_refcntc == -1) { 13489 /* 13490 * We should come here only if trap level tsb miss 13491 * handler is disabled. 13492 */ 13493 badstate |= (kp->kp_refcnt == 0 || kp->kp_refcnts > 0 || 13494 PP_ISKPMC(pp) || PP_ISKPMS(pp) || PP_ISNC(pp)); 13495 13496 if (badstate == 0) 13497 goto largeexit; 13498 } 13499 13500 if (badstate || kp->kp_refcntc < 0) 13501 goto badstate_exit; 13502 13503 /* 13504 * Combine the per kpm_page and per page kpm VAC states to 13505 * a summary state in order to make the kpm fault handling 13506 * more concise. 13507 */ 13508 tsbmcase = (((kp->kp_refcntc > 0) ? KPM_KC : 0) | 13509 ((kp->kp_refcnts > 0) ? KPM_KS : 0) | 13510 (PP_ISKPMC(pp) ? KPM_C : 0) | 13511 (PP_ISKPMS(pp) ? KPM_S : 0)); 13512 13513 switch (tsbmcase) { 13514 case KPM_TSBM_CONFL_GONE: /* - - - - */ 13515 /* 13516 * That's fine, we either have no more vac conflict in 13517 * this kpm page or someone raced in and has solved the 13518 * vac conflict for us -- call sfmmu_kpm_vac_conflict 13519 * to take care for correcting the vcolor and flushing 13520 * the dcache if required. 13521 */ 13522 mutex_exit(&kpmp->khl_mutex); 13523 sfmmu_kpm_vac_conflict(pp, vaddr); 13524 mutex_enter(&kpmp->khl_mutex); 13525 13526 if (PP_ISNC(pp) || kp->kp_refcnt <= 0 || 13527 addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) { 13528 panic("sfmmu_kpm_fault: inconsistent CONFL_GONE " 13529 "state, pp=%p", (void *)pp); 13530 } 13531 goto largeexit; 13532 13533 case KPM_TSBM_MAPS_RASM: /* - - ks - */ 13534 /* 13535 * All conflicts in this kpm page are gone but there are 13536 * already small mappings around, so we also map this 13537 * page small. This could be the trigger case for a 13538 * small mapping reaper, if this is really needed. 13539 * For now fall thru to the KPM_TSBM_MAPS handling. 13540 */ 13541 13542 case KPM_TSBM_MAPS: /* kc - ks - */ 13543 /* 13544 * Large page mapping is already broken, this page is not 13545 * conflicting, so map it small. Call sfmmu_kpm_vac_conflict 13546 * to take care for correcting the vcolor and flushing 13547 * the dcache if required. 13548 */ 13549 mutex_exit(&kpmp->khl_mutex); 13550 sfmmu_kpm_vac_conflict(pp, vaddr); 13551 mutex_enter(&kpmp->khl_mutex); 13552 13553 if (PP_ISNC(pp) || kp->kp_refcnt <= 0 || 13554 addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) { 13555 panic("sfmmu_kpm_fault: inconsistent MAPS state, " 13556 "pp=%p", (void *)pp); 13557 } 13558 kp->kp_refcnt--; 13559 kp->kp_refcnts++; 13560 pmtx = sfmmu_page_enter(pp); 13561 PP_SETKPMS(pp); 13562 sfmmu_page_exit(pmtx); 13563 goto smallexit; 13564 13565 case KPM_TSBM_RPLS_RASM: /* - - ks s */ 13566 /* 13567 * All conflicts in this kpm page are gone but this page 13568 * is mapped small. This could be the trigger case for a 13569 * small mapping reaper, if this is really needed. 13570 * For now we drop it in small again. Fall thru to the 13571 * KPM_TSBM_RPLS handling. 13572 */ 13573 13574 case KPM_TSBM_RPLS: /* kc - ks s */ 13575 /* 13576 * Large page mapping is already broken, this page is not 13577 * conflicting but already mapped small, so drop it in 13578 * small again. 13579 */ 13580 if (PP_ISNC(pp) || 13581 addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) { 13582 panic("sfmmu_kpm_fault: inconsistent RPLS state, " 13583 "pp=%p", (void *)pp); 13584 } 13585 goto smallexit; 13586 13587 case KPM_TSBM_MAPS_BRKO: /* kc - - - */ 13588 /* 13589 * The kpm page where we live in is marked conflicting 13590 * but this page is not conflicting. So we have to map it 13591 * in small. Call sfmmu_kpm_vac_conflict to take care for 13592 * correcting the vcolor and flushing the dcache if required. 13593 */ 13594 mutex_exit(&kpmp->khl_mutex); 13595 sfmmu_kpm_vac_conflict(pp, vaddr); 13596 mutex_enter(&kpmp->khl_mutex); 13597 13598 if (PP_ISNC(pp) || kp->kp_refcnt <= 0 || 13599 addr_to_vcolor(vaddr) != PP_GET_VCOLOR(pp)) { 13600 panic("sfmmu_kpm_fault: inconsistent MAPS_BRKO state, " 13601 "pp=%p", (void *)pp); 13602 } 13603 kp->kp_refcnt--; 13604 kp->kp_refcnts++; 13605 pmtx = sfmmu_page_enter(pp); 13606 PP_SETKPMS(pp); 13607 sfmmu_page_exit(pmtx); 13608 goto smallexit; 13609 13610 case KPM_TSBM_MAPS_BRKT: /* kc c - - */ 13611 case KPM_TSBM_MAPS_CONFL: /* kc c ks - */ 13612 if (!PP_ISMAPPED(pp)) { 13613 /* 13614 * We got a tsbmiss on kpm large page range that is 13615 * marked to contain vac conflicting pages introduced 13616 * by hme mappings. The hme mappings are all gone and 13617 * must have bypassed the kpm alias prevention logic. 13618 */ 13619 panic("sfmmu_kpm_fault: stale VAC conflict, pp=%p", 13620 (void *)pp); 13621 } 13622 13623 /* 13624 * Check for vcolor conflicts. Return here w/ either no 13625 * conflict (fast path), removed hme mapping chains 13626 * (unload conflict) or uncached (uncache conflict). 13627 * Dcache is cleaned and p_vcolor and P_TNC are set 13628 * accordingly. Drop kpmp for uncache conflict cases 13629 * since it will be grabbed within sfmmu_kpm_page_cache 13630 * in case of an uncache conflict. 13631 */ 13632 mutex_exit(&kpmp->khl_mutex); 13633 sfmmu_kpm_vac_conflict(pp, vaddr); 13634 mutex_enter(&kpmp->khl_mutex); 13635 13636 if (kp->kp_refcnt <= 0) 13637 panic("sfmmu_kpm_fault: bad refcnt kp=%p", (void *)kp); 13638 13639 if (PP_ISNC(pp)) { 13640 uncached = 1; 13641 } else { 13642 /* 13643 * When an unload conflict is solved and there are 13644 * no other small mappings around, we can resume 13645 * largepage mode. Otherwise we have to map or drop 13646 * in small. This could be a trigger for a small 13647 * mapping reaper when this was the last conflict 13648 * within the kpm page and when there are only 13649 * other small mappings around. 13650 */ 13651 ASSERT(addr_to_vcolor(vaddr) == PP_GET_VCOLOR(pp)); 13652 ASSERT(kp->kp_refcntc > 0); 13653 kp->kp_refcntc--; 13654 pmtx = sfmmu_page_enter(pp); 13655 PP_CLRKPMC(pp); 13656 sfmmu_page_exit(pmtx); 13657 ASSERT(PP_ISKPMS(pp) == 0); 13658 if (kp->kp_refcntc == 0 && kp->kp_refcnts == 0) 13659 goto largeexit; 13660 } 13661 13662 kp->kp_refcnt--; 13663 kp->kp_refcnts++; 13664 pmtx = sfmmu_page_enter(pp); 13665 PP_SETKPMS(pp); 13666 sfmmu_page_exit(pmtx); 13667 goto smallexit; 13668 13669 case KPM_TSBM_RPLS_CONFL: /* kc c ks s */ 13670 if (!PP_ISMAPPED(pp)) { 13671 /* 13672 * We got a tsbmiss on kpm large page range that is 13673 * marked to contain vac conflicting pages introduced 13674 * by hme mappings. They are all gone and must have 13675 * somehow bypassed the kpm alias prevention logic. 13676 */ 13677 panic("sfmmu_kpm_fault: stale VAC conflict, pp=%p", 13678 (void *)pp); 13679 } 13680 13681 /* 13682 * This state is only possible for an uncached mapping. 13683 */ 13684 if (!PP_ISNC(pp)) { 13685 panic("sfmmu_kpm_fault: page not uncached, pp=%p", 13686 (void *)pp); 13687 } 13688 uncached = 1; 13689 goto smallexit; 13690 13691 default: 13692 badstate_exit: 13693 panic("sfmmu_kpm_fault: inconsistent VAC state, vaddr=%p kp=%p " 13694 "pp=%p", (void *)vaddr, (void *)kp, (void *)pp); 13695 } 13696 13697 smallexit: 13698 /* tte assembly */ 13699 if (uncached == 0) 13700 KPM_TTE_VCACHED(tte.ll, pfn, TTE8K); 13701 else 13702 KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K); 13703 13704 /* tsb dropin */ 13705 sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT); 13706 13707 error = 0; 13708 goto exit; 13709 13710 largeexit: 13711 if (kp->kp_refcnt > 0) { 13712 13713 /* tte assembly */ 13714 KPM_TTE_VCACHED(tte.ll, pfn, TTE4M); 13715 13716 /* tsb dropin */ 13717 sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT4M); 13718 13719 if (kp->kp_refcntc == 0) { 13720 /* Set "go" flag for TL tsbmiss handler */ 13721 sfmmu_kpm_tsbmtl(&kp->kp_refcntc, &kpmp->khl_lock, 13722 KPMTSBM_START); 13723 } 13724 ASSERT(kp->kp_refcntc == -1); 13725 error = 0; 13726 13727 } else 13728 error = EFAULT; 13729 exit: 13730 mutex_exit(&kpmp->khl_mutex); 13731 sfmmu_mlist_exit(pml); 13732 return (error); 13733 } 13734 13735 /* 13736 * kpm fault handler for mappings with small page size. 13737 */ 13738 int 13739 sfmmu_kpm_fault_small(caddr_t vaddr, struct memseg *mseg, page_t *pp) 13740 { 13741 int error = 0; 13742 pgcnt_t inx; 13743 kpm_spage_t *ksp; 13744 kpm_shlk_t *kpmsp; 13745 kmutex_t *pml; 13746 pfn_t pfn = pp->p_pagenum; 13747 tte_t tte; 13748 kmutex_t *pmtx; 13749 int oldval; 13750 13751 inx = pfn - mseg->kpm_pbase; 13752 ksp = &mseg->kpm_spages[inx]; 13753 kpmsp = KPMP_SHASH(ksp); 13754 13755 pml = sfmmu_mlist_enter(pp); 13756 13757 if (!PP_ISMAPPED_KPM(pp)) { 13758 sfmmu_mlist_exit(pml); 13759 return (EFAULT); 13760 } 13761 13762 /* 13763 * kp_mapped lookup protected by mlist mutex 13764 */ 13765 if (ksp->kp_mapped == KPM_MAPPEDS) { 13766 /* 13767 * Fast path tsbmiss 13768 */ 13769 ASSERT(!PP_ISKPMC(pp)); 13770 ASSERT(!PP_ISNC(pp)); 13771 13772 /* tte assembly */ 13773 KPM_TTE_VCACHED(tte.ll, pfn, TTE8K); 13774 13775 /* tsb dropin */ 13776 sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT); 13777 13778 } else if (ksp->kp_mapped == KPM_MAPPEDSC) { 13779 /* 13780 * Got here due to existing or gone kpm/hme VAC conflict. 13781 * Recheck for vcolor conflicts. Return here w/ either 13782 * no conflict, removed hme mapping chain (unload 13783 * conflict) or uncached (uncache conflict). VACaches 13784 * are cleaned and p_vcolor and PP_TNC are set accordingly 13785 * for the conflict cases. 13786 */ 13787 sfmmu_kpm_vac_conflict(pp, vaddr); 13788 13789 if (PP_ISNC(pp)) { 13790 /* ASSERT(pp->p_share); XXX use hat_page_getshare */ 13791 13792 /* tte assembly */ 13793 KPM_TTE_VUNCACHED(tte.ll, pfn, TTE8K); 13794 13795 /* tsb dropin */ 13796 sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT); 13797 13798 } else { 13799 if (PP_ISKPMC(pp)) { 13800 pmtx = sfmmu_page_enter(pp); 13801 PP_CLRKPMC(pp); 13802 sfmmu_page_exit(pmtx); 13803 } 13804 13805 /* tte assembly */ 13806 KPM_TTE_VCACHED(tte.ll, pfn, TTE8K); 13807 13808 /* tsb dropin */ 13809 sfmmu_kpm_load_tsb(vaddr, &tte, MMU_PAGESHIFT); 13810 13811 oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped, 13812 &kpmsp->kshl_lock, KPM_MAPPEDS); 13813 13814 if (oldval != KPM_MAPPEDSC) 13815 panic("sfmmu_kpm_fault_small: " 13816 "stale smallpages mapping"); 13817 } 13818 13819 } else { 13820 /* 13821 * We got a tsbmiss on a not active kpm_page range. 13822 * Let decide segkpm_fault how to panic. 13823 */ 13824 error = EFAULT; 13825 } 13826 13827 sfmmu_mlist_exit(pml); 13828 return (error); 13829 } 13830 13831 /* 13832 * Check/handle potential hme/kpm mapping conflicts 13833 */ 13834 static void 13835 sfmmu_kpm_vac_conflict(page_t *pp, caddr_t vaddr) 13836 { 13837 int vcolor; 13838 struct sf_hment *sfhmep; 13839 struct hat *tmphat; 13840 struct sf_hment *tmphme = NULL; 13841 struct hme_blk *hmeblkp; 13842 tte_t tte; 13843 13844 ASSERT(sfmmu_mlist_held(pp)); 13845 13846 if (PP_ISNC(pp)) 13847 return; 13848 13849 vcolor = addr_to_vcolor(vaddr); 13850 if (PP_GET_VCOLOR(pp) == vcolor) 13851 return; 13852 13853 /* 13854 * There could be no vcolor conflict between a large cached 13855 * hme page and a non alias range kpm page (neither large nor 13856 * small mapped). So if a hme conflict already exists between 13857 * a constituent page of a large hme mapping and a shared small 13858 * conflicting hme mapping, both mappings must be already 13859 * uncached at this point. 13860 */ 13861 ASSERT(!PP_ISMAPPED_LARGE(pp)); 13862 13863 if (!PP_ISMAPPED(pp)) { 13864 /* 13865 * Previous hme user of page had a different color 13866 * but since there are no current users 13867 * we just flush the cache and change the color. 13868 */ 13869 SFMMU_STAT(sf_pgcolor_conflict); 13870 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 13871 PP_SET_VCOLOR(pp, vcolor); 13872 return; 13873 } 13874 13875 /* 13876 * If we get here we have a vac conflict with a current hme 13877 * mapping. This must have been established by forcing a wrong 13878 * colored mapping, e.g. by using mmap(2) with MAP_FIXED. 13879 */ 13880 13881 /* 13882 * Check if any mapping is in same as or if it is locked 13883 * since in that case we need to uncache. 13884 */ 13885 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 13886 tmphme = sfhmep->hme_next; 13887 hmeblkp = sfmmu_hmetohblk(sfhmep); 13888 if (hmeblkp->hblk_xhat_bit) 13889 continue; 13890 tmphat = hblktosfmmu(hmeblkp); 13891 sfmmu_copytte(&sfhmep->hme_tte, &tte); 13892 ASSERT(TTE_IS_VALID(&tte)); 13893 if ((tmphat == ksfmmup) || hmeblkp->hblk_lckcnt) { 13894 /* 13895 * We have an uncache conflict 13896 */ 13897 SFMMU_STAT(sf_uncache_conflict); 13898 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1); 13899 return; 13900 } 13901 } 13902 13903 /* 13904 * We have an unload conflict 13905 */ 13906 SFMMU_STAT(sf_unload_conflict); 13907 13908 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 13909 tmphme = sfhmep->hme_next; 13910 hmeblkp = sfmmu_hmetohblk(sfhmep); 13911 if (hmeblkp->hblk_xhat_bit) 13912 continue; 13913 (void) sfmmu_pageunload(pp, sfhmep, TTE8K); 13914 } 13915 13916 /* 13917 * Unloads only does tlb flushes so we need to flush the 13918 * dcache vcolor here. 13919 */ 13920 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 13921 PP_SET_VCOLOR(pp, vcolor); 13922 } 13923 13924 /* 13925 * Remove all kpm mappings using kpme's for pp and check that 13926 * all kpm mappings (w/ and w/o kpme's) are gone. 13927 */ 13928 static void 13929 sfmmu_kpm_pageunload(page_t *pp) 13930 { 13931 caddr_t vaddr; 13932 struct kpme *kpme, *nkpme; 13933 13934 ASSERT(pp != NULL); 13935 ASSERT(pp->p_kpmref); 13936 ASSERT(sfmmu_mlist_held(pp)); 13937 13938 vaddr = hat_kpm_page2va(pp, 1); 13939 13940 for (kpme = pp->p_kpmelist; kpme; kpme = nkpme) { 13941 ASSERT(kpme->kpe_page == pp); 13942 13943 if (pp->p_kpmref == 0) 13944 panic("sfmmu_kpm_pageunload: stale p_kpmref pp=%p " 13945 "kpme=%p", (void *)pp, (void *)kpme); 13946 13947 nkpme = kpme->kpe_next; 13948 13949 /* Add instance callback here here if needed later */ 13950 sfmmu_kpme_sub(kpme, pp); 13951 } 13952 13953 /* 13954 * Also correct after mixed kpme/nonkpme mappings. If nonkpme 13955 * segkpm clients have unlocked the page and forgot to mapout 13956 * we panic here. 13957 */ 13958 if (pp->p_kpmref != 0) 13959 panic("sfmmu_kpm_pageunload: bad refcnt pp=%p", (void *)pp); 13960 13961 sfmmu_kpm_mapout(pp, vaddr); 13962 } 13963 13964 /* 13965 * Remove a large kpm mapping from kernel TSB and all TLB's. 13966 */ 13967 static void 13968 sfmmu_kpm_demap_large(caddr_t vaddr) 13969 { 13970 sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT4M); 13971 sfmmu_kpm_demap_tlbs(vaddr, KCONTEXT); 13972 } 13973 13974 /* 13975 * Remove a small kpm mapping from kernel TSB and all TLB's. 13976 */ 13977 static void 13978 sfmmu_kpm_demap_small(caddr_t vaddr) 13979 { 13980 sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT); 13981 sfmmu_kpm_demap_tlbs(vaddr, KCONTEXT); 13982 } 13983 13984 /* 13985 * Demap a kpm mapping in all TLB's. 13986 */ 13987 static void 13988 sfmmu_kpm_demap_tlbs(caddr_t vaddr, int ctxnum) 13989 { 13990 cpuset_t cpuset; 13991 13992 kpreempt_disable(); 13993 cpuset = ksfmmup->sfmmu_cpusran; 13994 CPUSET_AND(cpuset, cpu_ready_set); 13995 CPUSET_DEL(cpuset, CPU->cpu_id); 13996 SFMMU_XCALL_STATS(ctxnum); 13997 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)vaddr, ctxnum); 13998 vtag_flushpage(vaddr, ctxnum); 13999 kpreempt_enable(); 14000 } 14001 14002 /* 14003 * Summary states used in sfmmu_kpm_vac_unload (KPM_VUL__*). 14004 * See also more detailed comments within in the sfmmu_kpm_vac_unload switch. 14005 * Abbreviations used: 14006 * BIG: Large page kpm mapping in use. 14007 * CONFL: VAC conflict(s) within a kpm_page. 14008 * INCR: Count of conflicts within a kpm_page is going to be incremented. 14009 * DECR: Count of conflicts within a kpm_page is going to be decremented. 14010 * UNMAP_SMALL: A small (regular page size) mapping is going to be unmapped. 14011 * TNC: Temporary non cached: a kpm mapped page is mapped in TNC state. 14012 */ 14013 #define KPM_VUL_BIG (0) 14014 #define KPM_VUL_CONFL_INCR1 (KPM_KS) 14015 #define KPM_VUL_UNMAP_SMALL1 (KPM_KS | KPM_S) 14016 #define KPM_VUL_CONFL_INCR2 (KPM_KC) 14017 #define KPM_VUL_CONFL_INCR3 (KPM_KC | KPM_KS) 14018 #define KPM_VUL_UNMAP_SMALL2 (KPM_KC | KPM_KS | KPM_S) 14019 #define KPM_VUL_CONFL_DECR1 (KPM_KC | KPM_C) 14020 #define KPM_VUL_CONFL_DECR2 (KPM_KC | KPM_C | KPM_KS) 14021 #define KPM_VUL_TNC (KPM_KC | KPM_C | KPM_KS | KPM_S) 14022 14023 /* 14024 * Handle VAC unload conflicts introduced by hme mappings or vice 14025 * versa when a hme conflict mapping is replaced by a non conflict 14026 * one. Perform actions and state transitions according to the 14027 * various page and kpm_page entry states. VACache flushes are in 14028 * the responsibiliy of the caller. We still hold the mlist lock. 14029 */ 14030 static void 14031 sfmmu_kpm_vac_unload(page_t *pp, caddr_t vaddr) 14032 { 14033 kpm_page_t *kp; 14034 kpm_hlk_t *kpmp; 14035 caddr_t kpmvaddr = hat_kpm_page2va(pp, 1); 14036 int newcolor; 14037 kmutex_t *pmtx; 14038 uint_t vacunlcase; 14039 int badstate = 0; 14040 kpm_spage_t *ksp; 14041 kpm_shlk_t *kpmsp; 14042 14043 ASSERT(PAGE_LOCKED(pp)); 14044 ASSERT(sfmmu_mlist_held(pp)); 14045 ASSERT(!PP_ISNC(pp)); 14046 14047 newcolor = addr_to_vcolor(kpmvaddr) != addr_to_vcolor(vaddr); 14048 if (kpm_smallpages) 14049 goto smallpages_vac_unload; 14050 14051 PP2KPMPG(pp, kp); 14052 kpmp = KPMP_HASH(kp); 14053 mutex_enter(&kpmp->khl_mutex); 14054 14055 if (IS_KPM_ALIAS_RANGE(kpmvaddr)) { 14056 if (kp->kp_refcnta < 1) { 14057 panic("sfmmu_kpm_vac_unload: bad refcnta kpm_page=%p\n", 14058 (void *)kp); 14059 } 14060 14061 if (PP_ISKPMC(pp) == 0) { 14062 if (newcolor == 0) 14063 goto exit; 14064 sfmmu_kpm_demap_small(kpmvaddr); 14065 pmtx = sfmmu_page_enter(pp); 14066 PP_SETKPMC(pp); 14067 sfmmu_page_exit(pmtx); 14068 14069 } else if (newcolor == 0) { 14070 pmtx = sfmmu_page_enter(pp); 14071 PP_CLRKPMC(pp); 14072 sfmmu_page_exit(pmtx); 14073 14074 } else { 14075 badstate++; 14076 } 14077 14078 goto exit; 14079 } 14080 14081 badstate = (kp->kp_refcnt < 0 || kp->kp_refcnts < 0); 14082 if (kp->kp_refcntc == -1) { 14083 /* 14084 * We should come here only if trap level tsb miss 14085 * handler is disabled. 14086 */ 14087 badstate |= (kp->kp_refcnt == 0 || kp->kp_refcnts > 0 || 14088 PP_ISKPMC(pp) || PP_ISKPMS(pp) || PP_ISNC(pp)); 14089 } else { 14090 badstate |= (kp->kp_refcntc < 0); 14091 } 14092 14093 if (badstate) 14094 goto exit; 14095 14096 if (PP_ISKPMC(pp) == 0 && newcolor == 0) { 14097 ASSERT(PP_ISKPMS(pp) == 0); 14098 goto exit; 14099 } 14100 14101 /* 14102 * Combine the per kpm_page and per page kpm VAC states 14103 * to a summary state in order to make the vac unload 14104 * handling more concise. 14105 */ 14106 vacunlcase = (((kp->kp_refcntc > 0) ? KPM_KC : 0) | 14107 ((kp->kp_refcnts > 0) ? KPM_KS : 0) | 14108 (PP_ISKPMC(pp) ? KPM_C : 0) | 14109 (PP_ISKPMS(pp) ? KPM_S : 0)); 14110 14111 switch (vacunlcase) { 14112 case KPM_VUL_BIG: /* - - - - */ 14113 /* 14114 * Have to breakup the large page mapping to be 14115 * able to handle the conflicting hme vaddr. 14116 */ 14117 if (kp->kp_refcntc == -1) { 14118 /* remove go indication */ 14119 sfmmu_kpm_tsbmtl(&kp->kp_refcntc, 14120 &kpmp->khl_lock, KPMTSBM_STOP); 14121 } 14122 sfmmu_kpm_demap_large(kpmvaddr); 14123 14124 ASSERT(kp->kp_refcntc == 0); 14125 kp->kp_refcntc++; 14126 pmtx = sfmmu_page_enter(pp); 14127 PP_SETKPMC(pp); 14128 sfmmu_page_exit(pmtx); 14129 break; 14130 14131 case KPM_VUL_UNMAP_SMALL1: /* - - ks s */ 14132 case KPM_VUL_UNMAP_SMALL2: /* kc - ks s */ 14133 /* 14134 * New conflict w/ an active kpm page, actually mapped 14135 * in by small TSB/TLB entries. Remove the mapping and 14136 * update states. 14137 */ 14138 ASSERT(newcolor); 14139 sfmmu_kpm_demap_small(kpmvaddr); 14140 kp->kp_refcnts--; 14141 kp->kp_refcnt++; 14142 kp->kp_refcntc++; 14143 pmtx = sfmmu_page_enter(pp); 14144 PP_CLRKPMS(pp); 14145 PP_SETKPMC(pp); 14146 sfmmu_page_exit(pmtx); 14147 break; 14148 14149 case KPM_VUL_CONFL_INCR1: /* - - ks - */ 14150 case KPM_VUL_CONFL_INCR2: /* kc - - - */ 14151 case KPM_VUL_CONFL_INCR3: /* kc - ks - */ 14152 /* 14153 * New conflict on a active kpm mapped page not yet in 14154 * TSB/TLB. Mark page and increment the kpm_page conflict 14155 * count. 14156 */ 14157 ASSERT(newcolor); 14158 kp->kp_refcntc++; 14159 pmtx = sfmmu_page_enter(pp); 14160 PP_SETKPMC(pp); 14161 sfmmu_page_exit(pmtx); 14162 break; 14163 14164 case KPM_VUL_CONFL_DECR1: /* kc c - - */ 14165 case KPM_VUL_CONFL_DECR2: /* kc c ks - */ 14166 /* 14167 * A conflicting hme mapping is removed for an active 14168 * kpm page not yet in TSB/TLB. Unmark page and decrement 14169 * the kpm_page conflict count. 14170 */ 14171 ASSERT(newcolor == 0); 14172 kp->kp_refcntc--; 14173 pmtx = sfmmu_page_enter(pp); 14174 PP_CLRKPMC(pp); 14175 sfmmu_page_exit(pmtx); 14176 break; 14177 14178 case KPM_VUL_TNC: /* kc c ks s */ 14179 cmn_err(CE_NOTE, "sfmmu_kpm_vac_unload: " 14180 "page not in NC state"); 14181 /* FALLTHRU */ 14182 14183 default: 14184 badstate++; 14185 } 14186 exit: 14187 if (badstate) { 14188 panic("sfmmu_kpm_vac_unload: inconsistent VAC state, " 14189 "kpmvaddr=%p kp=%p pp=%p", 14190 (void *)kpmvaddr, (void *)kp, (void *)pp); 14191 } 14192 mutex_exit(&kpmp->khl_mutex); 14193 14194 return; 14195 14196 smallpages_vac_unload: 14197 if (newcolor == 0) 14198 return; 14199 14200 PP2KPMSPG(pp, ksp); 14201 kpmsp = KPMP_SHASH(ksp); 14202 14203 if (PP_ISKPMC(pp) == 0) { 14204 if (ksp->kp_mapped == KPM_MAPPEDS) { 14205 /* 14206 * Stop TL tsbmiss handling 14207 */ 14208 (void) sfmmu_kpm_stsbmtl(&ksp->kp_mapped, 14209 &kpmsp->kshl_lock, KPM_MAPPEDSC); 14210 14211 sfmmu_kpm_demap_small(kpmvaddr); 14212 14213 } else if (ksp->kp_mapped != KPM_MAPPEDSC) { 14214 panic("sfmmu_kpm_vac_unload: inconsistent mapping"); 14215 } 14216 14217 pmtx = sfmmu_page_enter(pp); 14218 PP_SETKPMC(pp); 14219 sfmmu_page_exit(pmtx); 14220 14221 } else { 14222 if (ksp->kp_mapped != KPM_MAPPEDSC) 14223 panic("sfmmu_kpm_vac_unload: inconsistent mapping"); 14224 } 14225 } 14226 14227 /* 14228 * Page is marked to be in VAC conflict to an existing kpm mapping 14229 * or is kpm mapped using only the regular pagesize. Called from 14230 * sfmmu_hblk_unload when a mlist is completely removed. 14231 */ 14232 static void 14233 sfmmu_kpm_hme_unload(page_t *pp) 14234 { 14235 /* tte assembly */ 14236 kpm_page_t *kp; 14237 kpm_hlk_t *kpmp; 14238 caddr_t vaddr; 14239 kmutex_t *pmtx; 14240 uint_t flags; 14241 kpm_spage_t *ksp; 14242 14243 ASSERT(sfmmu_mlist_held(pp)); 14244 ASSERT(PP_ISMAPPED_KPM(pp)); 14245 14246 flags = pp->p_nrm & (P_KPMC | P_KPMS); 14247 if (kpm_smallpages) 14248 goto smallpages_hme_unload; 14249 14250 if (flags == (P_KPMC | P_KPMS)) { 14251 panic("sfmmu_kpm_hme_unload: page should be uncached"); 14252 14253 } else if (flags == P_KPMS) { 14254 /* 14255 * Page mapped small but not involved in VAC conflict 14256 */ 14257 return; 14258 } 14259 14260 vaddr = hat_kpm_page2va(pp, 1); 14261 14262 PP2KPMPG(pp, kp); 14263 kpmp = KPMP_HASH(kp); 14264 mutex_enter(&kpmp->khl_mutex); 14265 14266 if (IS_KPM_ALIAS_RANGE(vaddr)) { 14267 if (kp->kp_refcnta < 1) { 14268 panic("sfmmu_kpm_hme_unload: bad refcnta kpm_page=%p\n", 14269 (void *)kp); 14270 } 14271 14272 } else { 14273 if (kp->kp_refcntc < 1) { 14274 panic("sfmmu_kpm_hme_unload: bad refcntc kpm_page=%p\n", 14275 (void *)kp); 14276 } 14277 kp->kp_refcntc--; 14278 } 14279 14280 pmtx = sfmmu_page_enter(pp); 14281 PP_CLRKPMC(pp); 14282 sfmmu_page_exit(pmtx); 14283 14284 mutex_exit(&kpmp->khl_mutex); 14285 return; 14286 14287 smallpages_hme_unload: 14288 if (flags != P_KPMC) 14289 panic("sfmmu_kpm_hme_unload: page should be uncached"); 14290 14291 vaddr = hat_kpm_page2va(pp, 1); 14292 PP2KPMSPG(pp, ksp); 14293 14294 if (ksp->kp_mapped != KPM_MAPPEDSC) 14295 panic("sfmmu_kpm_hme_unload: inconsistent mapping"); 14296 14297 /* 14298 * Keep KPM_MAPPEDSC until the next kpm tsbmiss where it 14299 * prevents TL tsbmiss handling and force a hat_kpm_fault. 14300 * There we can start over again. 14301 */ 14302 14303 pmtx = sfmmu_page_enter(pp); 14304 PP_CLRKPMC(pp); 14305 sfmmu_page_exit(pmtx); 14306 } 14307 14308 /* 14309 * Special hooks for sfmmu_page_cache_array() when changing the 14310 * cacheability of a page. It is used to obey the hat_kpm lock 14311 * ordering (mlist -> kpmp -> spl, and back). 14312 */ 14313 static kpm_hlk_t * 14314 sfmmu_kpm_kpmp_enter(page_t *pp, pgcnt_t npages) 14315 { 14316 kpm_page_t *kp; 14317 kpm_hlk_t *kpmp; 14318 14319 ASSERT(sfmmu_mlist_held(pp)); 14320 14321 if (kpm_smallpages || PP_ISMAPPED_KPM(pp) == 0) 14322 return (NULL); 14323 14324 ASSERT(npages <= kpmpnpgs); 14325 14326 PP2KPMPG(pp, kp); 14327 kpmp = KPMP_HASH(kp); 14328 mutex_enter(&kpmp->khl_mutex); 14329 14330 return (kpmp); 14331 } 14332 14333 static void 14334 sfmmu_kpm_kpmp_exit(kpm_hlk_t *kpmp) 14335 { 14336 if (kpm_smallpages || kpmp == NULL) 14337 return; 14338 14339 mutex_exit(&kpmp->khl_mutex); 14340 } 14341 14342 /* 14343 * Summary states used in sfmmu_kpm_page_cache (KPM_*). 14344 * See also more detailed comments within in the sfmmu_kpm_page_cache switch. 14345 * Abbreviations used: 14346 * UNC: Input state for an uncache request. 14347 * BIG: Large page kpm mapping in use. 14348 * SMALL: Page has a small kpm mapping within a kpm_page range. 14349 * NODEMAP: No demap needed. 14350 * NOP: No operation needed on this input state. 14351 * CACHE: Input state for a re-cache request. 14352 * MAPS: Page is in TNC and kpm VAC conflict state and kpm mapped small. 14353 * NOMAP: Page is in TNC and kpm VAC conflict state, but not small kpm 14354 * mapped. 14355 * NOMAPO: Page is in TNC and kpm VAC conflict state, but not small kpm 14356 * mapped. There are also other small kpm mappings within this 14357 * kpm_page. 14358 */ 14359 #define KPM_UNC_BIG (0) 14360 #define KPM_UNC_NODEMAP1 (KPM_KS) 14361 #define KPM_UNC_SMALL1 (KPM_KS | KPM_S) 14362 #define KPM_UNC_NODEMAP2 (KPM_KC) 14363 #define KPM_UNC_NODEMAP3 (KPM_KC | KPM_KS) 14364 #define KPM_UNC_SMALL2 (KPM_KC | KPM_KS | KPM_S) 14365 #define KPM_UNC_NOP1 (KPM_KC | KPM_C) 14366 #define KPM_UNC_NOP2 (KPM_KC | KPM_C | KPM_KS) 14367 #define KPM_CACHE_NOMAP (KPM_KC | KPM_C) 14368 #define KPM_CACHE_NOMAPO (KPM_KC | KPM_C | KPM_KS) 14369 #define KPM_CACHE_MAPS (KPM_KC | KPM_C | KPM_KS | KPM_S) 14370 14371 /* 14372 * This function is called when the virtual cacheability of a page 14373 * is changed and the page has an actice kpm mapping. The mlist mutex, 14374 * the spl hash lock and the kpmp mutex (if needed) are already grabbed. 14375 */ 14376 static void 14377 sfmmu_kpm_page_cache(page_t *pp, int flags, int cache_flush_tag) 14378 { 14379 kpm_page_t *kp; 14380 kpm_hlk_t *kpmp; 14381 caddr_t kpmvaddr; 14382 int badstate = 0; 14383 uint_t pgcacase; 14384 kpm_spage_t *ksp; 14385 kpm_shlk_t *kpmsp; 14386 int oldval; 14387 14388 ASSERT(PP_ISMAPPED_KPM(pp)); 14389 ASSERT(sfmmu_mlist_held(pp)); 14390 ASSERT(sfmmu_page_spl_held(pp)); 14391 14392 if (flags != HAT_TMPNC && flags != HAT_CACHE) 14393 panic("sfmmu_kpm_page_cache: bad flags"); 14394 14395 kpmvaddr = hat_kpm_page2va(pp, 1); 14396 14397 if (flags == HAT_TMPNC && cache_flush_tag == CACHE_FLUSH) { 14398 pfn_t pfn = pp->p_pagenum; 14399 int vcolor = addr_to_vcolor(kpmvaddr); 14400 cpuset_t cpuset = cpu_ready_set; 14401 14402 /* Flush vcolor in DCache */ 14403 CPUSET_DEL(cpuset, CPU->cpu_id); 14404 SFMMU_XCALL_STATS(ksfmmup->sfmmu_cnum); 14405 xt_some(cpuset, vac_flushpage_tl1, pfn, vcolor); 14406 vac_flushpage(pfn, vcolor); 14407 } 14408 14409 if (kpm_smallpages) 14410 goto smallpages_page_cache; 14411 14412 PP2KPMPG(pp, kp); 14413 kpmp = KPMP_HASH(kp); 14414 ASSERT(MUTEX_HELD(&kpmp->khl_mutex)); 14415 14416 if (IS_KPM_ALIAS_RANGE(kpmvaddr)) { 14417 if (kp->kp_refcnta < 1) { 14418 panic("sfmmu_kpm_page_cache: bad refcnta " 14419 "kpm_page=%p\n", (void *)kp); 14420 } 14421 sfmmu_kpm_demap_small(kpmvaddr); 14422 if (flags == HAT_TMPNC) { 14423 PP_SETKPMC(pp); 14424 ASSERT(!PP_ISKPMS(pp)); 14425 } else { 14426 ASSERT(PP_ISKPMC(pp)); 14427 PP_CLRKPMC(pp); 14428 } 14429 goto exit; 14430 } 14431 14432 badstate = (kp->kp_refcnt < 0 || kp->kp_refcnts < 0); 14433 if (kp->kp_refcntc == -1) { 14434 /* 14435 * We should come here only if trap level tsb miss 14436 * handler is disabled. 14437 */ 14438 badstate |= (kp->kp_refcnt == 0 || kp->kp_refcnts > 0 || 14439 PP_ISKPMC(pp) || PP_ISKPMS(pp) || PP_ISNC(pp)); 14440 } else { 14441 badstate |= (kp->kp_refcntc < 0); 14442 } 14443 14444 if (badstate) 14445 goto exit; 14446 14447 /* 14448 * Combine the per kpm_page and per page kpm VAC states to 14449 * a summary state in order to make the VAC cache/uncache 14450 * handling more concise. 14451 */ 14452 pgcacase = (((kp->kp_refcntc > 0) ? KPM_KC : 0) | 14453 ((kp->kp_refcnts > 0) ? KPM_KS : 0) | 14454 (PP_ISKPMC(pp) ? KPM_C : 0) | 14455 (PP_ISKPMS(pp) ? KPM_S : 0)); 14456 14457 if (flags == HAT_CACHE) { 14458 switch (pgcacase) { 14459 case KPM_CACHE_MAPS: /* kc c ks s */ 14460 sfmmu_kpm_demap_small(kpmvaddr); 14461 if (kp->kp_refcnts < 1) { 14462 panic("sfmmu_kpm_page_cache: bad refcnts " 14463 "kpm_page=%p\n", (void *)kp); 14464 } 14465 kp->kp_refcnts--; 14466 kp->kp_refcnt++; 14467 PP_CLRKPMS(pp); 14468 /* FALLTHRU */ 14469 14470 case KPM_CACHE_NOMAP: /* kc c - - */ 14471 case KPM_CACHE_NOMAPO: /* kc c ks - */ 14472 kp->kp_refcntc--; 14473 PP_CLRKPMC(pp); 14474 break; 14475 14476 default: 14477 badstate++; 14478 } 14479 goto exit; 14480 } 14481 14482 switch (pgcacase) { 14483 case KPM_UNC_BIG: /* - - - - */ 14484 if (kp->kp_refcnt < 1) { 14485 panic("sfmmu_kpm_page_cache: bad refcnt " 14486 "kpm_page=%p\n", (void *)kp); 14487 } 14488 14489 /* 14490 * Have to breakup the large page mapping in preparation 14491 * to the upcoming TNC mode handled by small mappings. 14492 * The demap can already be done due to another conflict 14493 * within the kpm_page. 14494 */ 14495 if (kp->kp_refcntc == -1) { 14496 /* remove go indication */ 14497 sfmmu_kpm_tsbmtl(&kp->kp_refcntc, 14498 &kpmp->khl_lock, KPMTSBM_STOP); 14499 } 14500 ASSERT(kp->kp_refcntc == 0); 14501 sfmmu_kpm_demap_large(kpmvaddr); 14502 kp->kp_refcntc++; 14503 PP_SETKPMC(pp); 14504 break; 14505 14506 case KPM_UNC_SMALL1: /* - - ks s */ 14507 case KPM_UNC_SMALL2: /* kc - ks s */ 14508 /* 14509 * Have to demap an already small kpm mapping in preparation 14510 * to the upcoming TNC mode. The demap can already be done 14511 * due to another conflict within the kpm_page. 14512 */ 14513 sfmmu_kpm_demap_small(kpmvaddr); 14514 kp->kp_refcntc++; 14515 kp->kp_refcnts--; 14516 kp->kp_refcnt++; 14517 PP_CLRKPMS(pp); 14518 PP_SETKPMC(pp); 14519 break; 14520 14521 case KPM_UNC_NODEMAP1: /* - - ks - */ 14522 /* fallthru */ 14523 14524 case KPM_UNC_NODEMAP2: /* kc - - - */ 14525 case KPM_UNC_NODEMAP3: /* kc - ks - */ 14526 kp->kp_refcntc++; 14527 PP_SETKPMC(pp); 14528 break; 14529 14530 case KPM_UNC_NOP1: /* kc c - - */ 14531 case KPM_UNC_NOP2: /* kc c ks - */ 14532 break; 14533 14534 default: 14535 badstate++; 14536 } 14537 exit: 14538 if (badstate) { 14539 panic("sfmmu_kpm_page_cache: inconsistent VAC state " 14540 "kpmvaddr=%p kp=%p pp=%p", (void *)kpmvaddr, 14541 (void *)kp, (void *)pp); 14542 } 14543 return; 14544 14545 smallpages_page_cache: 14546 PP2KPMSPG(pp, ksp); 14547 kpmsp = KPMP_SHASH(ksp); 14548 14549 oldval = sfmmu_kpm_stsbmtl(&ksp->kp_mapped, 14550 &kpmsp->kshl_lock, KPM_MAPPEDSC); 14551 14552 if (!(oldval == KPM_MAPPEDS || oldval == KPM_MAPPEDSC)) 14553 panic("smallpages_page_cache: inconsistent mapping"); 14554 14555 sfmmu_kpm_demap_small(kpmvaddr); 14556 14557 if (flags == HAT_TMPNC) { 14558 PP_SETKPMC(pp); 14559 ASSERT(!PP_ISKPMS(pp)); 14560 14561 } else { 14562 ASSERT(PP_ISKPMC(pp)); 14563 PP_CLRKPMC(pp); 14564 } 14565 14566 /* 14567 * Keep KPM_MAPPEDSC until the next kpm tsbmiss where it 14568 * prevents TL tsbmiss handling and force a hat_kpm_fault. 14569 * There we can start over again. 14570 */ 14571 } 14572 14573 /* 14574 * unused in sfmmu 14575 */ 14576 void 14577 hat_dump(void) 14578 { 14579 } 14580 14581 /* 14582 * Called when a thread is exiting and we have switched to the kernel address 14583 * space. Perform the same VM initialization resume() uses when switching 14584 * processes. 14585 * 14586 * Note that sfmmu_load_mmustate() is currently a no-op for kernel threads, but 14587 * we call it anyway in case the semantics change in the future. 14588 */ 14589 /*ARGSUSED*/ 14590 void 14591 hat_thread_exit(kthread_t *thd) 14592 { 14593 ASSERT(thd->t_procp->p_as == &kas); 14594 14595 sfmmu_setctx_sec(KCONTEXT); 14596 sfmmu_load_mmustate(ksfmmup); 14597 } 14598