1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * VM - Hardware Address Translation management for Spitfire MMU. 30 * 31 * This file implements the machine specific hardware translation 32 * needed by the VM system. The machine independent interface is 33 * described in <vm/hat.h> while the machine dependent interface 34 * and data structures are described in <vm/hat_sfmmu.h>. 35 * 36 * The hat layer manages the address translation hardware as a cache 37 * driven by calls from the higher levels in the VM system. 38 */ 39 40 #include <sys/types.h> 41 #include <sys/kstat.h> 42 #include <vm/hat.h> 43 #include <vm/hat_sfmmu.h> 44 #include <vm/page.h> 45 #include <sys/pte.h> 46 #include <sys/systm.h> 47 #include <sys/mman.h> 48 #include <sys/sysmacros.h> 49 #include <sys/machparam.h> 50 #include <sys/vtrace.h> 51 #include <sys/kmem.h> 52 #include <sys/mmu.h> 53 #include <sys/cmn_err.h> 54 #include <sys/cpu.h> 55 #include <sys/cpuvar.h> 56 #include <sys/debug.h> 57 #include <sys/lgrp.h> 58 #include <sys/archsystm.h> 59 #include <sys/machsystm.h> 60 #include <sys/vmsystm.h> 61 #include <vm/as.h> 62 #include <vm/seg.h> 63 #include <vm/seg_kp.h> 64 #include <vm/seg_kmem.h> 65 #include <vm/seg_kpm.h> 66 #include <vm/rm.h> 67 #include <sys/t_lock.h> 68 #include <sys/obpdefs.h> 69 #include <sys/vm_machparam.h> 70 #include <sys/var.h> 71 #include <sys/trap.h> 72 #include <sys/machtrap.h> 73 #include <sys/scb.h> 74 #include <sys/bitmap.h> 75 #include <sys/machlock.h> 76 #include <sys/membar.h> 77 #include <sys/atomic.h> 78 #include <sys/cpu_module.h> 79 #include <sys/prom_debug.h> 80 #include <sys/ksynch.h> 81 #include <sys/mem_config.h> 82 #include <sys/mem_cage.h> 83 #include <sys/dtrace.h> 84 #include <vm/vm_dep.h> 85 #include <vm/xhat_sfmmu.h> 86 #include <sys/fpu/fpusystm.h> 87 #include <vm/mach_kpm.h> 88 89 #if defined(SF_ERRATA_57) 90 extern caddr_t errata57_limit; 91 #endif 92 93 #define HME8BLK_SZ_RND ((roundup(HME8BLK_SZ, sizeof (int64_t))) / \ 94 (sizeof (int64_t))) 95 #define HBLK_RESERVE ((struct hme_blk *)hblk_reserve) 96 97 #define HBLK_RESERVE_CNT 128 98 #define HBLK_RESERVE_MIN 20 99 100 static struct hme_blk *freehblkp; 101 static kmutex_t freehblkp_lock; 102 static int freehblkcnt; 103 104 static int64_t hblk_reserve[HME8BLK_SZ_RND]; 105 static kmutex_t hblk_reserve_lock; 106 static kthread_t *hblk_reserve_thread; 107 108 static nucleus_hblk8_info_t nucleus_hblk8; 109 static nucleus_hblk1_info_t nucleus_hblk1; 110 111 /* 112 * SFMMU specific hat functions 113 */ 114 void hat_pagecachectl(struct page *, int); 115 116 /* flags for hat_pagecachectl */ 117 #define HAT_CACHE 0x1 118 #define HAT_UNCACHE 0x2 119 #define HAT_TMPNC 0x4 120 121 /* 122 * Flag to allow the creation of non-cacheable translations 123 * to system memory. It is off by default. At the moment this 124 * flag is used by the ecache error injector. The error injector 125 * will turn it on when creating such a translation then shut it 126 * off when it's finished. 127 */ 128 129 int sfmmu_allow_nc_trans = 0; 130 131 /* 132 * Flag to disable large page support. 133 * value of 1 => disable all large pages. 134 * bits 1, 2, and 3 are to disable 64K, 512K and 4M pages respectively. 135 * 136 * For example, use the value 0x4 to disable 512K pages. 137 * 138 */ 139 #define LARGE_PAGES_OFF 0x1 140 141 /* 142 * WARNING: 512K pages MUST be disabled for ISM/DISM. If not 143 * a process would page fault indefinitely if it tried to 144 * access a 512K page. 145 */ 146 int disable_ism_large_pages = (1 << TTE512K); 147 int disable_large_pages = 0; 148 int disable_auto_large_pages = 0; 149 int disable_shm_large_pages = 0; 150 151 /* 152 * Private sfmmu data structures for hat management 153 */ 154 static struct kmem_cache *sfmmuid_cache; 155 static struct kmem_cache *mmuctxdom_cache; 156 157 /* 158 * Private sfmmu data structures for tsb management 159 */ 160 static struct kmem_cache *sfmmu_tsbinfo_cache; 161 static struct kmem_cache *sfmmu_tsb8k_cache; 162 static struct kmem_cache *sfmmu_tsb_cache[NLGRPS_MAX]; 163 static vmem_t *kmem_tsb_arena; 164 165 /* 166 * sfmmu static variables for hmeblk resource management. 167 */ 168 static vmem_t *hat_memload1_arena; /* HAT translation arena for sfmmu1_cache */ 169 static struct kmem_cache *sfmmu8_cache; 170 static struct kmem_cache *sfmmu1_cache; 171 static struct kmem_cache *pa_hment_cache; 172 173 static kmutex_t ism_mlist_lock; /* mutex for ism mapping list */ 174 /* 175 * private data for ism 176 */ 177 static struct kmem_cache *ism_blk_cache; 178 static struct kmem_cache *ism_ment_cache; 179 #define ISMID_STARTADDR NULL 180 181 /* 182 * Whether to delay TLB flushes and use Cheetah's flush-all support 183 * when removing contexts from the dirty list. 184 */ 185 int delay_tlb_flush; 186 int disable_delay_tlb_flush; 187 188 /* 189 * ``hat_lock'' is a hashed mutex lock for protecting sfmmu TSB lists, 190 * HAT flags, synchronizing TLB/TSB coherency, and context management. 191 * The lock is hashed on the sfmmup since the case where we need to lock 192 * all processes is rare but does occur (e.g. we need to unload a shared 193 * mapping from all processes using the mapping). We have a lot of buckets, 194 * and each slab of sfmmu_t's can use about a quarter of them, giving us 195 * a fairly good distribution without wasting too much space and overhead 196 * when we have to grab them all. 197 */ 198 #define SFMMU_NUM_LOCK 128 /* must be power of two */ 199 hatlock_t hat_lock[SFMMU_NUM_LOCK]; 200 201 /* 202 * Hash algorithm optimized for a small number of slabs. 203 * 7 is (highbit((sizeof sfmmu_t)) - 1) 204 * This hash algorithm is based upon the knowledge that sfmmu_t's come from a 205 * kmem_cache, and thus they will be sequential within that cache. In 206 * addition, each new slab will have a different "color" up to cache_maxcolor 207 * which will skew the hashing for each successive slab which is allocated. 208 * If the size of sfmmu_t changed to a larger size, this algorithm may need 209 * to be revisited. 210 */ 211 #define TSB_HASH_SHIFT_BITS (7) 212 #define PTR_HASH(x) ((uintptr_t)x >> TSB_HASH_SHIFT_BITS) 213 214 #ifdef DEBUG 215 int tsb_hash_debug = 0; 216 #define TSB_HASH(sfmmup) \ 217 (tsb_hash_debug ? &hat_lock[0] : \ 218 &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)]) 219 #else /* DEBUG */ 220 #define TSB_HASH(sfmmup) &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)] 221 #endif /* DEBUG */ 222 223 224 /* sfmmu_replace_tsb() return codes. */ 225 typedef enum tsb_replace_rc { 226 TSB_SUCCESS, 227 TSB_ALLOCFAIL, 228 TSB_LOSTRACE, 229 TSB_ALREADY_SWAPPED, 230 TSB_CANTGROW 231 } tsb_replace_rc_t; 232 233 /* 234 * Flags for TSB allocation routines. 235 */ 236 #define TSB_ALLOC 0x01 237 #define TSB_FORCEALLOC 0x02 238 #define TSB_GROW 0x04 239 #define TSB_SHRINK 0x08 240 #define TSB_SWAPIN 0x10 241 242 /* 243 * Support for HAT callbacks. 244 */ 245 #define SFMMU_MAX_RELOC_CALLBACKS 10 246 int sfmmu_max_cb_id = SFMMU_MAX_RELOC_CALLBACKS; 247 static id_t sfmmu_cb_nextid = 0; 248 static id_t sfmmu_tsb_cb_id; 249 struct sfmmu_callback *sfmmu_cb_table; 250 251 /* 252 * Kernel page relocation is enabled by default for non-caged 253 * kernel pages. This has little effect unless segkmem_reloc is 254 * set, since by default kernel memory comes from inside the 255 * kernel cage. 256 */ 257 int hat_kpr_enabled = 1; 258 259 kmutex_t kpr_mutex; 260 kmutex_t kpr_suspendlock; 261 kthread_t *kreloc_thread; 262 263 /* 264 * Enable VA->PA translation sanity checking on DEBUG kernels. 265 * Disabled by default. This is incompatible with some 266 * drivers (error injector, RSM) so if it breaks you get 267 * to keep both pieces. 268 */ 269 int hat_check_vtop = 0; 270 271 /* 272 * Private sfmmu routines (prototypes) 273 */ 274 static struct hme_blk *sfmmu_shadow_hcreate(sfmmu_t *, caddr_t, int, uint_t); 275 static struct hme_blk *sfmmu_hblk_alloc(sfmmu_t *, caddr_t, 276 struct hmehash_bucket *, uint_t, hmeblk_tag, uint_t); 277 static caddr_t sfmmu_hblk_unload(struct hat *, struct hme_blk *, caddr_t, 278 caddr_t, demap_range_t *, uint_t); 279 static caddr_t sfmmu_hblk_sync(struct hat *, struct hme_blk *, caddr_t, 280 caddr_t, int); 281 static void sfmmu_hblk_free(struct hmehash_bucket *, struct hme_blk *, 282 uint64_t, struct hme_blk **); 283 static void sfmmu_hblks_list_purge(struct hme_blk **); 284 static uint_t sfmmu_get_free_hblk(struct hme_blk **, uint_t); 285 static uint_t sfmmu_put_free_hblk(struct hme_blk *, uint_t); 286 static struct hme_blk *sfmmu_hblk_steal(int); 287 static int sfmmu_steal_this_hblk(struct hmehash_bucket *, 288 struct hme_blk *, uint64_t, uint64_t, 289 struct hme_blk *); 290 static caddr_t sfmmu_hblk_unlock(struct hme_blk *, caddr_t, caddr_t); 291 292 static void sfmmu_memload_batchsmall(struct hat *, caddr_t, page_t **, 293 uint_t, uint_t, pgcnt_t); 294 void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, 295 uint_t); 296 static int sfmmu_tteload_array(sfmmu_t *, tte_t *, caddr_t, page_t **, 297 uint_t); 298 static struct hmehash_bucket *sfmmu_tteload_acquire_hashbucket(sfmmu_t *, 299 caddr_t, int); 300 static struct hme_blk *sfmmu_tteload_find_hmeblk(sfmmu_t *, 301 struct hmehash_bucket *, caddr_t, uint_t, uint_t); 302 static int sfmmu_tteload_addentry(sfmmu_t *, struct hme_blk *, tte_t *, 303 caddr_t, page_t **, uint_t); 304 static void sfmmu_tteload_release_hashbucket(struct hmehash_bucket *); 305 306 static int sfmmu_pagearray_setup(caddr_t, page_t **, tte_t *, int); 307 pfn_t sfmmu_uvatopfn(caddr_t, sfmmu_t *); 308 void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 309 #ifdef VAC 310 static void sfmmu_vac_conflict(struct hat *, caddr_t, page_t *); 311 static int sfmmu_vacconflict_array(caddr_t, page_t *, int *); 312 int tst_tnc(page_t *pp, pgcnt_t); 313 void conv_tnc(page_t *pp, int); 314 #endif 315 316 static void sfmmu_get_ctx(sfmmu_t *); 317 static void sfmmu_free_sfmmu(sfmmu_t *); 318 319 static void sfmmu_gettte(struct hat *, caddr_t, tte_t *); 320 static void sfmmu_ttesync(struct hat *, caddr_t, tte_t *, page_t *); 321 static void sfmmu_chgattr(struct hat *, caddr_t, size_t, uint_t, int); 322 323 cpuset_t sfmmu_pageunload(page_t *, struct sf_hment *, int); 324 static void hat_pagereload(struct page *, struct page *); 325 static cpuset_t sfmmu_pagesync(page_t *, struct sf_hment *, uint_t); 326 #ifdef VAC 327 void sfmmu_page_cache_array(page_t *, int, int, pgcnt_t); 328 static void sfmmu_page_cache(page_t *, int, int, int); 329 #endif 330 331 static void sfmmu_tlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 332 pfn_t, int, int, int, int); 333 static void sfmmu_ismtlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, 334 pfn_t, int); 335 static void sfmmu_tlb_demap(caddr_t, sfmmu_t *, struct hme_blk *, int, int); 336 static void sfmmu_tlb_range_demap(demap_range_t *); 337 static void sfmmu_invalidate_ctx(sfmmu_t *); 338 static void sfmmu_sync_mmustate(sfmmu_t *); 339 340 static void sfmmu_tsbinfo_setup_phys(struct tsb_info *, pfn_t); 341 static int sfmmu_tsbinfo_alloc(struct tsb_info **, int, int, uint_t, 342 sfmmu_t *); 343 static void sfmmu_tsb_free(struct tsb_info *); 344 static void sfmmu_tsbinfo_free(struct tsb_info *); 345 static int sfmmu_init_tsbinfo(struct tsb_info *, int, int, uint_t, 346 sfmmu_t *); 347 348 static void sfmmu_tsb_swapin(sfmmu_t *, hatlock_t *); 349 static int sfmmu_select_tsb_szc(pgcnt_t); 350 static void sfmmu_mod_tsb(sfmmu_t *, caddr_t, tte_t *, int); 351 #define sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \ 352 sfmmu_mod_tsb(sfmmup, vaddr, tte, szc) 353 #define sfmmu_unload_tsb(sfmmup, vaddr, szc) \ 354 sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc) 355 static void sfmmu_copy_tsb(struct tsb_info *, struct tsb_info *); 356 static tsb_replace_rc_t sfmmu_replace_tsb(sfmmu_t *, struct tsb_info *, uint_t, 357 hatlock_t *, uint_t); 358 static void sfmmu_size_tsb(sfmmu_t *, int, uint64_t, uint64_t, int); 359 360 #ifdef VAC 361 void sfmmu_cache_flush(pfn_t, int); 362 void sfmmu_cache_flushcolor(int, pfn_t); 363 #endif 364 static caddr_t sfmmu_hblk_chgattr(sfmmu_t *, struct hme_blk *, caddr_t, 365 caddr_t, demap_range_t *, uint_t, int); 366 367 static uint64_t sfmmu_vtop_attr(uint_t, int mode, tte_t *); 368 static uint_t sfmmu_ptov_attr(tte_t *); 369 static caddr_t sfmmu_hblk_chgprot(sfmmu_t *, struct hme_blk *, caddr_t, 370 caddr_t, demap_range_t *, uint_t); 371 static uint_t sfmmu_vtop_prot(uint_t, uint_t *); 372 static int sfmmu_idcache_constructor(void *, void *, int); 373 static void sfmmu_idcache_destructor(void *, void *); 374 static int sfmmu_hblkcache_constructor(void *, void *, int); 375 static void sfmmu_hblkcache_destructor(void *, void *); 376 static void sfmmu_hblkcache_reclaim(void *); 377 static void sfmmu_shadow_hcleanup(sfmmu_t *, struct hme_blk *, 378 struct hmehash_bucket *); 379 static void sfmmu_free_hblks(sfmmu_t *, caddr_t, caddr_t, int); 380 static void sfmmu_rm_large_mappings(page_t *, int); 381 382 static void hat_lock_init(void); 383 static void hat_kstat_init(void); 384 static int sfmmu_kstat_percpu_update(kstat_t *ksp, int rw); 385 static void sfmmu_check_page_sizes(sfmmu_t *, int); 386 int fnd_mapping_sz(page_t *); 387 static void iment_add(struct ism_ment *, struct hat *); 388 static void iment_sub(struct ism_ment *, struct hat *); 389 static pgcnt_t ism_tsb_entries(sfmmu_t *, int szc); 390 extern void sfmmu_setup_tsbinfo(sfmmu_t *); 391 extern void sfmmu_clear_utsbinfo(void); 392 393 static void sfmmu_ctx_wrap_around(mmu_ctx_t *); 394 395 /* kpm globals */ 396 #ifdef DEBUG 397 /* 398 * Enable trap level tsbmiss handling 399 */ 400 int kpm_tsbmtl = 1; 401 402 /* 403 * Flush the TLB on kpm mapout. Note: Xcalls are used (again) for the 404 * required TLB shootdowns in this case, so handle w/ care. Off by default. 405 */ 406 int kpm_tlb_flush; 407 #endif /* DEBUG */ 408 409 static void *sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *, size_t, int); 410 411 #ifdef DEBUG 412 static void sfmmu_check_hblk_flist(); 413 #endif 414 415 /* 416 * Semi-private sfmmu data structures. Some of them are initialize in 417 * startup or in hat_init. Some of them are private but accessed by 418 * assembly code or mach_sfmmu.c 419 */ 420 struct hmehash_bucket *uhme_hash; /* user hmeblk hash table */ 421 struct hmehash_bucket *khme_hash; /* kernel hmeblk hash table */ 422 uint64_t uhme_hash_pa; /* PA of uhme_hash */ 423 uint64_t khme_hash_pa; /* PA of khme_hash */ 424 int uhmehash_num; /* # of buckets in user hash table */ 425 int khmehash_num; /* # of buckets in kernel hash table */ 426 427 uint_t max_mmu_ctxdoms = 0; /* max context domains in the system */ 428 mmu_ctx_t **mmu_ctxs_tbl; /* global array of context domains */ 429 uint64_t mmu_saved_gnum = 0; /* to init incoming MMUs' gnums */ 430 431 #define DEFAULT_NUM_CTXS_PER_MMU 8192 432 static uint_t nctxs = DEFAULT_NUM_CTXS_PER_MMU; 433 434 int cache; /* describes system cache */ 435 436 caddr_t ktsb_base; /* kernel 8k-indexed tsb base address */ 437 uint64_t ktsb_pbase; /* kernel 8k-indexed tsb phys address */ 438 int ktsb_szcode; /* kernel 8k-indexed tsb size code */ 439 int ktsb_sz; /* kernel 8k-indexed tsb size */ 440 441 caddr_t ktsb4m_base; /* kernel 4m-indexed tsb base address */ 442 uint64_t ktsb4m_pbase; /* kernel 4m-indexed tsb phys address */ 443 int ktsb4m_szcode; /* kernel 4m-indexed tsb size code */ 444 int ktsb4m_sz; /* kernel 4m-indexed tsb size */ 445 446 uint64_t kpm_tsbbase; /* kernel seg_kpm 4M TSB base address */ 447 int kpm_tsbsz; /* kernel seg_kpm 4M TSB size code */ 448 uint64_t kpmsm_tsbbase; /* kernel seg_kpm 8K TSB base address */ 449 int kpmsm_tsbsz; /* kernel seg_kpm 8K TSB size code */ 450 451 #ifndef sun4v 452 int utsb_dtlb_ttenum = -1; /* index in TLB for utsb locked TTE */ 453 int utsb4m_dtlb_ttenum = -1; /* index in TLB for 4M TSB TTE */ 454 int dtlb_resv_ttenum; /* index in TLB of first reserved TTE */ 455 caddr_t utsb_vabase; /* reserved kernel virtual memory */ 456 caddr_t utsb4m_vabase; /* for trap handler TSB accesses */ 457 #endif /* sun4v */ 458 uint64_t tsb_alloc_bytes = 0; /* bytes allocated to TSBs */ 459 vmem_t *kmem_tsb_default_arena[NLGRPS_MAX]; /* For dynamic TSBs */ 460 461 /* 462 * Size to use for TSB slabs. Future platforms that support page sizes 463 * larger than 4M may wish to change these values, and provide their own 464 * assembly macros for building and decoding the TSB base register contents. 465 * Note disable_large_pages will override the value set here. 466 */ 467 uint_t tsb_slab_ttesz = TTE4M; 468 uint_t tsb_slab_size; 469 uint_t tsb_slab_shift; 470 uint_t tsb_slab_mask; /* PFN mask for TTE */ 471 472 /* largest TSB size to grow to, will be smaller on smaller memory systems */ 473 int tsb_max_growsize = UTSB_MAX_SZCODE; 474 475 /* 476 * Tunable parameters dealing with TSB policies. 477 */ 478 479 /* 480 * This undocumented tunable forces all 8K TSBs to be allocated from 481 * the kernel heap rather than from the kmem_tsb_default_arena arenas. 482 */ 483 #ifdef DEBUG 484 int tsb_forceheap = 0; 485 #endif /* DEBUG */ 486 487 /* 488 * Decide whether to use per-lgroup arenas, or one global set of 489 * TSB arenas. The default is not to break up per-lgroup, since 490 * most platforms don't recognize any tangible benefit from it. 491 */ 492 int tsb_lgrp_affinity = 0; 493 494 /* 495 * Used for growing the TSB based on the process RSS. 496 * tsb_rss_factor is based on the smallest TSB, and is 497 * shifted by the TSB size to determine if we need to grow. 498 * The default will grow the TSB if the number of TTEs for 499 * this page size exceeds 75% of the number of TSB entries, 500 * which should _almost_ eliminate all conflict misses 501 * (at the expense of using up lots and lots of memory). 502 */ 503 #define TSB_RSS_FACTOR (TSB_ENTRIES(TSB_MIN_SZCODE) * 0.75) 504 #define SFMMU_RSS_TSBSIZE(tsbszc) (tsb_rss_factor << tsbszc) 505 #define SELECT_TSB_SIZECODE(pgcnt) ( \ 506 (enable_tsb_rss_sizing)? sfmmu_select_tsb_szc(pgcnt) : \ 507 default_tsb_size) 508 #define TSB_OK_SHRINK() \ 509 (tsb_alloc_bytes > tsb_alloc_hiwater || freemem < desfree) 510 #define TSB_OK_GROW() \ 511 (tsb_alloc_bytes < tsb_alloc_hiwater && freemem > desfree) 512 513 int enable_tsb_rss_sizing = 1; 514 int tsb_rss_factor = (int)TSB_RSS_FACTOR; 515 516 /* which TSB size code to use for new address spaces or if rss sizing off */ 517 int default_tsb_size = TSB_8K_SZCODE; 518 519 static uint64_t tsb_alloc_hiwater; /* limit TSB reserved memory */ 520 uint64_t tsb_alloc_hiwater_factor; /* tsb_alloc_hiwater = physmem / this */ 521 #define TSB_ALLOC_HIWATER_FACTOR_DEFAULT 32 522 523 #ifdef DEBUG 524 static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */ 525 static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */ 526 static int tsb_alloc_mtbf = 0; /* fail allocation every n attempts */ 527 static int tsb_alloc_fail_mtbf = 0; 528 static int tsb_alloc_count = 0; 529 #endif /* DEBUG */ 530 531 /* if set to 1, will remap valid TTEs when growing TSB. */ 532 int tsb_remap_ttes = 1; 533 534 /* 535 * If we have more than this many mappings, allocate a second TSB. 536 * This default is chosen because the I/D fully associative TLBs are 537 * assumed to have at least 8 available entries. Platforms with a 538 * larger fully-associative TLB could probably override the default. 539 */ 540 int tsb_sectsb_threshold = 8; 541 542 /* 543 * kstat data 544 */ 545 struct sfmmu_global_stat sfmmu_global_stat; 546 struct sfmmu_tsbsize_stat sfmmu_tsbsize_stat; 547 548 /* 549 * Global data 550 */ 551 sfmmu_t *ksfmmup; /* kernel's hat id */ 552 553 #ifdef DEBUG 554 static void chk_tte(tte_t *, tte_t *, tte_t *, struct hme_blk *); 555 #endif 556 557 /* sfmmu locking operations */ 558 static kmutex_t *sfmmu_mlspl_enter(struct page *, int); 559 static int sfmmu_mlspl_held(struct page *, int); 560 561 kmutex_t *sfmmu_page_enter(page_t *); 562 void sfmmu_page_exit(kmutex_t *); 563 int sfmmu_page_spl_held(struct page *); 564 565 /* sfmmu internal locking operations - accessed directly */ 566 static void sfmmu_mlist_reloc_enter(page_t *, page_t *, 567 kmutex_t **, kmutex_t **); 568 static void sfmmu_mlist_reloc_exit(kmutex_t *, kmutex_t *); 569 static hatlock_t * 570 sfmmu_hat_enter(sfmmu_t *); 571 static hatlock_t * 572 sfmmu_hat_tryenter(sfmmu_t *); 573 static void sfmmu_hat_exit(hatlock_t *); 574 static void sfmmu_hat_lock_all(void); 575 static void sfmmu_hat_unlock_all(void); 576 static void sfmmu_ismhat_enter(sfmmu_t *, int); 577 static void sfmmu_ismhat_exit(sfmmu_t *, int); 578 579 /* 580 * Array of mutexes protecting a page's mapping list and p_nrm field. 581 * 582 * The hash function looks complicated, but is made up so that: 583 * 584 * "pp" not shifted, so adjacent pp values will hash to different cache lines 585 * (8 byte alignment * 8 bytes/mutes == 64 byte coherency subblock) 586 * 587 * "pp" >> mml_shift, incorporates more source bits into the hash result 588 * 589 * "& (mml_table_size - 1), should be faster than using remainder "%" 590 * 591 * Hopefully, mml_table, mml_table_size and mml_shift are all in the same 592 * cacheline, since they get declared next to each other below. We'll trust 593 * ld not to do something random. 594 */ 595 #ifdef DEBUG 596 int mlist_hash_debug = 0; 597 #define MLIST_HASH(pp) (mlist_hash_debug ? &mml_table[0] : \ 598 &mml_table[((uintptr_t)(pp) + \ 599 ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)]) 600 #else /* !DEBUG */ 601 #define MLIST_HASH(pp) &mml_table[ \ 602 ((uintptr_t)(pp) + ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)] 603 #endif /* !DEBUG */ 604 605 kmutex_t *mml_table; 606 uint_t mml_table_sz; /* must be a power of 2 */ 607 uint_t mml_shift; /* log2(mml_table_sz) + 3 for align */ 608 609 kpm_hlk_t *kpmp_table; 610 uint_t kpmp_table_sz; /* must be a power of 2 */ 611 uchar_t kpmp_shift; 612 613 kpm_shlk_t *kpmp_stable; 614 uint_t kpmp_stable_sz; /* must be a power of 2 */ 615 616 /* 617 * SPL_HASH was improved to avoid false cache line sharing 618 */ 619 #define SPL_TABLE_SIZE 128 620 #define SPL_MASK (SPL_TABLE_SIZE - 1) 621 #define SPL_SHIFT 7 /* log2(SPL_TABLE_SIZE) */ 622 623 #define SPL_INDEX(pp) \ 624 ((((uintptr_t)(pp) >> SPL_SHIFT) ^ \ 625 ((uintptr_t)(pp) >> (SPL_SHIFT << 1))) & \ 626 (SPL_TABLE_SIZE - 1)) 627 628 #define SPL_HASH(pp) \ 629 (&sfmmu_page_lock[SPL_INDEX(pp) & SPL_MASK].pad_mutex) 630 631 static pad_mutex_t sfmmu_page_lock[SPL_TABLE_SIZE]; 632 633 634 /* 635 * hat_unload_callback() will group together callbacks in order 636 * to avoid xt_sync() calls. This is the maximum size of the group. 637 */ 638 #define MAX_CB_ADDR 32 639 640 tte_t hw_tte; 641 static ulong_t sfmmu_dmr_maxbit = DMR_MAXBIT; 642 643 static char *mmu_ctx_kstat_names[] = { 644 "mmu_ctx_tsb_exceptions", 645 "mmu_ctx_tsb_raise_exception", 646 "mmu_ctx_wrap_around", 647 }; 648 649 /* 650 * Wrapper for vmem_xalloc since vmem_create only allows limited 651 * parameters for vm_source_alloc functions. This function allows us 652 * to specify alignment consistent with the size of the object being 653 * allocated. 654 */ 655 static void * 656 sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *vmp, size_t size, int vmflag) 657 { 658 return (vmem_xalloc(vmp, size, size, 0, 0, NULL, NULL, vmflag)); 659 } 660 661 /* Common code for setting tsb_alloc_hiwater. */ 662 #define SFMMU_SET_TSB_ALLOC_HIWATER(pages) tsb_alloc_hiwater = \ 663 ptob(pages) / tsb_alloc_hiwater_factor 664 665 /* 666 * Set tsb_max_growsize to allow at most all of physical memory to be mapped by 667 * a single TSB. physmem is the number of physical pages so we need physmem 8K 668 * TTEs to represent all those physical pages. We round this up by using 669 * 1<<highbit(). To figure out which size code to use, remember that the size 670 * code is just an amount to shift the smallest TSB size to get the size of 671 * this TSB. So we subtract that size, TSB_START_SIZE, from highbit() (or 672 * highbit() - 1) to get the size code for the smallest TSB that can represent 673 * all of physical memory, while erring on the side of too much. 674 * 675 * If the computed size code is less than the current tsb_max_growsize, we set 676 * tsb_max_growsize to the computed size code. In the case where the computed 677 * size code is greater than tsb_max_growsize, we have these restrictions that 678 * apply to increasing tsb_max_growsize: 679 * 1) TSBs can't grow larger than the TSB slab size 680 * 2) TSBs can't grow larger than UTSB_MAX_SZCODE. 681 */ 682 #define SFMMU_SET_TSB_MAX_GROWSIZE(pages) { \ 683 int i, szc; \ 684 \ 685 i = highbit(pages); \ 686 if ((1 << (i - 1)) == (pages)) \ 687 i--; /* 2^n case, round down */ \ 688 szc = i - TSB_START_SIZE; \ 689 if (szc < tsb_max_growsize) \ 690 tsb_max_growsize = szc; \ 691 else if ((szc > tsb_max_growsize) && \ 692 (szc <= tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT))) \ 693 tsb_max_growsize = MIN(szc, UTSB_MAX_SZCODE); \ 694 } 695 696 /* 697 * Given a pointer to an sfmmu and a TTE size code, return a pointer to the 698 * tsb_info which handles that TTE size. 699 */ 700 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) \ 701 (tsbinfop) = (sfmmup)->sfmmu_tsb; \ 702 ASSERT(sfmmu_hat_lock_held(sfmmup)); \ 703 if ((tte_szc) >= TTE4M) \ 704 (tsbinfop) = (tsbinfop)->tsb_next; 705 706 /* 707 * Return the number of mappings present in the HAT 708 * for a particular process and page size. 709 */ 710 #define SFMMU_TTE_CNT(sfmmup, szc) \ 711 (sfmmup)->sfmmu_iblk? \ 712 (sfmmup)->sfmmu_ismttecnt[(szc)] + \ 713 (sfmmup)->sfmmu_ttecnt[(szc)] : \ 714 (sfmmup)->sfmmu_ttecnt[(szc)]; 715 716 /* 717 * Macro to use to unload entries from the TSB. 718 * It has knowledge of which page sizes get replicated in the TSB 719 * and will call the appropriate unload routine for the appropriate size. 720 */ 721 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp) \ 722 { \ 723 int ttesz = get_hblk_ttesz(hmeblkp); \ 724 if (ttesz == TTE8K || ttesz == TTE4M) { \ 725 sfmmu_unload_tsb(sfmmup, addr, ttesz); \ 726 } else { \ 727 caddr_t sva = (caddr_t)get_hblk_base(hmeblkp); \ 728 caddr_t eva = sva + get_hblk_span(hmeblkp); \ 729 ASSERT(addr >= sva && addr < eva); \ 730 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); \ 731 } \ 732 } 733 734 735 /* Update tsb_alloc_hiwater after memory is configured. */ 736 /*ARGSUSED*/ 737 static void 738 sfmmu_update_tsb_post_add(void *arg, pgcnt_t delta_pages) 739 { 740 /* Assumes physmem has already been updated. */ 741 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 742 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 743 } 744 745 /* 746 * Update tsb_alloc_hiwater before memory is deleted. We'll do nothing here 747 * and update tsb_alloc_hiwater and tsb_max_growsize after the memory is 748 * deleted. 749 */ 750 /*ARGSUSED*/ 751 static int 752 sfmmu_update_tsb_pre_del(void *arg, pgcnt_t delta_pages) 753 { 754 return (0); 755 } 756 757 /* Update tsb_alloc_hiwater after memory fails to be unconfigured. */ 758 /*ARGSUSED*/ 759 static void 760 sfmmu_update_tsb_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 761 { 762 /* 763 * Whether the delete was cancelled or not, just go ahead and update 764 * tsb_alloc_hiwater and tsb_max_growsize. 765 */ 766 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 767 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 768 } 769 770 static kphysm_setup_vector_t sfmmu_update_tsb_vec = { 771 KPHYSM_SETUP_VECTOR_VERSION, /* version */ 772 sfmmu_update_tsb_post_add, /* post_add */ 773 sfmmu_update_tsb_pre_del, /* pre_del */ 774 sfmmu_update_tsb_post_del /* post_del */ 775 }; 776 777 778 /* 779 * HME_BLK HASH PRIMITIVES 780 */ 781 782 /* 783 * Enter a hme on the mapping list for page pp. 784 * When large pages are more prevalent in the system we might want to 785 * keep the mapping list in ascending order by the hment size. For now, 786 * small pages are more frequent, so don't slow it down. 787 */ 788 #define HME_ADD(hme, pp) \ 789 { \ 790 ASSERT(sfmmu_mlist_held(pp)); \ 791 \ 792 hme->hme_prev = NULL; \ 793 hme->hme_next = pp->p_mapping; \ 794 hme->hme_page = pp; \ 795 if (pp->p_mapping) { \ 796 ((struct sf_hment *)(pp->p_mapping))->hme_prev = hme;\ 797 ASSERT(pp->p_share > 0); \ 798 } else { \ 799 /* EMPTY */ \ 800 ASSERT(pp->p_share == 0); \ 801 } \ 802 pp->p_mapping = hme; \ 803 pp->p_share++; \ 804 } 805 806 /* 807 * Enter a hme on the mapping list for page pp. 808 * If we are unmapping a large translation, we need to make sure that the 809 * change is reflect in the corresponding bit of the p_index field. 810 */ 811 #define HME_SUB(hme, pp) \ 812 { \ 813 ASSERT(sfmmu_mlist_held(pp)); \ 814 ASSERT(hme->hme_page == pp || IS_PAHME(hme)); \ 815 \ 816 if (pp->p_mapping == NULL) { \ 817 panic("hme_remove - no mappings"); \ 818 } \ 819 \ 820 membar_stst(); /* ensure previous stores finish */ \ 821 \ 822 ASSERT(pp->p_share > 0); \ 823 pp->p_share--; \ 824 \ 825 if (hme->hme_prev) { \ 826 ASSERT(pp->p_mapping != hme); \ 827 ASSERT(hme->hme_prev->hme_page == pp || \ 828 IS_PAHME(hme->hme_prev)); \ 829 hme->hme_prev->hme_next = hme->hme_next; \ 830 } else { \ 831 ASSERT(pp->p_mapping == hme); \ 832 pp->p_mapping = hme->hme_next; \ 833 ASSERT((pp->p_mapping == NULL) ? \ 834 (pp->p_share == 0) : 1); \ 835 } \ 836 \ 837 if (hme->hme_next) { \ 838 ASSERT(hme->hme_next->hme_page == pp || \ 839 IS_PAHME(hme->hme_next)); \ 840 hme->hme_next->hme_prev = hme->hme_prev; \ 841 } \ 842 \ 843 /* zero out the entry */ \ 844 hme->hme_next = NULL; \ 845 hme->hme_prev = NULL; \ 846 hme->hme_page = NULL; \ 847 \ 848 if (hme_size(hme) > TTE8K) { \ 849 /* remove mappings for remainder of large pg */ \ 850 sfmmu_rm_large_mappings(pp, hme_size(hme)); \ 851 } \ 852 } 853 854 /* 855 * This function returns the hment given the hme_blk and a vaddr. 856 * It assumes addr has already been checked to belong to hme_blk's 857 * range. 858 */ 859 #define HBLKTOHME(hment, hmeblkp, addr) \ 860 { \ 861 int index; \ 862 HBLKTOHME_IDX(hment, hmeblkp, addr, index) \ 863 } 864 865 /* 866 * Version of HBLKTOHME that also returns the index in hmeblkp 867 * of the hment. 868 */ 869 #define HBLKTOHME_IDX(hment, hmeblkp, addr, idx) \ 870 { \ 871 ASSERT(in_hblk_range((hmeblkp), (addr))); \ 872 \ 873 if (get_hblk_ttesz(hmeblkp) == TTE8K) { \ 874 idx = (((uintptr_t)(addr) >> MMU_PAGESHIFT) & (NHMENTS-1)); \ 875 } else \ 876 idx = 0; \ 877 \ 878 (hment) = &(hmeblkp)->hblk_hme[idx]; \ 879 } 880 881 /* 882 * Disable any page sizes not supported by the CPU 883 */ 884 void 885 hat_init_pagesizes() 886 { 887 int i; 888 889 mmu_exported_page_sizes = 0; 890 for (i = TTE8K; i < max_mmu_page_sizes; i++) { 891 extern int disable_text_largepages; 892 extern int disable_initdata_largepages; 893 894 szc_2_userszc[i] = (uint_t)-1; 895 userszc_2_szc[i] = (uint_t)-1; 896 897 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) { 898 disable_large_pages |= (1 << i); 899 disable_ism_large_pages |= (1 << i); 900 disable_text_largepages |= (1 << i); 901 disable_initdata_largepages |= (1 << i); 902 } else { 903 szc_2_userszc[i] = mmu_exported_page_sizes; 904 userszc_2_szc[mmu_exported_page_sizes] = i; 905 mmu_exported_page_sizes++; 906 } 907 } 908 909 disable_auto_large_pages = disable_large_pages; 910 911 /* 912 * Initialize mmu-specific large page sizes. 913 */ 914 if (&mmu_large_pages_disabled) { 915 disable_large_pages |= mmu_large_pages_disabled(HAT_LOAD); 916 disable_ism_large_pages |= 917 mmu_large_pages_disabled(HAT_LOAD_SHARE); 918 disable_auto_large_pages |= 919 mmu_large_pages_disabled(HAT_LOAD_AUTOLPG); 920 } 921 922 disable_shm_large_pages = disable_auto_large_pages; 923 } 924 925 /* 926 * Initialize the hardware address translation structures. 927 */ 928 void 929 hat_init(void) 930 { 931 int i; 932 uint_t sz; 933 uint_t maxtsb; 934 size_t size; 935 936 hat_lock_init(); 937 hat_kstat_init(); 938 939 /* 940 * Hardware-only bits in a TTE 941 */ 942 MAKE_TTE_MASK(&hw_tte); 943 944 hat_init_pagesizes(); 945 946 /* Initialize the hash locks */ 947 for (i = 0; i < khmehash_num; i++) { 948 mutex_init(&khme_hash[i].hmehash_mutex, NULL, 949 MUTEX_DEFAULT, NULL); 950 } 951 for (i = 0; i < uhmehash_num; i++) { 952 mutex_init(&uhme_hash[i].hmehash_mutex, NULL, 953 MUTEX_DEFAULT, NULL); 954 } 955 khmehash_num--; /* make sure counter starts from 0 */ 956 uhmehash_num--; /* make sure counter starts from 0 */ 957 958 /* 959 * Allocate context domain structures. 960 * 961 * A platform may choose to modify max_mmu_ctxdoms in 962 * set_platform_defaults(). If a platform does not define 963 * a set_platform_defaults() or does not choose to modify 964 * max_mmu_ctxdoms, it gets one MMU context domain for every CPU. 965 * 966 * For sun4v, there will be one global context domain, this is to 967 * avoid the ldom cpu substitution problem. 968 * 969 * For all platforms that have CPUs sharing MMUs, this 970 * value must be defined. 971 */ 972 if (max_mmu_ctxdoms == 0) { 973 #ifndef sun4v 974 max_mmu_ctxdoms = max_ncpus; 975 #else /* sun4v */ 976 max_mmu_ctxdoms = 1; 977 #endif /* sun4v */ 978 } 979 980 size = max_mmu_ctxdoms * sizeof (mmu_ctx_t *); 981 mmu_ctxs_tbl = kmem_zalloc(size, KM_SLEEP); 982 983 /* mmu_ctx_t is 64 bytes aligned */ 984 mmuctxdom_cache = kmem_cache_create("mmuctxdom_cache", 985 sizeof (mmu_ctx_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 986 /* 987 * MMU context domain initialization for the Boot CPU. 988 * This needs the context domains array allocated above. 989 */ 990 mutex_enter(&cpu_lock); 991 sfmmu_cpu_init(CPU); 992 mutex_exit(&cpu_lock); 993 994 /* 995 * Intialize ism mapping list lock. 996 */ 997 998 mutex_init(&ism_mlist_lock, NULL, MUTEX_DEFAULT, NULL); 999 1000 /* 1001 * Each sfmmu structure carries an array of MMU context info 1002 * structures, one per context domain. The size of this array depends 1003 * on the maximum number of context domains. So, the size of the 1004 * sfmmu structure varies per platform. 1005 * 1006 * sfmmu is allocated from static arena, because trap 1007 * handler at TL > 0 is not allowed to touch kernel relocatable 1008 * memory. sfmmu's alignment is changed to 64 bytes from 1009 * default 8 bytes, as the lower 6 bits will be used to pass 1010 * pgcnt to vtag_flush_pgcnt_tl1. 1011 */ 1012 size = sizeof (sfmmu_t) + sizeof (sfmmu_ctx_t) * (max_mmu_ctxdoms - 1); 1013 1014 sfmmuid_cache = kmem_cache_create("sfmmuid_cache", size, 1015 64, sfmmu_idcache_constructor, sfmmu_idcache_destructor, 1016 NULL, NULL, static_arena, 0); 1017 1018 sfmmu_tsbinfo_cache = kmem_cache_create("sfmmu_tsbinfo_cache", 1019 sizeof (struct tsb_info), 0, NULL, NULL, NULL, NULL, NULL, 0); 1020 1021 /* 1022 * Since we only use the tsb8k cache to "borrow" pages for TSBs 1023 * from the heap when low on memory or when TSB_FORCEALLOC is 1024 * specified, don't use magazines to cache them--we want to return 1025 * them to the system as quickly as possible. 1026 */ 1027 sfmmu_tsb8k_cache = kmem_cache_create("sfmmu_tsb8k_cache", 1028 MMU_PAGESIZE, MMU_PAGESIZE, NULL, NULL, NULL, NULL, 1029 static_arena, KMC_NOMAGAZINE); 1030 1031 /* 1032 * Set tsb_alloc_hiwater to 1/tsb_alloc_hiwater_factor of physical 1033 * memory, which corresponds to the old static reserve for TSBs. 1034 * tsb_alloc_hiwater_factor defaults to 32. This caps the amount of 1035 * memory we'll allocate for TSB slabs; beyond this point TSB 1036 * allocations will be taken from the kernel heap (via 1037 * sfmmu_tsb8k_cache) and will be throttled as would any other kmem 1038 * consumer. 1039 */ 1040 if (tsb_alloc_hiwater_factor == 0) { 1041 tsb_alloc_hiwater_factor = TSB_ALLOC_HIWATER_FACTOR_DEFAULT; 1042 } 1043 SFMMU_SET_TSB_ALLOC_HIWATER(physmem); 1044 1045 /* Set tsb_max_growsize. */ 1046 SFMMU_SET_TSB_MAX_GROWSIZE(physmem); 1047 1048 /* 1049 * On smaller memory systems, allocate TSB memory in smaller chunks 1050 * than the default 4M slab size. We also honor disable_large_pages 1051 * here. 1052 * 1053 * The trap handlers need to be patched with the final slab shift, 1054 * since they need to be able to construct the TSB pointer at runtime. 1055 */ 1056 if (tsb_max_growsize <= TSB_512K_SZCODE) 1057 tsb_slab_ttesz = TTE512K; 1058 1059 for (sz = tsb_slab_ttesz; sz > 0; sz--) { 1060 if (!(disable_large_pages & (1 << sz))) 1061 break; 1062 } 1063 1064 tsb_slab_ttesz = sz; 1065 tsb_slab_shift = MMU_PAGESHIFT + (sz << 1) + sz; 1066 tsb_slab_size = 1 << tsb_slab_shift; 1067 tsb_slab_mask = (1 << (tsb_slab_shift - MMU_PAGESHIFT)) - 1; 1068 1069 maxtsb = tsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT); 1070 if (tsb_max_growsize > maxtsb) 1071 tsb_max_growsize = maxtsb; 1072 1073 /* 1074 * Set up memory callback to update tsb_alloc_hiwater and 1075 * tsb_max_growsize. 1076 */ 1077 i = kphysm_setup_func_register(&sfmmu_update_tsb_vec, (void *) 0); 1078 ASSERT(i == 0); 1079 1080 /* 1081 * kmem_tsb_arena is the source from which large TSB slabs are 1082 * drawn. The quantum of this arena corresponds to the largest 1083 * TSB size we can dynamically allocate for user processes. 1084 * Currently it must also be a supported page size since we 1085 * use exactly one translation entry to map each slab page. 1086 * 1087 * The per-lgroup kmem_tsb_default_arena arenas are the arenas from 1088 * which most TSBs are allocated. Since most TSB allocations are 1089 * typically 8K we have a kmem cache we stack on top of each 1090 * kmem_tsb_default_arena to speed up those allocations. 1091 * 1092 * Note the two-level scheme of arenas is required only 1093 * because vmem_create doesn't allow us to specify alignment 1094 * requirements. If this ever changes the code could be 1095 * simplified to use only one level of arenas. 1096 */ 1097 kmem_tsb_arena = vmem_create("kmem_tsb", NULL, 0, tsb_slab_size, 1098 sfmmu_vmem_xalloc_aligned_wrapper, vmem_xfree, heap_arena, 1099 0, VM_SLEEP); 1100 1101 if (tsb_lgrp_affinity) { 1102 char s[50]; 1103 for (i = 0; i < NLGRPS_MAX; i++) { 1104 (void) sprintf(s, "kmem_tsb_lgrp%d", i); 1105 kmem_tsb_default_arena[i] = 1106 vmem_create(s, NULL, 0, PAGESIZE, 1107 sfmmu_tsb_segkmem_alloc, sfmmu_tsb_segkmem_free, 1108 kmem_tsb_arena, 0, VM_SLEEP | VM_BESTFIT); 1109 (void) sprintf(s, "sfmmu_tsb_lgrp%d_cache", i); 1110 sfmmu_tsb_cache[i] = kmem_cache_create(s, PAGESIZE, 1111 PAGESIZE, NULL, NULL, NULL, NULL, 1112 kmem_tsb_default_arena[i], 0); 1113 } 1114 } else { 1115 kmem_tsb_default_arena[0] = vmem_create("kmem_tsb_default", 1116 NULL, 0, PAGESIZE, sfmmu_tsb_segkmem_alloc, 1117 sfmmu_tsb_segkmem_free, kmem_tsb_arena, 0, 1118 VM_SLEEP | VM_BESTFIT); 1119 1120 sfmmu_tsb_cache[0] = kmem_cache_create("sfmmu_tsb_cache", 1121 PAGESIZE, PAGESIZE, NULL, NULL, NULL, NULL, 1122 kmem_tsb_default_arena[0], 0); 1123 } 1124 1125 sfmmu8_cache = kmem_cache_create("sfmmu8_cache", HME8BLK_SZ, 1126 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1127 sfmmu_hblkcache_destructor, 1128 sfmmu_hblkcache_reclaim, (void *)HME8BLK_SZ, 1129 hat_memload_arena, KMC_NOHASH); 1130 1131 hat_memload1_arena = vmem_create("hat_memload1", NULL, 0, PAGESIZE, 1132 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP); 1133 1134 sfmmu1_cache = kmem_cache_create("sfmmu1_cache", HME1BLK_SZ, 1135 HMEBLK_ALIGN, sfmmu_hblkcache_constructor, 1136 sfmmu_hblkcache_destructor, 1137 NULL, (void *)HME1BLK_SZ, 1138 hat_memload1_arena, KMC_NOHASH); 1139 1140 pa_hment_cache = kmem_cache_create("pa_hment_cache", PAHME_SZ, 1141 0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH); 1142 1143 ism_blk_cache = kmem_cache_create("ism_blk_cache", 1144 sizeof (ism_blk_t), ecache_alignsize, NULL, NULL, 1145 NULL, NULL, static_arena, KMC_NOHASH); 1146 1147 ism_ment_cache = kmem_cache_create("ism_ment_cache", 1148 sizeof (ism_ment_t), 0, NULL, NULL, 1149 NULL, NULL, NULL, 0); 1150 1151 /* 1152 * We grab the first hat for the kernel, 1153 */ 1154 AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER); 1155 kas.a_hat = hat_alloc(&kas); 1156 AS_LOCK_EXIT(&kas, &kas.a_lock); 1157 1158 /* 1159 * Initialize hblk_reserve. 1160 */ 1161 ((struct hme_blk *)hblk_reserve)->hblk_nextpa = 1162 va_to_pa((caddr_t)hblk_reserve); 1163 1164 #ifndef UTSB_PHYS 1165 /* 1166 * Reserve some kernel virtual address space for the locked TTEs 1167 * that allow us to probe the TSB from TL>0. 1168 */ 1169 utsb_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1170 0, 0, NULL, NULL, VM_SLEEP); 1171 utsb4m_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size, 1172 0, 0, NULL, NULL, VM_SLEEP); 1173 #endif 1174 1175 #ifdef VAC 1176 /* 1177 * The big page VAC handling code assumes VAC 1178 * will not be bigger than the smallest big 1179 * page- which is 64K. 1180 */ 1181 if (TTEPAGES(TTE64K) < CACHE_NUM_COLOR) { 1182 cmn_err(CE_PANIC, "VAC too big!"); 1183 } 1184 #endif 1185 1186 (void) xhat_init(); 1187 1188 uhme_hash_pa = va_to_pa(uhme_hash); 1189 khme_hash_pa = va_to_pa(khme_hash); 1190 1191 /* 1192 * Initialize relocation locks. kpr_suspendlock is held 1193 * at PIL_MAX to prevent interrupts from pinning the holder 1194 * of a suspended TTE which may access it leading to a 1195 * deadlock condition. 1196 */ 1197 mutex_init(&kpr_mutex, NULL, MUTEX_DEFAULT, NULL); 1198 mutex_init(&kpr_suspendlock, NULL, MUTEX_SPIN, (void *)PIL_MAX); 1199 } 1200 1201 /* 1202 * Initialize locking for the hat layer, called early during boot. 1203 */ 1204 static void 1205 hat_lock_init() 1206 { 1207 int i; 1208 1209 /* 1210 * initialize the array of mutexes protecting a page's mapping 1211 * list and p_nrm field. 1212 */ 1213 for (i = 0; i < mml_table_sz; i++) 1214 mutex_init(&mml_table[i], NULL, MUTEX_DEFAULT, NULL); 1215 1216 if (kpm_enable) { 1217 for (i = 0; i < kpmp_table_sz; i++) { 1218 mutex_init(&kpmp_table[i].khl_mutex, NULL, 1219 MUTEX_DEFAULT, NULL); 1220 } 1221 } 1222 1223 /* 1224 * Initialize array of mutex locks that protects sfmmu fields and 1225 * TSB lists. 1226 */ 1227 for (i = 0; i < SFMMU_NUM_LOCK; i++) 1228 mutex_init(HATLOCK_MUTEXP(&hat_lock[i]), NULL, MUTEX_DEFAULT, 1229 NULL); 1230 } 1231 1232 extern caddr_t kmem64_base, kmem64_end; 1233 1234 #define SFMMU_KERNEL_MAXVA \ 1235 (kmem64_base ? (uintptr_t)kmem64_end : (SYSLIMIT)) 1236 1237 /* 1238 * Allocate a hat structure. 1239 * Called when an address space first uses a hat. 1240 */ 1241 struct hat * 1242 hat_alloc(struct as *as) 1243 { 1244 sfmmu_t *sfmmup; 1245 int i; 1246 uint64_t cnum; 1247 extern uint_t get_color_start(struct as *); 1248 1249 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 1250 sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP); 1251 sfmmup->sfmmu_as = as; 1252 sfmmup->sfmmu_flags = 0; 1253 LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock); 1254 1255 if (as == &kas) { 1256 ksfmmup = sfmmup; 1257 sfmmup->sfmmu_cext = 0; 1258 cnum = KCONTEXT; 1259 1260 sfmmup->sfmmu_clrstart = 0; 1261 sfmmup->sfmmu_tsb = NULL; 1262 /* 1263 * hat_kern_setup() will call sfmmu_init_ktsbinfo() 1264 * to setup tsb_info for ksfmmup. 1265 */ 1266 } else { 1267 1268 /* 1269 * Just set to invalid ctx. When it faults, it will 1270 * get a valid ctx. This would avoid the situation 1271 * where we get a ctx, but it gets stolen and then 1272 * we fault when we try to run and so have to get 1273 * another ctx. 1274 */ 1275 sfmmup->sfmmu_cext = 0; 1276 cnum = INVALID_CONTEXT; 1277 1278 /* initialize original physical page coloring bin */ 1279 sfmmup->sfmmu_clrstart = get_color_start(as); 1280 #ifdef DEBUG 1281 if (tsb_random_size) { 1282 uint32_t randval = (uint32_t)gettick() >> 4; 1283 int size = randval % (tsb_max_growsize + 1); 1284 1285 /* chose a random tsb size for stress testing */ 1286 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size, 1287 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1288 } else 1289 #endif /* DEBUG */ 1290 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, 1291 default_tsb_size, 1292 TSB8K|TSB64K|TSB512K, 0, sfmmup); 1293 sfmmup->sfmmu_flags = HAT_SWAPPED; 1294 ASSERT(sfmmup->sfmmu_tsb != NULL); 1295 } 1296 1297 ASSERT(max_mmu_ctxdoms > 0); 1298 for (i = 0; i < max_mmu_ctxdoms; i++) { 1299 sfmmup->sfmmu_ctxs[i].cnum = cnum; 1300 sfmmup->sfmmu_ctxs[i].gnum = 0; 1301 } 1302 1303 sfmmu_setup_tsbinfo(sfmmup); 1304 for (i = 0; i < max_mmu_page_sizes; i++) { 1305 sfmmup->sfmmu_ttecnt[i] = 0; 1306 sfmmup->sfmmu_ismttecnt[i] = 0; 1307 sfmmup->sfmmu_pgsz[i] = TTE8K; 1308 } 1309 1310 sfmmup->sfmmu_iblk = NULL; 1311 sfmmup->sfmmu_ismhat = 0; 1312 sfmmup->sfmmu_ismblkpa = (uint64_t)-1; 1313 if (sfmmup == ksfmmup) { 1314 CPUSET_ALL(sfmmup->sfmmu_cpusran); 1315 } else { 1316 CPUSET_ZERO(sfmmup->sfmmu_cpusran); 1317 } 1318 sfmmup->sfmmu_free = 0; 1319 sfmmup->sfmmu_rmstat = 0; 1320 sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart; 1321 sfmmup->sfmmu_xhat_provider = NULL; 1322 cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL); 1323 return (sfmmup); 1324 } 1325 1326 /* 1327 * Create per-MMU context domain kstats for a given MMU ctx. 1328 */ 1329 static void 1330 sfmmu_mmu_kstat_create(mmu_ctx_t *mmu_ctxp) 1331 { 1332 mmu_ctx_stat_t stat; 1333 kstat_t *mmu_kstat; 1334 1335 ASSERT(MUTEX_HELD(&cpu_lock)); 1336 ASSERT(mmu_ctxp->mmu_kstat == NULL); 1337 1338 mmu_kstat = kstat_create("unix", mmu_ctxp->mmu_idx, "mmu_ctx", 1339 "hat", KSTAT_TYPE_NAMED, MMU_CTX_NUM_STATS, KSTAT_FLAG_VIRTUAL); 1340 1341 if (mmu_kstat == NULL) { 1342 cmn_err(CE_WARN, "kstat_create for MMU %d failed", 1343 mmu_ctxp->mmu_idx); 1344 } else { 1345 mmu_kstat->ks_data = mmu_ctxp->mmu_kstat_data; 1346 for (stat = 0; stat < MMU_CTX_NUM_STATS; stat++) 1347 kstat_named_init(&mmu_ctxp->mmu_kstat_data[stat], 1348 mmu_ctx_kstat_names[stat], KSTAT_DATA_INT64); 1349 mmu_ctxp->mmu_kstat = mmu_kstat; 1350 kstat_install(mmu_kstat); 1351 } 1352 } 1353 1354 /* 1355 * plat_cpuid_to_mmu_ctx_info() is a platform interface that returns MMU 1356 * context domain information for a given CPU. If a platform does not 1357 * specify that interface, then the function below is used instead to return 1358 * default information. The defaults are as follows: 1359 * 1360 * - For sun4u systems there's one MMU context domain per CPU. 1361 * This default is used by all sun4u systems except OPL. OPL systems 1362 * provide platform specific interface to map CPU ids to MMU ids 1363 * because on OPL more than 1 CPU shares a single MMU. 1364 * Note that on sun4v, there is one global context domain for 1365 * the entire system. This is to avoid running into potential problem 1366 * with ldom physical cpu substitution feature. 1367 * - The number of MMU context IDs supported on any CPU in the 1368 * system is 8K. 1369 */ 1370 /*ARGSUSED*/ 1371 static void 1372 sfmmu_cpuid_to_mmu_ctx_info(processorid_t cpuid, mmu_ctx_info_t *infop) 1373 { 1374 infop->mmu_nctxs = nctxs; 1375 #ifndef sun4v 1376 infop->mmu_idx = cpu[cpuid]->cpu_seqid; 1377 #else /* sun4v */ 1378 infop->mmu_idx = 0; 1379 #endif /* sun4v */ 1380 } 1381 1382 /* 1383 * Called during CPU initialization to set the MMU context-related information 1384 * for a CPU. 1385 * 1386 * cpu_lock serializes accesses to mmu_ctxs and mmu_saved_gnum. 1387 */ 1388 void 1389 sfmmu_cpu_init(cpu_t *cp) 1390 { 1391 mmu_ctx_info_t info; 1392 mmu_ctx_t *mmu_ctxp; 1393 1394 ASSERT(MUTEX_HELD(&cpu_lock)); 1395 1396 if (&plat_cpuid_to_mmu_ctx_info == NULL) 1397 sfmmu_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1398 else 1399 plat_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); 1400 1401 ASSERT(info.mmu_idx < max_mmu_ctxdoms); 1402 1403 if ((mmu_ctxp = mmu_ctxs_tbl[info.mmu_idx]) == NULL) { 1404 /* Each mmu_ctx is cacheline aligned. */ 1405 mmu_ctxp = kmem_cache_alloc(mmuctxdom_cache, KM_SLEEP); 1406 bzero(mmu_ctxp, sizeof (mmu_ctx_t)); 1407 1408 mutex_init(&mmu_ctxp->mmu_lock, NULL, MUTEX_SPIN, 1409 (void *)ipltospl(DISP_LEVEL)); 1410 mmu_ctxp->mmu_idx = info.mmu_idx; 1411 mmu_ctxp->mmu_nctxs = info.mmu_nctxs; 1412 /* 1413 * Globally for lifetime of a system, 1414 * gnum must always increase. 1415 * mmu_saved_gnum is protected by the cpu_lock. 1416 */ 1417 mmu_ctxp->mmu_gnum = mmu_saved_gnum + 1; 1418 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 1419 1420 sfmmu_mmu_kstat_create(mmu_ctxp); 1421 1422 mmu_ctxs_tbl[info.mmu_idx] = mmu_ctxp; 1423 } else { 1424 ASSERT(mmu_ctxp->mmu_idx == info.mmu_idx); 1425 } 1426 1427 /* 1428 * The mmu_lock is acquired here to prevent races with 1429 * the wrap-around code. 1430 */ 1431 mutex_enter(&mmu_ctxp->mmu_lock); 1432 1433 1434 mmu_ctxp->mmu_ncpus++; 1435 CPUSET_ADD(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1436 CPU_MMU_IDX(cp) = info.mmu_idx; 1437 CPU_MMU_CTXP(cp) = mmu_ctxp; 1438 1439 mutex_exit(&mmu_ctxp->mmu_lock); 1440 } 1441 1442 /* 1443 * Called to perform MMU context-related cleanup for a CPU. 1444 */ 1445 void 1446 sfmmu_cpu_cleanup(cpu_t *cp) 1447 { 1448 mmu_ctx_t *mmu_ctxp; 1449 1450 ASSERT(MUTEX_HELD(&cpu_lock)); 1451 1452 mmu_ctxp = CPU_MMU_CTXP(cp); 1453 ASSERT(mmu_ctxp != NULL); 1454 1455 /* 1456 * The mmu_lock is acquired here to prevent races with 1457 * the wrap-around code. 1458 */ 1459 mutex_enter(&mmu_ctxp->mmu_lock); 1460 1461 CPU_MMU_CTXP(cp) = NULL; 1462 1463 CPUSET_DEL(mmu_ctxp->mmu_cpuset, cp->cpu_id); 1464 if (--mmu_ctxp->mmu_ncpus == 0) { 1465 mmu_ctxs_tbl[mmu_ctxp->mmu_idx] = NULL; 1466 mutex_exit(&mmu_ctxp->mmu_lock); 1467 mutex_destroy(&mmu_ctxp->mmu_lock); 1468 1469 if (mmu_ctxp->mmu_kstat) 1470 kstat_delete(mmu_ctxp->mmu_kstat); 1471 1472 /* mmu_saved_gnum is protected by the cpu_lock. */ 1473 if (mmu_saved_gnum < mmu_ctxp->mmu_gnum) 1474 mmu_saved_gnum = mmu_ctxp->mmu_gnum; 1475 1476 kmem_cache_free(mmuctxdom_cache, mmu_ctxp); 1477 1478 return; 1479 } 1480 1481 mutex_exit(&mmu_ctxp->mmu_lock); 1482 } 1483 1484 /* 1485 * Hat_setup, makes an address space context the current active one. 1486 * In sfmmu this translates to setting the secondary context with the 1487 * corresponding context. 1488 */ 1489 void 1490 hat_setup(struct hat *sfmmup, int allocflag) 1491 { 1492 hatlock_t *hatlockp; 1493 1494 /* Init needs some special treatment. */ 1495 if (allocflag == HAT_INIT) { 1496 /* 1497 * Make sure that we have 1498 * 1. a TSB 1499 * 2. a valid ctx that doesn't get stolen after this point. 1500 */ 1501 hatlockp = sfmmu_hat_enter(sfmmup); 1502 1503 /* 1504 * Swap in the TSB. hat_init() allocates tsbinfos without 1505 * TSBs, but we need one for init, since the kernel does some 1506 * special things to set up its stack and needs the TSB to 1507 * resolve page faults. 1508 */ 1509 sfmmu_tsb_swapin(sfmmup, hatlockp); 1510 1511 sfmmu_get_ctx(sfmmup); 1512 1513 sfmmu_hat_exit(hatlockp); 1514 } else { 1515 ASSERT(allocflag == HAT_ALLOC); 1516 1517 hatlockp = sfmmu_hat_enter(sfmmup); 1518 kpreempt_disable(); 1519 1520 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); 1521 1522 /* 1523 * sfmmu_setctx_sec takes <pgsz|cnum> as a parameter, 1524 * pagesize bits don't matter in this case since we are passing 1525 * INVALID_CONTEXT to it. 1526 */ 1527 sfmmu_setctx_sec(INVALID_CONTEXT); 1528 sfmmu_clear_utsbinfo(); 1529 1530 kpreempt_enable(); 1531 sfmmu_hat_exit(hatlockp); 1532 } 1533 } 1534 1535 /* 1536 * Free all the translation resources for the specified address space. 1537 * Called from as_free when an address space is being destroyed. 1538 */ 1539 void 1540 hat_free_start(struct hat *sfmmup) 1541 { 1542 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 1543 ASSERT(sfmmup != ksfmmup); 1544 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1545 1546 sfmmup->sfmmu_free = 1; 1547 } 1548 1549 void 1550 hat_free_end(struct hat *sfmmup) 1551 { 1552 int i; 1553 1554 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1555 if (sfmmup->sfmmu_ismhat) { 1556 for (i = 0; i < mmu_page_sizes; i++) { 1557 sfmmup->sfmmu_ttecnt[i] = 0; 1558 sfmmup->sfmmu_ismttecnt[i] = 0; 1559 } 1560 } else { 1561 /* EMPTY */ 1562 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 1563 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 1564 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 1565 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 1566 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 1567 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 1568 } 1569 1570 if (sfmmup->sfmmu_rmstat) { 1571 hat_freestat(sfmmup->sfmmu_as, NULL); 1572 } 1573 1574 while (sfmmup->sfmmu_tsb != NULL) { 1575 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next; 1576 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb); 1577 sfmmup->sfmmu_tsb = next; 1578 } 1579 sfmmu_free_sfmmu(sfmmup); 1580 1581 kmem_cache_free(sfmmuid_cache, sfmmup); 1582 } 1583 1584 /* 1585 * Set up any translation structures, for the specified address space, 1586 * that are needed or preferred when the process is being swapped in. 1587 */ 1588 /* ARGSUSED */ 1589 void 1590 hat_swapin(struct hat *hat) 1591 { 1592 ASSERT(hat->sfmmu_xhat_provider == NULL); 1593 } 1594 1595 /* 1596 * Free all of the translation resources, for the specified address space, 1597 * that can be freed while the process is swapped out. Called from as_swapout. 1598 * Also, free up the ctx that this process was using. 1599 */ 1600 void 1601 hat_swapout(struct hat *sfmmup) 1602 { 1603 struct hmehash_bucket *hmebp; 1604 struct hme_blk *hmeblkp; 1605 struct hme_blk *pr_hblk = NULL; 1606 struct hme_blk *nx_hblk; 1607 int i; 1608 uint64_t hblkpa, prevpa, nx_pa; 1609 struct hme_blk *list = NULL; 1610 hatlock_t *hatlockp; 1611 struct tsb_info *tsbinfop; 1612 struct free_tsb { 1613 struct free_tsb *next; 1614 struct tsb_info *tsbinfop; 1615 }; /* free list of TSBs */ 1616 struct free_tsb *freelist, *last, *next; 1617 1618 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 1619 SFMMU_STAT(sf_swapout); 1620 1621 /* 1622 * There is no way to go from an as to all its translations in sfmmu. 1623 * Here is one of the times when we take the big hit and traverse 1624 * the hash looking for hme_blks to free up. Not only do we free up 1625 * this as hme_blks but all those that are free. We are obviously 1626 * swapping because we need memory so let's free up as much 1627 * as we can. 1628 * 1629 * Note that we don't flush TLB/TSB here -- it's not necessary 1630 * because: 1631 * 1) we free the ctx we're using and throw away the TSB(s); 1632 * 2) processes aren't runnable while being swapped out. 1633 */ 1634 ASSERT(sfmmup != KHATID); 1635 for (i = 0; i <= UHMEHASH_SZ; i++) { 1636 hmebp = &uhme_hash[i]; 1637 SFMMU_HASH_LOCK(hmebp); 1638 hmeblkp = hmebp->hmeblkp; 1639 hblkpa = hmebp->hmeh_nextpa; 1640 prevpa = 0; 1641 pr_hblk = NULL; 1642 while (hmeblkp) { 1643 1644 ASSERT(!hmeblkp->hblk_xhat_bit); 1645 1646 if ((hmeblkp->hblk_tag.htag_id == sfmmup) && 1647 !hmeblkp->hblk_shw_bit && !hmeblkp->hblk_lckcnt) { 1648 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 1649 (caddr_t)get_hblk_base(hmeblkp), 1650 get_hblk_endaddr(hmeblkp), 1651 NULL, HAT_UNLOAD); 1652 } 1653 nx_hblk = hmeblkp->hblk_next; 1654 nx_pa = hmeblkp->hblk_nextpa; 1655 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 1656 ASSERT(!hmeblkp->hblk_lckcnt); 1657 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 1658 prevpa, pr_hblk); 1659 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 1660 } else { 1661 pr_hblk = hmeblkp; 1662 prevpa = hblkpa; 1663 } 1664 hmeblkp = nx_hblk; 1665 hblkpa = nx_pa; 1666 } 1667 SFMMU_HASH_UNLOCK(hmebp); 1668 } 1669 1670 sfmmu_hblks_list_purge(&list); 1671 1672 /* 1673 * Now free up the ctx so that others can reuse it. 1674 */ 1675 hatlockp = sfmmu_hat_enter(sfmmup); 1676 1677 sfmmu_invalidate_ctx(sfmmup); 1678 1679 /* 1680 * Free TSBs, but not tsbinfos, and set SWAPPED flag. 1681 * If TSBs were never swapped in, just return. 1682 * This implies that we don't support partial swapping 1683 * of TSBs -- either all are swapped out, or none are. 1684 * 1685 * We must hold the HAT lock here to prevent racing with another 1686 * thread trying to unmap TTEs from the TSB or running the post- 1687 * relocator after relocating the TSB's memory. Unfortunately, we 1688 * can't free memory while holding the HAT lock or we could 1689 * deadlock, so we build a list of TSBs to be freed after marking 1690 * the tsbinfos as swapped out and free them after dropping the 1691 * lock. 1692 */ 1693 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 1694 sfmmu_hat_exit(hatlockp); 1695 return; 1696 } 1697 1698 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED); 1699 last = freelist = NULL; 1700 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 1701 tsbinfop = tsbinfop->tsb_next) { 1702 ASSERT((tsbinfop->tsb_flags & TSB_SWAPPED) == 0); 1703 1704 /* 1705 * Cast the TSB into a struct free_tsb and put it on the free 1706 * list. 1707 */ 1708 if (freelist == NULL) { 1709 last = freelist = (struct free_tsb *)tsbinfop->tsb_va; 1710 } else { 1711 last->next = (struct free_tsb *)tsbinfop->tsb_va; 1712 last = last->next; 1713 } 1714 last->next = NULL; 1715 last->tsbinfop = tsbinfop; 1716 tsbinfop->tsb_flags |= TSB_SWAPPED; 1717 /* 1718 * Zero out the TTE to clear the valid bit. 1719 * Note we can't use a value like 0xbad because we want to 1720 * ensure diagnostic bits are NEVER set on TTEs that might 1721 * be loaded. The intent is to catch any invalid access 1722 * to the swapped TSB, such as a thread running with a valid 1723 * context without first calling sfmmu_tsb_swapin() to 1724 * allocate TSB memory. 1725 */ 1726 tsbinfop->tsb_tte.ll = 0; 1727 } 1728 1729 /* Now we can drop the lock and free the TSB memory. */ 1730 sfmmu_hat_exit(hatlockp); 1731 for (; freelist != NULL; freelist = next) { 1732 next = freelist->next; 1733 sfmmu_tsb_free(freelist->tsbinfop); 1734 } 1735 } 1736 1737 /* 1738 * Duplicate the translations of an as into another newas 1739 */ 1740 /* ARGSUSED */ 1741 int 1742 hat_dup(struct hat *hat, struct hat *newhat, caddr_t addr, size_t len, 1743 uint_t flag) 1744 { 1745 ASSERT(hat->sfmmu_xhat_provider == NULL); 1746 ASSERT((flag == 0) || (flag == HAT_DUP_ALL) || (flag == HAT_DUP_COW)); 1747 1748 if (flag == HAT_DUP_COW) { 1749 panic("hat_dup: HAT_DUP_COW not supported"); 1750 } 1751 return (0); 1752 } 1753 1754 /* 1755 * Set up addr to map to page pp with protection prot. 1756 * As an optimization we also load the TSB with the 1757 * corresponding tte but it is no big deal if the tte gets kicked out. 1758 */ 1759 void 1760 hat_memload(struct hat *hat, caddr_t addr, struct page *pp, 1761 uint_t attr, uint_t flags) 1762 { 1763 tte_t tte; 1764 1765 1766 ASSERT(hat != NULL); 1767 ASSERT(PAGE_LOCKED(pp)); 1768 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 1769 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1770 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1771 1772 if (PP_ISFREE(pp)) { 1773 panic("hat_memload: loading a mapping to free page %p", 1774 (void *)pp); 1775 } 1776 1777 if (hat->sfmmu_xhat_provider) { 1778 XHAT_MEMLOAD(hat, addr, pp, attr, flags); 1779 return; 1780 } 1781 1782 ASSERT((hat == ksfmmup) || 1783 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1784 1785 if (flags & ~SFMMU_LOAD_ALLFLAG) 1786 cmn_err(CE_NOTE, "hat_memload: unsupported flags %d", 1787 flags & ~SFMMU_LOAD_ALLFLAG); 1788 1789 if (hat->sfmmu_rmstat) 1790 hat_resvstat(MMU_PAGESIZE, hat->sfmmu_as, addr); 1791 1792 #if defined(SF_ERRATA_57) 1793 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1794 (addr < errata57_limit) && (attr & PROT_EXEC) && 1795 !(flags & HAT_LOAD_SHARE)) { 1796 cmn_err(CE_WARN, "hat_memload: illegal attempt to make user " 1797 " page executable"); 1798 attr &= ~PROT_EXEC; 1799 } 1800 #endif 1801 1802 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 1803 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, flags); 1804 1805 /* 1806 * Check TSB and TLB page sizes. 1807 */ 1808 if ((flags & HAT_LOAD_SHARE) == 0) { 1809 sfmmu_check_page_sizes(hat, 1); 1810 } 1811 } 1812 1813 /* 1814 * hat_devload can be called to map real memory (e.g. 1815 * /dev/kmem) and even though hat_devload will determine pf is 1816 * for memory, it will be unable to get a shared lock on the 1817 * page (because someone else has it exclusively) and will 1818 * pass dp = NULL. If tteload doesn't get a non-NULL 1819 * page pointer it can't cache memory. 1820 */ 1821 void 1822 hat_devload(struct hat *hat, caddr_t addr, size_t len, pfn_t pfn, 1823 uint_t attr, int flags) 1824 { 1825 tte_t tte; 1826 struct page *pp = NULL; 1827 int use_lgpg = 0; 1828 1829 ASSERT(hat != NULL); 1830 1831 if (hat->sfmmu_xhat_provider) { 1832 XHAT_DEVLOAD(hat, addr, len, pfn, attr, flags); 1833 return; 1834 } 1835 1836 ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG)); 1837 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 1838 ASSERT((hat == ksfmmup) || 1839 AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock)); 1840 if (len == 0) 1841 panic("hat_devload: zero len"); 1842 if (flags & ~SFMMU_LOAD_ALLFLAG) 1843 cmn_err(CE_NOTE, "hat_devload: unsupported flags %d", 1844 flags & ~SFMMU_LOAD_ALLFLAG); 1845 1846 #if defined(SF_ERRATA_57) 1847 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 1848 (addr < errata57_limit) && (attr & PROT_EXEC) && 1849 !(flags & HAT_LOAD_SHARE)) { 1850 cmn_err(CE_WARN, "hat_devload: illegal attempt to make user " 1851 " page executable"); 1852 attr &= ~PROT_EXEC; 1853 } 1854 #endif 1855 1856 /* 1857 * If it's a memory page find its pp 1858 */ 1859 if (!(flags & HAT_LOAD_NOCONSIST) && pf_is_memory(pfn)) { 1860 pp = page_numtopp_nolock(pfn); 1861 if (pp == NULL) { 1862 flags |= HAT_LOAD_NOCONSIST; 1863 } else { 1864 if (PP_ISFREE(pp)) { 1865 panic("hat_memload: loading " 1866 "a mapping to free page %p", 1867 (void *)pp); 1868 } 1869 if (!PAGE_LOCKED(pp) && !PP_ISNORELOC(pp)) { 1870 panic("hat_memload: loading a mapping " 1871 "to unlocked relocatable page %p", 1872 (void *)pp); 1873 } 1874 ASSERT(len == MMU_PAGESIZE); 1875 } 1876 } 1877 1878 if (hat->sfmmu_rmstat) 1879 hat_resvstat(len, hat->sfmmu_as, addr); 1880 1881 if (flags & HAT_LOAD_NOCONSIST) { 1882 attr |= SFMMU_UNCACHEVTTE; 1883 use_lgpg = 1; 1884 } 1885 if (!pf_is_memory(pfn)) { 1886 attr |= SFMMU_UNCACHEPTTE | HAT_NOSYNC; 1887 use_lgpg = 1; 1888 switch (attr & HAT_ORDER_MASK) { 1889 case HAT_STRICTORDER: 1890 case HAT_UNORDERED_OK: 1891 /* 1892 * we set the side effect bit for all non 1893 * memory mappings unless merging is ok 1894 */ 1895 attr |= SFMMU_SIDEFFECT; 1896 break; 1897 case HAT_MERGING_OK: 1898 case HAT_LOADCACHING_OK: 1899 case HAT_STORECACHING_OK: 1900 break; 1901 default: 1902 panic("hat_devload: bad attr"); 1903 break; 1904 } 1905 } 1906 while (len) { 1907 if (!use_lgpg) { 1908 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1909 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1910 flags); 1911 len -= MMU_PAGESIZE; 1912 addr += MMU_PAGESIZE; 1913 pfn++; 1914 continue; 1915 } 1916 /* 1917 * try to use large pages, check va/pa alignments 1918 * Note that 32M/256M page sizes are not (yet) supported. 1919 */ 1920 if ((len >= MMU_PAGESIZE4M) && 1921 !((uintptr_t)addr & MMU_PAGEOFFSET4M) && 1922 !(disable_large_pages & (1 << TTE4M)) && 1923 !(mmu_ptob(pfn) & MMU_PAGEOFFSET4M)) { 1924 sfmmu_memtte(&tte, pfn, attr, TTE4M); 1925 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1926 flags); 1927 len -= MMU_PAGESIZE4M; 1928 addr += MMU_PAGESIZE4M; 1929 pfn += MMU_PAGESIZE4M / MMU_PAGESIZE; 1930 } else if ((len >= MMU_PAGESIZE512K) && 1931 !((uintptr_t)addr & MMU_PAGEOFFSET512K) && 1932 !(disable_large_pages & (1 << TTE512K)) && 1933 !(mmu_ptob(pfn) & MMU_PAGEOFFSET512K)) { 1934 sfmmu_memtte(&tte, pfn, attr, TTE512K); 1935 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1936 flags); 1937 len -= MMU_PAGESIZE512K; 1938 addr += MMU_PAGESIZE512K; 1939 pfn += MMU_PAGESIZE512K / MMU_PAGESIZE; 1940 } else if ((len >= MMU_PAGESIZE64K) && 1941 !((uintptr_t)addr & MMU_PAGEOFFSET64K) && 1942 !(disable_large_pages & (1 << TTE64K)) && 1943 !(mmu_ptob(pfn) & MMU_PAGEOFFSET64K)) { 1944 sfmmu_memtte(&tte, pfn, attr, TTE64K); 1945 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1946 flags); 1947 len -= MMU_PAGESIZE64K; 1948 addr += MMU_PAGESIZE64K; 1949 pfn += MMU_PAGESIZE64K / MMU_PAGESIZE; 1950 } else { 1951 sfmmu_memtte(&tte, pfn, attr, TTE8K); 1952 (void) sfmmu_tteload_array(hat, &tte, addr, &pp, 1953 flags); 1954 len -= MMU_PAGESIZE; 1955 addr += MMU_PAGESIZE; 1956 pfn++; 1957 } 1958 } 1959 1960 /* 1961 * Check TSB and TLB page sizes. 1962 */ 1963 if ((flags & HAT_LOAD_SHARE) == 0) { 1964 sfmmu_check_page_sizes(hat, 1); 1965 } 1966 } 1967 1968 /* 1969 * Map the largest extend possible out of the page array. The array may NOT 1970 * be in order. The largest possible mapping a page can have 1971 * is specified in the p_szc field. The p_szc field 1972 * cannot change as long as there any mappings (large or small) 1973 * to any of the pages that make up the large page. (ie. any 1974 * promotion/demotion of page size is not up to the hat but up to 1975 * the page free list manager). The array 1976 * should consist of properly aligned contigous pages that are 1977 * part of a big page for a large mapping to be created. 1978 */ 1979 void 1980 hat_memload_array(struct hat *hat, caddr_t addr, size_t len, 1981 struct page **pps, uint_t attr, uint_t flags) 1982 { 1983 int ttesz; 1984 size_t mapsz; 1985 pgcnt_t numpg, npgs; 1986 tte_t tte; 1987 page_t *pp; 1988 int large_pages_disable; 1989 1990 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 1991 1992 if (hat->sfmmu_xhat_provider) { 1993 XHAT_MEMLOAD_ARRAY(hat, addr, len, pps, attr, flags); 1994 return; 1995 } 1996 1997 if (hat->sfmmu_rmstat) 1998 hat_resvstat(len, hat->sfmmu_as, addr); 1999 2000 #if defined(SF_ERRATA_57) 2001 if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) && 2002 (addr < errata57_limit) && (attr & PROT_EXEC) && 2003 !(flags & HAT_LOAD_SHARE)) { 2004 cmn_err(CE_WARN, "hat_memload_array: illegal attempt to make " 2005 "user page executable"); 2006 attr &= ~PROT_EXEC; 2007 } 2008 #endif 2009 2010 /* Get number of pages */ 2011 npgs = len >> MMU_PAGESHIFT; 2012 2013 if (flags & HAT_LOAD_SHARE) { 2014 large_pages_disable = disable_ism_large_pages; 2015 } else { 2016 large_pages_disable = disable_large_pages; 2017 } 2018 2019 if (npgs < NHMENTS || large_pages_disable == LARGE_PAGES_OFF) { 2020 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2021 return; 2022 } 2023 2024 while (npgs >= NHMENTS) { 2025 pp = *pps; 2026 for (ttesz = pp->p_szc; ttesz != TTE8K; ttesz--) { 2027 /* 2028 * Check if this page size is disabled. 2029 */ 2030 if (large_pages_disable & (1 << ttesz)) 2031 continue; 2032 2033 numpg = TTEPAGES(ttesz); 2034 mapsz = numpg << MMU_PAGESHIFT; 2035 if ((npgs >= numpg) && 2036 IS_P2ALIGNED(addr, mapsz) && 2037 IS_P2ALIGNED(pp->p_pagenum, numpg)) { 2038 /* 2039 * At this point we have enough pages and 2040 * we know the virtual address and the pfn 2041 * are properly aligned. We still need 2042 * to check for physical contiguity but since 2043 * it is very likely that this is the case 2044 * we will assume they are so and undo 2045 * the request if necessary. It would 2046 * be great if we could get a hint flag 2047 * like HAT_CONTIG which would tell us 2048 * the pages are contigous for sure. 2049 */ 2050 sfmmu_memtte(&tte, (*pps)->p_pagenum, 2051 attr, ttesz); 2052 if (!sfmmu_tteload_array(hat, &tte, addr, 2053 pps, flags)) { 2054 break; 2055 } 2056 } 2057 } 2058 if (ttesz == TTE8K) { 2059 /* 2060 * We were not able to map array using a large page 2061 * batch a hmeblk or fraction at a time. 2062 */ 2063 numpg = ((uintptr_t)addr >> MMU_PAGESHIFT) 2064 & (NHMENTS-1); 2065 numpg = NHMENTS - numpg; 2066 ASSERT(numpg <= npgs); 2067 mapsz = numpg * MMU_PAGESIZE; 2068 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, 2069 numpg); 2070 } 2071 addr += mapsz; 2072 npgs -= numpg; 2073 pps += numpg; 2074 } 2075 2076 if (npgs) { 2077 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs); 2078 } 2079 2080 /* 2081 * Check TSB and TLB page sizes. 2082 */ 2083 if ((flags & HAT_LOAD_SHARE) == 0) { 2084 sfmmu_check_page_sizes(hat, 1); 2085 } 2086 } 2087 2088 /* 2089 * Function tries to batch 8K pages into the same hme blk. 2090 */ 2091 static void 2092 sfmmu_memload_batchsmall(struct hat *hat, caddr_t vaddr, page_t **pps, 2093 uint_t attr, uint_t flags, pgcnt_t npgs) 2094 { 2095 tte_t tte; 2096 page_t *pp; 2097 struct hmehash_bucket *hmebp; 2098 struct hme_blk *hmeblkp; 2099 int index; 2100 2101 while (npgs) { 2102 /* 2103 * Acquire the hash bucket. 2104 */ 2105 hmebp = sfmmu_tteload_acquire_hashbucket(hat, vaddr, TTE8K); 2106 ASSERT(hmebp); 2107 2108 /* 2109 * Find the hment block. 2110 */ 2111 hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr, 2112 TTE8K, flags); 2113 ASSERT(hmeblkp); 2114 2115 do { 2116 /* 2117 * Make the tte. 2118 */ 2119 pp = *pps; 2120 sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K); 2121 2122 /* 2123 * Add the translation. 2124 */ 2125 (void) sfmmu_tteload_addentry(hat, hmeblkp, &tte, 2126 vaddr, pps, flags); 2127 2128 /* 2129 * Goto next page. 2130 */ 2131 pps++; 2132 npgs--; 2133 2134 /* 2135 * Goto next address. 2136 */ 2137 vaddr += MMU_PAGESIZE; 2138 2139 /* 2140 * Don't crossover into a different hmentblk. 2141 */ 2142 index = (int)(((uintptr_t)vaddr >> MMU_PAGESHIFT) & 2143 (NHMENTS-1)); 2144 2145 } while (index != 0 && npgs != 0); 2146 2147 /* 2148 * Release the hash bucket. 2149 */ 2150 2151 sfmmu_tteload_release_hashbucket(hmebp); 2152 } 2153 } 2154 2155 /* 2156 * Construct a tte for a page: 2157 * 2158 * tte_valid = 1 2159 * tte_size2 = size & TTE_SZ2_BITS (Panther and Olympus-C only) 2160 * tte_size = size 2161 * tte_nfo = attr & HAT_NOFAULT 2162 * tte_ie = attr & HAT_STRUCTURE_LE 2163 * tte_hmenum = hmenum 2164 * tte_pahi = pp->p_pagenum >> TTE_PASHIFT; 2165 * tte_palo = pp->p_pagenum & TTE_PALOMASK; 2166 * tte_ref = 1 (optimization) 2167 * tte_wr_perm = attr & PROT_WRITE; 2168 * tte_no_sync = attr & HAT_NOSYNC 2169 * tte_lock = attr & SFMMU_LOCKTTE 2170 * tte_cp = !(attr & SFMMU_UNCACHEPTTE) 2171 * tte_cv = !(attr & SFMMU_UNCACHEVTTE) 2172 * tte_e = attr & SFMMU_SIDEFFECT 2173 * tte_priv = !(attr & PROT_USER) 2174 * tte_hwwr = if nosync is set and it is writable we set the mod bit (opt) 2175 * tte_glb = 0 2176 */ 2177 void 2178 sfmmu_memtte(tte_t *ttep, pfn_t pfn, uint_t attr, int tte_sz) 2179 { 2180 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 2181 2182 ttep->tte_inthi = MAKE_TTE_INTHI(pfn, attr, tte_sz, 0 /* hmenum */); 2183 ttep->tte_intlo = MAKE_TTE_INTLO(pfn, attr, tte_sz, 0 /* hmenum */); 2184 2185 if (TTE_IS_NOSYNC(ttep)) { 2186 TTE_SET_REF(ttep); 2187 if (TTE_IS_WRITABLE(ttep)) { 2188 TTE_SET_MOD(ttep); 2189 } 2190 } 2191 if (TTE_IS_NFO(ttep) && TTE_IS_EXECUTABLE(ttep)) { 2192 panic("sfmmu_memtte: can't set both NFO and EXEC bits"); 2193 } 2194 } 2195 2196 /* 2197 * This function will add a translation to the hme_blk and allocate the 2198 * hme_blk if one does not exist. 2199 * If a page structure is specified then it will add the 2200 * corresponding hment to the mapping list. 2201 * It will also update the hmenum field for the tte. 2202 */ 2203 void 2204 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp, 2205 uint_t flags) 2206 { 2207 (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags); 2208 } 2209 2210 /* 2211 * Load (ttep != NULL) or unload (ttep == NULL) one entry in the TSB. 2212 * Assumes that a particular page size may only be resident in one TSB. 2213 */ 2214 static void 2215 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz) 2216 { 2217 struct tsb_info *tsbinfop = NULL; 2218 uint64_t tag; 2219 struct tsbe *tsbe_addr; 2220 uint64_t tsb_base; 2221 uint_t tsb_size; 2222 int vpshift = MMU_PAGESHIFT; 2223 int phys = 0; 2224 2225 if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */ 2226 phys = ktsb_phys; 2227 if (ttesz >= TTE4M) { 2228 #ifndef sun4v 2229 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2230 #endif 2231 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2232 tsb_size = ktsb4m_szcode; 2233 } else { 2234 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2235 tsb_size = ktsb_szcode; 2236 } 2237 } else { 2238 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2239 2240 /* 2241 * If there isn't a TSB for this page size, or the TSB is 2242 * swapped out, there is nothing to do. Note that the latter 2243 * case seems impossible but can occur if hat_pageunload() 2244 * is called on an ISM mapping while the process is swapped 2245 * out. 2246 */ 2247 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2248 return; 2249 2250 /* 2251 * If another thread is in the middle of relocating a TSB 2252 * we can't unload the entry so set a flag so that the 2253 * TSB will be flushed before it can be accessed by the 2254 * process. 2255 */ 2256 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2257 if (ttep == NULL) 2258 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2259 return; 2260 } 2261 #if defined(UTSB_PHYS) 2262 phys = 1; 2263 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2264 #else 2265 tsb_base = (uint64_t)tsbinfop->tsb_va; 2266 #endif 2267 tsb_size = tsbinfop->tsb_szc; 2268 } 2269 if (ttesz >= TTE4M) 2270 vpshift = MMU_PAGESHIFT4M; 2271 2272 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2273 tag = sfmmu_make_tsbtag(vaddr); 2274 2275 if (ttep == NULL) { 2276 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2277 } else { 2278 if (ttesz >= TTE4M) { 2279 SFMMU_STAT(sf_tsb_load4m); 2280 } else { 2281 SFMMU_STAT(sf_tsb_load8k); 2282 } 2283 2284 sfmmu_load_tsbe(tsbe_addr, tag, ttep, phys); 2285 } 2286 } 2287 2288 /* 2289 * Unmap all entries from [start, end) matching the given page size. 2290 * 2291 * This function is used primarily to unmap replicated 64K or 512K entries 2292 * from the TSB that are inserted using the base page size TSB pointer, but 2293 * it may also be called to unmap a range of addresses from the TSB. 2294 */ 2295 void 2296 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz) 2297 { 2298 struct tsb_info *tsbinfop; 2299 uint64_t tag; 2300 struct tsbe *tsbe_addr; 2301 caddr_t vaddr; 2302 uint64_t tsb_base; 2303 int vpshift, vpgsz; 2304 uint_t tsb_size; 2305 int phys = 0; 2306 2307 /* 2308 * Assumptions: 2309 * If ttesz == 8K, 64K or 512K, we walk through the range 8K 2310 * at a time shooting down any valid entries we encounter. 2311 * 2312 * If ttesz >= 4M we walk the range 4M at a time shooting 2313 * down any valid mappings we find. 2314 */ 2315 if (sfmmup == ksfmmup) { 2316 phys = ktsb_phys; 2317 if (ttesz >= TTE4M) { 2318 #ifndef sun4v 2319 ASSERT((ttesz != TTE32M) && (ttesz != TTE256M)); 2320 #endif 2321 tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base; 2322 tsb_size = ktsb4m_szcode; 2323 } else { 2324 tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base; 2325 tsb_size = ktsb_szcode; 2326 } 2327 } else { 2328 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); 2329 2330 /* 2331 * If there isn't a TSB for this page size, or the TSB is 2332 * swapped out, there is nothing to do. Note that the latter 2333 * case seems impossible but can occur if hat_pageunload() 2334 * is called on an ISM mapping while the process is swapped 2335 * out. 2336 */ 2337 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED)) 2338 return; 2339 2340 /* 2341 * If another thread is in the middle of relocating a TSB 2342 * we can't unload the entry so set a flag so that the 2343 * TSB will be flushed before it can be accessed by the 2344 * process. 2345 */ 2346 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 2347 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED; 2348 return; 2349 } 2350 #if defined(UTSB_PHYS) 2351 phys = 1; 2352 tsb_base = (uint64_t)tsbinfop->tsb_pa; 2353 #else 2354 tsb_base = (uint64_t)tsbinfop->tsb_va; 2355 #endif 2356 tsb_size = tsbinfop->tsb_szc; 2357 } 2358 if (ttesz >= TTE4M) { 2359 vpshift = MMU_PAGESHIFT4M; 2360 vpgsz = MMU_PAGESIZE4M; 2361 } else { 2362 vpshift = MMU_PAGESHIFT; 2363 vpgsz = MMU_PAGESIZE; 2364 } 2365 2366 for (vaddr = start; vaddr < end; vaddr += vpgsz) { 2367 tag = sfmmu_make_tsbtag(vaddr); 2368 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size); 2369 sfmmu_unload_tsbe(tsbe_addr, tag, phys); 2370 } 2371 } 2372 2373 /* 2374 * Select the optimum TSB size given the number of mappings 2375 * that need to be cached. 2376 */ 2377 static int 2378 sfmmu_select_tsb_szc(pgcnt_t pgcnt) 2379 { 2380 int szc = 0; 2381 2382 #ifdef DEBUG 2383 if (tsb_grow_stress) { 2384 uint32_t randval = (uint32_t)gettick() >> 4; 2385 return (randval % (tsb_max_growsize + 1)); 2386 } 2387 #endif /* DEBUG */ 2388 2389 while ((szc < tsb_max_growsize) && (pgcnt > SFMMU_RSS_TSBSIZE(szc))) 2390 szc++; 2391 return (szc); 2392 } 2393 2394 /* 2395 * This function will add a translation to the hme_blk and allocate the 2396 * hme_blk if one does not exist. 2397 * If a page structure is specified then it will add the 2398 * corresponding hment to the mapping list. 2399 * It will also update the hmenum field for the tte. 2400 * Furthermore, it attempts to create a large page translation 2401 * for <addr,hat> at page array pps. It assumes addr and first 2402 * pp is correctly aligned. It returns 0 if successful and 1 otherwise. 2403 */ 2404 static int 2405 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr, 2406 page_t **pps, uint_t flags) 2407 { 2408 struct hmehash_bucket *hmebp; 2409 struct hme_blk *hmeblkp; 2410 int ret; 2411 uint_t size; 2412 2413 /* 2414 * Get mapping size. 2415 */ 2416 size = TTE_CSZ(ttep); 2417 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2418 2419 /* 2420 * Acquire the hash bucket. 2421 */ 2422 hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size); 2423 ASSERT(hmebp); 2424 2425 /* 2426 * Find the hment block. 2427 */ 2428 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags); 2429 ASSERT(hmeblkp); 2430 2431 /* 2432 * Add the translation. 2433 */ 2434 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags); 2435 2436 /* 2437 * Release the hash bucket. 2438 */ 2439 sfmmu_tteload_release_hashbucket(hmebp); 2440 2441 return (ret); 2442 } 2443 2444 /* 2445 * Function locks and returns a pointer to the hash bucket for vaddr and size. 2446 */ 2447 static struct hmehash_bucket * 2448 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size) 2449 { 2450 struct hmehash_bucket *hmebp; 2451 int hmeshift; 2452 2453 hmeshift = HME_HASH_SHIFT(size); 2454 2455 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 2456 2457 SFMMU_HASH_LOCK(hmebp); 2458 2459 return (hmebp); 2460 } 2461 2462 /* 2463 * Function returns a pointer to an hmeblk in the hash bucket, hmebp. If the 2464 * hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is 2465 * allocated. 2466 */ 2467 static struct hme_blk * 2468 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp, 2469 caddr_t vaddr, uint_t size, uint_t flags) 2470 { 2471 hmeblk_tag hblktag; 2472 int hmeshift; 2473 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 2474 uint64_t hblkpa, prevpa; 2475 struct kmem_cache *sfmmu_cache; 2476 uint_t forcefree; 2477 2478 hblktag.htag_id = sfmmup; 2479 hmeshift = HME_HASH_SHIFT(size); 2480 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 2481 hblktag.htag_rehash = HME_HASH_REHASH(size); 2482 2483 ttearray_realloc: 2484 2485 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, 2486 pr_hblk, prevpa, &list); 2487 2488 /* 2489 * We block until hblk_reserve_lock is released; it's held by 2490 * the thread, temporarily using hblk_reserve, until hblk_reserve is 2491 * replaced by a hblk from sfmmu8_cache. 2492 */ 2493 if (hmeblkp == (struct hme_blk *)hblk_reserve && 2494 hblk_reserve_thread != curthread) { 2495 SFMMU_HASH_UNLOCK(hmebp); 2496 mutex_enter(&hblk_reserve_lock); 2497 mutex_exit(&hblk_reserve_lock); 2498 SFMMU_STAT(sf_hblk_reserve_hit); 2499 SFMMU_HASH_LOCK(hmebp); 2500 goto ttearray_realloc; 2501 } 2502 2503 if (hmeblkp == NULL) { 2504 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 2505 hblktag, flags); 2506 } else { 2507 /* 2508 * It is possible for 8k and 64k hblks to collide since they 2509 * have the same rehash value. This is because we 2510 * lazily free hblks and 8K/64K blks could be lingering. 2511 * If we find size mismatch we free the block and & try again. 2512 */ 2513 if (get_hblk_ttesz(hmeblkp) != size) { 2514 ASSERT(!hmeblkp->hblk_vcnt); 2515 ASSERT(!hmeblkp->hblk_hmecnt); 2516 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 2517 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 2518 goto ttearray_realloc; 2519 } 2520 if (hmeblkp->hblk_shw_bit) { 2521 /* 2522 * if the hblk was previously used as a shadow hblk then 2523 * we will change it to a normal hblk 2524 */ 2525 if (hmeblkp->hblk_shw_mask) { 2526 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp); 2527 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2528 goto ttearray_realloc; 2529 } else { 2530 hmeblkp->hblk_shw_bit = 0; 2531 } 2532 } 2533 SFMMU_STAT(sf_hblk_hit); 2534 } 2535 2536 /* 2537 * hat_memload() should never call kmem_cache_free(); see block 2538 * comment showing the stacktrace in sfmmu_hblk_alloc(); 2539 * enqueue each hblk in the list to reserve list if it's created 2540 * from sfmmu8_cache *and* sfmmup == KHATID. 2541 */ 2542 forcefree = (sfmmup == KHATID) ? 1 : 0; 2543 while ((pr_hblk = list) != NULL) { 2544 list = pr_hblk->hblk_next; 2545 sfmmu_cache = get_hblk_cache(pr_hblk); 2546 if ((sfmmu_cache == sfmmu8_cache) && 2547 sfmmu_put_free_hblk(pr_hblk, forcefree)) 2548 continue; 2549 2550 ASSERT(sfmmup != KHATID); 2551 kmem_cache_free(sfmmu_cache, pr_hblk); 2552 } 2553 2554 ASSERT(get_hblk_ttesz(hmeblkp) == size); 2555 ASSERT(!hmeblkp->hblk_shw_bit); 2556 2557 return (hmeblkp); 2558 } 2559 2560 /* 2561 * Function adds a tte entry into the hmeblk. It returns 0 if successful and 1 2562 * otherwise. 2563 */ 2564 static int 2565 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep, 2566 caddr_t vaddr, page_t **pps, uint_t flags) 2567 { 2568 page_t *pp = *pps; 2569 int hmenum, size, remap; 2570 tte_t tteold, flush_tte; 2571 #ifdef DEBUG 2572 tte_t orig_old; 2573 #endif /* DEBUG */ 2574 struct sf_hment *sfhme; 2575 kmutex_t *pml, *pmtx; 2576 hatlock_t *hatlockp; 2577 2578 /* 2579 * remove this panic when we decide to let user virtual address 2580 * space be >= USERLIMIT. 2581 */ 2582 if (!TTE_IS_PRIVILEGED(ttep) && vaddr >= (caddr_t)USERLIMIT) 2583 panic("user addr %p in kernel space", vaddr); 2584 #if defined(TTE_IS_GLOBAL) 2585 if (TTE_IS_GLOBAL(ttep)) 2586 panic("sfmmu_tteload: creating global tte"); 2587 #endif 2588 2589 #ifdef DEBUG 2590 if (pf_is_memory(sfmmu_ttetopfn(ttep, vaddr)) && 2591 !TTE_IS_PCACHEABLE(ttep) && !sfmmu_allow_nc_trans) 2592 panic("sfmmu_tteload: non cacheable memory tte"); 2593 #endif /* DEBUG */ 2594 2595 if ((flags & HAT_LOAD_SHARE) || !TTE_IS_REF(ttep) || 2596 !TTE_IS_MOD(ttep)) { 2597 /* 2598 * Don't load TSB for dummy as in ISM. Also don't preload 2599 * the TSB if the TTE isn't writable since we're likely to 2600 * fault on it again -- preloading can be fairly expensive. 2601 */ 2602 flags |= SFMMU_NO_TSBLOAD; 2603 } 2604 2605 size = TTE_CSZ(ttep); 2606 switch (size) { 2607 case TTE8K: 2608 SFMMU_STAT(sf_tteload8k); 2609 break; 2610 case TTE64K: 2611 SFMMU_STAT(sf_tteload64k); 2612 break; 2613 case TTE512K: 2614 SFMMU_STAT(sf_tteload512k); 2615 break; 2616 case TTE4M: 2617 SFMMU_STAT(sf_tteload4m); 2618 break; 2619 case (TTE32M): 2620 SFMMU_STAT(sf_tteload32m); 2621 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2622 break; 2623 case (TTE256M): 2624 SFMMU_STAT(sf_tteload256m); 2625 ASSERT(mmu_page_sizes == max_mmu_page_sizes); 2626 break; 2627 } 2628 2629 ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size))); 2630 2631 HBLKTOHME_IDX(sfhme, hmeblkp, vaddr, hmenum); 2632 2633 /* 2634 * Need to grab mlist lock here so that pageunload 2635 * will not change tte behind us. 2636 */ 2637 if (pp) { 2638 pml = sfmmu_mlist_enter(pp); 2639 } 2640 2641 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2642 /* 2643 * Look for corresponding hment and if valid verify 2644 * pfns are equal. 2645 */ 2646 remap = TTE_IS_VALID(&tteold); 2647 if (remap) { 2648 pfn_t new_pfn, old_pfn; 2649 2650 old_pfn = TTE_TO_PFN(vaddr, &tteold); 2651 new_pfn = TTE_TO_PFN(vaddr, ttep); 2652 2653 if (flags & HAT_LOAD_REMAP) { 2654 /* make sure we are remapping same type of pages */ 2655 if (pf_is_memory(old_pfn) != pf_is_memory(new_pfn)) { 2656 panic("sfmmu_tteload - tte remap io<->memory"); 2657 } 2658 if (old_pfn != new_pfn && 2659 (pp != NULL || sfhme->hme_page != NULL)) { 2660 panic("sfmmu_tteload - tte remap pp != NULL"); 2661 } 2662 } else if (old_pfn != new_pfn) { 2663 panic("sfmmu_tteload - tte remap, hmeblkp 0x%p", 2664 (void *)hmeblkp); 2665 } 2666 ASSERT(TTE_CSZ(&tteold) == TTE_CSZ(ttep)); 2667 } 2668 2669 if (pp) { 2670 if (size == TTE8K) { 2671 #ifdef VAC 2672 /* 2673 * Handle VAC consistency 2674 */ 2675 if (!remap && (cache & CACHE_VAC) && !PP_ISNC(pp)) { 2676 sfmmu_vac_conflict(sfmmup, vaddr, pp); 2677 } 2678 #endif 2679 2680 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2681 pmtx = sfmmu_page_enter(pp); 2682 PP_CLRRO(pp); 2683 sfmmu_page_exit(pmtx); 2684 } else if (!PP_ISMAPPED(pp) && 2685 (!TTE_IS_WRITABLE(ttep)) && !(PP_ISMOD(pp))) { 2686 pmtx = sfmmu_page_enter(pp); 2687 if (!(PP_ISMOD(pp))) { 2688 PP_SETRO(pp); 2689 } 2690 sfmmu_page_exit(pmtx); 2691 } 2692 2693 } else if (sfmmu_pagearray_setup(vaddr, pps, ttep, remap)) { 2694 /* 2695 * sfmmu_pagearray_setup failed so return 2696 */ 2697 sfmmu_mlist_exit(pml); 2698 return (1); 2699 } 2700 } 2701 2702 /* 2703 * Make sure hment is not on a mapping list. 2704 */ 2705 ASSERT(remap || (sfhme->hme_page == NULL)); 2706 2707 /* if it is not a remap then hme->next better be NULL */ 2708 ASSERT((!remap) ? sfhme->hme_next == NULL : 1); 2709 2710 if (flags & HAT_LOAD_LOCK) { 2711 if (((int)hmeblkp->hblk_lckcnt + 1) >= MAX_HBLK_LCKCNT) { 2712 panic("too high lckcnt-hmeblk %p", 2713 (void *)hmeblkp); 2714 } 2715 atomic_add_16(&hmeblkp->hblk_lckcnt, 1); 2716 2717 HBLK_STACK_TRACE(hmeblkp, HBLK_LOCK); 2718 } 2719 2720 #ifdef VAC 2721 if (pp && PP_ISNC(pp)) { 2722 /* 2723 * If the physical page is marked to be uncacheable, like 2724 * by a vac conflict, make sure the new mapping is also 2725 * uncacheable. 2726 */ 2727 TTE_CLR_VCACHEABLE(ttep); 2728 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 2729 } 2730 #endif 2731 ttep->tte_hmenum = hmenum; 2732 2733 #ifdef DEBUG 2734 orig_old = tteold; 2735 #endif /* DEBUG */ 2736 2737 while (sfmmu_modifytte_try(&tteold, ttep, &sfhme->hme_tte) < 0) { 2738 if ((sfmmup == KHATID) && 2739 (flags & (HAT_LOAD_LOCK | HAT_LOAD_REMAP))) { 2740 sfmmu_copytte(&sfhme->hme_tte, &tteold); 2741 } 2742 #ifdef DEBUG 2743 chk_tte(&orig_old, &tteold, ttep, hmeblkp); 2744 #endif /* DEBUG */ 2745 } 2746 2747 if (!TTE_IS_VALID(&tteold)) { 2748 2749 atomic_add_16(&hmeblkp->hblk_vcnt, 1); 2750 atomic_add_long(&sfmmup->sfmmu_ttecnt[size], 1); 2751 2752 /* 2753 * HAT_RELOAD_SHARE has been deprecated with lpg DISM. 2754 */ 2755 2756 if (size > TTE8K && (flags & HAT_LOAD_SHARE) == 0 && 2757 sfmmup != ksfmmup) { 2758 /* 2759 * If this is the first large mapping for the process 2760 * we must force any CPUs running this process to TL=0 2761 * where they will reload the HAT flags from the 2762 * tsbmiss area. This is necessary to make the large 2763 * mappings we are about to load visible to those CPUs; 2764 * otherwise they'll loop forever calling pagefault() 2765 * since we don't search large hash chains by default. 2766 */ 2767 hatlockp = sfmmu_hat_enter(sfmmup); 2768 if (size == TTE512K && 2769 !SFMMU_FLAGS_ISSET(sfmmup, HAT_512K_FLAG)) { 2770 SFMMU_FLAGS_SET(sfmmup, HAT_512K_FLAG); 2771 sfmmu_sync_mmustate(sfmmup); 2772 } else if (size == TTE4M && 2773 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 2774 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 2775 sfmmu_sync_mmustate(sfmmup); 2776 } else if (size == TTE64K && 2777 !SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) { 2778 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 2779 /* no sync mmustate; 64K shares 8K hashes */ 2780 } else if (mmu_page_sizes == max_mmu_page_sizes) { 2781 if (size == TTE32M && 2782 !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 2783 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 2784 sfmmu_sync_mmustate(sfmmup); 2785 } else if (size == TTE256M && 2786 !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 2787 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 2788 sfmmu_sync_mmustate(sfmmup); 2789 } 2790 } 2791 if (size >= TTE4M && (flags & HAT_LOAD_TEXT) && 2792 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) { 2793 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG); 2794 } 2795 sfmmu_hat_exit(hatlockp); 2796 } 2797 } 2798 ASSERT(TTE_IS_VALID(&sfhme->hme_tte)); 2799 2800 flush_tte.tte_intlo = (tteold.tte_intlo ^ ttep->tte_intlo) & 2801 hw_tte.tte_intlo; 2802 flush_tte.tte_inthi = (tteold.tte_inthi ^ ttep->tte_inthi) & 2803 hw_tte.tte_inthi; 2804 2805 if (remap && (flush_tte.tte_inthi || flush_tte.tte_intlo)) { 2806 /* 2807 * If remap and new tte differs from old tte we need 2808 * to sync the mod bit and flush TLB/TSB. We don't 2809 * need to sync ref bit because we currently always set 2810 * ref bit in tteload. 2811 */ 2812 ASSERT(TTE_IS_REF(ttep)); 2813 if (TTE_IS_MOD(&tteold)) { 2814 sfmmu_ttesync(sfmmup, vaddr, &tteold, pp); 2815 } 2816 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0); 2817 xt_sync(sfmmup->sfmmu_cpusran); 2818 } 2819 2820 if ((flags & SFMMU_NO_TSBLOAD) == 0) { 2821 /* 2822 * We only preload 8K and 4M mappings into the TSB, since 2823 * 64K and 512K mappings are replicated and hence don't 2824 * have a single, unique TSB entry. Ditto for 32M/256M. 2825 */ 2826 if (size == TTE8K || size == TTE4M) { 2827 hatlockp = sfmmu_hat_enter(sfmmup); 2828 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte, size); 2829 sfmmu_hat_exit(hatlockp); 2830 } 2831 } 2832 if (pp) { 2833 if (!remap) { 2834 HME_ADD(sfhme, pp); 2835 atomic_add_16(&hmeblkp->hblk_hmecnt, 1); 2836 ASSERT(hmeblkp->hblk_hmecnt > 0); 2837 2838 /* 2839 * Cannot ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 2840 * see pageunload() for comment. 2841 */ 2842 } 2843 sfmmu_mlist_exit(pml); 2844 } 2845 2846 return (0); 2847 } 2848 /* 2849 * Function unlocks hash bucket. 2850 */ 2851 static void 2852 sfmmu_tteload_release_hashbucket(struct hmehash_bucket *hmebp) 2853 { 2854 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 2855 SFMMU_HASH_UNLOCK(hmebp); 2856 } 2857 2858 /* 2859 * function which checks and sets up page array for a large 2860 * translation. Will set p_vcolor, p_index, p_ro fields. 2861 * Assumes addr and pfnum of first page are properly aligned. 2862 * Will check for physical contiguity. If check fails it return 2863 * non null. 2864 */ 2865 static int 2866 sfmmu_pagearray_setup(caddr_t addr, page_t **pps, tte_t *ttep, int remap) 2867 { 2868 int i, index, ttesz; 2869 pfn_t pfnum; 2870 pgcnt_t npgs; 2871 page_t *pp, *pp1; 2872 kmutex_t *pmtx; 2873 #ifdef VAC 2874 int osz; 2875 int cflags = 0; 2876 int vac_err = 0; 2877 #endif 2878 int newidx = 0; 2879 2880 ttesz = TTE_CSZ(ttep); 2881 2882 ASSERT(ttesz > TTE8K); 2883 2884 npgs = TTEPAGES(ttesz); 2885 index = PAGESZ_TO_INDEX(ttesz); 2886 2887 pfnum = (*pps)->p_pagenum; 2888 ASSERT(IS_P2ALIGNED(pfnum, npgs)); 2889 2890 /* 2891 * Save the first pp so we can do HAT_TMPNC at the end. 2892 */ 2893 pp1 = *pps; 2894 #ifdef VAC 2895 osz = fnd_mapping_sz(pp1); 2896 #endif 2897 2898 for (i = 0; i < npgs; i++, pps++) { 2899 pp = *pps; 2900 ASSERT(PAGE_LOCKED(pp)); 2901 ASSERT(pp->p_szc >= ttesz); 2902 ASSERT(pp->p_szc == pp1->p_szc); 2903 ASSERT(sfmmu_mlist_held(pp)); 2904 2905 /* 2906 * XXX is it possible to maintain P_RO on the root only? 2907 */ 2908 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) { 2909 pmtx = sfmmu_page_enter(pp); 2910 PP_CLRRO(pp); 2911 sfmmu_page_exit(pmtx); 2912 } else if (!PP_ISMAPPED(pp) && !TTE_IS_WRITABLE(ttep) && 2913 !PP_ISMOD(pp)) { 2914 pmtx = sfmmu_page_enter(pp); 2915 if (!(PP_ISMOD(pp))) { 2916 PP_SETRO(pp); 2917 } 2918 sfmmu_page_exit(pmtx); 2919 } 2920 2921 /* 2922 * If this is a remap we skip vac & contiguity checks. 2923 */ 2924 if (remap) 2925 continue; 2926 2927 /* 2928 * set p_vcolor and detect any vac conflicts. 2929 */ 2930 #ifdef VAC 2931 if (vac_err == 0) { 2932 vac_err = sfmmu_vacconflict_array(addr, pp, &cflags); 2933 2934 } 2935 #endif 2936 2937 /* 2938 * Save current index in case we need to undo it. 2939 * Note: "PAGESZ_TO_INDEX(sz) (1 << (sz))" 2940 * "SFMMU_INDEX_SHIFT 6" 2941 * "SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1)" 2942 * "PP_MAPINDEX(p_index) (p_index & SFMMU_INDEX_MASK)" 2943 * 2944 * So: index = PAGESZ_TO_INDEX(ttesz); 2945 * if ttesz == 1 then index = 0x2 2946 * 2 then index = 0x4 2947 * 3 then index = 0x8 2948 * 4 then index = 0x10 2949 * 5 then index = 0x20 2950 * The code below checks if it's a new pagesize (ie, newidx) 2951 * in case we need to take it back out of p_index, 2952 * and then or's the new index into the existing index. 2953 */ 2954 if ((PP_MAPINDEX(pp) & index) == 0) 2955 newidx = 1; 2956 pp->p_index = (PP_MAPINDEX(pp) | index); 2957 2958 /* 2959 * contiguity check 2960 */ 2961 if (pp->p_pagenum != pfnum) { 2962 /* 2963 * If we fail the contiguity test then 2964 * the only thing we need to fix is the p_index field. 2965 * We might get a few extra flushes but since this 2966 * path is rare that is ok. The p_ro field will 2967 * get automatically fixed on the next tteload to 2968 * the page. NO TNC bit is set yet. 2969 */ 2970 while (i >= 0) { 2971 pp = *pps; 2972 if (newidx) 2973 pp->p_index = (PP_MAPINDEX(pp) & 2974 ~index); 2975 pps--; 2976 i--; 2977 } 2978 return (1); 2979 } 2980 pfnum++; 2981 addr += MMU_PAGESIZE; 2982 } 2983 2984 #ifdef VAC 2985 if (vac_err) { 2986 if (ttesz > osz) { 2987 /* 2988 * There are some smaller mappings that causes vac 2989 * conflicts. Convert all existing small mappings to 2990 * TNC. 2991 */ 2992 SFMMU_STAT_ADD(sf_uncache_conflict, npgs); 2993 sfmmu_page_cache_array(pp1, HAT_TMPNC, CACHE_FLUSH, 2994 npgs); 2995 } else { 2996 /* EMPTY */ 2997 /* 2998 * If there exists an big page mapping, 2999 * that means the whole existing big page 3000 * has TNC setting already. No need to covert to 3001 * TNC again. 3002 */ 3003 ASSERT(PP_ISTNC(pp1)); 3004 } 3005 } 3006 #endif /* VAC */ 3007 3008 return (0); 3009 } 3010 3011 #ifdef VAC 3012 /* 3013 * Routine that detects vac consistency for a large page. It also 3014 * sets virtual color for all pp's for this big mapping. 3015 */ 3016 static int 3017 sfmmu_vacconflict_array(caddr_t addr, page_t *pp, int *cflags) 3018 { 3019 int vcolor, ocolor; 3020 3021 ASSERT(sfmmu_mlist_held(pp)); 3022 3023 if (PP_ISNC(pp)) { 3024 return (HAT_TMPNC); 3025 } 3026 3027 vcolor = addr_to_vcolor(addr); 3028 if (PP_NEWPAGE(pp)) { 3029 PP_SET_VCOLOR(pp, vcolor); 3030 return (0); 3031 } 3032 3033 ocolor = PP_GET_VCOLOR(pp); 3034 if (ocolor == vcolor) { 3035 return (0); 3036 } 3037 3038 if (!PP_ISMAPPED(pp)) { 3039 /* 3040 * Previous user of page had a differnet color 3041 * but since there are no current users 3042 * we just flush the cache and change the color. 3043 * As an optimization for large pages we flush the 3044 * entire cache of that color and set a flag. 3045 */ 3046 SFMMU_STAT(sf_pgcolor_conflict); 3047 if (!CacheColor_IsFlushed(*cflags, ocolor)) { 3048 CacheColor_SetFlushed(*cflags, ocolor); 3049 sfmmu_cache_flushcolor(ocolor, pp->p_pagenum); 3050 } 3051 PP_SET_VCOLOR(pp, vcolor); 3052 return (0); 3053 } 3054 3055 /* 3056 * We got a real conflict with a current mapping. 3057 * set flags to start unencaching all mappings 3058 * and return failure so we restart looping 3059 * the pp array from the beginning. 3060 */ 3061 return (HAT_TMPNC); 3062 } 3063 #endif /* VAC */ 3064 3065 /* 3066 * creates a large page shadow hmeblk for a tte. 3067 * The purpose of this routine is to allow us to do quick unloads because 3068 * the vm layer can easily pass a very large but sparsely populated range. 3069 */ 3070 static struct hme_blk * 3071 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags) 3072 { 3073 struct hmehash_bucket *hmebp; 3074 hmeblk_tag hblktag; 3075 int hmeshift, size, vshift; 3076 uint_t shw_mask, newshw_mask; 3077 struct hme_blk *hmeblkp; 3078 3079 ASSERT(sfmmup != KHATID); 3080 if (mmu_page_sizes == max_mmu_page_sizes) { 3081 ASSERT(ttesz < TTE256M); 3082 } else { 3083 ASSERT(ttesz < TTE4M); 3084 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 3085 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 3086 } 3087 3088 if (ttesz == TTE8K) { 3089 size = TTE512K; 3090 } else { 3091 size = ++ttesz; 3092 } 3093 3094 hblktag.htag_id = sfmmup; 3095 hmeshift = HME_HASH_SHIFT(size); 3096 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 3097 hblktag.htag_rehash = HME_HASH_REHASH(size); 3098 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 3099 3100 SFMMU_HASH_LOCK(hmebp); 3101 3102 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3103 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 3104 if (hmeblkp == NULL) { 3105 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, 3106 hblktag, flags); 3107 } 3108 ASSERT(hmeblkp); 3109 if (!hmeblkp->hblk_shw_mask) { 3110 /* 3111 * if this is a unused hblk it was just allocated or could 3112 * potentially be a previous large page hblk so we need to 3113 * set the shadow bit. 3114 */ 3115 hmeblkp->hblk_shw_bit = 1; 3116 } 3117 ASSERT(hmeblkp->hblk_shw_bit == 1); 3118 vshift = vaddr_to_vshift(hblktag, vaddr, size); 3119 ASSERT(vshift < 8); 3120 /* 3121 * Atomically set shw mask bit 3122 */ 3123 do { 3124 shw_mask = hmeblkp->hblk_shw_mask; 3125 newshw_mask = shw_mask | (1 << vshift); 3126 newshw_mask = cas32(&hmeblkp->hblk_shw_mask, shw_mask, 3127 newshw_mask); 3128 } while (newshw_mask != shw_mask); 3129 3130 SFMMU_HASH_UNLOCK(hmebp); 3131 3132 return (hmeblkp); 3133 } 3134 3135 /* 3136 * This routine cleanup a previous shadow hmeblk and changes it to 3137 * a regular hblk. This happens rarely but it is possible 3138 * when a process wants to use large pages and there are hblks still 3139 * lying around from the previous as that used these hmeblks. 3140 * The alternative was to cleanup the shadow hblks at unload time 3141 * but since so few user processes actually use large pages, it is 3142 * better to be lazy and cleanup at this time. 3143 */ 3144 static void 3145 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 3146 struct hmehash_bucket *hmebp) 3147 { 3148 caddr_t addr, endaddr; 3149 int hashno, size; 3150 3151 ASSERT(hmeblkp->hblk_shw_bit); 3152 3153 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 3154 3155 if (!hmeblkp->hblk_shw_mask) { 3156 hmeblkp->hblk_shw_bit = 0; 3157 return; 3158 } 3159 addr = (caddr_t)get_hblk_base(hmeblkp); 3160 endaddr = get_hblk_endaddr(hmeblkp); 3161 size = get_hblk_ttesz(hmeblkp); 3162 hashno = size - 1; 3163 ASSERT(hashno > 0); 3164 SFMMU_HASH_UNLOCK(hmebp); 3165 3166 sfmmu_free_hblks(sfmmup, addr, endaddr, hashno); 3167 3168 SFMMU_HASH_LOCK(hmebp); 3169 } 3170 3171 static void 3172 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr, 3173 int hashno) 3174 { 3175 int hmeshift, shadow = 0; 3176 hmeblk_tag hblktag; 3177 struct hmehash_bucket *hmebp; 3178 struct hme_blk *hmeblkp; 3179 struct hme_blk *nx_hblk, *pr_hblk, *list = NULL; 3180 uint64_t hblkpa, prevpa, nx_pa; 3181 3182 ASSERT(hashno > 0); 3183 hblktag.htag_id = sfmmup; 3184 hblktag.htag_rehash = hashno; 3185 3186 hmeshift = HME_HASH_SHIFT(hashno); 3187 3188 while (addr < endaddr) { 3189 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3190 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3191 SFMMU_HASH_LOCK(hmebp); 3192 /* inline HME_HASH_SEARCH */ 3193 hmeblkp = hmebp->hmeblkp; 3194 hblkpa = hmebp->hmeh_nextpa; 3195 prevpa = 0; 3196 pr_hblk = NULL; 3197 while (hmeblkp) { 3198 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 3199 if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) { 3200 /* found hme_blk */ 3201 if (hmeblkp->hblk_shw_bit) { 3202 if (hmeblkp->hblk_shw_mask) { 3203 shadow = 1; 3204 sfmmu_shadow_hcleanup(sfmmup, 3205 hmeblkp, hmebp); 3206 break; 3207 } else { 3208 hmeblkp->hblk_shw_bit = 0; 3209 } 3210 } 3211 3212 /* 3213 * Hblk_hmecnt and hblk_vcnt could be non zero 3214 * since hblk_unload() does not gurantee that. 3215 * 3216 * XXX - this could cause tteload() to spin 3217 * where sfmmu_shadow_hcleanup() is called. 3218 */ 3219 } 3220 3221 nx_hblk = hmeblkp->hblk_next; 3222 nx_pa = hmeblkp->hblk_nextpa; 3223 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 3224 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 3225 pr_hblk); 3226 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 3227 } else { 3228 pr_hblk = hmeblkp; 3229 prevpa = hblkpa; 3230 } 3231 hmeblkp = nx_hblk; 3232 hblkpa = nx_pa; 3233 } 3234 3235 SFMMU_HASH_UNLOCK(hmebp); 3236 3237 if (shadow) { 3238 /* 3239 * We found another shadow hblk so cleaned its 3240 * children. We need to go back and cleanup 3241 * the original hblk so we don't change the 3242 * addr. 3243 */ 3244 shadow = 0; 3245 } else { 3246 addr = (caddr_t)roundup((uintptr_t)addr + 1, 3247 (1 << hmeshift)); 3248 } 3249 } 3250 sfmmu_hblks_list_purge(&list); 3251 } 3252 3253 /* 3254 * Release one hardware address translation lock on the given address range. 3255 */ 3256 void 3257 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len) 3258 { 3259 struct hmehash_bucket *hmebp; 3260 hmeblk_tag hblktag; 3261 int hmeshift, hashno = 1; 3262 struct hme_blk *hmeblkp, *list = NULL; 3263 caddr_t endaddr; 3264 3265 ASSERT(sfmmup != NULL); 3266 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3267 3268 ASSERT((sfmmup == ksfmmup) || 3269 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 3270 ASSERT((len & MMU_PAGEOFFSET) == 0); 3271 endaddr = addr + len; 3272 hblktag.htag_id = sfmmup; 3273 3274 /* 3275 * Spitfire supports 4 page sizes. 3276 * Most pages are expected to be of the smallest page size (8K) and 3277 * these will not need to be rehashed. 64K pages also don't need to be 3278 * rehashed because an hmeblk spans 64K of address space. 512K pages 3279 * might need 1 rehash and and 4M pages might need 2 rehashes. 3280 */ 3281 while (addr < endaddr) { 3282 hmeshift = HME_HASH_SHIFT(hashno); 3283 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 3284 hblktag.htag_rehash = hashno; 3285 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 3286 3287 SFMMU_HASH_LOCK(hmebp); 3288 3289 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 3290 if (hmeblkp != NULL) { 3291 /* 3292 * If we encounter a shadow hmeblk then 3293 * we know there are no valid hmeblks mapping 3294 * this address at this size or larger. 3295 * Just increment address by the smallest 3296 * page size. 3297 */ 3298 if (hmeblkp->hblk_shw_bit) { 3299 addr += MMU_PAGESIZE; 3300 } else { 3301 addr = sfmmu_hblk_unlock(hmeblkp, addr, 3302 endaddr); 3303 } 3304 SFMMU_HASH_UNLOCK(hmebp); 3305 hashno = 1; 3306 continue; 3307 } 3308 SFMMU_HASH_UNLOCK(hmebp); 3309 3310 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 3311 /* 3312 * We have traversed the whole list and rehashed 3313 * if necessary without finding the address to unlock 3314 * which should never happen. 3315 */ 3316 panic("sfmmu_unlock: addr not found. " 3317 "addr %p hat %p", (void *)addr, (void *)sfmmup); 3318 } else { 3319 hashno++; 3320 } 3321 } 3322 3323 sfmmu_hblks_list_purge(&list); 3324 } 3325 3326 /* 3327 * Function to unlock a range of addresses in an hmeblk. It returns the 3328 * next address that needs to be unlocked. 3329 * Should be called with the hash lock held. 3330 */ 3331 static caddr_t 3332 sfmmu_hblk_unlock(struct hme_blk *hmeblkp, caddr_t addr, caddr_t endaddr) 3333 { 3334 struct sf_hment *sfhme; 3335 tte_t tteold, ttemod; 3336 int ttesz, ret; 3337 3338 ASSERT(in_hblk_range(hmeblkp, addr)); 3339 ASSERT(hmeblkp->hblk_shw_bit == 0); 3340 3341 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 3342 ttesz = get_hblk_ttesz(hmeblkp); 3343 3344 HBLKTOHME(sfhme, hmeblkp, addr); 3345 while (addr < endaddr) { 3346 readtte: 3347 sfmmu_copytte(&sfhme->hme_tte, &tteold); 3348 if (TTE_IS_VALID(&tteold)) { 3349 3350 ttemod = tteold; 3351 3352 ret = sfmmu_modifytte_try(&tteold, &ttemod, 3353 &sfhme->hme_tte); 3354 3355 if (ret < 0) 3356 goto readtte; 3357 3358 if (hmeblkp->hblk_lckcnt == 0) 3359 panic("zero hblk lckcnt"); 3360 3361 if (((uintptr_t)addr + TTEBYTES(ttesz)) > 3362 (uintptr_t)endaddr) 3363 panic("can't unlock large tte"); 3364 3365 ASSERT(hmeblkp->hblk_lckcnt > 0); 3366 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 3367 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 3368 } else { 3369 panic("sfmmu_hblk_unlock: invalid tte"); 3370 } 3371 addr += TTEBYTES(ttesz); 3372 sfhme++; 3373 } 3374 return (addr); 3375 } 3376 3377 /* 3378 * Physical Address Mapping Framework 3379 * 3380 * General rules: 3381 * 3382 * (1) Applies only to seg_kmem memory pages. To make things easier, 3383 * seg_kpm addresses are also accepted by the routines, but nothing 3384 * is done with them since by definition their PA mappings are static. 3385 * (2) hat_add_callback() may only be called while holding the page lock 3386 * SE_SHARED or SE_EXCL of the underlying page (e.g., as_pagelock()), 3387 * or passing HAC_PAGELOCK flag. 3388 * (3) prehandler() and posthandler() may not call hat_add_callback() or 3389 * hat_delete_callback(), nor should they allocate memory. Post quiesce 3390 * callbacks may not sleep or acquire adaptive mutex locks. 3391 * (4) Either prehandler() or posthandler() (but not both) may be specified 3392 * as being NULL. Specifying an errhandler() is optional. 3393 * 3394 * Details of using the framework: 3395 * 3396 * registering a callback (hat_register_callback()) 3397 * 3398 * Pass prehandler, posthandler, errhandler addresses 3399 * as described below. If capture_cpus argument is nonzero, 3400 * suspend callback to the prehandler will occur with CPUs 3401 * captured and executing xc_loop() and CPUs will remain 3402 * captured until after the posthandler suspend callback 3403 * occurs. 3404 * 3405 * adding a callback (hat_add_callback()) 3406 * 3407 * as_pagelock(); 3408 * hat_add_callback(); 3409 * save returned pfn in private data structures or program registers; 3410 * as_pageunlock(); 3411 * 3412 * prehandler() 3413 * 3414 * Stop all accesses by physical address to this memory page. 3415 * Called twice: the first, PRESUSPEND, is a context safe to acquire 3416 * adaptive locks. The second, SUSPEND, is called at high PIL with 3417 * CPUs captured so adaptive locks may NOT be acquired (and all spin 3418 * locks must be XCALL_PIL or higher locks). 3419 * 3420 * May return the following errors: 3421 * EIO: A fatal error has occurred. This will result in panic. 3422 * EAGAIN: The page cannot be suspended. This will fail the 3423 * relocation. 3424 * 0: Success. 3425 * 3426 * posthandler() 3427 * 3428 * Save new pfn in private data structures or program registers; 3429 * not allowed to fail (non-zero return values will result in panic). 3430 * 3431 * errhandler() 3432 * 3433 * called when an error occurs related to the callback. Currently 3434 * the only such error is HAT_CB_ERR_LEAKED which indicates that 3435 * a page is being freed, but there are still outstanding callback(s) 3436 * registered on the page. 3437 * 3438 * removing a callback (hat_delete_callback(); e.g., prior to freeing memory) 3439 * 3440 * stop using physical address 3441 * hat_delete_callback(); 3442 * 3443 */ 3444 3445 /* 3446 * Register a callback class. Each subsystem should do this once and 3447 * cache the id_t returned for use in setting up and tearing down callbacks. 3448 * 3449 * There is no facility for removing callback IDs once they are created; 3450 * the "key" should be unique for each module, so in case a module is unloaded 3451 * and subsequently re-loaded, we can recycle the module's previous entry. 3452 */ 3453 id_t 3454 hat_register_callback(int key, 3455 int (*prehandler)(caddr_t, uint_t, uint_t, void *), 3456 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t), 3457 int (*errhandler)(caddr_t, uint_t, uint_t, void *), 3458 int capture_cpus) 3459 { 3460 id_t id; 3461 3462 /* 3463 * Search the table for a pre-existing callback associated with 3464 * the identifier "key". If one exists, we re-use that entry in 3465 * the table for this instance, otherwise we assign the next 3466 * available table slot. 3467 */ 3468 for (id = 0; id < sfmmu_max_cb_id; id++) { 3469 if (sfmmu_cb_table[id].key == key) 3470 break; 3471 } 3472 3473 if (id == sfmmu_max_cb_id) { 3474 id = sfmmu_cb_nextid++; 3475 if (id >= sfmmu_max_cb_id) 3476 panic("hat_register_callback: out of callback IDs"); 3477 } 3478 3479 ASSERT(prehandler != NULL || posthandler != NULL); 3480 3481 sfmmu_cb_table[id].key = key; 3482 sfmmu_cb_table[id].prehandler = prehandler; 3483 sfmmu_cb_table[id].posthandler = posthandler; 3484 sfmmu_cb_table[id].errhandler = errhandler; 3485 sfmmu_cb_table[id].capture_cpus = capture_cpus; 3486 3487 return (id); 3488 } 3489 3490 #define HAC_COOKIE_NONE (void *)-1 3491 3492 /* 3493 * Add relocation callbacks to the specified addr/len which will be called 3494 * when relocating the associated page. See the description of pre and 3495 * posthandler above for more details. 3496 * 3497 * If HAC_PAGELOCK is included in flags, the underlying memory page is 3498 * locked internally so the caller must be able to deal with the callback 3499 * running even before this function has returned. If HAC_PAGELOCK is not 3500 * set, it is assumed that the underlying memory pages are locked. 3501 * 3502 * Since the caller must track the individual page boundaries anyway, 3503 * we only allow a callback to be added to a single page (large 3504 * or small). Thus [addr, addr + len) MUST be contained within a single 3505 * page. 3506 * 3507 * Registering multiple callbacks on the same [addr, addr+len) is supported, 3508 * _provided_that_ a unique parameter is specified for each callback. 3509 * If multiple callbacks are registered on the same range the callback will 3510 * be invoked with each unique parameter. Registering the same callback with 3511 * the same argument more than once will result in corrupted kernel state. 3512 * 3513 * Returns the pfn of the underlying kernel page in *rpfn 3514 * on success, or PFN_INVALID on failure. 3515 * 3516 * cookiep (if passed) provides storage space for an opaque cookie 3517 * to return later to hat_delete_callback(). This cookie makes the callback 3518 * deletion significantly quicker by avoiding a potentially lengthy hash 3519 * search. 3520 * 3521 * Returns values: 3522 * 0: success 3523 * ENOMEM: memory allocation failure (e.g. flags was passed as HAC_NOSLEEP) 3524 * EINVAL: callback ID is not valid 3525 * ENXIO: ["vaddr", "vaddr" + len) is not mapped in the kernel's address 3526 * space 3527 * ERANGE: ["vaddr", "vaddr" + len) crosses a page boundary 3528 */ 3529 int 3530 hat_add_callback(id_t callback_id, caddr_t vaddr, uint_t len, uint_t flags, 3531 void *pvt, pfn_t *rpfn, void **cookiep) 3532 { 3533 struct hmehash_bucket *hmebp; 3534 hmeblk_tag hblktag; 3535 struct hme_blk *hmeblkp; 3536 int hmeshift, hashno; 3537 caddr_t saddr, eaddr, baseaddr; 3538 struct pa_hment *pahmep; 3539 struct sf_hment *sfhmep, *osfhmep; 3540 kmutex_t *pml; 3541 tte_t tte; 3542 page_t *pp; 3543 vnode_t *vp; 3544 u_offset_t off; 3545 pfn_t pfn; 3546 int kmflags = (flags & HAC_SLEEP)? KM_SLEEP : KM_NOSLEEP; 3547 int locked = 0; 3548 3549 /* 3550 * For KPM mappings, just return the physical address since we 3551 * don't need to register any callbacks. 3552 */ 3553 if (IS_KPM_ADDR(vaddr)) { 3554 uint64_t paddr; 3555 SFMMU_KPM_VTOP(vaddr, paddr); 3556 *rpfn = btop(paddr); 3557 if (cookiep != NULL) 3558 *cookiep = HAC_COOKIE_NONE; 3559 return (0); 3560 } 3561 3562 if (callback_id < (id_t)0 || callback_id >= sfmmu_cb_nextid) { 3563 *rpfn = PFN_INVALID; 3564 return (EINVAL); 3565 } 3566 3567 if ((pahmep = kmem_cache_alloc(pa_hment_cache, kmflags)) == NULL) { 3568 *rpfn = PFN_INVALID; 3569 return (ENOMEM); 3570 } 3571 3572 sfhmep = &pahmep->sfment; 3573 3574 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3575 eaddr = saddr + len; 3576 3577 rehash: 3578 /* Find the mapping(s) for this page */ 3579 for (hashno = TTE64K, hmeblkp = NULL; 3580 hmeblkp == NULL && hashno <= mmu_hashcnt; 3581 hashno++) { 3582 hmeshift = HME_HASH_SHIFT(hashno); 3583 hblktag.htag_id = ksfmmup; 3584 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3585 hblktag.htag_rehash = hashno; 3586 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3587 3588 SFMMU_HASH_LOCK(hmebp); 3589 3590 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3591 3592 if (hmeblkp == NULL) 3593 SFMMU_HASH_UNLOCK(hmebp); 3594 } 3595 3596 if (hmeblkp == NULL) { 3597 kmem_cache_free(pa_hment_cache, pahmep); 3598 *rpfn = PFN_INVALID; 3599 return (ENXIO); 3600 } 3601 3602 HBLKTOHME(osfhmep, hmeblkp, saddr); 3603 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3604 3605 if (!TTE_IS_VALID(&tte)) { 3606 SFMMU_HASH_UNLOCK(hmebp); 3607 kmem_cache_free(pa_hment_cache, pahmep); 3608 *rpfn = PFN_INVALID; 3609 return (ENXIO); 3610 } 3611 3612 /* 3613 * Make sure the boundaries for the callback fall within this 3614 * single mapping. 3615 */ 3616 baseaddr = (caddr_t)get_hblk_base(hmeblkp); 3617 ASSERT(saddr >= baseaddr); 3618 if (eaddr > saddr + TTEBYTES(TTE_CSZ(&tte))) { 3619 SFMMU_HASH_UNLOCK(hmebp); 3620 kmem_cache_free(pa_hment_cache, pahmep); 3621 *rpfn = PFN_INVALID; 3622 return (ERANGE); 3623 } 3624 3625 pfn = sfmmu_ttetopfn(&tte, vaddr); 3626 3627 /* 3628 * The pfn may not have a page_t underneath in which case we 3629 * just return it. This can happen if we are doing I/O to a 3630 * static portion of the kernel's address space, for instance. 3631 */ 3632 pp = osfhmep->hme_page; 3633 if (pp == NULL) { 3634 SFMMU_HASH_UNLOCK(hmebp); 3635 kmem_cache_free(pa_hment_cache, pahmep); 3636 *rpfn = pfn; 3637 if (cookiep) 3638 *cookiep = HAC_COOKIE_NONE; 3639 return (0); 3640 } 3641 ASSERT(pp == PP_PAGEROOT(pp)); 3642 3643 vp = pp->p_vnode; 3644 off = pp->p_offset; 3645 3646 pml = sfmmu_mlist_enter(pp); 3647 3648 if (flags & HAC_PAGELOCK) { 3649 if (!page_trylock(pp, SE_SHARED)) { 3650 /* 3651 * Somebody is holding SE_EXCL lock. Might 3652 * even be hat_page_relocate(). Drop all 3653 * our locks, lookup the page in &kvp, and 3654 * retry. If it doesn't exist in &kvp, then 3655 * we must be dealing with a kernel mapped 3656 * page which doesn't actually belong to 3657 * segkmem so we punt. 3658 */ 3659 sfmmu_mlist_exit(pml); 3660 SFMMU_HASH_UNLOCK(hmebp); 3661 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3662 if (pp == NULL) { 3663 kmem_cache_free(pa_hment_cache, pahmep); 3664 *rpfn = pfn; 3665 if (cookiep) 3666 *cookiep = HAC_COOKIE_NONE; 3667 return (0); 3668 } 3669 page_unlock(pp); 3670 goto rehash; 3671 } 3672 locked = 1; 3673 } 3674 3675 if (!PAGE_LOCKED(pp) && !panicstr) 3676 panic("hat_add_callback: page 0x%p not locked", pp); 3677 3678 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3679 pp->p_offset != off) { 3680 /* 3681 * The page moved before we got our hands on it. Drop 3682 * all the locks and try again. 3683 */ 3684 ASSERT((flags & HAC_PAGELOCK) != 0); 3685 sfmmu_mlist_exit(pml); 3686 SFMMU_HASH_UNLOCK(hmebp); 3687 page_unlock(pp); 3688 locked = 0; 3689 goto rehash; 3690 } 3691 3692 if (vp != &kvp) { 3693 /* 3694 * This is not a segkmem page but another page which 3695 * has been kernel mapped. It had better have at least 3696 * a share lock on it. Return the pfn. 3697 */ 3698 sfmmu_mlist_exit(pml); 3699 SFMMU_HASH_UNLOCK(hmebp); 3700 if (locked) 3701 page_unlock(pp); 3702 kmem_cache_free(pa_hment_cache, pahmep); 3703 ASSERT(PAGE_LOCKED(pp)); 3704 *rpfn = pfn; 3705 if (cookiep) 3706 *cookiep = HAC_COOKIE_NONE; 3707 return (0); 3708 } 3709 3710 /* 3711 * Setup this pa_hment and link its embedded dummy sf_hment into 3712 * the mapping list. 3713 */ 3714 pp->p_share++; 3715 pahmep->cb_id = callback_id; 3716 pahmep->addr = vaddr; 3717 pahmep->len = len; 3718 pahmep->refcnt = 1; 3719 pahmep->flags = 0; 3720 pahmep->pvt = pvt; 3721 3722 sfhmep->hme_tte.ll = 0; 3723 sfhmep->hme_data = pahmep; 3724 sfhmep->hme_prev = osfhmep; 3725 sfhmep->hme_next = osfhmep->hme_next; 3726 3727 if (osfhmep->hme_next) 3728 osfhmep->hme_next->hme_prev = sfhmep; 3729 3730 osfhmep->hme_next = sfhmep; 3731 3732 sfmmu_mlist_exit(pml); 3733 SFMMU_HASH_UNLOCK(hmebp); 3734 3735 if (locked) 3736 page_unlock(pp); 3737 3738 *rpfn = pfn; 3739 if (cookiep) 3740 *cookiep = (void *)pahmep; 3741 3742 return (0); 3743 } 3744 3745 /* 3746 * Remove the relocation callbacks from the specified addr/len. 3747 */ 3748 void 3749 hat_delete_callback(caddr_t vaddr, uint_t len, void *pvt, uint_t flags, 3750 void *cookie) 3751 { 3752 struct hmehash_bucket *hmebp; 3753 hmeblk_tag hblktag; 3754 struct hme_blk *hmeblkp; 3755 int hmeshift, hashno; 3756 caddr_t saddr; 3757 struct pa_hment *pahmep; 3758 struct sf_hment *sfhmep, *osfhmep; 3759 kmutex_t *pml; 3760 tte_t tte; 3761 page_t *pp; 3762 vnode_t *vp; 3763 u_offset_t off; 3764 int locked = 0; 3765 3766 /* 3767 * If the cookie is HAC_COOKIE_NONE then there is no pa_hment to 3768 * remove so just return. 3769 */ 3770 if (cookie == HAC_COOKIE_NONE || IS_KPM_ADDR(vaddr)) 3771 return; 3772 3773 saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK); 3774 3775 rehash: 3776 /* Find the mapping(s) for this page */ 3777 for (hashno = TTE64K, hmeblkp = NULL; 3778 hmeblkp == NULL && hashno <= mmu_hashcnt; 3779 hashno++) { 3780 hmeshift = HME_HASH_SHIFT(hashno); 3781 hblktag.htag_id = ksfmmup; 3782 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift); 3783 hblktag.htag_rehash = hashno; 3784 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift); 3785 3786 SFMMU_HASH_LOCK(hmebp); 3787 3788 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 3789 3790 if (hmeblkp == NULL) 3791 SFMMU_HASH_UNLOCK(hmebp); 3792 } 3793 3794 if (hmeblkp == NULL) 3795 return; 3796 3797 HBLKTOHME(osfhmep, hmeblkp, saddr); 3798 3799 sfmmu_copytte(&osfhmep->hme_tte, &tte); 3800 if (!TTE_IS_VALID(&tte)) { 3801 SFMMU_HASH_UNLOCK(hmebp); 3802 return; 3803 } 3804 3805 pp = osfhmep->hme_page; 3806 if (pp == NULL) { 3807 SFMMU_HASH_UNLOCK(hmebp); 3808 ASSERT(cookie == NULL); 3809 return; 3810 } 3811 3812 vp = pp->p_vnode; 3813 off = pp->p_offset; 3814 3815 pml = sfmmu_mlist_enter(pp); 3816 3817 if (flags & HAC_PAGELOCK) { 3818 if (!page_trylock(pp, SE_SHARED)) { 3819 /* 3820 * Somebody is holding SE_EXCL lock. Might 3821 * even be hat_page_relocate(). Drop all 3822 * our locks, lookup the page in &kvp, and 3823 * retry. If it doesn't exist in &kvp, then 3824 * we must be dealing with a kernel mapped 3825 * page which doesn't actually belong to 3826 * segkmem so we punt. 3827 */ 3828 sfmmu_mlist_exit(pml); 3829 SFMMU_HASH_UNLOCK(hmebp); 3830 pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED); 3831 if (pp == NULL) { 3832 ASSERT(cookie == NULL); 3833 return; 3834 } 3835 page_unlock(pp); 3836 goto rehash; 3837 } 3838 locked = 1; 3839 } 3840 3841 ASSERT(PAGE_LOCKED(pp)); 3842 3843 if (osfhmep->hme_page != pp || pp->p_vnode != vp || 3844 pp->p_offset != off) { 3845 /* 3846 * The page moved before we got our hands on it. Drop 3847 * all the locks and try again. 3848 */ 3849 ASSERT((flags & HAC_PAGELOCK) != 0); 3850 sfmmu_mlist_exit(pml); 3851 SFMMU_HASH_UNLOCK(hmebp); 3852 page_unlock(pp); 3853 locked = 0; 3854 goto rehash; 3855 } 3856 3857 if (vp != &kvp) { 3858 /* 3859 * This is not a segkmem page but another page which 3860 * has been kernel mapped. 3861 */ 3862 sfmmu_mlist_exit(pml); 3863 SFMMU_HASH_UNLOCK(hmebp); 3864 if (locked) 3865 page_unlock(pp); 3866 ASSERT(cookie == NULL); 3867 return; 3868 } 3869 3870 if (cookie != NULL) { 3871 pahmep = (struct pa_hment *)cookie; 3872 sfhmep = &pahmep->sfment; 3873 } else { 3874 for (sfhmep = pp->p_mapping; sfhmep != NULL; 3875 sfhmep = sfhmep->hme_next) { 3876 3877 /* 3878 * skip va<->pa mappings 3879 */ 3880 if (!IS_PAHME(sfhmep)) 3881 continue; 3882 3883 pahmep = sfhmep->hme_data; 3884 ASSERT(pahmep != NULL); 3885 3886 /* 3887 * if pa_hment matches, remove it 3888 */ 3889 if ((pahmep->pvt == pvt) && 3890 (pahmep->addr == vaddr) && 3891 (pahmep->len == len)) { 3892 break; 3893 } 3894 } 3895 } 3896 3897 if (sfhmep == NULL) { 3898 if (!panicstr) { 3899 panic("hat_delete_callback: pa_hment not found, pp %p", 3900 (void *)pp); 3901 } 3902 return; 3903 } 3904 3905 /* 3906 * Note: at this point a valid kernel mapping must still be 3907 * present on this page. 3908 */ 3909 pp->p_share--; 3910 if (pp->p_share <= 0) 3911 panic("hat_delete_callback: zero p_share"); 3912 3913 if (--pahmep->refcnt == 0) { 3914 if (pahmep->flags != 0) 3915 panic("hat_delete_callback: pa_hment is busy"); 3916 3917 /* 3918 * Remove sfhmep from the mapping list for the page. 3919 */ 3920 if (sfhmep->hme_prev) { 3921 sfhmep->hme_prev->hme_next = sfhmep->hme_next; 3922 } else { 3923 pp->p_mapping = sfhmep->hme_next; 3924 } 3925 3926 if (sfhmep->hme_next) 3927 sfhmep->hme_next->hme_prev = sfhmep->hme_prev; 3928 3929 sfmmu_mlist_exit(pml); 3930 SFMMU_HASH_UNLOCK(hmebp); 3931 3932 if (locked) 3933 page_unlock(pp); 3934 3935 kmem_cache_free(pa_hment_cache, pahmep); 3936 return; 3937 } 3938 3939 sfmmu_mlist_exit(pml); 3940 SFMMU_HASH_UNLOCK(hmebp); 3941 if (locked) 3942 page_unlock(pp); 3943 } 3944 3945 /* 3946 * hat_probe returns 1 if the translation for the address 'addr' is 3947 * loaded, zero otherwise. 3948 * 3949 * hat_probe should be used only for advisorary purposes because it may 3950 * occasionally return the wrong value. The implementation must guarantee that 3951 * returning the wrong value is a very rare event. hat_probe is used 3952 * to implement optimizations in the segment drivers. 3953 * 3954 */ 3955 int 3956 hat_probe(struct hat *sfmmup, caddr_t addr) 3957 { 3958 pfn_t pfn; 3959 tte_t tte; 3960 3961 ASSERT(sfmmup != NULL); 3962 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3963 3964 ASSERT((sfmmup == ksfmmup) || 3965 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 3966 3967 if (sfmmup == ksfmmup) { 3968 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte)) 3969 == PFN_SUSPENDED) { 3970 sfmmu_vatopfn_suspended(addr, sfmmup, &tte); 3971 } 3972 } else { 3973 pfn = sfmmu_uvatopfn(addr, sfmmup); 3974 } 3975 3976 if (pfn != PFN_INVALID) 3977 return (1); 3978 else 3979 return (0); 3980 } 3981 3982 ssize_t 3983 hat_getpagesize(struct hat *sfmmup, caddr_t addr) 3984 { 3985 tte_t tte; 3986 3987 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 3988 3989 sfmmu_gettte(sfmmup, addr, &tte); 3990 if (TTE_IS_VALID(&tte)) { 3991 return (TTEBYTES(TTE_CSZ(&tte))); 3992 } 3993 return (-1); 3994 } 3995 3996 static void 3997 sfmmu_gettte(struct hat *sfmmup, caddr_t addr, tte_t *ttep) 3998 { 3999 struct hmehash_bucket *hmebp; 4000 hmeblk_tag hblktag; 4001 int hmeshift, hashno = 1; 4002 struct hme_blk *hmeblkp, *list = NULL; 4003 struct sf_hment *sfhmep; 4004 4005 /* support for ISM */ 4006 ism_map_t *ism_map; 4007 ism_blk_t *ism_blkp; 4008 int i; 4009 sfmmu_t *ism_hatid = NULL; 4010 sfmmu_t *locked_hatid = NULL; 4011 4012 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 4013 4014 ism_blkp = sfmmup->sfmmu_iblk; 4015 if (ism_blkp) { 4016 sfmmu_ismhat_enter(sfmmup, 0); 4017 locked_hatid = sfmmup; 4018 } 4019 while (ism_blkp && ism_hatid == NULL) { 4020 ism_map = ism_blkp->iblk_maps; 4021 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 4022 if (addr >= ism_start(ism_map[i]) && 4023 addr < ism_end(ism_map[i])) { 4024 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 4025 addr = (caddr_t)(addr - 4026 ism_start(ism_map[i])); 4027 break; 4028 } 4029 } 4030 ism_blkp = ism_blkp->iblk_next; 4031 } 4032 if (locked_hatid) { 4033 sfmmu_ismhat_exit(locked_hatid, 0); 4034 } 4035 4036 hblktag.htag_id = sfmmup; 4037 ttep->ll = 0; 4038 4039 do { 4040 hmeshift = HME_HASH_SHIFT(hashno); 4041 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4042 hblktag.htag_rehash = hashno; 4043 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4044 4045 SFMMU_HASH_LOCK(hmebp); 4046 4047 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4048 if (hmeblkp != NULL) { 4049 HBLKTOHME(sfhmep, hmeblkp, addr); 4050 sfmmu_copytte(&sfhmep->hme_tte, ttep); 4051 SFMMU_HASH_UNLOCK(hmebp); 4052 break; 4053 } 4054 SFMMU_HASH_UNLOCK(hmebp); 4055 hashno++; 4056 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 4057 4058 sfmmu_hblks_list_purge(&list); 4059 } 4060 4061 uint_t 4062 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr) 4063 { 4064 tte_t tte; 4065 4066 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 4067 4068 sfmmu_gettte(sfmmup, addr, &tte); 4069 if (TTE_IS_VALID(&tte)) { 4070 *attr = sfmmu_ptov_attr(&tte); 4071 return (0); 4072 } 4073 *attr = 0; 4074 return ((uint_t)0xffffffff); 4075 } 4076 4077 /* 4078 * Enables more attributes on specified address range (ie. logical OR) 4079 */ 4080 void 4081 hat_setattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4082 { 4083 if (hat->sfmmu_xhat_provider) { 4084 XHAT_SETATTR(hat, addr, len, attr); 4085 return; 4086 } else { 4087 /* 4088 * This must be a CPU HAT. If the address space has 4089 * XHATs attached, change attributes for all of them, 4090 * just in case 4091 */ 4092 ASSERT(hat->sfmmu_as != NULL); 4093 if (hat->sfmmu_as->a_xhat != NULL) 4094 xhat_setattr_all(hat->sfmmu_as, addr, len, attr); 4095 } 4096 4097 sfmmu_chgattr(hat, addr, len, attr, SFMMU_SETATTR); 4098 } 4099 4100 /* 4101 * Assigns attributes to the specified address range. All the attributes 4102 * are specified. 4103 */ 4104 void 4105 hat_chgattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4106 { 4107 if (hat->sfmmu_xhat_provider) { 4108 XHAT_CHGATTR(hat, addr, len, attr); 4109 return; 4110 } else { 4111 /* 4112 * This must be a CPU HAT. If the address space has 4113 * XHATs attached, change attributes for all of them, 4114 * just in case 4115 */ 4116 ASSERT(hat->sfmmu_as != NULL); 4117 if (hat->sfmmu_as->a_xhat != NULL) 4118 xhat_chgattr_all(hat->sfmmu_as, addr, len, attr); 4119 } 4120 4121 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CHGATTR); 4122 } 4123 4124 /* 4125 * Remove attributes on the specified address range (ie. loginal NAND) 4126 */ 4127 void 4128 hat_clrattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr) 4129 { 4130 if (hat->sfmmu_xhat_provider) { 4131 XHAT_CLRATTR(hat, addr, len, attr); 4132 return; 4133 } else { 4134 /* 4135 * This must be a CPU HAT. If the address space has 4136 * XHATs attached, change attributes for all of them, 4137 * just in case 4138 */ 4139 ASSERT(hat->sfmmu_as != NULL); 4140 if (hat->sfmmu_as->a_xhat != NULL) 4141 xhat_clrattr_all(hat->sfmmu_as, addr, len, attr); 4142 } 4143 4144 sfmmu_chgattr(hat, addr, len, attr, SFMMU_CLRATTR); 4145 } 4146 4147 /* 4148 * Change attributes on an address range to that specified by attr and mode. 4149 */ 4150 static void 4151 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr, 4152 int mode) 4153 { 4154 struct hmehash_bucket *hmebp; 4155 hmeblk_tag hblktag; 4156 int hmeshift, hashno = 1; 4157 struct hme_blk *hmeblkp, *list = NULL; 4158 caddr_t endaddr; 4159 cpuset_t cpuset; 4160 demap_range_t dmr; 4161 4162 CPUSET_ZERO(cpuset); 4163 4164 ASSERT((sfmmup == ksfmmup) || 4165 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4166 ASSERT((len & MMU_PAGEOFFSET) == 0); 4167 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4168 4169 if ((attr & PROT_USER) && (mode != SFMMU_CLRATTR) && 4170 ((addr + len) > (caddr_t)USERLIMIT)) { 4171 panic("user addr %p in kernel space", 4172 (void *)addr); 4173 } 4174 4175 endaddr = addr + len; 4176 hblktag.htag_id = sfmmup; 4177 DEMAP_RANGE_INIT(sfmmup, &dmr); 4178 4179 while (addr < endaddr) { 4180 hmeshift = HME_HASH_SHIFT(hashno); 4181 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4182 hblktag.htag_rehash = hashno; 4183 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4184 4185 SFMMU_HASH_LOCK(hmebp); 4186 4187 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4188 if (hmeblkp != NULL) { 4189 /* 4190 * We've encountered a shadow hmeblk so skip the range 4191 * of the next smaller mapping size. 4192 */ 4193 if (hmeblkp->hblk_shw_bit) { 4194 ASSERT(sfmmup != ksfmmup); 4195 ASSERT(hashno > 1); 4196 addr = (caddr_t)P2END((uintptr_t)addr, 4197 TTEBYTES(hashno - 1)); 4198 } else { 4199 addr = sfmmu_hblk_chgattr(sfmmup, 4200 hmeblkp, addr, endaddr, &dmr, attr, mode); 4201 } 4202 SFMMU_HASH_UNLOCK(hmebp); 4203 hashno = 1; 4204 continue; 4205 } 4206 SFMMU_HASH_UNLOCK(hmebp); 4207 4208 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4209 /* 4210 * We have traversed the whole list and rehashed 4211 * if necessary without finding the address to chgattr. 4212 * This is ok, so we increment the address by the 4213 * smallest hmeblk range for kernel mappings or for 4214 * user mappings with no large pages, and the largest 4215 * hmeblk range, to account for shadow hmeblks, for 4216 * user mappings with large pages and continue. 4217 */ 4218 if (sfmmup == ksfmmup) 4219 addr = (caddr_t)P2END((uintptr_t)addr, 4220 TTEBYTES(1)); 4221 else 4222 addr = (caddr_t)P2END((uintptr_t)addr, 4223 TTEBYTES(hashno)); 4224 hashno = 1; 4225 } else { 4226 hashno++; 4227 } 4228 } 4229 4230 sfmmu_hblks_list_purge(&list); 4231 DEMAP_RANGE_FLUSH(&dmr); 4232 cpuset = sfmmup->sfmmu_cpusran; 4233 xt_sync(cpuset); 4234 } 4235 4236 /* 4237 * This function chgattr on a range of addresses in an hmeblk. It returns the 4238 * next addres that needs to be chgattr. 4239 * It should be called with the hash lock held. 4240 * XXX It should be possible to optimize chgattr by not flushing every time but 4241 * on the other hand: 4242 * 1. do one flush crosscall. 4243 * 2. only flush if we are increasing permissions (make sure this will work) 4244 */ 4245 static caddr_t 4246 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4247 caddr_t endaddr, demap_range_t *dmrp, uint_t attr, int mode) 4248 { 4249 tte_t tte, tteattr, tteflags, ttemod; 4250 struct sf_hment *sfhmep; 4251 int ttesz; 4252 struct page *pp = NULL; 4253 kmutex_t *pml, *pmtx; 4254 int ret; 4255 int use_demap_range; 4256 #if defined(SF_ERRATA_57) 4257 int check_exec; 4258 #endif 4259 4260 ASSERT(in_hblk_range(hmeblkp, addr)); 4261 ASSERT(hmeblkp->hblk_shw_bit == 0); 4262 4263 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4264 ttesz = get_hblk_ttesz(hmeblkp); 4265 4266 /* 4267 * Flush the current demap region if addresses have been 4268 * skipped or the page size doesn't match. 4269 */ 4270 use_demap_range = (TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp)); 4271 if (use_demap_range) { 4272 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4273 } else { 4274 DEMAP_RANGE_FLUSH(dmrp); 4275 } 4276 4277 tteattr.ll = sfmmu_vtop_attr(attr, mode, &tteflags); 4278 #if defined(SF_ERRATA_57) 4279 check_exec = (sfmmup != ksfmmup) && 4280 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4281 TTE_IS_EXECUTABLE(&tteattr); 4282 #endif 4283 HBLKTOHME(sfhmep, hmeblkp, addr); 4284 while (addr < endaddr) { 4285 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4286 if (TTE_IS_VALID(&tte)) { 4287 if ((tte.ll & tteflags.ll) == tteattr.ll) { 4288 /* 4289 * if the new attr is the same as old 4290 * continue 4291 */ 4292 goto next_addr; 4293 } 4294 if (!TTE_IS_WRITABLE(&tteattr)) { 4295 /* 4296 * make sure we clear hw modify bit if we 4297 * removing write protections 4298 */ 4299 tteflags.tte_intlo |= TTE_HWWR_INT; 4300 } 4301 4302 pml = NULL; 4303 pp = sfhmep->hme_page; 4304 if (pp) { 4305 pml = sfmmu_mlist_enter(pp); 4306 } 4307 4308 if (pp != sfhmep->hme_page) { 4309 /* 4310 * tte must have been unloaded. 4311 */ 4312 ASSERT(pml); 4313 sfmmu_mlist_exit(pml); 4314 continue; 4315 } 4316 4317 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4318 4319 ttemod = tte; 4320 ttemod.ll = (ttemod.ll & ~tteflags.ll) | tteattr.ll; 4321 ASSERT(TTE_TO_TTEPFN(&ttemod) == TTE_TO_TTEPFN(&tte)); 4322 4323 #if defined(SF_ERRATA_57) 4324 if (check_exec && addr < errata57_limit) 4325 ttemod.tte_exec_perm = 0; 4326 #endif 4327 ret = sfmmu_modifytte_try(&tte, &ttemod, 4328 &sfhmep->hme_tte); 4329 4330 if (ret < 0) { 4331 /* tte changed underneath us */ 4332 if (pml) { 4333 sfmmu_mlist_exit(pml); 4334 } 4335 continue; 4336 } 4337 4338 if (tteflags.tte_intlo & TTE_HWWR_INT) { 4339 /* 4340 * need to sync if we are clearing modify bit. 4341 */ 4342 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4343 } 4344 4345 if (pp && PP_ISRO(pp)) { 4346 if (tteattr.tte_intlo & TTE_WRPRM_INT) { 4347 pmtx = sfmmu_page_enter(pp); 4348 PP_CLRRO(pp); 4349 sfmmu_page_exit(pmtx); 4350 } 4351 } 4352 4353 if (ret > 0 && use_demap_range) { 4354 DEMAP_RANGE_MARKPG(dmrp, addr); 4355 } else if (ret > 0) { 4356 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4357 } 4358 4359 if (pml) { 4360 sfmmu_mlist_exit(pml); 4361 } 4362 } 4363 next_addr: 4364 addr += TTEBYTES(ttesz); 4365 sfhmep++; 4366 DEMAP_RANGE_NEXTPG(dmrp); 4367 } 4368 return (addr); 4369 } 4370 4371 /* 4372 * This routine converts virtual attributes to physical ones. It will 4373 * update the tteflags field with the tte mask corresponding to the attributes 4374 * affected and it returns the new attributes. It will also clear the modify 4375 * bit if we are taking away write permission. This is necessary since the 4376 * modify bit is the hardware permission bit and we need to clear it in order 4377 * to detect write faults. 4378 */ 4379 static uint64_t 4380 sfmmu_vtop_attr(uint_t attr, int mode, tte_t *ttemaskp) 4381 { 4382 tte_t ttevalue; 4383 4384 ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR)); 4385 4386 switch (mode) { 4387 case SFMMU_CHGATTR: 4388 /* all attributes specified */ 4389 ttevalue.tte_inthi = MAKE_TTEATTR_INTHI(attr); 4390 ttevalue.tte_intlo = MAKE_TTEATTR_INTLO(attr); 4391 ttemaskp->tte_inthi = TTEINTHI_ATTR; 4392 ttemaskp->tte_intlo = TTEINTLO_ATTR; 4393 break; 4394 case SFMMU_SETATTR: 4395 ASSERT(!(attr & ~HAT_PROT_MASK)); 4396 ttemaskp->ll = 0; 4397 ttevalue.ll = 0; 4398 /* 4399 * a valid tte implies exec and read for sfmmu 4400 * so no need to do anything about them. 4401 * since priviledged access implies user access 4402 * PROT_USER doesn't make sense either. 4403 */ 4404 if (attr & PROT_WRITE) { 4405 ttemaskp->tte_intlo |= TTE_WRPRM_INT; 4406 ttevalue.tte_intlo |= TTE_WRPRM_INT; 4407 } 4408 break; 4409 case SFMMU_CLRATTR: 4410 /* attributes will be nand with current ones */ 4411 if (attr & ~(PROT_WRITE | PROT_USER)) { 4412 panic("sfmmu: attr %x not supported", attr); 4413 } 4414 ttemaskp->ll = 0; 4415 ttevalue.ll = 0; 4416 if (attr & PROT_WRITE) { 4417 /* clear both writable and modify bit */ 4418 ttemaskp->tte_intlo |= TTE_WRPRM_INT | TTE_HWWR_INT; 4419 } 4420 if (attr & PROT_USER) { 4421 ttemaskp->tte_intlo |= TTE_PRIV_INT; 4422 ttevalue.tte_intlo |= TTE_PRIV_INT; 4423 } 4424 break; 4425 default: 4426 panic("sfmmu_vtop_attr: bad mode %x", mode); 4427 } 4428 ASSERT(TTE_TO_TTEPFN(&ttevalue) == 0); 4429 return (ttevalue.ll); 4430 } 4431 4432 static uint_t 4433 sfmmu_ptov_attr(tte_t *ttep) 4434 { 4435 uint_t attr; 4436 4437 ASSERT(TTE_IS_VALID(ttep)); 4438 4439 attr = PROT_READ; 4440 4441 if (TTE_IS_WRITABLE(ttep)) { 4442 attr |= PROT_WRITE; 4443 } 4444 if (TTE_IS_EXECUTABLE(ttep)) { 4445 attr |= PROT_EXEC; 4446 } 4447 if (!TTE_IS_PRIVILEGED(ttep)) { 4448 attr |= PROT_USER; 4449 } 4450 if (TTE_IS_NFO(ttep)) { 4451 attr |= HAT_NOFAULT; 4452 } 4453 if (TTE_IS_NOSYNC(ttep)) { 4454 attr |= HAT_NOSYNC; 4455 } 4456 if (TTE_IS_SIDEFFECT(ttep)) { 4457 attr |= SFMMU_SIDEFFECT; 4458 } 4459 if (!TTE_IS_VCACHEABLE(ttep)) { 4460 attr |= SFMMU_UNCACHEVTTE; 4461 } 4462 if (!TTE_IS_PCACHEABLE(ttep)) { 4463 attr |= SFMMU_UNCACHEPTTE; 4464 } 4465 return (attr); 4466 } 4467 4468 /* 4469 * hat_chgprot is a deprecated hat call. New segment drivers 4470 * should store all attributes and use hat_*attr calls. 4471 * 4472 * Change the protections in the virtual address range 4473 * given to the specified virtual protection. If vprot is ~PROT_WRITE, 4474 * then remove write permission, leaving the other 4475 * permissions unchanged. If vprot is ~PROT_USER, remove user permissions. 4476 * 4477 */ 4478 void 4479 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot) 4480 { 4481 struct hmehash_bucket *hmebp; 4482 hmeblk_tag hblktag; 4483 int hmeshift, hashno = 1; 4484 struct hme_blk *hmeblkp, *list = NULL; 4485 caddr_t endaddr; 4486 cpuset_t cpuset; 4487 demap_range_t dmr; 4488 4489 ASSERT((len & MMU_PAGEOFFSET) == 0); 4490 ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0); 4491 4492 if (sfmmup->sfmmu_xhat_provider) { 4493 XHAT_CHGPROT(sfmmup, addr, len, vprot); 4494 return; 4495 } else { 4496 /* 4497 * This must be a CPU HAT. If the address space has 4498 * XHATs attached, change attributes for all of them, 4499 * just in case 4500 */ 4501 ASSERT(sfmmup->sfmmu_as != NULL); 4502 if (sfmmup->sfmmu_as->a_xhat != NULL) 4503 xhat_chgprot_all(sfmmup->sfmmu_as, addr, len, vprot); 4504 } 4505 4506 CPUSET_ZERO(cpuset); 4507 4508 if ((vprot != (uint_t)~PROT_WRITE) && (vprot & PROT_USER) && 4509 ((addr + len) > (caddr_t)USERLIMIT)) { 4510 panic("user addr %p vprot %x in kernel space", 4511 (void *)addr, vprot); 4512 } 4513 endaddr = addr + len; 4514 hblktag.htag_id = sfmmup; 4515 DEMAP_RANGE_INIT(sfmmup, &dmr); 4516 4517 while (addr < endaddr) { 4518 hmeshift = HME_HASH_SHIFT(hashno); 4519 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 4520 hblktag.htag_rehash = hashno; 4521 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 4522 4523 SFMMU_HASH_LOCK(hmebp); 4524 4525 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 4526 if (hmeblkp != NULL) { 4527 /* 4528 * We've encountered a shadow hmeblk so skip the range 4529 * of the next smaller mapping size. 4530 */ 4531 if (hmeblkp->hblk_shw_bit) { 4532 ASSERT(sfmmup != ksfmmup); 4533 ASSERT(hashno > 1); 4534 addr = (caddr_t)P2END((uintptr_t)addr, 4535 TTEBYTES(hashno - 1)); 4536 } else { 4537 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp, 4538 addr, endaddr, &dmr, vprot); 4539 } 4540 SFMMU_HASH_UNLOCK(hmebp); 4541 hashno = 1; 4542 continue; 4543 } 4544 SFMMU_HASH_UNLOCK(hmebp); 4545 4546 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 4547 /* 4548 * We have traversed the whole list and rehashed 4549 * if necessary without finding the address to chgprot. 4550 * This is ok so we increment the address by the 4551 * smallest hmeblk range for kernel mappings and the 4552 * largest hmeblk range, to account for shadow hmeblks, 4553 * for user mappings and continue. 4554 */ 4555 if (sfmmup == ksfmmup) 4556 addr = (caddr_t)P2END((uintptr_t)addr, 4557 TTEBYTES(1)); 4558 else 4559 addr = (caddr_t)P2END((uintptr_t)addr, 4560 TTEBYTES(hashno)); 4561 hashno = 1; 4562 } else { 4563 hashno++; 4564 } 4565 } 4566 4567 sfmmu_hblks_list_purge(&list); 4568 DEMAP_RANGE_FLUSH(&dmr); 4569 cpuset = sfmmup->sfmmu_cpusran; 4570 xt_sync(cpuset); 4571 } 4572 4573 /* 4574 * This function chgprots a range of addresses in an hmeblk. It returns the 4575 * next addres that needs to be chgprot. 4576 * It should be called with the hash lock held. 4577 * XXX It shold be possible to optimize chgprot by not flushing every time but 4578 * on the other hand: 4579 * 1. do one flush crosscall. 4580 * 2. only flush if we are increasing permissions (make sure this will work) 4581 */ 4582 static caddr_t 4583 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 4584 caddr_t endaddr, demap_range_t *dmrp, uint_t vprot) 4585 { 4586 uint_t pprot; 4587 tte_t tte, ttemod; 4588 struct sf_hment *sfhmep; 4589 uint_t tteflags; 4590 int ttesz; 4591 struct page *pp = NULL; 4592 kmutex_t *pml, *pmtx; 4593 int ret; 4594 int use_demap_range; 4595 #if defined(SF_ERRATA_57) 4596 int check_exec; 4597 #endif 4598 4599 ASSERT(in_hblk_range(hmeblkp, addr)); 4600 ASSERT(hmeblkp->hblk_shw_bit == 0); 4601 4602 #ifdef DEBUG 4603 if (get_hblk_ttesz(hmeblkp) != TTE8K && 4604 (endaddr < get_hblk_endaddr(hmeblkp))) { 4605 panic("sfmmu_hblk_chgprot: partial chgprot of large page"); 4606 } 4607 #endif /* DEBUG */ 4608 4609 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 4610 ttesz = get_hblk_ttesz(hmeblkp); 4611 4612 pprot = sfmmu_vtop_prot(vprot, &tteflags); 4613 #if defined(SF_ERRATA_57) 4614 check_exec = (sfmmup != ksfmmup) && 4615 AS_TYPE_64BIT(sfmmup->sfmmu_as) && 4616 ((vprot & PROT_EXEC) == PROT_EXEC); 4617 #endif 4618 HBLKTOHME(sfhmep, hmeblkp, addr); 4619 4620 /* 4621 * Flush the current demap region if addresses have been 4622 * skipped or the page size doesn't match. 4623 */ 4624 use_demap_range = (TTEBYTES(ttesz) == MMU_PAGESIZE); 4625 if (use_demap_range) { 4626 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 4627 } else { 4628 DEMAP_RANGE_FLUSH(dmrp); 4629 } 4630 4631 while (addr < endaddr) { 4632 sfmmu_copytte(&sfhmep->hme_tte, &tte); 4633 if (TTE_IS_VALID(&tte)) { 4634 if (TTE_GET_LOFLAGS(&tte, tteflags) == pprot) { 4635 /* 4636 * if the new protection is the same as old 4637 * continue 4638 */ 4639 goto next_addr; 4640 } 4641 pml = NULL; 4642 pp = sfhmep->hme_page; 4643 if (pp) { 4644 pml = sfmmu_mlist_enter(pp); 4645 } 4646 if (pp != sfhmep->hme_page) { 4647 /* 4648 * tte most have been unloaded 4649 * underneath us. Recheck 4650 */ 4651 ASSERT(pml); 4652 sfmmu_mlist_exit(pml); 4653 continue; 4654 } 4655 4656 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 4657 4658 ttemod = tte; 4659 TTE_SET_LOFLAGS(&ttemod, tteflags, pprot); 4660 #if defined(SF_ERRATA_57) 4661 if (check_exec && addr < errata57_limit) 4662 ttemod.tte_exec_perm = 0; 4663 #endif 4664 ret = sfmmu_modifytte_try(&tte, &ttemod, 4665 &sfhmep->hme_tte); 4666 4667 if (ret < 0) { 4668 /* tte changed underneath us */ 4669 if (pml) { 4670 sfmmu_mlist_exit(pml); 4671 } 4672 continue; 4673 } 4674 4675 if (tteflags & TTE_HWWR_INT) { 4676 /* 4677 * need to sync if we are clearing modify bit. 4678 */ 4679 sfmmu_ttesync(sfmmup, addr, &tte, pp); 4680 } 4681 4682 if (pp && PP_ISRO(pp)) { 4683 if (pprot & TTE_WRPRM_INT) { 4684 pmtx = sfmmu_page_enter(pp); 4685 PP_CLRRO(pp); 4686 sfmmu_page_exit(pmtx); 4687 } 4688 } 4689 4690 if (ret > 0 && use_demap_range) { 4691 DEMAP_RANGE_MARKPG(dmrp, addr); 4692 } else if (ret > 0) { 4693 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 4694 } 4695 4696 if (pml) { 4697 sfmmu_mlist_exit(pml); 4698 } 4699 } 4700 next_addr: 4701 addr += TTEBYTES(ttesz); 4702 sfhmep++; 4703 DEMAP_RANGE_NEXTPG(dmrp); 4704 } 4705 return (addr); 4706 } 4707 4708 /* 4709 * This routine is deprecated and should only be used by hat_chgprot. 4710 * The correct routine is sfmmu_vtop_attr. 4711 * This routine converts virtual page protections to physical ones. It will 4712 * update the tteflags field with the tte mask corresponding to the protections 4713 * affected and it returns the new protections. It will also clear the modify 4714 * bit if we are taking away write permission. This is necessary since the 4715 * modify bit is the hardware permission bit and we need to clear it in order 4716 * to detect write faults. 4717 * It accepts the following special protections: 4718 * ~PROT_WRITE = remove write permissions. 4719 * ~PROT_USER = remove user permissions. 4720 */ 4721 static uint_t 4722 sfmmu_vtop_prot(uint_t vprot, uint_t *tteflagsp) 4723 { 4724 if (vprot == (uint_t)~PROT_WRITE) { 4725 *tteflagsp = TTE_WRPRM_INT | TTE_HWWR_INT; 4726 return (0); /* will cause wrprm to be cleared */ 4727 } 4728 if (vprot == (uint_t)~PROT_USER) { 4729 *tteflagsp = TTE_PRIV_INT; 4730 return (0); /* will cause privprm to be cleared */ 4731 } 4732 if ((vprot == 0) || (vprot == PROT_USER) || 4733 ((vprot & PROT_ALL) != vprot)) { 4734 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4735 } 4736 4737 switch (vprot) { 4738 case (PROT_READ): 4739 case (PROT_EXEC): 4740 case (PROT_EXEC | PROT_READ): 4741 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4742 return (TTE_PRIV_INT); /* set prv and clr wrt */ 4743 case (PROT_WRITE): 4744 case (PROT_WRITE | PROT_READ): 4745 case (PROT_EXEC | PROT_WRITE): 4746 case (PROT_EXEC | PROT_WRITE | PROT_READ): 4747 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4748 return (TTE_PRIV_INT | TTE_WRPRM_INT); /* set prv and wrt */ 4749 case (PROT_USER | PROT_READ): 4750 case (PROT_USER | PROT_EXEC): 4751 case (PROT_USER | PROT_EXEC | PROT_READ): 4752 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT; 4753 return (0); /* clr prv and wrt */ 4754 case (PROT_USER | PROT_WRITE): 4755 case (PROT_USER | PROT_WRITE | PROT_READ): 4756 case (PROT_USER | PROT_EXEC | PROT_WRITE): 4757 case (PROT_USER | PROT_EXEC | PROT_WRITE | PROT_READ): 4758 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT; 4759 return (TTE_WRPRM_INT); /* clr prv and set wrt */ 4760 default: 4761 panic("sfmmu_vtop_prot -- bad prot %x", vprot); 4762 } 4763 return (0); 4764 } 4765 4766 /* 4767 * Alternate unload for very large virtual ranges. With a true 64 bit VA, 4768 * the normal algorithm would take too long for a very large VA range with 4769 * few real mappings. This routine just walks thru all HMEs in the global 4770 * hash table to find and remove mappings. 4771 */ 4772 static void 4773 hat_unload_large_virtual( 4774 struct hat *sfmmup, 4775 caddr_t startaddr, 4776 size_t len, 4777 uint_t flags, 4778 hat_callback_t *callback) 4779 { 4780 struct hmehash_bucket *hmebp; 4781 struct hme_blk *hmeblkp; 4782 struct hme_blk *pr_hblk = NULL; 4783 struct hme_blk *nx_hblk; 4784 struct hme_blk *list = NULL; 4785 int i; 4786 uint64_t hblkpa, prevpa, nx_pa; 4787 demap_range_t dmr, *dmrp; 4788 cpuset_t cpuset; 4789 caddr_t endaddr = startaddr + len; 4790 caddr_t sa; 4791 caddr_t ea; 4792 caddr_t cb_sa[MAX_CB_ADDR]; 4793 caddr_t cb_ea[MAX_CB_ADDR]; 4794 int addr_cnt = 0; 4795 int a = 0; 4796 4797 if (sfmmup->sfmmu_free) { 4798 dmrp = NULL; 4799 } else { 4800 dmrp = &dmr; 4801 DEMAP_RANGE_INIT(sfmmup, dmrp); 4802 } 4803 4804 /* 4805 * Loop through all the hash buckets of HME blocks looking for matches. 4806 */ 4807 for (i = 0; i <= UHMEHASH_SZ; i++) { 4808 hmebp = &uhme_hash[i]; 4809 SFMMU_HASH_LOCK(hmebp); 4810 hmeblkp = hmebp->hmeblkp; 4811 hblkpa = hmebp->hmeh_nextpa; 4812 prevpa = 0; 4813 pr_hblk = NULL; 4814 while (hmeblkp) { 4815 nx_hblk = hmeblkp->hblk_next; 4816 nx_pa = hmeblkp->hblk_nextpa; 4817 4818 /* 4819 * skip if not this context, if a shadow block or 4820 * if the mapping is not in the requested range 4821 */ 4822 if (hmeblkp->hblk_tag.htag_id != sfmmup || 4823 hmeblkp->hblk_shw_bit || 4824 (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr || 4825 (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) { 4826 pr_hblk = hmeblkp; 4827 prevpa = hblkpa; 4828 goto next_block; 4829 } 4830 4831 /* 4832 * unload if there are any current valid mappings 4833 */ 4834 if (hmeblkp->hblk_vcnt != 0 || 4835 hmeblkp->hblk_hmecnt != 0) 4836 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 4837 sa, ea, dmrp, flags); 4838 4839 /* 4840 * on unmap we also release the HME block itself, once 4841 * all mappings are gone. 4842 */ 4843 if ((flags & HAT_UNLOAD_UNMAP) != 0 && 4844 !hmeblkp->hblk_vcnt && 4845 !hmeblkp->hblk_hmecnt) { 4846 ASSERT(!hmeblkp->hblk_lckcnt); 4847 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 4848 prevpa, pr_hblk); 4849 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 4850 } else { 4851 pr_hblk = hmeblkp; 4852 prevpa = hblkpa; 4853 } 4854 4855 if (callback == NULL) 4856 goto next_block; 4857 4858 /* 4859 * HME blocks may span more than one page, but we may be 4860 * unmapping only one page, so check for a smaller range 4861 * for the callback 4862 */ 4863 if (sa < startaddr) 4864 sa = startaddr; 4865 if (--ea > endaddr) 4866 ea = endaddr - 1; 4867 4868 cb_sa[addr_cnt] = sa; 4869 cb_ea[addr_cnt] = ea; 4870 if (++addr_cnt == MAX_CB_ADDR) { 4871 if (dmrp != NULL) { 4872 DEMAP_RANGE_FLUSH(dmrp); 4873 cpuset = sfmmup->sfmmu_cpusran; 4874 xt_sync(cpuset); 4875 } 4876 4877 for (a = 0; a < MAX_CB_ADDR; ++a) { 4878 callback->hcb_start_addr = cb_sa[a]; 4879 callback->hcb_end_addr = cb_ea[a]; 4880 callback->hcb_function(callback); 4881 } 4882 addr_cnt = 0; 4883 } 4884 4885 next_block: 4886 hmeblkp = nx_hblk; 4887 hblkpa = nx_pa; 4888 } 4889 SFMMU_HASH_UNLOCK(hmebp); 4890 } 4891 4892 sfmmu_hblks_list_purge(&list); 4893 if (dmrp != NULL) { 4894 DEMAP_RANGE_FLUSH(dmrp); 4895 cpuset = sfmmup->sfmmu_cpusran; 4896 xt_sync(cpuset); 4897 } 4898 4899 for (a = 0; a < addr_cnt; ++a) { 4900 callback->hcb_start_addr = cb_sa[a]; 4901 callback->hcb_end_addr = cb_ea[a]; 4902 callback->hcb_function(callback); 4903 } 4904 4905 /* 4906 * Check TSB and TLB page sizes if the process isn't exiting. 4907 */ 4908 if (!sfmmup->sfmmu_free) 4909 sfmmu_check_page_sizes(sfmmup, 0); 4910 } 4911 4912 /* 4913 * Unload all the mappings in the range [addr..addr+len). addr and len must 4914 * be MMU_PAGESIZE aligned. 4915 */ 4916 4917 extern struct seg *segkmap; 4918 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \ 4919 segkmap->s_base <= (addr) && (addr) < (segkmap->s_base + segkmap->s_size)) 4920 4921 4922 void 4923 hat_unload_callback( 4924 struct hat *sfmmup, 4925 caddr_t addr, 4926 size_t len, 4927 uint_t flags, 4928 hat_callback_t *callback) 4929 { 4930 struct hmehash_bucket *hmebp; 4931 hmeblk_tag hblktag; 4932 int hmeshift, hashno, iskernel; 4933 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; 4934 caddr_t endaddr; 4935 cpuset_t cpuset; 4936 uint64_t hblkpa, prevpa; 4937 int addr_count = 0; 4938 int a; 4939 caddr_t cb_start_addr[MAX_CB_ADDR]; 4940 caddr_t cb_end_addr[MAX_CB_ADDR]; 4941 int issegkmap = ISSEGKMAP(sfmmup, addr); 4942 demap_range_t dmr, *dmrp; 4943 4944 if (sfmmup->sfmmu_xhat_provider) { 4945 XHAT_UNLOAD_CALLBACK(sfmmup, addr, len, flags, callback); 4946 return; 4947 } else { 4948 /* 4949 * This must be a CPU HAT. If the address space has 4950 * XHATs attached, unload the mappings for all of them, 4951 * just in case 4952 */ 4953 ASSERT(sfmmup->sfmmu_as != NULL); 4954 if (sfmmup->sfmmu_as->a_xhat != NULL) 4955 xhat_unload_callback_all(sfmmup->sfmmu_as, addr, 4956 len, flags, callback); 4957 } 4958 4959 ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \ 4960 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 4961 4962 ASSERT(sfmmup != NULL); 4963 ASSERT((len & MMU_PAGEOFFSET) == 0); 4964 ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET)); 4965 4966 /* 4967 * Probing through a large VA range (say 63 bits) will be slow, even 4968 * at 4 Meg steps between the probes. So, when the virtual address range 4969 * is very large, search the HME entries for what to unload. 4970 * 4971 * len >> TTE_PAGE_SHIFT(TTE4M) is the # of 4Meg probes we'd need 4972 * 4973 * UHMEHASH_SZ is number of hash buckets to examine 4974 * 4975 */ 4976 if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) { 4977 hat_unload_large_virtual(sfmmup, addr, len, flags, callback); 4978 return; 4979 } 4980 4981 CPUSET_ZERO(cpuset); 4982 4983 /* 4984 * If the process is exiting, we can save a lot of fuss since 4985 * we'll flush the TLB when we free the ctx anyway. 4986 */ 4987 if (sfmmup->sfmmu_free) 4988 dmrp = NULL; 4989 else 4990 dmrp = &dmr; 4991 4992 DEMAP_RANGE_INIT(sfmmup, dmrp); 4993 endaddr = addr + len; 4994 hblktag.htag_id = sfmmup; 4995 4996 /* 4997 * It is likely for the vm to call unload over a wide range of 4998 * addresses that are actually very sparsely populated by 4999 * translations. In order to speed this up the sfmmu hat supports 5000 * the concept of shadow hmeblks. Dummy large page hmeblks that 5001 * correspond to actual small translations are allocated at tteload 5002 * time and are referred to as shadow hmeblks. Now, during unload 5003 * time, we first check if we have a shadow hmeblk for that 5004 * translation. The absence of one means the corresponding address 5005 * range is empty and can be skipped. 5006 * 5007 * The kernel is an exception to above statement and that is why 5008 * we don't use shadow hmeblks and hash starting from the smallest 5009 * page size. 5010 */ 5011 if (sfmmup == KHATID) { 5012 iskernel = 1; 5013 hashno = TTE64K; 5014 } else { 5015 iskernel = 0; 5016 if (mmu_page_sizes == max_mmu_page_sizes) { 5017 hashno = TTE256M; 5018 } else { 5019 hashno = TTE4M; 5020 } 5021 } 5022 while (addr < endaddr) { 5023 hmeshift = HME_HASH_SHIFT(hashno); 5024 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5025 hblktag.htag_rehash = hashno; 5026 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5027 5028 SFMMU_HASH_LOCK(hmebp); 5029 5030 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, hblkpa, pr_hblk, 5031 prevpa, &list); 5032 if (hmeblkp == NULL) { 5033 /* 5034 * didn't find an hmeblk. skip the appropiate 5035 * address range. 5036 */ 5037 SFMMU_HASH_UNLOCK(hmebp); 5038 if (iskernel) { 5039 if (hashno < mmu_hashcnt) { 5040 hashno++; 5041 continue; 5042 } else { 5043 hashno = TTE64K; 5044 addr = (caddr_t)roundup((uintptr_t)addr 5045 + 1, MMU_PAGESIZE64K); 5046 continue; 5047 } 5048 } 5049 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5050 (1 << hmeshift)); 5051 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5052 ASSERT(hashno == TTE64K); 5053 continue; 5054 } 5055 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5056 hashno = TTE512K; 5057 continue; 5058 } 5059 if (mmu_page_sizes == max_mmu_page_sizes) { 5060 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5061 hashno = TTE4M; 5062 continue; 5063 } 5064 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5065 hashno = TTE32M; 5066 continue; 5067 } 5068 hashno = TTE256M; 5069 continue; 5070 } else { 5071 hashno = TTE4M; 5072 continue; 5073 } 5074 } 5075 ASSERT(hmeblkp); 5076 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5077 /* 5078 * If the valid count is zero we can skip the range 5079 * mapped by this hmeblk. 5080 * We free hblks in the case of HAT_UNMAP. HAT_UNMAP 5081 * is used by segment drivers as a hint 5082 * that the mapping resource won't be used any longer. 5083 * The best example of this is during exit(). 5084 */ 5085 addr = (caddr_t)roundup((uintptr_t)addr + 1, 5086 get_hblk_span(hmeblkp)); 5087 if ((flags & HAT_UNLOAD_UNMAP) || 5088 (iskernel && !issegkmap)) { 5089 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5090 pr_hblk); 5091 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5092 } 5093 SFMMU_HASH_UNLOCK(hmebp); 5094 5095 if (iskernel) { 5096 hashno = TTE64K; 5097 continue; 5098 } 5099 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5100 ASSERT(hashno == TTE64K); 5101 continue; 5102 } 5103 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5104 hashno = TTE512K; 5105 continue; 5106 } 5107 if (mmu_page_sizes == max_mmu_page_sizes) { 5108 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5109 hashno = TTE4M; 5110 continue; 5111 } 5112 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5113 hashno = TTE32M; 5114 continue; 5115 } 5116 hashno = TTE256M; 5117 continue; 5118 } else { 5119 hashno = TTE4M; 5120 continue; 5121 } 5122 } 5123 if (hmeblkp->hblk_shw_bit) { 5124 /* 5125 * If we encounter a shadow hmeblk we know there is 5126 * smaller sized hmeblks mapping the same address space. 5127 * Decrement the hash size and rehash. 5128 */ 5129 ASSERT(sfmmup != KHATID); 5130 hashno--; 5131 SFMMU_HASH_UNLOCK(hmebp); 5132 continue; 5133 } 5134 5135 /* 5136 * track callback address ranges. 5137 * only start a new range when it's not contiguous 5138 */ 5139 if (callback != NULL) { 5140 if (addr_count > 0 && 5141 addr == cb_end_addr[addr_count - 1]) 5142 --addr_count; 5143 else 5144 cb_start_addr[addr_count] = addr; 5145 } 5146 5147 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr, 5148 dmrp, flags); 5149 5150 if (callback != NULL) 5151 cb_end_addr[addr_count++] = addr; 5152 5153 if (((flags & HAT_UNLOAD_UNMAP) || (iskernel && !issegkmap)) && 5154 !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { 5155 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, 5156 pr_hblk); 5157 sfmmu_hblk_free(hmebp, hmeblkp, hblkpa, &list); 5158 } 5159 SFMMU_HASH_UNLOCK(hmebp); 5160 5161 /* 5162 * Notify our caller as to exactly which pages 5163 * have been unloaded. We do these in clumps, 5164 * to minimize the number of xt_sync()s that need to occur. 5165 */ 5166 if (callback != NULL && addr_count == MAX_CB_ADDR) { 5167 DEMAP_RANGE_FLUSH(dmrp); 5168 if (dmrp != NULL) { 5169 cpuset = sfmmup->sfmmu_cpusran; 5170 xt_sync(cpuset); 5171 } 5172 5173 for (a = 0; a < MAX_CB_ADDR; ++a) { 5174 callback->hcb_start_addr = cb_start_addr[a]; 5175 callback->hcb_end_addr = cb_end_addr[a]; 5176 callback->hcb_function(callback); 5177 } 5178 addr_count = 0; 5179 } 5180 if (iskernel) { 5181 hashno = TTE64K; 5182 continue; 5183 } 5184 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) { 5185 ASSERT(hashno == TTE64K); 5186 continue; 5187 } 5188 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) { 5189 hashno = TTE512K; 5190 continue; 5191 } 5192 if (mmu_page_sizes == max_mmu_page_sizes) { 5193 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) { 5194 hashno = TTE4M; 5195 continue; 5196 } 5197 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) { 5198 hashno = TTE32M; 5199 continue; 5200 } 5201 hashno = TTE256M; 5202 } else { 5203 hashno = TTE4M; 5204 } 5205 } 5206 5207 sfmmu_hblks_list_purge(&list); 5208 DEMAP_RANGE_FLUSH(dmrp); 5209 if (dmrp != NULL) { 5210 cpuset = sfmmup->sfmmu_cpusran; 5211 xt_sync(cpuset); 5212 } 5213 if (callback && addr_count != 0) { 5214 for (a = 0; a < addr_count; ++a) { 5215 callback->hcb_start_addr = cb_start_addr[a]; 5216 callback->hcb_end_addr = cb_end_addr[a]; 5217 callback->hcb_function(callback); 5218 } 5219 } 5220 5221 /* 5222 * Check TSB and TLB page sizes if the process isn't exiting. 5223 */ 5224 if (!sfmmup->sfmmu_free) 5225 sfmmu_check_page_sizes(sfmmup, 0); 5226 } 5227 5228 /* 5229 * Unload all the mappings in the range [addr..addr+len). addr and len must 5230 * be MMU_PAGESIZE aligned. 5231 */ 5232 void 5233 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags) 5234 { 5235 if (sfmmup->sfmmu_xhat_provider) { 5236 XHAT_UNLOAD(sfmmup, addr, len, flags); 5237 return; 5238 } 5239 hat_unload_callback(sfmmup, addr, len, flags, NULL); 5240 } 5241 5242 5243 /* 5244 * Find the largest mapping size for this page. 5245 */ 5246 int 5247 fnd_mapping_sz(page_t *pp) 5248 { 5249 int sz; 5250 int p_index; 5251 5252 p_index = PP_MAPINDEX(pp); 5253 5254 sz = 0; 5255 p_index >>= 1; /* don't care about 8K bit */ 5256 for (; p_index; p_index >>= 1) { 5257 sz++; 5258 } 5259 5260 return (sz); 5261 } 5262 5263 /* 5264 * This function unloads a range of addresses for an hmeblk. 5265 * It returns the next address to be unloaded. 5266 * It should be called with the hash lock held. 5267 */ 5268 static caddr_t 5269 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5270 caddr_t endaddr, demap_range_t *dmrp, uint_t flags) 5271 { 5272 tte_t tte, ttemod; 5273 struct sf_hment *sfhmep; 5274 int ttesz; 5275 long ttecnt; 5276 page_t *pp; 5277 kmutex_t *pml; 5278 int ret; 5279 int use_demap_range; 5280 5281 ASSERT(in_hblk_range(hmeblkp, addr)); 5282 ASSERT(!hmeblkp->hblk_shw_bit); 5283 #ifdef DEBUG 5284 if (get_hblk_ttesz(hmeblkp) != TTE8K && 5285 (endaddr < get_hblk_endaddr(hmeblkp))) { 5286 panic("sfmmu_hblk_unload: partial unload of large page"); 5287 } 5288 #endif /* DEBUG */ 5289 5290 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5291 ttesz = get_hblk_ttesz(hmeblkp); 5292 5293 use_demap_range = (do_virtual_coloring && 5294 ((dmrp == NULL) || TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp))); 5295 if (use_demap_range) { 5296 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); 5297 } else { 5298 DEMAP_RANGE_FLUSH(dmrp); 5299 } 5300 ttecnt = 0; 5301 HBLKTOHME(sfhmep, hmeblkp, addr); 5302 5303 while (addr < endaddr) { 5304 pml = NULL; 5305 again: 5306 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5307 if (TTE_IS_VALID(&tte)) { 5308 pp = sfhmep->hme_page; 5309 if (pp && pml == NULL) { 5310 pml = sfmmu_mlist_enter(pp); 5311 } 5312 5313 /* 5314 * Verify if hme still points to 'pp' now that 5315 * we have p_mapping lock. 5316 */ 5317 if (sfhmep->hme_page != pp) { 5318 if (pp != NULL && sfhmep->hme_page != NULL) { 5319 if (pml) { 5320 sfmmu_mlist_exit(pml); 5321 } 5322 /* Re-start this iteration. */ 5323 continue; 5324 } 5325 ASSERT((pp != NULL) && 5326 (sfhmep->hme_page == NULL)); 5327 goto tte_unloaded; 5328 } 5329 5330 /* 5331 * This point on we have both HASH and p_mapping 5332 * lock. 5333 */ 5334 ASSERT(pp == sfhmep->hme_page); 5335 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5336 5337 /* 5338 * We need to loop on modify tte because it is 5339 * possible for pagesync to come along and 5340 * change the software bits beneath us. 5341 * 5342 * Page_unload can also invalidate the tte after 5343 * we read tte outside of p_mapping lock. 5344 */ 5345 ttemod = tte; 5346 5347 TTE_SET_INVALID(&ttemod); 5348 ret = sfmmu_modifytte_try(&tte, &ttemod, 5349 &sfhmep->hme_tte); 5350 5351 if (ret <= 0) { 5352 if (TTE_IS_VALID(&tte)) { 5353 goto again; 5354 } else { 5355 /* 5356 * We read in a valid pte, but it 5357 * is unloaded by page_unload. 5358 * hme_page has become NULL and 5359 * we hold no p_mapping lock. 5360 */ 5361 ASSERT(pp == NULL && pml == NULL); 5362 goto tte_unloaded; 5363 } 5364 } 5365 5366 if (!(flags & HAT_UNLOAD_NOSYNC)) { 5367 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5368 } 5369 5370 /* 5371 * Ok- we invalidated the tte. Do the rest of the job. 5372 */ 5373 ttecnt++; 5374 5375 if (flags & HAT_UNLOAD_UNLOCK) { 5376 ASSERT(hmeblkp->hblk_lckcnt > 0); 5377 atomic_add_16(&hmeblkp->hblk_lckcnt, -1); 5378 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); 5379 } 5380 5381 /* 5382 * Normally we would need to flush the page 5383 * from the virtual cache at this point in 5384 * order to prevent a potential cache alias 5385 * inconsistency. 5386 * The particular scenario we need to worry 5387 * about is: 5388 * Given: va1 and va2 are two virtual address 5389 * that alias and map the same physical 5390 * address. 5391 * 1. mapping exists from va1 to pa and data 5392 * has been read into the cache. 5393 * 2. unload va1. 5394 * 3. load va2 and modify data using va2. 5395 * 4 unload va2. 5396 * 5. load va1 and reference data. Unless we 5397 * flush the data cache when we unload we will 5398 * get stale data. 5399 * Fortunately, page coloring eliminates the 5400 * above scenario by remembering the color a 5401 * physical page was last or is currently 5402 * mapped to. Now, we delay the flush until 5403 * the loading of translations. Only when the 5404 * new translation is of a different color 5405 * are we forced to flush. 5406 */ 5407 if (use_demap_range) { 5408 /* 5409 * Mark this page as needing a demap. 5410 */ 5411 DEMAP_RANGE_MARKPG(dmrp, addr); 5412 } else { 5413 if (do_virtual_coloring) { 5414 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 5415 sfmmup->sfmmu_free, 0); 5416 } else { 5417 pfn_t pfnum; 5418 5419 pfnum = TTE_TO_PFN(addr, &tte); 5420 sfmmu_tlbcache_demap(addr, sfmmup, 5421 hmeblkp, pfnum, sfmmup->sfmmu_free, 5422 FLUSH_NECESSARY_CPUS, 5423 CACHE_FLUSH, 0); 5424 } 5425 } 5426 5427 if (pp) { 5428 /* 5429 * Remove the hment from the mapping list 5430 */ 5431 ASSERT(hmeblkp->hblk_hmecnt > 0); 5432 5433 /* 5434 * Again, we cannot 5435 * ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS); 5436 */ 5437 HME_SUB(sfhmep, pp); 5438 membar_stst(); 5439 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 5440 } 5441 5442 ASSERT(hmeblkp->hblk_vcnt > 0); 5443 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 5444 5445 ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 5446 !hmeblkp->hblk_lckcnt); 5447 5448 #ifdef VAC 5449 if (pp && (pp->p_nrm & (P_KPMC | P_KPMS | P_TNC))) { 5450 if (PP_ISTNC(pp)) { 5451 /* 5452 * If page was temporary 5453 * uncached, try to recache 5454 * it. Note that HME_SUB() was 5455 * called above so p_index and 5456 * mlist had been updated. 5457 */ 5458 conv_tnc(pp, ttesz); 5459 } else if (pp->p_mapping == NULL) { 5460 ASSERT(kpm_enable); 5461 /* 5462 * Page is marked to be in VAC conflict 5463 * to an existing kpm mapping and/or is 5464 * kpm mapped using only the regular 5465 * pagesize. 5466 */ 5467 sfmmu_kpm_hme_unload(pp); 5468 } 5469 } 5470 #endif /* VAC */ 5471 } else if ((pp = sfhmep->hme_page) != NULL) { 5472 /* 5473 * TTE is invalid but the hme 5474 * still exists. let pageunload 5475 * complete its job. 5476 */ 5477 ASSERT(pml == NULL); 5478 pml = sfmmu_mlist_enter(pp); 5479 if (sfhmep->hme_page != NULL) { 5480 sfmmu_mlist_exit(pml); 5481 pml = NULL; 5482 goto again; 5483 } 5484 ASSERT(sfhmep->hme_page == NULL); 5485 } else if (hmeblkp->hblk_hmecnt != 0) { 5486 /* 5487 * pageunload may have not finished decrementing 5488 * hblk_vcnt and hblk_hmecnt. Find page_t if any and 5489 * wait for pageunload to finish. Rely on pageunload 5490 * to decrement hblk_hmecnt after hblk_vcnt. 5491 */ 5492 pfn_t pfn = TTE_TO_TTEPFN(&tte); 5493 ASSERT(pml == NULL); 5494 if (pf_is_memory(pfn)) { 5495 pp = page_numtopp_nolock(pfn); 5496 if (pp != NULL) { 5497 pml = sfmmu_mlist_enter(pp); 5498 sfmmu_mlist_exit(pml); 5499 pml = NULL; 5500 } 5501 } 5502 } 5503 5504 tte_unloaded: 5505 /* 5506 * At this point, the tte we are looking at 5507 * should be unloaded, and hme has been unlinked 5508 * from page too. This is important because in 5509 * pageunload, it does ttesync() then HME_SUB. 5510 * We need to make sure HME_SUB has been completed 5511 * so we know ttesync() has been completed. Otherwise, 5512 * at exit time, after return from hat layer, VM will 5513 * release as structure which hat_setstat() (called 5514 * by ttesync()) needs. 5515 */ 5516 #ifdef DEBUG 5517 { 5518 tte_t dtte; 5519 5520 ASSERT(sfhmep->hme_page == NULL); 5521 5522 sfmmu_copytte(&sfhmep->hme_tte, &dtte); 5523 ASSERT(!TTE_IS_VALID(&dtte)); 5524 } 5525 #endif 5526 5527 if (pml) { 5528 sfmmu_mlist_exit(pml); 5529 } 5530 5531 addr += TTEBYTES(ttesz); 5532 sfhmep++; 5533 DEMAP_RANGE_NEXTPG(dmrp); 5534 } 5535 if (ttecnt > 0) 5536 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt); 5537 return (addr); 5538 } 5539 5540 /* 5541 * Synchronize all the mappings in the range [addr..addr+len). 5542 * Can be called with clearflag having two states: 5543 * HAT_SYNC_DONTZERO means just return the rm stats 5544 * HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats 5545 */ 5546 void 5547 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag) 5548 { 5549 struct hmehash_bucket *hmebp; 5550 hmeblk_tag hblktag; 5551 int hmeshift, hashno = 1; 5552 struct hme_blk *hmeblkp, *list = NULL; 5553 caddr_t endaddr; 5554 cpuset_t cpuset; 5555 5556 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 5557 ASSERT((sfmmup == ksfmmup) || 5558 AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock)); 5559 ASSERT((len & MMU_PAGEOFFSET) == 0); 5560 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 5561 (clearflag == HAT_SYNC_ZERORM)); 5562 5563 CPUSET_ZERO(cpuset); 5564 5565 endaddr = addr + len; 5566 hblktag.htag_id = sfmmup; 5567 /* 5568 * Spitfire supports 4 page sizes. 5569 * Most pages are expected to be of the smallest page 5570 * size (8K) and these will not need to be rehashed. 64K 5571 * pages also don't need to be rehashed because the an hmeblk 5572 * spans 64K of address space. 512K pages might need 1 rehash and 5573 * and 4M pages 2 rehashes. 5574 */ 5575 while (addr < endaddr) { 5576 hmeshift = HME_HASH_SHIFT(hashno); 5577 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift); 5578 hblktag.htag_rehash = hashno; 5579 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); 5580 5581 SFMMU_HASH_LOCK(hmebp); 5582 5583 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); 5584 if (hmeblkp != NULL) { 5585 /* 5586 * We've encountered a shadow hmeblk so skip the range 5587 * of the next smaller mapping size. 5588 */ 5589 if (hmeblkp->hblk_shw_bit) { 5590 ASSERT(sfmmup != ksfmmup); 5591 ASSERT(hashno > 1); 5592 addr = (caddr_t)P2END((uintptr_t)addr, 5593 TTEBYTES(hashno - 1)); 5594 } else { 5595 addr = sfmmu_hblk_sync(sfmmup, hmeblkp, 5596 addr, endaddr, clearflag); 5597 } 5598 SFMMU_HASH_UNLOCK(hmebp); 5599 hashno = 1; 5600 continue; 5601 } 5602 SFMMU_HASH_UNLOCK(hmebp); 5603 5604 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { 5605 /* 5606 * We have traversed the whole list and rehashed 5607 * if necessary without finding the address to sync. 5608 * This is ok so we increment the address by the 5609 * smallest hmeblk range for kernel mappings and the 5610 * largest hmeblk range, to account for shadow hmeblks, 5611 * for user mappings and continue. 5612 */ 5613 if (sfmmup == ksfmmup) 5614 addr = (caddr_t)P2END((uintptr_t)addr, 5615 TTEBYTES(1)); 5616 else 5617 addr = (caddr_t)P2END((uintptr_t)addr, 5618 TTEBYTES(hashno)); 5619 hashno = 1; 5620 } else { 5621 hashno++; 5622 } 5623 } 5624 sfmmu_hblks_list_purge(&list); 5625 cpuset = sfmmup->sfmmu_cpusran; 5626 xt_sync(cpuset); 5627 } 5628 5629 static caddr_t 5630 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, 5631 caddr_t endaddr, int clearflag) 5632 { 5633 tte_t tte, ttemod; 5634 struct sf_hment *sfhmep; 5635 int ttesz; 5636 struct page *pp; 5637 kmutex_t *pml; 5638 int ret; 5639 5640 ASSERT(hmeblkp->hblk_shw_bit == 0); 5641 5642 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); 5643 5644 ttesz = get_hblk_ttesz(hmeblkp); 5645 HBLKTOHME(sfhmep, hmeblkp, addr); 5646 5647 while (addr < endaddr) { 5648 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5649 if (TTE_IS_VALID(&tte)) { 5650 pml = NULL; 5651 pp = sfhmep->hme_page; 5652 if (pp) { 5653 pml = sfmmu_mlist_enter(pp); 5654 } 5655 if (pp != sfhmep->hme_page) { 5656 /* 5657 * tte most have been unloaded 5658 * underneath us. Recheck 5659 */ 5660 ASSERT(pml); 5661 sfmmu_mlist_exit(pml); 5662 continue; 5663 } 5664 5665 ASSERT(pp == NULL || sfmmu_mlist_held(pp)); 5666 5667 if (clearflag == HAT_SYNC_ZERORM) { 5668 ttemod = tte; 5669 TTE_CLR_RM(&ttemod); 5670 ret = sfmmu_modifytte_try(&tte, &ttemod, 5671 &sfhmep->hme_tte); 5672 if (ret < 0) { 5673 if (pml) { 5674 sfmmu_mlist_exit(pml); 5675 } 5676 continue; 5677 } 5678 5679 if (ret > 0) { 5680 sfmmu_tlb_demap(addr, sfmmup, 5681 hmeblkp, 0, 0); 5682 } 5683 } 5684 sfmmu_ttesync(sfmmup, addr, &tte, pp); 5685 if (pml) { 5686 sfmmu_mlist_exit(pml); 5687 } 5688 } 5689 addr += TTEBYTES(ttesz); 5690 sfhmep++; 5691 } 5692 return (addr); 5693 } 5694 5695 /* 5696 * This function will sync a tte to the page struct and it will 5697 * update the hat stats. Currently it allows us to pass a NULL pp 5698 * and we will simply update the stats. We may want to change this 5699 * so we only keep stats for pages backed by pp's. 5700 */ 5701 static void 5702 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp) 5703 { 5704 uint_t rm = 0; 5705 int sz; 5706 pgcnt_t npgs; 5707 5708 ASSERT(TTE_IS_VALID(ttep)); 5709 5710 if (TTE_IS_NOSYNC(ttep)) { 5711 return; 5712 } 5713 5714 if (TTE_IS_REF(ttep)) { 5715 rm = P_REF; 5716 } 5717 if (TTE_IS_MOD(ttep)) { 5718 rm |= P_MOD; 5719 } 5720 5721 if (rm == 0) { 5722 return; 5723 } 5724 5725 sz = TTE_CSZ(ttep); 5726 if (sfmmup->sfmmu_rmstat) { 5727 int i; 5728 caddr_t vaddr = addr; 5729 5730 for (i = 0; i < TTEPAGES(sz); i++, vaddr += MMU_PAGESIZE) { 5731 hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm); 5732 } 5733 5734 } 5735 5736 /* 5737 * XXX I want to use cas to update nrm bits but they 5738 * currently belong in common/vm and not in hat where 5739 * they should be. 5740 * The nrm bits are protected by the same mutex as 5741 * the one that protects the page's mapping list. 5742 */ 5743 if (!pp) 5744 return; 5745 ASSERT(sfmmu_mlist_held(pp)); 5746 /* 5747 * If the tte is for a large page, we need to sync all the 5748 * pages covered by the tte. 5749 */ 5750 if (sz != TTE8K) { 5751 ASSERT(pp->p_szc != 0); 5752 pp = PP_GROUPLEADER(pp, sz); 5753 ASSERT(sfmmu_mlist_held(pp)); 5754 } 5755 5756 /* Get number of pages from tte size. */ 5757 npgs = TTEPAGES(sz); 5758 5759 do { 5760 ASSERT(pp); 5761 ASSERT(sfmmu_mlist_held(pp)); 5762 if (((rm & P_REF) != 0 && !PP_ISREF(pp)) || 5763 ((rm & P_MOD) != 0 && !PP_ISMOD(pp))) 5764 hat_page_setattr(pp, rm); 5765 5766 /* 5767 * Are we done? If not, we must have a large mapping. 5768 * For large mappings we need to sync the rest of the pages 5769 * covered by this tte; goto the next page. 5770 */ 5771 } while (--npgs > 0 && (pp = PP_PAGENEXT(pp))); 5772 } 5773 5774 /* 5775 * Execute pre-callback handler of each pa_hment linked to pp 5776 * 5777 * Inputs: 5778 * flag: either HAT_PRESUSPEND or HAT_SUSPEND. 5779 * capture_cpus: pointer to return value (below) 5780 * 5781 * Returns: 5782 * Propagates the subsystem callback return values back to the caller; 5783 * returns 0 on success. If capture_cpus is non-NULL, the value returned 5784 * is zero if all of the pa_hments are of a type that do not require 5785 * capturing CPUs prior to suspending the mapping, else it is 1. 5786 */ 5787 static int 5788 hat_pageprocess_precallbacks(struct page *pp, uint_t flag, int *capture_cpus) 5789 { 5790 struct sf_hment *sfhmep; 5791 struct pa_hment *pahmep; 5792 int (*f)(caddr_t, uint_t, uint_t, void *); 5793 int ret; 5794 id_t id; 5795 int locked = 0; 5796 kmutex_t *pml; 5797 5798 ASSERT(PAGE_EXCL(pp)); 5799 if (!sfmmu_mlist_held(pp)) { 5800 pml = sfmmu_mlist_enter(pp); 5801 locked = 1; 5802 } 5803 5804 if (capture_cpus) 5805 *capture_cpus = 0; 5806 5807 top: 5808 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5809 /* 5810 * skip sf_hments corresponding to VA<->PA mappings; 5811 * for pa_hment's, hme_tte.ll is zero 5812 */ 5813 if (!IS_PAHME(sfhmep)) 5814 continue; 5815 5816 pahmep = sfhmep->hme_data; 5817 ASSERT(pahmep != NULL); 5818 5819 /* 5820 * skip if pre-handler has been called earlier in this loop 5821 */ 5822 if (pahmep->flags & flag) 5823 continue; 5824 5825 id = pahmep->cb_id; 5826 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5827 if (capture_cpus && sfmmu_cb_table[id].capture_cpus != 0) 5828 *capture_cpus = 1; 5829 if ((f = sfmmu_cb_table[id].prehandler) == NULL) { 5830 pahmep->flags |= flag; 5831 continue; 5832 } 5833 5834 /* 5835 * Drop the mapping list lock to avoid locking order issues. 5836 */ 5837 if (locked) 5838 sfmmu_mlist_exit(pml); 5839 5840 ret = f(pahmep->addr, pahmep->len, flag, pahmep->pvt); 5841 if (ret != 0) 5842 return (ret); /* caller must do the cleanup */ 5843 5844 if (locked) { 5845 pml = sfmmu_mlist_enter(pp); 5846 pahmep->flags |= flag; 5847 goto top; 5848 } 5849 5850 pahmep->flags |= flag; 5851 } 5852 5853 if (locked) 5854 sfmmu_mlist_exit(pml); 5855 5856 return (0); 5857 } 5858 5859 /* 5860 * Execute post-callback handler of each pa_hment linked to pp 5861 * 5862 * Same overall assumptions and restrictions apply as for 5863 * hat_pageprocess_precallbacks(). 5864 */ 5865 static void 5866 hat_pageprocess_postcallbacks(struct page *pp, uint_t flag) 5867 { 5868 pfn_t pgpfn = pp->p_pagenum; 5869 pfn_t pgmask = btop(page_get_pagesize(pp->p_szc)) - 1; 5870 pfn_t newpfn; 5871 struct sf_hment *sfhmep; 5872 struct pa_hment *pahmep; 5873 int (*f)(caddr_t, uint_t, uint_t, void *, pfn_t); 5874 id_t id; 5875 int locked = 0; 5876 kmutex_t *pml; 5877 5878 ASSERT(PAGE_EXCL(pp)); 5879 if (!sfmmu_mlist_held(pp)) { 5880 pml = sfmmu_mlist_enter(pp); 5881 locked = 1; 5882 } 5883 5884 top: 5885 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5886 /* 5887 * skip sf_hments corresponding to VA<->PA mappings; 5888 * for pa_hment's, hme_tte.ll is zero 5889 */ 5890 if (!IS_PAHME(sfhmep)) 5891 continue; 5892 5893 pahmep = sfhmep->hme_data; 5894 ASSERT(pahmep != NULL); 5895 5896 if ((pahmep->flags & flag) == 0) 5897 continue; 5898 5899 pahmep->flags &= ~flag; 5900 5901 id = pahmep->cb_id; 5902 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid); 5903 if ((f = sfmmu_cb_table[id].posthandler) == NULL) 5904 continue; 5905 5906 /* 5907 * Convert the base page PFN into the constituent PFN 5908 * which is needed by the callback handler. 5909 */ 5910 newpfn = pgpfn | (btop((uintptr_t)pahmep->addr) & pgmask); 5911 5912 /* 5913 * Drop the mapping list lock to avoid locking order issues. 5914 */ 5915 if (locked) 5916 sfmmu_mlist_exit(pml); 5917 5918 if (f(pahmep->addr, pahmep->len, flag, pahmep->pvt, newpfn) 5919 != 0) 5920 panic("sfmmu: posthandler failed"); 5921 5922 if (locked) { 5923 pml = sfmmu_mlist_enter(pp); 5924 goto top; 5925 } 5926 } 5927 5928 if (locked) 5929 sfmmu_mlist_exit(pml); 5930 } 5931 5932 /* 5933 * Suspend locked kernel mapping 5934 */ 5935 void 5936 hat_pagesuspend(struct page *pp) 5937 { 5938 struct sf_hment *sfhmep; 5939 sfmmu_t *sfmmup; 5940 tte_t tte, ttemod; 5941 struct hme_blk *hmeblkp; 5942 caddr_t addr; 5943 int index, cons; 5944 cpuset_t cpuset; 5945 5946 ASSERT(PAGE_EXCL(pp)); 5947 ASSERT(sfmmu_mlist_held(pp)); 5948 5949 mutex_enter(&kpr_suspendlock); 5950 5951 /* 5952 * Call into dtrace to tell it we're about to suspend a 5953 * kernel mapping. This prevents us from running into issues 5954 * with probe context trying to touch a suspended page 5955 * in the relocation codepath itself. 5956 */ 5957 if (dtrace_kreloc_init) 5958 (*dtrace_kreloc_init)(); 5959 5960 index = PP_MAPINDEX(pp); 5961 cons = TTE8K; 5962 5963 retry: 5964 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 5965 5966 if (IS_PAHME(sfhmep)) 5967 continue; 5968 5969 if (get_hblk_ttesz(sfmmu_hmetohblk(sfhmep)) != cons) 5970 continue; 5971 5972 /* 5973 * Loop until we successfully set the suspend bit in 5974 * the TTE. 5975 */ 5976 again: 5977 sfmmu_copytte(&sfhmep->hme_tte, &tte); 5978 ASSERT(TTE_IS_VALID(&tte)); 5979 5980 ttemod = tte; 5981 TTE_SET_SUSPEND(&ttemod); 5982 if (sfmmu_modifytte_try(&tte, &ttemod, 5983 &sfhmep->hme_tte) < 0) 5984 goto again; 5985 5986 /* 5987 * Invalidate TSB entry 5988 */ 5989 hmeblkp = sfmmu_hmetohblk(sfhmep); 5990 5991 sfmmup = hblktosfmmu(hmeblkp); 5992 ASSERT(sfmmup == ksfmmup); 5993 5994 addr = tte_to_vaddr(hmeblkp, tte); 5995 5996 /* 5997 * No need to make sure that the TSB for this sfmmu is 5998 * not being relocated since it is ksfmmup and thus it 5999 * will never be relocated. 6000 */ 6001 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 6002 6003 /* 6004 * Update xcall stats 6005 */ 6006 cpuset = cpu_ready_set; 6007 CPUSET_DEL(cpuset, CPU->cpu_id); 6008 6009 /* LINTED: constant in conditional context */ 6010 SFMMU_XCALL_STATS(ksfmmup); 6011 6012 /* 6013 * Flush TLB entry on remote CPU's 6014 */ 6015 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 6016 (uint64_t)ksfmmup); 6017 xt_sync(cpuset); 6018 6019 /* 6020 * Flush TLB entry on local CPU 6021 */ 6022 vtag_flushpage(addr, (uint64_t)ksfmmup); 6023 } 6024 6025 while (index != 0) { 6026 index = index >> 1; 6027 if (index != 0) 6028 cons++; 6029 if (index & 0x1) { 6030 pp = PP_GROUPLEADER(pp, cons); 6031 goto retry; 6032 } 6033 } 6034 } 6035 6036 #ifdef DEBUG 6037 6038 #define N_PRLE 1024 6039 struct prle { 6040 page_t *targ; 6041 page_t *repl; 6042 int status; 6043 int pausecpus; 6044 hrtime_t whence; 6045 }; 6046 6047 static struct prle page_relocate_log[N_PRLE]; 6048 static int prl_entry; 6049 static kmutex_t prl_mutex; 6050 6051 #define PAGE_RELOCATE_LOG(t, r, s, p) \ 6052 mutex_enter(&prl_mutex); \ 6053 page_relocate_log[prl_entry].targ = *(t); \ 6054 page_relocate_log[prl_entry].repl = *(r); \ 6055 page_relocate_log[prl_entry].status = (s); \ 6056 page_relocate_log[prl_entry].pausecpus = (p); \ 6057 page_relocate_log[prl_entry].whence = gethrtime(); \ 6058 prl_entry = (prl_entry == (N_PRLE - 1))? 0 : prl_entry + 1; \ 6059 mutex_exit(&prl_mutex); 6060 6061 #else /* !DEBUG */ 6062 #define PAGE_RELOCATE_LOG(t, r, s, p) 6063 #endif 6064 6065 /* 6066 * Core Kernel Page Relocation Algorithm 6067 * 6068 * Input: 6069 * 6070 * target : constituent pages are SE_EXCL locked. 6071 * replacement: constituent pages are SE_EXCL locked. 6072 * 6073 * Output: 6074 * 6075 * nrelocp: number of pages relocated 6076 */ 6077 int 6078 hat_page_relocate(page_t **target, page_t **replacement, spgcnt_t *nrelocp) 6079 { 6080 page_t *targ, *repl; 6081 page_t *tpp, *rpp; 6082 kmutex_t *low, *high; 6083 spgcnt_t npages, i; 6084 page_t *pl = NULL; 6085 int old_pil; 6086 cpuset_t cpuset; 6087 int cap_cpus; 6088 int ret; 6089 6090 if (hat_kpr_enabled == 0 || !kcage_on || PP_ISNORELOC(*target)) { 6091 PAGE_RELOCATE_LOG(target, replacement, EAGAIN, -1); 6092 return (EAGAIN); 6093 } 6094 6095 mutex_enter(&kpr_mutex); 6096 kreloc_thread = curthread; 6097 6098 targ = *target; 6099 repl = *replacement; 6100 ASSERT(repl != NULL); 6101 ASSERT(targ->p_szc == repl->p_szc); 6102 6103 npages = page_get_pagecnt(targ->p_szc); 6104 6105 /* 6106 * unload VA<->PA mappings that are not locked 6107 */ 6108 tpp = targ; 6109 for (i = 0; i < npages; i++) { 6110 (void) hat_pageunload(tpp, SFMMU_KERNEL_RELOC); 6111 tpp++; 6112 } 6113 6114 /* 6115 * Do "presuspend" callbacks, in a context from which we can still 6116 * block as needed. Note that we don't hold the mapping list lock 6117 * of "targ" at this point due to potential locking order issues; 6118 * we assume that between the hat_pageunload() above and holding 6119 * the SE_EXCL lock that the mapping list *cannot* change at this 6120 * point. 6121 */ 6122 ret = hat_pageprocess_precallbacks(targ, HAT_PRESUSPEND, &cap_cpus); 6123 if (ret != 0) { 6124 /* 6125 * EIO translates to fatal error, for all others cleanup 6126 * and return EAGAIN. 6127 */ 6128 ASSERT(ret != EIO); 6129 hat_pageprocess_postcallbacks(targ, HAT_POSTUNSUSPEND); 6130 PAGE_RELOCATE_LOG(target, replacement, ret, -1); 6131 kreloc_thread = NULL; 6132 mutex_exit(&kpr_mutex); 6133 return (EAGAIN); 6134 } 6135 6136 /* 6137 * acquire p_mapping list lock for both the target and replacement 6138 * root pages. 6139 * 6140 * low and high refer to the need to grab the mlist locks in a 6141 * specific order in order to prevent race conditions. Thus the 6142 * lower lock must be grabbed before the higher lock. 6143 * 6144 * This will block hat_unload's accessing p_mapping list. Since 6145 * we have SE_EXCL lock, hat_memload and hat_pageunload will be 6146 * blocked. Thus, no one else will be accessing the p_mapping list 6147 * while we suspend and reload the locked mapping below. 6148 */ 6149 tpp = targ; 6150 rpp = repl; 6151 sfmmu_mlist_reloc_enter(tpp, rpp, &low, &high); 6152 6153 kpreempt_disable(); 6154 6155 #ifdef VAC 6156 /* 6157 * If the replacement page is of a different virtual color 6158 * than the page it is replacing, we need to handle the VAC 6159 * consistency for it just as we would if we were setting up 6160 * a new mapping to a page. 6161 */ 6162 if ((tpp->p_szc == 0) && (PP_GET_VCOLOR(rpp) != NO_VCOLOR)) { 6163 if (tpp->p_vcolor != rpp->p_vcolor) { 6164 sfmmu_cache_flushcolor(PP_GET_VCOLOR(rpp), 6165 rpp->p_pagenum); 6166 } 6167 } 6168 #endif 6169 6170 /* 6171 * We raise our PIL to 13 so that we don't get captured by 6172 * another CPU or pinned by an interrupt thread. We can't go to 6173 * PIL 14 since the nexus driver(s) may need to interrupt at 6174 * that level in the case of IOMMU pseudo mappings. 6175 */ 6176 cpuset = cpu_ready_set; 6177 CPUSET_DEL(cpuset, CPU->cpu_id); 6178 if (!cap_cpus || CPUSET_ISNULL(cpuset)) { 6179 old_pil = splr(XCALL_PIL); 6180 } else { 6181 old_pil = -1; 6182 xc_attention(cpuset); 6183 } 6184 ASSERT(getpil() == XCALL_PIL); 6185 6186 /* 6187 * Now do suspend callbacks. In the case of an IOMMU mapping 6188 * this will suspend all DMA activity to the page while it is 6189 * being relocated. Since we are well above LOCK_LEVEL and CPUs 6190 * may be captured at this point we should have acquired any needed 6191 * locks in the presuspend callback. 6192 */ 6193 ret = hat_pageprocess_precallbacks(targ, HAT_SUSPEND, NULL); 6194 if (ret != 0) { 6195 repl = targ; 6196 goto suspend_fail; 6197 } 6198 6199 /* 6200 * Raise the PIL yet again, this time to block all high-level 6201 * interrupts on this CPU. This is necessary to prevent an 6202 * interrupt routine from pinning the thread which holds the 6203 * mapping suspended and then touching the suspended page. 6204 * 6205 * Once the page is suspended we also need to be careful to 6206 * avoid calling any functions which touch any seg_kmem memory 6207 * since that memory may be backed by the very page we are 6208 * relocating in here! 6209 */ 6210 hat_pagesuspend(targ); 6211 6212 /* 6213 * Now that we are confident everybody has stopped using this page, 6214 * copy the page contents. Note we use a physical copy to prevent 6215 * locking issues and to avoid fpRAS because we can't handle it in 6216 * this context. 6217 */ 6218 for (i = 0; i < npages; i++, tpp++, rpp++) { 6219 /* 6220 * Copy the contents of the page. 6221 */ 6222 ppcopy_kernel(tpp, rpp); 6223 } 6224 6225 tpp = targ; 6226 rpp = repl; 6227 for (i = 0; i < npages; i++, tpp++, rpp++) { 6228 /* 6229 * Copy attributes. VAC consistency was handled above, 6230 * if required. 6231 */ 6232 rpp->p_nrm = tpp->p_nrm; 6233 tpp->p_nrm = 0; 6234 rpp->p_index = tpp->p_index; 6235 tpp->p_index = 0; 6236 #ifdef VAC 6237 rpp->p_vcolor = tpp->p_vcolor; 6238 #endif 6239 } 6240 6241 /* 6242 * First, unsuspend the page, if we set the suspend bit, and transfer 6243 * the mapping list from the target page to the replacement page. 6244 * Next process postcallbacks; since pa_hment's are linked only to the 6245 * p_mapping list of root page, we don't iterate over the constituent 6246 * pages. 6247 */ 6248 hat_pagereload(targ, repl); 6249 6250 suspend_fail: 6251 hat_pageprocess_postcallbacks(repl, HAT_UNSUSPEND); 6252 6253 /* 6254 * Now lower our PIL and release any captured CPUs since we 6255 * are out of the "danger zone". After this it will again be 6256 * safe to acquire adaptive mutex locks, or to drop them... 6257 */ 6258 if (old_pil != -1) { 6259 splx(old_pil); 6260 } else { 6261 xc_dismissed(cpuset); 6262 } 6263 6264 kpreempt_enable(); 6265 6266 sfmmu_mlist_reloc_exit(low, high); 6267 6268 /* 6269 * Postsuspend callbacks should drop any locks held across 6270 * the suspend callbacks. As before, we don't hold the mapping 6271 * list lock at this point.. our assumption is that the mapping 6272 * list still can't change due to our holding SE_EXCL lock and 6273 * there being no unlocked mappings left. Hence the restriction 6274 * on calling context to hat_delete_callback() 6275 */ 6276 hat_pageprocess_postcallbacks(repl, HAT_POSTUNSUSPEND); 6277 if (ret != 0) { 6278 /* 6279 * The second presuspend call failed: we got here through 6280 * the suspend_fail label above. 6281 */ 6282 ASSERT(ret != EIO); 6283 PAGE_RELOCATE_LOG(target, replacement, ret, cap_cpus); 6284 kreloc_thread = NULL; 6285 mutex_exit(&kpr_mutex); 6286 return (EAGAIN); 6287 } 6288 6289 /* 6290 * Now that we're out of the performance critical section we can 6291 * take care of updating the hash table, since we still 6292 * hold all the pages locked SE_EXCL at this point we 6293 * needn't worry about things changing out from under us. 6294 */ 6295 tpp = targ; 6296 rpp = repl; 6297 for (i = 0; i < npages; i++, tpp++, rpp++) { 6298 6299 /* 6300 * replace targ with replacement in page_hash table 6301 */ 6302 targ = tpp; 6303 page_relocate_hash(rpp, targ); 6304 6305 /* 6306 * concatenate target; caller of platform_page_relocate() 6307 * expects target to be concatenated after returning. 6308 */ 6309 ASSERT(targ->p_next == targ); 6310 ASSERT(targ->p_prev == targ); 6311 page_list_concat(&pl, &targ); 6312 } 6313 6314 ASSERT(*target == pl); 6315 *nrelocp = npages; 6316 PAGE_RELOCATE_LOG(target, replacement, 0, cap_cpus); 6317 kreloc_thread = NULL; 6318 mutex_exit(&kpr_mutex); 6319 return (0); 6320 } 6321 6322 /* 6323 * Called when stray pa_hments are found attached to a page which is 6324 * being freed. Notify the subsystem which attached the pa_hment of 6325 * the error if it registered a suitable handler, else panic. 6326 */ 6327 static void 6328 sfmmu_pahment_leaked(struct pa_hment *pahmep) 6329 { 6330 id_t cb_id = pahmep->cb_id; 6331 6332 ASSERT(cb_id >= (id_t)0 && cb_id < sfmmu_cb_nextid); 6333 if (sfmmu_cb_table[cb_id].errhandler != NULL) { 6334 if (sfmmu_cb_table[cb_id].errhandler(pahmep->addr, pahmep->len, 6335 HAT_CB_ERR_LEAKED, pahmep->pvt) == 0) 6336 return; /* non-fatal */ 6337 } 6338 panic("pa_hment leaked: 0x%p", pahmep); 6339 } 6340 6341 /* 6342 * Remove all mappings to page 'pp'. 6343 */ 6344 int 6345 hat_pageunload(struct page *pp, uint_t forceflag) 6346 { 6347 struct page *origpp = pp; 6348 struct sf_hment *sfhme, *tmphme; 6349 struct hme_blk *hmeblkp; 6350 kmutex_t *pml; 6351 #ifdef VAC 6352 kmutex_t *pmtx; 6353 #endif 6354 cpuset_t cpuset, tset; 6355 int index, cons; 6356 int xhme_blks; 6357 int pa_hments; 6358 6359 ASSERT(PAGE_EXCL(pp)); 6360 6361 retry_xhat: 6362 tmphme = NULL; 6363 xhme_blks = 0; 6364 pa_hments = 0; 6365 CPUSET_ZERO(cpuset); 6366 6367 pml = sfmmu_mlist_enter(pp); 6368 6369 #ifdef VAC 6370 if (pp->p_kpmref) 6371 sfmmu_kpm_pageunload(pp); 6372 ASSERT(!PP_ISMAPPED_KPM(pp)); 6373 #endif 6374 6375 index = PP_MAPINDEX(pp); 6376 cons = TTE8K; 6377 retry: 6378 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6379 tmphme = sfhme->hme_next; 6380 6381 if (IS_PAHME(sfhme)) { 6382 ASSERT(sfhme->hme_data != NULL); 6383 pa_hments++; 6384 continue; 6385 } 6386 6387 hmeblkp = sfmmu_hmetohblk(sfhme); 6388 if (hmeblkp->hblk_xhat_bit) { 6389 struct xhat_hme_blk *xblk = 6390 (struct xhat_hme_blk *)hmeblkp; 6391 6392 (void) XHAT_PAGEUNLOAD(xblk->xhat_hme_blk_hat, 6393 pp, forceflag, XBLK2PROVBLK(xblk)); 6394 6395 xhme_blks = 1; 6396 continue; 6397 } 6398 6399 /* 6400 * If there are kernel mappings don't unload them, they will 6401 * be suspended. 6402 */ 6403 if (forceflag == SFMMU_KERNEL_RELOC && hmeblkp->hblk_lckcnt && 6404 hmeblkp->hblk_tag.htag_id == ksfmmup) 6405 continue; 6406 6407 tset = sfmmu_pageunload(pp, sfhme, cons); 6408 CPUSET_OR(cpuset, tset); 6409 } 6410 6411 while (index != 0) { 6412 index = index >> 1; 6413 if (index != 0) 6414 cons++; 6415 if (index & 0x1) { 6416 /* Go to leading page */ 6417 pp = PP_GROUPLEADER(pp, cons); 6418 ASSERT(sfmmu_mlist_held(pp)); 6419 goto retry; 6420 } 6421 } 6422 6423 /* 6424 * cpuset may be empty if the page was only mapped by segkpm, 6425 * in which case we won't actually cross-trap. 6426 */ 6427 xt_sync(cpuset); 6428 6429 /* 6430 * The page should have no mappings at this point, unless 6431 * we were called from hat_page_relocate() in which case we 6432 * leave the locked mappings which will be suspended later. 6433 */ 6434 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks || pa_hments || 6435 (forceflag == SFMMU_KERNEL_RELOC)); 6436 6437 #ifdef VAC 6438 if (PP_ISTNC(pp)) { 6439 if (cons == TTE8K) { 6440 pmtx = sfmmu_page_enter(pp); 6441 PP_CLRTNC(pp); 6442 sfmmu_page_exit(pmtx); 6443 } else { 6444 conv_tnc(pp, cons); 6445 } 6446 } 6447 #endif /* VAC */ 6448 6449 if (pa_hments && forceflag != SFMMU_KERNEL_RELOC) { 6450 /* 6451 * Unlink any pa_hments and free them, calling back 6452 * the responsible subsystem to notify it of the error. 6453 * This can occur in situations such as drivers leaking 6454 * DMA handles: naughty, but common enough that we'd like 6455 * to keep the system running rather than bringing it 6456 * down with an obscure error like "pa_hment leaked" 6457 * which doesn't aid the user in debugging their driver. 6458 */ 6459 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6460 tmphme = sfhme->hme_next; 6461 if (IS_PAHME(sfhme)) { 6462 struct pa_hment *pahmep = sfhme->hme_data; 6463 sfmmu_pahment_leaked(pahmep); 6464 HME_SUB(sfhme, pp); 6465 kmem_cache_free(pa_hment_cache, pahmep); 6466 } 6467 } 6468 6469 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks); 6470 } 6471 6472 sfmmu_mlist_exit(pml); 6473 6474 /* 6475 * XHAT may not have finished unloading pages 6476 * because some other thread was waiting for 6477 * mlist lock and XHAT_PAGEUNLOAD let it do 6478 * the job. 6479 */ 6480 if (xhme_blks) { 6481 pp = origpp; 6482 goto retry_xhat; 6483 } 6484 6485 return (0); 6486 } 6487 6488 cpuset_t 6489 sfmmu_pageunload(page_t *pp, struct sf_hment *sfhme, int cons) 6490 { 6491 struct hme_blk *hmeblkp; 6492 sfmmu_t *sfmmup; 6493 tte_t tte, ttemod; 6494 #ifdef DEBUG 6495 tte_t orig_old; 6496 #endif /* DEBUG */ 6497 caddr_t addr; 6498 int ttesz; 6499 int ret; 6500 cpuset_t cpuset; 6501 6502 ASSERT(pp != NULL); 6503 ASSERT(sfmmu_mlist_held(pp)); 6504 ASSERT(pp->p_vnode != &kvp); 6505 6506 CPUSET_ZERO(cpuset); 6507 6508 hmeblkp = sfmmu_hmetohblk(sfhme); 6509 6510 readtte: 6511 sfmmu_copytte(&sfhme->hme_tte, &tte); 6512 if (TTE_IS_VALID(&tte)) { 6513 sfmmup = hblktosfmmu(hmeblkp); 6514 ttesz = get_hblk_ttesz(hmeblkp); 6515 /* 6516 * Only unload mappings of 'cons' size. 6517 */ 6518 if (ttesz != cons) 6519 return (cpuset); 6520 6521 /* 6522 * Note that we have p_mapping lock, but no hash lock here. 6523 * hblk_unload() has to have both hash lock AND p_mapping 6524 * lock before it tries to modify tte. So, the tte could 6525 * not become invalid in the sfmmu_modifytte_try() below. 6526 */ 6527 ttemod = tte; 6528 #ifdef DEBUG 6529 orig_old = tte; 6530 #endif /* DEBUG */ 6531 6532 TTE_SET_INVALID(&ttemod); 6533 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6534 if (ret < 0) { 6535 #ifdef DEBUG 6536 /* only R/M bits can change. */ 6537 chk_tte(&orig_old, &tte, &ttemod, hmeblkp); 6538 #endif /* DEBUG */ 6539 goto readtte; 6540 } 6541 6542 if (ret == 0) { 6543 panic("pageunload: cas failed?"); 6544 } 6545 6546 addr = tte_to_vaddr(hmeblkp, tte); 6547 6548 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6549 6550 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -1); 6551 6552 /* 6553 * We need to flush the page from the virtual cache 6554 * in order to prevent a virtual cache alias 6555 * inconsistency. The particular scenario we need 6556 * to worry about is: 6557 * Given: va1 and va2 are two virtual address that 6558 * alias and will map the same physical address. 6559 * 1. mapping exists from va1 to pa and data has 6560 * been read into the cache. 6561 * 2. unload va1. 6562 * 3. load va2 and modify data using va2. 6563 * 4 unload va2. 6564 * 5. load va1 and reference data. Unless we flush 6565 * the data cache when we unload we will get 6566 * stale data. 6567 * This scenario is taken care of by using virtual 6568 * page coloring. 6569 */ 6570 if (sfmmup->sfmmu_ismhat) { 6571 /* 6572 * Flush TSBs, TLBs and caches 6573 * of every process 6574 * sharing this ism segment. 6575 */ 6576 sfmmu_hat_lock_all(); 6577 mutex_enter(&ism_mlist_lock); 6578 kpreempt_disable(); 6579 if (do_virtual_coloring) 6580 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6581 pp->p_pagenum, CACHE_NO_FLUSH); 6582 else 6583 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, 6584 pp->p_pagenum, CACHE_FLUSH); 6585 kpreempt_enable(); 6586 mutex_exit(&ism_mlist_lock); 6587 sfmmu_hat_unlock_all(); 6588 cpuset = cpu_ready_set; 6589 } else if (do_virtual_coloring) { 6590 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6591 cpuset = sfmmup->sfmmu_cpusran; 6592 } else { 6593 sfmmu_tlbcache_demap(addr, sfmmup, hmeblkp, 6594 pp->p_pagenum, 0, FLUSH_NECESSARY_CPUS, 6595 CACHE_FLUSH, 0); 6596 cpuset = sfmmup->sfmmu_cpusran; 6597 } 6598 6599 /* 6600 * Hme_sub has to run after ttesync() and a_rss update. 6601 * See hblk_unload(). 6602 */ 6603 HME_SUB(sfhme, pp); 6604 membar_stst(); 6605 6606 /* 6607 * We can not make ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS) 6608 * since pteload may have done a HME_ADD() right after 6609 * we did the HME_SUB() above. Hmecnt is now maintained 6610 * by cas only. no lock guranteed its value. The only 6611 * gurantee we have is the hmecnt should not be less than 6612 * what it should be so the hblk will not be taken away. 6613 * It's also important that we decremented the hmecnt after 6614 * we are done with hmeblkp so that this hmeblk won't be 6615 * stolen. 6616 */ 6617 ASSERT(hmeblkp->hblk_hmecnt > 0); 6618 ASSERT(hmeblkp->hblk_vcnt > 0); 6619 atomic_add_16(&hmeblkp->hblk_vcnt, -1); 6620 atomic_add_16(&hmeblkp->hblk_hmecnt, -1); 6621 /* 6622 * This is bug 4063182. 6623 * XXX: fixme 6624 * ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || 6625 * !hmeblkp->hblk_lckcnt); 6626 */ 6627 } else { 6628 panic("invalid tte? pp %p &tte %p", 6629 (void *)pp, (void *)&tte); 6630 } 6631 6632 return (cpuset); 6633 } 6634 6635 /* 6636 * While relocating a kernel page, this function will move the mappings 6637 * from tpp to dpp and modify any associated data with these mappings. 6638 * It also unsuspends the suspended kernel mapping. 6639 */ 6640 static void 6641 hat_pagereload(struct page *tpp, struct page *dpp) 6642 { 6643 struct sf_hment *sfhme; 6644 tte_t tte, ttemod; 6645 int index, cons; 6646 6647 ASSERT(getpil() == PIL_MAX); 6648 ASSERT(sfmmu_mlist_held(tpp)); 6649 ASSERT(sfmmu_mlist_held(dpp)); 6650 6651 index = PP_MAPINDEX(tpp); 6652 cons = TTE8K; 6653 6654 /* Update real mappings to the page */ 6655 retry: 6656 for (sfhme = tpp->p_mapping; sfhme != NULL; sfhme = sfhme->hme_next) { 6657 if (IS_PAHME(sfhme)) 6658 continue; 6659 sfmmu_copytte(&sfhme->hme_tte, &tte); 6660 ttemod = tte; 6661 6662 /* 6663 * replace old pfn with new pfn in TTE 6664 */ 6665 PFN_TO_TTE(ttemod, dpp->p_pagenum); 6666 6667 /* 6668 * clear suspend bit 6669 */ 6670 ASSERT(TTE_IS_SUSPEND(&ttemod)); 6671 TTE_CLR_SUSPEND(&ttemod); 6672 6673 if (sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte) < 0) 6674 panic("hat_pagereload(): sfmmu_modifytte_try() failed"); 6675 6676 /* 6677 * set hme_page point to new page 6678 */ 6679 sfhme->hme_page = dpp; 6680 } 6681 6682 /* 6683 * move p_mapping list from old page to new page 6684 */ 6685 dpp->p_mapping = tpp->p_mapping; 6686 tpp->p_mapping = NULL; 6687 dpp->p_share = tpp->p_share; 6688 tpp->p_share = 0; 6689 6690 while (index != 0) { 6691 index = index >> 1; 6692 if (index != 0) 6693 cons++; 6694 if (index & 0x1) { 6695 tpp = PP_GROUPLEADER(tpp, cons); 6696 dpp = PP_GROUPLEADER(dpp, cons); 6697 goto retry; 6698 } 6699 } 6700 6701 if (dtrace_kreloc_fini) 6702 (*dtrace_kreloc_fini)(); 6703 mutex_exit(&kpr_suspendlock); 6704 } 6705 6706 uint_t 6707 hat_pagesync(struct page *pp, uint_t clearflag) 6708 { 6709 struct sf_hment *sfhme, *tmphme = NULL; 6710 struct hme_blk *hmeblkp; 6711 kmutex_t *pml; 6712 cpuset_t cpuset, tset; 6713 int index, cons; 6714 extern ulong_t po_share; 6715 page_t *save_pp = pp; 6716 6717 CPUSET_ZERO(cpuset); 6718 6719 if (PP_ISRO(pp) && (clearflag & HAT_SYNC_STOPON_MOD)) { 6720 return (PP_GENERIC_ATTR(pp)); 6721 } 6722 6723 if ((clearflag == (HAT_SYNC_STOPON_REF | HAT_SYNC_DONTZERO)) && 6724 PP_ISREF(pp)) { 6725 return (PP_GENERIC_ATTR(pp)); 6726 } 6727 6728 if ((clearflag == (HAT_SYNC_STOPON_MOD | HAT_SYNC_DONTZERO)) && 6729 PP_ISMOD(pp)) { 6730 return (PP_GENERIC_ATTR(pp)); 6731 } 6732 6733 if ((clearflag & HAT_SYNC_STOPON_SHARED) != 0 && 6734 (pp->p_share > po_share) && 6735 !(clearflag & HAT_SYNC_ZERORM)) { 6736 if (PP_ISRO(pp)) 6737 hat_page_setattr(pp, P_REF); 6738 return (PP_GENERIC_ATTR(pp)); 6739 } 6740 6741 clearflag &= ~HAT_SYNC_STOPON_SHARED; 6742 pml = sfmmu_mlist_enter(pp); 6743 index = PP_MAPINDEX(pp); 6744 cons = TTE8K; 6745 retry: 6746 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6747 /* 6748 * We need to save the next hment on the list since 6749 * it is possible for pagesync to remove an invalid hment 6750 * from the list. 6751 */ 6752 tmphme = sfhme->hme_next; 6753 /* 6754 * If we are looking for large mappings and this hme doesn't 6755 * reach the range we are seeking, just ignore its. 6756 */ 6757 hmeblkp = sfmmu_hmetohblk(sfhme); 6758 if (hmeblkp->hblk_xhat_bit) 6759 continue; 6760 6761 if (hme_size(sfhme) < cons) 6762 continue; 6763 tset = sfmmu_pagesync(pp, sfhme, 6764 clearflag & ~HAT_SYNC_STOPON_RM); 6765 CPUSET_OR(cpuset, tset); 6766 /* 6767 * If clearflag is HAT_SYNC_DONTZERO, break out as soon 6768 * as the "ref" or "mod" is set. 6769 */ 6770 if ((clearflag & ~HAT_SYNC_STOPON_RM) == HAT_SYNC_DONTZERO && 6771 ((clearflag & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp)) || 6772 ((clearflag & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp))) { 6773 index = 0; 6774 break; 6775 } 6776 } 6777 6778 while (index) { 6779 index = index >> 1; 6780 cons++; 6781 if (index & 0x1) { 6782 /* Go to leading page */ 6783 pp = PP_GROUPLEADER(pp, cons); 6784 goto retry; 6785 } 6786 } 6787 6788 xt_sync(cpuset); 6789 sfmmu_mlist_exit(pml); 6790 return (PP_GENERIC_ATTR(save_pp)); 6791 } 6792 6793 /* 6794 * Get all the hardware dependent attributes for a page struct 6795 */ 6796 static cpuset_t 6797 sfmmu_pagesync(struct page *pp, struct sf_hment *sfhme, 6798 uint_t clearflag) 6799 { 6800 caddr_t addr; 6801 tte_t tte, ttemod; 6802 struct hme_blk *hmeblkp; 6803 int ret; 6804 sfmmu_t *sfmmup; 6805 cpuset_t cpuset; 6806 6807 ASSERT(pp != NULL); 6808 ASSERT(sfmmu_mlist_held(pp)); 6809 ASSERT((clearflag == HAT_SYNC_DONTZERO) || 6810 (clearflag == HAT_SYNC_ZERORM)); 6811 6812 SFMMU_STAT(sf_pagesync); 6813 6814 CPUSET_ZERO(cpuset); 6815 6816 sfmmu_pagesync_retry: 6817 6818 sfmmu_copytte(&sfhme->hme_tte, &tte); 6819 if (TTE_IS_VALID(&tte)) { 6820 hmeblkp = sfmmu_hmetohblk(sfhme); 6821 sfmmup = hblktosfmmu(hmeblkp); 6822 addr = tte_to_vaddr(hmeblkp, tte); 6823 if (clearflag == HAT_SYNC_ZERORM) { 6824 ttemod = tte; 6825 TTE_CLR_RM(&ttemod); 6826 ret = sfmmu_modifytte_try(&tte, &ttemod, 6827 &sfhme->hme_tte); 6828 if (ret < 0) { 6829 /* 6830 * cas failed and the new value is not what 6831 * we want. 6832 */ 6833 goto sfmmu_pagesync_retry; 6834 } 6835 6836 if (ret > 0) { 6837 /* we win the cas */ 6838 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6839 cpuset = sfmmup->sfmmu_cpusran; 6840 } 6841 } 6842 6843 sfmmu_ttesync(sfmmup, addr, &tte, pp); 6844 } 6845 return (cpuset); 6846 } 6847 6848 /* 6849 * Remove write permission from a mappings to a page, so that 6850 * we can detect the next modification of it. This requires modifying 6851 * the TTE then invalidating (demap) any TLB entry using that TTE. 6852 * This code is similar to sfmmu_pagesync(). 6853 */ 6854 static cpuset_t 6855 sfmmu_pageclrwrt(struct page *pp, struct sf_hment *sfhme) 6856 { 6857 caddr_t addr; 6858 tte_t tte; 6859 tte_t ttemod; 6860 struct hme_blk *hmeblkp; 6861 int ret; 6862 sfmmu_t *sfmmup; 6863 cpuset_t cpuset; 6864 6865 ASSERT(pp != NULL); 6866 ASSERT(sfmmu_mlist_held(pp)); 6867 6868 CPUSET_ZERO(cpuset); 6869 SFMMU_STAT(sf_clrwrt); 6870 6871 retry: 6872 6873 sfmmu_copytte(&sfhme->hme_tte, &tte); 6874 if (TTE_IS_VALID(&tte) && TTE_IS_WRITABLE(&tte)) { 6875 hmeblkp = sfmmu_hmetohblk(sfhme); 6876 6877 /* 6878 * xhat mappings should never be to a VMODSORT page. 6879 */ 6880 ASSERT(hmeblkp->hblk_xhat_bit == 0); 6881 6882 sfmmup = hblktosfmmu(hmeblkp); 6883 addr = tte_to_vaddr(hmeblkp, tte); 6884 6885 ttemod = tte; 6886 TTE_CLR_WRT(&ttemod); 6887 TTE_CLR_MOD(&ttemod); 6888 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 6889 6890 /* 6891 * if cas failed and the new value is not what 6892 * we want retry 6893 */ 6894 if (ret < 0) 6895 goto retry; 6896 6897 /* we win the cas */ 6898 if (ret > 0) { 6899 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); 6900 cpuset = sfmmup->sfmmu_cpusran; 6901 } 6902 } 6903 6904 return (cpuset); 6905 } 6906 6907 /* 6908 * Walk all mappings of a page, removing write permission and clearing the 6909 * ref/mod bits. This code is similar to hat_pagesync() 6910 */ 6911 static void 6912 hat_page_clrwrt(page_t *pp) 6913 { 6914 struct sf_hment *sfhme; 6915 struct sf_hment *tmphme = NULL; 6916 kmutex_t *pml; 6917 cpuset_t cpuset; 6918 cpuset_t tset; 6919 int index; 6920 int cons; 6921 6922 CPUSET_ZERO(cpuset); 6923 6924 pml = sfmmu_mlist_enter(pp); 6925 index = PP_MAPINDEX(pp); 6926 cons = TTE8K; 6927 retry: 6928 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) { 6929 tmphme = sfhme->hme_next; 6930 6931 /* 6932 * If we are looking for large mappings and this hme doesn't 6933 * reach the range we are seeking, just ignore its. 6934 */ 6935 6936 if (hme_size(sfhme) < cons) 6937 continue; 6938 6939 tset = sfmmu_pageclrwrt(pp, sfhme); 6940 CPUSET_OR(cpuset, tset); 6941 } 6942 6943 while (index) { 6944 index = index >> 1; 6945 cons++; 6946 if (index & 0x1) { 6947 /* Go to leading page */ 6948 pp = PP_GROUPLEADER(pp, cons); 6949 goto retry; 6950 } 6951 } 6952 6953 xt_sync(cpuset); 6954 sfmmu_mlist_exit(pml); 6955 } 6956 6957 /* 6958 * Set the given REF/MOD/RO bits for the given page. 6959 * For a vnode with a sorted v_pages list, we need to change 6960 * the attributes and the v_pages list together under page_vnode_mutex. 6961 */ 6962 void 6963 hat_page_setattr(page_t *pp, uint_t flag) 6964 { 6965 vnode_t *vp = pp->p_vnode; 6966 page_t **listp; 6967 kmutex_t *pmtx; 6968 kmutex_t *vphm = NULL; 6969 6970 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 6971 6972 /* 6973 * nothing to do if attribute already set 6974 */ 6975 if ((pp->p_nrm & flag) == flag) 6976 return; 6977 6978 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 6979 vphm = page_vnode_mutex(vp); 6980 mutex_enter(vphm); 6981 } 6982 6983 pmtx = sfmmu_page_enter(pp); 6984 pp->p_nrm |= flag; 6985 sfmmu_page_exit(pmtx); 6986 6987 if (vphm != NULL) { 6988 /* 6989 * Some File Systems examine v_pages for NULL w/o 6990 * grabbing the vphm mutex. Must not let it become NULL when 6991 * pp is the only page on the list. 6992 */ 6993 if (pp->p_vpnext != pp) { 6994 page_vpsub(&vp->v_pages, pp); 6995 if (vp->v_pages != NULL) 6996 listp = &vp->v_pages->p_vpprev->p_vpnext; 6997 else 6998 listp = &vp->v_pages; 6999 page_vpadd(listp, pp); 7000 } 7001 mutex_exit(vphm); 7002 } 7003 } 7004 7005 void 7006 hat_page_clrattr(page_t *pp, uint_t flag) 7007 { 7008 vnode_t *vp = pp->p_vnode; 7009 kmutex_t *vphm = NULL; 7010 kmutex_t *pmtx; 7011 7012 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7013 7014 /* 7015 * For vnode with a sorted v_pages list, we need to change 7016 * the attributes and the v_pages list together under page_vnode_mutex. 7017 */ 7018 if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) { 7019 vphm = page_vnode_mutex(vp); 7020 mutex_enter(vphm); 7021 } 7022 7023 pmtx = sfmmu_page_enter(pp); 7024 pp->p_nrm &= ~flag; 7025 sfmmu_page_exit(pmtx); 7026 7027 if (vphm != NULL) { 7028 /* 7029 * Some File Systems examine v_pages for NULL w/o 7030 * grabbing the vphm mutex. Must not let it become NULL when 7031 * pp is the only page on the list. 7032 */ 7033 if (pp->p_vpnext != pp) { 7034 page_vpsub(&vp->v_pages, pp); 7035 page_vpadd(&vp->v_pages, pp); 7036 } 7037 mutex_exit(vphm); 7038 7039 /* 7040 * VMODSORT works by removing write permissions and getting 7041 * a fault when a page is made dirty. At this point 7042 * we need to remove write permission from all mappings 7043 * to this page. 7044 */ 7045 hat_page_clrwrt(pp); 7046 } 7047 } 7048 7049 7050 uint_t 7051 hat_page_getattr(page_t *pp, uint_t flag) 7052 { 7053 ASSERT(!(flag & ~(P_MOD | P_REF | P_RO))); 7054 return ((uint_t)(pp->p_nrm & flag)); 7055 } 7056 7057 /* 7058 * DEBUG kernels: verify that a kernel va<->pa translation 7059 * is safe by checking the underlying page_t is in a page 7060 * relocation-safe state. 7061 */ 7062 #ifdef DEBUG 7063 void 7064 sfmmu_check_kpfn(pfn_t pfn) 7065 { 7066 page_t *pp; 7067 int index, cons; 7068 7069 if (hat_check_vtop == 0) 7070 return; 7071 7072 if (hat_kpr_enabled == 0 || kvseg.s_base == NULL || panicstr) 7073 return; 7074 7075 pp = page_numtopp_nolock(pfn); 7076 if (!pp) 7077 return; 7078 7079 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7080 return; 7081 7082 /* 7083 * Handed a large kernel page, we dig up the root page since we 7084 * know the root page might have the lock also. 7085 */ 7086 if (pp->p_szc != 0) { 7087 index = PP_MAPINDEX(pp); 7088 cons = TTE8K; 7089 again: 7090 while (index != 0) { 7091 index >>= 1; 7092 if (index != 0) 7093 cons++; 7094 if (index & 0x1) { 7095 pp = PP_GROUPLEADER(pp, cons); 7096 goto again; 7097 } 7098 } 7099 } 7100 7101 if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp)) 7102 return; 7103 7104 /* 7105 * Pages need to be locked or allocated "permanent" (either from 7106 * static_arena arena or explicitly setting PG_NORELOC when calling 7107 * page_create_va()) for VA->PA translations to be valid. 7108 */ 7109 if (!PP_ISNORELOC(pp)) 7110 panic("Illegal VA->PA translation, pp 0x%p not permanent", pp); 7111 else 7112 panic("Illegal VA->PA translation, pp 0x%p not locked", pp); 7113 } 7114 #endif /* DEBUG */ 7115 7116 /* 7117 * Returns a page frame number for a given virtual address. 7118 * Returns PFN_INVALID to indicate an invalid mapping 7119 */ 7120 pfn_t 7121 hat_getpfnum(struct hat *hat, caddr_t addr) 7122 { 7123 pfn_t pfn; 7124 tte_t tte; 7125 7126 /* 7127 * We would like to 7128 * ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 7129 * but we can't because the iommu driver will call this 7130 * routine at interrupt time and it can't grab the as lock 7131 * or it will deadlock: A thread could have the as lock 7132 * and be waiting for io. The io can't complete 7133 * because the interrupt thread is blocked trying to grab 7134 * the as lock. 7135 */ 7136 7137 ASSERT(hat->sfmmu_xhat_provider == NULL); 7138 7139 if (hat == ksfmmup) { 7140 if (segkpm && IS_KPM_ADDR(addr)) 7141 return (sfmmu_kpm_vatopfn(addr)); 7142 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7143 == PFN_SUSPENDED) { 7144 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7145 } 7146 sfmmu_check_kpfn(pfn); 7147 return (pfn); 7148 } else { 7149 return (sfmmu_uvatopfn(addr, hat)); 7150 } 7151 } 7152 7153 /* 7154 * hat_getkpfnum() is an obsolete DDI routine, and its use is discouraged. 7155 * Use hat_getpfnum(kas.a_hat, ...) instead. 7156 * 7157 * We'd like to return PFN_INVALID if the mappings have underlying page_t's 7158 * but can't right now due to the fact that some software has grown to use 7159 * this interface incorrectly. So for now when the interface is misused, 7160 * return a warning to the user that in the future it won't work in the 7161 * way they're abusing it, and carry on (after disabling page relocation). 7162 */ 7163 pfn_t 7164 hat_getkpfnum(caddr_t addr) 7165 { 7166 pfn_t pfn; 7167 tte_t tte; 7168 int badcaller = 0; 7169 extern int segkmem_reloc; 7170 7171 if (segkpm && IS_KPM_ADDR(addr)) { 7172 badcaller = 1; 7173 pfn = sfmmu_kpm_vatopfn(addr); 7174 } else { 7175 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte)) 7176 == PFN_SUSPENDED) { 7177 sfmmu_vatopfn_suspended(addr, ksfmmup, &tte); 7178 } 7179 badcaller = pf_is_memory(pfn); 7180 } 7181 7182 if (badcaller) { 7183 /* 7184 * We can't return PFN_INVALID or the caller may panic 7185 * or corrupt the system. The only alternative is to 7186 * disable page relocation at this point for all kernel 7187 * memory. This will impact any callers of page_relocate() 7188 * such as FMA or DR. 7189 * 7190 * RFE: Add junk here to spit out an ereport so the sysadmin 7191 * can be advised that he should upgrade his device driver 7192 * so that this doesn't happen. 7193 */ 7194 hat_getkpfnum_badcall(caller()); 7195 if (hat_kpr_enabled && segkmem_reloc) { 7196 hat_kpr_enabled = 0; 7197 segkmem_reloc = 0; 7198 cmn_err(CE_WARN, "Kernel Page Relocation is DISABLED"); 7199 } 7200 } 7201 return (pfn); 7202 } 7203 7204 pfn_t 7205 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup) 7206 { 7207 struct hmehash_bucket *hmebp; 7208 hmeblk_tag hblktag; 7209 int hmeshift, hashno = 1; 7210 struct hme_blk *hmeblkp = NULL; 7211 7212 struct sf_hment *sfhmep; 7213 tte_t tte; 7214 pfn_t pfn; 7215 7216 /* support for ISM */ 7217 ism_map_t *ism_map; 7218 ism_blk_t *ism_blkp; 7219 int i; 7220 sfmmu_t *ism_hatid = NULL; 7221 sfmmu_t *locked_hatid = NULL; 7222 7223 7224 ASSERT(sfmmup != ksfmmup); 7225 SFMMU_STAT(sf_user_vtop); 7226 /* 7227 * Set ism_hatid if vaddr falls in a ISM segment. 7228 */ 7229 ism_blkp = sfmmup->sfmmu_iblk; 7230 if (ism_blkp) { 7231 sfmmu_ismhat_enter(sfmmup, 0); 7232 locked_hatid = sfmmup; 7233 } 7234 while (ism_blkp && ism_hatid == NULL) { 7235 ism_map = ism_blkp->iblk_maps; 7236 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) { 7237 if (vaddr >= ism_start(ism_map[i]) && 7238 vaddr < ism_end(ism_map[i])) { 7239 sfmmup = ism_hatid = ism_map[i].imap_ismhat; 7240 vaddr = (caddr_t)(vaddr - 7241 ism_start(ism_map[i])); 7242 break; 7243 } 7244 } 7245 ism_blkp = ism_blkp->iblk_next; 7246 } 7247 if (locked_hatid) { 7248 sfmmu_ismhat_exit(locked_hatid, 0); 7249 } 7250 7251 hblktag.htag_id = sfmmup; 7252 do { 7253 hmeshift = HME_HASH_SHIFT(hashno); 7254 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift); 7255 hblktag.htag_rehash = hashno; 7256 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); 7257 7258 SFMMU_HASH_LOCK(hmebp); 7259 7260 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); 7261 if (hmeblkp != NULL) { 7262 HBLKTOHME(sfhmep, hmeblkp, vaddr); 7263 sfmmu_copytte(&sfhmep->hme_tte, &tte); 7264 if (TTE_IS_VALID(&tte)) { 7265 pfn = TTE_TO_PFN(vaddr, &tte); 7266 } else { 7267 pfn = PFN_INVALID; 7268 } 7269 SFMMU_HASH_UNLOCK(hmebp); 7270 return (pfn); 7271 } 7272 SFMMU_HASH_UNLOCK(hmebp); 7273 hashno++; 7274 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); 7275 return (PFN_INVALID); 7276 } 7277 7278 7279 /* 7280 * For compatability with AT&T and later optimizations 7281 */ 7282 /* ARGSUSED */ 7283 void 7284 hat_map(struct hat *hat, caddr_t addr, size_t len, uint_t flags) 7285 { 7286 ASSERT(hat != NULL); 7287 ASSERT(hat->sfmmu_xhat_provider == NULL); 7288 } 7289 7290 /* 7291 * Return the number of mappings to a particular page. 7292 * This number is an approximation of the number of 7293 * number of people sharing the page. 7294 */ 7295 ulong_t 7296 hat_page_getshare(page_t *pp) 7297 { 7298 page_t *spp = pp; /* start page */ 7299 kmutex_t *pml; 7300 ulong_t cnt; 7301 int index, sz = TTE64K; 7302 7303 /* 7304 * We need to grab the mlist lock to make sure any outstanding 7305 * load/unloads complete. Otherwise we could return zero 7306 * even though the unload(s) hasn't finished yet. 7307 */ 7308 pml = sfmmu_mlist_enter(spp); 7309 cnt = spp->p_share; 7310 7311 #ifdef VAC 7312 if (kpm_enable) 7313 cnt += spp->p_kpmref; 7314 #endif 7315 7316 /* 7317 * If we have any large mappings, we count the number of 7318 * mappings that this large page is part of. 7319 */ 7320 index = PP_MAPINDEX(spp); 7321 index >>= 1; 7322 while (index) { 7323 pp = PP_GROUPLEADER(spp, sz); 7324 if ((index & 0x1) && pp != spp) { 7325 cnt += pp->p_share; 7326 spp = pp; 7327 } 7328 index >>= 1; 7329 sz++; 7330 } 7331 sfmmu_mlist_exit(pml); 7332 return (cnt); 7333 } 7334 7335 /* 7336 * Unload all large mappings to the pp and reset the p_szc field of every 7337 * constituent page according to the remaining mappings. 7338 * 7339 * pp must be locked SE_EXCL. Even though no other constituent pages are 7340 * locked it's legal to unload the large mappings to the pp because all 7341 * constituent pages of large locked mappings have to be locked SE_SHARED. 7342 * This means if we have SE_EXCL lock on one of constituent pages none of the 7343 * large mappings to pp are locked. 7344 * 7345 * Decrease p_szc field starting from the last constituent page and ending 7346 * with the root page. This method is used because other threads rely on the 7347 * root's p_szc to find the lock to syncronize on. After a root page_t's p_szc 7348 * is demoted then other threads will succeed in sfmmu_mlspl_enter(). This 7349 * ensures that p_szc changes of the constituent pages appears atomic for all 7350 * threads that use sfmmu_mlspl_enter() to examine p_szc field. 7351 * 7352 * This mechanism is only used for file system pages where it's not always 7353 * possible to get SE_EXCL locks on all constituent pages to demote the size 7354 * code (as is done for anonymous or kernel large pages). 7355 * 7356 * See more comments in front of sfmmu_mlspl_enter(). 7357 */ 7358 void 7359 hat_page_demote(page_t *pp) 7360 { 7361 int index; 7362 int sz; 7363 cpuset_t cpuset; 7364 int sync = 0; 7365 page_t *rootpp; 7366 struct sf_hment *sfhme; 7367 struct sf_hment *tmphme = NULL; 7368 struct hme_blk *hmeblkp; 7369 uint_t pszc; 7370 page_t *lastpp; 7371 cpuset_t tset; 7372 pgcnt_t npgs; 7373 kmutex_t *pml; 7374 kmutex_t *pmtx = NULL; 7375 7376 ASSERT(PAGE_EXCL(pp)); 7377 ASSERT(!PP_ISFREE(pp)); 7378 ASSERT(page_szc_lock_assert(pp)); 7379 pml = sfmmu_mlist_enter(pp); 7380 7381 pszc = pp->p_szc; 7382 if (pszc == 0) { 7383 goto out; 7384 } 7385 7386 index = PP_MAPINDEX(pp) >> 1; 7387 7388 if (index) { 7389 CPUSET_ZERO(cpuset); 7390 sz = TTE64K; 7391 sync = 1; 7392 } 7393 7394 while (index) { 7395 if (!(index & 0x1)) { 7396 index >>= 1; 7397 sz++; 7398 continue; 7399 } 7400 ASSERT(sz <= pszc); 7401 rootpp = PP_GROUPLEADER(pp, sz); 7402 for (sfhme = rootpp->p_mapping; sfhme; sfhme = tmphme) { 7403 tmphme = sfhme->hme_next; 7404 hmeblkp = sfmmu_hmetohblk(sfhme); 7405 if (hme_size(sfhme) != sz) { 7406 continue; 7407 } 7408 if (hmeblkp->hblk_xhat_bit) { 7409 cmn_err(CE_PANIC, 7410 "hat_page_demote: xhat hmeblk"); 7411 } 7412 tset = sfmmu_pageunload(rootpp, sfhme, sz); 7413 CPUSET_OR(cpuset, tset); 7414 } 7415 if (index >>= 1) { 7416 sz++; 7417 } 7418 } 7419 7420 ASSERT(!PP_ISMAPPED_LARGE(pp)); 7421 7422 if (sync) { 7423 xt_sync(cpuset); 7424 #ifdef VAC 7425 if (PP_ISTNC(pp)) { 7426 conv_tnc(rootpp, sz); 7427 } 7428 #endif /* VAC */ 7429 } 7430 7431 pmtx = sfmmu_page_enter(pp); 7432 7433 ASSERT(pp->p_szc == pszc); 7434 rootpp = PP_PAGEROOT(pp); 7435 ASSERT(rootpp->p_szc == pszc); 7436 lastpp = PP_PAGENEXT_N(rootpp, TTEPAGES(pszc) - 1); 7437 7438 while (lastpp != rootpp) { 7439 sz = PP_MAPINDEX(lastpp) ? fnd_mapping_sz(lastpp) : 0; 7440 ASSERT(sz < pszc); 7441 npgs = (sz == 0) ? 1 : TTEPAGES(sz); 7442 ASSERT(P2PHASE(lastpp->p_pagenum, npgs) == npgs - 1); 7443 while (--npgs > 0) { 7444 lastpp->p_szc = (uchar_t)sz; 7445 lastpp = PP_PAGEPREV(lastpp); 7446 } 7447 if (sz) { 7448 /* 7449 * make sure before current root's pszc 7450 * is updated all updates to constituent pages pszc 7451 * fields are globally visible. 7452 */ 7453 membar_producer(); 7454 } 7455 lastpp->p_szc = sz; 7456 ASSERT(IS_P2ALIGNED(lastpp->p_pagenum, TTEPAGES(sz))); 7457 if (lastpp != rootpp) { 7458 lastpp = PP_PAGEPREV(lastpp); 7459 } 7460 } 7461 if (sz == 0) { 7462 /* the loop above doesn't cover this case */ 7463 rootpp->p_szc = 0; 7464 } 7465 out: 7466 ASSERT(pp->p_szc == 0); 7467 if (pmtx != NULL) { 7468 sfmmu_page_exit(pmtx); 7469 } 7470 sfmmu_mlist_exit(pml); 7471 } 7472 7473 /* 7474 * Refresh the HAT ismttecnt[] element for size szc. 7475 * Caller must have set ISM busy flag to prevent mapping 7476 * lists from changing while we're traversing them. 7477 */ 7478 pgcnt_t 7479 ism_tsb_entries(sfmmu_t *sfmmup, int szc) 7480 { 7481 ism_blk_t *ism_blkp = sfmmup->sfmmu_iblk; 7482 ism_map_t *ism_map; 7483 pgcnt_t npgs = 0; 7484 int j; 7485 7486 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 7487 for (; ism_blkp != NULL; ism_blkp = ism_blkp->iblk_next) { 7488 ism_map = ism_blkp->iblk_maps; 7489 for (j = 0; ism_map[j].imap_ismhat && j < ISM_MAP_SLOTS; j++) 7490 npgs += ism_map[j].imap_ismhat->sfmmu_ttecnt[szc]; 7491 } 7492 sfmmup->sfmmu_ismttecnt[szc] = npgs; 7493 return (npgs); 7494 } 7495 7496 /* 7497 * Yield the memory claim requirement for an address space. 7498 * 7499 * This is currently implemented as the number of bytes that have active 7500 * hardware translations that have page structures. Therefore, it can 7501 * underestimate the traditional resident set size, eg, if the 7502 * physical page is present and the hardware translation is missing; 7503 * and it can overestimate the rss, eg, if there are active 7504 * translations to a frame buffer with page structs. 7505 * Also, it does not take sharing into account. 7506 * 7507 * Note that we don't acquire locks here since this function is most often 7508 * called from the clock thread. 7509 */ 7510 size_t 7511 hat_get_mapped_size(struct hat *hat) 7512 { 7513 size_t assize = 0; 7514 int i; 7515 7516 if (hat == NULL) 7517 return (0); 7518 7519 ASSERT(hat->sfmmu_xhat_provider == NULL); 7520 7521 for (i = 0; i < mmu_page_sizes; i++) 7522 assize += (pgcnt_t)hat->sfmmu_ttecnt[i] * TTEBYTES(i); 7523 7524 if (hat->sfmmu_iblk == NULL) 7525 return (assize); 7526 7527 for (i = 0; i < mmu_page_sizes; i++) 7528 assize += (pgcnt_t)hat->sfmmu_ismttecnt[i] * TTEBYTES(i); 7529 7530 return (assize); 7531 } 7532 7533 int 7534 hat_stats_enable(struct hat *hat) 7535 { 7536 hatlock_t *hatlockp; 7537 7538 ASSERT(hat->sfmmu_xhat_provider == NULL); 7539 7540 hatlockp = sfmmu_hat_enter(hat); 7541 hat->sfmmu_rmstat++; 7542 sfmmu_hat_exit(hatlockp); 7543 return (1); 7544 } 7545 7546 void 7547 hat_stats_disable(struct hat *hat) 7548 { 7549 hatlock_t *hatlockp; 7550 7551 ASSERT(hat->sfmmu_xhat_provider == NULL); 7552 7553 hatlockp = sfmmu_hat_enter(hat); 7554 hat->sfmmu_rmstat--; 7555 sfmmu_hat_exit(hatlockp); 7556 } 7557 7558 /* 7559 * Routines for entering or removing ourselves from the 7560 * ism_hat's mapping list. 7561 */ 7562 static void 7563 iment_add(struct ism_ment *iment, struct hat *ism_hat) 7564 { 7565 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7566 7567 iment->iment_prev = NULL; 7568 iment->iment_next = ism_hat->sfmmu_iment; 7569 if (ism_hat->sfmmu_iment) { 7570 ism_hat->sfmmu_iment->iment_prev = iment; 7571 } 7572 ism_hat->sfmmu_iment = iment; 7573 } 7574 7575 static void 7576 iment_sub(struct ism_ment *iment, struct hat *ism_hat) 7577 { 7578 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 7579 7580 if (ism_hat->sfmmu_iment == NULL) { 7581 panic("ism map entry remove - no entries"); 7582 } 7583 7584 if (iment->iment_prev) { 7585 ASSERT(ism_hat->sfmmu_iment != iment); 7586 iment->iment_prev->iment_next = iment->iment_next; 7587 } else { 7588 ASSERT(ism_hat->sfmmu_iment == iment); 7589 ism_hat->sfmmu_iment = iment->iment_next; 7590 } 7591 7592 if (iment->iment_next) { 7593 iment->iment_next->iment_prev = iment->iment_prev; 7594 } 7595 7596 /* 7597 * zero out the entry 7598 */ 7599 iment->iment_next = NULL; 7600 iment->iment_prev = NULL; 7601 iment->iment_hat = NULL; 7602 } 7603 7604 /* 7605 * Hat_share()/unshare() return an (non-zero) error 7606 * when saddr and daddr are not properly aligned. 7607 * 7608 * The top level mapping element determines the alignment 7609 * requirement for saddr and daddr, depending on different 7610 * architectures. 7611 * 7612 * When hat_share()/unshare() are not supported, 7613 * HATOP_SHARE()/UNSHARE() return 0 7614 */ 7615 int 7616 hat_share(struct hat *sfmmup, caddr_t addr, 7617 struct hat *ism_hatid, caddr_t sptaddr, size_t len, uint_t ismszc) 7618 { 7619 ism_blk_t *ism_blkp; 7620 ism_blk_t *new_iblk; 7621 ism_map_t *ism_map; 7622 ism_ment_t *ism_ment; 7623 int i, added; 7624 hatlock_t *hatlockp; 7625 int reload_mmu = 0; 7626 uint_t ismshift = page_get_shift(ismszc); 7627 size_t ismpgsz = page_get_pagesize(ismszc); 7628 uint_t ismmask = (uint_t)ismpgsz - 1; 7629 size_t sh_size = ISM_SHIFT(ismshift, len); 7630 ushort_t ismhatflag; 7631 7632 #ifdef DEBUG 7633 caddr_t eaddr = addr + len; 7634 #endif /* DEBUG */ 7635 7636 ASSERT(ism_hatid != NULL && sfmmup != NULL); 7637 ASSERT(sptaddr == ISMID_STARTADDR); 7638 /* 7639 * Check the alignment. 7640 */ 7641 if (!ISM_ALIGNED(ismshift, addr) || !ISM_ALIGNED(ismshift, sptaddr)) 7642 return (EINVAL); 7643 7644 /* 7645 * Check size alignment. 7646 */ 7647 if (!ISM_ALIGNED(ismshift, len)) 7648 return (EINVAL); 7649 7650 ASSERT(sfmmup->sfmmu_xhat_provider == NULL); 7651 7652 /* 7653 * Allocate ism_ment for the ism_hat's mapping list, and an 7654 * ism map blk in case we need one. We must do our 7655 * allocations before acquiring locks to prevent a deadlock 7656 * in the kmem allocator on the mapping list lock. 7657 */ 7658 new_iblk = kmem_cache_alloc(ism_blk_cache, KM_SLEEP); 7659 ism_ment = kmem_cache_alloc(ism_ment_cache, KM_SLEEP); 7660 7661 /* 7662 * Serialize ISM mappings with the ISM busy flag, and also the 7663 * trap handlers. 7664 */ 7665 sfmmu_ismhat_enter(sfmmup, 0); 7666 7667 /* 7668 * Allocate an ism map blk if necessary. 7669 */ 7670 if (sfmmup->sfmmu_iblk == NULL) { 7671 sfmmup->sfmmu_iblk = new_iblk; 7672 bzero(new_iblk, sizeof (*new_iblk)); 7673 new_iblk->iblk_nextpa = (uint64_t)-1; 7674 membar_stst(); /* make sure next ptr visible to all CPUs */ 7675 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk); 7676 reload_mmu = 1; 7677 new_iblk = NULL; 7678 } 7679 7680 #ifdef DEBUG 7681 /* 7682 * Make sure mapping does not already exist. 7683 */ 7684 ism_blkp = sfmmup->sfmmu_iblk; 7685 while (ism_blkp) { 7686 ism_map = ism_blkp->iblk_maps; 7687 for (i = 0; i < ISM_MAP_SLOTS && ism_map[i].imap_ismhat; i++) { 7688 if ((addr >= ism_start(ism_map[i]) && 7689 addr < ism_end(ism_map[i])) || 7690 eaddr > ism_start(ism_map[i]) && 7691 eaddr <= ism_end(ism_map[i])) { 7692 panic("sfmmu_share: Already mapped!"); 7693 } 7694 } 7695 ism_blkp = ism_blkp->iblk_next; 7696 } 7697 #endif /* DEBUG */ 7698 7699 ASSERT(ismszc >= TTE4M); 7700 if (ismszc == TTE4M) { 7701 ismhatflag = HAT_4M_FLAG; 7702 } else if (ismszc == TTE32M) { 7703 ismhatflag = HAT_32M_FLAG; 7704 } else if (ismszc == TTE256M) { 7705 ismhatflag = HAT_256M_FLAG; 7706 } 7707 /* 7708 * Add mapping to first available mapping slot. 7709 */ 7710 ism_blkp = sfmmup->sfmmu_iblk; 7711 added = 0; 7712 while (!added) { 7713 ism_map = ism_blkp->iblk_maps; 7714 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7715 if (ism_map[i].imap_ismhat == NULL) { 7716 7717 ism_map[i].imap_ismhat = ism_hatid; 7718 ism_map[i].imap_vb_shift = (ushort_t)ismshift; 7719 ism_map[i].imap_hatflags = ismhatflag; 7720 ism_map[i].imap_sz_mask = ismmask; 7721 /* 7722 * imap_seg is checked in ISM_CHECK to see if 7723 * non-NULL, then other info assumed valid. 7724 */ 7725 membar_stst(); 7726 ism_map[i].imap_seg = (uintptr_t)addr | sh_size; 7727 ism_map[i].imap_ment = ism_ment; 7728 7729 /* 7730 * Now add ourselves to the ism_hat's 7731 * mapping list. 7732 */ 7733 ism_ment->iment_hat = sfmmup; 7734 ism_ment->iment_base_va = addr; 7735 ism_hatid->sfmmu_ismhat = 1; 7736 ism_hatid->sfmmu_flags = 0; 7737 mutex_enter(&ism_mlist_lock); 7738 iment_add(ism_ment, ism_hatid); 7739 mutex_exit(&ism_mlist_lock); 7740 added = 1; 7741 break; 7742 } 7743 } 7744 if (!added && ism_blkp->iblk_next == NULL) { 7745 ism_blkp->iblk_next = new_iblk; 7746 new_iblk = NULL; 7747 bzero(ism_blkp->iblk_next, 7748 sizeof (*ism_blkp->iblk_next)); 7749 ism_blkp->iblk_next->iblk_nextpa = (uint64_t)-1; 7750 membar_stst(); 7751 ism_blkp->iblk_nextpa = 7752 va_to_pa((caddr_t)ism_blkp->iblk_next); 7753 } 7754 ism_blkp = ism_blkp->iblk_next; 7755 } 7756 7757 /* 7758 * Update our counters for this sfmmup's ism mappings. 7759 */ 7760 for (i = 0; i <= ismszc; i++) { 7761 if (!(disable_ism_large_pages & (1 << i))) 7762 (void) ism_tsb_entries(sfmmup, i); 7763 } 7764 7765 hatlockp = sfmmu_hat_enter(sfmmup); 7766 7767 /* 7768 * For ISM and DISM we do not support 512K pages, so we only 7769 * only search the 4M and 8K/64K hashes for 4 pagesize cpus, and search 7770 * the 256M or 32M, and 4M and 8K/64K hashes for 6 pagesize cpus. 7771 */ 7772 ASSERT((disable_ism_large_pages & (1 << TTE512K)) != 0); 7773 7774 if (ismszc > TTE4M && !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) 7775 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7776 7777 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_64K_FLAG)) 7778 SFMMU_FLAGS_SET(sfmmup, HAT_64K_FLAG); 7779 7780 /* 7781 * If we updated the ismblkpa for this HAT or we need 7782 * to start searching the 256M or 32M or 4M hash, we must 7783 * make sure all CPUs running this process reload their 7784 * tsbmiss area. Otherwise they will fail to load the mappings 7785 * in the tsbmiss handler and will loop calling pagefault(). 7786 */ 7787 switch (ismszc) { 7788 case TTE256M: 7789 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_FLAG)) { 7790 SFMMU_FLAGS_SET(sfmmup, HAT_256M_FLAG); 7791 sfmmu_sync_mmustate(sfmmup); 7792 } 7793 break; 7794 case TTE32M: 7795 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_FLAG)) { 7796 SFMMU_FLAGS_SET(sfmmup, HAT_32M_FLAG); 7797 sfmmu_sync_mmustate(sfmmup); 7798 } 7799 break; 7800 case TTE4M: 7801 if (reload_mmu || !SFMMU_FLAGS_ISSET(sfmmup, HAT_4M_FLAG)) { 7802 SFMMU_FLAGS_SET(sfmmup, HAT_4M_FLAG); 7803 sfmmu_sync_mmustate(sfmmup); 7804 } 7805 break; 7806 default: 7807 break; 7808 } 7809 7810 /* 7811 * Now we can drop the locks. 7812 */ 7813 sfmmu_ismhat_exit(sfmmup, 1); 7814 sfmmu_hat_exit(hatlockp); 7815 7816 /* 7817 * Free up ismblk if we didn't use it. 7818 */ 7819 if (new_iblk != NULL) 7820 kmem_cache_free(ism_blk_cache, new_iblk); 7821 7822 /* 7823 * Check TSB and TLB page sizes. 7824 */ 7825 sfmmu_check_page_sizes(sfmmup, 1); 7826 7827 return (0); 7828 } 7829 7830 /* 7831 * hat_unshare removes exactly one ism_map from 7832 * this process's as. It expects multiple calls 7833 * to hat_unshare for multiple shm segments. 7834 */ 7835 void 7836 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) 7837 { 7838 ism_map_t *ism_map; 7839 ism_ment_t *free_ment = NULL; 7840 ism_blk_t *ism_blkp; 7841 struct hat *ism_hatid; 7842 int found, i; 7843 hatlock_t *hatlockp; 7844 struct tsb_info *tsbinfo; 7845 uint_t ismshift = page_get_shift(ismszc); 7846 size_t sh_size = ISM_SHIFT(ismshift, len); 7847 7848 ASSERT(ISM_ALIGNED(ismshift, addr)); 7849 ASSERT(ISM_ALIGNED(ismshift, len)); 7850 ASSERT(sfmmup != NULL); 7851 ASSERT(sfmmup != ksfmmup); 7852 7853 if (sfmmup->sfmmu_xhat_provider) { 7854 XHAT_UNSHARE(sfmmup, addr, len); 7855 return; 7856 } else { 7857 /* 7858 * This must be a CPU HAT. If the address space has 7859 * XHATs attached, inform all XHATs that ISM segment 7860 * is going away 7861 */ 7862 ASSERT(sfmmup->sfmmu_as != NULL); 7863 if (sfmmup->sfmmu_as->a_xhat != NULL) 7864 xhat_unshare_all(sfmmup->sfmmu_as, addr, len); 7865 } 7866 7867 /* 7868 * Make sure that during the entire time ISM mappings are removed, 7869 * the trap handlers serialize behind us, and that no one else 7870 * can be mucking with ISM mappings. This also lets us get away 7871 * with not doing expensive cross calls to flush the TLB -- we 7872 * just discard the context, flush the entire TSB, and call it 7873 * a day. 7874 */ 7875 sfmmu_ismhat_enter(sfmmup, 0); 7876 7877 /* 7878 * Remove the mapping. 7879 * 7880 * We can't have any holes in the ism map. 7881 * The tsb miss code while searching the ism map will 7882 * stop on an empty map slot. So we must move 7883 * everyone past the hole up 1 if any. 7884 * 7885 * Also empty ism map blks are not freed until the 7886 * process exits. This is to prevent a MT race condition 7887 * between sfmmu_unshare() and sfmmu_tsbmiss_exception(). 7888 */ 7889 found = 0; 7890 ism_blkp = sfmmup->sfmmu_iblk; 7891 while (!found && ism_blkp) { 7892 ism_map = ism_blkp->iblk_maps; 7893 for (i = 0; i < ISM_MAP_SLOTS; i++) { 7894 if (addr == ism_start(ism_map[i]) && 7895 sh_size == (size_t)(ism_size(ism_map[i]))) { 7896 found = 1; 7897 break; 7898 } 7899 } 7900 if (!found) 7901 ism_blkp = ism_blkp->iblk_next; 7902 } 7903 7904 if (found) { 7905 ism_hatid = ism_map[i].imap_ismhat; 7906 ASSERT(ism_hatid != NULL); 7907 ASSERT(ism_hatid->sfmmu_ismhat == 1); 7908 7909 /* 7910 * First remove ourselves from the ism mapping list. 7911 */ 7912 mutex_enter(&ism_mlist_lock); 7913 iment_sub(ism_map[i].imap_ment, ism_hatid); 7914 mutex_exit(&ism_mlist_lock); 7915 free_ment = ism_map[i].imap_ment; 7916 7917 /* 7918 * Now gurantee that any other cpu 7919 * that tries to process an ISM miss 7920 * will go to tl=0. 7921 */ 7922 hatlockp = sfmmu_hat_enter(sfmmup); 7923 7924 sfmmu_invalidate_ctx(sfmmup); 7925 7926 sfmmu_hat_exit(hatlockp); 7927 7928 /* 7929 * We delete the ism map by copying 7930 * the next map over the current one. 7931 * We will take the next one in the maps 7932 * array or from the next ism_blk. 7933 */ 7934 while (ism_blkp) { 7935 ism_map = ism_blkp->iblk_maps; 7936 while (i < (ISM_MAP_SLOTS - 1)) { 7937 ism_map[i] = ism_map[i + 1]; 7938 i++; 7939 } 7940 /* i == (ISM_MAP_SLOTS - 1) */ 7941 ism_blkp = ism_blkp->iblk_next; 7942 if (ism_blkp) { 7943 ism_map[i] = ism_blkp->iblk_maps[0]; 7944 i = 0; 7945 } else { 7946 ism_map[i].imap_seg = 0; 7947 ism_map[i].imap_vb_shift = 0; 7948 ism_map[i].imap_hatflags = 0; 7949 ism_map[i].imap_sz_mask = 0; 7950 ism_map[i].imap_ismhat = NULL; 7951 ism_map[i].imap_ment = NULL; 7952 } 7953 } 7954 7955 /* 7956 * Now flush entire TSB for the process, since 7957 * demapping page by page can be too expensive. 7958 * We don't have to flush the TLB here anymore 7959 * since we switch to a new TLB ctx instead. 7960 * Also, there is no need to flush if the process 7961 * is exiting since the TSB will be freed later. 7962 */ 7963 if (!sfmmup->sfmmu_free) { 7964 hatlockp = sfmmu_hat_enter(sfmmup); 7965 for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL; 7966 tsbinfo = tsbinfo->tsb_next) { 7967 if (tsbinfo->tsb_flags & TSB_SWAPPED) 7968 continue; 7969 sfmmu_inv_tsb(tsbinfo->tsb_va, 7970 TSB_BYTES(tsbinfo->tsb_szc)); 7971 } 7972 sfmmu_hat_exit(hatlockp); 7973 } 7974 } 7975 7976 /* 7977 * Update our counters for this sfmmup's ism mappings. 7978 */ 7979 for (i = 0; i <= ismszc; i++) { 7980 if (!(disable_ism_large_pages & (1 << i))) 7981 (void) ism_tsb_entries(sfmmup, i); 7982 } 7983 7984 sfmmu_ismhat_exit(sfmmup, 0); 7985 7986 /* 7987 * We must do our freeing here after dropping locks 7988 * to prevent a deadlock in the kmem allocator on the 7989 * mapping list lock. 7990 */ 7991 if (free_ment != NULL) 7992 kmem_cache_free(ism_ment_cache, free_ment); 7993 7994 /* 7995 * Check TSB and TLB page sizes if the process isn't exiting. 7996 */ 7997 if (!sfmmup->sfmmu_free) 7998 sfmmu_check_page_sizes(sfmmup, 0); 7999 } 8000 8001 /* ARGSUSED */ 8002 static int 8003 sfmmu_idcache_constructor(void *buf, void *cdrarg, int kmflags) 8004 { 8005 /* void *buf is sfmmu_t pointer */ 8006 return (0); 8007 } 8008 8009 /* ARGSUSED */ 8010 static void 8011 sfmmu_idcache_destructor(void *buf, void *cdrarg) 8012 { 8013 /* void *buf is sfmmu_t pointer */ 8014 } 8015 8016 /* 8017 * setup kmem hmeblks by bzeroing all members and initializing the nextpa 8018 * field to be the pa of this hmeblk 8019 */ 8020 /* ARGSUSED */ 8021 static int 8022 sfmmu_hblkcache_constructor(void *buf, void *cdrarg, int kmflags) 8023 { 8024 struct hme_blk *hmeblkp; 8025 8026 bzero(buf, (size_t)cdrarg); 8027 hmeblkp = (struct hme_blk *)buf; 8028 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); 8029 8030 #ifdef HBLK_TRACE 8031 mutex_init(&hmeblkp->hblk_audit_lock, NULL, MUTEX_DEFAULT, NULL); 8032 #endif /* HBLK_TRACE */ 8033 8034 return (0); 8035 } 8036 8037 /* ARGSUSED */ 8038 static void 8039 sfmmu_hblkcache_destructor(void *buf, void *cdrarg) 8040 { 8041 8042 #ifdef HBLK_TRACE 8043 8044 struct hme_blk *hmeblkp; 8045 8046 hmeblkp = (struct hme_blk *)buf; 8047 mutex_destroy(&hmeblkp->hblk_audit_lock); 8048 8049 #endif /* HBLK_TRACE */ 8050 } 8051 8052 #define SFMMU_CACHE_RECLAIM_SCAN_RATIO 8 8053 static int sfmmu_cache_reclaim_scan_ratio = SFMMU_CACHE_RECLAIM_SCAN_RATIO; 8054 /* 8055 * The kmem allocator will callback into our reclaim routine when the system 8056 * is running low in memory. We traverse the hash and free up all unused but 8057 * still cached hme_blks. We also traverse the free list and free them up 8058 * as well. 8059 */ 8060 /*ARGSUSED*/ 8061 static void 8062 sfmmu_hblkcache_reclaim(void *cdrarg) 8063 { 8064 int i; 8065 uint64_t hblkpa, prevpa, nx_pa; 8066 struct hmehash_bucket *hmebp; 8067 struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL; 8068 static struct hmehash_bucket *uhmehash_reclaim_hand; 8069 static struct hmehash_bucket *khmehash_reclaim_hand; 8070 struct hme_blk *list = NULL; 8071 8072 hmebp = uhmehash_reclaim_hand; 8073 if (hmebp == NULL || hmebp > &uhme_hash[UHMEHASH_SZ]) 8074 uhmehash_reclaim_hand = hmebp = uhme_hash; 8075 uhmehash_reclaim_hand += UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8076 8077 for (i = UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8078 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8079 hmeblkp = hmebp->hmeblkp; 8080 hblkpa = hmebp->hmeh_nextpa; 8081 prevpa = 0; 8082 pr_hblk = NULL; 8083 while (hmeblkp) { 8084 nx_hblk = hmeblkp->hblk_next; 8085 nx_pa = hmeblkp->hblk_nextpa; 8086 if (!hmeblkp->hblk_vcnt && 8087 !hmeblkp->hblk_hmecnt) { 8088 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8089 prevpa, pr_hblk); 8090 sfmmu_hblk_free(hmebp, hmeblkp, 8091 hblkpa, &list); 8092 } else { 8093 pr_hblk = hmeblkp; 8094 prevpa = hblkpa; 8095 } 8096 hmeblkp = nx_hblk; 8097 hblkpa = nx_pa; 8098 } 8099 SFMMU_HASH_UNLOCK(hmebp); 8100 } 8101 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 8102 hmebp = uhme_hash; 8103 } 8104 8105 hmebp = khmehash_reclaim_hand; 8106 if (hmebp == NULL || hmebp > &khme_hash[KHMEHASH_SZ]) 8107 khmehash_reclaim_hand = hmebp = khme_hash; 8108 khmehash_reclaim_hand += KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; 8109 8110 for (i = KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) { 8111 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) { 8112 hmeblkp = hmebp->hmeblkp; 8113 hblkpa = hmebp->hmeh_nextpa; 8114 prevpa = 0; 8115 pr_hblk = NULL; 8116 while (hmeblkp) { 8117 nx_hblk = hmeblkp->hblk_next; 8118 nx_pa = hmeblkp->hblk_nextpa; 8119 if (!hmeblkp->hblk_vcnt && 8120 !hmeblkp->hblk_hmecnt) { 8121 sfmmu_hblk_hash_rm(hmebp, hmeblkp, 8122 prevpa, pr_hblk); 8123 sfmmu_hblk_free(hmebp, hmeblkp, 8124 hblkpa, &list); 8125 } else { 8126 pr_hblk = hmeblkp; 8127 prevpa = hblkpa; 8128 } 8129 hmeblkp = nx_hblk; 8130 hblkpa = nx_pa; 8131 } 8132 SFMMU_HASH_UNLOCK(hmebp); 8133 } 8134 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 8135 hmebp = khme_hash; 8136 } 8137 sfmmu_hblks_list_purge(&list); 8138 } 8139 8140 /* 8141 * sfmmu_get_ppvcolor should become a vm_machdep or hatop interface. 8142 * same goes for sfmmu_get_addrvcolor(). 8143 * 8144 * This function will return the virtual color for the specified page. The 8145 * virtual color corresponds to this page current mapping or its last mapping. 8146 * It is used by memory allocators to choose addresses with the correct 8147 * alignment so vac consistency is automatically maintained. If the page 8148 * has no color it returns -1. 8149 */ 8150 /*ARGSUSED*/ 8151 int 8152 sfmmu_get_ppvcolor(struct page *pp) 8153 { 8154 #ifdef VAC 8155 int color; 8156 8157 if (!(cache & CACHE_VAC) || PP_NEWPAGE(pp)) { 8158 return (-1); 8159 } 8160 color = PP_GET_VCOLOR(pp); 8161 ASSERT(color < mmu_btop(shm_alignment)); 8162 return (color); 8163 #else 8164 return (-1); 8165 #endif /* VAC */ 8166 } 8167 8168 /* 8169 * This function will return the desired alignment for vac consistency 8170 * (vac color) given a virtual address. If no vac is present it returns -1. 8171 */ 8172 /*ARGSUSED*/ 8173 int 8174 sfmmu_get_addrvcolor(caddr_t vaddr) 8175 { 8176 #ifdef VAC 8177 if (cache & CACHE_VAC) { 8178 return (addr_to_vcolor(vaddr)); 8179 } else { 8180 return (-1); 8181 } 8182 #else 8183 return (-1); 8184 #endif /* VAC */ 8185 } 8186 8187 #ifdef VAC 8188 /* 8189 * Check for conflicts. 8190 * A conflict exists if the new and existent mappings do not match in 8191 * their "shm_alignment fields. If conflicts exist, the existant mappings 8192 * are flushed unless one of them is locked. If one of them is locked, then 8193 * the mappings are flushed and converted to non-cacheable mappings. 8194 */ 8195 static void 8196 sfmmu_vac_conflict(struct hat *hat, caddr_t addr, page_t *pp) 8197 { 8198 struct hat *tmphat; 8199 struct sf_hment *sfhmep, *tmphme = NULL; 8200 struct hme_blk *hmeblkp; 8201 int vcolor; 8202 tte_t tte; 8203 8204 ASSERT(sfmmu_mlist_held(pp)); 8205 ASSERT(!PP_ISNC(pp)); /* page better be cacheable */ 8206 8207 vcolor = addr_to_vcolor(addr); 8208 if (PP_NEWPAGE(pp)) { 8209 PP_SET_VCOLOR(pp, vcolor); 8210 return; 8211 } 8212 8213 if (PP_GET_VCOLOR(pp) == vcolor) { 8214 return; 8215 } 8216 8217 if (!PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp)) { 8218 /* 8219 * Previous user of page had a different color 8220 * but since there are no current users 8221 * we just flush the cache and change the color. 8222 */ 8223 SFMMU_STAT(sf_pgcolor_conflict); 8224 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8225 PP_SET_VCOLOR(pp, vcolor); 8226 return; 8227 } 8228 8229 /* 8230 * If we get here we have a vac conflict with a current 8231 * mapping. VAC conflict policy is as follows. 8232 * - The default is to unload the other mappings unless: 8233 * - If we have a large mapping we uncache the page. 8234 * We need to uncache the rest of the large page too. 8235 * - If any of the mappings are locked we uncache the page. 8236 * - If the requested mapping is inconsistent 8237 * with another mapping and that mapping 8238 * is in the same address space we have to 8239 * make it non-cached. The default thing 8240 * to do is unload the inconsistent mapping 8241 * but if they are in the same address space 8242 * we run the risk of unmapping the pc or the 8243 * stack which we will use as we return to the user, 8244 * in which case we can then fault on the thing 8245 * we just unloaded and get into an infinite loop. 8246 */ 8247 if (PP_ISMAPPED_LARGE(pp)) { 8248 int sz; 8249 8250 /* 8251 * Existing mapping is for big pages. We don't unload 8252 * existing big mappings to satisfy new mappings. 8253 * Always convert all mappings to TNC. 8254 */ 8255 sz = fnd_mapping_sz(pp); 8256 pp = PP_GROUPLEADER(pp, sz); 8257 SFMMU_STAT_ADD(sf_uncache_conflict, TTEPAGES(sz)); 8258 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 8259 TTEPAGES(sz)); 8260 8261 return; 8262 } 8263 8264 /* 8265 * check if any mapping is in same as or if it is locked 8266 * since in that case we need to uncache. 8267 */ 8268 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8269 tmphme = sfhmep->hme_next; 8270 hmeblkp = sfmmu_hmetohblk(sfhmep); 8271 if (hmeblkp->hblk_xhat_bit) 8272 continue; 8273 tmphat = hblktosfmmu(hmeblkp); 8274 sfmmu_copytte(&sfhmep->hme_tte, &tte); 8275 ASSERT(TTE_IS_VALID(&tte)); 8276 if ((tmphat == hat) || hmeblkp->hblk_lckcnt) { 8277 /* 8278 * We have an uncache conflict 8279 */ 8280 SFMMU_STAT(sf_uncache_conflict); 8281 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1); 8282 return; 8283 } 8284 } 8285 8286 /* 8287 * We have an unload conflict 8288 * We have already checked for LARGE mappings, therefore 8289 * the remaining mapping(s) must be TTE8K. 8290 */ 8291 SFMMU_STAT(sf_unload_conflict); 8292 8293 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) { 8294 tmphme = sfhmep->hme_next; 8295 hmeblkp = sfmmu_hmetohblk(sfhmep); 8296 if (hmeblkp->hblk_xhat_bit) 8297 continue; 8298 (void) sfmmu_pageunload(pp, sfhmep, TTE8K); 8299 } 8300 8301 if (PP_ISMAPPED_KPM(pp)) 8302 sfmmu_kpm_vac_unload(pp, addr); 8303 8304 /* 8305 * Unloads only do TLB flushes so we need to flush the 8306 * cache here. 8307 */ 8308 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp)); 8309 PP_SET_VCOLOR(pp, vcolor); 8310 } 8311 8312 /* 8313 * Whenever a mapping is unloaded and the page is in TNC state, 8314 * we see if the page can be made cacheable again. 'pp' is 8315 * the page that we just unloaded a mapping from, the size 8316 * of mapping that was unloaded is 'ottesz'. 8317 * Remark: 8318 * The recache policy for mpss pages can leave a performance problem 8319 * under the following circumstances: 8320 * . A large page in uncached mode has just been unmapped. 8321 * . All constituent pages are TNC due to a conflicting small mapping. 8322 * . There are many other, non conflicting, small mappings around for 8323 * a lot of the constituent pages. 8324 * . We're called w/ the "old" groupleader page and the old ottesz, 8325 * but this is irrelevant, since we're no more "PP_ISMAPPED_LARGE", so 8326 * we end up w/ TTE8K or npages == 1. 8327 * . We call tst_tnc w/ the old groupleader only, and if there is no 8328 * conflict, we re-cache only this page. 8329 * . All other small mappings are not checked and will be left in TNC mode. 8330 * The problem is not very serious because: 8331 * . mpss is actually only defined for heap and stack, so the probability 8332 * is not very high that a large page mapping exists in parallel to a small 8333 * one (this is possible, but seems to be bad programming style in the 8334 * appl). 8335 * . The problem gets a little bit more serious, when those TNC pages 8336 * have to be mapped into kernel space, e.g. for networking. 8337 * . When VAC alias conflicts occur in applications, this is regarded 8338 * as an application bug. So if kstat's show them, the appl should 8339 * be changed anyway. 8340 */ 8341 void 8342 conv_tnc(page_t *pp, int ottesz) 8343 { 8344 int cursz, dosz; 8345 pgcnt_t curnpgs, dopgs; 8346 pgcnt_t pg64k; 8347 page_t *pp2; 8348 8349 /* 8350 * Determine how big a range we check for TNC and find 8351 * leader page. cursz is the size of the biggest 8352 * mapping that still exist on 'pp'. 8353 */ 8354 if (PP_ISMAPPED_LARGE(pp)) { 8355 cursz = fnd_mapping_sz(pp); 8356 } else { 8357 cursz = TTE8K; 8358 } 8359 8360 if (ottesz >= cursz) { 8361 dosz = ottesz; 8362 pp2 = pp; 8363 } else { 8364 dosz = cursz; 8365 pp2 = PP_GROUPLEADER(pp, dosz); 8366 } 8367 8368 pg64k = TTEPAGES(TTE64K); 8369 dopgs = TTEPAGES(dosz); 8370 8371 ASSERT(dopgs == 1 || ((dopgs & (pg64k - 1)) == 0)); 8372 8373 while (dopgs != 0) { 8374 curnpgs = TTEPAGES(cursz); 8375 if (tst_tnc(pp2, curnpgs)) { 8376 SFMMU_STAT_ADD(sf_recache, curnpgs); 8377 sfmmu_page_cache_array(pp2, HAT_CACHE, CACHE_NO_FLUSH, 8378 curnpgs); 8379 } 8380 8381 ASSERT(dopgs >= curnpgs); 8382 dopgs -= curnpgs; 8383 8384 if (dopgs == 0) { 8385 break; 8386 } 8387 8388 pp2 = PP_PAGENEXT_N(pp2, curnpgs); 8389 if (((dopgs & (pg64k - 1)) == 0) && PP_ISMAPPED_LARGE(pp2)) { 8390 cursz = fnd_mapping_sz(pp2); 8391 } else { 8392 cursz = TTE8K; 8393 } 8394 } 8395 } 8396 8397 /* 8398 * Returns 1 if page(s) can be converted from TNC to cacheable setting, 8399 * returns 0 otherwise. Note that oaddr argument is valid for only 8400 * 8k pages. 8401 */ 8402 int 8403 tst_tnc(page_t *pp, pgcnt_t npages) 8404 { 8405 struct sf_hment *sfhme; 8406 struct hme_blk *hmeblkp; 8407 tte_t tte; 8408 caddr_t vaddr; 8409 int clr_valid = 0; 8410 int color, color1, bcolor; 8411 int i, ncolors; 8412 8413 ASSERT(pp != NULL); 8414 ASSERT(!(cache & CACHE_WRITEBACK)); 8415 8416 if (npages > 1) { 8417 ncolors = CACHE_NUM_COLOR; 8418 } 8419 8420 for (i = 0; i < npages; i++) { 8421 ASSERT(sfmmu_mlist_held(pp)); 8422 ASSERT(PP_ISTNC(pp)); 8423 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR); 8424 8425 if (PP_ISPNC(pp)) { 8426 return (0); 8427 } 8428 8429 clr_valid = 0; 8430 if (PP_ISMAPPED_KPM(pp)) { 8431 caddr_t kpmvaddr; 8432 8433 ASSERT(kpm_enable); 8434 kpmvaddr = hat_kpm_page2va(pp, 1); 8435 ASSERT(!(npages > 1 && IS_KPM_ALIAS_RANGE(kpmvaddr))); 8436 color1 = addr_to_vcolor(kpmvaddr); 8437 clr_valid = 1; 8438 } 8439 8440 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8441 hmeblkp = sfmmu_hmetohblk(sfhme); 8442 if (hmeblkp->hblk_xhat_bit) 8443 continue; 8444 8445 sfmmu_copytte(&sfhme->hme_tte, &tte); 8446 ASSERT(TTE_IS_VALID(&tte)); 8447 8448 vaddr = tte_to_vaddr(hmeblkp, tte); 8449 color = addr_to_vcolor(vaddr); 8450 8451 if (npages > 1) { 8452 /* 8453 * If there is a big mapping, make sure 8454 * 8K mapping is consistent with the big 8455 * mapping. 8456 */ 8457 bcolor = i % ncolors; 8458 if (color != bcolor) { 8459 return (0); 8460 } 8461 } 8462 if (!clr_valid) { 8463 clr_valid = 1; 8464 color1 = color; 8465 } 8466 8467 if (color1 != color) { 8468 return (0); 8469 } 8470 } 8471 8472 pp = PP_PAGENEXT(pp); 8473 } 8474 8475 return (1); 8476 } 8477 8478 void 8479 sfmmu_page_cache_array(page_t *pp, int flags, int cache_flush_flag, 8480 pgcnt_t npages) 8481 { 8482 kmutex_t *pmtx; 8483 int i, ncolors, bcolor; 8484 kpm_hlk_t *kpmp; 8485 cpuset_t cpuset; 8486 8487 ASSERT(pp != NULL); 8488 ASSERT(!(cache & CACHE_WRITEBACK)); 8489 8490 kpmp = sfmmu_kpm_kpmp_enter(pp, npages); 8491 pmtx = sfmmu_page_enter(pp); 8492 8493 /* 8494 * Fast path caching single unmapped page 8495 */ 8496 if (npages == 1 && !PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp) && 8497 flags == HAT_CACHE) { 8498 PP_CLRTNC(pp); 8499 PP_CLRPNC(pp); 8500 sfmmu_page_exit(pmtx); 8501 sfmmu_kpm_kpmp_exit(kpmp); 8502 return; 8503 } 8504 8505 /* 8506 * We need to capture all cpus in order to change cacheability 8507 * because we can't allow one cpu to access the same physical 8508 * page using a cacheable and a non-cachebale mapping at the same 8509 * time. Since we may end up walking the ism mapping list 8510 * have to grab it's lock now since we can't after all the 8511 * cpus have been captured. 8512 */ 8513 sfmmu_hat_lock_all(); 8514 mutex_enter(&ism_mlist_lock); 8515 kpreempt_disable(); 8516 cpuset = cpu_ready_set; 8517 xc_attention(cpuset); 8518 8519 if (npages > 1) { 8520 /* 8521 * Make sure all colors are flushed since the 8522 * sfmmu_page_cache() only flushes one color- 8523 * it does not know big pages. 8524 */ 8525 ncolors = CACHE_NUM_COLOR; 8526 if (flags & HAT_TMPNC) { 8527 for (i = 0; i < ncolors; i++) { 8528 sfmmu_cache_flushcolor(i, pp->p_pagenum); 8529 } 8530 cache_flush_flag = CACHE_NO_FLUSH; 8531 } 8532 } 8533 8534 for (i = 0; i < npages; i++) { 8535 8536 ASSERT(sfmmu_mlist_held(pp)); 8537 8538 if (!(flags == HAT_TMPNC && PP_ISTNC(pp))) { 8539 8540 if (npages > 1) { 8541 bcolor = i % ncolors; 8542 } else { 8543 bcolor = NO_VCOLOR; 8544 } 8545 8546 sfmmu_page_cache(pp, flags, cache_flush_flag, 8547 bcolor); 8548 } 8549 8550 pp = PP_PAGENEXT(pp); 8551 } 8552 8553 xt_sync(cpuset); 8554 xc_dismissed(cpuset); 8555 mutex_exit(&ism_mlist_lock); 8556 sfmmu_hat_unlock_all(); 8557 sfmmu_page_exit(pmtx); 8558 sfmmu_kpm_kpmp_exit(kpmp); 8559 kpreempt_enable(); 8560 } 8561 8562 /* 8563 * This function changes the virtual cacheability of all mappings to a 8564 * particular page. When changing from uncache to cacheable the mappings will 8565 * only be changed if all of them have the same virtual color. 8566 * We need to flush the cache in all cpus. It is possible that 8567 * a process referenced a page as cacheable but has sinced exited 8568 * and cleared the mapping list. We still to flush it but have no 8569 * state so all cpus is the only alternative. 8570 */ 8571 static void 8572 sfmmu_page_cache(page_t *pp, int flags, int cache_flush_flag, int bcolor) 8573 { 8574 struct sf_hment *sfhme; 8575 struct hme_blk *hmeblkp; 8576 sfmmu_t *sfmmup; 8577 tte_t tte, ttemod; 8578 caddr_t vaddr; 8579 int ret, color; 8580 pfn_t pfn; 8581 8582 color = bcolor; 8583 pfn = pp->p_pagenum; 8584 8585 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) { 8586 8587 hmeblkp = sfmmu_hmetohblk(sfhme); 8588 8589 if (hmeblkp->hblk_xhat_bit) 8590 continue; 8591 8592 sfmmu_copytte(&sfhme->hme_tte, &tte); 8593 ASSERT(TTE_IS_VALID(&tte)); 8594 vaddr = tte_to_vaddr(hmeblkp, tte); 8595 color = addr_to_vcolor(vaddr); 8596 8597 #ifdef DEBUG 8598 if ((flags & HAT_CACHE) && bcolor != NO_VCOLOR) { 8599 ASSERT(color == bcolor); 8600 } 8601 #endif 8602 8603 ASSERT(flags != HAT_TMPNC || color == PP_GET_VCOLOR(pp)); 8604 8605 ttemod = tte; 8606 if (flags & (HAT_UNCACHE | HAT_TMPNC)) { 8607 TTE_CLR_VCACHEABLE(&ttemod); 8608 } else { /* flags & HAT_CACHE */ 8609 TTE_SET_VCACHEABLE(&ttemod); 8610 } 8611 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte); 8612 if (ret < 0) { 8613 /* 8614 * Since all cpus are captured modifytte should not 8615 * fail. 8616 */ 8617 panic("sfmmu_page_cache: write to tte failed"); 8618 } 8619 8620 sfmmup = hblktosfmmu(hmeblkp); 8621 if (cache_flush_flag == CACHE_FLUSH) { 8622 /* 8623 * Flush TSBs, TLBs and caches 8624 */ 8625 if (sfmmup->sfmmu_ismhat) { 8626 if (flags & HAT_CACHE) { 8627 SFMMU_STAT(sf_ism_recache); 8628 } else { 8629 SFMMU_STAT(sf_ism_uncache); 8630 } 8631 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8632 pfn, CACHE_FLUSH); 8633 } else { 8634 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp, 8635 pfn, 0, FLUSH_ALL_CPUS, CACHE_FLUSH, 1); 8636 } 8637 8638 /* 8639 * all cache entries belonging to this pfn are 8640 * now flushed. 8641 */ 8642 cache_flush_flag = CACHE_NO_FLUSH; 8643 } else { 8644 8645 /* 8646 * Flush only TSBs and TLBs. 8647 */ 8648 if (sfmmup->sfmmu_ismhat) { 8649 if (flags & HAT_CACHE) { 8650 SFMMU_STAT(sf_ism_recache); 8651 } else { 8652 SFMMU_STAT(sf_ism_uncache); 8653 } 8654 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, 8655 pfn, CACHE_NO_FLUSH); 8656 } else { 8657 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1); 8658 } 8659 } 8660 } 8661 8662 if (PP_ISMAPPED_KPM(pp)) 8663 sfmmu_kpm_page_cache(pp, flags, cache_flush_flag); 8664 8665 switch (flags) { 8666 8667 default: 8668 panic("sfmmu_pagecache: unknown flags"); 8669 break; 8670 8671 case HAT_CACHE: 8672 PP_CLRTNC(pp); 8673 PP_CLRPNC(pp); 8674 PP_SET_VCOLOR(pp, color); 8675 break; 8676 8677 case HAT_TMPNC: 8678 PP_SETTNC(pp); 8679 PP_SET_VCOLOR(pp, NO_VCOLOR); 8680 break; 8681 8682 case HAT_UNCACHE: 8683 PP_SETPNC(pp); 8684 PP_CLRTNC(pp); 8685 PP_SET_VCOLOR(pp, NO_VCOLOR); 8686 break; 8687 } 8688 } 8689 #endif /* VAC */ 8690 8691 8692 /* 8693 * Wrapper routine used to return a context. 8694 * 8695 * It's the responsibility of the caller to guarantee that the 8696 * process serializes on calls here by taking the HAT lock for 8697 * the hat. 8698 * 8699 */ 8700 static void 8701 sfmmu_get_ctx(sfmmu_t *sfmmup) 8702 { 8703 mmu_ctx_t *mmu_ctxp; 8704 uint_t pstate_save; 8705 8706 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8707 ASSERT(sfmmup != ksfmmup); 8708 8709 kpreempt_disable(); 8710 8711 mmu_ctxp = CPU_MMU_CTXP(CPU); 8712 ASSERT(mmu_ctxp); 8713 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 8714 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 8715 8716 /* 8717 * Do a wrap-around if cnum reaches the max # cnum supported by a MMU. 8718 */ 8719 if (mmu_ctxp->mmu_cnum == mmu_ctxp->mmu_nctxs) 8720 sfmmu_ctx_wrap_around(mmu_ctxp); 8721 8722 /* 8723 * Let the MMU set up the page sizes to use for 8724 * this context in the TLB. Don't program 2nd dtlb for ism hat. 8725 */ 8726 if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) { 8727 mmu_set_ctx_page_sizes(sfmmup); 8728 } 8729 8730 /* 8731 * sfmmu_alloc_ctx and sfmmu_load_mmustate will be performed with 8732 * interrupts disabled to prevent race condition with wrap-around 8733 * ctx invalidatation. In sun4v, ctx invalidation also involves 8734 * a HV call to set the number of TSBs to 0. If interrupts are not 8735 * disabled until after sfmmu_load_mmustate is complete TSBs may 8736 * become assigned to INVALID_CONTEXT. This is not allowed. 8737 */ 8738 pstate_save = sfmmu_disable_intrs(); 8739 8740 sfmmu_alloc_ctx(sfmmup, 1, CPU); 8741 sfmmu_load_mmustate(sfmmup); 8742 8743 sfmmu_enable_intrs(pstate_save); 8744 8745 kpreempt_enable(); 8746 } 8747 8748 /* 8749 * When all cnums are used up in a MMU, cnum will wrap around to the 8750 * next generation and start from 2. 8751 */ 8752 static void 8753 sfmmu_ctx_wrap_around(mmu_ctx_t *mmu_ctxp) 8754 { 8755 8756 /* caller must have disabled the preemption */ 8757 ASSERT(curthread->t_preempt >= 1); 8758 ASSERT(mmu_ctxp != NULL); 8759 8760 /* acquire Per-MMU (PM) spin lock */ 8761 mutex_enter(&mmu_ctxp->mmu_lock); 8762 8763 /* re-check to see if wrap-around is needed */ 8764 if (mmu_ctxp->mmu_cnum < mmu_ctxp->mmu_nctxs) 8765 goto done; 8766 8767 SFMMU_MMU_STAT(mmu_wrap_around); 8768 8769 /* update gnum */ 8770 ASSERT(mmu_ctxp->mmu_gnum != 0); 8771 mmu_ctxp->mmu_gnum++; 8772 if (mmu_ctxp->mmu_gnum == 0 || 8773 mmu_ctxp->mmu_gnum > MAX_SFMMU_GNUM_VAL) { 8774 cmn_err(CE_PANIC, "mmu_gnum of mmu_ctx 0x%p is out of bound.", 8775 (void *)mmu_ctxp); 8776 } 8777 8778 if (mmu_ctxp->mmu_ncpus > 1) { 8779 cpuset_t cpuset; 8780 8781 membar_enter(); /* make sure updated gnum visible */ 8782 8783 SFMMU_XCALL_STATS(NULL); 8784 8785 /* xcall to others on the same MMU to invalidate ctx */ 8786 cpuset = mmu_ctxp->mmu_cpuset; 8787 ASSERT(CPU_IN_SET(cpuset, CPU->cpu_id)); 8788 CPUSET_DEL(cpuset, CPU->cpu_id); 8789 CPUSET_AND(cpuset, cpu_ready_set); 8790 8791 /* 8792 * Pass in INVALID_CONTEXT as the first parameter to 8793 * sfmmu_raise_tsb_exception, which invalidates the context 8794 * of any process running on the CPUs in the MMU. 8795 */ 8796 xt_some(cpuset, sfmmu_raise_tsb_exception, 8797 INVALID_CONTEXT, INVALID_CONTEXT); 8798 xt_sync(cpuset); 8799 8800 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 8801 } 8802 8803 if (sfmmu_getctx_sec() != INVALID_CONTEXT) { 8804 sfmmu_setctx_sec(INVALID_CONTEXT); 8805 sfmmu_clear_utsbinfo(); 8806 } 8807 8808 /* 8809 * No xcall is needed here. For sun4u systems all CPUs in context 8810 * domain share a single physical MMU therefore it's enough to flush 8811 * TLB on local CPU. On sun4v systems we use 1 global context 8812 * domain and flush all remote TLBs in sfmmu_raise_tsb_exception 8813 * handler. Note that vtag_flushall_uctxs() is called 8814 * for Ultra II machine, where the equivalent flushall functionality 8815 * is implemented in SW, and only user ctx TLB entries are flushed. 8816 */ 8817 if (&vtag_flushall_uctxs != NULL) { 8818 vtag_flushall_uctxs(); 8819 } else { 8820 vtag_flushall(); 8821 } 8822 8823 /* reset mmu cnum, skips cnum 0 and 1 */ 8824 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; 8825 8826 done: 8827 mutex_exit(&mmu_ctxp->mmu_lock); 8828 } 8829 8830 8831 /* 8832 * For multi-threaded process, set the process context to INVALID_CONTEXT 8833 * so that it faults and reloads the MMU state from TL=0. For single-threaded 8834 * process, we can just load the MMU state directly without having to 8835 * set context invalid. Caller must hold the hat lock since we don't 8836 * acquire it here. 8837 */ 8838 static void 8839 sfmmu_sync_mmustate(sfmmu_t *sfmmup) 8840 { 8841 uint_t cnum; 8842 uint_t pstate_save; 8843 8844 ASSERT(sfmmup != ksfmmup); 8845 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8846 8847 kpreempt_disable(); 8848 8849 /* 8850 * We check whether the pass'ed-in sfmmup is the same as the 8851 * current running proc. This is to makes sure the current proc 8852 * stays single-threaded if it already is. 8853 */ 8854 if ((sfmmup == curthread->t_procp->p_as->a_hat) && 8855 (curthread->t_procp->p_lwpcnt == 1)) { 8856 /* single-thread */ 8857 cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum; 8858 if (cnum != INVALID_CONTEXT) { 8859 uint_t curcnum; 8860 /* 8861 * Disable interrupts to prevent race condition 8862 * with sfmmu_ctx_wrap_around ctx invalidation. 8863 * In sun4v, ctx invalidation involves setting 8864 * TSB to NULL, hence, interrupts should be disabled 8865 * untill after sfmmu_load_mmustate is completed. 8866 */ 8867 pstate_save = sfmmu_disable_intrs(); 8868 curcnum = sfmmu_getctx_sec(); 8869 if (curcnum == cnum) 8870 sfmmu_load_mmustate(sfmmup); 8871 sfmmu_enable_intrs(pstate_save); 8872 ASSERT(curcnum == cnum || curcnum == INVALID_CONTEXT); 8873 } 8874 } else { 8875 /* 8876 * multi-thread 8877 * or when sfmmup is not the same as the curproc. 8878 */ 8879 sfmmu_invalidate_ctx(sfmmup); 8880 } 8881 8882 kpreempt_enable(); 8883 } 8884 8885 8886 /* 8887 * Replace the specified TSB with a new TSB. This function gets called when 8888 * we grow, shrink or swapin a TSB. When swapping in a TSB (TSB_SWAPIN), the 8889 * TSB_FORCEALLOC flag may be used to force allocation of a minimum-sized TSB 8890 * (8K). 8891 * 8892 * Caller must hold the HAT lock, but should assume any tsb_info 8893 * pointers it has are no longer valid after calling this function. 8894 * 8895 * Return values: 8896 * TSB_ALLOCFAIL Failed to allocate a TSB, due to memory constraints 8897 * TSB_LOSTRACE HAT is busy, i.e. another thread is already doing 8898 * something to this tsbinfo/TSB 8899 * TSB_SUCCESS Operation succeeded 8900 */ 8901 static tsb_replace_rc_t 8902 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, 8903 hatlock_t *hatlockp, uint_t flags) 8904 { 8905 struct tsb_info *new_tsbinfo = NULL; 8906 struct tsb_info *curtsb, *prevtsb; 8907 uint_t tte_sz_mask; 8908 int i; 8909 8910 ASSERT(sfmmup != ksfmmup); 8911 ASSERT(sfmmup->sfmmu_ismhat == 0); 8912 ASSERT(sfmmu_hat_lock_held(sfmmup)); 8913 ASSERT(szc <= tsb_max_growsize); 8914 8915 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY)) 8916 return (TSB_LOSTRACE); 8917 8918 /* 8919 * Find the tsb_info ahead of this one in the list, and 8920 * also make sure that the tsb_info passed in really 8921 * exists! 8922 */ 8923 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 8924 curtsb != old_tsbinfo && curtsb != NULL; 8925 prevtsb = curtsb, curtsb = curtsb->tsb_next); 8926 ASSERT(curtsb != NULL); 8927 8928 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8929 /* 8930 * The process is swapped out, so just set the new size 8931 * code. When it swaps back in, we'll allocate a new one 8932 * of the new chosen size. 8933 */ 8934 curtsb->tsb_szc = szc; 8935 return (TSB_SUCCESS); 8936 } 8937 SFMMU_FLAGS_SET(sfmmup, HAT_BUSY); 8938 8939 tte_sz_mask = old_tsbinfo->tsb_ttesz_mask; 8940 8941 /* 8942 * All initialization is done inside of sfmmu_tsbinfo_alloc(). 8943 * If we fail to allocate a TSB, exit. 8944 */ 8945 sfmmu_hat_exit(hatlockp); 8946 if (sfmmu_tsbinfo_alloc(&new_tsbinfo, szc, tte_sz_mask, 8947 flags, sfmmup)) { 8948 (void) sfmmu_hat_enter(sfmmup); 8949 if (!(flags & TSB_SWAPIN)) 8950 SFMMU_STAT(sf_tsb_resize_failures); 8951 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8952 return (TSB_ALLOCFAIL); 8953 } 8954 (void) sfmmu_hat_enter(sfmmup); 8955 8956 /* 8957 * Re-check to make sure somebody else didn't muck with us while we 8958 * didn't hold the HAT lock. If the process swapped out, fine, just 8959 * exit; this can happen if we try to shrink the TSB from the context 8960 * of another process (such as on an ISM unmap), though it is rare. 8961 */ 8962 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 8963 SFMMU_STAT(sf_tsb_resize_failures); 8964 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 8965 sfmmu_hat_exit(hatlockp); 8966 sfmmu_tsbinfo_free(new_tsbinfo); 8967 (void) sfmmu_hat_enter(sfmmup); 8968 return (TSB_LOSTRACE); 8969 } 8970 8971 #ifdef DEBUG 8972 /* Reverify that the tsb_info still exists.. for debugging only */ 8973 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; 8974 curtsb != old_tsbinfo && curtsb != NULL; 8975 prevtsb = curtsb, curtsb = curtsb->tsb_next); 8976 ASSERT(curtsb != NULL); 8977 #endif /* DEBUG */ 8978 8979 /* 8980 * Quiesce any CPUs running this process on their next TLB miss 8981 * so they atomically see the new tsb_info. We temporarily set the 8982 * context to invalid context so new threads that come on processor 8983 * after we do the xcall to cpusran will also serialize behind the 8984 * HAT lock on TLB miss and will see the new TSB. Since this short 8985 * race with a new thread coming on processor is relatively rare, 8986 * this synchronization mechanism should be cheaper than always 8987 * pausing all CPUs for the duration of the setup, which is what 8988 * the old implementation did. This is particuarly true if we are 8989 * copying a huge chunk of memory around during that window. 8990 * 8991 * The memory barriers are to make sure things stay consistent 8992 * with resume() since it does not hold the HAT lock while 8993 * walking the list of tsb_info structures. 8994 */ 8995 if ((flags & TSB_SWAPIN) != TSB_SWAPIN) { 8996 /* The TSB is either growing or shrinking. */ 8997 sfmmu_invalidate_ctx(sfmmup); 8998 } else { 8999 /* 9000 * It is illegal to swap in TSBs from a process other 9001 * than a process being swapped in. This in turn 9002 * implies we do not have a valid MMU context here 9003 * since a process needs one to resolve translation 9004 * misses. 9005 */ 9006 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup); 9007 } 9008 9009 #ifdef DEBUG 9010 ASSERT(max_mmu_ctxdoms > 0); 9011 9012 /* 9013 * Process should have INVALID_CONTEXT on all MMUs 9014 */ 9015 for (i = 0; i < max_mmu_ctxdoms; i++) { 9016 9017 ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); 9018 } 9019 #endif 9020 9021 new_tsbinfo->tsb_next = old_tsbinfo->tsb_next; 9022 membar_stst(); /* strict ordering required */ 9023 if (prevtsb) 9024 prevtsb->tsb_next = new_tsbinfo; 9025 else 9026 sfmmup->sfmmu_tsb = new_tsbinfo; 9027 membar_enter(); /* make sure new TSB globally visible */ 9028 sfmmu_setup_tsbinfo(sfmmup); 9029 9030 /* 9031 * We need to migrate TSB entries from the old TSB to the new TSB 9032 * if tsb_remap_ttes is set and the TSB is growing. 9033 */ 9034 if (tsb_remap_ttes && ((flags & TSB_GROW) == TSB_GROW)) 9035 sfmmu_copy_tsb(old_tsbinfo, new_tsbinfo); 9036 9037 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); 9038 9039 /* 9040 * Drop the HAT lock to free our old tsb_info. 9041 */ 9042 sfmmu_hat_exit(hatlockp); 9043 9044 if ((flags & TSB_GROW) == TSB_GROW) { 9045 SFMMU_STAT(sf_tsb_grow); 9046 } else if ((flags & TSB_SHRINK) == TSB_SHRINK) { 9047 SFMMU_STAT(sf_tsb_shrink); 9048 } 9049 9050 sfmmu_tsbinfo_free(old_tsbinfo); 9051 9052 (void) sfmmu_hat_enter(sfmmup); 9053 return (TSB_SUCCESS); 9054 } 9055 9056 /* 9057 * This function will re-program hat pgsz array, and invalidate the 9058 * process' context, forcing the process to switch to another 9059 * context on the next TLB miss, and therefore start using the 9060 * TLB that is reprogrammed for the new page sizes. 9061 */ 9062 void 9063 sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) 9064 { 9065 int i; 9066 hatlock_t *hatlockp = NULL; 9067 9068 hatlockp = sfmmu_hat_enter(sfmmup); 9069 /* USIII+-IV+ optimization, requires hat lock */ 9070 if (tmp_pgsz) { 9071 for (i = 0; i < mmu_page_sizes; i++) 9072 sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i]; 9073 } 9074 SFMMU_STAT(sf_tlb_reprog_pgsz); 9075 9076 sfmmu_invalidate_ctx(sfmmup); 9077 9078 sfmmu_hat_exit(hatlockp); 9079 } 9080 9081 /* 9082 * This function assumes that there are either four or six supported page 9083 * sizes and at most two programmable TLBs, so we need to decide which 9084 * page sizes are most important and then tell the MMU layer so it 9085 * can adjust the TLB page sizes accordingly (if supported). 9086 * 9087 * If these assumptions change, this function will need to be 9088 * updated to support whatever the new limits are. 9089 * 9090 * The growing flag is nonzero if we are growing the address space, 9091 * and zero if it is shrinking. This allows us to decide whether 9092 * to grow or shrink our TSB, depending upon available memory 9093 * conditions. 9094 */ 9095 static void 9096 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing) 9097 { 9098 uint64_t ttecnt[MMU_PAGE_SIZES]; 9099 uint64_t tte8k_cnt, tte4m_cnt; 9100 uint8_t i; 9101 int sectsb_thresh; 9102 9103 /* 9104 * Kernel threads, processes with small address spaces not using 9105 * large pages, and dummy ISM HATs need not apply. 9106 */ 9107 if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != NULL) 9108 return; 9109 9110 if ((sfmmup->sfmmu_flags & HAT_LGPG_FLAGS) == 0 && 9111 sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor) 9112 return; 9113 9114 for (i = 0; i < mmu_page_sizes; i++) { 9115 ttecnt[i] = SFMMU_TTE_CNT(sfmmup, i); 9116 } 9117 9118 /* Check pagesizes in use, and possibly reprogram DTLB. */ 9119 if (&mmu_check_page_sizes) 9120 mmu_check_page_sizes(sfmmup, ttecnt); 9121 9122 /* 9123 * Calculate the number of 8k ttes to represent the span of these 9124 * pages. 9125 */ 9126 tte8k_cnt = ttecnt[TTE8K] + 9127 (ttecnt[TTE64K] << (MMU_PAGESHIFT64K - MMU_PAGESHIFT)) + 9128 (ttecnt[TTE512K] << (MMU_PAGESHIFT512K - MMU_PAGESHIFT)); 9129 if (mmu_page_sizes == max_mmu_page_sizes) { 9130 tte4m_cnt = ttecnt[TTE4M] + 9131 (ttecnt[TTE32M] << (MMU_PAGESHIFT32M - MMU_PAGESHIFT4M)) + 9132 (ttecnt[TTE256M] << (MMU_PAGESHIFT256M - MMU_PAGESHIFT4M)); 9133 } else { 9134 tte4m_cnt = ttecnt[TTE4M]; 9135 } 9136 9137 /* 9138 * Inflate TSB sizes by a factor of 2 if this process 9139 * uses 4M text pages to minimize extra conflict misses 9140 * in the first TSB since without counting text pages 9141 * 8K TSB may become too small. 9142 * 9143 * Also double the size of the second TSB to minimize 9144 * extra conflict misses due to competition between 4M text pages 9145 * and data pages. 9146 * 9147 * We need to adjust the second TSB allocation threshold by the 9148 * inflation factor, since there is no point in creating a second 9149 * TSB when we know all the mappings can fit in the I/D TLBs. 9150 */ 9151 sectsb_thresh = tsb_sectsb_threshold; 9152 if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) { 9153 tte8k_cnt <<= 1; 9154 tte4m_cnt <<= 1; 9155 sectsb_thresh <<= 1; 9156 } 9157 9158 /* 9159 * Check to see if our TSB is the right size; we may need to 9160 * grow or shrink it. If the process is small, our work is 9161 * finished at this point. 9162 */ 9163 if (tte8k_cnt <= tsb_rss_factor && tte4m_cnt <= sectsb_thresh) { 9164 return; 9165 } 9166 sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh); 9167 } 9168 9169 static void 9170 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt, 9171 uint64_t tte4m_cnt, int sectsb_thresh) 9172 { 9173 int tsb_bits; 9174 uint_t tsb_szc; 9175 struct tsb_info *tsbinfop; 9176 hatlock_t *hatlockp = NULL; 9177 9178 hatlockp = sfmmu_hat_enter(sfmmup); 9179 ASSERT(hatlockp != NULL); 9180 tsbinfop = sfmmup->sfmmu_tsb; 9181 ASSERT(tsbinfop != NULL); 9182 9183 /* 9184 * If we're growing, select the size based on RSS. If we're 9185 * shrinking, leave some room so we don't have to turn around and 9186 * grow again immediately. 9187 */ 9188 if (growing) 9189 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt); 9190 else 9191 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt << 1); 9192 9193 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9194 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9195 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9196 hatlockp, TSB_SHRINK); 9197 } else if (growing && tsb_szc > tsbinfop->tsb_szc && TSB_OK_GROW()) { 9198 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, 9199 hatlockp, TSB_GROW); 9200 } 9201 tsbinfop = sfmmup->sfmmu_tsb; 9202 9203 /* 9204 * With the TLB and first TSB out of the way, we need to see if 9205 * we need a second TSB for 4M pages. If we managed to reprogram 9206 * the TLB page sizes above, the process will start using this new 9207 * TSB right away; otherwise, it will start using it on the next 9208 * context switch. Either way, it's no big deal so there's no 9209 * synchronization with the trap handlers here unless we grow the 9210 * TSB (in which case it's required to prevent using the old one 9211 * after it's freed). Note: second tsb is required for 32M/256M 9212 * page sizes. 9213 */ 9214 if (tte4m_cnt > sectsb_thresh) { 9215 /* 9216 * If we're growing, select the size based on RSS. If we're 9217 * shrinking, leave some room so we don't have to turn 9218 * around and grow again immediately. 9219 */ 9220 if (growing) 9221 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt); 9222 else 9223 tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt << 1); 9224 if (tsbinfop->tsb_next == NULL) { 9225 struct tsb_info *newtsb; 9226 int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)? 9227 0 : TSB_ALLOC; 9228 9229 sfmmu_hat_exit(hatlockp); 9230 9231 /* 9232 * Try to allocate a TSB for 4[32|256]M pages. If we 9233 * can't get the size we want, retry w/a minimum sized 9234 * TSB. If that still didn't work, give up; we can 9235 * still run without one. 9236 */ 9237 tsb_bits = (mmu_page_sizes == max_mmu_page_sizes)? 9238 TSB4M|TSB32M|TSB256M:TSB4M; 9239 if ((sfmmu_tsbinfo_alloc(&newtsb, tsb_szc, tsb_bits, 9240 allocflags, sfmmup) != 0) && 9241 (sfmmu_tsbinfo_alloc(&newtsb, TSB_MIN_SZCODE, 9242 tsb_bits, allocflags, sfmmup) != 0)) { 9243 return; 9244 } 9245 9246 hatlockp = sfmmu_hat_enter(sfmmup); 9247 9248 if (sfmmup->sfmmu_tsb->tsb_next == NULL) { 9249 sfmmup->sfmmu_tsb->tsb_next = newtsb; 9250 SFMMU_STAT(sf_tsb_sectsb_create); 9251 sfmmu_setup_tsbinfo(sfmmup); 9252 sfmmu_hat_exit(hatlockp); 9253 return; 9254 } else { 9255 /* 9256 * It's annoying, but possible for us 9257 * to get here.. we dropped the HAT lock 9258 * because of locking order in the kmem 9259 * allocator, and while we were off getting 9260 * our memory, some other thread decided to 9261 * do us a favor and won the race to get a 9262 * second TSB for this process. Sigh. 9263 */ 9264 sfmmu_hat_exit(hatlockp); 9265 sfmmu_tsbinfo_free(newtsb); 9266 return; 9267 } 9268 } 9269 9270 /* 9271 * We have a second TSB, see if it's big enough. 9272 */ 9273 tsbinfop = tsbinfop->tsb_next; 9274 9275 /* 9276 * Check to see if our second TSB is the right size; 9277 * we may need to grow or shrink it. 9278 * To prevent thrashing (e.g. growing the TSB on a 9279 * subsequent map operation), only try to shrink if 9280 * the TSB reach exceeds twice the virtual address 9281 * space size. 9282 */ 9283 if (!growing && (tsb_szc < tsbinfop->tsb_szc) && 9284 (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) { 9285 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9286 tsb_szc, hatlockp, TSB_SHRINK); 9287 } else if (growing && tsb_szc > tsbinfop->tsb_szc && 9288 TSB_OK_GROW()) { 9289 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, 9290 tsb_szc, hatlockp, TSB_GROW); 9291 } 9292 } 9293 9294 sfmmu_hat_exit(hatlockp); 9295 } 9296 9297 /* 9298 * Get the preferred page size code for a hat. 9299 * This is only advice, so locking is not done; 9300 * this transitory information could change 9301 * following the call anyway. This interface is 9302 * sun4 private. 9303 */ 9304 /*ARGSUSED*/ 9305 uint_t 9306 hat_preferred_pgsz(struct hat *hat, caddr_t vaddr, size_t maplen, int maptype) 9307 { 9308 sfmmu_t *sfmmup = (sfmmu_t *)hat; 9309 uint_t szc, maxszc = mmu_page_sizes - 1; 9310 size_t pgsz; 9311 9312 if (maptype == MAPPGSZ_ISM) { 9313 for (szc = maxszc; szc >= TTE4M; szc--) { 9314 if (disable_ism_large_pages & (1 << szc)) 9315 continue; 9316 9317 pgsz = hw_page_array[szc].hp_size; 9318 if ((maplen >= pgsz) && IS_P2ALIGNED(vaddr, pgsz)) 9319 return (szc); 9320 } 9321 return (TTE4M); 9322 } else if (&mmu_preferred_pgsz) { /* USIII+-USIV+ */ 9323 return (mmu_preferred_pgsz(sfmmup, vaddr, maplen)); 9324 } else { /* USIII, USII, Niagara */ 9325 for (szc = maxszc; szc > TTE8K; szc--) { 9326 if (disable_large_pages & (1 << szc)) 9327 continue; 9328 9329 pgsz = hw_page_array[szc].hp_size; 9330 if ((maplen >= pgsz) && IS_P2ALIGNED(vaddr, pgsz)) 9331 return (szc); 9332 } 9333 return (TTE8K); 9334 } 9335 } 9336 9337 /* 9338 * Free up a sfmmu 9339 * Since the sfmmu is currently embedded in the hat struct we simply zero 9340 * out our fields and free up the ism map blk list if any. 9341 */ 9342 static void 9343 sfmmu_free_sfmmu(sfmmu_t *sfmmup) 9344 { 9345 ism_blk_t *blkp, *nx_blkp; 9346 #ifdef DEBUG 9347 ism_map_t *map; 9348 int i; 9349 #endif 9350 9351 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); 9352 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); 9353 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); 9354 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); 9355 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); 9356 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); 9357 9358 sfmmup->sfmmu_free = 0; 9359 sfmmup->sfmmu_ismhat = 0; 9360 9361 blkp = sfmmup->sfmmu_iblk; 9362 sfmmup->sfmmu_iblk = NULL; 9363 9364 while (blkp) { 9365 #ifdef DEBUG 9366 map = blkp->iblk_maps; 9367 for (i = 0; i < ISM_MAP_SLOTS; i++) { 9368 ASSERT(map[i].imap_seg == 0); 9369 ASSERT(map[i].imap_ismhat == NULL); 9370 ASSERT(map[i].imap_ment == NULL); 9371 } 9372 #endif 9373 nx_blkp = blkp->iblk_next; 9374 blkp->iblk_next = NULL; 9375 blkp->iblk_nextpa = (uint64_t)-1; 9376 kmem_cache_free(ism_blk_cache, blkp); 9377 blkp = nx_blkp; 9378 } 9379 } 9380 9381 /* 9382 * Locking primitves accessed by HATLOCK macros 9383 */ 9384 9385 #define SFMMU_SPL_MTX (0x0) 9386 #define SFMMU_ML_MTX (0x1) 9387 9388 #define SFMMU_MLSPL_MTX(type, pg) (((type) == SFMMU_SPL_MTX) ? \ 9389 SPL_HASH(pg) : MLIST_HASH(pg)) 9390 9391 kmutex_t * 9392 sfmmu_page_enter(struct page *pp) 9393 { 9394 return (sfmmu_mlspl_enter(pp, SFMMU_SPL_MTX)); 9395 } 9396 9397 void 9398 sfmmu_page_exit(kmutex_t *spl) 9399 { 9400 mutex_exit(spl); 9401 } 9402 9403 int 9404 sfmmu_page_spl_held(struct page *pp) 9405 { 9406 return (sfmmu_mlspl_held(pp, SFMMU_SPL_MTX)); 9407 } 9408 9409 kmutex_t * 9410 sfmmu_mlist_enter(struct page *pp) 9411 { 9412 return (sfmmu_mlspl_enter(pp, SFMMU_ML_MTX)); 9413 } 9414 9415 void 9416 sfmmu_mlist_exit(kmutex_t *mml) 9417 { 9418 mutex_exit(mml); 9419 } 9420 9421 int 9422 sfmmu_mlist_held(struct page *pp) 9423 { 9424 9425 return (sfmmu_mlspl_held(pp, SFMMU_ML_MTX)); 9426 } 9427 9428 /* 9429 * Common code for sfmmu_mlist_enter() and sfmmu_page_enter(). For 9430 * sfmmu_mlist_enter() case mml_table lock array is used and for 9431 * sfmmu_page_enter() sfmmu_page_lock lock array is used. 9432 * 9433 * The lock is taken on a root page so that it protects an operation on all 9434 * constituent pages of a large page pp belongs to. 9435 * 9436 * The routine takes a lock from the appropriate array. The lock is determined 9437 * by hashing the root page. After taking the lock this routine checks if the 9438 * root page has the same size code that was used to determine the root (i.e 9439 * that root hasn't changed). If root page has the expected p_szc field we 9440 * have the right lock and it's returned to the caller. If root's p_szc 9441 * decreased we release the lock and retry from the beginning. This case can 9442 * happen due to hat_page_demote() decreasing p_szc between our load of p_szc 9443 * value and taking the lock. The number of retries due to p_szc decrease is 9444 * limited by the maximum p_szc value. If p_szc is 0 we return the lock 9445 * determined by hashing pp itself. 9446 * 9447 * If our caller doesn't hold a SE_SHARED or SE_EXCL lock on pp it's also 9448 * possible that p_szc can increase. To increase p_szc a thread has to lock 9449 * all constituent pages EXCL and do hat_pageunload() on all of them. All the 9450 * callers that don't hold a page locked recheck if hmeblk through which pp 9451 * was found still maps this pp. If it doesn't map it anymore returned lock 9452 * is immediately dropped. Therefore if sfmmu_mlspl_enter() hits the case of 9453 * p_szc increase after taking the lock it returns this lock without further 9454 * retries because in this case the caller doesn't care about which lock was 9455 * taken. The caller will drop it right away. 9456 * 9457 * After the routine returns it's guaranteed that hat_page_demote() can't 9458 * change p_szc field of any of constituent pages of a large page pp belongs 9459 * to as long as pp was either locked at least SHARED prior to this call or 9460 * the caller finds that hment that pointed to this pp still references this 9461 * pp (this also assumes that the caller holds hme hash bucket lock so that 9462 * the same pp can't be remapped into the same hmeblk after it was unmapped by 9463 * hat_pageunload()). 9464 */ 9465 static kmutex_t * 9466 sfmmu_mlspl_enter(struct page *pp, int type) 9467 { 9468 kmutex_t *mtx; 9469 uint_t prev_rszc = UINT_MAX; 9470 page_t *rootpp; 9471 uint_t szc; 9472 uint_t rszc; 9473 uint_t pszc = pp->p_szc; 9474 9475 ASSERT(pp != NULL); 9476 9477 again: 9478 if (pszc == 0) { 9479 mtx = SFMMU_MLSPL_MTX(type, pp); 9480 mutex_enter(mtx); 9481 return (mtx); 9482 } 9483 9484 /* The lock lives in the root page */ 9485 rootpp = PP_GROUPLEADER(pp, pszc); 9486 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9487 mutex_enter(mtx); 9488 9489 /* 9490 * Return mml in the following 3 cases: 9491 * 9492 * 1) If pp itself is root since if its p_szc decreased before we took 9493 * the lock pp is still the root of smaller szc page. And if its p_szc 9494 * increased it doesn't matter what lock we return (see comment in 9495 * front of this routine). 9496 * 9497 * 2) If pp's not root but rootpp is the root of a rootpp->p_szc size 9498 * large page we have the right lock since any previous potential 9499 * hat_page_demote() is done demoting from greater than current root's 9500 * p_szc because hat_page_demote() changes root's p_szc last. No 9501 * further hat_page_demote() can start or be in progress since it 9502 * would need the same lock we currently hold. 9503 * 9504 * 3) If rootpp's p_szc increased since previous iteration it doesn't 9505 * matter what lock we return (see comment in front of this routine). 9506 */ 9507 if (pp == rootpp || (rszc = rootpp->p_szc) == pszc || 9508 rszc >= prev_rszc) { 9509 return (mtx); 9510 } 9511 9512 /* 9513 * hat_page_demote() could have decreased root's p_szc. 9514 * In this case pp's p_szc must also be smaller than pszc. 9515 * Retry. 9516 */ 9517 if (rszc < pszc) { 9518 szc = pp->p_szc; 9519 if (szc < pszc) { 9520 mutex_exit(mtx); 9521 pszc = szc; 9522 goto again; 9523 } 9524 /* 9525 * pp's p_szc increased after it was decreased. 9526 * page cannot be mapped. Return current lock. The caller 9527 * will drop it right away. 9528 */ 9529 return (mtx); 9530 } 9531 9532 /* 9533 * root's p_szc is greater than pp's p_szc. 9534 * hat_page_demote() is not done with all pages 9535 * yet. Wait for it to complete. 9536 */ 9537 mutex_exit(mtx); 9538 rootpp = PP_GROUPLEADER(rootpp, rszc); 9539 mtx = SFMMU_MLSPL_MTX(type, rootpp); 9540 mutex_enter(mtx); 9541 mutex_exit(mtx); 9542 prev_rszc = rszc; 9543 goto again; 9544 } 9545 9546 static int 9547 sfmmu_mlspl_held(struct page *pp, int type) 9548 { 9549 kmutex_t *mtx; 9550 9551 ASSERT(pp != NULL); 9552 /* The lock lives in the root page */ 9553 pp = PP_PAGEROOT(pp); 9554 ASSERT(pp != NULL); 9555 9556 mtx = SFMMU_MLSPL_MTX(type, pp); 9557 return (MUTEX_HELD(mtx)); 9558 } 9559 9560 static uint_t 9561 sfmmu_get_free_hblk(struct hme_blk **hmeblkpp, uint_t critical) 9562 { 9563 struct hme_blk *hblkp; 9564 9565 if (freehblkp != NULL) { 9566 mutex_enter(&freehblkp_lock); 9567 if (freehblkp != NULL) { 9568 /* 9569 * If the current thread is owning hblk_reserve, 9570 * let it succede even if freehblkcnt is really low. 9571 */ 9572 if (freehblkcnt <= HBLK_RESERVE_MIN && !critical) { 9573 SFMMU_STAT(sf_get_free_throttle); 9574 mutex_exit(&freehblkp_lock); 9575 return (0); 9576 } 9577 freehblkcnt--; 9578 *hmeblkpp = freehblkp; 9579 hblkp = *hmeblkpp; 9580 freehblkp = hblkp->hblk_next; 9581 mutex_exit(&freehblkp_lock); 9582 hblkp->hblk_next = NULL; 9583 SFMMU_STAT(sf_get_free_success); 9584 return (1); 9585 } 9586 mutex_exit(&freehblkp_lock); 9587 } 9588 SFMMU_STAT(sf_get_free_fail); 9589 return (0); 9590 } 9591 9592 static uint_t 9593 sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical) 9594 { 9595 struct hme_blk *hblkp; 9596 9597 /* 9598 * If the current thread is mapping into kernel space, 9599 * let it succede even if freehblkcnt is max 9600 * so that it will avoid freeing it to kmem. 9601 * This will prevent stack overflow due to 9602 * possible recursion since kmem_cache_free() 9603 * might require creation of a slab which 9604 * in turn needs an hmeblk to map that slab; 9605 * let's break this vicious chain at the first 9606 * opportunity. 9607 */ 9608 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9609 mutex_enter(&freehblkp_lock); 9610 if (freehblkcnt < HBLK_RESERVE_CNT || critical) { 9611 SFMMU_STAT(sf_put_free_success); 9612 freehblkcnt++; 9613 hmeblkp->hblk_next = freehblkp; 9614 freehblkp = hmeblkp; 9615 mutex_exit(&freehblkp_lock); 9616 return (1); 9617 } 9618 mutex_exit(&freehblkp_lock); 9619 } 9620 9621 /* 9622 * Bring down freehblkcnt to HBLK_RESERVE_CNT. We are here 9623 * only if freehblkcnt is at least HBLK_RESERVE_CNT *and* 9624 * we are not in the process of mapping into kernel space. 9625 */ 9626 ASSERT(!critical); 9627 while (freehblkcnt > HBLK_RESERVE_CNT) { 9628 mutex_enter(&freehblkp_lock); 9629 if (freehblkcnt > HBLK_RESERVE_CNT) { 9630 freehblkcnt--; 9631 hblkp = freehblkp; 9632 freehblkp = hblkp->hblk_next; 9633 mutex_exit(&freehblkp_lock); 9634 ASSERT(get_hblk_cache(hblkp) == sfmmu8_cache); 9635 kmem_cache_free(sfmmu8_cache, hblkp); 9636 continue; 9637 } 9638 mutex_exit(&freehblkp_lock); 9639 } 9640 SFMMU_STAT(sf_put_free_fail); 9641 return (0); 9642 } 9643 9644 static void 9645 sfmmu_hblk_swap(struct hme_blk *new) 9646 { 9647 struct hme_blk *old, *hblkp, *prev; 9648 uint64_t hblkpa, prevpa, newpa; 9649 caddr_t base, vaddr, endaddr; 9650 struct hmehash_bucket *hmebp; 9651 struct sf_hment *osfhme, *nsfhme; 9652 page_t *pp; 9653 kmutex_t *pml; 9654 tte_t tte; 9655 9656 #ifdef DEBUG 9657 hmeblk_tag hblktag; 9658 struct hme_blk *found; 9659 #endif 9660 old = HBLK_RESERVE; 9661 9662 /* 9663 * save pa before bcopy clobbers it 9664 */ 9665 newpa = new->hblk_nextpa; 9666 9667 base = (caddr_t)get_hblk_base(old); 9668 endaddr = base + get_hblk_span(old); 9669 9670 /* 9671 * acquire hash bucket lock. 9672 */ 9673 hmebp = sfmmu_tteload_acquire_hashbucket(ksfmmup, base, TTE8K); 9674 9675 /* 9676 * copy contents from old to new 9677 */ 9678 bcopy((void *)old, (void *)new, HME8BLK_SZ); 9679 9680 /* 9681 * add new to hash chain 9682 */ 9683 sfmmu_hblk_hash_add(hmebp, new, newpa); 9684 9685 /* 9686 * search hash chain for hblk_reserve; this needs to be performed 9687 * after adding new, otherwise prevpa and prev won't correspond 9688 * to the hblk which is prior to old in hash chain when we call 9689 * sfmmu_hblk_hash_rm to remove old later. 9690 */ 9691 for (prevpa = 0, prev = NULL, 9692 hblkpa = hmebp->hmeh_nextpa, hblkp = hmebp->hmeblkp; 9693 hblkp != NULL && hblkp != old; 9694 prevpa = hblkpa, prev = hblkp, 9695 hblkpa = hblkp->hblk_nextpa, hblkp = hblkp->hblk_next); 9696 9697 if (hblkp != old) 9698 panic("sfmmu_hblk_swap: hblk_reserve not found"); 9699 9700 /* 9701 * p_mapping list is still pointing to hments in hblk_reserve; 9702 * fix up p_mapping list so that they point to hments in new. 9703 * 9704 * Since all these mappings are created by hblk_reserve_thread 9705 * on the way and it's using at least one of the buffers from each of 9706 * the newly minted slabs, there is no danger of any of these 9707 * mappings getting unloaded by another thread. 9708 * 9709 * tsbmiss could only modify ref/mod bits of hments in old/new. 9710 * Since all of these hments hold mappings established by segkmem 9711 * and mappings in segkmem are setup with HAT_NOSYNC, ref/mod bits 9712 * have no meaning for the mappings in hblk_reserve. hments in 9713 * old and new are identical except for ref/mod bits. 9714 */ 9715 for (vaddr = base; vaddr < endaddr; vaddr += TTEBYTES(TTE8K)) { 9716 9717 HBLKTOHME(osfhme, old, vaddr); 9718 sfmmu_copytte(&osfhme->hme_tte, &tte); 9719 9720 if (TTE_IS_VALID(&tte)) { 9721 if ((pp = osfhme->hme_page) == NULL) 9722 panic("sfmmu_hblk_swap: page not mapped"); 9723 9724 pml = sfmmu_mlist_enter(pp); 9725 9726 if (pp != osfhme->hme_page) 9727 panic("sfmmu_hblk_swap: mapping changed"); 9728 9729 HBLKTOHME(nsfhme, new, vaddr); 9730 9731 HME_ADD(nsfhme, pp); 9732 HME_SUB(osfhme, pp); 9733 9734 sfmmu_mlist_exit(pml); 9735 } 9736 } 9737 9738 /* 9739 * remove old from hash chain 9740 */ 9741 sfmmu_hblk_hash_rm(hmebp, old, prevpa, prev); 9742 9743 #ifdef DEBUG 9744 9745 hblktag.htag_id = ksfmmup; 9746 hblktag.htag_bspage = HME_HASH_BSPAGE(base, HME_HASH_SHIFT(TTE8K)); 9747 hblktag.htag_rehash = HME_HASH_REHASH(TTE8K); 9748 HME_HASH_FAST_SEARCH(hmebp, hblktag, found); 9749 9750 if (found != new) 9751 panic("sfmmu_hblk_swap: new hblk not found"); 9752 #endif 9753 9754 SFMMU_HASH_UNLOCK(hmebp); 9755 9756 /* 9757 * Reset hblk_reserve 9758 */ 9759 bzero((void *)old, HME8BLK_SZ); 9760 old->hblk_nextpa = va_to_pa((caddr_t)old); 9761 } 9762 9763 /* 9764 * Grab the mlist mutex for both pages passed in. 9765 * 9766 * low and high will be returned as pointers to the mutexes for these pages. 9767 * low refers to the mutex residing in the lower bin of the mlist hash, while 9768 * high refers to the mutex residing in the higher bin of the mlist hash. This 9769 * is due to the locking order restrictions on the same thread grabbing 9770 * multiple mlist mutexes. The low lock must be acquired before the high lock. 9771 * 9772 * If both pages hash to the same mutex, only grab that single mutex, and 9773 * high will be returned as NULL 9774 * If the pages hash to different bins in the hash, grab the lower addressed 9775 * lock first and then the higher addressed lock in order to follow the locking 9776 * rules involved with the same thread grabbing multiple mlist mutexes. 9777 * low and high will both have non-NULL values. 9778 */ 9779 static void 9780 sfmmu_mlist_reloc_enter(struct page *targ, struct page *repl, 9781 kmutex_t **low, kmutex_t **high) 9782 { 9783 kmutex_t *mml_targ, *mml_repl; 9784 9785 /* 9786 * no need to do the dance around szc as in sfmmu_mlist_enter() 9787 * because this routine is only called by hat_page_relocate() and all 9788 * targ and repl pages are already locked EXCL so szc can't change. 9789 */ 9790 9791 mml_targ = MLIST_HASH(PP_PAGEROOT(targ)); 9792 mml_repl = MLIST_HASH(PP_PAGEROOT(repl)); 9793 9794 if (mml_targ == mml_repl) { 9795 *low = mml_targ; 9796 *high = NULL; 9797 } else { 9798 if (mml_targ < mml_repl) { 9799 *low = mml_targ; 9800 *high = mml_repl; 9801 } else { 9802 *low = mml_repl; 9803 *high = mml_targ; 9804 } 9805 } 9806 9807 mutex_enter(*low); 9808 if (*high) 9809 mutex_enter(*high); 9810 } 9811 9812 static void 9813 sfmmu_mlist_reloc_exit(kmutex_t *low, kmutex_t *high) 9814 { 9815 if (high) 9816 mutex_exit(high); 9817 mutex_exit(low); 9818 } 9819 9820 static hatlock_t * 9821 sfmmu_hat_enter(sfmmu_t *sfmmup) 9822 { 9823 hatlock_t *hatlockp; 9824 9825 if (sfmmup != ksfmmup) { 9826 hatlockp = TSB_HASH(sfmmup); 9827 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 9828 return (hatlockp); 9829 } 9830 return (NULL); 9831 } 9832 9833 static hatlock_t * 9834 sfmmu_hat_tryenter(sfmmu_t *sfmmup) 9835 { 9836 hatlock_t *hatlockp; 9837 9838 if (sfmmup != ksfmmup) { 9839 hatlockp = TSB_HASH(sfmmup); 9840 if (mutex_tryenter(HATLOCK_MUTEXP(hatlockp)) == 0) 9841 return (NULL); 9842 return (hatlockp); 9843 } 9844 return (NULL); 9845 } 9846 9847 static void 9848 sfmmu_hat_exit(hatlock_t *hatlockp) 9849 { 9850 if (hatlockp != NULL) 9851 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 9852 } 9853 9854 static void 9855 sfmmu_hat_lock_all(void) 9856 { 9857 int i; 9858 for (i = 0; i < SFMMU_NUM_LOCK; i++) 9859 mutex_enter(HATLOCK_MUTEXP(&hat_lock[i])); 9860 } 9861 9862 static void 9863 sfmmu_hat_unlock_all(void) 9864 { 9865 int i; 9866 for (i = SFMMU_NUM_LOCK - 1; i >= 0; i--) 9867 mutex_exit(HATLOCK_MUTEXP(&hat_lock[i])); 9868 } 9869 9870 int 9871 sfmmu_hat_lock_held(sfmmu_t *sfmmup) 9872 { 9873 ASSERT(sfmmup != ksfmmup); 9874 return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup)))); 9875 } 9876 9877 /* 9878 * Locking primitives to provide consistency between ISM unmap 9879 * and other operations. Since ISM unmap can take a long time, we 9880 * use HAT_ISMBUSY flag (protected by the hatlock) to avoid creating 9881 * contention on the hatlock buckets while ISM segments are being 9882 * unmapped. The tradeoff is that the flags don't prevent priority 9883 * inversion from occurring, so we must request kernel priority in 9884 * case we have to sleep to keep from getting buried while holding 9885 * the HAT_ISMBUSY flag set, which in turn could block other kernel 9886 * threads from running (for example, in sfmmu_uvatopfn()). 9887 */ 9888 static void 9889 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held) 9890 { 9891 hatlock_t *hatlockp; 9892 9893 THREAD_KPRI_REQUEST(); 9894 if (!hatlock_held) 9895 hatlockp = sfmmu_hat_enter(sfmmup); 9896 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) 9897 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 9898 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY); 9899 if (!hatlock_held) 9900 sfmmu_hat_exit(hatlockp); 9901 } 9902 9903 static void 9904 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held) 9905 { 9906 hatlock_t *hatlockp; 9907 9908 if (!hatlock_held) 9909 hatlockp = sfmmu_hat_enter(sfmmup); 9910 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); 9911 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY); 9912 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 9913 if (!hatlock_held) 9914 sfmmu_hat_exit(hatlockp); 9915 THREAD_KPRI_RELEASE(); 9916 } 9917 9918 /* 9919 * 9920 * Algorithm: 9921 * 9922 * (1) if segkmem is not ready, allocate hblk from an array of pre-alloc'ed 9923 * hblks. 9924 * 9925 * (2) if we are allocating an hblk for mapping a slab in sfmmu_cache, 9926 * 9927 * (a) try to return an hblk from reserve pool of free hblks; 9928 * (b) if the reserve pool is empty, acquire hblk_reserve_lock 9929 * and return hblk_reserve. 9930 * 9931 * (3) call kmem_cache_alloc() to allocate hblk; 9932 * 9933 * (a) if hblk_reserve_lock is held by the current thread, 9934 * atomically replace hblk_reserve by the hblk that is 9935 * returned by kmem_cache_alloc; release hblk_reserve_lock 9936 * and call kmem_cache_alloc() again. 9937 * (b) if reserve pool is not full, add the hblk that is 9938 * returned by kmem_cache_alloc to reserve pool and 9939 * call kmem_cache_alloc again. 9940 * 9941 */ 9942 static struct hme_blk * 9943 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr, 9944 struct hmehash_bucket *hmebp, uint_t size, hmeblk_tag hblktag, 9945 uint_t flags) 9946 { 9947 struct hme_blk *hmeblkp = NULL; 9948 struct hme_blk *newhblkp; 9949 struct hme_blk *shw_hblkp = NULL; 9950 struct kmem_cache *sfmmu_cache = NULL; 9951 uint64_t hblkpa; 9952 ulong_t index; 9953 uint_t owner; /* set to 1 if using hblk_reserve */ 9954 uint_t forcefree; 9955 int sleep; 9956 9957 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 9958 9959 /* 9960 * If segkmem is not created yet, allocate from static hmeblks 9961 * created at the end of startup_modules(). See the block comment 9962 * in startup_modules() describing how we estimate the number of 9963 * static hmeblks that will be needed during re-map. 9964 */ 9965 if (!hblk_alloc_dynamic) { 9966 9967 if (size == TTE8K) { 9968 index = nucleus_hblk8.index; 9969 if (index >= nucleus_hblk8.len) { 9970 /* 9971 * If we panic here, see startup_modules() to 9972 * make sure that we are calculating the 9973 * number of hblk8's that we need correctly. 9974 */ 9975 panic("no nucleus hblk8 to allocate"); 9976 } 9977 hmeblkp = 9978 (struct hme_blk *)&nucleus_hblk8.list[index]; 9979 nucleus_hblk8.index++; 9980 SFMMU_STAT(sf_hblk8_nalloc); 9981 } else { 9982 index = nucleus_hblk1.index; 9983 if (nucleus_hblk1.index >= nucleus_hblk1.len) { 9984 /* 9985 * If we panic here, see startup_modules() 9986 * and H8TOH1; most likely you need to 9987 * update the calculation of the number 9988 * of hblk1's the kernel needs to boot. 9989 */ 9990 panic("no nucleus hblk1 to allocate"); 9991 } 9992 hmeblkp = 9993 (struct hme_blk *)&nucleus_hblk1.list[index]; 9994 nucleus_hblk1.index++; 9995 SFMMU_STAT(sf_hblk1_nalloc); 9996 } 9997 9998 goto hblk_init; 9999 } 10000 10001 SFMMU_HASH_UNLOCK(hmebp); 10002 10003 if (sfmmup != KHATID) { 10004 if (mmu_page_sizes == max_mmu_page_sizes) { 10005 if (size < TTE256M) 10006 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 10007 size, flags); 10008 } else { 10009 if (size < TTE4M) 10010 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, 10011 size, flags); 10012 } 10013 } 10014 10015 fill_hblk: 10016 owner = (hblk_reserve_thread == curthread) ? 1 : 0; 10017 10018 if (owner && size == TTE8K) { 10019 10020 /* 10021 * We are really in a tight spot. We already own 10022 * hblk_reserve and we need another hblk. In anticipation 10023 * of this kind of scenario, we specifically set aside 10024 * HBLK_RESERVE_MIN number of hblks to be used exclusively 10025 * by owner of hblk_reserve. 10026 */ 10027 SFMMU_STAT(sf_hblk_recurse_cnt); 10028 10029 if (!sfmmu_get_free_hblk(&hmeblkp, 1)) 10030 panic("sfmmu_hblk_alloc: reserve list is empty"); 10031 10032 goto hblk_verify; 10033 } 10034 10035 ASSERT(!owner); 10036 10037 if ((flags & HAT_NO_KALLOC) == 0) { 10038 10039 sfmmu_cache = ((size == TTE8K) ? sfmmu8_cache : sfmmu1_cache); 10040 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP); 10041 10042 if ((hmeblkp = kmem_cache_alloc(sfmmu_cache, sleep)) == NULL) { 10043 hmeblkp = sfmmu_hblk_steal(size); 10044 } else { 10045 /* 10046 * if we are the owner of hblk_reserve, 10047 * swap hblk_reserve with hmeblkp and 10048 * start a fresh life. Hope things go 10049 * better this time. 10050 */ 10051 if (hblk_reserve_thread == curthread) { 10052 ASSERT(sfmmu_cache == sfmmu8_cache); 10053 sfmmu_hblk_swap(hmeblkp); 10054 hblk_reserve_thread = NULL; 10055 mutex_exit(&hblk_reserve_lock); 10056 goto fill_hblk; 10057 } 10058 /* 10059 * let's donate this hblk to our reserve list if 10060 * we are not mapping kernel range 10061 */ 10062 if (size == TTE8K && sfmmup != KHATID) 10063 if (sfmmu_put_free_hblk(hmeblkp, 0)) 10064 goto fill_hblk; 10065 } 10066 } else { 10067 /* 10068 * We are here to map the slab in sfmmu8_cache; let's 10069 * check if we could tap our reserve list; if successful, 10070 * this will avoid the pain of going thru sfmmu_hblk_swap 10071 */ 10072 SFMMU_STAT(sf_hblk_slab_cnt); 10073 if (!sfmmu_get_free_hblk(&hmeblkp, 0)) { 10074 /* 10075 * let's start hblk_reserve dance 10076 */ 10077 SFMMU_STAT(sf_hblk_reserve_cnt); 10078 owner = 1; 10079 mutex_enter(&hblk_reserve_lock); 10080 hmeblkp = HBLK_RESERVE; 10081 hblk_reserve_thread = curthread; 10082 } 10083 } 10084 10085 hblk_verify: 10086 ASSERT(hmeblkp != NULL); 10087 set_hblk_sz(hmeblkp, size); 10088 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); 10089 SFMMU_HASH_LOCK(hmebp); 10090 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10091 if (newhblkp != NULL) { 10092 SFMMU_HASH_UNLOCK(hmebp); 10093 if (hmeblkp != HBLK_RESERVE) { 10094 /* 10095 * This is really tricky! 10096 * 10097 * vmem_alloc(vmem_seg_arena) 10098 * vmem_alloc(vmem_internal_arena) 10099 * segkmem_alloc(heap_arena) 10100 * vmem_alloc(heap_arena) 10101 * page_create() 10102 * hat_memload() 10103 * kmem_cache_free() 10104 * kmem_cache_alloc() 10105 * kmem_slab_create() 10106 * vmem_alloc(kmem_internal_arena) 10107 * segkmem_alloc(heap_arena) 10108 * vmem_alloc(heap_arena) 10109 * page_create() 10110 * hat_memload() 10111 * kmem_cache_free() 10112 * ... 10113 * 10114 * Thus, hat_memload() could call kmem_cache_free 10115 * for enough number of times that we could easily 10116 * hit the bottom of the stack or run out of reserve 10117 * list of vmem_seg structs. So, we must donate 10118 * this hblk to reserve list if it's allocated 10119 * from sfmmu8_cache *and* mapping kernel range. 10120 * We don't need to worry about freeing hmeblk1's 10121 * to kmem since they don't map any kmem slabs. 10122 * 10123 * Note: When segkmem supports largepages, we must 10124 * free hmeblk1's to reserve list as well. 10125 */ 10126 forcefree = (sfmmup == KHATID) ? 1 : 0; 10127 if (size == TTE8K && 10128 sfmmu_put_free_hblk(hmeblkp, forcefree)) { 10129 goto re_verify; 10130 } 10131 ASSERT(sfmmup != KHATID); 10132 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10133 } else { 10134 /* 10135 * Hey! we don't need hblk_reserve any more. 10136 */ 10137 ASSERT(owner); 10138 hblk_reserve_thread = NULL; 10139 mutex_exit(&hblk_reserve_lock); 10140 owner = 0; 10141 } 10142 re_verify: 10143 /* 10144 * let's check if the goodies are still present 10145 */ 10146 SFMMU_HASH_LOCK(hmebp); 10147 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp); 10148 if (newhblkp != NULL) { 10149 /* 10150 * return newhblkp if it's not hblk_reserve; 10151 * if newhblkp is hblk_reserve, return it 10152 * _only if_ we are the owner of hblk_reserve. 10153 */ 10154 if (newhblkp != HBLK_RESERVE || owner) { 10155 return (newhblkp); 10156 } else { 10157 /* 10158 * we just hit hblk_reserve in the hash and 10159 * we are not the owner of that; 10160 * 10161 * block until hblk_reserve_thread completes 10162 * swapping hblk_reserve and try the dance 10163 * once again. 10164 */ 10165 SFMMU_HASH_UNLOCK(hmebp); 10166 mutex_enter(&hblk_reserve_lock); 10167 mutex_exit(&hblk_reserve_lock); 10168 SFMMU_STAT(sf_hblk_reserve_hit); 10169 goto fill_hblk; 10170 } 10171 } else { 10172 /* 10173 * it's no more! try the dance once again. 10174 */ 10175 SFMMU_HASH_UNLOCK(hmebp); 10176 goto fill_hblk; 10177 } 10178 } 10179 10180 hblk_init: 10181 set_hblk_sz(hmeblkp, size); 10182 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10183 hmeblkp->hblk_next = (struct hme_blk *)NULL; 10184 hmeblkp->hblk_tag = hblktag; 10185 hmeblkp->hblk_shadow = shw_hblkp; 10186 hblkpa = hmeblkp->hblk_nextpa; 10187 hmeblkp->hblk_nextpa = 0; 10188 10189 ASSERT(get_hblk_ttesz(hmeblkp) == size); 10190 ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size)); 10191 ASSERT(hmeblkp->hblk_hmecnt == 0); 10192 ASSERT(hmeblkp->hblk_vcnt == 0); 10193 ASSERT(hmeblkp->hblk_lckcnt == 0); 10194 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10195 sfmmu_hblk_hash_add(hmebp, hmeblkp, hblkpa); 10196 return (hmeblkp); 10197 } 10198 10199 /* 10200 * This function performs any cleanup required on the hme_blk 10201 * and returns it to the free list. 10202 */ 10203 /* ARGSUSED */ 10204 static void 10205 sfmmu_hblk_free(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10206 uint64_t hblkpa, struct hme_blk **listp) 10207 { 10208 int shw_size, vshift; 10209 struct hme_blk *shw_hblkp; 10210 uint_t shw_mask, newshw_mask; 10211 uintptr_t vaddr; 10212 int size; 10213 uint_t critical; 10214 10215 ASSERT(hmeblkp); 10216 ASSERT(!hmeblkp->hblk_hmecnt); 10217 ASSERT(!hmeblkp->hblk_vcnt); 10218 ASSERT(!hmeblkp->hblk_lckcnt); 10219 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); 10220 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); 10221 10222 critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0; 10223 10224 size = get_hblk_ttesz(hmeblkp); 10225 shw_hblkp = hmeblkp->hblk_shadow; 10226 if (shw_hblkp) { 10227 ASSERT(hblktosfmmu(hmeblkp) != KHATID); 10228 if (mmu_page_sizes == max_mmu_page_sizes) { 10229 ASSERT(size < TTE256M); 10230 } else { 10231 ASSERT(size < TTE4M); 10232 } 10233 10234 shw_size = get_hblk_ttesz(shw_hblkp); 10235 vaddr = get_hblk_base(hmeblkp); 10236 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10237 ASSERT(vshift < 8); 10238 /* 10239 * Atomically clear shadow mask bit 10240 */ 10241 do { 10242 shw_mask = shw_hblkp->hblk_shw_mask; 10243 ASSERT(shw_mask & (1 << vshift)); 10244 newshw_mask = shw_mask & ~(1 << vshift); 10245 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10246 shw_mask, newshw_mask); 10247 } while (newshw_mask != shw_mask); 10248 hmeblkp->hblk_shadow = NULL; 10249 } 10250 hmeblkp->hblk_next = NULL; 10251 hmeblkp->hblk_nextpa = hblkpa; 10252 hmeblkp->hblk_shw_bit = 0; 10253 10254 if (hmeblkp->hblk_nuc_bit == 0) { 10255 10256 if (size == TTE8K && sfmmu_put_free_hblk(hmeblkp, critical)) 10257 return; 10258 10259 hmeblkp->hblk_next = *listp; 10260 *listp = hmeblkp; 10261 } 10262 } 10263 10264 static void 10265 sfmmu_hblks_list_purge(struct hme_blk **listp) 10266 { 10267 struct hme_blk *hmeblkp; 10268 10269 while ((hmeblkp = *listp) != NULL) { 10270 *listp = hmeblkp->hblk_next; 10271 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); 10272 } 10273 } 10274 10275 #define BUCKETS_TO_SEARCH_BEFORE_UNLOAD 30 10276 10277 static uint_t sfmmu_hblk_steal_twice; 10278 static uint_t sfmmu_hblk_steal_count, sfmmu_hblk_steal_unload_count; 10279 10280 /* 10281 * Steal a hmeblk 10282 * Enough hmeblks were allocated at startup (nucleus hmeblks) and also 10283 * hmeblks were added dynamically. We should never ever not be able to 10284 * find one. Look for an unused/unlocked hmeblk in user hash table. 10285 */ 10286 static struct hme_blk * 10287 sfmmu_hblk_steal(int size) 10288 { 10289 static struct hmehash_bucket *uhmehash_steal_hand = NULL; 10290 struct hmehash_bucket *hmebp; 10291 struct hme_blk *hmeblkp = NULL, *pr_hblk; 10292 uint64_t hblkpa, prevpa; 10293 int i; 10294 10295 for (;;) { 10296 hmebp = (uhmehash_steal_hand == NULL) ? uhme_hash : 10297 uhmehash_steal_hand; 10298 ASSERT(hmebp >= uhme_hash && hmebp <= &uhme_hash[UHMEHASH_SZ]); 10299 10300 for (i = 0; hmeblkp == NULL && i <= UHMEHASH_SZ + 10301 BUCKETS_TO_SEARCH_BEFORE_UNLOAD; i++) { 10302 SFMMU_HASH_LOCK(hmebp); 10303 hmeblkp = hmebp->hmeblkp; 10304 hblkpa = hmebp->hmeh_nextpa; 10305 prevpa = 0; 10306 pr_hblk = NULL; 10307 while (hmeblkp) { 10308 /* 10309 * check if it is a hmeblk that is not locked 10310 * and not shared. skip shadow hmeblks with 10311 * shadow_mask set i.e valid count non zero. 10312 */ 10313 if ((get_hblk_ttesz(hmeblkp) == size) && 10314 (hmeblkp->hblk_shw_bit == 0 || 10315 hmeblkp->hblk_vcnt == 0) && 10316 (hmeblkp->hblk_lckcnt == 0)) { 10317 /* 10318 * there is a high probability that we 10319 * will find a free one. search some 10320 * buckets for a free hmeblk initially 10321 * before unloading a valid hmeblk. 10322 */ 10323 if ((hmeblkp->hblk_vcnt == 0 && 10324 hmeblkp->hblk_hmecnt == 0) || (i >= 10325 BUCKETS_TO_SEARCH_BEFORE_UNLOAD)) { 10326 if (sfmmu_steal_this_hblk(hmebp, 10327 hmeblkp, hblkpa, prevpa, 10328 pr_hblk)) { 10329 /* 10330 * Hblk is unloaded 10331 * successfully 10332 */ 10333 break; 10334 } 10335 } 10336 } 10337 pr_hblk = hmeblkp; 10338 prevpa = hblkpa; 10339 hblkpa = hmeblkp->hblk_nextpa; 10340 hmeblkp = hmeblkp->hblk_next; 10341 } 10342 10343 SFMMU_HASH_UNLOCK(hmebp); 10344 if (hmebp++ == &uhme_hash[UHMEHASH_SZ]) 10345 hmebp = uhme_hash; 10346 } 10347 uhmehash_steal_hand = hmebp; 10348 10349 if (hmeblkp != NULL) 10350 break; 10351 10352 /* 10353 * in the worst case, look for a free one in the kernel 10354 * hash table. 10355 */ 10356 for (i = 0, hmebp = khme_hash; i <= KHMEHASH_SZ; i++) { 10357 SFMMU_HASH_LOCK(hmebp); 10358 hmeblkp = hmebp->hmeblkp; 10359 hblkpa = hmebp->hmeh_nextpa; 10360 prevpa = 0; 10361 pr_hblk = NULL; 10362 while (hmeblkp) { 10363 /* 10364 * check if it is free hmeblk 10365 */ 10366 if ((get_hblk_ttesz(hmeblkp) == size) && 10367 (hmeblkp->hblk_lckcnt == 0) && 10368 (hmeblkp->hblk_vcnt == 0) && 10369 (hmeblkp->hblk_hmecnt == 0)) { 10370 if (sfmmu_steal_this_hblk(hmebp, 10371 hmeblkp, hblkpa, prevpa, pr_hblk)) { 10372 break; 10373 } else { 10374 /* 10375 * Cannot fail since we have 10376 * hash lock. 10377 */ 10378 panic("fail to steal?"); 10379 } 10380 } 10381 10382 pr_hblk = hmeblkp; 10383 prevpa = hblkpa; 10384 hblkpa = hmeblkp->hblk_nextpa; 10385 hmeblkp = hmeblkp->hblk_next; 10386 } 10387 10388 SFMMU_HASH_UNLOCK(hmebp); 10389 if (hmebp++ == &khme_hash[KHMEHASH_SZ]) 10390 hmebp = khme_hash; 10391 } 10392 10393 if (hmeblkp != NULL) 10394 break; 10395 sfmmu_hblk_steal_twice++; 10396 } 10397 return (hmeblkp); 10398 } 10399 10400 /* 10401 * This routine does real work to prepare a hblk to be "stolen" by 10402 * unloading the mappings, updating shadow counts .... 10403 * It returns 1 if the block is ready to be reused (stolen), or 0 10404 * means the block cannot be stolen yet- pageunload is still working 10405 * on this hblk. 10406 */ 10407 static int 10408 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, 10409 uint64_t hblkpa, uint64_t prevpa, struct hme_blk *pr_hblk) 10410 { 10411 int shw_size, vshift; 10412 struct hme_blk *shw_hblkp; 10413 uintptr_t vaddr; 10414 uint_t shw_mask, newshw_mask; 10415 10416 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); 10417 10418 /* 10419 * check if the hmeblk is free, unload if necessary 10420 */ 10421 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10422 sfmmu_t *sfmmup; 10423 demap_range_t dmr; 10424 10425 sfmmup = hblktosfmmu(hmeblkp); 10426 DEMAP_RANGE_INIT(sfmmup, &dmr); 10427 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, 10428 (caddr_t)get_hblk_base(hmeblkp), 10429 get_hblk_endaddr(hmeblkp), &dmr, HAT_UNLOAD); 10430 DEMAP_RANGE_FLUSH(&dmr); 10431 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { 10432 /* 10433 * Pageunload is working on the same hblk. 10434 */ 10435 return (0); 10436 } 10437 10438 sfmmu_hblk_steal_unload_count++; 10439 } 10440 10441 ASSERT(hmeblkp->hblk_lckcnt == 0); 10442 ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0); 10443 10444 sfmmu_hblk_hash_rm(hmebp, hmeblkp, prevpa, pr_hblk); 10445 hmeblkp->hblk_nextpa = hblkpa; 10446 10447 shw_hblkp = hmeblkp->hblk_shadow; 10448 if (shw_hblkp) { 10449 shw_size = get_hblk_ttesz(shw_hblkp); 10450 vaddr = get_hblk_base(hmeblkp); 10451 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size); 10452 ASSERT(vshift < 8); 10453 /* 10454 * Atomically clear shadow mask bit 10455 */ 10456 do { 10457 shw_mask = shw_hblkp->hblk_shw_mask; 10458 ASSERT(shw_mask & (1 << vshift)); 10459 newshw_mask = shw_mask & ~(1 << vshift); 10460 newshw_mask = cas32(&shw_hblkp->hblk_shw_mask, 10461 shw_mask, newshw_mask); 10462 } while (newshw_mask != shw_mask); 10463 hmeblkp->hblk_shadow = NULL; 10464 } 10465 10466 /* 10467 * remove shadow bit if we are stealing an unused shadow hmeblk. 10468 * sfmmu_hblk_alloc needs it that way, will set shadow bit later if 10469 * we are indeed allocating a shadow hmeblk. 10470 */ 10471 hmeblkp->hblk_shw_bit = 0; 10472 10473 sfmmu_hblk_steal_count++; 10474 SFMMU_STAT(sf_steal_count); 10475 10476 return (1); 10477 } 10478 10479 struct hme_blk * 10480 sfmmu_hmetohblk(struct sf_hment *sfhme) 10481 { 10482 struct hme_blk *hmeblkp; 10483 struct sf_hment *sfhme0; 10484 struct hme_blk *hblk_dummy = 0; 10485 10486 /* 10487 * No dummy sf_hments, please. 10488 */ 10489 ASSERT(sfhme->hme_tte.ll != 0); 10490 10491 sfhme0 = sfhme - sfhme->hme_tte.tte_hmenum; 10492 hmeblkp = (struct hme_blk *)((uintptr_t)sfhme0 - 10493 (uintptr_t)&hblk_dummy->hblk_hme[0]); 10494 10495 return (hmeblkp); 10496 } 10497 10498 /* 10499 * On swapin, get appropriately sized TSB(s) and clear the HAT_SWAPPED flag. 10500 * If we can't get appropriately sized TSB(s), try for 8K TSB(s) using 10501 * KM_SLEEP allocation. 10502 * 10503 * Return 0 on success, -1 otherwise. 10504 */ 10505 static void 10506 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp) 10507 { 10508 struct tsb_info *tsbinfop, *next; 10509 tsb_replace_rc_t rc; 10510 boolean_t gotfirst = B_FALSE; 10511 10512 ASSERT(sfmmup != ksfmmup); 10513 ASSERT(sfmmu_hat_lock_held(sfmmup)); 10514 10515 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) { 10516 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); 10517 } 10518 10519 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10520 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN); 10521 } else { 10522 return; 10523 } 10524 10525 ASSERT(sfmmup->sfmmu_tsb != NULL); 10526 10527 /* 10528 * Loop over all tsbinfo's replacing them with ones that actually have 10529 * a TSB. If any of the replacements ever fail, bail out of the loop. 10530 */ 10531 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) { 10532 ASSERT(tsbinfop->tsb_flags & TSB_SWAPPED); 10533 next = tsbinfop->tsb_next; 10534 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc, 10535 hatlockp, TSB_SWAPIN); 10536 if (rc != TSB_SUCCESS) { 10537 break; 10538 } 10539 gotfirst = B_TRUE; 10540 } 10541 10542 switch (rc) { 10543 case TSB_SUCCESS: 10544 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10545 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10546 return; 10547 case TSB_ALLOCFAIL: 10548 break; 10549 default: 10550 panic("sfmmu_replace_tsb returned unrecognized failure code " 10551 "%d", rc); 10552 } 10553 10554 /* 10555 * In this case, we failed to get one of our TSBs. If we failed to 10556 * get the first TSB, get one of minimum size (8KB). Walk the list 10557 * and throw away the tsbinfos, starting where the allocation failed; 10558 * we can get by with just one TSB as long as we don't leave the 10559 * SWAPPED tsbinfo structures lying around. 10560 */ 10561 tsbinfop = sfmmup->sfmmu_tsb; 10562 next = tsbinfop->tsb_next; 10563 tsbinfop->tsb_next = NULL; 10564 10565 sfmmu_hat_exit(hatlockp); 10566 for (tsbinfop = next; tsbinfop != NULL; tsbinfop = next) { 10567 next = tsbinfop->tsb_next; 10568 sfmmu_tsbinfo_free(tsbinfop); 10569 } 10570 hatlockp = sfmmu_hat_enter(sfmmup); 10571 10572 /* 10573 * If we don't have any TSBs, get a single 8K TSB for 8K, 64K and 512K 10574 * pages. 10575 */ 10576 if (!gotfirst) { 10577 tsbinfop = sfmmup->sfmmu_tsb; 10578 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE, 10579 hatlockp, TSB_SWAPIN | TSB_FORCEALLOC); 10580 ASSERT(rc == TSB_SUCCESS); 10581 } 10582 10583 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); 10584 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 10585 } 10586 10587 /* 10588 * Handle exceptions for low level tsb_handler. 10589 * 10590 * There are many scenarios that could land us here: 10591 * 10592 * If the context is invalid we land here. The context can be invalid 10593 * for 3 reasons: 1) we couldn't allocate a new context and now need to 10594 * perform a wrap around operation in order to allocate a new context. 10595 * 2) Context was invalidated to change pagesize programming 3) ISMs or 10596 * TSBs configuration is changeing for this process and we are forced into 10597 * here to do a syncronization operation. If the context is valid we can 10598 * be here from window trap hanlder. In this case just call trap to handle 10599 * the fault. 10600 * 10601 * Note that the process will run in INVALID_CONTEXT before 10602 * faulting into here and subsequently loading the MMU registers 10603 * (including the TSB base register) associated with this process. 10604 * For this reason, the trap handlers must all test for 10605 * INVALID_CONTEXT before attempting to access any registers other 10606 * than the context registers. 10607 */ 10608 void 10609 sfmmu_tsbmiss_exception(struct regs *rp, uintptr_t tagaccess, uint_t traptype) 10610 { 10611 sfmmu_t *sfmmup; 10612 uint_t ctxnum; 10613 klwp_id_t lwp; 10614 char lwp_save_state; 10615 hatlock_t *hatlockp; 10616 struct tsb_info *tsbinfop; 10617 10618 SFMMU_STAT(sf_tsb_exceptions); 10619 SFMMU_MMU_STAT(mmu_tsb_exceptions); 10620 sfmmup = astosfmmu(curthread->t_procp->p_as); 10621 ctxnum = tagaccess & TAGACC_CTX_MASK; 10622 10623 ASSERT(sfmmup != ksfmmup && ctxnum != KCONTEXT); 10624 ASSERT(sfmmup->sfmmu_ismhat == 0); 10625 /* 10626 * First, make sure we come out of here with a valid ctx, 10627 * since if we don't get one we'll simply loop on the 10628 * faulting instruction. 10629 * 10630 * If the ISM mappings are changing, the TSB is being relocated, or 10631 * the process is swapped out we serialize behind the controlling 10632 * thread with the sfmmu_flags and sfmmu_tsb_cv condition variable. 10633 * Otherwise we synchronize with the context stealer or the thread 10634 * that required us to change out our MMU registers (such 10635 * as a thread changing out our TSB while we were running) by 10636 * locking the HAT and grabbing the rwlock on the context as a 10637 * reader temporarily. 10638 */ 10639 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) || 10640 ctxnum == INVALID_CONTEXT); 10641 10642 if (ctxnum == INVALID_CONTEXT) { 10643 /* 10644 * Must set lwp state to LWP_SYS before 10645 * trying to acquire any adaptive lock 10646 */ 10647 lwp = ttolwp(curthread); 10648 ASSERT(lwp); 10649 lwp_save_state = lwp->lwp_state; 10650 lwp->lwp_state = LWP_SYS; 10651 10652 hatlockp = sfmmu_hat_enter(sfmmup); 10653 retry: 10654 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; 10655 tsbinfop = tsbinfop->tsb_next) { 10656 if (tsbinfop->tsb_flags & TSB_RELOC_FLAG) { 10657 cv_wait(&sfmmup->sfmmu_tsb_cv, 10658 HATLOCK_MUTEXP(hatlockp)); 10659 goto retry; 10660 } 10661 } 10662 10663 /* 10664 * Wait for ISM maps to be updated. 10665 */ 10666 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) { 10667 cv_wait(&sfmmup->sfmmu_tsb_cv, 10668 HATLOCK_MUTEXP(hatlockp)); 10669 goto retry; 10670 } 10671 10672 /* 10673 * If we're swapping in, get TSB(s). Note that we must do 10674 * this before we get a ctx or load the MMU state. Once 10675 * we swap in we have to recheck to make sure the TSB(s) and 10676 * ISM mappings didn't change while we slept. 10677 */ 10678 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { 10679 sfmmu_tsb_swapin(sfmmup, hatlockp); 10680 goto retry; 10681 } 10682 10683 sfmmu_get_ctx(sfmmup); 10684 10685 sfmmu_hat_exit(hatlockp); 10686 /* 10687 * Must restore lwp_state if not calling 10688 * trap() for further processing. Restore 10689 * it anyway. 10690 */ 10691 lwp->lwp_state = lwp_save_state; 10692 if (sfmmup->sfmmu_ttecnt[TTE8K] != 0 || 10693 sfmmup->sfmmu_ttecnt[TTE64K] != 0 || 10694 sfmmup->sfmmu_ttecnt[TTE512K] != 0 || 10695 sfmmup->sfmmu_ttecnt[TTE4M] != 0 || 10696 sfmmup->sfmmu_ttecnt[TTE32M] != 0 || 10697 sfmmup->sfmmu_ttecnt[TTE256M] != 0) { 10698 return; 10699 } 10700 if (traptype == T_DATA_PROT) { 10701 traptype = T_DATA_MMU_MISS; 10702 } 10703 } 10704 trap(rp, (caddr_t)tagaccess, traptype, 0); 10705 } 10706 10707 /* 10708 * sfmmu_vatopfn_suspended is called from GET_TTE when TL=0 and 10709 * TTE_SUSPENDED bit set in tte we block on aquiring a page lock 10710 * rather than spinning to avoid send mondo timeouts with 10711 * interrupts enabled. When the lock is acquired it is immediately 10712 * released and we return back to sfmmu_vatopfn just after 10713 * the GET_TTE call. 10714 */ 10715 void 10716 sfmmu_vatopfn_suspended(caddr_t vaddr, sfmmu_t *sfmmu, tte_t *ttep) 10717 { 10718 struct page **pp; 10719 10720 (void) as_pagelock(sfmmu->sfmmu_as, &pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10721 as_pageunlock(sfmmu->sfmmu_as, pp, vaddr, TTE_CSZ(ttep), S_WRITE); 10722 } 10723 10724 /* 10725 * sfmmu_tsbmiss_suspended is called from GET_TTE when TL>0 and 10726 * TTE_SUSPENDED bit set in tte. We do this so that we can handle 10727 * cross traps which cannot be handled while spinning in the 10728 * trap handlers. Simply enter and exit the kpr_suspendlock spin 10729 * mutex, which is held by the holder of the suspend bit, and then 10730 * retry the trapped instruction after unwinding. 10731 */ 10732 /*ARGSUSED*/ 10733 void 10734 sfmmu_tsbmiss_suspended(struct regs *rp, uintptr_t tagacc, uint_t traptype) 10735 { 10736 ASSERT(curthread != kreloc_thread); 10737 mutex_enter(&kpr_suspendlock); 10738 mutex_exit(&kpr_suspendlock); 10739 } 10740 10741 /* 10742 * Special routine to flush out ism mappings- TSBs, TLBs and D-caches. 10743 * This routine may be called with all cpu's captured. Therefore, the 10744 * caller is responsible for holding all locks and disabling kernel 10745 * preemption. 10746 */ 10747 /* ARGSUSED */ 10748 static void 10749 sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, 10750 struct hme_blk *hmeblkp, pfn_t pfnum, int cache_flush_flag) 10751 { 10752 cpuset_t cpuset; 10753 caddr_t va; 10754 ism_ment_t *ment; 10755 sfmmu_t *sfmmup; 10756 #ifdef VAC 10757 int vcolor; 10758 #endif 10759 int ttesz; 10760 10761 /* 10762 * Walk the ism_hat's mapping list and flush the page 10763 * from every hat sharing this ism_hat. This routine 10764 * may be called while all cpu's have been captured. 10765 * Therefore we can't attempt to grab any locks. For now 10766 * this means we will protect the ism mapping list under 10767 * a single lock which will be grabbed by the caller. 10768 * If hat_share/unshare scalibility becomes a performance 10769 * problem then we may need to re-think ism mapping list locking. 10770 */ 10771 ASSERT(ism_sfmmup->sfmmu_ismhat); 10772 ASSERT(MUTEX_HELD(&ism_mlist_lock)); 10773 addr = addr - ISMID_STARTADDR; 10774 for (ment = ism_sfmmup->sfmmu_iment; ment; ment = ment->iment_next) { 10775 10776 sfmmup = ment->iment_hat; 10777 10778 va = ment->iment_base_va; 10779 va = (caddr_t)((uintptr_t)va + (uintptr_t)addr); 10780 10781 /* 10782 * Flush TSB of ISM mappings. 10783 */ 10784 ttesz = get_hblk_ttesz(hmeblkp); 10785 if (ttesz == TTE8K || ttesz == TTE4M) { 10786 sfmmu_unload_tsb(sfmmup, va, ttesz); 10787 } else { 10788 caddr_t sva = va; 10789 caddr_t eva; 10790 ASSERT(addr == (caddr_t)get_hblk_base(hmeblkp)); 10791 eva = sva + get_hblk_span(hmeblkp); 10792 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); 10793 } 10794 10795 cpuset = sfmmup->sfmmu_cpusran; 10796 CPUSET_AND(cpuset, cpu_ready_set); 10797 CPUSET_DEL(cpuset, CPU->cpu_id); 10798 10799 SFMMU_XCALL_STATS(sfmmup); 10800 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)va, 10801 (uint64_t)sfmmup); 10802 10803 vtag_flushpage(va, (uint64_t)sfmmup); 10804 10805 #ifdef VAC 10806 /* 10807 * Flush D$ 10808 * When flushing D$ we must flush all 10809 * cpu's. See sfmmu_cache_flush(). 10810 */ 10811 if (cache_flush_flag == CACHE_FLUSH) { 10812 cpuset = cpu_ready_set; 10813 CPUSET_DEL(cpuset, CPU->cpu_id); 10814 10815 SFMMU_XCALL_STATS(sfmmup); 10816 vcolor = addr_to_vcolor(va); 10817 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10818 vac_flushpage(pfnum, vcolor); 10819 } 10820 #endif /* VAC */ 10821 } 10822 } 10823 10824 /* 10825 * Demaps the TSB, CPU caches, and flushes all TLBs on all CPUs of 10826 * a particular virtual address and ctx. If noflush is set we do not 10827 * flush the TLB/TSB. This function may or may not be called with the 10828 * HAT lock held. 10829 */ 10830 static void 10831 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10832 pfn_t pfnum, int tlb_noflush, int cpu_flag, int cache_flush_flag, 10833 int hat_lock_held) 10834 { 10835 #ifdef VAC 10836 int vcolor; 10837 #endif 10838 cpuset_t cpuset; 10839 hatlock_t *hatlockp; 10840 10841 #if defined(lint) && !defined(VAC) 10842 pfnum = pfnum; 10843 cpu_flag = cpu_flag; 10844 cache_flush_flag = cache_flush_flag; 10845 #endif 10846 /* 10847 * There is no longer a need to protect against ctx being 10848 * stolen here since we don't store the ctx in the TSB anymore. 10849 */ 10850 #ifdef VAC 10851 vcolor = addr_to_vcolor(addr); 10852 #endif 10853 10854 /* 10855 * We must hold the hat lock during the flush of TLB, 10856 * to avoid a race with sfmmu_invalidate_ctx(), where 10857 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 10858 * causing TLB demap routine to skip flush on that MMU. 10859 * If the context on a MMU has already been set to 10860 * INVALID_CONTEXT, we just get an extra flush on 10861 * that MMU. 10862 */ 10863 if (!hat_lock_held && !tlb_noflush) 10864 hatlockp = sfmmu_hat_enter(sfmmup); 10865 10866 kpreempt_disable(); 10867 if (!tlb_noflush) { 10868 /* 10869 * Flush the TSB and TLB. 10870 */ 10871 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10872 10873 cpuset = sfmmup->sfmmu_cpusran; 10874 CPUSET_AND(cpuset, cpu_ready_set); 10875 CPUSET_DEL(cpuset, CPU->cpu_id); 10876 10877 SFMMU_XCALL_STATS(sfmmup); 10878 10879 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, 10880 (uint64_t)sfmmup); 10881 10882 vtag_flushpage(addr, (uint64_t)sfmmup); 10883 } 10884 10885 if (!hat_lock_held && !tlb_noflush) 10886 sfmmu_hat_exit(hatlockp); 10887 10888 #ifdef VAC 10889 /* 10890 * Flush the D$ 10891 * 10892 * Even if the ctx is stolen, we need to flush the 10893 * cache. Our ctx stealer only flushes the TLBs. 10894 */ 10895 if (cache_flush_flag == CACHE_FLUSH) { 10896 if (cpu_flag & FLUSH_ALL_CPUS) { 10897 cpuset = cpu_ready_set; 10898 } else { 10899 cpuset = sfmmup->sfmmu_cpusran; 10900 CPUSET_AND(cpuset, cpu_ready_set); 10901 } 10902 CPUSET_DEL(cpuset, CPU->cpu_id); 10903 SFMMU_XCALL_STATS(sfmmup); 10904 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 10905 vac_flushpage(pfnum, vcolor); 10906 } 10907 #endif /* VAC */ 10908 kpreempt_enable(); 10909 } 10910 10911 /* 10912 * Demaps the TSB and flushes all TLBs on all cpus for a particular virtual 10913 * address and ctx. If noflush is set we do not currently do anything. 10914 * This function may or may not be called with the HAT lock held. 10915 */ 10916 static void 10917 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, 10918 int tlb_noflush, int hat_lock_held) 10919 { 10920 cpuset_t cpuset; 10921 hatlock_t *hatlockp; 10922 10923 /* 10924 * If the process is exiting we have nothing to do. 10925 */ 10926 if (tlb_noflush) 10927 return; 10928 10929 /* 10930 * Flush TSB. 10931 */ 10932 if (!hat_lock_held) 10933 hatlockp = sfmmu_hat_enter(sfmmup); 10934 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); 10935 10936 kpreempt_disable(); 10937 10938 cpuset = sfmmup->sfmmu_cpusran; 10939 CPUSET_AND(cpuset, cpu_ready_set); 10940 CPUSET_DEL(cpuset, CPU->cpu_id); 10941 10942 SFMMU_XCALL_STATS(sfmmup); 10943 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup); 10944 10945 vtag_flushpage(addr, (uint64_t)sfmmup); 10946 10947 if (!hat_lock_held) 10948 sfmmu_hat_exit(hatlockp); 10949 10950 kpreempt_enable(); 10951 10952 } 10953 10954 /* 10955 * Special case of sfmmu_tlb_demap for MMU_PAGESIZE hblks. Use the xcall 10956 * call handler that can flush a range of pages to save on xcalls. 10957 */ 10958 static int sfmmu_xcall_save; 10959 10960 static void 10961 sfmmu_tlb_range_demap(demap_range_t *dmrp) 10962 { 10963 sfmmu_t *sfmmup = dmrp->dmr_sfmmup; 10964 hatlock_t *hatlockp; 10965 cpuset_t cpuset; 10966 uint64_t sfmmu_pgcnt; 10967 pgcnt_t pgcnt = 0; 10968 int pgunload = 0; 10969 int dirtypg = 0; 10970 caddr_t addr = dmrp->dmr_addr; 10971 caddr_t eaddr; 10972 uint64_t bitvec = dmrp->dmr_bitvec; 10973 10974 ASSERT(bitvec & 1); 10975 10976 /* 10977 * Flush TSB and calculate number of pages to flush. 10978 */ 10979 while (bitvec != 0) { 10980 dirtypg = 0; 10981 /* 10982 * Find the first page to flush and then count how many 10983 * pages there are after it that also need to be flushed. 10984 * This way the number of TSB flushes is minimized. 10985 */ 10986 while ((bitvec & 1) == 0) { 10987 pgcnt++; 10988 addr += MMU_PAGESIZE; 10989 bitvec >>= 1; 10990 } 10991 while (bitvec & 1) { 10992 dirtypg++; 10993 bitvec >>= 1; 10994 } 10995 eaddr = addr + ptob(dirtypg); 10996 hatlockp = sfmmu_hat_enter(sfmmup); 10997 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K); 10998 sfmmu_hat_exit(hatlockp); 10999 pgunload += dirtypg; 11000 addr = eaddr; 11001 pgcnt += dirtypg; 11002 } 11003 11004 ASSERT((pgcnt<<MMU_PAGESHIFT) <= dmrp->dmr_endaddr - dmrp->dmr_addr); 11005 if (sfmmup->sfmmu_free == 0) { 11006 addr = dmrp->dmr_addr; 11007 bitvec = dmrp->dmr_bitvec; 11008 11009 /* 11010 * make sure it has SFMMU_PGCNT_SHIFT bits only, 11011 * as it will be used to pack argument for xt_some 11012 */ 11013 ASSERT((pgcnt > 0) && 11014 (pgcnt <= (1 << SFMMU_PGCNT_SHIFT))); 11015 11016 /* 11017 * Encode pgcnt as (pgcnt -1 ), and pass (pgcnt - 1) in 11018 * the low 6 bits of sfmmup. This is doable since pgcnt 11019 * always >= 1. 11020 */ 11021 ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK)); 11022 sfmmu_pgcnt = (uint64_t)sfmmup | 11023 ((pgcnt - 1) & SFMMU_PGCNT_MASK); 11024 11025 /* 11026 * We must hold the hat lock during the flush of TLB, 11027 * to avoid a race with sfmmu_invalidate_ctx(), where 11028 * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, 11029 * causing TLB demap routine to skip flush on that MMU. 11030 * If the context on a MMU has already been set to 11031 * INVALID_CONTEXT, we just get an extra flush on 11032 * that MMU. 11033 */ 11034 hatlockp = sfmmu_hat_enter(sfmmup); 11035 kpreempt_disable(); 11036 11037 cpuset = sfmmup->sfmmu_cpusran; 11038 CPUSET_AND(cpuset, cpu_ready_set); 11039 CPUSET_DEL(cpuset, CPU->cpu_id); 11040 11041 SFMMU_XCALL_STATS(sfmmup); 11042 xt_some(cpuset, vtag_flush_pgcnt_tl1, (uint64_t)addr, 11043 sfmmu_pgcnt); 11044 11045 for (; bitvec != 0; bitvec >>= 1) { 11046 if (bitvec & 1) 11047 vtag_flushpage(addr, (uint64_t)sfmmup); 11048 addr += MMU_PAGESIZE; 11049 } 11050 kpreempt_enable(); 11051 sfmmu_hat_exit(hatlockp); 11052 11053 sfmmu_xcall_save += (pgunload-1); 11054 } 11055 dmrp->dmr_bitvec = 0; 11056 } 11057 11058 /* 11059 * In cases where we need to synchronize with TLB/TSB miss trap 11060 * handlers, _and_ need to flush the TLB, it's a lot easier to 11061 * throw away the context from the process than to do a 11062 * special song and dance to keep things consistent for the 11063 * handlers. 11064 * 11065 * Since the process suddenly ends up without a context and our caller 11066 * holds the hat lock, threads that fault after this function is called 11067 * will pile up on the lock. We can then do whatever we need to 11068 * atomically from the context of the caller. The first blocked thread 11069 * to resume executing will get the process a new context, and the 11070 * process will resume executing. 11071 * 11072 * One added advantage of this approach is that on MMUs that 11073 * support a "flush all" operation, we will delay the flush until 11074 * cnum wrap-around, and then flush the TLB one time. This 11075 * is rather rare, so it's a lot less expensive than making 8000 11076 * x-calls to flush the TLB 8000 times. 11077 * 11078 * A per-process (PP) lock is used to synchronize ctx allocations in 11079 * resume() and ctx invalidations here. 11080 */ 11081 static void 11082 sfmmu_invalidate_ctx(sfmmu_t *sfmmup) 11083 { 11084 cpuset_t cpuset; 11085 int cnum, currcnum; 11086 mmu_ctx_t *mmu_ctxp; 11087 int i; 11088 uint_t pstate_save; 11089 11090 SFMMU_STAT(sf_ctx_inv); 11091 11092 ASSERT(sfmmu_hat_lock_held(sfmmup)); 11093 ASSERT(sfmmup != ksfmmup); 11094 11095 kpreempt_disable(); 11096 11097 mmu_ctxp = CPU_MMU_CTXP(CPU); 11098 ASSERT(mmu_ctxp); 11099 ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); 11100 ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); 11101 11102 currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; 11103 11104 pstate_save = sfmmu_disable_intrs(); 11105 11106 lock_set(&sfmmup->sfmmu_ctx_lock); /* acquire PP lock */ 11107 /* set HAT cnum invalid across all context domains. */ 11108 for (i = 0; i < max_mmu_ctxdoms; i++) { 11109 11110 cnum = sfmmup->sfmmu_ctxs[i].cnum; 11111 if (cnum == INVALID_CONTEXT) { 11112 continue; 11113 } 11114 11115 sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT; 11116 } 11117 membar_enter(); /* make sure globally visible to all CPUs */ 11118 lock_clear(&sfmmup->sfmmu_ctx_lock); /* release PP lock */ 11119 11120 sfmmu_enable_intrs(pstate_save); 11121 11122 cpuset = sfmmup->sfmmu_cpusran; 11123 CPUSET_DEL(cpuset, CPU->cpu_id); 11124 CPUSET_AND(cpuset, cpu_ready_set); 11125 if (!CPUSET_ISNULL(cpuset)) { 11126 SFMMU_XCALL_STATS(sfmmup); 11127 xt_some(cpuset, sfmmu_raise_tsb_exception, 11128 (uint64_t)sfmmup, INVALID_CONTEXT); 11129 xt_sync(cpuset); 11130 SFMMU_STAT(sf_tsb_raise_exception); 11131 SFMMU_MMU_STAT(mmu_tsb_raise_exception); 11132 } 11133 11134 /* 11135 * If the hat to-be-invalidated is the same as the current 11136 * process on local CPU we need to invalidate 11137 * this CPU context as well. 11138 */ 11139 if ((sfmmu_getctx_sec() == currcnum) && 11140 (currcnum != INVALID_CONTEXT)) { 11141 sfmmu_setctx_sec(INVALID_CONTEXT); 11142 sfmmu_clear_utsbinfo(); 11143 } 11144 11145 kpreempt_enable(); 11146 11147 /* 11148 * we hold the hat lock, so nobody should allocate a context 11149 * for us yet 11150 */ 11151 ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT); 11152 } 11153 11154 #ifdef VAC 11155 /* 11156 * We need to flush the cache in all cpus. It is possible that 11157 * a process referenced a page as cacheable but has sinced exited 11158 * and cleared the mapping list. We still to flush it but have no 11159 * state so all cpus is the only alternative. 11160 */ 11161 void 11162 sfmmu_cache_flush(pfn_t pfnum, int vcolor) 11163 { 11164 cpuset_t cpuset; 11165 11166 kpreempt_disable(); 11167 cpuset = cpu_ready_set; 11168 CPUSET_DEL(cpuset, CPU->cpu_id); 11169 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11170 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); 11171 xt_sync(cpuset); 11172 vac_flushpage(pfnum, vcolor); 11173 kpreempt_enable(); 11174 } 11175 11176 void 11177 sfmmu_cache_flushcolor(int vcolor, pfn_t pfnum) 11178 { 11179 cpuset_t cpuset; 11180 11181 ASSERT(vcolor >= 0); 11182 11183 kpreempt_disable(); 11184 cpuset = cpu_ready_set; 11185 CPUSET_DEL(cpuset, CPU->cpu_id); 11186 SFMMU_XCALL_STATS(NULL); /* account to any ctx */ 11187 xt_some(cpuset, vac_flushcolor_tl1, vcolor, pfnum); 11188 xt_sync(cpuset); 11189 vac_flushcolor(vcolor, pfnum); 11190 kpreempt_enable(); 11191 } 11192 #endif /* VAC */ 11193 11194 /* 11195 * We need to prevent processes from accessing the TSB using a cached physical 11196 * address. It's alright if they try to access the TSB via virtual address 11197 * since they will just fault on that virtual address once the mapping has 11198 * been suspended. 11199 */ 11200 #pragma weak sendmondo_in_recover 11201 11202 /* ARGSUSED */ 11203 static int 11204 sfmmu_tsb_pre_relocator(caddr_t va, uint_t tsbsz, uint_t flags, void *tsbinfo) 11205 { 11206 hatlock_t *hatlockp; 11207 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11208 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11209 extern uint32_t sendmondo_in_recover; 11210 11211 if (flags != HAT_PRESUSPEND) 11212 return (0); 11213 11214 hatlockp = sfmmu_hat_enter(sfmmup); 11215 11216 tsbinfop->tsb_flags |= TSB_RELOC_FLAG; 11217 11218 /* 11219 * For Cheetah+ Erratum 25: 11220 * Wait for any active recovery to finish. We can't risk 11221 * relocating the TSB of the thread running mondo_recover_proc() 11222 * since, if we did that, we would deadlock. The scenario we are 11223 * trying to avoid is as follows: 11224 * 11225 * THIS CPU RECOVER CPU 11226 * -------- ----------- 11227 * Begins recovery, walking through TSB 11228 * hat_pagesuspend() TSB TTE 11229 * TLB miss on TSB TTE, spins at TL1 11230 * xt_sync() 11231 * send_mondo_timeout() 11232 * mondo_recover_proc() 11233 * ((deadlocked)) 11234 * 11235 * The second half of the workaround is that mondo_recover_proc() 11236 * checks to see if the tsb_info has the RELOC flag set, and if it 11237 * does, it skips over that TSB without ever touching tsbinfop->tsb_va 11238 * and hence avoiding the TLB miss that could result in a deadlock. 11239 */ 11240 if (&sendmondo_in_recover) { 11241 membar_enter(); /* make sure RELOC flag visible */ 11242 while (sendmondo_in_recover) { 11243 drv_usecwait(1); 11244 membar_consumer(); 11245 } 11246 } 11247 11248 sfmmu_invalidate_ctx(sfmmup); 11249 sfmmu_hat_exit(hatlockp); 11250 11251 return (0); 11252 } 11253 11254 /* ARGSUSED */ 11255 static int 11256 sfmmu_tsb_post_relocator(caddr_t va, uint_t tsbsz, uint_t flags, 11257 void *tsbinfo, pfn_t newpfn) 11258 { 11259 hatlock_t *hatlockp; 11260 struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; 11261 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; 11262 11263 if (flags != HAT_POSTUNSUSPEND) 11264 return (0); 11265 11266 hatlockp = sfmmu_hat_enter(sfmmup); 11267 11268 SFMMU_STAT(sf_tsb_reloc); 11269 11270 /* 11271 * The process may have swapped out while we were relocating one 11272 * of its TSBs. If so, don't bother doing the setup since the 11273 * process can't be using the memory anymore. 11274 */ 11275 if ((tsbinfop->tsb_flags & TSB_SWAPPED) == 0) { 11276 ASSERT(va == tsbinfop->tsb_va); 11277 sfmmu_tsbinfo_setup_phys(tsbinfop, newpfn); 11278 sfmmu_setup_tsbinfo(sfmmup); 11279 11280 if (tsbinfop->tsb_flags & TSB_FLUSH_NEEDED) { 11281 sfmmu_inv_tsb(tsbinfop->tsb_va, 11282 TSB_BYTES(tsbinfop->tsb_szc)); 11283 tsbinfop->tsb_flags &= ~TSB_FLUSH_NEEDED; 11284 } 11285 } 11286 11287 membar_exit(); 11288 tsbinfop->tsb_flags &= ~TSB_RELOC_FLAG; 11289 cv_broadcast(&sfmmup->sfmmu_tsb_cv); 11290 11291 sfmmu_hat_exit(hatlockp); 11292 11293 return (0); 11294 } 11295 11296 /* 11297 * Allocate and initialize a tsb_info structure. Note that we may or may not 11298 * allocate a TSB here, depending on the flags passed in. 11299 */ 11300 static int 11301 sfmmu_tsbinfo_alloc(struct tsb_info **tsbinfopp, int tsb_szc, int tte_sz_mask, 11302 uint_t flags, sfmmu_t *sfmmup) 11303 { 11304 int err; 11305 11306 *tsbinfopp = (struct tsb_info *)kmem_cache_alloc( 11307 sfmmu_tsbinfo_cache, KM_SLEEP); 11308 11309 if ((err = sfmmu_init_tsbinfo(*tsbinfopp, tte_sz_mask, 11310 tsb_szc, flags, sfmmup)) != 0) { 11311 kmem_cache_free(sfmmu_tsbinfo_cache, *tsbinfopp); 11312 SFMMU_STAT(sf_tsb_allocfail); 11313 *tsbinfopp = NULL; 11314 return (err); 11315 } 11316 SFMMU_STAT(sf_tsb_alloc); 11317 11318 /* 11319 * Bump the TSB size counters for this TSB size. 11320 */ 11321 (*(((int *)&sfmmu_tsbsize_stat) + tsb_szc))++; 11322 return (0); 11323 } 11324 11325 static void 11326 sfmmu_tsb_free(struct tsb_info *tsbinfo) 11327 { 11328 caddr_t tsbva = tsbinfo->tsb_va; 11329 uint_t tsb_size = TSB_BYTES(tsbinfo->tsb_szc); 11330 struct kmem_cache *kmem_cachep = tsbinfo->tsb_cache; 11331 vmem_t *vmp = tsbinfo->tsb_vmp; 11332 11333 /* 11334 * If we allocated this TSB from relocatable kernel memory, then we 11335 * need to uninstall the callback handler. 11336 */ 11337 if (tsbinfo->tsb_cache != sfmmu_tsb8k_cache) { 11338 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11339 caddr_t slab_vaddr = (caddr_t)((uintptr_t)tsbva & slab_mask); 11340 page_t **ppl; 11341 int ret; 11342 11343 ret = as_pagelock(&kas, &ppl, slab_vaddr, PAGESIZE, S_WRITE); 11344 ASSERT(ret == 0); 11345 hat_delete_callback(tsbva, (uint_t)tsb_size, (void *)tsbinfo, 11346 0, NULL); 11347 as_pageunlock(&kas, ppl, slab_vaddr, PAGESIZE, S_WRITE); 11348 } 11349 11350 if (kmem_cachep != NULL) { 11351 kmem_cache_free(kmem_cachep, tsbva); 11352 } else { 11353 vmem_xfree(vmp, (void *)tsbva, tsb_size); 11354 } 11355 tsbinfo->tsb_va = (caddr_t)0xbad00bad; 11356 atomic_add_64(&tsb_alloc_bytes, -(int64_t)tsb_size); 11357 } 11358 11359 static void 11360 sfmmu_tsbinfo_free(struct tsb_info *tsbinfo) 11361 { 11362 if ((tsbinfo->tsb_flags & TSB_SWAPPED) == 0) { 11363 sfmmu_tsb_free(tsbinfo); 11364 } 11365 kmem_cache_free(sfmmu_tsbinfo_cache, tsbinfo); 11366 11367 } 11368 11369 /* 11370 * Setup all the references to physical memory for this tsbinfo. 11371 * The underlying page(s) must be locked. 11372 */ 11373 static void 11374 sfmmu_tsbinfo_setup_phys(struct tsb_info *tsbinfo, pfn_t pfn) 11375 { 11376 ASSERT(pfn != PFN_INVALID); 11377 ASSERT(pfn == va_to_pfn(tsbinfo->tsb_va)); 11378 11379 #ifndef sun4v 11380 if (tsbinfo->tsb_szc == 0) { 11381 sfmmu_memtte(&tsbinfo->tsb_tte, pfn, 11382 PROT_WRITE|PROT_READ, TTE8K); 11383 } else { 11384 /* 11385 * Round down PA and use a large mapping; the handlers will 11386 * compute the TSB pointer at the correct offset into the 11387 * big virtual page. NOTE: this assumes all TSBs larger 11388 * than 8K must come from physically contiguous slabs of 11389 * size tsb_slab_size. 11390 */ 11391 sfmmu_memtte(&tsbinfo->tsb_tte, pfn & ~tsb_slab_mask, 11392 PROT_WRITE|PROT_READ, tsb_slab_ttesz); 11393 } 11394 tsbinfo->tsb_pa = ptob(pfn); 11395 11396 TTE_SET_LOCKED(&tsbinfo->tsb_tte); /* lock the tte into dtlb */ 11397 TTE_SET_MOD(&tsbinfo->tsb_tte); /* enable writes */ 11398 11399 ASSERT(TTE_IS_PRIVILEGED(&tsbinfo->tsb_tte)); 11400 ASSERT(TTE_IS_LOCKED(&tsbinfo->tsb_tte)); 11401 #else /* sun4v */ 11402 tsbinfo->tsb_pa = ptob(pfn); 11403 #endif /* sun4v */ 11404 } 11405 11406 11407 /* 11408 * Returns zero on success, ENOMEM if over the high water mark, 11409 * or EAGAIN if the caller needs to retry with a smaller TSB 11410 * size (or specify TSB_FORCEALLOC if the allocation can't fail). 11411 * 11412 * This call cannot fail to allocate a TSB if TSB_FORCEALLOC 11413 * is specified and the TSB requested is PAGESIZE, though it 11414 * may sleep waiting for memory if sufficient memory is not 11415 * available. 11416 */ 11417 static int 11418 sfmmu_init_tsbinfo(struct tsb_info *tsbinfo, int tteszmask, 11419 int tsbcode, uint_t flags, sfmmu_t *sfmmup) 11420 { 11421 caddr_t vaddr = NULL; 11422 caddr_t slab_vaddr; 11423 uintptr_t slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT; 11424 int tsbbytes = TSB_BYTES(tsbcode); 11425 int lowmem = 0; 11426 struct kmem_cache *kmem_cachep = NULL; 11427 vmem_t *vmp = NULL; 11428 lgrp_id_t lgrpid = LGRP_NONE; 11429 pfn_t pfn; 11430 uint_t cbflags = HAC_SLEEP; 11431 page_t **pplist; 11432 int ret; 11433 11434 if (flags & (TSB_FORCEALLOC | TSB_SWAPIN | TSB_GROW | TSB_SHRINK)) 11435 flags |= TSB_ALLOC; 11436 11437 ASSERT((flags & TSB_FORCEALLOC) == 0 || tsbcode == TSB_MIN_SZCODE); 11438 11439 tsbinfo->tsb_sfmmu = sfmmup; 11440 11441 /* 11442 * If not allocating a TSB, set up the tsbinfo, set TSB_SWAPPED, and 11443 * return. 11444 */ 11445 if ((flags & TSB_ALLOC) == 0) { 11446 tsbinfo->tsb_szc = tsbcode; 11447 tsbinfo->tsb_ttesz_mask = tteszmask; 11448 tsbinfo->tsb_va = (caddr_t)0xbadbadbeef; 11449 tsbinfo->tsb_pa = -1; 11450 tsbinfo->tsb_tte.ll = 0; 11451 tsbinfo->tsb_next = NULL; 11452 tsbinfo->tsb_flags = TSB_SWAPPED; 11453 tsbinfo->tsb_cache = NULL; 11454 tsbinfo->tsb_vmp = NULL; 11455 return (0); 11456 } 11457 11458 #ifdef DEBUG 11459 /* 11460 * For debugging: 11461 * Randomly force allocation failures every tsb_alloc_mtbf 11462 * tries if TSB_FORCEALLOC is not specified. This will 11463 * return ENOMEM if tsb_alloc_mtbf is odd, or EAGAIN if 11464 * it is even, to allow testing of both failure paths... 11465 */ 11466 if (tsb_alloc_mtbf && ((flags & TSB_FORCEALLOC) == 0) && 11467 (tsb_alloc_count++ == tsb_alloc_mtbf)) { 11468 tsb_alloc_count = 0; 11469 tsb_alloc_fail_mtbf++; 11470 return ((tsb_alloc_mtbf & 1)? ENOMEM : EAGAIN); 11471 } 11472 #endif /* DEBUG */ 11473 11474 /* 11475 * Enforce high water mark if we are not doing a forced allocation 11476 * and are not shrinking a process' TSB. 11477 */ 11478 if ((flags & TSB_SHRINK) == 0 && 11479 (tsbbytes + tsb_alloc_bytes) > tsb_alloc_hiwater) { 11480 if ((flags & TSB_FORCEALLOC) == 0) 11481 return (ENOMEM); 11482 lowmem = 1; 11483 } 11484 11485 /* 11486 * Allocate from the correct location based upon the size of the TSB 11487 * compared to the base page size, and what memory conditions dictate. 11488 * Note we always do nonblocking allocations from the TSB arena since 11489 * we don't want memory fragmentation to cause processes to block 11490 * indefinitely waiting for memory; until the kernel algorithms that 11491 * coalesce large pages are improved this is our best option. 11492 * 11493 * Algorithm: 11494 * If allocating a "large" TSB (>8K), allocate from the 11495 * appropriate kmem_tsb_default_arena vmem arena 11496 * else if low on memory or the TSB_FORCEALLOC flag is set or 11497 * tsb_forceheap is set 11498 * Allocate from kernel heap via sfmmu_tsb8k_cache with 11499 * KM_SLEEP (never fails) 11500 * else 11501 * Allocate from appropriate sfmmu_tsb_cache with 11502 * KM_NOSLEEP 11503 * endif 11504 */ 11505 if (tsb_lgrp_affinity) 11506 lgrpid = lgrp_home_id(curthread); 11507 if (lgrpid == LGRP_NONE) 11508 lgrpid = 0; /* use lgrp of boot CPU */ 11509 11510 if (tsbbytes > MMU_PAGESIZE) { 11511 vmp = kmem_tsb_default_arena[lgrpid]; 11512 vaddr = (caddr_t)vmem_xalloc(vmp, tsbbytes, tsbbytes, 0, 0, 11513 NULL, NULL, VM_NOSLEEP); 11514 #ifdef DEBUG 11515 } else if (lowmem || (flags & TSB_FORCEALLOC) || tsb_forceheap) { 11516 #else /* !DEBUG */ 11517 } else if (lowmem || (flags & TSB_FORCEALLOC)) { 11518 #endif /* DEBUG */ 11519 kmem_cachep = sfmmu_tsb8k_cache; 11520 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_SLEEP); 11521 ASSERT(vaddr != NULL); 11522 } else { 11523 kmem_cachep = sfmmu_tsb_cache[lgrpid]; 11524 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_NOSLEEP); 11525 } 11526 11527 tsbinfo->tsb_cache = kmem_cachep; 11528 tsbinfo->tsb_vmp = vmp; 11529 11530 if (vaddr == NULL) { 11531 return (EAGAIN); 11532 } 11533 11534 atomic_add_64(&tsb_alloc_bytes, (int64_t)tsbbytes); 11535 kmem_cachep = tsbinfo->tsb_cache; 11536 11537 /* 11538 * If we are allocating from outside the cage, then we need to 11539 * register a relocation callback handler. Note that for now 11540 * since pseudo mappings always hang off of the slab's root page, 11541 * we need only lock the first 8K of the TSB slab. This is a bit 11542 * hacky but it is good for performance. 11543 */ 11544 if (kmem_cachep != sfmmu_tsb8k_cache) { 11545 slab_vaddr = (caddr_t)((uintptr_t)vaddr & slab_mask); 11546 ret = as_pagelock(&kas, &pplist, slab_vaddr, PAGESIZE, S_WRITE); 11547 ASSERT(ret == 0); 11548 ret = hat_add_callback(sfmmu_tsb_cb_id, vaddr, (uint_t)tsbbytes, 11549 cbflags, (void *)tsbinfo, &pfn, NULL); 11550 11551 /* 11552 * Need to free up resources if we could not successfully 11553 * add the callback function and return an error condition. 11554 */ 11555 if (ret != 0) { 11556 if (kmem_cachep) { 11557 kmem_cache_free(kmem_cachep, vaddr); 11558 } else { 11559 vmem_xfree(vmp, (void *)vaddr, tsbbytes); 11560 } 11561 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, 11562 S_WRITE); 11563 return (EAGAIN); 11564 } 11565 } else { 11566 /* 11567 * Since allocation of 8K TSBs from heap is rare and occurs 11568 * during memory pressure we allocate them from permanent 11569 * memory rather than using callbacks to get the PFN. 11570 */ 11571 pfn = hat_getpfnum(kas.a_hat, vaddr); 11572 } 11573 11574 tsbinfo->tsb_va = vaddr; 11575 tsbinfo->tsb_szc = tsbcode; 11576 tsbinfo->tsb_ttesz_mask = tteszmask; 11577 tsbinfo->tsb_next = NULL; 11578 tsbinfo->tsb_flags = 0; 11579 11580 sfmmu_tsbinfo_setup_phys(tsbinfo, pfn); 11581 11582 if (kmem_cachep != sfmmu_tsb8k_cache) { 11583 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, S_WRITE); 11584 } 11585 11586 sfmmu_inv_tsb(vaddr, tsbbytes); 11587 return (0); 11588 } 11589 11590 /* 11591 * Initialize per cpu tsb and per cpu tsbmiss_area 11592 */ 11593 void 11594 sfmmu_init_tsbs(void) 11595 { 11596 int i; 11597 struct tsbmiss *tsbmissp; 11598 struct kpmtsbm *kpmtsbmp; 11599 #ifndef sun4v 11600 extern int dcache_line_mask; 11601 #endif /* sun4v */ 11602 extern uint_t vac_colors; 11603 11604 /* 11605 * Init. tsb miss area. 11606 */ 11607 tsbmissp = tsbmiss_area; 11608 11609 for (i = 0; i < NCPU; tsbmissp++, i++) { 11610 /* 11611 * initialize the tsbmiss area. 11612 * Do this for all possible CPUs as some may be added 11613 * while the system is running. There is no cost to this. 11614 */ 11615 tsbmissp->ksfmmup = ksfmmup; 11616 #ifndef sun4v 11617 tsbmissp->dcache_line_mask = (uint16_t)dcache_line_mask; 11618 #endif /* sun4v */ 11619 tsbmissp->khashstart = 11620 (struct hmehash_bucket *)va_to_pa((caddr_t)khme_hash); 11621 tsbmissp->uhashstart = 11622 (struct hmehash_bucket *)va_to_pa((caddr_t)uhme_hash); 11623 tsbmissp->khashsz = khmehash_num; 11624 tsbmissp->uhashsz = uhmehash_num; 11625 } 11626 11627 sfmmu_tsb_cb_id = hat_register_callback('T'<<16 | 'S' << 8 | 'B', 11628 sfmmu_tsb_pre_relocator, sfmmu_tsb_post_relocator, NULL, 0); 11629 11630 if (kpm_enable == 0) 11631 return; 11632 11633 /* -- Begin KPM specific init -- */ 11634 11635 if (kpm_smallpages) { 11636 /* 11637 * If we're using base pagesize pages for seg_kpm 11638 * mappings, we use the kernel TSB since we can't afford 11639 * to allocate a second huge TSB for these mappings. 11640 */ 11641 kpm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11642 kpm_tsbsz = ktsb_szcode; 11643 kpmsm_tsbbase = kpm_tsbbase; 11644 kpmsm_tsbsz = kpm_tsbsz; 11645 } else { 11646 /* 11647 * In VAC conflict case, just put the entries in the 11648 * kernel 8K indexed TSB for now so we can find them. 11649 * This could really be changed in the future if we feel 11650 * the need... 11651 */ 11652 kpmsm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base; 11653 kpmsm_tsbsz = ktsb_szcode; 11654 kpm_tsbbase = ktsb_phys? ktsb4m_pbase : (uint64_t)ktsb4m_base; 11655 kpm_tsbsz = ktsb4m_szcode; 11656 } 11657 11658 kpmtsbmp = kpmtsbm_area; 11659 for (i = 0; i < NCPU; kpmtsbmp++, i++) { 11660 /* 11661 * Initialize the kpmtsbm area. 11662 * Do this for all possible CPUs as some may be added 11663 * while the system is running. There is no cost to this. 11664 */ 11665 kpmtsbmp->vbase = kpm_vbase; 11666 kpmtsbmp->vend = kpm_vbase + kpm_size * vac_colors; 11667 kpmtsbmp->sz_shift = kpm_size_shift; 11668 kpmtsbmp->kpmp_shift = kpmp_shift; 11669 kpmtsbmp->kpmp2pshft = (uchar_t)kpmp2pshft; 11670 if (kpm_smallpages == 0) { 11671 kpmtsbmp->kpmp_table_sz = kpmp_table_sz; 11672 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_table); 11673 } else { 11674 kpmtsbmp->kpmp_table_sz = kpmp_stable_sz; 11675 kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_stable); 11676 } 11677 kpmtsbmp->msegphashpa = va_to_pa(memseg_phash); 11678 kpmtsbmp->flags = KPMTSBM_ENABLE_FLAG; 11679 #ifdef DEBUG 11680 kpmtsbmp->flags |= (kpm_tsbmtl) ? KPMTSBM_TLTSBM_FLAG : 0; 11681 #endif /* DEBUG */ 11682 if (ktsb_phys) 11683 kpmtsbmp->flags |= KPMTSBM_TSBPHYS_FLAG; 11684 } 11685 11686 /* -- End KPM specific init -- */ 11687 } 11688 11689 /* Avoid using sfmmu_tsbinfo_alloc() to avoid kmem_alloc - no real reason */ 11690 struct tsb_info ktsb_info[2]; 11691 11692 /* 11693 * Called from hat_kern_setup() to setup the tsb_info for ksfmmup. 11694 */ 11695 void 11696 sfmmu_init_ktsbinfo() 11697 { 11698 ASSERT(ksfmmup != NULL); 11699 ASSERT(ksfmmup->sfmmu_tsb == NULL); 11700 /* 11701 * Allocate tsbinfos for kernel and copy in data 11702 * to make debug easier and sun4v setup easier. 11703 */ 11704 ktsb_info[0].tsb_sfmmu = ksfmmup; 11705 ktsb_info[0].tsb_szc = ktsb_szcode; 11706 ktsb_info[0].tsb_ttesz_mask = TSB8K|TSB64K|TSB512K; 11707 ktsb_info[0].tsb_va = ktsb_base; 11708 ktsb_info[0].tsb_pa = ktsb_pbase; 11709 ktsb_info[0].tsb_flags = 0; 11710 ktsb_info[0].tsb_tte.ll = 0; 11711 ktsb_info[0].tsb_cache = NULL; 11712 11713 ktsb_info[1].tsb_sfmmu = ksfmmup; 11714 ktsb_info[1].tsb_szc = ktsb4m_szcode; 11715 ktsb_info[1].tsb_ttesz_mask = TSB4M; 11716 ktsb_info[1].tsb_va = ktsb4m_base; 11717 ktsb_info[1].tsb_pa = ktsb4m_pbase; 11718 ktsb_info[1].tsb_flags = 0; 11719 ktsb_info[1].tsb_tte.ll = 0; 11720 ktsb_info[1].tsb_cache = NULL; 11721 11722 /* Link them into ksfmmup. */ 11723 ktsb_info[0].tsb_next = &ktsb_info[1]; 11724 ktsb_info[1].tsb_next = NULL; 11725 ksfmmup->sfmmu_tsb = &ktsb_info[0]; 11726 11727 sfmmu_setup_tsbinfo(ksfmmup); 11728 } 11729 11730 /* 11731 * Cache the last value returned from va_to_pa(). If the VA specified 11732 * in the current call to cached_va_to_pa() maps to the same Page (as the 11733 * previous call to cached_va_to_pa()), then compute the PA using 11734 * cached info, else call va_to_pa(). 11735 * 11736 * Note: this function is neither MT-safe nor consistent in the presence 11737 * of multiple, interleaved threads. This function was created to enable 11738 * an optimization used during boot (at a point when there's only one thread 11739 * executing on the "boot CPU", and before startup_vm() has been called). 11740 */ 11741 static uint64_t 11742 cached_va_to_pa(void *vaddr) 11743 { 11744 static uint64_t prev_vaddr_base = 0; 11745 static uint64_t prev_pfn = 0; 11746 11747 if ((((uint64_t)vaddr) & MMU_PAGEMASK) == prev_vaddr_base) { 11748 return (prev_pfn | ((uint64_t)vaddr & MMU_PAGEOFFSET)); 11749 } else { 11750 uint64_t pa = va_to_pa(vaddr); 11751 11752 if (pa != ((uint64_t)-1)) { 11753 /* 11754 * Computed physical address is valid. Cache its 11755 * related info for the next cached_va_to_pa() call. 11756 */ 11757 prev_pfn = pa & MMU_PAGEMASK; 11758 prev_vaddr_base = ((uint64_t)vaddr) & MMU_PAGEMASK; 11759 } 11760 11761 return (pa); 11762 } 11763 } 11764 11765 /* 11766 * Carve up our nucleus hblk region. We may allocate more hblks than 11767 * asked due to rounding errors but we are guaranteed to have at least 11768 * enough space to allocate the requested number of hblk8's and hblk1's. 11769 */ 11770 void 11771 sfmmu_init_nucleus_hblks(caddr_t addr, size_t size, int nhblk8, int nhblk1) 11772 { 11773 struct hme_blk *hmeblkp; 11774 size_t hme8blk_sz, hme1blk_sz; 11775 size_t i; 11776 size_t hblk8_bound; 11777 ulong_t j = 0, k = 0; 11778 11779 ASSERT(addr != NULL && size != 0); 11780 11781 /* Need to use proper structure alignment */ 11782 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 11783 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 11784 11785 nucleus_hblk8.list = (void *)addr; 11786 nucleus_hblk8.index = 0; 11787 11788 /* 11789 * Use as much memory as possible for hblk8's since we 11790 * expect all bop_alloc'ed memory to be allocated in 8k chunks. 11791 * We need to hold back enough space for the hblk1's which 11792 * we'll allocate next. 11793 */ 11794 hblk8_bound = size - (nhblk1 * hme1blk_sz) - hme8blk_sz; 11795 for (i = 0; i <= hblk8_bound; i += hme8blk_sz, j++) { 11796 hmeblkp = (struct hme_blk *)addr; 11797 addr += hme8blk_sz; 11798 hmeblkp->hblk_nuc_bit = 1; 11799 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11800 } 11801 nucleus_hblk8.len = j; 11802 ASSERT(j >= nhblk8); 11803 SFMMU_STAT_ADD(sf_hblk8_ncreate, j); 11804 11805 nucleus_hblk1.list = (void *)addr; 11806 nucleus_hblk1.index = 0; 11807 for (; i <= (size - hme1blk_sz); i += hme1blk_sz, k++) { 11808 hmeblkp = (struct hme_blk *)addr; 11809 addr += hme1blk_sz; 11810 hmeblkp->hblk_nuc_bit = 1; 11811 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); 11812 } 11813 ASSERT(k >= nhblk1); 11814 nucleus_hblk1.len = k; 11815 SFMMU_STAT_ADD(sf_hblk1_ncreate, k); 11816 } 11817 11818 /* 11819 * This function is currently not supported on this platform. For what 11820 * it's supposed to do, see hat.c and hat_srmmu.c 11821 */ 11822 /* ARGSUSED */ 11823 faultcode_t 11824 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp, 11825 uint_t flags) 11826 { 11827 ASSERT(hat->sfmmu_xhat_provider == NULL); 11828 return (FC_NOSUPPORT); 11829 } 11830 11831 /* 11832 * Searchs the mapping list of the page for a mapping of the same size. If not 11833 * found the corresponding bit is cleared in the p_index field. When large 11834 * pages are more prevalent in the system, we can maintain the mapping list 11835 * in order and we don't have to traverse the list each time. Just check the 11836 * next and prev entries, and if both are of different size, we clear the bit. 11837 */ 11838 static void 11839 sfmmu_rm_large_mappings(page_t *pp, int ttesz) 11840 { 11841 struct sf_hment *sfhmep; 11842 struct hme_blk *hmeblkp; 11843 int index; 11844 pgcnt_t npgs; 11845 11846 ASSERT(ttesz > TTE8K); 11847 11848 ASSERT(sfmmu_mlist_held(pp)); 11849 11850 ASSERT(PP_ISMAPPED_LARGE(pp)); 11851 11852 /* 11853 * Traverse mapping list looking for another mapping of same size. 11854 * since we only want to clear index field if all mappings of 11855 * that size are gone. 11856 */ 11857 11858 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) { 11859 hmeblkp = sfmmu_hmetohblk(sfhmep); 11860 if (hmeblkp->hblk_xhat_bit) 11861 continue; 11862 if (hme_size(sfhmep) == ttesz) { 11863 /* 11864 * another mapping of the same size. don't clear index. 11865 */ 11866 return; 11867 } 11868 } 11869 11870 /* 11871 * Clear the p_index bit for large page. 11872 */ 11873 index = PAGESZ_TO_INDEX(ttesz); 11874 npgs = TTEPAGES(ttesz); 11875 while (npgs-- > 0) { 11876 ASSERT(pp->p_index & index); 11877 pp->p_index &= ~index; 11878 pp = PP_PAGENEXT(pp); 11879 } 11880 } 11881 11882 /* 11883 * return supported features 11884 */ 11885 /* ARGSUSED */ 11886 int 11887 hat_supported(enum hat_features feature, void *arg) 11888 { 11889 switch (feature) { 11890 case HAT_SHARED_PT: 11891 case HAT_DYNAMIC_ISM_UNMAP: 11892 case HAT_VMODSORT: 11893 return (1); 11894 default: 11895 return (0); 11896 } 11897 } 11898 11899 void 11900 hat_enter(struct hat *hat) 11901 { 11902 hatlock_t *hatlockp; 11903 11904 if (hat != ksfmmup) { 11905 hatlockp = TSB_HASH(hat); 11906 mutex_enter(HATLOCK_MUTEXP(hatlockp)); 11907 } 11908 } 11909 11910 void 11911 hat_exit(struct hat *hat) 11912 { 11913 hatlock_t *hatlockp; 11914 11915 if (hat != ksfmmup) { 11916 hatlockp = TSB_HASH(hat); 11917 mutex_exit(HATLOCK_MUTEXP(hatlockp)); 11918 } 11919 } 11920 11921 /*ARGSUSED*/ 11922 void 11923 hat_reserve(struct as *as, caddr_t addr, size_t len) 11924 { 11925 } 11926 11927 static void 11928 hat_kstat_init(void) 11929 { 11930 kstat_t *ksp; 11931 11932 ksp = kstat_create("unix", 0, "sfmmu_global_stat", "hat", 11933 KSTAT_TYPE_RAW, sizeof (struct sfmmu_global_stat), 11934 KSTAT_FLAG_VIRTUAL); 11935 if (ksp) { 11936 ksp->ks_data = (void *) &sfmmu_global_stat; 11937 kstat_install(ksp); 11938 } 11939 ksp = kstat_create("unix", 0, "sfmmu_tsbsize_stat", "hat", 11940 KSTAT_TYPE_RAW, sizeof (struct sfmmu_tsbsize_stat), 11941 KSTAT_FLAG_VIRTUAL); 11942 if (ksp) { 11943 ksp->ks_data = (void *) &sfmmu_tsbsize_stat; 11944 kstat_install(ksp); 11945 } 11946 ksp = kstat_create("unix", 0, "sfmmu_percpu_stat", "hat", 11947 KSTAT_TYPE_RAW, sizeof (struct sfmmu_percpu_stat) * NCPU, 11948 KSTAT_FLAG_WRITABLE); 11949 if (ksp) { 11950 ksp->ks_update = sfmmu_kstat_percpu_update; 11951 kstat_install(ksp); 11952 } 11953 } 11954 11955 /* ARGSUSED */ 11956 static int 11957 sfmmu_kstat_percpu_update(kstat_t *ksp, int rw) 11958 { 11959 struct sfmmu_percpu_stat *cpu_kstat = ksp->ks_data; 11960 struct tsbmiss *tsbm = tsbmiss_area; 11961 struct kpmtsbm *kpmtsbm = kpmtsbm_area; 11962 int i; 11963 11964 ASSERT(cpu_kstat); 11965 if (rw == KSTAT_READ) { 11966 for (i = 0; i < NCPU; cpu_kstat++, tsbm++, kpmtsbm++, i++) { 11967 cpu_kstat->sf_itlb_misses = tsbm->itlb_misses; 11968 cpu_kstat->sf_dtlb_misses = tsbm->dtlb_misses; 11969 cpu_kstat->sf_utsb_misses = tsbm->utsb_misses - 11970 tsbm->uprot_traps; 11971 cpu_kstat->sf_ktsb_misses = tsbm->ktsb_misses + 11972 kpmtsbm->kpm_tsb_misses - tsbm->kprot_traps; 11973 11974 if (tsbm->itlb_misses > 0 && tsbm->dtlb_misses > 0) { 11975 cpu_kstat->sf_tsb_hits = 11976 (tsbm->itlb_misses + tsbm->dtlb_misses) - 11977 (tsbm->utsb_misses + tsbm->ktsb_misses + 11978 kpmtsbm->kpm_tsb_misses); 11979 } else { 11980 cpu_kstat->sf_tsb_hits = 0; 11981 } 11982 cpu_kstat->sf_umod_faults = tsbm->uprot_traps; 11983 cpu_kstat->sf_kmod_faults = tsbm->kprot_traps; 11984 } 11985 } else { 11986 /* KSTAT_WRITE is used to clear stats */ 11987 for (i = 0; i < NCPU; tsbm++, kpmtsbm++, i++) { 11988 tsbm->itlb_misses = 0; 11989 tsbm->dtlb_misses = 0; 11990 tsbm->utsb_misses = 0; 11991 tsbm->ktsb_misses = 0; 11992 tsbm->uprot_traps = 0; 11993 tsbm->kprot_traps = 0; 11994 kpmtsbm->kpm_dtlb_misses = 0; 11995 kpmtsbm->kpm_tsb_misses = 0; 11996 } 11997 } 11998 return (0); 11999 } 12000 12001 #ifdef DEBUG 12002 12003 tte_t *gorig[NCPU], *gcur[NCPU], *gnew[NCPU]; 12004 12005 /* 12006 * A tte checker. *orig_old is the value we read before cas. 12007 * *cur is the value returned by cas. 12008 * *new is the desired value when we do the cas. 12009 * 12010 * *hmeblkp is currently unused. 12011 */ 12012 12013 /* ARGSUSED */ 12014 void 12015 chk_tte(tte_t *orig_old, tte_t *cur, tte_t *new, struct hme_blk *hmeblkp) 12016 { 12017 pfn_t i, j, k; 12018 int cpuid = CPU->cpu_id; 12019 12020 gorig[cpuid] = orig_old; 12021 gcur[cpuid] = cur; 12022 gnew[cpuid] = new; 12023 12024 #ifdef lint 12025 hmeblkp = hmeblkp; 12026 #endif 12027 12028 if (TTE_IS_VALID(orig_old)) { 12029 if (TTE_IS_VALID(cur)) { 12030 i = TTE_TO_TTEPFN(orig_old); 12031 j = TTE_TO_TTEPFN(cur); 12032 k = TTE_TO_TTEPFN(new); 12033 if (i != j) { 12034 /* remap error? */ 12035 panic("chk_tte: bad pfn, 0x%lx, 0x%lx", i, j); 12036 } 12037 12038 if (i != k) { 12039 /* remap error? */ 12040 panic("chk_tte: bad pfn2, 0x%lx, 0x%lx", i, k); 12041 } 12042 } else { 12043 if (TTE_IS_VALID(new)) { 12044 panic("chk_tte: invalid cur? "); 12045 } 12046 12047 i = TTE_TO_TTEPFN(orig_old); 12048 k = TTE_TO_TTEPFN(new); 12049 if (i != k) { 12050 panic("chk_tte: bad pfn3, 0x%lx, 0x%lx", i, k); 12051 } 12052 } 12053 } else { 12054 if (TTE_IS_VALID(cur)) { 12055 j = TTE_TO_TTEPFN(cur); 12056 if (TTE_IS_VALID(new)) { 12057 k = TTE_TO_TTEPFN(new); 12058 if (j != k) { 12059 panic("chk_tte: bad pfn4, 0x%lx, 0x%lx", 12060 j, k); 12061 } 12062 } else { 12063 panic("chk_tte: why here?"); 12064 } 12065 } else { 12066 if (!TTE_IS_VALID(new)) { 12067 panic("chk_tte: why here2 ?"); 12068 } 12069 } 12070 } 12071 } 12072 12073 #endif /* DEBUG */ 12074 12075 extern void prefetch_tsbe_read(struct tsbe *); 12076 extern void prefetch_tsbe_write(struct tsbe *); 12077 12078 12079 /* 12080 * We want to prefetch 7 cache lines ahead for our read prefetch. This gives 12081 * us optimal performance on Cheetah+. You can only have 8 outstanding 12082 * prefetches at any one time, so we opted for 7 read prefetches and 1 write 12083 * prefetch to make the most utilization of the prefetch capability. 12084 */ 12085 #define TSBE_PREFETCH_STRIDE (7) 12086 12087 void 12088 sfmmu_copy_tsb(struct tsb_info *old_tsbinfo, struct tsb_info *new_tsbinfo) 12089 { 12090 int old_bytes = TSB_BYTES(old_tsbinfo->tsb_szc); 12091 int new_bytes = TSB_BYTES(new_tsbinfo->tsb_szc); 12092 int old_entries = TSB_ENTRIES(old_tsbinfo->tsb_szc); 12093 int new_entries = TSB_ENTRIES(new_tsbinfo->tsb_szc); 12094 struct tsbe *old; 12095 struct tsbe *new; 12096 struct tsbe *new_base = (struct tsbe *)new_tsbinfo->tsb_va; 12097 uint64_t va; 12098 int new_offset; 12099 int i; 12100 int vpshift; 12101 int last_prefetch; 12102 12103 if (old_bytes == new_bytes) { 12104 bcopy(old_tsbinfo->tsb_va, new_tsbinfo->tsb_va, new_bytes); 12105 } else { 12106 12107 /* 12108 * A TSBE is 16 bytes which means there are four TSBE's per 12109 * P$ line (64 bytes), thus every 4 TSBE's we prefetch. 12110 */ 12111 old = (struct tsbe *)old_tsbinfo->tsb_va; 12112 last_prefetch = old_entries - (4*(TSBE_PREFETCH_STRIDE+1)); 12113 for (i = 0; i < old_entries; i++, old++) { 12114 if (((i & (4-1)) == 0) && (i < last_prefetch)) 12115 prefetch_tsbe_read(old); 12116 if (!old->tte_tag.tag_invalid) { 12117 /* 12118 * We have a valid TTE to remap. Check the 12119 * size. We won't remap 64K or 512K TTEs 12120 * because they span more than one TSB entry 12121 * and are indexed using an 8K virt. page. 12122 * Ditto for 32M and 256M TTEs. 12123 */ 12124 if (TTE_CSZ(&old->tte_data) == TTE64K || 12125 TTE_CSZ(&old->tte_data) == TTE512K) 12126 continue; 12127 if (mmu_page_sizes == max_mmu_page_sizes) { 12128 if (TTE_CSZ(&old->tte_data) == TTE32M || 12129 TTE_CSZ(&old->tte_data) == TTE256M) 12130 continue; 12131 } 12132 12133 /* clear the lower 22 bits of the va */ 12134 va = *(uint64_t *)old << 22; 12135 /* turn va into a virtual pfn */ 12136 va >>= 22 - TSB_START_SIZE; 12137 /* 12138 * or in bits from the offset in the tsb 12139 * to get the real virtual pfn. These 12140 * correspond to bits [21:13] in the va 12141 */ 12142 vpshift = 12143 TTE_BSZS_SHIFT(TTE_CSZ(&old->tte_data)) & 12144 0x1ff; 12145 va |= (i << vpshift); 12146 va >>= vpshift; 12147 new_offset = va & (new_entries - 1); 12148 new = new_base + new_offset; 12149 prefetch_tsbe_write(new); 12150 *new = *old; 12151 } 12152 } 12153 } 12154 } 12155 12156 /* 12157 * unused in sfmmu 12158 */ 12159 void 12160 hat_dump(void) 12161 { 12162 } 12163 12164 /* 12165 * Called when a thread is exiting and we have switched to the kernel address 12166 * space. Perform the same VM initialization resume() uses when switching 12167 * processes. 12168 * 12169 * Note that sfmmu_load_mmustate() is currently a no-op for kernel threads, but 12170 * we call it anyway in case the semantics change in the future. 12171 */ 12172 /*ARGSUSED*/ 12173 void 12174 hat_thread_exit(kthread_t *thd) 12175 { 12176 uint64_t pgsz_cnum; 12177 uint_t pstate_save; 12178 12179 ASSERT(thd->t_procp->p_as == &kas); 12180 12181 pgsz_cnum = KCONTEXT; 12182 #ifdef sun4u 12183 pgsz_cnum |= (ksfmmup->sfmmu_cext << CTXREG_EXT_SHIFT); 12184 #endif 12185 /* 12186 * Note that sfmmu_load_mmustate() is currently a no-op for 12187 * kernel threads. We need to disable interrupts here, 12188 * simply because otherwise sfmmu_load_mmustate() would panic 12189 * if the caller does not disable interrupts. 12190 */ 12191 pstate_save = sfmmu_disable_intrs(); 12192 sfmmu_setctx_sec(pgsz_cnum); 12193 sfmmu_load_mmustate(ksfmmup); 12194 sfmmu_enable_intrs(pstate_save); 12195 } 12196