1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * VM - Hardware Address Translation management. 29 * 30 * This file describes the contents of the sun-reference-mmu(sfmmu)- 31 * specific hat data structures and the sfmmu-specific hat procedures. 32 * The machine-independent interface is described in <vm/hat.h>. 33 */ 34 35 #ifndef _VM_HAT_SFMMU_H 36 #define _VM_HAT_SFMMU_H 37 38 #pragma ident "%Z%%M% %I% %E% SMI" 39 40 #ifdef __cplusplus 41 extern "C" { 42 #endif 43 44 #ifndef _ASM 45 46 #include <sys/types.h> 47 48 #endif /* _ASM */ 49 50 #ifdef _KERNEL 51 52 #include <sys/pte.h> 53 #include <vm/mach_sfmmu.h> 54 #include <sys/mmu.h> 55 56 /* 57 * Don't alter these without considering changes to ism_map_t. 58 */ 59 #define DEFAULT_ISM_PAGESIZE MMU_PAGESIZE4M 60 #define ISM_PG_SIZE(ism_vbshift) (1 << ism_vbshift) 61 #define ISM_SZ_MASK(ism_vbshift) (ISM_PG_SIZE(ism_vbshift) - 1) 62 #define ISM_MAP_SLOTS 8 /* Change this carefully. */ 63 64 #ifndef _ASM 65 66 #include <sys/t_lock.h> 67 #include <vm/hat.h> 68 #include <vm/seg.h> 69 #include <sys/machparam.h> 70 #include <sys/systm.h> 71 #include <sys/x_call.h> 72 #include <vm/page.h> 73 #include <sys/ksynch.h> 74 75 typedef struct hat sfmmu_t; 76 77 /* 78 * SFMMU attributes for hat_memload/hat_devload 79 */ 80 #define SFMMU_UNCACHEPTTE 0x01000000 /* unencache in physical $ */ 81 #define SFMMU_UNCACHEVTTE 0x02000000 /* unencache in virtual $ */ 82 #define SFMMU_SIDEFFECT 0x04000000 /* set side effect bit */ 83 #define SFMMU_LOAD_ALLATTR (HAT_PROT_MASK | HAT_ORDER_MASK | \ 84 HAT_ENDIAN_MASK | HAT_NOFAULT | HAT_NOSYNC | \ 85 SFMMU_UNCACHEPTTE | SFMMU_UNCACHEVTTE | SFMMU_SIDEFFECT) 86 87 88 /* 89 * sfmmu flags for hat_memload/hat_devload 90 */ 91 #define SFMMU_NO_TSBLOAD 0x08000000 /* do not preload tsb */ 92 #define SFMMU_LOAD_ALLFLAG (HAT_LOAD | HAT_LOAD_LOCK | \ 93 HAT_LOAD_ADV | HAT_LOAD_CONTIG | HAT_LOAD_NOCONSIST | \ 94 HAT_LOAD_SHARE | HAT_LOAD_REMAP | SFMMU_NO_TSBLOAD | \ 95 HAT_RELOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_TEXT) 96 97 /* 98 * sfmmu internal flag to hat_pageunload that spares locked mappings 99 */ 100 #define SFMMU_KERNEL_RELOC 0x8000 101 102 /* 103 * mode for sfmmu_chgattr 104 */ 105 #define SFMMU_SETATTR 0x0 106 #define SFMMU_CLRATTR 0x1 107 #define SFMMU_CHGATTR 0x2 108 109 /* 110 * sfmmu specific flags for page_t 111 */ 112 #define P_PNC 0x8 /* non-caching is permanent bit */ 113 #define P_TNC 0x10 /* non-caching is temporary bit */ 114 #define P_KPMS 0x20 /* kpm mapped small (vac alias prevention) */ 115 #define P_KPMC 0x40 /* kpm conflict page (vac alias prevention) */ 116 117 #define PP_GENERIC_ATTR(pp) ((pp)->p_nrm & (P_MOD | P_REF | P_RO)) 118 #define PP_ISMOD(pp) ((pp)->p_nrm & P_MOD) 119 #define PP_ISREF(pp) ((pp)->p_nrm & P_REF) 120 #define PP_ISRO(pp) ((pp)->p_nrm & P_RO) 121 #define PP_ISNC(pp) ((pp)->p_nrm & (P_PNC|P_TNC)) 122 #define PP_ISPNC(pp) ((pp)->p_nrm & P_PNC) 123 #define PP_ISTNC(pp) ((pp)->p_nrm & P_TNC) 124 #define PP_ISKPMS(pp) ((pp)->p_nrm & P_KPMS) 125 #define PP_ISKPMC(pp) ((pp)->p_nrm & P_KPMC) 126 127 #define PP_SETMOD(pp) ((pp)->p_nrm |= P_MOD) 128 #define PP_SETREF(pp) ((pp)->p_nrm |= P_REF) 129 #define PP_SETREFMOD(pp) ((pp)->p_nrm |= (P_REF|P_MOD)) 130 #define PP_SETRO(pp) ((pp)->p_nrm |= P_RO) 131 #define PP_SETREFRO(pp) ((pp)->p_nrm |= (P_REF|P_RO)) 132 #define PP_SETPNC(pp) ((pp)->p_nrm |= P_PNC) 133 #define PP_SETTNC(pp) ((pp)->p_nrm |= P_TNC) 134 #define PP_SETKPMS(pp) ((pp)->p_nrm |= P_KPMS) 135 #define PP_SETKPMC(pp) ((pp)->p_nrm |= P_KPMC) 136 137 #define PP_CLRMOD(pp) ((pp)->p_nrm &= ~P_MOD) 138 #define PP_CLRREF(pp) ((pp)->p_nrm &= ~P_REF) 139 #define PP_CLRREFMOD(pp) ((pp)->p_nrm &= ~(P_REF|P_MOD)) 140 #define PP_CLRRO(pp) ((pp)->p_nrm &= ~P_RO) 141 #define PP_CLRPNC(pp) ((pp)->p_nrm &= ~P_PNC) 142 #define PP_CLRTNC(pp) ((pp)->p_nrm &= ~P_TNC) 143 #define PP_CLRKPMS(pp) ((pp)->p_nrm &= ~P_KPMS) 144 #define PP_CLRKPMC(pp) ((pp)->p_nrm &= ~P_KPMC) 145 146 /* 147 * All shared memory segments attached with the SHM_SHARE_MMU flag (ISM) 148 * will be constrained to a 4M, 32M or 256M alignment. Also since every newly- 149 * created ISM segment is created out of a new address space at base va 150 * of 0 we don't need to store it. 151 */ 152 #define ISM_ALIGN(shift) (1 << shift) /* base va aligned to <n>M */ 153 #define ISM_ALIGNED(shift, va) (((uintptr_t)va & (ISM_ALIGN(shift) - 1)) == 0) 154 #define ISM_SHIFT(shift, x) ((uintptr_t)x >> (shift)) 155 156 /* 157 * Pad locks out to cache sub-block boundaries to prevent 158 * false sharing, so several processes don't contend for 159 * the same line if they aren't using the same lock. Since 160 * this is a typedef we also have a bit of freedom in 161 * changing lock implementations later if we decide it 162 * is necessary. 163 */ 164 typedef struct hat_lock { 165 kmutex_t hl_mutex; 166 uchar_t hl_pad[64 - sizeof (kmutex_t)]; 167 } hatlock_t; 168 169 #define HATLOCK_MUTEXP(hatlockp) (&((hatlockp)->hl_mutex)) 170 171 /* 172 * All segments mapped with ISM are guaranteed to be 4M, 32M or 256M aligned. 173 * Also size is guaranteed to be in 4M, 32M or 256M chunks. 174 * ism_seg consists of the following members: 175 * [XX..22] base address of ism segment. XX is 63 or 31 depending whether 176 * caddr_t is 64 bits or 32 bits. 177 * [21..0] size of segment. 178 * 179 * NOTE: Don't alter this structure without changing defines above and 180 * the tsb_miss and protection handlers. 181 */ 182 typedef struct ism_map { 183 uintptr_t imap_seg; /* base va + sz of ISM segment */ 184 ushort_t imap_vb_shift; /* mmu_pageshift for ism page size */ 185 ushort_t imap_hatflags; /* primary ism page size */ 186 uint_t imap_sz_mask; /* mmu_pagemask for ism page size */ 187 sfmmu_t *imap_ismhat; /* hat id of dummy ISM as */ 188 struct ism_ment *imap_ment; /* pointer to mapping list entry */ 189 } ism_map_t; 190 191 #define ism_start(map) ((caddr_t)((map).imap_seg & \ 192 ~ISM_SZ_MASK((map).imap_vb_shift))) 193 #define ism_size(map) ((map).imap_seg & ISM_SZ_MASK((map).imap_vb_shift)) 194 #define ism_end(map) ((caddr_t)(ism_start(map) + (ism_size(map) * \ 195 ISM_PG_SIZE((map).imap_vb_shift)))) 196 /* 197 * ISM mapping entry. Used to link all hat's sharing a ism_hat. 198 * Same function as the p_mapping list for a page. 199 */ 200 typedef struct ism_ment { 201 sfmmu_t *iment_hat; /* back pointer to hat_share() hat */ 202 caddr_t iment_base_va; /* hat's va base for this ism seg */ 203 struct ism_ment *iment_next; /* next ism map entry */ 204 struct ism_ment *iment_prev; /* prev ism map entry */ 205 } ism_ment_t; 206 207 /* 208 * ISM segment block. One will be hung off the sfmmu structure if a 209 * a process uses ISM. More will be linked using ismblk_next if more 210 * than ISM_MAP_SLOTS segments are attached to this proc. 211 * 212 * All modifications to fields in this structure will be protected 213 * by the hat mutex. In order to avoid grabbing this lock in low level 214 * routines (tsb miss/protection handlers and vatopfn) while not 215 * introducing any race conditions with hat_unshare, we will set 216 * CTX_ISM_BUSY bit in the ctx struct. Any mmu traps that occur 217 * for this ctx while this bit is set will be handled in sfmmu_tsb_excption 218 * where it will synchronize behind the hat mutex. 219 */ 220 typedef struct ism_blk { 221 ism_map_t iblk_maps[ISM_MAP_SLOTS]; 222 struct ism_blk *iblk_next; 223 uint64_t iblk_nextpa; 224 } ism_blk_t; 225 226 /* 227 * TSB access information. All fields are protected by the process's 228 * hat lock. 229 */ 230 231 struct tsb_info { 232 caddr_t tsb_va; /* tsb base virtual address */ 233 uint64_t tsb_pa; /* tsb base physical address */ 234 struct tsb_info *tsb_next; /* next tsb used by this process */ 235 uint16_t tsb_szc; /* tsb size code */ 236 uint16_t tsb_flags; /* flags for this tsb; see below */ 237 uint_t tsb_ttesz_mask; /* page size masks; see below */ 238 239 tte_t tsb_tte; /* tte to lock into DTLB */ 240 sfmmu_t *tsb_sfmmu; /* sfmmu */ 241 kmem_cache_t *tsb_cache; /* cache from which mem allocated */ 242 vmem_t *tsb_vmp; /* vmem arena from which mem alloc'd */ 243 }; 244 245 /* 246 * Values for "tsb_ttesz_mask" bitmask. 247 */ 248 #define TSB8K (1 << TTE8K) 249 #define TSB64K (1 << TTE64K) 250 #define TSB512K (1 << TTE512K) 251 #define TSB4M (1 << TTE4M) 252 #define TSB32M (1 << TTE32M) 253 #define TSB256M (1 << TTE256M) 254 255 /* 256 * Values for "tsb_flags" field. 257 */ 258 #define TSB_RELOC_FLAG 0x1 259 #define TSB_FLUSH_NEEDED 0x2 260 #define TSB_SWAPPED 0x4 261 262 /* 263 * The platform dependent hat structure. 264 * tte counts should be protected by cas. 265 * cpuset is protected by cas. 266 * 267 * Note that sfmmu_xhat_provider MUST be the first element. 268 */ 269 struct hat { 270 void *sfmmu_xhat_provider; /* NULL for CPU hat */ 271 cpuset_t sfmmu_cpusran; /* cpu bit mask for efficient xcalls */ 272 struct as *sfmmu_as; /* as this hat provides mapping for */ 273 ulong_t sfmmu_ttecnt[MMU_PAGE_SIZES]; /* per sz tte counts */ 274 ulong_t sfmmu_ismttecnt[MMU_PAGE_SIZES]; /* est. ism ttes */ 275 union _h_un { 276 ism_blk_t *sfmmu_iblkp; /* maps to ismhat(s) */ 277 ism_ment_t *sfmmu_imentp; /* ism hat's mapping list */ 278 } h_un; 279 uint_t sfmmu_free:1; /* hat to be freed - set on as_free */ 280 uint_t sfmmu_ismhat:1; /* hat is dummy ism hatid */ 281 uint_t sfmmu_ctxflushed:1; /* ctx has been flushed */ 282 uchar_t sfmmu_rmstat; /* refmod stats refcnt */ 283 uchar_t sfmmu_clrstart; /* start color bin for page coloring */ 284 ushort_t sfmmu_clrbin; /* per as phys page coloring bin */ 285 short sfmmu_cnum; /* context number */ 286 ushort_t sfmmu_flags; /* flags */ 287 struct tsb_info *sfmmu_tsb; /* list of per as tsbs */ 288 uint64_t sfmmu_ismblkpa; /* pa of sfmmu_iblkp, or -1 */ 289 kcondvar_t sfmmu_tsb_cv; /* signals TSB swapin or relocation */ 290 uchar_t sfmmu_cext; /* context page size encoding */ 291 uint8_t sfmmu_pgsz[MMU_PAGE_SIZES]; /* ranking for MMU */ 292 #ifdef sun4v 293 struct hv_tsb_block sfmmu_hvblock; 294 #endif 295 }; 296 297 #define sfmmu_iblk h_un.sfmmu_iblkp 298 #define sfmmu_iment h_un.sfmmu_imentp 299 300 /* 301 * bit mask for managing vac conflicts on large pages. 302 * bit 1 is for uncache flag. 303 * bits 2 through min(num of cache colors + 1,31) are 304 * for cache colors that have already been flushed. 305 */ 306 #define CACHE_UNCACHE 1 307 #define CACHE_NUM_COLOR (shm_alignment >> MMU_PAGESHIFT) 308 309 #define CACHE_VCOLOR_MASK(vcolor) (2 << (vcolor & (CACHE_NUM_COLOR - 1))) 310 311 #define CacheColor_IsFlushed(flag, vcolor) \ 312 ((flag) & CACHE_VCOLOR_MASK(vcolor)) 313 314 #define CacheColor_SetFlushed(flag, vcolor) \ 315 ((flag) |= CACHE_VCOLOR_MASK(vcolor)) 316 /* 317 * Flags passed to sfmmu_page_cache to flush page from vac or not. 318 */ 319 #define CACHE_FLUSH 0 320 #define CACHE_NO_FLUSH 1 321 322 /* 323 * Flags passed to sfmmu_tlbcache_demap 324 */ 325 #define FLUSH_NECESSARY_CPUS 0 326 #define FLUSH_ALL_CPUS 1 327 328 /* 329 * Software context structure. The size of this structure is currently 330 * hardwired into the tsb miss handlers in assembly code through the 331 * CTX_SZ_SHIFT define. Since this define is used in a shift we should keep 332 * this structure a power of two. 333 * 334 * ctx_flags: 335 * Bit 0 : Free flag. 336 */ 337 struct ctx { 338 union _ctx_un { 339 sfmmu_t *ctx_sfmmup; /* back pointer to hat id */ 340 struct ctx *ctx_freep; /* next ctx in freelist */ 341 } ctx_un; 342 krwlock_t ctx_rwlock; /* protect context from stealer */ 343 uint32_t ctx_flags; /* flags */ 344 uint8_t pad[12]; 345 }; 346 347 #define ctx_sfmmu ctx_un.ctx_sfmmup 348 #define ctx_free ctx_un.ctx_freep 349 350 #ifdef DEBUG 351 /* 352 * For debugging purpose only. Maybe removed later. 353 */ 354 struct ctx_trace { 355 sfmmu_t *sc_sfmmu_stolen; 356 sfmmu_t *sc_sfmmu_stealing; 357 clock_t sc_time; 358 ushort_t sc_type; 359 ushort_t sc_cnum; 360 }; 361 #define CTX_TRC_STEAL 0x1 362 #define CTX_TRC_FREE 0x0 363 #define TRSIZE 0x400 364 #define NEXT_CTXTR(ptr) (((ptr) >= ctx_trace_last) ? \ 365 ctx_trace_first : ((ptr) + 1)) 366 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) \ 367 mutex_enter(mutex); \ 368 (ptr)->sc_sfmmu_stolen = (stolen_sfmmu); \ 369 (ptr)->sc_sfmmu_stealing = (stealing_sfmmu); \ 370 (ptr)->sc_cnum = (cnum); \ 371 (ptr)->sc_type = (type); \ 372 (ptr)->sc_time = lbolt; \ 373 (ptr) = NEXT_CTXTR(ptr); \ 374 num_ctx_stolen += (type); \ 375 mutex_exit(mutex); 376 #else 377 378 #define TRACE_CTXS(mutex, ptr, cnum, stolen_sfmmu, stealing_sfmmu, type) 379 380 #endif /* DEBUG */ 381 382 #endif /* !_ASM */ 383 384 /* 385 * Macros for sfmmup->sfmmu_flags access. The macros that change the flags 386 * ASSERT() that we're holding the HAT lock before changing the flags; 387 * however callers that read the flags may do so without acquiring the lock 388 * in a fast path, and then recheck the flag after acquiring the lock in 389 * a slow path. 390 */ 391 #define SFMMU_FLAGS_ISSET(sfmmup, flags) \ 392 (((sfmmup)->sfmmu_flags & (flags)) == (flags)) 393 394 #define SFMMU_FLAGS_CLEAR(sfmmup, flags) \ 395 (ASSERT(sfmmu_hat_lock_held((sfmmup))), \ 396 (sfmmup)->sfmmu_flags &= ~(flags)) 397 398 #define SFMMU_FLAGS_SET(sfmmup, flags) \ 399 (ASSERT(sfmmu_hat_lock_held((sfmmup))), \ 400 (sfmmup)->sfmmu_flags |= (flags)) 401 402 /* 403 * sfmmu HAT flags 404 */ 405 #define HAT_64K_FLAG 0x01 406 #define HAT_512K_FLAG 0x02 407 #define HAT_4M_FLAG 0x04 408 #define HAT_32M_FLAG 0x08 409 #define HAT_256M_FLAG 0x10 410 #define HAT_4MTEXT_FLAG 0x80 411 #define HAT_SWAPPED 0x100 /* swapped out */ 412 #define HAT_SWAPIN 0x200 /* swapping in */ 413 #define HAT_BUSY 0x400 /* replacing TSB(s) */ 414 #define HAT_ISMBUSY 0x800 /* adding/removing/traversing ISM maps */ 415 416 #define HAT_LGPG_FLAGS \ 417 (HAT_64K_FLAG | HAT_512K_FLAG | HAT_4M_FLAG | \ 418 HAT_32M_FLAG | HAT_256M_FLAG) 419 420 #define HAT_FLAGS_MASK \ 421 (HAT_LGPG_FLAGS | HAT_4MTEXT_FLAG | HAT_SWAPPED | \ 422 HAT_SWAPIN | HAT_BUSY | HAT_ISMBUSY) 423 424 /* 425 * Context flags 426 */ 427 #define CTX_FREE_FLAG 0x1 428 #define CTX_FLAGS_MASK 0x1 429 430 #define CTX_SET_FLAGS(ctx, flag) \ 431 { \ 432 uint32_t old, new; \ 433 \ 434 do { \ 435 new = old = (ctx)->ctx_flags; \ 436 new &= CTX_FLAGS_MASK; \ 437 new |= flag; \ 438 new = cas32(&(ctx)->ctx_flags, old, new); \ 439 } while (new != old); \ 440 } 441 442 #define CTX_CLEAR_FLAGS(ctx, flag) \ 443 { \ 444 uint32_t old, new; \ 445 \ 446 do { \ 447 new = old = (ctx)->ctx_flags; \ 448 new &= CTX_FLAGS_MASK & ~(flag); \ 449 new = cas32(&(ctx)->ctx_flags, old, new); \ 450 } while (new != old); \ 451 } 452 453 #define ctxtoctxnum(ctx) ((ushort_t)((ctx) - ctxs)) 454 455 /* 456 * Defines needed for ctx stealing. 457 */ 458 #define GET_CTX_RETRY_CNT 100 459 460 /* 461 * Starting with context 0, the first NUM_LOCKED_CTXS contexts 462 * are locked so that sfmmu_getctx can't steal any of these 463 * contexts. At the time this software was being developed, the 464 * only context that needs to be locked is context 0 (the kernel 465 * context), and context 1 (reserved for stolen context). So this constant 466 * was originally defined to be 2. 467 */ 468 #define NUM_LOCKED_CTXS 2 469 #define INVALID_CONTEXT 1 470 471 #ifndef _ASM 472 473 /* 474 * Kernel page relocation stuff. 475 */ 476 struct sfmmu_callback { 477 int key; 478 int (*prehandler)(caddr_t, uint_t, uint_t, void *); 479 int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t); 480 int (*errhandler)(caddr_t, uint_t, uint_t, void *); 481 int capture_cpus; 482 }; 483 484 extern int sfmmu_max_cb_id; 485 extern struct sfmmu_callback *sfmmu_cb_table; 486 487 extern int hat_kpr_enabled; 488 489 struct pa_hment; 490 491 /* 492 * RFE: With multihat gone we gain back an int. We could use this to 493 * keep ref bits on a per cpu basis to eliminate xcalls. 494 */ 495 struct sf_hment { 496 tte_t hme_tte; /* tte for this hment */ 497 498 union { 499 struct page *page; /* what page this maps */ 500 struct pa_hment *data; /* pa_hment */ 501 } sf_hment_un; 502 503 struct sf_hment *hme_next; /* next hment */ 504 struct sf_hment *hme_prev; /* prev hment */ 505 }; 506 507 struct pa_hment { 508 caddr_t addr; /* va */ 509 uint_t len; /* bytes */ 510 ushort_t flags; /* internal flags */ 511 ushort_t refcnt; /* reference count */ 512 id_t cb_id; /* callback id, table index */ 513 void *pvt; /* handler's private data */ 514 struct sf_hment sfment; /* corresponding dummy sf_hment */ 515 }; 516 517 #define hme_page sf_hment_un.page 518 #define hme_data sf_hment_un.data 519 #define hme_size(sfhmep) ((int)(TTE_CSZ(&(sfhmep)->hme_tte))) 520 #define PAHME_SZ (sizeof (struct pa_hment)) 521 #define SFHME_SZ (sizeof (struct sf_hment)) 522 523 #define IS_PAHME(hme) ((hme)->hme_tte.ll == 0) 524 525 /* 526 * hmeblk_tag structure 527 * structure used to obtain a match on a hme_blk. Currently consists of 528 * the address of the sfmmu struct (or hatid), the base page address of the 529 * hme_blk, and the rehash count. The rehash count is actually only 2 bits 530 * and has the following meaning: 531 * 1 = 8k or 64k hash sequence. 532 * 2 = 512k hash sequence. 533 * 3 = 4M hash sequence. 534 * We require this count because we don't want to get a false hit on a 512K or 535 * 4M rehash with a base address corresponding to a 8k or 64k hmeblk. 536 * Note: The ordering and size of the hmeblk_tag members are implictly known 537 * by the tsb miss handlers written in assembly. Do not change this structure 538 * without checking those routines. See HTAG_SFMMUPSZ define. 539 */ 540 541 typedef union { 542 struct { 543 uint64_t hblk_basepg: 51, /* hme_blk base pg # */ 544 hblk_rehash: 13; /* rehash number */ 545 sfmmu_t *sfmmup; 546 } hblk_tag_un; 547 uint64_t htag_tag[2]; 548 } hmeblk_tag; 549 550 #define htag_id hblk_tag_un.sfmmup 551 #define htag_bspage hblk_tag_un.hblk_basepg 552 #define htag_rehash hblk_tag_un.hblk_rehash 553 554 #define HTAGS_EQ(tag1, tag2) (((tag1.htag_tag[0] ^ tag2.htag_tag[0]) | \ 555 (tag1.htag_tag[1] ^ tag2.htag_tag[1])) == 0) 556 #define HME_REHASH(sfmmup) \ 557 ((sfmmup)->sfmmu_ttecnt[TTE512K] != 0 || \ 558 (sfmmup)->sfmmu_ttecnt[TTE4M] != 0 || \ 559 (sfmmup)->sfmmu_ttecnt[TTE32M] != 0 || \ 560 (sfmmup)->sfmmu_ttecnt[TTE256M] != 0) 561 562 #endif /* !_ASM */ 563 564 #define NHMENTS 8 /* # of hments in an 8k hme_blk */ 565 /* needs to be multiple of 2 */ 566 #ifndef _ASM 567 568 #ifdef HBLK_TRACE 569 570 #define HBLK_LOCK 1 571 #define HBLK_UNLOCK 0 572 #define HBLK_STACK_DEPTH 6 573 #define HBLK_AUDIT_CACHE_SIZE 16 574 #define HBLK_LOCK_PATTERN 0xaaaaaaaa 575 #define HBLK_UNLOCK_PATTERN 0xbbbbbbbb 576 577 struct hblk_lockcnt_audit { 578 int flag; /* lock or unlock */ 579 kthread_id_t thread; 580 int depth; 581 pc_t stack[HBLK_STACK_DEPTH]; 582 }; 583 584 #endif /* HBLK_TRACE */ 585 586 587 /* 588 * Hment block structure. 589 * The hme_blk is the node data structure which the hash structure 590 * mantains. An hme_blk can have 2 different sizes depending on the 591 * number of hments it implicitly contains. When dealing with 64K, 512K, 592 * or 4M hments there is one hment per hme_blk. When dealing with 593 * 8k hments we allocate an hme_blk plus an additional 7 hments to 594 * give us a total of 8 (NHMENTS) hments that can be referenced through a 595 * hme_blk. 596 * 597 * The hmeblk structure contains 2 tte reference counters used to determine if 598 * it is ok to free up the hmeblk. Both counters have to be zero in order 599 * to be able to free up hmeblk. They are protected by cas. 600 * hblk_hmecnt is the number of hments present on pp mapping lists. 601 * hblk_vcnt reflects number of valid ttes in hmeblk. 602 * 603 * The hmeblk now also has per tte lock cnts. This is required because 604 * the counts can be high and there are not enough bits in the tte. When 605 * physio is fixed to not lock the translations we should be able to move 606 * the lock cnt back to the tte. See bug id 1198554. 607 * 608 * Note that xhat_hme_blk's layout follows this structure: hme_blk_misc 609 * and sf_hment are at the same offsets in both structures. Whenever 610 * hme_blk is changed, xhat_hme_blk may need to be updated as well. 611 */ 612 613 struct hme_blk_misc { 614 ushort_t locked_cnt; /* HAT_LOAD_LOCK ref cnt */ 615 uint_t notused:10; 616 uint_t xhat_bit:1; /* set for an xhat hme_blk */ 617 uint_t shadow_bit:1; /* set for a shadow hme_blk */ 618 uint_t nucleus_bit:1; /* set for a nucleus hme_blk */ 619 uint_t ttesize:3; /* contains ttesz of hmeblk */ 620 }; 621 622 struct hme_blk { 623 uint64_t hblk_nextpa; /* physical address for hash list */ 624 625 hmeblk_tag hblk_tag; /* tag used to obtain an hmeblk match */ 626 627 struct hme_blk *hblk_next; /* on free list or on hash list */ 628 /* protected by hash lock */ 629 630 struct hme_blk *hblk_shadow; /* pts to shadow hblk */ 631 /* protected by hash lock */ 632 uint_t hblk_span; /* span of memory hmeblk maps */ 633 634 struct hme_blk_misc hblk_misc; 635 636 union { 637 struct { 638 ushort_t hblk_hmecount; /* hment on mlists counter */ 639 ushort_t hblk_validcnt; /* valid tte reference count */ 640 } hblk_counts; 641 uint_t hblk_shadow_mask; 642 } hblk_un; 643 644 #ifdef HBLK_TRACE 645 kmutex_t hblk_audit_lock; /* lock to protect index */ 646 uint_t hblk_audit_index; /* index into audit_cache */ 647 struct hblk_lockcnt_audit hblk_audit_cache[HBLK_AUDIT_CACHE_SIZE]; 648 #endif /* HBLK_AUDIT */ 649 650 struct sf_hment hblk_hme[1]; /* hment array */ 651 }; 652 653 #define hblk_lckcnt hblk_misc.locked_cnt 654 #define hblk_xhat_bit hblk_misc.xhat_bit 655 #define hblk_shw_bit hblk_misc.shadow_bit 656 #define hblk_nuc_bit hblk_misc.nucleus_bit 657 #define hblk_ttesz hblk_misc.ttesize 658 #define hblk_hmecnt hblk_un.hblk_counts.hblk_hmecount 659 #define hblk_vcnt hblk_un.hblk_counts.hblk_validcnt 660 #define hblk_shw_mask hblk_un.hblk_shadow_mask 661 662 #define MAX_HBLK_LCKCNT 0xFFFF 663 #define HMEBLK_ALIGN 0x8 /* hmeblk has to be double aligned */ 664 665 #ifdef HBLK_TRACE 666 667 #define HBLK_STACK_TRACE(hmeblkp, lock) \ 668 { \ 669 int flag = lock; /* to pacify lint */ \ 670 int audit_index; \ 671 \ 672 mutex_enter(&hmeblkp->hblk_audit_lock); \ 673 audit_index = hmeblkp->hblk_audit_index; \ 674 hmeblkp->hblk_audit_index = ((hmeblkp->hblk_audit_index + 1) & \ 675 (HBLK_AUDIT_CACHE_SIZE - 1)); \ 676 mutex_exit(&hmeblkp->hblk_audit_lock); \ 677 \ 678 if (flag) \ 679 hmeblkp->hblk_audit_cache[audit_index].flag = \ 680 HBLK_LOCK_PATTERN; \ 681 else \ 682 hmeblkp->hblk_audit_cache[audit_index].flag = \ 683 HBLK_UNLOCK_PATTERN; \ 684 \ 685 hmeblkp->hblk_audit_cache[audit_index].thread = curthread; \ 686 hmeblkp->hblk_audit_cache[audit_index].depth = \ 687 getpcstack(hmeblkp->hblk_audit_cache[audit_index].stack, \ 688 HBLK_STACK_DEPTH); \ 689 } 690 691 #else 692 693 #define HBLK_STACK_TRACE(hmeblkp, lock) 694 695 #endif /* HBLK_TRACE */ 696 697 #define HMEHASH_FACTOR 16 /* used to calc # of buckets in hme hash */ 698 699 /* 700 * A maximum number of user hmeblks is defined in order to place an upper 701 * limit on how much nucleus memory is required and to avoid overflowing the 702 * tsbmiss uhashsz and khashsz data areas. The number below corresponds to 703 * the number of buckets required, for an average hash chain length of 4 on 704 * a 16TB machine. 705 */ 706 707 #define MAX_UHME_BUCKETS (0x1 << 30) 708 #define MAX_KHME_BUCKETS (0x1 << 30) 709 710 /* 711 * The minimum number of kernel hash buckets. 712 */ 713 #define MIN_KHME_BUCKETS 0x800 714 715 /* 716 * The number of hash buckets must be a power of 2. If the initial calculated 717 * value is less than USER_BUCKETS_THRESHOLD we round up to the next greater 718 * power of 2, otherwise we round down to avoid huge over allocations. 719 */ 720 #define USER_BUCKETS_THRESHOLD (1<<22) 721 722 #define MAX_NUCUHME_BUCKETS 0x4000 723 #define MAX_NUCKHME_BUCKETS 0x2000 724 725 /* 726 * There are 2 locks in the hmehash bucket. The hmehash_mutex is 727 * a regular mutex used to make sure operations on a hash link are only 728 * done by one thread. Any operation which comes into the hat with 729 * a <vaddr, as> will grab the hmehash_mutex. Normally one would expect 730 * the tsb miss handlers to grab the hash lock to make sure the hash list 731 * is consistent while we traverse it. Unfortunately this can lead to 732 * deadlocks or recursive mutex enters since it is possible for 733 * someone holding the lock to take a tlb/tsb miss. 734 * To solve this problem we have added the hmehash_listlock. This lock 735 * is only grabbed by the tsb miss handlers, vatopfn, and while 736 * adding/removing a hmeblk from the hash list. The code is written to 737 * guarantee we won't take a tlb miss while holding this lock. 738 */ 739 struct hmehash_bucket { 740 kmutex_t hmehash_mutex; 741 uint64_t hmeh_nextpa; /* physical address for hash list */ 742 struct hme_blk *hmeblkp; 743 uint_t hmeh_listlock; 744 }; 745 746 #endif /* !_ASM */ 747 748 749 /* 750 * The tsb miss handlers written in assembly know that sfmmup 751 * is a 64 bit ptr. 752 * 753 * The bspage and re-hash part is 64 bits, with the sfmmup being another 64 754 * bits. 755 */ 756 #define HTAG_SFMMUPSZ 0 /* Not really used for LP64 */ 757 #define HTAG_REHASHSZ 13 758 759 /* 760 * Assembly routines need to be able to get to ttesz 761 */ 762 #define HBLK_SZMASK 0x7 763 764 #ifndef _ASM 765 766 /* 767 * Returns the number of bytes that an hmeblk spans given its tte size 768 */ 769 #define get_hblk_span(hmeblkp) ((hmeblkp)->hblk_span) 770 #define get_hblk_ttesz(hmeblkp) ((hmeblkp)->hblk_ttesz) 771 #define get_hblk_cache(hmeblkp) (((hmeblkp)->hblk_ttesz == TTE8K) ? \ 772 sfmmu8_cache : sfmmu1_cache) 773 #define HMEBLK_SPAN(ttesz) \ 774 ((ttesz == TTE8K)? (TTEBYTES(ttesz) * NHMENTS) : TTEBYTES(ttesz)) 775 776 #define set_hblk_sz(hmeblkp, ttesz) \ 777 (hmeblkp)->hblk_ttesz = (ttesz); \ 778 (hmeblkp)->hblk_span = HMEBLK_SPAN(ttesz) 779 780 #define get_hblk_base(hmeblkp) \ 781 ((uintptr_t)(hmeblkp)->hblk_tag.htag_bspage << MMU_PAGESHIFT) 782 783 #define get_hblk_endaddr(hmeblkp) \ 784 ((caddr_t)(get_hblk_base(hmeblkp) + get_hblk_span(hmeblkp))) 785 786 #define in_hblk_range(hmeblkp, vaddr) \ 787 (((uintptr_t)(vaddr) >= get_hblk_base(hmeblkp)) && \ 788 ((uintptr_t)(vaddr) < (get_hblk_base(hmeblkp) + \ 789 get_hblk_span(hmeblkp)))) 790 791 #define tte_to_vaddr(hmeblkp, tte) ((caddr_t)(get_hblk_base(hmeblkp) \ 792 + (TTEBYTES(TTE_CSZ(&tte)) * (tte).tte_hmenum))) 793 794 #define vaddr_to_vshift(hblktag, vaddr, shwsz) \ 795 ((((uintptr_t)(vaddr) >> MMU_PAGESHIFT) - (hblktag.htag_bspage)) >>\ 796 TTE_BSZS_SHIFT((shwsz) - 1)) 797 798 #define HME8BLK_SZ (sizeof (struct hme_blk) + \ 799 (NHMENTS - 1) * sizeof (struct sf_hment)) 800 #define HME1BLK_SZ (sizeof (struct hme_blk)) 801 #define H8TOH1 (MMU_PAGESIZE4M / MMU_PAGESIZE) 802 #define H1MIN (2 + MAX_BIGKTSB_TTES) /* nucleus text+data, ktsb */ 803 804 /* 805 * Hme_blk hash structure 806 * Active mappings are kept in a hash structure of hme_blks. The hash 807 * function is based on (ctx, vaddr) The size of the hash table size is a 808 * power of 2 such that the average hash chain lenth is HMENT_HASHAVELEN. 809 * The hash actually consists of 2 separate hashes. One hash is for the user 810 * address space and the other hash is for the kernel address space. 811 * The number of buckets are calculated at boot time and stored in the global 812 * variables "uhmehash_num" and "khmehash_num". By making the hash table size 813 * a power of 2 we can use a simply & function to derive an index instead of 814 * a divide. 815 * 816 * HME_HASH_FUNCTION(hatid, vaddr, shift) returns a pointer to a hme_hash 817 * bucket. 818 * An hme hash bucket contains a pointer to an hme_blk and the mutex that 819 * protects the link list. 820 * Spitfire supports 4 page sizes. 8k and 64K pages only need one hash. 821 * 512K pages need 2 hashes and 4M pages need 3 hashes. 822 * The 'shift' parameter controls how many bits the vaddr will be shifted in 823 * the hash function. It is calculated in the HME_HASH_SHIFT(ttesz) function 824 * and it varies depending on the page size as follows: 825 * 8k pages: HBLK_RANGE_SHIFT 826 * 64k pages: MMU_PAGESHIFT64K 827 * 512K pages: MMU_PAGESHIFT512K 828 * 4M pages: MMU_PAGESHIFT4M 829 * An assembly version of the hash function exists in sfmmu_ktsb_miss(). All 830 * changes should be reflected in both versions. This function and the TSB 831 * miss handlers are the only places which know about the two hashes. 832 * 833 * HBLK_RANGE_SHIFT controls range of virtual addresses that will fall 834 * into the same bucket for a particular process. It is currently set to 835 * be equivalent to 64K range or one hme_blk. 836 * 837 * The hme_blks in the hash are protected by a per hash bucket mutex 838 * known as SFMMU_HASH_LOCK. 839 * You need to acquire this lock before traversing the hash bucket link 840 * list, while adding/removing a hme_blk to the list, and while 841 * modifying an hme_blk. A possible optimization is to replace these 842 * mutexes by readers/writer lock but right now it is not clear whether 843 * this is a win or not. 844 * 845 * The HME_HASH_TABLE_SEARCH will search the hash table for the 846 * hme_blk that contains the hment that corresponds to the passed 847 * ctx and vaddr. It assumed the SFMMU_HASH_LOCK is held. 848 */ 849 850 #endif /* ! _ASM */ 851 852 #define KHATID ksfmmup 853 #define UHMEHASH_SZ uhmehash_num 854 #define KHMEHASH_SZ khmehash_num 855 #define HMENT_HASHAVELEN 4 856 #define HBLK_RANGE_SHIFT MMU_PAGESHIFT64K /* shift for HBLK_BS_MASK */ 857 #define MAX_HASHCNT 5 858 #define DEFAULT_MAX_HASHCNT 3 859 860 #ifndef _ASM 861 862 #define HASHADDR_MASK(hashno) TTE_PAGEMASK(hashno) 863 864 #define HME_HASH_SHIFT(ttesz) \ 865 ((ttesz == TTE8K)? HBLK_RANGE_SHIFT : TTE_PAGE_SHIFT(ttesz)) \ 866 867 #define HME_HASH_ADDR(vaddr, hmeshift) \ 868 ((caddr_t)(((uintptr_t)(vaddr) >> (hmeshift)) << (hmeshift))) 869 870 #define HME_HASH_BSPAGE(vaddr, hmeshift) \ 871 (((uintptr_t)(vaddr) >> (hmeshift)) << ((hmeshift) - MMU_PAGESHIFT)) 872 873 #define HME_HASH_REHASH(ttesz) \ 874 (((ttesz) < TTE512K)? 1 : (ttesz)) 875 876 #define HME_HASH_FUNCTION(hatid, vaddr, shift) \ 877 ((hatid != KHATID)? \ 878 (&uhme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \ 879 UHMEHASH_SZ) ]): \ 880 (&khme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \ 881 KHMEHASH_SZ) ])) 882 883 /* 884 * This macro will traverse a hmeblk hash link list looking for an hme_blk 885 * that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp 886 * will be set to NULL, otherwise it will point to the correct hme_blk. 887 * This macro also cleans empty hblks. 888 */ 889 #define HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, \ 890 pr_hblk, prevpa, listp) \ 891 { \ 892 struct hme_blk *nx_hblk; \ 893 uint64_t nx_pa; \ 894 \ 895 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \ 896 hblkp = hmebp->hmeblkp; \ 897 hblkpa = hmebp->hmeh_nextpa; \ 898 prevpa = 0; \ 899 pr_hblk = NULL; \ 900 while (hblkp) { \ 901 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \ 902 /* found hme_blk */ \ 903 break; \ 904 } \ 905 nx_hblk = hblkp->hblk_next; \ 906 nx_pa = hblkp->hblk_nextpa; \ 907 if (!hblkp->hblk_vcnt && !hblkp->hblk_hmecnt) { \ 908 sfmmu_hblk_hash_rm(hmebp, hblkp, prevpa, pr_hblk); \ 909 sfmmu_hblk_free(hmebp, hblkp, hblkpa, listp); \ 910 } else { \ 911 pr_hblk = hblkp; \ 912 prevpa = hblkpa; \ 913 } \ 914 hblkp = nx_hblk; \ 915 hblkpa = nx_pa; \ 916 } \ 917 } 918 919 #define HME_HASH_SEARCH(hmebp, hblktag, hblkp, listp) \ 920 { \ 921 struct hme_blk *pr_hblk; \ 922 uint64_t hblkpa, prevpa; \ 923 \ 924 HME_HASH_SEARCH_PREV(hmebp, hblktag, hblkp, hblkpa, pr_hblk, \ 925 prevpa, listp); \ 926 } 927 928 /* 929 * This macro will traverse a hmeblk hash link list looking for an hme_blk 930 * that owns the specified vaddr and hatid. If if doesn't find one , hmeblkp 931 * will be set to NULL, otherwise it will point to the correct hme_blk. 932 * It doesn't remove empty hblks. 933 */ 934 #define HME_HASH_FAST_SEARCH(hmebp, hblktag, hblkp) \ 935 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp)); \ 936 for (hblkp = hmebp->hmeblkp; hblkp; \ 937 hblkp = hblkp->hblk_next) { \ 938 if (HTAGS_EQ(hblkp->hblk_tag, hblktag)) { \ 939 /* found hme_blk */ \ 940 break; \ 941 } \ 942 } \ 943 944 945 #define SFMMU_HASH_LOCK(hmebp) \ 946 (mutex_enter(&hmebp->hmehash_mutex)) 947 948 #define SFMMU_HASH_UNLOCK(hmebp) \ 949 (mutex_exit(&hmebp->hmehash_mutex)) 950 951 #define SFMMU_HASH_LOCK_TRYENTER(hmebp) \ 952 (mutex_tryenter(&hmebp->hmehash_mutex)) 953 954 #define SFMMU_HASH_LOCK_ISHELD(hmebp) \ 955 (mutex_owned(&hmebp->hmehash_mutex)) 956 957 #define SFMMU_XCALL_STATS(ctxnum) \ 958 { \ 959 if (ctxnum == KCONTEXT) { \ 960 SFMMU_STAT(sf_kernel_xcalls); \ 961 } else { \ 962 SFMMU_STAT(sf_user_xcalls); \ 963 } \ 964 } 965 966 #define astosfmmu(as) ((as)->a_hat) 967 #define sfmmutoctxnum(sfmmup) ((sfmmup)->sfmmu_cnum) 968 #define sfmmutoctx(sfmmup) (&ctxs[sfmmutoctxnum(sfmmup)]) 969 #define hblktosfmmu(hmeblkp) ((sfmmu_t *)(hmeblkp)->hblk_tag.htag_id) 970 #define sfmmutoas(sfmmup) ((sfmmup)->sfmmu_as) 971 #define ctxnumtoctx(ctxnum) (&ctxs[ctxnum]) 972 /* 973 * We use the sfmmu data structure to keep the per as page coloring info. 974 */ 975 #define as_color_bin(as) (astosfmmu(as)->sfmmu_clrbin) 976 #define as_color_start(as) (astosfmmu(as)->sfmmu_clrstart) 977 978 typedef struct { 979 char h8[HME8BLK_SZ]; 980 } hblk8_t; 981 982 typedef struct { 983 char h1[HME1BLK_SZ]; 984 } hblk1_t; 985 986 typedef struct { 987 ulong_t index; 988 ulong_t len; 989 hblk8_t *list; 990 } nucleus_hblk8_info_t; 991 992 typedef struct { 993 ulong_t index; 994 ulong_t len; 995 hblk1_t *list; 996 } nucleus_hblk1_info_t; 997 998 /* 999 * This struct is used for accumlating information about a range 1000 * of pages that are unloading so that a single xcall can flush 1001 * the entire range from remote tlbs. A function that must demap 1002 * a range of virtual addresses declares one of these structures 1003 * and initializes using DEMP_RANGE_INIT(). It then passes a pointer to this 1004 * struct to the appropriate sfmmu_hblk_* level function which does 1005 * all the bookkeeping using the other macros. When the function has 1006 * finished the virtual address range, it needs to call DEMAP_RANGE_FLUSH() 1007 * macro to take care of any remaining unflushed mappings. 1008 * 1009 * The maximum range this struct can represent is the number of bits 1010 * in the dmr_bitvec field times the pagesize in dmr_pgsz. Currently, only 1011 * MMU_PAGESIZE pages are supported. 1012 * 1013 * Since there are now cases where it's no longer necessary to do 1014 * flushes (e.g. when the process isn't runnable because it's swapping 1015 * out or exiting) we allow these macros to take a NULL dmr input and do 1016 * nothing in that case. 1017 */ 1018 typedef struct { 1019 sfmmu_t *dmr_sfmmup; /* relevent hat */ 1020 caddr_t dmr_addr; /* beginning address */ 1021 caddr_t dmr_endaddr; /* ending address */ 1022 ulong_t dmr_bitvec; /* valid pages found */ 1023 ulong_t dmr_bit; /* next page to examine */ 1024 ulong_t dmr_maxbit; /* highest page in range */ 1025 ulong_t dmr_pgsz; /* page size in range */ 1026 } demap_range_t; 1027 1028 #define DMR_MAXBIT ((ulong_t)1<<63) /* dmr_bit high bit */ 1029 1030 #define DEMAP_RANGE_INIT(sfmmup, dmrp) \ 1031 if ((dmrp) != NULL) { \ 1032 (dmrp)->dmr_sfmmup = (sfmmup); \ 1033 (dmrp)->dmr_bitvec = 0; \ 1034 (dmrp)->dmr_maxbit = sfmmu_dmr_maxbit; \ 1035 (dmrp)->dmr_pgsz = MMU_PAGESIZE; \ 1036 } 1037 1038 #define DEMAP_RANGE_PGSZ(dmrp) ((dmrp)? (dmrp)->dmr_pgsz : MMU_PAGESIZE) 1039 1040 #define DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr) \ 1041 if ((dmrp) != NULL) { \ 1042 if ((dmrp)->dmr_bitvec != 0 && (dmrp)->dmr_endaddr != (addr)) \ 1043 sfmmu_tlb_range_demap(dmrp); \ 1044 (dmrp)->dmr_endaddr = (endaddr); \ 1045 } 1046 1047 #define DEMAP_RANGE_FLUSH(dmrp) \ 1048 if ((dmrp) != NULL) { \ 1049 if ((dmrp)->dmr_bitvec != 0) \ 1050 sfmmu_tlb_range_demap(dmrp); \ 1051 } 1052 1053 #define DEMAP_RANGE_MARKPG(dmrp, addr) \ 1054 if ((dmrp) != NULL) { \ 1055 if ((dmrp)->dmr_bitvec == 0) { \ 1056 (dmrp)->dmr_addr = (addr); \ 1057 (dmrp)->dmr_bit = 1; \ 1058 } \ 1059 (dmrp)->dmr_bitvec |= (dmrp)->dmr_bit; \ 1060 } 1061 1062 #define DEMAP_RANGE_NEXTPG(dmrp) \ 1063 if ((dmrp) != NULL && (dmrp)->dmr_bitvec != 0) { \ 1064 if ((dmrp)->dmr_bit & (dmrp)->dmr_maxbit) { \ 1065 sfmmu_tlb_range_demap(dmrp); \ 1066 } else { \ 1067 (dmrp)->dmr_bit <<= 1; \ 1068 } \ 1069 } 1070 1071 /* 1072 * TSB related structures 1073 * 1074 * The TSB is made up of tte entries. Both the tag and data are present 1075 * in the TSB. The TSB locking is managed as follows: 1076 * A software bit in the tsb tag is used to indicate that entry is locked. 1077 * If a cpu servicing a tsb miss reads a locked entry the tag compare will 1078 * fail forcing the cpu to go to the hat hash for the translation. 1079 * The cpu who holds the lock can then modify the data side, and the tag side. 1080 * The last write should be to the word containing the lock bit which will 1081 * clear the lock and allow the tsb entry to be read. It is assumed that all 1082 * cpus reading the tsb will do so with atomic 128-bit loads. An atomic 128 1083 * bit load is required to prevent the following from happening: 1084 * 1085 * cpu 0 cpu 1 comments 1086 * 1087 * ldx tag tag unlocked 1088 * ldstub lock set lock 1089 * stx data 1090 * stx tag unlock 1091 * ldx tag incorrect tte!!! 1092 * 1093 * The software also maintains a bit in the tag to indicate an invalid 1094 * tsb entry. The purpose of this bit is to allow the tsb invalidate code 1095 * to invalidate a tsb entry with a single cas. See code for details. 1096 */ 1097 1098 union tsb_tag { 1099 struct { 1100 uint32_t tag_res0:16; /* reserved - context area */ 1101 uint32_t tag_inv:1; /* sw - invalid tsb entry */ 1102 uint32_t tag_lock:1; /* sw - locked tsb entry */ 1103 uint32_t tag_res1:4; /* reserved */ 1104 uint32_t tag_va_hi:10; /* va[63:54] */ 1105 uint32_t tag_va_lo; /* va[53:22] */ 1106 } tagbits; 1107 struct tsb_tagints { 1108 uint32_t inthi; 1109 uint32_t intlo; 1110 } tagints; 1111 }; 1112 #define tag_invalid tagbits.tag_inv 1113 #define tag_locked tagbits.tag_lock 1114 #define tag_vahi tagbits.tag_va_hi 1115 #define tag_valo tagbits.tag_va_lo 1116 #define tag_inthi tagints.inthi 1117 #define tag_intlo tagints.intlo 1118 1119 struct tsbe { 1120 union tsb_tag tte_tag; 1121 tte_t tte_data; 1122 }; 1123 1124 /* 1125 * A per cpu struct is kept that duplicates some info 1126 * used by the tl>0 tsb miss handlers plus it provides 1127 * a scratch area. Its purpose is to minimize cache misses 1128 * in the tsb miss handler and is 128 bytes (2 e$ lines). 1129 * 1130 * There should be one allocated per cpu in nucleus memory 1131 * and should be aligned on an ecache line boundary. 1132 */ 1133 struct tsbmiss { 1134 sfmmu_t *ksfmmup; /* kernel hat id */ 1135 sfmmu_t *usfmmup; /* user hat id */ 1136 struct tsbe *tsbptr; /* hardware computed ptr */ 1137 struct tsbe *tsbptr4m; /* hardware computed ptr */ 1138 uint64_t ismblkpa; 1139 struct hmehash_bucket *khashstart; 1140 struct hmehash_bucket *uhashstart; 1141 uint_t khashsz; 1142 uint_t uhashsz; 1143 uint16_t dcache_line_mask; /* used to flush dcache */ 1144 uint16_t hat_flags; 1145 uint32_t itlb_misses; 1146 uint32_t dtlb_misses; 1147 uint32_t utsb_misses; 1148 uint32_t ktsb_misses; 1149 uint16_t uprot_traps; 1150 uint16_t kprot_traps; 1151 1152 /* 1153 * scratch[0] -> TSB_TAGACC 1154 * scratch[1] -> TSBMISS_HMEBP 1155 * scratch[2] -> TSBMISS_HATID 1156 */ 1157 uintptr_t scratch[3]; 1158 uint8_t pad[0x10]; 1159 }; 1160 1161 /* 1162 * A per cpu struct is kept for the use within the tl>0 kpm tsb 1163 * miss handler. Some members are duplicates of common data or 1164 * the physical addresses of common data. A few members are also 1165 * written by the tl>0 kpm tsb miss handler. Its purpose is to 1166 * minimize cache misses in the kpm tsb miss handler and occupies 1167 * one ecache line. There should be one allocated per cpu in 1168 * nucleus memory and it should be aligned on an ecache line 1169 * boundary. It is not merged w/ struct tsbmiss since there is 1170 * not much to share and the tsbmiss pathes are different, so 1171 * a kpm tlbmiss/tsbmiss only touches one cacheline, except for 1172 * (DEBUG || SFMMU_STAT_GATHER) where the dtlb_misses counter 1173 * of struct tsbmiss is used on every dtlb miss. 1174 */ 1175 struct kpmtsbm { 1176 caddr_t vbase; /* start of address kpm range */ 1177 caddr_t vend; /* end of address kpm range */ 1178 uchar_t flags; /* flags needed in TL tsbmiss handler */ 1179 uchar_t sz_shift; /* for single kpm window */ 1180 uchar_t kpmp_shift; /* hash lock shift */ 1181 uchar_t kpmp2pshft; /* kpm page to page shift */ 1182 uint_t kpmp_table_sz; /* size of kpmp_table or kpmp_stable */ 1183 uint64_t kpmp_tablepa; /* paddr of kpmp_table or kpmp_stable */ 1184 uint64_t msegphashpa; /* paddr of memseg_phash */ 1185 struct tsbe *tsbptr; /* saved ktsb pointer */ 1186 uint_t kpm_dtlb_misses; /* kpm tlbmiss counter */ 1187 uint_t kpm_tsb_misses; /* kpm tsbmiss counter */ 1188 uintptr_t pad[1]; 1189 }; 1190 1191 extern uint_t tsb_slab_size; 1192 extern uint_t tsb_slab_shift; 1193 extern uint_t tsb_slab_ttesz; 1194 extern uint_t tsb_slab_pamask; 1195 1196 #endif /* !_ASM */ 1197 1198 /* 1199 * Flags for TL kpm tsbmiss handler 1200 */ 1201 #define KPMTSBM_ENABLE_FLAG 0x01 /* bit copy of kpm_enable */ 1202 #define KPMTSBM_TLTSBM_FLAG 0x02 /* use TL tsbmiss handler */ 1203 #define KPMTSBM_TSBPHYS_FLAG 0x04 /* use ASI_MEM for TSB update */ 1204 1205 /* 1206 * The TSB 1207 * All TSB sizes supported by the hardware are now supported (8K - 1M). 1208 * For kernel TSBs we may go beyond the hardware supported sizes and support 1209 * larger TSBs via software. 1210 * All TTE sizes are supported in the TSB; the manner in which this is 1211 * done is cpu dependent. 1212 */ 1213 #define TSB_MIN_SZCODE TSB_8K_SZCODE /* min. supported TSB size */ 1214 #define TSB_MIN_OFFSET_MASK (TSB_OFFSET_MASK(TSB_MIN_SZCODE)) 1215 1216 #define UTSB_MAX_SZCODE TSB_1M_SZCODE /* max. supported TSB size */ 1217 #define UTSB_MAX_OFFSET_MASK (TSB_OFFSET_MASK(UTSB_MAX_SZCODE)) 1218 1219 #define TSB_FREEMEM_MIN 0x1000 /* 32 mb */ 1220 #define TSB_FREEMEM_LARGE 0x10000 /* 512 mb */ 1221 #define TSB_8K_SZCODE 0 /* 512 entries */ 1222 #define TSB_16K_SZCODE 1 /* 1k entries */ 1223 #define TSB_32K_SZCODE 2 /* 2k entries */ 1224 #define TSB_64K_SZCODE 3 /* 4k entries */ 1225 #define TSB_128K_SZCODE 4 /* 8k entries */ 1226 #define TSB_256K_SZCODE 5 /* 16k entries */ 1227 #define TSB_512K_SZCODE 6 /* 32k entries */ 1228 #define TSB_1M_SZCODE 7 /* 64k entries */ 1229 #define TSB_2M_SZCODE 8 /* 128k entries */ 1230 #define TSB_4M_SZCODE 9 /* 256k entries */ 1231 #define TSB_ENTRY_SHIFT 4 /* each entry = 128 bits = 16 bytes */ 1232 #define TSB_ENTRY_SIZE (1 << 4) 1233 #define TSB_START_SIZE 9 1234 #define TSB_ENTRIES(tsbsz) (1 << (TSB_START_SIZE + tsbsz)) 1235 #define TSB_BYTES(tsbsz) (TSB_ENTRIES(tsbsz) << TSB_ENTRY_SHIFT) 1236 #define TSB_OFFSET_MASK(tsbsz) (TSB_ENTRIES(tsbsz) - 1) 1237 #define TSB_BASEADDR_MASK ((1 << 12) - 1) 1238 1239 /* 1240 * sun4u platforms 1241 * --------------- 1242 * We now support two user TSBs with one TSB base register. 1243 * Hence the TSB base register is split up as follows: 1244 * 1245 * When only one TSB present: 1246 * [63 62..42 41..13 12..4 3..0] 1247 * ^ ^ ^ ^ ^ 1248 * | | | | | 1249 * | | | | |_ TSB size code 1250 * | | | | 1251 * | | | |_ Reserved 0 1252 * | | | 1253 * | | |_ TSB VA[41..13] 1254 * | | 1255 * | |_ VA hole (Spitfire), zeros (Cheetah and beyond) 1256 * | 1257 * |_ 0 1258 * 1259 * When second TSB present: 1260 * [63 62..42 41..33 32..29 28..22 21..13 12..4 3..0] 1261 * ^ ^ ^ ^ ^ ^ ^ ^ 1262 * | | | | | | | | 1263 * | | | | | | | |_ First TSB size code 1264 * | | | | | | | 1265 * | | | | | | |_ Reserved 0 1266 * | | | | | | 1267 * | | | | | |_ First TSB's VA[21..13] 1268 * | | | | | 1269 * | | | | |_ Reserved for future use 1270 * | | | | 1271 * | | | |_ Second TSB's size code 1272 * | | | 1273 * | | |_ Second TSB's VA[21..13] 1274 * | | 1275 * | |_ VA hole (Spitfire) / ones (Cheetah and beyond) 1276 * | 1277 * |_ 1 1278 * 1279 * Note that since we store 21..13 of each TSB's VA, TSBs and their slabs 1280 * may be up to 4M in size. For now, only hardware supported TSB sizes 1281 * are supported, though the slabs are usually 4M in size. 1282 * 1283 * sun4v platforms 1284 * --------------- 1285 * On sun4v platforms, we use two dedicated scratchpad registers as pseudo 1286 * hardware TSB base registers to hold up to two different user TSBs. 1287 * 1288 * Each register contains TSB's physical base and size code information 1289 * as follows: 1290 * 1291 * [63..56 55..13 12..4 3..0] 1292 * ^ ^ ^ ^ 1293 * | | | | 1294 * | | | |_ TSB size code 1295 * | | | 1296 * | | |_ Reserved 0 1297 * | | 1298 * | |_ TSB PA[55..13] 1299 * | 1300 * | 1301 * | 1302 * |_ 0 for valid TSB 1303 * 1304 * Absence of a user TSB (primarily the second user TSB) is indicated by 1305 * storing a negative value in the TSB base register. This allows us to 1306 * check for presence of a user TSB by simply checking bit# 63. 1307 */ 1308 #define TSBREG_MSB_SHIFT 32 /* set upper bits */ 1309 #define TSBREG_MSB_CONST 0xfffff800 /* set bits 63..43 */ 1310 #define TSBREG_FIRTSB_SHIFT 42 /* to clear bits 63:22 */ 1311 #define TSBREG_SECTSB_MKSHIFT 20 /* 21:13 --> 41:33 */ 1312 #define TSBREG_SECTSB_LSHIFT 22 /* to clear bits 63:42 */ 1313 #define TSBREG_SECTSB_RSHIFT (TSBREG_SECTSB_MKSHIFT + TSBREG_SECTSB_LSHIFT) 1314 /* sectsb va -> bits 21:13 */ 1315 /* after clearing upper bits */ 1316 #define TSBREG_SECSZ_SHIFT 29 /* to get sectsb szc to 3:0 */ 1317 #define TSBREG_VAMASK_SHIFT 13 /* set up VA mask */ 1318 1319 #define BIGKTSB_SZ_MASK 0xf 1320 #define TSB_SOFTSZ_MASK BIGKTSB_SZ_MASK 1321 #define MIN_BIGKTSB_SZCODE 9 /* 256k entries */ 1322 #define MAX_BIGKTSB_SZCODE 11 /* 1024k entries */ 1323 #define MAX_BIGKTSB_TTES (TSB_BYTES(MAX_BIGKTSB_SZCODE) / MMU_PAGESIZE4M) 1324 1325 #define TAG_VALO_SHIFT 22 /* tag's va are bits 63-22 */ 1326 /* 1327 * sw bits used on tsb_tag - bit masks used only in assembly 1328 * use only a sethi for these fields. 1329 */ 1330 #define TSBTAG_INVALID 0x00008000 /* tsb_tag.tag_invalid */ 1331 #define TSBTAG_LOCKED 0x00004000 /* tsb_tag.tag_locked */ 1332 1333 #ifdef _ASM 1334 1335 /* 1336 * Marker to indicate that this instruction will be hot patched at runtime 1337 * to some other value. 1338 * This value must be zero since it fills in the imm bits of the target 1339 * instructions to be patched 1340 */ 1341 #define RUNTIME_PATCH (0) 1342 1343 /* 1344 * V9 defines nop instruction as the following, which we use 1345 * at runtime to nullify some instructions we don't want to 1346 * execute in the trap handlers on certain platforms. 1347 */ 1348 #define MAKE_NOP_INSTR(reg) \ 1349 sethi %hi(0x1000000), reg 1350 1351 /* 1352 * Macro to get this CPU's tsbmiss area. 1353 */ 1354 #define CPU_TSBMISS_AREA(tsbmiss, tmp1) \ 1355 CPU_INDEX(tmp1, tsbmiss); /* tmp1 = cpu idx */ \ 1356 sethi %hi(tsbmiss_area), tsbmiss; /* tsbmiss base ptr */ \ 1357 sllx tmp1, TSBMISS_SHIFT, tmp1; /* byte offset */ \ 1358 or tsbmiss, %lo(tsbmiss_area), tsbmiss; \ 1359 add tsbmiss, tmp1, tsbmiss /* tsbmiss area of CPU */ 1360 1361 1362 /* 1363 * Macro to set kernel context + page size codes in DMMU primary context 1364 * register. It is only necessary for sun4u because sun4v does not need 1365 * page size codes 1366 */ 1367 #ifdef sun4v 1368 1369 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) 1370 1371 #else 1372 1373 #define SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) \ 1374 sethi %hi(kcontextreg), reg0; \ 1375 ldx [reg0 + %lo(kcontextreg)], reg0; \ 1376 mov MMU_PCONTEXT, reg1; \ 1377 ldxa [reg1]ASI_MMU_CTX, reg2; \ 1378 xor reg0, reg2, reg2; \ 1379 brz reg2, label3; \ 1380 srlx reg2, CTXREG_NEXT_SHIFT, reg2; \ 1381 rdpr %pstate, reg3; /* disable interrupts */ \ 1382 btst PSTATE_IE, reg3; \ 1383 /*CSTYLED*/ \ 1384 bnz,a,pt %icc, label1; \ 1385 wrpr reg3, PSTATE_IE, %pstate; \ 1386 /*CSTYLED*/ \ 1387 label1:; \ 1388 brz reg2, label2; /* need demap if N_pgsz0/1 change */ \ 1389 sethi %hi(FLUSH_ADDR), reg4; \ 1390 mov DEMAP_ALL_TYPE, reg2; \ 1391 stxa %g0, [reg2]ASI_DTLB_DEMAP; \ 1392 stxa %g0, [reg2]ASI_ITLB_DEMAP; \ 1393 /*CSTYLED*/ \ 1394 label2:; \ 1395 stxa reg0, [reg1]ASI_MMU_CTX; \ 1396 flush reg4; \ 1397 btst PSTATE_IE, reg3; \ 1398 /*CSTYLED*/ \ 1399 bnz,a,pt %icc, label3; \ 1400 wrpr %g0, reg3, %pstate; /* restore interrupt state */ \ 1401 label3:; 1402 1403 #endif 1404 1405 #endif /* _ASM */ 1406 1407 #ifndef _ASM 1408 1409 /* 1410 * Page coloring 1411 * The p_vcolor field of the page struct (1 byte) is used to store the 1412 * virtual page color. This provides for 255 colors. The value zero is 1413 * used to mean the page has no color - never been mapped or somehow 1414 * purified. 1415 */ 1416 1417 #define PP_GET_VCOLOR(pp) (((pp)->p_vcolor) - 1) 1418 #define PP_NEWPAGE(pp) (!(pp)->p_vcolor) 1419 #define PP_SET_VCOLOR(pp, color) \ 1420 ((pp)->p_vcolor = ((color) + 1)) 1421 1422 /* 1423 * As mentioned p_vcolor == 0 means there is no color for this page. 1424 * But PP_SET_VCOLOR(pp, color) expects 'color' to be real color minus 1425 * one so we define this constant. 1426 */ 1427 #define NO_VCOLOR (-1) 1428 1429 #define addr_to_vcolor(addr) \ 1430 (((uint_t)(uintptr_t)(addr) >> MMU_PAGESHIFT) & vac_colors_mask) 1431 1432 /* 1433 * The field p_index in the psm page structure is for large pages support. 1434 * P_index is a bit-vector of the different mapping sizes that a given page 1435 * is part of. An hme structure for a large mapping is only added in the 1436 * group leader page (first page). All pages covered by a given large mapping 1437 * have the corrosponding mapping bit set in their p_index field. This allows 1438 * us to only store an explicit hme structure in the leading page which 1439 * simplifies the mapping link list management. Furthermore, it provides us 1440 * a fast mechanism for determining the largest mapping a page is part of. For 1441 * exmaple, a page with a 64K and a 4M mappings has a p_index value of 0x0A. 1442 * 1443 * Implementation note: even though the first bit in p_index is reserved 1444 * for 8K mappings, it is NOT USED by the code and SHOULD NOT be set. 1445 * In addition, the upper four bits of the p_index field are used by the 1446 * code as temporaries 1447 */ 1448 1449 /* 1450 * Defines for psm page struct fields and large page support 1451 */ 1452 #define SFMMU_INDEX_SHIFT 6 1453 #define SFMMU_INDEX_MASK ((1 << SFMMU_INDEX_SHIFT) - 1) 1454 1455 /* Return the mapping index */ 1456 #define PP_MAPINDEX(pp) ((pp)->p_index & SFMMU_INDEX_MASK) 1457 1458 /* 1459 * These macros rely on the following property: 1460 * All pages constituting a large page are covered by a virtually 1461 * contiguous set of page_t's. 1462 */ 1463 1464 /* Return the leader for this mapping size */ 1465 #define PP_GROUPLEADER(pp, sz) \ 1466 (&(pp)[-(int)(pp->p_pagenum & (TTEPAGES(sz)-1))]) 1467 1468 /* Return the root page for this page based on p_szc */ 1469 #define PP_PAGEROOT(pp) ((pp)->p_szc == 0 ? (pp) : \ 1470 PP_GROUPLEADER((pp), (pp)->p_szc)) 1471 1472 #define PP_PAGENEXT_N(pp, n) ((pp) + (n)) 1473 #define PP_PAGENEXT(pp) PP_PAGENEXT_N((pp), 1) 1474 1475 #define PP_PAGEPREV_N(pp, n) ((pp) - (n)) 1476 #define PP_PAGEPREV(pp) PP_PAGEPREV_N((pp), 1) 1477 1478 #define PP_ISMAPPED_LARGE(pp) (PP_MAPINDEX(pp) != 0) 1479 1480 /* Need function to test the page mappping which takes p_index into account */ 1481 #define PP_ISMAPPED(pp) ((pp)->p_mapping || PP_ISMAPPED_LARGE(pp)) 1482 1483 /* 1484 * Don't call this macro with sz equal to zero. 8K mappings SHOULD NOT 1485 * set p_index field. 1486 */ 1487 #define PAGESZ_TO_INDEX(sz) (1 << (sz)) 1488 1489 1490 /* 1491 * prototypes for hat assembly routines. Some of these are 1492 * known to machine dependent VM code. 1493 */ 1494 extern uint64_t sfmmu_make_tsbtag(caddr_t); 1495 extern struct tsbe * 1496 sfmmu_get_tsbe(uint64_t, caddr_t, int, int); 1497 extern void sfmmu_load_tsbe(struct tsbe *, uint64_t, tte_t *, int); 1498 extern void sfmmu_unload_tsbe(struct tsbe *, uint64_t, int); 1499 extern void sfmmu_load_mmustate(sfmmu_t *); 1500 extern void sfmmu_ctx_steal_tl1(uint64_t, uint64_t); 1501 extern void sfmmu_raise_tsb_exception(uint64_t, uint64_t); 1502 #ifndef sun4v 1503 extern void sfmmu_itlb_ld(caddr_t, int, tte_t *); 1504 extern void sfmmu_dtlb_ld(caddr_t, int, tte_t *); 1505 #endif /* sun4v */ 1506 extern void sfmmu_copytte(tte_t *, tte_t *); 1507 extern int sfmmu_modifytte(tte_t *, tte_t *, tte_t *); 1508 extern int sfmmu_modifytte_try(tte_t *, tte_t *, tte_t *); 1509 extern pfn_t sfmmu_ttetopfn(tte_t *, caddr_t); 1510 extern void sfmmu_hblk_hash_rm(struct hmehash_bucket *, 1511 struct hme_blk *, uint64_t, struct hme_blk *); 1512 extern void sfmmu_hblk_hash_add(struct hmehash_bucket *, struct hme_blk *, 1513 uint64_t); 1514 1515 /* 1516 * functions exported to machine dependent VM code 1517 */ 1518 extern void sfmmu_patch_ktsb(void); 1519 #ifndef sun4v 1520 extern void sfmmu_patch_utsb(void); 1521 #endif /* sun4v */ 1522 extern pfn_t sfmmu_vatopfn(caddr_t, sfmmu_t *, tte_t *); 1523 extern void sfmmu_vatopfn_suspended(caddr_t, sfmmu_t *, tte_t *); 1524 #ifdef DEBUG 1525 extern void sfmmu_check_kpfn(pfn_t); 1526 #else 1527 #define sfmmu_check_kpfn(pfn) /* disabled */ 1528 #endif /* DEBUG */ 1529 extern void sfmmu_memtte(tte_t *, pfn_t, uint_t, int); 1530 extern void sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *, uint_t); 1531 extern void sfmmu_tsbmiss_exception(struct regs *, uintptr_t, uint_t); 1532 extern void sfmmu_init_tsbs(void); 1533 extern caddr_t sfmmu_ktsb_alloc(caddr_t); 1534 extern int sfmmu_getctx_pri(void); 1535 extern int sfmmu_getctx_sec(void); 1536 extern void sfmmu_setctx_sec(int); 1537 extern void sfmmu_inv_tsb(caddr_t, uint_t); 1538 extern void sfmmu_init_ktsbinfo(void); 1539 extern int sfmmu_setup_4lp(void); 1540 extern void sfmmu_patch_mmu_asi(int); 1541 extern void sfmmu_init_nucleus_hblks(caddr_t, size_t, int, int); 1542 extern void sfmmu_cache_flushall(void); 1543 extern pgcnt_t sfmmu_tte_cnt(sfmmu_t *, uint_t); 1544 extern void *sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int); 1545 extern void sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t); 1546 extern void sfmmu_steal_context(sfmmu_t *, uint8_t *); 1547 1548 extern void hat_kern_setup(void); 1549 extern int hat_page_relocate(page_t **, page_t **, spgcnt_t *); 1550 extern uint_t hat_preferred_pgsz(struct hat *, caddr_t, size_t, int); 1551 extern int sfmmu_get_ppvcolor(struct page *); 1552 extern int sfmmu_get_addrvcolor(caddr_t); 1553 extern int sfmmu_hat_lock_held(sfmmu_t *); 1554 1555 /* 1556 * Functions exported to xhat_sfmmu.c 1557 */ 1558 extern kmutex_t *sfmmu_mlist_enter(page_t *); 1559 extern void sfmmu_mlist_exit(kmutex_t *); 1560 extern int sfmmu_mlist_held(struct page *); 1561 extern struct hme_blk *sfmmu_hmetohblk(struct sf_hment *); 1562 1563 /* 1564 * MMU-specific functions optionally imported from the CPU module 1565 */ 1566 #pragma weak mmu_large_pages_disabled 1567 #pragma weak mmu_set_ctx_page_sizes 1568 #pragma weak mmu_preferred_pgsz 1569 #pragma weak mmu_check_page_sizes 1570 1571 extern int mmu_large_pages_disabled(uint_t); 1572 extern void mmu_set_ctx_page_sizes(sfmmu_t *); 1573 extern uint_t mmu_preferred_pgsz(sfmmu_t *, caddr_t, size_t); 1574 extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *); 1575 1576 extern sfmmu_t *ksfmmup; 1577 extern struct ctx *ctxs; 1578 extern uint_t nctxs; 1579 extern caddr_t ktsb_base; 1580 extern uint64_t ktsb_pbase; 1581 extern int ktsb_sz; 1582 extern int ktsb_szcode; 1583 extern caddr_t ktsb4m_base; 1584 extern uint64_t ktsb4m_pbase; 1585 extern int ktsb4m_sz; 1586 extern int ktsb4m_szcode; 1587 extern uint64_t kpm_tsbbase; 1588 extern int kpm_tsbsz; 1589 extern int ktsb_phys; 1590 extern int enable_bigktsb; 1591 #ifndef sun4v 1592 extern int utsb_dtlb_ttenum; 1593 extern int utsb4m_dtlb_ttenum; 1594 #endif /* sun4v */ 1595 extern int uhmehash_num; 1596 extern int khmehash_num; 1597 extern struct hmehash_bucket *uhme_hash; 1598 extern struct hmehash_bucket *khme_hash; 1599 extern kmutex_t *mml_table; 1600 extern uint_t mml_table_sz; 1601 extern uint_t mml_shift; 1602 extern uint_t hblk_alloc_dynamic; 1603 extern struct tsbmiss tsbmiss_area[NCPU]; 1604 extern struct kpmtsbm kpmtsbm_area[NCPU]; 1605 extern int tsb_max_growsize; 1606 #ifndef sun4v 1607 extern int dtlb_resv_ttenum; 1608 extern caddr_t utsb_vabase; 1609 extern caddr_t utsb4m_vabase; 1610 #endif /* sun4v */ 1611 extern vmem_t *kmem_tsb_default_arena[]; 1612 extern int tsb_lgrp_affinity; 1613 1614 /* kpm externals */ 1615 extern pfn_t sfmmu_kpm_vatopfn(caddr_t); 1616 extern void sfmmu_kpm_patch_tlbm(void); 1617 extern void sfmmu_kpm_patch_tsbm(void); 1618 extern void sfmmu_kpm_load_tsb(caddr_t, tte_t *, int); 1619 extern void sfmmu_kpm_unload_tsb(caddr_t, int); 1620 extern void sfmmu_kpm_tsbmtl(short *, uint_t *, int); 1621 extern int sfmmu_kpm_stsbmtl(char *, uint_t *, int); 1622 extern caddr_t kpm_vbase; 1623 extern size_t kpm_size; 1624 extern struct memseg *memseg_hash[]; 1625 extern uint64_t memseg_phash[]; 1626 extern kpm_hlk_t *kpmp_table; 1627 extern kpm_shlk_t *kpmp_stable; 1628 extern uint_t kpmp_table_sz; 1629 extern uint_t kpmp_stable_sz; 1630 extern uchar_t kpmp_shift; 1631 1632 #define PP_ISMAPPED_KPM(pp) ((pp)->p_kpmref > 0) 1633 1634 #define IS_KPM_ALIAS_RANGE(vaddr) \ 1635 (((vaddr) - kpm_vbase) >> (uintptr_t)kpm_size_shift > 0) 1636 1637 #endif /* !_ASM */ 1638 1639 /* sfmmu_kpm_tsbmtl flags */ 1640 #define KPMTSBM_STOP 0 1641 #define KPMTSBM_START 1 1642 1643 /* kpm_smallpages kp_mapped values */ 1644 #define KPM_MAPPEDS -1 /* small mapping valid, no conflict */ 1645 #define KPM_MAPPEDSC 1 /* small mapping valid, conflict */ 1646 1647 /* Physical memseg address NULL marker */ 1648 #define MSEG_NULLPTR_PA -1 1649 1650 /* 1651 * Memseg hash defines for kpm trap level tsbmiss handler. 1652 * Must be in sync w/ page.h . 1653 */ 1654 #define SFMMU_MEM_HASH_SHIFT 0x9 1655 #define SFMMU_N_MEM_SLOTS 0x200 1656 #define SFMMU_MEM_HASH_ENTRY_SHIFT 3 1657 1658 #ifndef _ASM 1659 #if (SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT) 1660 #error SFMMU_MEM_HASH_SHIFT != MEM_HASH_SHIFT 1661 #endif 1662 #if (SFMMU_N_MEM_SLOTS != N_MEM_SLOTS) 1663 #error SFMMU_N_MEM_SLOTS != N_MEM_SLOTS 1664 #endif 1665 1666 /* Physical memseg address NULL marker */ 1667 #define SFMMU_MEMSEG_NULLPTR_PA -1 1668 1669 /* 1670 * Check KCONTEXT to be zero, asm parts depend on that assumption. 1671 */ 1672 #if (KCONTEXT != 0) 1673 #error KCONTEXT != 0 1674 #endif 1675 #endif /* !_ASM */ 1676 1677 1678 #endif /* _KERNEL */ 1679 1680 #ifndef _ASM 1681 /* 1682 * ctx, hmeblk, mlistlock and other stats for sfmmu 1683 */ 1684 struct sfmmu_global_stat { 1685 int sf_tsb_exceptions; /* # of tsb exceptions */ 1686 int sf_tsb_raise_exception; /* # tsb exc. w/o TLB flush */ 1687 1688 int sf_pagefaults; /* # of pagefaults */ 1689 1690 int sf_uhash_searches; /* # of user hash searches */ 1691 int sf_uhash_links; /* # of user hash links */ 1692 int sf_khash_searches; /* # of kernel hash searches */ 1693 int sf_khash_links; /* # of kernel hash links */ 1694 1695 int sf_swapout; /* # times hat swapped out */ 1696 1697 int sf_ctxfree; /* ctx alloc from free list */ 1698 int sf_ctxdirty; /* ctx alloc from dirty list */ 1699 int sf_ctxsteal; /* ctx allocated by steal */ 1700 1701 int sf_tsb_alloc; /* # TSB allocations */ 1702 int sf_tsb_allocfail; /* # times TSB alloc fail */ 1703 int sf_tsb_sectsb_create; /* # times second TSB added */ 1704 1705 int sf_tteload8k; /* calls to sfmmu_tteload */ 1706 int sf_tteload64k; /* calls to sfmmu_tteload */ 1707 int sf_tteload512k; /* calls to sfmmu_tteload */ 1708 int sf_tteload4m; /* calls to sfmmu_tteload */ 1709 int sf_tteload32m; /* calls to sfmmu_tteload */ 1710 int sf_tteload256m; /* calls to sfmmu_tteload */ 1711 1712 int sf_tsb_load8k; /* # times loaded 8K tsbent */ 1713 int sf_tsb_load4m; /* # times loaded 4M tsbent */ 1714 1715 int sf_hblk_hit; /* found hblk during tteload */ 1716 int sf_hblk8_ncreate; /* static hblk8's created */ 1717 int sf_hblk8_nalloc; /* static hblk8's allocated */ 1718 int sf_hblk1_ncreate; /* static hblk1's created */ 1719 int sf_hblk1_nalloc; /* static hblk1's allocated */ 1720 int sf_hblk_slab_cnt; /* sfmmu8_cache slab creates */ 1721 int sf_hblk_reserve_cnt; /* hblk_reserve usage */ 1722 int sf_hblk_recurse_cnt; /* hblk_reserve owner reqs */ 1723 int sf_hblk_reserve_hit; /* hblk_reserve hash hits */ 1724 int sf_get_free_success; /* reserve list allocs */ 1725 int sf_get_free_throttle; /* fails due to throttling */ 1726 int sf_get_free_fail; /* fails due to empty list */ 1727 int sf_put_free_success; /* reserve list frees */ 1728 int sf_put_free_fail; /* fails due to full list */ 1729 1730 int sf_pgcolor_conflict; /* VAC conflict resolution */ 1731 int sf_uncache_conflict; /* VAC conflict resolution */ 1732 int sf_unload_conflict; /* VAC unload resolution */ 1733 int sf_ism_uncache; /* VAC conflict resolution */ 1734 int sf_ism_recache; /* VAC conflict resolution */ 1735 int sf_recache; /* VAC conflict resolution */ 1736 1737 int sf_steal_count; /* # of hblks stolen */ 1738 1739 int sf_pagesync; /* # of pagesyncs */ 1740 int sf_clrwrt; /* # of clear write perms */ 1741 int sf_pagesync_invalid; /* pagesync with inv tte */ 1742 1743 int sf_kernel_xcalls; /* # of kernel cross calls */ 1744 int sf_user_xcalls; /* # of user cross calls */ 1745 1746 int sf_tsb_grow; /* # of user tsb grows */ 1747 int sf_tsb_shrink; /* # of user tsb shrinks */ 1748 int sf_tsb_resize_failures; /* # of user tsb resize */ 1749 int sf_tsb_reloc; /* # of user tsb relocations */ 1750 1751 int sf_user_vtop; /* # of user vatopfn calls */ 1752 1753 int sf_ctx_swap; /* # times switched MMU ctxs */ 1754 int sf_tlbflush_all; /* # times flush all TLBs */ 1755 int sf_tlbflush_ctx; /* # times flush TLB ctx */ 1756 int sf_tlbflush_deferred; /* # times !flush ctx imm. */ 1757 1758 int sf_tlb_reprog_pgsz; /* # times switch TLB pgsz */ 1759 }; 1760 1761 struct sfmmu_tsbsize_stat { 1762 int sf_tsbsz_8k; 1763 int sf_tsbsz_16k; 1764 int sf_tsbsz_32k; 1765 int sf_tsbsz_64k; 1766 int sf_tsbsz_128k; 1767 int sf_tsbsz_256k; 1768 int sf_tsbsz_512k; 1769 int sf_tsbsz_1m; 1770 int sf_tsbsz_2m; 1771 int sf_tsbsz_4m; 1772 }; 1773 1774 struct sfmmu_percpu_stat { 1775 int sf_itlb_misses; /* # of itlb misses */ 1776 int sf_dtlb_misses; /* # of dtlb misses */ 1777 int sf_utsb_misses; /* # of user tsb misses */ 1778 int sf_ktsb_misses; /* # of kernel tsb misses */ 1779 int sf_tsb_hits; /* # of tsb hits */ 1780 int sf_umod_faults; /* # of mod (prot viol) flts */ 1781 int sf_kmod_faults; /* # of mod (prot viol) flts */ 1782 }; 1783 1784 #define SFMMU_STAT(stat) sfmmu_global_stat.stat++; 1785 #define SFMMU_STAT_ADD(stat, amount) sfmmu_global_stat.stat += amount; 1786 #define SFMMU_STAT_SET(stat, count) sfmmu_global_stat.stat = count; 1787 1788 #endif /* !_ASM */ 1789 1790 #ifdef __cplusplus 1791 } 1792 #endif 1793 1794 #endif /* _VM_HAT_SFMMU_H */ 1795