1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/t_lock.h> 27 #include <sys/memlist.h> 28 #include <sys/cpuvar.h> 29 #include <sys/vmem.h> 30 #include <sys/mman.h> 31 #include <sys/vm.h> 32 #include <sys/kmem.h> 33 #include <sys/cmn_err.h> 34 #include <sys/debug.h> 35 #include <sys/vm_machparam.h> 36 #include <sys/tss.h> 37 #include <sys/vnode.h> 38 #include <vm/hat.h> 39 #include <vm/anon.h> 40 #include <vm/as.h> 41 #include <vm/page.h> 42 #include <vm/seg.h> 43 #include <vm/seg_kmem.h> 44 #include <vm/seg_map.h> 45 #include <vm/hat_i86.h> 46 #include <sys/promif.h> 47 #include <sys/x86_archext.h> 48 #include <sys/systm.h> 49 #include <sys/archsystm.h> 50 #include <sys/sunddi.h> 51 #include <sys/ddidmareq.h> 52 #include <sys/controlregs.h> 53 #include <sys/reboot.h> 54 #include <sys/kdi.h> 55 #include <sys/bootconf.h> 56 #include <sys/bootsvcs.h> 57 #include <sys/bootinfo.h> 58 #include <vm/kboot_mmu.h> 59 60 #ifdef __xpv 61 #include <sys/hypervisor.h> 62 #endif 63 64 caddr_t 65 i86devmap(pfn_t pf, pgcnt_t pgcnt, uint_t prot) 66 { 67 caddr_t addr; 68 caddr_t addr1; 69 page_t *pp; 70 71 addr1 = addr = vmem_alloc(heap_arena, mmu_ptob(pgcnt), VM_SLEEP); 72 73 for (; pgcnt != 0; addr += MMU_PAGESIZE, ++pf, --pgcnt) { 74 pp = page_numtopp_nolock(pf); 75 if (pp == NULL) { 76 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pf, 77 prot | HAT_NOSYNC, HAT_LOAD_LOCK); 78 } else { 79 hat_memload(kas.a_hat, addr, pp, 80 prot | HAT_NOSYNC, HAT_LOAD_LOCK); 81 } 82 } 83 84 return (addr1); 85 } 86 87 /* 88 * This routine is like page_numtopp, but accepts only free pages, which 89 * it allocates (unfrees) and returns with the exclusive lock held. 90 * It is used by machdep.c/dma_init() to find contiguous free pages. 91 * 92 * XXX this and some others should probably be in vm_machdep.c 93 */ 94 page_t * 95 page_numtopp_alloc(pfn_t pfnum) 96 { 97 page_t *pp; 98 99 retry: 100 pp = page_numtopp_nolock(pfnum); 101 if (pp == NULL) { 102 return (NULL); 103 } 104 105 if (!page_trylock(pp, SE_EXCL)) { 106 return (NULL); 107 } 108 109 if (page_pptonum(pp) != pfnum) { 110 page_unlock(pp); 111 goto retry; 112 } 113 114 if (!PP_ISFREE(pp)) { 115 page_unlock(pp); 116 return (NULL); 117 } 118 if (pp->p_szc) { 119 page_demote_free_pages(pp); 120 page_unlock(pp); 121 goto retry; 122 } 123 124 /* If associated with a vnode, destroy mappings */ 125 126 if (pp->p_vnode) { 127 128 page_destroy_free(pp); 129 130 if (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_NO_RECLAIM)) { 131 return (NULL); 132 } 133 134 if (page_pptonum(pp) != pfnum) { 135 page_unlock(pp); 136 goto retry; 137 } 138 } 139 140 if (!PP_ISFREE(pp)) { 141 page_unlock(pp); 142 return (NULL); 143 } 144 145 if (!page_reclaim(pp, (kmutex_t *)NULL)) 146 return (NULL); 147 148 return (pp); 149 } 150 151 /* 152 * Flag is not set early in boot. Once it is set we are no longer 153 * using boot's page tables. 154 */ 155 uint_t khat_running = 0; 156 157 /* 158 * This procedure is callable only while the boot loader is in charge of the 159 * MMU. It assumes that PA == VA for page table pointers. It doesn't live in 160 * kboot_mmu.c since it's used from common code. 161 */ 162 pfn_t 163 va_to_pfn(void *vaddr) 164 { 165 uintptr_t des_va = ALIGN2PAGE(vaddr); 166 uintptr_t va = des_va; 167 size_t len; 168 uint_t prot; 169 pfn_t pfn; 170 171 if (khat_running) 172 panic("va_to_pfn(): called too late\n"); 173 174 if (kbm_probe(&va, &len, &pfn, &prot) == 0) 175 return (PFN_INVALID); 176 if (va > des_va) 177 return (PFN_INVALID); 178 if (va < des_va) 179 pfn += mmu_btop(des_va - va); 180 return (pfn); 181 } 182 183 /* 184 * Initialize a special area in the kernel that always holds some PTEs for 185 * faster performance. This always holds segmap's PTEs. 186 * In the 32 bit kernel this maps the kernel heap too. 187 */ 188 void 189 hat_kmap_init(uintptr_t base, size_t len) 190 { 191 uintptr_t map_addr; /* base rounded down to large page size */ 192 uintptr_t map_eaddr; /* base + len rounded up */ 193 size_t map_len; 194 caddr_t ptes; /* mapping area in kernel for kmap ptes */ 195 size_t window_size; /* size of mapping area for ptes */ 196 ulong_t htable_cnt; /* # of page tables to cover map_len */ 197 ulong_t i; 198 htable_t *ht; 199 uintptr_t va; 200 201 /* 202 * We have to map in an area that matches an entire page table. 203 * The PTEs are large page aligned to avoid spurious pagefaults 204 * on the hypervisor. 205 */ 206 map_addr = base & LEVEL_MASK(1); 207 map_eaddr = (base + len + LEVEL_SIZE(1) - 1) & LEVEL_MASK(1); 208 map_len = map_eaddr - map_addr; 209 window_size = mmu_btop(map_len) * mmu.pte_size; 210 window_size = (window_size + LEVEL_SIZE(1)) & LEVEL_MASK(1); 211 htable_cnt = map_len >> LEVEL_SHIFT(1); 212 213 /* 214 * allocate vmem for the kmap_ptes 215 */ 216 ptes = vmem_xalloc(heap_arena, window_size, LEVEL_SIZE(1), 0, 217 0, NULL, NULL, VM_SLEEP); 218 mmu.kmap_htables = 219 kmem_alloc(htable_cnt * sizeof (htable_t *), KM_SLEEP); 220 221 /* 222 * Map the page tables that cover kmap into the allocated range. 223 * Note we don't ever htable_release() the kmap page tables - they 224 * can't ever be stolen, freed, etc. 225 */ 226 for (va = map_addr, i = 0; i < htable_cnt; va += LEVEL_SIZE(1), ++i) { 227 ht = htable_create(kas.a_hat, va, 0, NULL); 228 if (ht == NULL) 229 panic("hat_kmap_init: ht == NULL"); 230 mmu.kmap_htables[i] = ht; 231 232 hat_devload(kas.a_hat, ptes + i * MMU_PAGESIZE, 233 MMU_PAGESIZE, ht->ht_pfn, 234 #ifdef __xpv 235 PROT_READ | HAT_NOSYNC | HAT_UNORDERED_OK, 236 #else 237 PROT_READ | PROT_WRITE | HAT_NOSYNC | HAT_UNORDERED_OK, 238 #endif 239 HAT_LOAD | HAT_LOAD_NOCONSIST); 240 } 241 242 /* 243 * set information in mmu to activate handling of kmap 244 */ 245 mmu.kmap_addr = map_addr; 246 mmu.kmap_eaddr = map_eaddr; 247 mmu.kmap_ptes = (x86pte_t *)ptes; 248 } 249 250 extern caddr_t kpm_vbase; 251 extern size_t kpm_size; 252 253 #ifdef __xpv 254 /* 255 * Create the initial segkpm mappings for the hypervisor. To avoid having 256 * to deal with page tables being read only, we make all mappings 257 * read only at first. 258 */ 259 static void 260 xen_kpm_create(paddr_t paddr, level_t lvl) 261 { 262 ulong_t pg_off; 263 264 for (pg_off = 0; pg_off < LEVEL_SIZE(lvl); pg_off += MMU_PAGESIZE) { 265 kbm_map((uintptr_t)kpm_vbase + paddr, (paddr_t)0, 0, 1); 266 kbm_read_only((uintptr_t)kpm_vbase + paddr + pg_off, 267 paddr + pg_off); 268 } 269 } 270 271 /* 272 * Try to make all kpm mappings writable. Failures are ok, as those 273 * are just pagetable, GDT, etc. pages. 274 */ 275 static void 276 xen_kpm_finish_init(void) 277 { 278 pfn_t gdtpfn = mmu_btop(CPU->cpu_m.mcpu_gdtpa); 279 pfn_t pfn; 280 page_t *pp; 281 282 for (pfn = 0; pfn < mfn_count; ++pfn) { 283 /* 284 * skip gdt 285 */ 286 if (pfn == gdtpfn) 287 continue; 288 289 /* 290 * p_index is a hint that this is a pagetable 291 */ 292 pp = page_numtopp_nolock(pfn); 293 if (pp && pp->p_index) { 294 pp->p_index = 0; 295 continue; 296 } 297 (void) xen_kpm_page(pfn, PT_VALID | PT_WRITABLE); 298 } 299 } 300 #endif 301 302 /* 303 * Routine to pre-allocate data structures for hat_kern_setup(). It computes 304 * how many pagetables it needs by walking the boot loader's page tables. 305 */ 306 /*ARGSUSED*/ 307 void 308 hat_kern_alloc( 309 caddr_t segmap_base, 310 size_t segmap_size, 311 caddr_t ekernelheap) 312 { 313 uintptr_t last_va = (uintptr_t)-1; /* catch 1st time */ 314 uintptr_t va = 0; 315 size_t size; 316 pfn_t pfn; 317 uint_t prot; 318 uint_t table_cnt = 1; 319 uint_t mapping_cnt; 320 level_t start_level; 321 level_t l; 322 struct memlist *pmem; 323 level_t lpagel = mmu.max_page_level; 324 uint64_t paddr; 325 int64_t psize; 326 int nwindows; 327 328 if (kpm_size > 0) { 329 /* 330 * Create the kpm page tables. When running on the 331 * hypervisor these are made read/only at first. 332 * Later we'll add write permission where possible. 333 */ 334 for (pmem = phys_install; pmem; pmem = pmem->ml_next) { 335 paddr = pmem->ml_address; 336 psize = pmem->ml_size; 337 while (psize >= MMU_PAGESIZE) { 338 /* find the largest page size */ 339 for (l = lpagel; l > 0; l--) { 340 if ((paddr & LEVEL_OFFSET(l)) == 0 && 341 psize > LEVEL_SIZE(l)) 342 break; 343 } 344 345 #if defined(__xpv) 346 /* 347 * Create read/only mappings to avoid 348 * conflicting with pagetable usage 349 */ 350 xen_kpm_create(paddr, l); 351 #else 352 kbm_map((uintptr_t)kpm_vbase + paddr, paddr, 353 l, 1); 354 #endif 355 paddr += LEVEL_SIZE(l); 356 psize -= LEVEL_SIZE(l); 357 } 358 } 359 } 360 361 /* 362 * If this machine doesn't have a kpm segment, we need to allocate 363 * a small number of 'windows' which can be used to map pagetables. 364 */ 365 nwindows = (kpm_size == 0) ? 2 * NCPU : 0; 366 367 #if defined(__xpv) 368 /* 369 * On a hypervisor, these windows are also used by the xpv_panic 370 * code, where we need one window for each level of the pagetable 371 * hierarchy. 372 */ 373 nwindows = MAX(nwindows, mmu.max_level); 374 #endif 375 376 if (nwindows != 0) { 377 /* 378 * Create the page windows and 1 page of VA in 379 * which we map the PTEs of those windows. 380 */ 381 mmu.pwin_base = vmem_xalloc(heap_arena, nwindows * MMU_PAGESIZE, 382 LEVEL_SIZE(1), 0, 0, NULL, NULL, VM_SLEEP); 383 ASSERT(nwindows <= MMU_PAGESIZE / mmu.pte_size); 384 mmu.pwin_pte_va = vmem_xalloc(heap_arena, MMU_PAGESIZE, 385 MMU_PAGESIZE, 0, 0, NULL, NULL, VM_SLEEP); 386 387 /* 388 * Find/Create the page table window mappings. 389 */ 390 paddr = 0; 391 (void) find_pte((uintptr_t)mmu.pwin_base, &paddr, 0, 0); 392 ASSERT(paddr != 0); 393 ASSERT((paddr & MMU_PAGEOFFSET) == 0); 394 mmu.pwin_pte_pa = paddr; 395 #ifdef __xpv 396 (void) find_pte((uintptr_t)mmu.pwin_pte_va, NULL, 0, 0); 397 kbm_read_only((uintptr_t)mmu.pwin_pte_va, mmu.pwin_pte_pa); 398 #else 399 kbm_map((uintptr_t)mmu.pwin_pte_va, mmu.pwin_pte_pa, 0, 1); 400 #endif 401 } 402 403 /* 404 * Walk the boot loader's page tables and figure out 405 * how many tables and page mappings there will be. 406 */ 407 while (kbm_probe(&va, &size, &pfn, &prot) != 0) { 408 /* 409 * At each level, if the last_va falls into a new htable, 410 * increment table_cnt. We can stop at the 1st level where 411 * they are in the same htable. 412 */ 413 start_level = 0; 414 while (start_level <= mmu.max_page_level) { 415 if (size == LEVEL_SIZE(start_level)) 416 break; 417 start_level++; 418 } 419 420 for (l = start_level; l < mmu.max_level; ++l) { 421 if (va >> LEVEL_SHIFT(l + 1) == 422 last_va >> LEVEL_SHIFT(l + 1)) 423 break; 424 ++table_cnt; 425 } 426 last_va = va; 427 l = (start_level == 0) ? 1 : start_level; 428 va = (va & LEVEL_MASK(l)) + LEVEL_SIZE(l); 429 } 430 431 /* 432 * Besides the boot loader mappings, we're going to fill in 433 * the entire top level page table for the kernel. Make sure there's 434 * enough reserve for that too. 435 */ 436 table_cnt += mmu.top_level_count - ((kernelbase >> 437 LEVEL_SHIFT(mmu.max_level)) & (mmu.top_level_count - 1)); 438 439 #if defined(__i386) 440 /* 441 * The 32 bit PAE hat allocates tables one level below the top when 442 * kernelbase isn't 1 Gig aligned. We'll just be sloppy and allocate 443 * a bunch more to the reserve. Any unused will be returned later. 444 * Note we've already counted these mappings, just not the extra 445 * pagetables. 446 */ 447 if (mmu.pae_hat != 0 && (kernelbase & LEVEL_OFFSET(mmu.max_level)) != 0) 448 table_cnt += mmu.ptes_per_table - 449 ((kernelbase & LEVEL_OFFSET(mmu.max_level)) >> 450 LEVEL_SHIFT(mmu.max_level - 1)); 451 #endif 452 453 /* 454 * Add 1/4 more into table_cnt for extra slop. The unused 455 * slop is freed back when we htable_adjust_reserve() later. 456 */ 457 table_cnt += table_cnt >> 2; 458 459 /* 460 * We only need mapping entries (hments) for shared pages. 461 * This should be far, far fewer than the total possible, 462 * We'll allocate enough for 1/16 of all possible PTEs. 463 */ 464 mapping_cnt = (table_cnt * mmu.ptes_per_table) >> 4; 465 466 /* 467 * Now create the initial htable/hment reserves 468 */ 469 htable_initial_reserve(table_cnt); 470 hment_reserve(mapping_cnt); 471 x86pte_cpu_init(CPU); 472 } 473 474 475 /* 476 * This routine handles the work of creating the kernel's initial mappings 477 * by deciphering the mappings in the page tables created by the boot program. 478 * 479 * We maintain large page mappings, but only to a level 1 pagesize. 480 * The boot loader can only add new mappings once this function starts. 481 * In particular it can not change the pagesize used for any existing 482 * mappings or this code breaks! 483 */ 484 485 void 486 hat_kern_setup(void) 487 { 488 /* 489 * Attach htables to the existing pagetables 490 */ 491 /* BEGIN CSTYLED */ 492 htable_attach(kas.a_hat, 0, mmu.max_level, NULL, 493 #ifdef __xpv 494 mmu_btop(xen_info->pt_base - ONE_GIG)); 495 #else 496 mmu_btop(getcr3())); 497 #endif 498 /* END CSTYLED */ 499 500 #if defined(__i386) && !defined(__xpv) 501 CPU->cpu_tss->tss_cr3 = dftss0->tss_cr3 = getcr3(); 502 #endif /* __i386 */ 503 504 #if defined(__xpv) && defined(__amd64) 505 /* 506 * Try to make the kpm mappings r/w. Failures here are OK, as 507 * it's probably just a pagetable 508 */ 509 xen_kpm_finish_init(); 510 #endif 511 512 /* 513 * The kernel HAT is now officially open for business. 514 */ 515 khat_running = 1; 516 517 CPUSET_ATOMIC_ADD(kas.a_hat->hat_cpus, CPU->cpu_id); 518 CPU->cpu_current_hat = kas.a_hat; 519 } 520