1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/systm.h> 29 #include <sys/archsystm.h> 30 #include <sys/debug.h> 31 #include <sys/bootconf.h> 32 #include <sys/bootsvcs.h> 33 #include <sys/bootinfo.h> 34 #include <sys/mman.h> 35 #include <sys/cmn_err.h> 36 #include <sys/param.h> 37 #include <sys/machparam.h> 38 #include <sys/machsystm.h> 39 #include <sys/promif.h> 40 #include <sys/kobj.h> 41 #ifdef __xpv 42 #include <sys/hypervisor.h> 43 #endif 44 #include <vm/kboot_mmu.h> 45 #include <vm/hat_pte.h> 46 #include <vm/hat_i86.h> 47 #include <vm/seg_kmem.h> 48 49 #if 0 50 /* 51 * Joe's debug printing 52 */ 53 #define DBG(x) \ 54 bop_printf(NULL, "kboot_mmu.c: %s is %" PRIx64 "\n", #x, (uint64_t)(x)); 55 #else 56 #define DBG(x) /* naught */ 57 #endif 58 59 /* 60 * Page table and memory stuff. 61 */ 62 static caddr_t window; 63 static caddr_t pte_to_window; 64 65 /* 66 * this are needed by mmu_init() 67 */ 68 int kbm_nx_support = 0; /* NX bit in PTEs is in use */ 69 int kbm_pae_support = 0; /* PAE is 64 bit Page table entries */ 70 int kbm_pge_support = 0; /* PGE is Page table global bit enabled */ 71 int kbm_largepage_support = 0; 72 uint_t kbm_nucleus_size = 0; 73 74 #define BOOT_SHIFT(l) (shift_amt[l]) 75 #define BOOT_SZ(l) ((size_t)1 << BOOT_SHIFT(l)) 76 #define BOOT_OFFSET(l) (BOOT_SZ(l) - 1) 77 #define BOOT_MASK(l) (~BOOT_OFFSET(l)) 78 79 /* 80 * Initialize memory management parameters for boot time page table management 81 */ 82 void 83 kbm_init(struct xboot_info *bi) 84 { 85 /* 86 * configure mmu information 87 */ 88 kbm_nucleus_size = (uintptr_t)bi->bi_kseg_size; 89 kbm_largepage_support = bi->bi_use_largepage; 90 kbm_nx_support = bi->bi_use_nx; 91 kbm_pae_support = bi->bi_use_pae; 92 kbm_pge_support = bi->bi_use_pge; 93 window = bi->bi_pt_window; 94 DBG(window); 95 pte_to_window = bi->bi_pte_to_pt_window; 96 DBG(pte_to_window); 97 if (kbm_pae_support) { 98 shift_amt = shift_amt_pae; 99 ptes_per_table = 512; 100 pte_size = 8; 101 lpagesize = TWO_MEG; 102 #ifdef __amd64 103 top_level = 3; 104 #else 105 top_level = 2; 106 #endif 107 } else { 108 shift_amt = shift_amt_nopae; 109 ptes_per_table = 1024; 110 pte_size = 4; 111 lpagesize = FOUR_MEG; 112 top_level = 1; 113 } 114 115 #ifdef __xpv 116 xen_info = bi->bi_xen_start_info; 117 mfn_list = (mfn_t *)xen_info->mfn_list; 118 DBG(mfn_list); 119 mfn_count = xen_info->nr_pages; 120 DBG(mfn_count); 121 #endif 122 top_page_table = bi->bi_top_page_table; 123 DBG(top_page_table); 124 } 125 126 /* 127 * Change the addressible page table window to point at a given page 128 */ 129 /*ARGSUSED*/ 130 void * 131 kbm_remap_window(paddr_t physaddr, int writeable) 132 { 133 x86pte_t pt_bits = PT_NOCONSIST | PT_VALID | PT_WRITABLE; 134 135 DBG(physaddr); 136 137 #ifdef __xpv 138 if (!writeable) 139 pt_bits &= ~PT_WRITABLE; 140 if (HYPERVISOR_update_va_mapping((uintptr_t)window, 141 pa_to_ma(physaddr) | pt_bits, UVMF_INVLPG | UVMF_LOCAL) < 0) 142 bop_panic("HYPERVISOR_update_va_mapping() failed"); 143 #else 144 if (kbm_pae_support) 145 *((x86pte_t *)pte_to_window) = physaddr | pt_bits; 146 else 147 *((x86pte32_t *)pte_to_window) = physaddr | pt_bits; 148 mmu_tlbflush_entry(window); 149 #endif 150 DBG(window); 151 return (window); 152 } 153 154 /* 155 * Add a mapping for the physical page at the given virtual address. 156 */ 157 void 158 kbm_map(uintptr_t va, paddr_t pa, uint_t level, uint_t is_kernel) 159 { 160 x86pte_t *ptep; 161 paddr_t pte_physaddr; 162 x86pte_t pteval; 163 164 if (khat_running) 165 panic("kbm_map() called too late"); 166 167 pteval = pa_to_ma(pa) | PT_NOCONSIST | PT_VALID | PT_WRITABLE; 168 if (level >= 1) 169 pteval |= PT_PAGESIZE; 170 if (kbm_pge_support && is_kernel) 171 pteval |= PT_GLOBAL; 172 173 #ifdef __xpv 174 /* 175 * try update_va_mapping first - fails if page table is missing. 176 */ 177 if (HYPERVISOR_update_va_mapping(va, pteval, 178 UVMF_INVLPG | UVMF_LOCAL) == 0) 179 return; 180 #endif 181 182 /* 183 * Find the pte that will map this address. This creates any 184 * missing intermediate level page tables. 185 */ 186 ptep = find_pte(va, &pte_physaddr, level, 0); 187 if (ptep == NULL) 188 bop_panic("kbm_map: find_pte returned NULL"); 189 190 #ifdef __xpv 191 if (HYPERVISOR_update_va_mapping(va, pteval, UVMF_INVLPG | UVMF_LOCAL)) 192 bop_panic("HYPERVISOR_update_va_mapping() failed"); 193 #else 194 if (kbm_pae_support) 195 *ptep = pteval; 196 else 197 *((x86pte32_t *)ptep) = pteval; 198 mmu_tlbflush_entry((caddr_t)va); 199 #endif 200 } 201 202 #ifdef __xpv 203 204 /* 205 * Add a mapping for the machine page at the given virtual address. 206 */ 207 void 208 kbm_map_ma(maddr_t ma, uintptr_t va, uint_t level) 209 { 210 paddr_t pte_physaddr; 211 x86pte_t pteval; 212 213 pteval = ma | PT_NOCONSIST | PT_VALID | PT_REF | PT_WRITABLE; 214 if (level == 1) 215 pteval |= PT_PAGESIZE; 216 217 /* 218 * try update_va_mapping first - fails if page table is missing. 219 */ 220 if (HYPERVISOR_update_va_mapping(va, 221 pteval, UVMF_INVLPG | UVMF_LOCAL) == 0) 222 return; 223 224 /* 225 * Find the pte that will map this address. This creates any 226 * missing intermediate level page tables 227 */ 228 (void) find_pte(va, &pte_physaddr, level, 0); 229 230 if (HYPERVISOR_update_va_mapping(va, 231 pteval, UVMF_INVLPG | UVMF_LOCAL) != 0) 232 bop_panic("HYPERVISOR_update_va_mapping failed"); 233 } 234 235 #endif /* __xpv */ 236 237 238 /* 239 * Probe the boot time page tables to find the first mapping 240 * including va (or higher) and return non-zero if one is found. 241 * va is updated to the starting address and len to the pagesize. 242 * pp will be set to point to the 1st page_t of the mapped page(s). 243 * 244 * Note that if va is in the middle of a large page, the returned va 245 * will be less than what was asked for. 246 */ 247 int 248 kbm_probe(uintptr_t *va, size_t *len, pfn_t *pfn, uint_t *prot) 249 { 250 uintptr_t probe_va; 251 x86pte_t *ptep; 252 paddr_t pte_physaddr; 253 x86pte_t pte_val; 254 level_t l; 255 256 if (khat_running) 257 panic("kbm_probe() called too late"); 258 *len = 0; 259 *pfn = PFN_INVALID; 260 *prot = 0; 261 probe_va = *va; 262 restart_new_va: 263 l = top_level; 264 for (;;) { 265 if (IN_VA_HOLE(probe_va)) 266 probe_va = mmu.hole_end; 267 268 if (IN_HYPERVISOR_VA(probe_va)) 269 #if defined(__amd64) && defined(__xpv) 270 probe_va = HYPERVISOR_VIRT_END; 271 #else 272 return (0); 273 #endif 274 275 /* 276 * If we don't have a valid PTP/PTE at this level 277 * then we can bump VA by this level's pagesize and try again. 278 * When the probe_va wraps around, we are done. 279 */ 280 ptep = find_pte(probe_va, &pte_physaddr, l, 1); 281 if (ptep == NULL) 282 bop_panic("kbm_probe: find_pte returned NULL"); 283 if (kbm_pae_support) 284 pte_val = *ptep; 285 else 286 pte_val = *((x86pte32_t *)ptep); 287 if (!PTE_ISVALID(pte_val)) { 288 probe_va = (probe_va & BOOT_MASK(l)) + BOOT_SZ(l); 289 if (probe_va <= *va) 290 return (0); 291 goto restart_new_va; 292 } 293 294 /* 295 * If this entry is a pointer to a lower level page table 296 * go down to it. 297 */ 298 if (!PTE_ISPAGE(pte_val, l)) { 299 ASSERT(l > 0); 300 --l; 301 continue; 302 } 303 304 /* 305 * We found a boot level page table entry 306 */ 307 *len = BOOT_SZ(l); 308 *va = probe_va & ~(*len - 1); 309 *pfn = PTE2PFN(pte_val, l); 310 311 312 *prot = PROT_READ | PROT_EXEC; 313 if (PTE_GET(pte_val, PT_WRITABLE)) 314 *prot |= PROT_WRITE; 315 316 /* 317 * pt_nx is cleared if processor doesn't support NX bit 318 */ 319 if (PTE_GET(pte_val, mmu.pt_nx)) 320 *prot &= ~PROT_EXEC; 321 322 return (1); 323 } 324 } 325 326 327 /* 328 * Destroy a boot loader page table 4K mapping. 329 */ 330 void 331 kbm_unmap(uintptr_t va) 332 { 333 if (khat_running) 334 panic("kbm_unmap() called too late"); 335 else { 336 #ifdef __xpv 337 (void) HYPERVISOR_update_va_mapping(va, 0, 338 UVMF_INVLPG | UVMF_LOCAL); 339 #else 340 x86pte_t *ptep; 341 level_t level = 0; 342 uint_t probe_only = 1; 343 344 ptep = find_pte(va, NULL, level, probe_only); 345 if (ptep == NULL) 346 return; 347 348 if (kbm_pae_support) 349 *ptep = 0; 350 else 351 *((x86pte32_t *)ptep) = 0; 352 mmu_tlbflush_entry((caddr_t)va); 353 #endif 354 } 355 } 356 357 358 /* 359 * Change a boot loader page table 4K mapping. 360 * Returns the pfn of the old mapping. 361 */ 362 pfn_t 363 kbm_remap(uintptr_t va, pfn_t pfn) 364 { 365 x86pte_t *ptep; 366 level_t level = 0; 367 uint_t probe_only = 1; 368 x86pte_t pte_val = pa_to_ma(pfn_to_pa(pfn)) | PT_WRITABLE | 369 PT_NOCONSIST | PT_VALID; 370 x86pte_t old_pte; 371 372 if (khat_running) 373 panic("kbm_remap() called too late"); 374 ptep = find_pte(va, NULL, level, probe_only); 375 if (ptep == NULL) 376 bop_panic("kbm_remap: find_pte returned NULL"); 377 378 if (kbm_pae_support) 379 old_pte = *ptep; 380 else 381 old_pte = *((x86pte32_t *)ptep); 382 383 #ifdef __xpv 384 if (HYPERVISOR_update_va_mapping(va, pte_val, UVMF_INVLPG | UVMF_LOCAL)) 385 bop_panic("HYPERVISOR_update_va_mapping() failed"); 386 #else 387 if (kbm_pae_support) 388 *((x86pte_t *)ptep) = pte_val; 389 else 390 *((x86pte32_t *)ptep) = pte_val; 391 mmu_tlbflush_entry((caddr_t)va); 392 #endif 393 394 if (!(old_pte & PT_VALID) || ma_to_pa(old_pte) == -1) 395 return (PFN_INVALID); 396 return (mmu_btop(ma_to_pa(old_pte))); 397 } 398 399 400 /* 401 * Change a boot loader page table 4K mapping to read only. 402 */ 403 void 404 kbm_read_only(uintptr_t va, paddr_t pa) 405 { 406 x86pte_t pte_val = pa_to_ma(pa) | 407 PT_NOCONSIST | PT_REF | PT_MOD | PT_VALID; 408 409 #ifdef __xpv 410 if (HYPERVISOR_update_va_mapping(va, pte_val, UVMF_INVLPG | UVMF_LOCAL)) 411 bop_panic("HYPERVISOR_update_va_mapping() failed"); 412 #else 413 x86pte_t *ptep; 414 level_t level = 0; 415 416 ptep = find_pte(va, NULL, level, 0); 417 if (ptep == NULL) 418 bop_panic("kbm_read_only: find_pte returned NULL"); 419 420 if (kbm_pae_support) 421 *ptep = pte_val; 422 else 423 *((x86pte32_t *)ptep) = pte_val; 424 mmu_tlbflush_entry((caddr_t)va); 425 #endif 426 } 427 428 /* 429 * interfaces for kernel debugger to access physical memory 430 */ 431 static x86pte_t save_pte; 432 433 void * 434 kbm_push(paddr_t pa) 435 { 436 static int first_time = 1; 437 438 if (first_time) { 439 first_time = 0; 440 return (window); 441 } 442 443 if (kbm_pae_support) 444 save_pte = *((x86pte_t *)pte_to_window); 445 else 446 save_pte = *((x86pte32_t *)pte_to_window); 447 return (kbm_remap_window(pa, 0)); 448 } 449 450 void 451 kbm_pop(void) 452 { 453 #ifdef __xpv 454 if (HYPERVISOR_update_va_mapping((uintptr_t)window, save_pte, 455 UVMF_INVLPG | UVMF_LOCAL) < 0) 456 bop_panic("HYPERVISOR_update_va_mapping() failed"); 457 #else 458 if (kbm_pae_support) 459 *((x86pte_t *)pte_to_window) = save_pte; 460 else 461 *((x86pte32_t *)pte_to_window) = save_pte; 462 mmu_tlbflush_entry(window); 463 #endif 464 } 465 466 x86pte_t 467 get_pteval(paddr_t table, uint_t index) 468 { 469 void *table_ptr = kbm_remap_window(table, 0); 470 471 if (kbm_pae_support) 472 return (((x86pte_t *)table_ptr)[index]); 473 return (((x86pte32_t *)table_ptr)[index]); 474 } 475 476 #ifndef __xpv 477 void 478 set_pteval(paddr_t table, uint_t index, uint_t level, x86pte_t pteval) 479 { 480 void *table_ptr = kbm_remap_window(table, 0); 481 if (kbm_pae_support) 482 ((x86pte_t *)table_ptr)[index] = pteval; 483 else 484 ((x86pte32_t *)table_ptr)[index] = pteval; 485 if (level == top_level && level == 2) 486 reload_cr3(); 487 } 488 #endif 489 490 paddr_t 491 make_ptable(x86pte_t *pteval, uint_t level) 492 { 493 paddr_t new_table; 494 void *table_ptr; 495 496 new_table = do_bop_phys_alloc(MMU_PAGESIZE, MMU_PAGESIZE); 497 table_ptr = kbm_remap_window(new_table, 1); 498 bzero(table_ptr, MMU_PAGESIZE); 499 #ifdef __xpv 500 /* Remove write permission to the new page table. */ 501 (void) kbm_remap_window(new_table, 0); 502 #endif 503 504 if (level == top_level && level == 2) 505 *pteval = pa_to_ma(new_table) | PT_VALID; 506 else 507 *pteval = pa_to_ma(new_table) | 508 PT_VALID | PT_REF | PT_USER | PT_WRITABLE; 509 510 return (new_table); 511 } 512 513 x86pte_t * 514 map_pte(paddr_t table, uint_t index) 515 { 516 void *table_ptr = kbm_remap_window(table, 0); 517 return ((x86pte_t *)((caddr_t)table_ptr + index * pte_size)); 518 } 519