1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 28 #include <sys/mach_mmu.h> 29 #include <sys/machsystm.h> 30 #include <sys/cmn_err.h> 31 #include <sys/promif.h> 32 #include <sys/hypervisor.h> 33 #include <sys/bootconf.h> 34 #include <sys/ontrap.h> 35 #include <sys/rwlock.h> 36 #include <sys/sysmacros.h> 37 #include <vm/seg_kmem.h> 38 #include <vm/kboot_mmu.h> 39 #include <vm/hat_pte.h> 40 #include <vm/hat.h> 41 #include <vm/htable.h> 42 #include <vm/hat_i86.h> 43 44 start_info_t *xen_info; 45 ulong_t mfn_count; 46 mfn_t *mfn_list; 47 mfn_t *mfn_list_pages; /* pages that make a table of mfn's */ 48 /* that make up the pa_to_ma table */ 49 mfn_t *mfn_list_pages_page; /* page of mfn's for mfn_list_pages */ 50 mfn_t cached_max_mfn; 51 uintptr_t xen_virt_start; 52 pfn_t *mfn_to_pfn_mapping; 53 caddr_t xb_addr; /* virtual addr for the store_mfn page */ 54 55 56 /* 57 * We need to prevent migration or suspension of a domU while it's 58 * manipulating MFN values, as the MFN values will spontaneously 59 * change. The next 4 routines provide a mechanism for that. 60 * The basic idea is to use reader/writer mutex, readers are any thread 61 * that is manipulating MFNs. Only the thread which is going to actually call 62 * HYPERVISOR_suspend() will become a writer. 63 * 64 * Since various places need to manipulate MFNs and also call the HAT, 65 * we track if a thread acquires reader status and allow it to recursively 66 * do so again. This prevents deadlocks if a migration request 67 * is started and waits for some reader, but then the previous reader needs 68 * to call into the HAT. 69 */ 70 #define NUM_M2P_LOCKS 128 71 static struct { 72 krwlock_t m2p_rwlock; 73 char m2p_pad[64 - sizeof (krwlock_t)]; /* 64 byte cache line size */ 74 } m2p_lock[NUM_M2P_LOCKS]; 75 76 #define XM2P_HASH ((uintptr_t)curthread->t_tid & (NUM_M2P_LOCKS - 1)) 77 78 void 79 xen_block_migrate(void) 80 { 81 if (!DOMAIN_IS_INITDOMAIN(xen_info) && 82 ++curthread->t_xpvcntr == 1) 83 rw_enter(&m2p_lock[XM2P_HASH].m2p_rwlock, RW_READER); 84 } 85 86 void 87 xen_allow_migrate(void) 88 { 89 if (!DOMAIN_IS_INITDOMAIN(xen_info) && 90 --curthread->t_xpvcntr == 0) 91 rw_exit(&m2p_lock[XM2P_HASH].m2p_rwlock); 92 } 93 94 void 95 xen_start_migrate(void) 96 { 97 int i; 98 99 ASSERT(curthread->t_xpvcntr == 0); 100 ++curthread->t_xpvcntr; /* this allows calls into HAT */ 101 for (i = 0; i < NUM_M2P_LOCKS; ++i) 102 rw_enter(&m2p_lock[i].m2p_rwlock, RW_WRITER); 103 } 104 105 void 106 xen_end_migrate(void) 107 { 108 int i; 109 110 for (i = 0; i < NUM_M2P_LOCKS; ++i) 111 rw_exit(&m2p_lock[i].m2p_rwlock); 112 ASSERT(curthread->t_xpvcntr == 1); 113 --curthread->t_xpvcntr; 114 } 115 116 /*ARGSUSED*/ 117 void 118 set_pteval(paddr_t table, uint_t index, uint_t level, x86pte_t pteval) 119 { 120 mmu_update_t t; 121 maddr_t mtable = pa_to_ma(table); 122 int retcnt; 123 124 t.ptr = (mtable + index * pte_size) | MMU_NORMAL_PT_UPDATE; 125 t.val = pteval; 126 if (HYPERVISOR_mmu_update(&t, 1, &retcnt, DOMID_SELF) || retcnt != 1) 127 bop_panic("HYPERVISOR_mmu_update() failed"); 128 } 129 130 /* 131 * The start_info_t and mfn_list are initially mapped in low "boot" memory. 132 * Each has a page aligned address and size. We relocate them up into the 133 * kernel's normal address space at this point in time. We also create 134 * the arrays that let the hypervisor suspend/resume a domain. 135 */ 136 void 137 xen_relocate_start_info(void) 138 { 139 maddr_t mach_addr; 140 size_t sz; 141 size_t sz2; 142 offset_t off; 143 uintptr_t addr; 144 uintptr_t old; 145 int i, j; 146 147 /* 148 * In dom0, we have to account for the console_info structure 149 * which might immediately follow the start_info in memory. 150 */ 151 sz = sizeof (start_info_t); 152 if (DOMAIN_IS_INITDOMAIN(xen_info) && 153 xen_info->console.dom0.info_off >= sizeof (start_info_t)) { 154 sz += xen_info->console.dom0.info_off - sizeof (start_info_t) + 155 xen_info->console.dom0.info_size; 156 } 157 sz = P2ROUNDUP(sz, MMU_PAGESIZE); 158 addr = (uintptr_t)vmem_alloc(heap_arena, sz, VM_SLEEP); 159 for (off = 0; off < sz; off += MMU_PAGESIZE) { 160 mach_addr = pa_to_ma(pfn_to_pa(va_to_pfn( 161 (caddr_t)xen_info + off))); 162 kbm_map_ma(mach_addr + off, addr + off, 0); 163 } 164 boot_mapin((caddr_t)addr, sz); 165 old = (uintptr_t)xen_info; 166 xen_info = (start_info_t *)addr; 167 for (off = 0; off < sz; off += MMU_PAGESIZE) 168 kbm_unmap(old + off); 169 170 /* 171 * Relocate the mfn_list, any number of pages. 172 */ 173 sz = P2ROUNDUP(mfn_count * sizeof (mfn_t), MMU_PAGESIZE); 174 addr = (uintptr_t)vmem_xalloc(heap_arena, sz, MMU_PAGESIZE, 0, 175 0, 0, 0, VM_SLEEP); 176 for (off = 0; off < sz; off += MMU_PAGESIZE) { 177 mach_addr = 178 pa_to_ma(pfn_to_pa(va_to_pfn((caddr_t)mfn_list + off))); 179 kbm_map_ma(mach_addr, addr + off, 0); 180 } 181 boot_mapin((caddr_t)addr, sz); 182 old = (uintptr_t)mfn_list; 183 mfn_list = (mfn_t *)addr; 184 xen_info->mfn_list = (mfn_t)addr; 185 for (off = 0; off < sz; off += MMU_PAGESIZE) 186 kbm_unmap(old + off); 187 188 /* 189 * Create the lists of mfn_list pages needed by suspend/resume. 190 * Note we skip this for domain 0 as it can't suspend/resume. 191 */ 192 if (!DOMAIN_IS_INITDOMAIN(xen_info)) { 193 sz2 = P2ROUNDUP(mmu_btop(sz) * sizeof (mfn_t), MMU_PAGESIZE); 194 mfn_list_pages = kmem_zalloc(sz2, VM_SLEEP); 195 mfn_list_pages_page = kmem_zalloc(MMU_PAGESIZE, VM_SLEEP); 196 i = 0; 197 for (off = 0; off < sz; off += MMU_PAGESIZE) { 198 j = mmu_btop(off); 199 if (((j * sizeof (mfn_t)) & MMU_PAGEOFFSET) == 0) { 200 mfn_list_pages_page[i++] = 201 pfn_to_mfn(va_to_pfn(&mfn_list_pages[j])); 202 } 203 mfn_list_pages[j] = 204 pfn_to_mfn(va_to_pfn((caddr_t)mfn_list + off)); 205 } 206 HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = 207 pfn_to_mfn(va_to_pfn(mfn_list_pages_page)); 208 HYPERVISOR_shared_info->arch.max_pfn = xen_info->nr_pages; 209 } 210 211 /* 212 * Remap the shared info (for I/O) into high memory, too. 213 */ 214 sz = MMU_PAGESIZE; 215 addr = (uintptr_t)vmem_alloc(heap_arena, sz, VM_SLEEP); 216 kbm_map_ma(xen_info->shared_info, addr, 0); 217 /* shared info has no PFN so don't do: boot_mapin((caddr_t)addr, sz) */ 218 old = (uintptr_t)HYPERVISOR_shared_info; 219 HYPERVISOR_shared_info = (void *)addr; 220 kbm_unmap(old); 221 222 /* 223 * Remap the console info into high memory, too. 224 */ 225 if (!DOMAIN_IS_INITDOMAIN(xen_info)) { 226 sz = MMU_PAGESIZE; 227 addr = (uintptr_t)vmem_alloc(heap_arena, sz, VM_SLEEP); 228 kbm_map_ma(pfn_to_pa(xen_info->console.domU.mfn), addr, 0); 229 boot_mapin((caddr_t)addr, sz); 230 old = (uintptr_t)HYPERVISOR_console_page; 231 HYPERVISOR_console_page = (void *)addr; 232 kbm_unmap(old); 233 } else { 234 HYPERVISOR_console_page = NULL; 235 } 236 237 /* 238 * On domUs we need to have the xenbus page (store_mfn) mapped into 239 * the kernel. This is referenced as xb_addr. 240 */ 241 if (!DOMAIN_IS_INITDOMAIN(xen_info)) { 242 xb_addr = vmem_alloc(heap_arena, MMU_PAGESIZE, VM_SLEEP); 243 kbm_map_ma(mfn_to_ma(xen_info->store_mfn), 244 (uintptr_t)xb_addr, 0); 245 boot_mapin(xb_addr, MMU_PAGESIZE); 246 } 247 } 248 249 /* 250 * Generate the pfn value to use for a foreign mfn. 251 */ 252 pfn_t 253 xen_assign_pfn(mfn_t mfn) 254 { 255 pfn_t pfn; 256 257 #ifdef DEBUG 258 /* 259 * make sure this MFN isn't in our list of MFNs 260 */ 261 on_trap_data_t otd; 262 uint_t on_trap_ready = (t0.t_stk != NULL); 263 264 if (on_trap_ready) { 265 if (on_trap(&otd, OT_DATA_ACCESS) == 0) { 266 pfn = mfn_to_pfn_mapping[mfn]; 267 if (pfn < mfn_count && mfn_list[pfn] == mfn) 268 panic("xen_assign_pfn() mfn belongs to us"); 269 } 270 no_trap(); 271 } 272 #endif /* DEBUG */ 273 274 if (mfn == MFN_INVALID) 275 panic("xen_assign_pfn(MFN_INVALID) not allowed"); 276 pfn = (pfn_t)mfn | PFN_IS_FOREIGN_MFN; 277 if (pfn == mfn) 278 panic("xen_assign_pfn(mfn) PFN_IS_FOREIGN_MFN bit already set"); 279 return (pfn); 280 } 281 282 void 283 xen_release_pfn(pfn_t pfn) 284 { 285 if (pfn == PFN_INVALID) 286 panic("xen_release_pfn(PFN_INVALID) not allowed"); 287 if ((pfn & PFN_IS_FOREIGN_MFN) == 0) 288 panic("mfn high bit not set"); 289 } 290 291 uint_t 292 pfn_is_foreign(pfn_t pfn) 293 { 294 if (pfn == PFN_INVALID) 295 return (0); 296 return ((pfn & PFN_IS_FOREIGN_MFN) != 0); 297 } 298 299 pfn_t 300 pte2pfn(x86pte_t pte, level_t l) 301 { 302 mfn_t mfn = PTE2MFN(pte, l); 303 304 if ((pte & PT_SOFTWARE) >= PT_FOREIGN) 305 return ((pfn_t)mfn | PFN_IS_FOREIGN_MFN); 306 return (mfn_to_pfn(mfn)); 307 } 308 309 mfn_t 310 pfn_to_mfn(pfn_t pfn) 311 { 312 if (pfn == PFN_INVALID) 313 panic("pfn_to_mfn(PFN_INVALID) not allowed"); 314 315 if (pfn & PFN_IS_FOREIGN_MFN) 316 return (pfn & ~PFN_IS_FOREIGN_MFN); 317 318 if (pfn >= mfn_count) 319 panic("pfn_to_mfn(): illegal PFN 0x%lx", pfn); 320 321 return (mfn_list[pfn]); 322 } 323 324 /* 325 * This routine translates an MFN back into the corresponding PFN value. 326 * It has to be careful since the mfn_to_pfn_mapping[] might fault 327 * as that table is sparse. It also has to check for non-faulting, but out of 328 * range that exceed the table. 329 */ 330 pfn_t 331 mfn_to_pfn(mfn_t mfn) 332 { 333 pfn_t pfn; 334 on_trap_data_t otd; 335 uint_t on_trap_ready = (t0.t_stk != NULL); 336 337 /* 338 * Cleared at a suspend or migrate 339 */ 340 if (cached_max_mfn == 0) 341 cached_max_mfn = 342 HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL); 343 344 if (cached_max_mfn < mfn) 345 return ((pfn_t)mfn | PFN_IS_FOREIGN_MFN); 346 347 if (on_trap_ready && on_trap(&otd, OT_DATA_ACCESS)) { 348 pfn = (pfn_t)mfn | PFN_IS_FOREIGN_MFN; 349 } else { 350 pfn = mfn_to_pfn_mapping[mfn]; 351 352 if (pfn == PFN_INVALID || pfn >= mfn_count || 353 pfn_to_mfn(pfn) != mfn) 354 pfn = (pfn_t)mfn | PFN_IS_FOREIGN_MFN; 355 } 356 357 if (on_trap_ready) 358 no_trap(); 359 360 /* 361 * If khat_running is set then we should be checking 362 * in domUs that migration is blocked while using the 363 * mfn_to_pfn_mapping[] table. 364 */ 365 ASSERT(!khat_running || DOMAIN_IS_INITDOMAIN(xen_info) || 366 rw_read_held(&m2p_lock[XM2P_HASH].m2p_rwlock)); 367 368 return (pfn); 369 } 370 371 /* 372 * From a pseudo-physical address, find the corresponding machine address. 373 */ 374 maddr_t 375 pa_to_ma(paddr_t pa) 376 { 377 mfn_t mfn = pfn_to_mfn(mmu_btop(pa)); 378 379 if (mfn == MFN_INVALID) 380 panic("pa_to_ma() got MFN_INVALID"); 381 return (mfn_to_ma(mfn) + (pa & MMU_PAGEOFFSET)); 382 } 383 384 /* 385 * From a machine address, find the corresponding pseudo-physical address. 386 */ 387 paddr_t 388 ma_to_pa(maddr_t ma) 389 { 390 pfn_t pfn = mfn_to_pfn(mmu_btop(ma)); 391 392 if (pfn == PFN_INVALID) 393 panic("ma_to_pa() got PFN_INVALID"); 394 return (pfn_to_pa(pfn) + (ma & MMU_PAGEOFFSET)); 395 } 396 397 /* 398 * When calling reassign_pfn(), the page must be (at least) read locked 399 * to make sure swrand does not try to grab it. 400 */ 401 #ifdef DEBUG 402 #define CHECK_PAGE_LOCK(pfn) { \ 403 page_t *pp = page_numtopp_nolock(pfn); \ 404 if ((pp != NULL) && (!PAGE_LOCKED(pp))) { \ 405 panic("reassign_pfn() called with unlocked page (pfn 0x%lx)", \ 406 pfn); \ 407 } \ 408 } 409 #else /* DEBUG */ 410 #define CHECK_PAGE_LOCK(pfn) 411 #endif /* DEBUG */ 412 413 /* 414 * Reassign a new machine page to back a physical address. 415 */ 416 void 417 reassign_pfn(pfn_t pfn, mfn_t mfn) 418 { 419 int mmu_update_return; 420 mmu_update_t t; 421 extern void update_contig_pfnlist(pfn_t, mfn_t, mfn_t); 422 423 ASSERT(pfn != PFN_INVALID); 424 ASSERT(!pfn_is_foreign(pfn)); 425 426 ASSERT(pfn < mfn_count); 427 update_contig_pfnlist(pfn, mfn_list[pfn], mfn); 428 if (mfn == MFN_INVALID) { 429 CHECK_PAGE_LOCK(pfn); 430 if (kpm_vbase != NULL && xen_kpm_page(pfn, 0) < 0) 431 panic("reassign_pfn(): failed to remove kpm mapping"); 432 mfn_list[pfn] = mfn; 433 return; 434 } 435 436 /* 437 * Verify that previously given away pages are still page locked. 438 */ 439 if (mfn_list[pfn] == MFN_INVALID) { 440 CHECK_PAGE_LOCK(pfn); 441 } 442 mfn_list[pfn] = mfn; 443 444 t.ptr = mfn_to_ma(mfn) | MMU_MACHPHYS_UPDATE; 445 t.val = pfn; 446 447 if (HYPERVISOR_mmu_update(&t, 1, &mmu_update_return, DOMID_SELF)) 448 panic("HYPERVISOR_mmu_update() failed"); 449 ASSERT(mmu_update_return == 1); 450 451 if (kpm_vbase != NULL && xen_kpm_page(pfn, PT_VALID | PT_WRITABLE) < 0) 452 panic("reassign_pfn(): failed to enable kpm mapping"); 453 } 454