1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright (c) 2010, Intel Corporation. 27 * All rights reserved. 28 */ 29 30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 31 /* All Rights Reserved */ 32 33 /* 34 * Portions of this source code were derived from Berkeley 4.3 BSD 35 * under license from the Regents of the University of California. 36 */ 37 38 /* 39 * UNIX machine dependent virtual memory support. 40 */ 41 42 #include <sys/types.h> 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/user.h> 46 #include <sys/proc.h> 47 #include <sys/kmem.h> 48 #include <sys/vmem.h> 49 #include <sys/buf.h> 50 #include <sys/cpuvar.h> 51 #include <sys/lgrp.h> 52 #include <sys/disp.h> 53 #include <sys/vm.h> 54 #include <sys/mman.h> 55 #include <sys/vnode.h> 56 #include <sys/cred.h> 57 #include <sys/exec.h> 58 #include <sys/exechdr.h> 59 #include <sys/debug.h> 60 #include <sys/vmsystm.h> 61 62 #include <vm/hat.h> 63 #include <vm/as.h> 64 #include <vm/seg.h> 65 #include <vm/seg_kp.h> 66 #include <vm/seg_vn.h> 67 #include <vm/page.h> 68 #include <vm/seg_kmem.h> 69 #include <vm/seg_kpm.h> 70 #include <vm/vm_dep.h> 71 72 #include <sys/cpu.h> 73 #include <sys/vm_machparam.h> 74 #include <sys/memlist.h> 75 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */ 76 #include <vm/hat_i86.h> 77 #include <sys/x86_archext.h> 78 #include <sys/elf_386.h> 79 #include <sys/cmn_err.h> 80 #include <sys/archsystm.h> 81 #include <sys/machsystm.h> 82 83 #include <sys/vtrace.h> 84 #include <sys/ddidmareq.h> 85 #include <sys/promif.h> 86 #include <sys/memnode.h> 87 #include <sys/stack.h> 88 #include <util/qsort.h> 89 #include <sys/taskq.h> 90 91 #ifdef __xpv 92 93 #include <sys/hypervisor.h> 94 #include <sys/xen_mmu.h> 95 #include <sys/balloon_impl.h> 96 97 /* 98 * domain 0 pages usable for DMA are kept pre-allocated and kept in 99 * distinct lists, ordered by increasing mfn. 100 */ 101 static kmutex_t io_pool_lock; 102 static kmutex_t contig_list_lock; 103 static page_t *io_pool_4g; /* pool for 32 bit dma limited devices */ 104 static page_t *io_pool_16m; /* pool for 24 bit dma limited legacy devices */ 105 static long io_pool_cnt; 106 static long io_pool_cnt_max = 0; 107 #define DEFAULT_IO_POOL_MIN 128 108 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN; 109 static long io_pool_cnt_lowater = 0; 110 static long io_pool_shrink_attempts; /* how many times did we try to shrink */ 111 static long io_pool_shrinks; /* how many times did we really shrink */ 112 static long io_pool_grows; /* how many times did we grow */ 113 static mfn_t start_mfn = 1; 114 static caddr_t io_pool_kva; /* use to alloc pages when needed */ 115 116 static int create_contig_pfnlist(uint_t); 117 118 /* 119 * percentage of phys mem to hold in the i/o pool 120 */ 121 #define DEFAULT_IO_POOL_PCT 2 122 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT; 123 static void page_io_pool_sub(page_t **, page_t *, page_t *); 124 int ioalloc_dbg = 0; 125 126 #endif /* __xpv */ 127 128 uint_t vac_colors = 1; 129 130 int largepagesupport = 0; 131 extern uint_t page_create_new; 132 extern uint_t page_create_exists; 133 extern uint_t page_create_putbacks; 134 /* 135 * Allow users to disable the kernel's use of SSE. 136 */ 137 extern int use_sse_pagecopy, use_sse_pagezero; 138 139 /* 140 * combined memory ranges from mnode and memranges[] to manage single 141 * mnode/mtype dimension in the page lists. 142 */ 143 typedef struct { 144 pfn_t mnr_pfnlo; 145 pfn_t mnr_pfnhi; 146 int mnr_mnode; 147 int mnr_memrange; /* index into memranges[] */ 148 int mnr_next; /* next lower PA mnoderange */ 149 int mnr_exists; 150 /* maintain page list stats */ 151 pgcnt_t mnr_mt_clpgcnt; /* cache list cnt */ 152 pgcnt_t mnr_mt_flpgcnt[MMU_PAGE_SIZES]; /* free list cnt per szc */ 153 pgcnt_t mnr_mt_totcnt; /* sum of cache and free lists */ 154 #ifdef DEBUG 155 struct mnr_mts { /* mnode/mtype szc stats */ 156 pgcnt_t mnr_mts_pgcnt; 157 int mnr_mts_colors; 158 pgcnt_t *mnr_mtsc_pgcnt; 159 } *mnr_mts; 160 #endif 161 } mnoderange_t; 162 163 #define MEMRANGEHI(mtype) \ 164 ((mtype > 0) ? memranges[mtype - 1] - 1: physmax) 165 #define MEMRANGELO(mtype) (memranges[mtype]) 166 167 #define MTYPE_FREEMEM(mt) (mnoderanges[mt].mnr_mt_totcnt) 168 169 /* 170 * As the PC architecture evolved memory up was clumped into several 171 * ranges for various historical I/O devices to do DMA. 172 * < 16Meg - ISA bus 173 * < 2Gig - ??? 174 * < 4Gig - PCI bus or drivers that don't understand PAE mode 175 * 176 * These are listed in reverse order, so that we can skip over unused 177 * ranges on machines with small memories. 178 * 179 * For now under the Hypervisor, we'll only ever have one memrange. 180 */ 181 #define PFN_4GIG 0x100000 182 #define PFN_16MEG 0x1000 183 /* Indices into the memory range (arch_memranges) array. */ 184 #define MRI_4G 0 185 #define MRI_2G 1 186 #define MRI_16M 2 187 #define MRI_0 3 188 static pfn_t arch_memranges[NUM_MEM_RANGES] = { 189 PFN_4GIG, /* pfn range for 4G and above */ 190 0x80000, /* pfn range for 2G-4G */ 191 PFN_16MEG, /* pfn range for 16M-2G */ 192 0x00000, /* pfn range for 0-16M */ 193 }; 194 pfn_t *memranges = &arch_memranges[0]; 195 int nranges = NUM_MEM_RANGES; 196 197 /* 198 * This combines mem_node_config and memranges into one data 199 * structure to be used for page list management. 200 */ 201 mnoderange_t *mnoderanges; 202 int mnoderangecnt; 203 int mtype4g; 204 int mtype16m; 205 int mtypetop; /* index of highest pfn'ed mnoderange */ 206 207 /* 208 * 4g memory management variables for systems with more than 4g of memory: 209 * 210 * physical memory below 4g is required for 32bit dma devices and, currently, 211 * for kmem memory. On systems with more than 4g of memory, the pool of memory 212 * below 4g can be depleted without any paging activity given that there is 213 * likely to be sufficient memory above 4g. 214 * 215 * physmax4g is set true if the largest pfn is over 4g. The rest of the 216 * 4g memory management code is enabled only when physmax4g is true. 217 * 218 * maxmem4g is the count of the maximum number of pages on the page lists 219 * with physical addresses below 4g. It can be a lot less then 4g given that 220 * BIOS may reserve large chunks of space below 4g for hot plug pci devices, 221 * agp aperture etc. 222 * 223 * freemem4g maintains the count of the number of available pages on the 224 * page lists with physical addresses below 4g. 225 * 226 * DESFREE4G specifies the desired amount of below 4g memory. It defaults to 227 * 6% (desfree4gshift = 4) of maxmem4g. 228 * 229 * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G 230 * and the amount of physical memory above 4g is greater than freemem4g. 231 * In this case, page_get_* routines will restrict below 4g allocations 232 * for requests that don't specifically require it. 233 */ 234 235 #define DESFREE4G (maxmem4g >> desfree4gshift) 236 237 #define RESTRICT4G_ALLOC \ 238 (physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem)) 239 240 static pgcnt_t maxmem4g; 241 static pgcnt_t freemem4g; 242 static int physmax4g; 243 static int desfree4gshift = 4; /* maxmem4g shift to derive DESFREE4G */ 244 245 /* 246 * 16m memory management: 247 * 248 * reserve some amount of physical memory below 16m for legacy devices. 249 * 250 * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above 251 * 16m or if the 16m pool drops below DESFREE16M. 252 * 253 * In this case, general page allocations via page_get_{free,cache}list 254 * routines will be restricted from allocating from the 16m pool. Allocations 255 * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations 256 * are not restricted. 257 */ 258 259 #define FREEMEM16M MTYPE_FREEMEM(mtype16m) 260 #define DESFREE16M desfree16m 261 #define RESTRICT16M_ALLOC(freemem, pgcnt, flags) \ 262 ((freemem != 0) && ((flags & PG_PANIC) == 0) && \ 263 ((freemem >= (FREEMEM16M)) || \ 264 (FREEMEM16M < (DESFREE16M + pgcnt)))) 265 266 static pgcnt_t desfree16m = 0x380; 267 268 /* 269 * This can be patched via /etc/system to allow old non-PAE aware device 270 * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM. 271 */ 272 int restricted_kmemalloc = 0; 273 274 #ifdef VM_STATS 275 struct { 276 ulong_t pga_alloc; 277 ulong_t pga_notfullrange; 278 ulong_t pga_nulldmaattr; 279 ulong_t pga_allocok; 280 ulong_t pga_allocfailed; 281 ulong_t pgma_alloc; 282 ulong_t pgma_allocok; 283 ulong_t pgma_allocfailed; 284 ulong_t pgma_allocempty; 285 } pga_vmstats; 286 #endif 287 288 uint_t mmu_page_sizes; 289 290 /* How many page sizes the users can see */ 291 uint_t mmu_exported_page_sizes; 292 293 /* page sizes that legacy applications can see */ 294 uint_t mmu_legacy_page_sizes; 295 296 /* 297 * Number of pages in 1 GB. Don't enable automatic large pages if we have 298 * fewer than this many pages. 299 */ 300 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT); 301 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT); 302 303 /* 304 * Maximum and default segment size tunables for user private 305 * and shared anon memory, and user text and initialized data. 306 * These can be patched via /etc/system to allow large pages 307 * to be used for mapping application private and shared anon memory. 308 */ 309 size_t mcntl0_lpsize = MMU_PAGESIZE; 310 size_t max_uheap_lpsize = MMU_PAGESIZE; 311 size_t default_uheap_lpsize = MMU_PAGESIZE; 312 size_t max_ustack_lpsize = MMU_PAGESIZE; 313 size_t default_ustack_lpsize = MMU_PAGESIZE; 314 size_t max_privmap_lpsize = MMU_PAGESIZE; 315 size_t max_uidata_lpsize = MMU_PAGESIZE; 316 size_t max_utext_lpsize = MMU_PAGESIZE; 317 size_t max_shm_lpsize = MMU_PAGESIZE; 318 319 320 /* 321 * initialized by page_coloring_init(). 322 */ 323 uint_t page_colors; 324 uint_t page_colors_mask; 325 uint_t page_coloring_shift; 326 int cpu_page_colors; 327 static uint_t l2_colors; 328 329 /* 330 * Page freelists and cachelists are dynamically allocated once mnoderangecnt 331 * and page_colors are calculated from the l2 cache n-way set size. Within a 332 * mnode range, the page freelist and cachelist are hashed into bins based on 333 * color. This makes it easier to search for a page within a specific memory 334 * range. 335 */ 336 #define PAGE_COLORS_MIN 16 337 338 page_t ****page_freelists; 339 page_t ***page_cachelists; 340 341 342 /* 343 * Used by page layer to know about page sizes 344 */ 345 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1]; 346 347 kmutex_t *fpc_mutex[NPC_MUTEX]; 348 kmutex_t *cpc_mutex[NPC_MUTEX]; 349 350 /* Lock to protect mnoderanges array for memory DR operations. */ 351 static kmutex_t mnoderange_lock; 352 353 /* 354 * Only let one thread at a time try to coalesce large pages, to 355 * prevent them from working against each other. 356 */ 357 static kmutex_t contig_lock; 358 #define CONTIG_LOCK() mutex_enter(&contig_lock); 359 #define CONTIG_UNLOCK() mutex_exit(&contig_lock); 360 361 #define PFN_16M (mmu_btop((uint64_t)0x1000000)) 362 363 /* 364 * Return the optimum page size for a given mapping 365 */ 366 /*ARGSUSED*/ 367 size_t 368 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl) 369 { 370 level_t l = 0; 371 size_t pgsz = MMU_PAGESIZE; 372 size_t max_lpsize; 373 uint_t mszc; 374 375 ASSERT(maptype != MAPPGSZ_VA); 376 377 if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) { 378 return (MMU_PAGESIZE); 379 } 380 381 switch (maptype) { 382 case MAPPGSZ_HEAP: 383 case MAPPGSZ_STK: 384 max_lpsize = memcntl ? mcntl0_lpsize : (maptype == 385 MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize); 386 if (max_lpsize == MMU_PAGESIZE) { 387 return (MMU_PAGESIZE); 388 } 389 if (len == 0) { 390 len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase + 391 p->p_brksize - p->p_bssbase : p->p_stksize; 392 } 393 len = (maptype == MAPPGSZ_HEAP) ? MAX(len, 394 default_uheap_lpsize) : MAX(len, default_ustack_lpsize); 395 396 /* 397 * use the pages size that best fits len 398 */ 399 for (l = mmu.umax_page_level; l > 0; --l) { 400 if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) { 401 continue; 402 } else { 403 pgsz = LEVEL_SIZE(l); 404 } 405 break; 406 } 407 408 mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc : 409 p->p_stkpageszc); 410 if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) { 411 pgsz = hw_page_array[mszc].hp_size; 412 } 413 return (pgsz); 414 415 case MAPPGSZ_ISM: 416 for (l = mmu.umax_page_level; l > 0; --l) { 417 if (len >= LEVEL_SIZE(l)) 418 return (LEVEL_SIZE(l)); 419 } 420 return (LEVEL_SIZE(0)); 421 } 422 return (pgsz); 423 } 424 425 static uint_t 426 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize, 427 size_t min_physmem) 428 { 429 caddr_t eaddr = addr + size; 430 uint_t szcvec = 0; 431 caddr_t raddr; 432 caddr_t readdr; 433 size_t pgsz; 434 int i; 435 436 if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) { 437 return (0); 438 } 439 440 for (i = mmu_exported_page_sizes - 1; i > 0; i--) { 441 pgsz = page_get_pagesize(i); 442 if (pgsz > max_lpsize) { 443 continue; 444 } 445 raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 446 readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 447 if (raddr < addr || raddr >= readdr) { 448 continue; 449 } 450 if (P2PHASE((uintptr_t)addr ^ off, pgsz)) { 451 continue; 452 } 453 /* 454 * Set szcvec to the remaining page sizes. 455 */ 456 szcvec = ((1 << (i + 1)) - 1) & ~1; 457 break; 458 } 459 return (szcvec); 460 } 461 462 /* 463 * Return a bit vector of large page size codes that 464 * can be used to map [addr, addr + len) region. 465 */ 466 /*ARGSUSED*/ 467 uint_t 468 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type, 469 int memcntl) 470 { 471 size_t max_lpsize = mcntl0_lpsize; 472 473 if (mmu.max_page_level == 0) 474 return (0); 475 476 if (flags & MAP_TEXT) { 477 if (!memcntl) 478 max_lpsize = max_utext_lpsize; 479 return (map_szcvec(addr, size, off, max_lpsize, 480 shm_lpg_min_physmem)); 481 482 } else if (flags & MAP_INITDATA) { 483 if (!memcntl) 484 max_lpsize = max_uidata_lpsize; 485 return (map_szcvec(addr, size, off, max_lpsize, 486 privm_lpg_min_physmem)); 487 488 } else if (type == MAPPGSZC_SHM) { 489 if (!memcntl) 490 max_lpsize = max_shm_lpsize; 491 return (map_szcvec(addr, size, off, max_lpsize, 492 shm_lpg_min_physmem)); 493 494 } else if (type == MAPPGSZC_HEAP) { 495 if (!memcntl) 496 max_lpsize = max_uheap_lpsize; 497 return (map_szcvec(addr, size, off, max_lpsize, 498 privm_lpg_min_physmem)); 499 500 } else if (type == MAPPGSZC_STACK) { 501 if (!memcntl) 502 max_lpsize = max_ustack_lpsize; 503 return (map_szcvec(addr, size, off, max_lpsize, 504 privm_lpg_min_physmem)); 505 506 } else { 507 if (!memcntl) 508 max_lpsize = max_privmap_lpsize; 509 return (map_szcvec(addr, size, off, max_lpsize, 510 privm_lpg_min_physmem)); 511 } 512 } 513 514 /* 515 * Handle a pagefault. 516 */ 517 faultcode_t 518 pagefault( 519 caddr_t addr, 520 enum fault_type type, 521 enum seg_rw rw, 522 int iskernel) 523 { 524 struct as *as; 525 struct hat *hat; 526 struct proc *p; 527 kthread_t *t; 528 faultcode_t res; 529 caddr_t base; 530 size_t len; 531 int err; 532 int mapped_red; 533 uintptr_t ea; 534 535 ASSERT_STACK_ALIGNED(); 536 537 if (INVALID_VADDR(addr)) 538 return (FC_NOMAP); 539 540 mapped_red = segkp_map_red(); 541 542 if (iskernel) { 543 as = &kas; 544 hat = as->a_hat; 545 } else { 546 t = curthread; 547 p = ttoproc(t); 548 as = p->p_as; 549 hat = as->a_hat; 550 } 551 552 /* 553 * Dispatch pagefault. 554 */ 555 res = as_fault(hat, as, addr, 1, type, rw); 556 557 /* 558 * If this isn't a potential unmapped hole in the user's 559 * UNIX data or stack segments, just return status info. 560 */ 561 if (res != FC_NOMAP || iskernel) 562 goto out; 563 564 /* 565 * Check to see if we happened to faulted on a currently unmapped 566 * part of the UNIX data or stack segments. If so, create a zfod 567 * mapping there and then try calling the fault routine again. 568 */ 569 base = p->p_brkbase; 570 len = p->p_brksize; 571 572 if (addr < base || addr >= base + len) { /* data seg? */ 573 base = (caddr_t)p->p_usrstack - p->p_stksize; 574 len = p->p_stksize; 575 if (addr < base || addr >= p->p_usrstack) { /* stack seg? */ 576 /* not in either UNIX data or stack segments */ 577 res = FC_NOMAP; 578 goto out; 579 } 580 } 581 582 /* 583 * the rest of this function implements a 3.X 4.X 5.X compatibility 584 * This code is probably not needed anymore 585 */ 586 if (p->p_model == DATAMODEL_ILP32) { 587 588 /* expand the gap to the page boundaries on each side */ 589 ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE); 590 base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE); 591 len = ea - (uintptr_t)base; 592 593 as_rangelock(as); 594 if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) == 595 0) { 596 err = as_map(as, base, len, segvn_create, zfod_argsp); 597 as_rangeunlock(as); 598 if (err) { 599 res = FC_MAKE_ERR(err); 600 goto out; 601 } 602 } else { 603 /* 604 * This page is already mapped by another thread after 605 * we returned from as_fault() above. We just fall 606 * through as_fault() below. 607 */ 608 as_rangeunlock(as); 609 } 610 611 res = as_fault(hat, as, addr, 1, F_INVAL, rw); 612 } 613 614 out: 615 if (mapped_red) 616 segkp_unmap_red(); 617 618 return (res); 619 } 620 621 void 622 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags) 623 { 624 struct proc *p = curproc; 625 caddr_t userlimit = (flags & _MAP_LOW32) ? 626 (caddr_t)_userlimit32 : p->p_as->a_userlimit; 627 628 map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags); 629 } 630 631 /*ARGSUSED*/ 632 int 633 map_addr_vacalign_check(caddr_t addr, u_offset_t off) 634 { 635 return (0); 636 } 637 638 /* 639 * map_addr_proc() is the routine called when the system is to 640 * choose an address for the user. We will pick an address 641 * range which is the highest available below userlimit. 642 * 643 * Every mapping will have a redzone of a single page on either side of 644 * the request. This is done to leave one page unmapped between segments. 645 * This is not required, but it's useful for the user because if their 646 * program strays across a segment boundary, it will catch a fault 647 * immediately making debugging a little easier. Currently the redzone 648 * is mandatory. 649 * 650 * addrp is a value/result parameter. 651 * On input it is a hint from the user to be used in a completely 652 * machine dependent fashion. We decide to completely ignore this hint. 653 * If MAP_ALIGN was specified, addrp contains the minimal alignment, which 654 * must be some "power of two" multiple of pagesize. 655 * 656 * On output it is NULL if no address can be found in the current 657 * processes address space or else an address that is currently 658 * not mapped for len bytes with a page of red zone on either side. 659 * 660 * vacalign is not needed on x86 (it's for viturally addressed caches) 661 */ 662 /*ARGSUSED*/ 663 void 664 map_addr_proc( 665 caddr_t *addrp, 666 size_t len, 667 offset_t off, 668 int vacalign, 669 caddr_t userlimit, 670 struct proc *p, 671 uint_t flags) 672 { 673 struct as *as = p->p_as; 674 caddr_t addr; 675 caddr_t base; 676 size_t slen; 677 size_t align_amount; 678 679 ASSERT32(userlimit == as->a_userlimit); 680 681 base = p->p_brkbase; 682 #if defined(__amd64) 683 /* 684 * XX64 Yes, this needs more work. 685 */ 686 if (p->p_model == DATAMODEL_NATIVE) { 687 if (userlimit < as->a_userlimit) { 688 /* 689 * This happens when a program wants to map 690 * something in a range that's accessible to a 691 * program in a smaller address space. For example, 692 * a 64-bit program calling mmap32(2) to guarantee 693 * that the returned address is below 4Gbytes. 694 */ 695 ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff)); 696 697 if (userlimit > base) 698 slen = userlimit - base; 699 else { 700 *addrp = NULL; 701 return; 702 } 703 } else { 704 /* 705 * XX64 This layout is probably wrong .. but in 706 * the event we make the amd64 address space look 707 * like sparcv9 i.e. with the stack -above- the 708 * heap, this bit of code might even be correct. 709 */ 710 slen = p->p_usrstack - base - 711 ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK); 712 } 713 } else 714 #endif 715 slen = userlimit - base; 716 717 /* Make len be a multiple of PAGESIZE */ 718 len = (len + PAGEOFFSET) & PAGEMASK; 719 720 /* 721 * figure out what the alignment should be 722 * 723 * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same???? 724 */ 725 if (len <= ELF_386_MAXPGSZ) { 726 /* 727 * Align virtual addresses to ensure that ELF shared libraries 728 * are mapped with the appropriate alignment constraints by 729 * the run-time linker. 730 */ 731 align_amount = ELF_386_MAXPGSZ; 732 } else { 733 /* 734 * For 32-bit processes, only those which have specified 735 * MAP_ALIGN and an addr will be aligned on a larger page size. 736 * Not doing so can potentially waste up to 1G of process 737 * address space. 738 */ 739 int lvl = (p->p_model == DATAMODEL_ILP32) ? 1 : 740 mmu.umax_page_level; 741 742 while (lvl && len < LEVEL_SIZE(lvl)) 743 --lvl; 744 745 align_amount = LEVEL_SIZE(lvl); 746 } 747 if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount)) 748 align_amount = (uintptr_t)*addrp; 749 750 ASSERT(ISP2(align_amount)); 751 ASSERT(align_amount == 0 || align_amount >= PAGESIZE); 752 753 off = off & (align_amount - 1); 754 /* 755 * Look for a large enough hole starting below userlimit. 756 * After finding it, use the upper part. 757 */ 758 if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount, 759 PAGESIZE, off) == 0) { 760 caddr_t as_addr; 761 762 /* 763 * addr is the highest possible address to use since we have 764 * a PAGESIZE redzone at the beginning and end. 765 */ 766 addr = base + slen - (PAGESIZE + len); 767 as_addr = addr; 768 /* 769 * Round address DOWN to the alignment amount and 770 * add the offset in. 771 * If addr is greater than as_addr, len would not be large 772 * enough to include the redzone, so we must adjust down 773 * by the alignment amount. 774 */ 775 addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1))); 776 addr += (uintptr_t)off; 777 if (addr > as_addr) { 778 addr -= align_amount; 779 } 780 781 ASSERT(addr > base); 782 ASSERT(addr + len < base + slen); 783 ASSERT(((uintptr_t)addr & (align_amount - 1)) == 784 ((uintptr_t)(off))); 785 *addrp = addr; 786 } else { 787 *addrp = NULL; /* no more virtual space */ 788 } 789 } 790 791 int valid_va_range_aligned_wraparound; 792 793 /* 794 * Determine whether [*basep, *basep + *lenp) contains a mappable range of 795 * addresses at least "minlen" long, where the base of the range is at "off" 796 * phase from an "align" boundary and there is space for a "redzone"-sized 797 * redzone on either side of the range. On success, 1 is returned and *basep 798 * and *lenp are adjusted to describe the acceptable range (including 799 * the redzone). On failure, 0 is returned. 800 */ 801 /*ARGSUSED3*/ 802 int 803 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir, 804 size_t align, size_t redzone, size_t off) 805 { 806 uintptr_t hi, lo; 807 size_t tot_len; 808 809 ASSERT(align == 0 ? off == 0 : off < align); 810 ASSERT(ISP2(align)); 811 ASSERT(align == 0 || align >= PAGESIZE); 812 813 lo = (uintptr_t)*basep; 814 hi = lo + *lenp; 815 tot_len = minlen + 2 * redzone; /* need at least this much space */ 816 817 /* 818 * If hi rolled over the top, try cutting back. 819 */ 820 if (hi < lo) { 821 *lenp = 0UL - lo - 1UL; 822 /* See if this really happens. If so, then we figure out why */ 823 valid_va_range_aligned_wraparound++; 824 hi = lo + *lenp; 825 } 826 if (*lenp < tot_len) { 827 return (0); 828 } 829 830 #if defined(__amd64) 831 /* 832 * Deal with a possible hole in the address range between 833 * hole_start and hole_end that should never be mapped. 834 */ 835 if (lo < hole_start) { 836 if (hi > hole_start) { 837 if (hi < hole_end) { 838 hi = hole_start; 839 } else { 840 /* lo < hole_start && hi >= hole_end */ 841 if (dir == AH_LO) { 842 /* 843 * prefer lowest range 844 */ 845 if (hole_start - lo >= tot_len) 846 hi = hole_start; 847 else if (hi - hole_end >= tot_len) 848 lo = hole_end; 849 else 850 return (0); 851 } else { 852 /* 853 * prefer highest range 854 */ 855 if (hi - hole_end >= tot_len) 856 lo = hole_end; 857 else if (hole_start - lo >= tot_len) 858 hi = hole_start; 859 else 860 return (0); 861 } 862 } 863 } 864 } else { 865 /* lo >= hole_start */ 866 if (hi < hole_end) 867 return (0); 868 if (lo < hole_end) 869 lo = hole_end; 870 } 871 #endif 872 873 if (hi - lo < tot_len) 874 return (0); 875 876 if (align > 1) { 877 uintptr_t tlo = lo + redzone; 878 uintptr_t thi = hi - redzone; 879 tlo = (uintptr_t)P2PHASEUP(tlo, align, off); 880 if (tlo < lo + redzone) { 881 return (0); 882 } 883 if (thi < tlo || thi - tlo < minlen) { 884 return (0); 885 } 886 } 887 888 *basep = (caddr_t)lo; 889 *lenp = hi - lo; 890 return (1); 891 } 892 893 /* 894 * Determine whether [*basep, *basep + *lenp) contains a mappable range of 895 * addresses at least "minlen" long. On success, 1 is returned and *basep 896 * and *lenp are adjusted to describe the acceptable range. On failure, 0 897 * is returned. 898 */ 899 int 900 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir) 901 { 902 return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0)); 903 } 904 905 /* 906 * Determine whether [addr, addr+len] are valid user addresses. 907 */ 908 /*ARGSUSED*/ 909 int 910 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as, 911 caddr_t userlimit) 912 { 913 caddr_t eaddr = addr + len; 914 915 if (eaddr <= addr || addr >= userlimit || eaddr > userlimit) 916 return (RANGE_BADADDR); 917 918 #if defined(__amd64) 919 /* 920 * Check for the VA hole 921 */ 922 if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end) 923 return (RANGE_BADADDR); 924 #endif 925 926 return (RANGE_OKAY); 927 } 928 929 /* 930 * Return 1 if the page frame is onboard memory, else 0. 931 */ 932 int 933 pf_is_memory(pfn_t pf) 934 { 935 if (pfn_is_foreign(pf)) 936 return (0); 937 return (address_in_memlist(phys_install, pfn_to_pa(pf), 1)); 938 } 939 940 /* 941 * return the memrange containing pfn 942 */ 943 int 944 memrange_num(pfn_t pfn) 945 { 946 int n; 947 948 for (n = 0; n < nranges - 1; ++n) { 949 if (pfn >= memranges[n]) 950 break; 951 } 952 return (n); 953 } 954 955 /* 956 * return the mnoderange containing pfn 957 */ 958 /*ARGSUSED*/ 959 int 960 pfn_2_mtype(pfn_t pfn) 961 { 962 #if defined(__xpv) 963 return (0); 964 #else 965 int n; 966 967 /* Always start from highest pfn and work our way down */ 968 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) { 969 if (pfn >= mnoderanges[n].mnr_pfnlo) { 970 break; 971 } 972 } 973 return (n); 974 #endif 975 } 976 977 #if !defined(__xpv) 978 /* 979 * is_contigpage_free: 980 * returns a page list of contiguous pages. It minimally has to return 981 * minctg pages. Caller determines minctg based on the scatter-gather 982 * list length. 983 * 984 * pfnp is set to the next page frame to search on return. 985 */ 986 static page_t * 987 is_contigpage_free( 988 pfn_t *pfnp, 989 pgcnt_t *pgcnt, 990 pgcnt_t minctg, 991 uint64_t pfnseg, 992 int iolock) 993 { 994 int i = 0; 995 pfn_t pfn = *pfnp; 996 page_t *pp; 997 page_t *plist = NULL; 998 999 /* 1000 * fail if pfn + minctg crosses a segment boundary. 1001 * Adjust for next starting pfn to begin at segment boundary. 1002 */ 1003 1004 if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) { 1005 *pfnp = roundup(*pfnp, pfnseg + 1); 1006 return (NULL); 1007 } 1008 1009 do { 1010 retry: 1011 pp = page_numtopp_nolock(pfn + i); 1012 if ((pp == NULL) || 1013 (page_trylock(pp, SE_EXCL) == 0)) { 1014 (*pfnp)++; 1015 break; 1016 } 1017 if (page_pptonum(pp) != pfn + i) { 1018 page_unlock(pp); 1019 goto retry; 1020 } 1021 1022 if (!(PP_ISFREE(pp))) { 1023 page_unlock(pp); 1024 (*pfnp)++; 1025 break; 1026 } 1027 1028 if (!PP_ISAGED(pp)) { 1029 page_list_sub(pp, PG_CACHE_LIST); 1030 page_hashout(pp, (kmutex_t *)NULL); 1031 } else { 1032 page_list_sub(pp, PG_FREE_LIST); 1033 } 1034 1035 if (iolock) 1036 page_io_lock(pp); 1037 page_list_concat(&plist, &pp); 1038 1039 /* 1040 * exit loop when pgcnt satisfied or segment boundary reached. 1041 */ 1042 1043 } while ((++i < *pgcnt) && ((pfn + i) & pfnseg)); 1044 1045 *pfnp += i; /* set to next pfn to search */ 1046 1047 if (i >= minctg) { 1048 *pgcnt -= i; 1049 return (plist); 1050 } 1051 1052 /* 1053 * failure: minctg not satisfied. 1054 * 1055 * if next request crosses segment boundary, set next pfn 1056 * to search from the segment boundary. 1057 */ 1058 if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) 1059 *pfnp = roundup(*pfnp, pfnseg + 1); 1060 1061 /* clean up any pages already allocated */ 1062 1063 while (plist) { 1064 pp = plist; 1065 page_sub(&plist, pp); 1066 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 1067 if (iolock) 1068 page_io_unlock(pp); 1069 page_unlock(pp); 1070 } 1071 1072 return (NULL); 1073 } 1074 #endif /* !__xpv */ 1075 1076 /* 1077 * verify that pages being returned from allocator have correct DMA attribute 1078 */ 1079 #ifndef DEBUG 1080 #define check_dma(a, b, c) (void)(0) 1081 #else 1082 static void 1083 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt) 1084 { 1085 if (dma_attr == NULL) 1086 return; 1087 1088 while (cnt-- > 0) { 1089 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) < 1090 dma_attr->dma_attr_addr_lo) 1091 panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp); 1092 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >= 1093 dma_attr->dma_attr_addr_hi) 1094 panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp); 1095 pp = pp->p_next; 1096 } 1097 } 1098 #endif 1099 1100 #if !defined(__xpv) 1101 static page_t * 1102 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock) 1103 { 1104 pfn_t pfn; 1105 int sgllen; 1106 uint64_t pfnseg; 1107 pgcnt_t minctg; 1108 page_t *pplist = NULL, *plist; 1109 uint64_t lo, hi; 1110 pgcnt_t pfnalign = 0; 1111 static pfn_t startpfn; 1112 static pgcnt_t lastctgcnt; 1113 uintptr_t align; 1114 1115 CONTIG_LOCK(); 1116 1117 if (mattr) { 1118 lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET)); 1119 hi = mmu_btop(mattr->dma_attr_addr_hi); 1120 if (hi >= physmax) 1121 hi = physmax - 1; 1122 sgllen = mattr->dma_attr_sgllen; 1123 pfnseg = mmu_btop(mattr->dma_attr_seg); 1124 1125 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer); 1126 if (align > MMU_PAGESIZE) 1127 pfnalign = mmu_btop(align); 1128 1129 /* 1130 * in order to satisfy the request, must minimally 1131 * acquire minctg contiguous pages 1132 */ 1133 minctg = howmany(*pgcnt, sgllen); 1134 1135 ASSERT(hi >= lo); 1136 1137 /* 1138 * start from where last searched if the minctg >= lastctgcnt 1139 */ 1140 if (minctg < lastctgcnt || startpfn < lo || startpfn > hi) 1141 startpfn = lo; 1142 } else { 1143 hi = physmax - 1; 1144 lo = 0; 1145 sgllen = 1; 1146 pfnseg = mmu.highest_pfn; 1147 minctg = *pgcnt; 1148 1149 if (minctg < lastctgcnt) 1150 startpfn = lo; 1151 } 1152 lastctgcnt = minctg; 1153 1154 ASSERT(pfnseg + 1 >= (uint64_t)minctg); 1155 1156 /* conserve 16m memory - start search above 16m when possible */ 1157 if (hi > PFN_16M && startpfn < PFN_16M) 1158 startpfn = PFN_16M; 1159 1160 pfn = startpfn; 1161 if (pfnalign) 1162 pfn = P2ROUNDUP(pfn, pfnalign); 1163 1164 while (pfn + minctg - 1 <= hi) { 1165 1166 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock); 1167 if (plist) { 1168 page_list_concat(&pplist, &plist); 1169 sgllen--; 1170 /* 1171 * return when contig pages no longer needed 1172 */ 1173 if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) { 1174 startpfn = pfn; 1175 CONTIG_UNLOCK(); 1176 check_dma(mattr, pplist, *pgcnt); 1177 return (pplist); 1178 } 1179 minctg = howmany(*pgcnt, sgllen); 1180 } 1181 if (pfnalign) 1182 pfn = P2ROUNDUP(pfn, pfnalign); 1183 } 1184 1185 /* cannot find contig pages in specified range */ 1186 if (startpfn == lo) { 1187 CONTIG_UNLOCK(); 1188 return (NULL); 1189 } 1190 1191 /* did not start with lo previously */ 1192 pfn = lo; 1193 if (pfnalign) 1194 pfn = P2ROUNDUP(pfn, pfnalign); 1195 1196 /* allow search to go above startpfn */ 1197 while (pfn < startpfn) { 1198 1199 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock); 1200 if (plist != NULL) { 1201 1202 page_list_concat(&pplist, &plist); 1203 sgllen--; 1204 1205 /* 1206 * return when contig pages no longer needed 1207 */ 1208 if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) { 1209 startpfn = pfn; 1210 CONTIG_UNLOCK(); 1211 check_dma(mattr, pplist, *pgcnt); 1212 return (pplist); 1213 } 1214 minctg = howmany(*pgcnt, sgllen); 1215 } 1216 if (pfnalign) 1217 pfn = P2ROUNDUP(pfn, pfnalign); 1218 } 1219 CONTIG_UNLOCK(); 1220 return (NULL); 1221 } 1222 #endif /* !__xpv */ 1223 1224 /* 1225 * mnode_range_cnt() calculates the number of memory ranges for mnode and 1226 * memranges[]. Used to determine the size of page lists and mnoderanges. 1227 */ 1228 int 1229 mnode_range_cnt(int mnode) 1230 { 1231 #if defined(__xpv) 1232 ASSERT(mnode == 0); 1233 return (1); 1234 #else /* __xpv */ 1235 int mri; 1236 int mnrcnt = 0; 1237 1238 if (mem_node_config[mnode].exists != 0) { 1239 mri = nranges - 1; 1240 1241 /* find the memranges index below contained in mnode range */ 1242 1243 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase) 1244 mri--; 1245 1246 /* 1247 * increment mnode range counter when memranges or mnode 1248 * boundary is reached. 1249 */ 1250 while (mri >= 0 && 1251 mem_node_config[mnode].physmax >= MEMRANGELO(mri)) { 1252 mnrcnt++; 1253 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri)) 1254 mri--; 1255 else 1256 break; 1257 } 1258 } 1259 ASSERT(mnrcnt <= MAX_MNODE_MRANGES); 1260 return (mnrcnt); 1261 #endif /* __xpv */ 1262 } 1263 1264 /* 1265 * mnode_range_setup() initializes mnoderanges. 1266 */ 1267 void 1268 mnode_range_setup(mnoderange_t *mnoderanges) 1269 { 1270 mnoderange_t *mp = mnoderanges; 1271 int mnode, mri; 1272 int mindex = 0; /* current index into mnoderanges array */ 1273 int i, j; 1274 pfn_t hipfn; 1275 int last, hi; 1276 1277 for (mnode = 0; mnode < max_mem_nodes; mnode++) { 1278 if (mem_node_config[mnode].exists == 0) 1279 continue; 1280 1281 mri = nranges - 1; 1282 1283 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase) 1284 mri--; 1285 1286 while (mri >= 0 && mem_node_config[mnode].physmax >= 1287 MEMRANGELO(mri)) { 1288 mnoderanges->mnr_pfnlo = MAX(MEMRANGELO(mri), 1289 mem_node_config[mnode].physbase); 1290 mnoderanges->mnr_pfnhi = MIN(MEMRANGEHI(mri), 1291 mem_node_config[mnode].physmax); 1292 mnoderanges->mnr_mnode = mnode; 1293 mnoderanges->mnr_memrange = mri; 1294 mnoderanges->mnr_exists = 1; 1295 mnoderanges++; 1296 mindex++; 1297 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri)) 1298 mri--; 1299 else 1300 break; 1301 } 1302 } 1303 1304 /* 1305 * For now do a simple sort of the mnoderanges array to fill in 1306 * the mnr_next fields. Since mindex is expected to be relatively 1307 * small, using a simple O(N^2) algorithm. 1308 */ 1309 for (i = 0; i < mindex; i++) { 1310 if (mp[i].mnr_pfnlo == 0) /* find lowest */ 1311 break; 1312 } 1313 ASSERT(i < mindex); 1314 last = i; 1315 mtype16m = last; 1316 mp[last].mnr_next = -1; 1317 for (i = 0; i < mindex - 1; i++) { 1318 hipfn = (pfn_t)(-1); 1319 hi = -1; 1320 /* find next highest mnode range */ 1321 for (j = 0; j < mindex; j++) { 1322 if (mp[j].mnr_pfnlo > mp[last].mnr_pfnlo && 1323 mp[j].mnr_pfnlo < hipfn) { 1324 hipfn = mp[j].mnr_pfnlo; 1325 hi = j; 1326 } 1327 } 1328 mp[hi].mnr_next = last; 1329 last = hi; 1330 } 1331 mtypetop = last; 1332 } 1333 1334 #ifndef __xpv 1335 /* 1336 * Update mnoderanges for memory hot-add DR operations. 1337 */ 1338 static void 1339 mnode_range_add(int mnode) 1340 { 1341 int *prev; 1342 int n, mri; 1343 pfn_t start, end; 1344 extern void membar_sync(void); 1345 1346 ASSERT(0 <= mnode && mnode < max_mem_nodes); 1347 ASSERT(mem_node_config[mnode].exists); 1348 start = mem_node_config[mnode].physbase; 1349 end = mem_node_config[mnode].physmax; 1350 ASSERT(start <= end); 1351 mutex_enter(&mnoderange_lock); 1352 1353 #ifdef DEBUG 1354 /* Check whether it interleaves with other memory nodes. */ 1355 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) { 1356 ASSERT(mnoderanges[n].mnr_exists); 1357 if (mnoderanges[n].mnr_mnode == mnode) 1358 continue; 1359 ASSERT(start > mnoderanges[n].mnr_pfnhi || 1360 end < mnoderanges[n].mnr_pfnlo); 1361 } 1362 #endif /* DEBUG */ 1363 1364 mri = nranges - 1; 1365 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase) 1366 mri--; 1367 while (mri >= 0 && mem_node_config[mnode].physmax >= MEMRANGELO(mri)) { 1368 /* Check whether mtype already exists. */ 1369 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) { 1370 if (mnoderanges[n].mnr_mnode == mnode && 1371 mnoderanges[n].mnr_memrange == mri) { 1372 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), 1373 start); 1374 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), 1375 end); 1376 break; 1377 } 1378 } 1379 1380 /* Add a new entry if it doesn't exist yet. */ 1381 if (n == -1) { 1382 /* Try to find an unused entry in mnoderanges array. */ 1383 for (n = 0; n < mnoderangecnt; n++) { 1384 if (mnoderanges[n].mnr_exists == 0) 1385 break; 1386 } 1387 ASSERT(n < mnoderangecnt); 1388 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), start); 1389 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), end); 1390 mnoderanges[n].mnr_mnode = mnode; 1391 mnoderanges[n].mnr_memrange = mri; 1392 mnoderanges[n].mnr_exists = 1; 1393 /* Page 0 should always be present. */ 1394 for (prev = &mtypetop; 1395 mnoderanges[*prev].mnr_pfnlo > start; 1396 prev = &mnoderanges[*prev].mnr_next) { 1397 ASSERT(mnoderanges[*prev].mnr_next >= 0); 1398 ASSERT(mnoderanges[*prev].mnr_pfnlo > end); 1399 } 1400 mnoderanges[n].mnr_next = *prev; 1401 membar_sync(); 1402 *prev = n; 1403 } 1404 1405 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri)) 1406 mri--; 1407 else 1408 break; 1409 } 1410 1411 mutex_exit(&mnoderange_lock); 1412 } 1413 1414 /* 1415 * Update mnoderanges for memory hot-removal DR operations. 1416 */ 1417 static void 1418 mnode_range_del(int mnode) 1419 { 1420 _NOTE(ARGUNUSED(mnode)); 1421 ASSERT(0 <= mnode && mnode < max_mem_nodes); 1422 /* TODO: support deletion operation. */ 1423 ASSERT(0); 1424 } 1425 1426 void 1427 plat_slice_add(pfn_t start, pfn_t end) 1428 { 1429 mem_node_add_slice(start, end); 1430 if (plat_dr_enabled()) { 1431 mnode_range_add(PFN_2_MEM_NODE(start)); 1432 } 1433 } 1434 1435 void 1436 plat_slice_del(pfn_t start, pfn_t end) 1437 { 1438 ASSERT(PFN_2_MEM_NODE(start) == PFN_2_MEM_NODE(end)); 1439 ASSERT(plat_dr_enabled()); 1440 mnode_range_del(PFN_2_MEM_NODE(start)); 1441 mem_node_del_slice(start, end); 1442 } 1443 #endif /* __xpv */ 1444 1445 /*ARGSUSED*/ 1446 int 1447 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz) 1448 { 1449 int mtype = mtypetop; 1450 1451 #if !defined(__xpv) 1452 #if defined(__i386) 1453 /* 1454 * set the mtype range 1455 * - kmem requests need to be below 4g if restricted_kmemalloc is set. 1456 * - for non kmem requests, set range to above 4g if memory below 4g 1457 * runs low. 1458 */ 1459 if (restricted_kmemalloc && VN_ISKAS(vp) && 1460 (caddr_t)(vaddr) >= kernelheap && 1461 (caddr_t)(vaddr) < ekernelheap) { 1462 ASSERT(physmax4g); 1463 mtype = mtype4g; 1464 if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz), 1465 btop(pgsz), *flags)) { 1466 *flags |= PGI_MT_RANGE16M; 1467 } else { 1468 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt); 1469 VM_STAT_COND_ADD((*flags & PG_PANIC), 1470 vmm_vmstats.pgpanicalloc); 1471 *flags |= PGI_MT_RANGE0; 1472 } 1473 return (mtype); 1474 } 1475 #endif /* __i386 */ 1476 1477 if (RESTRICT4G_ALLOC) { 1478 VM_STAT_ADD(vmm_vmstats.restrict4gcnt); 1479 /* here only for > 4g systems */ 1480 *flags |= PGI_MT_RANGE4G; 1481 } else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) { 1482 *flags |= PGI_MT_RANGE16M; 1483 } else { 1484 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt); 1485 VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc); 1486 *flags |= PGI_MT_RANGE0; 1487 } 1488 #endif /* !__xpv */ 1489 return (mtype); 1490 } 1491 1492 1493 /* mtype init for page_get_replacement_page */ 1494 /*ARGSUSED*/ 1495 int 1496 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt) 1497 { 1498 int mtype = mtypetop; 1499 #if !defined(__xpv) 1500 if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) { 1501 *flags |= PGI_MT_RANGE16M; 1502 } else { 1503 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt); 1504 *flags |= PGI_MT_RANGE0; 1505 } 1506 #endif 1507 return (mtype); 1508 } 1509 1510 /* 1511 * Determine if the mnode range specified in mtype contains memory belonging 1512 * to memory node mnode. If flags & PGI_MT_RANGE is set then mtype contains 1513 * the range from high pfn to 0, 16m or 4g. 1514 * 1515 * Return first mnode range type index found otherwise return -1 if none found. 1516 */ 1517 int 1518 mtype_func(int mnode, int mtype, uint_t flags) 1519 { 1520 if (flags & PGI_MT_RANGE) { 1521 int mnr_lim = MRI_0; 1522 1523 if (flags & PGI_MT_NEXT) { 1524 mtype = mnoderanges[mtype].mnr_next; 1525 } 1526 if (flags & PGI_MT_RANGE4G) 1527 mnr_lim = MRI_4G; /* exclude 0-4g range */ 1528 else if (flags & PGI_MT_RANGE16M) 1529 mnr_lim = MRI_16M; /* exclude 0-16m range */ 1530 while (mtype != -1 && 1531 mnoderanges[mtype].mnr_memrange <= mnr_lim) { 1532 if (mnoderanges[mtype].mnr_mnode == mnode) 1533 return (mtype); 1534 mtype = mnoderanges[mtype].mnr_next; 1535 } 1536 } else if (mnoderanges[mtype].mnr_mnode == mnode) { 1537 return (mtype); 1538 } 1539 return (-1); 1540 } 1541 1542 /* 1543 * Update the page list max counts with the pfn range specified by the 1544 * input parameters. 1545 */ 1546 void 1547 mtype_modify_max(pfn_t startpfn, long cnt) 1548 { 1549 int mtype; 1550 pgcnt_t inc; 1551 spgcnt_t scnt = (spgcnt_t)(cnt); 1552 pgcnt_t acnt = ABS(scnt); 1553 pfn_t endpfn = startpfn + acnt; 1554 pfn_t pfn, lo; 1555 1556 if (!physmax4g) 1557 return; 1558 1559 mtype = mtypetop; 1560 for (pfn = endpfn; pfn > startpfn; ) { 1561 ASSERT(mtype != -1); 1562 lo = mnoderanges[mtype].mnr_pfnlo; 1563 if (pfn > lo) { 1564 if (startpfn >= lo) { 1565 inc = pfn - startpfn; 1566 } else { 1567 inc = pfn - lo; 1568 } 1569 if (mnoderanges[mtype].mnr_memrange != MRI_4G) { 1570 if (scnt > 0) 1571 maxmem4g += inc; 1572 else 1573 maxmem4g -= inc; 1574 } 1575 pfn -= inc; 1576 } 1577 mtype = mnoderanges[mtype].mnr_next; 1578 } 1579 } 1580 1581 int 1582 mtype_2_mrange(int mtype) 1583 { 1584 return (mnoderanges[mtype].mnr_memrange); 1585 } 1586 1587 void 1588 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi) 1589 { 1590 _NOTE(ARGUNUSED(mnode)); 1591 ASSERT(mnoderanges[mtype].mnr_mnode == mnode); 1592 *pfnlo = mnoderanges[mtype].mnr_pfnlo; 1593 *pfnhi = mnoderanges[mtype].mnr_pfnhi; 1594 } 1595 1596 size_t 1597 plcnt_sz(size_t ctrs_sz) 1598 { 1599 #ifdef DEBUG 1600 int szc, colors; 1601 1602 ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes; 1603 for (szc = 0; szc < mmu_page_sizes; szc++) { 1604 colors = page_get_pagecolors(szc); 1605 ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors; 1606 } 1607 #endif 1608 return (ctrs_sz); 1609 } 1610 1611 caddr_t 1612 plcnt_init(caddr_t addr) 1613 { 1614 #ifdef DEBUG 1615 int mt, szc, colors; 1616 1617 for (mt = 0; mt < mnoderangecnt; mt++) { 1618 mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr; 1619 addr += (sizeof (struct mnr_mts) * mmu_page_sizes); 1620 for (szc = 0; szc < mmu_page_sizes; szc++) { 1621 colors = page_get_pagecolors(szc); 1622 mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors; 1623 mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt = 1624 (pgcnt_t *)addr; 1625 addr += (sizeof (pgcnt_t) * colors); 1626 } 1627 } 1628 #endif 1629 return (addr); 1630 } 1631 1632 void 1633 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags) 1634 { 1635 _NOTE(ARGUNUSED(pp)); 1636 #ifdef DEBUG 1637 int bin = PP_2_BIN(pp); 1638 1639 atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt); 1640 atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin], 1641 cnt); 1642 #endif 1643 ASSERT(mtype == PP_2_MTYPE(pp)); 1644 if (physmax4g && mnoderanges[mtype].mnr_memrange != MRI_4G) 1645 atomic_add_long(&freemem4g, cnt); 1646 if (flags & PG_CACHE_LIST) 1647 atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt); 1648 else 1649 atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt); 1650 atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt); 1651 } 1652 1653 /* 1654 * Returns the free page count for mnode 1655 */ 1656 int 1657 mnode_pgcnt(int mnode) 1658 { 1659 int mtype = mtypetop; 1660 int flags = PGI_MT_RANGE0; 1661 pgcnt_t pgcnt = 0; 1662 1663 mtype = mtype_func(mnode, mtype, flags); 1664 1665 while (mtype != -1) { 1666 pgcnt += MTYPE_FREEMEM(mtype); 1667 mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT); 1668 } 1669 return (pgcnt); 1670 } 1671 1672 /* 1673 * Initialize page coloring variables based on the l2 cache parameters. 1674 * Calculate and return memory needed for page coloring data structures. 1675 */ 1676 size_t 1677 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc) 1678 { 1679 _NOTE(ARGUNUSED(l2_linesz)); 1680 size_t colorsz = 0; 1681 int i; 1682 int colors; 1683 1684 #if defined(__xpv) 1685 /* 1686 * Hypervisor domains currently don't have any concept of NUMA. 1687 * Hence we'll act like there is only 1 memrange. 1688 */ 1689 i = memrange_num(1); 1690 #else /* !__xpv */ 1691 /* 1692 * Reduce the memory ranges lists if we don't have large amounts 1693 * of memory. This avoids searching known empty free lists. 1694 * To support memory DR operations, we need to keep memory ranges 1695 * for possible memory hot-add operations. 1696 */ 1697 if (plat_dr_physmax > physmax) 1698 i = memrange_num(plat_dr_physmax); 1699 else 1700 i = memrange_num(physmax); 1701 #if defined(__i386) 1702 if (i > MRI_4G) 1703 restricted_kmemalloc = 0; 1704 #endif 1705 /* physmax greater than 4g */ 1706 if (i == MRI_4G) 1707 physmax4g = 1; 1708 #endif /* !__xpv */ 1709 memranges += i; 1710 nranges -= i; 1711 1712 ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES); 1713 1714 ASSERT(ISP2(l2_linesz)); 1715 ASSERT(l2_sz > MMU_PAGESIZE); 1716 1717 /* l2_assoc is 0 for fully associative l2 cache */ 1718 if (l2_assoc) 1719 l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE)); 1720 else 1721 l2_colors = 1; 1722 1723 ASSERT(ISP2(l2_colors)); 1724 1725 /* for scalability, configure at least PAGE_COLORS_MIN color bins */ 1726 page_colors = MAX(l2_colors, PAGE_COLORS_MIN); 1727 1728 /* 1729 * cpu_page_colors is non-zero when a page color may be spread across 1730 * multiple bins. 1731 */ 1732 if (l2_colors < page_colors) 1733 cpu_page_colors = l2_colors; 1734 1735 ASSERT(ISP2(page_colors)); 1736 1737 page_colors_mask = page_colors - 1; 1738 1739 ASSERT(ISP2(CPUSETSIZE())); 1740 page_coloring_shift = lowbit(CPUSETSIZE()); 1741 1742 /* initialize number of colors per page size */ 1743 for (i = 0; i <= mmu.max_page_level; i++) { 1744 hw_page_array[i].hp_size = LEVEL_SIZE(i); 1745 hw_page_array[i].hp_shift = LEVEL_SHIFT(i); 1746 hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0); 1747 hw_page_array[i].hp_colors = (page_colors_mask >> 1748 (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift)) 1749 + 1; 1750 colorequivszc[i] = 0; 1751 } 1752 1753 /* 1754 * The value of cpu_page_colors determines if additional color bins 1755 * need to be checked for a particular color in the page_get routines. 1756 */ 1757 if (cpu_page_colors != 0) { 1758 1759 int a = lowbit(page_colors) - lowbit(cpu_page_colors); 1760 ASSERT(a > 0); 1761 ASSERT(a < 16); 1762 1763 for (i = 0; i <= mmu.max_page_level; i++) { 1764 if ((colors = hw_page_array[i].hp_colors) <= 1) { 1765 colorequivszc[i] = 0; 1766 continue; 1767 } 1768 while ((colors >> a) == 0) 1769 a--; 1770 ASSERT(a >= 0); 1771 1772 /* higher 4 bits encodes color equiv mask */ 1773 colorequivszc[i] = (a << 4); 1774 } 1775 } 1776 1777 /* factor in colorequiv to check additional 'equivalent' bins. */ 1778 if (colorequiv > 1) { 1779 1780 int a = lowbit(colorequiv) - 1; 1781 if (a > 15) 1782 a = 15; 1783 1784 for (i = 0; i <= mmu.max_page_level; i++) { 1785 if ((colors = hw_page_array[i].hp_colors) <= 1) { 1786 continue; 1787 } 1788 while ((colors >> a) == 0) 1789 a--; 1790 if ((a << 4) > colorequivszc[i]) { 1791 colorequivszc[i] = (a << 4); 1792 } 1793 } 1794 } 1795 1796 /* size for mnoderanges */ 1797 for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++) 1798 mnoderangecnt += mnode_range_cnt(i); 1799 if (plat_dr_support_memory()) { 1800 /* 1801 * Reserve enough space for memory DR operations. 1802 * Two extra mnoderanges for possbile fragmentations, 1803 * one for the 2G boundary and the other for the 4G boundary. 1804 * We don't expect a memory board crossing the 16M boundary 1805 * for memory hot-add operations on x86 platforms. 1806 */ 1807 mnoderangecnt += 2 + max_mem_nodes - lgrp_plat_node_cnt; 1808 } 1809 colorsz = mnoderangecnt * sizeof (mnoderange_t); 1810 1811 /* size for fpc_mutex and cpc_mutex */ 1812 colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX); 1813 1814 /* size of page_freelists */ 1815 colorsz += mnoderangecnt * sizeof (page_t ***); 1816 colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **); 1817 1818 for (i = 0; i < mmu_page_sizes; i++) { 1819 colors = page_get_pagecolors(i); 1820 colorsz += mnoderangecnt * colors * sizeof (page_t *); 1821 } 1822 1823 /* size of page_cachelists */ 1824 colorsz += mnoderangecnt * sizeof (page_t **); 1825 colorsz += mnoderangecnt * page_colors * sizeof (page_t *); 1826 1827 return (colorsz); 1828 } 1829 1830 /* 1831 * Called once at startup to configure page_coloring data structures and 1832 * does the 1st page_free()/page_freelist_add(). 1833 */ 1834 void 1835 page_coloring_setup(caddr_t pcmemaddr) 1836 { 1837 int i; 1838 int j; 1839 int k; 1840 caddr_t addr; 1841 int colors; 1842 1843 /* 1844 * do page coloring setup 1845 */ 1846 addr = pcmemaddr; 1847 1848 mnoderanges = (mnoderange_t *)addr; 1849 addr += (mnoderangecnt * sizeof (mnoderange_t)); 1850 1851 mnode_range_setup(mnoderanges); 1852 1853 if (physmax4g) 1854 mtype4g = pfn_2_mtype(0xfffff); 1855 1856 for (k = 0; k < NPC_MUTEX; k++) { 1857 fpc_mutex[k] = (kmutex_t *)addr; 1858 addr += (max_mem_nodes * sizeof (kmutex_t)); 1859 } 1860 for (k = 0; k < NPC_MUTEX; k++) { 1861 cpc_mutex[k] = (kmutex_t *)addr; 1862 addr += (max_mem_nodes * sizeof (kmutex_t)); 1863 } 1864 page_freelists = (page_t ****)addr; 1865 addr += (mnoderangecnt * sizeof (page_t ***)); 1866 1867 page_cachelists = (page_t ***)addr; 1868 addr += (mnoderangecnt * sizeof (page_t **)); 1869 1870 for (i = 0; i < mnoderangecnt; i++) { 1871 page_freelists[i] = (page_t ***)addr; 1872 addr += (mmu_page_sizes * sizeof (page_t **)); 1873 1874 for (j = 0; j < mmu_page_sizes; j++) { 1875 colors = page_get_pagecolors(j); 1876 page_freelists[i][j] = (page_t **)addr; 1877 addr += (colors * sizeof (page_t *)); 1878 } 1879 page_cachelists[i] = (page_t **)addr; 1880 addr += (page_colors * sizeof (page_t *)); 1881 } 1882 } 1883 1884 #if defined(__xpv) 1885 /* 1886 * Give back 10% of the io_pool pages to the free list. 1887 * Don't shrink the pool below some absolute minimum. 1888 */ 1889 static void 1890 page_io_pool_shrink() 1891 { 1892 int retcnt; 1893 page_t *pp, *pp_first, *pp_last, **curpool; 1894 mfn_t mfn; 1895 int bothpools = 0; 1896 1897 mutex_enter(&io_pool_lock); 1898 io_pool_shrink_attempts++; /* should be a kstat? */ 1899 retcnt = io_pool_cnt / 10; 1900 if (io_pool_cnt - retcnt < io_pool_cnt_min) 1901 retcnt = io_pool_cnt - io_pool_cnt_min; 1902 if (retcnt <= 0) 1903 goto done; 1904 io_pool_shrinks++; /* should be a kstat? */ 1905 curpool = &io_pool_4g; 1906 domore: 1907 /* 1908 * Loop through taking pages from the end of the list 1909 * (highest mfns) till amount to return reached. 1910 */ 1911 for (pp = *curpool; pp && retcnt > 0; ) { 1912 pp_first = pp_last = pp->p_prev; 1913 if (pp_first == *curpool) 1914 break; 1915 retcnt--; 1916 io_pool_cnt--; 1917 page_io_pool_sub(curpool, pp_first, pp_last); 1918 if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn) 1919 start_mfn = mfn; 1920 page_free(pp_first, 1); 1921 pp = *curpool; 1922 } 1923 if (retcnt != 0 && !bothpools) { 1924 /* 1925 * If not enough found in less constrained pool try the 1926 * more constrained one. 1927 */ 1928 curpool = &io_pool_16m; 1929 bothpools = 1; 1930 goto domore; 1931 } 1932 done: 1933 mutex_exit(&io_pool_lock); 1934 } 1935 1936 #endif /* __xpv */ 1937 1938 uint_t 1939 page_create_update_flags_x86(uint_t flags) 1940 { 1941 #if defined(__xpv) 1942 /* 1943 * Check this is an urgent allocation and free pages are depleted. 1944 */ 1945 if (!(flags & PG_WAIT) && freemem < desfree) 1946 page_io_pool_shrink(); 1947 #else /* !__xpv */ 1948 /* 1949 * page_create_get_something may call this because 4g memory may be 1950 * depleted. Set flags to allow for relocation of base page below 1951 * 4g if necessary. 1952 */ 1953 if (physmax4g) 1954 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI); 1955 #endif /* __xpv */ 1956 return (flags); 1957 } 1958 1959 /*ARGSUSED*/ 1960 int 1961 bp_color(struct buf *bp) 1962 { 1963 return (0); 1964 } 1965 1966 #if defined(__xpv) 1967 1968 /* 1969 * Take pages out of an io_pool 1970 */ 1971 static void 1972 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last) 1973 { 1974 if (*poolp == pp_first) { 1975 *poolp = pp_last->p_next; 1976 if (*poolp == pp_first) 1977 *poolp = NULL; 1978 } 1979 pp_first->p_prev->p_next = pp_last->p_next; 1980 pp_last->p_next->p_prev = pp_first->p_prev; 1981 pp_first->p_prev = pp_last; 1982 pp_last->p_next = pp_first; 1983 } 1984 1985 /* 1986 * Put a page on the io_pool list. The list is ordered by increasing MFN. 1987 */ 1988 static void 1989 page_io_pool_add(page_t **poolp, page_t *pp) 1990 { 1991 page_t *look; 1992 mfn_t mfn = mfn_list[pp->p_pagenum]; 1993 1994 if (*poolp == NULL) { 1995 *poolp = pp; 1996 pp->p_next = pp; 1997 pp->p_prev = pp; 1998 return; 1999 } 2000 2001 /* 2002 * Since we try to take pages from the high end of the pool 2003 * chances are good that the pages to be put on the list will 2004 * go at or near the end of the list. so start at the end and 2005 * work backwards. 2006 */ 2007 look = (*poolp)->p_prev; 2008 while (mfn < mfn_list[look->p_pagenum]) { 2009 look = look->p_prev; 2010 if (look == (*poolp)->p_prev) 2011 break; /* backed all the way to front of list */ 2012 } 2013 2014 /* insert after look */ 2015 pp->p_prev = look; 2016 pp->p_next = look->p_next; 2017 pp->p_next->p_prev = pp; 2018 look->p_next = pp; 2019 if (mfn < mfn_list[(*poolp)->p_pagenum]) { 2020 /* 2021 * we inserted a new first list element 2022 * adjust pool pointer to newly inserted element 2023 */ 2024 *poolp = pp; 2025 } 2026 } 2027 2028 /* 2029 * Add a page to the io_pool. Setting the force flag will force the page 2030 * into the io_pool no matter what. 2031 */ 2032 static void 2033 add_page_to_pool(page_t *pp, int force) 2034 { 2035 page_t *highest; 2036 page_t *freep = NULL; 2037 2038 mutex_enter(&io_pool_lock); 2039 /* 2040 * Always keep the scarce low memory pages 2041 */ 2042 if (mfn_list[pp->p_pagenum] < PFN_16MEG) { 2043 ++io_pool_cnt; 2044 page_io_pool_add(&io_pool_16m, pp); 2045 goto done; 2046 } 2047 if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) { 2048 ++io_pool_cnt; 2049 page_io_pool_add(&io_pool_4g, pp); 2050 } else { 2051 highest = io_pool_4g->p_prev; 2052 if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) { 2053 page_io_pool_sub(&io_pool_4g, highest, highest); 2054 page_io_pool_add(&io_pool_4g, pp); 2055 freep = highest; 2056 } else { 2057 freep = pp; 2058 } 2059 } 2060 done: 2061 mutex_exit(&io_pool_lock); 2062 if (freep) 2063 page_free(freep, 1); 2064 } 2065 2066 2067 int contig_pfn_cnt; /* no of pfns in the contig pfn list */ 2068 int contig_pfn_max; /* capacity of the contig pfn list */ 2069 int next_alloc_pfn; /* next position in list to start a contig search */ 2070 int contig_pfnlist_updates; /* pfn list update count */ 2071 int contig_pfnlist_builds; /* how many times have we (re)built list */ 2072 int contig_pfnlist_buildfailed; /* how many times has list build failed */ 2073 int create_contig_pending; /* nonzero means taskq creating contig list */ 2074 pfn_t *contig_pfn_list = NULL; /* list of contig pfns in ascending mfn order */ 2075 2076 /* 2077 * Function to use in sorting a list of pfns by their underlying mfns. 2078 */ 2079 static int 2080 mfn_compare(const void *pfnp1, const void *pfnp2) 2081 { 2082 mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1]; 2083 mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2]; 2084 2085 if (mfn1 > mfn2) 2086 return (1); 2087 if (mfn1 < mfn2) 2088 return (-1); 2089 return (0); 2090 } 2091 2092 /* 2093 * Compact the contig_pfn_list by tossing all the non-contiguous 2094 * elements from the list. 2095 */ 2096 static void 2097 compact_contig_pfn_list(void) 2098 { 2099 pfn_t pfn, lapfn, prev_lapfn; 2100 mfn_t mfn; 2101 int i, newcnt = 0; 2102 2103 prev_lapfn = 0; 2104 for (i = 0; i < contig_pfn_cnt - 1; i++) { 2105 pfn = contig_pfn_list[i]; 2106 lapfn = contig_pfn_list[i + 1]; 2107 mfn = mfn_list[pfn]; 2108 /* 2109 * See if next pfn is for a contig mfn 2110 */ 2111 if (mfn_list[lapfn] != mfn + 1) 2112 continue; 2113 /* 2114 * pfn and lookahead are both put in list 2115 * unless pfn is the previous lookahead. 2116 */ 2117 if (pfn != prev_lapfn) 2118 contig_pfn_list[newcnt++] = pfn; 2119 contig_pfn_list[newcnt++] = lapfn; 2120 prev_lapfn = lapfn; 2121 } 2122 for (i = newcnt; i < contig_pfn_cnt; i++) 2123 contig_pfn_list[i] = 0; 2124 contig_pfn_cnt = newcnt; 2125 } 2126 2127 /*ARGSUSED*/ 2128 static void 2129 call_create_contiglist(void *arg) 2130 { 2131 (void) create_contig_pfnlist(PG_WAIT); 2132 } 2133 2134 /* 2135 * Create list of freelist pfns that have underlying 2136 * contiguous mfns. The list is kept in ascending mfn order. 2137 * returns 1 if list created else 0. 2138 */ 2139 static int 2140 create_contig_pfnlist(uint_t flags) 2141 { 2142 pfn_t pfn; 2143 page_t *pp; 2144 int ret = 1; 2145 2146 mutex_enter(&contig_list_lock); 2147 if (contig_pfn_list != NULL) 2148 goto out; 2149 contig_pfn_max = freemem + (freemem / 10); 2150 contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t), 2151 (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP); 2152 if (contig_pfn_list == NULL) { 2153 /* 2154 * If we could not create the contig list (because 2155 * we could not sleep for memory). Dispatch a taskq that can 2156 * sleep to get the memory. 2157 */ 2158 if (!create_contig_pending) { 2159 if (taskq_dispatch(system_taskq, call_create_contiglist, 2160 NULL, TQ_NOSLEEP) != NULL) 2161 create_contig_pending = 1; 2162 } 2163 contig_pfnlist_buildfailed++; /* count list build failures */ 2164 ret = 0; 2165 goto out; 2166 } 2167 create_contig_pending = 0; 2168 ASSERT(contig_pfn_cnt == 0); 2169 for (pfn = 0; pfn < mfn_count; pfn++) { 2170 pp = page_numtopp_nolock(pfn); 2171 if (pp == NULL || !PP_ISFREE(pp)) 2172 continue; 2173 contig_pfn_list[contig_pfn_cnt] = pfn; 2174 if (++contig_pfn_cnt == contig_pfn_max) 2175 break; 2176 } 2177 /* 2178 * Sanity check the new list. 2179 */ 2180 if (contig_pfn_cnt < 2) { /* no contig pfns */ 2181 contig_pfn_cnt = 0; 2182 contig_pfnlist_buildfailed++; 2183 kmem_free(contig_pfn_list, contig_pfn_max * sizeof (pfn_t)); 2184 contig_pfn_list = NULL; 2185 contig_pfn_max = 0; 2186 ret = 0; 2187 goto out; 2188 } 2189 qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare); 2190 compact_contig_pfn_list(); 2191 /* 2192 * Make sure next search of the newly created contiguous pfn 2193 * list starts at the beginning of the list. 2194 */ 2195 next_alloc_pfn = 0; 2196 contig_pfnlist_builds++; /* count list builds */ 2197 out: 2198 mutex_exit(&contig_list_lock); 2199 return (ret); 2200 } 2201 2202 2203 /* 2204 * Toss the current contig pfnlist. Someone is about to do a massive 2205 * update to pfn<->mfn mappings. So we have them destroy the list and lock 2206 * it till they are done with their update. 2207 */ 2208 void 2209 clear_and_lock_contig_pfnlist() 2210 { 2211 pfn_t *listp = NULL; 2212 size_t listsize; 2213 2214 mutex_enter(&contig_list_lock); 2215 if (contig_pfn_list != NULL) { 2216 listp = contig_pfn_list; 2217 listsize = contig_pfn_max * sizeof (pfn_t); 2218 contig_pfn_list = NULL; 2219 contig_pfn_max = contig_pfn_cnt = 0; 2220 } 2221 if (listp != NULL) 2222 kmem_free(listp, listsize); 2223 } 2224 2225 /* 2226 * Unlock the contig_pfn_list. The next attempted use of it will cause 2227 * it to be re-created. 2228 */ 2229 void 2230 unlock_contig_pfnlist() 2231 { 2232 mutex_exit(&contig_list_lock); 2233 } 2234 2235 /* 2236 * Update the contiguous pfn list in response to a pfn <-> mfn reassignment 2237 */ 2238 void 2239 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn) 2240 { 2241 int probe_hi, probe_lo, probe_pos, insert_after, insert_point; 2242 pfn_t probe_pfn; 2243 mfn_t probe_mfn; 2244 int drop_lock = 0; 2245 2246 if (mutex_owner(&contig_list_lock) != curthread) { 2247 drop_lock = 1; 2248 mutex_enter(&contig_list_lock); 2249 } 2250 if (contig_pfn_list == NULL) 2251 goto done; 2252 contig_pfnlist_updates++; 2253 /* 2254 * Find the pfn in the current list. Use a binary chop to locate it. 2255 */ 2256 probe_hi = contig_pfn_cnt - 1; 2257 probe_lo = 0; 2258 probe_pos = (probe_hi + probe_lo) / 2; 2259 while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) { 2260 if (probe_pos == probe_lo) { /* pfn not in list */ 2261 probe_pos = -1; 2262 break; 2263 } 2264 if (pfn_to_mfn(probe_pfn) <= oldmfn) 2265 probe_lo = probe_pos; 2266 else 2267 probe_hi = probe_pos; 2268 probe_pos = (probe_hi + probe_lo) / 2; 2269 } 2270 if (probe_pos >= 0) { 2271 /* 2272 * Remove pfn from list and ensure next alloc 2273 * position stays in bounds. 2274 */ 2275 if (--contig_pfn_cnt <= next_alloc_pfn) 2276 next_alloc_pfn = 0; 2277 if (contig_pfn_cnt < 2) { /* no contig pfns */ 2278 contig_pfn_cnt = 0; 2279 kmem_free(contig_pfn_list, 2280 contig_pfn_max * sizeof (pfn_t)); 2281 contig_pfn_list = NULL; 2282 contig_pfn_max = 0; 2283 goto done; 2284 } 2285 ovbcopy(&contig_pfn_list[probe_pos + 1], 2286 &contig_pfn_list[probe_pos], 2287 (contig_pfn_cnt - probe_pos) * sizeof (pfn_t)); 2288 } 2289 if (newmfn == MFN_INVALID) 2290 goto done; 2291 /* 2292 * Check if new mfn has adjacent mfns in the list 2293 */ 2294 probe_hi = contig_pfn_cnt - 1; 2295 probe_lo = 0; 2296 insert_after = -2; 2297 do { 2298 probe_pos = (probe_hi + probe_lo) / 2; 2299 probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]); 2300 if (newmfn == probe_mfn + 1) 2301 insert_after = probe_pos; 2302 else if (newmfn == probe_mfn - 1) 2303 insert_after = probe_pos - 1; 2304 if (probe_pos == probe_lo) 2305 break; 2306 if (probe_mfn <= newmfn) 2307 probe_lo = probe_pos; 2308 else 2309 probe_hi = probe_pos; 2310 } while (insert_after == -2); 2311 /* 2312 * If there is space in the list and there are adjacent mfns 2313 * insert the pfn in to its proper place in the list. 2314 */ 2315 if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) { 2316 insert_point = insert_after + 1; 2317 ovbcopy(&contig_pfn_list[insert_point], 2318 &contig_pfn_list[insert_point + 1], 2319 (contig_pfn_cnt - insert_point) * sizeof (pfn_t)); 2320 contig_pfn_list[insert_point] = pfn; 2321 contig_pfn_cnt++; 2322 } 2323 done: 2324 if (drop_lock) 2325 mutex_exit(&contig_list_lock); 2326 } 2327 2328 /* 2329 * Called to (re-)populate the io_pool from the free page lists. 2330 */ 2331 long 2332 populate_io_pool(void) 2333 { 2334 pfn_t pfn; 2335 mfn_t mfn, max_mfn; 2336 page_t *pp; 2337 2338 /* 2339 * Figure out the bounds of the pool on first invocation. 2340 * We use a percentage of memory for the io pool size. 2341 * we allow that to shrink, but not to less than a fixed minimum 2342 */ 2343 if (io_pool_cnt_max == 0) { 2344 io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct); 2345 io_pool_cnt_lowater = io_pool_cnt_max; 2346 /* 2347 * This is the first time in populate_io_pool, grab a va to use 2348 * when we need to allocate pages. 2349 */ 2350 io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); 2351 } 2352 /* 2353 * If we are out of pages in the pool, then grow the size of the pool 2354 */ 2355 if (io_pool_cnt == 0) { 2356 /* 2357 * Grow the max size of the io pool by 5%, but never more than 2358 * 25% of physical memory. 2359 */ 2360 if (io_pool_cnt_max < physmem / 4) 2361 io_pool_cnt_max += io_pool_cnt_max / 20; 2362 } 2363 io_pool_grows++; /* should be a kstat? */ 2364 2365 /* 2366 * Get highest mfn on this platform, but limit to the 32 bit DMA max. 2367 */ 2368 (void) mfn_to_pfn(start_mfn); 2369 max_mfn = MIN(cached_max_mfn, PFN_4GIG); 2370 for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) { 2371 pfn = mfn_to_pfn(mfn); 2372 if (pfn & PFN_IS_FOREIGN_MFN) 2373 continue; 2374 /* 2375 * try to allocate it from free pages 2376 */ 2377 pp = page_numtopp_alloc(pfn); 2378 if (pp == NULL) 2379 continue; 2380 PP_CLRFREE(pp); 2381 add_page_to_pool(pp, 1); 2382 if (io_pool_cnt >= io_pool_cnt_max) 2383 break; 2384 } 2385 2386 return (io_pool_cnt); 2387 } 2388 2389 /* 2390 * Destroy a page that was being used for DMA I/O. It may or 2391 * may not actually go back to the io_pool. 2392 */ 2393 void 2394 page_destroy_io(page_t *pp) 2395 { 2396 mfn_t mfn = mfn_list[pp->p_pagenum]; 2397 2398 /* 2399 * When the page was alloc'd a reservation was made, release it now 2400 */ 2401 page_unresv(1); 2402 /* 2403 * Unload translations, if any, then hash out the 2404 * page to erase its identity. 2405 */ 2406 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 2407 page_hashout(pp, NULL); 2408 2409 /* 2410 * If the page came from the free lists, just put it back to them. 2411 * DomU pages always go on the free lists as well. 2412 */ 2413 if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) { 2414 page_free(pp, 1); 2415 return; 2416 } 2417 2418 add_page_to_pool(pp, 0); 2419 } 2420 2421 2422 long contig_searches; /* count of times contig pages requested */ 2423 long contig_search_restarts; /* count of contig ranges tried */ 2424 long contig_search_failed; /* count of contig alloc failures */ 2425 2426 /* 2427 * Free partial page list 2428 */ 2429 static void 2430 free_partial_list(page_t **pplist) 2431 { 2432 page_t *pp; 2433 2434 while (*pplist != NULL) { 2435 pp = *pplist; 2436 page_io_pool_sub(pplist, pp, pp); 2437 page_free(pp, 1); 2438 } 2439 } 2440 2441 /* 2442 * Look thru the contiguous pfns that are not part of the io_pool for 2443 * contiguous free pages. Return a list of the found pages or NULL. 2444 */ 2445 page_t * 2446 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg, 2447 pgcnt_t pfnalign) 2448 { 2449 page_t *pp, *plist = NULL; 2450 mfn_t mfn, prev_mfn, start_mfn; 2451 pfn_t pfn; 2452 int pages_needed, pages_requested; 2453 int search_start; 2454 2455 /* 2456 * create the contig pfn list if not already done 2457 */ 2458 retry: 2459 mutex_enter(&contig_list_lock); 2460 if (contig_pfn_list == NULL) { 2461 mutex_exit(&contig_list_lock); 2462 if (!create_contig_pfnlist(flags)) { 2463 return (NULL); 2464 } 2465 goto retry; 2466 } 2467 contig_searches++; 2468 /* 2469 * Search contiguous pfn list for physically contiguous pages not in 2470 * the io_pool. Start the search where the last search left off. 2471 */ 2472 pages_requested = pages_needed = npages; 2473 search_start = next_alloc_pfn; 2474 start_mfn = prev_mfn = 0; 2475 while (pages_needed) { 2476 pfn = contig_pfn_list[next_alloc_pfn]; 2477 mfn = pfn_to_mfn(pfn); 2478 /* 2479 * Check if mfn is first one or contig to previous one and 2480 * if page corresponding to mfn is free and that mfn 2481 * range is not crossing a segment boundary. 2482 */ 2483 if ((prev_mfn == 0 || mfn == prev_mfn + 1) && 2484 (pp = page_numtopp_alloc(pfn)) != NULL && 2485 !((mfn & pfnseg) < (start_mfn & pfnseg))) { 2486 PP_CLRFREE(pp); 2487 page_io_pool_add(&plist, pp); 2488 pages_needed--; 2489 if (prev_mfn == 0) { 2490 if (pfnalign && 2491 mfn != P2ROUNDUP(mfn, pfnalign)) { 2492 /* 2493 * not properly aligned 2494 */ 2495 contig_search_restarts++; 2496 free_partial_list(&plist); 2497 pages_needed = pages_requested; 2498 start_mfn = prev_mfn = 0; 2499 goto skip; 2500 } 2501 start_mfn = mfn; 2502 } 2503 prev_mfn = mfn; 2504 } else { 2505 contig_search_restarts++; 2506 free_partial_list(&plist); 2507 pages_needed = pages_requested; 2508 start_mfn = prev_mfn = 0; 2509 } 2510 skip: 2511 if (++next_alloc_pfn == contig_pfn_cnt) 2512 next_alloc_pfn = 0; 2513 if (next_alloc_pfn == search_start) 2514 break; /* all pfns searched */ 2515 } 2516 mutex_exit(&contig_list_lock); 2517 if (pages_needed) { 2518 contig_search_failed++; 2519 /* 2520 * Failed to find enough contig pages. 2521 * free partial page list 2522 */ 2523 free_partial_list(&plist); 2524 } 2525 return (plist); 2526 } 2527 2528 /* 2529 * Search the reserved io pool pages for a page range with the 2530 * desired characteristics. 2531 */ 2532 page_t * 2533 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg) 2534 { 2535 page_t *pp_first, *pp_last; 2536 page_t *pp, **poolp; 2537 pgcnt_t nwanted, pfnalign; 2538 uint64_t pfnseg; 2539 mfn_t mfn, tmfn, hi_mfn, lo_mfn; 2540 int align, attempt = 0; 2541 2542 if (minctg == 1) 2543 contig = 0; 2544 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo); 2545 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi); 2546 pfnseg = mmu_btop(mattr->dma_attr_seg); 2547 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer); 2548 if (align > MMU_PAGESIZE) 2549 pfnalign = mmu_btop(align); 2550 else 2551 pfnalign = 0; 2552 2553 try_again: 2554 /* 2555 * See if we want pages for a legacy device 2556 */ 2557 if (hi_mfn < PFN_16MEG) 2558 poolp = &io_pool_16m; 2559 else 2560 poolp = &io_pool_4g; 2561 try_smaller: 2562 /* 2563 * Take pages from I/O pool. We'll use pages from the highest 2564 * MFN range possible. 2565 */ 2566 pp_first = pp_last = NULL; 2567 mutex_enter(&io_pool_lock); 2568 nwanted = minctg; 2569 for (pp = *poolp; pp && nwanted > 0; ) { 2570 pp = pp->p_prev; 2571 2572 /* 2573 * skip pages above allowable range 2574 */ 2575 mfn = mfn_list[pp->p_pagenum]; 2576 if (hi_mfn < mfn) 2577 goto skip; 2578 2579 /* 2580 * stop at pages below allowable range 2581 */ 2582 if (lo_mfn > mfn) 2583 break; 2584 restart: 2585 if (pp_last == NULL) { 2586 /* 2587 * Check alignment 2588 */ 2589 tmfn = mfn - (minctg - 1); 2590 if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign)) 2591 goto skip; /* not properly aligned */ 2592 /* 2593 * Check segment 2594 */ 2595 if ((mfn & pfnseg) < (tmfn & pfnseg)) 2596 goto skip; /* crosses seg boundary */ 2597 /* 2598 * Start building page list 2599 */ 2600 pp_first = pp_last = pp; 2601 nwanted--; 2602 } else { 2603 /* 2604 * check physical contiguity if required 2605 */ 2606 if (contig && 2607 mfn_list[pp_first->p_pagenum] != mfn + 1) { 2608 /* 2609 * not a contiguous page, restart list. 2610 */ 2611 pp_last = NULL; 2612 nwanted = minctg; 2613 goto restart; 2614 } else { /* add page to list */ 2615 pp_first = pp; 2616 nwanted--; 2617 } 2618 } 2619 skip: 2620 if (pp == *poolp) 2621 break; 2622 } 2623 2624 /* 2625 * If we didn't find memory. Try the more constrained pool, then 2626 * sweep free pages into the DMA pool and try again. 2627 */ 2628 if (nwanted != 0) { 2629 mutex_exit(&io_pool_lock); 2630 /* 2631 * If we were looking in the less constrained pool and 2632 * didn't find pages, try the more constrained pool. 2633 */ 2634 if (poolp == &io_pool_4g) { 2635 poolp = &io_pool_16m; 2636 goto try_smaller; 2637 } 2638 kmem_reap(); 2639 if (++attempt < 4) { 2640 /* 2641 * Grab some more io_pool pages 2642 */ 2643 (void) populate_io_pool(); 2644 goto try_again; /* go around and retry */ 2645 } 2646 return (NULL); 2647 } 2648 /* 2649 * Found the pages, now snip them from the list 2650 */ 2651 page_io_pool_sub(poolp, pp_first, pp_last); 2652 io_pool_cnt -= minctg; 2653 /* 2654 * reset low water mark 2655 */ 2656 if (io_pool_cnt < io_pool_cnt_lowater) 2657 io_pool_cnt_lowater = io_pool_cnt; 2658 mutex_exit(&io_pool_lock); 2659 return (pp_first); 2660 } 2661 2662 page_t * 2663 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr, 2664 ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg) 2665 { 2666 uint_t kflags; 2667 int order, extra, extpages, i, contig, nbits, extents; 2668 page_t *pp, *expp, *pp_first, **pplist = NULL; 2669 mfn_t *mfnlist = NULL; 2670 2671 contig = flags & PG_PHYSCONTIG; 2672 if (minctg == 1) 2673 contig = 0; 2674 flags &= ~PG_PHYSCONTIG; 2675 kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP; 2676 /* 2677 * Hypervisor will allocate extents, if we want contig 2678 * pages extent must be >= minctg 2679 */ 2680 if (contig) { 2681 order = highbit(minctg) - 1; 2682 if (minctg & ((1 << order) - 1)) 2683 order++; 2684 extpages = 1 << order; 2685 } else { 2686 order = 0; 2687 extpages = minctg; 2688 } 2689 if (extpages > minctg) { 2690 extra = extpages - minctg; 2691 if (!page_resv(extra, kflags)) 2692 return (NULL); 2693 } 2694 pp_first = NULL; 2695 pplist = kmem_alloc(extpages * sizeof (page_t *), kflags); 2696 if (pplist == NULL) 2697 goto balloon_fail; 2698 mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags); 2699 if (mfnlist == NULL) 2700 goto balloon_fail; 2701 pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr); 2702 if (pp == NULL) 2703 goto balloon_fail; 2704 pp_first = pp; 2705 if (extpages > minctg) { 2706 /* 2707 * fill out the rest of extent pages to swap 2708 * with the hypervisor 2709 */ 2710 for (i = 0; i < extra; i++) { 2711 expp = page_create_va(vp, 2712 (u_offset_t)(uintptr_t)io_pool_kva, 2713 PAGESIZE, flags, &kvseg, io_pool_kva); 2714 if (expp == NULL) 2715 goto balloon_fail; 2716 (void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD); 2717 page_io_unlock(expp); 2718 page_hashout(expp, NULL); 2719 page_io_lock(expp); 2720 /* 2721 * add page to end of list 2722 */ 2723 expp->p_prev = pp_first->p_prev; 2724 expp->p_next = pp_first; 2725 expp->p_prev->p_next = expp; 2726 pp_first->p_prev = expp; 2727 } 2728 2729 } 2730 for (i = 0; i < extpages; i++) { 2731 pplist[i] = pp; 2732 pp = pp->p_next; 2733 } 2734 nbits = highbit(mattr->dma_attr_addr_hi); 2735 extents = contig ? 1 : minctg; 2736 if (balloon_replace_pages(extents, pplist, nbits, order, 2737 mfnlist) != extents) { 2738 if (ioalloc_dbg) 2739 cmn_err(CE_NOTE, "request to hypervisor" 2740 " for %d pages, maxaddr %" PRIx64 " failed", 2741 extpages, mattr->dma_attr_addr_hi); 2742 goto balloon_fail; 2743 } 2744 2745 kmem_free(pplist, extpages * sizeof (page_t *)); 2746 kmem_free(mfnlist, extpages * sizeof (mfn_t)); 2747 /* 2748 * Return any excess pages to free list 2749 */ 2750 if (extpages > minctg) { 2751 for (i = 0; i < extra; i++) { 2752 pp = pp_first->p_prev; 2753 page_sub(&pp_first, pp); 2754 page_io_unlock(pp); 2755 page_unresv(1); 2756 page_free(pp, 1); 2757 } 2758 } 2759 return (pp_first); 2760 balloon_fail: 2761 /* 2762 * Return pages to free list and return failure 2763 */ 2764 while (pp_first != NULL) { 2765 pp = pp_first; 2766 page_sub(&pp_first, pp); 2767 page_io_unlock(pp); 2768 if (pp->p_vnode != NULL) 2769 page_hashout(pp, NULL); 2770 page_free(pp, 1); 2771 } 2772 if (pplist) 2773 kmem_free(pplist, extpages * sizeof (page_t *)); 2774 if (mfnlist) 2775 kmem_free(mfnlist, extpages * sizeof (mfn_t)); 2776 page_unresv(extpages - minctg); 2777 return (NULL); 2778 } 2779 2780 static void 2781 return_partial_alloc(page_t *plist) 2782 { 2783 page_t *pp; 2784 2785 while (plist != NULL) { 2786 pp = plist; 2787 page_sub(&plist, pp); 2788 page_io_unlock(pp); 2789 page_destroy_io(pp); 2790 } 2791 } 2792 2793 static page_t * 2794 page_get_contigpages( 2795 struct vnode *vp, 2796 u_offset_t off, 2797 int *npagesp, 2798 uint_t flags, 2799 caddr_t vaddr, 2800 ddi_dma_attr_t *mattr) 2801 { 2802 mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL); 2803 page_t *plist; /* list to return */ 2804 page_t *pp, *mcpl; 2805 int contig, anyaddr, npages, getone = 0; 2806 mfn_t lo_mfn; 2807 mfn_t hi_mfn; 2808 pgcnt_t pfnalign = 0; 2809 int align, sgllen; 2810 uint64_t pfnseg; 2811 pgcnt_t minctg; 2812 2813 npages = *npagesp; 2814 ASSERT(mattr != NULL); 2815 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo); 2816 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi); 2817 sgllen = mattr->dma_attr_sgllen; 2818 pfnseg = mmu_btop(mattr->dma_attr_seg); 2819 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer); 2820 if (align > MMU_PAGESIZE) 2821 pfnalign = mmu_btop(align); 2822 2823 contig = flags & PG_PHYSCONTIG; 2824 if (npages == -1) { 2825 npages = 1; 2826 pfnalign = 0; 2827 } 2828 /* 2829 * Clear the contig flag if only one page is needed. 2830 */ 2831 if (npages == 1) { 2832 getone = 1; 2833 contig = 0; 2834 } 2835 2836 /* 2837 * Check if any page in the system is fine. 2838 */ 2839 anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn; 2840 if (!contig && anyaddr && !pfnalign) { 2841 flags &= ~PG_PHYSCONTIG; 2842 plist = page_create_va(vp, off, npages * MMU_PAGESIZE, 2843 flags, &kvseg, vaddr); 2844 if (plist != NULL) { 2845 *npagesp = 0; 2846 return (plist); 2847 } 2848 } 2849 plist = NULL; 2850 minctg = howmany(npages, sgllen); 2851 while (npages > sgllen || getone) { 2852 if (minctg > npages) 2853 minctg = npages; 2854 mcpl = NULL; 2855 /* 2856 * We could want contig pages with no address range limits. 2857 */ 2858 if (anyaddr && contig) { 2859 /* 2860 * Look for free contig pages to satisfy the request. 2861 */ 2862 mcpl = find_contig_free(minctg, flags, pfnseg, 2863 pfnalign); 2864 } 2865 /* 2866 * Try the reserved io pools next 2867 */ 2868 if (mcpl == NULL) 2869 mcpl = page_io_pool_alloc(mattr, contig, minctg); 2870 if (mcpl != NULL) { 2871 pp = mcpl; 2872 do { 2873 if (!page_hashin(pp, vp, off, NULL)) { 2874 panic("page_get_contigpages:" 2875 " hashin failed" 2876 " pp %p, vp %p, off %llx", 2877 (void *)pp, (void *)vp, off); 2878 } 2879 off += MMU_PAGESIZE; 2880 PP_CLRFREE(pp); 2881 PP_CLRAGED(pp); 2882 page_set_props(pp, P_REF); 2883 page_io_lock(pp); 2884 pp = pp->p_next; 2885 } while (pp != mcpl); 2886 } else { 2887 /* 2888 * Hypervisor exchange doesn't handle segment or 2889 * alignment constraints 2890 */ 2891 if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi || 2892 pfnalign) 2893 goto fail; 2894 /* 2895 * Try exchanging pages with the hypervisor 2896 */ 2897 mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr, 2898 flags, minctg); 2899 if (mcpl == NULL) 2900 goto fail; 2901 off += minctg * MMU_PAGESIZE; 2902 } 2903 check_dma(mattr, mcpl, minctg); 2904 /* 2905 * Here with a minctg run of contiguous pages, add them to the 2906 * list we will return for this request. 2907 */ 2908 page_list_concat(&plist, &mcpl); 2909 npages -= minctg; 2910 *npagesp = npages; 2911 sgllen--; 2912 if (getone) 2913 break; 2914 } 2915 return (plist); 2916 fail: 2917 return_partial_alloc(plist); 2918 return (NULL); 2919 } 2920 2921 /* 2922 * Allocator for domain 0 I/O pages. We match the required 2923 * DMA attributes and contiguity constraints. 2924 */ 2925 /*ARGSUSED*/ 2926 page_t * 2927 page_create_io( 2928 struct vnode *vp, 2929 u_offset_t off, 2930 uint_t bytes, 2931 uint_t flags, 2932 struct as *as, 2933 caddr_t vaddr, 2934 ddi_dma_attr_t *mattr) 2935 { 2936 page_t *plist = NULL, *pp; 2937 int npages = 0, contig, anyaddr, pages_req; 2938 mfn_t lo_mfn; 2939 mfn_t hi_mfn; 2940 pgcnt_t pfnalign = 0; 2941 int align; 2942 int is_domu = 0; 2943 int dummy, bytes_got; 2944 mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL); 2945 2946 ASSERT(mattr != NULL); 2947 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo); 2948 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi); 2949 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer); 2950 if (align > MMU_PAGESIZE) 2951 pfnalign = mmu_btop(align); 2952 2953 /* 2954 * Clear the contig flag if only one page is needed or the scatter 2955 * gather list length is >= npages. 2956 */ 2957 pages_req = npages = mmu_btopr(bytes); 2958 contig = (flags & PG_PHYSCONTIG); 2959 bytes = P2ROUNDUP(bytes, MMU_PAGESIZE); 2960 if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages) 2961 contig = 0; 2962 2963 /* 2964 * Check if any old page in the system is fine. 2965 * DomU should always go down this path. 2966 */ 2967 is_domu = !DOMAIN_IS_INITDOMAIN(xen_info); 2968 anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign; 2969 if ((!contig && anyaddr) || is_domu) { 2970 flags &= ~PG_PHYSCONTIG; 2971 plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr); 2972 if (plist != NULL) 2973 return (plist); 2974 else if (is_domu) 2975 return (NULL); /* no memory available */ 2976 } 2977 /* 2978 * DomU should never reach here 2979 */ 2980 if (contig) { 2981 plist = page_get_contigpages(vp, off, &npages, flags, vaddr, 2982 mattr); 2983 if (plist == NULL) 2984 goto fail; 2985 bytes_got = (pages_req - npages) << MMU_PAGESHIFT; 2986 vaddr += bytes_got; 2987 off += bytes_got; 2988 /* 2989 * We now have all the contiguous pages we need, but 2990 * we may still need additional non-contiguous pages. 2991 */ 2992 } 2993 /* 2994 * now loop collecting the requested number of pages, these do 2995 * not have to be contiguous pages but we will use the contig 2996 * page alloc code to get the pages since it will honor any 2997 * other constraints the pages may have. 2998 */ 2999 while (npages--) { 3000 dummy = -1; 3001 pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr); 3002 if (pp == NULL) 3003 goto fail; 3004 page_add(&plist, pp); 3005 vaddr += MMU_PAGESIZE; 3006 off += MMU_PAGESIZE; 3007 } 3008 return (plist); 3009 fail: 3010 /* 3011 * Failed to get enough pages, return ones we did get 3012 */ 3013 return_partial_alloc(plist); 3014 return (NULL); 3015 } 3016 3017 /* 3018 * Lock and return the page with the highest mfn that we can find. last_mfn 3019 * holds the last one found, so the next search can start from there. We 3020 * also keep a counter so that we don't loop forever if the machine has no 3021 * free pages. 3022 * 3023 * This is called from the balloon thread to find pages to give away. new_high 3024 * is used when new mfn's have been added to the system - we will reset our 3025 * search if the new mfn's are higher than our current search position. 3026 */ 3027 page_t * 3028 page_get_high_mfn(mfn_t new_high) 3029 { 3030 static mfn_t last_mfn = 0; 3031 pfn_t pfn; 3032 page_t *pp; 3033 ulong_t loop_count = 0; 3034 3035 if (new_high > last_mfn) 3036 last_mfn = new_high; 3037 3038 for (; loop_count < mfn_count; loop_count++, last_mfn--) { 3039 if (last_mfn == 0) { 3040 last_mfn = cached_max_mfn; 3041 } 3042 3043 pfn = mfn_to_pfn(last_mfn); 3044 if (pfn & PFN_IS_FOREIGN_MFN) 3045 continue; 3046 3047 /* See if the page is free. If so, lock it. */ 3048 pp = page_numtopp_alloc(pfn); 3049 if (pp == NULL) 3050 continue; 3051 PP_CLRFREE(pp); 3052 3053 ASSERT(PAGE_EXCL(pp)); 3054 ASSERT(pp->p_vnode == NULL); 3055 ASSERT(!hat_page_is_mapped(pp)); 3056 last_mfn--; 3057 return (pp); 3058 } 3059 return (NULL); 3060 } 3061 3062 #else /* !__xpv */ 3063 3064 /* 3065 * get a page from any list with the given mnode 3066 */ 3067 static page_t * 3068 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags, 3069 int mnode, int mtype, ddi_dma_attr_t *dma_attr) 3070 { 3071 kmutex_t *pcm; 3072 int i; 3073 page_t *pp; 3074 page_t *first_pp; 3075 uint64_t pgaddr; 3076 ulong_t bin; 3077 int mtypestart; 3078 int plw_initialized; 3079 page_list_walker_t plw; 3080 3081 VM_STAT_ADD(pga_vmstats.pgma_alloc); 3082 3083 ASSERT((flags & PG_MATCH_COLOR) == 0); 3084 ASSERT(szc == 0); 3085 ASSERT(dma_attr != NULL); 3086 3087 MTYPE_START(mnode, mtype, flags); 3088 if (mtype < 0) { 3089 VM_STAT_ADD(pga_vmstats.pgma_allocempty); 3090 return (NULL); 3091 } 3092 3093 mtypestart = mtype; 3094 3095 bin = origbin; 3096 3097 /* 3098 * check up to page_colors + 1 bins - origbin may be checked twice 3099 * because of BIN_STEP skip 3100 */ 3101 do { 3102 plw_initialized = 0; 3103 3104 for (plw.plw_count = 0; 3105 plw.plw_count < page_colors; plw.plw_count++) { 3106 3107 if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL) 3108 goto nextfreebin; 3109 3110 pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 3111 mutex_enter(pcm); 3112 pp = PAGE_FREELISTS(mnode, szc, bin, mtype); 3113 first_pp = pp; 3114 while (pp != NULL) { 3115 if (page_trylock(pp, SE_EXCL) == 0) { 3116 pp = pp->p_next; 3117 if (pp == first_pp) { 3118 pp = NULL; 3119 } 3120 continue; 3121 } 3122 3123 ASSERT(PP_ISFREE(pp)); 3124 ASSERT(PP_ISAGED(pp)); 3125 ASSERT(pp->p_vnode == NULL); 3126 ASSERT(pp->p_hash == NULL); 3127 ASSERT(pp->p_offset == (u_offset_t)-1); 3128 ASSERT(pp->p_szc == szc); 3129 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 3130 /* check if page within DMA attributes */ 3131 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum)); 3132 if ((pgaddr >= dma_attr->dma_attr_addr_lo) && 3133 (pgaddr + MMU_PAGESIZE - 1 <= 3134 dma_attr->dma_attr_addr_hi)) { 3135 break; 3136 } 3137 3138 /* continue looking */ 3139 page_unlock(pp); 3140 pp = pp->p_next; 3141 if (pp == first_pp) 3142 pp = NULL; 3143 3144 } 3145 if (pp != NULL) { 3146 ASSERT(mtype == PP_2_MTYPE(pp)); 3147 ASSERT(pp->p_szc == 0); 3148 3149 /* found a page with specified DMA attributes */ 3150 page_sub(&PAGE_FREELISTS(mnode, szc, bin, 3151 mtype), pp); 3152 page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 3153 3154 if ((PP_ISFREE(pp) == 0) || 3155 (PP_ISAGED(pp) == 0)) { 3156 cmn_err(CE_PANIC, "page %p is not free", 3157 (void *)pp); 3158 } 3159 3160 mutex_exit(pcm); 3161 check_dma(dma_attr, pp, 1); 3162 VM_STAT_ADD(pga_vmstats.pgma_allocok); 3163 return (pp); 3164 } 3165 mutex_exit(pcm); 3166 nextfreebin: 3167 if (plw_initialized == 0) { 3168 page_list_walk_init(szc, 0, bin, 1, 0, &plw); 3169 ASSERT(plw.plw_ceq_dif == page_colors); 3170 plw_initialized = 1; 3171 } 3172 3173 if (plw.plw_do_split) { 3174 pp = page_freelist_split(szc, bin, mnode, 3175 mtype, 3176 mmu_btop(dma_attr->dma_attr_addr_lo), 3177 mmu_btop(dma_attr->dma_attr_addr_hi + 1), 3178 &plw); 3179 if (pp != NULL) { 3180 check_dma(dma_attr, pp, 1); 3181 return (pp); 3182 } 3183 } 3184 3185 bin = page_list_walk_next_bin(szc, bin, &plw); 3186 } 3187 3188 MTYPE_NEXT(mnode, mtype, flags); 3189 } while (mtype >= 0); 3190 3191 /* failed to find a page in the freelist; try it in the cachelist */ 3192 3193 /* reset mtype start for cachelist search */ 3194 mtype = mtypestart; 3195 ASSERT(mtype >= 0); 3196 3197 /* start with the bin of matching color */ 3198 bin = origbin; 3199 3200 do { 3201 for (i = 0; i <= page_colors; i++) { 3202 if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL) 3203 goto nextcachebin; 3204 pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST); 3205 mutex_enter(pcm); 3206 pp = PAGE_CACHELISTS(mnode, bin, mtype); 3207 first_pp = pp; 3208 while (pp != NULL) { 3209 if (page_trylock(pp, SE_EXCL) == 0) { 3210 pp = pp->p_next; 3211 if (pp == first_pp) 3212 pp = NULL; 3213 continue; 3214 } 3215 ASSERT(pp->p_vnode); 3216 ASSERT(PP_ISAGED(pp) == 0); 3217 ASSERT(pp->p_szc == 0); 3218 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 3219 3220 /* check if page within DMA attributes */ 3221 3222 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum)); 3223 if ((pgaddr >= dma_attr->dma_attr_addr_lo) && 3224 (pgaddr + MMU_PAGESIZE - 1 <= 3225 dma_attr->dma_attr_addr_hi)) { 3226 break; 3227 } 3228 3229 /* continue looking */ 3230 page_unlock(pp); 3231 pp = pp->p_next; 3232 if (pp == first_pp) 3233 pp = NULL; 3234 } 3235 3236 if (pp != NULL) { 3237 ASSERT(mtype == PP_2_MTYPE(pp)); 3238 ASSERT(pp->p_szc == 0); 3239 3240 /* found a page with specified DMA attributes */ 3241 page_sub(&PAGE_CACHELISTS(mnode, bin, 3242 mtype), pp); 3243 page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST); 3244 3245 mutex_exit(pcm); 3246 ASSERT(pp->p_vnode); 3247 ASSERT(PP_ISAGED(pp) == 0); 3248 check_dma(dma_attr, pp, 1); 3249 VM_STAT_ADD(pga_vmstats.pgma_allocok); 3250 return (pp); 3251 } 3252 mutex_exit(pcm); 3253 nextcachebin: 3254 bin += (i == 0) ? BIN_STEP : 1; 3255 bin &= page_colors_mask; 3256 } 3257 MTYPE_NEXT(mnode, mtype, flags); 3258 } while (mtype >= 0); 3259 3260 VM_STAT_ADD(pga_vmstats.pgma_allocfailed); 3261 return (NULL); 3262 } 3263 3264 /* 3265 * This function is similar to page_get_freelist()/page_get_cachelist() 3266 * but it searches both the lists to find a page with the specified 3267 * color (or no color) and DMA attributes. The search is done in the 3268 * freelist first and then in the cache list within the highest memory 3269 * range (based on DMA attributes) before searching in the lower 3270 * memory ranges. 3271 * 3272 * Note: This function is called only by page_create_io(). 3273 */ 3274 /*ARGSUSED*/ 3275 static page_t * 3276 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr, 3277 size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t *lgrp) 3278 { 3279 uint_t bin; 3280 int mtype; 3281 page_t *pp; 3282 int n; 3283 int m; 3284 int szc; 3285 int fullrange; 3286 int mnode; 3287 int local_failed_stat = 0; 3288 lgrp_mnode_cookie_t lgrp_cookie; 3289 3290 VM_STAT_ADD(pga_vmstats.pga_alloc); 3291 3292 /* only base pagesize currently supported */ 3293 if (size != MMU_PAGESIZE) 3294 return (NULL); 3295 3296 /* 3297 * If we're passed a specific lgroup, we use it. Otherwise, 3298 * assume first-touch placement is desired. 3299 */ 3300 if (!LGRP_EXISTS(lgrp)) 3301 lgrp = lgrp_home_lgrp(); 3302 3303 /* LINTED */ 3304 AS_2_BIN(as, seg, vp, vaddr, bin, 0); 3305 3306 /* 3307 * Only hold one freelist or cachelist lock at a time, that way we 3308 * can start anywhere and not have to worry about lock 3309 * ordering. 3310 */ 3311 if (dma_attr == NULL) { 3312 n = mtype16m; 3313 m = mtypetop; 3314 fullrange = 1; 3315 VM_STAT_ADD(pga_vmstats.pga_nulldmaattr); 3316 } else { 3317 pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo); 3318 pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi); 3319 3320 /* 3321 * We can guarantee alignment only for page boundary. 3322 */ 3323 if (dma_attr->dma_attr_align > MMU_PAGESIZE) 3324 return (NULL); 3325 3326 /* Sanity check the dma_attr */ 3327 if (pfnlo > pfnhi) 3328 return (NULL); 3329 3330 n = pfn_2_mtype(pfnlo); 3331 m = pfn_2_mtype(pfnhi); 3332 3333 fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) && 3334 (pfnhi >= mnoderanges[m].mnr_pfnhi)); 3335 } 3336 VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange); 3337 3338 szc = 0; 3339 3340 /* cylcing thru mtype handled by RANGE0 if n == mtype16m */ 3341 if (n == mtype16m) { 3342 flags |= PGI_MT_RANGE0; 3343 n = m; 3344 } 3345 3346 /* 3347 * Try local memory node first, but try remote if we can't 3348 * get a page of the right color. 3349 */ 3350 LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER); 3351 while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 3352 /* 3353 * allocate pages from high pfn to low. 3354 */ 3355 mtype = m; 3356 do { 3357 if (fullrange != 0) { 3358 pp = page_get_mnode_freelist(mnode, 3359 bin, mtype, szc, flags); 3360 if (pp == NULL) { 3361 pp = page_get_mnode_cachelist( 3362 bin, flags, mnode, mtype); 3363 } 3364 } else { 3365 pp = page_get_mnode_anylist(bin, szc, 3366 flags, mnode, mtype, dma_attr); 3367 } 3368 if (pp != NULL) { 3369 VM_STAT_ADD(pga_vmstats.pga_allocok); 3370 check_dma(dma_attr, pp, 1); 3371 return (pp); 3372 } 3373 } while (mtype != n && 3374 (mtype = mnoderanges[mtype].mnr_next) != -1); 3375 if (!local_failed_stat) { 3376 lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 3377 local_failed_stat = 1; 3378 } 3379 } 3380 VM_STAT_ADD(pga_vmstats.pga_allocfailed); 3381 3382 return (NULL); 3383 } 3384 3385 /* 3386 * page_create_io() 3387 * 3388 * This function is a copy of page_create_va() with an additional 3389 * argument 'mattr' that specifies DMA memory requirements to 3390 * the page list functions. This function is used by the segkmem 3391 * allocator so it is only to create new pages (i.e PG_EXCL is 3392 * set). 3393 * 3394 * Note: This interface is currently used by x86 PSM only and is 3395 * not fully specified so the commitment level is only for 3396 * private interface specific to x86. This interface uses PSM 3397 * specific page_get_anylist() interface. 3398 */ 3399 3400 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 3401 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 3402 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 3403 break; \ 3404 } \ 3405 } 3406 3407 3408 page_t * 3409 page_create_io( 3410 struct vnode *vp, 3411 u_offset_t off, 3412 uint_t bytes, 3413 uint_t flags, 3414 struct as *as, 3415 caddr_t vaddr, 3416 ddi_dma_attr_t *mattr) /* DMA memory attributes if any */ 3417 { 3418 page_t *plist = NULL; 3419 uint_t plist_len = 0; 3420 pgcnt_t npages; 3421 page_t *npp = NULL; 3422 uint_t pages_req; 3423 page_t *pp; 3424 kmutex_t *phm = NULL; 3425 uint_t index; 3426 3427 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 3428 "page_create_start:vp %p off %llx bytes %u flags %x", 3429 vp, off, bytes, flags); 3430 3431 ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0); 3432 3433 pages_req = npages = mmu_btopr(bytes); 3434 3435 /* 3436 * Do the freemem and pcf accounting. 3437 */ 3438 if (!page_create_wait(npages, flags)) { 3439 return (NULL); 3440 } 3441 3442 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 3443 "page_create_success:vp %p off %llx", vp, off); 3444 3445 /* 3446 * If satisfying this request has left us with too little 3447 * memory, start the wheels turning to get some back. The 3448 * first clause of the test prevents waking up the pageout 3449 * daemon in situations where it would decide that there's 3450 * nothing to do. 3451 */ 3452 if (nscan < desscan && freemem < minfree) { 3453 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 3454 "pageout_cv_signal:freemem %ld", freemem); 3455 cv_signal(&proc_pageout->p_cv); 3456 } 3457 3458 if (flags & PG_PHYSCONTIG) { 3459 3460 plist = page_get_contigpage(&npages, mattr, 1); 3461 if (plist == NULL) { 3462 page_create_putback(npages); 3463 return (NULL); 3464 } 3465 3466 pp = plist; 3467 3468 do { 3469 if (!page_hashin(pp, vp, off, NULL)) { 3470 panic("pg_creat_io: hashin failed %p %p %llx", 3471 (void *)pp, (void *)vp, off); 3472 } 3473 VM_STAT_ADD(page_create_new); 3474 off += MMU_PAGESIZE; 3475 PP_CLRFREE(pp); 3476 PP_CLRAGED(pp); 3477 page_set_props(pp, P_REF); 3478 pp = pp->p_next; 3479 } while (pp != plist); 3480 3481 if (!npages) { 3482 check_dma(mattr, plist, pages_req); 3483 return (plist); 3484 } else { 3485 vaddr += (pages_req - npages) << MMU_PAGESHIFT; 3486 } 3487 3488 /* 3489 * fall-thru: 3490 * 3491 * page_get_contigpage returns when npages <= sgllen. 3492 * Grab the rest of the non-contig pages below from anylist. 3493 */ 3494 } 3495 3496 /* 3497 * Loop around collecting the requested number of pages. 3498 * Most of the time, we have to `create' a new page. With 3499 * this in mind, pull the page off the free list before 3500 * getting the hash lock. This will minimize the hash 3501 * lock hold time, nesting, and the like. If it turns 3502 * out we don't need the page, we put it back at the end. 3503 */ 3504 while (npages--) { 3505 phm = NULL; 3506 3507 index = PAGE_HASH_FUNC(vp, off); 3508 top: 3509 ASSERT(phm == NULL); 3510 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 3511 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3512 3513 if (npp == NULL) { 3514 /* 3515 * Try to get the page of any color either from 3516 * the freelist or from the cache list. 3517 */ 3518 npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE, 3519 flags & ~PG_MATCH_COLOR, mattr, NULL); 3520 if (npp == NULL) { 3521 if (mattr == NULL) { 3522 /* 3523 * Not looking for a special page; 3524 * panic! 3525 */ 3526 panic("no page found %d", (int)npages); 3527 } 3528 /* 3529 * No page found! This can happen 3530 * if we are looking for a page 3531 * within a specific memory range 3532 * for DMA purposes. If PG_WAIT is 3533 * specified then we wait for a 3534 * while and then try again. The 3535 * wait could be forever if we 3536 * don't get the page(s) we need. 3537 * 3538 * Note: XXX We really need a mechanism 3539 * to wait for pages in the desired 3540 * range. For now, we wait for any 3541 * pages and see if we can use it. 3542 */ 3543 3544 if ((mattr != NULL) && (flags & PG_WAIT)) { 3545 delay(10); 3546 goto top; 3547 } 3548 goto fail; /* undo accounting stuff */ 3549 } 3550 3551 if (PP_ISAGED(npp) == 0) { 3552 /* 3553 * Since this page came from the 3554 * cachelist, we must destroy the 3555 * old vnode association. 3556 */ 3557 page_hashout(npp, (kmutex_t *)NULL); 3558 } 3559 } 3560 3561 /* 3562 * We own this page! 3563 */ 3564 ASSERT(PAGE_EXCL(npp)); 3565 ASSERT(npp->p_vnode == NULL); 3566 ASSERT(!hat_page_is_mapped(npp)); 3567 PP_CLRFREE(npp); 3568 PP_CLRAGED(npp); 3569 3570 /* 3571 * Here we have a page in our hot little mits and are 3572 * just waiting to stuff it on the appropriate lists. 3573 * Get the mutex and check to see if it really does 3574 * not exist. 3575 */ 3576 phm = PAGE_HASH_MUTEX(index); 3577 mutex_enter(phm); 3578 PAGE_HASH_SEARCH(index, pp, vp, off); 3579 if (pp == NULL) { 3580 VM_STAT_ADD(page_create_new); 3581 pp = npp; 3582 npp = NULL; 3583 if (!page_hashin(pp, vp, off, phm)) { 3584 /* 3585 * Since we hold the page hash mutex and 3586 * just searched for this page, page_hashin 3587 * had better not fail. If it does, that 3588 * means somethread did not follow the 3589 * page hash mutex rules. Panic now and 3590 * get it over with. As usual, go down 3591 * holding all the locks. 3592 */ 3593 ASSERT(MUTEX_HELD(phm)); 3594 panic("page_create: hashin fail %p %p %llx %p", 3595 (void *)pp, (void *)vp, off, (void *)phm); 3596 3597 } 3598 ASSERT(MUTEX_HELD(phm)); 3599 mutex_exit(phm); 3600 phm = NULL; 3601 3602 /* 3603 * Hat layer locking need not be done to set 3604 * the following bits since the page is not hashed 3605 * and was on the free list (i.e., had no mappings). 3606 * 3607 * Set the reference bit to protect 3608 * against immediate pageout 3609 * 3610 * XXXmh modify freelist code to set reference 3611 * bit so we don't have to do it here. 3612 */ 3613 page_set_props(pp, P_REF); 3614 } else { 3615 ASSERT(MUTEX_HELD(phm)); 3616 mutex_exit(phm); 3617 phm = NULL; 3618 /* 3619 * NOTE: This should not happen for pages associated 3620 * with kernel vnode 'kvp'. 3621 */ 3622 /* XX64 - to debug why this happens! */ 3623 ASSERT(!VN_ISKAS(vp)); 3624 if (VN_ISKAS(vp)) 3625 cmn_err(CE_NOTE, 3626 "page_create: page not expected " 3627 "in hash list for kernel vnode - pp 0x%p", 3628 (void *)pp); 3629 VM_STAT_ADD(page_create_exists); 3630 goto fail; 3631 } 3632 3633 /* 3634 * Got a page! It is locked. Acquire the i/o 3635 * lock since we are going to use the p_next and 3636 * p_prev fields to link the requested pages together. 3637 */ 3638 page_io_lock(pp); 3639 page_add(&plist, pp); 3640 plist = plist->p_next; 3641 off += MMU_PAGESIZE; 3642 vaddr += MMU_PAGESIZE; 3643 } 3644 3645 check_dma(mattr, plist, pages_req); 3646 return (plist); 3647 3648 fail: 3649 if (npp != NULL) { 3650 /* 3651 * Did not need this page after all. 3652 * Put it back on the free list. 3653 */ 3654 VM_STAT_ADD(page_create_putbacks); 3655 PP_SETFREE(npp); 3656 PP_SETAGED(npp); 3657 npp->p_offset = (u_offset_t)-1; 3658 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 3659 page_unlock(npp); 3660 } 3661 3662 /* 3663 * Give up the pages we already got. 3664 */ 3665 while (plist != NULL) { 3666 pp = plist; 3667 page_sub(&plist, pp); 3668 page_io_unlock(pp); 3669 plist_len++; 3670 /*LINTED: constant in conditional ctx*/ 3671 VN_DISPOSE(pp, B_INVAL, 0, kcred); 3672 } 3673 3674 /* 3675 * VN_DISPOSE does freemem accounting for the pages in plist 3676 * by calling page_free. So, we need to undo the pcf accounting 3677 * for only the remaining pages. 3678 */ 3679 VM_STAT_ADD(page_create_putbacks); 3680 page_create_putback(pages_req - plist_len); 3681 3682 return (NULL); 3683 } 3684 #endif /* !__xpv */ 3685 3686 3687 /* 3688 * Copy the data from the physical page represented by "frompp" to 3689 * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and 3690 * CPU->cpu_caddr2. It assumes that no one uses either map at interrupt 3691 * level and no one sleeps with an active mapping there. 3692 * 3693 * Note that the ref/mod bits in the page_t's are not affected by 3694 * this operation, hence it is up to the caller to update them appropriately. 3695 */ 3696 int 3697 ppcopy(page_t *frompp, page_t *topp) 3698 { 3699 caddr_t pp_addr1; 3700 caddr_t pp_addr2; 3701 hat_mempte_t pte1; 3702 hat_mempte_t pte2; 3703 kmutex_t *ppaddr_mutex; 3704 label_t ljb; 3705 int ret = 1; 3706 3707 ASSERT_STACK_ALIGNED(); 3708 ASSERT(PAGE_LOCKED(frompp)); 3709 ASSERT(PAGE_LOCKED(topp)); 3710 3711 if (kpm_enable) { 3712 pp_addr1 = hat_kpm_page2va(frompp, 0); 3713 pp_addr2 = hat_kpm_page2va(topp, 0); 3714 kpreempt_disable(); 3715 } else { 3716 /* 3717 * disable pre-emption so that CPU can't change 3718 */ 3719 kpreempt_disable(); 3720 3721 pp_addr1 = CPU->cpu_caddr1; 3722 pp_addr2 = CPU->cpu_caddr2; 3723 pte1 = CPU->cpu_caddr1pte; 3724 pte2 = CPU->cpu_caddr2pte; 3725 3726 ppaddr_mutex = &CPU->cpu_ppaddr_mutex; 3727 mutex_enter(ppaddr_mutex); 3728 3729 hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1, 3730 PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST); 3731 hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2, 3732 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 3733 HAT_LOAD_NOCONSIST); 3734 } 3735 3736 if (on_fault(&ljb)) { 3737 ret = 0; 3738 goto faulted; 3739 } 3740 if (use_sse_pagecopy) 3741 #ifdef __xpv 3742 page_copy_no_xmm(pp_addr2, pp_addr1); 3743 #else 3744 hwblkpagecopy(pp_addr1, pp_addr2); 3745 #endif 3746 else 3747 bcopy(pp_addr1, pp_addr2, PAGESIZE); 3748 3749 no_fault(); 3750 faulted: 3751 if (!kpm_enable) { 3752 #ifdef __xpv 3753 /* 3754 * We can't leave unused mappings laying about under the 3755 * hypervisor, so blow them away. 3756 */ 3757 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0, 3758 UVMF_INVLPG | UVMF_LOCAL) < 0) 3759 panic("HYPERVISOR_update_va_mapping() failed"); 3760 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0, 3761 UVMF_INVLPG | UVMF_LOCAL) < 0) 3762 panic("HYPERVISOR_update_va_mapping() failed"); 3763 #endif 3764 mutex_exit(ppaddr_mutex); 3765 } 3766 kpreempt_enable(); 3767 return (ret); 3768 } 3769 3770 void 3771 pagezero(page_t *pp, uint_t off, uint_t len) 3772 { 3773 ASSERT(PAGE_LOCKED(pp)); 3774 pfnzero(page_pptonum(pp), off, len); 3775 } 3776 3777 /* 3778 * Zero the physical page from off to off + len given by pfn 3779 * without changing the reference and modified bits of page. 3780 * 3781 * We use this using CPU private page address #2, see ppcopy() for more info. 3782 * pfnzero() must not be called at interrupt level. 3783 */ 3784 void 3785 pfnzero(pfn_t pfn, uint_t off, uint_t len) 3786 { 3787 caddr_t pp_addr2; 3788 hat_mempte_t pte2; 3789 kmutex_t *ppaddr_mutex = NULL; 3790 3791 ASSERT_STACK_ALIGNED(); 3792 ASSERT(len <= MMU_PAGESIZE); 3793 ASSERT(off <= MMU_PAGESIZE); 3794 ASSERT(off + len <= MMU_PAGESIZE); 3795 3796 if (kpm_enable && !pfn_is_foreign(pfn)) { 3797 pp_addr2 = hat_kpm_pfn2va(pfn); 3798 kpreempt_disable(); 3799 } else { 3800 kpreempt_disable(); 3801 3802 pp_addr2 = CPU->cpu_caddr2; 3803 pte2 = CPU->cpu_caddr2pte; 3804 3805 ppaddr_mutex = &CPU->cpu_ppaddr_mutex; 3806 mutex_enter(ppaddr_mutex); 3807 3808 hat_mempte_remap(pfn, pp_addr2, pte2, 3809 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 3810 HAT_LOAD_NOCONSIST); 3811 } 3812 3813 if (use_sse_pagezero) { 3814 #ifdef __xpv 3815 uint_t rem; 3816 3817 /* 3818 * zero a byte at a time until properly aligned for 3819 * block_zero_no_xmm(). 3820 */ 3821 while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0) 3822 pp_addr2[off++] = 0; 3823 3824 /* 3825 * Now use faster block_zero_no_xmm() for any range 3826 * that is properly aligned and sized. 3827 */ 3828 rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN)); 3829 len -= rem; 3830 if (len != 0) { 3831 block_zero_no_xmm(pp_addr2 + off, len); 3832 off += len; 3833 } 3834 3835 /* 3836 * zero remainder with byte stores. 3837 */ 3838 while (rem-- > 0) 3839 pp_addr2[off++] = 0; 3840 #else 3841 hwblkclr(pp_addr2 + off, len); 3842 #endif 3843 } else { 3844 bzero(pp_addr2 + off, len); 3845 } 3846 3847 if (!kpm_enable || pfn_is_foreign(pfn)) { 3848 #ifdef __xpv 3849 /* 3850 * On the hypervisor this page might get used for a page 3851 * table before any intervening change to this mapping, 3852 * so blow it away. 3853 */ 3854 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0, 3855 UVMF_INVLPG) < 0) 3856 panic("HYPERVISOR_update_va_mapping() failed"); 3857 #endif 3858 mutex_exit(ppaddr_mutex); 3859 } 3860 3861 kpreempt_enable(); 3862 } 3863 3864 /* 3865 * Platform-dependent page scrub call. 3866 */ 3867 void 3868 pagescrub(page_t *pp, uint_t off, uint_t len) 3869 { 3870 /* 3871 * For now, we rely on the fact that pagezero() will 3872 * always clear UEs. 3873 */ 3874 pagezero(pp, off, len); 3875 } 3876 3877 /* 3878 * set up two private addresses for use on a given CPU for use in ppcopy() 3879 */ 3880 void 3881 setup_vaddr_for_ppcopy(struct cpu *cpup) 3882 { 3883 void *addr; 3884 hat_mempte_t pte_pa; 3885 3886 addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP); 3887 pte_pa = hat_mempte_setup(addr); 3888 cpup->cpu_caddr1 = addr; 3889 cpup->cpu_caddr1pte = pte_pa; 3890 3891 addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP); 3892 pte_pa = hat_mempte_setup(addr); 3893 cpup->cpu_caddr2 = addr; 3894 cpup->cpu_caddr2pte = pte_pa; 3895 3896 mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL); 3897 } 3898 3899 /* 3900 * Undo setup_vaddr_for_ppcopy 3901 */ 3902 void 3903 teardown_vaddr_for_ppcopy(struct cpu *cpup) 3904 { 3905 mutex_destroy(&cpup->cpu_ppaddr_mutex); 3906 3907 hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte); 3908 cpup->cpu_caddr2pte = 0; 3909 vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1)); 3910 cpup->cpu_caddr2 = 0; 3911 3912 hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte); 3913 cpup->cpu_caddr1pte = 0; 3914 vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1)); 3915 cpup->cpu_caddr1 = 0; 3916 } 3917 3918 /* 3919 * Function for flushing D-cache when performing module relocations 3920 * to an alternate mapping. Unnecessary on Intel / AMD platforms. 3921 */ 3922 void 3923 dcache_flushall() 3924 {} 3925 3926 size_t 3927 exec_get_spslew(void) 3928 { 3929 return (0); 3930 } 3931 3932 /* 3933 * Allocate a memory page. The argument 'seed' can be any pseudo-random 3934 * number to vary where the pages come from. This is quite a hacked up 3935 * method -- it works for now, but really needs to be fixed up a bit. 3936 * 3937 * We currently use page_create_va() on the kvp with fake offsets, 3938 * segments and virt address. This is pretty bogus, but was copied from the 3939 * old hat_i86.c code. A better approach would be to specify either mnode 3940 * random or mnode local and takes a page from whatever color has the MOST 3941 * available - this would have a minimal impact on page coloring. 3942 */ 3943 page_t * 3944 page_get_physical(uintptr_t seed) 3945 { 3946 page_t *pp; 3947 u_offset_t offset; 3948 static struct seg tmpseg; 3949 static uintptr_t ctr = 0; 3950 3951 /* 3952 * This code is gross, we really need a simpler page allocator. 3953 * 3954 * We need to assign an offset for the page to call page_create_va() 3955 * To avoid conflicts with other pages, we get creative with the offset. 3956 * For 32 bits, we need an offset > 4Gig 3957 * For 64 bits, need an offset somewhere in the VA hole. 3958 */ 3959 offset = seed; 3960 if (offset > kernelbase) 3961 offset -= kernelbase; 3962 offset <<= MMU_PAGESHIFT; 3963 #if defined(__amd64) 3964 offset += mmu.hole_start; /* something in VA hole */ 3965 #else 3966 offset += 1ULL << 40; /* something > 4 Gig */ 3967 #endif 3968 3969 if (page_resv(1, KM_NOSLEEP) == 0) 3970 return (NULL); 3971 3972 #ifdef DEBUG 3973 pp = page_exists(&kvp, offset); 3974 if (pp != NULL) 3975 panic("page already exists %p", (void *)pp); 3976 #endif 3977 3978 pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL, 3979 &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE)); /* changing VA usage */ 3980 if (pp != NULL) { 3981 page_io_unlock(pp); 3982 page_hashout(pp, NULL); 3983 page_downgrade(pp); 3984 } 3985 return (pp); 3986 } 3987