1 /****************************************************************************** 2 * Xen balloon driver - enables returning/claiming memory to/from Xen. 3 * 4 * Copyright (c) 2003, B Dragovic 5 * Copyright (c) 2003-2004, M Williamson, K Fraser 6 * Copyright (c) 2005 Dan M. Smith, IBM Corporation 7 * Copyright (c) 2010 Daniel Kiper 8 * 9 * Memory hotplug support was written by Daniel Kiper. Work on 10 * it was sponsored by Google under Google Summer of Code 2010 11 * program. Jeremy Fitzhardinge from Citrix was the mentor for 12 * this project. 13 * 14 * This program is free software; you can redistribute it and/or 15 * modify it under the terms of the GNU General Public License version 2 16 * as published by the Free Software Foundation; or, when distributed 17 * separately from the Linux kernel or incorporated into other 18 * software packages, subject to the following license: 19 * 20 * Permission is hereby granted, free of charge, to any person obtaining a copy 21 * of this source file (the "Software"), to deal in the Software without 22 * restriction, including without limitation the rights to use, copy, modify, 23 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 24 * and to permit persons to whom the Software is furnished to do so, subject to 25 * the following conditions: 26 * 27 * The above copyright notice and this permission notice shall be included in 28 * all copies or substantial portions of the Software. 29 * 30 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 31 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 32 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 33 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 34 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 35 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 36 * IN THE SOFTWARE. 37 */ 38 39 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 40 41 #include <linux/cpu.h> 42 #include <linux/kernel.h> 43 #include <linux/sched.h> 44 #include <linux/errno.h> 45 #include <linux/module.h> 46 #include <linux/mm.h> 47 #include <linux/bootmem.h> 48 #include <linux/pagemap.h> 49 #include <linux/highmem.h> 50 #include <linux/mutex.h> 51 #include <linux/list.h> 52 #include <linux/gfp.h> 53 #include <linux/notifier.h> 54 #include <linux/memory.h> 55 #include <linux/memory_hotplug.h> 56 #include <linux/percpu-defs.h> 57 58 #include <asm/page.h> 59 #include <asm/pgalloc.h> 60 #include <asm/pgtable.h> 61 #include <asm/tlb.h> 62 63 #include <asm/xen/hypervisor.h> 64 #include <asm/xen/hypercall.h> 65 66 #include <xen/xen.h> 67 #include <xen/interface/xen.h> 68 #include <xen/interface/memory.h> 69 #include <xen/balloon.h> 70 #include <xen/features.h> 71 #include <xen/page.h> 72 73 /* 74 * balloon_process() state: 75 * 76 * BP_DONE: done or nothing to do, 77 * BP_EAGAIN: error, go to sleep, 78 * BP_ECANCELED: error, balloon operation canceled. 79 */ 80 81 enum bp_state { 82 BP_DONE, 83 BP_EAGAIN, 84 BP_ECANCELED 85 }; 86 87 88 static DEFINE_MUTEX(balloon_mutex); 89 90 struct balloon_stats balloon_stats; 91 EXPORT_SYMBOL_GPL(balloon_stats); 92 93 /* We increase/decrease in batches which fit in a page */ 94 static xen_pfn_t frame_list[PAGE_SIZE / sizeof(unsigned long)]; 95 static DEFINE_PER_CPU(struct page *, balloon_scratch_page); 96 97 98 /* List of ballooned pages, threaded through the mem_map array. */ 99 static LIST_HEAD(ballooned_pages); 100 101 /* Main work function, always executed in process context. */ 102 static void balloon_process(struct work_struct *work); 103 static DECLARE_DELAYED_WORK(balloon_worker, balloon_process); 104 105 /* When ballooning out (allocating memory to return to Xen) we don't really 106 want the kernel to try too hard since that can trigger the oom killer. */ 107 #define GFP_BALLOON \ 108 (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC) 109 110 static void scrub_page(struct page *page) 111 { 112 #ifdef CONFIG_XEN_SCRUB_PAGES 113 clear_highpage(page); 114 #endif 115 } 116 117 /* balloon_append: add the given page to the balloon. */ 118 static void __balloon_append(struct page *page) 119 { 120 /* Lowmem is re-populated first, so highmem pages go at list tail. */ 121 if (PageHighMem(page)) { 122 list_add_tail(&page->lru, &ballooned_pages); 123 balloon_stats.balloon_high++; 124 } else { 125 list_add(&page->lru, &ballooned_pages); 126 balloon_stats.balloon_low++; 127 } 128 } 129 130 static void balloon_append(struct page *page) 131 { 132 __balloon_append(page); 133 adjust_managed_page_count(page, -1); 134 } 135 136 /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */ 137 static struct page *balloon_retrieve(bool prefer_highmem) 138 { 139 struct page *page; 140 141 if (list_empty(&ballooned_pages)) 142 return NULL; 143 144 if (prefer_highmem) 145 page = list_entry(ballooned_pages.prev, struct page, lru); 146 else 147 page = list_entry(ballooned_pages.next, struct page, lru); 148 list_del(&page->lru); 149 150 if (PageHighMem(page)) 151 balloon_stats.balloon_high--; 152 else 153 balloon_stats.balloon_low--; 154 155 adjust_managed_page_count(page, 1); 156 157 return page; 158 } 159 160 static struct page *balloon_next_page(struct page *page) 161 { 162 struct list_head *next = page->lru.next; 163 if (next == &ballooned_pages) 164 return NULL; 165 return list_entry(next, struct page, lru); 166 } 167 168 static enum bp_state update_schedule(enum bp_state state) 169 { 170 if (state == BP_ECANCELED) 171 return BP_ECANCELED; 172 173 if (state == BP_DONE) { 174 balloon_stats.schedule_delay = 1; 175 balloon_stats.retry_count = 1; 176 return BP_DONE; 177 } 178 179 ++balloon_stats.retry_count; 180 181 if (balloon_stats.max_retry_count != RETRY_UNLIMITED && 182 balloon_stats.retry_count > balloon_stats.max_retry_count) { 183 balloon_stats.schedule_delay = 1; 184 balloon_stats.retry_count = 1; 185 return BP_ECANCELED; 186 } 187 188 balloon_stats.schedule_delay <<= 1; 189 190 if (balloon_stats.schedule_delay > balloon_stats.max_schedule_delay) 191 balloon_stats.schedule_delay = balloon_stats.max_schedule_delay; 192 193 return BP_EAGAIN; 194 } 195 196 #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG 197 static long current_credit(void) 198 { 199 return balloon_stats.target_pages - balloon_stats.current_pages - 200 balloon_stats.hotplug_pages; 201 } 202 203 static bool balloon_is_inflated(void) 204 { 205 if (balloon_stats.balloon_low || balloon_stats.balloon_high || 206 balloon_stats.balloon_hotplug) 207 return true; 208 else 209 return false; 210 } 211 212 /* 213 * reserve_additional_memory() adds memory region of size >= credit above 214 * max_pfn. New region is section aligned and size is modified to be multiple 215 * of section size. Those features allow optimal use of address space and 216 * establish proper alignment when this function is called first time after 217 * boot (last section not fully populated at boot time contains unused memory 218 * pages with PG_reserved bit not set; online_pages_range() does not allow page 219 * onlining in whole range if first onlined page does not have PG_reserved 220 * bit set). Real size of added memory is established at page onlining stage. 221 */ 222 223 static enum bp_state reserve_additional_memory(long credit) 224 { 225 int nid, rc; 226 u64 hotplug_start_paddr; 227 unsigned long balloon_hotplug = credit; 228 229 hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn)); 230 balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION); 231 nid = memory_add_physaddr_to_nid(hotplug_start_paddr); 232 233 rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug << PAGE_SHIFT); 234 235 if (rc) { 236 pr_warn("Cannot add additional memory (%i)\n", rc); 237 return BP_ECANCELED; 238 } 239 240 balloon_hotplug -= credit; 241 242 balloon_stats.hotplug_pages += credit; 243 balloon_stats.balloon_hotplug = balloon_hotplug; 244 245 return BP_DONE; 246 } 247 248 static void xen_online_page(struct page *page) 249 { 250 __online_page_set_limits(page); 251 252 mutex_lock(&balloon_mutex); 253 254 __balloon_append(page); 255 256 if (balloon_stats.hotplug_pages) 257 --balloon_stats.hotplug_pages; 258 else 259 --balloon_stats.balloon_hotplug; 260 261 mutex_unlock(&balloon_mutex); 262 } 263 264 static int xen_memory_notifier(struct notifier_block *nb, unsigned long val, void *v) 265 { 266 if (val == MEM_ONLINE) 267 schedule_delayed_work(&balloon_worker, 0); 268 269 return NOTIFY_OK; 270 } 271 272 static struct notifier_block xen_memory_nb = { 273 .notifier_call = xen_memory_notifier, 274 .priority = 0 275 }; 276 #else 277 static long current_credit(void) 278 { 279 unsigned long target = balloon_stats.target_pages; 280 281 target = min(target, 282 balloon_stats.current_pages + 283 balloon_stats.balloon_low + 284 balloon_stats.balloon_high); 285 286 return target - balloon_stats.current_pages; 287 } 288 289 static bool balloon_is_inflated(void) 290 { 291 if (balloon_stats.balloon_low || balloon_stats.balloon_high) 292 return true; 293 else 294 return false; 295 } 296 297 static enum bp_state reserve_additional_memory(long credit) 298 { 299 balloon_stats.target_pages = balloon_stats.current_pages; 300 return BP_DONE; 301 } 302 #endif /* CONFIG_XEN_BALLOON_MEMORY_HOTPLUG */ 303 304 static enum bp_state increase_reservation(unsigned long nr_pages) 305 { 306 int rc; 307 unsigned long pfn, i; 308 struct page *page; 309 struct xen_memory_reservation reservation = { 310 .address_bits = 0, 311 .extent_order = 0, 312 .domid = DOMID_SELF 313 }; 314 315 #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG 316 if (!balloon_stats.balloon_low && !balloon_stats.balloon_high) { 317 nr_pages = min(nr_pages, balloon_stats.balloon_hotplug); 318 balloon_stats.hotplug_pages += nr_pages; 319 balloon_stats.balloon_hotplug -= nr_pages; 320 return BP_DONE; 321 } 322 #endif 323 324 if (nr_pages > ARRAY_SIZE(frame_list)) 325 nr_pages = ARRAY_SIZE(frame_list); 326 327 page = list_first_entry_or_null(&ballooned_pages, struct page, lru); 328 for (i = 0; i < nr_pages; i++) { 329 if (!page) { 330 nr_pages = i; 331 break; 332 } 333 frame_list[i] = page_to_pfn(page); 334 page = balloon_next_page(page); 335 } 336 337 set_xen_guest_handle(reservation.extent_start, frame_list); 338 reservation.nr_extents = nr_pages; 339 rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, &reservation); 340 if (rc <= 0) 341 return BP_EAGAIN; 342 343 for (i = 0; i < rc; i++) { 344 page = balloon_retrieve(false); 345 BUG_ON(page == NULL); 346 347 pfn = page_to_pfn(page); 348 349 #ifdef CONFIG_XEN_HAVE_PVMMU 350 if (!xen_feature(XENFEAT_auto_translated_physmap)) { 351 set_phys_to_machine(pfn, frame_list[i]); 352 353 /* Link back into the page tables if not highmem. */ 354 if (!PageHighMem(page)) { 355 int ret; 356 ret = HYPERVISOR_update_va_mapping( 357 (unsigned long)__va(pfn << PAGE_SHIFT), 358 mfn_pte(frame_list[i], PAGE_KERNEL), 359 0); 360 BUG_ON(ret); 361 } 362 } 363 #endif 364 365 /* Relinquish the page back to the allocator. */ 366 __free_reserved_page(page); 367 } 368 369 balloon_stats.current_pages += rc; 370 371 return BP_DONE; 372 } 373 374 static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp) 375 { 376 enum bp_state state = BP_DONE; 377 unsigned long pfn, i; 378 struct page *page; 379 int ret; 380 struct xen_memory_reservation reservation = { 381 .address_bits = 0, 382 .extent_order = 0, 383 .domid = DOMID_SELF 384 }; 385 386 #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG 387 if (balloon_stats.hotplug_pages) { 388 nr_pages = min(nr_pages, balloon_stats.hotplug_pages); 389 balloon_stats.hotplug_pages -= nr_pages; 390 balloon_stats.balloon_hotplug += nr_pages; 391 return BP_DONE; 392 } 393 #endif 394 395 if (nr_pages > ARRAY_SIZE(frame_list)) 396 nr_pages = ARRAY_SIZE(frame_list); 397 398 for (i = 0; i < nr_pages; i++) { 399 page = alloc_page(gfp); 400 if (page == NULL) { 401 nr_pages = i; 402 state = BP_EAGAIN; 403 break; 404 } 405 scrub_page(page); 406 407 frame_list[i] = page_to_pfn(page); 408 } 409 410 /* 411 * Ensure that ballooned highmem pages don't have kmaps. 412 * 413 * Do this before changing the p2m as kmap_flush_unused() 414 * reads PTEs to obtain pages (and hence needs the original 415 * p2m entry). 416 */ 417 kmap_flush_unused(); 418 419 /* Update direct mapping, invalidate P2M, and add to balloon. */ 420 for (i = 0; i < nr_pages; i++) { 421 pfn = frame_list[i]; 422 frame_list[i] = pfn_to_mfn(pfn); 423 page = pfn_to_page(pfn); 424 425 #ifdef CONFIG_XEN_HAVE_PVMMU 426 /* 427 * Ballooned out frames are effectively replaced with 428 * a scratch frame. Ensure direct mappings and the 429 * p2m are consistent. 430 */ 431 if (!xen_feature(XENFEAT_auto_translated_physmap)) { 432 if (!PageHighMem(page)) { 433 struct page *scratch_page = get_balloon_scratch_page(); 434 435 ret = HYPERVISOR_update_va_mapping( 436 (unsigned long)__va(pfn << PAGE_SHIFT), 437 pfn_pte(page_to_pfn(scratch_page), 438 PAGE_KERNEL_RO), 0); 439 BUG_ON(ret); 440 441 put_balloon_scratch_page(); 442 } 443 __set_phys_to_machine(pfn, INVALID_P2M_ENTRY); 444 } 445 #endif 446 447 balloon_append(page); 448 } 449 450 flush_tlb_all(); 451 452 set_xen_guest_handle(reservation.extent_start, frame_list); 453 reservation.nr_extents = nr_pages; 454 ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation); 455 BUG_ON(ret != nr_pages); 456 457 balloon_stats.current_pages -= nr_pages; 458 459 return state; 460 } 461 462 /* 463 * We avoid multiple worker processes conflicting via the balloon mutex. 464 * We may of course race updates of the target counts (which are protected 465 * by the balloon lock), or with changes to the Xen hard limit, but we will 466 * recover from these in time. 467 */ 468 static void balloon_process(struct work_struct *work) 469 { 470 enum bp_state state = BP_DONE; 471 long credit; 472 473 mutex_lock(&balloon_mutex); 474 475 do { 476 credit = current_credit(); 477 478 if (credit > 0) { 479 if (balloon_is_inflated()) 480 state = increase_reservation(credit); 481 else 482 state = reserve_additional_memory(credit); 483 } 484 485 if (credit < 0) 486 state = decrease_reservation(-credit, GFP_BALLOON); 487 488 state = update_schedule(state); 489 490 #ifndef CONFIG_PREEMPT 491 if (need_resched()) 492 schedule(); 493 #endif 494 } while (credit && state == BP_DONE); 495 496 /* Schedule more work if there is some still to be done. */ 497 if (state == BP_EAGAIN) 498 schedule_delayed_work(&balloon_worker, balloon_stats.schedule_delay * HZ); 499 500 mutex_unlock(&balloon_mutex); 501 } 502 503 struct page *get_balloon_scratch_page(void) 504 { 505 struct page *ret = get_cpu_var(balloon_scratch_page); 506 BUG_ON(ret == NULL); 507 return ret; 508 } 509 510 void put_balloon_scratch_page(void) 511 { 512 put_cpu_var(balloon_scratch_page); 513 } 514 515 /* Resets the Xen limit, sets new target, and kicks off processing. */ 516 void balloon_set_new_target(unsigned long target) 517 { 518 /* No need for lock. Not read-modify-write updates. */ 519 balloon_stats.target_pages = target; 520 schedule_delayed_work(&balloon_worker, 0); 521 } 522 EXPORT_SYMBOL_GPL(balloon_set_new_target); 523 524 /** 525 * alloc_xenballooned_pages - get pages that have been ballooned out 526 * @nr_pages: Number of pages to get 527 * @pages: pages returned 528 * @highmem: allow highmem pages 529 * @return 0 on success, error otherwise 530 */ 531 int alloc_xenballooned_pages(int nr_pages, struct page **pages, bool highmem) 532 { 533 int pgno = 0; 534 struct page *page; 535 mutex_lock(&balloon_mutex); 536 while (pgno < nr_pages) { 537 page = balloon_retrieve(highmem); 538 if (page && (highmem || !PageHighMem(page))) { 539 pages[pgno++] = page; 540 } else { 541 enum bp_state st; 542 if (page) 543 balloon_append(page); 544 st = decrease_reservation(nr_pages - pgno, 545 highmem ? GFP_HIGHUSER : GFP_USER); 546 if (st != BP_DONE) 547 goto out_undo; 548 } 549 } 550 mutex_unlock(&balloon_mutex); 551 return 0; 552 out_undo: 553 while (pgno) 554 balloon_append(pages[--pgno]); 555 /* Free the memory back to the kernel soon */ 556 schedule_delayed_work(&balloon_worker, 0); 557 mutex_unlock(&balloon_mutex); 558 return -ENOMEM; 559 } 560 EXPORT_SYMBOL(alloc_xenballooned_pages); 561 562 /** 563 * free_xenballooned_pages - return pages retrieved with get_ballooned_pages 564 * @nr_pages: Number of pages 565 * @pages: pages to return 566 */ 567 void free_xenballooned_pages(int nr_pages, struct page **pages) 568 { 569 int i; 570 571 mutex_lock(&balloon_mutex); 572 573 for (i = 0; i < nr_pages; i++) { 574 if (pages[i]) 575 balloon_append(pages[i]); 576 } 577 578 /* The balloon may be too large now. Shrink it if needed. */ 579 if (current_credit()) 580 schedule_delayed_work(&balloon_worker, 0); 581 582 mutex_unlock(&balloon_mutex); 583 } 584 EXPORT_SYMBOL(free_xenballooned_pages); 585 586 static void __init balloon_add_region(unsigned long start_pfn, 587 unsigned long pages) 588 { 589 unsigned long pfn, extra_pfn_end; 590 struct page *page; 591 592 /* 593 * If the amount of usable memory has been limited (e.g., with 594 * the 'mem' command line parameter), don't add pages beyond 595 * this limit. 596 */ 597 extra_pfn_end = min(max_pfn, start_pfn + pages); 598 599 for (pfn = start_pfn; pfn < extra_pfn_end; pfn++) { 600 page = pfn_to_page(pfn); 601 /* totalram_pages and totalhigh_pages do not 602 include the boot-time balloon extension, so 603 don't subtract from it. */ 604 __balloon_append(page); 605 } 606 } 607 608 static int alloc_balloon_scratch_page(int cpu) 609 { 610 if (per_cpu(balloon_scratch_page, cpu) != NULL) 611 return 0; 612 613 per_cpu(balloon_scratch_page, cpu) = alloc_page(GFP_KERNEL); 614 if (per_cpu(balloon_scratch_page, cpu) == NULL) { 615 pr_warn("Failed to allocate balloon_scratch_page for cpu %d\n", cpu); 616 return -ENOMEM; 617 } 618 619 return 0; 620 } 621 622 623 static int balloon_cpu_notify(struct notifier_block *self, 624 unsigned long action, void *hcpu) 625 { 626 int cpu = (long)hcpu; 627 switch (action) { 628 case CPU_UP_PREPARE: 629 if (alloc_balloon_scratch_page(cpu)) 630 return NOTIFY_BAD; 631 break; 632 default: 633 break; 634 } 635 return NOTIFY_OK; 636 } 637 638 static struct notifier_block balloon_cpu_notifier = { 639 .notifier_call = balloon_cpu_notify, 640 }; 641 642 static int __init balloon_init(void) 643 { 644 int i, cpu; 645 646 if (!xen_domain()) 647 return -ENODEV; 648 649 if (!xen_feature(XENFEAT_auto_translated_physmap)) { 650 register_cpu_notifier(&balloon_cpu_notifier); 651 652 get_online_cpus(); 653 for_each_online_cpu(cpu) { 654 if (alloc_balloon_scratch_page(cpu)) { 655 put_online_cpus(); 656 unregister_cpu_notifier(&balloon_cpu_notifier); 657 return -ENOMEM; 658 } 659 } 660 put_online_cpus(); 661 } 662 663 pr_info("Initialising balloon driver\n"); 664 665 balloon_stats.current_pages = xen_pv_domain() 666 ? min(xen_start_info->nr_pages - xen_released_pages, max_pfn) 667 : get_num_physpages(); 668 balloon_stats.target_pages = balloon_stats.current_pages; 669 balloon_stats.balloon_low = 0; 670 balloon_stats.balloon_high = 0; 671 672 balloon_stats.schedule_delay = 1; 673 balloon_stats.max_schedule_delay = 32; 674 balloon_stats.retry_count = 1; 675 balloon_stats.max_retry_count = RETRY_UNLIMITED; 676 677 #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG 678 balloon_stats.hotplug_pages = 0; 679 balloon_stats.balloon_hotplug = 0; 680 681 set_online_page_callback(&xen_online_page); 682 register_memory_notifier(&xen_memory_nb); 683 #endif 684 685 /* 686 * Initialize the balloon with pages from the extra memory 687 * regions (see arch/x86/xen/setup.c). 688 */ 689 for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) 690 if (xen_extra_mem[i].size) 691 balloon_add_region(PFN_UP(xen_extra_mem[i].start), 692 PFN_DOWN(xen_extra_mem[i].size)); 693 694 return 0; 695 } 696 697 subsys_initcall(balloon_init); 698 699 static int __init balloon_clear(void) 700 { 701 int cpu; 702 703 for_each_possible_cpu(cpu) 704 per_cpu(balloon_scratch_page, cpu) = NULL; 705 706 return 0; 707 } 708 early_initcall(balloon_clear); 709 710 MODULE_LICENSE("GPL"); 711