1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Collaborative memory management interface. 4 * 5 * Copyright (C) 2008 IBM Corporation 6 * Author(s): Brian King (brking@linux.vnet.ibm.com), 7 */ 8 9 #include <linux/ctype.h> 10 #include <linux/delay.h> 11 #include <linux/errno.h> 12 #include <linux/fs.h> 13 #include <linux/gfp.h> 14 #include <linux/kthread.h> 15 #include <linux/module.h> 16 #include <linux/oom.h> 17 #include <linux/reboot.h> 18 #include <linux/sched.h> 19 #include <linux/stringify.h> 20 #include <linux/swap.h> 21 #include <linux/device.h> 22 #include <linux/mount.h> 23 #include <linux/pseudo_fs.h> 24 #include <linux/magic.h> 25 #include <linux/balloon_compaction.h> 26 #include <asm/firmware.h> 27 #include <asm/hvcall.h> 28 #include <asm/mmu.h> 29 #include <asm/pgalloc.h> 30 #include <linux/uaccess.h> 31 #include <linux/memory.h> 32 #include <asm/plpar_wrappers.h> 33 34 #include "pseries.h" 35 36 #define CMM_DRIVER_VERSION "1.0.0" 37 #define CMM_DEFAULT_DELAY 1 38 #define CMM_HOTPLUG_DELAY 5 39 #define CMM_DEBUG 0 40 #define CMM_DISABLE 0 41 #define CMM_OOM_KB 1024 42 #define CMM_MIN_MEM_MB 256 43 #define KB2PAGES(_p) ((_p)>>(PAGE_SHIFT-10)) 44 #define PAGES2KB(_p) ((_p)<<(PAGE_SHIFT-10)) 45 46 #define CMM_MEM_HOTPLUG_PRI 1 47 48 static unsigned int delay = CMM_DEFAULT_DELAY; 49 static unsigned int hotplug_delay = CMM_HOTPLUG_DELAY; 50 static unsigned int oom_kb = CMM_OOM_KB; 51 static unsigned int cmm_debug = CMM_DEBUG; 52 static unsigned int cmm_disabled = CMM_DISABLE; 53 static unsigned long min_mem_mb = CMM_MIN_MEM_MB; 54 static bool __read_mostly simulate; 55 static unsigned long simulate_loan_target_kb; 56 static struct device cmm_dev; 57 58 MODULE_AUTHOR("Brian King <brking@linux.vnet.ibm.com>"); 59 MODULE_DESCRIPTION("IBM System p Collaborative Memory Manager"); 60 MODULE_LICENSE("GPL"); 61 MODULE_VERSION(CMM_DRIVER_VERSION); 62 63 module_param_named(delay, delay, uint, 0644); 64 MODULE_PARM_DESC(delay, "Delay (in seconds) between polls to query hypervisor paging requests. " 65 "[Default=" __stringify(CMM_DEFAULT_DELAY) "]"); 66 module_param_named(hotplug_delay, hotplug_delay, uint, 0644); 67 MODULE_PARM_DESC(hotplug_delay, "Delay (in seconds) after memory hotplug remove " 68 "before loaning resumes. " 69 "[Default=" __stringify(CMM_HOTPLUG_DELAY) "]"); 70 module_param_named(oom_kb, oom_kb, uint, 0644); 71 MODULE_PARM_DESC(oom_kb, "Amount of memory in kb to free on OOM. " 72 "[Default=" __stringify(CMM_OOM_KB) "]"); 73 module_param_named(min_mem_mb, min_mem_mb, ulong, 0644); 74 MODULE_PARM_DESC(min_mem_mb, "Minimum amount of memory (in MB) to not balloon. " 75 "[Default=" __stringify(CMM_MIN_MEM_MB) "]"); 76 module_param_named(debug, cmm_debug, uint, 0644); 77 MODULE_PARM_DESC(debug, "Enable module debugging logging. Set to 1 to enable. " 78 "[Default=" __stringify(CMM_DEBUG) "]"); 79 module_param_named(simulate, simulate, bool, 0444); 80 MODULE_PARM_DESC(simulate, "Enable simulation mode (no communication with hw)."); 81 82 #define cmm_dbg(...) if (cmm_debug) { printk(KERN_INFO "cmm: "__VA_ARGS__); } 83 84 static atomic_long_t loaned_pages; 85 static unsigned long loaned_pages_target; 86 static unsigned long oom_freed_pages; 87 88 static DEFINE_MUTEX(hotplug_mutex); 89 static int hotplug_occurred; /* protected by the hotplug mutex */ 90 91 static struct task_struct *cmm_thread_ptr; 92 static struct balloon_dev_info b_dev_info; 93 94 static long plpar_page_set_loaned(struct page *page) 95 { 96 const unsigned long vpa = page_to_phys(page); 97 unsigned long cmo_page_sz = cmo_get_page_size(); 98 long rc = 0; 99 int i; 100 101 if (unlikely(simulate)) 102 return 0; 103 104 for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz) 105 rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED, vpa + i, 0); 106 107 for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz) 108 plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE, 109 vpa + i - cmo_page_sz, 0); 110 111 return rc; 112 } 113 114 static long plpar_page_set_active(struct page *page) 115 { 116 const unsigned long vpa = page_to_phys(page); 117 unsigned long cmo_page_sz = cmo_get_page_size(); 118 long rc = 0; 119 int i; 120 121 if (unlikely(simulate)) 122 return 0; 123 124 for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz) 125 rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE, vpa + i, 0); 126 127 for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz) 128 plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED, 129 vpa + i - cmo_page_sz, 0); 130 131 return rc; 132 } 133 134 /** 135 * cmm_alloc_pages - Allocate pages and mark them as loaned 136 * @nr: number of pages to allocate 137 * 138 * Return value: 139 * number of pages requested to be allocated which were not 140 **/ 141 static long cmm_alloc_pages(long nr) 142 { 143 struct page *page; 144 long rc; 145 146 cmm_dbg("Begin request for %ld pages\n", nr); 147 148 while (nr) { 149 /* Exit if a hotplug operation is in progress or occurred */ 150 if (mutex_trylock(&hotplug_mutex)) { 151 if (hotplug_occurred) { 152 mutex_unlock(&hotplug_mutex); 153 break; 154 } 155 mutex_unlock(&hotplug_mutex); 156 } else { 157 break; 158 } 159 160 page = balloon_page_alloc(); 161 if (!page) 162 break; 163 rc = plpar_page_set_loaned(page); 164 if (rc) { 165 pr_err("%s: Can not set page to loaned. rc=%ld\n", __func__, rc); 166 __free_page(page); 167 break; 168 } 169 170 balloon_page_enqueue(&b_dev_info, page); 171 atomic_long_inc(&loaned_pages); 172 adjust_managed_page_count(page, -1); 173 nr--; 174 } 175 176 cmm_dbg("End request with %ld pages unfulfilled\n", nr); 177 return nr; 178 } 179 180 /** 181 * cmm_free_pages - Free pages and mark them as active 182 * @nr: number of pages to free 183 * 184 * Return value: 185 * number of pages requested to be freed which were not 186 **/ 187 static long cmm_free_pages(long nr) 188 { 189 struct page *page; 190 191 cmm_dbg("Begin free of %ld pages.\n", nr); 192 while (nr) { 193 page = balloon_page_dequeue(&b_dev_info); 194 if (!page) 195 break; 196 plpar_page_set_active(page); 197 adjust_managed_page_count(page, 1); 198 __free_page(page); 199 atomic_long_dec(&loaned_pages); 200 nr--; 201 } 202 cmm_dbg("End request with %ld pages unfulfilled\n", nr); 203 return nr; 204 } 205 206 /** 207 * cmm_oom_notify - OOM notifier 208 * @self: notifier block struct 209 * @dummy: not used 210 * @parm: returned - number of pages freed 211 * 212 * Return value: 213 * NOTIFY_OK 214 **/ 215 static int cmm_oom_notify(struct notifier_block *self, 216 unsigned long dummy, void *parm) 217 { 218 unsigned long *freed = parm; 219 long nr = KB2PAGES(oom_kb); 220 221 cmm_dbg("OOM processing started\n"); 222 nr = cmm_free_pages(nr); 223 loaned_pages_target = atomic_long_read(&loaned_pages); 224 *freed += KB2PAGES(oom_kb) - nr; 225 oom_freed_pages += KB2PAGES(oom_kb) - nr; 226 cmm_dbg("OOM processing complete\n"); 227 return NOTIFY_OK; 228 } 229 230 /** 231 * cmm_get_mpp - Read memory performance parameters 232 * 233 * Makes hcall to query the current page loan request from the hypervisor. 234 * 235 * Return value: 236 * nothing 237 **/ 238 static void cmm_get_mpp(void) 239 { 240 const long __loaned_pages = atomic_long_read(&loaned_pages); 241 const long total_pages = totalram_pages() + __loaned_pages; 242 int rc; 243 struct hvcall_mpp_data mpp_data; 244 signed long active_pages_target, page_loan_request, target; 245 signed long min_mem_pages = (min_mem_mb * 1024 * 1024) / PAGE_SIZE; 246 247 if (likely(!simulate)) { 248 rc = h_get_mpp(&mpp_data); 249 if (rc != H_SUCCESS) 250 return; 251 page_loan_request = div_s64((s64)mpp_data.loan_request, 252 PAGE_SIZE); 253 target = page_loan_request + __loaned_pages; 254 } else { 255 target = KB2PAGES(simulate_loan_target_kb); 256 page_loan_request = target - __loaned_pages; 257 } 258 259 if (target < 0 || total_pages < min_mem_pages) 260 target = 0; 261 262 if (target > oom_freed_pages) 263 target -= oom_freed_pages; 264 else 265 target = 0; 266 267 active_pages_target = total_pages - target; 268 269 if (min_mem_pages > active_pages_target) 270 target = total_pages - min_mem_pages; 271 272 if (target < 0) 273 target = 0; 274 275 loaned_pages_target = target; 276 277 cmm_dbg("delta = %ld, loaned = %lu, target = %lu, oom = %lu, totalram = %lu\n", 278 page_loan_request, __loaned_pages, loaned_pages_target, 279 oom_freed_pages, totalram_pages()); 280 } 281 282 static struct notifier_block cmm_oom_nb = { 283 .notifier_call = cmm_oom_notify 284 }; 285 286 /** 287 * cmm_thread - CMM task thread 288 * @dummy: not used 289 * 290 * Return value: 291 * 0 292 **/ 293 static int cmm_thread(void *dummy) 294 { 295 unsigned long timeleft; 296 long __loaned_pages; 297 298 while (1) { 299 timeleft = msleep_interruptible(delay * 1000); 300 301 if (kthread_should_stop() || timeleft) 302 break; 303 304 if (mutex_trylock(&hotplug_mutex)) { 305 if (hotplug_occurred) { 306 hotplug_occurred = 0; 307 mutex_unlock(&hotplug_mutex); 308 cmm_dbg("Hotplug operation has occurred, " 309 "loaning activity suspended " 310 "for %d seconds.\n", 311 hotplug_delay); 312 timeleft = msleep_interruptible(hotplug_delay * 313 1000); 314 if (kthread_should_stop() || timeleft) 315 break; 316 continue; 317 } 318 mutex_unlock(&hotplug_mutex); 319 } else { 320 cmm_dbg("Hotplug operation in progress, activity " 321 "suspended\n"); 322 continue; 323 } 324 325 cmm_get_mpp(); 326 327 __loaned_pages = atomic_long_read(&loaned_pages); 328 if (loaned_pages_target > __loaned_pages) { 329 if (cmm_alloc_pages(loaned_pages_target - __loaned_pages)) 330 loaned_pages_target = __loaned_pages; 331 } else if (loaned_pages_target < __loaned_pages) 332 cmm_free_pages(__loaned_pages - loaned_pages_target); 333 } 334 return 0; 335 } 336 337 #define CMM_SHOW(name, format, args...) \ 338 static ssize_t show_##name(struct device *dev, \ 339 struct device_attribute *attr, \ 340 char *buf) \ 341 { \ 342 return sprintf(buf, format, ##args); \ 343 } \ 344 static DEVICE_ATTR(name, 0444, show_##name, NULL) 345 346 CMM_SHOW(loaned_kb, "%lu\n", PAGES2KB(atomic_long_read(&loaned_pages))); 347 CMM_SHOW(loaned_target_kb, "%lu\n", PAGES2KB(loaned_pages_target)); 348 349 static ssize_t show_oom_pages(struct device *dev, 350 struct device_attribute *attr, char *buf) 351 { 352 return sprintf(buf, "%lu\n", PAGES2KB(oom_freed_pages)); 353 } 354 355 static ssize_t store_oom_pages(struct device *dev, 356 struct device_attribute *attr, 357 const char *buf, size_t count) 358 { 359 unsigned long val = simple_strtoul (buf, NULL, 10); 360 361 if (!capable(CAP_SYS_ADMIN)) 362 return -EPERM; 363 if (val != 0) 364 return -EBADMSG; 365 366 oom_freed_pages = 0; 367 return count; 368 } 369 370 static DEVICE_ATTR(oom_freed_kb, 0644, 371 show_oom_pages, store_oom_pages); 372 373 static struct device_attribute *cmm_attrs[] = { 374 &dev_attr_loaned_kb, 375 &dev_attr_loaned_target_kb, 376 &dev_attr_oom_freed_kb, 377 }; 378 379 static DEVICE_ULONG_ATTR(simulate_loan_target_kb, 0644, 380 simulate_loan_target_kb); 381 382 static struct bus_type cmm_subsys = { 383 .name = "cmm", 384 .dev_name = "cmm", 385 }; 386 387 static void cmm_release_device(struct device *dev) 388 { 389 } 390 391 /** 392 * cmm_sysfs_register - Register with sysfs 393 * 394 * Return value: 395 * 0 on success / other on failure 396 **/ 397 static int cmm_sysfs_register(struct device *dev) 398 { 399 int i, rc; 400 401 if ((rc = subsys_system_register(&cmm_subsys, NULL))) 402 return rc; 403 404 dev->id = 0; 405 dev->bus = &cmm_subsys; 406 dev->release = cmm_release_device; 407 408 if ((rc = device_register(dev))) 409 goto subsys_unregister; 410 411 for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++) { 412 if ((rc = device_create_file(dev, cmm_attrs[i]))) 413 goto fail; 414 } 415 416 if (!simulate) 417 return 0; 418 rc = device_create_file(dev, &dev_attr_simulate_loan_target_kb.attr); 419 if (rc) 420 goto fail; 421 return 0; 422 423 fail: 424 while (--i >= 0) 425 device_remove_file(dev, cmm_attrs[i]); 426 device_unregister(dev); 427 subsys_unregister: 428 bus_unregister(&cmm_subsys); 429 return rc; 430 } 431 432 /** 433 * cmm_unregister_sysfs - Unregister from sysfs 434 * 435 **/ 436 static void cmm_unregister_sysfs(struct device *dev) 437 { 438 int i; 439 440 for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++) 441 device_remove_file(dev, cmm_attrs[i]); 442 device_unregister(dev); 443 bus_unregister(&cmm_subsys); 444 } 445 446 /** 447 * cmm_reboot_notifier - Make sure pages are not still marked as "loaned" 448 * 449 **/ 450 static int cmm_reboot_notifier(struct notifier_block *nb, 451 unsigned long action, void *unused) 452 { 453 if (action == SYS_RESTART) { 454 if (cmm_thread_ptr) 455 kthread_stop(cmm_thread_ptr); 456 cmm_thread_ptr = NULL; 457 cmm_free_pages(atomic_long_read(&loaned_pages)); 458 } 459 return NOTIFY_DONE; 460 } 461 462 static struct notifier_block cmm_reboot_nb = { 463 .notifier_call = cmm_reboot_notifier, 464 }; 465 466 /** 467 * cmm_memory_cb - Handle memory hotplug notifier calls 468 * @self: notifier block struct 469 * @action: action to take 470 * @arg: struct memory_notify data for handler 471 * 472 * Return value: 473 * NOTIFY_OK or notifier error based on subfunction return value 474 * 475 **/ 476 static int cmm_memory_cb(struct notifier_block *self, 477 unsigned long action, void *arg) 478 { 479 int ret = 0; 480 481 switch (action) { 482 case MEM_GOING_OFFLINE: 483 mutex_lock(&hotplug_mutex); 484 hotplug_occurred = 1; 485 break; 486 case MEM_OFFLINE: 487 case MEM_CANCEL_OFFLINE: 488 mutex_unlock(&hotplug_mutex); 489 cmm_dbg("Memory offline operation complete.\n"); 490 break; 491 case MEM_GOING_ONLINE: 492 case MEM_ONLINE: 493 case MEM_CANCEL_ONLINE: 494 break; 495 } 496 497 return notifier_from_errno(ret); 498 } 499 500 static struct notifier_block cmm_mem_nb = { 501 .notifier_call = cmm_memory_cb, 502 .priority = CMM_MEM_HOTPLUG_PRI 503 }; 504 505 #ifdef CONFIG_BALLOON_COMPACTION 506 static struct vfsmount *balloon_mnt; 507 508 static int cmm_init_fs_context(struct fs_context *fc) 509 { 510 return init_pseudo(fc, PPC_CMM_MAGIC) ? 0 : -ENOMEM; 511 } 512 513 static struct file_system_type balloon_fs = { 514 .name = "ppc-cmm", 515 .init_fs_context = cmm_init_fs_context, 516 .kill_sb = kill_anon_super, 517 }; 518 519 static int cmm_migratepage(struct balloon_dev_info *b_dev_info, 520 struct page *newpage, struct page *page, 521 enum migrate_mode mode) 522 { 523 unsigned long flags; 524 525 /* 526 * loan/"inflate" the newpage first. 527 * 528 * We might race against the cmm_thread who might discover after our 529 * loan request that another page is to be unloaned. However, once 530 * the cmm_thread runs again later, this error will automatically 531 * be corrected. 532 */ 533 if (plpar_page_set_loaned(newpage)) { 534 /* Unlikely, but possible. Tell the caller not to retry now. */ 535 pr_err_ratelimited("%s: Cannot set page to loaned.", __func__); 536 return -EBUSY; 537 } 538 539 /* balloon page list reference */ 540 get_page(newpage); 541 542 spin_lock_irqsave(&b_dev_info->pages_lock, flags); 543 balloon_page_insert(b_dev_info, newpage); 544 balloon_page_delete(page); 545 b_dev_info->isolated_pages--; 546 spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); 547 548 /* 549 * activate/"deflate" the old page. We ignore any errors just like the 550 * other callers. 551 */ 552 plpar_page_set_active(page); 553 554 /* balloon page list reference */ 555 put_page(page); 556 557 return MIGRATEPAGE_SUCCESS; 558 } 559 560 static int cmm_balloon_compaction_init(void) 561 { 562 int rc; 563 564 balloon_devinfo_init(&b_dev_info); 565 b_dev_info.migratepage = cmm_migratepage; 566 567 balloon_mnt = kern_mount(&balloon_fs); 568 if (IS_ERR(balloon_mnt)) { 569 rc = PTR_ERR(balloon_mnt); 570 balloon_mnt = NULL; 571 return rc; 572 } 573 574 b_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb); 575 if (IS_ERR(b_dev_info.inode)) { 576 rc = PTR_ERR(b_dev_info.inode); 577 b_dev_info.inode = NULL; 578 kern_unmount(balloon_mnt); 579 balloon_mnt = NULL; 580 return rc; 581 } 582 583 b_dev_info.inode->i_mapping->a_ops = &balloon_aops; 584 return 0; 585 } 586 static void cmm_balloon_compaction_deinit(void) 587 { 588 if (b_dev_info.inode) 589 iput(b_dev_info.inode); 590 b_dev_info.inode = NULL; 591 kern_unmount(balloon_mnt); 592 balloon_mnt = NULL; 593 } 594 #else /* CONFIG_BALLOON_COMPACTION */ 595 static int cmm_balloon_compaction_init(void) 596 { 597 return 0; 598 } 599 600 static void cmm_balloon_compaction_deinit(void) 601 { 602 } 603 #endif /* CONFIG_BALLOON_COMPACTION */ 604 605 /** 606 * cmm_init - Module initialization 607 * 608 * Return value: 609 * 0 on success / other on failure 610 **/ 611 static int cmm_init(void) 612 { 613 int rc; 614 615 if (!firmware_has_feature(FW_FEATURE_CMO) && !simulate) 616 return -EOPNOTSUPP; 617 618 rc = cmm_balloon_compaction_init(); 619 if (rc) 620 return rc; 621 622 rc = register_oom_notifier(&cmm_oom_nb); 623 if (rc < 0) 624 goto out_balloon_compaction; 625 626 if ((rc = register_reboot_notifier(&cmm_reboot_nb))) 627 goto out_oom_notifier; 628 629 if ((rc = cmm_sysfs_register(&cmm_dev))) 630 goto out_reboot_notifier; 631 632 rc = register_memory_notifier(&cmm_mem_nb); 633 if (rc) 634 goto out_unregister_notifier; 635 636 if (cmm_disabled) 637 return 0; 638 639 cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread"); 640 if (IS_ERR(cmm_thread_ptr)) { 641 rc = PTR_ERR(cmm_thread_ptr); 642 goto out_unregister_notifier; 643 } 644 645 return 0; 646 out_unregister_notifier: 647 unregister_memory_notifier(&cmm_mem_nb); 648 cmm_unregister_sysfs(&cmm_dev); 649 out_reboot_notifier: 650 unregister_reboot_notifier(&cmm_reboot_nb); 651 out_oom_notifier: 652 unregister_oom_notifier(&cmm_oom_nb); 653 out_balloon_compaction: 654 cmm_balloon_compaction_deinit(); 655 return rc; 656 } 657 658 /** 659 * cmm_exit - Module exit 660 * 661 * Return value: 662 * nothing 663 **/ 664 static void cmm_exit(void) 665 { 666 if (cmm_thread_ptr) 667 kthread_stop(cmm_thread_ptr); 668 unregister_oom_notifier(&cmm_oom_nb); 669 unregister_reboot_notifier(&cmm_reboot_nb); 670 unregister_memory_notifier(&cmm_mem_nb); 671 cmm_free_pages(atomic_long_read(&loaned_pages)); 672 cmm_unregister_sysfs(&cmm_dev); 673 cmm_balloon_compaction_deinit(); 674 } 675 676 /** 677 * cmm_set_disable - Disable/Enable CMM 678 * 679 * Return value: 680 * 0 on success / other on failure 681 **/ 682 static int cmm_set_disable(const char *val, const struct kernel_param *kp) 683 { 684 int disable = simple_strtoul(val, NULL, 10); 685 686 if (disable != 0 && disable != 1) 687 return -EINVAL; 688 689 if (disable && !cmm_disabled) { 690 if (cmm_thread_ptr) 691 kthread_stop(cmm_thread_ptr); 692 cmm_thread_ptr = NULL; 693 cmm_free_pages(atomic_long_read(&loaned_pages)); 694 } else if (!disable && cmm_disabled) { 695 cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread"); 696 if (IS_ERR(cmm_thread_ptr)) 697 return PTR_ERR(cmm_thread_ptr); 698 } 699 700 cmm_disabled = disable; 701 return 0; 702 } 703 704 module_param_call(disable, cmm_set_disable, param_get_uint, 705 &cmm_disabled, 0644); 706 MODULE_PARM_DESC(disable, "Disable CMM. Set to 1 to disable. " 707 "[Default=" __stringify(CMM_DISABLE) "]"); 708 709 module_init(cmm_init); 710 module_exit(cmm_exit); 711