cmm.c (1ef2f06b71792c2efaa4fb4aef8f1fc2a115ee1f) cmm.c (fe030c9b85e6783bc52fe86449c0a4b8aa16c753)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Collaborative memory management interface.
4 *
5 * Copyright (C) 2008 IBM Corporation
6 * Author(s): Brian King (brking@linux.vnet.ibm.com),
7 */
8

--- 5 unchanged lines hidden (view full) ---

14#include <linux/kthread.h>
15#include <linux/module.h>
16#include <linux/oom.h>
17#include <linux/reboot.h>
18#include <linux/sched.h>
19#include <linux/stringify.h>
20#include <linux/swap.h>
21#include <linux/device.h>
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Collaborative memory management interface.
4 *
5 * Copyright (C) 2008 IBM Corporation
6 * Author(s): Brian King (brking@linux.vnet.ibm.com),
7 */
8

--- 5 unchanged lines hidden (view full) ---

14#include <linux/kthread.h>
15#include <linux/module.h>
16#include <linux/oom.h>
17#include <linux/reboot.h>
18#include <linux/sched.h>
19#include <linux/stringify.h>
20#include <linux/swap.h>
21#include <linux/device.h>
22#include <linux/mount.h>
23#include <linux/pseudo_fs.h>
24#include <linux/magic.h>
25#include <linux/balloon_compaction.h>
22#include <asm/firmware.h>
23#include <asm/hvcall.h>
24#include <asm/mmu.h>
25#include <asm/pgalloc.h>
26#include <linux/uaccess.h>
27#include <linux/memory.h>
28#include <asm/plpar_wrappers.h>
29

--- 42 unchanged lines hidden (view full) ---

72 "[Default=" __stringify(CMM_DEBUG) "]");
73
74#define cmm_dbg(...) if (cmm_debug) { printk(KERN_INFO "cmm: "__VA_ARGS__); }
75
76static atomic_long_t loaned_pages;
77static unsigned long loaned_pages_target;
78static unsigned long oom_freed_pages;
79
26#include <asm/firmware.h>
27#include <asm/hvcall.h>
28#include <asm/mmu.h>
29#include <asm/pgalloc.h>
30#include <linux/uaccess.h>
31#include <linux/memory.h>
32#include <asm/plpar_wrappers.h>
33

--- 42 unchanged lines hidden (view full) ---

76 "[Default=" __stringify(CMM_DEBUG) "]");
77
78#define cmm_dbg(...) if (cmm_debug) { printk(KERN_INFO "cmm: "__VA_ARGS__); }
79
80static atomic_long_t loaned_pages;
81static unsigned long loaned_pages_target;
82static unsigned long oom_freed_pages;
83
80static LIST_HEAD(cmm_page_list);
81static DEFINE_SPINLOCK(cmm_lock);
82
83static DEFINE_MUTEX(hotplug_mutex);
84static int hotplug_occurred; /* protected by the hotplug mutex */
85
86static struct task_struct *cmm_thread_ptr;
84static DEFINE_MUTEX(hotplug_mutex);
85static int hotplug_occurred; /* protected by the hotplug mutex */
86
87static struct task_struct *cmm_thread_ptr;
88static struct balloon_dev_info b_dev_info;
87
88static long plpar_page_set_loaned(struct page *page)
89{
90 const unsigned long vpa = page_to_phys(page);
91 unsigned long cmo_page_sz = cmo_get_page_size();
92 long rc = 0;
93 int i;
94

--- 49 unchanged lines hidden (view full) ---

144 } else {
145 break;
146 }
147
148 page = alloc_page(GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY |
149 __GFP_NOMEMALLOC);
150 if (!page)
151 break;
89
90static long plpar_page_set_loaned(struct page *page)
91{
92 const unsigned long vpa = page_to_phys(page);
93 unsigned long cmo_page_sz = cmo_get_page_size();
94 long rc = 0;
95 int i;
96

--- 49 unchanged lines hidden (view full) ---

146 } else {
147 break;
148 }
149
150 page = alloc_page(GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY |
151 __GFP_NOMEMALLOC);
152 if (!page)
153 break;
152 spin_lock(&cmm_lock);
153 rc = plpar_page_set_loaned(page);
154 if (rc) {
155 pr_err("%s: Can not set page to loaned. rc=%ld\n", __func__, rc);
154 rc = plpar_page_set_loaned(page);
155 if (rc) {
156 pr_err("%s: Can not set page to loaned. rc=%ld\n", __func__, rc);
156 spin_unlock(&cmm_lock);
157 __free_page(page);
158 break;
159 }
160
157 __free_page(page);
158 break;
159 }
160
161 list_add(&page->lru, &cmm_page_list);
161 balloon_page_enqueue(&b_dev_info, page);
162 atomic_long_inc(&loaned_pages);
163 adjust_managed_page_count(page, -1);
162 atomic_long_inc(&loaned_pages);
163 adjust_managed_page_count(page, -1);
164 spin_unlock(&cmm_lock);
165 nr--;
166 }
167
168 cmm_dbg("End request with %ld pages unfulfilled\n", nr);
169 return nr;
170}
171
172/**
173 * cmm_free_pages - Free pages and mark them as active
174 * @nr: number of pages to free
175 *
176 * Return value:
177 * number of pages requested to be freed which were not
178 **/
179static long cmm_free_pages(long nr)
180{
164 nr--;
165 }
166
167 cmm_dbg("End request with %ld pages unfulfilled\n", nr);
168 return nr;
169}
170
171/**
172 * cmm_free_pages - Free pages and mark them as active
173 * @nr: number of pages to free
174 *
175 * Return value:
176 * number of pages requested to be freed which were not
177 **/
178static long cmm_free_pages(long nr)
179{
181 struct page *page, *tmp;
180 struct page *page;
182
183 cmm_dbg("Begin free of %ld pages.\n", nr);
181
182 cmm_dbg("Begin free of %ld pages.\n", nr);
184 spin_lock(&cmm_lock);
185 list_for_each_entry_safe(page, tmp, &cmm_page_list, lru) {
186 if (!nr)
183 while (nr) {
184 page = balloon_page_dequeue(&b_dev_info);
185 if (!page)
187 break;
188 plpar_page_set_active(page);
186 break;
187 plpar_page_set_active(page);
189 list_del(&page->lru);
190 adjust_managed_page_count(page, 1);
191 __free_page(page);
192 atomic_long_dec(&loaned_pages);
193 nr--;
194 }
188 adjust_managed_page_count(page, 1);
189 __free_page(page);
190 atomic_long_dec(&loaned_pages);
191 nr--;
192 }
195 spin_unlock(&cmm_lock);
196 cmm_dbg("End request with %ld pages unfulfilled\n", nr);
197 return nr;
198}
199
200/**
201 * cmm_oom_notify - OOM notifier
202 * @self: notifier block struct
203 * @dummy: not used

--- 275 unchanged lines hidden (view full) ---

479 return notifier_from_errno(ret);
480}
481
482static struct notifier_block cmm_mem_nb = {
483 .notifier_call = cmm_memory_cb,
484 .priority = CMM_MEM_HOTPLUG_PRI
485};
486
193 cmm_dbg("End request with %ld pages unfulfilled\n", nr);
194 return nr;
195}
196
197/**
198 * cmm_oom_notify - OOM notifier
199 * @self: notifier block struct
200 * @dummy: not used

--- 275 unchanged lines hidden (view full) ---

476 return notifier_from_errno(ret);
477}
478
479static struct notifier_block cmm_mem_nb = {
480 .notifier_call = cmm_memory_cb,
481 .priority = CMM_MEM_HOTPLUG_PRI
482};
483
484#ifdef CONFIG_BALLOON_COMPACTION
485static struct vfsmount *balloon_mnt;
486
487static int cmm_init_fs_context(struct fs_context *fc)
488{
489 return init_pseudo(fc, PPC_CMM_MAGIC) ? 0 : -ENOMEM;
490}
491
492static struct file_system_type balloon_fs = {
493 .name = "ppc-cmm",
494 .init_fs_context = cmm_init_fs_context,
495 .kill_sb = kill_anon_super,
496};
497
498static int cmm_migratepage(struct balloon_dev_info *b_dev_info,
499 struct page *newpage, struct page *page,
500 enum migrate_mode mode)
501{
502 unsigned long flags;
503
504 /*
505 * loan/"inflate" the newpage first.
506 *
507 * We might race against the cmm_thread who might discover after our
508 * loan request that another page is to be unloaned. However, once
509 * the cmm_thread runs again later, this error will automatically
510 * be corrected.
511 */
512 if (plpar_page_set_loaned(newpage)) {
513 /* Unlikely, but possible. Tell the caller not to retry now. */
514 pr_err_ratelimited("%s: Cannot set page to loaned.", __func__);
515 return -EBUSY;
516 }
517
518 /* balloon page list reference */
519 get_page(newpage);
520
521 spin_lock_irqsave(&b_dev_info->pages_lock, flags);
522 balloon_page_insert(b_dev_info, newpage);
523 balloon_page_delete(page);
524 b_dev_info->isolated_pages--;
525 spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
526
527 /*
528 * activate/"deflate" the old page. We ignore any errors just like the
529 * other callers.
530 */
531 plpar_page_set_active(page);
532
533 /* balloon page list reference */
534 put_page(page);
535
536 return MIGRATEPAGE_SUCCESS;
537}
538
539static int cmm_balloon_compaction_init(void)
540{
541 int rc;
542
543 balloon_devinfo_init(&b_dev_info);
544 b_dev_info.migratepage = cmm_migratepage;
545
546 balloon_mnt = kern_mount(&balloon_fs);
547 if (IS_ERR(balloon_mnt)) {
548 rc = PTR_ERR(balloon_mnt);
549 balloon_mnt = NULL;
550 return rc;
551 }
552
553 b_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
554 if (IS_ERR(b_dev_info.inode)) {
555 rc = PTR_ERR(b_dev_info.inode);
556 b_dev_info.inode = NULL;
557 kern_unmount(balloon_mnt);
558 balloon_mnt = NULL;
559 return rc;
560 }
561
562 b_dev_info.inode->i_mapping->a_ops = &balloon_aops;
563 return 0;
564}
565static void cmm_balloon_compaction_deinit(void)
566{
567 if (b_dev_info.inode)
568 iput(b_dev_info.inode);
569 b_dev_info.inode = NULL;
570 kern_unmount(balloon_mnt);
571 balloon_mnt = NULL;
572}
573#else /* CONFIG_BALLOON_COMPACTION */
574static int cmm_balloon_compaction_init(void)
575{
576 return 0;
577}
578
579static void cmm_balloon_compaction_deinit(void)
580{
581}
582#endif /* CONFIG_BALLOON_COMPACTION */
583
487/**
488 * cmm_init - Module initialization
489 *
490 * Return value:
491 * 0 on success / other on failure
492 **/
493static int cmm_init(void)
494{
495 int rc;
496
497 if (!firmware_has_feature(FW_FEATURE_CMO))
498 return -EOPNOTSUPP;
499
584/**
585 * cmm_init - Module initialization
586 *
587 * Return value:
588 * 0 on success / other on failure
589 **/
590static int cmm_init(void)
591{
592 int rc;
593
594 if (!firmware_has_feature(FW_FEATURE_CMO))
595 return -EOPNOTSUPP;
596
500 if ((rc = register_oom_notifier(&cmm_oom_nb)) < 0)
597 rc = cmm_balloon_compaction_init();
598 if (rc)
501 return rc;
502
599 return rc;
600
601 rc = register_oom_notifier(&cmm_oom_nb);
602 if (rc < 0)
603 goto out_balloon_compaction;
604
503 if ((rc = register_reboot_notifier(&cmm_reboot_nb)))
504 goto out_oom_notifier;
505
506 if ((rc = cmm_sysfs_register(&cmm_dev)))
507 goto out_reboot_notifier;
508
509 rc = register_memory_notifier(&cmm_mem_nb);
510 if (rc)

--- 11 unchanged lines hidden (view full) ---

522 return 0;
523out_unregister_notifier:
524 unregister_memory_notifier(&cmm_mem_nb);
525 cmm_unregister_sysfs(&cmm_dev);
526out_reboot_notifier:
527 unregister_reboot_notifier(&cmm_reboot_nb);
528out_oom_notifier:
529 unregister_oom_notifier(&cmm_oom_nb);
605 if ((rc = register_reboot_notifier(&cmm_reboot_nb)))
606 goto out_oom_notifier;
607
608 if ((rc = cmm_sysfs_register(&cmm_dev)))
609 goto out_reboot_notifier;
610
611 rc = register_memory_notifier(&cmm_mem_nb);
612 if (rc)

--- 11 unchanged lines hidden (view full) ---

624 return 0;
625out_unregister_notifier:
626 unregister_memory_notifier(&cmm_mem_nb);
627 cmm_unregister_sysfs(&cmm_dev);
628out_reboot_notifier:
629 unregister_reboot_notifier(&cmm_reboot_nb);
630out_oom_notifier:
631 unregister_oom_notifier(&cmm_oom_nb);
632out_balloon_compaction:
633 cmm_balloon_compaction_deinit();
530 return rc;
531}
532
533/**
534 * cmm_exit - Module exit
535 *
536 * Return value:
537 * nothing
538 **/
539static void cmm_exit(void)
540{
541 if (cmm_thread_ptr)
542 kthread_stop(cmm_thread_ptr);
543 unregister_oom_notifier(&cmm_oom_nb);
544 unregister_reboot_notifier(&cmm_reboot_nb);
545 unregister_memory_notifier(&cmm_mem_nb);
546 cmm_free_pages(atomic_long_read(&loaned_pages));
547 cmm_unregister_sysfs(&cmm_dev);
634 return rc;
635}
636
637/**
638 * cmm_exit - Module exit
639 *
640 * Return value:
641 * nothing
642 **/
643static void cmm_exit(void)
644{
645 if (cmm_thread_ptr)
646 kthread_stop(cmm_thread_ptr);
647 unregister_oom_notifier(&cmm_oom_nb);
648 unregister_reboot_notifier(&cmm_reboot_nb);
649 unregister_memory_notifier(&cmm_mem_nb);
650 cmm_free_pages(atomic_long_read(&loaned_pages));
651 cmm_unregister_sysfs(&cmm_dev);
652 cmm_balloon_compaction_deinit();
548}
549
550/**
551 * cmm_set_disable - Disable/Enable CMM
552 *
553 * Return value:
554 * 0 on success / other on failure
555 **/

--- 29 unchanged lines hidden --- 		653}
654
655/**
656 * cmm_set_disable - Disable/Enable CMM
657 *
658 * Return value:
659 * 0 on success / other on failure
660 **/

--- 29 unchanged lines hidden ---