1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Common interface for implementing a memory balloon, including support 4 * for migration of pages inflated in a memory balloon. 5 * 6 * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <aquini@redhat.com> 7 */ 8 #include <linux/mm.h> 9 #include <linux/slab.h> 10 #include <linux/export.h> 11 #include <linux/balloon.h> 12 13 /* 14 * Lock protecting the balloon_dev_info of all devices. We don't really 15 * expect more than one device. 16 */ 17 static DEFINE_SPINLOCK(balloon_pages_lock); 18 19 /** 20 * balloon_page_insert - insert a page into the balloon's page list and make 21 * the page->private assignment accordingly. 22 * @balloon : pointer to balloon device 23 * @page : page to be assigned as a 'balloon page' 24 * 25 * Caller must ensure the balloon_pages_lock is held. 26 */ 27 static void balloon_page_insert(struct balloon_dev_info *balloon, 28 struct page *page) 29 { 30 lockdep_assert_held(&balloon_pages_lock); 31 __SetPageOffline(page); 32 if (IS_ENABLED(CONFIG_BALLOON_MIGRATION)) { 33 SetPageMovableOps(page); 34 set_page_private(page, (unsigned long)balloon); 35 } 36 list_add(&page->lru, &balloon->pages); 37 } 38 39 /** 40 * balloon_page_finalize - prepare a balloon page that was removed from the 41 * balloon list for release to the page allocator 42 * @page: page to be released to the page allocator 43 * 44 * Caller must ensure the balloon_pages_lock is held. 45 */ 46 static void balloon_page_finalize(struct page *page) 47 { 48 lockdep_assert_held(&balloon_pages_lock); 49 if (IS_ENABLED(CONFIG_BALLOON_MIGRATION)) 50 set_page_private(page, 0); 51 /* PageOffline is sticky until the page is freed to the buddy. */ 52 } 53 54 static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info, 55 struct page *page) 56 { 57 balloon_page_insert(b_dev_info, page); 58 if (b_dev_info->adjust_managed_page_count) 59 adjust_managed_page_count(page, -1); 60 __count_vm_event(BALLOON_INFLATE); 61 inc_node_page_state(page, NR_BALLOON_PAGES); 62 } 63 64 /** 65 * balloon_page_list_enqueue() - inserts a list of pages into the balloon page 66 * list. 67 * @b_dev_info: balloon device descriptor where we will insert a new page to 68 * @pages: pages to enqueue - allocated using balloon_page_alloc. 69 * 70 * Driver must call this function to properly enqueue balloon pages before 71 * definitively removing them from the guest system. 72 * 73 * Return: number of pages that were enqueued. 74 */ 75 size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info, 76 struct list_head *pages) 77 { 78 struct page *page, *tmp; 79 unsigned long flags; 80 size_t n_pages = 0; 81 82 spin_lock_irqsave(&balloon_pages_lock, flags); 83 list_for_each_entry_safe(page, tmp, pages, lru) { 84 list_del(&page->lru); 85 balloon_page_enqueue_one(b_dev_info, page); 86 n_pages++; 87 } 88 spin_unlock_irqrestore(&balloon_pages_lock, flags); 89 return n_pages; 90 } 91 EXPORT_SYMBOL_GPL(balloon_page_list_enqueue); 92 93 /** 94 * balloon_page_list_dequeue() - removes pages from balloon's page list and 95 * returns a list of the pages. 96 * @b_dev_info: balloon device descriptor where we will grab a page from. 97 * @pages: pointer to the list of pages that would be returned to the caller. 98 * @n_req_pages: number of requested pages. 99 * 100 * Driver must call this function to properly de-allocate a previous enlisted 101 * balloon pages before definitively releasing it back to the guest system. 102 * This function tries to remove @n_req_pages from the ballooned pages and 103 * return them to the caller in the @pages list. 104 * 105 * Note that this function may fail to dequeue some pages even if the balloon 106 * isn't empty - since the page list can be temporarily empty due to compaction 107 * of isolated pages. 108 * 109 * Return: number of pages that were added to the @pages list. 110 */ 111 size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info, 112 struct list_head *pages, size_t n_req_pages) 113 { 114 struct page *page, *tmp; 115 unsigned long flags; 116 size_t n_pages = 0; 117 118 spin_lock_irqsave(&balloon_pages_lock, flags); 119 list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) { 120 if (n_pages == n_req_pages) 121 break; 122 list_del(&page->lru); 123 if (b_dev_info->adjust_managed_page_count) 124 adjust_managed_page_count(page, 1); 125 balloon_page_finalize(page); 126 __count_vm_event(BALLOON_DEFLATE); 127 list_add(&page->lru, pages); 128 dec_node_page_state(page, NR_BALLOON_PAGES); 129 n_pages++; 130 } 131 spin_unlock_irqrestore(&balloon_pages_lock, flags); 132 133 return n_pages; 134 } 135 EXPORT_SYMBOL_GPL(balloon_page_list_dequeue); 136 137 /** 138 * balloon_page_alloc - allocates a new page for insertion into the balloon 139 * page list. 140 * 141 * Driver must call this function to properly allocate a new balloon page. 142 * Driver must call balloon_page_enqueue before definitively removing the page 143 * from the guest system. 144 * 145 * Return: struct page for the allocated page or NULL on allocation failure. 146 */ 147 struct page *balloon_page_alloc(void) 148 { 149 gfp_t gfp_flags = __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 150 151 if (IS_ENABLED(CONFIG_BALLOON_MIGRATION)) 152 gfp_flags |= GFP_HIGHUSER_MOVABLE; 153 else 154 gfp_flags |= GFP_HIGHUSER; 155 156 return alloc_page(gfp_flags); 157 } 158 EXPORT_SYMBOL_GPL(balloon_page_alloc); 159 160 /** 161 * balloon_page_enqueue - inserts a new page into the balloon page list. 162 * 163 * @b_dev_info: balloon device descriptor where we will insert a new page 164 * @page: new page to enqueue - allocated using balloon_page_alloc. 165 * 166 * Drivers must call this function to properly enqueue a new allocated balloon 167 * page before definitively removing the page from the guest system. 168 * 169 * Drivers must not enqueue pages while page->lru is still in 170 * use, and must not use page->lru until a page was unqueued again. 171 */ 172 void balloon_page_enqueue(struct balloon_dev_info *b_dev_info, 173 struct page *page) 174 { 175 unsigned long flags; 176 177 spin_lock_irqsave(&balloon_pages_lock, flags); 178 balloon_page_enqueue_one(b_dev_info, page); 179 spin_unlock_irqrestore(&balloon_pages_lock, flags); 180 } 181 EXPORT_SYMBOL_GPL(balloon_page_enqueue); 182 183 /** 184 * balloon_page_dequeue - removes a page from balloon's page list and returns 185 * its address to allow the driver to release the page. 186 * @b_dev_info: balloon device descriptor where we will grab a page from. 187 * 188 * Driver must call this function to properly dequeue a previously enqueued page 189 * before definitively releasing it back to the guest system. 190 * 191 * Caller must perform its own accounting to ensure that this 192 * function is called only if some pages are actually enqueued. 193 * 194 * Note that this function may fail to dequeue some pages even if there are 195 * some enqueued pages - since the page list can be temporarily empty due to 196 * the compaction of isolated pages. 197 * 198 * TODO: remove the caller accounting requirements, and allow caller to wait 199 * until all pages can be dequeued. 200 * 201 * Return: struct page for the dequeued page, or NULL if no page was dequeued. 202 */ 203 struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info) 204 { 205 unsigned long flags; 206 LIST_HEAD(pages); 207 int n_pages; 208 209 n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1); 210 211 if (n_pages != 1) { 212 /* 213 * If we are unable to dequeue a balloon page because the page 214 * list is empty and there are no isolated pages, then something 215 * went out of track and some balloon pages are lost. 216 * BUG() here, otherwise the balloon driver may get stuck in 217 * an infinite loop while attempting to release all its pages. 218 */ 219 spin_lock_irqsave(&balloon_pages_lock, flags); 220 if (unlikely(list_empty(&b_dev_info->pages) && 221 !b_dev_info->isolated_pages)) 222 BUG(); 223 spin_unlock_irqrestore(&balloon_pages_lock, flags); 224 return NULL; 225 } 226 return list_first_entry(&pages, struct page, lru); 227 } 228 EXPORT_SYMBOL_GPL(balloon_page_dequeue); 229 230 #ifdef CONFIG_BALLOON_MIGRATION 231 static struct balloon_dev_info *balloon_page_device(struct page *page) 232 { 233 return (struct balloon_dev_info *)page_private(page); 234 } 235 236 static bool balloon_page_isolate(struct page *page, isolate_mode_t mode) 237 238 { 239 struct balloon_dev_info *b_dev_info; 240 unsigned long flags; 241 242 spin_lock_irqsave(&balloon_pages_lock, flags); 243 b_dev_info = balloon_page_device(page); 244 if (!b_dev_info) { 245 /* 246 * The page already got deflated and removed from the 247 * balloon list. 248 */ 249 spin_unlock_irqrestore(&balloon_pages_lock, flags); 250 return false; 251 } 252 list_del(&page->lru); 253 b_dev_info->isolated_pages++; 254 spin_unlock_irqrestore(&balloon_pages_lock, flags); 255 256 return true; 257 } 258 259 static void balloon_page_putback(struct page *page) 260 { 261 struct balloon_dev_info *b_dev_info = balloon_page_device(page); 262 unsigned long flags; 263 264 /* 265 * When we isolated the page, the page was still inflated in a balloon 266 * device. As isolated balloon pages cannot get deflated, we still have 267 * a balloon device here. 268 */ 269 if (WARN_ON_ONCE(!b_dev_info)) 270 return; 271 272 spin_lock_irqsave(&balloon_pages_lock, flags); 273 list_add(&page->lru, &b_dev_info->pages); 274 b_dev_info->isolated_pages--; 275 spin_unlock_irqrestore(&balloon_pages_lock, flags); 276 } 277 278 static int balloon_page_migrate(struct page *newpage, struct page *page, 279 enum migrate_mode mode) 280 { 281 struct balloon_dev_info *b_dev_info = balloon_page_device(page); 282 unsigned long flags; 283 int rc; 284 285 /* 286 * When we isolated the page, the page was still inflated in a balloon 287 * device. As isolated balloon pages cannot get deflated, we still have 288 * a balloon device here. 289 */ 290 if (WARN_ON_ONCE(!b_dev_info)) 291 return -EAGAIN; 292 293 rc = b_dev_info->migratepage(b_dev_info, newpage, page, mode); 294 if (rc < 0 && rc != -ENOENT) 295 return rc; 296 297 spin_lock_irqsave(&balloon_pages_lock, flags); 298 if (!rc) { 299 /* Insert the new page into the balloon list. */ 300 get_page(newpage); 301 balloon_page_insert(b_dev_info, newpage); 302 __count_vm_event(BALLOON_MIGRATE); 303 304 if (b_dev_info->adjust_managed_page_count && 305 page_zone(page) != page_zone(newpage)) { 306 /* 307 * When we migrate a page to a different zone we 308 * have to fixup the count of both involved zones. 309 */ 310 adjust_managed_page_count(page, 1); 311 adjust_managed_page_count(newpage, -1); 312 } 313 } else { 314 /* Old page was deflated but new page not inflated. */ 315 __count_vm_event(BALLOON_DEFLATE); 316 317 if (b_dev_info->adjust_managed_page_count) 318 adjust_managed_page_count(page, 1); 319 } 320 321 b_dev_info->isolated_pages--; 322 323 /* Free the now-deflated page we isolated in balloon_page_isolate(). */ 324 balloon_page_finalize(page); 325 spin_unlock_irqrestore(&balloon_pages_lock, flags); 326 327 put_page(page); 328 329 return 0; 330 } 331 332 static const struct movable_operations balloon_mops = { 333 .migrate_page = balloon_page_migrate, 334 .isolate_page = balloon_page_isolate, 335 .putback_page = balloon_page_putback, 336 }; 337 338 static int __init balloon_init(void) 339 { 340 return set_movable_ops(&balloon_mops, PGTY_offline); 341 } 342 core_initcall(balloon_init); 343 344 #endif /* CONFIG_BALLOON_MIGRATION */ 345