1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* memcontrol.h - Memory Controller 3 * 4 * Copyright IBM Corporation, 2007 5 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 6 * 7 * Copyright 2007 OpenVZ SWsoft Inc 8 * Author: Pavel Emelianov <xemul@openvz.org> 9 */ 10 11 #ifndef _LINUX_MEMCONTROL_H 12 #define _LINUX_MEMCONTROL_H 13 #include <linux/cgroup.h> 14 #include <linux/vm_event_item.h> 15 #include <linux/hardirq.h> 16 #include <linux/jump_label.h> 17 #include <linux/kernel.h> 18 #include <linux/page_counter.h> 19 #include <linux/vmpressure.h> 20 #include <linux/eventfd.h> 21 #include <linux/mm.h> 22 #include <linux/vmstat.h> 23 #include <linux/writeback.h> 24 #include <linux/page-flags.h> 25 #include <linux/shrinker.h> 26 27 struct mem_cgroup; 28 struct obj_cgroup; 29 struct page; 30 struct mm_struct; 31 struct kmem_cache; 32 33 /* Cgroup-specific page state, on top of universal node page state */ 34 enum memcg_stat_item { 35 MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS, 36 MEMCG_SOCK, 37 MEMCG_PERCPU_B, 38 MEMCG_VMALLOC, 39 MEMCG_KMEM, 40 MEMCG_ZSWAP_B, 41 MEMCG_ZSWAPPED, 42 MEMCG_NR_STAT, 43 }; 44 45 enum memcg_memory_event { 46 MEMCG_LOW, 47 MEMCG_HIGH, 48 MEMCG_MAX, 49 MEMCG_OOM, 50 MEMCG_OOM_KILL, 51 MEMCG_OOM_GROUP_KILL, 52 MEMCG_SWAP_HIGH, 53 MEMCG_SWAP_MAX, 54 MEMCG_SWAP_FAIL, 55 MEMCG_NR_MEMORY_EVENTS, 56 }; 57 58 struct mem_cgroup_reclaim_cookie { 59 pg_data_t *pgdat; 60 unsigned int generation; 61 }; 62 63 #ifdef CONFIG_MEMCG 64 65 #define MEM_CGROUP_ID_SHIFT 16 66 67 struct mem_cgroup_id { 68 int id; 69 refcount_t ref; 70 }; 71 72 /* 73 * Per memcg event counter is incremented at every pagein/pageout. With THP, 74 * it will be incremented by the number of pages. This counter is used 75 * to trigger some periodic events. This is straightforward and better 76 * than using jiffies etc. to handle periodic memcg event. 77 */ 78 enum mem_cgroup_events_target { 79 MEM_CGROUP_TARGET_THRESH, 80 MEM_CGROUP_TARGET_SOFTLIMIT, 81 MEM_CGROUP_NTARGETS, 82 }; 83 84 struct memcg_vmstats_percpu; 85 struct memcg_vmstats; 86 struct lruvec_stats_percpu; 87 struct lruvec_stats; 88 89 struct mem_cgroup_reclaim_iter { 90 struct mem_cgroup *position; 91 /* scan generation, increased every round-trip */ 92 unsigned int generation; 93 }; 94 95 /* 96 * per-node information in memory controller. 97 */ 98 struct mem_cgroup_per_node { 99 struct lruvec lruvec; 100 101 struct lruvec_stats_percpu __percpu *lruvec_stats_percpu; 102 struct lruvec_stats *lruvec_stats; 103 104 unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; 105 106 struct mem_cgroup_reclaim_iter iter; 107 108 struct shrinker_info __rcu *shrinker_info; 109 110 struct rb_node tree_node; /* RB tree node */ 111 unsigned long usage_in_excess;/* Set to the value by which */ 112 /* the soft limit is exceeded*/ 113 bool on_tree; 114 struct mem_cgroup *memcg; /* Back pointer, we cannot */ 115 /* use container_of */ 116 }; 117 118 struct mem_cgroup_threshold { 119 struct eventfd_ctx *eventfd; 120 unsigned long threshold; 121 }; 122 123 /* For threshold */ 124 struct mem_cgroup_threshold_ary { 125 /* An array index points to threshold just below or equal to usage. */ 126 int current_threshold; 127 /* Size of entries[] */ 128 unsigned int size; 129 /* Array of thresholds */ 130 struct mem_cgroup_threshold entries[] __counted_by(size); 131 }; 132 133 struct mem_cgroup_thresholds { 134 /* Primary thresholds array */ 135 struct mem_cgroup_threshold_ary *primary; 136 /* 137 * Spare threshold array. 138 * This is needed to make mem_cgroup_unregister_event() "never fail". 139 * It must be able to store at least primary->size - 1 entries. 140 */ 141 struct mem_cgroup_threshold_ary *spare; 142 }; 143 144 /* 145 * Remember four most recent foreign writebacks with dirty pages in this 146 * cgroup. Inode sharing is expected to be uncommon and, even if we miss 147 * one in a given round, we're likely to catch it later if it keeps 148 * foreign-dirtying, so a fairly low count should be enough. 149 * 150 * See mem_cgroup_track_foreign_dirty_slowpath() for details. 151 */ 152 #define MEMCG_CGWB_FRN_CNT 4 153 154 struct memcg_cgwb_frn { 155 u64 bdi_id; /* bdi->id of the foreign inode */ 156 int memcg_id; /* memcg->css.id of foreign inode */ 157 u64 at; /* jiffies_64 at the time of dirtying */ 158 struct wb_completion done; /* tracks in-flight foreign writebacks */ 159 }; 160 161 /* 162 * Bucket for arbitrarily byte-sized objects charged to a memory 163 * cgroup. The bucket can be reparented in one piece when the cgroup 164 * is destroyed, without having to round up the individual references 165 * of all live memory objects in the wild. 166 */ 167 struct obj_cgroup { 168 struct percpu_ref refcnt; 169 struct mem_cgroup *memcg; 170 atomic_t nr_charged_bytes; 171 union { 172 struct list_head list; /* protected by objcg_lock */ 173 struct rcu_head rcu; 174 }; 175 }; 176 177 /* 178 * The memory controller data structure. The memory controller controls both 179 * page cache and RSS per cgroup. We would eventually like to provide 180 * statistics based on the statistics developed by Rik Van Riel for clock-pro, 181 * to help the administrator determine what knobs to tune. 182 */ 183 struct mem_cgroup { 184 struct cgroup_subsys_state css; 185 186 /* Private memcg ID. Used to ID objects that outlive the cgroup */ 187 struct mem_cgroup_id id; 188 189 /* Accounted resources */ 190 struct page_counter memory; /* Both v1 & v2 */ 191 192 union { 193 struct page_counter swap; /* v2 only */ 194 struct page_counter memsw; /* v1 only */ 195 }; 196 197 /* Legacy consumer-oriented counters */ 198 struct page_counter kmem; /* v1 only */ 199 struct page_counter tcpmem; /* v1 only */ 200 201 /* Range enforcement for interrupt charges */ 202 struct work_struct high_work; 203 204 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 205 unsigned long zswap_max; 206 207 /* 208 * Prevent pages from this memcg from being written back from zswap to 209 * swap, and from being swapped out on zswap store failures. 210 */ 211 bool zswap_writeback; 212 #endif 213 214 unsigned long soft_limit; 215 216 /* vmpressure notifications */ 217 struct vmpressure vmpressure; 218 219 /* 220 * Should the OOM killer kill all belonging tasks, had it kill one? 221 */ 222 bool oom_group; 223 224 /* protected by memcg_oom_lock */ 225 bool oom_lock; 226 int under_oom; 227 228 int swappiness; 229 /* OOM-Killer disable */ 230 int oom_kill_disable; 231 232 /* memory.events and memory.events.local */ 233 struct cgroup_file events_file; 234 struct cgroup_file events_local_file; 235 236 /* handle for "memory.swap.events" */ 237 struct cgroup_file swap_events_file; 238 239 /* protect arrays of thresholds */ 240 struct mutex thresholds_lock; 241 242 /* thresholds for memory usage. RCU-protected */ 243 struct mem_cgroup_thresholds thresholds; 244 245 /* thresholds for mem+swap usage. RCU-protected */ 246 struct mem_cgroup_thresholds memsw_thresholds; 247 248 /* For oom notifier event fd */ 249 struct list_head oom_notify; 250 251 /* 252 * Should we move charges of a task when a task is moved into this 253 * mem_cgroup ? And what type of charges should we move ? 254 */ 255 unsigned long move_charge_at_immigrate; 256 /* taken only while moving_account > 0 */ 257 spinlock_t move_lock; 258 unsigned long move_lock_flags; 259 260 CACHELINE_PADDING(_pad1_); 261 262 /* memory.stat */ 263 struct memcg_vmstats *vmstats; 264 265 /* memory.events */ 266 atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; 267 atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS]; 268 269 /* 270 * Hint of reclaim pressure for socket memroy management. Note 271 * that this indicator should NOT be used in legacy cgroup mode 272 * where socket memory is accounted/charged separately. 273 */ 274 unsigned long socket_pressure; 275 276 /* Legacy tcp memory accounting */ 277 bool tcpmem_active; 278 int tcpmem_pressure; 279 280 #ifdef CONFIG_MEMCG_KMEM 281 int kmemcg_id; 282 /* 283 * memcg->objcg is wiped out as a part of the objcg repaprenting 284 * process. memcg->orig_objcg preserves a pointer (and a reference) 285 * to the original objcg until the end of live of memcg. 286 */ 287 struct obj_cgroup __rcu *objcg; 288 struct obj_cgroup *orig_objcg; 289 /* list of inherited objcgs, protected by objcg_lock */ 290 struct list_head objcg_list; 291 #endif 292 293 CACHELINE_PADDING(_pad2_); 294 295 /* 296 * set > 0 if pages under this cgroup are moving to other cgroup. 297 */ 298 atomic_t moving_account; 299 struct task_struct *move_lock_task; 300 301 struct memcg_vmstats_percpu __percpu *vmstats_percpu; 302 303 #ifdef CONFIG_CGROUP_WRITEBACK 304 struct list_head cgwb_list; 305 struct wb_domain cgwb_domain; 306 struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; 307 #endif 308 309 /* List of events which userspace want to receive */ 310 struct list_head event_list; 311 spinlock_t event_list_lock; 312 313 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 314 struct deferred_split deferred_split_queue; 315 #endif 316 317 #ifdef CONFIG_LRU_GEN_WALKS_MMU 318 /* per-memcg mm_struct list */ 319 struct lru_gen_mm_list mm_list; 320 #endif 321 322 struct mem_cgroup_per_node *nodeinfo[]; 323 }; 324 325 /* 326 * size of first charge trial. 327 * TODO: maybe necessary to use big numbers in big irons or dynamic based of the 328 * workload. 329 */ 330 #define MEMCG_CHARGE_BATCH 64U 331 332 extern struct mem_cgroup *root_mem_cgroup; 333 334 enum page_memcg_data_flags { 335 /* page->memcg_data is a pointer to an slabobj_ext vector */ 336 MEMCG_DATA_OBJEXTS = (1UL << 0), 337 /* page has been accounted as a non-slab kernel page */ 338 MEMCG_DATA_KMEM = (1UL << 1), 339 /* the next bit after the last actual flag */ 340 __NR_MEMCG_DATA_FLAGS = (1UL << 2), 341 }; 342 343 #define __FIRST_OBJEXT_FLAG __NR_MEMCG_DATA_FLAGS 344 345 #else /* CONFIG_MEMCG */ 346 347 #define __FIRST_OBJEXT_FLAG (1UL << 0) 348 349 #endif /* CONFIG_MEMCG */ 350 351 enum objext_flags { 352 /* slabobj_ext vector failed to allocate */ 353 OBJEXTS_ALLOC_FAIL = __FIRST_OBJEXT_FLAG, 354 /* the next bit after the last actual flag */ 355 __NR_OBJEXTS_FLAGS = (__FIRST_OBJEXT_FLAG << 1), 356 }; 357 358 #define OBJEXTS_FLAGS_MASK (__NR_OBJEXTS_FLAGS - 1) 359 360 #ifdef CONFIG_MEMCG 361 362 static inline bool folio_memcg_kmem(struct folio *folio); 363 364 /* 365 * After the initialization objcg->memcg is always pointing at 366 * a valid memcg, but can be atomically swapped to the parent memcg. 367 * 368 * The caller must ensure that the returned memcg won't be released: 369 * e.g. acquire the rcu_read_lock or css_set_lock. 370 */ 371 static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) 372 { 373 return READ_ONCE(objcg->memcg); 374 } 375 376 /* 377 * __folio_memcg - Get the memory cgroup associated with a non-kmem folio 378 * @folio: Pointer to the folio. 379 * 380 * Returns a pointer to the memory cgroup associated with the folio, 381 * or NULL. This function assumes that the folio is known to have a 382 * proper memory cgroup pointer. It's not safe to call this function 383 * against some type of folios, e.g. slab folios or ex-slab folios or 384 * kmem folios. 385 */ 386 static inline struct mem_cgroup *__folio_memcg(struct folio *folio) 387 { 388 unsigned long memcg_data = folio->memcg_data; 389 390 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); 391 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio); 392 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio); 393 394 return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); 395 } 396 397 /* 398 * __folio_objcg - get the object cgroup associated with a kmem folio. 399 * @folio: Pointer to the folio. 400 * 401 * Returns a pointer to the object cgroup associated with the folio, 402 * or NULL. This function assumes that the folio is known to have a 403 * proper object cgroup pointer. It's not safe to call this function 404 * against some type of folios, e.g. slab folios or ex-slab folios or 405 * LRU folios. 406 */ 407 static inline struct obj_cgroup *__folio_objcg(struct folio *folio) 408 { 409 unsigned long memcg_data = folio->memcg_data; 410 411 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); 412 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio); 413 VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio); 414 415 return (struct obj_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); 416 } 417 418 /* 419 * folio_memcg - Get the memory cgroup associated with a folio. 420 * @folio: Pointer to the folio. 421 * 422 * Returns a pointer to the memory cgroup associated with the folio, 423 * or NULL. This function assumes that the folio is known to have a 424 * proper memory cgroup pointer. It's not safe to call this function 425 * against some type of folios, e.g. slab folios or ex-slab folios. 426 * 427 * For a non-kmem folio any of the following ensures folio and memcg binding 428 * stability: 429 * 430 * - the folio lock 431 * - LRU isolation 432 * - folio_memcg_lock() 433 * - exclusive reference 434 * - mem_cgroup_trylock_pages() 435 * 436 * For a kmem folio a caller should hold an rcu read lock to protect memcg 437 * associated with a kmem folio from being released. 438 */ 439 static inline struct mem_cgroup *folio_memcg(struct folio *folio) 440 { 441 if (folio_memcg_kmem(folio)) 442 return obj_cgroup_memcg(__folio_objcg(folio)); 443 return __folio_memcg(folio); 444 } 445 446 static inline struct mem_cgroup *page_memcg(struct page *page) 447 { 448 return folio_memcg(page_folio(page)); 449 } 450 451 /** 452 * folio_memcg_rcu - Locklessly get the memory cgroup associated with a folio. 453 * @folio: Pointer to the folio. 454 * 455 * This function assumes that the folio is known to have a 456 * proper memory cgroup pointer. It's not safe to call this function 457 * against some type of folios, e.g. slab folios or ex-slab folios. 458 * 459 * Return: A pointer to the memory cgroup associated with the folio, 460 * or NULL. 461 */ 462 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) 463 { 464 unsigned long memcg_data = READ_ONCE(folio->memcg_data); 465 466 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); 467 WARN_ON_ONCE(!rcu_read_lock_held()); 468 469 if (memcg_data & MEMCG_DATA_KMEM) { 470 struct obj_cgroup *objcg; 471 472 objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); 473 return obj_cgroup_memcg(objcg); 474 } 475 476 return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); 477 } 478 479 /* 480 * folio_memcg_check - Get the memory cgroup associated with a folio. 481 * @folio: Pointer to the folio. 482 * 483 * Returns a pointer to the memory cgroup associated with the folio, 484 * or NULL. This function unlike folio_memcg() can take any folio 485 * as an argument. It has to be used in cases when it's not known if a folio 486 * has an associated memory cgroup pointer or an object cgroups vector or 487 * an object cgroup. 488 * 489 * For a non-kmem folio any of the following ensures folio and memcg binding 490 * stability: 491 * 492 * - the folio lock 493 * - LRU isolation 494 * - lock_folio_memcg() 495 * - exclusive reference 496 * - mem_cgroup_trylock_pages() 497 * 498 * For a kmem folio a caller should hold an rcu read lock to protect memcg 499 * associated with a kmem folio from being released. 500 */ 501 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) 502 { 503 /* 504 * Because folio->memcg_data might be changed asynchronously 505 * for slabs, READ_ONCE() should be used here. 506 */ 507 unsigned long memcg_data = READ_ONCE(folio->memcg_data); 508 509 if (memcg_data & MEMCG_DATA_OBJEXTS) 510 return NULL; 511 512 if (memcg_data & MEMCG_DATA_KMEM) { 513 struct obj_cgroup *objcg; 514 515 objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); 516 return obj_cgroup_memcg(objcg); 517 } 518 519 return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); 520 } 521 522 static inline struct mem_cgroup *page_memcg_check(struct page *page) 523 { 524 if (PageTail(page)) 525 return NULL; 526 return folio_memcg_check((struct folio *)page); 527 } 528 529 static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) 530 { 531 struct mem_cgroup *memcg; 532 533 rcu_read_lock(); 534 retry: 535 memcg = obj_cgroup_memcg(objcg); 536 if (unlikely(!css_tryget(&memcg->css))) 537 goto retry; 538 rcu_read_unlock(); 539 540 return memcg; 541 } 542 543 #ifdef CONFIG_MEMCG_KMEM 544 /* 545 * folio_memcg_kmem - Check if the folio has the memcg_kmem flag set. 546 * @folio: Pointer to the folio. 547 * 548 * Checks if the folio has MemcgKmem flag set. The caller must ensure 549 * that the folio has an associated memory cgroup. It's not safe to call 550 * this function against some types of folios, e.g. slab folios. 551 */ 552 static inline bool folio_memcg_kmem(struct folio *folio) 553 { 554 VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page); 555 VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio); 556 return folio->memcg_data & MEMCG_DATA_KMEM; 557 } 558 559 560 #else 561 static inline bool folio_memcg_kmem(struct folio *folio) 562 { 563 return false; 564 } 565 566 #endif 567 568 static inline bool PageMemcgKmem(struct page *page) 569 { 570 return folio_memcg_kmem(page_folio(page)); 571 } 572 573 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) 574 { 575 return (memcg == root_mem_cgroup); 576 } 577 578 static inline bool mem_cgroup_disabled(void) 579 { 580 return !cgroup_subsys_enabled(memory_cgrp_subsys); 581 } 582 583 static inline void mem_cgroup_protection(struct mem_cgroup *root, 584 struct mem_cgroup *memcg, 585 unsigned long *min, 586 unsigned long *low) 587 { 588 *min = *low = 0; 589 590 if (mem_cgroup_disabled()) 591 return; 592 593 /* 594 * There is no reclaim protection applied to a targeted reclaim. 595 * We are special casing this specific case here because 596 * mem_cgroup_calculate_protection is not robust enough to keep 597 * the protection invariant for calculated effective values for 598 * parallel reclaimers with different reclaim target. This is 599 * especially a problem for tail memcgs (as they have pages on LRU) 600 * which would want to have effective values 0 for targeted reclaim 601 * but a different value for external reclaim. 602 * 603 * Example 604 * Let's have global and A's reclaim in parallel: 605 * | 606 * A (low=2G, usage = 3G, max = 3G, children_low_usage = 1.5G) 607 * |\ 608 * | C (low = 1G, usage = 2.5G) 609 * B (low = 1G, usage = 0.5G) 610 * 611 * For the global reclaim 612 * A.elow = A.low 613 * B.elow = min(B.usage, B.low) because children_low_usage <= A.elow 614 * C.elow = min(C.usage, C.low) 615 * 616 * With the effective values resetting we have A reclaim 617 * A.elow = 0 618 * B.elow = B.low 619 * C.elow = C.low 620 * 621 * If the global reclaim races with A's reclaim then 622 * B.elow = C.elow = 0 because children_low_usage > A.elow) 623 * is possible and reclaiming B would be violating the protection. 624 * 625 */ 626 if (root == memcg) 627 return; 628 629 *min = READ_ONCE(memcg->memory.emin); 630 *low = READ_ONCE(memcg->memory.elow); 631 } 632 633 void mem_cgroup_calculate_protection(struct mem_cgroup *root, 634 struct mem_cgroup *memcg); 635 636 static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, 637 struct mem_cgroup *memcg) 638 { 639 /* 640 * The root memcg doesn't account charges, and doesn't support 641 * protection. The target memcg's protection is ignored, see 642 * mem_cgroup_calculate_protection() and mem_cgroup_protection() 643 */ 644 return mem_cgroup_disabled() || mem_cgroup_is_root(memcg) || 645 memcg == target; 646 } 647 648 static inline bool mem_cgroup_below_low(struct mem_cgroup *target, 649 struct mem_cgroup *memcg) 650 { 651 if (mem_cgroup_unprotected(target, memcg)) 652 return false; 653 654 return READ_ONCE(memcg->memory.elow) >= 655 page_counter_read(&memcg->memory); 656 } 657 658 static inline bool mem_cgroup_below_min(struct mem_cgroup *target, 659 struct mem_cgroup *memcg) 660 { 661 if (mem_cgroup_unprotected(target, memcg)) 662 return false; 663 664 return READ_ONCE(memcg->memory.emin) >= 665 page_counter_read(&memcg->memory); 666 } 667 668 void mem_cgroup_commit_charge(struct folio *folio, struct mem_cgroup *memcg); 669 670 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp); 671 672 /** 673 * mem_cgroup_charge - Charge a newly allocated folio to a cgroup. 674 * @folio: Folio to charge. 675 * @mm: mm context of the allocating task. 676 * @gfp: Reclaim mode. 677 * 678 * Try to charge @folio to the memcg that @mm belongs to, reclaiming 679 * pages according to @gfp if necessary. If @mm is NULL, try to 680 * charge to the active memcg. 681 * 682 * Do not use this for folios allocated for swapin. 683 * 684 * Return: 0 on success. Otherwise, an error code is returned. 685 */ 686 static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, 687 gfp_t gfp) 688 { 689 if (mem_cgroup_disabled()) 690 return 0; 691 return __mem_cgroup_charge(folio, mm, gfp); 692 } 693 694 int mem_cgroup_hugetlb_try_charge(struct mem_cgroup *memcg, gfp_t gfp, 695 long nr_pages); 696 697 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm, 698 gfp_t gfp, swp_entry_t entry); 699 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry); 700 701 void __mem_cgroup_uncharge(struct folio *folio); 702 703 /** 704 * mem_cgroup_uncharge - Uncharge a folio. 705 * @folio: Folio to uncharge. 706 * 707 * Uncharge a folio previously charged with mem_cgroup_charge(). 708 */ 709 static inline void mem_cgroup_uncharge(struct folio *folio) 710 { 711 if (mem_cgroup_disabled()) 712 return; 713 __mem_cgroup_uncharge(folio); 714 } 715 716 void __mem_cgroup_uncharge_folios(struct folio_batch *folios); 717 static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) 718 { 719 if (mem_cgroup_disabled()) 720 return; 721 __mem_cgroup_uncharge_folios(folios); 722 } 723 724 void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages); 725 void mem_cgroup_replace_folio(struct folio *old, struct folio *new); 726 void mem_cgroup_migrate(struct folio *old, struct folio *new); 727 728 /** 729 * mem_cgroup_lruvec - get the lru list vector for a memcg & node 730 * @memcg: memcg of the wanted lruvec 731 * @pgdat: pglist_data 732 * 733 * Returns the lru list vector holding pages for a given @memcg & 734 * @pgdat combination. This can be the node lruvec, if the memory 735 * controller is disabled. 736 */ 737 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, 738 struct pglist_data *pgdat) 739 { 740 struct mem_cgroup_per_node *mz; 741 struct lruvec *lruvec; 742 743 if (mem_cgroup_disabled()) { 744 lruvec = &pgdat->__lruvec; 745 goto out; 746 } 747 748 if (!memcg) 749 memcg = root_mem_cgroup; 750 751 mz = memcg->nodeinfo[pgdat->node_id]; 752 lruvec = &mz->lruvec; 753 out: 754 /* 755 * Since a node can be onlined after the mem_cgroup was created, 756 * we have to be prepared to initialize lruvec->pgdat here; 757 * and if offlined then reonlined, we need to reinitialize it. 758 */ 759 if (unlikely(lruvec->pgdat != pgdat)) 760 lruvec->pgdat = pgdat; 761 return lruvec; 762 } 763 764 /** 765 * folio_lruvec - return lruvec for isolating/putting an LRU folio 766 * @folio: Pointer to the folio. 767 * 768 * This function relies on folio->mem_cgroup being stable. 769 */ 770 static inline struct lruvec *folio_lruvec(struct folio *folio) 771 { 772 struct mem_cgroup *memcg = folio_memcg(folio); 773 774 VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio); 775 return mem_cgroup_lruvec(memcg, folio_pgdat(folio)); 776 } 777 778 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p); 779 780 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm); 781 782 struct mem_cgroup *get_mem_cgroup_from_current(void); 783 784 struct lruvec *folio_lruvec_lock(struct folio *folio); 785 struct lruvec *folio_lruvec_lock_irq(struct folio *folio); 786 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, 787 unsigned long *flags); 788 789 #ifdef CONFIG_DEBUG_VM 790 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio); 791 #else 792 static inline 793 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) 794 { 795 } 796 #endif 797 798 static inline 799 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ 800 return css ? container_of(css, struct mem_cgroup, css) : NULL; 801 } 802 803 static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg) 804 { 805 return percpu_ref_tryget(&objcg->refcnt); 806 } 807 808 static inline void obj_cgroup_get(struct obj_cgroup *objcg) 809 { 810 percpu_ref_get(&objcg->refcnt); 811 } 812 813 static inline void obj_cgroup_get_many(struct obj_cgroup *objcg, 814 unsigned long nr) 815 { 816 percpu_ref_get_many(&objcg->refcnt, nr); 817 } 818 819 static inline void obj_cgroup_put(struct obj_cgroup *objcg) 820 { 821 if (objcg) 822 percpu_ref_put(&objcg->refcnt); 823 } 824 825 static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg) 826 { 827 return !memcg || css_tryget(&memcg->css); 828 } 829 830 static inline bool mem_cgroup_tryget_online(struct mem_cgroup *memcg) 831 { 832 return !memcg || css_tryget_online(&memcg->css); 833 } 834 835 static inline void mem_cgroup_put(struct mem_cgroup *memcg) 836 { 837 if (memcg) 838 css_put(&memcg->css); 839 } 840 841 #define mem_cgroup_from_counter(counter, member) \ 842 container_of(counter, struct mem_cgroup, member) 843 844 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *, 845 struct mem_cgroup *, 846 struct mem_cgroup_reclaim_cookie *); 847 void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); 848 void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 849 int (*)(struct task_struct *, void *), void *arg); 850 851 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg) 852 { 853 if (mem_cgroup_disabled()) 854 return 0; 855 856 return memcg->id.id; 857 } 858 struct mem_cgroup *mem_cgroup_from_id(unsigned short id); 859 860 #ifdef CONFIG_SHRINKER_DEBUG 861 static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg) 862 { 863 return memcg ? cgroup_ino(memcg->css.cgroup) : 0; 864 } 865 866 struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino); 867 #endif 868 869 static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) 870 { 871 return mem_cgroup_from_css(seq_css(m)); 872 } 873 874 static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) 875 { 876 struct mem_cgroup_per_node *mz; 877 878 if (mem_cgroup_disabled()) 879 return NULL; 880 881 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 882 return mz->memcg; 883 } 884 885 /** 886 * parent_mem_cgroup - find the accounting parent of a memcg 887 * @memcg: memcg whose parent to find 888 * 889 * Returns the parent memcg, or NULL if this is the root. 890 */ 891 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) 892 { 893 return mem_cgroup_from_css(memcg->css.parent); 894 } 895 896 static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, 897 struct mem_cgroup *root) 898 { 899 if (root == memcg) 900 return true; 901 return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup); 902 } 903 904 static inline bool mm_match_cgroup(struct mm_struct *mm, 905 struct mem_cgroup *memcg) 906 { 907 struct mem_cgroup *task_memcg; 908 bool match = false; 909 910 rcu_read_lock(); 911 task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 912 if (task_memcg) 913 match = mem_cgroup_is_descendant(task_memcg, memcg); 914 rcu_read_unlock(); 915 return match; 916 } 917 918 struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio); 919 ino_t page_cgroup_ino(struct page *page); 920 921 static inline bool mem_cgroup_online(struct mem_cgroup *memcg) 922 { 923 if (mem_cgroup_disabled()) 924 return true; 925 return !!(memcg->css.flags & CSS_ONLINE); 926 } 927 928 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 929 int zid, int nr_pages); 930 931 static inline 932 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, 933 enum lru_list lru, int zone_idx) 934 { 935 struct mem_cgroup_per_node *mz; 936 937 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 938 return READ_ONCE(mz->lru_zone_size[zone_idx][lru]); 939 } 940 941 void mem_cgroup_handle_over_high(gfp_t gfp_mask); 942 943 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg); 944 945 unsigned long mem_cgroup_size(struct mem_cgroup *memcg); 946 947 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, 948 struct task_struct *p); 949 950 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg); 951 952 static inline void mem_cgroup_enter_user_fault(void) 953 { 954 WARN_ON(current->in_user_fault); 955 current->in_user_fault = 1; 956 } 957 958 static inline void mem_cgroup_exit_user_fault(void) 959 { 960 WARN_ON(!current->in_user_fault); 961 current->in_user_fault = 0; 962 } 963 964 static inline bool task_in_memcg_oom(struct task_struct *p) 965 { 966 return p->memcg_in_oom; 967 } 968 969 bool mem_cgroup_oom_synchronize(bool wait); 970 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim, 971 struct mem_cgroup *oom_domain); 972 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg); 973 974 void folio_memcg_lock(struct folio *folio); 975 void folio_memcg_unlock(struct folio *folio); 976 977 void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, 978 int val); 979 980 /* try to stablize folio_memcg() for all the pages in a memcg */ 981 static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) 982 { 983 rcu_read_lock(); 984 985 if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account)) 986 return true; 987 988 rcu_read_unlock(); 989 return false; 990 } 991 992 static inline void mem_cgroup_unlock_pages(void) 993 { 994 rcu_read_unlock(); 995 } 996 997 /* idx can be of type enum memcg_stat_item or node_stat_item */ 998 static inline void mod_memcg_state(struct mem_cgroup *memcg, 999 enum memcg_stat_item idx, int val) 1000 { 1001 unsigned long flags; 1002 1003 local_irq_save(flags); 1004 __mod_memcg_state(memcg, idx, val); 1005 local_irq_restore(flags); 1006 } 1007 1008 static inline void mod_memcg_page_state(struct page *page, 1009 enum memcg_stat_item idx, int val) 1010 { 1011 struct mem_cgroup *memcg; 1012 1013 if (mem_cgroup_disabled()) 1014 return; 1015 1016 rcu_read_lock(); 1017 memcg = page_memcg(page); 1018 if (memcg) 1019 mod_memcg_state(memcg, idx, val); 1020 rcu_read_unlock(); 1021 } 1022 1023 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx); 1024 unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx); 1025 unsigned long lruvec_page_state_local(struct lruvec *lruvec, 1026 enum node_stat_item idx); 1027 1028 void mem_cgroup_flush_stats(struct mem_cgroup *memcg); 1029 void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg); 1030 1031 void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val); 1032 1033 static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, 1034 int val) 1035 { 1036 unsigned long flags; 1037 1038 local_irq_save(flags); 1039 __mod_lruvec_kmem_state(p, idx, val); 1040 local_irq_restore(flags); 1041 } 1042 1043 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, 1044 unsigned long count); 1045 1046 static inline void count_memcg_events(struct mem_cgroup *memcg, 1047 enum vm_event_item idx, 1048 unsigned long count) 1049 { 1050 unsigned long flags; 1051 1052 local_irq_save(flags); 1053 __count_memcg_events(memcg, idx, count); 1054 local_irq_restore(flags); 1055 } 1056 1057 static inline void count_memcg_folio_events(struct folio *folio, 1058 enum vm_event_item idx, unsigned long nr) 1059 { 1060 struct mem_cgroup *memcg = folio_memcg(folio); 1061 1062 if (memcg) 1063 count_memcg_events(memcg, idx, nr); 1064 } 1065 1066 static inline void count_memcg_event_mm(struct mm_struct *mm, 1067 enum vm_event_item idx) 1068 { 1069 struct mem_cgroup *memcg; 1070 1071 if (mem_cgroup_disabled()) 1072 return; 1073 1074 rcu_read_lock(); 1075 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 1076 if (likely(memcg)) 1077 count_memcg_events(memcg, idx, 1); 1078 rcu_read_unlock(); 1079 } 1080 1081 static inline void memcg_memory_event(struct mem_cgroup *memcg, 1082 enum memcg_memory_event event) 1083 { 1084 bool swap_event = event == MEMCG_SWAP_HIGH || event == MEMCG_SWAP_MAX || 1085 event == MEMCG_SWAP_FAIL; 1086 1087 atomic_long_inc(&memcg->memory_events_local[event]); 1088 if (!swap_event) 1089 cgroup_file_notify(&memcg->events_local_file); 1090 1091 do { 1092 atomic_long_inc(&memcg->memory_events[event]); 1093 if (swap_event) 1094 cgroup_file_notify(&memcg->swap_events_file); 1095 else 1096 cgroup_file_notify(&memcg->events_file); 1097 1098 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 1099 break; 1100 if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS) 1101 break; 1102 } while ((memcg = parent_mem_cgroup(memcg)) && 1103 !mem_cgroup_is_root(memcg)); 1104 } 1105 1106 static inline void memcg_memory_event_mm(struct mm_struct *mm, 1107 enum memcg_memory_event event) 1108 { 1109 struct mem_cgroup *memcg; 1110 1111 if (mem_cgroup_disabled()) 1112 return; 1113 1114 rcu_read_lock(); 1115 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 1116 if (likely(memcg)) 1117 memcg_memory_event(memcg, event); 1118 rcu_read_unlock(); 1119 } 1120 1121 void split_page_memcg(struct page *head, int old_order, int new_order); 1122 1123 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 1124 gfp_t gfp_mask, 1125 unsigned long *total_scanned); 1126 1127 #else /* CONFIG_MEMCG */ 1128 1129 #define MEM_CGROUP_ID_SHIFT 0 1130 1131 static inline struct mem_cgroup *folio_memcg(struct folio *folio) 1132 { 1133 return NULL; 1134 } 1135 1136 static inline struct mem_cgroup *page_memcg(struct page *page) 1137 { 1138 return NULL; 1139 } 1140 1141 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) 1142 { 1143 WARN_ON_ONCE(!rcu_read_lock_held()); 1144 return NULL; 1145 } 1146 1147 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) 1148 { 1149 return NULL; 1150 } 1151 1152 static inline struct mem_cgroup *page_memcg_check(struct page *page) 1153 { 1154 return NULL; 1155 } 1156 1157 static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) 1158 { 1159 return NULL; 1160 } 1161 1162 static inline bool folio_memcg_kmem(struct folio *folio) 1163 { 1164 return false; 1165 } 1166 1167 static inline bool PageMemcgKmem(struct page *page) 1168 { 1169 return false; 1170 } 1171 1172 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) 1173 { 1174 return true; 1175 } 1176 1177 static inline bool mem_cgroup_disabled(void) 1178 { 1179 return true; 1180 } 1181 1182 static inline void memcg_memory_event(struct mem_cgroup *memcg, 1183 enum memcg_memory_event event) 1184 { 1185 } 1186 1187 static inline void memcg_memory_event_mm(struct mm_struct *mm, 1188 enum memcg_memory_event event) 1189 { 1190 } 1191 1192 static inline void mem_cgroup_protection(struct mem_cgroup *root, 1193 struct mem_cgroup *memcg, 1194 unsigned long *min, 1195 unsigned long *low) 1196 { 1197 *min = *low = 0; 1198 } 1199 1200 static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root, 1201 struct mem_cgroup *memcg) 1202 { 1203 } 1204 1205 static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, 1206 struct mem_cgroup *memcg) 1207 { 1208 return true; 1209 } 1210 static inline bool mem_cgroup_below_low(struct mem_cgroup *target, 1211 struct mem_cgroup *memcg) 1212 { 1213 return false; 1214 } 1215 1216 static inline bool mem_cgroup_below_min(struct mem_cgroup *target, 1217 struct mem_cgroup *memcg) 1218 { 1219 return false; 1220 } 1221 1222 static inline void mem_cgroup_commit_charge(struct folio *folio, 1223 struct mem_cgroup *memcg) 1224 { 1225 } 1226 1227 static inline int mem_cgroup_charge(struct folio *folio, 1228 struct mm_struct *mm, gfp_t gfp) 1229 { 1230 return 0; 1231 } 1232 1233 static inline int mem_cgroup_hugetlb_try_charge(struct mem_cgroup *memcg, 1234 gfp_t gfp, long nr_pages) 1235 { 1236 return 0; 1237 } 1238 1239 static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, 1240 struct mm_struct *mm, gfp_t gfp, swp_entry_t entry) 1241 { 1242 return 0; 1243 } 1244 1245 static inline void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry) 1246 { 1247 } 1248 1249 static inline void mem_cgroup_uncharge(struct folio *folio) 1250 { 1251 } 1252 1253 static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) 1254 { 1255 } 1256 1257 static inline void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, 1258 unsigned int nr_pages) 1259 { 1260 } 1261 1262 static inline void mem_cgroup_replace_folio(struct folio *old, 1263 struct folio *new) 1264 { 1265 } 1266 1267 static inline void mem_cgroup_migrate(struct folio *old, struct folio *new) 1268 { 1269 } 1270 1271 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, 1272 struct pglist_data *pgdat) 1273 { 1274 return &pgdat->__lruvec; 1275 } 1276 1277 static inline struct lruvec *folio_lruvec(struct folio *folio) 1278 { 1279 struct pglist_data *pgdat = folio_pgdat(folio); 1280 return &pgdat->__lruvec; 1281 } 1282 1283 static inline 1284 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) 1285 { 1286 } 1287 1288 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) 1289 { 1290 return NULL; 1291 } 1292 1293 static inline bool mm_match_cgroup(struct mm_struct *mm, 1294 struct mem_cgroup *memcg) 1295 { 1296 return true; 1297 } 1298 1299 static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) 1300 { 1301 return NULL; 1302 } 1303 1304 static inline struct mem_cgroup *get_mem_cgroup_from_current(void) 1305 { 1306 return NULL; 1307 } 1308 1309 static inline 1310 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css) 1311 { 1312 return NULL; 1313 } 1314 1315 static inline void obj_cgroup_put(struct obj_cgroup *objcg) 1316 { 1317 } 1318 1319 static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg) 1320 { 1321 return true; 1322 } 1323 1324 static inline bool mem_cgroup_tryget_online(struct mem_cgroup *memcg) 1325 { 1326 return true; 1327 } 1328 1329 static inline void mem_cgroup_put(struct mem_cgroup *memcg) 1330 { 1331 } 1332 1333 static inline struct lruvec *folio_lruvec_lock(struct folio *folio) 1334 { 1335 struct pglist_data *pgdat = folio_pgdat(folio); 1336 1337 spin_lock(&pgdat->__lruvec.lru_lock); 1338 return &pgdat->__lruvec; 1339 } 1340 1341 static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) 1342 { 1343 struct pglist_data *pgdat = folio_pgdat(folio); 1344 1345 spin_lock_irq(&pgdat->__lruvec.lru_lock); 1346 return &pgdat->__lruvec; 1347 } 1348 1349 static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, 1350 unsigned long *flagsp) 1351 { 1352 struct pglist_data *pgdat = folio_pgdat(folio); 1353 1354 spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp); 1355 return &pgdat->__lruvec; 1356 } 1357 1358 static inline struct mem_cgroup * 1359 mem_cgroup_iter(struct mem_cgroup *root, 1360 struct mem_cgroup *prev, 1361 struct mem_cgroup_reclaim_cookie *reclaim) 1362 { 1363 return NULL; 1364 } 1365 1366 static inline void mem_cgroup_iter_break(struct mem_cgroup *root, 1367 struct mem_cgroup *prev) 1368 { 1369 } 1370 1371 static inline void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 1372 int (*fn)(struct task_struct *, void *), void *arg) 1373 { 1374 } 1375 1376 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg) 1377 { 1378 return 0; 1379 } 1380 1381 static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id) 1382 { 1383 WARN_ON_ONCE(id); 1384 /* XXX: This should always return root_mem_cgroup */ 1385 return NULL; 1386 } 1387 1388 #ifdef CONFIG_SHRINKER_DEBUG 1389 static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg) 1390 { 1391 return 0; 1392 } 1393 1394 static inline struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino) 1395 { 1396 return NULL; 1397 } 1398 #endif 1399 1400 static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) 1401 { 1402 return NULL; 1403 } 1404 1405 static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) 1406 { 1407 return NULL; 1408 } 1409 1410 static inline bool mem_cgroup_online(struct mem_cgroup *memcg) 1411 { 1412 return true; 1413 } 1414 1415 static inline 1416 unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, 1417 enum lru_list lru, int zone_idx) 1418 { 1419 return 0; 1420 } 1421 1422 static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) 1423 { 1424 return 0; 1425 } 1426 1427 static inline unsigned long mem_cgroup_size(struct mem_cgroup *memcg) 1428 { 1429 return 0; 1430 } 1431 1432 static inline void 1433 mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) 1434 { 1435 } 1436 1437 static inline void 1438 mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) 1439 { 1440 } 1441 1442 static inline void folio_memcg_lock(struct folio *folio) 1443 { 1444 } 1445 1446 static inline void folio_memcg_unlock(struct folio *folio) 1447 { 1448 } 1449 1450 static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) 1451 { 1452 /* to match folio_memcg_rcu() */ 1453 rcu_read_lock(); 1454 return true; 1455 } 1456 1457 static inline void mem_cgroup_unlock_pages(void) 1458 { 1459 rcu_read_unlock(); 1460 } 1461 1462 static inline void mem_cgroup_handle_over_high(gfp_t gfp_mask) 1463 { 1464 } 1465 1466 static inline void mem_cgroup_enter_user_fault(void) 1467 { 1468 } 1469 1470 static inline void mem_cgroup_exit_user_fault(void) 1471 { 1472 } 1473 1474 static inline bool task_in_memcg_oom(struct task_struct *p) 1475 { 1476 return false; 1477 } 1478 1479 static inline bool mem_cgroup_oom_synchronize(bool wait) 1480 { 1481 return false; 1482 } 1483 1484 static inline struct mem_cgroup *mem_cgroup_get_oom_group( 1485 struct task_struct *victim, struct mem_cgroup *oom_domain) 1486 { 1487 return NULL; 1488 } 1489 1490 static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) 1491 { 1492 } 1493 1494 static inline void __mod_memcg_state(struct mem_cgroup *memcg, 1495 enum memcg_stat_item idx, 1496 int nr) 1497 { 1498 } 1499 1500 static inline void mod_memcg_state(struct mem_cgroup *memcg, 1501 enum memcg_stat_item idx, 1502 int nr) 1503 { 1504 } 1505 1506 static inline void mod_memcg_page_state(struct page *page, 1507 enum memcg_stat_item idx, int val) 1508 { 1509 } 1510 1511 static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) 1512 { 1513 return 0; 1514 } 1515 1516 static inline unsigned long lruvec_page_state(struct lruvec *lruvec, 1517 enum node_stat_item idx) 1518 { 1519 return node_page_state(lruvec_pgdat(lruvec), idx); 1520 } 1521 1522 static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, 1523 enum node_stat_item idx) 1524 { 1525 return node_page_state(lruvec_pgdat(lruvec), idx); 1526 } 1527 1528 static inline void mem_cgroup_flush_stats(struct mem_cgroup *memcg) 1529 { 1530 } 1531 1532 static inline void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg) 1533 { 1534 } 1535 1536 static inline void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, 1537 int val) 1538 { 1539 struct page *page = virt_to_head_page(p); 1540 1541 __mod_node_page_state(page_pgdat(page), idx, val); 1542 } 1543 1544 static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, 1545 int val) 1546 { 1547 struct page *page = virt_to_head_page(p); 1548 1549 mod_node_page_state(page_pgdat(page), idx, val); 1550 } 1551 1552 static inline void count_memcg_events(struct mem_cgroup *memcg, 1553 enum vm_event_item idx, 1554 unsigned long count) 1555 { 1556 } 1557 1558 static inline void __count_memcg_events(struct mem_cgroup *memcg, 1559 enum vm_event_item idx, 1560 unsigned long count) 1561 { 1562 } 1563 1564 static inline void count_memcg_folio_events(struct folio *folio, 1565 enum vm_event_item idx, unsigned long nr) 1566 { 1567 } 1568 1569 static inline 1570 void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx) 1571 { 1572 } 1573 1574 static inline void split_page_memcg(struct page *head, int old_order, int new_order) 1575 { 1576 } 1577 1578 static inline 1579 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 1580 gfp_t gfp_mask, 1581 unsigned long *total_scanned) 1582 { 1583 return 0; 1584 } 1585 #endif /* CONFIG_MEMCG */ 1586 1587 /* 1588 * Extended information for slab objects stored as an array in page->memcg_data 1589 * if MEMCG_DATA_OBJEXTS is set. 1590 */ 1591 struct slabobj_ext { 1592 #ifdef CONFIG_MEMCG_KMEM 1593 struct obj_cgroup *objcg; 1594 #endif 1595 #ifdef CONFIG_MEM_ALLOC_PROFILING 1596 union codetag_ref ref; 1597 #endif 1598 } __aligned(8); 1599 1600 static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx) 1601 { 1602 __mod_lruvec_kmem_state(p, idx, 1); 1603 } 1604 1605 static inline void __dec_lruvec_kmem_state(void *p, enum node_stat_item idx) 1606 { 1607 __mod_lruvec_kmem_state(p, idx, -1); 1608 } 1609 1610 static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) 1611 { 1612 struct mem_cgroup *memcg; 1613 1614 memcg = lruvec_memcg(lruvec); 1615 if (!memcg) 1616 return NULL; 1617 memcg = parent_mem_cgroup(memcg); 1618 if (!memcg) 1619 return NULL; 1620 return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec)); 1621 } 1622 1623 static inline void unlock_page_lruvec(struct lruvec *lruvec) 1624 { 1625 spin_unlock(&lruvec->lru_lock); 1626 } 1627 1628 static inline void unlock_page_lruvec_irq(struct lruvec *lruvec) 1629 { 1630 spin_unlock_irq(&lruvec->lru_lock); 1631 } 1632 1633 static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec, 1634 unsigned long flags) 1635 { 1636 spin_unlock_irqrestore(&lruvec->lru_lock, flags); 1637 } 1638 1639 /* Test requires a stable page->memcg binding, see page_memcg() */ 1640 static inline bool folio_matches_lruvec(struct folio *folio, 1641 struct lruvec *lruvec) 1642 { 1643 return lruvec_pgdat(lruvec) == folio_pgdat(folio) && 1644 lruvec_memcg(lruvec) == folio_memcg(folio); 1645 } 1646 1647 /* Don't lock again iff page's lruvec locked */ 1648 static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, 1649 struct lruvec *locked_lruvec) 1650 { 1651 if (locked_lruvec) { 1652 if (folio_matches_lruvec(folio, locked_lruvec)) 1653 return locked_lruvec; 1654 1655 unlock_page_lruvec_irq(locked_lruvec); 1656 } 1657 1658 return folio_lruvec_lock_irq(folio); 1659 } 1660 1661 /* Don't lock again iff folio's lruvec locked */ 1662 static inline void folio_lruvec_relock_irqsave(struct folio *folio, 1663 struct lruvec **lruvecp, unsigned long *flags) 1664 { 1665 if (*lruvecp) { 1666 if (folio_matches_lruvec(folio, *lruvecp)) 1667 return; 1668 1669 unlock_page_lruvec_irqrestore(*lruvecp, *flags); 1670 } 1671 1672 *lruvecp = folio_lruvec_lock_irqsave(folio, flags); 1673 } 1674 1675 #ifdef CONFIG_CGROUP_WRITEBACK 1676 1677 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb); 1678 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, 1679 unsigned long *pheadroom, unsigned long *pdirty, 1680 unsigned long *pwriteback); 1681 1682 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio, 1683 struct bdi_writeback *wb); 1684 1685 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio, 1686 struct bdi_writeback *wb) 1687 { 1688 struct mem_cgroup *memcg; 1689 1690 if (mem_cgroup_disabled()) 1691 return; 1692 1693 memcg = folio_memcg(folio); 1694 if (unlikely(memcg && &memcg->css != wb->memcg_css)) 1695 mem_cgroup_track_foreign_dirty_slowpath(folio, wb); 1696 } 1697 1698 void mem_cgroup_flush_foreign(struct bdi_writeback *wb); 1699 1700 #else /* CONFIG_CGROUP_WRITEBACK */ 1701 1702 static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) 1703 { 1704 return NULL; 1705 } 1706 1707 static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb, 1708 unsigned long *pfilepages, 1709 unsigned long *pheadroom, 1710 unsigned long *pdirty, 1711 unsigned long *pwriteback) 1712 { 1713 } 1714 1715 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio, 1716 struct bdi_writeback *wb) 1717 { 1718 } 1719 1720 static inline void mem_cgroup_flush_foreign(struct bdi_writeback *wb) 1721 { 1722 } 1723 1724 #endif /* CONFIG_CGROUP_WRITEBACK */ 1725 1726 struct sock; 1727 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages, 1728 gfp_t gfp_mask); 1729 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages); 1730 #ifdef CONFIG_MEMCG 1731 extern struct static_key_false memcg_sockets_enabled_key; 1732 #define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key) 1733 void mem_cgroup_sk_alloc(struct sock *sk); 1734 void mem_cgroup_sk_free(struct sock *sk); 1735 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg) 1736 { 1737 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 1738 return !!memcg->tcpmem_pressure; 1739 do { 1740 if (time_before(jiffies, READ_ONCE(memcg->socket_pressure))) 1741 return true; 1742 } while ((memcg = parent_mem_cgroup(memcg))); 1743 return false; 1744 } 1745 1746 int alloc_shrinker_info(struct mem_cgroup *memcg); 1747 void free_shrinker_info(struct mem_cgroup *memcg); 1748 void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id); 1749 void reparent_shrinker_deferred(struct mem_cgroup *memcg); 1750 #else 1751 #define mem_cgroup_sockets_enabled 0 1752 static inline void mem_cgroup_sk_alloc(struct sock *sk) { }; 1753 static inline void mem_cgroup_sk_free(struct sock *sk) { }; 1754 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg) 1755 { 1756 return false; 1757 } 1758 1759 static inline void set_shrinker_bit(struct mem_cgroup *memcg, 1760 int nid, int shrinker_id) 1761 { 1762 } 1763 #endif 1764 1765 #ifdef CONFIG_MEMCG_KMEM 1766 bool mem_cgroup_kmem_disabled(void); 1767 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order); 1768 void __memcg_kmem_uncharge_page(struct page *page, int order); 1769 1770 /* 1771 * The returned objcg pointer is safe to use without additional 1772 * protection within a scope. The scope is defined either by 1773 * the current task (similar to the "current" global variable) 1774 * or by set_active_memcg() pair. 1775 * Please, use obj_cgroup_get() to get a reference if the pointer 1776 * needs to be used outside of the local scope. 1777 */ 1778 struct obj_cgroup *current_obj_cgroup(void); 1779 struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio); 1780 1781 static inline struct obj_cgroup *get_obj_cgroup_from_current(void) 1782 { 1783 struct obj_cgroup *objcg = current_obj_cgroup(); 1784 1785 if (objcg) 1786 obj_cgroup_get(objcg); 1787 1788 return objcg; 1789 } 1790 1791 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size); 1792 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size); 1793 1794 extern struct static_key_false memcg_bpf_enabled_key; 1795 static inline bool memcg_bpf_enabled(void) 1796 { 1797 return static_branch_likely(&memcg_bpf_enabled_key); 1798 } 1799 1800 extern struct static_key_false memcg_kmem_online_key; 1801 1802 static inline bool memcg_kmem_online(void) 1803 { 1804 return static_branch_likely(&memcg_kmem_online_key); 1805 } 1806 1807 static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp, 1808 int order) 1809 { 1810 if (memcg_kmem_online()) 1811 return __memcg_kmem_charge_page(page, gfp, order); 1812 return 0; 1813 } 1814 1815 static inline void memcg_kmem_uncharge_page(struct page *page, int order) 1816 { 1817 if (memcg_kmem_online()) 1818 __memcg_kmem_uncharge_page(page, order); 1819 } 1820 1821 /* 1822 * A helper for accessing memcg's kmem_id, used for getting 1823 * corresponding LRU lists. 1824 */ 1825 static inline int memcg_kmem_id(struct mem_cgroup *memcg) 1826 { 1827 return memcg ? memcg->kmemcg_id : -1; 1828 } 1829 1830 struct mem_cgroup *mem_cgroup_from_obj(void *p); 1831 struct mem_cgroup *mem_cgroup_from_slab_obj(void *p); 1832 1833 static inline void count_objcg_event(struct obj_cgroup *objcg, 1834 enum vm_event_item idx) 1835 { 1836 struct mem_cgroup *memcg; 1837 1838 if (!memcg_kmem_online()) 1839 return; 1840 1841 rcu_read_lock(); 1842 memcg = obj_cgroup_memcg(objcg); 1843 count_memcg_events(memcg, idx, 1); 1844 rcu_read_unlock(); 1845 } 1846 1847 #else 1848 static inline bool mem_cgroup_kmem_disabled(void) 1849 { 1850 return true; 1851 } 1852 1853 static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp, 1854 int order) 1855 { 1856 return 0; 1857 } 1858 1859 static inline void memcg_kmem_uncharge_page(struct page *page, int order) 1860 { 1861 } 1862 1863 static inline int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, 1864 int order) 1865 { 1866 return 0; 1867 } 1868 1869 static inline void __memcg_kmem_uncharge_page(struct page *page, int order) 1870 { 1871 } 1872 1873 static inline struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio) 1874 { 1875 return NULL; 1876 } 1877 1878 static inline bool memcg_bpf_enabled(void) 1879 { 1880 return false; 1881 } 1882 1883 static inline bool memcg_kmem_online(void) 1884 { 1885 return false; 1886 } 1887 1888 static inline int memcg_kmem_id(struct mem_cgroup *memcg) 1889 { 1890 return -1; 1891 } 1892 1893 static inline struct mem_cgroup *mem_cgroup_from_obj(void *p) 1894 { 1895 return NULL; 1896 } 1897 1898 static inline struct mem_cgroup *mem_cgroup_from_slab_obj(void *p) 1899 { 1900 return NULL; 1901 } 1902 1903 static inline void count_objcg_event(struct obj_cgroup *objcg, 1904 enum vm_event_item idx) 1905 { 1906 } 1907 1908 #endif /* CONFIG_MEMCG_KMEM */ 1909 1910 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 1911 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg); 1912 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size); 1913 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size); 1914 bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg); 1915 #else 1916 static inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg) 1917 { 1918 return true; 1919 } 1920 static inline void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, 1921 size_t size) 1922 { 1923 } 1924 static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, 1925 size_t size) 1926 { 1927 } 1928 static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) 1929 { 1930 /* if zswap is disabled, do not block pages going to the swapping device */ 1931 return true; 1932 } 1933 #endif 1934 1935 #endif /* _LINUX_MEMCONTROL_H */ 1936