xref: /linux/mm/memcontrol-v1.c (revision f858cc9eed5b05cbe38d7ffd2787c21e3718eb7d)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 
3 #include <linux/memcontrol.h>
4 #include <linux/swap.h>
5 #include <linux/mm_inline.h>
6 #include <linux/pagewalk.h>
7 #include <linux/backing-dev.h>
8 #include <linux/swap_cgroup.h>
9 #include <linux/eventfd.h>
10 #include <linux/poll.h>
11 #include <linux/sort.h>
12 #include <linux/file.h>
13 #include <linux/seq_buf.h>
14 
15 #include "internal.h"
16 #include "swap.h"
17 #include "memcontrol-v1.h"
18 
19 /*
20  * Cgroups above their limits are maintained in a RB-Tree, independent of
21  * their hierarchy representation
22  */
23 
24 struct mem_cgroup_tree_per_node {
25 	struct rb_root rb_root;
26 	struct rb_node *rb_rightmost;
27 	spinlock_t lock;
28 };
29 
30 struct mem_cgroup_tree {
31 	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
32 };
33 
34 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
35 
36 /*
37  * Maximum loops in mem_cgroup_soft_reclaim(), used for soft
38  * limit reclaim to prevent infinite loops, if they ever occur.
39  */
40 #define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
41 #define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
42 
43 /* Stuffs for move charges at task migration. */
44 /*
45  * Types of charges to be moved.
46  */
47 #define MOVE_ANON	0x1ULL
48 #define MOVE_FILE	0x2ULL
49 #define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
50 
51 /* "mc" and its members are protected by cgroup_mutex */
52 static struct move_charge_struct {
53 	spinlock_t	  lock; /* for from, to */
54 	struct mm_struct  *mm;
55 	struct mem_cgroup *from;
56 	struct mem_cgroup *to;
57 	unsigned long flags;
58 	unsigned long precharge;
59 	unsigned long moved_charge;
60 	unsigned long moved_swap;
61 	struct task_struct *moving_task;	/* a task moving charges */
62 	wait_queue_head_t waitq;		/* a waitq for other context */
63 } mc = {
64 	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
65 	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
66 };
67 
68 /* for OOM */
69 struct mem_cgroup_eventfd_list {
70 	struct list_head list;
71 	struct eventfd_ctx *eventfd;
72 };
73 
74 /*
75  * cgroup_event represents events which userspace want to receive.
76  */
77 struct mem_cgroup_event {
78 	/*
79 	 * memcg which the event belongs to.
80 	 */
81 	struct mem_cgroup *memcg;
82 	/*
83 	 * eventfd to signal userspace about the event.
84 	 */
85 	struct eventfd_ctx *eventfd;
86 	/*
87 	 * Each of these stored in a list by the cgroup.
88 	 */
89 	struct list_head list;
90 	/*
91 	 * register_event() callback will be used to add new userspace
92 	 * waiter for changes related to this event.  Use eventfd_signal()
93 	 * on eventfd to send notification to userspace.
94 	 */
95 	int (*register_event)(struct mem_cgroup *memcg,
96 			      struct eventfd_ctx *eventfd, const char *args);
97 	/*
98 	 * unregister_event() callback will be called when userspace closes
99 	 * the eventfd or on cgroup removing.  This callback must be set,
100 	 * if you want provide notification functionality.
101 	 */
102 	void (*unregister_event)(struct mem_cgroup *memcg,
103 				 struct eventfd_ctx *eventfd);
104 	/*
105 	 * All fields below needed to unregister event when
106 	 * userspace closes eventfd.
107 	 */
108 	poll_table pt;
109 	wait_queue_head_t *wqh;
110 	wait_queue_entry_t wait;
111 	struct work_struct remove;
112 };
113 
114 #define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
115 #define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
116 #define MEMFILE_ATTR(val)	((val) & 0xffff)
117 
118 enum {
119 	RES_USAGE,
120 	RES_LIMIT,
121 	RES_MAX_USAGE,
122 	RES_FAILCNT,
123 	RES_SOFT_LIMIT,
124 };
125 
126 #ifdef CONFIG_LOCKDEP
127 static struct lockdep_map memcg_oom_lock_dep_map = {
128 	.name = "memcg_oom_lock",
129 };
130 #endif
131 
132 DEFINE_SPINLOCK(memcg_oom_lock);
133 
134 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
135 					 struct mem_cgroup_tree_per_node *mctz,
136 					 unsigned long new_usage_in_excess)
137 {
138 	struct rb_node **p = &mctz->rb_root.rb_node;
139 	struct rb_node *parent = NULL;
140 	struct mem_cgroup_per_node *mz_node;
141 	bool rightmost = true;
142 
143 	if (mz->on_tree)
144 		return;
145 
146 	mz->usage_in_excess = new_usage_in_excess;
147 	if (!mz->usage_in_excess)
148 		return;
149 	while (*p) {
150 		parent = *p;
151 		mz_node = rb_entry(parent, struct mem_cgroup_per_node,
152 					tree_node);
153 		if (mz->usage_in_excess < mz_node->usage_in_excess) {
154 			p = &(*p)->rb_left;
155 			rightmost = false;
156 		} else {
157 			p = &(*p)->rb_right;
158 		}
159 	}
160 
161 	if (rightmost)
162 		mctz->rb_rightmost = &mz->tree_node;
163 
164 	rb_link_node(&mz->tree_node, parent, p);
165 	rb_insert_color(&mz->tree_node, &mctz->rb_root);
166 	mz->on_tree = true;
167 }
168 
169 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
170 					 struct mem_cgroup_tree_per_node *mctz)
171 {
172 	if (!mz->on_tree)
173 		return;
174 
175 	if (&mz->tree_node == mctz->rb_rightmost)
176 		mctz->rb_rightmost = rb_prev(&mz->tree_node);
177 
178 	rb_erase(&mz->tree_node, &mctz->rb_root);
179 	mz->on_tree = false;
180 }
181 
182 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
183 				       struct mem_cgroup_tree_per_node *mctz)
184 {
185 	unsigned long flags;
186 
187 	spin_lock_irqsave(&mctz->lock, flags);
188 	__mem_cgroup_remove_exceeded(mz, mctz);
189 	spin_unlock_irqrestore(&mctz->lock, flags);
190 }
191 
192 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
193 {
194 	unsigned long nr_pages = page_counter_read(&memcg->memory);
195 	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
196 	unsigned long excess = 0;
197 
198 	if (nr_pages > soft_limit)
199 		excess = nr_pages - soft_limit;
200 
201 	return excess;
202 }
203 
204 static void memcg1_update_tree(struct mem_cgroup *memcg, int nid)
205 {
206 	unsigned long excess;
207 	struct mem_cgroup_per_node *mz;
208 	struct mem_cgroup_tree_per_node *mctz;
209 
210 	if (lru_gen_enabled()) {
211 		if (soft_limit_excess(memcg))
212 			lru_gen_soft_reclaim(memcg, nid);
213 		return;
214 	}
215 
216 	mctz = soft_limit_tree.rb_tree_per_node[nid];
217 	if (!mctz)
218 		return;
219 	/*
220 	 * Necessary to update all ancestors when hierarchy is used.
221 	 * because their event counter is not touched.
222 	 */
223 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
224 		mz = memcg->nodeinfo[nid];
225 		excess = soft_limit_excess(memcg);
226 		/*
227 		 * We have to update the tree if mz is on RB-tree or
228 		 * mem is over its softlimit.
229 		 */
230 		if (excess || mz->on_tree) {
231 			unsigned long flags;
232 
233 			spin_lock_irqsave(&mctz->lock, flags);
234 			/* if on-tree, remove it */
235 			if (mz->on_tree)
236 				__mem_cgroup_remove_exceeded(mz, mctz);
237 			/*
238 			 * Insert again. mz->usage_in_excess will be updated.
239 			 * If excess is 0, no tree ops.
240 			 */
241 			__mem_cgroup_insert_exceeded(mz, mctz, excess);
242 			spin_unlock_irqrestore(&mctz->lock, flags);
243 		}
244 	}
245 }
246 
247 void memcg1_remove_from_trees(struct mem_cgroup *memcg)
248 {
249 	struct mem_cgroup_tree_per_node *mctz;
250 	struct mem_cgroup_per_node *mz;
251 	int nid;
252 
253 	for_each_node(nid) {
254 		mz = memcg->nodeinfo[nid];
255 		mctz = soft_limit_tree.rb_tree_per_node[nid];
256 		if (mctz)
257 			mem_cgroup_remove_exceeded(mz, mctz);
258 	}
259 }
260 
261 static struct mem_cgroup_per_node *
262 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
263 {
264 	struct mem_cgroup_per_node *mz;
265 
266 retry:
267 	mz = NULL;
268 	if (!mctz->rb_rightmost)
269 		goto done;		/* Nothing to reclaim from */
270 
271 	mz = rb_entry(mctz->rb_rightmost,
272 		      struct mem_cgroup_per_node, tree_node);
273 	/*
274 	 * Remove the node now but someone else can add it back,
275 	 * we will to add it back at the end of reclaim to its correct
276 	 * position in the tree.
277 	 */
278 	__mem_cgroup_remove_exceeded(mz, mctz);
279 	if (!soft_limit_excess(mz->memcg) ||
280 	    !css_tryget(&mz->memcg->css))
281 		goto retry;
282 done:
283 	return mz;
284 }
285 
286 static struct mem_cgroup_per_node *
287 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
288 {
289 	struct mem_cgroup_per_node *mz;
290 
291 	spin_lock_irq(&mctz->lock);
292 	mz = __mem_cgroup_largest_soft_limit_node(mctz);
293 	spin_unlock_irq(&mctz->lock);
294 	return mz;
295 }
296 
297 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
298 				   pg_data_t *pgdat,
299 				   gfp_t gfp_mask,
300 				   unsigned long *total_scanned)
301 {
302 	struct mem_cgroup *victim = NULL;
303 	int total = 0;
304 	int loop = 0;
305 	unsigned long excess;
306 	unsigned long nr_scanned;
307 	struct mem_cgroup_reclaim_cookie reclaim = {
308 		.pgdat = pgdat,
309 	};
310 
311 	excess = soft_limit_excess(root_memcg);
312 
313 	while (1) {
314 		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
315 		if (!victim) {
316 			loop++;
317 			if (loop >= 2) {
318 				/*
319 				 * If we have not been able to reclaim
320 				 * anything, it might because there are
321 				 * no reclaimable pages under this hierarchy
322 				 */
323 				if (!total)
324 					break;
325 				/*
326 				 * We want to do more targeted reclaim.
327 				 * excess >> 2 is not to excessive so as to
328 				 * reclaim too much, nor too less that we keep
329 				 * coming back to reclaim from this cgroup
330 				 */
331 				if (total >= (excess >> 2) ||
332 					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
333 					break;
334 			}
335 			continue;
336 		}
337 		total += mem_cgroup_shrink_node(victim, gfp_mask, false,
338 					pgdat, &nr_scanned);
339 		*total_scanned += nr_scanned;
340 		if (!soft_limit_excess(root_memcg))
341 			break;
342 	}
343 	mem_cgroup_iter_break(root_memcg, victim);
344 	return total;
345 }
346 
347 unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order,
348 					    gfp_t gfp_mask,
349 					    unsigned long *total_scanned)
350 {
351 	unsigned long nr_reclaimed = 0;
352 	struct mem_cgroup_per_node *mz, *next_mz = NULL;
353 	unsigned long reclaimed;
354 	int loop = 0;
355 	struct mem_cgroup_tree_per_node *mctz;
356 	unsigned long excess;
357 
358 	if (lru_gen_enabled())
359 		return 0;
360 
361 	if (order > 0)
362 		return 0;
363 
364 	mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id];
365 
366 	/*
367 	 * Do not even bother to check the largest node if the root
368 	 * is empty. Do it lockless to prevent lock bouncing. Races
369 	 * are acceptable as soft limit is best effort anyway.
370 	 */
371 	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
372 		return 0;
373 
374 	/*
375 	 * This loop can run a while, specially if mem_cgroup's continuously
376 	 * keep exceeding their soft limit and putting the system under
377 	 * pressure
378 	 */
379 	do {
380 		if (next_mz)
381 			mz = next_mz;
382 		else
383 			mz = mem_cgroup_largest_soft_limit_node(mctz);
384 		if (!mz)
385 			break;
386 
387 		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
388 						    gfp_mask, total_scanned);
389 		nr_reclaimed += reclaimed;
390 		spin_lock_irq(&mctz->lock);
391 
392 		/*
393 		 * If we failed to reclaim anything from this memory cgroup
394 		 * it is time to move on to the next cgroup
395 		 */
396 		next_mz = NULL;
397 		if (!reclaimed)
398 			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
399 
400 		excess = soft_limit_excess(mz->memcg);
401 		/*
402 		 * One school of thought says that we should not add
403 		 * back the node to the tree if reclaim returns 0.
404 		 * But our reclaim could return 0, simply because due
405 		 * to priority we are exposing a smaller subset of
406 		 * memory to reclaim from. Consider this as a longer
407 		 * term TODO.
408 		 */
409 		/* If excess == 0, no tree ops */
410 		__mem_cgroup_insert_exceeded(mz, mctz, excess);
411 		spin_unlock_irq(&mctz->lock);
412 		css_put(&mz->memcg->css);
413 		loop++;
414 		/*
415 		 * Could not reclaim anything and there are no more
416 		 * mem cgroups to try or we seem to be looping without
417 		 * reclaiming anything.
418 		 */
419 		if (!nr_reclaimed &&
420 			(next_mz == NULL ||
421 			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
422 			break;
423 	} while (!nr_reclaimed);
424 	if (next_mz)
425 		css_put(&next_mz->memcg->css);
426 	return nr_reclaimed;
427 }
428 
429 /*
430  * A routine for checking "mem" is under move_account() or not.
431  *
432  * Checking a cgroup is mc.from or mc.to or under hierarchy of
433  * moving cgroups. This is for waiting at high-memory pressure
434  * caused by "move".
435  */
436 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
437 {
438 	struct mem_cgroup *from;
439 	struct mem_cgroup *to;
440 	bool ret = false;
441 	/*
442 	 * Unlike task_move routines, we access mc.to, mc.from not under
443 	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
444 	 */
445 	spin_lock(&mc.lock);
446 	from = mc.from;
447 	to = mc.to;
448 	if (!from)
449 		goto unlock;
450 
451 	ret = mem_cgroup_is_descendant(from, memcg) ||
452 		mem_cgroup_is_descendant(to, memcg);
453 unlock:
454 	spin_unlock(&mc.lock);
455 	return ret;
456 }
457 
458 bool memcg1_wait_acct_move(struct mem_cgroup *memcg)
459 {
460 	if (mc.moving_task && current != mc.moving_task) {
461 		if (mem_cgroup_under_move(memcg)) {
462 			DEFINE_WAIT(wait);
463 			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
464 			/* moving charge context might have finished. */
465 			if (mc.moving_task)
466 				schedule();
467 			finish_wait(&mc.waitq, &wait);
468 			return true;
469 		}
470 	}
471 	return false;
472 }
473 
474 /**
475  * folio_memcg_lock - Bind a folio to its memcg.
476  * @folio: The folio.
477  *
478  * This function prevents unlocked LRU folios from being moved to
479  * another cgroup.
480  *
481  * It ensures lifetime of the bound memcg.  The caller is responsible
482  * for the lifetime of the folio.
483  */
484 void folio_memcg_lock(struct folio *folio)
485 {
486 	struct mem_cgroup *memcg;
487 	unsigned long flags;
488 
489 	/*
490 	 * The RCU lock is held throughout the transaction.  The fast
491 	 * path can get away without acquiring the memcg->move_lock
492 	 * because page moving starts with an RCU grace period.
493          */
494 	rcu_read_lock();
495 
496 	if (mem_cgroup_disabled())
497 		return;
498 again:
499 	memcg = folio_memcg(folio);
500 	if (unlikely(!memcg))
501 		return;
502 
503 #ifdef CONFIG_PROVE_LOCKING
504 	local_irq_save(flags);
505 	might_lock(&memcg->move_lock);
506 	local_irq_restore(flags);
507 #endif
508 
509 	if (atomic_read(&memcg->moving_account) <= 0)
510 		return;
511 
512 	spin_lock_irqsave(&memcg->move_lock, flags);
513 	if (memcg != folio_memcg(folio)) {
514 		spin_unlock_irqrestore(&memcg->move_lock, flags);
515 		goto again;
516 	}
517 
518 	/*
519 	 * When charge migration first begins, we can have multiple
520 	 * critical sections holding the fast-path RCU lock and one
521 	 * holding the slowpath move_lock. Track the task who has the
522 	 * move_lock for folio_memcg_unlock().
523 	 */
524 	memcg->move_lock_task = current;
525 	memcg->move_lock_flags = flags;
526 }
527 
528 static void __folio_memcg_unlock(struct mem_cgroup *memcg)
529 {
530 	if (memcg && memcg->move_lock_task == current) {
531 		unsigned long flags = memcg->move_lock_flags;
532 
533 		memcg->move_lock_task = NULL;
534 		memcg->move_lock_flags = 0;
535 
536 		spin_unlock_irqrestore(&memcg->move_lock, flags);
537 	}
538 
539 	rcu_read_unlock();
540 }
541 
542 /**
543  * folio_memcg_unlock - Release the binding between a folio and its memcg.
544  * @folio: The folio.
545  *
546  * This releases the binding created by folio_memcg_lock().  This does
547  * not change the accounting of this folio to its memcg, but it does
548  * permit others to change it.
549  */
550 void folio_memcg_unlock(struct folio *folio)
551 {
552 	__folio_memcg_unlock(folio_memcg(folio));
553 }
554 
555 #ifdef CONFIG_SWAP
556 /**
557  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
558  * @entry: swap entry to be moved
559  * @from:  mem_cgroup which the entry is moved from
560  * @to:  mem_cgroup which the entry is moved to
561  *
562  * It succeeds only when the swap_cgroup's record for this entry is the same
563  * as the mem_cgroup's id of @from.
564  *
565  * Returns 0 on success, -EINVAL on failure.
566  *
567  * The caller must have charged to @to, IOW, called page_counter_charge() about
568  * both res and memsw, and called css_get().
569  */
570 static int mem_cgroup_move_swap_account(swp_entry_t entry,
571 				struct mem_cgroup *from, struct mem_cgroup *to)
572 {
573 	unsigned short old_id, new_id;
574 
575 	old_id = mem_cgroup_id(from);
576 	new_id = mem_cgroup_id(to);
577 
578 	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
579 		mod_memcg_state(from, MEMCG_SWAP, -1);
580 		mod_memcg_state(to, MEMCG_SWAP, 1);
581 		return 0;
582 	}
583 	return -EINVAL;
584 }
585 #else
586 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
587 				struct mem_cgroup *from, struct mem_cgroup *to)
588 {
589 	return -EINVAL;
590 }
591 #endif
592 
593 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
594 				struct cftype *cft)
595 {
596 	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
597 }
598 
599 #ifdef CONFIG_MMU
600 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
601 				 struct cftype *cft, u64 val)
602 {
603 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
604 
605 	pr_warn_once("Cgroup memory moving (move_charge_at_immigrate) is deprecated. "
606 		     "Please report your usecase to linux-mm@kvack.org if you "
607 		     "depend on this functionality.\n");
608 
609 	if (val & ~MOVE_MASK)
610 		return -EINVAL;
611 
612 	/*
613 	 * No kind of locking is needed in here, because ->can_attach() will
614 	 * check this value once in the beginning of the process, and then carry
615 	 * on with stale data. This means that changes to this value will only
616 	 * affect task migrations starting after the change.
617 	 */
618 	memcg->move_charge_at_immigrate = val;
619 	return 0;
620 }
621 #else
622 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
623 				 struct cftype *cft, u64 val)
624 {
625 	return -ENOSYS;
626 }
627 #endif
628 
629 #ifdef CONFIG_MMU
630 /* Handlers for move charge at task migration. */
631 static int mem_cgroup_do_precharge(unsigned long count)
632 {
633 	int ret;
634 
635 	/* Try a single bulk charge without reclaim first, kswapd may wake */
636 	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
637 	if (!ret) {
638 		mc.precharge += count;
639 		return ret;
640 	}
641 
642 	/* Try charges one by one with reclaim, but do not retry */
643 	while (count--) {
644 		ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
645 		if (ret)
646 			return ret;
647 		mc.precharge++;
648 		cond_resched();
649 	}
650 	return 0;
651 }
652 
653 union mc_target {
654 	struct folio	*folio;
655 	swp_entry_t	ent;
656 };
657 
658 enum mc_target_type {
659 	MC_TARGET_NONE = 0,
660 	MC_TARGET_PAGE,
661 	MC_TARGET_SWAP,
662 	MC_TARGET_DEVICE,
663 };
664 
665 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
666 						unsigned long addr, pte_t ptent)
667 {
668 	struct page *page = vm_normal_page(vma, addr, ptent);
669 
670 	if (!page)
671 		return NULL;
672 	if (PageAnon(page)) {
673 		if (!(mc.flags & MOVE_ANON))
674 			return NULL;
675 	} else {
676 		if (!(mc.flags & MOVE_FILE))
677 			return NULL;
678 	}
679 	get_page(page);
680 
681 	return page;
682 }
683 
684 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
685 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
686 			pte_t ptent, swp_entry_t *entry)
687 {
688 	struct page *page = NULL;
689 	swp_entry_t ent = pte_to_swp_entry(ptent);
690 
691 	if (!(mc.flags & MOVE_ANON))
692 		return NULL;
693 
694 	/*
695 	 * Handle device private pages that are not accessible by the CPU, but
696 	 * stored as special swap entries in the page table.
697 	 */
698 	if (is_device_private_entry(ent)) {
699 		page = pfn_swap_entry_to_page(ent);
700 		if (!get_page_unless_zero(page))
701 			return NULL;
702 		return page;
703 	}
704 
705 	if (non_swap_entry(ent))
706 		return NULL;
707 
708 	/*
709 	 * Because swap_cache_get_folio() updates some statistics counter,
710 	 * we call find_get_page() with swapper_space directly.
711 	 */
712 	page = find_get_page(swap_address_space(ent), swap_cache_index(ent));
713 	entry->val = ent.val;
714 
715 	return page;
716 }
717 #else
718 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
719 			pte_t ptent, swp_entry_t *entry)
720 {
721 	return NULL;
722 }
723 #endif
724 
725 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
726 			unsigned long addr, pte_t ptent)
727 {
728 	unsigned long index;
729 	struct folio *folio;
730 
731 	if (!vma->vm_file) /* anonymous vma */
732 		return NULL;
733 	if (!(mc.flags & MOVE_FILE))
734 		return NULL;
735 
736 	/* folio is moved even if it's not RSS of this task(page-faulted). */
737 	/* shmem/tmpfs may report page out on swap: account for that too. */
738 	index = linear_page_index(vma, addr);
739 	folio = filemap_get_incore_folio(vma->vm_file->f_mapping, index);
740 	if (IS_ERR(folio))
741 		return NULL;
742 	return folio_file_page(folio, index);
743 }
744 
745 static void memcg1_check_events(struct mem_cgroup *memcg, int nid);
746 static void memcg1_charge_statistics(struct mem_cgroup *memcg, int nr_pages);
747 
748 /**
749  * mem_cgroup_move_account - move account of the folio
750  * @folio: The folio.
751  * @compound: charge the page as compound or small page
752  * @from: mem_cgroup which the folio is moved from.
753  * @to:	mem_cgroup which the folio is moved to. @from != @to.
754  *
755  * The folio must be locked and not on the LRU.
756  *
757  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
758  * from old cgroup.
759  */
760 static int mem_cgroup_move_account(struct folio *folio,
761 				   bool compound,
762 				   struct mem_cgroup *from,
763 				   struct mem_cgroup *to)
764 {
765 	struct lruvec *from_vec, *to_vec;
766 	struct pglist_data *pgdat;
767 	unsigned int nr_pages = compound ? folio_nr_pages(folio) : 1;
768 	int nid, ret;
769 
770 	VM_BUG_ON(from == to);
771 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
772 	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
773 	VM_BUG_ON(compound && !folio_test_large(folio));
774 
775 	ret = -EINVAL;
776 	if (folio_memcg(folio) != from)
777 		goto out;
778 
779 	pgdat = folio_pgdat(folio);
780 	from_vec = mem_cgroup_lruvec(from, pgdat);
781 	to_vec = mem_cgroup_lruvec(to, pgdat);
782 
783 	folio_memcg_lock(folio);
784 
785 	if (folio_test_anon(folio)) {
786 		if (folio_mapped(folio)) {
787 			__mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
788 			__mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
789 			if (folio_test_pmd_mappable(folio)) {
790 				__mod_lruvec_state(from_vec, NR_ANON_THPS,
791 						   -nr_pages);
792 				__mod_lruvec_state(to_vec, NR_ANON_THPS,
793 						   nr_pages);
794 			}
795 		}
796 	} else {
797 		__mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
798 		__mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
799 
800 		if (folio_test_swapbacked(folio)) {
801 			__mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
802 			__mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
803 		}
804 
805 		if (folio_mapped(folio)) {
806 			__mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
807 			__mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
808 		}
809 
810 		if (folio_test_dirty(folio)) {
811 			struct address_space *mapping = folio_mapping(folio);
812 
813 			if (mapping_can_writeback(mapping)) {
814 				__mod_lruvec_state(from_vec, NR_FILE_DIRTY,
815 						   -nr_pages);
816 				__mod_lruvec_state(to_vec, NR_FILE_DIRTY,
817 						   nr_pages);
818 			}
819 		}
820 	}
821 
822 #ifdef CONFIG_SWAP
823 	if (folio_test_swapcache(folio)) {
824 		__mod_lruvec_state(from_vec, NR_SWAPCACHE, -nr_pages);
825 		__mod_lruvec_state(to_vec, NR_SWAPCACHE, nr_pages);
826 	}
827 #endif
828 	if (folio_test_writeback(folio)) {
829 		__mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
830 		__mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
831 	}
832 
833 	/*
834 	 * All state has been migrated, let's switch to the new memcg.
835 	 *
836 	 * It is safe to change page's memcg here because the page
837 	 * is referenced, charged, isolated, and locked: we can't race
838 	 * with (un)charging, migration, LRU putback, or anything else
839 	 * that would rely on a stable page's memory cgroup.
840 	 *
841 	 * Note that folio_memcg_lock is a memcg lock, not a page lock,
842 	 * to save space. As soon as we switch page's memory cgroup to a
843 	 * new memcg that isn't locked, the above state can change
844 	 * concurrently again. Make sure we're truly done with it.
845 	 */
846 	smp_mb();
847 
848 	css_get(&to->css);
849 	css_put(&from->css);
850 
851 	folio->memcg_data = (unsigned long)to;
852 
853 	__folio_memcg_unlock(from);
854 
855 	ret = 0;
856 	nid = folio_nid(folio);
857 
858 	local_irq_disable();
859 	memcg1_charge_statistics(to, nr_pages);
860 	memcg1_check_events(to, nid);
861 	memcg1_charge_statistics(from, -nr_pages);
862 	memcg1_check_events(from, nid);
863 	local_irq_enable();
864 out:
865 	return ret;
866 }
867 
868 /**
869  * get_mctgt_type - get target type of moving charge
870  * @vma: the vma the pte to be checked belongs
871  * @addr: the address corresponding to the pte to be checked
872  * @ptent: the pte to be checked
873  * @target: the pointer the target page or swap ent will be stored(can be NULL)
874  *
875  * Context: Called with pte lock held.
876  * Return:
877  * * MC_TARGET_NONE - If the pte is not a target for move charge.
878  * * MC_TARGET_PAGE - If the page corresponding to this pte is a target for
879  *   move charge. If @target is not NULL, the folio is stored in target->folio
880  *   with extra refcnt taken (Caller should release it).
881  * * MC_TARGET_SWAP - If the swap entry corresponding to this pte is a
882  *   target for charge migration.  If @target is not NULL, the entry is
883  *   stored in target->ent.
884  * * MC_TARGET_DEVICE - Like MC_TARGET_PAGE but page is device memory and
885  *   thus not on the lru.  For now such page is charged like a regular page
886  *   would be as it is just special memory taking the place of a regular page.
887  *   See Documentations/vm/hmm.txt and include/linux/hmm.h
888  */
889 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
890 		unsigned long addr, pte_t ptent, union mc_target *target)
891 {
892 	struct page *page = NULL;
893 	struct folio *folio;
894 	enum mc_target_type ret = MC_TARGET_NONE;
895 	swp_entry_t ent = { .val = 0 };
896 
897 	if (pte_present(ptent))
898 		page = mc_handle_present_pte(vma, addr, ptent);
899 	else if (pte_none_mostly(ptent))
900 		/*
901 		 * PTE markers should be treated as a none pte here, separated
902 		 * from other swap handling below.
903 		 */
904 		page = mc_handle_file_pte(vma, addr, ptent);
905 	else if (is_swap_pte(ptent))
906 		page = mc_handle_swap_pte(vma, ptent, &ent);
907 
908 	if (page)
909 		folio = page_folio(page);
910 	if (target && page) {
911 		if (!folio_trylock(folio)) {
912 			folio_put(folio);
913 			return ret;
914 		}
915 		/*
916 		 * page_mapped() must be stable during the move. This
917 		 * pte is locked, so if it's present, the page cannot
918 		 * become unmapped. If it isn't, we have only partial
919 		 * control over the mapped state: the page lock will
920 		 * prevent new faults against pagecache and swapcache,
921 		 * so an unmapped page cannot become mapped. However,
922 		 * if the page is already mapped elsewhere, it can
923 		 * unmap, and there is nothing we can do about it.
924 		 * Alas, skip moving the page in this case.
925 		 */
926 		if (!pte_present(ptent) && page_mapped(page)) {
927 			folio_unlock(folio);
928 			folio_put(folio);
929 			return ret;
930 		}
931 	}
932 
933 	if (!page && !ent.val)
934 		return ret;
935 	if (page) {
936 		/*
937 		 * Do only loose check w/o serialization.
938 		 * mem_cgroup_move_account() checks the page is valid or
939 		 * not under LRU exclusion.
940 		 */
941 		if (folio_memcg(folio) == mc.from) {
942 			ret = MC_TARGET_PAGE;
943 			if (folio_is_device_private(folio) ||
944 			    folio_is_device_coherent(folio))
945 				ret = MC_TARGET_DEVICE;
946 			if (target)
947 				target->folio = folio;
948 		}
949 		if (!ret || !target) {
950 			if (target)
951 				folio_unlock(folio);
952 			folio_put(folio);
953 		}
954 	}
955 	/*
956 	 * There is a swap entry and a page doesn't exist or isn't charged.
957 	 * But we cannot move a tail-page in a THP.
958 	 */
959 	if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
960 	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
961 		ret = MC_TARGET_SWAP;
962 		if (target)
963 			target->ent = ent;
964 	}
965 	return ret;
966 }
967 
968 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
969 /*
970  * We don't consider PMD mapped swapping or file mapped pages because THP does
971  * not support them for now.
972  * Caller should make sure that pmd_trans_huge(pmd) is true.
973  */
974 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
975 		unsigned long addr, pmd_t pmd, union mc_target *target)
976 {
977 	struct page *page = NULL;
978 	struct folio *folio;
979 	enum mc_target_type ret = MC_TARGET_NONE;
980 
981 	if (unlikely(is_swap_pmd(pmd))) {
982 		VM_BUG_ON(thp_migration_supported() &&
983 				  !is_pmd_migration_entry(pmd));
984 		return ret;
985 	}
986 	page = pmd_page(pmd);
987 	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
988 	folio = page_folio(page);
989 	if (!(mc.flags & MOVE_ANON))
990 		return ret;
991 	if (folio_memcg(folio) == mc.from) {
992 		ret = MC_TARGET_PAGE;
993 		if (target) {
994 			folio_get(folio);
995 			if (!folio_trylock(folio)) {
996 				folio_put(folio);
997 				return MC_TARGET_NONE;
998 			}
999 			target->folio = folio;
1000 		}
1001 	}
1002 	return ret;
1003 }
1004 #else
1005 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
1006 		unsigned long addr, pmd_t pmd, union mc_target *target)
1007 {
1008 	return MC_TARGET_NONE;
1009 }
1010 #endif
1011 
1012 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
1013 					unsigned long addr, unsigned long end,
1014 					struct mm_walk *walk)
1015 {
1016 	struct vm_area_struct *vma = walk->vma;
1017 	pte_t *pte;
1018 	spinlock_t *ptl;
1019 
1020 	ptl = pmd_trans_huge_lock(pmd, vma);
1021 	if (ptl) {
1022 		/*
1023 		 * Note their can not be MC_TARGET_DEVICE for now as we do not
1024 		 * support transparent huge page with MEMORY_DEVICE_PRIVATE but
1025 		 * this might change.
1026 		 */
1027 		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
1028 			mc.precharge += HPAGE_PMD_NR;
1029 		spin_unlock(ptl);
1030 		return 0;
1031 	}
1032 
1033 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1034 	if (!pte)
1035 		return 0;
1036 	for (; addr != end; pte++, addr += PAGE_SIZE)
1037 		if (get_mctgt_type(vma, addr, ptep_get(pte), NULL))
1038 			mc.precharge++;	/* increment precharge temporarily */
1039 	pte_unmap_unlock(pte - 1, ptl);
1040 	cond_resched();
1041 
1042 	return 0;
1043 }
1044 
1045 static const struct mm_walk_ops precharge_walk_ops = {
1046 	.pmd_entry	= mem_cgroup_count_precharge_pte_range,
1047 	.walk_lock	= PGWALK_RDLOCK,
1048 };
1049 
1050 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
1051 {
1052 	unsigned long precharge;
1053 
1054 	mmap_read_lock(mm);
1055 	walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL);
1056 	mmap_read_unlock(mm);
1057 
1058 	precharge = mc.precharge;
1059 	mc.precharge = 0;
1060 
1061 	return precharge;
1062 }
1063 
1064 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
1065 {
1066 	unsigned long precharge = mem_cgroup_count_precharge(mm);
1067 
1068 	VM_BUG_ON(mc.moving_task);
1069 	mc.moving_task = current;
1070 	return mem_cgroup_do_precharge(precharge);
1071 }
1072 
1073 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
1074 static void __mem_cgroup_clear_mc(void)
1075 {
1076 	struct mem_cgroup *from = mc.from;
1077 	struct mem_cgroup *to = mc.to;
1078 
1079 	/* we must uncharge all the leftover precharges from mc.to */
1080 	if (mc.precharge) {
1081 		mem_cgroup_cancel_charge(mc.to, mc.precharge);
1082 		mc.precharge = 0;
1083 	}
1084 	/*
1085 	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
1086 	 * we must uncharge here.
1087 	 */
1088 	if (mc.moved_charge) {
1089 		mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
1090 		mc.moved_charge = 0;
1091 	}
1092 	/* we must fixup refcnts and charges */
1093 	if (mc.moved_swap) {
1094 		/* uncharge swap account from the old cgroup */
1095 		if (!mem_cgroup_is_root(mc.from))
1096 			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
1097 
1098 		mem_cgroup_id_put_many(mc.from, mc.moved_swap);
1099 
1100 		/*
1101 		 * we charged both to->memory and to->memsw, so we
1102 		 * should uncharge to->memory.
1103 		 */
1104 		if (!mem_cgroup_is_root(mc.to))
1105 			page_counter_uncharge(&mc.to->memory, mc.moved_swap);
1106 
1107 		mc.moved_swap = 0;
1108 	}
1109 	memcg1_oom_recover(from);
1110 	memcg1_oom_recover(to);
1111 	wake_up_all(&mc.waitq);
1112 }
1113 
1114 static void mem_cgroup_clear_mc(void)
1115 {
1116 	struct mm_struct *mm = mc.mm;
1117 
1118 	/*
1119 	 * we must clear moving_task before waking up waiters at the end of
1120 	 * task migration.
1121 	 */
1122 	mc.moving_task = NULL;
1123 	__mem_cgroup_clear_mc();
1124 	spin_lock(&mc.lock);
1125 	mc.from = NULL;
1126 	mc.to = NULL;
1127 	mc.mm = NULL;
1128 	spin_unlock(&mc.lock);
1129 
1130 	mmput(mm);
1131 }
1132 
1133 int memcg1_can_attach(struct cgroup_taskset *tset)
1134 {
1135 	struct cgroup_subsys_state *css;
1136 	struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
1137 	struct mem_cgroup *from;
1138 	struct task_struct *leader, *p;
1139 	struct mm_struct *mm;
1140 	unsigned long move_flags;
1141 	int ret = 0;
1142 
1143 	/* charge immigration isn't supported on the default hierarchy */
1144 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1145 		return 0;
1146 
1147 	/*
1148 	 * Multi-process migrations only happen on the default hierarchy
1149 	 * where charge immigration is not used.  Perform charge
1150 	 * immigration if @tset contains a leader and whine if there are
1151 	 * multiple.
1152 	 */
1153 	p = NULL;
1154 	cgroup_taskset_for_each_leader(leader, css, tset) {
1155 		WARN_ON_ONCE(p);
1156 		p = leader;
1157 		memcg = mem_cgroup_from_css(css);
1158 	}
1159 	if (!p)
1160 		return 0;
1161 
1162 	/*
1163 	 * We are now committed to this value whatever it is. Changes in this
1164 	 * tunable will only affect upcoming migrations, not the current one.
1165 	 * So we need to save it, and keep it going.
1166 	 */
1167 	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
1168 	if (!move_flags)
1169 		return 0;
1170 
1171 	from = mem_cgroup_from_task(p);
1172 
1173 	VM_BUG_ON(from == memcg);
1174 
1175 	mm = get_task_mm(p);
1176 	if (!mm)
1177 		return 0;
1178 	/* We move charges only when we move a owner of the mm */
1179 	if (mm->owner == p) {
1180 		VM_BUG_ON(mc.from);
1181 		VM_BUG_ON(mc.to);
1182 		VM_BUG_ON(mc.precharge);
1183 		VM_BUG_ON(mc.moved_charge);
1184 		VM_BUG_ON(mc.moved_swap);
1185 
1186 		spin_lock(&mc.lock);
1187 		mc.mm = mm;
1188 		mc.from = from;
1189 		mc.to = memcg;
1190 		mc.flags = move_flags;
1191 		spin_unlock(&mc.lock);
1192 		/* We set mc.moving_task later */
1193 
1194 		ret = mem_cgroup_precharge_mc(mm);
1195 		if (ret)
1196 			mem_cgroup_clear_mc();
1197 	} else {
1198 		mmput(mm);
1199 	}
1200 	return ret;
1201 }
1202 
1203 void memcg1_cancel_attach(struct cgroup_taskset *tset)
1204 {
1205 	if (mc.to)
1206 		mem_cgroup_clear_mc();
1207 }
1208 
1209 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
1210 				unsigned long addr, unsigned long end,
1211 				struct mm_walk *walk)
1212 {
1213 	int ret = 0;
1214 	struct vm_area_struct *vma = walk->vma;
1215 	pte_t *pte;
1216 	spinlock_t *ptl;
1217 	enum mc_target_type target_type;
1218 	union mc_target target;
1219 	struct folio *folio;
1220 
1221 	ptl = pmd_trans_huge_lock(pmd, vma);
1222 	if (ptl) {
1223 		if (mc.precharge < HPAGE_PMD_NR) {
1224 			spin_unlock(ptl);
1225 			return 0;
1226 		}
1227 		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
1228 		if (target_type == MC_TARGET_PAGE) {
1229 			folio = target.folio;
1230 			if (folio_isolate_lru(folio)) {
1231 				if (!mem_cgroup_move_account(folio, true,
1232 							     mc.from, mc.to)) {
1233 					mc.precharge -= HPAGE_PMD_NR;
1234 					mc.moved_charge += HPAGE_PMD_NR;
1235 				}
1236 				folio_putback_lru(folio);
1237 			}
1238 			folio_unlock(folio);
1239 			folio_put(folio);
1240 		} else if (target_type == MC_TARGET_DEVICE) {
1241 			folio = target.folio;
1242 			if (!mem_cgroup_move_account(folio, true,
1243 						     mc.from, mc.to)) {
1244 				mc.precharge -= HPAGE_PMD_NR;
1245 				mc.moved_charge += HPAGE_PMD_NR;
1246 			}
1247 			folio_unlock(folio);
1248 			folio_put(folio);
1249 		}
1250 		spin_unlock(ptl);
1251 		return 0;
1252 	}
1253 
1254 retry:
1255 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1256 	if (!pte)
1257 		return 0;
1258 	for (; addr != end; addr += PAGE_SIZE) {
1259 		pte_t ptent = ptep_get(pte++);
1260 		bool device = false;
1261 		swp_entry_t ent;
1262 
1263 		if (!mc.precharge)
1264 			break;
1265 
1266 		switch (get_mctgt_type(vma, addr, ptent, &target)) {
1267 		case MC_TARGET_DEVICE:
1268 			device = true;
1269 			fallthrough;
1270 		case MC_TARGET_PAGE:
1271 			folio = target.folio;
1272 			/*
1273 			 * We can have a part of the split pmd here. Moving it
1274 			 * can be done but it would be too convoluted so simply
1275 			 * ignore such a partial THP and keep it in original
1276 			 * memcg. There should be somebody mapping the head.
1277 			 */
1278 			if (folio_test_large(folio))
1279 				goto put;
1280 			if (!device && !folio_isolate_lru(folio))
1281 				goto put;
1282 			if (!mem_cgroup_move_account(folio, false,
1283 						mc.from, mc.to)) {
1284 				mc.precharge--;
1285 				/* we uncharge from mc.from later. */
1286 				mc.moved_charge++;
1287 			}
1288 			if (!device)
1289 				folio_putback_lru(folio);
1290 put:			/* get_mctgt_type() gets & locks the page */
1291 			folio_unlock(folio);
1292 			folio_put(folio);
1293 			break;
1294 		case MC_TARGET_SWAP:
1295 			ent = target.ent;
1296 			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
1297 				mc.precharge--;
1298 				mem_cgroup_id_get_many(mc.to, 1);
1299 				/* we fixup other refcnts and charges later. */
1300 				mc.moved_swap++;
1301 			}
1302 			break;
1303 		default:
1304 			break;
1305 		}
1306 	}
1307 	pte_unmap_unlock(pte - 1, ptl);
1308 	cond_resched();
1309 
1310 	if (addr != end) {
1311 		/*
1312 		 * We have consumed all precharges we got in can_attach().
1313 		 * We try charge one by one, but don't do any additional
1314 		 * charges to mc.to if we have failed in charge once in attach()
1315 		 * phase.
1316 		 */
1317 		ret = mem_cgroup_do_precharge(1);
1318 		if (!ret)
1319 			goto retry;
1320 	}
1321 
1322 	return ret;
1323 }
1324 
1325 static const struct mm_walk_ops charge_walk_ops = {
1326 	.pmd_entry	= mem_cgroup_move_charge_pte_range,
1327 	.walk_lock	= PGWALK_RDLOCK,
1328 };
1329 
1330 static void mem_cgroup_move_charge(void)
1331 {
1332 	lru_add_drain_all();
1333 	/*
1334 	 * Signal folio_memcg_lock() to take the memcg's move_lock
1335 	 * while we're moving its pages to another memcg. Then wait
1336 	 * for already started RCU-only updates to finish.
1337 	 */
1338 	atomic_inc(&mc.from->moving_account);
1339 	synchronize_rcu();
1340 retry:
1341 	if (unlikely(!mmap_read_trylock(mc.mm))) {
1342 		/*
1343 		 * Someone who are holding the mmap_lock might be waiting in
1344 		 * waitq. So we cancel all extra charges, wake up all waiters,
1345 		 * and retry. Because we cancel precharges, we might not be able
1346 		 * to move enough charges, but moving charge is a best-effort
1347 		 * feature anyway, so it wouldn't be a big problem.
1348 		 */
1349 		__mem_cgroup_clear_mc();
1350 		cond_resched();
1351 		goto retry;
1352 	}
1353 	/*
1354 	 * When we have consumed all precharges and failed in doing
1355 	 * additional charge, the page walk just aborts.
1356 	 */
1357 	walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL);
1358 	mmap_read_unlock(mc.mm);
1359 	atomic_dec(&mc.from->moving_account);
1360 }
1361 
1362 void memcg1_move_task(void)
1363 {
1364 	if (mc.to) {
1365 		mem_cgroup_move_charge();
1366 		mem_cgroup_clear_mc();
1367 	}
1368 }
1369 
1370 #else	/* !CONFIG_MMU */
1371 int memcg1_can_attach(struct cgroup_taskset *tset)
1372 {
1373 	return 0;
1374 }
1375 void memcg1_cancel_attach(struct cgroup_taskset *tset)
1376 {
1377 }
1378 void memcg1_move_task(void)
1379 {
1380 }
1381 #endif
1382 
1383 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
1384 {
1385 	struct mem_cgroup_threshold_ary *t;
1386 	unsigned long usage;
1387 	int i;
1388 
1389 	rcu_read_lock();
1390 	if (!swap)
1391 		t = rcu_dereference(memcg->thresholds.primary);
1392 	else
1393 		t = rcu_dereference(memcg->memsw_thresholds.primary);
1394 
1395 	if (!t)
1396 		goto unlock;
1397 
1398 	usage = mem_cgroup_usage(memcg, swap);
1399 
1400 	/*
1401 	 * current_threshold points to threshold just below or equal to usage.
1402 	 * If it's not true, a threshold was crossed after last
1403 	 * call of __mem_cgroup_threshold().
1404 	 */
1405 	i = t->current_threshold;
1406 
1407 	/*
1408 	 * Iterate backward over array of thresholds starting from
1409 	 * current_threshold and check if a threshold is crossed.
1410 	 * If none of thresholds below usage is crossed, we read
1411 	 * only one element of the array here.
1412 	 */
1413 	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
1414 		eventfd_signal(t->entries[i].eventfd);
1415 
1416 	/* i = current_threshold + 1 */
1417 	i++;
1418 
1419 	/*
1420 	 * Iterate forward over array of thresholds starting from
1421 	 * current_threshold+1 and check if a threshold is crossed.
1422 	 * If none of thresholds above usage is crossed, we read
1423 	 * only one element of the array here.
1424 	 */
1425 	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
1426 		eventfd_signal(t->entries[i].eventfd);
1427 
1428 	/* Update current_threshold */
1429 	t->current_threshold = i - 1;
1430 unlock:
1431 	rcu_read_unlock();
1432 }
1433 
1434 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
1435 {
1436 	while (memcg) {
1437 		__mem_cgroup_threshold(memcg, false);
1438 		if (do_memsw_account())
1439 			__mem_cgroup_threshold(memcg, true);
1440 
1441 		memcg = parent_mem_cgroup(memcg);
1442 	}
1443 }
1444 
1445 /* Cgroup1: threshold notifications & softlimit tree updates */
1446 struct memcg1_events_percpu {
1447 	unsigned long nr_page_events;
1448 	unsigned long targets[MEM_CGROUP_NTARGETS];
1449 };
1450 
1451 static void memcg1_charge_statistics(struct mem_cgroup *memcg, int nr_pages)
1452 {
1453 	/* pagein of a big page is an event. So, ignore page size */
1454 	if (nr_pages > 0)
1455 		__count_memcg_events(memcg, PGPGIN, 1);
1456 	else {
1457 		__count_memcg_events(memcg, PGPGOUT, 1);
1458 		nr_pages = -nr_pages; /* for event */
1459 	}
1460 
1461 	__this_cpu_add(memcg->events_percpu->nr_page_events, nr_pages);
1462 }
1463 
1464 #define THRESHOLDS_EVENTS_TARGET 128
1465 #define SOFTLIMIT_EVENTS_TARGET 1024
1466 
1467 static bool memcg1_event_ratelimit(struct mem_cgroup *memcg,
1468 				enum mem_cgroup_events_target target)
1469 {
1470 	unsigned long val, next;
1471 
1472 	val = __this_cpu_read(memcg->events_percpu->nr_page_events);
1473 	next = __this_cpu_read(memcg->events_percpu->targets[target]);
1474 	/* from time_after() in jiffies.h */
1475 	if ((long)(next - val) < 0) {
1476 		switch (target) {
1477 		case MEM_CGROUP_TARGET_THRESH:
1478 			next = val + THRESHOLDS_EVENTS_TARGET;
1479 			break;
1480 		case MEM_CGROUP_TARGET_SOFTLIMIT:
1481 			next = val + SOFTLIMIT_EVENTS_TARGET;
1482 			break;
1483 		default:
1484 			break;
1485 		}
1486 		__this_cpu_write(memcg->events_percpu->targets[target], next);
1487 		return true;
1488 	}
1489 	return false;
1490 }
1491 
1492 /*
1493  * Check events in order.
1494  *
1495  */
1496 static void memcg1_check_events(struct mem_cgroup *memcg, int nid)
1497 {
1498 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
1499 		return;
1500 
1501 	/* threshold event is triggered in finer grain than soft limit */
1502 	if (unlikely(memcg1_event_ratelimit(memcg,
1503 						MEM_CGROUP_TARGET_THRESH))) {
1504 		bool do_softlimit;
1505 
1506 		do_softlimit = memcg1_event_ratelimit(memcg,
1507 						MEM_CGROUP_TARGET_SOFTLIMIT);
1508 		mem_cgroup_threshold(memcg);
1509 		if (unlikely(do_softlimit))
1510 			memcg1_update_tree(memcg, nid);
1511 	}
1512 }
1513 
1514 void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg)
1515 {
1516 	unsigned long flags;
1517 
1518 	local_irq_save(flags);
1519 	memcg1_charge_statistics(memcg, folio_nr_pages(folio));
1520 	memcg1_check_events(memcg, folio_nid(folio));
1521 	local_irq_restore(flags);
1522 }
1523 
1524 void memcg1_swapout(struct folio *folio, struct mem_cgroup *memcg)
1525 {
1526 	/*
1527 	 * Interrupts should be disabled here because the caller holds the
1528 	 * i_pages lock which is taken with interrupts-off. It is
1529 	 * important here to have the interrupts disabled because it is the
1530 	 * only synchronisation we have for updating the per-CPU variables.
1531 	 */
1532 	preempt_disable_nested();
1533 	VM_WARN_ON_IRQS_ENABLED();
1534 	memcg1_charge_statistics(memcg, -folio_nr_pages(folio));
1535 	preempt_enable_nested();
1536 	memcg1_check_events(memcg, folio_nid(folio));
1537 }
1538 
1539 void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
1540 			   unsigned long nr_memory, int nid)
1541 {
1542 	unsigned long flags;
1543 
1544 	local_irq_save(flags);
1545 	__count_memcg_events(memcg, PGPGOUT, pgpgout);
1546 	__this_cpu_add(memcg->events_percpu->nr_page_events, nr_memory);
1547 	memcg1_check_events(memcg, nid);
1548 	local_irq_restore(flags);
1549 }
1550 
1551 static int compare_thresholds(const void *a, const void *b)
1552 {
1553 	const struct mem_cgroup_threshold *_a = a;
1554 	const struct mem_cgroup_threshold *_b = b;
1555 
1556 	if (_a->threshold > _b->threshold)
1557 		return 1;
1558 
1559 	if (_a->threshold < _b->threshold)
1560 		return -1;
1561 
1562 	return 0;
1563 }
1564 
1565 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
1566 {
1567 	struct mem_cgroup_eventfd_list *ev;
1568 
1569 	spin_lock(&memcg_oom_lock);
1570 
1571 	list_for_each_entry(ev, &memcg->oom_notify, list)
1572 		eventfd_signal(ev->eventfd);
1573 
1574 	spin_unlock(&memcg_oom_lock);
1575 	return 0;
1576 }
1577 
1578 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
1579 {
1580 	struct mem_cgroup *iter;
1581 
1582 	for_each_mem_cgroup_tree(iter, memcg)
1583 		mem_cgroup_oom_notify_cb(iter);
1584 }
1585 
1586 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
1587 	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
1588 {
1589 	struct mem_cgroup_thresholds *thresholds;
1590 	struct mem_cgroup_threshold_ary *new;
1591 	unsigned long threshold;
1592 	unsigned long usage;
1593 	int i, size, ret;
1594 
1595 	ret = page_counter_memparse(args, "-1", &threshold);
1596 	if (ret)
1597 		return ret;
1598 
1599 	mutex_lock(&memcg->thresholds_lock);
1600 
1601 	if (type == _MEM) {
1602 		thresholds = &memcg->thresholds;
1603 		usage = mem_cgroup_usage(memcg, false);
1604 	} else if (type == _MEMSWAP) {
1605 		thresholds = &memcg->memsw_thresholds;
1606 		usage = mem_cgroup_usage(memcg, true);
1607 	} else
1608 		BUG();
1609 
1610 	/* Check if a threshold crossed before adding a new one */
1611 	if (thresholds->primary)
1612 		__mem_cgroup_threshold(memcg, type == _MEMSWAP);
1613 
1614 	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
1615 
1616 	/* Allocate memory for new array of thresholds */
1617 	new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
1618 	if (!new) {
1619 		ret = -ENOMEM;
1620 		goto unlock;
1621 	}
1622 	new->size = size;
1623 
1624 	/* Copy thresholds (if any) to new array */
1625 	if (thresholds->primary)
1626 		memcpy(new->entries, thresholds->primary->entries,
1627 		       flex_array_size(new, entries, size - 1));
1628 
1629 	/* Add new threshold */
1630 	new->entries[size - 1].eventfd = eventfd;
1631 	new->entries[size - 1].threshold = threshold;
1632 
1633 	/* Sort thresholds. Registering of new threshold isn't time-critical */
1634 	sort(new->entries, size, sizeof(*new->entries),
1635 			compare_thresholds, NULL);
1636 
1637 	/* Find current threshold */
1638 	new->current_threshold = -1;
1639 	for (i = 0; i < size; i++) {
1640 		if (new->entries[i].threshold <= usage) {
1641 			/*
1642 			 * new->current_threshold will not be used until
1643 			 * rcu_assign_pointer(), so it's safe to increment
1644 			 * it here.
1645 			 */
1646 			++new->current_threshold;
1647 		} else
1648 			break;
1649 	}
1650 
1651 	/* Free old spare buffer and save old primary buffer as spare */
1652 	kfree(thresholds->spare);
1653 	thresholds->spare = thresholds->primary;
1654 
1655 	rcu_assign_pointer(thresholds->primary, new);
1656 
1657 	/* To be sure that nobody uses thresholds */
1658 	synchronize_rcu();
1659 
1660 unlock:
1661 	mutex_unlock(&memcg->thresholds_lock);
1662 
1663 	return ret;
1664 }
1665 
1666 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
1667 	struct eventfd_ctx *eventfd, const char *args)
1668 {
1669 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
1670 }
1671 
1672 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
1673 	struct eventfd_ctx *eventfd, const char *args)
1674 {
1675 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
1676 }
1677 
1678 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
1679 	struct eventfd_ctx *eventfd, enum res_type type)
1680 {
1681 	struct mem_cgroup_thresholds *thresholds;
1682 	struct mem_cgroup_threshold_ary *new;
1683 	unsigned long usage;
1684 	int i, j, size, entries;
1685 
1686 	mutex_lock(&memcg->thresholds_lock);
1687 
1688 	if (type == _MEM) {
1689 		thresholds = &memcg->thresholds;
1690 		usage = mem_cgroup_usage(memcg, false);
1691 	} else if (type == _MEMSWAP) {
1692 		thresholds = &memcg->memsw_thresholds;
1693 		usage = mem_cgroup_usage(memcg, true);
1694 	} else
1695 		BUG();
1696 
1697 	if (!thresholds->primary)
1698 		goto unlock;
1699 
1700 	/* Check if a threshold crossed before removing */
1701 	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
1702 
1703 	/* Calculate new number of threshold */
1704 	size = entries = 0;
1705 	for (i = 0; i < thresholds->primary->size; i++) {
1706 		if (thresholds->primary->entries[i].eventfd != eventfd)
1707 			size++;
1708 		else
1709 			entries++;
1710 	}
1711 
1712 	new = thresholds->spare;
1713 
1714 	/* If no items related to eventfd have been cleared, nothing to do */
1715 	if (!entries)
1716 		goto unlock;
1717 
1718 	/* Set thresholds array to NULL if we don't have thresholds */
1719 	if (!size) {
1720 		kfree(new);
1721 		new = NULL;
1722 		goto swap_buffers;
1723 	}
1724 
1725 	new->size = size;
1726 
1727 	/* Copy thresholds and find current threshold */
1728 	new->current_threshold = -1;
1729 	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
1730 		if (thresholds->primary->entries[i].eventfd == eventfd)
1731 			continue;
1732 
1733 		new->entries[j] = thresholds->primary->entries[i];
1734 		if (new->entries[j].threshold <= usage) {
1735 			/*
1736 			 * new->current_threshold will not be used
1737 			 * until rcu_assign_pointer(), so it's safe to increment
1738 			 * it here.
1739 			 */
1740 			++new->current_threshold;
1741 		}
1742 		j++;
1743 	}
1744 
1745 swap_buffers:
1746 	/* Swap primary and spare array */
1747 	thresholds->spare = thresholds->primary;
1748 
1749 	rcu_assign_pointer(thresholds->primary, new);
1750 
1751 	/* To be sure that nobody uses thresholds */
1752 	synchronize_rcu();
1753 
1754 	/* If all events are unregistered, free the spare array */
1755 	if (!new) {
1756 		kfree(thresholds->spare);
1757 		thresholds->spare = NULL;
1758 	}
1759 unlock:
1760 	mutex_unlock(&memcg->thresholds_lock);
1761 }
1762 
1763 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
1764 	struct eventfd_ctx *eventfd)
1765 {
1766 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
1767 }
1768 
1769 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
1770 	struct eventfd_ctx *eventfd)
1771 {
1772 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
1773 }
1774 
1775 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
1776 	struct eventfd_ctx *eventfd, const char *args)
1777 {
1778 	struct mem_cgroup_eventfd_list *event;
1779 
1780 	event = kmalloc(sizeof(*event),	GFP_KERNEL);
1781 	if (!event)
1782 		return -ENOMEM;
1783 
1784 	spin_lock(&memcg_oom_lock);
1785 
1786 	event->eventfd = eventfd;
1787 	list_add(&event->list, &memcg->oom_notify);
1788 
1789 	/* already in OOM ? */
1790 	if (memcg->under_oom)
1791 		eventfd_signal(eventfd);
1792 	spin_unlock(&memcg_oom_lock);
1793 
1794 	return 0;
1795 }
1796 
1797 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
1798 	struct eventfd_ctx *eventfd)
1799 {
1800 	struct mem_cgroup_eventfd_list *ev, *tmp;
1801 
1802 	spin_lock(&memcg_oom_lock);
1803 
1804 	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
1805 		if (ev->eventfd == eventfd) {
1806 			list_del(&ev->list);
1807 			kfree(ev);
1808 		}
1809 	}
1810 
1811 	spin_unlock(&memcg_oom_lock);
1812 }
1813 
1814 /*
1815  * DO NOT USE IN NEW FILES.
1816  *
1817  * "cgroup.event_control" implementation.
1818  *
1819  * This is way over-engineered.  It tries to support fully configurable
1820  * events for each user.  Such level of flexibility is completely
1821  * unnecessary especially in the light of the planned unified hierarchy.
1822  *
1823  * Please deprecate this and replace with something simpler if at all
1824  * possible.
1825  */
1826 
1827 /*
1828  * Unregister event and free resources.
1829  *
1830  * Gets called from workqueue.
1831  */
1832 static void memcg_event_remove(struct work_struct *work)
1833 {
1834 	struct mem_cgroup_event *event =
1835 		container_of(work, struct mem_cgroup_event, remove);
1836 	struct mem_cgroup *memcg = event->memcg;
1837 
1838 	remove_wait_queue(event->wqh, &event->wait);
1839 
1840 	event->unregister_event(memcg, event->eventfd);
1841 
1842 	/* Notify userspace the event is going away. */
1843 	eventfd_signal(event->eventfd);
1844 
1845 	eventfd_ctx_put(event->eventfd);
1846 	kfree(event);
1847 	css_put(&memcg->css);
1848 }
1849 
1850 /*
1851  * Gets called on EPOLLHUP on eventfd when user closes it.
1852  *
1853  * Called with wqh->lock held and interrupts disabled.
1854  */
1855 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
1856 			    int sync, void *key)
1857 {
1858 	struct mem_cgroup_event *event =
1859 		container_of(wait, struct mem_cgroup_event, wait);
1860 	struct mem_cgroup *memcg = event->memcg;
1861 	__poll_t flags = key_to_poll(key);
1862 
1863 	if (flags & EPOLLHUP) {
1864 		/*
1865 		 * If the event has been detached at cgroup removal, we
1866 		 * can simply return knowing the other side will cleanup
1867 		 * for us.
1868 		 *
1869 		 * We can't race against event freeing since the other
1870 		 * side will require wqh->lock via remove_wait_queue(),
1871 		 * which we hold.
1872 		 */
1873 		spin_lock(&memcg->event_list_lock);
1874 		if (!list_empty(&event->list)) {
1875 			list_del_init(&event->list);
1876 			/*
1877 			 * We are in atomic context, but cgroup_event_remove()
1878 			 * may sleep, so we have to call it in workqueue.
1879 			 */
1880 			schedule_work(&event->remove);
1881 		}
1882 		spin_unlock(&memcg->event_list_lock);
1883 	}
1884 
1885 	return 0;
1886 }
1887 
1888 static void memcg_event_ptable_queue_proc(struct file *file,
1889 		wait_queue_head_t *wqh, poll_table *pt)
1890 {
1891 	struct mem_cgroup_event *event =
1892 		container_of(pt, struct mem_cgroup_event, pt);
1893 
1894 	event->wqh = wqh;
1895 	add_wait_queue(wqh, &event->wait);
1896 }
1897 
1898 /*
1899  * DO NOT USE IN NEW FILES.
1900  *
1901  * Parse input and register new cgroup event handler.
1902  *
1903  * Input must be in format '<event_fd> <control_fd> <args>'.
1904  * Interpretation of args is defined by control file implementation.
1905  */
1906 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
1907 					 char *buf, size_t nbytes, loff_t off)
1908 {
1909 	struct cgroup_subsys_state *css = of_css(of);
1910 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
1911 	struct mem_cgroup_event *event;
1912 	struct cgroup_subsys_state *cfile_css;
1913 	unsigned int efd, cfd;
1914 	struct fd efile;
1915 	struct fd cfile;
1916 	struct dentry *cdentry;
1917 	const char *name;
1918 	char *endp;
1919 	int ret;
1920 
1921 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
1922 		return -EOPNOTSUPP;
1923 
1924 	buf = strstrip(buf);
1925 
1926 	efd = simple_strtoul(buf, &endp, 10);
1927 	if (*endp != ' ')
1928 		return -EINVAL;
1929 	buf = endp + 1;
1930 
1931 	cfd = simple_strtoul(buf, &endp, 10);
1932 	if (*endp == '\0')
1933 		buf = endp;
1934 	else if (*endp == ' ')
1935 		buf = endp + 1;
1936 	else
1937 		return -EINVAL;
1938 
1939 	event = kzalloc(sizeof(*event), GFP_KERNEL);
1940 	if (!event)
1941 		return -ENOMEM;
1942 
1943 	event->memcg = memcg;
1944 	INIT_LIST_HEAD(&event->list);
1945 	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
1946 	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
1947 	INIT_WORK(&event->remove, memcg_event_remove);
1948 
1949 	efile = fdget(efd);
1950 	if (!fd_file(efile)) {
1951 		ret = -EBADF;
1952 		goto out_kfree;
1953 	}
1954 
1955 	event->eventfd = eventfd_ctx_fileget(fd_file(efile));
1956 	if (IS_ERR(event->eventfd)) {
1957 		ret = PTR_ERR(event->eventfd);
1958 		goto out_put_efile;
1959 	}
1960 
1961 	cfile = fdget(cfd);
1962 	if (!fd_file(cfile)) {
1963 		ret = -EBADF;
1964 		goto out_put_eventfd;
1965 	}
1966 
1967 	/* the process need read permission on control file */
1968 	/* AV: shouldn't we check that it's been opened for read instead? */
1969 	ret = file_permission(fd_file(cfile), MAY_READ);
1970 	if (ret < 0)
1971 		goto out_put_cfile;
1972 
1973 	/*
1974 	 * The control file must be a regular cgroup1 file. As a regular cgroup
1975 	 * file can't be renamed, it's safe to access its name afterwards.
1976 	 */
1977 	cdentry = fd_file(cfile)->f_path.dentry;
1978 	if (cdentry->d_sb->s_type != &cgroup_fs_type || !d_is_reg(cdentry)) {
1979 		ret = -EINVAL;
1980 		goto out_put_cfile;
1981 	}
1982 
1983 	/*
1984 	 * Determine the event callbacks and set them in @event.  This used
1985 	 * to be done via struct cftype but cgroup core no longer knows
1986 	 * about these events.  The following is crude but the whole thing
1987 	 * is for compatibility anyway.
1988 	 *
1989 	 * DO NOT ADD NEW FILES.
1990 	 */
1991 	name = cdentry->d_name.name;
1992 
1993 	if (!strcmp(name, "memory.usage_in_bytes")) {
1994 		event->register_event = mem_cgroup_usage_register_event;
1995 		event->unregister_event = mem_cgroup_usage_unregister_event;
1996 	} else if (!strcmp(name, "memory.oom_control")) {
1997 		pr_warn_once("oom_control is deprecated and will be removed. "
1998 			     "Please report your usecase to linux-mm-@kvack.org"
1999 			     " if you depend on this functionality. \n");
2000 		event->register_event = mem_cgroup_oom_register_event;
2001 		event->unregister_event = mem_cgroup_oom_unregister_event;
2002 	} else if (!strcmp(name, "memory.pressure_level")) {
2003 		pr_warn_once("pressure_level is deprecated and will be removed. "
2004 			     "Please report your usecase to linux-mm-@kvack.org "
2005 			     "if you depend on this functionality. \n");
2006 		event->register_event = vmpressure_register_event;
2007 		event->unregister_event = vmpressure_unregister_event;
2008 	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
2009 		event->register_event = memsw_cgroup_usage_register_event;
2010 		event->unregister_event = memsw_cgroup_usage_unregister_event;
2011 	} else {
2012 		ret = -EINVAL;
2013 		goto out_put_cfile;
2014 	}
2015 
2016 	/*
2017 	 * Verify @cfile should belong to @css.  Also, remaining events are
2018 	 * automatically removed on cgroup destruction but the removal is
2019 	 * asynchronous, so take an extra ref on @css.
2020 	 */
2021 	cfile_css = css_tryget_online_from_dir(cdentry->d_parent,
2022 					       &memory_cgrp_subsys);
2023 	ret = -EINVAL;
2024 	if (IS_ERR(cfile_css))
2025 		goto out_put_cfile;
2026 	if (cfile_css != css) {
2027 		css_put(cfile_css);
2028 		goto out_put_cfile;
2029 	}
2030 
2031 	ret = event->register_event(memcg, event->eventfd, buf);
2032 	if (ret)
2033 		goto out_put_css;
2034 
2035 	vfs_poll(fd_file(efile), &event->pt);
2036 
2037 	spin_lock_irq(&memcg->event_list_lock);
2038 	list_add(&event->list, &memcg->event_list);
2039 	spin_unlock_irq(&memcg->event_list_lock);
2040 
2041 	fdput(cfile);
2042 	fdput(efile);
2043 
2044 	return nbytes;
2045 
2046 out_put_css:
2047 	css_put(css);
2048 out_put_cfile:
2049 	fdput(cfile);
2050 out_put_eventfd:
2051 	eventfd_ctx_put(event->eventfd);
2052 out_put_efile:
2053 	fdput(efile);
2054 out_kfree:
2055 	kfree(event);
2056 
2057 	return ret;
2058 }
2059 
2060 void memcg1_memcg_init(struct mem_cgroup *memcg)
2061 {
2062 	INIT_LIST_HEAD(&memcg->oom_notify);
2063 	mutex_init(&memcg->thresholds_lock);
2064 	spin_lock_init(&memcg->move_lock);
2065 	INIT_LIST_HEAD(&memcg->event_list);
2066 	spin_lock_init(&memcg->event_list_lock);
2067 }
2068 
2069 void memcg1_css_offline(struct mem_cgroup *memcg)
2070 {
2071 	struct mem_cgroup_event *event, *tmp;
2072 
2073 	/*
2074 	 * Unregister events and notify userspace.
2075 	 * Notify userspace about cgroup removing only after rmdir of cgroup
2076 	 * directory to avoid race between userspace and kernelspace.
2077 	 */
2078 	spin_lock_irq(&memcg->event_list_lock);
2079 	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
2080 		list_del_init(&event->list);
2081 		schedule_work(&event->remove);
2082 	}
2083 	spin_unlock_irq(&memcg->event_list_lock);
2084 }
2085 
2086 /*
2087  * Check OOM-Killer is already running under our hierarchy.
2088  * If someone is running, return false.
2089  */
2090 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
2091 {
2092 	struct mem_cgroup *iter, *failed = NULL;
2093 
2094 	spin_lock(&memcg_oom_lock);
2095 
2096 	for_each_mem_cgroup_tree(iter, memcg) {
2097 		if (iter->oom_lock) {
2098 			/*
2099 			 * this subtree of our hierarchy is already locked
2100 			 * so we cannot give a lock.
2101 			 */
2102 			failed = iter;
2103 			mem_cgroup_iter_break(memcg, iter);
2104 			break;
2105 		} else
2106 			iter->oom_lock = true;
2107 	}
2108 
2109 	if (failed) {
2110 		/*
2111 		 * OK, we failed to lock the whole subtree so we have
2112 		 * to clean up what we set up to the failing subtree
2113 		 */
2114 		for_each_mem_cgroup_tree(iter, memcg) {
2115 			if (iter == failed) {
2116 				mem_cgroup_iter_break(memcg, iter);
2117 				break;
2118 			}
2119 			iter->oom_lock = false;
2120 		}
2121 	} else
2122 		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
2123 
2124 	spin_unlock(&memcg_oom_lock);
2125 
2126 	return !failed;
2127 }
2128 
2129 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
2130 {
2131 	struct mem_cgroup *iter;
2132 
2133 	spin_lock(&memcg_oom_lock);
2134 	mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
2135 	for_each_mem_cgroup_tree(iter, memcg)
2136 		iter->oom_lock = false;
2137 	spin_unlock(&memcg_oom_lock);
2138 }
2139 
2140 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
2141 {
2142 	struct mem_cgroup *iter;
2143 
2144 	spin_lock(&memcg_oom_lock);
2145 	for_each_mem_cgroup_tree(iter, memcg)
2146 		iter->under_oom++;
2147 	spin_unlock(&memcg_oom_lock);
2148 }
2149 
2150 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
2151 {
2152 	struct mem_cgroup *iter;
2153 
2154 	/*
2155 	 * Be careful about under_oom underflows because a child memcg
2156 	 * could have been added after mem_cgroup_mark_under_oom.
2157 	 */
2158 	spin_lock(&memcg_oom_lock);
2159 	for_each_mem_cgroup_tree(iter, memcg)
2160 		if (iter->under_oom > 0)
2161 			iter->under_oom--;
2162 	spin_unlock(&memcg_oom_lock);
2163 }
2164 
2165 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
2166 
2167 struct oom_wait_info {
2168 	struct mem_cgroup *memcg;
2169 	wait_queue_entry_t	wait;
2170 };
2171 
2172 static int memcg_oom_wake_function(wait_queue_entry_t *wait,
2173 	unsigned mode, int sync, void *arg)
2174 {
2175 	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
2176 	struct mem_cgroup *oom_wait_memcg;
2177 	struct oom_wait_info *oom_wait_info;
2178 
2179 	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
2180 	oom_wait_memcg = oom_wait_info->memcg;
2181 
2182 	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
2183 	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
2184 		return 0;
2185 	return autoremove_wake_function(wait, mode, sync, arg);
2186 }
2187 
2188 void memcg1_oom_recover(struct mem_cgroup *memcg)
2189 {
2190 	/*
2191 	 * For the following lockless ->under_oom test, the only required
2192 	 * guarantee is that it must see the state asserted by an OOM when
2193 	 * this function is called as a result of userland actions
2194 	 * triggered by the notification of the OOM.  This is trivially
2195 	 * achieved by invoking mem_cgroup_mark_under_oom() before
2196 	 * triggering notification.
2197 	 */
2198 	if (memcg && memcg->under_oom)
2199 		__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
2200 }
2201 
2202 /**
2203  * mem_cgroup_oom_synchronize - complete memcg OOM handling
2204  * @handle: actually kill/wait or just clean up the OOM state
2205  *
2206  * This has to be called at the end of a page fault if the memcg OOM
2207  * handler was enabled.
2208  *
2209  * Memcg supports userspace OOM handling where failed allocations must
2210  * sleep on a waitqueue until the userspace task resolves the
2211  * situation.  Sleeping directly in the charge context with all kinds
2212  * of locks held is not a good idea, instead we remember an OOM state
2213  * in the task and mem_cgroup_oom_synchronize() has to be called at
2214  * the end of the page fault to complete the OOM handling.
2215  *
2216  * Returns %true if an ongoing memcg OOM situation was detected and
2217  * completed, %false otherwise.
2218  */
2219 bool mem_cgroup_oom_synchronize(bool handle)
2220 {
2221 	struct mem_cgroup *memcg = current->memcg_in_oom;
2222 	struct oom_wait_info owait;
2223 	bool locked;
2224 
2225 	/* OOM is global, do not handle */
2226 	if (!memcg)
2227 		return false;
2228 
2229 	if (!handle)
2230 		goto cleanup;
2231 
2232 	owait.memcg = memcg;
2233 	owait.wait.flags = 0;
2234 	owait.wait.func = memcg_oom_wake_function;
2235 	owait.wait.private = current;
2236 	INIT_LIST_HEAD(&owait.wait.entry);
2237 
2238 	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
2239 	mem_cgroup_mark_under_oom(memcg);
2240 
2241 	locked = mem_cgroup_oom_trylock(memcg);
2242 
2243 	if (locked)
2244 		mem_cgroup_oom_notify(memcg);
2245 
2246 	schedule();
2247 	mem_cgroup_unmark_under_oom(memcg);
2248 	finish_wait(&memcg_oom_waitq, &owait.wait);
2249 
2250 	if (locked)
2251 		mem_cgroup_oom_unlock(memcg);
2252 cleanup:
2253 	current->memcg_in_oom = NULL;
2254 	css_put(&memcg->css);
2255 	return true;
2256 }
2257 
2258 
2259 bool memcg1_oom_prepare(struct mem_cgroup *memcg, bool *locked)
2260 {
2261 	/*
2262 	 * We are in the middle of the charge context here, so we
2263 	 * don't want to block when potentially sitting on a callstack
2264 	 * that holds all kinds of filesystem and mm locks.
2265 	 *
2266 	 * cgroup1 allows disabling the OOM killer and waiting for outside
2267 	 * handling until the charge can succeed; remember the context and put
2268 	 * the task to sleep at the end of the page fault when all locks are
2269 	 * released.
2270 	 *
2271 	 * On the other hand, in-kernel OOM killer allows for an async victim
2272 	 * memory reclaim (oom_reaper) and that means that we are not solely
2273 	 * relying on the oom victim to make a forward progress and we can
2274 	 * invoke the oom killer here.
2275 	 *
2276 	 * Please note that mem_cgroup_out_of_memory might fail to find a
2277 	 * victim and then we have to bail out from the charge path.
2278 	 */
2279 	if (READ_ONCE(memcg->oom_kill_disable)) {
2280 		if (current->in_user_fault) {
2281 			css_get(&memcg->css);
2282 			current->memcg_in_oom = memcg;
2283 		}
2284 		return false;
2285 	}
2286 
2287 	mem_cgroup_mark_under_oom(memcg);
2288 
2289 	*locked = mem_cgroup_oom_trylock(memcg);
2290 
2291 	if (*locked)
2292 		mem_cgroup_oom_notify(memcg);
2293 
2294 	mem_cgroup_unmark_under_oom(memcg);
2295 
2296 	return true;
2297 }
2298 
2299 void memcg1_oom_finish(struct mem_cgroup *memcg, bool locked)
2300 {
2301 	if (locked)
2302 		mem_cgroup_oom_unlock(memcg);
2303 }
2304 
2305 static DEFINE_MUTEX(memcg_max_mutex);
2306 
2307 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
2308 				 unsigned long max, bool memsw)
2309 {
2310 	bool enlarge = false;
2311 	bool drained = false;
2312 	int ret;
2313 	bool limits_invariant;
2314 	struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
2315 
2316 	do {
2317 		if (signal_pending(current)) {
2318 			ret = -EINTR;
2319 			break;
2320 		}
2321 
2322 		mutex_lock(&memcg_max_mutex);
2323 		/*
2324 		 * Make sure that the new limit (memsw or memory limit) doesn't
2325 		 * break our basic invariant rule memory.max <= memsw.max.
2326 		 */
2327 		limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
2328 					   max <= memcg->memsw.max;
2329 		if (!limits_invariant) {
2330 			mutex_unlock(&memcg_max_mutex);
2331 			ret = -EINVAL;
2332 			break;
2333 		}
2334 		if (max > counter->max)
2335 			enlarge = true;
2336 		ret = page_counter_set_max(counter, max);
2337 		mutex_unlock(&memcg_max_mutex);
2338 
2339 		if (!ret)
2340 			break;
2341 
2342 		if (!drained) {
2343 			drain_all_stock(memcg);
2344 			drained = true;
2345 			continue;
2346 		}
2347 
2348 		if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
2349 				memsw ? 0 : MEMCG_RECLAIM_MAY_SWAP, NULL)) {
2350 			ret = -EBUSY;
2351 			break;
2352 		}
2353 	} while (true);
2354 
2355 	if (!ret && enlarge)
2356 		memcg1_oom_recover(memcg);
2357 
2358 	return ret;
2359 }
2360 
2361 /*
2362  * Reclaims as many pages from the given memcg as possible.
2363  *
2364  * Caller is responsible for holding css reference for memcg.
2365  */
2366 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2367 {
2368 	int nr_retries = MAX_RECLAIM_RETRIES;
2369 
2370 	/* we call try-to-free pages for make this cgroup empty */
2371 	lru_add_drain_all();
2372 
2373 	drain_all_stock(memcg);
2374 
2375 	/* try to free all pages in this cgroup */
2376 	while (nr_retries && page_counter_read(&memcg->memory)) {
2377 		if (signal_pending(current))
2378 			return -EINTR;
2379 
2380 		if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
2381 						  MEMCG_RECLAIM_MAY_SWAP, NULL))
2382 			nr_retries--;
2383 	}
2384 
2385 	return 0;
2386 }
2387 
2388 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2389 					    char *buf, size_t nbytes,
2390 					    loff_t off)
2391 {
2392 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2393 
2394 	if (mem_cgroup_is_root(memcg))
2395 		return -EINVAL;
2396 	return mem_cgroup_force_empty(memcg) ?: nbytes;
2397 }
2398 
2399 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2400 				     struct cftype *cft)
2401 {
2402 	return 1;
2403 }
2404 
2405 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2406 				      struct cftype *cft, u64 val)
2407 {
2408 	if (val == 1)
2409 		return 0;
2410 
2411 	pr_warn_once("Non-hierarchical mode is deprecated. "
2412 		     "Please report your usecase to linux-mm@kvack.org if you "
2413 		     "depend on this functionality.\n");
2414 
2415 	return -EINVAL;
2416 }
2417 
2418 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
2419 			       struct cftype *cft)
2420 {
2421 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2422 	struct page_counter *counter;
2423 
2424 	switch (MEMFILE_TYPE(cft->private)) {
2425 	case _MEM:
2426 		counter = &memcg->memory;
2427 		break;
2428 	case _MEMSWAP:
2429 		counter = &memcg->memsw;
2430 		break;
2431 	case _KMEM:
2432 		counter = &memcg->kmem;
2433 		break;
2434 	case _TCP:
2435 		counter = &memcg->tcpmem;
2436 		break;
2437 	default:
2438 		BUG();
2439 	}
2440 
2441 	switch (MEMFILE_ATTR(cft->private)) {
2442 	case RES_USAGE:
2443 		if (counter == &memcg->memory)
2444 			return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
2445 		if (counter == &memcg->memsw)
2446 			return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
2447 		return (u64)page_counter_read(counter) * PAGE_SIZE;
2448 	case RES_LIMIT:
2449 		return (u64)counter->max * PAGE_SIZE;
2450 	case RES_MAX_USAGE:
2451 		return (u64)counter->watermark * PAGE_SIZE;
2452 	case RES_FAILCNT:
2453 		return counter->failcnt;
2454 	case RES_SOFT_LIMIT:
2455 		return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE;
2456 	default:
2457 		BUG();
2458 	}
2459 }
2460 
2461 /*
2462  * This function doesn't do anything useful. Its only job is to provide a read
2463  * handler for a file so that cgroup_file_mode() will add read permissions.
2464  */
2465 static int mem_cgroup_dummy_seq_show(__always_unused struct seq_file *m,
2466 				     __always_unused void *v)
2467 {
2468 	return -EINVAL;
2469 }
2470 
2471 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
2472 {
2473 	int ret;
2474 
2475 	mutex_lock(&memcg_max_mutex);
2476 
2477 	ret = page_counter_set_max(&memcg->tcpmem, max);
2478 	if (ret)
2479 		goto out;
2480 
2481 	if (!memcg->tcpmem_active) {
2482 		/*
2483 		 * The active flag needs to be written after the static_key
2484 		 * update. This is what guarantees that the socket activation
2485 		 * function is the last one to run. See mem_cgroup_sk_alloc()
2486 		 * for details, and note that we don't mark any socket as
2487 		 * belonging to this memcg until that flag is up.
2488 		 *
2489 		 * We need to do this, because static_keys will span multiple
2490 		 * sites, but we can't control their order. If we mark a socket
2491 		 * as accounted, but the accounting functions are not patched in
2492 		 * yet, we'll lose accounting.
2493 		 *
2494 		 * We never race with the readers in mem_cgroup_sk_alloc(),
2495 		 * because when this value change, the code to process it is not
2496 		 * patched in yet.
2497 		 */
2498 		static_branch_inc(&memcg_sockets_enabled_key);
2499 		memcg->tcpmem_active = true;
2500 	}
2501 out:
2502 	mutex_unlock(&memcg_max_mutex);
2503 	return ret;
2504 }
2505 
2506 /*
2507  * The user of this function is...
2508  * RES_LIMIT.
2509  */
2510 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
2511 				char *buf, size_t nbytes, loff_t off)
2512 {
2513 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2514 	unsigned long nr_pages;
2515 	int ret;
2516 
2517 	buf = strstrip(buf);
2518 	ret = page_counter_memparse(buf, "-1", &nr_pages);
2519 	if (ret)
2520 		return ret;
2521 
2522 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
2523 	case RES_LIMIT:
2524 		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
2525 			ret = -EINVAL;
2526 			break;
2527 		}
2528 		switch (MEMFILE_TYPE(of_cft(of)->private)) {
2529 		case _MEM:
2530 			ret = mem_cgroup_resize_max(memcg, nr_pages, false);
2531 			break;
2532 		case _MEMSWAP:
2533 			ret = mem_cgroup_resize_max(memcg, nr_pages, true);
2534 			break;
2535 		case _KMEM:
2536 			pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
2537 				     "Writing any value to this file has no effect. "
2538 				     "Please report your usecase to linux-mm@kvack.org if you "
2539 				     "depend on this functionality.\n");
2540 			ret = 0;
2541 			break;
2542 		case _TCP:
2543 			pr_warn_once("kmem.tcp.limit_in_bytes is deprecated and will be removed. "
2544 				     "Please report your usecase to linux-mm@kvack.org if you "
2545 				     "depend on this functionality.\n");
2546 			ret = memcg_update_tcp_max(memcg, nr_pages);
2547 			break;
2548 		}
2549 		break;
2550 	case RES_SOFT_LIMIT:
2551 		if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
2552 			ret = -EOPNOTSUPP;
2553 		} else {
2554 			pr_warn_once("soft_limit_in_bytes is deprecated and will be removed. "
2555 				     "Please report your usecase to linux-mm@kvack.org if you "
2556 				     "depend on this functionality.\n");
2557 			WRITE_ONCE(memcg->soft_limit, nr_pages);
2558 			ret = 0;
2559 		}
2560 		break;
2561 	}
2562 	return ret ?: nbytes;
2563 }
2564 
2565 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
2566 				size_t nbytes, loff_t off)
2567 {
2568 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2569 	struct page_counter *counter;
2570 
2571 	switch (MEMFILE_TYPE(of_cft(of)->private)) {
2572 	case _MEM:
2573 		counter = &memcg->memory;
2574 		break;
2575 	case _MEMSWAP:
2576 		counter = &memcg->memsw;
2577 		break;
2578 	case _KMEM:
2579 		counter = &memcg->kmem;
2580 		break;
2581 	case _TCP:
2582 		counter = &memcg->tcpmem;
2583 		break;
2584 	default:
2585 		BUG();
2586 	}
2587 
2588 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
2589 	case RES_MAX_USAGE:
2590 		page_counter_reset_watermark(counter);
2591 		break;
2592 	case RES_FAILCNT:
2593 		counter->failcnt = 0;
2594 		break;
2595 	default:
2596 		BUG();
2597 	}
2598 
2599 	return nbytes;
2600 }
2601 
2602 #ifdef CONFIG_NUMA
2603 
2604 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
2605 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
2606 #define LRU_ALL	     ((1 << NR_LRU_LISTS) - 1)
2607 
2608 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
2609 				int nid, unsigned int lru_mask, bool tree)
2610 {
2611 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
2612 	unsigned long nr = 0;
2613 	enum lru_list lru;
2614 
2615 	VM_BUG_ON((unsigned)nid >= nr_node_ids);
2616 
2617 	for_each_lru(lru) {
2618 		if (!(BIT(lru) & lru_mask))
2619 			continue;
2620 		if (tree)
2621 			nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
2622 		else
2623 			nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
2624 	}
2625 	return nr;
2626 }
2627 
2628 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
2629 					     unsigned int lru_mask,
2630 					     bool tree)
2631 {
2632 	unsigned long nr = 0;
2633 	enum lru_list lru;
2634 
2635 	for_each_lru(lru) {
2636 		if (!(BIT(lru) & lru_mask))
2637 			continue;
2638 		if (tree)
2639 			nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
2640 		else
2641 			nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
2642 	}
2643 	return nr;
2644 }
2645 
2646 static int memcg_numa_stat_show(struct seq_file *m, void *v)
2647 {
2648 	struct numa_stat {
2649 		const char *name;
2650 		unsigned int lru_mask;
2651 	};
2652 
2653 	static const struct numa_stat stats[] = {
2654 		{ "total", LRU_ALL },
2655 		{ "file", LRU_ALL_FILE },
2656 		{ "anon", LRU_ALL_ANON },
2657 		{ "unevictable", BIT(LRU_UNEVICTABLE) },
2658 	};
2659 	const struct numa_stat *stat;
2660 	int nid;
2661 	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
2662 
2663 	mem_cgroup_flush_stats(memcg);
2664 
2665 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
2666 		seq_printf(m, "%s=%lu", stat->name,
2667 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
2668 						   false));
2669 		for_each_node_state(nid, N_MEMORY)
2670 			seq_printf(m, " N%d=%lu", nid,
2671 				   mem_cgroup_node_nr_lru_pages(memcg, nid,
2672 							stat->lru_mask, false));
2673 		seq_putc(m, '\n');
2674 	}
2675 
2676 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
2677 
2678 		seq_printf(m, "hierarchical_%s=%lu", stat->name,
2679 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
2680 						   true));
2681 		for_each_node_state(nid, N_MEMORY)
2682 			seq_printf(m, " N%d=%lu", nid,
2683 				   mem_cgroup_node_nr_lru_pages(memcg, nid,
2684 							stat->lru_mask, true));
2685 		seq_putc(m, '\n');
2686 	}
2687 
2688 	return 0;
2689 }
2690 #endif /* CONFIG_NUMA */
2691 
2692 static const unsigned int memcg1_stats[] = {
2693 	NR_FILE_PAGES,
2694 	NR_ANON_MAPPED,
2695 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2696 	NR_ANON_THPS,
2697 #endif
2698 	NR_SHMEM,
2699 	NR_FILE_MAPPED,
2700 	NR_FILE_DIRTY,
2701 	NR_WRITEBACK,
2702 	WORKINGSET_REFAULT_ANON,
2703 	WORKINGSET_REFAULT_FILE,
2704 #ifdef CONFIG_SWAP
2705 	MEMCG_SWAP,
2706 	NR_SWAPCACHE,
2707 #endif
2708 };
2709 
2710 static const char *const memcg1_stat_names[] = {
2711 	"cache",
2712 	"rss",
2713 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2714 	"rss_huge",
2715 #endif
2716 	"shmem",
2717 	"mapped_file",
2718 	"dirty",
2719 	"writeback",
2720 	"workingset_refault_anon",
2721 	"workingset_refault_file",
2722 #ifdef CONFIG_SWAP
2723 	"swap",
2724 	"swapcached",
2725 #endif
2726 };
2727 
2728 /* Universal VM events cgroup1 shows, original sort order */
2729 static const unsigned int memcg1_events[] = {
2730 	PGPGIN,
2731 	PGPGOUT,
2732 	PGFAULT,
2733 	PGMAJFAULT,
2734 };
2735 
2736 void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
2737 {
2738 	unsigned long memory, memsw;
2739 	struct mem_cgroup *mi;
2740 	unsigned int i;
2741 
2742 	BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
2743 
2744 	mem_cgroup_flush_stats(memcg);
2745 
2746 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
2747 		unsigned long nr;
2748 
2749 		nr = memcg_page_state_local_output(memcg, memcg1_stats[i]);
2750 		seq_buf_printf(s, "%s %lu\n", memcg1_stat_names[i], nr);
2751 	}
2752 
2753 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
2754 		seq_buf_printf(s, "%s %lu\n", vm_event_name(memcg1_events[i]),
2755 			       memcg_events_local(memcg, memcg1_events[i]));
2756 
2757 	for (i = 0; i < NR_LRU_LISTS; i++)
2758 		seq_buf_printf(s, "%s %lu\n", lru_list_name(i),
2759 			       memcg_page_state_local(memcg, NR_LRU_BASE + i) *
2760 			       PAGE_SIZE);
2761 
2762 	/* Hierarchical information */
2763 	memory = memsw = PAGE_COUNTER_MAX;
2764 	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
2765 		memory = min(memory, READ_ONCE(mi->memory.max));
2766 		memsw = min(memsw, READ_ONCE(mi->memsw.max));
2767 	}
2768 	seq_buf_printf(s, "hierarchical_memory_limit %llu\n",
2769 		       (u64)memory * PAGE_SIZE);
2770 	seq_buf_printf(s, "hierarchical_memsw_limit %llu\n",
2771 		       (u64)memsw * PAGE_SIZE);
2772 
2773 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
2774 		unsigned long nr;
2775 
2776 		nr = memcg_page_state_output(memcg, memcg1_stats[i]);
2777 		seq_buf_printf(s, "total_%s %llu\n", memcg1_stat_names[i],
2778 			       (u64)nr);
2779 	}
2780 
2781 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
2782 		seq_buf_printf(s, "total_%s %llu\n",
2783 			       vm_event_name(memcg1_events[i]),
2784 			       (u64)memcg_events(memcg, memcg1_events[i]));
2785 
2786 	for (i = 0; i < NR_LRU_LISTS; i++)
2787 		seq_buf_printf(s, "total_%s %llu\n", lru_list_name(i),
2788 			       (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
2789 			       PAGE_SIZE);
2790 
2791 #ifdef CONFIG_DEBUG_VM
2792 	{
2793 		pg_data_t *pgdat;
2794 		struct mem_cgroup_per_node *mz;
2795 		unsigned long anon_cost = 0;
2796 		unsigned long file_cost = 0;
2797 
2798 		for_each_online_pgdat(pgdat) {
2799 			mz = memcg->nodeinfo[pgdat->node_id];
2800 
2801 			anon_cost += mz->lruvec.anon_cost;
2802 			file_cost += mz->lruvec.file_cost;
2803 		}
2804 		seq_buf_printf(s, "anon_cost %lu\n", anon_cost);
2805 		seq_buf_printf(s, "file_cost %lu\n", file_cost);
2806 	}
2807 #endif
2808 }
2809 
2810 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
2811 				      struct cftype *cft)
2812 {
2813 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2814 
2815 	return mem_cgroup_swappiness(memcg);
2816 }
2817 
2818 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
2819 				       struct cftype *cft, u64 val)
2820 {
2821 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2822 
2823 	if (val > MAX_SWAPPINESS)
2824 		return -EINVAL;
2825 
2826 	if (!mem_cgroup_is_root(memcg))
2827 		WRITE_ONCE(memcg->swappiness, val);
2828 	else
2829 		WRITE_ONCE(vm_swappiness, val);
2830 
2831 	return 0;
2832 }
2833 
2834 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
2835 {
2836 	struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
2837 
2838 	seq_printf(sf, "oom_kill_disable %d\n", READ_ONCE(memcg->oom_kill_disable));
2839 	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
2840 	seq_printf(sf, "oom_kill %lu\n",
2841 		   atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
2842 	return 0;
2843 }
2844 
2845 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
2846 	struct cftype *cft, u64 val)
2847 {
2848 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2849 
2850 	pr_warn_once("oom_control is deprecated and will be removed. "
2851 		     "Please report your usecase to linux-mm-@kvack.org if you "
2852 		     "depend on this functionality. \n");
2853 
2854 	/* cannot set to root cgroup and only 0 and 1 are allowed */
2855 	if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1)))
2856 		return -EINVAL;
2857 
2858 	WRITE_ONCE(memcg->oom_kill_disable, val);
2859 	if (!val)
2860 		memcg1_oom_recover(memcg);
2861 
2862 	return 0;
2863 }
2864 
2865 #ifdef CONFIG_SLUB_DEBUG
2866 static int mem_cgroup_slab_show(struct seq_file *m, void *p)
2867 {
2868 	/*
2869 	 * Deprecated.
2870 	 * Please, take a look at tools/cgroup/memcg_slabinfo.py .
2871 	 */
2872 	return 0;
2873 }
2874 #endif
2875 
2876 struct cftype mem_cgroup_legacy_files[] = {
2877 	{
2878 		.name = "usage_in_bytes",
2879 		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
2880 		.read_u64 = mem_cgroup_read_u64,
2881 	},
2882 	{
2883 		.name = "max_usage_in_bytes",
2884 		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
2885 		.write = mem_cgroup_reset,
2886 		.read_u64 = mem_cgroup_read_u64,
2887 	},
2888 	{
2889 		.name = "limit_in_bytes",
2890 		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
2891 		.write = mem_cgroup_write,
2892 		.read_u64 = mem_cgroup_read_u64,
2893 	},
2894 	{
2895 		.name = "soft_limit_in_bytes",
2896 		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
2897 		.write = mem_cgroup_write,
2898 		.read_u64 = mem_cgroup_read_u64,
2899 	},
2900 	{
2901 		.name = "failcnt",
2902 		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
2903 		.write = mem_cgroup_reset,
2904 		.read_u64 = mem_cgroup_read_u64,
2905 	},
2906 	{
2907 		.name = "stat",
2908 		.seq_show = memory_stat_show,
2909 	},
2910 	{
2911 		.name = "force_empty",
2912 		.write = mem_cgroup_force_empty_write,
2913 	},
2914 	{
2915 		.name = "use_hierarchy",
2916 		.write_u64 = mem_cgroup_hierarchy_write,
2917 		.read_u64 = mem_cgroup_hierarchy_read,
2918 	},
2919 	{
2920 		.name = "cgroup.event_control",		/* XXX: for compat */
2921 		.write = memcg_write_event_control,
2922 		.flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
2923 	},
2924 	{
2925 		.name = "swappiness",
2926 		.read_u64 = mem_cgroup_swappiness_read,
2927 		.write_u64 = mem_cgroup_swappiness_write,
2928 	},
2929 	{
2930 		.name = "move_charge_at_immigrate",
2931 		.read_u64 = mem_cgroup_move_charge_read,
2932 		.write_u64 = mem_cgroup_move_charge_write,
2933 	},
2934 	{
2935 		.name = "oom_control",
2936 		.seq_show = mem_cgroup_oom_control_read,
2937 		.write_u64 = mem_cgroup_oom_control_write,
2938 	},
2939 	{
2940 		.name = "pressure_level",
2941 		.seq_show = mem_cgroup_dummy_seq_show,
2942 	},
2943 #ifdef CONFIG_NUMA
2944 	{
2945 		.name = "numa_stat",
2946 		.seq_show = memcg_numa_stat_show,
2947 	},
2948 #endif
2949 	{
2950 		.name = "kmem.limit_in_bytes",
2951 		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
2952 		.write = mem_cgroup_write,
2953 		.read_u64 = mem_cgroup_read_u64,
2954 	},
2955 	{
2956 		.name = "kmem.usage_in_bytes",
2957 		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
2958 		.read_u64 = mem_cgroup_read_u64,
2959 	},
2960 	{
2961 		.name = "kmem.failcnt",
2962 		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
2963 		.write = mem_cgroup_reset,
2964 		.read_u64 = mem_cgroup_read_u64,
2965 	},
2966 	{
2967 		.name = "kmem.max_usage_in_bytes",
2968 		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
2969 		.write = mem_cgroup_reset,
2970 		.read_u64 = mem_cgroup_read_u64,
2971 	},
2972 #ifdef CONFIG_SLUB_DEBUG
2973 	{
2974 		.name = "kmem.slabinfo",
2975 		.seq_show = mem_cgroup_slab_show,
2976 	},
2977 #endif
2978 	{
2979 		.name = "kmem.tcp.limit_in_bytes",
2980 		.private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
2981 		.write = mem_cgroup_write,
2982 		.read_u64 = mem_cgroup_read_u64,
2983 	},
2984 	{
2985 		.name = "kmem.tcp.usage_in_bytes",
2986 		.private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
2987 		.read_u64 = mem_cgroup_read_u64,
2988 	},
2989 	{
2990 		.name = "kmem.tcp.failcnt",
2991 		.private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
2992 		.write = mem_cgroup_reset,
2993 		.read_u64 = mem_cgroup_read_u64,
2994 	},
2995 	{
2996 		.name = "kmem.tcp.max_usage_in_bytes",
2997 		.private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
2998 		.write = mem_cgroup_reset,
2999 		.read_u64 = mem_cgroup_read_u64,
3000 	},
3001 	{ },	/* terminate */
3002 };
3003 
3004 struct cftype memsw_files[] = {
3005 	{
3006 		.name = "memsw.usage_in_bytes",
3007 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
3008 		.read_u64 = mem_cgroup_read_u64,
3009 	},
3010 	{
3011 		.name = "memsw.max_usage_in_bytes",
3012 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
3013 		.write = mem_cgroup_reset,
3014 		.read_u64 = mem_cgroup_read_u64,
3015 	},
3016 	{
3017 		.name = "memsw.limit_in_bytes",
3018 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
3019 		.write = mem_cgroup_write,
3020 		.read_u64 = mem_cgroup_read_u64,
3021 	},
3022 	{
3023 		.name = "memsw.failcnt",
3024 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
3025 		.write = mem_cgroup_reset,
3026 		.read_u64 = mem_cgroup_read_u64,
3027 	},
3028 	{ },	/* terminate */
3029 };
3030 
3031 void memcg1_account_kmem(struct mem_cgroup *memcg, int nr_pages)
3032 {
3033 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
3034 		if (nr_pages > 0)
3035 			page_counter_charge(&memcg->kmem, nr_pages);
3036 		else
3037 			page_counter_uncharge(&memcg->kmem, -nr_pages);
3038 	}
3039 }
3040 
3041 bool memcg1_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
3042 			 gfp_t gfp_mask)
3043 {
3044 	struct page_counter *fail;
3045 
3046 	if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
3047 		memcg->tcpmem_pressure = 0;
3048 		return true;
3049 	}
3050 	memcg->tcpmem_pressure = 1;
3051 	if (gfp_mask & __GFP_NOFAIL) {
3052 		page_counter_charge(&memcg->tcpmem, nr_pages);
3053 		return true;
3054 	}
3055 	return false;
3056 }
3057 
3058 bool memcg1_alloc_events(struct mem_cgroup *memcg)
3059 {
3060 	memcg->events_percpu = alloc_percpu_gfp(struct memcg1_events_percpu,
3061 						GFP_KERNEL_ACCOUNT);
3062 	return !!memcg->events_percpu;
3063 }
3064 
3065 void memcg1_free_events(struct mem_cgroup *memcg)
3066 {
3067 	if (memcg->events_percpu)
3068 		free_percpu(memcg->events_percpu);
3069 }
3070 
3071 static int __init memcg1_init(void)
3072 {
3073 	int node;
3074 
3075 	for_each_node(node) {
3076 		struct mem_cgroup_tree_per_node *rtpn;
3077 
3078 		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, node);
3079 
3080 		rtpn->rb_root = RB_ROOT;
3081 		rtpn->rb_rightmost = NULL;
3082 		spin_lock_init(&rtpn->lock);
3083 		soft_limit_tree.rb_tree_per_node[node] = rtpn;
3084 	}
3085 
3086 	return 0;
3087 }
3088 subsys_initcall(memcg1_init);
3089