xref: /linux/mm/memcontrol.c (revision 9fb29c734f9e98adc1f2f3c4629fe487cb93f2dd)
1 /* memcontrol.c - Memory Controller
2  *
3  * Copyright IBM Corporation, 2007
4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5  *
6  * Copyright 2007 OpenVZ SWsoft Inc
7  * Author: Pavel Emelianov <xemul@openvz.org>
8  *
9  * Memory thresholds
10  * Copyright (C) 2009 Nokia Corporation
11  * Author: Kirill A. Shutemov
12  *
13  * Kernel Memory Controller
14  * Copyright (C) 2012 Parallels Inc. and Google Inc.
15  * Authors: Glauber Costa and Suleiman Souhlal
16  *
17  * Native page reclaim
18  * Charge lifetime sanitation
19  * Lockless page tracking & accounting
20  * Unified hierarchy configuration model
21  * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
22  *
23  * This program is free software; you can redistribute it and/or modify
24  * it under the terms of the GNU General Public License as published by
25  * the Free Software Foundation; either version 2 of the License, or
26  * (at your option) any later version.
27  *
28  * This program is distributed in the hope that it will be useful,
29  * but WITHOUT ANY WARRANTY; without even the implied warranty of
30  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
31  * GNU General Public License for more details.
32  */
33 
34 #include <linux/page_counter.h>
35 #include <linux/memcontrol.h>
36 #include <linux/cgroup.h>
37 #include <linux/mm.h>
38 #include <linux/sched/mm.h>
39 #include <linux/shmem_fs.h>
40 #include <linux/hugetlb.h>
41 #include <linux/pagemap.h>
42 #include <linux/smp.h>
43 #include <linux/page-flags.h>
44 #include <linux/backing-dev.h>
45 #include <linux/bit_spinlock.h>
46 #include <linux/rcupdate.h>
47 #include <linux/limits.h>
48 #include <linux/export.h>
49 #include <linux/mutex.h>
50 #include <linux/rbtree.h>
51 #include <linux/slab.h>
52 #include <linux/swap.h>
53 #include <linux/swapops.h>
54 #include <linux/spinlock.h>
55 #include <linux/eventfd.h>
56 #include <linux/poll.h>
57 #include <linux/sort.h>
58 #include <linux/fs.h>
59 #include <linux/seq_file.h>
60 #include <linux/vmpressure.h>
61 #include <linux/mm_inline.h>
62 #include <linux/swap_cgroup.h>
63 #include <linux/cpu.h>
64 #include <linux/oom.h>
65 #include <linux/lockdep.h>
66 #include <linux/file.h>
67 #include <linux/tracehook.h>
68 #include "internal.h"
69 #include <net/sock.h>
70 #include <net/ip.h>
71 #include "slab.h"
72 
73 #include <linux/uaccess.h>
74 
75 #include <trace/events/vmscan.h>
76 
77 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
78 EXPORT_SYMBOL(memory_cgrp_subsys);
79 
80 struct mem_cgroup *root_mem_cgroup __read_mostly;
81 
82 #define MEM_CGROUP_RECLAIM_RETRIES	5
83 
84 /* Socket memory accounting disabled? */
85 static bool cgroup_memory_nosocket;
86 
87 /* Kernel memory accounting disabled? */
88 static bool cgroup_memory_nokmem;
89 
90 /* Whether the swap controller is active */
91 #ifdef CONFIG_MEMCG_SWAP
92 int do_swap_account __read_mostly;
93 #else
94 #define do_swap_account		0
95 #endif
96 
97 /* Whether legacy memory+swap accounting is active */
98 static bool do_memsw_account(void)
99 {
100 	return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
101 }
102 
103 static const char *const mem_cgroup_lru_names[] = {
104 	"inactive_anon",
105 	"active_anon",
106 	"inactive_file",
107 	"active_file",
108 	"unevictable",
109 };
110 
111 #define THRESHOLDS_EVENTS_TARGET 128
112 #define SOFTLIMIT_EVENTS_TARGET 1024
113 #define NUMAINFO_EVENTS_TARGET	1024
114 
115 /*
116  * Cgroups above their limits are maintained in a RB-Tree, independent of
117  * their hierarchy representation
118  */
119 
120 struct mem_cgroup_tree_per_node {
121 	struct rb_root rb_root;
122 	struct rb_node *rb_rightmost;
123 	spinlock_t lock;
124 };
125 
126 struct mem_cgroup_tree {
127 	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
128 };
129 
130 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
131 
132 /* for OOM */
133 struct mem_cgroup_eventfd_list {
134 	struct list_head list;
135 	struct eventfd_ctx *eventfd;
136 };
137 
138 /*
139  * cgroup_event represents events which userspace want to receive.
140  */
141 struct mem_cgroup_event {
142 	/*
143 	 * memcg which the event belongs to.
144 	 */
145 	struct mem_cgroup *memcg;
146 	/*
147 	 * eventfd to signal userspace about the event.
148 	 */
149 	struct eventfd_ctx *eventfd;
150 	/*
151 	 * Each of these stored in a list by the cgroup.
152 	 */
153 	struct list_head list;
154 	/*
155 	 * register_event() callback will be used to add new userspace
156 	 * waiter for changes related to this event.  Use eventfd_signal()
157 	 * on eventfd to send notification to userspace.
158 	 */
159 	int (*register_event)(struct mem_cgroup *memcg,
160 			      struct eventfd_ctx *eventfd, const char *args);
161 	/*
162 	 * unregister_event() callback will be called when userspace closes
163 	 * the eventfd or on cgroup removing.  This callback must be set,
164 	 * if you want provide notification functionality.
165 	 */
166 	void (*unregister_event)(struct mem_cgroup *memcg,
167 				 struct eventfd_ctx *eventfd);
168 	/*
169 	 * All fields below needed to unregister event when
170 	 * userspace closes eventfd.
171 	 */
172 	poll_table pt;
173 	wait_queue_head_t *wqh;
174 	wait_queue_entry_t wait;
175 	struct work_struct remove;
176 };
177 
178 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
179 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
180 
181 /* Stuffs for move charges at task migration. */
182 /*
183  * Types of charges to be moved.
184  */
185 #define MOVE_ANON	0x1U
186 #define MOVE_FILE	0x2U
187 #define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
188 
189 /* "mc" and its members are protected by cgroup_mutex */
190 static struct move_charge_struct {
191 	spinlock_t	  lock; /* for from, to */
192 	struct mm_struct  *mm;
193 	struct mem_cgroup *from;
194 	struct mem_cgroup *to;
195 	unsigned long flags;
196 	unsigned long precharge;
197 	unsigned long moved_charge;
198 	unsigned long moved_swap;
199 	struct task_struct *moving_task;	/* a task moving charges */
200 	wait_queue_head_t waitq;		/* a waitq for other context */
201 } mc = {
202 	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
203 	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
204 };
205 
206 /*
207  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
208  * limit reclaim to prevent infinite loops, if they ever occur.
209  */
210 #define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
211 #define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
212 
213 enum charge_type {
214 	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
215 	MEM_CGROUP_CHARGE_TYPE_ANON,
216 	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
217 	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
218 	NR_CHARGE_TYPE,
219 };
220 
221 /* for encoding cft->private value on file */
222 enum res_type {
223 	_MEM,
224 	_MEMSWAP,
225 	_OOM_TYPE,
226 	_KMEM,
227 	_TCP,
228 };
229 
230 #define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
231 #define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
232 #define MEMFILE_ATTR(val)	((val) & 0xffff)
233 /* Used for OOM nofiier */
234 #define OOM_CONTROL		(0)
235 
236 /*
237  * Iteration constructs for visiting all cgroups (under a tree).  If
238  * loops are exited prematurely (break), mem_cgroup_iter_break() must
239  * be used for reference counting.
240  */
241 #define for_each_mem_cgroup_tree(iter, root)		\
242 	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
243 	     iter != NULL;				\
244 	     iter = mem_cgroup_iter(root, iter, NULL))
245 
246 #define for_each_mem_cgroup(iter)			\
247 	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
248 	     iter != NULL;				\
249 	     iter = mem_cgroup_iter(NULL, iter, NULL))
250 
251 /* Some nice accessors for the vmpressure. */
252 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
253 {
254 	if (!memcg)
255 		memcg = root_mem_cgroup;
256 	return &memcg->vmpressure;
257 }
258 
259 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
260 {
261 	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
262 }
263 
264 #ifdef CONFIG_MEMCG_KMEM
265 /*
266  * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
267  * The main reason for not using cgroup id for this:
268  *  this works better in sparse environments, where we have a lot of memcgs,
269  *  but only a few kmem-limited. Or also, if we have, for instance, 200
270  *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
271  *  200 entry array for that.
272  *
273  * The current size of the caches array is stored in memcg_nr_cache_ids. It
274  * will double each time we have to increase it.
275  */
276 static DEFINE_IDA(memcg_cache_ida);
277 int memcg_nr_cache_ids;
278 
279 /* Protects memcg_nr_cache_ids */
280 static DECLARE_RWSEM(memcg_cache_ids_sem);
281 
282 void memcg_get_cache_ids(void)
283 {
284 	down_read(&memcg_cache_ids_sem);
285 }
286 
287 void memcg_put_cache_ids(void)
288 {
289 	up_read(&memcg_cache_ids_sem);
290 }
291 
292 /*
293  * MIN_SIZE is different than 1, because we would like to avoid going through
294  * the alloc/free process all the time. In a small machine, 4 kmem-limited
295  * cgroups is a reasonable guess. In the future, it could be a parameter or
296  * tunable, but that is strictly not necessary.
297  *
298  * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
299  * this constant directly from cgroup, but it is understandable that this is
300  * better kept as an internal representation in cgroup.c. In any case, the
301  * cgrp_id space is not getting any smaller, and we don't have to necessarily
302  * increase ours as well if it increases.
303  */
304 #define MEMCG_CACHES_MIN_SIZE 4
305 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
306 
307 /*
308  * A lot of the calls to the cache allocation functions are expected to be
309  * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
310  * conditional to this static branch, we'll have to allow modules that does
311  * kmem_cache_alloc and the such to see this symbol as well
312  */
313 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
314 EXPORT_SYMBOL(memcg_kmem_enabled_key);
315 
316 struct workqueue_struct *memcg_kmem_cache_wq;
317 
318 static int memcg_shrinker_map_size;
319 static DEFINE_MUTEX(memcg_shrinker_map_mutex);
320 
321 static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
322 {
323 	kvfree(container_of(head, struct memcg_shrinker_map, rcu));
324 }
325 
326 static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
327 					 int size, int old_size)
328 {
329 	struct memcg_shrinker_map *new, *old;
330 	int nid;
331 
332 	lockdep_assert_held(&memcg_shrinker_map_mutex);
333 
334 	for_each_node(nid) {
335 		old = rcu_dereference_protected(
336 			mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
337 		/* Not yet online memcg */
338 		if (!old)
339 			return 0;
340 
341 		new = kvmalloc(sizeof(*new) + size, GFP_KERNEL);
342 		if (!new)
343 			return -ENOMEM;
344 
345 		/* Set all old bits, clear all new bits */
346 		memset(new->map, (int)0xff, old_size);
347 		memset((void *)new->map + old_size, 0, size - old_size);
348 
349 		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
350 		call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
351 	}
352 
353 	return 0;
354 }
355 
356 static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
357 {
358 	struct mem_cgroup_per_node *pn;
359 	struct memcg_shrinker_map *map;
360 	int nid;
361 
362 	if (mem_cgroup_is_root(memcg))
363 		return;
364 
365 	for_each_node(nid) {
366 		pn = mem_cgroup_nodeinfo(memcg, nid);
367 		map = rcu_dereference_protected(pn->shrinker_map, true);
368 		if (map)
369 			kvfree(map);
370 		rcu_assign_pointer(pn->shrinker_map, NULL);
371 	}
372 }
373 
374 static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
375 {
376 	struct memcg_shrinker_map *map;
377 	int nid, size, ret = 0;
378 
379 	if (mem_cgroup_is_root(memcg))
380 		return 0;
381 
382 	mutex_lock(&memcg_shrinker_map_mutex);
383 	size = memcg_shrinker_map_size;
384 	for_each_node(nid) {
385 		map = kvzalloc(sizeof(*map) + size, GFP_KERNEL);
386 		if (!map) {
387 			memcg_free_shrinker_maps(memcg);
388 			ret = -ENOMEM;
389 			break;
390 		}
391 		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
392 	}
393 	mutex_unlock(&memcg_shrinker_map_mutex);
394 
395 	return ret;
396 }
397 
398 int memcg_expand_shrinker_maps(int new_id)
399 {
400 	int size, old_size, ret = 0;
401 	struct mem_cgroup *memcg;
402 
403 	size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
404 	old_size = memcg_shrinker_map_size;
405 	if (size <= old_size)
406 		return 0;
407 
408 	mutex_lock(&memcg_shrinker_map_mutex);
409 	if (!root_mem_cgroup)
410 		goto unlock;
411 
412 	for_each_mem_cgroup(memcg) {
413 		if (mem_cgroup_is_root(memcg))
414 			continue;
415 		ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
416 		if (ret)
417 			goto unlock;
418 	}
419 unlock:
420 	if (!ret)
421 		memcg_shrinker_map_size = size;
422 	mutex_unlock(&memcg_shrinker_map_mutex);
423 	return ret;
424 }
425 
426 void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
427 {
428 	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
429 		struct memcg_shrinker_map *map;
430 
431 		rcu_read_lock();
432 		map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
433 		/* Pairs with smp mb in shrink_slab() */
434 		smp_mb__before_atomic();
435 		set_bit(shrinker_id, map->map);
436 		rcu_read_unlock();
437 	}
438 }
439 
440 #else /* CONFIG_MEMCG_KMEM */
441 static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
442 {
443 	return 0;
444 }
445 static void memcg_free_shrinker_maps(struct mem_cgroup *memcg) { }
446 #endif /* CONFIG_MEMCG_KMEM */
447 
448 /**
449  * mem_cgroup_css_from_page - css of the memcg associated with a page
450  * @page: page of interest
451  *
452  * If memcg is bound to the default hierarchy, css of the memcg associated
453  * with @page is returned.  The returned css remains associated with @page
454  * until it is released.
455  *
456  * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
457  * is returned.
458  */
459 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
460 {
461 	struct mem_cgroup *memcg;
462 
463 	memcg = page->mem_cgroup;
464 
465 	if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
466 		memcg = root_mem_cgroup;
467 
468 	return &memcg->css;
469 }
470 
471 /**
472  * page_cgroup_ino - return inode number of the memcg a page is charged to
473  * @page: the page
474  *
475  * Look up the closest online ancestor of the memory cgroup @page is charged to
476  * and return its inode number or 0 if @page is not charged to any cgroup. It
477  * is safe to call this function without holding a reference to @page.
478  *
479  * Note, this function is inherently racy, because there is nothing to prevent
480  * the cgroup inode from getting torn down and potentially reallocated a moment
481  * after page_cgroup_ino() returns, so it only should be used by callers that
482  * do not care (such as procfs interfaces).
483  */
484 ino_t page_cgroup_ino(struct page *page)
485 {
486 	struct mem_cgroup *memcg;
487 	unsigned long ino = 0;
488 
489 	rcu_read_lock();
490 	memcg = READ_ONCE(page->mem_cgroup);
491 	while (memcg && !(memcg->css.flags & CSS_ONLINE))
492 		memcg = parent_mem_cgroup(memcg);
493 	if (memcg)
494 		ino = cgroup_ino(memcg->css.cgroup);
495 	rcu_read_unlock();
496 	return ino;
497 }
498 
499 static struct mem_cgroup_per_node *
500 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
501 {
502 	int nid = page_to_nid(page);
503 
504 	return memcg->nodeinfo[nid];
505 }
506 
507 static struct mem_cgroup_tree_per_node *
508 soft_limit_tree_node(int nid)
509 {
510 	return soft_limit_tree.rb_tree_per_node[nid];
511 }
512 
513 static struct mem_cgroup_tree_per_node *
514 soft_limit_tree_from_page(struct page *page)
515 {
516 	int nid = page_to_nid(page);
517 
518 	return soft_limit_tree.rb_tree_per_node[nid];
519 }
520 
521 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
522 					 struct mem_cgroup_tree_per_node *mctz,
523 					 unsigned long new_usage_in_excess)
524 {
525 	struct rb_node **p = &mctz->rb_root.rb_node;
526 	struct rb_node *parent = NULL;
527 	struct mem_cgroup_per_node *mz_node;
528 	bool rightmost = true;
529 
530 	if (mz->on_tree)
531 		return;
532 
533 	mz->usage_in_excess = new_usage_in_excess;
534 	if (!mz->usage_in_excess)
535 		return;
536 	while (*p) {
537 		parent = *p;
538 		mz_node = rb_entry(parent, struct mem_cgroup_per_node,
539 					tree_node);
540 		if (mz->usage_in_excess < mz_node->usage_in_excess) {
541 			p = &(*p)->rb_left;
542 			rightmost = false;
543 		}
544 
545 		/*
546 		 * We can't avoid mem cgroups that are over their soft
547 		 * limit by the same amount
548 		 */
549 		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
550 			p = &(*p)->rb_right;
551 	}
552 
553 	if (rightmost)
554 		mctz->rb_rightmost = &mz->tree_node;
555 
556 	rb_link_node(&mz->tree_node, parent, p);
557 	rb_insert_color(&mz->tree_node, &mctz->rb_root);
558 	mz->on_tree = true;
559 }
560 
561 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
562 					 struct mem_cgroup_tree_per_node *mctz)
563 {
564 	if (!mz->on_tree)
565 		return;
566 
567 	if (&mz->tree_node == mctz->rb_rightmost)
568 		mctz->rb_rightmost = rb_prev(&mz->tree_node);
569 
570 	rb_erase(&mz->tree_node, &mctz->rb_root);
571 	mz->on_tree = false;
572 }
573 
574 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
575 				       struct mem_cgroup_tree_per_node *mctz)
576 {
577 	unsigned long flags;
578 
579 	spin_lock_irqsave(&mctz->lock, flags);
580 	__mem_cgroup_remove_exceeded(mz, mctz);
581 	spin_unlock_irqrestore(&mctz->lock, flags);
582 }
583 
584 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
585 {
586 	unsigned long nr_pages = page_counter_read(&memcg->memory);
587 	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
588 	unsigned long excess = 0;
589 
590 	if (nr_pages > soft_limit)
591 		excess = nr_pages - soft_limit;
592 
593 	return excess;
594 }
595 
596 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
597 {
598 	unsigned long excess;
599 	struct mem_cgroup_per_node *mz;
600 	struct mem_cgroup_tree_per_node *mctz;
601 
602 	mctz = soft_limit_tree_from_page(page);
603 	if (!mctz)
604 		return;
605 	/*
606 	 * Necessary to update all ancestors when hierarchy is used.
607 	 * because their event counter is not touched.
608 	 */
609 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
610 		mz = mem_cgroup_page_nodeinfo(memcg, page);
611 		excess = soft_limit_excess(memcg);
612 		/*
613 		 * We have to update the tree if mz is on RB-tree or
614 		 * mem is over its softlimit.
615 		 */
616 		if (excess || mz->on_tree) {
617 			unsigned long flags;
618 
619 			spin_lock_irqsave(&mctz->lock, flags);
620 			/* if on-tree, remove it */
621 			if (mz->on_tree)
622 				__mem_cgroup_remove_exceeded(mz, mctz);
623 			/*
624 			 * Insert again. mz->usage_in_excess will be updated.
625 			 * If excess is 0, no tree ops.
626 			 */
627 			__mem_cgroup_insert_exceeded(mz, mctz, excess);
628 			spin_unlock_irqrestore(&mctz->lock, flags);
629 		}
630 	}
631 }
632 
633 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
634 {
635 	struct mem_cgroup_tree_per_node *mctz;
636 	struct mem_cgroup_per_node *mz;
637 	int nid;
638 
639 	for_each_node(nid) {
640 		mz = mem_cgroup_nodeinfo(memcg, nid);
641 		mctz = soft_limit_tree_node(nid);
642 		if (mctz)
643 			mem_cgroup_remove_exceeded(mz, mctz);
644 	}
645 }
646 
647 static struct mem_cgroup_per_node *
648 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
649 {
650 	struct mem_cgroup_per_node *mz;
651 
652 retry:
653 	mz = NULL;
654 	if (!mctz->rb_rightmost)
655 		goto done;		/* Nothing to reclaim from */
656 
657 	mz = rb_entry(mctz->rb_rightmost,
658 		      struct mem_cgroup_per_node, tree_node);
659 	/*
660 	 * Remove the node now but someone else can add it back,
661 	 * we will to add it back at the end of reclaim to its correct
662 	 * position in the tree.
663 	 */
664 	__mem_cgroup_remove_exceeded(mz, mctz);
665 	if (!soft_limit_excess(mz->memcg) ||
666 	    !css_tryget_online(&mz->memcg->css))
667 		goto retry;
668 done:
669 	return mz;
670 }
671 
672 static struct mem_cgroup_per_node *
673 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
674 {
675 	struct mem_cgroup_per_node *mz;
676 
677 	spin_lock_irq(&mctz->lock);
678 	mz = __mem_cgroup_largest_soft_limit_node(mctz);
679 	spin_unlock_irq(&mctz->lock);
680 	return mz;
681 }
682 
683 static unsigned long memcg_sum_events(struct mem_cgroup *memcg,
684 				      int event)
685 {
686 	return atomic_long_read(&memcg->events[event]);
687 }
688 
689 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
690 					 struct page *page,
691 					 bool compound, int nr_pages)
692 {
693 	/*
694 	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
695 	 * counted as CACHE even if it's on ANON LRU.
696 	 */
697 	if (PageAnon(page))
698 		__mod_memcg_state(memcg, MEMCG_RSS, nr_pages);
699 	else {
700 		__mod_memcg_state(memcg, MEMCG_CACHE, nr_pages);
701 		if (PageSwapBacked(page))
702 			__mod_memcg_state(memcg, NR_SHMEM, nr_pages);
703 	}
704 
705 	if (compound) {
706 		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
707 		__mod_memcg_state(memcg, MEMCG_RSS_HUGE, nr_pages);
708 	}
709 
710 	/* pagein of a big page is an event. So, ignore page size */
711 	if (nr_pages > 0)
712 		__count_memcg_events(memcg, PGPGIN, 1);
713 	else {
714 		__count_memcg_events(memcg, PGPGOUT, 1);
715 		nr_pages = -nr_pages; /* for event */
716 	}
717 
718 	__this_cpu_add(memcg->stat_cpu->nr_page_events, nr_pages);
719 }
720 
721 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
722 					   int nid, unsigned int lru_mask)
723 {
724 	struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
725 	unsigned long nr = 0;
726 	enum lru_list lru;
727 
728 	VM_BUG_ON((unsigned)nid >= nr_node_ids);
729 
730 	for_each_lru(lru) {
731 		if (!(BIT(lru) & lru_mask))
732 			continue;
733 		nr += mem_cgroup_get_lru_size(lruvec, lru);
734 	}
735 	return nr;
736 }
737 
738 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
739 			unsigned int lru_mask)
740 {
741 	unsigned long nr = 0;
742 	int nid;
743 
744 	for_each_node_state(nid, N_MEMORY)
745 		nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
746 	return nr;
747 }
748 
749 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
750 				       enum mem_cgroup_events_target target)
751 {
752 	unsigned long val, next;
753 
754 	val = __this_cpu_read(memcg->stat_cpu->nr_page_events);
755 	next = __this_cpu_read(memcg->stat_cpu->targets[target]);
756 	/* from time_after() in jiffies.h */
757 	if ((long)(next - val) < 0) {
758 		switch (target) {
759 		case MEM_CGROUP_TARGET_THRESH:
760 			next = val + THRESHOLDS_EVENTS_TARGET;
761 			break;
762 		case MEM_CGROUP_TARGET_SOFTLIMIT:
763 			next = val + SOFTLIMIT_EVENTS_TARGET;
764 			break;
765 		case MEM_CGROUP_TARGET_NUMAINFO:
766 			next = val + NUMAINFO_EVENTS_TARGET;
767 			break;
768 		default:
769 			break;
770 		}
771 		__this_cpu_write(memcg->stat_cpu->targets[target], next);
772 		return true;
773 	}
774 	return false;
775 }
776 
777 /*
778  * Check events in order.
779  *
780  */
781 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
782 {
783 	/* threshold event is triggered in finer grain than soft limit */
784 	if (unlikely(mem_cgroup_event_ratelimit(memcg,
785 						MEM_CGROUP_TARGET_THRESH))) {
786 		bool do_softlimit;
787 		bool do_numainfo __maybe_unused;
788 
789 		do_softlimit = mem_cgroup_event_ratelimit(memcg,
790 						MEM_CGROUP_TARGET_SOFTLIMIT);
791 #if MAX_NUMNODES > 1
792 		do_numainfo = mem_cgroup_event_ratelimit(memcg,
793 						MEM_CGROUP_TARGET_NUMAINFO);
794 #endif
795 		mem_cgroup_threshold(memcg);
796 		if (unlikely(do_softlimit))
797 			mem_cgroup_update_tree(memcg, page);
798 #if MAX_NUMNODES > 1
799 		if (unlikely(do_numainfo))
800 			atomic_inc(&memcg->numainfo_events);
801 #endif
802 	}
803 }
804 
805 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
806 {
807 	/*
808 	 * mm_update_next_owner() may clear mm->owner to NULL
809 	 * if it races with swapoff, page migration, etc.
810 	 * So this can be called with p == NULL.
811 	 */
812 	if (unlikely(!p))
813 		return NULL;
814 
815 	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
816 }
817 EXPORT_SYMBOL(mem_cgroup_from_task);
818 
819 /**
820  * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
821  * @mm: mm from which memcg should be extracted. It can be NULL.
822  *
823  * Obtain a reference on mm->memcg and returns it if successful. Otherwise
824  * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
825  * returned.
826  */
827 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
828 {
829 	struct mem_cgroup *memcg;
830 
831 	if (mem_cgroup_disabled())
832 		return NULL;
833 
834 	rcu_read_lock();
835 	do {
836 		/*
837 		 * Page cache insertions can happen withou an
838 		 * actual mm context, e.g. during disk probing
839 		 * on boot, loopback IO, acct() writes etc.
840 		 */
841 		if (unlikely(!mm))
842 			memcg = root_mem_cgroup;
843 		else {
844 			memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
845 			if (unlikely(!memcg))
846 				memcg = root_mem_cgroup;
847 		}
848 	} while (!css_tryget_online(&memcg->css));
849 	rcu_read_unlock();
850 	return memcg;
851 }
852 EXPORT_SYMBOL(get_mem_cgroup_from_mm);
853 
854 /**
855  * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
856  * @page: page from which memcg should be extracted.
857  *
858  * Obtain a reference on page->memcg and returns it if successful. Otherwise
859  * root_mem_cgroup is returned.
860  */
861 struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
862 {
863 	struct mem_cgroup *memcg = page->mem_cgroup;
864 
865 	if (mem_cgroup_disabled())
866 		return NULL;
867 
868 	rcu_read_lock();
869 	if (!memcg || !css_tryget_online(&memcg->css))
870 		memcg = root_mem_cgroup;
871 	rcu_read_unlock();
872 	return memcg;
873 }
874 EXPORT_SYMBOL(get_mem_cgroup_from_page);
875 
876 /**
877  * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg.
878  */
879 static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
880 {
881 	if (unlikely(current->active_memcg)) {
882 		struct mem_cgroup *memcg = root_mem_cgroup;
883 
884 		rcu_read_lock();
885 		if (css_tryget_online(&current->active_memcg->css))
886 			memcg = current->active_memcg;
887 		rcu_read_unlock();
888 		return memcg;
889 	}
890 	return get_mem_cgroup_from_mm(current->mm);
891 }
892 
893 /**
894  * mem_cgroup_iter - iterate over memory cgroup hierarchy
895  * @root: hierarchy root
896  * @prev: previously returned memcg, NULL on first invocation
897  * @reclaim: cookie for shared reclaim walks, NULL for full walks
898  *
899  * Returns references to children of the hierarchy below @root, or
900  * @root itself, or %NULL after a full round-trip.
901  *
902  * Caller must pass the return value in @prev on subsequent
903  * invocations for reference counting, or use mem_cgroup_iter_break()
904  * to cancel a hierarchy walk before the round-trip is complete.
905  *
906  * Reclaimers can specify a node and a priority level in @reclaim to
907  * divide up the memcgs in the hierarchy among all concurrent
908  * reclaimers operating on the same node and priority.
909  */
910 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
911 				   struct mem_cgroup *prev,
912 				   struct mem_cgroup_reclaim_cookie *reclaim)
913 {
914 	struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
915 	struct cgroup_subsys_state *css = NULL;
916 	struct mem_cgroup *memcg = NULL;
917 	struct mem_cgroup *pos = NULL;
918 
919 	if (mem_cgroup_disabled())
920 		return NULL;
921 
922 	if (!root)
923 		root = root_mem_cgroup;
924 
925 	if (prev && !reclaim)
926 		pos = prev;
927 
928 	if (!root->use_hierarchy && root != root_mem_cgroup) {
929 		if (prev)
930 			goto out;
931 		return root;
932 	}
933 
934 	rcu_read_lock();
935 
936 	if (reclaim) {
937 		struct mem_cgroup_per_node *mz;
938 
939 		mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
940 		iter = &mz->iter[reclaim->priority];
941 
942 		if (prev && reclaim->generation != iter->generation)
943 			goto out_unlock;
944 
945 		while (1) {
946 			pos = READ_ONCE(iter->position);
947 			if (!pos || css_tryget(&pos->css))
948 				break;
949 			/*
950 			 * css reference reached zero, so iter->position will
951 			 * be cleared by ->css_released. However, we should not
952 			 * rely on this happening soon, because ->css_released
953 			 * is called from a work queue, and by busy-waiting we
954 			 * might block it. So we clear iter->position right
955 			 * away.
956 			 */
957 			(void)cmpxchg(&iter->position, pos, NULL);
958 		}
959 	}
960 
961 	if (pos)
962 		css = &pos->css;
963 
964 	for (;;) {
965 		css = css_next_descendant_pre(css, &root->css);
966 		if (!css) {
967 			/*
968 			 * Reclaimers share the hierarchy walk, and a
969 			 * new one might jump in right at the end of
970 			 * the hierarchy - make sure they see at least
971 			 * one group and restart from the beginning.
972 			 */
973 			if (!prev)
974 				continue;
975 			break;
976 		}
977 
978 		/*
979 		 * Verify the css and acquire a reference.  The root
980 		 * is provided by the caller, so we know it's alive
981 		 * and kicking, and don't take an extra reference.
982 		 */
983 		memcg = mem_cgroup_from_css(css);
984 
985 		if (css == &root->css)
986 			break;
987 
988 		if (css_tryget(css))
989 			break;
990 
991 		memcg = NULL;
992 	}
993 
994 	if (reclaim) {
995 		/*
996 		 * The position could have already been updated by a competing
997 		 * thread, so check that the value hasn't changed since we read
998 		 * it to avoid reclaiming from the same cgroup twice.
999 		 */
1000 		(void)cmpxchg(&iter->position, pos, memcg);
1001 
1002 		if (pos)
1003 			css_put(&pos->css);
1004 
1005 		if (!memcg)
1006 			iter->generation++;
1007 		else if (!prev)
1008 			reclaim->generation = iter->generation;
1009 	}
1010 
1011 out_unlock:
1012 	rcu_read_unlock();
1013 out:
1014 	if (prev && prev != root)
1015 		css_put(&prev->css);
1016 
1017 	return memcg;
1018 }
1019 
1020 /**
1021  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1022  * @root: hierarchy root
1023  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1024  */
1025 void mem_cgroup_iter_break(struct mem_cgroup *root,
1026 			   struct mem_cgroup *prev)
1027 {
1028 	if (!root)
1029 		root = root_mem_cgroup;
1030 	if (prev && prev != root)
1031 		css_put(&prev->css);
1032 }
1033 
1034 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1035 {
1036 	struct mem_cgroup *memcg = dead_memcg;
1037 	struct mem_cgroup_reclaim_iter *iter;
1038 	struct mem_cgroup_per_node *mz;
1039 	int nid;
1040 	int i;
1041 
1042 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
1043 		for_each_node(nid) {
1044 			mz = mem_cgroup_nodeinfo(memcg, nid);
1045 			for (i = 0; i <= DEF_PRIORITY; i++) {
1046 				iter = &mz->iter[i];
1047 				cmpxchg(&iter->position,
1048 					dead_memcg, NULL);
1049 			}
1050 		}
1051 	}
1052 }
1053 
1054 /**
1055  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1056  * @memcg: hierarchy root
1057  * @fn: function to call for each task
1058  * @arg: argument passed to @fn
1059  *
1060  * This function iterates over tasks attached to @memcg or to any of its
1061  * descendants and calls @fn for each task. If @fn returns a non-zero
1062  * value, the function breaks the iteration loop and returns the value.
1063  * Otherwise, it will iterate over all tasks and return 0.
1064  *
1065  * This function must not be called for the root memory cgroup.
1066  */
1067 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1068 			  int (*fn)(struct task_struct *, void *), void *arg)
1069 {
1070 	struct mem_cgroup *iter;
1071 	int ret = 0;
1072 
1073 	BUG_ON(memcg == root_mem_cgroup);
1074 
1075 	for_each_mem_cgroup_tree(iter, memcg) {
1076 		struct css_task_iter it;
1077 		struct task_struct *task;
1078 
1079 		css_task_iter_start(&iter->css, 0, &it);
1080 		while (!ret && (task = css_task_iter_next(&it)))
1081 			ret = fn(task, arg);
1082 		css_task_iter_end(&it);
1083 		if (ret) {
1084 			mem_cgroup_iter_break(memcg, iter);
1085 			break;
1086 		}
1087 	}
1088 	return ret;
1089 }
1090 
1091 /**
1092  * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
1093  * @page: the page
1094  * @pgdat: pgdat of the page
1095  *
1096  * This function is only safe when following the LRU page isolation
1097  * and putback protocol: the LRU lock must be held, and the page must
1098  * either be PageLRU() or the caller must have isolated/allocated it.
1099  */
1100 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
1101 {
1102 	struct mem_cgroup_per_node *mz;
1103 	struct mem_cgroup *memcg;
1104 	struct lruvec *lruvec;
1105 
1106 	if (mem_cgroup_disabled()) {
1107 		lruvec = &pgdat->lruvec;
1108 		goto out;
1109 	}
1110 
1111 	memcg = page->mem_cgroup;
1112 	/*
1113 	 * Swapcache readahead pages are added to the LRU - and
1114 	 * possibly migrated - before they are charged.
1115 	 */
1116 	if (!memcg)
1117 		memcg = root_mem_cgroup;
1118 
1119 	mz = mem_cgroup_page_nodeinfo(memcg, page);
1120 	lruvec = &mz->lruvec;
1121 out:
1122 	/*
1123 	 * Since a node can be onlined after the mem_cgroup was created,
1124 	 * we have to be prepared to initialize lruvec->zone here;
1125 	 * and if offlined then reonlined, we need to reinitialize it.
1126 	 */
1127 	if (unlikely(lruvec->pgdat != pgdat))
1128 		lruvec->pgdat = pgdat;
1129 	return lruvec;
1130 }
1131 
1132 /**
1133  * mem_cgroup_update_lru_size - account for adding or removing an lru page
1134  * @lruvec: mem_cgroup per zone lru vector
1135  * @lru: index of lru list the page is sitting on
1136  * @zid: zone id of the accounted pages
1137  * @nr_pages: positive when adding or negative when removing
1138  *
1139  * This function must be called under lru_lock, just before a page is added
1140  * to or just after a page is removed from an lru list (that ordering being
1141  * so as to allow it to check that lru_size 0 is consistent with list_empty).
1142  */
1143 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1144 				int zid, int nr_pages)
1145 {
1146 	struct mem_cgroup_per_node *mz;
1147 	unsigned long *lru_size;
1148 	long size;
1149 
1150 	if (mem_cgroup_disabled())
1151 		return;
1152 
1153 	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1154 	lru_size = &mz->lru_zone_size[zid][lru];
1155 
1156 	if (nr_pages < 0)
1157 		*lru_size += nr_pages;
1158 
1159 	size = *lru_size;
1160 	if (WARN_ONCE(size < 0,
1161 		"%s(%p, %d, %d): lru_size %ld\n",
1162 		__func__, lruvec, lru, nr_pages, size)) {
1163 		VM_BUG_ON(1);
1164 		*lru_size = 0;
1165 	}
1166 
1167 	if (nr_pages > 0)
1168 		*lru_size += nr_pages;
1169 }
1170 
1171 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1172 {
1173 	struct mem_cgroup *task_memcg;
1174 	struct task_struct *p;
1175 	bool ret;
1176 
1177 	p = find_lock_task_mm(task);
1178 	if (p) {
1179 		task_memcg = get_mem_cgroup_from_mm(p->mm);
1180 		task_unlock(p);
1181 	} else {
1182 		/*
1183 		 * All threads may have already detached their mm's, but the oom
1184 		 * killer still needs to detect if they have already been oom
1185 		 * killed to prevent needlessly killing additional tasks.
1186 		 */
1187 		rcu_read_lock();
1188 		task_memcg = mem_cgroup_from_task(task);
1189 		css_get(&task_memcg->css);
1190 		rcu_read_unlock();
1191 	}
1192 	ret = mem_cgroup_is_descendant(task_memcg, memcg);
1193 	css_put(&task_memcg->css);
1194 	return ret;
1195 }
1196 
1197 /**
1198  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1199  * @memcg: the memory cgroup
1200  *
1201  * Returns the maximum amount of memory @mem can be charged with, in
1202  * pages.
1203  */
1204 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1205 {
1206 	unsigned long margin = 0;
1207 	unsigned long count;
1208 	unsigned long limit;
1209 
1210 	count = page_counter_read(&memcg->memory);
1211 	limit = READ_ONCE(memcg->memory.max);
1212 	if (count < limit)
1213 		margin = limit - count;
1214 
1215 	if (do_memsw_account()) {
1216 		count = page_counter_read(&memcg->memsw);
1217 		limit = READ_ONCE(memcg->memsw.max);
1218 		if (count <= limit)
1219 			margin = min(margin, limit - count);
1220 		else
1221 			margin = 0;
1222 	}
1223 
1224 	return margin;
1225 }
1226 
1227 /*
1228  * A routine for checking "mem" is under move_account() or not.
1229  *
1230  * Checking a cgroup is mc.from or mc.to or under hierarchy of
1231  * moving cgroups. This is for waiting at high-memory pressure
1232  * caused by "move".
1233  */
1234 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1235 {
1236 	struct mem_cgroup *from;
1237 	struct mem_cgroup *to;
1238 	bool ret = false;
1239 	/*
1240 	 * Unlike task_move routines, we access mc.to, mc.from not under
1241 	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1242 	 */
1243 	spin_lock(&mc.lock);
1244 	from = mc.from;
1245 	to = mc.to;
1246 	if (!from)
1247 		goto unlock;
1248 
1249 	ret = mem_cgroup_is_descendant(from, memcg) ||
1250 		mem_cgroup_is_descendant(to, memcg);
1251 unlock:
1252 	spin_unlock(&mc.lock);
1253 	return ret;
1254 }
1255 
1256 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1257 {
1258 	if (mc.moving_task && current != mc.moving_task) {
1259 		if (mem_cgroup_under_move(memcg)) {
1260 			DEFINE_WAIT(wait);
1261 			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1262 			/* moving charge context might have finished. */
1263 			if (mc.moving_task)
1264 				schedule();
1265 			finish_wait(&mc.waitq, &wait);
1266 			return true;
1267 		}
1268 	}
1269 	return false;
1270 }
1271 
1272 static const unsigned int memcg1_stats[] = {
1273 	MEMCG_CACHE,
1274 	MEMCG_RSS,
1275 	MEMCG_RSS_HUGE,
1276 	NR_SHMEM,
1277 	NR_FILE_MAPPED,
1278 	NR_FILE_DIRTY,
1279 	NR_WRITEBACK,
1280 	MEMCG_SWAP,
1281 };
1282 
1283 static const char *const memcg1_stat_names[] = {
1284 	"cache",
1285 	"rss",
1286 	"rss_huge",
1287 	"shmem",
1288 	"mapped_file",
1289 	"dirty",
1290 	"writeback",
1291 	"swap",
1292 };
1293 
1294 #define K(x) ((x) << (PAGE_SHIFT-10))
1295 /**
1296  * mem_cgroup_print_oom_context: Print OOM information relevant to
1297  * memory controller.
1298  * @memcg: The memory cgroup that went over limit
1299  * @p: Task that is going to be killed
1300  *
1301  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1302  * enabled
1303  */
1304 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1305 {
1306 	rcu_read_lock();
1307 
1308 	if (memcg) {
1309 		pr_cont(",oom_memcg=");
1310 		pr_cont_cgroup_path(memcg->css.cgroup);
1311 	} else
1312 		pr_cont(",global_oom");
1313 	if (p) {
1314 		pr_cont(",task_memcg=");
1315 		pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1316 	}
1317 	rcu_read_unlock();
1318 }
1319 
1320 /**
1321  * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
1322  * memory controller.
1323  * @memcg: The memory cgroup that went over limit
1324  */
1325 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1326 {
1327 	struct mem_cgroup *iter;
1328 	unsigned int i;
1329 
1330 	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1331 		K((u64)page_counter_read(&memcg->memory)),
1332 		K((u64)memcg->memory.max), memcg->memory.failcnt);
1333 	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1334 		K((u64)page_counter_read(&memcg->memsw)),
1335 		K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1336 	pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1337 		K((u64)page_counter_read(&memcg->kmem)),
1338 		K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1339 
1340 	for_each_mem_cgroup_tree(iter, memcg) {
1341 		pr_info("Memory cgroup stats for ");
1342 		pr_cont_cgroup_path(iter->css.cgroup);
1343 		pr_cont(":");
1344 
1345 		for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
1346 			if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)
1347 				continue;
1348 			pr_cont(" %s:%luKB", memcg1_stat_names[i],
1349 				K(memcg_page_state(iter, memcg1_stats[i])));
1350 		}
1351 
1352 		for (i = 0; i < NR_LRU_LISTS; i++)
1353 			pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1354 				K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1355 
1356 		pr_cont("\n");
1357 	}
1358 }
1359 
1360 /*
1361  * Return the memory (and swap, if configured) limit for a memcg.
1362  */
1363 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1364 {
1365 	unsigned long max;
1366 
1367 	max = memcg->memory.max;
1368 	if (mem_cgroup_swappiness(memcg)) {
1369 		unsigned long memsw_max;
1370 		unsigned long swap_max;
1371 
1372 		memsw_max = memcg->memsw.max;
1373 		swap_max = memcg->swap.max;
1374 		swap_max = min(swap_max, (unsigned long)total_swap_pages);
1375 		max = min(max + swap_max, memsw_max);
1376 	}
1377 	return max;
1378 }
1379 
1380 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1381 				     int order)
1382 {
1383 	struct oom_control oc = {
1384 		.zonelist = NULL,
1385 		.nodemask = NULL,
1386 		.memcg = memcg,
1387 		.gfp_mask = gfp_mask,
1388 		.order = order,
1389 	};
1390 	bool ret;
1391 
1392 	mutex_lock(&oom_lock);
1393 	ret = out_of_memory(&oc);
1394 	mutex_unlock(&oom_lock);
1395 	return ret;
1396 }
1397 
1398 #if MAX_NUMNODES > 1
1399 
1400 /**
1401  * test_mem_cgroup_node_reclaimable
1402  * @memcg: the target memcg
1403  * @nid: the node ID to be checked.
1404  * @noswap : specify true here if the user wants flle only information.
1405  *
1406  * This function returns whether the specified memcg contains any
1407  * reclaimable pages on a node. Returns true if there are any reclaimable
1408  * pages in the node.
1409  */
1410 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1411 		int nid, bool noswap)
1412 {
1413 	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1414 		return true;
1415 	if (noswap || !total_swap_pages)
1416 		return false;
1417 	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1418 		return true;
1419 	return false;
1420 
1421 }
1422 
1423 /*
1424  * Always updating the nodemask is not very good - even if we have an empty
1425  * list or the wrong list here, we can start from some node and traverse all
1426  * nodes based on the zonelist. So update the list loosely once per 10 secs.
1427  *
1428  */
1429 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1430 {
1431 	int nid;
1432 	/*
1433 	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1434 	 * pagein/pageout changes since the last update.
1435 	 */
1436 	if (!atomic_read(&memcg->numainfo_events))
1437 		return;
1438 	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1439 		return;
1440 
1441 	/* make a nodemask where this memcg uses memory from */
1442 	memcg->scan_nodes = node_states[N_MEMORY];
1443 
1444 	for_each_node_mask(nid, node_states[N_MEMORY]) {
1445 
1446 		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1447 			node_clear(nid, memcg->scan_nodes);
1448 	}
1449 
1450 	atomic_set(&memcg->numainfo_events, 0);
1451 	atomic_set(&memcg->numainfo_updating, 0);
1452 }
1453 
1454 /*
1455  * Selecting a node where we start reclaim from. Because what we need is just
1456  * reducing usage counter, start from anywhere is O,K. Considering
1457  * memory reclaim from current node, there are pros. and cons.
1458  *
1459  * Freeing memory from current node means freeing memory from a node which
1460  * we'll use or we've used. So, it may make LRU bad. And if several threads
1461  * hit limits, it will see a contention on a node. But freeing from remote
1462  * node means more costs for memory reclaim because of memory latency.
1463  *
1464  * Now, we use round-robin. Better algorithm is welcomed.
1465  */
1466 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1467 {
1468 	int node;
1469 
1470 	mem_cgroup_may_update_nodemask(memcg);
1471 	node = memcg->last_scanned_node;
1472 
1473 	node = next_node_in(node, memcg->scan_nodes);
1474 	/*
1475 	 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1476 	 * last time it really checked all the LRUs due to rate limiting.
1477 	 * Fallback to the current node in that case for simplicity.
1478 	 */
1479 	if (unlikely(node == MAX_NUMNODES))
1480 		node = numa_node_id();
1481 
1482 	memcg->last_scanned_node = node;
1483 	return node;
1484 }
1485 #else
1486 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1487 {
1488 	return 0;
1489 }
1490 #endif
1491 
1492 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1493 				   pg_data_t *pgdat,
1494 				   gfp_t gfp_mask,
1495 				   unsigned long *total_scanned)
1496 {
1497 	struct mem_cgroup *victim = NULL;
1498 	int total = 0;
1499 	int loop = 0;
1500 	unsigned long excess;
1501 	unsigned long nr_scanned;
1502 	struct mem_cgroup_reclaim_cookie reclaim = {
1503 		.pgdat = pgdat,
1504 		.priority = 0,
1505 	};
1506 
1507 	excess = soft_limit_excess(root_memcg);
1508 
1509 	while (1) {
1510 		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1511 		if (!victim) {
1512 			loop++;
1513 			if (loop >= 2) {
1514 				/*
1515 				 * If we have not been able to reclaim
1516 				 * anything, it might because there are
1517 				 * no reclaimable pages under this hierarchy
1518 				 */
1519 				if (!total)
1520 					break;
1521 				/*
1522 				 * We want to do more targeted reclaim.
1523 				 * excess >> 2 is not to excessive so as to
1524 				 * reclaim too much, nor too less that we keep
1525 				 * coming back to reclaim from this cgroup
1526 				 */
1527 				if (total >= (excess >> 2) ||
1528 					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1529 					break;
1530 			}
1531 			continue;
1532 		}
1533 		total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1534 					pgdat, &nr_scanned);
1535 		*total_scanned += nr_scanned;
1536 		if (!soft_limit_excess(root_memcg))
1537 			break;
1538 	}
1539 	mem_cgroup_iter_break(root_memcg, victim);
1540 	return total;
1541 }
1542 
1543 #ifdef CONFIG_LOCKDEP
1544 static struct lockdep_map memcg_oom_lock_dep_map = {
1545 	.name = "memcg_oom_lock",
1546 };
1547 #endif
1548 
1549 static DEFINE_SPINLOCK(memcg_oom_lock);
1550 
1551 /*
1552  * Check OOM-Killer is already running under our hierarchy.
1553  * If someone is running, return false.
1554  */
1555 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1556 {
1557 	struct mem_cgroup *iter, *failed = NULL;
1558 
1559 	spin_lock(&memcg_oom_lock);
1560 
1561 	for_each_mem_cgroup_tree(iter, memcg) {
1562 		if (iter->oom_lock) {
1563 			/*
1564 			 * this subtree of our hierarchy is already locked
1565 			 * so we cannot give a lock.
1566 			 */
1567 			failed = iter;
1568 			mem_cgroup_iter_break(memcg, iter);
1569 			break;
1570 		} else
1571 			iter->oom_lock = true;
1572 	}
1573 
1574 	if (failed) {
1575 		/*
1576 		 * OK, we failed to lock the whole subtree so we have
1577 		 * to clean up what we set up to the failing subtree
1578 		 */
1579 		for_each_mem_cgroup_tree(iter, memcg) {
1580 			if (iter == failed) {
1581 				mem_cgroup_iter_break(memcg, iter);
1582 				break;
1583 			}
1584 			iter->oom_lock = false;
1585 		}
1586 	} else
1587 		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1588 
1589 	spin_unlock(&memcg_oom_lock);
1590 
1591 	return !failed;
1592 }
1593 
1594 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1595 {
1596 	struct mem_cgroup *iter;
1597 
1598 	spin_lock(&memcg_oom_lock);
1599 	mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1600 	for_each_mem_cgroup_tree(iter, memcg)
1601 		iter->oom_lock = false;
1602 	spin_unlock(&memcg_oom_lock);
1603 }
1604 
1605 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1606 {
1607 	struct mem_cgroup *iter;
1608 
1609 	spin_lock(&memcg_oom_lock);
1610 	for_each_mem_cgroup_tree(iter, memcg)
1611 		iter->under_oom++;
1612 	spin_unlock(&memcg_oom_lock);
1613 }
1614 
1615 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1616 {
1617 	struct mem_cgroup *iter;
1618 
1619 	/*
1620 	 * When a new child is created while the hierarchy is under oom,
1621 	 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
1622 	 */
1623 	spin_lock(&memcg_oom_lock);
1624 	for_each_mem_cgroup_tree(iter, memcg)
1625 		if (iter->under_oom > 0)
1626 			iter->under_oom--;
1627 	spin_unlock(&memcg_oom_lock);
1628 }
1629 
1630 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1631 
1632 struct oom_wait_info {
1633 	struct mem_cgroup *memcg;
1634 	wait_queue_entry_t	wait;
1635 };
1636 
1637 static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1638 	unsigned mode, int sync, void *arg)
1639 {
1640 	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1641 	struct mem_cgroup *oom_wait_memcg;
1642 	struct oom_wait_info *oom_wait_info;
1643 
1644 	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1645 	oom_wait_memcg = oom_wait_info->memcg;
1646 
1647 	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1648 	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1649 		return 0;
1650 	return autoremove_wake_function(wait, mode, sync, arg);
1651 }
1652 
1653 static void memcg_oom_recover(struct mem_cgroup *memcg)
1654 {
1655 	/*
1656 	 * For the following lockless ->under_oom test, the only required
1657 	 * guarantee is that it must see the state asserted by an OOM when
1658 	 * this function is called as a result of userland actions
1659 	 * triggered by the notification of the OOM.  This is trivially
1660 	 * achieved by invoking mem_cgroup_mark_under_oom() before
1661 	 * triggering notification.
1662 	 */
1663 	if (memcg && memcg->under_oom)
1664 		__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1665 }
1666 
1667 enum oom_status {
1668 	OOM_SUCCESS,
1669 	OOM_FAILED,
1670 	OOM_ASYNC,
1671 	OOM_SKIPPED
1672 };
1673 
1674 static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1675 {
1676 	enum oom_status ret;
1677 	bool locked;
1678 
1679 	if (order > PAGE_ALLOC_COSTLY_ORDER)
1680 		return OOM_SKIPPED;
1681 
1682 	memcg_memory_event(memcg, MEMCG_OOM);
1683 
1684 	/*
1685 	 * We are in the middle of the charge context here, so we
1686 	 * don't want to block when potentially sitting on a callstack
1687 	 * that holds all kinds of filesystem and mm locks.
1688 	 *
1689 	 * cgroup1 allows disabling the OOM killer and waiting for outside
1690 	 * handling until the charge can succeed; remember the context and put
1691 	 * the task to sleep at the end of the page fault when all locks are
1692 	 * released.
1693 	 *
1694 	 * On the other hand, in-kernel OOM killer allows for an async victim
1695 	 * memory reclaim (oom_reaper) and that means that we are not solely
1696 	 * relying on the oom victim to make a forward progress and we can
1697 	 * invoke the oom killer here.
1698 	 *
1699 	 * Please note that mem_cgroup_out_of_memory might fail to find a
1700 	 * victim and then we have to bail out from the charge path.
1701 	 */
1702 	if (memcg->oom_kill_disable) {
1703 		if (!current->in_user_fault)
1704 			return OOM_SKIPPED;
1705 		css_get(&memcg->css);
1706 		current->memcg_in_oom = memcg;
1707 		current->memcg_oom_gfp_mask = mask;
1708 		current->memcg_oom_order = order;
1709 
1710 		return OOM_ASYNC;
1711 	}
1712 
1713 	mem_cgroup_mark_under_oom(memcg);
1714 
1715 	locked = mem_cgroup_oom_trylock(memcg);
1716 
1717 	if (locked)
1718 		mem_cgroup_oom_notify(memcg);
1719 
1720 	mem_cgroup_unmark_under_oom(memcg);
1721 	if (mem_cgroup_out_of_memory(memcg, mask, order))
1722 		ret = OOM_SUCCESS;
1723 	else
1724 		ret = OOM_FAILED;
1725 
1726 	if (locked)
1727 		mem_cgroup_oom_unlock(memcg);
1728 
1729 	return ret;
1730 }
1731 
1732 /**
1733  * mem_cgroup_oom_synchronize - complete memcg OOM handling
1734  * @handle: actually kill/wait or just clean up the OOM state
1735  *
1736  * This has to be called at the end of a page fault if the memcg OOM
1737  * handler was enabled.
1738  *
1739  * Memcg supports userspace OOM handling where failed allocations must
1740  * sleep on a waitqueue until the userspace task resolves the
1741  * situation.  Sleeping directly in the charge context with all kinds
1742  * of locks held is not a good idea, instead we remember an OOM state
1743  * in the task and mem_cgroup_oom_synchronize() has to be called at
1744  * the end of the page fault to complete the OOM handling.
1745  *
1746  * Returns %true if an ongoing memcg OOM situation was detected and
1747  * completed, %false otherwise.
1748  */
1749 bool mem_cgroup_oom_synchronize(bool handle)
1750 {
1751 	struct mem_cgroup *memcg = current->memcg_in_oom;
1752 	struct oom_wait_info owait;
1753 	bool locked;
1754 
1755 	/* OOM is global, do not handle */
1756 	if (!memcg)
1757 		return false;
1758 
1759 	if (!handle)
1760 		goto cleanup;
1761 
1762 	owait.memcg = memcg;
1763 	owait.wait.flags = 0;
1764 	owait.wait.func = memcg_oom_wake_function;
1765 	owait.wait.private = current;
1766 	INIT_LIST_HEAD(&owait.wait.entry);
1767 
1768 	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1769 	mem_cgroup_mark_under_oom(memcg);
1770 
1771 	locked = mem_cgroup_oom_trylock(memcg);
1772 
1773 	if (locked)
1774 		mem_cgroup_oom_notify(memcg);
1775 
1776 	if (locked && !memcg->oom_kill_disable) {
1777 		mem_cgroup_unmark_under_oom(memcg);
1778 		finish_wait(&memcg_oom_waitq, &owait.wait);
1779 		mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1780 					 current->memcg_oom_order);
1781 	} else {
1782 		schedule();
1783 		mem_cgroup_unmark_under_oom(memcg);
1784 		finish_wait(&memcg_oom_waitq, &owait.wait);
1785 	}
1786 
1787 	if (locked) {
1788 		mem_cgroup_oom_unlock(memcg);
1789 		/*
1790 		 * There is no guarantee that an OOM-lock contender
1791 		 * sees the wakeups triggered by the OOM kill
1792 		 * uncharges.  Wake any sleepers explicitely.
1793 		 */
1794 		memcg_oom_recover(memcg);
1795 	}
1796 cleanup:
1797 	current->memcg_in_oom = NULL;
1798 	css_put(&memcg->css);
1799 	return true;
1800 }
1801 
1802 /**
1803  * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
1804  * @victim: task to be killed by the OOM killer
1805  * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
1806  *
1807  * Returns a pointer to a memory cgroup, which has to be cleaned up
1808  * by killing all belonging OOM-killable tasks.
1809  *
1810  * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
1811  */
1812 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
1813 					    struct mem_cgroup *oom_domain)
1814 {
1815 	struct mem_cgroup *oom_group = NULL;
1816 	struct mem_cgroup *memcg;
1817 
1818 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1819 		return NULL;
1820 
1821 	if (!oom_domain)
1822 		oom_domain = root_mem_cgroup;
1823 
1824 	rcu_read_lock();
1825 
1826 	memcg = mem_cgroup_from_task(victim);
1827 	if (memcg == root_mem_cgroup)
1828 		goto out;
1829 
1830 	/*
1831 	 * Traverse the memory cgroup hierarchy from the victim task's
1832 	 * cgroup up to the OOMing cgroup (or root) to find the
1833 	 * highest-level memory cgroup with oom.group set.
1834 	 */
1835 	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
1836 		if (memcg->oom_group)
1837 			oom_group = memcg;
1838 
1839 		if (memcg == oom_domain)
1840 			break;
1841 	}
1842 
1843 	if (oom_group)
1844 		css_get(&oom_group->css);
1845 out:
1846 	rcu_read_unlock();
1847 
1848 	return oom_group;
1849 }
1850 
1851 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
1852 {
1853 	pr_info("Tasks in ");
1854 	pr_cont_cgroup_path(memcg->css.cgroup);
1855 	pr_cont(" are going to be killed due to memory.oom.group set\n");
1856 }
1857 
1858 /**
1859  * lock_page_memcg - lock a page->mem_cgroup binding
1860  * @page: the page
1861  *
1862  * This function protects unlocked LRU pages from being moved to
1863  * another cgroup.
1864  *
1865  * It ensures lifetime of the returned memcg. Caller is responsible
1866  * for the lifetime of the page; __unlock_page_memcg() is available
1867  * when @page might get freed inside the locked section.
1868  */
1869 struct mem_cgroup *lock_page_memcg(struct page *page)
1870 {
1871 	struct mem_cgroup *memcg;
1872 	unsigned long flags;
1873 
1874 	/*
1875 	 * The RCU lock is held throughout the transaction.  The fast
1876 	 * path can get away without acquiring the memcg->move_lock
1877 	 * because page moving starts with an RCU grace period.
1878 	 *
1879 	 * The RCU lock also protects the memcg from being freed when
1880 	 * the page state that is going to change is the only thing
1881 	 * preventing the page itself from being freed. E.g. writeback
1882 	 * doesn't hold a page reference and relies on PG_writeback to
1883 	 * keep off truncation, migration and so forth.
1884          */
1885 	rcu_read_lock();
1886 
1887 	if (mem_cgroup_disabled())
1888 		return NULL;
1889 again:
1890 	memcg = page->mem_cgroup;
1891 	if (unlikely(!memcg))
1892 		return NULL;
1893 
1894 	if (atomic_read(&memcg->moving_account) <= 0)
1895 		return memcg;
1896 
1897 	spin_lock_irqsave(&memcg->move_lock, flags);
1898 	if (memcg != page->mem_cgroup) {
1899 		spin_unlock_irqrestore(&memcg->move_lock, flags);
1900 		goto again;
1901 	}
1902 
1903 	/*
1904 	 * When charge migration first begins, we can have locked and
1905 	 * unlocked page stat updates happening concurrently.  Track
1906 	 * the task who has the lock for unlock_page_memcg().
1907 	 */
1908 	memcg->move_lock_task = current;
1909 	memcg->move_lock_flags = flags;
1910 
1911 	return memcg;
1912 }
1913 EXPORT_SYMBOL(lock_page_memcg);
1914 
1915 /**
1916  * __unlock_page_memcg - unlock and unpin a memcg
1917  * @memcg: the memcg
1918  *
1919  * Unlock and unpin a memcg returned by lock_page_memcg().
1920  */
1921 void __unlock_page_memcg(struct mem_cgroup *memcg)
1922 {
1923 	if (memcg && memcg->move_lock_task == current) {
1924 		unsigned long flags = memcg->move_lock_flags;
1925 
1926 		memcg->move_lock_task = NULL;
1927 		memcg->move_lock_flags = 0;
1928 
1929 		spin_unlock_irqrestore(&memcg->move_lock, flags);
1930 	}
1931 
1932 	rcu_read_unlock();
1933 }
1934 
1935 /**
1936  * unlock_page_memcg - unlock a page->mem_cgroup binding
1937  * @page: the page
1938  */
1939 void unlock_page_memcg(struct page *page)
1940 {
1941 	__unlock_page_memcg(page->mem_cgroup);
1942 }
1943 EXPORT_SYMBOL(unlock_page_memcg);
1944 
1945 struct memcg_stock_pcp {
1946 	struct mem_cgroup *cached; /* this never be root cgroup */
1947 	unsigned int nr_pages;
1948 	struct work_struct work;
1949 	unsigned long flags;
1950 #define FLUSHING_CACHED_CHARGE	0
1951 };
1952 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1953 static DEFINE_MUTEX(percpu_charge_mutex);
1954 
1955 /**
1956  * consume_stock: Try to consume stocked charge on this cpu.
1957  * @memcg: memcg to consume from.
1958  * @nr_pages: how many pages to charge.
1959  *
1960  * The charges will only happen if @memcg matches the current cpu's memcg
1961  * stock, and at least @nr_pages are available in that stock.  Failure to
1962  * service an allocation will refill the stock.
1963  *
1964  * returns true if successful, false otherwise.
1965  */
1966 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1967 {
1968 	struct memcg_stock_pcp *stock;
1969 	unsigned long flags;
1970 	bool ret = false;
1971 
1972 	if (nr_pages > MEMCG_CHARGE_BATCH)
1973 		return ret;
1974 
1975 	local_irq_save(flags);
1976 
1977 	stock = this_cpu_ptr(&memcg_stock);
1978 	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1979 		stock->nr_pages -= nr_pages;
1980 		ret = true;
1981 	}
1982 
1983 	local_irq_restore(flags);
1984 
1985 	return ret;
1986 }
1987 
1988 /*
1989  * Returns stocks cached in percpu and reset cached information.
1990  */
1991 static void drain_stock(struct memcg_stock_pcp *stock)
1992 {
1993 	struct mem_cgroup *old = stock->cached;
1994 
1995 	if (stock->nr_pages) {
1996 		page_counter_uncharge(&old->memory, stock->nr_pages);
1997 		if (do_memsw_account())
1998 			page_counter_uncharge(&old->memsw, stock->nr_pages);
1999 		css_put_many(&old->css, stock->nr_pages);
2000 		stock->nr_pages = 0;
2001 	}
2002 	stock->cached = NULL;
2003 }
2004 
2005 static void drain_local_stock(struct work_struct *dummy)
2006 {
2007 	struct memcg_stock_pcp *stock;
2008 	unsigned long flags;
2009 
2010 	/*
2011 	 * The only protection from memory hotplug vs. drain_stock races is
2012 	 * that we always operate on local CPU stock here with IRQ disabled
2013 	 */
2014 	local_irq_save(flags);
2015 
2016 	stock = this_cpu_ptr(&memcg_stock);
2017 	drain_stock(stock);
2018 	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2019 
2020 	local_irq_restore(flags);
2021 }
2022 
2023 /*
2024  * Cache charges(val) to local per_cpu area.
2025  * This will be consumed by consume_stock() function, later.
2026  */
2027 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2028 {
2029 	struct memcg_stock_pcp *stock;
2030 	unsigned long flags;
2031 
2032 	local_irq_save(flags);
2033 
2034 	stock = this_cpu_ptr(&memcg_stock);
2035 	if (stock->cached != memcg) { /* reset if necessary */
2036 		drain_stock(stock);
2037 		stock->cached = memcg;
2038 	}
2039 	stock->nr_pages += nr_pages;
2040 
2041 	if (stock->nr_pages > MEMCG_CHARGE_BATCH)
2042 		drain_stock(stock);
2043 
2044 	local_irq_restore(flags);
2045 }
2046 
2047 /*
2048  * Drains all per-CPU charge caches for given root_memcg resp. subtree
2049  * of the hierarchy under it.
2050  */
2051 static void drain_all_stock(struct mem_cgroup *root_memcg)
2052 {
2053 	int cpu, curcpu;
2054 
2055 	/* If someone's already draining, avoid adding running more workers. */
2056 	if (!mutex_trylock(&percpu_charge_mutex))
2057 		return;
2058 	/*
2059 	 * Notify other cpus that system-wide "drain" is running
2060 	 * We do not care about races with the cpu hotplug because cpu down
2061 	 * as well as workers from this path always operate on the local
2062 	 * per-cpu data. CPU up doesn't touch memcg_stock at all.
2063 	 */
2064 	curcpu = get_cpu();
2065 	for_each_online_cpu(cpu) {
2066 		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2067 		struct mem_cgroup *memcg;
2068 
2069 		memcg = stock->cached;
2070 		if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css))
2071 			continue;
2072 		if (!mem_cgroup_is_descendant(memcg, root_memcg)) {
2073 			css_put(&memcg->css);
2074 			continue;
2075 		}
2076 		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2077 			if (cpu == curcpu)
2078 				drain_local_stock(&stock->work);
2079 			else
2080 				schedule_work_on(cpu, &stock->work);
2081 		}
2082 		css_put(&memcg->css);
2083 	}
2084 	put_cpu();
2085 	mutex_unlock(&percpu_charge_mutex);
2086 }
2087 
2088 static int memcg_hotplug_cpu_dead(unsigned int cpu)
2089 {
2090 	struct memcg_stock_pcp *stock;
2091 	struct mem_cgroup *memcg;
2092 
2093 	stock = &per_cpu(memcg_stock, cpu);
2094 	drain_stock(stock);
2095 
2096 	for_each_mem_cgroup(memcg) {
2097 		int i;
2098 
2099 		for (i = 0; i < MEMCG_NR_STAT; i++) {
2100 			int nid;
2101 			long x;
2102 
2103 			x = this_cpu_xchg(memcg->stat_cpu->count[i], 0);
2104 			if (x)
2105 				atomic_long_add(x, &memcg->stat[i]);
2106 
2107 			if (i >= NR_VM_NODE_STAT_ITEMS)
2108 				continue;
2109 
2110 			for_each_node(nid) {
2111 				struct mem_cgroup_per_node *pn;
2112 
2113 				pn = mem_cgroup_nodeinfo(memcg, nid);
2114 				x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
2115 				if (x)
2116 					atomic_long_add(x, &pn->lruvec_stat[i]);
2117 			}
2118 		}
2119 
2120 		for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
2121 			long x;
2122 
2123 			x = this_cpu_xchg(memcg->stat_cpu->events[i], 0);
2124 			if (x)
2125 				atomic_long_add(x, &memcg->events[i]);
2126 		}
2127 	}
2128 
2129 	return 0;
2130 }
2131 
2132 static void reclaim_high(struct mem_cgroup *memcg,
2133 			 unsigned int nr_pages,
2134 			 gfp_t gfp_mask)
2135 {
2136 	do {
2137 		if (page_counter_read(&memcg->memory) <= memcg->high)
2138 			continue;
2139 		memcg_memory_event(memcg, MEMCG_HIGH);
2140 		try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
2141 	} while ((memcg = parent_mem_cgroup(memcg)));
2142 }
2143 
2144 static void high_work_func(struct work_struct *work)
2145 {
2146 	struct mem_cgroup *memcg;
2147 
2148 	memcg = container_of(work, struct mem_cgroup, high_work);
2149 	reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2150 }
2151 
2152 /*
2153  * Scheduled by try_charge() to be executed from the userland return path
2154  * and reclaims memory over the high limit.
2155  */
2156 void mem_cgroup_handle_over_high(void)
2157 {
2158 	unsigned int nr_pages = current->memcg_nr_pages_over_high;
2159 	struct mem_cgroup *memcg;
2160 
2161 	if (likely(!nr_pages))
2162 		return;
2163 
2164 	memcg = get_mem_cgroup_from_mm(current->mm);
2165 	reclaim_high(memcg, nr_pages, GFP_KERNEL);
2166 	css_put(&memcg->css);
2167 	current->memcg_nr_pages_over_high = 0;
2168 }
2169 
2170 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2171 		      unsigned int nr_pages)
2172 {
2173 	unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2174 	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2175 	struct mem_cgroup *mem_over_limit;
2176 	struct page_counter *counter;
2177 	unsigned long nr_reclaimed;
2178 	bool may_swap = true;
2179 	bool drained = false;
2180 	bool oomed = false;
2181 	enum oom_status oom_status;
2182 
2183 	if (mem_cgroup_is_root(memcg))
2184 		return 0;
2185 retry:
2186 	if (consume_stock(memcg, nr_pages))
2187 		return 0;
2188 
2189 	if (!do_memsw_account() ||
2190 	    page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2191 		if (page_counter_try_charge(&memcg->memory, batch, &counter))
2192 			goto done_restock;
2193 		if (do_memsw_account())
2194 			page_counter_uncharge(&memcg->memsw, batch);
2195 		mem_over_limit = mem_cgroup_from_counter(counter, memory);
2196 	} else {
2197 		mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2198 		may_swap = false;
2199 	}
2200 
2201 	if (batch > nr_pages) {
2202 		batch = nr_pages;
2203 		goto retry;
2204 	}
2205 
2206 	/*
2207 	 * Unlike in global OOM situations, memcg is not in a physical
2208 	 * memory shortage.  Allow dying and OOM-killed tasks to
2209 	 * bypass the last charges so that they can exit quickly and
2210 	 * free their memory.
2211 	 */
2212 	if (unlikely(tsk_is_oom_victim(current) ||
2213 		     fatal_signal_pending(current) ||
2214 		     current->flags & PF_EXITING))
2215 		goto force;
2216 
2217 	/*
2218 	 * Prevent unbounded recursion when reclaim operations need to
2219 	 * allocate memory. This might exceed the limits temporarily,
2220 	 * but we prefer facilitating memory reclaim and getting back
2221 	 * under the limit over triggering OOM kills in these cases.
2222 	 */
2223 	if (unlikely(current->flags & PF_MEMALLOC))
2224 		goto force;
2225 
2226 	if (unlikely(task_in_memcg_oom(current)))
2227 		goto nomem;
2228 
2229 	if (!gfpflags_allow_blocking(gfp_mask))
2230 		goto nomem;
2231 
2232 	memcg_memory_event(mem_over_limit, MEMCG_MAX);
2233 
2234 	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2235 						    gfp_mask, may_swap);
2236 
2237 	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2238 		goto retry;
2239 
2240 	if (!drained) {
2241 		drain_all_stock(mem_over_limit);
2242 		drained = true;
2243 		goto retry;
2244 	}
2245 
2246 	if (gfp_mask & __GFP_NORETRY)
2247 		goto nomem;
2248 	/*
2249 	 * Even though the limit is exceeded at this point, reclaim
2250 	 * may have been able to free some pages.  Retry the charge
2251 	 * before killing the task.
2252 	 *
2253 	 * Only for regular pages, though: huge pages are rather
2254 	 * unlikely to succeed so close to the limit, and we fall back
2255 	 * to regular pages anyway in case of failure.
2256 	 */
2257 	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2258 		goto retry;
2259 	/*
2260 	 * At task move, charge accounts can be doubly counted. So, it's
2261 	 * better to wait until the end of task_move if something is going on.
2262 	 */
2263 	if (mem_cgroup_wait_acct_move(mem_over_limit))
2264 		goto retry;
2265 
2266 	if (nr_retries--)
2267 		goto retry;
2268 
2269 	if (gfp_mask & __GFP_RETRY_MAYFAIL && oomed)
2270 		goto nomem;
2271 
2272 	if (gfp_mask & __GFP_NOFAIL)
2273 		goto force;
2274 
2275 	if (fatal_signal_pending(current))
2276 		goto force;
2277 
2278 	/*
2279 	 * keep retrying as long as the memcg oom killer is able to make
2280 	 * a forward progress or bypass the charge if the oom killer
2281 	 * couldn't make any progress.
2282 	 */
2283 	oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
2284 		       get_order(nr_pages * PAGE_SIZE));
2285 	switch (oom_status) {
2286 	case OOM_SUCCESS:
2287 		nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2288 		oomed = true;
2289 		goto retry;
2290 	case OOM_FAILED:
2291 		goto force;
2292 	default:
2293 		goto nomem;
2294 	}
2295 nomem:
2296 	if (!(gfp_mask & __GFP_NOFAIL))
2297 		return -ENOMEM;
2298 force:
2299 	/*
2300 	 * The allocation either can't fail or will lead to more memory
2301 	 * being freed very soon.  Allow memory usage go over the limit
2302 	 * temporarily by force charging it.
2303 	 */
2304 	page_counter_charge(&memcg->memory, nr_pages);
2305 	if (do_memsw_account())
2306 		page_counter_charge(&memcg->memsw, nr_pages);
2307 	css_get_many(&memcg->css, nr_pages);
2308 
2309 	return 0;
2310 
2311 done_restock:
2312 	css_get_many(&memcg->css, batch);
2313 	if (batch > nr_pages)
2314 		refill_stock(memcg, batch - nr_pages);
2315 
2316 	/*
2317 	 * If the hierarchy is above the normal consumption range, schedule
2318 	 * reclaim on returning to userland.  We can perform reclaim here
2319 	 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2320 	 * GFP_KERNEL can consistently be used during reclaim.  @memcg is
2321 	 * not recorded as it most likely matches current's and won't
2322 	 * change in the meantime.  As high limit is checked again before
2323 	 * reclaim, the cost of mismatch is negligible.
2324 	 */
2325 	do {
2326 		if (page_counter_read(&memcg->memory) > memcg->high) {
2327 			/* Don't bother a random interrupted task */
2328 			if (in_interrupt()) {
2329 				schedule_work(&memcg->high_work);
2330 				break;
2331 			}
2332 			current->memcg_nr_pages_over_high += batch;
2333 			set_notify_resume(current);
2334 			break;
2335 		}
2336 	} while ((memcg = parent_mem_cgroup(memcg)));
2337 
2338 	return 0;
2339 }
2340 
2341 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2342 {
2343 	if (mem_cgroup_is_root(memcg))
2344 		return;
2345 
2346 	page_counter_uncharge(&memcg->memory, nr_pages);
2347 	if (do_memsw_account())
2348 		page_counter_uncharge(&memcg->memsw, nr_pages);
2349 
2350 	css_put_many(&memcg->css, nr_pages);
2351 }
2352 
2353 static void lock_page_lru(struct page *page, int *isolated)
2354 {
2355 	struct zone *zone = page_zone(page);
2356 
2357 	spin_lock_irq(zone_lru_lock(zone));
2358 	if (PageLRU(page)) {
2359 		struct lruvec *lruvec;
2360 
2361 		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2362 		ClearPageLRU(page);
2363 		del_page_from_lru_list(page, lruvec, page_lru(page));
2364 		*isolated = 1;
2365 	} else
2366 		*isolated = 0;
2367 }
2368 
2369 static void unlock_page_lru(struct page *page, int isolated)
2370 {
2371 	struct zone *zone = page_zone(page);
2372 
2373 	if (isolated) {
2374 		struct lruvec *lruvec;
2375 
2376 		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2377 		VM_BUG_ON_PAGE(PageLRU(page), page);
2378 		SetPageLRU(page);
2379 		add_page_to_lru_list(page, lruvec, page_lru(page));
2380 	}
2381 	spin_unlock_irq(zone_lru_lock(zone));
2382 }
2383 
2384 static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2385 			  bool lrucare)
2386 {
2387 	int isolated;
2388 
2389 	VM_BUG_ON_PAGE(page->mem_cgroup, page);
2390 
2391 	/*
2392 	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2393 	 * may already be on some other mem_cgroup's LRU.  Take care of it.
2394 	 */
2395 	if (lrucare)
2396 		lock_page_lru(page, &isolated);
2397 
2398 	/*
2399 	 * Nobody should be changing or seriously looking at
2400 	 * page->mem_cgroup at this point:
2401 	 *
2402 	 * - the page is uncharged
2403 	 *
2404 	 * - the page is off-LRU
2405 	 *
2406 	 * - an anonymous fault has exclusive page access, except for
2407 	 *   a locked page table
2408 	 *
2409 	 * - a page cache insertion, a swapin fault, or a migration
2410 	 *   have the page locked
2411 	 */
2412 	page->mem_cgroup = memcg;
2413 
2414 	if (lrucare)
2415 		unlock_page_lru(page, isolated);
2416 }
2417 
2418 #ifdef CONFIG_MEMCG_KMEM
2419 static int memcg_alloc_cache_id(void)
2420 {
2421 	int id, size;
2422 	int err;
2423 
2424 	id = ida_simple_get(&memcg_cache_ida,
2425 			    0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2426 	if (id < 0)
2427 		return id;
2428 
2429 	if (id < memcg_nr_cache_ids)
2430 		return id;
2431 
2432 	/*
2433 	 * There's no space for the new id in memcg_caches arrays,
2434 	 * so we have to grow them.
2435 	 */
2436 	down_write(&memcg_cache_ids_sem);
2437 
2438 	size = 2 * (id + 1);
2439 	if (size < MEMCG_CACHES_MIN_SIZE)
2440 		size = MEMCG_CACHES_MIN_SIZE;
2441 	else if (size > MEMCG_CACHES_MAX_SIZE)
2442 		size = MEMCG_CACHES_MAX_SIZE;
2443 
2444 	err = memcg_update_all_caches(size);
2445 	if (!err)
2446 		err = memcg_update_all_list_lrus(size);
2447 	if (!err)
2448 		memcg_nr_cache_ids = size;
2449 
2450 	up_write(&memcg_cache_ids_sem);
2451 
2452 	if (err) {
2453 		ida_simple_remove(&memcg_cache_ida, id);
2454 		return err;
2455 	}
2456 	return id;
2457 }
2458 
2459 static void memcg_free_cache_id(int id)
2460 {
2461 	ida_simple_remove(&memcg_cache_ida, id);
2462 }
2463 
2464 struct memcg_kmem_cache_create_work {
2465 	struct mem_cgroup *memcg;
2466 	struct kmem_cache *cachep;
2467 	struct work_struct work;
2468 };
2469 
2470 static void memcg_kmem_cache_create_func(struct work_struct *w)
2471 {
2472 	struct memcg_kmem_cache_create_work *cw =
2473 		container_of(w, struct memcg_kmem_cache_create_work, work);
2474 	struct mem_cgroup *memcg = cw->memcg;
2475 	struct kmem_cache *cachep = cw->cachep;
2476 
2477 	memcg_create_kmem_cache(memcg, cachep);
2478 
2479 	css_put(&memcg->css);
2480 	kfree(cw);
2481 }
2482 
2483 /*
2484  * Enqueue the creation of a per-memcg kmem_cache.
2485  */
2486 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2487 					       struct kmem_cache *cachep)
2488 {
2489 	struct memcg_kmem_cache_create_work *cw;
2490 
2491 	cw = kmalloc(sizeof(*cw), GFP_NOWAIT | __GFP_NOWARN);
2492 	if (!cw)
2493 		return;
2494 
2495 	css_get(&memcg->css);
2496 
2497 	cw->memcg = memcg;
2498 	cw->cachep = cachep;
2499 	INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2500 
2501 	queue_work(memcg_kmem_cache_wq, &cw->work);
2502 }
2503 
2504 static inline bool memcg_kmem_bypass(void)
2505 {
2506 	if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2507 		return true;
2508 	return false;
2509 }
2510 
2511 /**
2512  * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2513  * @cachep: the original global kmem cache
2514  *
2515  * Return the kmem_cache we're supposed to use for a slab allocation.
2516  * We try to use the current memcg's version of the cache.
2517  *
2518  * If the cache does not exist yet, if we are the first user of it, we
2519  * create it asynchronously in a workqueue and let the current allocation
2520  * go through with the original cache.
2521  *
2522  * This function takes a reference to the cache it returns to assure it
2523  * won't get destroyed while we are working with it. Once the caller is
2524  * done with it, memcg_kmem_put_cache() must be called to release the
2525  * reference.
2526  */
2527 struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2528 {
2529 	struct mem_cgroup *memcg;
2530 	struct kmem_cache *memcg_cachep;
2531 	int kmemcg_id;
2532 
2533 	VM_BUG_ON(!is_root_cache(cachep));
2534 
2535 	if (memcg_kmem_bypass())
2536 		return cachep;
2537 
2538 	memcg = get_mem_cgroup_from_current();
2539 	kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2540 	if (kmemcg_id < 0)
2541 		goto out;
2542 
2543 	memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2544 	if (likely(memcg_cachep))
2545 		return memcg_cachep;
2546 
2547 	/*
2548 	 * If we are in a safe context (can wait, and not in interrupt
2549 	 * context), we could be be predictable and return right away.
2550 	 * This would guarantee that the allocation being performed
2551 	 * already belongs in the new cache.
2552 	 *
2553 	 * However, there are some clashes that can arrive from locking.
2554 	 * For instance, because we acquire the slab_mutex while doing
2555 	 * memcg_create_kmem_cache, this means no further allocation
2556 	 * could happen with the slab_mutex held. So it's better to
2557 	 * defer everything.
2558 	 */
2559 	memcg_schedule_kmem_cache_create(memcg, cachep);
2560 out:
2561 	css_put(&memcg->css);
2562 	return cachep;
2563 }
2564 
2565 /**
2566  * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2567  * @cachep: the cache returned by memcg_kmem_get_cache
2568  */
2569 void memcg_kmem_put_cache(struct kmem_cache *cachep)
2570 {
2571 	if (!is_root_cache(cachep))
2572 		css_put(&cachep->memcg_params.memcg->css);
2573 }
2574 
2575 /**
2576  * memcg_kmem_charge_memcg: charge a kmem page
2577  * @page: page to charge
2578  * @gfp: reclaim mode
2579  * @order: allocation order
2580  * @memcg: memory cgroup to charge
2581  *
2582  * Returns 0 on success, an error code on failure.
2583  */
2584 int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2585 			    struct mem_cgroup *memcg)
2586 {
2587 	unsigned int nr_pages = 1 << order;
2588 	struct page_counter *counter;
2589 	int ret;
2590 
2591 	ret = try_charge(memcg, gfp, nr_pages);
2592 	if (ret)
2593 		return ret;
2594 
2595 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2596 	    !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2597 		cancel_charge(memcg, nr_pages);
2598 		return -ENOMEM;
2599 	}
2600 
2601 	page->mem_cgroup = memcg;
2602 
2603 	return 0;
2604 }
2605 
2606 /**
2607  * memcg_kmem_charge: charge a kmem page to the current memory cgroup
2608  * @page: page to charge
2609  * @gfp: reclaim mode
2610  * @order: allocation order
2611  *
2612  * Returns 0 on success, an error code on failure.
2613  */
2614 int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2615 {
2616 	struct mem_cgroup *memcg;
2617 	int ret = 0;
2618 
2619 	if (mem_cgroup_disabled() || memcg_kmem_bypass())
2620 		return 0;
2621 
2622 	memcg = get_mem_cgroup_from_current();
2623 	if (!mem_cgroup_is_root(memcg)) {
2624 		ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
2625 		if (!ret)
2626 			__SetPageKmemcg(page);
2627 	}
2628 	css_put(&memcg->css);
2629 	return ret;
2630 }
2631 /**
2632  * memcg_kmem_uncharge: uncharge a kmem page
2633  * @page: page to uncharge
2634  * @order: allocation order
2635  */
2636 void memcg_kmem_uncharge(struct page *page, int order)
2637 {
2638 	struct mem_cgroup *memcg = page->mem_cgroup;
2639 	unsigned int nr_pages = 1 << order;
2640 
2641 	if (!memcg)
2642 		return;
2643 
2644 	VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2645 
2646 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2647 		page_counter_uncharge(&memcg->kmem, nr_pages);
2648 
2649 	page_counter_uncharge(&memcg->memory, nr_pages);
2650 	if (do_memsw_account())
2651 		page_counter_uncharge(&memcg->memsw, nr_pages);
2652 
2653 	page->mem_cgroup = NULL;
2654 
2655 	/* slab pages do not have PageKmemcg flag set */
2656 	if (PageKmemcg(page))
2657 		__ClearPageKmemcg(page);
2658 
2659 	css_put_many(&memcg->css, nr_pages);
2660 }
2661 #endif /* CONFIG_MEMCG_KMEM */
2662 
2663 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2664 
2665 /*
2666  * Because tail pages are not marked as "used", set it. We're under
2667  * zone_lru_lock and migration entries setup in all page mappings.
2668  */
2669 void mem_cgroup_split_huge_fixup(struct page *head)
2670 {
2671 	int i;
2672 
2673 	if (mem_cgroup_disabled())
2674 		return;
2675 
2676 	for (i = 1; i < HPAGE_PMD_NR; i++)
2677 		head[i].mem_cgroup = head->mem_cgroup;
2678 
2679 	__mod_memcg_state(head->mem_cgroup, MEMCG_RSS_HUGE, -HPAGE_PMD_NR);
2680 }
2681 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2682 
2683 #ifdef CONFIG_MEMCG_SWAP
2684 /**
2685  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2686  * @entry: swap entry to be moved
2687  * @from:  mem_cgroup which the entry is moved from
2688  * @to:  mem_cgroup which the entry is moved to
2689  *
2690  * It succeeds only when the swap_cgroup's record for this entry is the same
2691  * as the mem_cgroup's id of @from.
2692  *
2693  * Returns 0 on success, -EINVAL on failure.
2694  *
2695  * The caller must have charged to @to, IOW, called page_counter_charge() about
2696  * both res and memsw, and called css_get().
2697  */
2698 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2699 				struct mem_cgroup *from, struct mem_cgroup *to)
2700 {
2701 	unsigned short old_id, new_id;
2702 
2703 	old_id = mem_cgroup_id(from);
2704 	new_id = mem_cgroup_id(to);
2705 
2706 	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2707 		mod_memcg_state(from, MEMCG_SWAP, -1);
2708 		mod_memcg_state(to, MEMCG_SWAP, 1);
2709 		return 0;
2710 	}
2711 	return -EINVAL;
2712 }
2713 #else
2714 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2715 				struct mem_cgroup *from, struct mem_cgroup *to)
2716 {
2717 	return -EINVAL;
2718 }
2719 #endif
2720 
2721 static DEFINE_MUTEX(memcg_max_mutex);
2722 
2723 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
2724 				 unsigned long max, bool memsw)
2725 {
2726 	bool enlarge = false;
2727 	bool drained = false;
2728 	int ret;
2729 	bool limits_invariant;
2730 	struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
2731 
2732 	do {
2733 		if (signal_pending(current)) {
2734 			ret = -EINTR;
2735 			break;
2736 		}
2737 
2738 		mutex_lock(&memcg_max_mutex);
2739 		/*
2740 		 * Make sure that the new limit (memsw or memory limit) doesn't
2741 		 * break our basic invariant rule memory.max <= memsw.max.
2742 		 */
2743 		limits_invariant = memsw ? max >= memcg->memory.max :
2744 					   max <= memcg->memsw.max;
2745 		if (!limits_invariant) {
2746 			mutex_unlock(&memcg_max_mutex);
2747 			ret = -EINVAL;
2748 			break;
2749 		}
2750 		if (max > counter->max)
2751 			enlarge = true;
2752 		ret = page_counter_set_max(counter, max);
2753 		mutex_unlock(&memcg_max_mutex);
2754 
2755 		if (!ret)
2756 			break;
2757 
2758 		if (!drained) {
2759 			drain_all_stock(memcg);
2760 			drained = true;
2761 			continue;
2762 		}
2763 
2764 		if (!try_to_free_mem_cgroup_pages(memcg, 1,
2765 					GFP_KERNEL, !memsw)) {
2766 			ret = -EBUSY;
2767 			break;
2768 		}
2769 	} while (true);
2770 
2771 	if (!ret && enlarge)
2772 		memcg_oom_recover(memcg);
2773 
2774 	return ret;
2775 }
2776 
2777 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2778 					    gfp_t gfp_mask,
2779 					    unsigned long *total_scanned)
2780 {
2781 	unsigned long nr_reclaimed = 0;
2782 	struct mem_cgroup_per_node *mz, *next_mz = NULL;
2783 	unsigned long reclaimed;
2784 	int loop = 0;
2785 	struct mem_cgroup_tree_per_node *mctz;
2786 	unsigned long excess;
2787 	unsigned long nr_scanned;
2788 
2789 	if (order > 0)
2790 		return 0;
2791 
2792 	mctz = soft_limit_tree_node(pgdat->node_id);
2793 
2794 	/*
2795 	 * Do not even bother to check the largest node if the root
2796 	 * is empty. Do it lockless to prevent lock bouncing. Races
2797 	 * are acceptable as soft limit is best effort anyway.
2798 	 */
2799 	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
2800 		return 0;
2801 
2802 	/*
2803 	 * This loop can run a while, specially if mem_cgroup's continuously
2804 	 * keep exceeding their soft limit and putting the system under
2805 	 * pressure
2806 	 */
2807 	do {
2808 		if (next_mz)
2809 			mz = next_mz;
2810 		else
2811 			mz = mem_cgroup_largest_soft_limit_node(mctz);
2812 		if (!mz)
2813 			break;
2814 
2815 		nr_scanned = 0;
2816 		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2817 						    gfp_mask, &nr_scanned);
2818 		nr_reclaimed += reclaimed;
2819 		*total_scanned += nr_scanned;
2820 		spin_lock_irq(&mctz->lock);
2821 		__mem_cgroup_remove_exceeded(mz, mctz);
2822 
2823 		/*
2824 		 * If we failed to reclaim anything from this memory cgroup
2825 		 * it is time to move on to the next cgroup
2826 		 */
2827 		next_mz = NULL;
2828 		if (!reclaimed)
2829 			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2830 
2831 		excess = soft_limit_excess(mz->memcg);
2832 		/*
2833 		 * One school of thought says that we should not add
2834 		 * back the node to the tree if reclaim returns 0.
2835 		 * But our reclaim could return 0, simply because due
2836 		 * to priority we are exposing a smaller subset of
2837 		 * memory to reclaim from. Consider this as a longer
2838 		 * term TODO.
2839 		 */
2840 		/* If excess == 0, no tree ops */
2841 		__mem_cgroup_insert_exceeded(mz, mctz, excess);
2842 		spin_unlock_irq(&mctz->lock);
2843 		css_put(&mz->memcg->css);
2844 		loop++;
2845 		/*
2846 		 * Could not reclaim anything and there are no more
2847 		 * mem cgroups to try or we seem to be looping without
2848 		 * reclaiming anything.
2849 		 */
2850 		if (!nr_reclaimed &&
2851 			(next_mz == NULL ||
2852 			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2853 			break;
2854 	} while (!nr_reclaimed);
2855 	if (next_mz)
2856 		css_put(&next_mz->memcg->css);
2857 	return nr_reclaimed;
2858 }
2859 
2860 /*
2861  * Test whether @memcg has children, dead or alive.  Note that this
2862  * function doesn't care whether @memcg has use_hierarchy enabled and
2863  * returns %true if there are child csses according to the cgroup
2864  * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
2865  */
2866 static inline bool memcg_has_children(struct mem_cgroup *memcg)
2867 {
2868 	bool ret;
2869 
2870 	rcu_read_lock();
2871 	ret = css_next_child(NULL, &memcg->css);
2872 	rcu_read_unlock();
2873 	return ret;
2874 }
2875 
2876 /*
2877  * Reclaims as many pages from the given memcg as possible.
2878  *
2879  * Caller is responsible for holding css reference for memcg.
2880  */
2881 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2882 {
2883 	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2884 
2885 	/* we call try-to-free pages for make this cgroup empty */
2886 	lru_add_drain_all();
2887 
2888 	drain_all_stock(memcg);
2889 
2890 	/* try to free all pages in this cgroup */
2891 	while (nr_retries && page_counter_read(&memcg->memory)) {
2892 		int progress;
2893 
2894 		if (signal_pending(current))
2895 			return -EINTR;
2896 
2897 		progress = try_to_free_mem_cgroup_pages(memcg, 1,
2898 							GFP_KERNEL, true);
2899 		if (!progress) {
2900 			nr_retries--;
2901 			/* maybe some writeback is necessary */
2902 			congestion_wait(BLK_RW_ASYNC, HZ/10);
2903 		}
2904 
2905 	}
2906 
2907 	return 0;
2908 }
2909 
2910 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2911 					    char *buf, size_t nbytes,
2912 					    loff_t off)
2913 {
2914 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2915 
2916 	if (mem_cgroup_is_root(memcg))
2917 		return -EINVAL;
2918 	return mem_cgroup_force_empty(memcg) ?: nbytes;
2919 }
2920 
2921 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2922 				     struct cftype *cft)
2923 {
2924 	return mem_cgroup_from_css(css)->use_hierarchy;
2925 }
2926 
2927 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2928 				      struct cftype *cft, u64 val)
2929 {
2930 	int retval = 0;
2931 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2932 	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2933 
2934 	if (memcg->use_hierarchy == val)
2935 		return 0;
2936 
2937 	/*
2938 	 * If parent's use_hierarchy is set, we can't make any modifications
2939 	 * in the child subtrees. If it is unset, then the change can
2940 	 * occur, provided the current cgroup has no children.
2941 	 *
2942 	 * For the root cgroup, parent_mem is NULL, we allow value to be
2943 	 * set if there are no children.
2944 	 */
2945 	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2946 				(val == 1 || val == 0)) {
2947 		if (!memcg_has_children(memcg))
2948 			memcg->use_hierarchy = val;
2949 		else
2950 			retval = -EBUSY;
2951 	} else
2952 		retval = -EINVAL;
2953 
2954 	return retval;
2955 }
2956 
2957 struct accumulated_stats {
2958 	unsigned long stat[MEMCG_NR_STAT];
2959 	unsigned long events[NR_VM_EVENT_ITEMS];
2960 	unsigned long lru_pages[NR_LRU_LISTS];
2961 	const unsigned int *stats_array;
2962 	const unsigned int *events_array;
2963 	int stats_size;
2964 	int events_size;
2965 };
2966 
2967 static void accumulate_memcg_tree(struct mem_cgroup *memcg,
2968 				  struct accumulated_stats *acc)
2969 {
2970 	struct mem_cgroup *mi;
2971 	int i;
2972 
2973 	for_each_mem_cgroup_tree(mi, memcg) {
2974 		for (i = 0; i < acc->stats_size; i++)
2975 			acc->stat[i] += memcg_page_state(mi,
2976 				acc->stats_array ? acc->stats_array[i] : i);
2977 
2978 		for (i = 0; i < acc->events_size; i++)
2979 			acc->events[i] += memcg_sum_events(mi,
2980 				acc->events_array ? acc->events_array[i] : i);
2981 
2982 		for (i = 0; i < NR_LRU_LISTS; i++)
2983 			acc->lru_pages[i] +=
2984 				mem_cgroup_nr_lru_pages(mi, BIT(i));
2985 	}
2986 }
2987 
2988 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2989 {
2990 	unsigned long val = 0;
2991 
2992 	if (mem_cgroup_is_root(memcg)) {
2993 		struct mem_cgroup *iter;
2994 
2995 		for_each_mem_cgroup_tree(iter, memcg) {
2996 			val += memcg_page_state(iter, MEMCG_CACHE);
2997 			val += memcg_page_state(iter, MEMCG_RSS);
2998 			if (swap)
2999 				val += memcg_page_state(iter, MEMCG_SWAP);
3000 		}
3001 	} else {
3002 		if (!swap)
3003 			val = page_counter_read(&memcg->memory);
3004 		else
3005 			val = page_counter_read(&memcg->memsw);
3006 	}
3007 	return val;
3008 }
3009 
3010 enum {
3011 	RES_USAGE,
3012 	RES_LIMIT,
3013 	RES_MAX_USAGE,
3014 	RES_FAILCNT,
3015 	RES_SOFT_LIMIT,
3016 };
3017 
3018 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3019 			       struct cftype *cft)
3020 {
3021 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3022 	struct page_counter *counter;
3023 
3024 	switch (MEMFILE_TYPE(cft->private)) {
3025 	case _MEM:
3026 		counter = &memcg->memory;
3027 		break;
3028 	case _MEMSWAP:
3029 		counter = &memcg->memsw;
3030 		break;
3031 	case _KMEM:
3032 		counter = &memcg->kmem;
3033 		break;
3034 	case _TCP:
3035 		counter = &memcg->tcpmem;
3036 		break;
3037 	default:
3038 		BUG();
3039 	}
3040 
3041 	switch (MEMFILE_ATTR(cft->private)) {
3042 	case RES_USAGE:
3043 		if (counter == &memcg->memory)
3044 			return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3045 		if (counter == &memcg->memsw)
3046 			return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3047 		return (u64)page_counter_read(counter) * PAGE_SIZE;
3048 	case RES_LIMIT:
3049 		return (u64)counter->max * PAGE_SIZE;
3050 	case RES_MAX_USAGE:
3051 		return (u64)counter->watermark * PAGE_SIZE;
3052 	case RES_FAILCNT:
3053 		return counter->failcnt;
3054 	case RES_SOFT_LIMIT:
3055 		return (u64)memcg->soft_limit * PAGE_SIZE;
3056 	default:
3057 		BUG();
3058 	}
3059 }
3060 
3061 #ifdef CONFIG_MEMCG_KMEM
3062 static int memcg_online_kmem(struct mem_cgroup *memcg)
3063 {
3064 	int memcg_id;
3065 
3066 	if (cgroup_memory_nokmem)
3067 		return 0;
3068 
3069 	BUG_ON(memcg->kmemcg_id >= 0);
3070 	BUG_ON(memcg->kmem_state);
3071 
3072 	memcg_id = memcg_alloc_cache_id();
3073 	if (memcg_id < 0)
3074 		return memcg_id;
3075 
3076 	static_branch_inc(&memcg_kmem_enabled_key);
3077 	/*
3078 	 * A memory cgroup is considered kmem-online as soon as it gets
3079 	 * kmemcg_id. Setting the id after enabling static branching will
3080 	 * guarantee no one starts accounting before all call sites are
3081 	 * patched.
3082 	 */
3083 	memcg->kmemcg_id = memcg_id;
3084 	memcg->kmem_state = KMEM_ONLINE;
3085 	INIT_LIST_HEAD(&memcg->kmem_caches);
3086 
3087 	return 0;
3088 }
3089 
3090 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3091 {
3092 	struct cgroup_subsys_state *css;
3093 	struct mem_cgroup *parent, *child;
3094 	int kmemcg_id;
3095 
3096 	if (memcg->kmem_state != KMEM_ONLINE)
3097 		return;
3098 	/*
3099 	 * Clear the online state before clearing memcg_caches array
3100 	 * entries. The slab_mutex in memcg_deactivate_kmem_caches()
3101 	 * guarantees that no cache will be created for this cgroup
3102 	 * after we are done (see memcg_create_kmem_cache()).
3103 	 */
3104 	memcg->kmem_state = KMEM_ALLOCATED;
3105 
3106 	memcg_deactivate_kmem_caches(memcg);
3107 
3108 	kmemcg_id = memcg->kmemcg_id;
3109 	BUG_ON(kmemcg_id < 0);
3110 
3111 	parent = parent_mem_cgroup(memcg);
3112 	if (!parent)
3113 		parent = root_mem_cgroup;
3114 
3115 	/*
3116 	 * Change kmemcg_id of this cgroup and all its descendants to the
3117 	 * parent's id, and then move all entries from this cgroup's list_lrus
3118 	 * to ones of the parent. After we have finished, all list_lrus
3119 	 * corresponding to this cgroup are guaranteed to remain empty. The
3120 	 * ordering is imposed by list_lru_node->lock taken by
3121 	 * memcg_drain_all_list_lrus().
3122 	 */
3123 	rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
3124 	css_for_each_descendant_pre(css, &memcg->css) {
3125 		child = mem_cgroup_from_css(css);
3126 		BUG_ON(child->kmemcg_id != kmemcg_id);
3127 		child->kmemcg_id = parent->kmemcg_id;
3128 		if (!memcg->use_hierarchy)
3129 			break;
3130 	}
3131 	rcu_read_unlock();
3132 
3133 	memcg_drain_all_list_lrus(kmemcg_id, parent);
3134 
3135 	memcg_free_cache_id(kmemcg_id);
3136 }
3137 
3138 static void memcg_free_kmem(struct mem_cgroup *memcg)
3139 {
3140 	/* css_alloc() failed, offlining didn't happen */
3141 	if (unlikely(memcg->kmem_state == KMEM_ONLINE))
3142 		memcg_offline_kmem(memcg);
3143 
3144 	if (memcg->kmem_state == KMEM_ALLOCATED) {
3145 		memcg_destroy_kmem_caches(memcg);
3146 		static_branch_dec(&memcg_kmem_enabled_key);
3147 		WARN_ON(page_counter_read(&memcg->kmem));
3148 	}
3149 }
3150 #else
3151 static int memcg_online_kmem(struct mem_cgroup *memcg)
3152 {
3153 	return 0;
3154 }
3155 static void memcg_offline_kmem(struct mem_cgroup *memcg)
3156 {
3157 }
3158 static void memcg_free_kmem(struct mem_cgroup *memcg)
3159 {
3160 }
3161 #endif /* CONFIG_MEMCG_KMEM */
3162 
3163 static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3164 				 unsigned long max)
3165 {
3166 	int ret;
3167 
3168 	mutex_lock(&memcg_max_mutex);
3169 	ret = page_counter_set_max(&memcg->kmem, max);
3170 	mutex_unlock(&memcg_max_mutex);
3171 	return ret;
3172 }
3173 
3174 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3175 {
3176 	int ret;
3177 
3178 	mutex_lock(&memcg_max_mutex);
3179 
3180 	ret = page_counter_set_max(&memcg->tcpmem, max);
3181 	if (ret)
3182 		goto out;
3183 
3184 	if (!memcg->tcpmem_active) {
3185 		/*
3186 		 * The active flag needs to be written after the static_key
3187 		 * update. This is what guarantees that the socket activation
3188 		 * function is the last one to run. See mem_cgroup_sk_alloc()
3189 		 * for details, and note that we don't mark any socket as
3190 		 * belonging to this memcg until that flag is up.
3191 		 *
3192 		 * We need to do this, because static_keys will span multiple
3193 		 * sites, but we can't control their order. If we mark a socket
3194 		 * as accounted, but the accounting functions are not patched in
3195 		 * yet, we'll lose accounting.
3196 		 *
3197 		 * We never race with the readers in mem_cgroup_sk_alloc(),
3198 		 * because when this value change, the code to process it is not
3199 		 * patched in yet.
3200 		 */
3201 		static_branch_inc(&memcg_sockets_enabled_key);
3202 		memcg->tcpmem_active = true;
3203 	}
3204 out:
3205 	mutex_unlock(&memcg_max_mutex);
3206 	return ret;
3207 }
3208 
3209 /*
3210  * The user of this function is...
3211  * RES_LIMIT.
3212  */
3213 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3214 				char *buf, size_t nbytes, loff_t off)
3215 {
3216 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3217 	unsigned long nr_pages;
3218 	int ret;
3219 
3220 	buf = strstrip(buf);
3221 	ret = page_counter_memparse(buf, "-1", &nr_pages);
3222 	if (ret)
3223 		return ret;
3224 
3225 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3226 	case RES_LIMIT:
3227 		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3228 			ret = -EINVAL;
3229 			break;
3230 		}
3231 		switch (MEMFILE_TYPE(of_cft(of)->private)) {
3232 		case _MEM:
3233 			ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3234 			break;
3235 		case _MEMSWAP:
3236 			ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3237 			break;
3238 		case _KMEM:
3239 			ret = memcg_update_kmem_max(memcg, nr_pages);
3240 			break;
3241 		case _TCP:
3242 			ret = memcg_update_tcp_max(memcg, nr_pages);
3243 			break;
3244 		}
3245 		break;
3246 	case RES_SOFT_LIMIT:
3247 		memcg->soft_limit = nr_pages;
3248 		ret = 0;
3249 		break;
3250 	}
3251 	return ret ?: nbytes;
3252 }
3253 
3254 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3255 				size_t nbytes, loff_t off)
3256 {
3257 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3258 	struct page_counter *counter;
3259 
3260 	switch (MEMFILE_TYPE(of_cft(of)->private)) {
3261 	case _MEM:
3262 		counter = &memcg->memory;
3263 		break;
3264 	case _MEMSWAP:
3265 		counter = &memcg->memsw;
3266 		break;
3267 	case _KMEM:
3268 		counter = &memcg->kmem;
3269 		break;
3270 	case _TCP:
3271 		counter = &memcg->tcpmem;
3272 		break;
3273 	default:
3274 		BUG();
3275 	}
3276 
3277 	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3278 	case RES_MAX_USAGE:
3279 		page_counter_reset_watermark(counter);
3280 		break;
3281 	case RES_FAILCNT:
3282 		counter->failcnt = 0;
3283 		break;
3284 	default:
3285 		BUG();
3286 	}
3287 
3288 	return nbytes;
3289 }
3290 
3291 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3292 					struct cftype *cft)
3293 {
3294 	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3295 }
3296 
3297 #ifdef CONFIG_MMU
3298 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3299 					struct cftype *cft, u64 val)
3300 {
3301 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3302 
3303 	if (val & ~MOVE_MASK)
3304 		return -EINVAL;
3305 
3306 	/*
3307 	 * No kind of locking is needed in here, because ->can_attach() will
3308 	 * check this value once in the beginning of the process, and then carry
3309 	 * on with stale data. This means that changes to this value will only
3310 	 * affect task migrations starting after the change.
3311 	 */
3312 	memcg->move_charge_at_immigrate = val;
3313 	return 0;
3314 }
3315 #else
3316 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3317 					struct cftype *cft, u64 val)
3318 {
3319 	return -ENOSYS;
3320 }
3321 #endif
3322 
3323 #ifdef CONFIG_NUMA
3324 static int memcg_numa_stat_show(struct seq_file *m, void *v)
3325 {
3326 	struct numa_stat {
3327 		const char *name;
3328 		unsigned int lru_mask;
3329 	};
3330 
3331 	static const struct numa_stat stats[] = {
3332 		{ "total", LRU_ALL },
3333 		{ "file", LRU_ALL_FILE },
3334 		{ "anon", LRU_ALL_ANON },
3335 		{ "unevictable", BIT(LRU_UNEVICTABLE) },
3336 	};
3337 	const struct numa_stat *stat;
3338 	int nid;
3339 	unsigned long nr;
3340 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3341 
3342 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3343 		nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3344 		seq_printf(m, "%s=%lu", stat->name, nr);
3345 		for_each_node_state(nid, N_MEMORY) {
3346 			nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3347 							  stat->lru_mask);
3348 			seq_printf(m, " N%d=%lu", nid, nr);
3349 		}
3350 		seq_putc(m, '\n');
3351 	}
3352 
3353 	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3354 		struct mem_cgroup *iter;
3355 
3356 		nr = 0;
3357 		for_each_mem_cgroup_tree(iter, memcg)
3358 			nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3359 		seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3360 		for_each_node_state(nid, N_MEMORY) {
3361 			nr = 0;
3362 			for_each_mem_cgroup_tree(iter, memcg)
3363 				nr += mem_cgroup_node_nr_lru_pages(
3364 					iter, nid, stat->lru_mask);
3365 			seq_printf(m, " N%d=%lu", nid, nr);
3366 		}
3367 		seq_putc(m, '\n');
3368 	}
3369 
3370 	return 0;
3371 }
3372 #endif /* CONFIG_NUMA */
3373 
3374 /* Universal VM events cgroup1 shows, original sort order */
3375 static const unsigned int memcg1_events[] = {
3376 	PGPGIN,
3377 	PGPGOUT,
3378 	PGFAULT,
3379 	PGMAJFAULT,
3380 };
3381 
3382 static const char *const memcg1_event_names[] = {
3383 	"pgpgin",
3384 	"pgpgout",
3385 	"pgfault",
3386 	"pgmajfault",
3387 };
3388 
3389 static int memcg_stat_show(struct seq_file *m, void *v)
3390 {
3391 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3392 	unsigned long memory, memsw;
3393 	struct mem_cgroup *mi;
3394 	unsigned int i;
3395 	struct accumulated_stats acc;
3396 
3397 	BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
3398 	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3399 
3400 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3401 		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3402 			continue;
3403 		seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
3404 			   memcg_page_state(memcg, memcg1_stats[i]) *
3405 			   PAGE_SIZE);
3406 	}
3407 
3408 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3409 		seq_printf(m, "%s %lu\n", memcg1_event_names[i],
3410 			   memcg_sum_events(memcg, memcg1_events[i]));
3411 
3412 	for (i = 0; i < NR_LRU_LISTS; i++)
3413 		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3414 			   mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3415 
3416 	/* Hierarchical information */
3417 	memory = memsw = PAGE_COUNTER_MAX;
3418 	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3419 		memory = min(memory, mi->memory.max);
3420 		memsw = min(memsw, mi->memsw.max);
3421 	}
3422 	seq_printf(m, "hierarchical_memory_limit %llu\n",
3423 		   (u64)memory * PAGE_SIZE);
3424 	if (do_memsw_account())
3425 		seq_printf(m, "hierarchical_memsw_limit %llu\n",
3426 			   (u64)memsw * PAGE_SIZE);
3427 
3428 	memset(&acc, 0, sizeof(acc));
3429 	acc.stats_size = ARRAY_SIZE(memcg1_stats);
3430 	acc.stats_array = memcg1_stats;
3431 	acc.events_size = ARRAY_SIZE(memcg1_events);
3432 	acc.events_array = memcg1_events;
3433 	accumulate_memcg_tree(memcg, &acc);
3434 
3435 	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3436 		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3437 			continue;
3438 		seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
3439 			   (u64)acc.stat[i] * PAGE_SIZE);
3440 	}
3441 
3442 	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3443 		seq_printf(m, "total_%s %llu\n", memcg1_event_names[i],
3444 			   (u64)acc.events[i]);
3445 
3446 	for (i = 0; i < NR_LRU_LISTS; i++)
3447 		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i],
3448 			   (u64)acc.lru_pages[i] * PAGE_SIZE);
3449 
3450 #ifdef CONFIG_DEBUG_VM
3451 	{
3452 		pg_data_t *pgdat;
3453 		struct mem_cgroup_per_node *mz;
3454 		struct zone_reclaim_stat *rstat;
3455 		unsigned long recent_rotated[2] = {0, 0};
3456 		unsigned long recent_scanned[2] = {0, 0};
3457 
3458 		for_each_online_pgdat(pgdat) {
3459 			mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3460 			rstat = &mz->lruvec.reclaim_stat;
3461 
3462 			recent_rotated[0] += rstat->recent_rotated[0];
3463 			recent_rotated[1] += rstat->recent_rotated[1];
3464 			recent_scanned[0] += rstat->recent_scanned[0];
3465 			recent_scanned[1] += rstat->recent_scanned[1];
3466 		}
3467 		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3468 		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3469 		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3470 		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3471 	}
3472 #endif
3473 
3474 	return 0;
3475 }
3476 
3477 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3478 				      struct cftype *cft)
3479 {
3480 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3481 
3482 	return mem_cgroup_swappiness(memcg);
3483 }
3484 
3485 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3486 				       struct cftype *cft, u64 val)
3487 {
3488 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3489 
3490 	if (val > 100)
3491 		return -EINVAL;
3492 
3493 	if (css->parent)
3494 		memcg->swappiness = val;
3495 	else
3496 		vm_swappiness = val;
3497 
3498 	return 0;
3499 }
3500 
3501 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3502 {
3503 	struct mem_cgroup_threshold_ary *t;
3504 	unsigned long usage;
3505 	int i;
3506 
3507 	rcu_read_lock();
3508 	if (!swap)
3509 		t = rcu_dereference(memcg->thresholds.primary);
3510 	else
3511 		t = rcu_dereference(memcg->memsw_thresholds.primary);
3512 
3513 	if (!t)
3514 		goto unlock;
3515 
3516 	usage = mem_cgroup_usage(memcg, swap);
3517 
3518 	/*
3519 	 * current_threshold points to threshold just below or equal to usage.
3520 	 * If it's not true, a threshold was crossed after last
3521 	 * call of __mem_cgroup_threshold().
3522 	 */
3523 	i = t->current_threshold;
3524 
3525 	/*
3526 	 * Iterate backward over array of thresholds starting from
3527 	 * current_threshold and check if a threshold is crossed.
3528 	 * If none of thresholds below usage is crossed, we read
3529 	 * only one element of the array here.
3530 	 */
3531 	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3532 		eventfd_signal(t->entries[i].eventfd, 1);
3533 
3534 	/* i = current_threshold + 1 */
3535 	i++;
3536 
3537 	/*
3538 	 * Iterate forward over array of thresholds starting from
3539 	 * current_threshold+1 and check if a threshold is crossed.
3540 	 * If none of thresholds above usage is crossed, we read
3541 	 * only one element of the array here.
3542 	 */
3543 	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3544 		eventfd_signal(t->entries[i].eventfd, 1);
3545 
3546 	/* Update current_threshold */
3547 	t->current_threshold = i - 1;
3548 unlock:
3549 	rcu_read_unlock();
3550 }
3551 
3552 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3553 {
3554 	while (memcg) {
3555 		__mem_cgroup_threshold(memcg, false);
3556 		if (do_memsw_account())
3557 			__mem_cgroup_threshold(memcg, true);
3558 
3559 		memcg = parent_mem_cgroup(memcg);
3560 	}
3561 }
3562 
3563 static int compare_thresholds(const void *a, const void *b)
3564 {
3565 	const struct mem_cgroup_threshold *_a = a;
3566 	const struct mem_cgroup_threshold *_b = b;
3567 
3568 	if (_a->threshold > _b->threshold)
3569 		return 1;
3570 
3571 	if (_a->threshold < _b->threshold)
3572 		return -1;
3573 
3574 	return 0;
3575 }
3576 
3577 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3578 {
3579 	struct mem_cgroup_eventfd_list *ev;
3580 
3581 	spin_lock(&memcg_oom_lock);
3582 
3583 	list_for_each_entry(ev, &memcg->oom_notify, list)
3584 		eventfd_signal(ev->eventfd, 1);
3585 
3586 	spin_unlock(&memcg_oom_lock);
3587 	return 0;
3588 }
3589 
3590 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3591 {
3592 	struct mem_cgroup *iter;
3593 
3594 	for_each_mem_cgroup_tree(iter, memcg)
3595 		mem_cgroup_oom_notify_cb(iter);
3596 }
3597 
3598 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3599 	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3600 {
3601 	struct mem_cgroup_thresholds *thresholds;
3602 	struct mem_cgroup_threshold_ary *new;
3603 	unsigned long threshold;
3604 	unsigned long usage;
3605 	int i, size, ret;
3606 
3607 	ret = page_counter_memparse(args, "-1", &threshold);
3608 	if (ret)
3609 		return ret;
3610 
3611 	mutex_lock(&memcg->thresholds_lock);
3612 
3613 	if (type == _MEM) {
3614 		thresholds = &memcg->thresholds;
3615 		usage = mem_cgroup_usage(memcg, false);
3616 	} else if (type == _MEMSWAP) {
3617 		thresholds = &memcg->memsw_thresholds;
3618 		usage = mem_cgroup_usage(memcg, true);
3619 	} else
3620 		BUG();
3621 
3622 	/* Check if a threshold crossed before adding a new one */
3623 	if (thresholds->primary)
3624 		__mem_cgroup_threshold(memcg, type == _MEMSWAP);
3625 
3626 	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3627 
3628 	/* Allocate memory for new array of thresholds */
3629 	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3630 			GFP_KERNEL);
3631 	if (!new) {
3632 		ret = -ENOMEM;
3633 		goto unlock;
3634 	}
3635 	new->size = size;
3636 
3637 	/* Copy thresholds (if any) to new array */
3638 	if (thresholds->primary) {
3639 		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3640 				sizeof(struct mem_cgroup_threshold));
3641 	}
3642 
3643 	/* Add new threshold */
3644 	new->entries[size - 1].eventfd = eventfd;
3645 	new->entries[size - 1].threshold = threshold;
3646 
3647 	/* Sort thresholds. Registering of new threshold isn't time-critical */
3648 	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3649 			compare_thresholds, NULL);
3650 
3651 	/* Find current threshold */
3652 	new->current_threshold = -1;
3653 	for (i = 0; i < size; i++) {
3654 		if (new->entries[i].threshold <= usage) {
3655 			/*
3656 			 * new->current_threshold will not be used until
3657 			 * rcu_assign_pointer(), so it's safe to increment
3658 			 * it here.
3659 			 */
3660 			++new->current_threshold;
3661 		} else
3662 			break;
3663 	}
3664 
3665 	/* Free old spare buffer and save old primary buffer as spare */
3666 	kfree(thresholds->spare);
3667 	thresholds->spare = thresholds->primary;
3668 
3669 	rcu_assign_pointer(thresholds->primary, new);
3670 
3671 	/* To be sure that nobody uses thresholds */
3672 	synchronize_rcu();
3673 
3674 unlock:
3675 	mutex_unlock(&memcg->thresholds_lock);
3676 
3677 	return ret;
3678 }
3679 
3680 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3681 	struct eventfd_ctx *eventfd, const char *args)
3682 {
3683 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3684 }
3685 
3686 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3687 	struct eventfd_ctx *eventfd, const char *args)
3688 {
3689 	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3690 }
3691 
3692 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3693 	struct eventfd_ctx *eventfd, enum res_type type)
3694 {
3695 	struct mem_cgroup_thresholds *thresholds;
3696 	struct mem_cgroup_threshold_ary *new;
3697 	unsigned long usage;
3698 	int i, j, size;
3699 
3700 	mutex_lock(&memcg->thresholds_lock);
3701 
3702 	if (type == _MEM) {
3703 		thresholds = &memcg->thresholds;
3704 		usage = mem_cgroup_usage(memcg, false);
3705 	} else if (type == _MEMSWAP) {
3706 		thresholds = &memcg->memsw_thresholds;
3707 		usage = mem_cgroup_usage(memcg, true);
3708 	} else
3709 		BUG();
3710 
3711 	if (!thresholds->primary)
3712 		goto unlock;
3713 
3714 	/* Check if a threshold crossed before removing */
3715 	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
3716 
3717 	/* Calculate new number of threshold */
3718 	size = 0;
3719 	for (i = 0; i < thresholds->primary->size; i++) {
3720 		if (thresholds->primary->entries[i].eventfd != eventfd)
3721 			size++;
3722 	}
3723 
3724 	new = thresholds->spare;
3725 
3726 	/* Set thresholds array to NULL if we don't have thresholds */
3727 	if (!size) {
3728 		kfree(new);
3729 		new = NULL;
3730 		goto swap_buffers;
3731 	}
3732 
3733 	new->size = size;
3734 
3735 	/* Copy thresholds and find current threshold */
3736 	new->current_threshold = -1;
3737 	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3738 		if (thresholds->primary->entries[i].eventfd == eventfd)
3739 			continue;
3740 
3741 		new->entries[j] = thresholds->primary->entries[i];
3742 		if (new->entries[j].threshold <= usage) {
3743 			/*
3744 			 * new->current_threshold will not be used
3745 			 * until rcu_assign_pointer(), so it's safe to increment
3746 			 * it here.
3747 			 */
3748 			++new->current_threshold;
3749 		}
3750 		j++;
3751 	}
3752 
3753 swap_buffers:
3754 	/* Swap primary and spare array */
3755 	thresholds->spare = thresholds->primary;
3756 
3757 	rcu_assign_pointer(thresholds->primary, new);
3758 
3759 	/* To be sure that nobody uses thresholds */
3760 	synchronize_rcu();
3761 
3762 	/* If all events are unregistered, free the spare array */
3763 	if (!new) {
3764 		kfree(thresholds->spare);
3765 		thresholds->spare = NULL;
3766 	}
3767 unlock:
3768 	mutex_unlock(&memcg->thresholds_lock);
3769 }
3770 
3771 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3772 	struct eventfd_ctx *eventfd)
3773 {
3774 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
3775 }
3776 
3777 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3778 	struct eventfd_ctx *eventfd)
3779 {
3780 	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
3781 }
3782 
3783 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
3784 	struct eventfd_ctx *eventfd, const char *args)
3785 {
3786 	struct mem_cgroup_eventfd_list *event;
3787 
3788 	event = kmalloc(sizeof(*event),	GFP_KERNEL);
3789 	if (!event)
3790 		return -ENOMEM;
3791 
3792 	spin_lock(&memcg_oom_lock);
3793 
3794 	event->eventfd = eventfd;
3795 	list_add(&event->list, &memcg->oom_notify);
3796 
3797 	/* already in OOM ? */
3798 	if (memcg->under_oom)
3799 		eventfd_signal(eventfd, 1);
3800 	spin_unlock(&memcg_oom_lock);
3801 
3802 	return 0;
3803 }
3804 
3805 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
3806 	struct eventfd_ctx *eventfd)
3807 {
3808 	struct mem_cgroup_eventfd_list *ev, *tmp;
3809 
3810 	spin_lock(&memcg_oom_lock);
3811 
3812 	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
3813 		if (ev->eventfd == eventfd) {
3814 			list_del(&ev->list);
3815 			kfree(ev);
3816 		}
3817 	}
3818 
3819 	spin_unlock(&memcg_oom_lock);
3820 }
3821 
3822 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3823 {
3824 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3825 
3826 	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3827 	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3828 	seq_printf(sf, "oom_kill %lu\n",
3829 		   atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
3830 	return 0;
3831 }
3832 
3833 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3834 	struct cftype *cft, u64 val)
3835 {
3836 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3837 
3838 	/* cannot set to root cgroup and only 0 and 1 are allowed */
3839 	if (!css->parent || !((val == 0) || (val == 1)))
3840 		return -EINVAL;
3841 
3842 	memcg->oom_kill_disable = val;
3843 	if (!val)
3844 		memcg_oom_recover(memcg);
3845 
3846 	return 0;
3847 }
3848 
3849 #ifdef CONFIG_CGROUP_WRITEBACK
3850 
3851 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3852 {
3853 	return wb_domain_init(&memcg->cgwb_domain, gfp);
3854 }
3855 
3856 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3857 {
3858 	wb_domain_exit(&memcg->cgwb_domain);
3859 }
3860 
3861 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3862 {
3863 	wb_domain_size_changed(&memcg->cgwb_domain);
3864 }
3865 
3866 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3867 {
3868 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3869 
3870 	if (!memcg->css.parent)
3871 		return NULL;
3872 
3873 	return &memcg->cgwb_domain;
3874 }
3875 
3876 /**
3877  * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
3878  * @wb: bdi_writeback in question
3879  * @pfilepages: out parameter for number of file pages
3880  * @pheadroom: out parameter for number of allocatable pages according to memcg
3881  * @pdirty: out parameter for number of dirty pages
3882  * @pwriteback: out parameter for number of pages under writeback
3883  *
3884  * Determine the numbers of file, headroom, dirty, and writeback pages in
3885  * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
3886  * is a bit more involved.
3887  *
3888  * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
3889  * headroom is calculated as the lowest headroom of itself and the
3890  * ancestors.  Note that this doesn't consider the actual amount of
3891  * available memory in the system.  The caller should further cap
3892  * *@pheadroom accordingly.
3893  */
3894 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3895 			 unsigned long *pheadroom, unsigned long *pdirty,
3896 			 unsigned long *pwriteback)
3897 {
3898 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3899 	struct mem_cgroup *parent;
3900 
3901 	*pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
3902 
3903 	/* this should eventually include NR_UNSTABLE_NFS */
3904 	*pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
3905 	*pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
3906 						     (1 << LRU_ACTIVE_FILE));
3907 	*pheadroom = PAGE_COUNTER_MAX;
3908 
3909 	while ((parent = parent_mem_cgroup(memcg))) {
3910 		unsigned long ceiling = min(memcg->memory.max, memcg->high);
3911 		unsigned long used = page_counter_read(&memcg->memory);
3912 
3913 		*pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3914 		memcg = parent;
3915 	}
3916 }
3917 
3918 #else	/* CONFIG_CGROUP_WRITEBACK */
3919 
3920 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3921 {
3922 	return 0;
3923 }
3924 
3925 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3926 {
3927 }
3928 
3929 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3930 {
3931 }
3932 
3933 #endif	/* CONFIG_CGROUP_WRITEBACK */
3934 
3935 /*
3936  * DO NOT USE IN NEW FILES.
3937  *
3938  * "cgroup.event_control" implementation.
3939  *
3940  * This is way over-engineered.  It tries to support fully configurable
3941  * events for each user.  Such level of flexibility is completely
3942  * unnecessary especially in the light of the planned unified hierarchy.
3943  *
3944  * Please deprecate this and replace with something simpler if at all
3945  * possible.
3946  */
3947 
3948 /*
3949  * Unregister event and free resources.
3950  *
3951  * Gets called from workqueue.
3952  */
3953 static void memcg_event_remove(struct work_struct *work)
3954 {
3955 	struct mem_cgroup_event *event =
3956 		container_of(work, struct mem_cgroup_event, remove);
3957 	struct mem_cgroup *memcg = event->memcg;
3958 
3959 	remove_wait_queue(event->wqh, &event->wait);
3960 
3961 	event->unregister_event(memcg, event->eventfd);
3962 
3963 	/* Notify userspace the event is going away. */
3964 	eventfd_signal(event->eventfd, 1);
3965 
3966 	eventfd_ctx_put(event->eventfd);
3967 	kfree(event);
3968 	css_put(&memcg->css);
3969 }
3970 
3971 /*
3972  * Gets called on EPOLLHUP on eventfd when user closes it.
3973  *
3974  * Called with wqh->lock held and interrupts disabled.
3975  */
3976 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
3977 			    int sync, void *key)
3978 {
3979 	struct mem_cgroup_event *event =
3980 		container_of(wait, struct mem_cgroup_event, wait);
3981 	struct mem_cgroup *memcg = event->memcg;
3982 	__poll_t flags = key_to_poll(key);
3983 
3984 	if (flags & EPOLLHUP) {
3985 		/*
3986 		 * If the event has been detached at cgroup removal, we
3987 		 * can simply return knowing the other side will cleanup
3988 		 * for us.
3989 		 *
3990 		 * We can't race against event freeing since the other
3991 		 * side will require wqh->lock via remove_wait_queue(),
3992 		 * which we hold.
3993 		 */
3994 		spin_lock(&memcg->event_list_lock);
3995 		if (!list_empty(&event->list)) {
3996 			list_del_init(&event->list);
3997 			/*
3998 			 * We are in atomic context, but cgroup_event_remove()
3999 			 * may sleep, so we have to call it in workqueue.
4000 			 */
4001 			schedule_work(&event->remove);
4002 		}
4003 		spin_unlock(&memcg->event_list_lock);
4004 	}
4005 
4006 	return 0;
4007 }
4008 
4009 static void memcg_event_ptable_queue_proc(struct file *file,
4010 		wait_queue_head_t *wqh, poll_table *pt)
4011 {
4012 	struct mem_cgroup_event *event =
4013 		container_of(pt, struct mem_cgroup_event, pt);
4014 
4015 	event->wqh = wqh;
4016 	add_wait_queue(wqh, &event->wait);
4017 }
4018 
4019 /*
4020  * DO NOT USE IN NEW FILES.
4021  *
4022  * Parse input and register new cgroup event handler.
4023  *
4024  * Input must be in format '<event_fd> <control_fd> <args>'.
4025  * Interpretation of args is defined by control file implementation.
4026  */
4027 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4028 					 char *buf, size_t nbytes, loff_t off)
4029 {
4030 	struct cgroup_subsys_state *css = of_css(of);
4031 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4032 	struct mem_cgroup_event *event;
4033 	struct cgroup_subsys_state *cfile_css;
4034 	unsigned int efd, cfd;
4035 	struct fd efile;
4036 	struct fd cfile;
4037 	const char *name;
4038 	char *endp;
4039 	int ret;
4040 
4041 	buf = strstrip(buf);
4042 
4043 	efd = simple_strtoul(buf, &endp, 10);
4044 	if (*endp != ' ')
4045 		return -EINVAL;
4046 	buf = endp + 1;
4047 
4048 	cfd = simple_strtoul(buf, &endp, 10);
4049 	if ((*endp != ' ') && (*endp != '\0'))
4050 		return -EINVAL;
4051 	buf = endp + 1;
4052 
4053 	event = kzalloc(sizeof(*event), GFP_KERNEL);
4054 	if (!event)
4055 		return -ENOMEM;
4056 
4057 	event->memcg = memcg;
4058 	INIT_LIST_HEAD(&event->list);
4059 	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
4060 	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
4061 	INIT_WORK(&event->remove, memcg_event_remove);
4062 
4063 	efile = fdget(efd);
4064 	if (!efile.file) {
4065 		ret = -EBADF;
4066 		goto out_kfree;
4067 	}
4068 
4069 	event->eventfd = eventfd_ctx_fileget(efile.file);
4070 	if (IS_ERR(event->eventfd)) {
4071 		ret = PTR_ERR(event->eventfd);
4072 		goto out_put_efile;
4073 	}
4074 
4075 	cfile = fdget(cfd);
4076 	if (!cfile.file) {
4077 		ret = -EBADF;
4078 		goto out_put_eventfd;
4079 	}
4080 
4081 	/* the process need read permission on control file */
4082 	/* AV: shouldn't we check that it's been opened for read instead? */
4083 	ret = inode_permission(file_inode(cfile.file), MAY_READ);
4084 	if (ret < 0)
4085 		goto out_put_cfile;
4086 
4087 	/*
4088 	 * Determine the event callbacks and set them in @event.  This used
4089 	 * to be done via struct cftype but cgroup core no longer knows
4090 	 * about these events.  The following is crude but the whole thing
4091 	 * is for compatibility anyway.
4092 	 *
4093 	 * DO NOT ADD NEW FILES.
4094 	 */
4095 	name = cfile.file->f_path.dentry->d_name.name;
4096 
4097 	if (!strcmp(name, "memory.usage_in_bytes")) {
4098 		event->register_event = mem_cgroup_usage_register_event;
4099 		event->unregister_event = mem_cgroup_usage_unregister_event;
4100 	} else if (!strcmp(name, "memory.oom_control")) {
4101 		event->register_event = mem_cgroup_oom_register_event;
4102 		event->unregister_event = mem_cgroup_oom_unregister_event;
4103 	} else if (!strcmp(name, "memory.pressure_level")) {
4104 		event->register_event = vmpressure_register_event;
4105 		event->unregister_event = vmpressure_unregister_event;
4106 	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
4107 		event->register_event = memsw_cgroup_usage_register_event;
4108 		event->unregister_event = memsw_cgroup_usage_unregister_event;
4109 	} else {
4110 		ret = -EINVAL;
4111 		goto out_put_cfile;
4112 	}
4113 
4114 	/*
4115 	 * Verify @cfile should belong to @css.  Also, remaining events are
4116 	 * automatically removed on cgroup destruction but the removal is
4117 	 * asynchronous, so take an extra ref on @css.
4118 	 */
4119 	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
4120 					       &memory_cgrp_subsys);
4121 	ret = -EINVAL;
4122 	if (IS_ERR(cfile_css))
4123 		goto out_put_cfile;
4124 	if (cfile_css != css) {
4125 		css_put(cfile_css);
4126 		goto out_put_cfile;
4127 	}
4128 
4129 	ret = event->register_event(memcg, event->eventfd, buf);
4130 	if (ret)
4131 		goto out_put_css;
4132 
4133 	vfs_poll(efile.file, &event->pt);
4134 
4135 	spin_lock(&memcg->event_list_lock);
4136 	list_add(&event->list, &memcg->event_list);
4137 	spin_unlock(&memcg->event_list_lock);
4138 
4139 	fdput(cfile);
4140 	fdput(efile);
4141 
4142 	return nbytes;
4143 
4144 out_put_css:
4145 	css_put(css);
4146 out_put_cfile:
4147 	fdput(cfile);
4148 out_put_eventfd:
4149 	eventfd_ctx_put(event->eventfd);
4150 out_put_efile:
4151 	fdput(efile);
4152 out_kfree:
4153 	kfree(event);
4154 
4155 	return ret;
4156 }
4157 
4158 static struct cftype mem_cgroup_legacy_files[] = {
4159 	{
4160 		.name = "usage_in_bytes",
4161 		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4162 		.read_u64 = mem_cgroup_read_u64,
4163 	},
4164 	{
4165 		.name = "max_usage_in_bytes",
4166 		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4167 		.write = mem_cgroup_reset,
4168 		.read_u64 = mem_cgroup_read_u64,
4169 	},
4170 	{
4171 		.name = "limit_in_bytes",
4172 		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4173 		.write = mem_cgroup_write,
4174 		.read_u64 = mem_cgroup_read_u64,
4175 	},
4176 	{
4177 		.name = "soft_limit_in_bytes",
4178 		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4179 		.write = mem_cgroup_write,
4180 		.read_u64 = mem_cgroup_read_u64,
4181 	},
4182 	{
4183 		.name = "failcnt",
4184 		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4185 		.write = mem_cgroup_reset,
4186 		.read_u64 = mem_cgroup_read_u64,
4187 	},
4188 	{
4189 		.name = "stat",
4190 		.seq_show = memcg_stat_show,
4191 	},
4192 	{
4193 		.name = "force_empty",
4194 		.write = mem_cgroup_force_empty_write,
4195 	},
4196 	{
4197 		.name = "use_hierarchy",
4198 		.write_u64 = mem_cgroup_hierarchy_write,
4199 		.read_u64 = mem_cgroup_hierarchy_read,
4200 	},
4201 	{
4202 		.name = "cgroup.event_control",		/* XXX: for compat */
4203 		.write = memcg_write_event_control,
4204 		.flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
4205 	},
4206 	{
4207 		.name = "swappiness",
4208 		.read_u64 = mem_cgroup_swappiness_read,
4209 		.write_u64 = mem_cgroup_swappiness_write,
4210 	},
4211 	{
4212 		.name = "move_charge_at_immigrate",
4213 		.read_u64 = mem_cgroup_move_charge_read,
4214 		.write_u64 = mem_cgroup_move_charge_write,
4215 	},
4216 	{
4217 		.name = "oom_control",
4218 		.seq_show = mem_cgroup_oom_control_read,
4219 		.write_u64 = mem_cgroup_oom_control_write,
4220 		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4221 	},
4222 	{
4223 		.name = "pressure_level",
4224 	},
4225 #ifdef CONFIG_NUMA
4226 	{
4227 		.name = "numa_stat",
4228 		.seq_show = memcg_numa_stat_show,
4229 	},
4230 #endif
4231 	{
4232 		.name = "kmem.limit_in_bytes",
4233 		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4234 		.write = mem_cgroup_write,
4235 		.read_u64 = mem_cgroup_read_u64,
4236 	},
4237 	{
4238 		.name = "kmem.usage_in_bytes",
4239 		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4240 		.read_u64 = mem_cgroup_read_u64,
4241 	},
4242 	{
4243 		.name = "kmem.failcnt",
4244 		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4245 		.write = mem_cgroup_reset,
4246 		.read_u64 = mem_cgroup_read_u64,
4247 	},
4248 	{
4249 		.name = "kmem.max_usage_in_bytes",
4250 		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4251 		.write = mem_cgroup_reset,
4252 		.read_u64 = mem_cgroup_read_u64,
4253 	},
4254 #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
4255 	{
4256 		.name = "kmem.slabinfo",
4257 		.seq_start = memcg_slab_start,
4258 		.seq_next = memcg_slab_next,
4259 		.seq_stop = memcg_slab_stop,
4260 		.seq_show = memcg_slab_show,
4261 	},
4262 #endif
4263 	{
4264 		.name = "kmem.tcp.limit_in_bytes",
4265 		.private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4266 		.write = mem_cgroup_write,
4267 		.read_u64 = mem_cgroup_read_u64,
4268 	},
4269 	{
4270 		.name = "kmem.tcp.usage_in_bytes",
4271 		.private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4272 		.read_u64 = mem_cgroup_read_u64,
4273 	},
4274 	{
4275 		.name = "kmem.tcp.failcnt",
4276 		.private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4277 		.write = mem_cgroup_reset,
4278 		.read_u64 = mem_cgroup_read_u64,
4279 	},
4280 	{
4281 		.name = "kmem.tcp.max_usage_in_bytes",
4282 		.private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4283 		.write = mem_cgroup_reset,
4284 		.read_u64 = mem_cgroup_read_u64,
4285 	},
4286 	{ },	/* terminate */
4287 };
4288 
4289 /*
4290  * Private memory cgroup IDR
4291  *
4292  * Swap-out records and page cache shadow entries need to store memcg
4293  * references in constrained space, so we maintain an ID space that is
4294  * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4295  * memory-controlled cgroups to 64k.
4296  *
4297  * However, there usually are many references to the oflline CSS after
4298  * the cgroup has been destroyed, such as page cache or reclaimable
4299  * slab objects, that don't need to hang on to the ID. We want to keep
4300  * those dead CSS from occupying IDs, or we might quickly exhaust the
4301  * relatively small ID space and prevent the creation of new cgroups
4302  * even when there are much fewer than 64k cgroups - possibly none.
4303  *
4304  * Maintain a private 16-bit ID space for memcg, and allow the ID to
4305  * be freed and recycled when it's no longer needed, which is usually
4306  * when the CSS is offlined.
4307  *
4308  * The only exception to that are records of swapped out tmpfs/shmem
4309  * pages that need to be attributed to live ancestors on swapin. But
4310  * those references are manageable from userspace.
4311  */
4312 
4313 static DEFINE_IDR(mem_cgroup_idr);
4314 
4315 static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
4316 {
4317 	if (memcg->id.id > 0) {
4318 		idr_remove(&mem_cgroup_idr, memcg->id.id);
4319 		memcg->id.id = 0;
4320 	}
4321 }
4322 
4323 static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
4324 {
4325 	refcount_add(n, &memcg->id.ref);
4326 }
4327 
4328 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
4329 {
4330 	if (refcount_sub_and_test(n, &memcg->id.ref)) {
4331 		mem_cgroup_id_remove(memcg);
4332 
4333 		/* Memcg ID pins CSS */
4334 		css_put(&memcg->css);
4335 	}
4336 }
4337 
4338 static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
4339 {
4340 	mem_cgroup_id_get_many(memcg, 1);
4341 }
4342 
4343 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
4344 {
4345 	mem_cgroup_id_put_many(memcg, 1);
4346 }
4347 
4348 /**
4349  * mem_cgroup_from_id - look up a memcg from a memcg id
4350  * @id: the memcg id to look up
4351  *
4352  * Caller must hold rcu_read_lock().
4353  */
4354 struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4355 {
4356 	WARN_ON_ONCE(!rcu_read_lock_held());
4357 	return idr_find(&mem_cgroup_idr, id);
4358 }
4359 
4360 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4361 {
4362 	struct mem_cgroup_per_node *pn;
4363 	int tmp = node;
4364 	/*
4365 	 * This routine is called against possible nodes.
4366 	 * But it's BUG to call kmalloc() against offline node.
4367 	 *
4368 	 * TODO: this routine can waste much memory for nodes which will
4369 	 *       never be onlined. It's better to use memory hotplug callback
4370 	 *       function.
4371 	 */
4372 	if (!node_state(node, N_NORMAL_MEMORY))
4373 		tmp = -1;
4374 	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4375 	if (!pn)
4376 		return 1;
4377 
4378 	pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
4379 	if (!pn->lruvec_stat_cpu) {
4380 		kfree(pn);
4381 		return 1;
4382 	}
4383 
4384 	lruvec_init(&pn->lruvec);
4385 	pn->usage_in_excess = 0;
4386 	pn->on_tree = false;
4387 	pn->memcg = memcg;
4388 
4389 	memcg->nodeinfo[node] = pn;
4390 	return 0;
4391 }
4392 
4393 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4394 {
4395 	struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
4396 
4397 	if (!pn)
4398 		return;
4399 
4400 	free_percpu(pn->lruvec_stat_cpu);
4401 	kfree(pn);
4402 }
4403 
4404 static void __mem_cgroup_free(struct mem_cgroup *memcg)
4405 {
4406 	int node;
4407 
4408 	for_each_node(node)
4409 		free_mem_cgroup_per_node_info(memcg, node);
4410 	free_percpu(memcg->stat_cpu);
4411 	kfree(memcg);
4412 }
4413 
4414 static void mem_cgroup_free(struct mem_cgroup *memcg)
4415 {
4416 	memcg_wb_domain_exit(memcg);
4417 	__mem_cgroup_free(memcg);
4418 }
4419 
4420 static struct mem_cgroup *mem_cgroup_alloc(void)
4421 {
4422 	struct mem_cgroup *memcg;
4423 	size_t size;
4424 	int node;
4425 
4426 	size = sizeof(struct mem_cgroup);
4427 	size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4428 
4429 	memcg = kzalloc(size, GFP_KERNEL);
4430 	if (!memcg)
4431 		return NULL;
4432 
4433 	memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4434 				 1, MEM_CGROUP_ID_MAX,
4435 				 GFP_KERNEL);
4436 	if (memcg->id.id < 0)
4437 		goto fail;
4438 
4439 	memcg->stat_cpu = alloc_percpu(struct mem_cgroup_stat_cpu);
4440 	if (!memcg->stat_cpu)
4441 		goto fail;
4442 
4443 	for_each_node(node)
4444 		if (alloc_mem_cgroup_per_node_info(memcg, node))
4445 			goto fail;
4446 
4447 	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4448 		goto fail;
4449 
4450 	INIT_WORK(&memcg->high_work, high_work_func);
4451 	memcg->last_scanned_node = MAX_NUMNODES;
4452 	INIT_LIST_HEAD(&memcg->oom_notify);
4453 	mutex_init(&memcg->thresholds_lock);
4454 	spin_lock_init(&memcg->move_lock);
4455 	vmpressure_init(&memcg->vmpressure);
4456 	INIT_LIST_HEAD(&memcg->event_list);
4457 	spin_lock_init(&memcg->event_list_lock);
4458 	memcg->socket_pressure = jiffies;
4459 #ifdef CONFIG_MEMCG_KMEM
4460 	memcg->kmemcg_id = -1;
4461 #endif
4462 #ifdef CONFIG_CGROUP_WRITEBACK
4463 	INIT_LIST_HEAD(&memcg->cgwb_list);
4464 #endif
4465 	idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
4466 	return memcg;
4467 fail:
4468 	mem_cgroup_id_remove(memcg);
4469 	__mem_cgroup_free(memcg);
4470 	return NULL;
4471 }
4472 
4473 static struct cgroup_subsys_state * __ref
4474 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4475 {
4476 	struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4477 	struct mem_cgroup *memcg;
4478 	long error = -ENOMEM;
4479 
4480 	memcg = mem_cgroup_alloc();
4481 	if (!memcg)
4482 		return ERR_PTR(error);
4483 
4484 	memcg->high = PAGE_COUNTER_MAX;
4485 	memcg->soft_limit = PAGE_COUNTER_MAX;
4486 	if (parent) {
4487 		memcg->swappiness = mem_cgroup_swappiness(parent);
4488 		memcg->oom_kill_disable = parent->oom_kill_disable;
4489 	}
4490 	if (parent && parent->use_hierarchy) {
4491 		memcg->use_hierarchy = true;
4492 		page_counter_init(&memcg->memory, &parent->memory);
4493 		page_counter_init(&memcg->swap, &parent->swap);
4494 		page_counter_init(&memcg->memsw, &parent->memsw);
4495 		page_counter_init(&memcg->kmem, &parent->kmem);
4496 		page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4497 	} else {
4498 		page_counter_init(&memcg->memory, NULL);
4499 		page_counter_init(&memcg->swap, NULL);
4500 		page_counter_init(&memcg->memsw, NULL);
4501 		page_counter_init(&memcg->kmem, NULL);
4502 		page_counter_init(&memcg->tcpmem, NULL);
4503 		/*
4504 		 * Deeper hierachy with use_hierarchy == false doesn't make
4505 		 * much sense so let cgroup subsystem know about this
4506 		 * unfortunate state in our controller.
4507 		 */
4508 		if (parent != root_mem_cgroup)
4509 			memory_cgrp_subsys.broken_hierarchy = true;
4510 	}
4511 
4512 	/* The following stuff does not apply to the root */
4513 	if (!parent) {
4514 		root_mem_cgroup = memcg;
4515 		return &memcg->css;
4516 	}
4517 
4518 	error = memcg_online_kmem(memcg);
4519 	if (error)
4520 		goto fail;
4521 
4522 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4523 		static_branch_inc(&memcg_sockets_enabled_key);
4524 
4525 	return &memcg->css;
4526 fail:
4527 	mem_cgroup_id_remove(memcg);
4528 	mem_cgroup_free(memcg);
4529 	return ERR_PTR(-ENOMEM);
4530 }
4531 
4532 static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4533 {
4534 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4535 
4536 	/*
4537 	 * A memcg must be visible for memcg_expand_shrinker_maps()
4538 	 * by the time the maps are allocated. So, we allocate maps
4539 	 * here, when for_each_mem_cgroup() can't skip it.
4540 	 */
4541 	if (memcg_alloc_shrinker_maps(memcg)) {
4542 		mem_cgroup_id_remove(memcg);
4543 		return -ENOMEM;
4544 	}
4545 
4546 	/* Online state pins memcg ID, memcg ID pins CSS */
4547 	refcount_set(&memcg->id.ref, 1);
4548 	css_get(css);
4549 	return 0;
4550 }
4551 
4552 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4553 {
4554 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4555 	struct mem_cgroup_event *event, *tmp;
4556 
4557 	/*
4558 	 * Unregister events and notify userspace.
4559 	 * Notify userspace about cgroup removing only after rmdir of cgroup
4560 	 * directory to avoid race between userspace and kernelspace.
4561 	 */
4562 	spin_lock(&memcg->event_list_lock);
4563 	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4564 		list_del_init(&event->list);
4565 		schedule_work(&event->remove);
4566 	}
4567 	spin_unlock(&memcg->event_list_lock);
4568 
4569 	page_counter_set_min(&memcg->memory, 0);
4570 	page_counter_set_low(&memcg->memory, 0);
4571 
4572 	memcg_offline_kmem(memcg);
4573 	wb_memcg_offline(memcg);
4574 
4575 	drain_all_stock(memcg);
4576 
4577 	mem_cgroup_id_put(memcg);
4578 }
4579 
4580 static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4581 {
4582 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4583 
4584 	invalidate_reclaim_iterators(memcg);
4585 }
4586 
4587 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4588 {
4589 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4590 
4591 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4592 		static_branch_dec(&memcg_sockets_enabled_key);
4593 
4594 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
4595 		static_branch_dec(&memcg_sockets_enabled_key);
4596 
4597 	vmpressure_cleanup(&memcg->vmpressure);
4598 	cancel_work_sync(&memcg->high_work);
4599 	mem_cgroup_remove_from_trees(memcg);
4600 	memcg_free_shrinker_maps(memcg);
4601 	memcg_free_kmem(memcg);
4602 	mem_cgroup_free(memcg);
4603 }
4604 
4605 /**
4606  * mem_cgroup_css_reset - reset the states of a mem_cgroup
4607  * @css: the target css
4608  *
4609  * Reset the states of the mem_cgroup associated with @css.  This is
4610  * invoked when the userland requests disabling on the default hierarchy
4611  * but the memcg is pinned through dependency.  The memcg should stop
4612  * applying policies and should revert to the vanilla state as it may be
4613  * made visible again.
4614  *
4615  * The current implementation only resets the essential configurations.
4616  * This needs to be expanded to cover all the visible parts.
4617  */
4618 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4619 {
4620 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4621 
4622 	page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
4623 	page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
4624 	page_counter_set_max(&memcg->memsw, PAGE_COUNTER_MAX);
4625 	page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
4626 	page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
4627 	page_counter_set_min(&memcg->memory, 0);
4628 	page_counter_set_low(&memcg->memory, 0);
4629 	memcg->high = PAGE_COUNTER_MAX;
4630 	memcg->soft_limit = PAGE_COUNTER_MAX;
4631 	memcg_wb_domain_size_changed(memcg);
4632 }
4633 
4634 #ifdef CONFIG_MMU
4635 /* Handlers for move charge at task migration. */
4636 static int mem_cgroup_do_precharge(unsigned long count)
4637 {
4638 	int ret;
4639 
4640 	/* Try a single bulk charge without reclaim first, kswapd may wake */
4641 	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4642 	if (!ret) {
4643 		mc.precharge += count;
4644 		return ret;
4645 	}
4646 
4647 	/* Try charges one by one with reclaim, but do not retry */
4648 	while (count--) {
4649 		ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
4650 		if (ret)
4651 			return ret;
4652 		mc.precharge++;
4653 		cond_resched();
4654 	}
4655 	return 0;
4656 }
4657 
4658 union mc_target {
4659 	struct page	*page;
4660 	swp_entry_t	ent;
4661 };
4662 
4663 enum mc_target_type {
4664 	MC_TARGET_NONE = 0,
4665 	MC_TARGET_PAGE,
4666 	MC_TARGET_SWAP,
4667 	MC_TARGET_DEVICE,
4668 };
4669 
4670 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4671 						unsigned long addr, pte_t ptent)
4672 {
4673 	struct page *page = _vm_normal_page(vma, addr, ptent, true);
4674 
4675 	if (!page || !page_mapped(page))
4676 		return NULL;
4677 	if (PageAnon(page)) {
4678 		if (!(mc.flags & MOVE_ANON))
4679 			return NULL;
4680 	} else {
4681 		if (!(mc.flags & MOVE_FILE))
4682 			return NULL;
4683 	}
4684 	if (!get_page_unless_zero(page))
4685 		return NULL;
4686 
4687 	return page;
4688 }
4689 
4690 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
4691 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4692 			pte_t ptent, swp_entry_t *entry)
4693 {
4694 	struct page *page = NULL;
4695 	swp_entry_t ent = pte_to_swp_entry(ptent);
4696 
4697 	if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
4698 		return NULL;
4699 
4700 	/*
4701 	 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
4702 	 * a device and because they are not accessible by CPU they are store
4703 	 * as special swap entry in the CPU page table.
4704 	 */
4705 	if (is_device_private_entry(ent)) {
4706 		page = device_private_entry_to_page(ent);
4707 		/*
4708 		 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
4709 		 * a refcount of 1 when free (unlike normal page)
4710 		 */
4711 		if (!page_ref_add_unless(page, 1, 1))
4712 			return NULL;
4713 		return page;
4714 	}
4715 
4716 	/*
4717 	 * Because lookup_swap_cache() updates some statistics counter,
4718 	 * we call find_get_page() with swapper_space directly.
4719 	 */
4720 	page = find_get_page(swap_address_space(ent), swp_offset(ent));
4721 	if (do_memsw_account())
4722 		entry->val = ent.val;
4723 
4724 	return page;
4725 }
4726 #else
4727 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4728 			pte_t ptent, swp_entry_t *entry)
4729 {
4730 	return NULL;
4731 }
4732 #endif
4733 
4734 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4735 			unsigned long addr, pte_t ptent, swp_entry_t *entry)
4736 {
4737 	struct page *page = NULL;
4738 	struct address_space *mapping;
4739 	pgoff_t pgoff;
4740 
4741 	if (!vma->vm_file) /* anonymous vma */
4742 		return NULL;
4743 	if (!(mc.flags & MOVE_FILE))
4744 		return NULL;
4745 
4746 	mapping = vma->vm_file->f_mapping;
4747 	pgoff = linear_page_index(vma, addr);
4748 
4749 	/* page is moved even if it's not RSS of this task(page-faulted). */
4750 #ifdef CONFIG_SWAP
4751 	/* shmem/tmpfs may report page out on swap: account for that too. */
4752 	if (shmem_mapping(mapping)) {
4753 		page = find_get_entry(mapping, pgoff);
4754 		if (xa_is_value(page)) {
4755 			swp_entry_t swp = radix_to_swp_entry(page);
4756 			if (do_memsw_account())
4757 				*entry = swp;
4758 			page = find_get_page(swap_address_space(swp),
4759 					     swp_offset(swp));
4760 		}
4761 	} else
4762 		page = find_get_page(mapping, pgoff);
4763 #else
4764 	page = find_get_page(mapping, pgoff);
4765 #endif
4766 	return page;
4767 }
4768 
4769 /**
4770  * mem_cgroup_move_account - move account of the page
4771  * @page: the page
4772  * @compound: charge the page as compound or small page
4773  * @from: mem_cgroup which the page is moved from.
4774  * @to:	mem_cgroup which the page is moved to. @from != @to.
4775  *
4776  * The caller must make sure the page is not on LRU (isolate_page() is useful.)
4777  *
4778  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
4779  * from old cgroup.
4780  */
4781 static int mem_cgroup_move_account(struct page *page,
4782 				   bool compound,
4783 				   struct mem_cgroup *from,
4784 				   struct mem_cgroup *to)
4785 {
4786 	unsigned long flags;
4787 	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4788 	int ret;
4789 	bool anon;
4790 
4791 	VM_BUG_ON(from == to);
4792 	VM_BUG_ON_PAGE(PageLRU(page), page);
4793 	VM_BUG_ON(compound && !PageTransHuge(page));
4794 
4795 	/*
4796 	 * Prevent mem_cgroup_migrate() from looking at
4797 	 * page->mem_cgroup of its source page while we change it.
4798 	 */
4799 	ret = -EBUSY;
4800 	if (!trylock_page(page))
4801 		goto out;
4802 
4803 	ret = -EINVAL;
4804 	if (page->mem_cgroup != from)
4805 		goto out_unlock;
4806 
4807 	anon = PageAnon(page);
4808 
4809 	spin_lock_irqsave(&from->move_lock, flags);
4810 
4811 	if (!anon && page_mapped(page)) {
4812 		__mod_memcg_state(from, NR_FILE_MAPPED, -nr_pages);
4813 		__mod_memcg_state(to, NR_FILE_MAPPED, nr_pages);
4814 	}
4815 
4816 	/*
4817 	 * move_lock grabbed above and caller set from->moving_account, so
4818 	 * mod_memcg_page_state will serialize updates to PageDirty.
4819 	 * So mapping should be stable for dirty pages.
4820 	 */
4821 	if (!anon && PageDirty(page)) {
4822 		struct address_space *mapping = page_mapping(page);
4823 
4824 		if (mapping_cap_account_dirty(mapping)) {
4825 			__mod_memcg_state(from, NR_FILE_DIRTY, -nr_pages);
4826 			__mod_memcg_state(to, NR_FILE_DIRTY, nr_pages);
4827 		}
4828 	}
4829 
4830 	if (PageWriteback(page)) {
4831 		__mod_memcg_state(from, NR_WRITEBACK, -nr_pages);
4832 		__mod_memcg_state(to, NR_WRITEBACK, nr_pages);
4833 	}
4834 
4835 	/*
4836 	 * It is safe to change page->mem_cgroup here because the page
4837 	 * is referenced, charged, and isolated - we can't race with
4838 	 * uncharging, charging, migration, or LRU putback.
4839 	 */
4840 
4841 	/* caller should have done css_get */
4842 	page->mem_cgroup = to;
4843 	spin_unlock_irqrestore(&from->move_lock, flags);
4844 
4845 	ret = 0;
4846 
4847 	local_irq_disable();
4848 	mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4849 	memcg_check_events(to, page);
4850 	mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4851 	memcg_check_events(from, page);
4852 	local_irq_enable();
4853 out_unlock:
4854 	unlock_page(page);
4855 out:
4856 	return ret;
4857 }
4858 
4859 /**
4860  * get_mctgt_type - get target type of moving charge
4861  * @vma: the vma the pte to be checked belongs
4862  * @addr: the address corresponding to the pte to be checked
4863  * @ptent: the pte to be checked
4864  * @target: the pointer the target page or swap ent will be stored(can be NULL)
4865  *
4866  * Returns
4867  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
4868  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4869  *     move charge. if @target is not NULL, the page is stored in target->page
4870  *     with extra refcnt got(Callers should handle it).
4871  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4872  *     target for charge migration. if @target is not NULL, the entry is stored
4873  *     in target->ent.
4874  *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is MEMORY_DEVICE_PUBLIC
4875  *     or MEMORY_DEVICE_PRIVATE (so ZONE_DEVICE page and thus not on the lru).
4876  *     For now we such page is charge like a regular page would be as for all
4877  *     intent and purposes it is just special memory taking the place of a
4878  *     regular page.
4879  *
4880  *     See Documentations/vm/hmm.txt and include/linux/hmm.h
4881  *
4882  * Called with pte lock held.
4883  */
4884 
4885 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
4886 		unsigned long addr, pte_t ptent, union mc_target *target)
4887 {
4888 	struct page *page = NULL;
4889 	enum mc_target_type ret = MC_TARGET_NONE;
4890 	swp_entry_t ent = { .val = 0 };
4891 
4892 	if (pte_present(ptent))
4893 		page = mc_handle_present_pte(vma, addr, ptent);
4894 	else if (is_swap_pte(ptent))
4895 		page = mc_handle_swap_pte(vma, ptent, &ent);
4896 	else if (pte_none(ptent))
4897 		page = mc_handle_file_pte(vma, addr, ptent, &ent);
4898 
4899 	if (!page && !ent.val)
4900 		return ret;
4901 	if (page) {
4902 		/*
4903 		 * Do only loose check w/o serialization.
4904 		 * mem_cgroup_move_account() checks the page is valid or
4905 		 * not under LRU exclusion.
4906 		 */
4907 		if (page->mem_cgroup == mc.from) {
4908 			ret = MC_TARGET_PAGE;
4909 			if (is_device_private_page(page) ||
4910 			    is_device_public_page(page))
4911 				ret = MC_TARGET_DEVICE;
4912 			if (target)
4913 				target->page = page;
4914 		}
4915 		if (!ret || !target)
4916 			put_page(page);
4917 	}
4918 	/*
4919 	 * There is a swap entry and a page doesn't exist or isn't charged.
4920 	 * But we cannot move a tail-page in a THP.
4921 	 */
4922 	if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
4923 	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4924 		ret = MC_TARGET_SWAP;
4925 		if (target)
4926 			target->ent = ent;
4927 	}
4928 	return ret;
4929 }
4930 
4931 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4932 /*
4933  * We don't consider PMD mapped swapping or file mapped pages because THP does
4934  * not support them for now.
4935  * Caller should make sure that pmd_trans_huge(pmd) is true.
4936  */
4937 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4938 		unsigned long addr, pmd_t pmd, union mc_target *target)
4939 {
4940 	struct page *page = NULL;
4941 	enum mc_target_type ret = MC_TARGET_NONE;
4942 
4943 	if (unlikely(is_swap_pmd(pmd))) {
4944 		VM_BUG_ON(thp_migration_supported() &&
4945 				  !is_pmd_migration_entry(pmd));
4946 		return ret;
4947 	}
4948 	page = pmd_page(pmd);
4949 	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4950 	if (!(mc.flags & MOVE_ANON))
4951 		return ret;
4952 	if (page->mem_cgroup == mc.from) {
4953 		ret = MC_TARGET_PAGE;
4954 		if (target) {
4955 			get_page(page);
4956 			target->page = page;
4957 		}
4958 	}
4959 	return ret;
4960 }
4961 #else
4962 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4963 		unsigned long addr, pmd_t pmd, union mc_target *target)
4964 {
4965 	return MC_TARGET_NONE;
4966 }
4967 #endif
4968 
4969 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4970 					unsigned long addr, unsigned long end,
4971 					struct mm_walk *walk)
4972 {
4973 	struct vm_area_struct *vma = walk->vma;
4974 	pte_t *pte;
4975 	spinlock_t *ptl;
4976 
4977 	ptl = pmd_trans_huge_lock(pmd, vma);
4978 	if (ptl) {
4979 		/*
4980 		 * Note their can not be MC_TARGET_DEVICE for now as we do not
4981 		 * support transparent huge page with MEMORY_DEVICE_PUBLIC or
4982 		 * MEMORY_DEVICE_PRIVATE but this might change.
4983 		 */
4984 		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4985 			mc.precharge += HPAGE_PMD_NR;
4986 		spin_unlock(ptl);
4987 		return 0;
4988 	}
4989 
4990 	if (pmd_trans_unstable(pmd))
4991 		return 0;
4992 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4993 	for (; addr != end; pte++, addr += PAGE_SIZE)
4994 		if (get_mctgt_type(vma, addr, *pte, NULL))
4995 			mc.precharge++;	/* increment precharge temporarily */
4996 	pte_unmap_unlock(pte - 1, ptl);
4997 	cond_resched();
4998 
4999 	return 0;
5000 }
5001 
5002 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
5003 {
5004 	unsigned long precharge;
5005 
5006 	struct mm_walk mem_cgroup_count_precharge_walk = {
5007 		.pmd_entry = mem_cgroup_count_precharge_pte_range,
5008 		.mm = mm,
5009 	};
5010 	down_read(&mm->mmap_sem);
5011 	walk_page_range(0, mm->highest_vm_end,
5012 			&mem_cgroup_count_precharge_walk);
5013 	up_read(&mm->mmap_sem);
5014 
5015 	precharge = mc.precharge;
5016 	mc.precharge = 0;
5017 
5018 	return precharge;
5019 }
5020 
5021 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
5022 {
5023 	unsigned long precharge = mem_cgroup_count_precharge(mm);
5024 
5025 	VM_BUG_ON(mc.moving_task);
5026 	mc.moving_task = current;
5027 	return mem_cgroup_do_precharge(precharge);
5028 }
5029 
5030 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
5031 static void __mem_cgroup_clear_mc(void)
5032 {
5033 	struct mem_cgroup *from = mc.from;
5034 	struct mem_cgroup *to = mc.to;
5035 
5036 	/* we must uncharge all the leftover precharges from mc.to */
5037 	if (mc.precharge) {
5038 		cancel_charge(mc.to, mc.precharge);
5039 		mc.precharge = 0;
5040 	}
5041 	/*
5042 	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
5043 	 * we must uncharge here.
5044 	 */
5045 	if (mc.moved_charge) {
5046 		cancel_charge(mc.from, mc.moved_charge);
5047 		mc.moved_charge = 0;
5048 	}
5049 	/* we must fixup refcnts and charges */
5050 	if (mc.moved_swap) {
5051 		/* uncharge swap account from the old cgroup */
5052 		if (!mem_cgroup_is_root(mc.from))
5053 			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
5054 
5055 		mem_cgroup_id_put_many(mc.from, mc.moved_swap);
5056 
5057 		/*
5058 		 * we charged both to->memory and to->memsw, so we
5059 		 * should uncharge to->memory.
5060 		 */
5061 		if (!mem_cgroup_is_root(mc.to))
5062 			page_counter_uncharge(&mc.to->memory, mc.moved_swap);
5063 
5064 		mem_cgroup_id_get_many(mc.to, mc.moved_swap);
5065 		css_put_many(&mc.to->css, mc.moved_swap);
5066 
5067 		mc.moved_swap = 0;
5068 	}
5069 	memcg_oom_recover(from);
5070 	memcg_oom_recover(to);
5071 	wake_up_all(&mc.waitq);
5072 }
5073 
5074 static void mem_cgroup_clear_mc(void)
5075 {
5076 	struct mm_struct *mm = mc.mm;
5077 
5078 	/*
5079 	 * we must clear moving_task before waking up waiters at the end of
5080 	 * task migration.
5081 	 */
5082 	mc.moving_task = NULL;
5083 	__mem_cgroup_clear_mc();
5084 	spin_lock(&mc.lock);
5085 	mc.from = NULL;
5086 	mc.to = NULL;
5087 	mc.mm = NULL;
5088 	spin_unlock(&mc.lock);
5089 
5090 	mmput(mm);
5091 }
5092 
5093 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5094 {
5095 	struct cgroup_subsys_state *css;
5096 	struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
5097 	struct mem_cgroup *from;
5098 	struct task_struct *leader, *p;
5099 	struct mm_struct *mm;
5100 	unsigned long move_flags;
5101 	int ret = 0;
5102 
5103 	/* charge immigration isn't supported on the default hierarchy */
5104 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5105 		return 0;
5106 
5107 	/*
5108 	 * Multi-process migrations only happen on the default hierarchy
5109 	 * where charge immigration is not used.  Perform charge
5110 	 * immigration if @tset contains a leader and whine if there are
5111 	 * multiple.
5112 	 */
5113 	p = NULL;
5114 	cgroup_taskset_for_each_leader(leader, css, tset) {
5115 		WARN_ON_ONCE(p);
5116 		p = leader;
5117 		memcg = mem_cgroup_from_css(css);
5118 	}
5119 	if (!p)
5120 		return 0;
5121 
5122 	/*
5123 	 * We are now commited to this value whatever it is. Changes in this
5124 	 * tunable will only affect upcoming migrations, not the current one.
5125 	 * So we need to save it, and keep it going.
5126 	 */
5127 	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
5128 	if (!move_flags)
5129 		return 0;
5130 
5131 	from = mem_cgroup_from_task(p);
5132 
5133 	VM_BUG_ON(from == memcg);
5134 
5135 	mm = get_task_mm(p);
5136 	if (!mm)
5137 		return 0;
5138 	/* We move charges only when we move a owner of the mm */
5139 	if (mm->owner == p) {
5140 		VM_BUG_ON(mc.from);
5141 		VM_BUG_ON(mc.to);
5142 		VM_BUG_ON(mc.precharge);
5143 		VM_BUG_ON(mc.moved_charge);
5144 		VM_BUG_ON(mc.moved_swap);
5145 
5146 		spin_lock(&mc.lock);
5147 		mc.mm = mm;
5148 		mc.from = from;
5149 		mc.to = memcg;
5150 		mc.flags = move_flags;
5151 		spin_unlock(&mc.lock);
5152 		/* We set mc.moving_task later */
5153 
5154 		ret = mem_cgroup_precharge_mc(mm);
5155 		if (ret)
5156 			mem_cgroup_clear_mc();
5157 	} else {
5158 		mmput(mm);
5159 	}
5160 	return ret;
5161 }
5162 
5163 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5164 {
5165 	if (mc.to)
5166 		mem_cgroup_clear_mc();
5167 }
5168 
5169 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
5170 				unsigned long addr, unsigned long end,
5171 				struct mm_walk *walk)
5172 {
5173 	int ret = 0;
5174 	struct vm_area_struct *vma = walk->vma;
5175 	pte_t *pte;
5176 	spinlock_t *ptl;
5177 	enum mc_target_type target_type;
5178 	union mc_target target;
5179 	struct page *page;
5180 
5181 	ptl = pmd_trans_huge_lock(pmd, vma);
5182 	if (ptl) {
5183 		if (mc.precharge < HPAGE_PMD_NR) {
5184 			spin_unlock(ptl);
5185 			return 0;
5186 		}
5187 		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
5188 		if (target_type == MC_TARGET_PAGE) {
5189 			page = target.page;
5190 			if (!isolate_lru_page(page)) {
5191 				if (!mem_cgroup_move_account(page, true,
5192 							     mc.from, mc.to)) {
5193 					mc.precharge -= HPAGE_PMD_NR;
5194 					mc.moved_charge += HPAGE_PMD_NR;
5195 				}
5196 				putback_lru_page(page);
5197 			}
5198 			put_page(page);
5199 		} else if (target_type == MC_TARGET_DEVICE) {
5200 			page = target.page;
5201 			if (!mem_cgroup_move_account(page, true,
5202 						     mc.from, mc.to)) {
5203 				mc.precharge -= HPAGE_PMD_NR;
5204 				mc.moved_charge += HPAGE_PMD_NR;
5205 			}
5206 			put_page(page);
5207 		}
5208 		spin_unlock(ptl);
5209 		return 0;
5210 	}
5211 
5212 	if (pmd_trans_unstable(pmd))
5213 		return 0;
5214 retry:
5215 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5216 	for (; addr != end; addr += PAGE_SIZE) {
5217 		pte_t ptent = *(pte++);
5218 		bool device = false;
5219 		swp_entry_t ent;
5220 
5221 		if (!mc.precharge)
5222 			break;
5223 
5224 		switch (get_mctgt_type(vma, addr, ptent, &target)) {
5225 		case MC_TARGET_DEVICE:
5226 			device = true;
5227 			/* fall through */
5228 		case MC_TARGET_PAGE:
5229 			page = target.page;
5230 			/*
5231 			 * We can have a part of the split pmd here. Moving it
5232 			 * can be done but it would be too convoluted so simply
5233 			 * ignore such a partial THP and keep it in original
5234 			 * memcg. There should be somebody mapping the head.
5235 			 */
5236 			if (PageTransCompound(page))
5237 				goto put;
5238 			if (!device && isolate_lru_page(page))
5239 				goto put;
5240 			if (!mem_cgroup_move_account(page, false,
5241 						mc.from, mc.to)) {
5242 				mc.precharge--;
5243 				/* we uncharge from mc.from later. */
5244 				mc.moved_charge++;
5245 			}
5246 			if (!device)
5247 				putback_lru_page(page);
5248 put:			/* get_mctgt_type() gets the page */
5249 			put_page(page);
5250 			break;
5251 		case MC_TARGET_SWAP:
5252 			ent = target.ent;
5253 			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
5254 				mc.precharge--;
5255 				/* we fixup refcnts and charges later. */
5256 				mc.moved_swap++;
5257 			}
5258 			break;
5259 		default:
5260 			break;
5261 		}
5262 	}
5263 	pte_unmap_unlock(pte - 1, ptl);
5264 	cond_resched();
5265 
5266 	if (addr != end) {
5267 		/*
5268 		 * We have consumed all precharges we got in can_attach().
5269 		 * We try charge one by one, but don't do any additional
5270 		 * charges to mc.to if we have failed in charge once in attach()
5271 		 * phase.
5272 		 */
5273 		ret = mem_cgroup_do_precharge(1);
5274 		if (!ret)
5275 			goto retry;
5276 	}
5277 
5278 	return ret;
5279 }
5280 
5281 static void mem_cgroup_move_charge(void)
5282 {
5283 	struct mm_walk mem_cgroup_move_charge_walk = {
5284 		.pmd_entry = mem_cgroup_move_charge_pte_range,
5285 		.mm = mc.mm,
5286 	};
5287 
5288 	lru_add_drain_all();
5289 	/*
5290 	 * Signal lock_page_memcg() to take the memcg's move_lock
5291 	 * while we're moving its pages to another memcg. Then wait
5292 	 * for already started RCU-only updates to finish.
5293 	 */
5294 	atomic_inc(&mc.from->moving_account);
5295 	synchronize_rcu();
5296 retry:
5297 	if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
5298 		/*
5299 		 * Someone who are holding the mmap_sem might be waiting in
5300 		 * waitq. So we cancel all extra charges, wake up all waiters,
5301 		 * and retry. Because we cancel precharges, we might not be able
5302 		 * to move enough charges, but moving charge is a best-effort
5303 		 * feature anyway, so it wouldn't be a big problem.
5304 		 */
5305 		__mem_cgroup_clear_mc();
5306 		cond_resched();
5307 		goto retry;
5308 	}
5309 	/*
5310 	 * When we have consumed all precharges and failed in doing
5311 	 * additional charge, the page walk just aborts.
5312 	 */
5313 	walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);
5314 
5315 	up_read(&mc.mm->mmap_sem);
5316 	atomic_dec(&mc.from->moving_account);
5317 }
5318 
5319 static void mem_cgroup_move_task(void)
5320 {
5321 	if (mc.to) {
5322 		mem_cgroup_move_charge();
5323 		mem_cgroup_clear_mc();
5324 	}
5325 }
5326 #else	/* !CONFIG_MMU */
5327 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5328 {
5329 	return 0;
5330 }
5331 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5332 {
5333 }
5334 static void mem_cgroup_move_task(void)
5335 {
5336 }
5337 #endif
5338 
5339 /*
5340  * Cgroup retains root cgroups across [un]mount cycles making it necessary
5341  * to verify whether we're attached to the default hierarchy on each mount
5342  * attempt.
5343  */
5344 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5345 {
5346 	/*
5347 	 * use_hierarchy is forced on the default hierarchy.  cgroup core
5348 	 * guarantees that @root doesn't have any children, so turning it
5349 	 * on for the root memcg is enough.
5350 	 */
5351 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5352 		root_mem_cgroup->use_hierarchy = true;
5353 	else
5354 		root_mem_cgroup->use_hierarchy = false;
5355 }
5356 
5357 static u64 memory_current_read(struct cgroup_subsys_state *css,
5358 			       struct cftype *cft)
5359 {
5360 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5361 
5362 	return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5363 }
5364 
5365 static int memory_min_show(struct seq_file *m, void *v)
5366 {
5367 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5368 	unsigned long min = READ_ONCE(memcg->memory.min);
5369 
5370 	if (min == PAGE_COUNTER_MAX)
5371 		seq_puts(m, "max\n");
5372 	else
5373 		seq_printf(m, "%llu\n", (u64)min * PAGE_SIZE);
5374 
5375 	return 0;
5376 }
5377 
5378 static ssize_t memory_min_write(struct kernfs_open_file *of,
5379 				char *buf, size_t nbytes, loff_t off)
5380 {
5381 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5382 	unsigned long min;
5383 	int err;
5384 
5385 	buf = strstrip(buf);
5386 	err = page_counter_memparse(buf, "max", &min);
5387 	if (err)
5388 		return err;
5389 
5390 	page_counter_set_min(&memcg->memory, min);
5391 
5392 	return nbytes;
5393 }
5394 
5395 static int memory_low_show(struct seq_file *m, void *v)
5396 {
5397 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5398 	unsigned long low = READ_ONCE(memcg->memory.low);
5399 
5400 	if (low == PAGE_COUNTER_MAX)
5401 		seq_puts(m, "max\n");
5402 	else
5403 		seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5404 
5405 	return 0;
5406 }
5407 
5408 static ssize_t memory_low_write(struct kernfs_open_file *of,
5409 				char *buf, size_t nbytes, loff_t off)
5410 {
5411 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5412 	unsigned long low;
5413 	int err;
5414 
5415 	buf = strstrip(buf);
5416 	err = page_counter_memparse(buf, "max", &low);
5417 	if (err)
5418 		return err;
5419 
5420 	page_counter_set_low(&memcg->memory, low);
5421 
5422 	return nbytes;
5423 }
5424 
5425 static int memory_high_show(struct seq_file *m, void *v)
5426 {
5427 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5428 	unsigned long high = READ_ONCE(memcg->high);
5429 
5430 	if (high == PAGE_COUNTER_MAX)
5431 		seq_puts(m, "max\n");
5432 	else
5433 		seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5434 
5435 	return 0;
5436 }
5437 
5438 static ssize_t memory_high_write(struct kernfs_open_file *of,
5439 				 char *buf, size_t nbytes, loff_t off)
5440 {
5441 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5442 	unsigned long nr_pages;
5443 	unsigned long high;
5444 	int err;
5445 
5446 	buf = strstrip(buf);
5447 	err = page_counter_memparse(buf, "max", &high);
5448 	if (err)
5449 		return err;
5450 
5451 	memcg->high = high;
5452 
5453 	nr_pages = page_counter_read(&memcg->memory);
5454 	if (nr_pages > high)
5455 		try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5456 					     GFP_KERNEL, true);
5457 
5458 	memcg_wb_domain_size_changed(memcg);
5459 	return nbytes;
5460 }
5461 
5462 static int memory_max_show(struct seq_file *m, void *v)
5463 {
5464 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5465 	unsigned long max = READ_ONCE(memcg->memory.max);
5466 
5467 	if (max == PAGE_COUNTER_MAX)
5468 		seq_puts(m, "max\n");
5469 	else
5470 		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5471 
5472 	return 0;
5473 }
5474 
5475 static ssize_t memory_max_write(struct kernfs_open_file *of,
5476 				char *buf, size_t nbytes, loff_t off)
5477 {
5478 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5479 	unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5480 	bool drained = false;
5481 	unsigned long max;
5482 	int err;
5483 
5484 	buf = strstrip(buf);
5485 	err = page_counter_memparse(buf, "max", &max);
5486 	if (err)
5487 		return err;
5488 
5489 	xchg(&memcg->memory.max, max);
5490 
5491 	for (;;) {
5492 		unsigned long nr_pages = page_counter_read(&memcg->memory);
5493 
5494 		if (nr_pages <= max)
5495 			break;
5496 
5497 		if (signal_pending(current)) {
5498 			err = -EINTR;
5499 			break;
5500 		}
5501 
5502 		if (!drained) {
5503 			drain_all_stock(memcg);
5504 			drained = true;
5505 			continue;
5506 		}
5507 
5508 		if (nr_reclaims) {
5509 			if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5510 							  GFP_KERNEL, true))
5511 				nr_reclaims--;
5512 			continue;
5513 		}
5514 
5515 		memcg_memory_event(memcg, MEMCG_OOM);
5516 		if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5517 			break;
5518 	}
5519 
5520 	memcg_wb_domain_size_changed(memcg);
5521 	return nbytes;
5522 }
5523 
5524 static int memory_events_show(struct seq_file *m, void *v)
5525 {
5526 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5527 
5528 	seq_printf(m, "low %lu\n",
5529 		   atomic_long_read(&memcg->memory_events[MEMCG_LOW]));
5530 	seq_printf(m, "high %lu\n",
5531 		   atomic_long_read(&memcg->memory_events[MEMCG_HIGH]));
5532 	seq_printf(m, "max %lu\n",
5533 		   atomic_long_read(&memcg->memory_events[MEMCG_MAX]));
5534 	seq_printf(m, "oom %lu\n",
5535 		   atomic_long_read(&memcg->memory_events[MEMCG_OOM]));
5536 	seq_printf(m, "oom_kill %lu\n",
5537 		   atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
5538 
5539 	return 0;
5540 }
5541 
5542 static int memory_stat_show(struct seq_file *m, void *v)
5543 {
5544 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5545 	struct accumulated_stats acc;
5546 	int i;
5547 
5548 	/*
5549 	 * Provide statistics on the state of the memory subsystem as
5550 	 * well as cumulative event counters that show past behavior.
5551 	 *
5552 	 * This list is ordered following a combination of these gradients:
5553 	 * 1) generic big picture -> specifics and details
5554 	 * 2) reflecting userspace activity -> reflecting kernel heuristics
5555 	 *
5556 	 * Current memory state:
5557 	 */
5558 
5559 	memset(&acc, 0, sizeof(acc));
5560 	acc.stats_size = MEMCG_NR_STAT;
5561 	acc.events_size = NR_VM_EVENT_ITEMS;
5562 	accumulate_memcg_tree(memcg, &acc);
5563 
5564 	seq_printf(m, "anon %llu\n",
5565 		   (u64)acc.stat[MEMCG_RSS] * PAGE_SIZE);
5566 	seq_printf(m, "file %llu\n",
5567 		   (u64)acc.stat[MEMCG_CACHE] * PAGE_SIZE);
5568 	seq_printf(m, "kernel_stack %llu\n",
5569 		   (u64)acc.stat[MEMCG_KERNEL_STACK_KB] * 1024);
5570 	seq_printf(m, "slab %llu\n",
5571 		   (u64)(acc.stat[NR_SLAB_RECLAIMABLE] +
5572 			 acc.stat[NR_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5573 	seq_printf(m, "sock %llu\n",
5574 		   (u64)acc.stat[MEMCG_SOCK] * PAGE_SIZE);
5575 
5576 	seq_printf(m, "shmem %llu\n",
5577 		   (u64)acc.stat[NR_SHMEM] * PAGE_SIZE);
5578 	seq_printf(m, "file_mapped %llu\n",
5579 		   (u64)acc.stat[NR_FILE_MAPPED] * PAGE_SIZE);
5580 	seq_printf(m, "file_dirty %llu\n",
5581 		   (u64)acc.stat[NR_FILE_DIRTY] * PAGE_SIZE);
5582 	seq_printf(m, "file_writeback %llu\n",
5583 		   (u64)acc.stat[NR_WRITEBACK] * PAGE_SIZE);
5584 
5585 	for (i = 0; i < NR_LRU_LISTS; i++)
5586 		seq_printf(m, "%s %llu\n", mem_cgroup_lru_names[i],
5587 			   (u64)acc.lru_pages[i] * PAGE_SIZE);
5588 
5589 	seq_printf(m, "slab_reclaimable %llu\n",
5590 		   (u64)acc.stat[NR_SLAB_RECLAIMABLE] * PAGE_SIZE);
5591 	seq_printf(m, "slab_unreclaimable %llu\n",
5592 		   (u64)acc.stat[NR_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5593 
5594 	/* Accumulated memory events */
5595 
5596 	seq_printf(m, "pgfault %lu\n", acc.events[PGFAULT]);
5597 	seq_printf(m, "pgmajfault %lu\n", acc.events[PGMAJFAULT]);
5598 
5599 	seq_printf(m, "workingset_refault %lu\n",
5600 		   acc.stat[WORKINGSET_REFAULT]);
5601 	seq_printf(m, "workingset_activate %lu\n",
5602 		   acc.stat[WORKINGSET_ACTIVATE]);
5603 	seq_printf(m, "workingset_nodereclaim %lu\n",
5604 		   acc.stat[WORKINGSET_NODERECLAIM]);
5605 
5606 	seq_printf(m, "pgrefill %lu\n", acc.events[PGREFILL]);
5607 	seq_printf(m, "pgscan %lu\n", acc.events[PGSCAN_KSWAPD] +
5608 		   acc.events[PGSCAN_DIRECT]);
5609 	seq_printf(m, "pgsteal %lu\n", acc.events[PGSTEAL_KSWAPD] +
5610 		   acc.events[PGSTEAL_DIRECT]);
5611 	seq_printf(m, "pgactivate %lu\n", acc.events[PGACTIVATE]);
5612 	seq_printf(m, "pgdeactivate %lu\n", acc.events[PGDEACTIVATE]);
5613 	seq_printf(m, "pglazyfree %lu\n", acc.events[PGLAZYFREE]);
5614 	seq_printf(m, "pglazyfreed %lu\n", acc.events[PGLAZYFREED]);
5615 
5616 	return 0;
5617 }
5618 
5619 static int memory_oom_group_show(struct seq_file *m, void *v)
5620 {
5621 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5622 
5623 	seq_printf(m, "%d\n", memcg->oom_group);
5624 
5625 	return 0;
5626 }
5627 
5628 static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
5629 				      char *buf, size_t nbytes, loff_t off)
5630 {
5631 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5632 	int ret, oom_group;
5633 
5634 	buf = strstrip(buf);
5635 	if (!buf)
5636 		return -EINVAL;
5637 
5638 	ret = kstrtoint(buf, 0, &oom_group);
5639 	if (ret)
5640 		return ret;
5641 
5642 	if (oom_group != 0 && oom_group != 1)
5643 		return -EINVAL;
5644 
5645 	memcg->oom_group = oom_group;
5646 
5647 	return nbytes;
5648 }
5649 
5650 static struct cftype memory_files[] = {
5651 	{
5652 		.name = "current",
5653 		.flags = CFTYPE_NOT_ON_ROOT,
5654 		.read_u64 = memory_current_read,
5655 	},
5656 	{
5657 		.name = "min",
5658 		.flags = CFTYPE_NOT_ON_ROOT,
5659 		.seq_show = memory_min_show,
5660 		.write = memory_min_write,
5661 	},
5662 	{
5663 		.name = "low",
5664 		.flags = CFTYPE_NOT_ON_ROOT,
5665 		.seq_show = memory_low_show,
5666 		.write = memory_low_write,
5667 	},
5668 	{
5669 		.name = "high",
5670 		.flags = CFTYPE_NOT_ON_ROOT,
5671 		.seq_show = memory_high_show,
5672 		.write = memory_high_write,
5673 	},
5674 	{
5675 		.name = "max",
5676 		.flags = CFTYPE_NOT_ON_ROOT,
5677 		.seq_show = memory_max_show,
5678 		.write = memory_max_write,
5679 	},
5680 	{
5681 		.name = "events",
5682 		.flags = CFTYPE_NOT_ON_ROOT,
5683 		.file_offset = offsetof(struct mem_cgroup, events_file),
5684 		.seq_show = memory_events_show,
5685 	},
5686 	{
5687 		.name = "stat",
5688 		.flags = CFTYPE_NOT_ON_ROOT,
5689 		.seq_show = memory_stat_show,
5690 	},
5691 	{
5692 		.name = "oom.group",
5693 		.flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
5694 		.seq_show = memory_oom_group_show,
5695 		.write = memory_oom_group_write,
5696 	},
5697 	{ }	/* terminate */
5698 };
5699 
5700 struct cgroup_subsys memory_cgrp_subsys = {
5701 	.css_alloc = mem_cgroup_css_alloc,
5702 	.css_online = mem_cgroup_css_online,
5703 	.css_offline = mem_cgroup_css_offline,
5704 	.css_released = mem_cgroup_css_released,
5705 	.css_free = mem_cgroup_css_free,
5706 	.css_reset = mem_cgroup_css_reset,
5707 	.can_attach = mem_cgroup_can_attach,
5708 	.cancel_attach = mem_cgroup_cancel_attach,
5709 	.post_attach = mem_cgroup_move_task,
5710 	.bind = mem_cgroup_bind,
5711 	.dfl_cftypes = memory_files,
5712 	.legacy_cftypes = mem_cgroup_legacy_files,
5713 	.early_init = 0,
5714 };
5715 
5716 /**
5717  * mem_cgroup_protected - check if memory consumption is in the normal range
5718  * @root: the top ancestor of the sub-tree being checked
5719  * @memcg: the memory cgroup to check
5720  *
5721  * WARNING: This function is not stateless! It can only be used as part
5722  *          of a top-down tree iteration, not for isolated queries.
5723  *
5724  * Returns one of the following:
5725  *   MEMCG_PROT_NONE: cgroup memory is not protected
5726  *   MEMCG_PROT_LOW: cgroup memory is protected as long there is
5727  *     an unprotected supply of reclaimable memory from other cgroups.
5728  *   MEMCG_PROT_MIN: cgroup memory is protected
5729  *
5730  * @root is exclusive; it is never protected when looked at directly
5731  *
5732  * To provide a proper hierarchical behavior, effective memory.min/low values
5733  * are used. Below is the description of how effective memory.low is calculated.
5734  * Effective memory.min values is calculated in the same way.
5735  *
5736  * Effective memory.low is always equal or less than the original memory.low.
5737  * If there is no memory.low overcommittment (which is always true for
5738  * top-level memory cgroups), these two values are equal.
5739  * Otherwise, it's a part of parent's effective memory.low,
5740  * calculated as a cgroup's memory.low usage divided by sum of sibling's
5741  * memory.low usages, where memory.low usage is the size of actually
5742  * protected memory.
5743  *
5744  *                                             low_usage
5745  * elow = min( memory.low, parent->elow * ------------------ ),
5746  *                                        siblings_low_usage
5747  *
5748  *             | memory.current, if memory.current < memory.low
5749  * low_usage = |
5750 	       | 0, otherwise.
5751  *
5752  *
5753  * Such definition of the effective memory.low provides the expected
5754  * hierarchical behavior: parent's memory.low value is limiting
5755  * children, unprotected memory is reclaimed first and cgroups,
5756  * which are not using their guarantee do not affect actual memory
5757  * distribution.
5758  *
5759  * For example, if there are memcgs A, A/B, A/C, A/D and A/E:
5760  *
5761  *     A      A/memory.low = 2G, A/memory.current = 6G
5762  *    //\\
5763  *   BC  DE   B/memory.low = 3G  B/memory.current = 2G
5764  *            C/memory.low = 1G  C/memory.current = 2G
5765  *            D/memory.low = 0   D/memory.current = 2G
5766  *            E/memory.low = 10G E/memory.current = 0
5767  *
5768  * and the memory pressure is applied, the following memory distribution
5769  * is expected (approximately):
5770  *
5771  *     A/memory.current = 2G
5772  *
5773  *     B/memory.current = 1.3G
5774  *     C/memory.current = 0.6G
5775  *     D/memory.current = 0
5776  *     E/memory.current = 0
5777  *
5778  * These calculations require constant tracking of the actual low usages
5779  * (see propagate_protected_usage()), as well as recursive calculation of
5780  * effective memory.low values. But as we do call mem_cgroup_protected()
5781  * path for each memory cgroup top-down from the reclaim,
5782  * it's possible to optimize this part, and save calculated elow
5783  * for next usage. This part is intentionally racy, but it's ok,
5784  * as memory.low is a best-effort mechanism.
5785  */
5786 enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root,
5787 						struct mem_cgroup *memcg)
5788 {
5789 	struct mem_cgroup *parent;
5790 	unsigned long emin, parent_emin;
5791 	unsigned long elow, parent_elow;
5792 	unsigned long usage;
5793 
5794 	if (mem_cgroup_disabled())
5795 		return MEMCG_PROT_NONE;
5796 
5797 	if (!root)
5798 		root = root_mem_cgroup;
5799 	if (memcg == root)
5800 		return MEMCG_PROT_NONE;
5801 
5802 	usage = page_counter_read(&memcg->memory);
5803 	if (!usage)
5804 		return MEMCG_PROT_NONE;
5805 
5806 	emin = memcg->memory.min;
5807 	elow = memcg->memory.low;
5808 
5809 	parent = parent_mem_cgroup(memcg);
5810 	/* No parent means a non-hierarchical mode on v1 memcg */
5811 	if (!parent)
5812 		return MEMCG_PROT_NONE;
5813 
5814 	if (parent == root)
5815 		goto exit;
5816 
5817 	parent_emin = READ_ONCE(parent->memory.emin);
5818 	emin = min(emin, parent_emin);
5819 	if (emin && parent_emin) {
5820 		unsigned long min_usage, siblings_min_usage;
5821 
5822 		min_usage = min(usage, memcg->memory.min);
5823 		siblings_min_usage = atomic_long_read(
5824 			&parent->memory.children_min_usage);
5825 
5826 		if (min_usage && siblings_min_usage)
5827 			emin = min(emin, parent_emin * min_usage /
5828 				   siblings_min_usage);
5829 	}
5830 
5831 	parent_elow = READ_ONCE(parent->memory.elow);
5832 	elow = min(elow, parent_elow);
5833 	if (elow && parent_elow) {
5834 		unsigned long low_usage, siblings_low_usage;
5835 
5836 		low_usage = min(usage, memcg->memory.low);
5837 		siblings_low_usage = atomic_long_read(
5838 			&parent->memory.children_low_usage);
5839 
5840 		if (low_usage && siblings_low_usage)
5841 			elow = min(elow, parent_elow * low_usage /
5842 				   siblings_low_usage);
5843 	}
5844 
5845 exit:
5846 	memcg->memory.emin = emin;
5847 	memcg->memory.elow = elow;
5848 
5849 	if (usage <= emin)
5850 		return MEMCG_PROT_MIN;
5851 	else if (usage <= elow)
5852 		return MEMCG_PROT_LOW;
5853 	else
5854 		return MEMCG_PROT_NONE;
5855 }
5856 
5857 /**
5858  * mem_cgroup_try_charge - try charging a page
5859  * @page: page to charge
5860  * @mm: mm context of the victim
5861  * @gfp_mask: reclaim mode
5862  * @memcgp: charged memcg return
5863  * @compound: charge the page as compound or small page
5864  *
5865  * Try to charge @page to the memcg that @mm belongs to, reclaiming
5866  * pages according to @gfp_mask if necessary.
5867  *
5868  * Returns 0 on success, with *@memcgp pointing to the charged memcg.
5869  * Otherwise, an error code is returned.
5870  *
5871  * After page->mapping has been set up, the caller must finalize the
5872  * charge with mem_cgroup_commit_charge().  Or abort the transaction
5873  * with mem_cgroup_cancel_charge() in case page instantiation fails.
5874  */
5875 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5876 			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
5877 			  bool compound)
5878 {
5879 	struct mem_cgroup *memcg = NULL;
5880 	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5881 	int ret = 0;
5882 
5883 	if (mem_cgroup_disabled())
5884 		goto out;
5885 
5886 	if (PageSwapCache(page)) {
5887 		/*
5888 		 * Every swap fault against a single page tries to charge the
5889 		 * page, bail as early as possible.  shmem_unuse() encounters
5890 		 * already charged pages, too.  The USED bit is protected by
5891 		 * the page lock, which serializes swap cache removal, which
5892 		 * in turn serializes uncharging.
5893 		 */
5894 		VM_BUG_ON_PAGE(!PageLocked(page), page);
5895 		if (compound_head(page)->mem_cgroup)
5896 			goto out;
5897 
5898 		if (do_swap_account) {
5899 			swp_entry_t ent = { .val = page_private(page), };
5900 			unsigned short id = lookup_swap_cgroup_id(ent);
5901 
5902 			rcu_read_lock();
5903 			memcg = mem_cgroup_from_id(id);
5904 			if (memcg && !css_tryget_online(&memcg->css))
5905 				memcg = NULL;
5906 			rcu_read_unlock();
5907 		}
5908 	}
5909 
5910 	if (!memcg)
5911 		memcg = get_mem_cgroup_from_mm(mm);
5912 
5913 	ret = try_charge(memcg, gfp_mask, nr_pages);
5914 
5915 	css_put(&memcg->css);
5916 out:
5917 	*memcgp = memcg;
5918 	return ret;
5919 }
5920 
5921 int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm,
5922 			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
5923 			  bool compound)
5924 {
5925 	struct mem_cgroup *memcg;
5926 	int ret;
5927 
5928 	ret = mem_cgroup_try_charge(page, mm, gfp_mask, memcgp, compound);
5929 	memcg = *memcgp;
5930 	mem_cgroup_throttle_swaprate(memcg, page_to_nid(page), gfp_mask);
5931 	return ret;
5932 }
5933 
5934 /**
5935  * mem_cgroup_commit_charge - commit a page charge
5936  * @page: page to charge
5937  * @memcg: memcg to charge the page to
5938  * @lrucare: page might be on LRU already
5939  * @compound: charge the page as compound or small page
5940  *
5941  * Finalize a charge transaction started by mem_cgroup_try_charge(),
5942  * after page->mapping has been set up.  This must happen atomically
5943  * as part of the page instantiation, i.e. under the page table lock
5944  * for anonymous pages, under the page lock for page and swap cache.
5945  *
5946  * In addition, the page must not be on the LRU during the commit, to
5947  * prevent racing with task migration.  If it might be, use @lrucare.
5948  *
5949  * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
5950  */
5951 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5952 			      bool lrucare, bool compound)
5953 {
5954 	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5955 
5956 	VM_BUG_ON_PAGE(!page->mapping, page);
5957 	VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5958 
5959 	if (mem_cgroup_disabled())
5960 		return;
5961 	/*
5962 	 * Swap faults will attempt to charge the same page multiple
5963 	 * times.  But reuse_swap_page() might have removed the page
5964 	 * from swapcache already, so we can't check PageSwapCache().
5965 	 */
5966 	if (!memcg)
5967 		return;
5968 
5969 	commit_charge(page, memcg, lrucare);
5970 
5971 	local_irq_disable();
5972 	mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
5973 	memcg_check_events(memcg, page);
5974 	local_irq_enable();
5975 
5976 	if (do_memsw_account() && PageSwapCache(page)) {
5977 		swp_entry_t entry = { .val = page_private(page) };
5978 		/*
5979 		 * The swap entry might not get freed for a long time,
5980 		 * let's not wait for it.  The page already received a
5981 		 * memory+swap charge, drop the swap entry duplicate.
5982 		 */
5983 		mem_cgroup_uncharge_swap(entry, nr_pages);
5984 	}
5985 }
5986 
5987 /**
5988  * mem_cgroup_cancel_charge - cancel a page charge
5989  * @page: page to charge
5990  * @memcg: memcg to charge the page to
5991  * @compound: charge the page as compound or small page
5992  *
5993  * Cancel a charge transaction started by mem_cgroup_try_charge().
5994  */
5995 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
5996 		bool compound)
5997 {
5998 	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5999 
6000 	if (mem_cgroup_disabled())
6001 		return;
6002 	/*
6003 	 * Swap faults will attempt to charge the same page multiple
6004 	 * times.  But reuse_swap_page() might have removed the page
6005 	 * from swapcache already, so we can't check PageSwapCache().
6006 	 */
6007 	if (!memcg)
6008 		return;
6009 
6010 	cancel_charge(memcg, nr_pages);
6011 }
6012 
6013 struct uncharge_gather {
6014 	struct mem_cgroup *memcg;
6015 	unsigned long pgpgout;
6016 	unsigned long nr_anon;
6017 	unsigned long nr_file;
6018 	unsigned long nr_kmem;
6019 	unsigned long nr_huge;
6020 	unsigned long nr_shmem;
6021 	struct page *dummy_page;
6022 };
6023 
6024 static inline void uncharge_gather_clear(struct uncharge_gather *ug)
6025 {
6026 	memset(ug, 0, sizeof(*ug));
6027 }
6028 
6029 static void uncharge_batch(const struct uncharge_gather *ug)
6030 {
6031 	unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem;
6032 	unsigned long flags;
6033 
6034 	if (!mem_cgroup_is_root(ug->memcg)) {
6035 		page_counter_uncharge(&ug->memcg->memory, nr_pages);
6036 		if (do_memsw_account())
6037 			page_counter_uncharge(&ug->memcg->memsw, nr_pages);
6038 		if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
6039 			page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
6040 		memcg_oom_recover(ug->memcg);
6041 	}
6042 
6043 	local_irq_save(flags);
6044 	__mod_memcg_state(ug->memcg, MEMCG_RSS, -ug->nr_anon);
6045 	__mod_memcg_state(ug->memcg, MEMCG_CACHE, -ug->nr_file);
6046 	__mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge);
6047 	__mod_memcg_state(ug->memcg, NR_SHMEM, -ug->nr_shmem);
6048 	__count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
6049 	__this_cpu_add(ug->memcg->stat_cpu->nr_page_events, nr_pages);
6050 	memcg_check_events(ug->memcg, ug->dummy_page);
6051 	local_irq_restore(flags);
6052 
6053 	if (!mem_cgroup_is_root(ug->memcg))
6054 		css_put_many(&ug->memcg->css, nr_pages);
6055 }
6056 
6057 static void uncharge_page(struct page *page, struct uncharge_gather *ug)
6058 {
6059 	VM_BUG_ON_PAGE(PageLRU(page), page);
6060 	VM_BUG_ON_PAGE(page_count(page) && !is_zone_device_page(page) &&
6061 			!PageHWPoison(page) , page);
6062 
6063 	if (!page->mem_cgroup)
6064 		return;
6065 
6066 	/*
6067 	 * Nobody should be changing or seriously looking at
6068 	 * page->mem_cgroup at this point, we have fully
6069 	 * exclusive access to the page.
6070 	 */
6071 
6072 	if (ug->memcg != page->mem_cgroup) {
6073 		if (ug->memcg) {
6074 			uncharge_batch(ug);
6075 			uncharge_gather_clear(ug);
6076 		}
6077 		ug->memcg = page->mem_cgroup;
6078 	}
6079 
6080 	if (!PageKmemcg(page)) {
6081 		unsigned int nr_pages = 1;
6082 
6083 		if (PageTransHuge(page)) {
6084 			nr_pages <<= compound_order(page);
6085 			ug->nr_huge += nr_pages;
6086 		}
6087 		if (PageAnon(page))
6088 			ug->nr_anon += nr_pages;
6089 		else {
6090 			ug->nr_file += nr_pages;
6091 			if (PageSwapBacked(page))
6092 				ug->nr_shmem += nr_pages;
6093 		}
6094 		ug->pgpgout++;
6095 	} else {
6096 		ug->nr_kmem += 1 << compound_order(page);
6097 		__ClearPageKmemcg(page);
6098 	}
6099 
6100 	ug->dummy_page = page;
6101 	page->mem_cgroup = NULL;
6102 }
6103 
6104 static void uncharge_list(struct list_head *page_list)
6105 {
6106 	struct uncharge_gather ug;
6107 	struct list_head *next;
6108 
6109 	uncharge_gather_clear(&ug);
6110 
6111 	/*
6112 	 * Note that the list can be a single page->lru; hence the
6113 	 * do-while loop instead of a simple list_for_each_entry().
6114 	 */
6115 	next = page_list->next;
6116 	do {
6117 		struct page *page;
6118 
6119 		page = list_entry(next, struct page, lru);
6120 		next = page->lru.next;
6121 
6122 		uncharge_page(page, &ug);
6123 	} while (next != page_list);
6124 
6125 	if (ug.memcg)
6126 		uncharge_batch(&ug);
6127 }
6128 
6129 /**
6130  * mem_cgroup_uncharge - uncharge a page
6131  * @page: page to uncharge
6132  *
6133  * Uncharge a page previously charged with mem_cgroup_try_charge() and
6134  * mem_cgroup_commit_charge().
6135  */
6136 void mem_cgroup_uncharge(struct page *page)
6137 {
6138 	struct uncharge_gather ug;
6139 
6140 	if (mem_cgroup_disabled())
6141 		return;
6142 
6143 	/* Don't touch page->lru of any random page, pre-check: */
6144 	if (!page->mem_cgroup)
6145 		return;
6146 
6147 	uncharge_gather_clear(&ug);
6148 	uncharge_page(page, &ug);
6149 	uncharge_batch(&ug);
6150 }
6151 
6152 /**
6153  * mem_cgroup_uncharge_list - uncharge a list of page
6154  * @page_list: list of pages to uncharge
6155  *
6156  * Uncharge a list of pages previously charged with
6157  * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
6158  */
6159 void mem_cgroup_uncharge_list(struct list_head *page_list)
6160 {
6161 	if (mem_cgroup_disabled())
6162 		return;
6163 
6164 	if (!list_empty(page_list))
6165 		uncharge_list(page_list);
6166 }
6167 
6168 /**
6169  * mem_cgroup_migrate - charge a page's replacement
6170  * @oldpage: currently circulating page
6171  * @newpage: replacement page
6172  *
6173  * Charge @newpage as a replacement page for @oldpage. @oldpage will
6174  * be uncharged upon free.
6175  *
6176  * Both pages must be locked, @newpage->mapping must be set up.
6177  */
6178 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
6179 {
6180 	struct mem_cgroup *memcg;
6181 	unsigned int nr_pages;
6182 	bool compound;
6183 	unsigned long flags;
6184 
6185 	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
6186 	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
6187 	VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
6188 	VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
6189 		       newpage);
6190 
6191 	if (mem_cgroup_disabled())
6192 		return;
6193 
6194 	/* Page cache replacement: new page already charged? */
6195 	if (newpage->mem_cgroup)
6196 		return;
6197 
6198 	/* Swapcache readahead pages can get replaced before being charged */
6199 	memcg = oldpage->mem_cgroup;
6200 	if (!memcg)
6201 		return;
6202 
6203 	/* Force-charge the new page. The old one will be freed soon */
6204 	compound = PageTransHuge(newpage);
6205 	nr_pages = compound ? hpage_nr_pages(newpage) : 1;
6206 
6207 	page_counter_charge(&memcg->memory, nr_pages);
6208 	if (do_memsw_account())
6209 		page_counter_charge(&memcg->memsw, nr_pages);
6210 	css_get_many(&memcg->css, nr_pages);
6211 
6212 	commit_charge(newpage, memcg, false);
6213 
6214 	local_irq_save(flags);
6215 	mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
6216 	memcg_check_events(memcg, newpage);
6217 	local_irq_restore(flags);
6218 }
6219 
6220 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
6221 EXPORT_SYMBOL(memcg_sockets_enabled_key);
6222 
6223 void mem_cgroup_sk_alloc(struct sock *sk)
6224 {
6225 	struct mem_cgroup *memcg;
6226 
6227 	if (!mem_cgroup_sockets_enabled)
6228 		return;
6229 
6230 	/*
6231 	 * Socket cloning can throw us here with sk_memcg already
6232 	 * filled. It won't however, necessarily happen from
6233 	 * process context. So the test for root memcg given
6234 	 * the current task's memcg won't help us in this case.
6235 	 *
6236 	 * Respecting the original socket's memcg is a better
6237 	 * decision in this case.
6238 	 */
6239 	if (sk->sk_memcg) {
6240 		css_get(&sk->sk_memcg->css);
6241 		return;
6242 	}
6243 
6244 	rcu_read_lock();
6245 	memcg = mem_cgroup_from_task(current);
6246 	if (memcg == root_mem_cgroup)
6247 		goto out;
6248 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
6249 		goto out;
6250 	if (css_tryget_online(&memcg->css))
6251 		sk->sk_memcg = memcg;
6252 out:
6253 	rcu_read_unlock();
6254 }
6255 
6256 void mem_cgroup_sk_free(struct sock *sk)
6257 {
6258 	if (sk->sk_memcg)
6259 		css_put(&sk->sk_memcg->css);
6260 }
6261 
6262 /**
6263  * mem_cgroup_charge_skmem - charge socket memory
6264  * @memcg: memcg to charge
6265  * @nr_pages: number of pages to charge
6266  *
6267  * Charges @nr_pages to @memcg. Returns %true if the charge fit within
6268  * @memcg's configured limit, %false if the charge had to be forced.
6269  */
6270 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
6271 {
6272 	gfp_t gfp_mask = GFP_KERNEL;
6273 
6274 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
6275 		struct page_counter *fail;
6276 
6277 		if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
6278 			memcg->tcpmem_pressure = 0;
6279 			return true;
6280 		}
6281 		page_counter_charge(&memcg->tcpmem, nr_pages);
6282 		memcg->tcpmem_pressure = 1;
6283 		return false;
6284 	}
6285 
6286 	/* Don't block in the packet receive path */
6287 	if (in_softirq())
6288 		gfp_mask = GFP_NOWAIT;
6289 
6290 	mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
6291 
6292 	if (try_charge(memcg, gfp_mask, nr_pages) == 0)
6293 		return true;
6294 
6295 	try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
6296 	return false;
6297 }
6298 
6299 /**
6300  * mem_cgroup_uncharge_skmem - uncharge socket memory
6301  * @memcg: memcg to uncharge
6302  * @nr_pages: number of pages to uncharge
6303  */
6304 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
6305 {
6306 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
6307 		page_counter_uncharge(&memcg->tcpmem, nr_pages);
6308 		return;
6309 	}
6310 
6311 	mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
6312 
6313 	refill_stock(memcg, nr_pages);
6314 }
6315 
6316 static int __init cgroup_memory(char *s)
6317 {
6318 	char *token;
6319 
6320 	while ((token = strsep(&s, ",")) != NULL) {
6321 		if (!*token)
6322 			continue;
6323 		if (!strcmp(token, "nosocket"))
6324 			cgroup_memory_nosocket = true;
6325 		if (!strcmp(token, "nokmem"))
6326 			cgroup_memory_nokmem = true;
6327 	}
6328 	return 0;
6329 }
6330 __setup("cgroup.memory=", cgroup_memory);
6331 
6332 /*
6333  * subsys_initcall() for memory controller.
6334  *
6335  * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
6336  * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
6337  * basically everything that doesn't depend on a specific mem_cgroup structure
6338  * should be initialized from here.
6339  */
6340 static int __init mem_cgroup_init(void)
6341 {
6342 	int cpu, node;
6343 
6344 #ifdef CONFIG_MEMCG_KMEM
6345 	/*
6346 	 * Kmem cache creation is mostly done with the slab_mutex held,
6347 	 * so use a workqueue with limited concurrency to avoid stalling
6348 	 * all worker threads in case lots of cgroups are created and
6349 	 * destroyed simultaneously.
6350 	 */
6351 	memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1);
6352 	BUG_ON(!memcg_kmem_cache_wq);
6353 #endif
6354 
6355 	cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
6356 				  memcg_hotplug_cpu_dead);
6357 
6358 	for_each_possible_cpu(cpu)
6359 		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
6360 			  drain_local_stock);
6361 
6362 	for_each_node(node) {
6363 		struct mem_cgroup_tree_per_node *rtpn;
6364 
6365 		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
6366 				    node_online(node) ? node : NUMA_NO_NODE);
6367 
6368 		rtpn->rb_root = RB_ROOT;
6369 		rtpn->rb_rightmost = NULL;
6370 		spin_lock_init(&rtpn->lock);
6371 		soft_limit_tree.rb_tree_per_node[node] = rtpn;
6372 	}
6373 
6374 	return 0;
6375 }
6376 subsys_initcall(mem_cgroup_init);
6377 
6378 #ifdef CONFIG_MEMCG_SWAP
6379 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
6380 {
6381 	while (!refcount_inc_not_zero(&memcg->id.ref)) {
6382 		/*
6383 		 * The root cgroup cannot be destroyed, so it's refcount must
6384 		 * always be >= 1.
6385 		 */
6386 		if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
6387 			VM_BUG_ON(1);
6388 			break;
6389 		}
6390 		memcg = parent_mem_cgroup(memcg);
6391 		if (!memcg)
6392 			memcg = root_mem_cgroup;
6393 	}
6394 	return memcg;
6395 }
6396 
6397 /**
6398  * mem_cgroup_swapout - transfer a memsw charge to swap
6399  * @page: page whose memsw charge to transfer
6400  * @entry: swap entry to move the charge to
6401  *
6402  * Transfer the memsw charge of @page to @entry.
6403  */
6404 void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
6405 {
6406 	struct mem_cgroup *memcg, *swap_memcg;
6407 	unsigned int nr_entries;
6408 	unsigned short oldid;
6409 
6410 	VM_BUG_ON_PAGE(PageLRU(page), page);
6411 	VM_BUG_ON_PAGE(page_count(page), page);
6412 
6413 	if (!do_memsw_account())
6414 		return;
6415 
6416 	memcg = page->mem_cgroup;
6417 
6418 	/* Readahead page, never charged */
6419 	if (!memcg)
6420 		return;
6421 
6422 	/*
6423 	 * In case the memcg owning these pages has been offlined and doesn't
6424 	 * have an ID allocated to it anymore, charge the closest online
6425 	 * ancestor for the swap instead and transfer the memory+swap charge.
6426 	 */
6427 	swap_memcg = mem_cgroup_id_get_online(memcg);
6428 	nr_entries = hpage_nr_pages(page);
6429 	/* Get references for the tail pages, too */
6430 	if (nr_entries > 1)
6431 		mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
6432 	oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
6433 				   nr_entries);
6434 	VM_BUG_ON_PAGE(oldid, page);
6435 	mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
6436 
6437 	page->mem_cgroup = NULL;
6438 
6439 	if (!mem_cgroup_is_root(memcg))
6440 		page_counter_uncharge(&memcg->memory, nr_entries);
6441 
6442 	if (memcg != swap_memcg) {
6443 		if (!mem_cgroup_is_root(swap_memcg))
6444 			page_counter_charge(&swap_memcg->memsw, nr_entries);
6445 		page_counter_uncharge(&memcg->memsw, nr_entries);
6446 	}
6447 
6448 	/*
6449 	 * Interrupts should be disabled here because the caller holds the
6450 	 * i_pages lock which is taken with interrupts-off. It is
6451 	 * important here to have the interrupts disabled because it is the
6452 	 * only synchronisation we have for updating the per-CPU variables.
6453 	 */
6454 	VM_BUG_ON(!irqs_disabled());
6455 	mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page),
6456 				     -nr_entries);
6457 	memcg_check_events(memcg, page);
6458 
6459 	if (!mem_cgroup_is_root(memcg))
6460 		css_put_many(&memcg->css, nr_entries);
6461 }
6462 
6463 /**
6464  * mem_cgroup_try_charge_swap - try charging swap space for a page
6465  * @page: page being added to swap
6466  * @entry: swap entry to charge
6467  *
6468  * Try to charge @page's memcg for the swap space at @entry.
6469  *
6470  * Returns 0 on success, -ENOMEM on failure.
6471  */
6472 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
6473 {
6474 	unsigned int nr_pages = hpage_nr_pages(page);
6475 	struct page_counter *counter;
6476 	struct mem_cgroup *memcg;
6477 	unsigned short oldid;
6478 
6479 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
6480 		return 0;
6481 
6482 	memcg = page->mem_cgroup;
6483 
6484 	/* Readahead page, never charged */
6485 	if (!memcg)
6486 		return 0;
6487 
6488 	if (!entry.val) {
6489 		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
6490 		return 0;
6491 	}
6492 
6493 	memcg = mem_cgroup_id_get_online(memcg);
6494 
6495 	if (!mem_cgroup_is_root(memcg) &&
6496 	    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
6497 		memcg_memory_event(memcg, MEMCG_SWAP_MAX);
6498 		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
6499 		mem_cgroup_id_put(memcg);
6500 		return -ENOMEM;
6501 	}
6502 
6503 	/* Get references for the tail pages, too */
6504 	if (nr_pages > 1)
6505 		mem_cgroup_id_get_many(memcg, nr_pages - 1);
6506 	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
6507 	VM_BUG_ON_PAGE(oldid, page);
6508 	mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
6509 
6510 	return 0;
6511 }
6512 
6513 /**
6514  * mem_cgroup_uncharge_swap - uncharge swap space
6515  * @entry: swap entry to uncharge
6516  * @nr_pages: the amount of swap space to uncharge
6517  */
6518 void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
6519 {
6520 	struct mem_cgroup *memcg;
6521 	unsigned short id;
6522 
6523 	if (!do_swap_account)
6524 		return;
6525 
6526 	id = swap_cgroup_record(entry, 0, nr_pages);
6527 	rcu_read_lock();
6528 	memcg = mem_cgroup_from_id(id);
6529 	if (memcg) {
6530 		if (!mem_cgroup_is_root(memcg)) {
6531 			if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
6532 				page_counter_uncharge(&memcg->swap, nr_pages);
6533 			else
6534 				page_counter_uncharge(&memcg->memsw, nr_pages);
6535 		}
6536 		mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
6537 		mem_cgroup_id_put_many(memcg, nr_pages);
6538 	}
6539 	rcu_read_unlock();
6540 }
6541 
6542 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
6543 {
6544 	long nr_swap_pages = get_nr_swap_pages();
6545 
6546 	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6547 		return nr_swap_pages;
6548 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6549 		nr_swap_pages = min_t(long, nr_swap_pages,
6550 				      READ_ONCE(memcg->swap.max) -
6551 				      page_counter_read(&memcg->swap));
6552 	return nr_swap_pages;
6553 }
6554 
6555 bool mem_cgroup_swap_full(struct page *page)
6556 {
6557 	struct mem_cgroup *memcg;
6558 
6559 	VM_BUG_ON_PAGE(!PageLocked(page), page);
6560 
6561 	if (vm_swap_full())
6562 		return true;
6563 	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6564 		return false;
6565 
6566 	memcg = page->mem_cgroup;
6567 	if (!memcg)
6568 		return false;
6569 
6570 	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6571 		if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.max)
6572 			return true;
6573 
6574 	return false;
6575 }
6576 
6577 /* for remember boot option*/
6578 #ifdef CONFIG_MEMCG_SWAP_ENABLED
6579 static int really_do_swap_account __initdata = 1;
6580 #else
6581 static int really_do_swap_account __initdata;
6582 #endif
6583 
6584 static int __init enable_swap_account(char *s)
6585 {
6586 	if (!strcmp(s, "1"))
6587 		really_do_swap_account = 1;
6588 	else if (!strcmp(s, "0"))
6589 		really_do_swap_account = 0;
6590 	return 1;
6591 }
6592 __setup("swapaccount=", enable_swap_account);
6593 
6594 static u64 swap_current_read(struct cgroup_subsys_state *css,
6595 			     struct cftype *cft)
6596 {
6597 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6598 
6599 	return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
6600 }
6601 
6602 static int swap_max_show(struct seq_file *m, void *v)
6603 {
6604 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6605 	unsigned long max = READ_ONCE(memcg->swap.max);
6606 
6607 	if (max == PAGE_COUNTER_MAX)
6608 		seq_puts(m, "max\n");
6609 	else
6610 		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
6611 
6612 	return 0;
6613 }
6614 
6615 static ssize_t swap_max_write(struct kernfs_open_file *of,
6616 			      char *buf, size_t nbytes, loff_t off)
6617 {
6618 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6619 	unsigned long max;
6620 	int err;
6621 
6622 	buf = strstrip(buf);
6623 	err = page_counter_memparse(buf, "max", &max);
6624 	if (err)
6625 		return err;
6626 
6627 	xchg(&memcg->swap.max, max);
6628 
6629 	return nbytes;
6630 }
6631 
6632 static int swap_events_show(struct seq_file *m, void *v)
6633 {
6634 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6635 
6636 	seq_printf(m, "max %lu\n",
6637 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
6638 	seq_printf(m, "fail %lu\n",
6639 		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
6640 
6641 	return 0;
6642 }
6643 
6644 static struct cftype swap_files[] = {
6645 	{
6646 		.name = "swap.current",
6647 		.flags = CFTYPE_NOT_ON_ROOT,
6648 		.read_u64 = swap_current_read,
6649 	},
6650 	{
6651 		.name = "swap.max",
6652 		.flags = CFTYPE_NOT_ON_ROOT,
6653 		.seq_show = swap_max_show,
6654 		.write = swap_max_write,
6655 	},
6656 	{
6657 		.name = "swap.events",
6658 		.flags = CFTYPE_NOT_ON_ROOT,
6659 		.file_offset = offsetof(struct mem_cgroup, swap_events_file),
6660 		.seq_show = swap_events_show,
6661 	},
6662 	{ }	/* terminate */
6663 };
6664 
6665 static struct cftype memsw_cgroup_files[] = {
6666 	{
6667 		.name = "memsw.usage_in_bytes",
6668 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6669 		.read_u64 = mem_cgroup_read_u64,
6670 	},
6671 	{
6672 		.name = "memsw.max_usage_in_bytes",
6673 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6674 		.write = mem_cgroup_reset,
6675 		.read_u64 = mem_cgroup_read_u64,
6676 	},
6677 	{
6678 		.name = "memsw.limit_in_bytes",
6679 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6680 		.write = mem_cgroup_write,
6681 		.read_u64 = mem_cgroup_read_u64,
6682 	},
6683 	{
6684 		.name = "memsw.failcnt",
6685 		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6686 		.write = mem_cgroup_reset,
6687 		.read_u64 = mem_cgroup_read_u64,
6688 	},
6689 	{ },	/* terminate */
6690 };
6691 
6692 static int __init mem_cgroup_swap_init(void)
6693 {
6694 	if (!mem_cgroup_disabled() && really_do_swap_account) {
6695 		do_swap_account = 1;
6696 		WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
6697 					       swap_files));
6698 		WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
6699 						  memsw_cgroup_files));
6700 	}
6701 	return 0;
6702 }
6703 subsys_initcall(mem_cgroup_swap_init);
6704 
6705 #endif /* CONFIG_MEMCG_SWAP */
6706