1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
25d1ea48bSJohannes Weiner #include <linux/swap_cgroup.h>
35d1ea48bSJohannes Weiner #include <linux/vmalloc.h>
45d1ea48bSJohannes Weiner #include <linux/mm.h>
55d1ea48bSJohannes Weiner
65d1ea48bSJohannes Weiner #include <linux/swapops.h> /* depends on mm.h include */
75d1ea48bSJohannes Weiner
85d1ea48bSJohannes Weiner static DEFINE_MUTEX(swap_cgroup_mutex);
95d1ea48bSJohannes Weiner struct swap_cgroup_ctrl {
105d1ea48bSJohannes Weiner struct page **map;
115d1ea48bSJohannes Weiner unsigned long length;
125d1ea48bSJohannes Weiner spinlock_t lock;
135d1ea48bSJohannes Weiner };
145d1ea48bSJohannes Weiner
155d1ea48bSJohannes Weiner static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
165d1ea48bSJohannes Weiner
175d1ea48bSJohannes Weiner struct swap_cgroup {
185d1ea48bSJohannes Weiner unsigned short id;
195d1ea48bSJohannes Weiner };
205d1ea48bSJohannes Weiner #define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
215d1ea48bSJohannes Weiner
225d1ea48bSJohannes Weiner /*
235d1ea48bSJohannes Weiner * SwapCgroup implements "lookup" and "exchange" operations.
245d1ea48bSJohannes Weiner * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
255d1ea48bSJohannes Weiner * against SwapCache. At swap_free(), this is accessed directly from swap.
265d1ea48bSJohannes Weiner *
275d1ea48bSJohannes Weiner * This means,
285d1ea48bSJohannes Weiner * - we have no race in "exchange" when we're accessed via SwapCache because
295d1ea48bSJohannes Weiner * SwapCache(and its swp_entry) is under lock.
305d1ea48bSJohannes Weiner * - When called via swap_free(), there is no user of this entry and no race.
315d1ea48bSJohannes Weiner * Then, we don't need lock around "exchange".
325d1ea48bSJohannes Weiner *
335d1ea48bSJohannes Weiner * TODO: we can push these buffers out to HIGHMEM.
345d1ea48bSJohannes Weiner */
355d1ea48bSJohannes Weiner
365d1ea48bSJohannes Weiner /*
375d1ea48bSJohannes Weiner * allocate buffer for swap_cgroup.
385d1ea48bSJohannes Weiner */
swap_cgroup_prepare(int type)395d1ea48bSJohannes Weiner static int swap_cgroup_prepare(int type)
405d1ea48bSJohannes Weiner {
415d1ea48bSJohannes Weiner struct page *page;
425d1ea48bSJohannes Weiner struct swap_cgroup_ctrl *ctrl;
435d1ea48bSJohannes Weiner unsigned long idx, max;
445d1ea48bSJohannes Weiner
455d1ea48bSJohannes Weiner ctrl = &swap_cgroup_ctrl[type];
465d1ea48bSJohannes Weiner
475d1ea48bSJohannes Weiner for (idx = 0; idx < ctrl->length; idx++) {
485d1ea48bSJohannes Weiner page = alloc_page(GFP_KERNEL | __GFP_ZERO);
495d1ea48bSJohannes Weiner if (!page)
505d1ea48bSJohannes Weiner goto not_enough_page;
515d1ea48bSJohannes Weiner ctrl->map[idx] = page;
52ef707629SYu Zhao
53ef707629SYu Zhao if (!(idx % SWAP_CLUSTER_MAX))
54ef707629SYu Zhao cond_resched();
555d1ea48bSJohannes Weiner }
565d1ea48bSJohannes Weiner return 0;
575d1ea48bSJohannes Weiner not_enough_page:
585d1ea48bSJohannes Weiner max = idx;
595d1ea48bSJohannes Weiner for (idx = 0; idx < max; idx++)
605d1ea48bSJohannes Weiner __free_page(ctrl->map[idx]);
615d1ea48bSJohannes Weiner
625d1ea48bSJohannes Weiner return -ENOMEM;
635d1ea48bSJohannes Weiner }
645d1ea48bSJohannes Weiner
__lookup_swap_cgroup(struct swap_cgroup_ctrl * ctrl,pgoff_t offset)6538d8b4e6SHuang Ying static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl,
6638d8b4e6SHuang Ying pgoff_t offset)
6738d8b4e6SHuang Ying {
6838d8b4e6SHuang Ying struct page *mappage;
6938d8b4e6SHuang Ying struct swap_cgroup *sc;
7038d8b4e6SHuang Ying
7138d8b4e6SHuang Ying mappage = ctrl->map[offset / SC_PER_PAGE];
7238d8b4e6SHuang Ying sc = page_address(mappage);
7338d8b4e6SHuang Ying return sc + offset % SC_PER_PAGE;
7438d8b4e6SHuang Ying }
7538d8b4e6SHuang Ying
lookup_swap_cgroup(swp_entry_t ent,struct swap_cgroup_ctrl ** ctrlp)765d1ea48bSJohannes Weiner static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
775d1ea48bSJohannes Weiner struct swap_cgroup_ctrl **ctrlp)
785d1ea48bSJohannes Weiner {
795d1ea48bSJohannes Weiner pgoff_t offset = swp_offset(ent);
805d1ea48bSJohannes Weiner struct swap_cgroup_ctrl *ctrl;
815d1ea48bSJohannes Weiner
825d1ea48bSJohannes Weiner ctrl = &swap_cgroup_ctrl[swp_type(ent)];
835d1ea48bSJohannes Weiner if (ctrlp)
845d1ea48bSJohannes Weiner *ctrlp = ctrl;
8538d8b4e6SHuang Ying return __lookup_swap_cgroup(ctrl, offset);
865d1ea48bSJohannes Weiner }
875d1ea48bSJohannes Weiner
885d1ea48bSJohannes Weiner /**
895d1ea48bSJohannes Weiner * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
905d1ea48bSJohannes Weiner * @ent: swap entry to be cmpxchged
915d1ea48bSJohannes Weiner * @old: old id
925d1ea48bSJohannes Weiner * @new: new id
935d1ea48bSJohannes Weiner *
945d1ea48bSJohannes Weiner * Returns old id at success, 0 at failure.
955d1ea48bSJohannes Weiner * (There is no mem_cgroup using 0 as its id)
965d1ea48bSJohannes Weiner */
swap_cgroup_cmpxchg(swp_entry_t ent,unsigned short old,unsigned short new)975d1ea48bSJohannes Weiner unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
985d1ea48bSJohannes Weiner unsigned short old, unsigned short new)
995d1ea48bSJohannes Weiner {
1005d1ea48bSJohannes Weiner struct swap_cgroup_ctrl *ctrl;
1015d1ea48bSJohannes Weiner struct swap_cgroup *sc;
1025d1ea48bSJohannes Weiner unsigned long flags;
1035d1ea48bSJohannes Weiner unsigned short retval;
1045d1ea48bSJohannes Weiner
1055d1ea48bSJohannes Weiner sc = lookup_swap_cgroup(ent, &ctrl);
1065d1ea48bSJohannes Weiner
1075d1ea48bSJohannes Weiner spin_lock_irqsave(&ctrl->lock, flags);
1085d1ea48bSJohannes Weiner retval = sc->id;
1095d1ea48bSJohannes Weiner if (retval == old)
1105d1ea48bSJohannes Weiner sc->id = new;
1115d1ea48bSJohannes Weiner else
1125d1ea48bSJohannes Weiner retval = 0;
1135d1ea48bSJohannes Weiner spin_unlock_irqrestore(&ctrl->lock, flags);
1145d1ea48bSJohannes Weiner return retval;
1155d1ea48bSJohannes Weiner }
1165d1ea48bSJohannes Weiner
1175d1ea48bSJohannes Weiner /**
11838d8b4e6SHuang Ying * swap_cgroup_record - record mem_cgroup for a set of swap entries
11938d8b4e6SHuang Ying * @ent: the first swap entry to be recorded into
1205d1ea48bSJohannes Weiner * @id: mem_cgroup to be recorded
12138d8b4e6SHuang Ying * @nr_ents: number of swap entries to be recorded
1225d1ea48bSJohannes Weiner *
1235d1ea48bSJohannes Weiner * Returns old value at success, 0 at failure.
1245d1ea48bSJohannes Weiner * (Of course, old value can be 0.)
1255d1ea48bSJohannes Weiner */
swap_cgroup_record(swp_entry_t ent,unsigned short id,unsigned int nr_ents)12638d8b4e6SHuang Ying unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id,
12738d8b4e6SHuang Ying unsigned int nr_ents)
1285d1ea48bSJohannes Weiner {
1295d1ea48bSJohannes Weiner struct swap_cgroup_ctrl *ctrl;
1305d1ea48bSJohannes Weiner struct swap_cgroup *sc;
1315d1ea48bSJohannes Weiner unsigned short old;
1325d1ea48bSJohannes Weiner unsigned long flags;
13338d8b4e6SHuang Ying pgoff_t offset = swp_offset(ent);
13438d8b4e6SHuang Ying pgoff_t end = offset + nr_ents;
1355d1ea48bSJohannes Weiner
1365d1ea48bSJohannes Weiner sc = lookup_swap_cgroup(ent, &ctrl);
1375d1ea48bSJohannes Weiner
1385d1ea48bSJohannes Weiner spin_lock_irqsave(&ctrl->lock, flags);
1395d1ea48bSJohannes Weiner old = sc->id;
14038d8b4e6SHuang Ying for (;;) {
14138d8b4e6SHuang Ying VM_BUG_ON(sc->id != old);
1425d1ea48bSJohannes Weiner sc->id = id;
14338d8b4e6SHuang Ying offset++;
14438d8b4e6SHuang Ying if (offset == end)
14538d8b4e6SHuang Ying break;
14638d8b4e6SHuang Ying if (offset % SC_PER_PAGE)
14738d8b4e6SHuang Ying sc++;
14838d8b4e6SHuang Ying else
14938d8b4e6SHuang Ying sc = __lookup_swap_cgroup(ctrl, offset);
15038d8b4e6SHuang Ying }
1515d1ea48bSJohannes Weiner spin_unlock_irqrestore(&ctrl->lock, flags);
1525d1ea48bSJohannes Weiner
1535d1ea48bSJohannes Weiner return old;
1545d1ea48bSJohannes Weiner }
1555d1ea48bSJohannes Weiner
1565d1ea48bSJohannes Weiner /**
1575d1ea48bSJohannes Weiner * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
1585d1ea48bSJohannes Weiner * @ent: swap entry to be looked up.
1595d1ea48bSJohannes Weiner *
1605d1ea48bSJohannes Weiner * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
1615d1ea48bSJohannes Weiner */
lookup_swap_cgroup_id(swp_entry_t ent)1625d1ea48bSJohannes Weiner unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
1635d1ea48bSJohannes Weiner {
1645d1ea48bSJohannes Weiner return lookup_swap_cgroup(ent, NULL)->id;
1655d1ea48bSJohannes Weiner }
1665d1ea48bSJohannes Weiner
swap_cgroup_swapon(int type,unsigned long max_pages)1675d1ea48bSJohannes Weiner int swap_cgroup_swapon(int type, unsigned long max_pages)
1685d1ea48bSJohannes Weiner {
1695d1ea48bSJohannes Weiner void *array;
1705d1ea48bSJohannes Weiner unsigned long length;
1715d1ea48bSJohannes Weiner struct swap_cgroup_ctrl *ctrl;
1725d1ea48bSJohannes Weiner
173*c91bdc93SJohannes Weiner if (mem_cgroup_disabled())
174*c91bdc93SJohannes Weiner return 0;
175*c91bdc93SJohannes Weiner
1765d1ea48bSJohannes Weiner length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
1775d1ea48bSJohannes Weiner
1783000f2e2SPaolo Bonzini array = vcalloc(length, sizeof(void *));
1795d1ea48bSJohannes Weiner if (!array)
1805d1ea48bSJohannes Weiner goto nomem;
1815d1ea48bSJohannes Weiner
1825d1ea48bSJohannes Weiner ctrl = &swap_cgroup_ctrl[type];
1835d1ea48bSJohannes Weiner mutex_lock(&swap_cgroup_mutex);
1845d1ea48bSJohannes Weiner ctrl->length = length;
1855d1ea48bSJohannes Weiner ctrl->map = array;
1865d1ea48bSJohannes Weiner spin_lock_init(&ctrl->lock);
1875d1ea48bSJohannes Weiner if (swap_cgroup_prepare(type)) {
1885d1ea48bSJohannes Weiner /* memory shortage */
1895d1ea48bSJohannes Weiner ctrl->map = NULL;
1905d1ea48bSJohannes Weiner ctrl->length = 0;
1915d1ea48bSJohannes Weiner mutex_unlock(&swap_cgroup_mutex);
1925d1ea48bSJohannes Weiner vfree(array);
1935d1ea48bSJohannes Weiner goto nomem;
1945d1ea48bSJohannes Weiner }
1955d1ea48bSJohannes Weiner mutex_unlock(&swap_cgroup_mutex);
1965d1ea48bSJohannes Weiner
1975d1ea48bSJohannes Weiner return 0;
1985d1ea48bSJohannes Weiner nomem:
1991170532bSJoe Perches pr_info("couldn't allocate enough memory for swap_cgroup\n");
2001170532bSJoe Perches pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
2015d1ea48bSJohannes Weiner return -ENOMEM;
2025d1ea48bSJohannes Weiner }
2035d1ea48bSJohannes Weiner
swap_cgroup_swapoff(int type)2045d1ea48bSJohannes Weiner void swap_cgroup_swapoff(int type)
2055d1ea48bSJohannes Weiner {
2065d1ea48bSJohannes Weiner struct page **map;
2075d1ea48bSJohannes Weiner unsigned long i, length;
2085d1ea48bSJohannes Weiner struct swap_cgroup_ctrl *ctrl;
2095d1ea48bSJohannes Weiner
210*c91bdc93SJohannes Weiner if (mem_cgroup_disabled())
211*c91bdc93SJohannes Weiner return;
212*c91bdc93SJohannes Weiner
2135d1ea48bSJohannes Weiner mutex_lock(&swap_cgroup_mutex);
2145d1ea48bSJohannes Weiner ctrl = &swap_cgroup_ctrl[type];
2155d1ea48bSJohannes Weiner map = ctrl->map;
2165d1ea48bSJohannes Weiner length = ctrl->length;
2175d1ea48bSJohannes Weiner ctrl->map = NULL;
2185d1ea48bSJohannes Weiner ctrl->length = 0;
2195d1ea48bSJohannes Weiner mutex_unlock(&swap_cgroup_mutex);
2205d1ea48bSJohannes Weiner
2215d1ea48bSJohannes Weiner if (map) {
2225d1ea48bSJohannes Weiner for (i = 0; i < length; i++) {
2235d1ea48bSJohannes Weiner struct page *page = map[i];
2245d1ea48bSJohannes Weiner if (page)
2255d1ea48bSJohannes Weiner __free_page(page);
226460bcec8SDavid Rientjes if (!(i % SWAP_CLUSTER_MAX))
227460bcec8SDavid Rientjes cond_resched();
2285d1ea48bSJohannes Weiner }
2295d1ea48bSJohannes Weiner vfree(map);
2305d1ea48bSJohannes Weiner }
2315d1ea48bSJohannes Weiner }
232