1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/swap_cgroup.h>
3 #include <linux/vmalloc.h>
4 #include <linux/mm.h>
5
6 #include <linux/swapops.h> /* depends on mm.h include */
7
8 static DEFINE_MUTEX(swap_cgroup_mutex);
9
10 /* Pack two cgroup id (short) of two entries in one swap_cgroup (atomic_t) */
11 #define ID_PER_SC (sizeof(struct swap_cgroup) / sizeof(unsigned short))
12 #define ID_SHIFT (BITS_PER_TYPE(unsigned short))
13 #define ID_MASK (BIT(ID_SHIFT) - 1)
14 struct swap_cgroup {
15 atomic_t ids;
16 };
17
18 struct swap_cgroup_ctrl {
19 struct swap_cgroup *map;
20 };
21
22 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
23
__swap_cgroup_id_lookup(struct swap_cgroup * map,pgoff_t offset)24 static unsigned short __swap_cgroup_id_lookup(struct swap_cgroup *map,
25 pgoff_t offset)
26 {
27 unsigned int shift = (offset % ID_PER_SC) * ID_SHIFT;
28 unsigned int old_ids = atomic_read(&map[offset / ID_PER_SC].ids);
29
30 BUILD_BUG_ON(!is_power_of_2(ID_PER_SC));
31 BUILD_BUG_ON(sizeof(struct swap_cgroup) != sizeof(atomic_t));
32
33 return (old_ids >> shift) & ID_MASK;
34 }
35
__swap_cgroup_id_xchg(struct swap_cgroup * map,pgoff_t offset,unsigned short new_id)36 static unsigned short __swap_cgroup_id_xchg(struct swap_cgroup *map,
37 pgoff_t offset,
38 unsigned short new_id)
39 {
40 unsigned short old_id;
41 struct swap_cgroup *sc = &map[offset / ID_PER_SC];
42 unsigned int shift = (offset % ID_PER_SC) * ID_SHIFT;
43 unsigned int new_ids, old_ids = atomic_read(&sc->ids);
44
45 do {
46 old_id = (old_ids >> shift) & ID_MASK;
47 new_ids = (old_ids & ~(ID_MASK << shift));
48 new_ids |= ((unsigned int)new_id) << shift;
49 } while (!atomic_try_cmpxchg(&sc->ids, &old_ids, new_ids));
50
51 return old_id;
52 }
53
54 /**
55 * swap_cgroup_record - record mem_cgroup for a set of swap entries.
56 * These entries must belong to one single folio, and that folio
57 * must be being charged for swap space (swap out), and these
58 * entries must not have been charged
59 *
60 * @folio: the folio that the swap entry belongs to
61 * @id: mem_cgroup ID to be recorded
62 * @ent: the first swap entry to be recorded
63 */
swap_cgroup_record(struct folio * folio,unsigned short id,swp_entry_t ent)64 void swap_cgroup_record(struct folio *folio, unsigned short id,
65 swp_entry_t ent)
66 {
67 unsigned int nr_ents = folio_nr_pages(folio);
68 struct swap_cgroup *map;
69 pgoff_t offset, end;
70 unsigned short old;
71
72 offset = swp_offset(ent);
73 end = offset + nr_ents;
74 map = swap_cgroup_ctrl[swp_type(ent)].map;
75
76 do {
77 old = __swap_cgroup_id_xchg(map, offset, id);
78 VM_BUG_ON(old);
79 } while (++offset != end);
80 }
81
82 /**
83 * swap_cgroup_clear - clear mem_cgroup for a set of swap entries.
84 * These entries must be being uncharged from swap. They either
85 * belongs to one single folio in the swap cache (swap in for
86 * cgroup v1), or no longer have any users (slot freeing).
87 *
88 * @ent: the first swap entry to be recorded into
89 * @nr_ents: number of swap entries to be recorded
90 *
91 * Returns the existing old value.
92 */
swap_cgroup_clear(swp_entry_t ent,unsigned int nr_ents)93 unsigned short swap_cgroup_clear(swp_entry_t ent, unsigned int nr_ents)
94 {
95 pgoff_t offset = swp_offset(ent);
96 pgoff_t end = offset + nr_ents;
97 struct swap_cgroup *map;
98 unsigned short old, iter = 0;
99
100 offset = swp_offset(ent);
101 end = offset + nr_ents;
102 map = swap_cgroup_ctrl[swp_type(ent)].map;
103
104 do {
105 old = __swap_cgroup_id_xchg(map, offset, 0);
106 if (!iter)
107 iter = old;
108 VM_BUG_ON(iter != old);
109 } while (++offset != end);
110
111 return old;
112 }
113
114 /**
115 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
116 * @ent: swap entry to be looked up.
117 *
118 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
119 */
lookup_swap_cgroup_id(swp_entry_t ent)120 unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
121 {
122 struct swap_cgroup_ctrl *ctrl;
123
124 if (mem_cgroup_disabled())
125 return 0;
126
127 ctrl = &swap_cgroup_ctrl[swp_type(ent)];
128 return __swap_cgroup_id_lookup(ctrl->map, swp_offset(ent));
129 }
130
swap_cgroup_swapon(int type,unsigned long max_pages)131 int swap_cgroup_swapon(int type, unsigned long max_pages)
132 {
133 struct swap_cgroup *map;
134 struct swap_cgroup_ctrl *ctrl;
135
136 if (mem_cgroup_disabled())
137 return 0;
138
139 BUILD_BUG_ON(sizeof(unsigned short) * ID_PER_SC !=
140 sizeof(struct swap_cgroup));
141 map = vzalloc(DIV_ROUND_UP(max_pages, ID_PER_SC) *
142 sizeof(struct swap_cgroup));
143 if (!map)
144 goto nomem;
145
146 ctrl = &swap_cgroup_ctrl[type];
147 mutex_lock(&swap_cgroup_mutex);
148 ctrl->map = map;
149 mutex_unlock(&swap_cgroup_mutex);
150
151 return 0;
152 nomem:
153 pr_info("couldn't allocate enough memory for swap_cgroup\n");
154 pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
155 return -ENOMEM;
156 }
157
swap_cgroup_swapoff(int type)158 void swap_cgroup_swapoff(int type)
159 {
160 struct swap_cgroup *map;
161 struct swap_cgroup_ctrl *ctrl;
162
163 if (mem_cgroup_disabled())
164 return;
165
166 mutex_lock(&swap_cgroup_mutex);
167 ctrl = &swap_cgroup_ctrl[type];
168 map = ctrl->map;
169 ctrl->map = NULL;
170 mutex_unlock(&swap_cgroup_mutex);
171
172 vfree(map);
173 }
174