1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/swap_cgroup.h> 3 #include <linux/vmalloc.h> 4 #include <linux/mm.h> 5 6 #include <linux/swapops.h> /* depends on mm.h include */ 7 8 static DEFINE_MUTEX(swap_cgroup_mutex); 9 10 /* Pack two cgroup id (short) of two entries in one swap_cgroup (atomic_t) */ 11 #define ID_PER_SC (sizeof(struct swap_cgroup) / sizeof(unsigned short)) 12 #define ID_SHIFT (BITS_PER_TYPE(unsigned short)) 13 #define ID_MASK (BIT(ID_SHIFT) - 1) 14 struct swap_cgroup { 15 atomic_t ids; 16 }; 17 18 struct swap_cgroup_ctrl { 19 struct swap_cgroup *map; 20 }; 21 22 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES]; 23 24 static unsigned short __swap_cgroup_id_lookup(struct swap_cgroup *map, 25 pgoff_t offset) 26 { 27 unsigned int shift = (offset % ID_PER_SC) * ID_SHIFT; 28 unsigned int old_ids = atomic_read(&map[offset / ID_PER_SC].ids); 29 30 BUILD_BUG_ON(!is_power_of_2(ID_PER_SC)); 31 BUILD_BUG_ON(sizeof(struct swap_cgroup) != sizeof(atomic_t)); 32 33 return (old_ids >> shift) & ID_MASK; 34 } 35 36 static unsigned short __swap_cgroup_id_xchg(struct swap_cgroup *map, 37 pgoff_t offset, 38 unsigned short new_id) 39 { 40 unsigned short old_id; 41 struct swap_cgroup *sc = &map[offset / ID_PER_SC]; 42 unsigned int shift = (offset % ID_PER_SC) * ID_SHIFT; 43 unsigned int new_ids, old_ids = atomic_read(&sc->ids); 44 45 do { 46 old_id = (old_ids >> shift) & ID_MASK; 47 new_ids = (old_ids & ~(ID_MASK << shift)); 48 new_ids |= ((unsigned int)new_id) << shift; 49 } while (!atomic_try_cmpxchg(&sc->ids, &old_ids, new_ids)); 50 51 return old_id; 52 } 53 54 /** 55 * swap_cgroup_record - record mem_cgroup for a set of swap entries. 56 * These entries must belong to one single folio, and that folio 57 * must be being charged for swap space (swap out), and these 58 * entries must not have been charged 59 * 60 * @folio: the folio that the swap entry belongs to 61 * @ent: the first swap entry to be recorded 62 */ 63 void swap_cgroup_record(struct folio *folio, swp_entry_t ent) 64 { 65 unsigned int nr_ents = folio_nr_pages(folio); 66 struct swap_cgroup *map; 67 pgoff_t offset, end; 68 unsigned short old; 69 70 offset = swp_offset(ent); 71 end = offset + nr_ents; 72 map = swap_cgroup_ctrl[swp_type(ent)].map; 73 74 do { 75 old = __swap_cgroup_id_xchg(map, offset, 76 mem_cgroup_id(folio_memcg(folio))); 77 VM_BUG_ON(old); 78 } while (++offset != end); 79 } 80 81 /** 82 * swap_cgroup_clear - clear mem_cgroup for a set of swap entries. 83 * These entries must be being uncharged from swap. They either 84 * belongs to one single folio in the swap cache (swap in for 85 * cgroup v1), or no longer have any users (slot freeing). 86 * 87 * @ent: the first swap entry to be recorded into 88 * @nr_ents: number of swap entries to be recorded 89 * 90 * Returns the existing old value. 91 */ 92 unsigned short swap_cgroup_clear(swp_entry_t ent, unsigned int nr_ents) 93 { 94 pgoff_t offset = swp_offset(ent); 95 pgoff_t end = offset + nr_ents; 96 struct swap_cgroup *map; 97 unsigned short old, iter = 0; 98 99 offset = swp_offset(ent); 100 end = offset + nr_ents; 101 map = swap_cgroup_ctrl[swp_type(ent)].map; 102 103 do { 104 old = __swap_cgroup_id_xchg(map, offset, 0); 105 if (!iter) 106 iter = old; 107 VM_BUG_ON(iter != old); 108 } while (++offset != end); 109 110 return old; 111 } 112 113 /** 114 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry 115 * @ent: swap entry to be looked up. 116 * 117 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID) 118 */ 119 unsigned short lookup_swap_cgroup_id(swp_entry_t ent) 120 { 121 struct swap_cgroup_ctrl *ctrl; 122 123 if (mem_cgroup_disabled()) 124 return 0; 125 126 ctrl = &swap_cgroup_ctrl[swp_type(ent)]; 127 return __swap_cgroup_id_lookup(ctrl->map, swp_offset(ent)); 128 } 129 130 int swap_cgroup_swapon(int type, unsigned long max_pages) 131 { 132 struct swap_cgroup *map; 133 struct swap_cgroup_ctrl *ctrl; 134 135 if (mem_cgroup_disabled()) 136 return 0; 137 138 BUILD_BUG_ON(sizeof(unsigned short) * ID_PER_SC != 139 sizeof(struct swap_cgroup)); 140 map = vzalloc(DIV_ROUND_UP(max_pages, ID_PER_SC) * 141 sizeof(struct swap_cgroup)); 142 if (!map) 143 goto nomem; 144 145 ctrl = &swap_cgroup_ctrl[type]; 146 mutex_lock(&swap_cgroup_mutex); 147 ctrl->map = map; 148 mutex_unlock(&swap_cgroup_mutex); 149 150 return 0; 151 nomem: 152 pr_info("couldn't allocate enough memory for swap_cgroup\n"); 153 pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n"); 154 return -ENOMEM; 155 } 156 157 void swap_cgroup_swapoff(int type) 158 { 159 struct swap_cgroup *map; 160 struct swap_cgroup_ctrl *ctrl; 161 162 if (mem_cgroup_disabled()) 163 return; 164 165 mutex_lock(&swap_cgroup_mutex); 166 ctrl = &swap_cgroup_ctrl[type]; 167 map = ctrl->map; 168 ctrl->map = NULL; 169 mutex_unlock(&swap_cgroup_mutex); 170 171 vfree(map); 172 } 173