1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 * Copyright (C) 2024 Alibaba Cloud 6 */ 7 #include "internal.h" 8 9 struct z_erofs_gbuf { 10 spinlock_t lock; 11 void *ptr; 12 struct page **pages; 13 unsigned int nrpages; 14 }; 15 16 static struct z_erofs_gbuf *z_erofs_gbufpool, *z_erofs_rsvbuf; 17 static unsigned int z_erofs_gbuf_count, z_erofs_gbuf_nrpages, 18 z_erofs_rsv_nrpages; 19 20 module_param_named(global_buffers, z_erofs_gbuf_count, uint, 0444); 21 module_param_named(reserved_pages, z_erofs_rsv_nrpages, uint, 0444); 22 23 atomic_long_t erofs_global_shrink_cnt; /* for all mounted instances */ 24 25 /* protects `erofs_sb_list_lock` and the mounted `erofs_sb_list` */ 26 static DEFINE_SPINLOCK(erofs_sb_list_lock); 27 static LIST_HEAD(erofs_sb_list); 28 static unsigned int shrinker_run_no; 29 static struct shrinker *erofs_shrinker_info; 30 31 static unsigned int z_erofs_gbuf_id(void) 32 { 33 return raw_smp_processor_id() % z_erofs_gbuf_count; 34 } 35 36 void *z_erofs_get_gbuf(unsigned int requiredpages) 37 __acquires(gbuf->lock) 38 { 39 struct z_erofs_gbuf *gbuf; 40 41 migrate_disable(); 42 gbuf = &z_erofs_gbufpool[z_erofs_gbuf_id()]; 43 spin_lock(&gbuf->lock); 44 /* check if the buffer is too small */ 45 if (requiredpages > gbuf->nrpages) { 46 spin_unlock(&gbuf->lock); 47 migrate_enable(); 48 /* (for sparse checker) pretend gbuf->lock is still taken */ 49 __acquire(gbuf->lock); 50 return NULL; 51 } 52 return gbuf->ptr; 53 } 54 55 void z_erofs_put_gbuf(void *ptr) __releases(gbuf->lock) 56 { 57 struct z_erofs_gbuf *gbuf; 58 59 gbuf = &z_erofs_gbufpool[z_erofs_gbuf_id()]; 60 DBG_BUGON(gbuf->ptr != ptr); 61 spin_unlock(&gbuf->lock); 62 migrate_enable(); 63 } 64 65 int z_erofs_gbuf_growsize(unsigned int nrpages) 66 { 67 static DEFINE_MUTEX(gbuf_resize_mutex); 68 struct page **tmp_pages = NULL; 69 struct z_erofs_gbuf *gbuf; 70 void *ptr, *old_ptr; 71 int last, i, j; 72 73 mutex_lock(&gbuf_resize_mutex); 74 /* avoid shrinking gbufs, since no idea how many fses rely on */ 75 if (nrpages <= z_erofs_gbuf_nrpages) { 76 mutex_unlock(&gbuf_resize_mutex); 77 return 0; 78 } 79 80 for (i = 0; i < z_erofs_gbuf_count; ++i) { 81 gbuf = &z_erofs_gbufpool[i]; 82 tmp_pages = kcalloc(nrpages, sizeof(*tmp_pages), GFP_KERNEL); 83 if (!tmp_pages) 84 goto out; 85 86 for (j = 0; j < gbuf->nrpages; ++j) 87 tmp_pages[j] = gbuf->pages[j]; 88 do { 89 last = j; 90 j = alloc_pages_bulk_array(GFP_KERNEL, nrpages, 91 tmp_pages); 92 if (last == j) 93 goto out; 94 } while (j != nrpages); 95 96 ptr = vmap(tmp_pages, nrpages, VM_MAP, PAGE_KERNEL); 97 if (!ptr) 98 goto out; 99 100 spin_lock(&gbuf->lock); 101 kfree(gbuf->pages); 102 gbuf->pages = tmp_pages; 103 old_ptr = gbuf->ptr; 104 gbuf->ptr = ptr; 105 gbuf->nrpages = nrpages; 106 spin_unlock(&gbuf->lock); 107 if (old_ptr) 108 vunmap(old_ptr); 109 } 110 z_erofs_gbuf_nrpages = nrpages; 111 out: 112 if (i < z_erofs_gbuf_count && tmp_pages) { 113 for (j = 0; j < nrpages; ++j) 114 if (tmp_pages[j] && (j >= gbuf->nrpages || 115 tmp_pages[j] != gbuf->pages[j])) 116 __free_page(tmp_pages[j]); 117 kfree(tmp_pages); 118 } 119 mutex_unlock(&gbuf_resize_mutex); 120 return i < z_erofs_gbuf_count ? -ENOMEM : 0; 121 } 122 123 int __init z_erofs_gbuf_init(void) 124 { 125 unsigned int i, total = num_possible_cpus(); 126 127 if (z_erofs_gbuf_count) 128 total = min(z_erofs_gbuf_count, total); 129 z_erofs_gbuf_count = total; 130 131 /* The last (special) global buffer is the reserved buffer */ 132 total += !!z_erofs_rsv_nrpages; 133 134 z_erofs_gbufpool = kcalloc(total, sizeof(*z_erofs_gbufpool), 135 GFP_KERNEL); 136 if (!z_erofs_gbufpool) 137 return -ENOMEM; 138 139 if (z_erofs_rsv_nrpages) { 140 z_erofs_rsvbuf = &z_erofs_gbufpool[total - 1]; 141 z_erofs_rsvbuf->pages = kcalloc(z_erofs_rsv_nrpages, 142 sizeof(*z_erofs_rsvbuf->pages), GFP_KERNEL); 143 if (!z_erofs_rsvbuf->pages) { 144 z_erofs_rsvbuf = NULL; 145 z_erofs_rsv_nrpages = 0; 146 } 147 } 148 for (i = 0; i < total; ++i) 149 spin_lock_init(&z_erofs_gbufpool[i].lock); 150 return 0; 151 } 152 153 void z_erofs_gbuf_exit(void) 154 { 155 int i, j; 156 157 for (i = 0; i < z_erofs_gbuf_count + (!!z_erofs_rsvbuf); ++i) { 158 struct z_erofs_gbuf *gbuf = &z_erofs_gbufpool[i]; 159 160 if (gbuf->ptr) { 161 vunmap(gbuf->ptr); 162 gbuf->ptr = NULL; 163 } 164 165 if (!gbuf->pages) 166 continue; 167 168 for (j = 0; j < gbuf->nrpages; ++j) 169 if (gbuf->pages[j]) 170 put_page(gbuf->pages[j]); 171 kfree(gbuf->pages); 172 gbuf->pages = NULL; 173 } 174 kfree(z_erofs_gbufpool); 175 } 176 177 struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp, bool tryrsv) 178 { 179 struct page *page = *pagepool; 180 181 if (page) { 182 *pagepool = (struct page *)page_private(page); 183 } else if (tryrsv && z_erofs_rsvbuf && z_erofs_rsvbuf->nrpages) { 184 spin_lock(&z_erofs_rsvbuf->lock); 185 if (z_erofs_rsvbuf->nrpages) 186 page = z_erofs_rsvbuf->pages[--z_erofs_rsvbuf->nrpages]; 187 spin_unlock(&z_erofs_rsvbuf->lock); 188 } 189 if (!page) 190 page = alloc_page(gfp); 191 DBG_BUGON(page && page_ref_count(page) != 1); 192 return page; 193 } 194 195 void erofs_release_pages(struct page **pagepool) 196 { 197 while (*pagepool) { 198 struct page *page = *pagepool; 199 200 *pagepool = (struct page *)page_private(page); 201 /* try to fill reserved global pool first */ 202 if (z_erofs_rsvbuf && z_erofs_rsvbuf->nrpages < 203 z_erofs_rsv_nrpages) { 204 spin_lock(&z_erofs_rsvbuf->lock); 205 if (z_erofs_rsvbuf->nrpages < z_erofs_rsv_nrpages) { 206 z_erofs_rsvbuf->pages[z_erofs_rsvbuf->nrpages++] 207 = page; 208 spin_unlock(&z_erofs_rsvbuf->lock); 209 continue; 210 } 211 spin_unlock(&z_erofs_rsvbuf->lock); 212 } 213 put_page(page); 214 } 215 } 216 217 void erofs_shrinker_register(struct super_block *sb) 218 { 219 struct erofs_sb_info *sbi = EROFS_SB(sb); 220 221 mutex_init(&sbi->umount_mutex); 222 223 spin_lock(&erofs_sb_list_lock); 224 list_add(&sbi->list, &erofs_sb_list); 225 spin_unlock(&erofs_sb_list_lock); 226 } 227 228 void erofs_shrinker_unregister(struct super_block *sb) 229 { 230 struct erofs_sb_info *const sbi = EROFS_SB(sb); 231 232 mutex_lock(&sbi->umount_mutex); 233 /* clean up all remaining pclusters in memory */ 234 z_erofs_shrink_scan(sbi, ~0UL); 235 236 spin_lock(&erofs_sb_list_lock); 237 list_del(&sbi->list); 238 spin_unlock(&erofs_sb_list_lock); 239 mutex_unlock(&sbi->umount_mutex); 240 } 241 242 static unsigned long erofs_shrink_count(struct shrinker *shrink, 243 struct shrink_control *sc) 244 { 245 return atomic_long_read(&erofs_global_shrink_cnt); 246 } 247 248 static unsigned long erofs_shrink_scan(struct shrinker *shrink, 249 struct shrink_control *sc) 250 { 251 struct erofs_sb_info *sbi; 252 struct list_head *p; 253 254 unsigned long nr = sc->nr_to_scan; 255 unsigned int run_no; 256 unsigned long freed = 0; 257 258 spin_lock(&erofs_sb_list_lock); 259 do { 260 run_no = ++shrinker_run_no; 261 } while (run_no == 0); 262 263 /* Iterate over all mounted superblocks and try to shrink them */ 264 p = erofs_sb_list.next; 265 while (p != &erofs_sb_list) { 266 sbi = list_entry(p, struct erofs_sb_info, list); 267 268 /* 269 * We move the ones we do to the end of the list, so we stop 270 * when we see one we have already done. 271 */ 272 if (sbi->shrinker_run_no == run_no) 273 break; 274 275 if (!mutex_trylock(&sbi->umount_mutex)) { 276 p = p->next; 277 continue; 278 } 279 280 spin_unlock(&erofs_sb_list_lock); 281 sbi->shrinker_run_no = run_no; 282 freed += z_erofs_shrink_scan(sbi, nr - freed); 283 spin_lock(&erofs_sb_list_lock); 284 /* Get the next list element before we move this one */ 285 p = p->next; 286 287 /* 288 * Move this one to the end of the list to provide some 289 * fairness. 290 */ 291 list_move_tail(&sbi->list, &erofs_sb_list); 292 mutex_unlock(&sbi->umount_mutex); 293 294 if (freed >= nr) 295 break; 296 } 297 spin_unlock(&erofs_sb_list_lock); 298 return freed; 299 } 300 301 int __init erofs_init_shrinker(void) 302 { 303 erofs_shrinker_info = shrinker_alloc(0, "erofs-shrinker"); 304 if (!erofs_shrinker_info) 305 return -ENOMEM; 306 307 erofs_shrinker_info->count_objects = erofs_shrink_count; 308 erofs_shrinker_info->scan_objects = erofs_shrink_scan; 309 shrinker_register(erofs_shrinker_info); 310 return 0; 311 } 312 313 void erofs_exit_shrinker(void) 314 { 315 shrinker_free(erofs_shrinker_info); 316 } 317