1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 * Copyright (C) 2022 Alibaba Cloud 6 */ 7 #include "compress.h" 8 #include <linux/psi.h> 9 #include <linux/cpuhotplug.h> 10 #include <trace/events/erofs.h> 11 12 #define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE) 13 #define Z_EROFS_INLINE_BVECS 2 14 15 struct z_erofs_bvec { 16 struct page *page; 17 int offset; 18 unsigned int end; 19 }; 20 21 #define __Z_EROFS_BVSET(name, total) \ 22 struct name { \ 23 /* point to the next page which contains the following bvecs */ \ 24 struct page *nextpage; \ 25 struct z_erofs_bvec bvec[total]; \ 26 } 27 __Z_EROFS_BVSET(z_erofs_bvset,); 28 __Z_EROFS_BVSET(z_erofs_bvset_inline, Z_EROFS_INLINE_BVECS); 29 30 /* 31 * Structure fields follow one of the following exclusion rules. 32 * 33 * I: Modifiable by initialization/destruction paths and read-only 34 * for everyone else; 35 * 36 * L: Field should be protected by the pcluster lock; 37 * 38 * A: Field should be accessed / updated in atomic for parallelized code. 39 */ 40 struct z_erofs_pcluster { 41 struct mutex lock; 42 struct lockref lockref; 43 44 /* A: point to next chained pcluster or TAILs */ 45 struct z_erofs_pcluster *next; 46 47 /* I: start physical position of this pcluster */ 48 erofs_off_t pos; 49 50 /* L: the maximum decompression size of this round */ 51 unsigned int length; 52 53 /* L: total number of bvecs */ 54 unsigned int vcnt; 55 56 /* I: pcluster size (compressed size) in bytes */ 57 unsigned int pclustersize; 58 59 /* I: page offset of start position of decompression */ 60 unsigned short pageofs_out; 61 62 /* I: page offset of inline compressed data */ 63 unsigned short pageofs_in; 64 65 union { 66 /* L: inline a certain number of bvec for bootstrap */ 67 struct z_erofs_bvset_inline bvset; 68 69 /* I: can be used to free the pcluster by RCU. */ 70 struct rcu_head rcu; 71 }; 72 73 /* I: compression algorithm format */ 74 unsigned char algorithmformat; 75 76 /* I: whether compressed data is in-lined or not */ 77 bool from_meta; 78 79 /* L: whether partial decompression or not */ 80 bool partial; 81 82 /* L: indicate several pageofs_outs or not */ 83 bool multibases; 84 85 /* L: whether extra buffer allocations are best-effort */ 86 bool besteffort; 87 88 /* A: compressed bvecs (can be cached or inplaced pages) */ 89 struct z_erofs_bvec compressed_bvecs[]; 90 }; 91 92 /* the end of a chain of pclusters */ 93 #define Z_EROFS_PCLUSTER_TAIL ((void *) 0x700 + POISON_POINTER_DELTA) 94 95 struct z_erofs_decompressqueue { 96 struct super_block *sb; 97 struct z_erofs_pcluster *head; 98 atomic_t pending_bios; 99 100 union { 101 struct completion done; 102 struct work_struct work; 103 struct kthread_work kthread_work; 104 } u; 105 bool eio, sync; 106 }; 107 108 static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl) 109 { 110 return PAGE_ALIGN(pcl->pageofs_in + pcl->pclustersize) >> PAGE_SHIFT; 111 } 112 113 static bool erofs_folio_is_managed(struct erofs_sb_info *sbi, struct folio *fo) 114 { 115 return fo->mapping == MNGD_MAPPING(sbi); 116 } 117 118 #define Z_EROFS_ONSTACK_PAGES 32 119 120 /* 121 * since pclustersize is variable for big pcluster feature, introduce slab 122 * pools implementation for different pcluster sizes. 123 */ 124 struct z_erofs_pcluster_slab { 125 struct kmem_cache *slab; 126 unsigned int maxpages; 127 char name[48]; 128 }; 129 130 #define _PCLP(n) { .maxpages = n } 131 132 static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { 133 _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), 134 _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES + 1) 135 }; 136 137 struct z_erofs_bvec_iter { 138 struct page *bvpage; 139 struct z_erofs_bvset *bvset; 140 unsigned int nr, cur; 141 }; 142 143 static struct page *z_erofs_bvec_iter_end(struct z_erofs_bvec_iter *iter) 144 { 145 if (iter->bvpage) 146 kunmap_local(iter->bvset); 147 return iter->bvpage; 148 } 149 150 static struct page *z_erofs_bvset_flip(struct z_erofs_bvec_iter *iter) 151 { 152 unsigned long base = (unsigned long)((struct z_erofs_bvset *)0)->bvec; 153 /* have to access nextpage in advance, otherwise it will be unmapped */ 154 struct page *nextpage = iter->bvset->nextpage; 155 struct page *oldpage; 156 157 DBG_BUGON(!nextpage); 158 oldpage = z_erofs_bvec_iter_end(iter); 159 iter->bvpage = nextpage; 160 iter->bvset = kmap_local_page(nextpage); 161 iter->nr = (PAGE_SIZE - base) / sizeof(struct z_erofs_bvec); 162 iter->cur = 0; 163 return oldpage; 164 } 165 166 static void z_erofs_bvec_iter_begin(struct z_erofs_bvec_iter *iter, 167 struct z_erofs_bvset_inline *bvset, 168 unsigned int bootstrap_nr, 169 unsigned int cur) 170 { 171 *iter = (struct z_erofs_bvec_iter) { 172 .nr = bootstrap_nr, 173 .bvset = (struct z_erofs_bvset *)bvset, 174 }; 175 176 while (cur > iter->nr) { 177 cur -= iter->nr; 178 z_erofs_bvset_flip(iter); 179 } 180 iter->cur = cur; 181 } 182 183 static int z_erofs_bvec_enqueue(struct z_erofs_bvec_iter *iter, 184 struct z_erofs_bvec *bvec, 185 struct page **candidate_bvpage, 186 struct page **pagepool) 187 { 188 if (iter->cur >= iter->nr) { 189 struct page *nextpage = *candidate_bvpage; 190 191 if (!nextpage) { 192 nextpage = __erofs_allocpage(pagepool, GFP_KERNEL, 193 true); 194 if (!nextpage) 195 return -ENOMEM; 196 set_page_private(nextpage, Z_EROFS_SHORTLIVED_PAGE); 197 } 198 DBG_BUGON(iter->bvset->nextpage); 199 iter->bvset->nextpage = nextpage; 200 z_erofs_bvset_flip(iter); 201 202 iter->bvset->nextpage = NULL; 203 *candidate_bvpage = NULL; 204 } 205 iter->bvset->bvec[iter->cur++] = *bvec; 206 return 0; 207 } 208 209 static void z_erofs_bvec_dequeue(struct z_erofs_bvec_iter *iter, 210 struct z_erofs_bvec *bvec, 211 struct page **old_bvpage) 212 { 213 if (iter->cur == iter->nr) 214 *old_bvpage = z_erofs_bvset_flip(iter); 215 else 216 *old_bvpage = NULL; 217 *bvec = iter->bvset->bvec[iter->cur++]; 218 } 219 220 static void z_erofs_destroy_pcluster_pool(void) 221 { 222 int i; 223 224 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 225 if (!pcluster_pool[i].slab) 226 continue; 227 kmem_cache_destroy(pcluster_pool[i].slab); 228 pcluster_pool[i].slab = NULL; 229 } 230 } 231 232 static int z_erofs_create_pcluster_pool(void) 233 { 234 struct z_erofs_pcluster_slab *pcs; 235 struct z_erofs_pcluster *a; 236 unsigned int size; 237 238 for (pcs = pcluster_pool; 239 pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) { 240 size = struct_size(a, compressed_bvecs, pcs->maxpages); 241 242 sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages); 243 pcs->slab = kmem_cache_create(pcs->name, size, 0, 244 SLAB_RECLAIM_ACCOUNT, NULL); 245 if (pcs->slab) 246 continue; 247 248 z_erofs_destroy_pcluster_pool(); 249 return -ENOMEM; 250 } 251 return 0; 252 } 253 254 static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int size) 255 { 256 unsigned int nrpages = PAGE_ALIGN(size) >> PAGE_SHIFT; 257 struct z_erofs_pcluster_slab *pcs = pcluster_pool; 258 259 for (; pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) { 260 struct z_erofs_pcluster *pcl; 261 262 if (nrpages > pcs->maxpages) 263 continue; 264 265 pcl = kmem_cache_zalloc(pcs->slab, GFP_KERNEL); 266 if (!pcl) 267 return ERR_PTR(-ENOMEM); 268 return pcl; 269 } 270 return ERR_PTR(-EINVAL); 271 } 272 273 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) 274 { 275 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 276 int i; 277 278 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 279 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; 280 281 if (pclusterpages > pcs->maxpages) 282 continue; 283 284 kmem_cache_free(pcs->slab, pcl); 285 return; 286 } 287 DBG_BUGON(1); 288 } 289 290 static struct workqueue_struct *z_erofs_workqueue __read_mostly; 291 292 #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD 293 static struct kthread_worker __rcu **z_erofs_pcpu_workers; 294 295 static void erofs_destroy_percpu_workers(void) 296 { 297 struct kthread_worker *worker; 298 unsigned int cpu; 299 300 for_each_possible_cpu(cpu) { 301 worker = rcu_dereference_protected( 302 z_erofs_pcpu_workers[cpu], 1); 303 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL); 304 if (worker) 305 kthread_destroy_worker(worker); 306 } 307 kfree(z_erofs_pcpu_workers); 308 } 309 310 static struct kthread_worker *erofs_init_percpu_worker(int cpu) 311 { 312 struct kthread_worker *worker = 313 kthread_run_worker_on_cpu(cpu, 0, "erofs_worker/%u"); 314 315 if (IS_ERR(worker)) 316 return worker; 317 if (IS_ENABLED(CONFIG_EROFS_FS_PCPU_KTHREAD_HIPRI)) 318 sched_set_fifo_low(worker->task); 319 return worker; 320 } 321 322 static int erofs_init_percpu_workers(void) 323 { 324 struct kthread_worker *worker; 325 unsigned int cpu; 326 327 z_erofs_pcpu_workers = kcalloc(num_possible_cpus(), 328 sizeof(struct kthread_worker *), GFP_ATOMIC); 329 if (!z_erofs_pcpu_workers) 330 return -ENOMEM; 331 332 for_each_online_cpu(cpu) { /* could miss cpu{off,on}line? */ 333 worker = erofs_init_percpu_worker(cpu); 334 if (!IS_ERR(worker)) 335 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker); 336 } 337 return 0; 338 } 339 #else 340 static inline void erofs_destroy_percpu_workers(void) {} 341 static inline int erofs_init_percpu_workers(void) { return 0; } 342 #endif 343 344 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_EROFS_FS_PCPU_KTHREAD) 345 static DEFINE_SPINLOCK(z_erofs_pcpu_worker_lock); 346 static enum cpuhp_state erofs_cpuhp_state; 347 348 static int erofs_cpu_online(unsigned int cpu) 349 { 350 struct kthread_worker *worker, *old; 351 352 worker = erofs_init_percpu_worker(cpu); 353 if (IS_ERR(worker)) 354 return PTR_ERR(worker); 355 356 spin_lock(&z_erofs_pcpu_worker_lock); 357 old = rcu_dereference_protected(z_erofs_pcpu_workers[cpu], 358 lockdep_is_held(&z_erofs_pcpu_worker_lock)); 359 if (!old) 360 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker); 361 spin_unlock(&z_erofs_pcpu_worker_lock); 362 if (old) 363 kthread_destroy_worker(worker); 364 return 0; 365 } 366 367 static int erofs_cpu_offline(unsigned int cpu) 368 { 369 struct kthread_worker *worker; 370 371 spin_lock(&z_erofs_pcpu_worker_lock); 372 worker = rcu_dereference_protected(z_erofs_pcpu_workers[cpu], 373 lockdep_is_held(&z_erofs_pcpu_worker_lock)); 374 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL); 375 spin_unlock(&z_erofs_pcpu_worker_lock); 376 377 synchronize_rcu(); 378 if (worker) 379 kthread_destroy_worker(worker); 380 return 0; 381 } 382 383 static int erofs_cpu_hotplug_init(void) 384 { 385 int state; 386 387 state = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 388 "fs/erofs:online", erofs_cpu_online, erofs_cpu_offline); 389 if (state < 0) 390 return state; 391 392 erofs_cpuhp_state = state; 393 return 0; 394 } 395 396 static void erofs_cpu_hotplug_destroy(void) 397 { 398 if (erofs_cpuhp_state) 399 cpuhp_remove_state_nocalls(erofs_cpuhp_state); 400 } 401 #else /* !CONFIG_HOTPLUG_CPU || !CONFIG_EROFS_FS_PCPU_KTHREAD */ 402 static inline int erofs_cpu_hotplug_init(void) { return 0; } 403 static inline void erofs_cpu_hotplug_destroy(void) {} 404 #endif 405 406 void z_erofs_exit_subsystem(void) 407 { 408 erofs_cpu_hotplug_destroy(); 409 erofs_destroy_percpu_workers(); 410 destroy_workqueue(z_erofs_workqueue); 411 z_erofs_destroy_pcluster_pool(); 412 z_erofs_exit_decompressor(); 413 } 414 415 int __init z_erofs_init_subsystem(void) 416 { 417 int err = z_erofs_init_decompressor(); 418 419 if (err) 420 goto err_decompressor; 421 422 err = z_erofs_create_pcluster_pool(); 423 if (err) 424 goto err_pcluster_pool; 425 426 z_erofs_workqueue = alloc_workqueue("erofs_worker", 427 WQ_UNBOUND | WQ_HIGHPRI, num_possible_cpus()); 428 if (!z_erofs_workqueue) { 429 err = -ENOMEM; 430 goto err_workqueue_init; 431 } 432 433 err = erofs_init_percpu_workers(); 434 if (err) 435 goto err_pcpu_worker; 436 437 err = erofs_cpu_hotplug_init(); 438 if (err < 0) 439 goto err_cpuhp_init; 440 return err; 441 442 err_cpuhp_init: 443 erofs_destroy_percpu_workers(); 444 err_pcpu_worker: 445 destroy_workqueue(z_erofs_workqueue); 446 err_workqueue_init: 447 z_erofs_destroy_pcluster_pool(); 448 err_pcluster_pool: 449 z_erofs_exit_decompressor(); 450 err_decompressor: 451 return err; 452 } 453 454 enum z_erofs_pclustermode { 455 /* It has previously been linked into another processing chain */ 456 Z_EROFS_PCLUSTER_INFLIGHT, 457 /* 458 * A weaker form of Z_EROFS_PCLUSTER_FOLLOWED; the difference is that it 459 * may be dispatched to the bypass queue later due to uptodated managed 460 * folios. All file-backed folios related to this pcluster cannot be 461 * reused for in-place I/O (or bvpage) since the pcluster may be decoded 462 * in a separate queue (and thus out of order). 463 */ 464 Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE, 465 /* 466 * The pcluster has just been linked to our processing chain. 467 * File-backed folios (except for the head page) related to it can be 468 * used for in-place I/O (or bvpage). 469 */ 470 Z_EROFS_PCLUSTER_FOLLOWED, 471 }; 472 473 struct z_erofs_frontend { 474 struct inode *const inode; 475 struct erofs_map_blocks map; 476 struct z_erofs_bvec_iter biter; 477 478 struct page *pagepool; 479 struct page *candidate_bvpage; 480 struct z_erofs_pcluster *pcl, *head; 481 enum z_erofs_pclustermode mode; 482 483 erofs_off_t headoffset; 484 485 /* a pointer used to pick up inplace I/O pages */ 486 unsigned int icur; 487 }; 488 489 #define Z_EROFS_DEFINE_FRONTEND(fe, i, ho) struct z_erofs_frontend fe = { \ 490 .inode = i, .head = Z_EROFS_PCLUSTER_TAIL, \ 491 .mode = Z_EROFS_PCLUSTER_FOLLOWED, .headoffset = ho } 492 493 static bool z_erofs_should_alloc_cache(struct z_erofs_frontend *fe) 494 { 495 unsigned int cachestrategy = EROFS_I_SB(fe->inode)->opt.cache_strategy; 496 497 if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED) 498 return false; 499 500 if (!(fe->map.m_flags & EROFS_MAP_FULL_MAPPED)) 501 return true; 502 503 if (cachestrategy >= EROFS_ZIP_CACHE_READAROUND && 504 fe->map.m_la < fe->headoffset) 505 return true; 506 507 return false; 508 } 509 510 static void z_erofs_bind_cache(struct z_erofs_frontend *fe) 511 { 512 struct address_space *mc = MNGD_MAPPING(EROFS_I_SB(fe->inode)); 513 struct z_erofs_pcluster *pcl = fe->pcl; 514 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 515 bool shouldalloc = z_erofs_should_alloc_cache(fe); 516 pgoff_t poff = pcl->pos >> PAGE_SHIFT; 517 bool may_bypass = true; 518 /* Optimistic allocation, as in-place I/O can be used as a fallback */ 519 gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) | 520 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 521 struct folio *folio, *newfolio; 522 unsigned int i; 523 524 if (i_blocksize(fe->inode) != PAGE_SIZE || 525 fe->mode < Z_EROFS_PCLUSTER_FOLLOWED) 526 return; 527 528 for (i = 0; i < pclusterpages; ++i) { 529 /* Inaccurate check w/o locking to avoid unneeded lookups */ 530 if (READ_ONCE(pcl->compressed_bvecs[i].page)) 531 continue; 532 533 folio = filemap_get_folio(mc, poff + i); 534 if (IS_ERR(folio)) { 535 may_bypass = false; 536 if (!shouldalloc) 537 continue; 538 539 /* 540 * Allocate a managed folio for cached I/O, or it may be 541 * then filled with a file-backed folio for in-place I/O 542 */ 543 newfolio = filemap_alloc_folio(gfp, 0); 544 if (!newfolio) 545 continue; 546 newfolio->private = Z_EROFS_PREALLOCATED_FOLIO; 547 folio = NULL; 548 } 549 spin_lock(&pcl->lockref.lock); 550 if (!pcl->compressed_bvecs[i].page) { 551 pcl->compressed_bvecs[i].page = 552 folio_page(folio ?: newfolio, 0); 553 spin_unlock(&pcl->lockref.lock); 554 continue; 555 } 556 spin_unlock(&pcl->lockref.lock); 557 folio_put(folio ?: newfolio); 558 } 559 560 /* 561 * Don't perform in-place I/O if all compressed pages are available in 562 * the managed cache, as the pcluster can be moved to the bypass queue. 563 */ 564 if (may_bypass) 565 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; 566 } 567 568 /* (erofs_shrinker) disconnect cached encoded data with pclusters */ 569 static int erofs_try_to_free_all_cached_folios(struct erofs_sb_info *sbi, 570 struct z_erofs_pcluster *pcl) 571 { 572 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 573 struct folio *folio; 574 int i; 575 576 DBG_BUGON(pcl->from_meta); 577 /* Each cached folio contains one page unless bs > ps is supported */ 578 for (i = 0; i < pclusterpages; ++i) { 579 if (pcl->compressed_bvecs[i].page) { 580 folio = page_folio(pcl->compressed_bvecs[i].page); 581 /* Avoid reclaiming or migrating this folio */ 582 if (!folio_trylock(folio)) 583 return -EBUSY; 584 585 if (!erofs_folio_is_managed(sbi, folio)) 586 continue; 587 pcl->compressed_bvecs[i].page = NULL; 588 folio_detach_private(folio); 589 folio_unlock(folio); 590 } 591 } 592 return 0; 593 } 594 595 static bool z_erofs_cache_release_folio(struct folio *folio, gfp_t gfp) 596 { 597 struct z_erofs_pcluster *pcl = folio_get_private(folio); 598 struct z_erofs_bvec *bvec = pcl->compressed_bvecs; 599 struct z_erofs_bvec *end = bvec + z_erofs_pclusterpages(pcl); 600 bool ret; 601 602 if (!folio_test_private(folio)) 603 return true; 604 605 ret = false; 606 spin_lock(&pcl->lockref.lock); 607 if (pcl->lockref.count <= 0) { 608 DBG_BUGON(pcl->from_meta); 609 for (; bvec < end; ++bvec) { 610 if (bvec->page && page_folio(bvec->page) == folio) { 611 bvec->page = NULL; 612 folio_detach_private(folio); 613 ret = true; 614 break; 615 } 616 } 617 } 618 spin_unlock(&pcl->lockref.lock); 619 return ret; 620 } 621 622 /* 623 * It will be called only on inode eviction. In case that there are still some 624 * decompression requests in progress, wait with rescheduling for a bit here. 625 * An extra lock could be introduced instead but it seems unnecessary. 626 */ 627 static void z_erofs_cache_invalidate_folio(struct folio *folio, 628 size_t offset, size_t length) 629 { 630 const size_t stop = length + offset; 631 632 /* Check for potential overflow in debug mode */ 633 DBG_BUGON(stop > folio_size(folio) || stop < length); 634 635 if (offset == 0 && stop == folio_size(folio)) 636 while (!z_erofs_cache_release_folio(folio, 0)) 637 cond_resched(); 638 } 639 640 static const struct address_space_operations z_erofs_cache_aops = { 641 .release_folio = z_erofs_cache_release_folio, 642 .invalidate_folio = z_erofs_cache_invalidate_folio, 643 }; 644 645 int z_erofs_init_super(struct super_block *sb) 646 { 647 struct inode *const inode = new_inode(sb); 648 649 if (!inode) 650 return -ENOMEM; 651 set_nlink(inode, 1); 652 inode->i_size = OFFSET_MAX; 653 inode->i_mapping->a_ops = &z_erofs_cache_aops; 654 mapping_set_gfp_mask(inode->i_mapping, GFP_KERNEL); 655 EROFS_SB(sb)->managed_cache = inode; 656 xa_init(&EROFS_SB(sb)->managed_pslots); 657 return 0; 658 } 659 660 /* callers must be with pcluster lock held */ 661 static int z_erofs_attach_page(struct z_erofs_frontend *fe, 662 struct z_erofs_bvec *bvec, bool exclusive) 663 { 664 struct z_erofs_pcluster *pcl = fe->pcl; 665 int ret; 666 667 if (exclusive) { 668 /* Inplace I/O is limited to one page for uncompressed data */ 669 if (pcl->algorithmformat < Z_EROFS_COMPRESSION_MAX || 670 fe->icur <= 1) { 671 /* Try to prioritize inplace I/O here */ 672 spin_lock(&pcl->lockref.lock); 673 while (fe->icur > 0) { 674 if (pcl->compressed_bvecs[--fe->icur].page) 675 continue; 676 pcl->compressed_bvecs[fe->icur] = *bvec; 677 spin_unlock(&pcl->lockref.lock); 678 return 0; 679 } 680 spin_unlock(&pcl->lockref.lock); 681 } 682 683 /* otherwise, check if it can be used as a bvpage */ 684 if (fe->mode >= Z_EROFS_PCLUSTER_FOLLOWED && 685 !fe->candidate_bvpage) 686 fe->candidate_bvpage = bvec->page; 687 } 688 ret = z_erofs_bvec_enqueue(&fe->biter, bvec, &fe->candidate_bvpage, 689 &fe->pagepool); 690 fe->pcl->vcnt += (ret >= 0); 691 return ret; 692 } 693 694 static bool z_erofs_get_pcluster(struct z_erofs_pcluster *pcl) 695 { 696 if (lockref_get_not_zero(&pcl->lockref)) 697 return true; 698 699 spin_lock(&pcl->lockref.lock); 700 if (__lockref_is_dead(&pcl->lockref)) { 701 spin_unlock(&pcl->lockref.lock); 702 return false; 703 } 704 705 if (!pcl->lockref.count++) 706 atomic_long_dec(&erofs_global_shrink_cnt); 707 spin_unlock(&pcl->lockref.lock); 708 return true; 709 } 710 711 static int z_erofs_register_pcluster(struct z_erofs_frontend *fe) 712 { 713 struct erofs_map_blocks *map = &fe->map; 714 struct super_block *sb = fe->inode->i_sb; 715 struct erofs_sb_info *sbi = EROFS_SB(sb); 716 struct z_erofs_pcluster *pcl, *pre; 717 unsigned int pageofs_in; 718 int err; 719 720 pageofs_in = erofs_blkoff(sb, map->m_pa); 721 pcl = z_erofs_alloc_pcluster(pageofs_in + map->m_plen); 722 if (IS_ERR(pcl)) 723 return PTR_ERR(pcl); 724 725 lockref_init(&pcl->lockref); /* one ref for this request */ 726 pcl->algorithmformat = map->m_algorithmformat; 727 pcl->pclustersize = map->m_plen; 728 pcl->pageofs_in = pageofs_in; 729 pcl->length = 0; 730 pcl->partial = true; 731 pcl->next = fe->head; 732 pcl->pos = map->m_pa; 733 pcl->pageofs_in = pageofs_in; 734 pcl->pageofs_out = map->m_la & ~PAGE_MASK; 735 pcl->from_meta = map->m_flags & EROFS_MAP_META; 736 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED; 737 738 /* 739 * lock all primary followed works before visible to others 740 * and mutex_trylock *never* fails for a new pcluster. 741 */ 742 mutex_init(&pcl->lock); 743 DBG_BUGON(!mutex_trylock(&pcl->lock)); 744 745 if (!pcl->from_meta) { 746 while (1) { 747 xa_lock(&sbi->managed_pslots); 748 pre = __xa_cmpxchg(&sbi->managed_pslots, pcl->pos, 749 NULL, pcl, GFP_KERNEL); 750 if (!pre || xa_is_err(pre) || z_erofs_get_pcluster(pre)) { 751 xa_unlock(&sbi->managed_pslots); 752 break; 753 } 754 /* try to legitimize the current in-tree one */ 755 xa_unlock(&sbi->managed_pslots); 756 cond_resched(); 757 } 758 if (xa_is_err(pre)) { 759 err = xa_err(pre); 760 goto err_out; 761 } else if (pre) { 762 fe->pcl = pre; 763 err = -EEXIST; 764 goto err_out; 765 } 766 } 767 fe->head = fe->pcl = pcl; 768 return 0; 769 770 err_out: 771 mutex_unlock(&pcl->lock); 772 z_erofs_free_pcluster(pcl); 773 return err; 774 } 775 776 static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) 777 { 778 struct erofs_map_blocks *map = &fe->map; 779 struct super_block *sb = fe->inode->i_sb; 780 struct z_erofs_pcluster *pcl = NULL; 781 int ret; 782 783 DBG_BUGON(fe->pcl); 784 /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */ 785 DBG_BUGON(!fe->head); 786 787 if (!(map->m_flags & EROFS_MAP_META)) { 788 while (1) { 789 rcu_read_lock(); 790 pcl = xa_load(&EROFS_SB(sb)->managed_pslots, map->m_pa); 791 if (!pcl || z_erofs_get_pcluster(pcl)) { 792 DBG_BUGON(pcl && map->m_pa != pcl->pos); 793 rcu_read_unlock(); 794 break; 795 } 796 rcu_read_unlock(); 797 } 798 } else if ((map->m_pa & ~PAGE_MASK) + map->m_plen > PAGE_SIZE) { 799 DBG_BUGON(1); 800 return -EFSCORRUPTED; 801 } 802 803 if (pcl) { 804 fe->pcl = pcl; 805 ret = -EEXIST; 806 } else { 807 ret = z_erofs_register_pcluster(fe); 808 } 809 810 if (ret == -EEXIST) { 811 mutex_lock(&fe->pcl->lock); 812 /* check if this pcluster hasn't been linked into any chain. */ 813 if (!cmpxchg(&fe->pcl->next, NULL, fe->head)) { 814 /* .. so it can be attached to our submission chain */ 815 fe->head = fe->pcl; 816 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED; 817 } else { /* otherwise, it belongs to an inflight chain */ 818 fe->mode = Z_EROFS_PCLUSTER_INFLIGHT; 819 } 820 } else if (ret) { 821 return ret; 822 } 823 824 z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset, 825 Z_EROFS_INLINE_BVECS, fe->pcl->vcnt); 826 if (!fe->pcl->from_meta) { 827 /* bind cache first when cached decompression is preferred */ 828 z_erofs_bind_cache(fe); 829 } else { 830 void *mptr; 831 832 mptr = erofs_read_metabuf(&map->buf, sb, map->m_pa, false); 833 if (IS_ERR(mptr)) { 834 ret = PTR_ERR(mptr); 835 erofs_err(sb, "failed to get inline data %d", ret); 836 return ret; 837 } 838 get_page(map->buf.page); 839 WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, map->buf.page); 840 fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK; 841 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; 842 } 843 /* file-backed inplace I/O pages are traversed in reverse order */ 844 fe->icur = z_erofs_pclusterpages(fe->pcl); 845 return 0; 846 } 847 848 static void z_erofs_rcu_callback(struct rcu_head *head) 849 { 850 z_erofs_free_pcluster(container_of(head, struct z_erofs_pcluster, rcu)); 851 } 852 853 static bool __erofs_try_to_release_pcluster(struct erofs_sb_info *sbi, 854 struct z_erofs_pcluster *pcl) 855 { 856 if (pcl->lockref.count) 857 return false; 858 859 /* 860 * Note that all cached folios should be detached before deleted from 861 * the XArray. Otherwise some folios could be still attached to the 862 * orphan old pcluster when the new one is available in the tree. 863 */ 864 if (erofs_try_to_free_all_cached_folios(sbi, pcl)) 865 return false; 866 867 /* 868 * It's impossible to fail after the pcluster is freezed, but in order 869 * to avoid some race conditions, add a DBG_BUGON to observe this. 870 */ 871 DBG_BUGON(__xa_erase(&sbi->managed_pslots, pcl->pos) != pcl); 872 873 lockref_mark_dead(&pcl->lockref); 874 return true; 875 } 876 877 static bool erofs_try_to_release_pcluster(struct erofs_sb_info *sbi, 878 struct z_erofs_pcluster *pcl) 879 { 880 bool free; 881 882 spin_lock(&pcl->lockref.lock); 883 free = __erofs_try_to_release_pcluster(sbi, pcl); 884 spin_unlock(&pcl->lockref.lock); 885 if (free) { 886 atomic_long_dec(&erofs_global_shrink_cnt); 887 call_rcu(&pcl->rcu, z_erofs_rcu_callback); 888 } 889 return free; 890 } 891 892 unsigned long z_erofs_shrink_scan(struct erofs_sb_info *sbi, unsigned long nr) 893 { 894 struct z_erofs_pcluster *pcl; 895 unsigned long index, freed = 0; 896 897 xa_lock(&sbi->managed_pslots); 898 xa_for_each(&sbi->managed_pslots, index, pcl) { 899 /* try to shrink each valid pcluster */ 900 if (!erofs_try_to_release_pcluster(sbi, pcl)) 901 continue; 902 xa_unlock(&sbi->managed_pslots); 903 904 ++freed; 905 if (!--nr) 906 return freed; 907 xa_lock(&sbi->managed_pslots); 908 } 909 xa_unlock(&sbi->managed_pslots); 910 return freed; 911 } 912 913 static void z_erofs_put_pcluster(struct erofs_sb_info *sbi, 914 struct z_erofs_pcluster *pcl, bool try_free) 915 { 916 bool free = false; 917 918 if (lockref_put_or_lock(&pcl->lockref)) 919 return; 920 921 DBG_BUGON(__lockref_is_dead(&pcl->lockref)); 922 if (!--pcl->lockref.count) { 923 if (try_free && xa_trylock(&sbi->managed_pslots)) { 924 free = __erofs_try_to_release_pcluster(sbi, pcl); 925 xa_unlock(&sbi->managed_pslots); 926 } 927 atomic_long_add(!free, &erofs_global_shrink_cnt); 928 } 929 spin_unlock(&pcl->lockref.lock); 930 if (free) 931 call_rcu(&pcl->rcu, z_erofs_rcu_callback); 932 } 933 934 static void z_erofs_pcluster_end(struct z_erofs_frontend *fe) 935 { 936 struct z_erofs_pcluster *pcl = fe->pcl; 937 938 if (!pcl) 939 return; 940 941 z_erofs_bvec_iter_end(&fe->biter); 942 mutex_unlock(&pcl->lock); 943 944 if (fe->candidate_bvpage) 945 fe->candidate_bvpage = NULL; 946 947 /* Drop refcount if it doesn't belong to our processing chain */ 948 if (fe->mode < Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE) 949 z_erofs_put_pcluster(EROFS_I_SB(fe->inode), pcl, false); 950 fe->pcl = NULL; 951 } 952 953 static int z_erofs_read_fragment(struct super_block *sb, struct folio *folio, 954 unsigned int cur, unsigned int end, erofs_off_t pos) 955 { 956 struct inode *packed_inode = EROFS_SB(sb)->packed_inode; 957 struct erofs_buf buf = __EROFS_BUF_INITIALIZER; 958 unsigned int cnt; 959 u8 *src; 960 961 if (!packed_inode) 962 return -EFSCORRUPTED; 963 964 buf.mapping = packed_inode->i_mapping; 965 for (; cur < end; cur += cnt, pos += cnt) { 966 cnt = min(end - cur, sb->s_blocksize - erofs_blkoff(sb, pos)); 967 src = erofs_bread(&buf, pos, true); 968 if (IS_ERR(src)) { 969 erofs_put_metabuf(&buf); 970 return PTR_ERR(src); 971 } 972 memcpy_to_folio(folio, cur, src, cnt); 973 } 974 erofs_put_metabuf(&buf); 975 return 0; 976 } 977 978 static int z_erofs_scan_folio(struct z_erofs_frontend *f, 979 struct folio *folio, bool ra) 980 { 981 struct inode *const inode = f->inode; 982 struct erofs_map_blocks *const map = &f->map; 983 const loff_t offset = folio_pos(folio); 984 const unsigned int bs = i_blocksize(inode); 985 unsigned int end = folio_size(folio), split = 0, cur, pgs; 986 bool tight, excl; 987 int err = 0; 988 989 tight = (bs == PAGE_SIZE); 990 erofs_onlinefolio_init(folio); 991 do { 992 if (offset + end - 1 < map->m_la || 993 offset + end - 1 >= map->m_la + map->m_llen) { 994 z_erofs_pcluster_end(f); 995 map->m_la = offset + end - 1; 996 map->m_llen = 0; 997 err = z_erofs_map_blocks_iter(inode, map, 0); 998 if (err) 999 break; 1000 } 1001 1002 cur = offset > map->m_la ? 0 : map->m_la - offset; 1003 pgs = round_down(cur, PAGE_SIZE); 1004 /* bump split parts first to avoid several separate cases */ 1005 ++split; 1006 1007 if (!(map->m_flags & EROFS_MAP_MAPPED)) { 1008 folio_zero_segment(folio, cur, end); 1009 tight = false; 1010 } else if (map->m_flags & EROFS_MAP_FRAGMENT) { 1011 erofs_off_t fpos = offset + cur - map->m_la; 1012 1013 err = z_erofs_read_fragment(inode->i_sb, folio, cur, 1014 cur + min(map->m_llen - fpos, end - cur), 1015 EROFS_I(inode)->z_fragmentoff + fpos); 1016 if (err) 1017 break; 1018 tight = false; 1019 } else { 1020 if (!f->pcl) { 1021 err = z_erofs_pcluster_begin(f); 1022 if (err) 1023 break; 1024 f->pcl->besteffort |= !ra; 1025 } 1026 1027 pgs = round_down(end - 1, PAGE_SIZE); 1028 /* 1029 * Ensure this partial page belongs to this submit chain 1030 * rather than other concurrent submit chains or 1031 * noio(bypass) chains since those chains are handled 1032 * asynchronously thus it cannot be used for inplace I/O 1033 * or bvpage (should be processed in the strict order.) 1034 */ 1035 tight &= (f->mode >= Z_EROFS_PCLUSTER_FOLLOWED); 1036 excl = false; 1037 if (cur <= pgs) { 1038 excl = (split <= 1) || tight; 1039 cur = pgs; 1040 } 1041 1042 err = z_erofs_attach_page(f, &((struct z_erofs_bvec) { 1043 .page = folio_page(folio, pgs >> PAGE_SHIFT), 1044 .offset = offset + pgs - map->m_la, 1045 .end = end - pgs, }), excl); 1046 if (err) 1047 break; 1048 1049 erofs_onlinefolio_split(folio); 1050 if (f->pcl->pageofs_out != (map->m_la & ~PAGE_MASK)) 1051 f->pcl->multibases = true; 1052 if (f->pcl->length < offset + end - map->m_la) { 1053 f->pcl->length = offset + end - map->m_la; 1054 f->pcl->pageofs_out = map->m_la & ~PAGE_MASK; 1055 } 1056 if ((map->m_flags & EROFS_MAP_FULL_MAPPED) && 1057 !(map->m_flags & EROFS_MAP_PARTIAL_REF) && 1058 f->pcl->length == map->m_llen) 1059 f->pcl->partial = false; 1060 } 1061 /* shorten the remaining extent to update progress */ 1062 map->m_llen = offset + cur - map->m_la; 1063 map->m_flags &= ~EROFS_MAP_FULL_MAPPED; 1064 if (cur <= pgs) { 1065 split = cur < pgs; 1066 tight = (bs == PAGE_SIZE); 1067 } 1068 } while ((end = cur) > 0); 1069 erofs_onlinefolio_end(folio, err); 1070 return err; 1071 } 1072 1073 static bool z_erofs_is_sync_decompress(struct erofs_sb_info *sbi, 1074 unsigned int readahead_pages) 1075 { 1076 /* auto: enable for read_folio, disable for readahead */ 1077 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) && 1078 !readahead_pages) 1079 return true; 1080 1081 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_FORCE_ON) && 1082 (readahead_pages <= sbi->opt.max_sync_decompress_pages)) 1083 return true; 1084 1085 return false; 1086 } 1087 1088 static bool z_erofs_page_is_invalidated(struct page *page) 1089 { 1090 return !page_folio(page)->mapping && !z_erofs_is_shortlived_page(page); 1091 } 1092 1093 struct z_erofs_backend { 1094 struct page *onstack_pages[Z_EROFS_ONSTACK_PAGES]; 1095 struct super_block *sb; 1096 struct z_erofs_pcluster *pcl; 1097 1098 /* pages with the longest decompressed length for deduplication */ 1099 struct page **decompressed_pages; 1100 /* pages to keep the compressed data */ 1101 struct page **compressed_pages; 1102 1103 struct list_head decompressed_secondary_bvecs; 1104 struct page **pagepool; 1105 unsigned int onstack_used, nr_pages; 1106 }; 1107 1108 struct z_erofs_bvec_item { 1109 struct z_erofs_bvec bvec; 1110 struct list_head list; 1111 }; 1112 1113 static void z_erofs_do_decompressed_bvec(struct z_erofs_backend *be, 1114 struct z_erofs_bvec *bvec) 1115 { 1116 struct z_erofs_bvec_item *item; 1117 unsigned int pgnr; 1118 1119 if (!((bvec->offset + be->pcl->pageofs_out) & ~PAGE_MASK) && 1120 (bvec->end == PAGE_SIZE || 1121 bvec->offset + bvec->end == be->pcl->length)) { 1122 pgnr = (bvec->offset + be->pcl->pageofs_out) >> PAGE_SHIFT; 1123 DBG_BUGON(pgnr >= be->nr_pages); 1124 if (!be->decompressed_pages[pgnr]) { 1125 be->decompressed_pages[pgnr] = bvec->page; 1126 return; 1127 } 1128 } 1129 1130 /* (cold path) one pcluster is requested multiple times */ 1131 item = kmalloc(sizeof(*item), GFP_KERNEL | __GFP_NOFAIL); 1132 item->bvec = *bvec; 1133 list_add(&item->list, &be->decompressed_secondary_bvecs); 1134 } 1135 1136 static void z_erofs_fill_other_copies(struct z_erofs_backend *be, int err) 1137 { 1138 unsigned int off0 = be->pcl->pageofs_out; 1139 struct list_head *p, *n; 1140 1141 list_for_each_safe(p, n, &be->decompressed_secondary_bvecs) { 1142 struct z_erofs_bvec_item *bvi; 1143 unsigned int end, cur; 1144 void *dst, *src; 1145 1146 bvi = container_of(p, struct z_erofs_bvec_item, list); 1147 cur = bvi->bvec.offset < 0 ? -bvi->bvec.offset : 0; 1148 end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset, 1149 bvi->bvec.end); 1150 dst = kmap_local_page(bvi->bvec.page); 1151 while (cur < end) { 1152 unsigned int pgnr, scur, len; 1153 1154 pgnr = (bvi->bvec.offset + cur + off0) >> PAGE_SHIFT; 1155 DBG_BUGON(pgnr >= be->nr_pages); 1156 1157 scur = bvi->bvec.offset + cur - 1158 ((pgnr << PAGE_SHIFT) - off0); 1159 len = min_t(unsigned int, end - cur, PAGE_SIZE - scur); 1160 if (!be->decompressed_pages[pgnr]) { 1161 err = -EFSCORRUPTED; 1162 cur += len; 1163 continue; 1164 } 1165 src = kmap_local_page(be->decompressed_pages[pgnr]); 1166 memcpy(dst + cur, src + scur, len); 1167 kunmap_local(src); 1168 cur += len; 1169 } 1170 kunmap_local(dst); 1171 erofs_onlinefolio_end(page_folio(bvi->bvec.page), err); 1172 list_del(p); 1173 kfree(bvi); 1174 } 1175 } 1176 1177 static void z_erofs_parse_out_bvecs(struct z_erofs_backend *be) 1178 { 1179 struct z_erofs_pcluster *pcl = be->pcl; 1180 struct z_erofs_bvec_iter biter; 1181 struct page *old_bvpage; 1182 int i; 1183 1184 z_erofs_bvec_iter_begin(&biter, &pcl->bvset, Z_EROFS_INLINE_BVECS, 0); 1185 for (i = 0; i < pcl->vcnt; ++i) { 1186 struct z_erofs_bvec bvec; 1187 1188 z_erofs_bvec_dequeue(&biter, &bvec, &old_bvpage); 1189 1190 if (old_bvpage) 1191 z_erofs_put_shortlivedpage(be->pagepool, old_bvpage); 1192 1193 DBG_BUGON(z_erofs_page_is_invalidated(bvec.page)); 1194 z_erofs_do_decompressed_bvec(be, &bvec); 1195 } 1196 1197 old_bvpage = z_erofs_bvec_iter_end(&biter); 1198 if (old_bvpage) 1199 z_erofs_put_shortlivedpage(be->pagepool, old_bvpage); 1200 } 1201 1202 static int z_erofs_parse_in_bvecs(struct z_erofs_backend *be, bool *overlapped) 1203 { 1204 struct z_erofs_pcluster *pcl = be->pcl; 1205 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 1206 int i, err = 0; 1207 1208 *overlapped = false; 1209 for (i = 0; i < pclusterpages; ++i) { 1210 struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i]; 1211 struct page *page = bvec->page; 1212 1213 /* compressed data ought to be valid when decompressing */ 1214 if (IS_ERR(page) || !page) { 1215 bvec->page = NULL; /* clear the failure reason */ 1216 err = page ? PTR_ERR(page) : -EIO; 1217 continue; 1218 } 1219 be->compressed_pages[i] = page; 1220 1221 if (pcl->from_meta || 1222 erofs_folio_is_managed(EROFS_SB(be->sb), page_folio(page))) { 1223 if (!PageUptodate(page)) 1224 err = -EIO; 1225 continue; 1226 } 1227 1228 DBG_BUGON(z_erofs_page_is_invalidated(page)); 1229 if (z_erofs_is_shortlived_page(page)) 1230 continue; 1231 z_erofs_do_decompressed_bvec(be, bvec); 1232 *overlapped = true; 1233 } 1234 return err; 1235 } 1236 1237 static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, int err) 1238 { 1239 struct erofs_sb_info *const sbi = EROFS_SB(be->sb); 1240 struct z_erofs_pcluster *pcl = be->pcl; 1241 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 1242 const struct z_erofs_decompressor *decomp = 1243 z_erofs_decomp[pcl->algorithmformat]; 1244 int i, j, jtop, err2; 1245 struct page *page; 1246 bool overlapped; 1247 bool try_free = true; 1248 1249 mutex_lock(&pcl->lock); 1250 be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT; 1251 1252 /* allocate (de)compressed page arrays if cannot be kept on stack */ 1253 be->decompressed_pages = NULL; 1254 be->compressed_pages = NULL; 1255 be->onstack_used = 0; 1256 if (be->nr_pages <= Z_EROFS_ONSTACK_PAGES) { 1257 be->decompressed_pages = be->onstack_pages; 1258 be->onstack_used = be->nr_pages; 1259 memset(be->decompressed_pages, 0, 1260 sizeof(struct page *) * be->nr_pages); 1261 } 1262 1263 if (pclusterpages + be->onstack_used <= Z_EROFS_ONSTACK_PAGES) 1264 be->compressed_pages = be->onstack_pages + be->onstack_used; 1265 1266 if (!be->decompressed_pages) 1267 be->decompressed_pages = 1268 kvcalloc(be->nr_pages, sizeof(struct page *), 1269 GFP_KERNEL | __GFP_NOFAIL); 1270 if (!be->compressed_pages) 1271 be->compressed_pages = 1272 kvcalloc(pclusterpages, sizeof(struct page *), 1273 GFP_KERNEL | __GFP_NOFAIL); 1274 1275 z_erofs_parse_out_bvecs(be); 1276 err2 = z_erofs_parse_in_bvecs(be, &overlapped); 1277 if (err2) 1278 err = err2; 1279 if (!err) 1280 err = decomp->decompress(&(struct z_erofs_decompress_req) { 1281 .sb = be->sb, 1282 .in = be->compressed_pages, 1283 .out = be->decompressed_pages, 1284 .inpages = pclusterpages, 1285 .outpages = be->nr_pages, 1286 .pageofs_in = pcl->pageofs_in, 1287 .pageofs_out = pcl->pageofs_out, 1288 .inputsize = pcl->pclustersize, 1289 .outputsize = pcl->length, 1290 .alg = pcl->algorithmformat, 1291 .inplace_io = overlapped, 1292 .partial_decoding = pcl->partial, 1293 .fillgaps = pcl->multibases, 1294 .gfp = pcl->besteffort ? GFP_KERNEL : 1295 GFP_NOWAIT | __GFP_NORETRY 1296 }, be->pagepool); 1297 1298 /* must handle all compressed pages before actual file pages */ 1299 if (pcl->from_meta) { 1300 page = pcl->compressed_bvecs[0].page; 1301 WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL); 1302 put_page(page); 1303 } else { 1304 /* managed folios are still left in compressed_bvecs[] */ 1305 for (i = 0; i < pclusterpages; ++i) { 1306 page = be->compressed_pages[i]; 1307 if (!page) 1308 continue; 1309 if (erofs_folio_is_managed(sbi, page_folio(page))) { 1310 try_free = false; 1311 continue; 1312 } 1313 (void)z_erofs_put_shortlivedpage(be->pagepool, page); 1314 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); 1315 } 1316 } 1317 if (be->compressed_pages < be->onstack_pages || 1318 be->compressed_pages >= be->onstack_pages + Z_EROFS_ONSTACK_PAGES) 1319 kvfree(be->compressed_pages); 1320 1321 jtop = 0; 1322 z_erofs_fill_other_copies(be, err); 1323 for (i = 0; i < be->nr_pages; ++i) { 1324 page = be->decompressed_pages[i]; 1325 if (!page) 1326 continue; 1327 1328 DBG_BUGON(z_erofs_page_is_invalidated(page)); 1329 if (!z_erofs_is_shortlived_page(page)) { 1330 erofs_onlinefolio_end(page_folio(page), err); 1331 continue; 1332 } 1333 if (pcl->algorithmformat != Z_EROFS_COMPRESSION_LZ4) { 1334 erofs_pagepool_add(be->pagepool, page); 1335 continue; 1336 } 1337 for (j = 0; j < jtop && be->decompressed_pages[j] != page; ++j) 1338 ; 1339 if (j >= jtop) /* this bounce page is newly detected */ 1340 be->decompressed_pages[jtop++] = page; 1341 } 1342 while (jtop) 1343 erofs_pagepool_add(be->pagepool, 1344 be->decompressed_pages[--jtop]); 1345 if (be->decompressed_pages != be->onstack_pages) 1346 kvfree(be->decompressed_pages); 1347 1348 pcl->length = 0; 1349 pcl->partial = true; 1350 pcl->multibases = false; 1351 pcl->besteffort = false; 1352 pcl->bvset.nextpage = NULL; 1353 pcl->vcnt = 0; 1354 1355 /* pcluster lock MUST be taken before the following line */ 1356 WRITE_ONCE(pcl->next, NULL); 1357 mutex_unlock(&pcl->lock); 1358 1359 if (pcl->from_meta) 1360 z_erofs_free_pcluster(pcl); 1361 else 1362 z_erofs_put_pcluster(sbi, pcl, try_free); 1363 return err; 1364 } 1365 1366 static int z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io, 1367 struct page **pagepool) 1368 { 1369 struct z_erofs_backend be = { 1370 .sb = io->sb, 1371 .pagepool = pagepool, 1372 .decompressed_secondary_bvecs = 1373 LIST_HEAD_INIT(be.decompressed_secondary_bvecs), 1374 .pcl = io->head, 1375 }; 1376 struct z_erofs_pcluster *next; 1377 int err = io->eio ? -EIO : 0; 1378 1379 for (; be.pcl != Z_EROFS_PCLUSTER_TAIL; be.pcl = next) { 1380 DBG_BUGON(!be.pcl); 1381 next = READ_ONCE(be.pcl->next); 1382 err = z_erofs_decompress_pcluster(&be, err) ?: err; 1383 } 1384 return err; 1385 } 1386 1387 static void z_erofs_decompressqueue_work(struct work_struct *work) 1388 { 1389 struct z_erofs_decompressqueue *bgq = 1390 container_of(work, struct z_erofs_decompressqueue, u.work); 1391 struct page *pagepool = NULL; 1392 1393 DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL); 1394 z_erofs_decompress_queue(bgq, &pagepool); 1395 erofs_release_pages(&pagepool); 1396 kvfree(bgq); 1397 } 1398 1399 #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD 1400 static void z_erofs_decompressqueue_kthread_work(struct kthread_work *work) 1401 { 1402 z_erofs_decompressqueue_work((struct work_struct *)work); 1403 } 1404 #endif 1405 1406 static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, 1407 int bios) 1408 { 1409 struct erofs_sb_info *const sbi = EROFS_SB(io->sb); 1410 1411 /* wake up the caller thread for sync decompression */ 1412 if (io->sync) { 1413 if (!atomic_add_return(bios, &io->pending_bios)) 1414 complete(&io->u.done); 1415 return; 1416 } 1417 1418 if (atomic_add_return(bios, &io->pending_bios)) 1419 return; 1420 /* Use (kthread_)work and sync decompression for atomic contexts only */ 1421 if (!in_task() || irqs_disabled() || rcu_read_lock_any_held()) { 1422 #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD 1423 struct kthread_worker *worker; 1424 1425 rcu_read_lock(); 1426 worker = rcu_dereference( 1427 z_erofs_pcpu_workers[raw_smp_processor_id()]); 1428 if (!worker) { 1429 INIT_WORK(&io->u.work, z_erofs_decompressqueue_work); 1430 queue_work(z_erofs_workqueue, &io->u.work); 1431 } else { 1432 kthread_queue_work(worker, &io->u.kthread_work); 1433 } 1434 rcu_read_unlock(); 1435 #else 1436 queue_work(z_erofs_workqueue, &io->u.work); 1437 #endif 1438 /* enable sync decompression for readahead */ 1439 if (sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) 1440 sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; 1441 return; 1442 } 1443 z_erofs_decompressqueue_work(&io->u.work); 1444 } 1445 1446 static void z_erofs_fill_bio_vec(struct bio_vec *bvec, 1447 struct z_erofs_frontend *f, 1448 struct z_erofs_pcluster *pcl, 1449 unsigned int nr, 1450 struct address_space *mc) 1451 { 1452 gfp_t gfp = mapping_gfp_mask(mc); 1453 bool tocache = false; 1454 struct z_erofs_bvec zbv; 1455 struct address_space *mapping; 1456 struct folio *folio; 1457 struct page *page; 1458 int bs = i_blocksize(f->inode); 1459 1460 /* Except for inplace folios, the entire folio can be used for I/Os */ 1461 bvec->bv_offset = 0; 1462 bvec->bv_len = PAGE_SIZE; 1463 repeat: 1464 spin_lock(&pcl->lockref.lock); 1465 zbv = pcl->compressed_bvecs[nr]; 1466 spin_unlock(&pcl->lockref.lock); 1467 if (!zbv.page) 1468 goto out_allocfolio; 1469 1470 bvec->bv_page = zbv.page; 1471 DBG_BUGON(z_erofs_is_shortlived_page(bvec->bv_page)); 1472 1473 folio = page_folio(zbv.page); 1474 /* For preallocated managed folios, add them to page cache here */ 1475 if (folio->private == Z_EROFS_PREALLOCATED_FOLIO) { 1476 tocache = true; 1477 goto out_tocache; 1478 } 1479 1480 mapping = READ_ONCE(folio->mapping); 1481 /* 1482 * File-backed folios for inplace I/Os are all locked steady, 1483 * therefore it is impossible for `mapping` to be NULL. 1484 */ 1485 if (mapping && mapping != mc) { 1486 if (zbv.offset < 0) 1487 bvec->bv_offset = round_up(-zbv.offset, bs); 1488 bvec->bv_len = round_up(zbv.end, bs) - bvec->bv_offset; 1489 return; 1490 } 1491 1492 folio_lock(folio); 1493 if (likely(folio->mapping == mc)) { 1494 /* 1495 * The cached folio is still in managed cache but without 1496 * a valid `->private` pcluster hint. Let's reconnect them. 1497 */ 1498 if (!folio_test_private(folio)) { 1499 folio_attach_private(folio, pcl); 1500 /* compressed_bvecs[] already takes a ref before */ 1501 folio_put(folio); 1502 } 1503 if (likely(folio->private == pcl)) { 1504 /* don't submit cache I/Os again if already uptodate */ 1505 if (folio_test_uptodate(folio)) { 1506 folio_unlock(folio); 1507 bvec->bv_page = NULL; 1508 } 1509 return; 1510 } 1511 /* 1512 * Already linked with another pcluster, which only appears in 1513 * crafted images by fuzzers for now. But handle this anyway. 1514 */ 1515 tocache = false; /* use temporary short-lived pages */ 1516 } else { 1517 DBG_BUGON(1); /* referenced managed folios can't be truncated */ 1518 tocache = true; 1519 } 1520 folio_unlock(folio); 1521 folio_put(folio); 1522 out_allocfolio: 1523 page = __erofs_allocpage(&f->pagepool, gfp, true); 1524 spin_lock(&pcl->lockref.lock); 1525 if (unlikely(pcl->compressed_bvecs[nr].page != zbv.page)) { 1526 if (page) 1527 erofs_pagepool_add(&f->pagepool, page); 1528 spin_unlock(&pcl->lockref.lock); 1529 cond_resched(); 1530 goto repeat; 1531 } 1532 pcl->compressed_bvecs[nr].page = page ? page : ERR_PTR(-ENOMEM); 1533 spin_unlock(&pcl->lockref.lock); 1534 bvec->bv_page = page; 1535 if (!page) 1536 return; 1537 folio = page_folio(page); 1538 out_tocache: 1539 if (!tocache || bs != PAGE_SIZE || 1540 filemap_add_folio(mc, folio, (pcl->pos >> PAGE_SHIFT) + nr, gfp)) { 1541 /* turn into a temporary shortlived folio (1 ref) */ 1542 folio->private = (void *)Z_EROFS_SHORTLIVED_PAGE; 1543 return; 1544 } 1545 folio_attach_private(folio, pcl); 1546 /* drop a refcount added by allocpage (then 2 refs in total here) */ 1547 folio_put(folio); 1548 } 1549 1550 static struct z_erofs_decompressqueue *jobqueue_init(struct super_block *sb, 1551 struct z_erofs_decompressqueue *fgq, bool *fg) 1552 { 1553 struct z_erofs_decompressqueue *q; 1554 1555 if (fg && !*fg) { 1556 q = kvzalloc(sizeof(*q), GFP_KERNEL | __GFP_NOWARN); 1557 if (!q) { 1558 *fg = true; 1559 goto fg_out; 1560 } 1561 #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD 1562 kthread_init_work(&q->u.kthread_work, 1563 z_erofs_decompressqueue_kthread_work); 1564 #else 1565 INIT_WORK(&q->u.work, z_erofs_decompressqueue_work); 1566 #endif 1567 } else { 1568 fg_out: 1569 q = fgq; 1570 init_completion(&fgq->u.done); 1571 atomic_set(&fgq->pending_bios, 0); 1572 q->eio = false; 1573 q->sync = true; 1574 } 1575 q->sb = sb; 1576 q->head = Z_EROFS_PCLUSTER_TAIL; 1577 return q; 1578 } 1579 1580 /* define decompression jobqueue types */ 1581 enum { 1582 JQ_BYPASS, 1583 JQ_SUBMIT, 1584 NR_JOBQUEUES, 1585 }; 1586 1587 static void z_erofs_move_to_bypass_queue(struct z_erofs_pcluster *pcl, 1588 struct z_erofs_pcluster *next, 1589 struct z_erofs_pcluster **qtail[]) 1590 { 1591 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL); 1592 WRITE_ONCE(*qtail[JQ_SUBMIT], next); 1593 WRITE_ONCE(*qtail[JQ_BYPASS], pcl); 1594 qtail[JQ_BYPASS] = &pcl->next; 1595 } 1596 1597 static void z_erofs_endio(struct bio *bio) 1598 { 1599 struct z_erofs_decompressqueue *q = bio->bi_private; 1600 blk_status_t err = bio->bi_status; 1601 struct folio_iter fi; 1602 1603 bio_for_each_folio_all(fi, bio) { 1604 struct folio *folio = fi.folio; 1605 1606 DBG_BUGON(folio_test_uptodate(folio)); 1607 DBG_BUGON(z_erofs_page_is_invalidated(&folio->page)); 1608 if (!erofs_folio_is_managed(EROFS_SB(q->sb), folio)) 1609 continue; 1610 1611 if (!err) 1612 folio_mark_uptodate(folio); 1613 folio_unlock(folio); 1614 } 1615 if (err) 1616 q->eio = true; 1617 z_erofs_decompress_kickoff(q, -1); 1618 if (bio->bi_bdev) 1619 bio_put(bio); 1620 } 1621 1622 static void z_erofs_submit_queue(struct z_erofs_frontend *f, 1623 struct z_erofs_decompressqueue *fgq, 1624 bool *force_fg, bool readahead) 1625 { 1626 struct super_block *sb = f->inode->i_sb; 1627 struct address_space *mc = MNGD_MAPPING(EROFS_SB(sb)); 1628 struct z_erofs_pcluster **qtail[NR_JOBQUEUES]; 1629 struct z_erofs_decompressqueue *q[NR_JOBQUEUES]; 1630 struct z_erofs_pcluster *pcl, *next; 1631 /* bio is NULL initially, so no need to initialize last_{index,bdev} */ 1632 erofs_off_t last_pa; 1633 unsigned int nr_bios = 0; 1634 struct bio *bio = NULL; 1635 unsigned long pflags; 1636 int memstall = 0; 1637 1638 /* No need to read from device for pclusters in the bypass queue. */ 1639 q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL); 1640 q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, force_fg); 1641 1642 qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; 1643 qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head; 1644 1645 /* by default, all need io submission */ 1646 q[JQ_SUBMIT]->head = next = f->head; 1647 1648 do { 1649 struct erofs_map_dev mdev; 1650 erofs_off_t cur, end; 1651 struct bio_vec bvec; 1652 unsigned int i = 0; 1653 bool bypass = true; 1654 1655 pcl = next; 1656 next = READ_ONCE(pcl->next); 1657 if (pcl->from_meta) { 1658 z_erofs_move_to_bypass_queue(pcl, next, qtail); 1659 continue; 1660 } 1661 1662 /* no device id here, thus it will always succeed */ 1663 mdev = (struct erofs_map_dev) { 1664 .m_pa = round_down(pcl->pos, sb->s_blocksize), 1665 }; 1666 (void)erofs_map_dev(sb, &mdev); 1667 1668 cur = mdev.m_pa; 1669 end = round_up(cur + pcl->pageofs_in + pcl->pclustersize, 1670 sb->s_blocksize); 1671 do { 1672 bvec.bv_page = NULL; 1673 if (bio && (cur != last_pa || 1674 bio->bi_bdev != mdev.m_bdev)) { 1675 drain_io: 1676 if (erofs_is_fileio_mode(EROFS_SB(sb))) 1677 erofs_fileio_submit_bio(bio); 1678 else if (erofs_is_fscache_mode(sb)) 1679 erofs_fscache_submit_bio(bio); 1680 else 1681 submit_bio(bio); 1682 1683 if (memstall) { 1684 psi_memstall_leave(&pflags); 1685 memstall = 0; 1686 } 1687 bio = NULL; 1688 } 1689 1690 if (!bvec.bv_page) { 1691 z_erofs_fill_bio_vec(&bvec, f, pcl, i++, mc); 1692 if (!bvec.bv_page) 1693 continue; 1694 if (cur + bvec.bv_len > end) 1695 bvec.bv_len = end - cur; 1696 DBG_BUGON(bvec.bv_len < sb->s_blocksize); 1697 } 1698 1699 if (unlikely(PageWorkingset(bvec.bv_page)) && 1700 !memstall) { 1701 psi_memstall_enter(&pflags); 1702 memstall = 1; 1703 } 1704 1705 if (!bio) { 1706 if (erofs_is_fileio_mode(EROFS_SB(sb))) 1707 bio = erofs_fileio_bio_alloc(&mdev); 1708 else if (erofs_is_fscache_mode(sb)) 1709 bio = erofs_fscache_bio_alloc(&mdev); 1710 else 1711 bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS, 1712 REQ_OP_READ, GFP_NOIO); 1713 bio->bi_end_io = z_erofs_endio; 1714 bio->bi_iter.bi_sector = cur >> 9; 1715 bio->bi_private = q[JQ_SUBMIT]; 1716 if (readahead) 1717 bio->bi_opf |= REQ_RAHEAD; 1718 ++nr_bios; 1719 } 1720 1721 if (!bio_add_page(bio, bvec.bv_page, bvec.bv_len, 1722 bvec.bv_offset)) 1723 goto drain_io; 1724 last_pa = cur + bvec.bv_len; 1725 bypass = false; 1726 } while ((cur += bvec.bv_len) < end); 1727 1728 if (!bypass) 1729 qtail[JQ_SUBMIT] = &pcl->next; 1730 else 1731 z_erofs_move_to_bypass_queue(pcl, next, qtail); 1732 } while (next != Z_EROFS_PCLUSTER_TAIL); 1733 1734 if (bio) { 1735 if (erofs_is_fileio_mode(EROFS_SB(sb))) 1736 erofs_fileio_submit_bio(bio); 1737 else if (erofs_is_fscache_mode(sb)) 1738 erofs_fscache_submit_bio(bio); 1739 else 1740 submit_bio(bio); 1741 } 1742 if (memstall) 1743 psi_memstall_leave(&pflags); 1744 1745 /* 1746 * although background is preferred, no one is pending for submission. 1747 * don't issue decompression but drop it directly instead. 1748 */ 1749 if (!*force_fg && !nr_bios) { 1750 kvfree(q[JQ_SUBMIT]); 1751 return; 1752 } 1753 z_erofs_decompress_kickoff(q[JQ_SUBMIT], nr_bios); 1754 } 1755 1756 static int z_erofs_runqueue(struct z_erofs_frontend *f, unsigned int rapages) 1757 { 1758 struct z_erofs_decompressqueue io[NR_JOBQUEUES]; 1759 struct erofs_sb_info *sbi = EROFS_I_SB(f->inode); 1760 bool force_fg = z_erofs_is_sync_decompress(sbi, rapages); 1761 int err; 1762 1763 if (f->head == Z_EROFS_PCLUSTER_TAIL) 1764 return 0; 1765 z_erofs_submit_queue(f, io, &force_fg, !!rapages); 1766 1767 /* handle bypass queue (no i/o pclusters) immediately */ 1768 err = z_erofs_decompress_queue(&io[JQ_BYPASS], &f->pagepool); 1769 if (!force_fg) 1770 return err; 1771 1772 /* wait until all bios are completed */ 1773 wait_for_completion_io(&io[JQ_SUBMIT].u.done); 1774 1775 /* handle synchronous decompress queue in the caller context */ 1776 return z_erofs_decompress_queue(&io[JQ_SUBMIT], &f->pagepool) ?: err; 1777 } 1778 1779 /* 1780 * Since partial uptodate is still unimplemented for now, we have to use 1781 * approximate readmore strategies as a start. 1782 */ 1783 static void z_erofs_pcluster_readmore(struct z_erofs_frontend *f, 1784 struct readahead_control *rac, bool backmost) 1785 { 1786 struct inode *inode = f->inode; 1787 struct erofs_map_blocks *map = &f->map; 1788 erofs_off_t cur, end, headoffset = f->headoffset; 1789 int err; 1790 1791 if (backmost) { 1792 if (rac) 1793 end = headoffset + readahead_length(rac) - 1; 1794 else 1795 end = headoffset + PAGE_SIZE - 1; 1796 map->m_la = end; 1797 err = z_erofs_map_blocks_iter(inode, map, 1798 EROFS_GET_BLOCKS_READMORE); 1799 if (err) 1800 return; 1801 1802 /* expand ra for the trailing edge if readahead */ 1803 if (rac) { 1804 cur = round_up(map->m_la + map->m_llen, PAGE_SIZE); 1805 readahead_expand(rac, headoffset, cur - headoffset); 1806 return; 1807 } 1808 end = round_up(end, PAGE_SIZE); 1809 } else { 1810 end = round_up(map->m_la, PAGE_SIZE); 1811 if (!map->m_llen) 1812 return; 1813 } 1814 1815 cur = map->m_la + map->m_llen - 1; 1816 while ((cur >= end) && (cur < i_size_read(inode))) { 1817 pgoff_t index = cur >> PAGE_SHIFT; 1818 struct folio *folio; 1819 1820 folio = erofs_grab_folio_nowait(inode->i_mapping, index); 1821 if (!IS_ERR_OR_NULL(folio)) { 1822 if (folio_test_uptodate(folio)) 1823 folio_unlock(folio); 1824 else 1825 z_erofs_scan_folio(f, folio, !!rac); 1826 folio_put(folio); 1827 } 1828 1829 if (cur < PAGE_SIZE) 1830 break; 1831 cur = (index << PAGE_SHIFT) - 1; 1832 } 1833 } 1834 1835 static int z_erofs_read_folio(struct file *file, struct folio *folio) 1836 { 1837 struct inode *const inode = folio->mapping->host; 1838 Z_EROFS_DEFINE_FRONTEND(f, inode, folio_pos(folio)); 1839 int err; 1840 1841 trace_erofs_read_folio(folio, false); 1842 z_erofs_pcluster_readmore(&f, NULL, true); 1843 err = z_erofs_scan_folio(&f, folio, false); 1844 z_erofs_pcluster_readmore(&f, NULL, false); 1845 z_erofs_pcluster_end(&f); 1846 1847 /* if some pclusters are ready, need submit them anyway */ 1848 err = z_erofs_runqueue(&f, 0) ?: err; 1849 if (err && err != -EINTR) 1850 erofs_err(inode->i_sb, "read error %d @ %lu of nid %llu", 1851 err, folio->index, EROFS_I(inode)->nid); 1852 1853 erofs_put_metabuf(&f.map.buf); 1854 erofs_release_pages(&f.pagepool); 1855 return err; 1856 } 1857 1858 static void z_erofs_readahead(struct readahead_control *rac) 1859 { 1860 struct inode *const inode = rac->mapping->host; 1861 Z_EROFS_DEFINE_FRONTEND(f, inode, readahead_pos(rac)); 1862 struct folio *head = NULL, *folio; 1863 unsigned int nrpages = readahead_count(rac); 1864 int err; 1865 1866 z_erofs_pcluster_readmore(&f, rac, true); 1867 nrpages = readahead_count(rac); 1868 trace_erofs_readpages(inode, readahead_index(rac), nrpages, false); 1869 while ((folio = readahead_folio(rac))) { 1870 folio->private = head; 1871 head = folio; 1872 } 1873 1874 /* traverse in reverse order for best metadata I/O performance */ 1875 while (head) { 1876 folio = head; 1877 head = folio_get_private(folio); 1878 1879 err = z_erofs_scan_folio(&f, folio, true); 1880 if (err && err != -EINTR) 1881 erofs_err(inode->i_sb, "readahead error at folio %lu @ nid %llu", 1882 folio->index, EROFS_I(inode)->nid); 1883 } 1884 z_erofs_pcluster_readmore(&f, rac, false); 1885 z_erofs_pcluster_end(&f); 1886 1887 (void)z_erofs_runqueue(&f, nrpages); 1888 erofs_put_metabuf(&f.map.buf); 1889 erofs_release_pages(&f.pagepool); 1890 } 1891 1892 const struct address_space_operations z_erofs_aops = { 1893 .read_folio = z_erofs_read_folio, 1894 .readahead = z_erofs_readahead, 1895 }; 1896