1128 static void z_erofs_do_decompressed_bvec(struct z_erofs_backend *be,  in z_erofs_do_decompressed_bvec()  argument
1131 	int poff = bvec->offset + be->pcl->pageofs_out;  in z_erofs_do_decompressed_bvec()
1136 			bvec->offset + bvec->end == be->pcl->length)) {  in z_erofs_do_decompressed_bvec()
1137 		DBG_BUGON((poff >> PAGE_SHIFT) >= be->nr_pages);  in z_erofs_do_decompressed_bvec()
1138 		page = be->decompressed_pages + (poff >> PAGE_SHIFT);  in z_erofs_do_decompressed_bvec()
1144 		be->keepxcpy = true;  in z_erofs_do_decompressed_bvec()
1150 	list_add(&item->list, &be->decompressed_secondary_bvecs);  in z_erofs_do_decompressed_bvec()
1153 static void z_erofs_fill_other_copies(struct z_erofs_backend *be, int err)  in z_erofs_fill_other_copies()  argument
1155 	unsigned int off0 = be->pcl->pageofs_out;  in z_erofs_fill_other_copies()
1158 	list_for_each_safe(p, n, &be->decompressed_secondary_bvecs) {  in z_erofs_fill_other_copies()
1165 		end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset,  in z_erofs_fill_other_copies()
1172 			DBG_BUGON(pgnr >= be->nr_pages);  in z_erofs_fill_other_copies()
1177 			if (!be->decompressed_pages[pgnr]) {  in z_erofs_fill_other_copies()
1182 			src = kmap_local_page(be->decompressed_pages[pgnr]);  in z_erofs_fill_other_copies()
1194 static void z_erofs_parse_out_bvecs(struct z_erofs_backend *be)  in z_erofs_parse_out_bvecs()  argument
1196 	struct z_erofs_pcluster *pcl = be->pcl;  in z_erofs_parse_out_bvecs()
1208 			z_erofs_put_shortlivedpage(be->pagepool, old_bvpage);  in z_erofs_parse_out_bvecs()
1211 		z_erofs_do_decompressed_bvec(be, &bvec);  in z_erofs_parse_out_bvecs()
1216 		z_erofs_put_shortlivedpage(be->pagepool, old_bvpage);  in z_erofs_parse_out_bvecs()
1219 static int z_erofs_parse_in_bvecs(struct z_erofs_backend *be, bool *overlapped)  in z_erofs_parse_in_bvecs()  argument
1221 	struct z_erofs_pcluster *pcl = be->pcl;  in z_erofs_parse_in_bvecs()
1236 		be->compressed_pages[i] = page;  in z_erofs_parse_in_bvecs()
1239 		    erofs_folio_is_managed(EROFS_SB(be->sb), page_folio(page))) {  in z_erofs_parse_in_bvecs()
1248 		z_erofs_do_decompressed_bvec(be, bvec);  in z_erofs_parse_in_bvecs()
1254 static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio)  in z_erofs_decompress_pcluster()  argument
1256 	struct erofs_sb_info *const sbi = EROFS_SB(be->sb);  in z_erofs_decompress_pcluster()
1257 	struct z_erofs_pcluster *pcl = be->pcl;  in z_erofs_decompress_pcluster()
1268 	be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT;  in z_erofs_decompress_pcluster()
1271 	be->decompressed_pages = NULL;  in z_erofs_decompress_pcluster()
1272 	be->compressed_pages = NULL;  in z_erofs_decompress_pcluster()
1273 	be->onstack_used = 0;  in z_erofs_decompress_pcluster()
1274 	if (be->nr_pages <= Z_EROFS_ONSTACK_PAGES) {  in z_erofs_decompress_pcluster()
1275 		be->decompressed_pages = be->onstack_pages;  in z_erofs_decompress_pcluster()
1276 		be->onstack_used = be->nr_pages;  in z_erofs_decompress_pcluster()
1277 		memset(be->decompressed_pages, 0,  in z_erofs_decompress_pcluster()
1278 		       sizeof(struct page *) * be->nr_pages);  in z_erofs_decompress_pcluster()
1281 	if (pclusterpages + be->onstack_used <= Z_EROFS_ONSTACK_PAGES)  in z_erofs_decompress_pcluster()
1282 		be->compressed_pages = be->onstack_pages + be->onstack_used;  in z_erofs_decompress_pcluster()
1284 	if (!be->decompressed_pages)  in z_erofs_decompress_pcluster()
1285 		be->decompressed_pages =  in z_erofs_decompress_pcluster()
1286 			kvzalloc_objs(struct page *, be->nr_pages,  in z_erofs_decompress_pcluster()
1288 	if (!be->compressed_pages)  in z_erofs_decompress_pcluster()
1289 		be->compressed_pages =  in z_erofs_decompress_pcluster()
1293 	z_erofs_parse_out_bvecs(be);  in z_erofs_decompress_pcluster()
1294 	err2 = z_erofs_parse_in_bvecs(be, &overlapped);  in z_erofs_decompress_pcluster()
1299 					.sb = be->sb,  in z_erofs_decompress_pcluster()
1300 					.in = be->compressed_pages,  in z_erofs_decompress_pcluster()
1301 					.out = be->decompressed_pages,  in z_erofs_decompress_pcluster()
1303 					.outpages = be->nr_pages,  in z_erofs_decompress_pcluster()
1311 					.fillgaps = be->keepxcpy,  in z_erofs_decompress_pcluster()
1314 				 }, be->pagepool);  in z_erofs_decompress_pcluster()
1317 				erofs_err(be->sb, "failed to decompress (%s) %pe @ pa %llu size %u => %u",  in z_erofs_decompress_pcluster()
1322 			erofs_err(be->sb, "failed to decompress (%s) %s @ pa %llu size %u => %u",  in z_erofs_decompress_pcluster()
1336 			page = be->compressed_pages[i];  in z_erofs_decompress_pcluster()
1343 			(void)z_erofs_put_shortlivedpage(be->pagepool, page);  in z_erofs_decompress_pcluster()
1347 	if (be->compressed_pages < be->onstack_pages ||  in z_erofs_decompress_pcluster()
1348 	    be->compressed_pages >= be->onstack_pages + Z_EROFS_ONSTACK_PAGES)  in z_erofs_decompress_pcluster()
1349 		kvfree(be->compressed_pages);  in z_erofs_decompress_pcluster()
1351 	z_erofs_fill_other_copies(be, err);  in z_erofs_decompress_pcluster()
1352 	for (i = 0; i < be->nr_pages; ++i) {  in z_erofs_decompress_pcluster()
1353 		page = be->decompressed_pages[i];  in z_erofs_decompress_pcluster()
1361 			erofs_pagepool_add(be->pagepool, page);  in z_erofs_decompress_pcluster()
1363 	if (be->decompressed_pages != be->onstack_pages)  in z_erofs_decompress_pcluster()
1364 		kvfree(be->decompressed_pages);  in z_erofs_decompress_pcluster()
1386 	struct z_erofs_backend be = {  in z_erofs_decompress_queue()  local
1390 			LIST_HEAD_INIT(be.decompressed_secondary_bvecs),  in z_erofs_decompress_queue()
1396 	for (; be.pcl != Z_EROFS_PCLUSTER_TAIL; be.pcl = next) {  in z_erofs_decompress_queue()
1397 		DBG_BUGON(!be.pcl);  in z_erofs_decompress_queue()
1398 		next = READ_ONCE(be.pcl->next);  in z_erofs_decompress_queue()
1399 		err = z_erofs_decompress_pcluster(&be, io->eio) ?: err;  in z_erofs_decompress_queue()