1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2019 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 * Copyright (C) 2024 Alibaba Cloud 6 */ 7 #include "compress.h" 8 #include <linux/lz4.h> 9 10 static int z_erofs_load_lz4_config(struct super_block *sb, 11 struct erofs_super_block *dsb, void *data, int size) 12 { 13 struct erofs_sb_info *sbi = EROFS_SB(sb); 14 struct z_erofs_lz4_cfgs *lz4 = data; 15 u16 distance; 16 17 if (lz4) { 18 if (size < sizeof(struct z_erofs_lz4_cfgs)) { 19 erofs_err(sb, "invalid lz4 cfgs, size=%u", size); 20 return -EINVAL; 21 } 22 sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks); 23 if (!sbi->lz4.max_pclusterblks) { 24 sbi->lz4.max_pclusterblks = 1; /* reserved case */ 25 } else if (sbi->lz4.max_pclusterblks > 26 erofs_blknr(sb, Z_EROFS_PCLUSTER_MAX_SIZE)) { 27 erofs_err(sb, "too large lz4 pclusterblks %u", 28 sbi->lz4.max_pclusterblks); 29 return -EINVAL; 30 } 31 } else { 32 distance = le16_to_cpu(dsb->u1.lz4_max_distance); 33 if (!distance && !erofs_sb_has_lz4_0padding(sbi)) 34 return 0; 35 sbi->lz4.max_pclusterblks = 1; 36 sbi->available_compr_algs = 1 << Z_EROFS_COMPRESSION_LZ4; 37 } 38 return z_erofs_gbuf_growsize(sbi->lz4.max_pclusterblks); 39 } 40 41 /* 42 * Fill all gaps with bounce pages if it's a sparse page list (for example some 43 * folios are already uptodate and thus can be mapped into userspace). Also 44 * check if pages are physically consecutive, which can be seen for moderate CR. 45 */ 46 static int z_erofs_oneshot_prepare_dstpages(struct z_erofs_decompress_req *rq, 47 struct page **pagepool) 48 { 49 void *kaddr = NULL; 50 unsigned int i; 51 52 for (i = 0; i < rq->outpages; ++i) { 53 struct page *page, *victim; 54 55 page = rq->out[i]; 56 if (page) { 57 if (!PageHighMem(page)) { 58 if (!i) { 59 kaddr = page_address(page); 60 continue; 61 } 62 if (kaddr && 63 kaddr + PAGE_SIZE == page_address(page)) { 64 kaddr += PAGE_SIZE; 65 continue; 66 } 67 } 68 } else { 69 victim = __erofs_allocpage(pagepool, rq->gfp, true); 70 if (!victim) 71 return -ENOMEM; 72 set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); 73 rq->out[i] = victim; 74 } 75 kaddr = NULL; 76 } 77 return kaddr ? 1 : 0; 78 } 79 80 static void *z_erofs_lz4_handle_overlap(const struct z_erofs_decompress_req *rq, 81 void *inpage, void *out, unsigned int *inputmargin, 82 int *maptype, bool may_inplace) 83 { 84 unsigned int oend, omargin, cnt, i; 85 struct page **in; 86 void *src; 87 88 /* 89 * If in-place I/O isn't used, for example, the bounce compressed cache 90 * can hold data for incomplete read requests. Just map the compressed 91 * buffer as well and decompress directly. 92 */ 93 if (!rq->inplace_io) { 94 if (rq->inpages <= 1) { 95 *maptype = 0; 96 return inpage; 97 } 98 kunmap_local(inpage); 99 src = erofs_vm_map_ram(rq->in, rq->inpages); 100 if (!src) 101 return ERR_PTR(-ENOMEM); 102 *maptype = 1; 103 return src; 104 } 105 /* 106 * Then, deal with in-place I/Os. The reasons why in-place I/O is useful 107 * are: (1) It minimizes memory footprint during the I/O submission, 108 * which is useful for slow storage (including network devices and 109 * low-end HDDs/eMMCs) but with a lot inflight I/Os; (2) If in-place 110 * decompression can also be applied, it will reuse the unique buffer so 111 * that no extra CPU D-cache is polluted with temporary compressed data 112 * for extreme performance. 113 */ 114 oend = rq->pageofs_out + rq->outputsize; 115 omargin = PAGE_ALIGN(oend) - oend; 116 if (!rq->partial_decoding && may_inplace && 117 omargin >= LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize)) { 118 for (i = 0; i < rq->inpages; ++i) 119 if (rq->out[rq->outpages - rq->inpages + i] != 120 rq->in[i]) 121 break; 122 if (i >= rq->inpages) { 123 kunmap_local(inpage); 124 *maptype = 3; 125 return out + ((rq->outpages - rq->inpages) << PAGE_SHIFT); 126 } 127 } 128 /* 129 * If in-place decompression can't be applied, copy compressed data that 130 * may potentially overlap during decompression to a per-CPU buffer. 131 */ 132 src = z_erofs_get_gbuf(rq->inpages); 133 if (!src) { 134 DBG_BUGON(1); 135 kunmap_local(inpage); 136 return ERR_PTR(-EFAULT); 137 } 138 139 for (i = 0, in = rq->in; i < rq->inputsize; i += cnt, ++in) { 140 cnt = min_t(u32, rq->inputsize - i, PAGE_SIZE - *inputmargin); 141 if (!inpage) 142 inpage = kmap_local_page(*in); 143 memcpy(src + i, inpage + *inputmargin, cnt); 144 kunmap_local(inpage); 145 inpage = NULL; 146 *inputmargin = 0; 147 } 148 *maptype = 2; 149 return src; 150 } 151 152 /* 153 * Get the exact on-disk size of the compressed data: 154 * - For LZ4, it should apply if the zero_padding feature is on (5.3+); 155 * - For others, zero_padding is enabled all the time. 156 */ 157 const char *z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, 158 const char *padbuf, unsigned int padbufsize) 159 { 160 const char *padend; 161 162 padend = memchr_inv(padbuf, 0, padbufsize); 163 if (!padend) 164 return "compressed data start not found"; 165 rq->inputsize -= padend - padbuf; 166 rq->pageofs_in += padend - padbuf; 167 return NULL; 168 } 169 170 static const char *__z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, 171 u8 *dst) 172 { 173 bool may_inplace = false; 174 unsigned int inputmargin; 175 u8 *out, *headpage, *src; 176 const char *reason; 177 int ret, maptype; 178 179 headpage = kmap_local_page(*rq->in); 180 reason = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in, 181 min_t(unsigned int, rq->inputsize, 182 rq->sb->s_blocksize - rq->pageofs_in)); 183 if (reason) { 184 kunmap_local(headpage); 185 return reason; 186 } 187 may_inplace = !((rq->pageofs_in + rq->inputsize) & 188 (rq->sb->s_blocksize - 1)); 189 190 inputmargin = rq->pageofs_in; 191 src = z_erofs_lz4_handle_overlap(rq, headpage, dst, &inputmargin, 192 &maptype, may_inplace); 193 if (IS_ERR(src)) 194 return ERR_CAST(src); 195 196 out = dst + rq->pageofs_out; 197 if (rq->partial_decoding) 198 ret = LZ4_decompress_safe_partial(src + inputmargin, out, 199 rq->inputsize, rq->outputsize, rq->outputsize); 200 else 201 ret = LZ4_decompress_safe(src + inputmargin, out, 202 rq->inputsize, rq->outputsize); 203 if (ret == rq->outputsize) 204 reason = NULL; 205 else if (ret < 0) 206 reason = "corrupted compressed data"; 207 else 208 reason = "unexpected end of stream"; 209 210 if (!maptype) { 211 kunmap_local(headpage); 212 } else if (maptype == 1) { 213 vm_unmap_ram(src, rq->inpages); 214 } else if (maptype == 2) { 215 z_erofs_put_gbuf(src); 216 } else if (maptype != 3) { 217 DBG_BUGON(1); 218 return ERR_PTR(-EFAULT); 219 } 220 return reason; 221 } 222 223 static const char *z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, 224 struct page **pagepool) 225 { 226 unsigned int dst_maptype; 227 const char *reason; 228 void *dst; 229 int ret; 230 231 /* one optimized fast path only for non bigpcluster cases yet */ 232 if (rq->inpages == 1 && rq->outpages == 1 && !rq->inplace_io) { 233 DBG_BUGON(!*rq->out); 234 dst = kmap_local_page(*rq->out); 235 dst_maptype = 0; 236 } else { 237 /* general decoding path which can be used for all cases */ 238 ret = z_erofs_oneshot_prepare_dstpages(rq, pagepool); 239 if (ret < 0) 240 return ERR_PTR(ret); 241 if (ret > 0) { 242 dst = page_address(*rq->out); 243 dst_maptype = 1; 244 } else { 245 dst = erofs_vm_map_ram(rq->out, rq->outpages); 246 if (!dst) 247 return ERR_PTR(-ENOMEM); 248 dst_maptype = 2; 249 } 250 } 251 reason = __z_erofs_lz4_decompress(rq, dst); 252 if (!dst_maptype) 253 kunmap_local(dst); 254 else if (dst_maptype == 2) 255 vm_unmap_ram(dst, rq->outpages); 256 return reason; 257 } 258 259 static const char *z_erofs_transform_plain(struct z_erofs_decompress_req *rq, 260 struct page **pagepool) 261 { 262 const unsigned int nrpages_in = rq->inpages, nrpages_out = rq->outpages; 263 const unsigned int bs = rq->sb->s_blocksize; 264 unsigned int cur = 0, ni = 0, no, pi, po, insz, cnt; 265 u8 *kin; 266 267 if (rq->outputsize > rq->inputsize) 268 return ERR_PTR(-EOPNOTSUPP); 269 if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) { 270 cur = bs - (rq->pageofs_out & (bs - 1)); 271 DBG_BUGON(rq->pageofs_in & (bs - 1)); 272 pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK; 273 cur = min(cur, rq->outputsize); 274 if (cur && rq->out[0]) { 275 kin = kmap_local_page(rq->in[nrpages_in - 1]); 276 if (rq->out[0] == rq->in[nrpages_in - 1]) 277 memmove(kin + rq->pageofs_out, kin + pi, cur); 278 else 279 memcpy_to_page(rq->out[0], rq->pageofs_out, 280 kin + pi, cur); 281 kunmap_local(kin); 282 } 283 rq->outputsize -= cur; 284 } 285 286 for (; rq->outputsize; rq->pageofs_in = 0, cur += insz, ni++) { 287 insz = min(PAGE_SIZE - rq->pageofs_in, rq->outputsize); 288 rq->outputsize -= insz; 289 if (!rq->in[ni]) 290 continue; 291 kin = kmap_local_page(rq->in[ni]); 292 pi = 0; 293 do { 294 no = (rq->pageofs_out + cur + pi) >> PAGE_SHIFT; 295 po = (rq->pageofs_out + cur + pi) & ~PAGE_MASK; 296 DBG_BUGON(no >= nrpages_out); 297 cnt = min(insz - pi, PAGE_SIZE - po); 298 if (rq->out[no] == rq->in[ni]) 299 memmove(kin + po, 300 kin + rq->pageofs_in + pi, cnt); 301 else if (rq->out[no]) 302 memcpy_to_page(rq->out[no], po, 303 kin + rq->pageofs_in + pi, cnt); 304 pi += cnt; 305 } while (pi < insz); 306 kunmap_local(kin); 307 } 308 DBG_BUGON(ni > nrpages_in); 309 return NULL; 310 } 311 312 const char *z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, 313 void **dst, void **src, struct page **pgpl) 314 { 315 struct z_erofs_decompress_req *rq = dctx->rq; 316 struct page **pgo, *tmppage; 317 unsigned int j; 318 319 if (!dctx->avail_out) { 320 if (++dctx->no >= rq->outpages || !rq->outputsize) 321 return "insufficient space for decompressed data"; 322 323 if (dctx->kout) 324 kunmap_local(dctx->kout); 325 dctx->avail_out = min(rq->outputsize, PAGE_SIZE - rq->pageofs_out); 326 rq->outputsize -= dctx->avail_out; 327 pgo = &rq->out[dctx->no]; 328 if (!*pgo && rq->fillgaps) { /* deduped */ 329 *pgo = erofs_allocpage(pgpl, rq->gfp); 330 if (!*pgo) { 331 dctx->kout = NULL; 332 return ERR_PTR(-ENOMEM); 333 } 334 set_page_private(*pgo, Z_EROFS_SHORTLIVED_PAGE); 335 } 336 if (*pgo) { 337 dctx->kout = kmap_local_page(*pgo); 338 *dst = dctx->kout + rq->pageofs_out; 339 } else { 340 *dst = dctx->kout = NULL; 341 } 342 rq->pageofs_out = 0; 343 } 344 345 if (dctx->inbuf_pos == dctx->inbuf_sz && rq->inputsize) { 346 if (++dctx->ni >= rq->inpages) 347 return "invalid compressed data"; 348 if (dctx->kout) /* unlike kmap(), take care of the orders */ 349 kunmap_local(dctx->kout); 350 kunmap_local(dctx->kin); 351 352 dctx->inbuf_sz = min_t(u32, rq->inputsize, PAGE_SIZE); 353 rq->inputsize -= dctx->inbuf_sz; 354 dctx->kin = kmap_local_page(rq->in[dctx->ni]); 355 *src = dctx->kin; 356 dctx->bounced = false; 357 if (dctx->kout) { 358 j = (u8 *)*dst - dctx->kout; 359 dctx->kout = kmap_local_page(rq->out[dctx->no]); 360 *dst = dctx->kout + j; 361 } 362 dctx->inbuf_pos = 0; 363 } 364 365 /* 366 * Handle overlapping: Use the given bounce buffer if the input data is 367 * under processing; Or utilize short-lived pages from the on-stack page 368 * pool, where pages are shared among the same request. Note that only 369 * a few inplace I/O pages need to be doubled. 370 */ 371 if (!dctx->bounced && rq->out[dctx->no] == rq->in[dctx->ni]) { 372 memcpy(dctx->bounce, *src, dctx->inbuf_sz); 373 *src = dctx->bounce; 374 dctx->bounced = true; 375 } 376 377 for (j = dctx->ni + 1; j < rq->inpages; ++j) { 378 if (rq->out[dctx->no] != rq->in[j]) 379 continue; 380 tmppage = erofs_allocpage(pgpl, rq->gfp); 381 if (!tmppage) 382 return ERR_PTR(-ENOMEM); 383 set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); 384 copy_highpage(tmppage, rq->in[j]); 385 rq->in[j] = tmppage; 386 } 387 return NULL; 388 } 389 390 const struct z_erofs_decompressor *z_erofs_decomp[] = { 391 [Z_EROFS_COMPRESSION_SHIFTED] = &(const struct z_erofs_decompressor) { 392 .decompress = z_erofs_transform_plain, 393 .name = "shifted" 394 }, 395 [Z_EROFS_COMPRESSION_INTERLACED] = &(const struct z_erofs_decompressor) { 396 .decompress = z_erofs_transform_plain, 397 .name = "interlaced" 398 }, 399 [Z_EROFS_COMPRESSION_LZ4] = &(const struct z_erofs_decompressor) { 400 .config = z_erofs_load_lz4_config, 401 .decompress = z_erofs_lz4_decompress, 402 .init = z_erofs_gbuf_init, 403 .exit = z_erofs_gbuf_exit, 404 .name = "lz4" 405 }, 406 #ifdef CONFIG_EROFS_FS_ZIP_LZMA 407 [Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp, 408 #endif 409 #ifdef CONFIG_EROFS_FS_ZIP_DEFLATE 410 [Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp, 411 #endif 412 #ifdef CONFIG_EROFS_FS_ZIP_ZSTD 413 [Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp, 414 #endif 415 }; 416 417 int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb) 418 { 419 struct erofs_sb_info *sbi = EROFS_SB(sb); 420 struct erofs_buf buf = __EROFS_BUF_INITIALIZER; 421 unsigned long algs, alg; 422 erofs_off_t offset; 423 int size, ret = 0; 424 425 if (!erofs_sb_has_compr_cfgs(sbi)) 426 return z_erofs_load_lz4_config(sb, dsb, NULL, 0); 427 428 algs = le16_to_cpu(dsb->u1.available_compr_algs); 429 sbi->available_compr_algs = algs; 430 if (algs & ~Z_EROFS_ALL_COMPR_ALGS) { 431 erofs_err(sb, "unidentified algorithms %lx, please upgrade kernel", 432 algs & ~Z_EROFS_ALL_COMPR_ALGS); 433 return -EOPNOTSUPP; 434 } 435 436 (void)erofs_init_metabuf(&buf, sb, false); 437 offset = EROFS_SUPER_OFFSET + sbi->sb_size; 438 for_each_set_bit(alg, &algs, Z_EROFS_COMPRESSION_MAX) { 439 const struct z_erofs_decompressor *dec = z_erofs_decomp[alg]; 440 void *data; 441 442 data = erofs_read_metadata(sb, &buf, &offset, &size); 443 if (IS_ERR(data)) { 444 ret = PTR_ERR(data); 445 break; 446 } 447 448 if (dec && dec->config) { 449 ret = dec->config(sb, dsb, data, size); 450 } else { 451 erofs_err(sb, "algorithm %ld isn't enabled on this kernel", 452 alg); 453 ret = -EOPNOTSUPP; 454 } 455 kfree(data); 456 if (ret) 457 break; 458 } 459 erofs_put_metabuf(&buf); 460 return ret; 461 } 462 463 int __init z_erofs_init_decompressor(void) 464 { 465 int i, err; 466 467 for (i = 0; i < Z_EROFS_COMPRESSION_MAX; ++i) { 468 err = z_erofs_decomp[i] ? z_erofs_decomp[i]->init() : 0; 469 if (err) { 470 while (i--) 471 if (z_erofs_decomp[i]) 472 z_erofs_decomp[i]->exit(); 473 return err; 474 } 475 } 476 return 0; 477 } 478 479 void z_erofs_exit_decompressor(void) 480 { 481 int i; 482 483 for (i = 0; i < Z_EROFS_COMPRESSION_MAX; ++i) 484 if (z_erofs_decomp[i]) 485 z_erofs_decomp[i]->exit(); 486 } 487