1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/xz.h> 3 #include "compress.h" 4 5 struct z_erofs_lzma { 6 struct z_erofs_lzma *next; 7 struct xz_dec_microlzma *state; 8 struct xz_buf buf; 9 u8 bounce[PAGE_SIZE]; 10 }; 11 12 /* considering the LZMA performance, no need to use a lockless list for now */ 13 static DEFINE_SPINLOCK(z_erofs_lzma_lock); 14 static unsigned int z_erofs_lzma_max_dictsize; 15 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; 16 static struct z_erofs_lzma *z_erofs_lzma_head; 17 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); 18 19 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); 20 21 void z_erofs_lzma_exit(void) 22 { 23 /* there should be no running fs instance */ 24 while (z_erofs_lzma_avail_strms) { 25 struct z_erofs_lzma *strm; 26 27 spin_lock(&z_erofs_lzma_lock); 28 strm = z_erofs_lzma_head; 29 if (!strm) { 30 spin_unlock(&z_erofs_lzma_lock); 31 DBG_BUGON(1); 32 return; 33 } 34 z_erofs_lzma_head = NULL; 35 spin_unlock(&z_erofs_lzma_lock); 36 37 while (strm) { 38 struct z_erofs_lzma *n = strm->next; 39 40 if (strm->state) 41 xz_dec_microlzma_end(strm->state); 42 kfree(strm); 43 --z_erofs_lzma_avail_strms; 44 strm = n; 45 } 46 } 47 } 48 49 int __init z_erofs_lzma_init(void) 50 { 51 unsigned int i; 52 53 /* by default, use # of possible CPUs instead */ 54 if (!z_erofs_lzma_nstrms) 55 z_erofs_lzma_nstrms = num_possible_cpus(); 56 57 for (i = 0; i < z_erofs_lzma_nstrms; ++i) { 58 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL); 59 60 if (!strm) { 61 z_erofs_lzma_exit(); 62 return -ENOMEM; 63 } 64 spin_lock(&z_erofs_lzma_lock); 65 strm->next = z_erofs_lzma_head; 66 z_erofs_lzma_head = strm; 67 spin_unlock(&z_erofs_lzma_lock); 68 ++z_erofs_lzma_avail_strms; 69 } 70 return 0; 71 } 72 73 int z_erofs_load_lzma_config(struct super_block *sb, 74 struct erofs_super_block *dsb, void *data, int size) 75 { 76 static DEFINE_MUTEX(lzma_resize_mutex); 77 struct z_erofs_lzma_cfgs *lzma = data; 78 unsigned int dict_size, i; 79 struct z_erofs_lzma *strm, *head = NULL; 80 int err; 81 82 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { 83 erofs_err(sb, "invalid lzma cfgs, size=%u", size); 84 return -EINVAL; 85 } 86 if (lzma->format) { 87 erofs_err(sb, "unidentified lzma format %x, please check kernel version", 88 le16_to_cpu(lzma->format)); 89 return -EINVAL; 90 } 91 dict_size = le32_to_cpu(lzma->dict_size); 92 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { 93 erofs_err(sb, "unsupported lzma dictionary size %u", 94 dict_size); 95 return -EINVAL; 96 } 97 98 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ 99 mutex_lock(&lzma_resize_mutex); 100 101 if (z_erofs_lzma_max_dictsize >= dict_size) { 102 mutex_unlock(&lzma_resize_mutex); 103 return 0; 104 } 105 106 /* 1. collect/isolate all streams for the following check */ 107 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { 108 struct z_erofs_lzma *last; 109 110 again: 111 spin_lock(&z_erofs_lzma_lock); 112 strm = z_erofs_lzma_head; 113 if (!strm) { 114 spin_unlock(&z_erofs_lzma_lock); 115 wait_event(z_erofs_lzma_wq, 116 READ_ONCE(z_erofs_lzma_head)); 117 goto again; 118 } 119 z_erofs_lzma_head = NULL; 120 spin_unlock(&z_erofs_lzma_lock); 121 122 for (last = strm; last->next; last = last->next) 123 ++i; 124 last->next = head; 125 head = strm; 126 } 127 128 err = 0; 129 /* 2. walk each isolated stream and grow max dict_size if needed */ 130 for (strm = head; strm; strm = strm->next) { 131 if (strm->state) 132 xz_dec_microlzma_end(strm->state); 133 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); 134 if (!strm->state) 135 err = -ENOMEM; 136 } 137 138 /* 3. push back all to the global list and update max dict_size */ 139 spin_lock(&z_erofs_lzma_lock); 140 DBG_BUGON(z_erofs_lzma_head); 141 z_erofs_lzma_head = head; 142 spin_unlock(&z_erofs_lzma_lock); 143 wake_up_all(&z_erofs_lzma_wq); 144 145 z_erofs_lzma_max_dictsize = dict_size; 146 mutex_unlock(&lzma_resize_mutex); 147 return err; 148 } 149 150 int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, 151 struct page **pgpl) 152 { 153 const unsigned int nrpages_out = 154 PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; 155 const unsigned int nrpages_in = 156 PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; 157 unsigned int inlen, outlen, pageofs; 158 struct z_erofs_lzma *strm; 159 u8 *kin; 160 bool bounced = false; 161 int no, ni, j, err = 0; 162 163 /* 1. get the exact LZMA compressed size */ 164 kin = kmap(*rq->in); 165 err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in, 166 min_t(unsigned int, rq->inputsize, 167 rq->sb->s_blocksize - rq->pageofs_in)); 168 if (err) { 169 kunmap(*rq->in); 170 return err; 171 } 172 173 /* 2. get an available lzma context */ 174 again: 175 spin_lock(&z_erofs_lzma_lock); 176 strm = z_erofs_lzma_head; 177 if (!strm) { 178 spin_unlock(&z_erofs_lzma_lock); 179 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); 180 goto again; 181 } 182 z_erofs_lzma_head = strm->next; 183 spin_unlock(&z_erofs_lzma_lock); 184 185 /* 3. multi-call decompress */ 186 inlen = rq->inputsize; 187 outlen = rq->outputsize; 188 xz_dec_microlzma_reset(strm->state, inlen, outlen, 189 !rq->partial_decoding); 190 pageofs = rq->pageofs_out; 191 strm->buf.in = kin + rq->pageofs_in; 192 strm->buf.in_pos = 0; 193 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in); 194 inlen -= strm->buf.in_size; 195 strm->buf.out = NULL; 196 strm->buf.out_pos = 0; 197 strm->buf.out_size = 0; 198 199 for (ni = 0, no = -1;;) { 200 enum xz_ret xz_err; 201 202 if (strm->buf.out_pos == strm->buf.out_size) { 203 if (strm->buf.out) { 204 kunmap(rq->out[no]); 205 strm->buf.out = NULL; 206 } 207 208 if (++no >= nrpages_out || !outlen) { 209 erofs_err(rq->sb, "decompressed buf out of bound"); 210 err = -EFSCORRUPTED; 211 break; 212 } 213 strm->buf.out_pos = 0; 214 strm->buf.out_size = min_t(u32, outlen, 215 PAGE_SIZE - pageofs); 216 outlen -= strm->buf.out_size; 217 if (!rq->out[no] && rq->fillgaps) { /* deduped */ 218 rq->out[no] = erofs_allocpage(pgpl, rq->gfp); 219 if (!rq->out[no]) { 220 err = -ENOMEM; 221 break; 222 } 223 set_page_private(rq->out[no], 224 Z_EROFS_SHORTLIVED_PAGE); 225 } 226 if (rq->out[no]) 227 strm->buf.out = kmap(rq->out[no]) + pageofs; 228 pageofs = 0; 229 } else if (strm->buf.in_pos == strm->buf.in_size) { 230 kunmap(rq->in[ni]); 231 232 if (++ni >= nrpages_in || !inlen) { 233 erofs_err(rq->sb, "compressed buf out of bound"); 234 err = -EFSCORRUPTED; 235 break; 236 } 237 strm->buf.in_pos = 0; 238 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE); 239 inlen -= strm->buf.in_size; 240 kin = kmap(rq->in[ni]); 241 strm->buf.in = kin; 242 bounced = false; 243 } 244 245 /* 246 * Handle overlapping: Use bounced buffer if the compressed 247 * data is under processing; Otherwise, Use short-lived pages 248 * from the on-stack pagepool where pages share with the same 249 * request. 250 */ 251 if (!bounced && rq->out[no] == rq->in[ni]) { 252 memcpy(strm->bounce, strm->buf.in, strm->buf.in_size); 253 strm->buf.in = strm->bounce; 254 bounced = true; 255 } 256 for (j = ni + 1; j < nrpages_in; ++j) { 257 struct page *tmppage; 258 259 if (rq->out[no] != rq->in[j]) 260 continue; 261 tmppage = erofs_allocpage(pgpl, rq->gfp); 262 if (!tmppage) { 263 err = -ENOMEM; 264 goto failed; 265 } 266 set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); 267 copy_highpage(tmppage, rq->in[j]); 268 rq->in[j] = tmppage; 269 } 270 xz_err = xz_dec_microlzma_run(strm->state, &strm->buf); 271 DBG_BUGON(strm->buf.out_pos > strm->buf.out_size); 272 DBG_BUGON(strm->buf.in_pos > strm->buf.in_size); 273 274 if (xz_err != XZ_OK) { 275 if (xz_err == XZ_STREAM_END && !outlen) 276 break; 277 erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]", 278 xz_err, rq->inputsize, rq->outputsize); 279 err = -EFSCORRUPTED; 280 break; 281 } 282 } 283 failed: 284 if (no < nrpages_out && strm->buf.out) 285 kunmap(rq->out[no]); 286 if (ni < nrpages_in) 287 kunmap(rq->in[ni]); 288 /* 4. push back LZMA stream context to the global list */ 289 spin_lock(&z_erofs_lzma_lock); 290 strm->next = z_erofs_lzma_head; 291 z_erofs_lzma_head = strm; 292 spin_unlock(&z_erofs_lzma_lock); 293 wake_up(&z_erofs_lzma_wq); 294 return err; 295 } 296