1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/xz.h> 3 #include "compress.h" 4 5 struct z_erofs_lzma { 6 struct z_erofs_lzma *next; 7 struct xz_dec_microlzma *state; 8 u8 bounce[PAGE_SIZE]; 9 }; 10 11 /* considering the LZMA performance, no need to use a lockless list for now */ 12 static DEFINE_SPINLOCK(z_erofs_lzma_lock); 13 static unsigned int z_erofs_lzma_max_dictsize; 14 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; 15 static struct z_erofs_lzma *z_erofs_lzma_head; 16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); 17 18 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); 19 20 static void z_erofs_lzma_exit(void) 21 { 22 /* there should be no running fs instance */ 23 while (z_erofs_lzma_avail_strms) { 24 struct z_erofs_lzma *strm; 25 26 spin_lock(&z_erofs_lzma_lock); 27 strm = z_erofs_lzma_head; 28 if (!strm) { 29 spin_unlock(&z_erofs_lzma_lock); 30 DBG_BUGON(1); 31 return; 32 } 33 z_erofs_lzma_head = NULL; 34 spin_unlock(&z_erofs_lzma_lock); 35 36 while (strm) { 37 struct z_erofs_lzma *n = strm->next; 38 39 if (strm->state) 40 xz_dec_microlzma_end(strm->state); 41 kfree(strm); 42 --z_erofs_lzma_avail_strms; 43 strm = n; 44 } 45 } 46 } 47 48 static int __init z_erofs_lzma_init(void) 49 { 50 unsigned int i; 51 52 /* by default, use # of possible CPUs instead */ 53 if (!z_erofs_lzma_nstrms) 54 z_erofs_lzma_nstrms = min_t(unsigned int, num_possible_cpus(), 55 CONFIG_EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS); 56 57 for (i = 0; i < z_erofs_lzma_nstrms; ++i) { 58 struct z_erofs_lzma *strm = kzalloc_obj(*strm); 59 60 if (!strm) { 61 z_erofs_lzma_exit(); 62 return -ENOMEM; 63 } 64 spin_lock(&z_erofs_lzma_lock); 65 strm->next = z_erofs_lzma_head; 66 z_erofs_lzma_head = strm; 67 spin_unlock(&z_erofs_lzma_lock); 68 ++z_erofs_lzma_avail_strms; 69 } 70 return 0; 71 } 72 73 static int z_erofs_load_lzma_config(struct super_block *sb, 74 struct erofs_super_block *dsb, void *data, int size) 75 { 76 static DEFINE_MUTEX(lzma_resize_mutex); 77 struct z_erofs_lzma_cfgs *lzma = data; 78 unsigned int dict_size, i; 79 struct z_erofs_lzma *strm, *head = NULL; 80 int err; 81 82 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { 83 erofs_err(sb, "invalid lzma cfgs, size=%u", size); 84 return -EINVAL; 85 } 86 if (lzma->format) { 87 erofs_err(sb, "unidentified lzma format %x, please check kernel version", 88 le16_to_cpu(lzma->format)); 89 return -EINVAL; 90 } 91 dict_size = le32_to_cpu(lzma->dict_size); 92 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { 93 erofs_err(sb, "unsupported lzma dictionary size %u", 94 dict_size); 95 return -EINVAL; 96 } 97 98 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ 99 mutex_lock(&lzma_resize_mutex); 100 101 if (z_erofs_lzma_max_dictsize >= dict_size) { 102 mutex_unlock(&lzma_resize_mutex); 103 return 0; 104 } 105 106 /* 1. collect/isolate all streams for the following check */ 107 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { 108 struct z_erofs_lzma *last; 109 110 again: 111 spin_lock(&z_erofs_lzma_lock); 112 strm = z_erofs_lzma_head; 113 if (!strm) { 114 spin_unlock(&z_erofs_lzma_lock); 115 wait_event(z_erofs_lzma_wq, 116 READ_ONCE(z_erofs_lzma_head)); 117 goto again; 118 } 119 z_erofs_lzma_head = NULL; 120 spin_unlock(&z_erofs_lzma_lock); 121 122 for (last = strm; last->next; last = last->next) 123 ++i; 124 last->next = head; 125 head = strm; 126 } 127 128 err = 0; 129 /* 2. walk each isolated stream and grow max dict_size if needed */ 130 for (strm = head; strm; strm = strm->next) { 131 if (strm->state) 132 xz_dec_microlzma_end(strm->state); 133 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); 134 if (!strm->state) 135 err = -ENOMEM; 136 } 137 138 /* 3. push back all to the global list and update max dict_size */ 139 spin_lock(&z_erofs_lzma_lock); 140 DBG_BUGON(z_erofs_lzma_head); 141 z_erofs_lzma_head = head; 142 spin_unlock(&z_erofs_lzma_lock); 143 wake_up_all(&z_erofs_lzma_wq); 144 145 z_erofs_lzma_max_dictsize = dict_size; 146 mutex_unlock(&lzma_resize_mutex); 147 return err; 148 } 149 150 static const char *z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, 151 struct page **pgpl) 152 { 153 struct super_block *sb = rq->sb; 154 struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; 155 struct xz_buf buf = {}; 156 struct z_erofs_lzma *strm; 157 enum xz_ret xz_err; 158 const char *reason; 159 160 /* 1. get the exact LZMA compressed size */ 161 dctx.kin = kmap_local_page(*rq->in); 162 reason = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, 163 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); 164 if (reason) { 165 kunmap_local(dctx.kin); 166 return reason; 167 } 168 169 /* 2. get an available lzma context */ 170 again: 171 spin_lock(&z_erofs_lzma_lock); 172 strm = z_erofs_lzma_head; 173 if (!strm) { 174 spin_unlock(&z_erofs_lzma_lock); 175 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); 176 goto again; 177 } 178 z_erofs_lzma_head = strm->next; 179 spin_unlock(&z_erofs_lzma_lock); 180 181 /* 3. multi-call decompress */ 182 xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize, 183 !rq->partial_decoding); 184 buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); 185 rq->inputsize -= buf.in_size; 186 buf.in = dctx.kin + rq->pageofs_in; 187 dctx.bounce = strm->bounce; 188 do { 189 dctx.avail_out = buf.out_size - buf.out_pos; 190 dctx.inbuf_sz = buf.in_size; 191 dctx.inbuf_pos = buf.in_pos; 192 reason = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out, 193 (void **)&buf.in, pgpl); 194 if (reason) 195 break; 196 197 if (buf.out_size == buf.out_pos) { 198 buf.out_size = dctx.avail_out; 199 buf.out_pos = 0; 200 } 201 buf.in_size = dctx.inbuf_sz; 202 buf.in_pos = dctx.inbuf_pos; 203 204 xz_err = xz_dec_microlzma_run(strm->state, &buf); 205 DBG_BUGON(buf.out_pos > buf.out_size); 206 DBG_BUGON(buf.in_pos > buf.in_size); 207 208 if (xz_err != XZ_OK) { 209 if (xz_err == XZ_STREAM_END && !rq->outputsize) 210 break; 211 reason = (xz_err == XZ_DATA_ERROR ? 212 "corrupted compressed data" : 213 "unexpected end of stream"); 214 break; 215 } 216 } while (1); 217 218 if (dctx.kout) 219 kunmap_local(dctx.kout); 220 kunmap_local(dctx.kin); 221 /* 4. push back LZMA stream context to the global list */ 222 spin_lock(&z_erofs_lzma_lock); 223 strm->next = z_erofs_lzma_head; 224 z_erofs_lzma_head = strm; 225 spin_unlock(&z_erofs_lzma_lock); 226 wake_up(&z_erofs_lzma_wq); 227 return reason; 228 } 229 230 const struct z_erofs_decompressor z_erofs_lzma_decomp = { 231 .config = z_erofs_load_lzma_config, 232 .decompress = z_erofs_lzma_decompress, 233 .init = z_erofs_lzma_init, 234 .exit = z_erofs_lzma_exit, 235 .name = "lzma" 236 }; 237