1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/xz.h> 3 #include "compress.h" 4 5 struct z_erofs_lzma { 6 struct z_erofs_lzma *next; 7 struct xz_dec_microlzma *state; 8 unsigned int dict_size; 9 u8 bounce[PAGE_SIZE]; 10 }; 11 12 /* considering the LZMA performance, no need to use a lockless list for now */ 13 static DEFINE_SPINLOCK(z_erofs_lzma_lock); 14 static unsigned int z_erofs_lzma_max_dictsize; 15 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; 16 static struct z_erofs_lzma *z_erofs_lzma_head; 17 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); 18 19 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); 20 21 static void z_erofs_lzma_exit(void) 22 { 23 /* there should be no running fs instance */ 24 while (z_erofs_lzma_avail_strms) { 25 struct z_erofs_lzma *strm; 26 27 spin_lock(&z_erofs_lzma_lock); 28 strm = z_erofs_lzma_head; 29 if (!strm) { 30 spin_unlock(&z_erofs_lzma_lock); 31 DBG_BUGON(1); 32 return; 33 } 34 z_erofs_lzma_head = NULL; 35 spin_unlock(&z_erofs_lzma_lock); 36 37 while (strm) { 38 struct z_erofs_lzma *n = strm->next; 39 40 if (strm->state) 41 xz_dec_microlzma_end(strm->state); 42 kfree(strm); 43 --z_erofs_lzma_avail_strms; 44 strm = n; 45 } 46 } 47 } 48 49 static int __init z_erofs_lzma_init(void) 50 { 51 unsigned int i; 52 53 /* by default, use # of possible CPUs instead */ 54 if (!z_erofs_lzma_nstrms) 55 z_erofs_lzma_nstrms = min_t(unsigned int, num_possible_cpus(), 56 CONFIG_EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS); 57 58 for (i = 0; i < z_erofs_lzma_nstrms; ++i) { 59 struct z_erofs_lzma *strm = kzalloc_obj(*strm); 60 61 if (!strm) { 62 z_erofs_lzma_exit(); 63 return -ENOMEM; 64 } 65 spin_lock(&z_erofs_lzma_lock); 66 strm->next = z_erofs_lzma_head; 67 z_erofs_lzma_head = strm; 68 spin_unlock(&z_erofs_lzma_lock); 69 ++z_erofs_lzma_avail_strms; 70 } 71 return 0; 72 } 73 74 static int z_erofs_load_lzma_config(struct super_block *sb, 75 struct erofs_super_block *dsb, void *data, int size) 76 { 77 static DEFINE_MUTEX(lzma_resize_mutex); 78 struct z_erofs_lzma_cfgs *lzma = data; 79 unsigned int dict_size, i; 80 struct z_erofs_lzma *strm, *head = NULL; 81 int err; 82 83 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { 84 erofs_err(sb, "invalid lzma cfgs, size=%u", size); 85 return -EINVAL; 86 } 87 if (lzma->format) { 88 erofs_err(sb, "unidentified lzma format %x, please check kernel version", 89 le16_to_cpu(lzma->format)); 90 return -EINVAL; 91 } 92 dict_size = le32_to_cpu(lzma->dict_size); 93 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { 94 erofs_err(sb, "unsupported lzma dictionary size %u", 95 dict_size); 96 return -EINVAL; 97 } 98 99 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ 100 mutex_lock(&lzma_resize_mutex); 101 102 if (z_erofs_lzma_max_dictsize >= dict_size) { 103 mutex_unlock(&lzma_resize_mutex); 104 return 0; 105 } 106 107 /* 1. collect/isolate all streams for the following check */ 108 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { 109 struct z_erofs_lzma *last; 110 111 again: 112 spin_lock(&z_erofs_lzma_lock); 113 strm = z_erofs_lzma_head; 114 if (!strm) { 115 spin_unlock(&z_erofs_lzma_lock); 116 wait_event(z_erofs_lzma_wq, 117 READ_ONCE(z_erofs_lzma_head)); 118 goto again; 119 } 120 z_erofs_lzma_head = NULL; 121 spin_unlock(&z_erofs_lzma_lock); 122 123 for (last = strm; last->next; last = last->next) 124 ++i; 125 last->next = head; 126 head = strm; 127 } 128 129 err = 0; 130 /* 2. walk each isolated stream and grow max dict_size if needed */ 131 for (strm = head; strm; strm = strm->next) { 132 struct xz_dec_microlzma *state; 133 134 if (strm->dict_size >= dict_size) 135 continue; 136 state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); 137 if (!state) { 138 err = -ENOMEM; 139 break; 140 } 141 if (strm->state) 142 xz_dec_microlzma_end(strm->state); 143 strm->state = state; 144 strm->dict_size = dict_size; 145 } 146 147 /* 3. push back all to the global list and update max dict_size */ 148 spin_lock(&z_erofs_lzma_lock); 149 DBG_BUGON(z_erofs_lzma_head); 150 z_erofs_lzma_head = head; 151 spin_unlock(&z_erofs_lzma_lock); 152 wake_up_all(&z_erofs_lzma_wq); 153 154 if (!err) 155 z_erofs_lzma_max_dictsize = dict_size; 156 mutex_unlock(&lzma_resize_mutex); 157 return err; 158 } 159 160 static const char *z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, 161 struct page **pgpl) 162 { 163 struct super_block *sb = rq->sb; 164 struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; 165 struct xz_buf buf = {}; 166 struct z_erofs_lzma *strm; 167 enum xz_ret xz_err; 168 const char *reason; 169 170 /* 1. get the exact LZMA compressed size */ 171 dctx.kin = kmap_local_page(*rq->in); 172 reason = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, 173 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); 174 if (reason) { 175 kunmap_local(dctx.kin); 176 return reason; 177 } 178 179 /* 2. get an available lzma context */ 180 again: 181 spin_lock(&z_erofs_lzma_lock); 182 strm = z_erofs_lzma_head; 183 if (!strm) { 184 spin_unlock(&z_erofs_lzma_lock); 185 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); 186 goto again; 187 } 188 z_erofs_lzma_head = strm->next; 189 spin_unlock(&z_erofs_lzma_lock); 190 191 /* 3. multi-call decompress */ 192 xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize, 193 !rq->partial_decoding); 194 buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); 195 rq->inputsize -= buf.in_size; 196 buf.in = dctx.kin + rq->pageofs_in; 197 dctx.bounce = strm->bounce; 198 do { 199 dctx.avail_out = buf.out_size - buf.out_pos; 200 dctx.inbuf_sz = buf.in_size; 201 dctx.inbuf_pos = buf.in_pos; 202 reason = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out, 203 (void **)&buf.in, pgpl); 204 if (reason) 205 break; 206 207 if (buf.out_size == buf.out_pos) { 208 buf.out_size = dctx.avail_out; 209 buf.out_pos = 0; 210 } 211 buf.in_size = dctx.inbuf_sz; 212 buf.in_pos = dctx.inbuf_pos; 213 214 xz_err = xz_dec_microlzma_run(strm->state, &buf); 215 DBG_BUGON(buf.out_pos > buf.out_size); 216 DBG_BUGON(buf.in_pos > buf.in_size); 217 218 if (xz_err != XZ_OK) { 219 if (xz_err == XZ_STREAM_END && !rq->outputsize) 220 break; 221 reason = (xz_err == XZ_DATA_ERROR ? 222 "corrupted compressed data" : 223 "unexpected end of stream"); 224 break; 225 } 226 } while (1); 227 228 if (dctx.kout) 229 kunmap_local(dctx.kout); 230 kunmap_local(dctx.kin); 231 /* 4. push back LZMA stream context to the global list */ 232 spin_lock(&z_erofs_lzma_lock); 233 strm->next = z_erofs_lzma_head; 234 z_erofs_lzma_head = strm; 235 spin_unlock(&z_erofs_lzma_lock); 236 wake_up(&z_erofs_lzma_wq); 237 return reason; 238 } 239 240 const struct z_erofs_decompressor z_erofs_lzma_decomp = { 241 .config = z_erofs_load_lzma_config, 242 .decompress = z_erofs_lzma_decompress, 243 .init = z_erofs_lzma_init, 244 .exit = z_erofs_lzma_exit, 245 .name = "lzma" 246 }; 247