1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/xz.h> 3 #include "compress.h" 4 5 struct z_erofs_lzma { 6 struct z_erofs_lzma *next; 7 struct xz_dec_microlzma *state; 8 u8 bounce[PAGE_SIZE]; 9 }; 10 11 /* considering the LZMA performance, no need to use a lockless list for now */ 12 static DEFINE_SPINLOCK(z_erofs_lzma_lock); 13 static unsigned int z_erofs_lzma_max_dictsize; 14 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; 15 static struct z_erofs_lzma *z_erofs_lzma_head; 16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); 17 18 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); 19 20 static void z_erofs_lzma_exit(void) 21 { 22 /* there should be no running fs instance */ 23 while (z_erofs_lzma_avail_strms) { 24 struct z_erofs_lzma *strm; 25 26 spin_lock(&z_erofs_lzma_lock); 27 strm = z_erofs_lzma_head; 28 if (!strm) { 29 spin_unlock(&z_erofs_lzma_lock); 30 DBG_BUGON(1); 31 return; 32 } 33 z_erofs_lzma_head = NULL; 34 spin_unlock(&z_erofs_lzma_lock); 35 36 while (strm) { 37 struct z_erofs_lzma *n = strm->next; 38 39 if (strm->state) 40 xz_dec_microlzma_end(strm->state); 41 kfree(strm); 42 --z_erofs_lzma_avail_strms; 43 strm = n; 44 } 45 } 46 } 47 48 static int __init z_erofs_lzma_init(void) 49 { 50 unsigned int i; 51 52 /* by default, use # of possible CPUs instead */ 53 if (!z_erofs_lzma_nstrms) 54 z_erofs_lzma_nstrms = num_possible_cpus(); 55 56 for (i = 0; i < z_erofs_lzma_nstrms; ++i) { 57 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL); 58 59 if (!strm) { 60 z_erofs_lzma_exit(); 61 return -ENOMEM; 62 } 63 spin_lock(&z_erofs_lzma_lock); 64 strm->next = z_erofs_lzma_head; 65 z_erofs_lzma_head = strm; 66 spin_unlock(&z_erofs_lzma_lock); 67 ++z_erofs_lzma_avail_strms; 68 } 69 return 0; 70 } 71 72 static int z_erofs_load_lzma_config(struct super_block *sb, 73 struct erofs_super_block *dsb, void *data, int size) 74 { 75 static DEFINE_MUTEX(lzma_resize_mutex); 76 struct z_erofs_lzma_cfgs *lzma = data; 77 unsigned int dict_size, i; 78 struct z_erofs_lzma *strm, *head = NULL; 79 int err; 80 81 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { 82 erofs_err(sb, "invalid lzma cfgs, size=%u", size); 83 return -EINVAL; 84 } 85 if (lzma->format) { 86 erofs_err(sb, "unidentified lzma format %x, please check kernel version", 87 le16_to_cpu(lzma->format)); 88 return -EINVAL; 89 } 90 dict_size = le32_to_cpu(lzma->dict_size); 91 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { 92 erofs_err(sb, "unsupported lzma dictionary size %u", 93 dict_size); 94 return -EINVAL; 95 } 96 97 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ 98 mutex_lock(&lzma_resize_mutex); 99 100 if (z_erofs_lzma_max_dictsize >= dict_size) { 101 mutex_unlock(&lzma_resize_mutex); 102 return 0; 103 } 104 105 /* 1. collect/isolate all streams for the following check */ 106 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { 107 struct z_erofs_lzma *last; 108 109 again: 110 spin_lock(&z_erofs_lzma_lock); 111 strm = z_erofs_lzma_head; 112 if (!strm) { 113 spin_unlock(&z_erofs_lzma_lock); 114 wait_event(z_erofs_lzma_wq, 115 READ_ONCE(z_erofs_lzma_head)); 116 goto again; 117 } 118 z_erofs_lzma_head = NULL; 119 spin_unlock(&z_erofs_lzma_lock); 120 121 for (last = strm; last->next; last = last->next) 122 ++i; 123 last->next = head; 124 head = strm; 125 } 126 127 err = 0; 128 /* 2. walk each isolated stream and grow max dict_size if needed */ 129 for (strm = head; strm; strm = strm->next) { 130 if (strm->state) 131 xz_dec_microlzma_end(strm->state); 132 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); 133 if (!strm->state) 134 err = -ENOMEM; 135 } 136 137 /* 3. push back all to the global list and update max dict_size */ 138 spin_lock(&z_erofs_lzma_lock); 139 DBG_BUGON(z_erofs_lzma_head); 140 z_erofs_lzma_head = head; 141 spin_unlock(&z_erofs_lzma_lock); 142 wake_up_all(&z_erofs_lzma_wq); 143 144 z_erofs_lzma_max_dictsize = dict_size; 145 mutex_unlock(&lzma_resize_mutex); 146 return err; 147 } 148 149 static int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, 150 struct page **pgpl) 151 { 152 struct super_block *sb = rq->sb; 153 struct z_erofs_stream_dctx dctx = { 154 .rq = rq, 155 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT, 156 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize) 157 >> PAGE_SHIFT, 158 .no = -1, .ni = 0, 159 }; 160 struct xz_buf buf = {}; 161 struct z_erofs_lzma *strm; 162 enum xz_ret xz_err; 163 int err; 164 165 /* 1. get the exact LZMA compressed size */ 166 dctx.kin = kmap_local_page(*rq->in); 167 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, 168 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); 169 if (err) { 170 kunmap_local(dctx.kin); 171 return err; 172 } 173 174 /* 2. get an available lzma context */ 175 again: 176 spin_lock(&z_erofs_lzma_lock); 177 strm = z_erofs_lzma_head; 178 if (!strm) { 179 spin_unlock(&z_erofs_lzma_lock); 180 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); 181 goto again; 182 } 183 z_erofs_lzma_head = strm->next; 184 spin_unlock(&z_erofs_lzma_lock); 185 186 /* 3. multi-call decompress */ 187 xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize, 188 !rq->partial_decoding); 189 buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); 190 rq->inputsize -= buf.in_size; 191 buf.in = dctx.kin + rq->pageofs_in; 192 dctx.bounce = strm->bounce; 193 do { 194 dctx.avail_out = buf.out_size - buf.out_pos; 195 dctx.inbuf_sz = buf.in_size; 196 dctx.inbuf_pos = buf.in_pos; 197 err = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out, 198 (void **)&buf.in, pgpl); 199 if (err) 200 break; 201 202 if (buf.out_size == buf.out_pos) { 203 buf.out_size = dctx.avail_out; 204 buf.out_pos = 0; 205 } 206 buf.in_size = dctx.inbuf_sz; 207 buf.in_pos = dctx.inbuf_pos; 208 209 xz_err = xz_dec_microlzma_run(strm->state, &buf); 210 DBG_BUGON(buf.out_pos > buf.out_size); 211 DBG_BUGON(buf.in_pos > buf.in_size); 212 213 if (xz_err != XZ_OK) { 214 if (xz_err == XZ_STREAM_END && !rq->outputsize) 215 break; 216 erofs_err(sb, "failed to decompress %d in[%u] out[%u]", 217 xz_err, rq->inputsize, rq->outputsize); 218 err = -EFSCORRUPTED; 219 break; 220 } 221 } while (1); 222 223 if (dctx.kout) 224 kunmap_local(dctx.kout); 225 kunmap_local(dctx.kin); 226 /* 4. push back LZMA stream context to the global list */ 227 spin_lock(&z_erofs_lzma_lock); 228 strm->next = z_erofs_lzma_head; 229 z_erofs_lzma_head = strm; 230 spin_unlock(&z_erofs_lzma_lock); 231 wake_up(&z_erofs_lzma_wq); 232 return err; 233 } 234 235 const struct z_erofs_decompressor z_erofs_lzma_decomp = { 236 .config = z_erofs_load_lzma_config, 237 .decompress = z_erofs_lzma_decompress, 238 .init = z_erofs_lzma_init, 239 .exit = z_erofs_lzma_exit, 240 .name = "lzma" 241 }; 242