1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/zstd.h> 3 #include "compress.h" 4 5 struct z_erofs_zstd { 6 struct z_erofs_zstd *next; 7 u8 bounce[PAGE_SIZE]; 8 void *wksp; 9 unsigned int wkspsz; 10 }; 11 12 static DEFINE_SPINLOCK(z_erofs_zstd_lock); 13 static unsigned int z_erofs_zstd_max_dictsize; 14 static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms; 15 static struct z_erofs_zstd *z_erofs_zstd_head; 16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq); 17 18 module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444); 19 20 static struct z_erofs_zstd *z_erofs_isolate_strms(bool all) 21 { 22 struct z_erofs_zstd *strm; 23 24 again: 25 spin_lock(&z_erofs_zstd_lock); 26 strm = z_erofs_zstd_head; 27 if (!strm) { 28 spin_unlock(&z_erofs_zstd_lock); 29 wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head)); 30 goto again; 31 } 32 z_erofs_zstd_head = all ? NULL : strm->next; 33 spin_unlock(&z_erofs_zstd_lock); 34 return strm; 35 } 36 37 void z_erofs_zstd_exit(void) 38 { 39 while (z_erofs_zstd_avail_strms) { 40 struct z_erofs_zstd *strm, *n; 41 42 for (strm = z_erofs_isolate_strms(true); strm; strm = n) { 43 n = strm->next; 44 45 kvfree(strm->wksp); 46 kfree(strm); 47 --z_erofs_zstd_avail_strms; 48 } 49 } 50 } 51 52 int __init z_erofs_zstd_init(void) 53 { 54 /* by default, use # of possible CPUs instead */ 55 if (!z_erofs_zstd_nstrms) 56 z_erofs_zstd_nstrms = num_possible_cpus(); 57 58 for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms; 59 ++z_erofs_zstd_avail_strms) { 60 struct z_erofs_zstd *strm; 61 62 strm = kzalloc(sizeof(*strm), GFP_KERNEL); 63 if (!strm) { 64 z_erofs_zstd_exit(); 65 return -ENOMEM; 66 } 67 spin_lock(&z_erofs_zstd_lock); 68 strm->next = z_erofs_zstd_head; 69 z_erofs_zstd_head = strm; 70 spin_unlock(&z_erofs_zstd_lock); 71 } 72 return 0; 73 } 74 75 int z_erofs_load_zstd_config(struct super_block *sb, 76 struct erofs_super_block *dsb, void *data, int size) 77 { 78 static DEFINE_MUTEX(zstd_resize_mutex); 79 struct z_erofs_zstd_cfgs *zstd = data; 80 unsigned int dict_size, wkspsz; 81 struct z_erofs_zstd *strm, *head = NULL; 82 void *wksp; 83 84 if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) { 85 erofs_err(sb, "unsupported zstd format, size=%u", size); 86 return -EINVAL; 87 } 88 89 if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) { 90 erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog); 91 return -EINVAL; 92 } 93 dict_size = 1U << (zstd->windowlog + 10); 94 95 /* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */ 96 mutex_lock(&zstd_resize_mutex); 97 if (z_erofs_zstd_max_dictsize >= dict_size) { 98 mutex_unlock(&zstd_resize_mutex); 99 return 0; 100 } 101 102 /* 1. collect/isolate all streams for the following check */ 103 while (z_erofs_zstd_avail_strms) { 104 struct z_erofs_zstd *n; 105 106 for (strm = z_erofs_isolate_strms(true); strm; strm = n) { 107 n = strm->next; 108 strm->next = head; 109 head = strm; 110 --z_erofs_zstd_avail_strms; 111 } 112 } 113 114 /* 2. walk each isolated stream and grow max dict_size if needed */ 115 wkspsz = zstd_dstream_workspace_bound(dict_size); 116 for (strm = head; strm; strm = strm->next) { 117 wksp = kvmalloc(wkspsz, GFP_KERNEL); 118 if (!wksp) 119 break; 120 kvfree(strm->wksp); 121 strm->wksp = wksp; 122 strm->wkspsz = wkspsz; 123 } 124 125 /* 3. push back all to the global list and update max dict_size */ 126 spin_lock(&z_erofs_zstd_lock); 127 DBG_BUGON(z_erofs_zstd_head); 128 z_erofs_zstd_head = head; 129 spin_unlock(&z_erofs_zstd_lock); 130 z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms; 131 wake_up_all(&z_erofs_zstd_wq); 132 if (!strm) 133 z_erofs_zstd_max_dictsize = dict_size; 134 mutex_unlock(&zstd_resize_mutex); 135 return strm ? -ENOMEM : 0; 136 } 137 138 int z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq, 139 struct page **pgpl) 140 { 141 const unsigned int nrpages_out = 142 PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; 143 const unsigned int nrpages_in = 144 PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; 145 zstd_dstream *stream; 146 struct super_block *sb = rq->sb; 147 unsigned int insz, outsz, pofs; 148 struct z_erofs_zstd *strm; 149 zstd_in_buffer in_buf = { NULL, 0, 0 }; 150 zstd_out_buffer out_buf = { NULL, 0, 0 }; 151 u8 *kin, *kout = NULL; 152 bool bounced = false; 153 int no = -1, ni = 0, j = 0, zerr, err; 154 155 /* 1. get the exact compressed size */ 156 kin = kmap_local_page(*rq->in); 157 err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in, 158 min_t(unsigned int, rq->inputsize, 159 sb->s_blocksize - rq->pageofs_in)); 160 if (err) { 161 kunmap_local(kin); 162 return err; 163 } 164 165 /* 2. get an available ZSTD context */ 166 strm = z_erofs_isolate_strms(false); 167 168 /* 3. multi-call decompress */ 169 insz = rq->inputsize; 170 outsz = rq->outputsize; 171 stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz); 172 if (!stream) { 173 err = -EIO; 174 goto failed_zinit; 175 } 176 177 pofs = rq->pageofs_out; 178 in_buf.size = min_t(u32, insz, PAGE_SIZE - rq->pageofs_in); 179 insz -= in_buf.size; 180 in_buf.src = kin + rq->pageofs_in; 181 do { 182 if (out_buf.size == out_buf.pos) { 183 if (++no >= nrpages_out || !outsz) { 184 erofs_err(sb, "insufficient space for decompressed data"); 185 err = -EFSCORRUPTED; 186 break; 187 } 188 189 if (kout) 190 kunmap_local(kout); 191 out_buf.size = min_t(u32, outsz, PAGE_SIZE - pofs); 192 outsz -= out_buf.size; 193 if (!rq->out[no]) { 194 rq->out[no] = erofs_allocpage(pgpl, rq->gfp); 195 if (!rq->out[no]) { 196 kout = NULL; 197 err = -ENOMEM; 198 break; 199 } 200 set_page_private(rq->out[no], 201 Z_EROFS_SHORTLIVED_PAGE); 202 } 203 kout = kmap_local_page(rq->out[no]); 204 out_buf.dst = kout + pofs; 205 out_buf.pos = 0; 206 pofs = 0; 207 } 208 209 if (in_buf.size == in_buf.pos && insz) { 210 if (++ni >= nrpages_in) { 211 erofs_err(sb, "invalid compressed data"); 212 err = -EFSCORRUPTED; 213 break; 214 } 215 216 if (kout) /* unlike kmap(), take care of the orders */ 217 kunmap_local(kout); 218 kunmap_local(kin); 219 in_buf.size = min_t(u32, insz, PAGE_SIZE); 220 insz -= in_buf.size; 221 kin = kmap_local_page(rq->in[ni]); 222 in_buf.src = kin; 223 in_buf.pos = 0; 224 bounced = false; 225 if (kout) { 226 j = (u8 *)out_buf.dst - kout; 227 kout = kmap_local_page(rq->out[no]); 228 out_buf.dst = kout + j; 229 } 230 } 231 232 /* 233 * Handle overlapping: Use bounced buffer if the compressed 234 * data is under processing; Or use short-lived pages from the 235 * on-stack pagepool where pages share among the same request 236 * and not _all_ inplace I/O pages are needed to be doubled. 237 */ 238 if (!bounced && rq->out[no] == rq->in[ni]) { 239 memcpy(strm->bounce, in_buf.src, in_buf.size); 240 in_buf.src = strm->bounce; 241 bounced = true; 242 } 243 244 for (j = ni + 1; j < nrpages_in; ++j) { 245 struct page *tmppage; 246 247 if (rq->out[no] != rq->in[j]) 248 continue; 249 tmppage = erofs_allocpage(pgpl, rq->gfp); 250 if (!tmppage) { 251 err = -ENOMEM; 252 goto failed; 253 } 254 set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); 255 copy_highpage(tmppage, rq->in[j]); 256 rq->in[j] = tmppage; 257 } 258 zerr = zstd_decompress_stream(stream, &out_buf, &in_buf); 259 if (zstd_is_error(zerr) || (!zerr && outsz)) { 260 erofs_err(sb, "failed to decompress in[%u] out[%u]: %s", 261 rq->inputsize, rq->outputsize, 262 zerr ? zstd_get_error_name(zerr) : "unexpected end of stream"); 263 err = -EFSCORRUPTED; 264 break; 265 } 266 } while (outsz || out_buf.pos < out_buf.size); 267 failed: 268 if (kout) 269 kunmap_local(kout); 270 failed_zinit: 271 kunmap_local(kin); 272 /* 4. push back ZSTD stream context to the global list */ 273 spin_lock(&z_erofs_zstd_lock); 274 strm->next = z_erofs_zstd_head; 275 z_erofs_zstd_head = strm; 276 spin_unlock(&z_erofs_zstd_lock); 277 wake_up(&z_erofs_zstd_wq); 278 return err; 279 } 280