1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/zstd.h> 3 #include "compress.h" 4 5 struct z_erofs_zstd { 6 struct z_erofs_zstd *next; 7 u8 bounce[PAGE_SIZE]; 8 void *wksp; 9 unsigned int wkspsz; 10 }; 11 12 static DEFINE_SPINLOCK(z_erofs_zstd_lock); 13 static unsigned int z_erofs_zstd_max_dictsize; 14 static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms; 15 static struct z_erofs_zstd *z_erofs_zstd_head; 16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq); 17 18 module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444); 19 20 static struct z_erofs_zstd *z_erofs_isolate_strms(bool all) 21 { 22 struct z_erofs_zstd *strm; 23 24 again: 25 spin_lock(&z_erofs_zstd_lock); 26 strm = z_erofs_zstd_head; 27 if (!strm) { 28 spin_unlock(&z_erofs_zstd_lock); 29 wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head)); 30 goto again; 31 } 32 z_erofs_zstd_head = all ? NULL : strm->next; 33 spin_unlock(&z_erofs_zstd_lock); 34 return strm; 35 } 36 37 static void z_erofs_zstd_exit(void) 38 { 39 while (z_erofs_zstd_avail_strms) { 40 struct z_erofs_zstd *strm, *n; 41 42 for (strm = z_erofs_isolate_strms(true); strm; strm = n) { 43 n = strm->next; 44 45 kvfree(strm->wksp); 46 kfree(strm); 47 --z_erofs_zstd_avail_strms; 48 } 49 } 50 } 51 52 static int __init z_erofs_zstd_init(void) 53 { 54 /* by default, use # of possible CPUs instead */ 55 if (!z_erofs_zstd_nstrms) 56 z_erofs_zstd_nstrms = num_possible_cpus(); 57 58 for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms; 59 ++z_erofs_zstd_avail_strms) { 60 struct z_erofs_zstd *strm; 61 62 strm = kzalloc(sizeof(*strm), GFP_KERNEL); 63 if (!strm) { 64 z_erofs_zstd_exit(); 65 return -ENOMEM; 66 } 67 spin_lock(&z_erofs_zstd_lock); 68 strm->next = z_erofs_zstd_head; 69 z_erofs_zstd_head = strm; 70 spin_unlock(&z_erofs_zstd_lock); 71 } 72 return 0; 73 } 74 75 static int z_erofs_load_zstd_config(struct super_block *sb, 76 struct erofs_super_block *dsb, void *data, int size) 77 { 78 static DEFINE_MUTEX(zstd_resize_mutex); 79 struct z_erofs_zstd_cfgs *zstd = data; 80 unsigned int dict_size, wkspsz; 81 struct z_erofs_zstd *strm, *head = NULL; 82 void *wksp; 83 84 if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) { 85 erofs_err(sb, "unsupported zstd format, size=%u", size); 86 return -EINVAL; 87 } 88 89 if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) { 90 erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog); 91 return -EINVAL; 92 } 93 dict_size = 1U << (zstd->windowlog + 10); 94 95 /* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */ 96 mutex_lock(&zstd_resize_mutex); 97 if (z_erofs_zstd_max_dictsize >= dict_size) { 98 mutex_unlock(&zstd_resize_mutex); 99 return 0; 100 } 101 102 /* 1. collect/isolate all streams for the following check */ 103 while (z_erofs_zstd_avail_strms) { 104 struct z_erofs_zstd *n; 105 106 for (strm = z_erofs_isolate_strms(true); strm; strm = n) { 107 n = strm->next; 108 strm->next = head; 109 head = strm; 110 --z_erofs_zstd_avail_strms; 111 } 112 } 113 114 /* 2. walk each isolated stream and grow max dict_size if needed */ 115 wkspsz = zstd_dstream_workspace_bound(dict_size); 116 for (strm = head; strm; strm = strm->next) { 117 wksp = kvmalloc(wkspsz, GFP_KERNEL); 118 if (!wksp) 119 break; 120 kvfree(strm->wksp); 121 strm->wksp = wksp; 122 strm->wkspsz = wkspsz; 123 } 124 125 /* 3. push back all to the global list and update max dict_size */ 126 spin_lock(&z_erofs_zstd_lock); 127 DBG_BUGON(z_erofs_zstd_head); 128 z_erofs_zstd_head = head; 129 spin_unlock(&z_erofs_zstd_lock); 130 z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms; 131 wake_up_all(&z_erofs_zstd_wq); 132 if (!strm) 133 z_erofs_zstd_max_dictsize = dict_size; 134 mutex_unlock(&zstd_resize_mutex); 135 return strm ? -ENOMEM : 0; 136 } 137 138 static int z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq, 139 struct page **pgpl) 140 { 141 struct super_block *sb = rq->sb; 142 struct z_erofs_stream_dctx dctx = { 143 .rq = rq, 144 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT, 145 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize) 146 >> PAGE_SHIFT, 147 .no = -1, .ni = 0, 148 }; 149 zstd_in_buffer in_buf = { NULL, 0, 0 }; 150 zstd_out_buffer out_buf = { NULL, 0, 0 }; 151 struct z_erofs_zstd *strm; 152 zstd_dstream *stream; 153 int zerr, err; 154 155 /* 1. get the exact compressed size */ 156 dctx.kin = kmap_local_page(*rq->in); 157 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, 158 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); 159 if (err) { 160 kunmap_local(dctx.kin); 161 return err; 162 } 163 164 /* 2. get an available ZSTD context */ 165 strm = z_erofs_isolate_strms(false); 166 167 /* 3. multi-call decompress */ 168 stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz); 169 if (!stream) { 170 err = -EIO; 171 goto failed_zinit; 172 } 173 174 rq->fillgaps = true; /* ZSTD doesn't support NULL output buffer */ 175 in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in); 176 rq->inputsize -= in_buf.size; 177 in_buf.src = dctx.kin + rq->pageofs_in; 178 dctx.bounce = strm->bounce; 179 180 do { 181 dctx.avail_out = out_buf.size - out_buf.pos; 182 dctx.inbuf_sz = in_buf.size; 183 dctx.inbuf_pos = in_buf.pos; 184 err = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst, 185 (void **)&in_buf.src, pgpl); 186 if (err) 187 break; 188 189 if (out_buf.size == out_buf.pos) { 190 out_buf.size = dctx.avail_out; 191 out_buf.pos = 0; 192 } 193 in_buf.size = dctx.inbuf_sz; 194 in_buf.pos = dctx.inbuf_pos; 195 196 zerr = zstd_decompress_stream(stream, &out_buf, &in_buf); 197 if (zstd_is_error(zerr) || (!zerr && rq->outputsize)) { 198 erofs_err(sb, "failed to decompress in[%u] out[%u]: %s", 199 rq->inputsize, rq->outputsize, 200 zerr ? zstd_get_error_name(zerr) : "unexpected end of stream"); 201 err = -EFSCORRUPTED; 202 break; 203 } 204 } while (rq->outputsize || out_buf.pos < out_buf.size); 205 206 if (dctx.kout) 207 kunmap_local(dctx.kout); 208 failed_zinit: 209 kunmap_local(dctx.kin); 210 /* 4. push back ZSTD stream context to the global list */ 211 spin_lock(&z_erofs_zstd_lock); 212 strm->next = z_erofs_zstd_head; 213 z_erofs_zstd_head = strm; 214 spin_unlock(&z_erofs_zstd_lock); 215 wake_up(&z_erofs_zstd_wq); 216 return err; 217 } 218 219 const struct z_erofs_decompressor z_erofs_zstd_decomp = { 220 .config = z_erofs_load_zstd_config, 221 .decompress = z_erofs_zstd_decompress, 222 .init = z_erofs_zstd_init, 223 .exit = z_erofs_zstd_exit, 224 .name = "zstd", 225 }; 226