1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/zlib.h> 3 #include "compress.h" 4 5 struct z_erofs_deflate { 6 struct z_erofs_deflate *next; 7 struct z_stream_s z; 8 u8 bounce[PAGE_SIZE]; 9 }; 10 11 static DEFINE_SPINLOCK(z_erofs_deflate_lock); 12 static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms; 13 static struct z_erofs_deflate *z_erofs_deflate_head; 14 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq); 15 16 module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444); 17 18 static void z_erofs_deflate_exit(void) 19 { 20 /* there should be no running fs instance */ 21 while (z_erofs_deflate_avail_strms) { 22 struct z_erofs_deflate *strm; 23 24 spin_lock(&z_erofs_deflate_lock); 25 strm = z_erofs_deflate_head; 26 if (!strm) { 27 spin_unlock(&z_erofs_deflate_lock); 28 continue; 29 } 30 z_erofs_deflate_head = NULL; 31 spin_unlock(&z_erofs_deflate_lock); 32 33 while (strm) { 34 struct z_erofs_deflate *n = strm->next; 35 36 vfree(strm->z.workspace); 37 kfree(strm); 38 --z_erofs_deflate_avail_strms; 39 strm = n; 40 } 41 } 42 } 43 44 static int __init z_erofs_deflate_init(void) 45 { 46 /* by default, use # of possible CPUs instead */ 47 if (!z_erofs_deflate_nstrms) 48 z_erofs_deflate_nstrms = num_possible_cpus(); 49 return 0; 50 } 51 52 static int z_erofs_load_deflate_config(struct super_block *sb, 53 struct erofs_super_block *dsb, void *data, int size) 54 { 55 struct z_erofs_deflate_cfgs *dfl = data; 56 static DEFINE_MUTEX(deflate_resize_mutex); 57 static bool inited; 58 59 if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) { 60 erofs_err(sb, "invalid deflate cfgs, size=%u", size); 61 return -EINVAL; 62 } 63 64 if (dfl->windowbits > MAX_WBITS) { 65 erofs_err(sb, "unsupported windowbits %u", dfl->windowbits); 66 return -EOPNOTSUPP; 67 } 68 mutex_lock(&deflate_resize_mutex); 69 if (!inited) { 70 for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms; 71 ++z_erofs_deflate_avail_strms) { 72 struct z_erofs_deflate *strm; 73 74 strm = kzalloc(sizeof(*strm), GFP_KERNEL); 75 if (!strm) 76 goto failed; 77 /* XXX: in-kernel zlib cannot customize windowbits */ 78 strm->z.workspace = vmalloc(zlib_inflate_workspacesize()); 79 if (!strm->z.workspace) { 80 kfree(strm); 81 goto failed; 82 } 83 84 spin_lock(&z_erofs_deflate_lock); 85 strm->next = z_erofs_deflate_head; 86 z_erofs_deflate_head = strm; 87 spin_unlock(&z_erofs_deflate_lock); 88 } 89 inited = true; 90 } 91 mutex_unlock(&deflate_resize_mutex); 92 erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!"); 93 return 0; 94 failed: 95 mutex_unlock(&deflate_resize_mutex); 96 z_erofs_deflate_exit(); 97 return -ENOMEM; 98 } 99 100 static const char *__z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, 101 struct page **pgpl) 102 { 103 struct super_block *sb = rq->sb; 104 struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; 105 struct z_erofs_deflate *strm; 106 const char *reason; 107 int zerr; 108 109 /* 1. get the exact DEFLATE compressed size */ 110 dctx.kin = kmap_local_page(*rq->in); 111 reason = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, 112 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); 113 if (reason) { 114 kunmap_local(dctx.kin); 115 return reason; 116 } 117 118 /* 2. get an available DEFLATE context */ 119 again: 120 spin_lock(&z_erofs_deflate_lock); 121 strm = z_erofs_deflate_head; 122 if (!strm) { 123 spin_unlock(&z_erofs_deflate_lock); 124 wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head)); 125 goto again; 126 } 127 z_erofs_deflate_head = strm->next; 128 spin_unlock(&z_erofs_deflate_lock); 129 130 /* 3. multi-call decompress */ 131 zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS); 132 if (zerr != Z_OK) { 133 reason = ERR_PTR(-EINVAL); 134 goto failed_zinit; 135 } 136 137 rq->fillgaps = true; /* DEFLATE doesn't support NULL output buffer */ 138 strm->z.avail_in = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); 139 rq->inputsize -= strm->z.avail_in; 140 strm->z.next_in = dctx.kin + rq->pageofs_in; 141 strm->z.avail_out = 0; 142 dctx.bounce = strm->bounce; 143 144 while (1) { 145 dctx.avail_out = strm->z.avail_out; 146 dctx.inbuf_sz = strm->z.avail_in; 147 reason = z_erofs_stream_switch_bufs(&dctx, 148 (void **)&strm->z.next_out, 149 (void **)&strm->z.next_in, pgpl); 150 if (reason) 151 break; 152 strm->z.avail_out = dctx.avail_out; 153 strm->z.avail_in = dctx.inbuf_sz; 154 155 zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH); 156 if (zerr != Z_OK || !(rq->outputsize + strm->z.avail_out)) { 157 if (zerr == Z_OK && rq->partial_decoding) 158 break; 159 if (zerr == Z_STREAM_END && !rq->outputsize) 160 break; 161 reason = (zerr == Z_DATA_ERROR ? 162 "corrupted compressed data" : 163 "unexpected end of stream"); 164 break; 165 } 166 } 167 if (zlib_inflateEnd(&strm->z) != Z_OK && !reason) 168 reason = ERR_PTR(-EIO); 169 if (dctx.kout) 170 kunmap_local(dctx.kout); 171 failed_zinit: 172 kunmap_local(dctx.kin); 173 /* 4. push back DEFLATE stream context to the global list */ 174 spin_lock(&z_erofs_deflate_lock); 175 strm->next = z_erofs_deflate_head; 176 z_erofs_deflate_head = strm; 177 spin_unlock(&z_erofs_deflate_lock); 178 wake_up(&z_erofs_deflate_wq); 179 return reason; 180 } 181 182 static const char *z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, 183 struct page **pgpl) 184 { 185 #ifdef CONFIG_EROFS_FS_ZIP_ACCEL 186 int err; 187 188 if (!rq->partial_decoding) { 189 err = z_erofs_crypto_decompress(rq, pgpl); 190 if (err != -EOPNOTSUPP) 191 return ERR_PTR(err); 192 193 } 194 #endif 195 return __z_erofs_deflate_decompress(rq, pgpl); 196 } 197 198 const struct z_erofs_decompressor z_erofs_deflate_decomp = { 199 .config = z_erofs_load_deflate_config, 200 .decompress = z_erofs_deflate_decompress, 201 .init = z_erofs_deflate_init, 202 .exit = z_erofs_deflate_exit, 203 .name = "deflate", 204 }; 205