1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/zlib.h> 3 #include "compress.h" 4 5 struct z_erofs_deflate { 6 struct z_erofs_deflate *next; 7 struct z_stream_s z; 8 u8 bounce[PAGE_SIZE]; 9 }; 10 11 static DEFINE_SPINLOCK(z_erofs_deflate_lock); 12 static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms; 13 static struct z_erofs_deflate *z_erofs_deflate_head; 14 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq); 15 16 module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444); 17 18 static void z_erofs_deflate_exit(void) 19 { 20 /* there should be no running fs instance */ 21 while (z_erofs_deflate_avail_strms) { 22 struct z_erofs_deflate *strm; 23 24 spin_lock(&z_erofs_deflate_lock); 25 strm = z_erofs_deflate_head; 26 if (!strm) { 27 spin_unlock(&z_erofs_deflate_lock); 28 continue; 29 } 30 z_erofs_deflate_head = NULL; 31 spin_unlock(&z_erofs_deflate_lock); 32 33 while (strm) { 34 struct z_erofs_deflate *n = strm->next; 35 36 vfree(strm->z.workspace); 37 kfree(strm); 38 --z_erofs_deflate_avail_strms; 39 strm = n; 40 } 41 } 42 } 43 44 static int __init z_erofs_deflate_init(void) 45 { 46 /* by default, use # of possible CPUs instead */ 47 if (!z_erofs_deflate_nstrms) 48 z_erofs_deflate_nstrms = num_possible_cpus(); 49 return 0; 50 } 51 52 static int z_erofs_load_deflate_config(struct super_block *sb, 53 struct erofs_super_block *dsb, void *data, int size) 54 { 55 struct z_erofs_deflate_cfgs *dfl = data; 56 static DEFINE_MUTEX(deflate_resize_mutex); 57 static bool inited; 58 59 if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) { 60 erofs_err(sb, "invalid deflate cfgs, size=%u", size); 61 return -EINVAL; 62 } 63 64 if (dfl->windowbits > MAX_WBITS) { 65 erofs_err(sb, "unsupported windowbits %u", dfl->windowbits); 66 return -EOPNOTSUPP; 67 } 68 mutex_lock(&deflate_resize_mutex); 69 if (!inited) { 70 for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms; 71 ++z_erofs_deflate_avail_strms) { 72 struct z_erofs_deflate *strm; 73 74 strm = kzalloc(sizeof(*strm), GFP_KERNEL); 75 if (!strm) 76 goto failed; 77 /* XXX: in-kernel zlib cannot customize windowbits */ 78 strm->z.workspace = vmalloc(zlib_inflate_workspacesize()); 79 if (!strm->z.workspace) { 80 kfree(strm); 81 goto failed; 82 } 83 84 spin_lock(&z_erofs_deflate_lock); 85 strm->next = z_erofs_deflate_head; 86 z_erofs_deflate_head = strm; 87 spin_unlock(&z_erofs_deflate_lock); 88 } 89 inited = true; 90 } 91 mutex_unlock(&deflate_resize_mutex); 92 erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!"); 93 return 0; 94 failed: 95 mutex_unlock(&deflate_resize_mutex); 96 z_erofs_deflate_exit(); 97 return -ENOMEM; 98 } 99 100 static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, 101 struct page **pgpl) 102 { 103 struct super_block *sb = rq->sb; 104 struct z_erofs_stream_dctx dctx = { 105 .rq = rq, 106 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT, 107 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize) 108 >> PAGE_SHIFT, 109 .no = -1, .ni = 0, 110 }; 111 struct z_erofs_deflate *strm; 112 int zerr, err; 113 114 /* 1. get the exact DEFLATE compressed size */ 115 dctx.kin = kmap_local_page(*rq->in); 116 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, 117 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); 118 if (err) { 119 kunmap_local(dctx.kin); 120 return err; 121 } 122 123 /* 2. get an available DEFLATE context */ 124 again: 125 spin_lock(&z_erofs_deflate_lock); 126 strm = z_erofs_deflate_head; 127 if (!strm) { 128 spin_unlock(&z_erofs_deflate_lock); 129 wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head)); 130 goto again; 131 } 132 z_erofs_deflate_head = strm->next; 133 spin_unlock(&z_erofs_deflate_lock); 134 135 /* 3. multi-call decompress */ 136 zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS); 137 if (zerr != Z_OK) { 138 err = -EIO; 139 goto failed_zinit; 140 } 141 142 rq->fillgaps = true; /* DEFLATE doesn't support NULL output buffer */ 143 strm->z.avail_in = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); 144 rq->inputsize -= strm->z.avail_in; 145 strm->z.next_in = dctx.kin + rq->pageofs_in; 146 strm->z.avail_out = 0; 147 dctx.bounce = strm->bounce; 148 149 while (1) { 150 dctx.avail_out = strm->z.avail_out; 151 dctx.inbuf_sz = strm->z.avail_in; 152 err = z_erofs_stream_switch_bufs(&dctx, 153 (void **)&strm->z.next_out, 154 (void **)&strm->z.next_in, pgpl); 155 if (err) 156 break; 157 strm->z.avail_out = dctx.avail_out; 158 strm->z.avail_in = dctx.inbuf_sz; 159 160 zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH); 161 if (zerr != Z_OK || !(rq->outputsize + strm->z.avail_out)) { 162 if (zerr == Z_OK && rq->partial_decoding) 163 break; 164 if (zerr == Z_STREAM_END && !rq->outputsize) 165 break; 166 erofs_err(sb, "failed to decompress %d in[%u] out[%u]", 167 zerr, rq->inputsize, rq->outputsize); 168 err = -EFSCORRUPTED; 169 break; 170 } 171 } 172 if (zlib_inflateEnd(&strm->z) != Z_OK && !err) 173 err = -EIO; 174 if (dctx.kout) 175 kunmap_local(dctx.kout); 176 failed_zinit: 177 kunmap_local(dctx.kin); 178 /* 4. push back DEFLATE stream context to the global list */ 179 spin_lock(&z_erofs_deflate_lock); 180 strm->next = z_erofs_deflate_head; 181 z_erofs_deflate_head = strm; 182 spin_unlock(&z_erofs_deflate_lock); 183 wake_up(&z_erofs_deflate_wq); 184 return err; 185 } 186 187 const struct z_erofs_decompressor z_erofs_deflate_decomp = { 188 .config = z_erofs_load_deflate_config, 189 .decompress = z_erofs_deflate_decompress, 190 .init = z_erofs_deflate_init, 191 .exit = z_erofs_deflate_exit, 192 .name = "deflate", 193 }; 194