1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/zlib.h> 3 #include "compress.h" 4 5 struct z_erofs_deflate { 6 struct z_erofs_deflate *next; 7 struct z_stream_s z; 8 u8 bounce[PAGE_SIZE]; 9 }; 10 11 static DEFINE_SPINLOCK(z_erofs_deflate_lock); 12 static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms; 13 static struct z_erofs_deflate *z_erofs_deflate_head; 14 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq); 15 16 module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444); 17 18 static void z_erofs_deflate_exit(void) 19 { 20 /* there should be no running fs instance */ 21 while (z_erofs_deflate_avail_strms) { 22 struct z_erofs_deflate *strm; 23 24 spin_lock(&z_erofs_deflate_lock); 25 strm = z_erofs_deflate_head; 26 if (!strm) { 27 spin_unlock(&z_erofs_deflate_lock); 28 continue; 29 } 30 z_erofs_deflate_head = NULL; 31 spin_unlock(&z_erofs_deflate_lock); 32 33 while (strm) { 34 struct z_erofs_deflate *n = strm->next; 35 36 vfree(strm->z.workspace); 37 kfree(strm); 38 --z_erofs_deflate_avail_strms; 39 strm = n; 40 } 41 } 42 } 43 44 static int __init z_erofs_deflate_init(void) 45 { 46 /* by default, use # of possible CPUs instead */ 47 if (!z_erofs_deflate_nstrms) 48 z_erofs_deflate_nstrms = num_possible_cpus(); 49 return 0; 50 } 51 52 static int z_erofs_load_deflate_config(struct super_block *sb, 53 struct erofs_super_block *dsb, void *data, int size) 54 { 55 struct z_erofs_deflate_cfgs *dfl = data; 56 static DEFINE_MUTEX(deflate_resize_mutex); 57 static bool inited; 58 59 if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) { 60 erofs_err(sb, "invalid deflate cfgs, size=%u", size); 61 return -EINVAL; 62 } 63 64 if (dfl->windowbits > MAX_WBITS) { 65 erofs_err(sb, "unsupported windowbits %u", dfl->windowbits); 66 return -EOPNOTSUPP; 67 } 68 mutex_lock(&deflate_resize_mutex); 69 if (!inited) { 70 for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms; 71 ++z_erofs_deflate_avail_strms) { 72 struct z_erofs_deflate *strm; 73 74 strm = kzalloc(sizeof(*strm), GFP_KERNEL); 75 if (!strm) 76 goto failed; 77 /* XXX: in-kernel zlib cannot customize windowbits */ 78 strm->z.workspace = vmalloc(zlib_inflate_workspacesize()); 79 if (!strm->z.workspace) { 80 kfree(strm); 81 goto failed; 82 } 83 84 spin_lock(&z_erofs_deflate_lock); 85 strm->next = z_erofs_deflate_head; 86 z_erofs_deflate_head = strm; 87 spin_unlock(&z_erofs_deflate_lock); 88 } 89 inited = true; 90 } 91 mutex_unlock(&deflate_resize_mutex); 92 erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!"); 93 return 0; 94 failed: 95 mutex_unlock(&deflate_resize_mutex); 96 z_erofs_deflate_exit(); 97 return -ENOMEM; 98 } 99 100 static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, 101 struct page **pgpl) 102 { 103 struct super_block *sb = rq->sb; 104 struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; 105 struct z_erofs_deflate *strm; 106 int zerr, err; 107 108 /* 1. get the exact DEFLATE compressed size */ 109 dctx.kin = kmap_local_page(*rq->in); 110 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, 111 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); 112 if (err) { 113 kunmap_local(dctx.kin); 114 return err; 115 } 116 117 /* 2. get an available DEFLATE context */ 118 again: 119 spin_lock(&z_erofs_deflate_lock); 120 strm = z_erofs_deflate_head; 121 if (!strm) { 122 spin_unlock(&z_erofs_deflate_lock); 123 wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head)); 124 goto again; 125 } 126 z_erofs_deflate_head = strm->next; 127 spin_unlock(&z_erofs_deflate_lock); 128 129 /* 3. multi-call decompress */ 130 zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS); 131 if (zerr != Z_OK) { 132 err = -EIO; 133 goto failed_zinit; 134 } 135 136 rq->fillgaps = true; /* DEFLATE doesn't support NULL output buffer */ 137 strm->z.avail_in = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); 138 rq->inputsize -= strm->z.avail_in; 139 strm->z.next_in = dctx.kin + rq->pageofs_in; 140 strm->z.avail_out = 0; 141 dctx.bounce = strm->bounce; 142 143 while (1) { 144 dctx.avail_out = strm->z.avail_out; 145 dctx.inbuf_sz = strm->z.avail_in; 146 err = z_erofs_stream_switch_bufs(&dctx, 147 (void **)&strm->z.next_out, 148 (void **)&strm->z.next_in, pgpl); 149 if (err) 150 break; 151 strm->z.avail_out = dctx.avail_out; 152 strm->z.avail_in = dctx.inbuf_sz; 153 154 zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH); 155 if (zerr != Z_OK || !(rq->outputsize + strm->z.avail_out)) { 156 if (zerr == Z_OK && rq->partial_decoding) 157 break; 158 if (zerr == Z_STREAM_END && !rq->outputsize) 159 break; 160 erofs_err(sb, "failed to decompress %d in[%u] out[%u]", 161 zerr, rq->inputsize, rq->outputsize); 162 err = -EFSCORRUPTED; 163 break; 164 } 165 } 166 if (zlib_inflateEnd(&strm->z) != Z_OK && !err) 167 err = -EIO; 168 if (dctx.kout) 169 kunmap_local(dctx.kout); 170 failed_zinit: 171 kunmap_local(dctx.kin); 172 /* 4. push back DEFLATE stream context to the global list */ 173 spin_lock(&z_erofs_deflate_lock); 174 strm->next = z_erofs_deflate_head; 175 z_erofs_deflate_head = strm; 176 spin_unlock(&z_erofs_deflate_lock); 177 wake_up(&z_erofs_deflate_wq); 178 return err; 179 } 180 181 const struct z_erofs_decompressor z_erofs_deflate_decomp = { 182 .config = z_erofs_load_deflate_config, 183 .decompress = z_erofs_deflate_decompress, 184 .init = z_erofs_deflate_init, 185 .exit = z_erofs_deflate_exit, 186 .name = "deflate", 187 }; 188