1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/zlib.h>
3 #include "compress.h"
4
5 struct z_erofs_deflate {
6 struct z_erofs_deflate *next;
7 struct z_stream_s z;
8 u8 bounce[PAGE_SIZE];
9 };
10
11 static DEFINE_SPINLOCK(z_erofs_deflate_lock);
12 static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms;
13 static struct z_erofs_deflate *z_erofs_deflate_head;
14 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq);
15
16 module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444);
17
z_erofs_deflate_exit(void)18 static void z_erofs_deflate_exit(void)
19 {
20 /* there should be no running fs instance */
21 while (z_erofs_deflate_avail_strms) {
22 struct z_erofs_deflate *strm;
23
24 spin_lock(&z_erofs_deflate_lock);
25 strm = z_erofs_deflate_head;
26 if (!strm) {
27 spin_unlock(&z_erofs_deflate_lock);
28 continue;
29 }
30 z_erofs_deflate_head = NULL;
31 spin_unlock(&z_erofs_deflate_lock);
32
33 while (strm) {
34 struct z_erofs_deflate *n = strm->next;
35
36 vfree(strm->z.workspace);
37 kfree(strm);
38 --z_erofs_deflate_avail_strms;
39 strm = n;
40 }
41 }
42 }
43
z_erofs_deflate_init(void)44 static int __init z_erofs_deflate_init(void)
45 {
46 /* by default, use # of possible CPUs instead */
47 if (!z_erofs_deflate_nstrms)
48 z_erofs_deflate_nstrms = num_possible_cpus();
49 return 0;
50 }
51
z_erofs_load_deflate_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)52 static int z_erofs_load_deflate_config(struct super_block *sb,
53 struct erofs_super_block *dsb, void *data, int size)
54 {
55 struct z_erofs_deflate_cfgs *dfl = data;
56 static DEFINE_MUTEX(deflate_resize_mutex);
57 static bool inited;
58
59 if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) {
60 erofs_err(sb, "invalid deflate cfgs, size=%u", size);
61 return -EINVAL;
62 }
63
64 if (dfl->windowbits > MAX_WBITS) {
65 erofs_err(sb, "unsupported windowbits %u", dfl->windowbits);
66 return -EOPNOTSUPP;
67 }
68 mutex_lock(&deflate_resize_mutex);
69 if (!inited) {
70 for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms;
71 ++z_erofs_deflate_avail_strms) {
72 struct z_erofs_deflate *strm;
73
74 strm = kzalloc(sizeof(*strm), GFP_KERNEL);
75 if (!strm)
76 goto failed;
77 /* XXX: in-kernel zlib cannot customize windowbits */
78 strm->z.workspace = vmalloc(zlib_inflate_workspacesize());
79 if (!strm->z.workspace) {
80 kfree(strm);
81 goto failed;
82 }
83
84 spin_lock(&z_erofs_deflate_lock);
85 strm->next = z_erofs_deflate_head;
86 z_erofs_deflate_head = strm;
87 spin_unlock(&z_erofs_deflate_lock);
88 }
89 inited = true;
90 }
91 mutex_unlock(&deflate_resize_mutex);
92 erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!");
93 return 0;
94 failed:
95 mutex_unlock(&deflate_resize_mutex);
96 z_erofs_deflate_exit();
97 return -ENOMEM;
98 }
99
z_erofs_deflate_decompress(struct z_erofs_decompress_req * rq,struct page ** pgpl)100 static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
101 struct page **pgpl)
102 {
103 struct super_block *sb = rq->sb;
104 struct z_erofs_stream_dctx dctx = {
105 .rq = rq,
106 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,
107 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)
108 >> PAGE_SHIFT,
109 .no = -1, .ni = 0,
110 };
111 struct z_erofs_deflate *strm;
112 int zerr, err;
113
114 /* 1. get the exact DEFLATE compressed size */
115 dctx.kin = kmap_local_page(*rq->in);
116 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
117 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
118 if (err) {
119 kunmap_local(dctx.kin);
120 return err;
121 }
122
123 /* 2. get an available DEFLATE context */
124 again:
125 spin_lock(&z_erofs_deflate_lock);
126 strm = z_erofs_deflate_head;
127 if (!strm) {
128 spin_unlock(&z_erofs_deflate_lock);
129 wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head));
130 goto again;
131 }
132 z_erofs_deflate_head = strm->next;
133 spin_unlock(&z_erofs_deflate_lock);
134
135 /* 3. multi-call decompress */
136 zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS);
137 if (zerr != Z_OK) {
138 err = -EIO;
139 goto failed_zinit;
140 }
141
142 rq->fillgaps = true; /* DEFLATE doesn't support NULL output buffer */
143 strm->z.avail_in = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);
144 rq->inputsize -= strm->z.avail_in;
145 strm->z.next_in = dctx.kin + rq->pageofs_in;
146 strm->z.avail_out = 0;
147 dctx.bounce = strm->bounce;
148
149 while (1) {
150 dctx.avail_out = strm->z.avail_out;
151 dctx.inbuf_sz = strm->z.avail_in;
152 err = z_erofs_stream_switch_bufs(&dctx,
153 (void **)&strm->z.next_out,
154 (void **)&strm->z.next_in, pgpl);
155 if (err)
156 break;
157 strm->z.avail_out = dctx.avail_out;
158 strm->z.avail_in = dctx.inbuf_sz;
159
160 zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH);
161 if (zerr != Z_OK || !(rq->outputsize + strm->z.avail_out)) {
162 if (zerr == Z_OK && rq->partial_decoding)
163 break;
164 if (zerr == Z_STREAM_END && !rq->outputsize)
165 break;
166 erofs_err(sb, "failed to decompress %d in[%u] out[%u]",
167 zerr, rq->inputsize, rq->outputsize);
168 err = -EFSCORRUPTED;
169 break;
170 }
171 }
172 if (zlib_inflateEnd(&strm->z) != Z_OK && !err)
173 err = -EIO;
174 if (dctx.kout)
175 kunmap_local(dctx.kout);
176 failed_zinit:
177 kunmap_local(dctx.kin);
178 /* 4. push back DEFLATE stream context to the global list */
179 spin_lock(&z_erofs_deflate_lock);
180 strm->next = z_erofs_deflate_head;
181 z_erofs_deflate_head = strm;
182 spin_unlock(&z_erofs_deflate_lock);
183 wake_up(&z_erofs_deflate_wq);
184 return err;
185 }
186
187 const struct z_erofs_decompressor z_erofs_deflate_decomp = {
188 .config = z_erofs_load_deflate_config,
189 .decompress = z_erofs_deflate_decompress,
190 .init = z_erofs_deflate_init,
191 .exit = z_erofs_deflate_exit,
192 .name = "deflate",
193 };
194