1 #include <linux/device.h>
2 #include <linux/init.h>
3 #include <linux/kernel.h>
4 #include <linux/miscdevice.h>
5 #include <linux/module.h>
6 #include <linux/anon_inodes.h>
7 #include <linux/ktime.h>
8 #include <linux/hrtimer.h>
9 #include <linux/poll.h>
10
11 #include <linux/io_uring/cmd.h>
12 #include <linux/io_uring_types.h>
13 #include <uapi/linux/io_uring/mock_file.h>
14
15 struct io_mock_iocb {
16 struct kiocb *iocb;
17 struct hrtimer timer;
18 int res;
19 };
20
21 struct io_mock_file {
22 size_t size;
23 u64 rw_delay_ns;
24 bool pollable;
25 struct wait_queue_head poll_wq;
26 };
27
28 #define IO_VALID_COPY_CMD_FLAGS IORING_MOCK_COPY_FROM
29
io_copy_regbuf(struct iov_iter * reg_iter,void __user * ubuf)30 static int io_copy_regbuf(struct iov_iter *reg_iter, void __user *ubuf)
31 {
32 size_t ret, copied = 0;
33 size_t buflen = PAGE_SIZE;
34 void *tmp_buf;
35
36 tmp_buf = kzalloc(buflen, GFP_KERNEL);
37 if (!tmp_buf)
38 return -ENOMEM;
39
40 while (iov_iter_count(reg_iter)) {
41 size_t len = min(iov_iter_count(reg_iter), buflen);
42
43 if (iov_iter_rw(reg_iter) == ITER_SOURCE) {
44 ret = copy_from_iter(tmp_buf, len, reg_iter);
45 if (ret <= 0)
46 break;
47 if (copy_to_user(ubuf, tmp_buf, ret))
48 break;
49 } else {
50 if (copy_from_user(tmp_buf, ubuf, len))
51 break;
52 ret = copy_to_iter(tmp_buf, len, reg_iter);
53 if (ret <= 0)
54 break;
55 }
56 ubuf += ret;
57 copied += ret;
58 }
59
60 kfree(tmp_buf);
61 return copied;
62 }
63
io_cmd_copy_regbuf(struct io_uring_cmd * cmd,unsigned int issue_flags)64 static int io_cmd_copy_regbuf(struct io_uring_cmd *cmd, unsigned int issue_flags)
65 {
66 const struct io_uring_sqe *sqe = cmd->sqe;
67 const struct iovec __user *iovec;
68 unsigned flags, iovec_len;
69 struct iov_iter iter;
70 void __user *ubuf;
71 int dir, ret;
72
73 ubuf = u64_to_user_ptr(READ_ONCE(sqe->addr3));
74 iovec = u64_to_user_ptr(READ_ONCE(sqe->addr));
75 iovec_len = READ_ONCE(sqe->len);
76 flags = READ_ONCE(sqe->file_index);
77
78 if (unlikely(sqe->ioprio || sqe->__pad1))
79 return -EINVAL;
80 if (flags & ~IO_VALID_COPY_CMD_FLAGS)
81 return -EINVAL;
82
83 dir = (flags & IORING_MOCK_COPY_FROM) ? ITER_SOURCE : ITER_DEST;
84 ret = io_uring_cmd_import_fixed_vec(cmd, iovec, iovec_len, dir, &iter,
85 issue_flags);
86 if (ret)
87 return ret;
88 ret = io_copy_regbuf(&iter, ubuf);
89 return ret ? ret : -EFAULT;
90 }
91
io_mock_cmd(struct io_uring_cmd * cmd,unsigned int issue_flags)92 static int io_mock_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
93 {
94 switch (cmd->cmd_op) {
95 case IORING_MOCK_CMD_COPY_REGBUF:
96 return io_cmd_copy_regbuf(cmd, issue_flags);
97 }
98 return -ENOTSUPP;
99 }
100
io_mock_rw_timer_expired(struct hrtimer * timer)101 static enum hrtimer_restart io_mock_rw_timer_expired(struct hrtimer *timer)
102 {
103 struct io_mock_iocb *mio = container_of(timer, struct io_mock_iocb, timer);
104 struct kiocb *iocb = mio->iocb;
105
106 WRITE_ONCE(iocb->private, NULL);
107 iocb->ki_complete(iocb, mio->res);
108 kfree(mio);
109 return HRTIMER_NORESTART;
110 }
111
io_mock_delay_rw(struct kiocb * iocb,size_t len)112 static ssize_t io_mock_delay_rw(struct kiocb *iocb, size_t len)
113 {
114 struct io_mock_file *mf = iocb->ki_filp->private_data;
115 struct io_mock_iocb *mio;
116
117 mio = kzalloc(sizeof(*mio), GFP_KERNEL);
118 if (!mio)
119 return -ENOMEM;
120
121 mio->iocb = iocb;
122 mio->res = len;
123 hrtimer_setup(&mio->timer, io_mock_rw_timer_expired,
124 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
125 hrtimer_start(&mio->timer, ns_to_ktime(mf->rw_delay_ns),
126 HRTIMER_MODE_REL);
127 return -EIOCBQUEUED;
128 }
129
io_mock_read_iter(struct kiocb * iocb,struct iov_iter * to)130 static ssize_t io_mock_read_iter(struct kiocb *iocb, struct iov_iter *to)
131 {
132 struct io_mock_file *mf = iocb->ki_filp->private_data;
133 size_t len = iov_iter_count(to);
134 size_t nr_zeroed;
135
136 if (iocb->ki_pos + len > mf->size)
137 return -EINVAL;
138 nr_zeroed = iov_iter_zero(len, to);
139 if (!mf->rw_delay_ns || nr_zeroed != len)
140 return nr_zeroed;
141
142 return io_mock_delay_rw(iocb, len);
143 }
144
io_mock_write_iter(struct kiocb * iocb,struct iov_iter * from)145 static ssize_t io_mock_write_iter(struct kiocb *iocb, struct iov_iter *from)
146 {
147 struct io_mock_file *mf = iocb->ki_filp->private_data;
148 size_t len = iov_iter_count(from);
149
150 if (iocb->ki_pos + len > mf->size)
151 return -EINVAL;
152 if (!mf->rw_delay_ns) {
153 iov_iter_advance(from, len);
154 return len;
155 }
156
157 return io_mock_delay_rw(iocb, len);
158 }
159
io_mock_llseek(struct file * file,loff_t offset,int whence)160 static loff_t io_mock_llseek(struct file *file, loff_t offset, int whence)
161 {
162 struct io_mock_file *mf = file->private_data;
163
164 return fixed_size_llseek(file, offset, whence, mf->size);
165 }
166
io_mock_poll(struct file * file,struct poll_table_struct * pt)167 static __poll_t io_mock_poll(struct file *file, struct poll_table_struct *pt)
168 {
169 struct io_mock_file *mf = file->private_data;
170 __poll_t mask = 0;
171
172 poll_wait(file, &mf->poll_wq, pt);
173
174 mask |= EPOLLOUT | EPOLLWRNORM;
175 mask |= EPOLLIN | EPOLLRDNORM;
176 return mask;
177 }
178
io_mock_release(struct inode * inode,struct file * file)179 static int io_mock_release(struct inode *inode, struct file *file)
180 {
181 struct io_mock_file *mf = file->private_data;
182
183 kfree(mf);
184 return 0;
185 }
186
187 static const struct file_operations io_mock_fops = {
188 .owner = THIS_MODULE,
189 .release = io_mock_release,
190 .uring_cmd = io_mock_cmd,
191 .read_iter = io_mock_read_iter,
192 .write_iter = io_mock_write_iter,
193 .llseek = io_mock_llseek,
194 };
195
196 static const struct file_operations io_mock_poll_fops = {
197 .owner = THIS_MODULE,
198 .release = io_mock_release,
199 .uring_cmd = io_mock_cmd,
200 .read_iter = io_mock_read_iter,
201 .write_iter = io_mock_write_iter,
202 .llseek = io_mock_llseek,
203 .poll = io_mock_poll,
204 };
205
206 #define IO_VALID_CREATE_FLAGS (IORING_MOCK_CREATE_F_SUPPORT_NOWAIT | \
207 IORING_MOCK_CREATE_F_POLL)
208
io_create_mock_file(struct io_uring_cmd * cmd,unsigned int issue_flags)209 static int io_create_mock_file(struct io_uring_cmd *cmd, unsigned int issue_flags)
210 {
211 const struct file_operations *fops = &io_mock_fops;
212 const struct io_uring_sqe *sqe = cmd->sqe;
213 struct io_uring_mock_create mc, __user *uarg;
214 struct io_mock_file *mf = NULL;
215 struct file *file = NULL;
216 size_t uarg_size;
217 int fd = -1, ret;
218
219 /*
220 * It's a testing only driver that allows exercising edge cases
221 * that wouldn't be possible to hit otherwise.
222 */
223 add_taint(TAINT_TEST, LOCKDEP_STILL_OK);
224
225 uarg = u64_to_user_ptr(READ_ONCE(sqe->addr));
226 uarg_size = READ_ONCE(sqe->len);
227
228 if (sqe->ioprio || sqe->__pad1 || sqe->addr3 || sqe->file_index)
229 return -EINVAL;
230 if (uarg_size != sizeof(mc))
231 return -EINVAL;
232
233 memset(&mc, 0, sizeof(mc));
234 if (copy_from_user(&mc, uarg, uarg_size))
235 return -EFAULT;
236 if (!mem_is_zero(mc.__resv, sizeof(mc.__resv)))
237 return -EINVAL;
238 if (mc.flags & ~IO_VALID_CREATE_FLAGS)
239 return -EINVAL;
240 if (mc.file_size > SZ_1G)
241 return -EINVAL;
242 if (mc.rw_delay_ns > NSEC_PER_SEC)
243 return -EINVAL;
244
245 mf = kzalloc(sizeof(*mf), GFP_KERNEL_ACCOUNT);
246 if (!mf)
247 return -ENOMEM;
248
249 ret = fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
250 if (fd < 0)
251 goto fail;
252
253 init_waitqueue_head(&mf->poll_wq);
254 mf->size = mc.file_size;
255 mf->rw_delay_ns = mc.rw_delay_ns;
256 if (mc.flags & IORING_MOCK_CREATE_F_POLL) {
257 fops = &io_mock_poll_fops;
258 mf->pollable = true;
259 }
260
261 file = anon_inode_create_getfile("[io_uring_mock]", fops,
262 mf, O_RDWR | O_CLOEXEC, NULL);
263 if (IS_ERR(file)) {
264 ret = PTR_ERR(file);
265 goto fail;
266 }
267
268 file->f_mode |= FMODE_READ | FMODE_CAN_READ |
269 FMODE_WRITE | FMODE_CAN_WRITE |
270 FMODE_LSEEK;
271 if (mc.flags & IORING_MOCK_CREATE_F_SUPPORT_NOWAIT)
272 file->f_mode |= FMODE_NOWAIT;
273
274 mc.out_fd = fd;
275 if (copy_to_user(uarg, &mc, uarg_size)) {
276 fput(file);
277 ret = -EFAULT;
278 goto fail;
279 }
280
281 fd_install(fd, file);
282 return 0;
283 fail:
284 if (fd >= 0)
285 put_unused_fd(fd);
286 kfree(mf);
287 return ret;
288 }
289
io_probe_mock(struct io_uring_cmd * cmd)290 static int io_probe_mock(struct io_uring_cmd *cmd)
291 {
292 const struct io_uring_sqe *sqe = cmd->sqe;
293 struct io_uring_mock_probe mp, __user *uarg;
294 size_t uarg_size;
295
296 uarg = u64_to_user_ptr(READ_ONCE(sqe->addr));
297 uarg_size = READ_ONCE(sqe->len);
298
299 if (sqe->ioprio || sqe->__pad1 || sqe->addr3 || sqe->file_index ||
300 uarg_size != sizeof(mp))
301 return -EINVAL;
302
303 memset(&mp, 0, sizeof(mp));
304 if (copy_from_user(&mp, uarg, uarg_size))
305 return -EFAULT;
306 if (!mem_is_zero(&mp, sizeof(mp)))
307 return -EINVAL;
308
309 mp.features = IORING_MOCK_FEAT_END;
310
311 if (copy_to_user(uarg, &mp, uarg_size))
312 return -EFAULT;
313 return 0;
314 }
315
iou_mock_mgr_cmd(struct io_uring_cmd * cmd,unsigned int issue_flags)316 static int iou_mock_mgr_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
317 {
318 if (!capable(CAP_SYS_ADMIN))
319 return -EPERM;
320
321 switch (cmd->cmd_op) {
322 case IORING_MOCK_MGR_CMD_PROBE:
323 return io_probe_mock(cmd);
324 case IORING_MOCK_MGR_CMD_CREATE:
325 return io_create_mock_file(cmd, issue_flags);
326 }
327 return -EOPNOTSUPP;
328 }
329
330 static const struct file_operations iou_mock_dev_fops = {
331 .owner = THIS_MODULE,
332 .uring_cmd = iou_mock_mgr_cmd,
333 };
334
335 static struct miscdevice iou_mock_miscdev = {
336 .minor = MISC_DYNAMIC_MINOR,
337 .name = "io_uring_mock",
338 .fops = &iou_mock_dev_fops,
339 };
340
io_mock_init(void)341 static int __init io_mock_init(void)
342 {
343 int ret;
344
345 ret = misc_register(&iou_mock_miscdev);
346 if (ret < 0) {
347 pr_err("Could not initialize io_uring mock device\n");
348 return ret;
349 }
350 return 0;
351 }
352
io_mock_exit(void)353 static void __exit io_mock_exit(void)
354 {
355 misc_deregister(&iou_mock_miscdev);
356 }
357
358 module_init(io_mock_init)
359 module_exit(io_mock_exit)
360
361 MODULE_AUTHOR("Pavel Begunkov <asml.silence@gmail.com>");
362 MODULE_DESCRIPTION("io_uring mock file");
363 MODULE_LICENSE("GPL");
364