xref: /linux/fs/read_write.c (revision 827634added7f38b7d724cab1dccdb2b004c13c3)
1 /*
2  *  linux/fs/read_write.c
3  *
4  *  Copyright (C) 1991, 1992  Linus Torvalds
5  */
6 
7 #include <linux/slab.h>
8 #include <linux/stat.h>
9 #include <linux/fcntl.h>
10 #include <linux/file.h>
11 #include <linux/uio.h>
12 #include <linux/fsnotify.h>
13 #include <linux/security.h>
14 #include <linux/export.h>
15 #include <linux/syscalls.h>
16 #include <linux/pagemap.h>
17 #include <linux/splice.h>
18 #include <linux/compat.h>
19 #include "internal.h"
20 
21 #include <asm/uaccess.h>
22 #include <asm/unistd.h>
23 
24 typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *);
25 typedef ssize_t (*iter_fn_t)(struct kiocb *, struct iov_iter *);
26 
27 const struct file_operations generic_ro_fops = {
28 	.llseek		= generic_file_llseek,
29 	.read_iter	= generic_file_read_iter,
30 	.mmap		= generic_file_readonly_mmap,
31 	.splice_read	= generic_file_splice_read,
32 };
33 
34 EXPORT_SYMBOL(generic_ro_fops);
35 
36 static inline int unsigned_offsets(struct file *file)
37 {
38 	return file->f_mode & FMODE_UNSIGNED_OFFSET;
39 }
40 
41 /**
42  * vfs_setpos - update the file offset for lseek
43  * @file:	file structure in question
44  * @offset:	file offset to seek to
45  * @maxsize:	maximum file size
46  *
47  * This is a low-level filesystem helper for updating the file offset to
48  * the value specified by @offset if the given offset is valid and it is
49  * not equal to the current file offset.
50  *
51  * Return the specified offset on success and -EINVAL on invalid offset.
52  */
53 loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
54 {
55 	if (offset < 0 && !unsigned_offsets(file))
56 		return -EINVAL;
57 	if (offset > maxsize)
58 		return -EINVAL;
59 
60 	if (offset != file->f_pos) {
61 		file->f_pos = offset;
62 		file->f_version = 0;
63 	}
64 	return offset;
65 }
66 EXPORT_SYMBOL(vfs_setpos);
67 
68 /**
69  * generic_file_llseek_size - generic llseek implementation for regular files
70  * @file:	file structure to seek on
71  * @offset:	file offset to seek to
72  * @whence:	type of seek
73  * @size:	max size of this file in file system
74  * @eof:	offset used for SEEK_END position
75  *
76  * This is a variant of generic_file_llseek that allows passing in a custom
77  * maximum file size and a custom EOF position, for e.g. hashed directories
78  *
79  * Synchronization:
80  * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
81  * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
82  * read/writes behave like SEEK_SET against seeks.
83  */
84 loff_t
85 generic_file_llseek_size(struct file *file, loff_t offset, int whence,
86 		loff_t maxsize, loff_t eof)
87 {
88 	switch (whence) {
89 	case SEEK_END:
90 		offset += eof;
91 		break;
92 	case SEEK_CUR:
93 		/*
94 		 * Here we special-case the lseek(fd, 0, SEEK_CUR)
95 		 * position-querying operation.  Avoid rewriting the "same"
96 		 * f_pos value back to the file because a concurrent read(),
97 		 * write() or lseek() might have altered it
98 		 */
99 		if (offset == 0)
100 			return file->f_pos;
101 		/*
102 		 * f_lock protects against read/modify/write race with other
103 		 * SEEK_CURs. Note that parallel writes and reads behave
104 		 * like SEEK_SET.
105 		 */
106 		spin_lock(&file->f_lock);
107 		offset = vfs_setpos(file, file->f_pos + offset, maxsize);
108 		spin_unlock(&file->f_lock);
109 		return offset;
110 	case SEEK_DATA:
111 		/*
112 		 * In the generic case the entire file is data, so as long as
113 		 * offset isn't at the end of the file then the offset is data.
114 		 */
115 		if (offset >= eof)
116 			return -ENXIO;
117 		break;
118 	case SEEK_HOLE:
119 		/*
120 		 * There is a virtual hole at the end of the file, so as long as
121 		 * offset isn't i_size or larger, return i_size.
122 		 */
123 		if (offset >= eof)
124 			return -ENXIO;
125 		offset = eof;
126 		break;
127 	}
128 
129 	return vfs_setpos(file, offset, maxsize);
130 }
131 EXPORT_SYMBOL(generic_file_llseek_size);
132 
133 /**
134  * generic_file_llseek - generic llseek implementation for regular files
135  * @file:	file structure to seek on
136  * @offset:	file offset to seek to
137  * @whence:	type of seek
138  *
139  * This is a generic implemenation of ->llseek useable for all normal local
140  * filesystems.  It just updates the file offset to the value specified by
141  * @offset and @whence.
142  */
143 loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
144 {
145 	struct inode *inode = file->f_mapping->host;
146 
147 	return generic_file_llseek_size(file, offset, whence,
148 					inode->i_sb->s_maxbytes,
149 					i_size_read(inode));
150 }
151 EXPORT_SYMBOL(generic_file_llseek);
152 
153 /**
154  * fixed_size_llseek - llseek implementation for fixed-sized devices
155  * @file:	file structure to seek on
156  * @offset:	file offset to seek to
157  * @whence:	type of seek
158  * @size:	size of the file
159  *
160  */
161 loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
162 {
163 	switch (whence) {
164 	case SEEK_SET: case SEEK_CUR: case SEEK_END:
165 		return generic_file_llseek_size(file, offset, whence,
166 						size, size);
167 	default:
168 		return -EINVAL;
169 	}
170 }
171 EXPORT_SYMBOL(fixed_size_llseek);
172 
173 /**
174  * noop_llseek - No Operation Performed llseek implementation
175  * @file:	file structure to seek on
176  * @offset:	file offset to seek to
177  * @whence:	type of seek
178  *
179  * This is an implementation of ->llseek useable for the rare special case when
180  * userspace expects the seek to succeed but the (device) file is actually not
181  * able to perform the seek. In this case you use noop_llseek() instead of
182  * falling back to the default implementation of ->llseek.
183  */
184 loff_t noop_llseek(struct file *file, loff_t offset, int whence)
185 {
186 	return file->f_pos;
187 }
188 EXPORT_SYMBOL(noop_llseek);
189 
190 loff_t no_llseek(struct file *file, loff_t offset, int whence)
191 {
192 	return -ESPIPE;
193 }
194 EXPORT_SYMBOL(no_llseek);
195 
196 loff_t default_llseek(struct file *file, loff_t offset, int whence)
197 {
198 	struct inode *inode = file_inode(file);
199 	loff_t retval;
200 
201 	mutex_lock(&inode->i_mutex);
202 	switch (whence) {
203 		case SEEK_END:
204 			offset += i_size_read(inode);
205 			break;
206 		case SEEK_CUR:
207 			if (offset == 0) {
208 				retval = file->f_pos;
209 				goto out;
210 			}
211 			offset += file->f_pos;
212 			break;
213 		case SEEK_DATA:
214 			/*
215 			 * In the generic case the entire file is data, so as
216 			 * long as offset isn't at the end of the file then the
217 			 * offset is data.
218 			 */
219 			if (offset >= inode->i_size) {
220 				retval = -ENXIO;
221 				goto out;
222 			}
223 			break;
224 		case SEEK_HOLE:
225 			/*
226 			 * There is a virtual hole at the end of the file, so
227 			 * as long as offset isn't i_size or larger, return
228 			 * i_size.
229 			 */
230 			if (offset >= inode->i_size) {
231 				retval = -ENXIO;
232 				goto out;
233 			}
234 			offset = inode->i_size;
235 			break;
236 	}
237 	retval = -EINVAL;
238 	if (offset >= 0 || unsigned_offsets(file)) {
239 		if (offset != file->f_pos) {
240 			file->f_pos = offset;
241 			file->f_version = 0;
242 		}
243 		retval = offset;
244 	}
245 out:
246 	mutex_unlock(&inode->i_mutex);
247 	return retval;
248 }
249 EXPORT_SYMBOL(default_llseek);
250 
251 loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
252 {
253 	loff_t (*fn)(struct file *, loff_t, int);
254 
255 	fn = no_llseek;
256 	if (file->f_mode & FMODE_LSEEK) {
257 		if (file->f_op->llseek)
258 			fn = file->f_op->llseek;
259 	}
260 	return fn(file, offset, whence);
261 }
262 EXPORT_SYMBOL(vfs_llseek);
263 
264 static inline struct fd fdget_pos(int fd)
265 {
266 	return __to_fd(__fdget_pos(fd));
267 }
268 
269 static inline void fdput_pos(struct fd f)
270 {
271 	if (f.flags & FDPUT_POS_UNLOCK)
272 		mutex_unlock(&f.file->f_pos_lock);
273 	fdput(f);
274 }
275 
276 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
277 {
278 	off_t retval;
279 	struct fd f = fdget_pos(fd);
280 	if (!f.file)
281 		return -EBADF;
282 
283 	retval = -EINVAL;
284 	if (whence <= SEEK_MAX) {
285 		loff_t res = vfs_llseek(f.file, offset, whence);
286 		retval = res;
287 		if (res != (loff_t)retval)
288 			retval = -EOVERFLOW;	/* LFS: should only happen on 32 bit platforms */
289 	}
290 	fdput_pos(f);
291 	return retval;
292 }
293 
294 #ifdef CONFIG_COMPAT
295 COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
296 {
297 	return sys_lseek(fd, offset, whence);
298 }
299 #endif
300 
301 #ifdef __ARCH_WANT_SYS_LLSEEK
302 SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
303 		unsigned long, offset_low, loff_t __user *, result,
304 		unsigned int, whence)
305 {
306 	int retval;
307 	struct fd f = fdget_pos(fd);
308 	loff_t offset;
309 
310 	if (!f.file)
311 		return -EBADF;
312 
313 	retval = -EINVAL;
314 	if (whence > SEEK_MAX)
315 		goto out_putf;
316 
317 	offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
318 			whence);
319 
320 	retval = (int)offset;
321 	if (offset >= 0) {
322 		retval = -EFAULT;
323 		if (!copy_to_user(result, &offset, sizeof(offset)))
324 			retval = 0;
325 	}
326 out_putf:
327 	fdput_pos(f);
328 	return retval;
329 }
330 #endif
331 
332 ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos)
333 {
334 	struct kiocb kiocb;
335 	ssize_t ret;
336 
337 	if (!file->f_op->read_iter)
338 		return -EINVAL;
339 
340 	init_sync_kiocb(&kiocb, file);
341 	kiocb.ki_pos = *ppos;
342 
343 	iter->type |= READ;
344 	ret = file->f_op->read_iter(&kiocb, iter);
345 	BUG_ON(ret == -EIOCBQUEUED);
346 	if (ret > 0)
347 		*ppos = kiocb.ki_pos;
348 	return ret;
349 }
350 EXPORT_SYMBOL(vfs_iter_read);
351 
352 ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos)
353 {
354 	struct kiocb kiocb;
355 	ssize_t ret;
356 
357 	if (!file->f_op->write_iter)
358 		return -EINVAL;
359 
360 	init_sync_kiocb(&kiocb, file);
361 	kiocb.ki_pos = *ppos;
362 
363 	iter->type |= WRITE;
364 	ret = file->f_op->write_iter(&kiocb, iter);
365 	BUG_ON(ret == -EIOCBQUEUED);
366 	if (ret > 0)
367 		*ppos = kiocb.ki_pos;
368 	return ret;
369 }
370 EXPORT_SYMBOL(vfs_iter_write);
371 
372 /*
373  * rw_verify_area doesn't like huge counts. We limit
374  * them to something that fits in "int" so that others
375  * won't have to do range checks all the time.
376  */
377 int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
378 {
379 	struct inode *inode;
380 	loff_t pos;
381 	int retval = -EINVAL;
382 
383 	inode = file_inode(file);
384 	if (unlikely((ssize_t) count < 0))
385 		return retval;
386 	pos = *ppos;
387 	if (unlikely(pos < 0)) {
388 		if (!unsigned_offsets(file))
389 			return retval;
390 		if (count >= -pos) /* both values are in 0..LLONG_MAX */
391 			return -EOVERFLOW;
392 	} else if (unlikely((loff_t) (pos + count) < 0)) {
393 		if (!unsigned_offsets(file))
394 			return retval;
395 	}
396 
397 	if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
398 		retval = locks_mandatory_area(
399 			read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
400 			inode, file, pos, count);
401 		if (retval < 0)
402 			return retval;
403 	}
404 	retval = security_file_permission(file,
405 				read_write == READ ? MAY_READ : MAY_WRITE);
406 	if (retval)
407 		return retval;
408 	return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
409 }
410 
411 static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
412 {
413 	struct iovec iov = { .iov_base = buf, .iov_len = len };
414 	struct kiocb kiocb;
415 	struct iov_iter iter;
416 	ssize_t ret;
417 
418 	init_sync_kiocb(&kiocb, filp);
419 	kiocb.ki_pos = *ppos;
420 	iov_iter_init(&iter, READ, &iov, 1, len);
421 
422 	ret = filp->f_op->read_iter(&kiocb, &iter);
423 	BUG_ON(ret == -EIOCBQUEUED);
424 	*ppos = kiocb.ki_pos;
425 	return ret;
426 }
427 
428 ssize_t __vfs_read(struct file *file, char __user *buf, size_t count,
429 		   loff_t *pos)
430 {
431 	if (file->f_op->read)
432 		return file->f_op->read(file, buf, count, pos);
433 	else if (file->f_op->read_iter)
434 		return new_sync_read(file, buf, count, pos);
435 	else
436 		return -EINVAL;
437 }
438 EXPORT_SYMBOL(__vfs_read);
439 
440 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
441 {
442 	ssize_t ret;
443 
444 	if (!(file->f_mode & FMODE_READ))
445 		return -EBADF;
446 	if (!(file->f_mode & FMODE_CAN_READ))
447 		return -EINVAL;
448 	if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
449 		return -EFAULT;
450 
451 	ret = rw_verify_area(READ, file, pos, count);
452 	if (ret >= 0) {
453 		count = ret;
454 		ret = __vfs_read(file, buf, count, pos);
455 		if (ret > 0) {
456 			fsnotify_access(file);
457 			add_rchar(current, ret);
458 		}
459 		inc_syscr(current);
460 	}
461 
462 	return ret;
463 }
464 
465 EXPORT_SYMBOL(vfs_read);
466 
467 static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
468 {
469 	struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
470 	struct kiocb kiocb;
471 	struct iov_iter iter;
472 	ssize_t ret;
473 
474 	init_sync_kiocb(&kiocb, filp);
475 	kiocb.ki_pos = *ppos;
476 	iov_iter_init(&iter, WRITE, &iov, 1, len);
477 
478 	ret = filp->f_op->write_iter(&kiocb, &iter);
479 	BUG_ON(ret == -EIOCBQUEUED);
480 	if (ret > 0)
481 		*ppos = kiocb.ki_pos;
482 	return ret;
483 }
484 
485 ssize_t __vfs_write(struct file *file, const char __user *p, size_t count,
486 		    loff_t *pos)
487 {
488 	if (file->f_op->write)
489 		return file->f_op->write(file, p, count, pos);
490 	else if (file->f_op->write_iter)
491 		return new_sync_write(file, p, count, pos);
492 	else
493 		return -EINVAL;
494 }
495 EXPORT_SYMBOL(__vfs_write);
496 
497 ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos)
498 {
499 	mm_segment_t old_fs;
500 	const char __user *p;
501 	ssize_t ret;
502 
503 	if (!(file->f_mode & FMODE_CAN_WRITE))
504 		return -EINVAL;
505 
506 	old_fs = get_fs();
507 	set_fs(get_ds());
508 	p = (__force const char __user *)buf;
509 	if (count > MAX_RW_COUNT)
510 		count =  MAX_RW_COUNT;
511 	ret = __vfs_write(file, p, count, pos);
512 	set_fs(old_fs);
513 	if (ret > 0) {
514 		fsnotify_modify(file);
515 		add_wchar(current, ret);
516 	}
517 	inc_syscw(current);
518 	return ret;
519 }
520 
521 EXPORT_SYMBOL(__kernel_write);
522 
523 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
524 {
525 	ssize_t ret;
526 
527 	if (!(file->f_mode & FMODE_WRITE))
528 		return -EBADF;
529 	if (!(file->f_mode & FMODE_CAN_WRITE))
530 		return -EINVAL;
531 	if (unlikely(!access_ok(VERIFY_READ, buf, count)))
532 		return -EFAULT;
533 
534 	ret = rw_verify_area(WRITE, file, pos, count);
535 	if (ret >= 0) {
536 		count = ret;
537 		file_start_write(file);
538 		ret = __vfs_write(file, buf, count, pos);
539 		if (ret > 0) {
540 			fsnotify_modify(file);
541 			add_wchar(current, ret);
542 		}
543 		inc_syscw(current);
544 		file_end_write(file);
545 	}
546 
547 	return ret;
548 }
549 
550 EXPORT_SYMBOL(vfs_write);
551 
552 static inline loff_t file_pos_read(struct file *file)
553 {
554 	return file->f_pos;
555 }
556 
557 static inline void file_pos_write(struct file *file, loff_t pos)
558 {
559 	file->f_pos = pos;
560 }
561 
562 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
563 {
564 	struct fd f = fdget_pos(fd);
565 	ssize_t ret = -EBADF;
566 
567 	if (f.file) {
568 		loff_t pos = file_pos_read(f.file);
569 		ret = vfs_read(f.file, buf, count, &pos);
570 		if (ret >= 0)
571 			file_pos_write(f.file, pos);
572 		fdput_pos(f);
573 	}
574 	return ret;
575 }
576 
577 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
578 		size_t, count)
579 {
580 	struct fd f = fdget_pos(fd);
581 	ssize_t ret = -EBADF;
582 
583 	if (f.file) {
584 		loff_t pos = file_pos_read(f.file);
585 		ret = vfs_write(f.file, buf, count, &pos);
586 		if (ret >= 0)
587 			file_pos_write(f.file, pos);
588 		fdput_pos(f);
589 	}
590 
591 	return ret;
592 }
593 
594 SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
595 			size_t, count, loff_t, pos)
596 {
597 	struct fd f;
598 	ssize_t ret = -EBADF;
599 
600 	if (pos < 0)
601 		return -EINVAL;
602 
603 	f = fdget(fd);
604 	if (f.file) {
605 		ret = -ESPIPE;
606 		if (f.file->f_mode & FMODE_PREAD)
607 			ret = vfs_read(f.file, buf, count, &pos);
608 		fdput(f);
609 	}
610 
611 	return ret;
612 }
613 
614 SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
615 			 size_t, count, loff_t, pos)
616 {
617 	struct fd f;
618 	ssize_t ret = -EBADF;
619 
620 	if (pos < 0)
621 		return -EINVAL;
622 
623 	f = fdget(fd);
624 	if (f.file) {
625 		ret = -ESPIPE;
626 		if (f.file->f_mode & FMODE_PWRITE)
627 			ret = vfs_write(f.file, buf, count, &pos);
628 		fdput(f);
629 	}
630 
631 	return ret;
632 }
633 
634 /*
635  * Reduce an iovec's length in-place.  Return the resulting number of segments
636  */
637 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
638 {
639 	unsigned long seg = 0;
640 	size_t len = 0;
641 
642 	while (seg < nr_segs) {
643 		seg++;
644 		if (len + iov->iov_len >= to) {
645 			iov->iov_len = to - len;
646 			break;
647 		}
648 		len += iov->iov_len;
649 		iov++;
650 	}
651 	return seg;
652 }
653 EXPORT_SYMBOL(iov_shorten);
654 
655 static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
656 		loff_t *ppos, iter_fn_t fn)
657 {
658 	struct kiocb kiocb;
659 	ssize_t ret;
660 
661 	init_sync_kiocb(&kiocb, filp);
662 	kiocb.ki_pos = *ppos;
663 
664 	ret = fn(&kiocb, iter);
665 	BUG_ON(ret == -EIOCBQUEUED);
666 	*ppos = kiocb.ki_pos;
667 	return ret;
668 }
669 
670 /* Do it by hand, with file-ops */
671 static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
672 		loff_t *ppos, io_fn_t fn)
673 {
674 	ssize_t ret = 0;
675 
676 	while (iov_iter_count(iter)) {
677 		struct iovec iovec = iov_iter_iovec(iter);
678 		ssize_t nr;
679 
680 		nr = fn(filp, iovec.iov_base, iovec.iov_len, ppos);
681 
682 		if (nr < 0) {
683 			if (!ret)
684 				ret = nr;
685 			break;
686 		}
687 		ret += nr;
688 		if (nr != iovec.iov_len)
689 			break;
690 		iov_iter_advance(iter, nr);
691 	}
692 
693 	return ret;
694 }
695 
696 /* A write operation does a read from user space and vice versa */
697 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
698 
699 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
700 			      unsigned long nr_segs, unsigned long fast_segs,
701 			      struct iovec *fast_pointer,
702 			      struct iovec **ret_pointer)
703 {
704 	unsigned long seg;
705 	ssize_t ret;
706 	struct iovec *iov = fast_pointer;
707 
708 	/*
709 	 * SuS says "The readv() function *may* fail if the iovcnt argument
710 	 * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
711 	 * traditionally returned zero for zero segments, so...
712 	 */
713 	if (nr_segs == 0) {
714 		ret = 0;
715 		goto out;
716 	}
717 
718 	/*
719 	 * First get the "struct iovec" from user memory and
720 	 * verify all the pointers
721 	 */
722 	if (nr_segs > UIO_MAXIOV) {
723 		ret = -EINVAL;
724 		goto out;
725 	}
726 	if (nr_segs > fast_segs) {
727 		iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
728 		if (iov == NULL) {
729 			ret = -ENOMEM;
730 			goto out;
731 		}
732 	}
733 	if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
734 		ret = -EFAULT;
735 		goto out;
736 	}
737 
738 	/*
739 	 * According to the Single Unix Specification we should return EINVAL
740 	 * if an element length is < 0 when cast to ssize_t or if the
741 	 * total length would overflow the ssize_t return value of the
742 	 * system call.
743 	 *
744 	 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
745 	 * overflow case.
746 	 */
747 	ret = 0;
748 	for (seg = 0; seg < nr_segs; seg++) {
749 		void __user *buf = iov[seg].iov_base;
750 		ssize_t len = (ssize_t)iov[seg].iov_len;
751 
752 		/* see if we we're about to use an invalid len or if
753 		 * it's about to overflow ssize_t */
754 		if (len < 0) {
755 			ret = -EINVAL;
756 			goto out;
757 		}
758 		if (type >= 0
759 		    && unlikely(!access_ok(vrfy_dir(type), buf, len))) {
760 			ret = -EFAULT;
761 			goto out;
762 		}
763 		if (len > MAX_RW_COUNT - ret) {
764 			len = MAX_RW_COUNT - ret;
765 			iov[seg].iov_len = len;
766 		}
767 		ret += len;
768 	}
769 out:
770 	*ret_pointer = iov;
771 	return ret;
772 }
773 
774 static ssize_t do_readv_writev(int type, struct file *file,
775 			       const struct iovec __user * uvector,
776 			       unsigned long nr_segs, loff_t *pos)
777 {
778 	size_t tot_len;
779 	struct iovec iovstack[UIO_FASTIOV];
780 	struct iovec *iov = iovstack;
781 	struct iov_iter iter;
782 	ssize_t ret;
783 	io_fn_t fn;
784 	iter_fn_t iter_fn;
785 
786 	ret = import_iovec(type, uvector, nr_segs,
787 			   ARRAY_SIZE(iovstack), &iov, &iter);
788 	if (ret < 0)
789 		return ret;
790 
791 	tot_len = iov_iter_count(&iter);
792 	if (!tot_len)
793 		goto out;
794 	ret = rw_verify_area(type, file, pos, tot_len);
795 	if (ret < 0)
796 		goto out;
797 
798 	if (type == READ) {
799 		fn = file->f_op->read;
800 		iter_fn = file->f_op->read_iter;
801 	} else {
802 		fn = (io_fn_t)file->f_op->write;
803 		iter_fn = file->f_op->write_iter;
804 		file_start_write(file);
805 	}
806 
807 	if (iter_fn)
808 		ret = do_iter_readv_writev(file, &iter, pos, iter_fn);
809 	else
810 		ret = do_loop_readv_writev(file, &iter, pos, fn);
811 
812 	if (type != READ)
813 		file_end_write(file);
814 
815 out:
816 	kfree(iov);
817 	if ((ret + (type == READ)) > 0) {
818 		if (type == READ)
819 			fsnotify_access(file);
820 		else
821 			fsnotify_modify(file);
822 	}
823 	return ret;
824 }
825 
826 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
827 		  unsigned long vlen, loff_t *pos)
828 {
829 	if (!(file->f_mode & FMODE_READ))
830 		return -EBADF;
831 	if (!(file->f_mode & FMODE_CAN_READ))
832 		return -EINVAL;
833 
834 	return do_readv_writev(READ, file, vec, vlen, pos);
835 }
836 
837 EXPORT_SYMBOL(vfs_readv);
838 
839 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
840 		   unsigned long vlen, loff_t *pos)
841 {
842 	if (!(file->f_mode & FMODE_WRITE))
843 		return -EBADF;
844 	if (!(file->f_mode & FMODE_CAN_WRITE))
845 		return -EINVAL;
846 
847 	return do_readv_writev(WRITE, file, vec, vlen, pos);
848 }
849 
850 EXPORT_SYMBOL(vfs_writev);
851 
852 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
853 		unsigned long, vlen)
854 {
855 	struct fd f = fdget_pos(fd);
856 	ssize_t ret = -EBADF;
857 
858 	if (f.file) {
859 		loff_t pos = file_pos_read(f.file);
860 		ret = vfs_readv(f.file, vec, vlen, &pos);
861 		if (ret >= 0)
862 			file_pos_write(f.file, pos);
863 		fdput_pos(f);
864 	}
865 
866 	if (ret > 0)
867 		add_rchar(current, ret);
868 	inc_syscr(current);
869 	return ret;
870 }
871 
872 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
873 		unsigned long, vlen)
874 {
875 	struct fd f = fdget_pos(fd);
876 	ssize_t ret = -EBADF;
877 
878 	if (f.file) {
879 		loff_t pos = file_pos_read(f.file);
880 		ret = vfs_writev(f.file, vec, vlen, &pos);
881 		if (ret >= 0)
882 			file_pos_write(f.file, pos);
883 		fdput_pos(f);
884 	}
885 
886 	if (ret > 0)
887 		add_wchar(current, ret);
888 	inc_syscw(current);
889 	return ret;
890 }
891 
892 static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
893 {
894 #define HALF_LONG_BITS (BITS_PER_LONG / 2)
895 	return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
896 }
897 
898 SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
899 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
900 {
901 	loff_t pos = pos_from_hilo(pos_h, pos_l);
902 	struct fd f;
903 	ssize_t ret = -EBADF;
904 
905 	if (pos < 0)
906 		return -EINVAL;
907 
908 	f = fdget(fd);
909 	if (f.file) {
910 		ret = -ESPIPE;
911 		if (f.file->f_mode & FMODE_PREAD)
912 			ret = vfs_readv(f.file, vec, vlen, &pos);
913 		fdput(f);
914 	}
915 
916 	if (ret > 0)
917 		add_rchar(current, ret);
918 	inc_syscr(current);
919 	return ret;
920 }
921 
922 SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
923 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
924 {
925 	loff_t pos = pos_from_hilo(pos_h, pos_l);
926 	struct fd f;
927 	ssize_t ret = -EBADF;
928 
929 	if (pos < 0)
930 		return -EINVAL;
931 
932 	f = fdget(fd);
933 	if (f.file) {
934 		ret = -ESPIPE;
935 		if (f.file->f_mode & FMODE_PWRITE)
936 			ret = vfs_writev(f.file, vec, vlen, &pos);
937 		fdput(f);
938 	}
939 
940 	if (ret > 0)
941 		add_wchar(current, ret);
942 	inc_syscw(current);
943 	return ret;
944 }
945 
946 #ifdef CONFIG_COMPAT
947 
948 static ssize_t compat_do_readv_writev(int type, struct file *file,
949 			       const struct compat_iovec __user *uvector,
950 			       unsigned long nr_segs, loff_t *pos)
951 {
952 	compat_ssize_t tot_len;
953 	struct iovec iovstack[UIO_FASTIOV];
954 	struct iovec *iov = iovstack;
955 	struct iov_iter iter;
956 	ssize_t ret;
957 	io_fn_t fn;
958 	iter_fn_t iter_fn;
959 
960 	ret = compat_import_iovec(type, uvector, nr_segs,
961 				  UIO_FASTIOV, &iov, &iter);
962 	if (ret < 0)
963 		return ret;
964 
965 	tot_len = iov_iter_count(&iter);
966 	if (!tot_len)
967 		goto out;
968 	ret = rw_verify_area(type, file, pos, tot_len);
969 	if (ret < 0)
970 		goto out;
971 
972 	if (type == READ) {
973 		fn = file->f_op->read;
974 		iter_fn = file->f_op->read_iter;
975 	} else {
976 		fn = (io_fn_t)file->f_op->write;
977 		iter_fn = file->f_op->write_iter;
978 		file_start_write(file);
979 	}
980 
981 	if (iter_fn)
982 		ret = do_iter_readv_writev(file, &iter, pos, iter_fn);
983 	else
984 		ret = do_loop_readv_writev(file, &iter, pos, fn);
985 
986 	if (type != READ)
987 		file_end_write(file);
988 
989 out:
990 	kfree(iov);
991 	if ((ret + (type == READ)) > 0) {
992 		if (type == READ)
993 			fsnotify_access(file);
994 		else
995 			fsnotify_modify(file);
996 	}
997 	return ret;
998 }
999 
1000 static size_t compat_readv(struct file *file,
1001 			   const struct compat_iovec __user *vec,
1002 			   unsigned long vlen, loff_t *pos)
1003 {
1004 	ssize_t ret = -EBADF;
1005 
1006 	if (!(file->f_mode & FMODE_READ))
1007 		goto out;
1008 
1009 	ret = -EINVAL;
1010 	if (!(file->f_mode & FMODE_CAN_READ))
1011 		goto out;
1012 
1013 	ret = compat_do_readv_writev(READ, file, vec, vlen, pos);
1014 
1015 out:
1016 	if (ret > 0)
1017 		add_rchar(current, ret);
1018 	inc_syscr(current);
1019 	return ret;
1020 }
1021 
1022 COMPAT_SYSCALL_DEFINE3(readv, compat_ulong_t, fd,
1023 		const struct compat_iovec __user *,vec,
1024 		compat_ulong_t, vlen)
1025 {
1026 	struct fd f = fdget_pos(fd);
1027 	ssize_t ret;
1028 	loff_t pos;
1029 
1030 	if (!f.file)
1031 		return -EBADF;
1032 	pos = f.file->f_pos;
1033 	ret = compat_readv(f.file, vec, vlen, &pos);
1034 	if (ret >= 0)
1035 		f.file->f_pos = pos;
1036 	fdput_pos(f);
1037 	return ret;
1038 }
1039 
1040 static long __compat_sys_preadv64(unsigned long fd,
1041 				  const struct compat_iovec __user *vec,
1042 				  unsigned long vlen, loff_t pos)
1043 {
1044 	struct fd f;
1045 	ssize_t ret;
1046 
1047 	if (pos < 0)
1048 		return -EINVAL;
1049 	f = fdget(fd);
1050 	if (!f.file)
1051 		return -EBADF;
1052 	ret = -ESPIPE;
1053 	if (f.file->f_mode & FMODE_PREAD)
1054 		ret = compat_readv(f.file, vec, vlen, &pos);
1055 	fdput(f);
1056 	return ret;
1057 }
1058 
1059 #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
1060 COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1061 		const struct compat_iovec __user *,vec,
1062 		unsigned long, vlen, loff_t, pos)
1063 {
1064 	return __compat_sys_preadv64(fd, vec, vlen, pos);
1065 }
1066 #endif
1067 
1068 COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
1069 		const struct compat_iovec __user *,vec,
1070 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1071 {
1072 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1073 
1074 	return __compat_sys_preadv64(fd, vec, vlen, pos);
1075 }
1076 
1077 static size_t compat_writev(struct file *file,
1078 			    const struct compat_iovec __user *vec,
1079 			    unsigned long vlen, loff_t *pos)
1080 {
1081 	ssize_t ret = -EBADF;
1082 
1083 	if (!(file->f_mode & FMODE_WRITE))
1084 		goto out;
1085 
1086 	ret = -EINVAL;
1087 	if (!(file->f_mode & FMODE_CAN_WRITE))
1088 		goto out;
1089 
1090 	ret = compat_do_readv_writev(WRITE, file, vec, vlen, pos);
1091 
1092 out:
1093 	if (ret > 0)
1094 		add_wchar(current, ret);
1095 	inc_syscw(current);
1096 	return ret;
1097 }
1098 
1099 COMPAT_SYSCALL_DEFINE3(writev, compat_ulong_t, fd,
1100 		const struct compat_iovec __user *, vec,
1101 		compat_ulong_t, vlen)
1102 {
1103 	struct fd f = fdget_pos(fd);
1104 	ssize_t ret;
1105 	loff_t pos;
1106 
1107 	if (!f.file)
1108 		return -EBADF;
1109 	pos = f.file->f_pos;
1110 	ret = compat_writev(f.file, vec, vlen, &pos);
1111 	if (ret >= 0)
1112 		f.file->f_pos = pos;
1113 	fdput_pos(f);
1114 	return ret;
1115 }
1116 
1117 static long __compat_sys_pwritev64(unsigned long fd,
1118 				   const struct compat_iovec __user *vec,
1119 				   unsigned long vlen, loff_t pos)
1120 {
1121 	struct fd f;
1122 	ssize_t ret;
1123 
1124 	if (pos < 0)
1125 		return -EINVAL;
1126 	f = fdget(fd);
1127 	if (!f.file)
1128 		return -EBADF;
1129 	ret = -ESPIPE;
1130 	if (f.file->f_mode & FMODE_PWRITE)
1131 		ret = compat_writev(f.file, vec, vlen, &pos);
1132 	fdput(f);
1133 	return ret;
1134 }
1135 
1136 #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
1137 COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1138 		const struct compat_iovec __user *,vec,
1139 		unsigned long, vlen, loff_t, pos)
1140 {
1141 	return __compat_sys_pwritev64(fd, vec, vlen, pos);
1142 }
1143 #endif
1144 
1145 COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
1146 		const struct compat_iovec __user *,vec,
1147 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1148 {
1149 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1150 
1151 	return __compat_sys_pwritev64(fd, vec, vlen, pos);
1152 }
1153 #endif
1154 
1155 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1156 		  	   size_t count, loff_t max)
1157 {
1158 	struct fd in, out;
1159 	struct inode *in_inode, *out_inode;
1160 	loff_t pos;
1161 	loff_t out_pos;
1162 	ssize_t retval;
1163 	int fl;
1164 
1165 	/*
1166 	 * Get input file, and verify that it is ok..
1167 	 */
1168 	retval = -EBADF;
1169 	in = fdget(in_fd);
1170 	if (!in.file)
1171 		goto out;
1172 	if (!(in.file->f_mode & FMODE_READ))
1173 		goto fput_in;
1174 	retval = -ESPIPE;
1175 	if (!ppos) {
1176 		pos = in.file->f_pos;
1177 	} else {
1178 		pos = *ppos;
1179 		if (!(in.file->f_mode & FMODE_PREAD))
1180 			goto fput_in;
1181 	}
1182 	retval = rw_verify_area(READ, in.file, &pos, count);
1183 	if (retval < 0)
1184 		goto fput_in;
1185 	count = retval;
1186 
1187 	/*
1188 	 * Get output file, and verify that it is ok..
1189 	 */
1190 	retval = -EBADF;
1191 	out = fdget(out_fd);
1192 	if (!out.file)
1193 		goto fput_in;
1194 	if (!(out.file->f_mode & FMODE_WRITE))
1195 		goto fput_out;
1196 	retval = -EINVAL;
1197 	in_inode = file_inode(in.file);
1198 	out_inode = file_inode(out.file);
1199 	out_pos = out.file->f_pos;
1200 	retval = rw_verify_area(WRITE, out.file, &out_pos, count);
1201 	if (retval < 0)
1202 		goto fput_out;
1203 	count = retval;
1204 
1205 	if (!max)
1206 		max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1207 
1208 	if (unlikely(pos + count > max)) {
1209 		retval = -EOVERFLOW;
1210 		if (pos >= max)
1211 			goto fput_out;
1212 		count = max - pos;
1213 	}
1214 
1215 	fl = 0;
1216 #if 0
1217 	/*
1218 	 * We need to debate whether we can enable this or not. The
1219 	 * man page documents EAGAIN return for the output at least,
1220 	 * and the application is arguably buggy if it doesn't expect
1221 	 * EAGAIN on a non-blocking file descriptor.
1222 	 */
1223 	if (in.file->f_flags & O_NONBLOCK)
1224 		fl = SPLICE_F_NONBLOCK;
1225 #endif
1226 	file_start_write(out.file);
1227 	retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
1228 	file_end_write(out.file);
1229 
1230 	if (retval > 0) {
1231 		add_rchar(current, retval);
1232 		add_wchar(current, retval);
1233 		fsnotify_access(in.file);
1234 		fsnotify_modify(out.file);
1235 		out.file->f_pos = out_pos;
1236 		if (ppos)
1237 			*ppos = pos;
1238 		else
1239 			in.file->f_pos = pos;
1240 	}
1241 
1242 	inc_syscr(current);
1243 	inc_syscw(current);
1244 	if (pos > max)
1245 		retval = -EOVERFLOW;
1246 
1247 fput_out:
1248 	fdput(out);
1249 fput_in:
1250 	fdput(in);
1251 out:
1252 	return retval;
1253 }
1254 
1255 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
1256 {
1257 	loff_t pos;
1258 	off_t off;
1259 	ssize_t ret;
1260 
1261 	if (offset) {
1262 		if (unlikely(get_user(off, offset)))
1263 			return -EFAULT;
1264 		pos = off;
1265 		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1266 		if (unlikely(put_user(pos, offset)))
1267 			return -EFAULT;
1268 		return ret;
1269 	}
1270 
1271 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1272 }
1273 
1274 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
1275 {
1276 	loff_t pos;
1277 	ssize_t ret;
1278 
1279 	if (offset) {
1280 		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1281 			return -EFAULT;
1282 		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1283 		if (unlikely(put_user(pos, offset)))
1284 			return -EFAULT;
1285 		return ret;
1286 	}
1287 
1288 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1289 }
1290 
1291 #ifdef CONFIG_COMPAT
1292 COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1293 		compat_off_t __user *, offset, compat_size_t, count)
1294 {
1295 	loff_t pos;
1296 	off_t off;
1297 	ssize_t ret;
1298 
1299 	if (offset) {
1300 		if (unlikely(get_user(off, offset)))
1301 			return -EFAULT;
1302 		pos = off;
1303 		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1304 		if (unlikely(put_user(pos, offset)))
1305 			return -EFAULT;
1306 		return ret;
1307 	}
1308 
1309 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1310 }
1311 
1312 COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1313 		compat_loff_t __user *, offset, compat_size_t, count)
1314 {
1315 	loff_t pos;
1316 	ssize_t ret;
1317 
1318 	if (offset) {
1319 		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1320 			return -EFAULT;
1321 		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1322 		if (unlikely(put_user(pos, offset)))
1323 			return -EFAULT;
1324 		return ret;
1325 	}
1326 
1327 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1328 }
1329 #endif
1330