103 static void __free_fdtable(struct fdtable *fdt)  in __free_fdtable()  argument
105 	kvfree(fdt->fd);  in __free_fdtable()
106 	kvfree(fdt->open_fds);  in __free_fdtable()
107 	kfree(fdt);  in __free_fdtable()
118 #define fdt_words(fdt) ((fdt)->max_fds / BITS_PER_LONG) // words in ->open_fds  argument
164 	struct fdtable *fdt;  in alloc_fdtable()  local
215 	fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL_ACCOUNT);  in alloc_fdtable()
216 	if (!fdt)  in alloc_fdtable()
218 	fdt->max_fds = nr;  in alloc_fdtable()
222 	fdt->fd = data;  in alloc_fdtable()
229 	fdt->open_fds = data;  in alloc_fdtable()
231 	fdt->close_on_exec = data;  in alloc_fdtable()
233 	fdt->full_fds_bits = data;  in alloc_fdtable()
235 	return fdt;  in alloc_fdtable()
238 	kvfree(fdt->fd);  in alloc_fdtable()
240 	kfree(fdt);  in alloc_fdtable()
273 	rcu_assign_pointer(files->fdt, new_fdt);  in expand_fdtable()
292 	struct fdtable *fdt;  in expand_files()  local
296 	fdt = files_fdtable(files);  in expand_files()
299 	if (nr < fdt->max_fds)  in expand_files()
322 static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt,  in __set_close_on_exec()  argument
326 		__set_bit(fd, fdt->close_on_exec);  in __set_close_on_exec()
328 		if (test_bit(fd, fdt->close_on_exec))  in __set_close_on_exec()
329 			__clear_bit(fd, fdt->close_on_exec);  in __set_close_on_exec()
333 static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt, bool set)  in __set_open_fd()  argument
335 	__set_bit(fd, fdt->open_fds);  in __set_open_fd()
336 	__set_close_on_exec(fd, fdt, set);  in __set_open_fd()
338 	if (!~fdt->open_fds[fd])  in __set_open_fd()
339 		__set_bit(fd, fdt->full_fds_bits);  in __set_open_fd()
342 static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt)  in __clear_open_fd()  argument
344 	__clear_bit(fd, fdt->open_fds);  in __clear_open_fd()
346 	if (test_bit(fd, fdt->full_fds_bits))  in __clear_open_fd()
347 		__clear_bit(fd, fdt->full_fds_bits);  in __clear_open_fd()
350 static inline bool fd_is_open(unsigned int fd, const struct fdtable *fdt)  in fd_is_open()  argument
352 	return test_bit(fd, fdt->open_fds);  in fd_is_open()
364 static unsigned int sane_fdtable_size(struct fdtable *fdt, struct fd_range *punch_hole)  in sane_fdtable_size()  argument
366 	unsigned int last = find_last_bit(fdt->open_fds, fdt->max_fds);  in sane_fdtable_size()
368 	if (last == fdt->max_fds)  in sane_fdtable_size()
371 		last = find_last_bit(fdt->open_fds, punch_hole->from);  in sane_fdtable_size()
469 	rcu_assign_pointer(newf->fdt, new_fdt);  in dup_fd()
481 	struct fdtable *fdt = rcu_dereference_raw(files->fdt);  in close_files()  local
487 		if (i >= fdt->max_fds)  in close_files()
489 		set = fdt->open_fds[j++];  in close_files()
492 				struct file *file = fdt->fd[i];  in close_files()
503 	return fdt;  in close_files()
509 		struct fdtable *fdt = close_files(files);  in put_files_struct()  local
512 		if (fdt != &files->fdtab)  in put_files_struct()
513 			__free_fdtable(fdt);  in put_files_struct()
532 	.fdt		= &init_files.fdtab,
544 static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)  in find_next_fd()  argument
546 	unsigned int maxfd = fdt->max_fds; /* always multiple of BITS_PER_LONG */  in find_next_fd()
554 	bit = find_next_zero_bit(&fdt->open_fds[bitbit], BITS_PER_LONG,  in find_next_fd()
559 	bitbit = find_next_zero_bit(fdt->full_fds_bits, maxbit, bitbit) * BITS_PER_LONG;  in find_next_fd()
564 	return find_next_zero_bit(fdt->open_fds, maxfd, start);  in find_next_fd()
575 	struct fdtable *fdt;  in alloc_fd()  local
579 	fdt = files_fdtable(files);  in alloc_fd()
584 	if (likely(fd < fdt->max_fds))  in alloc_fd()
585 		fd = find_next_fd(fdt, fd);  in alloc_fd()
595 	if (unlikely(fd >= fdt->max_fds)) {  in alloc_fd()
606 	__set_open_fd(fd, fdt, flags & O_CLOEXEC);  in alloc_fd()
608 	VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL);  in alloc_fd()
628 	struct fdtable *fdt = files_fdtable(files);  in __put_unused_fd()  local
629 	__clear_open_fd(fd, fdt);  in __put_unused_fd()
655 	struct fdtable *fdt;  in fd_install()  local
665 		fdt = files_fdtable(files);  in fd_install()
666 		VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL);  in fd_install()
667 		rcu_assign_pointer(fdt->fd[fd], file);  in fd_install()
673 	fdt = rcu_dereference_sched(files->fdt);  in fd_install()
674 	VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL);  in fd_install()
675 	rcu_assign_pointer(fdt->fd[fd], file);  in fd_install()
694 	struct fdtable *fdt = files_fdtable(files);  in file_close_fd_locked()  local
699 	if (fd >= fdt->max_fds)  in file_close_fd_locked()
702 	fd = array_index_nospec(fd, fdt->max_fds);  in file_close_fd_locked()
703 	file = rcu_dereference_raw(fdt->fd[fd]);  in file_close_fd_locked()
705 		rcu_assign_pointer(fdt->fd[fd], NULL);  in file_close_fd_locked()
728  * @fdt: File descriptor table.
734 static inline unsigned last_fd(struct fdtable *fdt)  in last_fd()  argument
736 	return fdt->max_fds - 1;  in last_fd()
742 	struct fdtable *fdt;  in __range_cloexec()  local
746 	fdt = files_fdtable(cur_fds);  in __range_cloexec()
747 	max_fd = min(last_fd(fdt), max_fd);  in __range_cloexec()
749 		bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1);  in __range_cloexec()
865 	struct fdtable *fdt;  in do_close_on_exec()  local
872 		fdt = files_fdtable(files);  in do_close_on_exec()
873 		if (fd >= fdt->max_fds)  in do_close_on_exec()
875 		set = fdt->close_on_exec[i];  in do_close_on_exec()
878 		fdt->close_on_exec[i] = 0;  in do_close_on_exec()
883 			file = fdt->fd[fd];  in do_close_on_exec()
886 			rcu_assign_pointer(fdt->fd[fd], NULL);  in do_close_on_exec()
994 		struct fdtable *fdt = rcu_dereference_raw(files->fdt);  in __fget_files_rcu()  local
999 		nospec_mask = array_index_mask_nospec(fd, fdt->max_fds);  in __fget_files_rcu()
1002 		 * fdentry points to the 'fd' offset, or fdt->fd[0].  in __fget_files_rcu()
1003 		 * Loading from fdt->fd[0] is always safe, because the  in __fget_files_rcu()
1006 		fdentry = fdt->fd + (fd & nospec_mask);  in __fget_files_rcu()
1036 		 *       Note that we don't need to re-check the 'fdt->fd'  in __fget_files_rcu()
1038 		 *       hand-in-hand with 'fdt'.  in __fget_files_rcu()
1043 		    unlikely(rcu_dereference_raw(files->fdt) != fdt)) {  in __fget_files_rcu()
1270 	struct fdtable *fdt;  in do_dup2()  local
1298 	fdt = files_fdtable(files);  in do_dup2()
1299 	fd = array_index_nospec(fd, fdt->max_fds);  in do_dup2()
1300 	tofree = rcu_dereference_raw(fdt->fd[fd]);  in do_dup2()
1301 	if (!tofree && fd_is_open(fd, fdt))  in do_dup2()
1304 	rcu_assign_pointer(fdt->fd[fd], file);  in do_dup2()
1305 	__set_open_fd(fd, fdt, flags & O_CLOEXEC);  in do_dup2()
1490 	struct fdtable *fdt;  in iterate_fd()  local
1495 	for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {  in iterate_fd()
1497 		file = rcu_dereference_check_fdtable(files, fdt->fd[n]);  in iterate_fd()