1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/mm.h> 6 #include <linux/slab.h> 7 #include <linux/nospec.h> 8 #include <linux/io_uring.h> 9 10 #include <uapi/linux/io_uring.h> 11 12 #include "io_uring.h" 13 #include "rsrc.h" 14 #include "filetable.h" 15 16 static int io_file_bitmap_get(struct io_ring_ctx *ctx) 17 { 18 struct io_file_table *table = &ctx->file_table; 19 unsigned long nr = ctx->file_alloc_end; 20 int ret; 21 22 if (!table->bitmap) 23 return -ENFILE; 24 25 do { 26 ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint); 27 if (ret != nr) 28 return ret; 29 30 if (table->alloc_hint == ctx->file_alloc_start) 31 break; 32 nr = table->alloc_hint; 33 table->alloc_hint = ctx->file_alloc_start; 34 } while (1); 35 36 return -ENFILE; 37 } 38 39 bool io_alloc_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table, 40 unsigned nr_files) 41 { 42 if (io_rsrc_data_alloc(&table->data, nr_files)) 43 return false; 44 table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT); 45 if (table->bitmap) 46 return true; 47 io_rsrc_data_free(ctx, &table->data); 48 return false; 49 } 50 51 void io_free_file_tables(struct io_ring_ctx *ctx, struct io_file_table *table) 52 { 53 io_rsrc_data_free(ctx, &table->data); 54 bitmap_free(table->bitmap); 55 table->bitmap = NULL; 56 } 57 58 static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, 59 u32 slot_index) 60 __must_hold(&req->ctx->uring_lock) 61 { 62 struct io_rsrc_node *node; 63 64 if (io_is_uring_fops(file)) 65 return -EBADF; 66 if (!ctx->file_table.data.nr) 67 return -ENXIO; 68 if (slot_index >= ctx->file_table.data.nr) 69 return -EINVAL; 70 71 node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE); 72 if (!node) 73 return -ENOMEM; 74 75 if (!io_reset_rsrc_node(ctx, &ctx->file_table.data, slot_index)) 76 io_file_bitmap_set(&ctx->file_table, slot_index); 77 78 ctx->file_table.data.nodes[slot_index] = node; 79 io_fixed_file_set(node, file); 80 return 0; 81 } 82 83 int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file, 84 unsigned int file_slot) 85 { 86 bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC; 87 int ret; 88 89 if (alloc_slot) { 90 ret = io_file_bitmap_get(ctx); 91 if (unlikely(ret < 0)) 92 return ret; 93 file_slot = ret; 94 } else { 95 file_slot--; 96 } 97 98 ret = io_install_fixed_file(ctx, file, file_slot); 99 if (!ret && alloc_slot) 100 ret = file_slot; 101 return ret; 102 } 103 /* 104 * Note when io_fixed_fd_install() returns error value, it will ensure 105 * fput() is called correspondingly. 106 */ 107 int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags, 108 struct file *file, unsigned int file_slot) 109 { 110 struct io_ring_ctx *ctx = req->ctx; 111 int ret; 112 113 io_ring_submit_lock(ctx, issue_flags); 114 ret = __io_fixed_fd_install(ctx, file, file_slot); 115 io_ring_submit_unlock(ctx, issue_flags); 116 117 if (unlikely(ret < 0)) 118 fput(file); 119 return ret; 120 } 121 122 int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset) 123 { 124 struct io_rsrc_node *node; 125 126 if (unlikely(!ctx->file_table.data.nr)) 127 return -ENXIO; 128 if (offset >= ctx->file_table.data.nr) 129 return -EINVAL; 130 131 node = io_rsrc_node_lookup(&ctx->file_table.data, offset); 132 if (!node) 133 return -EBADF; 134 io_reset_rsrc_node(ctx, &ctx->file_table.data, offset); 135 io_file_bitmap_clear(&ctx->file_table, offset); 136 return 0; 137 } 138 139 int io_register_file_alloc_range(struct io_ring_ctx *ctx, 140 struct io_uring_file_index_range __user *arg) 141 { 142 struct io_uring_file_index_range range; 143 u32 end; 144 145 if (copy_from_user(&range, arg, sizeof(range))) 146 return -EFAULT; 147 if (check_add_overflow(range.off, range.len, &end)) 148 return -EOVERFLOW; 149 if (range.resv || end > ctx->file_table.data.nr) 150 return -EINVAL; 151 152 io_file_table_set_alloc_range(ctx, range.off, range.len); 153 return 0; 154 } 155