1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #include <linux/mutex.h> 3 #include <linux/bpf.h> 4 #include <linux/bpf_verifier.h> 5 6 #include "io_uring.h" 7 #include "register.h" 8 #include "loop.h" 9 #include "memmap.h" 10 #include "bpf-ops.h" 11 12 static DEFINE_MUTEX(io_bpf_ctrl_mutex); 13 static const struct btf_type *loop_params_type; 14 15 __bpf_kfunc_start_defs(); 16 17 __bpf_kfunc int bpf_io_uring_submit_sqes(struct iou_ctx *loop_ctx, u32 nr) 18 { 19 struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx); 20 21 return io_submit_sqes(ctx, nr); 22 } 23 24 __bpf_kfunc 25 __u8 *bpf_io_uring_get_region(struct iou_ctx *loop_ctx, __u32 region_id, 26 const size_t rdwr_buf_size) 27 { 28 struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx); 29 struct io_mapped_region *r; 30 31 lockdep_assert_held(&ctx->uring_lock); 32 33 switch (region_id) { 34 case IOU_REGION_MEM: 35 r = &ctx->param_region; 36 break; 37 case IOU_REGION_CQ: 38 r = &ctx->ring_region; 39 break; 40 case IOU_REGION_SQ: 41 r = &ctx->sq_region; 42 break; 43 default: 44 return NULL; 45 } 46 47 if (unlikely(rdwr_buf_size > io_region_size(r))) 48 return NULL; 49 return io_region_get_ptr(r); 50 } 51 52 __bpf_kfunc_end_defs(); 53 54 BTF_KFUNCS_START(io_uring_kfunc_set) 55 BTF_ID_FLAGS(func, bpf_io_uring_submit_sqes, KF_SLEEPABLE); 56 BTF_ID_FLAGS(func, bpf_io_uring_get_region, KF_RET_NULL); 57 BTF_KFUNCS_END(io_uring_kfunc_set) 58 59 static const struct btf_kfunc_id_set bpf_io_uring_kfunc_set = { 60 .owner = THIS_MODULE, 61 .set = &io_uring_kfunc_set, 62 }; 63 64 static int io_bpf_ops__loop_step(struct iou_ctx *ctx, 65 struct iou_loop_params *lp) 66 { 67 return IOU_LOOP_STOP; 68 } 69 70 static struct io_uring_bpf_ops io_bpf_ops_stubs = { 71 .loop_step = io_bpf_ops__loop_step, 72 }; 73 74 static bool bpf_io_is_valid_access(int off, int size, 75 enum bpf_access_type type, 76 const struct bpf_prog *prog, 77 struct bpf_insn_access_aux *info) 78 { 79 if (type != BPF_READ) 80 return false; 81 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 82 return false; 83 if (off % size != 0) 84 return false; 85 86 return btf_ctx_access(off, size, type, prog, info); 87 } 88 89 static int bpf_io_btf_struct_access(struct bpf_verifier_log *log, 90 const struct bpf_reg_state *reg, int off, 91 int size) 92 { 93 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 94 95 if (t == loop_params_type) { 96 if (off + size <= offsetofend(struct iou_loop_params, cq_wait_idx)) 97 return SCALAR_VALUE; 98 } 99 100 return -EACCES; 101 } 102 103 static const struct bpf_verifier_ops bpf_io_verifier_ops = { 104 .get_func_proto = bpf_base_func_proto, 105 .is_valid_access = bpf_io_is_valid_access, 106 .btf_struct_access = bpf_io_btf_struct_access, 107 }; 108 109 static const struct btf_type * 110 io_lookup_struct_type(struct btf *btf, const char *name) 111 { 112 s32 type_id; 113 114 type_id = btf_find_by_name_kind(btf, name, BTF_KIND_STRUCT); 115 if (type_id < 0) 116 return NULL; 117 return btf_type_by_id(btf, type_id); 118 } 119 120 static int bpf_io_init(struct btf *btf) 121 { 122 int ret; 123 124 loop_params_type = io_lookup_struct_type(btf, "iou_loop_params"); 125 if (!loop_params_type) { 126 pr_err("io_uring: Failed to locate iou_loop_params\n"); 127 return -EINVAL; 128 } 129 130 ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 131 &bpf_io_uring_kfunc_set); 132 if (ret) { 133 pr_err("io_uring: Failed to register kfuncs (%d)\n", ret); 134 return ret; 135 } 136 return 0; 137 } 138 139 static int bpf_io_check_member(const struct btf_type *t, 140 const struct btf_member *member, 141 const struct bpf_prog *prog) 142 { 143 return 0; 144 } 145 146 static int bpf_io_init_member(const struct btf_type *t, 147 const struct btf_member *member, 148 void *kdata, const void *udata) 149 { 150 u32 moff = __btf_member_bit_offset(t, member) / 8; 151 const struct io_uring_bpf_ops *uops = udata; 152 struct io_uring_bpf_ops *ops = kdata; 153 154 switch (moff) { 155 case offsetof(struct io_uring_bpf_ops, ring_fd): 156 ops->ring_fd = uops->ring_fd; 157 return 1; 158 } 159 return 0; 160 } 161 162 static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops) 163 { 164 if (ctx->flags & (IORING_SETUP_SQPOLL | IORING_SETUP_IOPOLL)) 165 return -EOPNOTSUPP; 166 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 167 return -EOPNOTSUPP; 168 169 if (ctx->bpf_ops) 170 return -EBUSY; 171 if (WARN_ON_ONCE(!ops->loop_step)) 172 return -EINVAL; 173 174 ops->priv = ctx; 175 ctx->bpf_ops = ops; 176 ctx->loop_step = ops->loop_step; 177 return 0; 178 } 179 180 static int bpf_io_reg(void *kdata, struct bpf_link *link) 181 { 182 struct io_uring_bpf_ops *ops = kdata; 183 struct io_ring_ctx *ctx; 184 struct file *file; 185 int ret = -EBUSY; 186 187 file = io_uring_ctx_get_file(ops->ring_fd, false); 188 if (IS_ERR(file)) 189 return PTR_ERR(file); 190 ctx = file->private_data; 191 192 scoped_guard(mutex, &io_bpf_ctrl_mutex) { 193 guard(mutex)(&ctx->uring_lock); 194 ret = io_install_bpf(ctx, ops); 195 } 196 197 fput(file); 198 return ret; 199 } 200 201 static void io_eject_bpf(struct io_ring_ctx *ctx) 202 { 203 struct io_uring_bpf_ops *ops = ctx->bpf_ops; 204 205 if (WARN_ON_ONCE(!ops)) 206 return; 207 if (WARN_ON_ONCE(ops->priv != ctx)) 208 return; 209 210 ops->priv = NULL; 211 ctx->bpf_ops = NULL; 212 ctx->loop_step = NULL; 213 } 214 215 static void bpf_io_unreg(void *kdata, struct bpf_link *link) 216 { 217 struct io_uring_bpf_ops *ops = kdata; 218 struct io_ring_ctx *ctx; 219 220 guard(mutex)(&io_bpf_ctrl_mutex); 221 ctx = ops->priv; 222 if (ctx) { 223 guard(mutex)(&ctx->uring_lock); 224 if (WARN_ON_ONCE(ctx->bpf_ops != ops)) 225 return; 226 227 io_eject_bpf(ctx); 228 } 229 } 230 231 void io_unregister_bpf_ops(struct io_ring_ctx *ctx) 232 { 233 /* 234 * ->bpf_ops is write protected by io_bpf_ctrl_mutex and uring_lock, 235 * and read protected by either. Try to avoid taking the global lock 236 * for rings that never had any bpf installed. 237 */ 238 scoped_guard(mutex, &ctx->uring_lock) { 239 if (!ctx->bpf_ops) 240 return; 241 } 242 243 guard(mutex)(&io_bpf_ctrl_mutex); 244 guard(mutex)(&ctx->uring_lock); 245 if (ctx->bpf_ops) 246 io_eject_bpf(ctx); 247 } 248 249 static struct bpf_struct_ops bpf_ring_ops = { 250 .verifier_ops = &bpf_io_verifier_ops, 251 .reg = bpf_io_reg, 252 .unreg = bpf_io_unreg, 253 .check_member = bpf_io_check_member, 254 .init_member = bpf_io_init_member, 255 .init = bpf_io_init, 256 .cfi_stubs = &io_bpf_ops_stubs, 257 .name = "io_uring_bpf_ops", 258 .owner = THIS_MODULE, 259 }; 260 261 static int __init io_uring_bpf_init(void) 262 { 263 int ret; 264 265 ret = register_bpf_struct_ops(&bpf_ring_ops, io_uring_bpf_ops); 266 if (ret) { 267 pr_err("io_uring: Failed to register struct_ops (%d)\n", ret); 268 return ret; 269 } 270 271 return 0; 272 } 273 __initcall(io_uring_bpf_init); 274