1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #include <linux/mutex.h> 3 #include <linux/bpf.h> 4 #include <linux/bpf_verifier.h> 5 6 #include "io_uring.h" 7 #include "register.h" 8 #include "loop.h" 9 #include "memmap.h" 10 #include "bpf-ops.h" 11 12 static DEFINE_MUTEX(io_bpf_ctrl_mutex); 13 static const struct btf_type *loop_params_type; 14 15 __bpf_kfunc_start_defs(); 16 17 __bpf_kfunc int bpf_io_uring_submit_sqes(struct iou_ctx *loop_ctx, u32 nr) 18 { 19 struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx); 20 21 return io_submit_sqes(ctx, nr); 22 } 23 24 __bpf_kfunc 25 __u8 *bpf_io_uring_get_region(struct iou_ctx *loop_ctx, __u32 region_id, 26 const size_t rdwr_buf_size) 27 { 28 struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx); 29 struct io_mapped_region *r; 30 31 lockdep_assert_held(&ctx->uring_lock); 32 33 switch (region_id) { 34 case IOU_REGION_MEM: 35 r = &ctx->param_region; 36 break; 37 case IOU_REGION_CQ: 38 r = &ctx->ring_region; 39 break; 40 case IOU_REGION_SQ: 41 r = &ctx->sq_region; 42 break; 43 default: 44 return NULL; 45 } 46 47 if (unlikely(rdwr_buf_size > io_region_size(r))) 48 return NULL; 49 return io_region_get_ptr(r); 50 } 51 52 __bpf_kfunc_end_defs(); 53 54 BTF_KFUNCS_START(io_uring_kfunc_set) 55 BTF_ID_FLAGS(func, bpf_io_uring_submit_sqes, KF_SLEEPABLE); 56 BTF_ID_FLAGS(func, bpf_io_uring_get_region, KF_RET_NULL); 57 BTF_KFUNCS_END(io_uring_kfunc_set) 58 59 static const struct btf_kfunc_id_set bpf_io_uring_kfunc_set = { 60 .owner = THIS_MODULE, 61 .set = &io_uring_kfunc_set, 62 }; 63 64 static int io_bpf_ops__loop_step(struct iou_ctx *ctx, 65 struct iou_loop_params *lp) 66 { 67 return IOU_LOOP_STOP; 68 } 69 70 static struct io_uring_bpf_ops io_bpf_ops_stubs = { 71 .loop_step = io_bpf_ops__loop_step, 72 }; 73 74 static bool bpf_io_is_valid_access(int off, int size, 75 enum bpf_access_type type, 76 const struct bpf_prog *prog, 77 struct bpf_insn_access_aux *info) 78 { 79 if (type != BPF_READ) 80 return false; 81 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 82 return false; 83 if (off % size != 0) 84 return false; 85 86 return btf_ctx_access(off, size, type, prog, info); 87 } 88 89 static int bpf_io_btf_struct_access(struct bpf_verifier_log *log, 90 const struct bpf_reg_state *reg, int off, 91 int size) 92 { 93 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 94 95 if (t == loop_params_type) { 96 if (off + size <= offsetofend(struct iou_loop_params, cq_wait_idx)) 97 return SCALAR_VALUE; 98 } 99 100 return -EACCES; 101 } 102 103 static const struct bpf_verifier_ops bpf_io_verifier_ops = { 104 .get_func_proto = bpf_base_func_proto, 105 .is_valid_access = bpf_io_is_valid_access, 106 .btf_struct_access = bpf_io_btf_struct_access, 107 }; 108 109 static const struct btf_type * 110 io_lookup_struct_type(struct btf *btf, const char *name) 111 { 112 s32 type_id; 113 114 type_id = btf_find_by_name_kind(btf, name, BTF_KIND_STRUCT); 115 if (type_id < 0) 116 return NULL; 117 return btf_type_by_id(btf, type_id); 118 } 119 120 static int bpf_io_init(struct btf *btf) 121 { 122 int ret; 123 124 loop_params_type = io_lookup_struct_type(btf, "iou_loop_params"); 125 if (!loop_params_type) { 126 pr_err("io_uring: Failed to locate iou_loop_params\n"); 127 return -EINVAL; 128 } 129 130 ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 131 &bpf_io_uring_kfunc_set); 132 if (ret) { 133 pr_err("io_uring: Failed to register kfuncs (%d)\n", ret); 134 return ret; 135 } 136 return 0; 137 } 138 139 static int bpf_io_check_member(const struct btf_type *t, 140 const struct btf_member *member, 141 const struct bpf_prog *prog) 142 { 143 return 0; 144 } 145 146 static int bpf_io_init_member(const struct btf_type *t, 147 const struct btf_member *member, 148 void *kdata, const void *udata) 149 { 150 u32 moff = __btf_member_bit_offset(t, member) / 8; 151 const struct io_uring_bpf_ops *uops = udata; 152 struct io_uring_bpf_ops *ops = kdata; 153 154 switch (moff) { 155 case offsetof(struct io_uring_bpf_ops, ring_fd): 156 ops->ring_fd = uops->ring_fd; 157 return 1; 158 } 159 return 0; 160 } 161 162 static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops) 163 { 164 if (ctx->flags & (IORING_SETUP_SQPOLL | IORING_SETUP_IOPOLL)) 165 return -EOPNOTSUPP; 166 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 167 return -EOPNOTSUPP; 168 169 if (ctx->bpf_ops) 170 return -EBUSY; 171 if (ops->priv) 172 return -EBUSY; 173 if (WARN_ON_ONCE(!ops->loop_step)) 174 return -EINVAL; 175 176 ops->priv = ctx; 177 ctx->bpf_ops = ops; 178 ctx->loop_step = ops->loop_step; 179 return 0; 180 } 181 182 static int bpf_io_reg(void *kdata, struct bpf_link *link) 183 { 184 struct io_uring_bpf_ops *ops = kdata; 185 struct io_ring_ctx *ctx; 186 struct file *file; 187 int ret = -EBUSY; 188 189 file = io_uring_ctx_get_file(ops->ring_fd, false); 190 if (IS_ERR(file)) 191 return PTR_ERR(file); 192 ctx = file->private_data; 193 194 scoped_guard(mutex, &io_bpf_ctrl_mutex) { 195 guard(mutex)(&ctx->uring_lock); 196 ret = io_install_bpf(ctx, ops); 197 } 198 199 fput(file); 200 return ret; 201 } 202 203 static void io_eject_bpf(struct io_ring_ctx *ctx) 204 { 205 struct io_uring_bpf_ops *ops = ctx->bpf_ops; 206 207 if (WARN_ON_ONCE(!ops)) 208 return; 209 if (WARN_ON_ONCE(ops->priv != ctx)) 210 return; 211 212 ops->priv = NULL; 213 ctx->bpf_ops = NULL; 214 ctx->loop_step = NULL; 215 } 216 217 static void bpf_io_unreg(void *kdata, struct bpf_link *link) 218 { 219 struct io_uring_bpf_ops *ops = kdata; 220 struct io_ring_ctx *ctx; 221 222 guard(mutex)(&io_bpf_ctrl_mutex); 223 ctx = ops->priv; 224 if (ctx) { 225 guard(mutex)(&ctx->uring_lock); 226 if (WARN_ON_ONCE(ctx->bpf_ops != ops)) 227 return; 228 229 io_eject_bpf(ctx); 230 } 231 } 232 233 void io_unregister_bpf_ops(struct io_ring_ctx *ctx) 234 { 235 /* 236 * ->bpf_ops is write protected by io_bpf_ctrl_mutex and uring_lock, 237 * and read protected by either. Try to avoid taking the global lock 238 * for rings that never had any bpf installed. 239 */ 240 scoped_guard(mutex, &ctx->uring_lock) { 241 if (!ctx->bpf_ops) 242 return; 243 } 244 245 guard(mutex)(&io_bpf_ctrl_mutex); 246 guard(mutex)(&ctx->uring_lock); 247 if (ctx->bpf_ops) 248 io_eject_bpf(ctx); 249 } 250 251 static struct bpf_struct_ops bpf_ring_ops = { 252 .verifier_ops = &bpf_io_verifier_ops, 253 .reg = bpf_io_reg, 254 .unreg = bpf_io_unreg, 255 .check_member = bpf_io_check_member, 256 .init_member = bpf_io_init_member, 257 .init = bpf_io_init, 258 .cfi_stubs = &io_bpf_ops_stubs, 259 .name = "io_uring_bpf_ops", 260 .owner = THIS_MODULE, 261 }; 262 263 static int __init io_uring_bpf_init(void) 264 { 265 int ret; 266 267 ret = register_bpf_struct_ops(&bpf_ring_ops, io_uring_bpf_ops); 268 if (ret) { 269 pr_err("io_uring: Failed to register struct_ops (%d)\n", ret); 270 return ret; 271 } 272 273 return 0; 274 } 275 __initcall(io_uring_bpf_init); 276