1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/types.h> 4 #include <linux/bpf_verifier.h> 5 #include <linux/bpf.h> 6 #include <linux/btf.h> 7 #include <linux/filter.h> 8 #include <net/pkt_sched.h> 9 #include <net/pkt_cls.h> 10 11 #define QDISC_OP_IDX(op) (offsetof(struct Qdisc_ops, op) / sizeof(void (*)(void))) 12 #define QDISC_MOFF_IDX(moff) (moff / sizeof(void (*)(void))) 13 14 static struct bpf_struct_ops bpf_Qdisc_ops; 15 16 struct bpf_sched_data { 17 struct qdisc_watchdog watchdog; 18 }; 19 20 struct bpf_sk_buff_ptr { 21 struct sk_buff *skb; 22 }; 23 24 static int bpf_qdisc_init(struct btf *btf) 25 { 26 return 0; 27 } 28 29 BTF_ID_LIST_SINGLE(bpf_qdisc_ids, struct, Qdisc) 30 BTF_ID_LIST_SINGLE(bpf_sk_buff_ids, struct, sk_buff) 31 BTF_ID_LIST_SINGLE(bpf_sk_buff_ptr_ids, struct, bpf_sk_buff_ptr) 32 33 static bool bpf_qdisc_is_valid_access(int off, int size, 34 enum bpf_access_type type, 35 const struct bpf_prog *prog, 36 struct bpf_insn_access_aux *info) 37 { 38 struct btf *btf = prog->aux->attach_btf; 39 u32 arg; 40 41 arg = btf_ctx_arg_idx(btf, prog->aux->attach_func_proto, off); 42 if (prog->aux->attach_st_ops_member_off == offsetof(struct Qdisc_ops, enqueue)) { 43 if (arg == 2 && type == BPF_READ) { 44 info->reg_type = PTR_TO_BTF_ID | PTR_TRUSTED; 45 info->btf = btf; 46 info->btf_id = bpf_sk_buff_ptr_ids[0]; 47 return true; 48 } 49 } 50 51 return bpf_tracing_btf_ctx_access(off, size, type, prog, info); 52 } 53 54 static int bpf_qdisc_qdisc_access(struct bpf_verifier_log *log, 55 const struct bpf_reg_state *reg, 56 int off, size_t *end) 57 { 58 switch (off) { 59 case offsetof(struct Qdisc, limit): 60 *end = offsetofend(struct Qdisc, limit); 61 break; 62 case offsetof(struct Qdisc, q) + offsetof(struct qdisc_skb_head, qlen): 63 *end = offsetof(struct Qdisc, q) + offsetofend(struct qdisc_skb_head, qlen); 64 break; 65 case offsetof(struct Qdisc, qstats) ... offsetofend(struct Qdisc, qstats) - 1: 66 *end = offsetofend(struct Qdisc, qstats); 67 break; 68 default: 69 return -EACCES; 70 } 71 72 return 0; 73 } 74 75 static int bpf_qdisc_sk_buff_access(struct bpf_verifier_log *log, 76 const struct bpf_reg_state *reg, 77 int off, size_t *end) 78 { 79 switch (off) { 80 case offsetof(struct sk_buff, tstamp): 81 *end = offsetofend(struct sk_buff, tstamp); 82 break; 83 case offsetof(struct sk_buff, cb) + offsetof(struct qdisc_skb_cb, data[0]) ... 84 offsetof(struct sk_buff, cb) + offsetof(struct qdisc_skb_cb, 85 data[QDISC_CB_PRIV_LEN - 1]): 86 *end = offsetof(struct sk_buff, cb) + 87 offsetofend(struct qdisc_skb_cb, data[QDISC_CB_PRIV_LEN - 1]); 88 break; 89 default: 90 return -EACCES; 91 } 92 93 return 0; 94 } 95 96 static int bpf_qdisc_btf_struct_access(struct bpf_verifier_log *log, 97 const struct bpf_reg_state *reg, 98 int off, int size) 99 { 100 const struct btf_type *t, *skbt, *qdisct; 101 size_t end; 102 int err; 103 104 skbt = btf_type_by_id(reg->btf, bpf_sk_buff_ids[0]); 105 qdisct = btf_type_by_id(reg->btf, bpf_qdisc_ids[0]); 106 t = btf_type_by_id(reg->btf, reg->btf_id); 107 108 if (t == skbt) { 109 err = bpf_qdisc_sk_buff_access(log, reg, off, &end); 110 } else if (t == qdisct) { 111 err = bpf_qdisc_qdisc_access(log, reg, off, &end); 112 } else { 113 bpf_log(log, "only read is supported\n"); 114 return -EACCES; 115 } 116 117 if (err) { 118 bpf_log(log, "no write support to %s at off %d\n", 119 btf_name_by_offset(reg->btf, t->name_off), off); 120 return -EACCES; 121 } 122 123 if (off + size > end) { 124 bpf_log(log, 125 "write access at off %d with size %d beyond the member of %s ended at %zu\n", 126 off, size, btf_name_by_offset(reg->btf, t->name_off), end); 127 return -EACCES; 128 } 129 130 return 0; 131 } 132 133 BTF_ID_LIST_SINGLE(bpf_qdisc_init_prologue_ids, func, bpf_qdisc_init_prologue) 134 135 static int bpf_qdisc_gen_prologue(struct bpf_insn *insn_buf, bool direct_write, 136 const struct bpf_prog *prog) 137 { 138 struct bpf_insn *insn = insn_buf; 139 140 if (prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, init)) 141 return 0; 142 143 /* r6 = r1; // r6 will be "u64 *ctx". r1 is "u64 *ctx". 144 * r2 = r1[16]; // r2 will be "struct netlink_ext_ack *extack" 145 * r1 = r1[0]; // r1 will be "struct Qdisc *sch" 146 * r0 = bpf_qdisc_init_prologue(r1, r2); 147 * if r0 == 0 goto pc+1; 148 * BPF_EXIT; 149 * r1 = r6; // r1 will be "u64 *ctx". 150 */ 151 *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); 152 *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 16); 153 *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 154 *insn++ = BPF_CALL_KFUNC(0, bpf_qdisc_init_prologue_ids[0]); 155 *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1); 156 *insn++ = BPF_EXIT_INSN(); 157 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6); 158 *insn++ = prog->insnsi[0]; 159 160 return insn - insn_buf; 161 } 162 163 BTF_ID_LIST_SINGLE(bpf_qdisc_reset_destroy_epilogue_ids, func, bpf_qdisc_reset_destroy_epilogue) 164 165 static int bpf_qdisc_gen_epilogue(struct bpf_insn *insn_buf, const struct bpf_prog *prog, 166 s16 ctx_stack_off) 167 { 168 struct bpf_insn *insn = insn_buf; 169 170 if (prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, reset) && 171 prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, destroy)) 172 return 0; 173 174 /* r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx" 175 * r1 = r1[0]; // r1 will be "struct Qdisc *sch" 176 * r0 = bpf_qdisc_reset_destroy_epilogue(r1); 177 * BPF_EXIT; 178 */ 179 *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_FP, ctx_stack_off); 180 *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 181 *insn++ = BPF_CALL_KFUNC(0, bpf_qdisc_reset_destroy_epilogue_ids[0]); 182 *insn++ = BPF_EXIT_INSN(); 183 184 return insn - insn_buf; 185 } 186 187 __bpf_kfunc_start_defs(); 188 189 /* bpf_skb_get_hash - Get the flow hash of an skb. 190 * @skb: The skb to get the flow hash from. 191 */ 192 __bpf_kfunc u32 bpf_skb_get_hash(struct sk_buff *skb) 193 { 194 return skb_get_hash(skb); 195 } 196 197 /* bpf_kfree_skb - Release an skb's reference and drop it immediately. 198 * @skb: The skb whose reference to be released and dropped. 199 */ 200 __bpf_kfunc void bpf_kfree_skb(struct sk_buff *skb) 201 { 202 kfree_skb(skb); 203 } 204 205 __bpf_kfunc void bpf_kfree_skb_dtor(void *skb) 206 { 207 bpf_kfree_skb(skb); 208 } 209 CFI_NOSEAL(bpf_kfree_skb_dtor); 210 211 /* bpf_qdisc_skb_drop - Drop an skb by adding it to a deferred free list. 212 * @skb: The skb whose reference to be released and dropped. 213 * @to_free_list: The list of skbs to be dropped. 214 */ 215 __bpf_kfunc void bpf_qdisc_skb_drop(struct sk_buff *skb, 216 struct bpf_sk_buff_ptr *to_free_list) 217 { 218 __qdisc_drop(skb, (struct sk_buff **)to_free_list); 219 } 220 221 /* bpf_qdisc_watchdog_schedule - Schedule a qdisc to a later time using a timer. 222 * @sch: The qdisc to be scheduled. 223 * @expire: The expiry time of the timer. 224 * @delta_ns: The slack range of the timer. 225 */ 226 __bpf_kfunc void bpf_qdisc_watchdog_schedule(struct Qdisc *sch, u64 expire, u64 delta_ns) 227 { 228 struct bpf_sched_data *q = qdisc_priv(sch); 229 230 qdisc_watchdog_schedule_range_ns(&q->watchdog, expire, delta_ns); 231 } 232 233 /* bpf_qdisc_init_prologue - Hidden kfunc called in prologue of .init. */ 234 __bpf_kfunc int bpf_qdisc_init_prologue(struct Qdisc *sch, 235 struct netlink_ext_ack *extack) 236 { 237 struct bpf_sched_data *q = qdisc_priv(sch); 238 struct net_device *dev = qdisc_dev(sch); 239 struct Qdisc *p; 240 241 qdisc_watchdog_init(&q->watchdog, sch); 242 243 if (sch->parent != TC_H_ROOT) { 244 /* If qdisc_lookup() returns NULL, it means .init is called by 245 * qdisc_create_dflt() in mq/mqprio_init and the parent qdisc 246 * has not been added to qdisc_hash yet. 247 */ 248 p = qdisc_lookup(dev, TC_H_MAJ(sch->parent)); 249 if (p && !(p->flags & TCQ_F_MQROOT)) { 250 NL_SET_ERR_MSG(extack, "BPF qdisc only supported on root or mq"); 251 return -EINVAL; 252 } 253 } 254 255 return 0; 256 } 257 258 /* bpf_qdisc_reset_destroy_epilogue - Hidden kfunc called in epilogue of .reset 259 * and .destroy 260 */ 261 __bpf_kfunc void bpf_qdisc_reset_destroy_epilogue(struct Qdisc *sch) 262 { 263 struct bpf_sched_data *q = qdisc_priv(sch); 264 265 qdisc_watchdog_cancel(&q->watchdog); 266 } 267 268 /* bpf_qdisc_bstats_update - Update Qdisc basic statistics 269 * @sch: The qdisc from which an skb is dequeued. 270 * @skb: The skb to be dequeued. 271 */ 272 __bpf_kfunc void bpf_qdisc_bstats_update(struct Qdisc *sch, const struct sk_buff *skb) 273 { 274 bstats_update(&sch->bstats, skb); 275 } 276 277 __bpf_kfunc_end_defs(); 278 279 BTF_KFUNCS_START(qdisc_kfunc_ids) 280 BTF_ID_FLAGS(func, bpf_skb_get_hash) 281 BTF_ID_FLAGS(func, bpf_kfree_skb, KF_RELEASE) 282 BTF_ID_FLAGS(func, bpf_qdisc_skb_drop, KF_RELEASE) 283 BTF_ID_FLAGS(func, bpf_dynptr_from_skb) 284 BTF_ID_FLAGS(func, bpf_qdisc_watchdog_schedule) 285 BTF_ID_FLAGS(func, bpf_qdisc_init_prologue) 286 BTF_ID_FLAGS(func, bpf_qdisc_reset_destroy_epilogue) 287 BTF_ID_FLAGS(func, bpf_qdisc_bstats_update) 288 BTF_KFUNCS_END(qdisc_kfunc_ids) 289 290 BTF_SET_START(qdisc_common_kfunc_set) 291 BTF_ID(func, bpf_skb_get_hash) 292 BTF_ID(func, bpf_kfree_skb) 293 BTF_ID(func, bpf_dynptr_from_skb) 294 BTF_SET_END(qdisc_common_kfunc_set) 295 296 BTF_SET_START(qdisc_enqueue_kfunc_set) 297 BTF_ID(func, bpf_qdisc_skb_drop) 298 BTF_ID(func, bpf_qdisc_watchdog_schedule) 299 BTF_SET_END(qdisc_enqueue_kfunc_set) 300 301 BTF_SET_START(qdisc_dequeue_kfunc_set) 302 BTF_ID(func, bpf_qdisc_watchdog_schedule) 303 BTF_ID(func, bpf_qdisc_bstats_update) 304 BTF_SET_END(qdisc_dequeue_kfunc_set) 305 306 enum qdisc_ops_kf_flags { 307 QDISC_OPS_KF_COMMON = 0, 308 QDISC_OPS_KF_ENQUEUE = 1 << 0, 309 QDISC_OPS_KF_DEQUEUE = 1 << 1, 310 }; 311 312 static const u32 qdisc_ops_context_flags[] = { 313 [QDISC_OP_IDX(enqueue)] = QDISC_OPS_KF_ENQUEUE, 314 [QDISC_OP_IDX(dequeue)] = QDISC_OPS_KF_DEQUEUE, 315 [QDISC_OP_IDX(init)] = QDISC_OPS_KF_COMMON, 316 [QDISC_OP_IDX(reset)] = QDISC_OPS_KF_COMMON, 317 [QDISC_OP_IDX(destroy)] = QDISC_OPS_KF_COMMON, 318 }; 319 320 static int bpf_qdisc_kfunc_filter(const struct bpf_prog *prog, u32 kfunc_id) 321 { 322 u32 moff, flags; 323 324 if (!btf_id_set8_contains(&qdisc_kfunc_ids, kfunc_id)) 325 return 0; 326 327 if (prog->aux->st_ops != &bpf_Qdisc_ops) 328 return -EACCES; 329 330 moff = prog->aux->attach_st_ops_member_off; 331 flags = qdisc_ops_context_flags[QDISC_MOFF_IDX(moff)]; 332 333 if ((flags & QDISC_OPS_KF_ENQUEUE) && 334 btf_id_set_contains(&qdisc_enqueue_kfunc_set, kfunc_id)) 335 return 0; 336 337 if ((flags & QDISC_OPS_KF_DEQUEUE) && 338 btf_id_set_contains(&qdisc_dequeue_kfunc_set, kfunc_id)) 339 return 0; 340 341 if (btf_id_set_contains(&qdisc_common_kfunc_set, kfunc_id)) 342 return 0; 343 344 return -EACCES; 345 } 346 347 static const struct btf_kfunc_id_set bpf_qdisc_kfunc_set = { 348 .owner = THIS_MODULE, 349 .set = &qdisc_kfunc_ids, 350 .filter = bpf_qdisc_kfunc_filter, 351 }; 352 353 static const struct bpf_verifier_ops bpf_qdisc_verifier_ops = { 354 .get_func_proto = bpf_base_func_proto, 355 .is_valid_access = bpf_qdisc_is_valid_access, 356 .btf_struct_access = bpf_qdisc_btf_struct_access, 357 .gen_prologue = bpf_qdisc_gen_prologue, 358 .gen_epilogue = bpf_qdisc_gen_epilogue, 359 }; 360 361 static int bpf_qdisc_init_member(const struct btf_type *t, 362 const struct btf_member *member, 363 void *kdata, const void *udata) 364 { 365 const struct Qdisc_ops *uqdisc_ops; 366 struct Qdisc_ops *qdisc_ops; 367 u32 moff; 368 369 uqdisc_ops = (const struct Qdisc_ops *)udata; 370 qdisc_ops = (struct Qdisc_ops *)kdata; 371 372 moff = __btf_member_bit_offset(t, member) / 8; 373 switch (moff) { 374 case offsetof(struct Qdisc_ops, priv_size): 375 if (uqdisc_ops->priv_size) 376 return -EINVAL; 377 qdisc_ops->priv_size = sizeof(struct bpf_sched_data); 378 return 1; 379 case offsetof(struct Qdisc_ops, peek): 380 qdisc_ops->peek = qdisc_peek_dequeued; 381 return 0; 382 case offsetof(struct Qdisc_ops, id): 383 if (bpf_obj_name_cpy(qdisc_ops->id, uqdisc_ops->id, 384 sizeof(qdisc_ops->id)) <= 0) 385 return -EINVAL; 386 return 1; 387 } 388 389 return 0; 390 } 391 392 static int bpf_qdisc_reg(void *kdata, struct bpf_link *link) 393 { 394 return register_qdisc(kdata); 395 } 396 397 static void bpf_qdisc_unreg(void *kdata, struct bpf_link *link) 398 { 399 return unregister_qdisc(kdata); 400 } 401 402 static int bpf_qdisc_validate(void *kdata) 403 { 404 struct Qdisc_ops *ops = (struct Qdisc_ops *)kdata; 405 406 if (!ops->enqueue || !ops->dequeue || !ops->init || 407 !ops->reset || !ops->destroy) 408 return -EINVAL; 409 410 return 0; 411 } 412 413 static int Qdisc_ops__enqueue(struct sk_buff *skb__ref, struct Qdisc *sch, 414 struct sk_buff **to_free) 415 { 416 return 0; 417 } 418 419 static struct sk_buff *Qdisc_ops__dequeue(struct Qdisc *sch) 420 { 421 return NULL; 422 } 423 424 static int Qdisc_ops__init(struct Qdisc *sch, struct nlattr *arg, 425 struct netlink_ext_ack *extack) 426 { 427 return 0; 428 } 429 430 static void Qdisc_ops__reset(struct Qdisc *sch) 431 { 432 } 433 434 static void Qdisc_ops__destroy(struct Qdisc *sch) 435 { 436 } 437 438 static struct Qdisc_ops __bpf_ops_qdisc_ops = { 439 .enqueue = Qdisc_ops__enqueue, 440 .dequeue = Qdisc_ops__dequeue, 441 .init = Qdisc_ops__init, 442 .reset = Qdisc_ops__reset, 443 .destroy = Qdisc_ops__destroy, 444 }; 445 446 static struct bpf_struct_ops bpf_Qdisc_ops = { 447 .verifier_ops = &bpf_qdisc_verifier_ops, 448 .reg = bpf_qdisc_reg, 449 .unreg = bpf_qdisc_unreg, 450 .validate = bpf_qdisc_validate, 451 .init_member = bpf_qdisc_init_member, 452 .init = bpf_qdisc_init, 453 .name = "Qdisc_ops", 454 .cfi_stubs = &__bpf_ops_qdisc_ops, 455 .owner = THIS_MODULE, 456 }; 457 458 BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb_dtor) 459 460 static int __init bpf_qdisc_kfunc_init(void) 461 { 462 int ret; 463 const struct btf_id_dtor_kfunc skb_kfunc_dtors[] = { 464 { 465 .btf_id = bpf_sk_buff_ids[0], 466 .kfunc_btf_id = bpf_sk_buff_dtor_ids[0] 467 }, 468 }; 469 470 ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_qdisc_kfunc_set); 471 ret = ret ?: register_btf_id_dtor_kfuncs(skb_kfunc_dtors, 472 ARRAY_SIZE(skb_kfunc_dtors), 473 THIS_MODULE); 474 ret = ret ?: register_bpf_struct_ops(&bpf_Qdisc_ops, Qdisc_ops); 475 476 return ret; 477 } 478 late_initcall(bpf_qdisc_kfunc_init); 479