1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/types.h>
4 #include <linux/bpf_verifier.h>
5 #include <linux/bpf.h>
6 #include <linux/btf.h>
7 #include <linux/filter.h>
8 #include <net/pkt_sched.h>
9 #include <net/pkt_cls.h>
10
11 #define QDISC_OP_IDX(op) (offsetof(struct Qdisc_ops, op) / sizeof(void (*)(void)))
12 #define QDISC_MOFF_IDX(moff) (moff / sizeof(void (*)(void)))
13
14 static struct bpf_struct_ops bpf_Qdisc_ops;
15
16 struct bpf_sched_data {
17 struct qdisc_watchdog watchdog;
18 };
19
20 struct bpf_sk_buff_ptr {
21 struct sk_buff *skb;
22 };
23
bpf_qdisc_init(struct btf * btf)24 static int bpf_qdisc_init(struct btf *btf)
25 {
26 return 0;
27 }
28
BTF_ID_LIST_SINGLE(bpf_qdisc_ids,struct,Qdisc)29 BTF_ID_LIST_SINGLE(bpf_qdisc_ids, struct, Qdisc)
30 BTF_ID_LIST_SINGLE(bpf_sk_buff_ids, struct, sk_buff)
31 BTF_ID_LIST_SINGLE(bpf_sk_buff_ptr_ids, struct, bpf_sk_buff_ptr)
32
33 static bool bpf_qdisc_is_valid_access(int off, int size,
34 enum bpf_access_type type,
35 const struct bpf_prog *prog,
36 struct bpf_insn_access_aux *info)
37 {
38 struct btf *btf = prog->aux->attach_btf;
39 u32 arg;
40
41 arg = btf_ctx_arg_idx(btf, prog->aux->attach_func_proto, off);
42 if (prog->aux->attach_st_ops_member_off == offsetof(struct Qdisc_ops, enqueue)) {
43 if (arg == 2 && type == BPF_READ) {
44 info->reg_type = PTR_TO_BTF_ID | PTR_TRUSTED;
45 info->btf = btf;
46 info->btf_id = bpf_sk_buff_ptr_ids[0];
47 return true;
48 }
49 }
50
51 return bpf_tracing_btf_ctx_access(off, size, type, prog, info);
52 }
53
bpf_qdisc_qdisc_access(struct bpf_verifier_log * log,const struct bpf_reg_state * reg,int off,size_t * end)54 static int bpf_qdisc_qdisc_access(struct bpf_verifier_log *log,
55 const struct bpf_reg_state *reg,
56 int off, size_t *end)
57 {
58 switch (off) {
59 case offsetof(struct Qdisc, limit):
60 *end = offsetofend(struct Qdisc, limit);
61 break;
62 case offsetof(struct Qdisc, q) + offsetof(struct qdisc_skb_head, qlen):
63 *end = offsetof(struct Qdisc, q) + offsetofend(struct qdisc_skb_head, qlen);
64 break;
65 case offsetof(struct Qdisc, qstats) ... offsetofend(struct Qdisc, qstats) - 1:
66 *end = offsetofend(struct Qdisc, qstats);
67 break;
68 default:
69 return -EACCES;
70 }
71
72 return 0;
73 }
74
bpf_qdisc_sk_buff_access(struct bpf_verifier_log * log,const struct bpf_reg_state * reg,int off,size_t * end)75 static int bpf_qdisc_sk_buff_access(struct bpf_verifier_log *log,
76 const struct bpf_reg_state *reg,
77 int off, size_t *end)
78 {
79 switch (off) {
80 case offsetof(struct sk_buff, tstamp):
81 *end = offsetofend(struct sk_buff, tstamp);
82 break;
83 case offsetof(struct sk_buff, cb) + offsetof(struct qdisc_skb_cb, data[0]) ...
84 offsetof(struct sk_buff, cb) + offsetof(struct qdisc_skb_cb,
85 data[QDISC_CB_PRIV_LEN - 1]):
86 *end = offsetof(struct sk_buff, cb) +
87 offsetofend(struct qdisc_skb_cb, data[QDISC_CB_PRIV_LEN - 1]);
88 break;
89 default:
90 return -EACCES;
91 }
92
93 return 0;
94 }
95
bpf_qdisc_btf_struct_access(struct bpf_verifier_log * log,const struct bpf_reg_state * reg,int off,int size)96 static int bpf_qdisc_btf_struct_access(struct bpf_verifier_log *log,
97 const struct bpf_reg_state *reg,
98 int off, int size)
99 {
100 const struct btf_type *t, *skbt, *qdisct;
101 size_t end;
102 int err;
103
104 skbt = btf_type_by_id(reg->btf, bpf_sk_buff_ids[0]);
105 qdisct = btf_type_by_id(reg->btf, bpf_qdisc_ids[0]);
106 t = btf_type_by_id(reg->btf, reg->btf_id);
107
108 if (t == skbt) {
109 err = bpf_qdisc_sk_buff_access(log, reg, off, &end);
110 } else if (t == qdisct) {
111 err = bpf_qdisc_qdisc_access(log, reg, off, &end);
112 } else {
113 bpf_log(log, "only read is supported\n");
114 return -EACCES;
115 }
116
117 if (err) {
118 bpf_log(log, "no write support to %s at off %d\n",
119 btf_name_by_offset(reg->btf, t->name_off), off);
120 return -EACCES;
121 }
122
123 if (off + size > end) {
124 bpf_log(log,
125 "write access at off %d with size %d beyond the member of %s ended at %zu\n",
126 off, size, btf_name_by_offset(reg->btf, t->name_off), end);
127 return -EACCES;
128 }
129
130 return 0;
131 }
132
BTF_ID_LIST_SINGLE(bpf_qdisc_init_prologue_ids,func,bpf_qdisc_init_prologue)133 BTF_ID_LIST_SINGLE(bpf_qdisc_init_prologue_ids, func, bpf_qdisc_init_prologue)
134
135 static int bpf_qdisc_gen_prologue(struct bpf_insn *insn_buf, bool direct_write,
136 const struct bpf_prog *prog)
137 {
138 struct bpf_insn *insn = insn_buf;
139
140 if (prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, init))
141 return 0;
142
143 /* r6 = r1; // r6 will be "u64 *ctx". r1 is "u64 *ctx".
144 * r2 = r1[16]; // r2 will be "struct netlink_ext_ack *extack"
145 * r1 = r1[0]; // r1 will be "struct Qdisc *sch"
146 * r0 = bpf_qdisc_init_prologue(r1, r2);
147 * if r0 == 0 goto pc+1;
148 * BPF_EXIT;
149 * r1 = r6; // r1 will be "u64 *ctx".
150 */
151 *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
152 *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 16);
153 *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
154 *insn++ = BPF_CALL_KFUNC(0, bpf_qdisc_init_prologue_ids[0]);
155 *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1);
156 *insn++ = BPF_EXIT_INSN();
157 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
158 *insn++ = prog->insnsi[0];
159
160 return insn - insn_buf;
161 }
162
BTF_ID_LIST_SINGLE(bpf_qdisc_reset_destroy_epilogue_ids,func,bpf_qdisc_reset_destroy_epilogue)163 BTF_ID_LIST_SINGLE(bpf_qdisc_reset_destroy_epilogue_ids, func, bpf_qdisc_reset_destroy_epilogue)
164
165 static int bpf_qdisc_gen_epilogue(struct bpf_insn *insn_buf, const struct bpf_prog *prog,
166 s16 ctx_stack_off)
167 {
168 struct bpf_insn *insn = insn_buf;
169
170 if (prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, reset) &&
171 prog->aux->attach_st_ops_member_off != offsetof(struct Qdisc_ops, destroy))
172 return 0;
173
174 /* r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"
175 * r1 = r1[0]; // r1 will be "struct Qdisc *sch"
176 * r0 = bpf_qdisc_reset_destroy_epilogue(r1);
177 * BPF_EXIT;
178 */
179 *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_FP, ctx_stack_off);
180 *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
181 *insn++ = BPF_CALL_KFUNC(0, bpf_qdisc_reset_destroy_epilogue_ids[0]);
182 *insn++ = BPF_EXIT_INSN();
183
184 return insn - insn_buf;
185 }
186
187 __bpf_kfunc_start_defs();
188
189 /* bpf_skb_get_hash - Get the flow hash of an skb.
190 * @skb: The skb to get the flow hash from.
191 */
bpf_skb_get_hash(struct sk_buff * skb)192 __bpf_kfunc u32 bpf_skb_get_hash(struct sk_buff *skb)
193 {
194 return skb_get_hash(skb);
195 }
196
197 /* bpf_kfree_skb - Release an skb's reference and drop it immediately.
198 * @skb: The skb whose reference to be released and dropped.
199 */
bpf_kfree_skb(struct sk_buff * skb)200 __bpf_kfunc void bpf_kfree_skb(struct sk_buff *skb)
201 {
202 kfree_skb(skb);
203 }
204
205 /* bpf_qdisc_skb_drop - Drop an skb by adding it to a deferred free list.
206 * @skb: The skb whose reference to be released and dropped.
207 * @to_free_list: The list of skbs to be dropped.
208 */
bpf_qdisc_skb_drop(struct sk_buff * skb,struct bpf_sk_buff_ptr * to_free_list)209 __bpf_kfunc void bpf_qdisc_skb_drop(struct sk_buff *skb,
210 struct bpf_sk_buff_ptr *to_free_list)
211 {
212 __qdisc_drop(skb, (struct sk_buff **)to_free_list);
213 }
214
215 /* bpf_qdisc_watchdog_schedule - Schedule a qdisc to a later time using a timer.
216 * @sch: The qdisc to be scheduled.
217 * @expire: The expiry time of the timer.
218 * @delta_ns: The slack range of the timer.
219 */
bpf_qdisc_watchdog_schedule(struct Qdisc * sch,u64 expire,u64 delta_ns)220 __bpf_kfunc void bpf_qdisc_watchdog_schedule(struct Qdisc *sch, u64 expire, u64 delta_ns)
221 {
222 struct bpf_sched_data *q = qdisc_priv(sch);
223
224 qdisc_watchdog_schedule_range_ns(&q->watchdog, expire, delta_ns);
225 }
226
227 /* bpf_qdisc_init_prologue - Hidden kfunc called in prologue of .init. */
bpf_qdisc_init_prologue(struct Qdisc * sch,struct netlink_ext_ack * extack)228 __bpf_kfunc int bpf_qdisc_init_prologue(struct Qdisc *sch,
229 struct netlink_ext_ack *extack)
230 {
231 struct bpf_sched_data *q = qdisc_priv(sch);
232 struct net_device *dev = qdisc_dev(sch);
233 struct Qdisc *p;
234
235 qdisc_watchdog_init(&q->watchdog, sch);
236
237 if (sch->parent != TC_H_ROOT) {
238 /* If qdisc_lookup() returns NULL, it means .init is called by
239 * qdisc_create_dflt() in mq/mqprio_init and the parent qdisc
240 * has not been added to qdisc_hash yet.
241 */
242 p = qdisc_lookup(dev, TC_H_MAJ(sch->parent));
243 if (p && !(p->flags & TCQ_F_MQROOT)) {
244 NL_SET_ERR_MSG(extack, "BPF qdisc only supported on root or mq");
245 return -EINVAL;
246 }
247 }
248
249 return 0;
250 }
251
252 /* bpf_qdisc_reset_destroy_epilogue - Hidden kfunc called in epilogue of .reset
253 * and .destroy
254 */
bpf_qdisc_reset_destroy_epilogue(struct Qdisc * sch)255 __bpf_kfunc void bpf_qdisc_reset_destroy_epilogue(struct Qdisc *sch)
256 {
257 struct bpf_sched_data *q = qdisc_priv(sch);
258
259 qdisc_watchdog_cancel(&q->watchdog);
260 }
261
262 /* bpf_qdisc_bstats_update - Update Qdisc basic statistics
263 * @sch: The qdisc from which an skb is dequeued.
264 * @skb: The skb to be dequeued.
265 */
bpf_qdisc_bstats_update(struct Qdisc * sch,const struct sk_buff * skb)266 __bpf_kfunc void bpf_qdisc_bstats_update(struct Qdisc *sch, const struct sk_buff *skb)
267 {
268 bstats_update(&sch->bstats, skb);
269 }
270
271 __bpf_kfunc_end_defs();
272
273 BTF_KFUNCS_START(qdisc_kfunc_ids)
274 BTF_ID_FLAGS(func, bpf_skb_get_hash, KF_TRUSTED_ARGS)
275 BTF_ID_FLAGS(func, bpf_kfree_skb, KF_RELEASE)
276 BTF_ID_FLAGS(func, bpf_qdisc_skb_drop, KF_RELEASE)
277 BTF_ID_FLAGS(func, bpf_dynptr_from_skb, KF_TRUSTED_ARGS)
278 BTF_ID_FLAGS(func, bpf_qdisc_watchdog_schedule, KF_TRUSTED_ARGS)
279 BTF_ID_FLAGS(func, bpf_qdisc_init_prologue, KF_TRUSTED_ARGS)
280 BTF_ID_FLAGS(func, bpf_qdisc_reset_destroy_epilogue, KF_TRUSTED_ARGS)
281 BTF_ID_FLAGS(func, bpf_qdisc_bstats_update, KF_TRUSTED_ARGS)
282 BTF_KFUNCS_END(qdisc_kfunc_ids)
283
284 BTF_SET_START(qdisc_common_kfunc_set)
285 BTF_ID(func, bpf_skb_get_hash)
286 BTF_ID(func, bpf_kfree_skb)
287 BTF_ID(func, bpf_dynptr_from_skb)
288 BTF_SET_END(qdisc_common_kfunc_set)
289
290 BTF_SET_START(qdisc_enqueue_kfunc_set)
291 BTF_ID(func, bpf_qdisc_skb_drop)
292 BTF_ID(func, bpf_qdisc_watchdog_schedule)
293 BTF_SET_END(qdisc_enqueue_kfunc_set)
294
295 BTF_SET_START(qdisc_dequeue_kfunc_set)
296 BTF_ID(func, bpf_qdisc_watchdog_schedule)
297 BTF_ID(func, bpf_qdisc_bstats_update)
298 BTF_SET_END(qdisc_dequeue_kfunc_set)
299
300 enum qdisc_ops_kf_flags {
301 QDISC_OPS_KF_COMMON = 0,
302 QDISC_OPS_KF_ENQUEUE = 1 << 0,
303 QDISC_OPS_KF_DEQUEUE = 1 << 1,
304 };
305
306 static const u32 qdisc_ops_context_flags[] = {
307 [QDISC_OP_IDX(enqueue)] = QDISC_OPS_KF_ENQUEUE,
308 [QDISC_OP_IDX(dequeue)] = QDISC_OPS_KF_DEQUEUE,
309 [QDISC_OP_IDX(init)] = QDISC_OPS_KF_COMMON,
310 [QDISC_OP_IDX(reset)] = QDISC_OPS_KF_COMMON,
311 [QDISC_OP_IDX(destroy)] = QDISC_OPS_KF_COMMON,
312 };
313
bpf_qdisc_kfunc_filter(const struct bpf_prog * prog,u32 kfunc_id)314 static int bpf_qdisc_kfunc_filter(const struct bpf_prog *prog, u32 kfunc_id)
315 {
316 u32 moff, flags;
317
318 if (!btf_id_set8_contains(&qdisc_kfunc_ids, kfunc_id))
319 return 0;
320
321 if (prog->aux->st_ops != &bpf_Qdisc_ops)
322 return -EACCES;
323
324 moff = prog->aux->attach_st_ops_member_off;
325 flags = qdisc_ops_context_flags[QDISC_MOFF_IDX(moff)];
326
327 if ((flags & QDISC_OPS_KF_ENQUEUE) &&
328 btf_id_set_contains(&qdisc_enqueue_kfunc_set, kfunc_id))
329 return 0;
330
331 if ((flags & QDISC_OPS_KF_DEQUEUE) &&
332 btf_id_set_contains(&qdisc_dequeue_kfunc_set, kfunc_id))
333 return 0;
334
335 if (btf_id_set_contains(&qdisc_common_kfunc_set, kfunc_id))
336 return 0;
337
338 return -EACCES;
339 }
340
341 static const struct btf_kfunc_id_set bpf_qdisc_kfunc_set = {
342 .owner = THIS_MODULE,
343 .set = &qdisc_kfunc_ids,
344 .filter = bpf_qdisc_kfunc_filter,
345 };
346
347 static const struct bpf_verifier_ops bpf_qdisc_verifier_ops = {
348 .get_func_proto = bpf_base_func_proto,
349 .is_valid_access = bpf_qdisc_is_valid_access,
350 .btf_struct_access = bpf_qdisc_btf_struct_access,
351 .gen_prologue = bpf_qdisc_gen_prologue,
352 .gen_epilogue = bpf_qdisc_gen_epilogue,
353 };
354
bpf_qdisc_init_member(const struct btf_type * t,const struct btf_member * member,void * kdata,const void * udata)355 static int bpf_qdisc_init_member(const struct btf_type *t,
356 const struct btf_member *member,
357 void *kdata, const void *udata)
358 {
359 const struct Qdisc_ops *uqdisc_ops;
360 struct Qdisc_ops *qdisc_ops;
361 u32 moff;
362
363 uqdisc_ops = (const struct Qdisc_ops *)udata;
364 qdisc_ops = (struct Qdisc_ops *)kdata;
365
366 moff = __btf_member_bit_offset(t, member) / 8;
367 switch (moff) {
368 case offsetof(struct Qdisc_ops, priv_size):
369 if (uqdisc_ops->priv_size)
370 return -EINVAL;
371 qdisc_ops->priv_size = sizeof(struct bpf_sched_data);
372 return 1;
373 case offsetof(struct Qdisc_ops, peek):
374 qdisc_ops->peek = qdisc_peek_dequeued;
375 return 0;
376 case offsetof(struct Qdisc_ops, id):
377 if (bpf_obj_name_cpy(qdisc_ops->id, uqdisc_ops->id,
378 sizeof(qdisc_ops->id)) <= 0)
379 return -EINVAL;
380 return 1;
381 }
382
383 return 0;
384 }
385
bpf_qdisc_reg(void * kdata,struct bpf_link * link)386 static int bpf_qdisc_reg(void *kdata, struct bpf_link *link)
387 {
388 return register_qdisc(kdata);
389 }
390
bpf_qdisc_unreg(void * kdata,struct bpf_link * link)391 static void bpf_qdisc_unreg(void *kdata, struct bpf_link *link)
392 {
393 return unregister_qdisc(kdata);
394 }
395
bpf_qdisc_validate(void * kdata)396 static int bpf_qdisc_validate(void *kdata)
397 {
398 struct Qdisc_ops *ops = (struct Qdisc_ops *)kdata;
399
400 if (!ops->enqueue || !ops->dequeue || !ops->init ||
401 !ops->reset || !ops->destroy)
402 return -EINVAL;
403
404 return 0;
405 }
406
Qdisc_ops__enqueue(struct sk_buff * skb__ref,struct Qdisc * sch,struct sk_buff ** to_free)407 static int Qdisc_ops__enqueue(struct sk_buff *skb__ref, struct Qdisc *sch,
408 struct sk_buff **to_free)
409 {
410 return 0;
411 }
412
Qdisc_ops__dequeue(struct Qdisc * sch)413 static struct sk_buff *Qdisc_ops__dequeue(struct Qdisc *sch)
414 {
415 return NULL;
416 }
417
Qdisc_ops__init(struct Qdisc * sch,struct nlattr * arg,struct netlink_ext_ack * extack)418 static int Qdisc_ops__init(struct Qdisc *sch, struct nlattr *arg,
419 struct netlink_ext_ack *extack)
420 {
421 return 0;
422 }
423
Qdisc_ops__reset(struct Qdisc * sch)424 static void Qdisc_ops__reset(struct Qdisc *sch)
425 {
426 }
427
Qdisc_ops__destroy(struct Qdisc * sch)428 static void Qdisc_ops__destroy(struct Qdisc *sch)
429 {
430 }
431
432 static struct Qdisc_ops __bpf_ops_qdisc_ops = {
433 .enqueue = Qdisc_ops__enqueue,
434 .dequeue = Qdisc_ops__dequeue,
435 .init = Qdisc_ops__init,
436 .reset = Qdisc_ops__reset,
437 .destroy = Qdisc_ops__destroy,
438 };
439
440 static struct bpf_struct_ops bpf_Qdisc_ops = {
441 .verifier_ops = &bpf_qdisc_verifier_ops,
442 .reg = bpf_qdisc_reg,
443 .unreg = bpf_qdisc_unreg,
444 .validate = bpf_qdisc_validate,
445 .init_member = bpf_qdisc_init_member,
446 .init = bpf_qdisc_init,
447 .name = "Qdisc_ops",
448 .cfi_stubs = &__bpf_ops_qdisc_ops,
449 .owner = THIS_MODULE,
450 };
451
BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids,func,bpf_kfree_skb)452 BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb)
453
454 static int __init bpf_qdisc_kfunc_init(void)
455 {
456 int ret;
457 const struct btf_id_dtor_kfunc skb_kfunc_dtors[] = {
458 {
459 .btf_id = bpf_sk_buff_ids[0],
460 .kfunc_btf_id = bpf_sk_buff_dtor_ids[0]
461 },
462 };
463
464 ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_qdisc_kfunc_set);
465 ret = ret ?: register_btf_id_dtor_kfuncs(skb_kfunc_dtors,
466 ARRAY_SIZE(skb_kfunc_dtors),
467 THIS_MODULE);
468 ret = ret ?: register_bpf_struct_ops(&bpf_Qdisc_ops, Qdisc_ops);
469
470 return ret;
471 }
472 late_initcall(bpf_qdisc_kfunc_init);
473