1 // SPDX-License-Identifier: GPL-2.0-only
5  * Might be used to classify traffic through flexible, user-defined and
6  * possibly JIT-ed BPF filters for traffic control as an alternative to
40 	struct list_head link;  member
50 	struct tcf_proto *tp;  member
82 				       const struct tcf_proto *tp,  in cls_bpf_classify()  argument
85 	struct cls_bpf_head *head = rcu_dereference_bh(tp->root);  in cls_bpf_classify()
88 	int ret = -1;  in cls_bpf_classify()
90 	list_for_each_entry_rcu(prog, &head->plist, link) {  in cls_bpf_classify()
93 		qdisc_skb_cb(skb)->tc_classid = prog->res.classid;  in cls_bpf_classify()
95 		if (tc_skip_sw(prog->gen_flags)) {  in cls_bpf_classify()
96 			filter_res = prog->exts_integrated ? TC_ACT_UNSPEC : 0;  in cls_bpf_classify()
99 			__skb_push(skb, skb->mac_len);  in cls_bpf_classify()
101 			filter_res = bpf_prog_run(prog->filter, skb);  in cls_bpf_classify()
102 			__skb_pull(skb, skb->mac_len);  in cls_bpf_classify()
105 			filter_res = bpf_prog_run(prog->filter, skb);  in cls_bpf_classify()
107 		if (unlikely(!skb->tstamp && skb->tstamp_type))  in cls_bpf_classify()
108 			skb->tstamp_type = SKB_CLOCK_REALTIME;  in cls_bpf_classify()
110 		if (prog->exts_integrated) {  in cls_bpf_classify()
111 			res->class   = 0;  in cls_bpf_classify()
112 			res->classid = TC_H_MAJ(prog->res.classid) |  in cls_bpf_classify()
113 				       qdisc_skb_cb(skb)->tc_classid;  in cls_bpf_classify()
123 		if (filter_res != -1) {  in cls_bpf_classify()
124 			res->class   = 0;  in cls_bpf_classify()
125 			res->classid = filter_res;  in cls_bpf_classify()
127 			*res = prog->res;  in cls_bpf_classify()
130 		ret = tcf_exts_exec(skb, &prog->exts, res);  in cls_bpf_classify()
142 	return !prog->bpf_ops;  in cls_bpf_is_ebpf()
145 static int cls_bpf_offload_cmd(struct tcf_proto *tp, struct cls_bpf_prog *prog,  in cls_bpf_offload_cmd()  argument
149 	struct tcf_block *block = tp->chain->block;  in cls_bpf_offload_cmd()
155 	skip_sw = prog && tc_skip_sw(prog->gen_flags);  in cls_bpf_offload_cmd()
158 	tc_cls_common_offload_init(&cls_bpf.common, tp, obj->gen_flags, extack);  in cls_bpf_offload_cmd()
160 	cls_bpf.exts = &obj->exts;  in cls_bpf_offload_cmd()
161 	cls_bpf.prog = prog ? prog->filter : NULL;  in cls_bpf_offload_cmd()
162 	cls_bpf.oldprog = oldprog ? oldprog->filter : NULL;  in cls_bpf_offload_cmd()
163 	cls_bpf.name = obj->bpf_name;  in cls_bpf_offload_cmd()
164 	cls_bpf.exts_integrated = obj->exts_integrated;  in cls_bpf_offload_cmd()
167 		err = tc_setup_cb_replace(block, tp, TC_SETUP_CLSBPF, &cls_bpf,  in cls_bpf_offload_cmd()
168 					  skip_sw, &oldprog->gen_flags,  in cls_bpf_offload_cmd()
169 					  &oldprog->in_hw_count,  in cls_bpf_offload_cmd()
170 					  &prog->gen_flags, &prog->in_hw_count,  in cls_bpf_offload_cmd()
173 		err = tc_setup_cb_add(block, tp, TC_SETUP_CLSBPF, &cls_bpf,  in cls_bpf_offload_cmd()
174 				      skip_sw, &prog->gen_flags,  in cls_bpf_offload_cmd()
175 				      &prog->in_hw_count, true);  in cls_bpf_offload_cmd()
177 		err = tc_setup_cb_destroy(block, tp, TC_SETUP_CLSBPF, &cls_bpf,  in cls_bpf_offload_cmd()
178 					  skip_sw, &oldprog->gen_flags,  in cls_bpf_offload_cmd()
179 					  &oldprog->in_hw_count, true);  in cls_bpf_offload_cmd()
182 		cls_bpf_offload_cmd(tp, oldprog, prog, extack);  in cls_bpf_offload_cmd()
186 	if (prog && skip_sw && !(prog->gen_flags & TCA_CLS_FLAGS_IN_HW))  in cls_bpf_offload_cmd()
187 		return -EINVAL;  in cls_bpf_offload_cmd()
197 static int cls_bpf_offload(struct tcf_proto *tp, struct cls_bpf_prog *prog,  in cls_bpf_offload()  argument
202 	    cls_bpf_flags(prog->gen_flags) !=  in cls_bpf_offload()
203 	    cls_bpf_flags(oldprog->gen_flags))  in cls_bpf_offload()
204 		return -EINVAL;  in cls_bpf_offload()
206 	if (prog && tc_skip_hw(prog->gen_flags))  in cls_bpf_offload()
208 	if (oldprog && tc_skip_hw(oldprog->gen_flags))  in cls_bpf_offload()
213 	return cls_bpf_offload_cmd(tp, prog, oldprog, extack);  in cls_bpf_offload()
216 static void cls_bpf_stop_offload(struct tcf_proto *tp,  in cls_bpf_stop_offload()  argument
222 	err = cls_bpf_offload_cmd(tp, NULL, prog, extack);  in cls_bpf_stop_offload()
227 static void cls_bpf_offload_update_stats(struct tcf_proto *tp,  in cls_bpf_offload_update_stats()  argument
230 	struct tcf_block *block = tp->chain->block;  in cls_bpf_offload_update_stats()
233 	tc_cls_common_offload_init(&cls_bpf.common, tp, prog->gen_flags, NULL);  in cls_bpf_offload_update_stats()
235 	cls_bpf.exts = &prog->exts;  in cls_bpf_offload_update_stats()
236 	cls_bpf.prog = prog->filter;  in cls_bpf_offload_update_stats()
237 	cls_bpf.name = prog->bpf_name;  in cls_bpf_offload_update_stats()
238 	cls_bpf.exts_integrated = prog->exts_integrated;  in cls_bpf_offload_update_stats()
243 static int cls_bpf_init(struct tcf_proto *tp)  in cls_bpf_init()  argument
249 		return -ENOBUFS;  in cls_bpf_init()
251 	INIT_LIST_HEAD_RCU(&head->plist);  in cls_bpf_init()
252 	idr_init(&head->handle_idr);  in cls_bpf_init()
253 	rcu_assign_pointer(tp->root, head);  in cls_bpf_init()
261 		bpf_prog_put(prog->filter);  in cls_bpf_free_parms()
263 		bpf_prog_destroy(prog->filter);  in cls_bpf_free_parms()
265 	kfree(prog->bpf_name);  in cls_bpf_free_parms()
266 	kfree(prog->bpf_ops);  in cls_bpf_free_parms()
271 	tcf_exts_destroy(&prog->exts);  in __cls_bpf_delete_prog()
272 	tcf_exts_put_net(&prog->exts);  in __cls_bpf_delete_prog()
288 static void __cls_bpf_delete(struct tcf_proto *tp, struct cls_bpf_prog *prog,  in __cls_bpf_delete()  argument
291 	struct cls_bpf_head *head = rtnl_dereference(tp->root);  in __cls_bpf_delete()
293 	idr_remove(&head->handle_idr, prog->handle);  in __cls_bpf_delete()
294 	cls_bpf_stop_offload(tp, prog, extack);  in __cls_bpf_delete()
295 	list_del_rcu(&prog->link);  in __cls_bpf_delete()
296 	tcf_unbind_filter(tp, &prog->res);  in __cls_bpf_delete()
297 	if (tcf_exts_get_net(&prog->exts))  in __cls_bpf_delete()
298 		tcf_queue_work(&prog->rwork, cls_bpf_delete_prog_work);  in __cls_bpf_delete()
303 static int cls_bpf_delete(struct tcf_proto *tp, void *arg, bool *last,  in cls_bpf_delete()  argument
306 	struct cls_bpf_head *head = rtnl_dereference(tp->root);  in cls_bpf_delete()
308 	__cls_bpf_delete(tp, arg, extack);  in cls_bpf_delete()
309 	*last = list_empty(&head->plist);  in cls_bpf_delete()
313 static void cls_bpf_destroy(struct tcf_proto *tp, bool rtnl_held,  in cls_bpf_destroy()  argument
316 	struct cls_bpf_head *head = rtnl_dereference(tp->root);  in cls_bpf_destroy()
319 	list_for_each_entry_safe(prog, tmp, &head->plist, link)  in cls_bpf_destroy()
320 		__cls_bpf_delete(tp, prog, extack);  in cls_bpf_destroy()
322 	idr_destroy(&head->handle_idr);  in cls_bpf_destroy()
326 static void *cls_bpf_get(struct tcf_proto *tp, u32 handle)  in cls_bpf_get()  argument
328 	struct cls_bpf_head *head = rtnl_dereference(tp->root);  in cls_bpf_get()
331 	list_for_each_entry(prog, &head->plist, link) {  in cls_bpf_get()
332 		if (prog->handle == handle)  in cls_bpf_get()
349 		return -EINVAL;  in cls_bpf_prog_from_ops()
353 		return -EINVAL;  in cls_bpf_prog_from_ops()
357 		return -ENOMEM;  in cls_bpf_prog_from_ops()
368 	prog->bpf_ops = bpf_ops;  in cls_bpf_prog_from_ops()
369 	prog->bpf_num_ops = bpf_num_ops;  in cls_bpf_prog_from_ops()
370 	prog->bpf_name = NULL;  in cls_bpf_prog_from_ops()
371 	prog->filter = fp;  in cls_bpf_prog_from_ops()
377 				 u32 gen_flags, const struct tcf_proto *tp)  in cls_bpf_prog_from_efd()  argument
395 			return -ENOMEM;  in cls_bpf_prog_from_efd()
399 	prog->bpf_ops = NULL;  in cls_bpf_prog_from_efd()
400 	prog->bpf_name = name;  in cls_bpf_prog_from_efd()
401 	prog->filter = fp;  in cls_bpf_prog_from_efd()
403 	if (fp->dst_needed)  in cls_bpf_prog_from_efd()
404 		tcf_block_netif_keep_dst(tp->chain->block);  in cls_bpf_prog_from_efd()
410 			  struct tcf_proto *tp, unsigned long base,  in cls_bpf_change()  argument
415 	struct cls_bpf_head *head = rtnl_dereference(tp->root);  in cls_bpf_change()
425 		return -EINVAL;  in cls_bpf_change()
434 		return -ENOBUFS;  in cls_bpf_change()
436 	ret = tcf_exts_init(&prog->exts, net, TCA_BPF_ACT, TCA_BPF_POLICE);  in cls_bpf_change()
441 		if (handle && oldprog->handle != handle) {  in cls_bpf_change()
442 			ret = -EINVAL;  in cls_bpf_change()
449 		ret = idr_alloc_u32(&head->handle_idr, prog, &handle,  in cls_bpf_change()
452 		ret = idr_alloc_u32(&head->handle_idr, prog, &handle,  in cls_bpf_change()
458 	prog->handle = handle;  in cls_bpf_change()
463 		ret = -EINVAL;  in cls_bpf_change()
467 	ret = tcf_exts_validate(net, tp, tb, tca[TCA_RATE], &prog->exts,  in cls_bpf_change()
476 			ret = -EINVAL;  in cls_bpf_change()
486 			ret = -EINVAL;  in cls_bpf_change()
491 	prog->exts_integrated = have_exts;  in cls_bpf_change()
492 	prog->gen_flags = gen_flags;  in cls_bpf_change()
495 		cls_bpf_prog_from_efd(tb, prog, gen_flags, tp);  in cls_bpf_change()
500 		prog->res.classid = nla_get_u32(tb[TCA_BPF_CLASSID]);  in cls_bpf_change()
501 		tcf_bind_filter(tp, &prog->res, base);  in cls_bpf_change()
505 	ret = cls_bpf_offload(tp, prog, oldprog, extack);  in cls_bpf_change()
509 	if (!tc_in_hw(prog->gen_flags))  in cls_bpf_change()
510 		prog->gen_flags |= TCA_CLS_FLAGS_NOT_IN_HW;  in cls_bpf_change()
513 		idr_replace(&head->handle_idr, prog, handle);  in cls_bpf_change()
514 		list_replace_rcu(&oldprog->link, &prog->link);  in cls_bpf_change()
515 		tcf_unbind_filter(tp, &oldprog->res);  in cls_bpf_change()
516 		tcf_exts_get_net(&oldprog->exts);  in cls_bpf_change()
517 		tcf_queue_work(&oldprog->rwork, cls_bpf_delete_prog_work);  in cls_bpf_change()
519 		list_add_rcu(&prog->link, &head->plist);  in cls_bpf_change()
527 		tcf_unbind_filter(tp, &prog->res);  in cls_bpf_change()
531 		idr_remove(&head->handle_idr, prog->handle);  in cls_bpf_change()
533 	tcf_exts_destroy(&prog->exts);  in cls_bpf_change()
543 	if (nla_put_u16(skb, TCA_BPF_OPS_LEN, prog->bpf_num_ops))  in cls_bpf_dump_bpf_info()
544 		return -EMSGSIZE;  in cls_bpf_dump_bpf_info()
546 	nla = nla_reserve(skb, TCA_BPF_OPS, prog->bpf_num_ops *  in cls_bpf_dump_bpf_info()
549 		return -EMSGSIZE;  in cls_bpf_dump_bpf_info()
551 	memcpy(nla_data(nla), prog->bpf_ops, nla_len(nla));  in cls_bpf_dump_bpf_info()
561 	if (prog->bpf_name &&  in cls_bpf_dump_ebpf_info()
562 	    nla_put_string(skb, TCA_BPF_NAME, prog->bpf_name))  in cls_bpf_dump_ebpf_info()
563 		return -EMSGSIZE;  in cls_bpf_dump_ebpf_info()
565 	if (nla_put_u32(skb, TCA_BPF_ID, prog->filter->aux->id))  in cls_bpf_dump_ebpf_info()
566 		return -EMSGSIZE;  in cls_bpf_dump_ebpf_info()
568 	nla = nla_reserve(skb, TCA_BPF_TAG, sizeof(prog->filter->tag));  in cls_bpf_dump_ebpf_info()
570 		return -EMSGSIZE;  in cls_bpf_dump_ebpf_info()
572 	memcpy(nla_data(nla), prog->filter->tag, nla_len(nla));  in cls_bpf_dump_ebpf_info()
577 static int cls_bpf_dump(struct net *net, struct tcf_proto *tp, void *fh,  in cls_bpf_dump()  argument
586 		return skb->len;  in cls_bpf_dump()
588 	tm->tcm_handle = prog->handle;  in cls_bpf_dump()
590 	cls_bpf_offload_update_stats(tp, prog);  in cls_bpf_dump()
596 	if (prog->res.classid &&  in cls_bpf_dump()
597 	    nla_put_u32(skb, TCA_BPF_CLASSID, prog->res.classid))  in cls_bpf_dump()
607 	if (tcf_exts_dump(skb, &prog->exts) < 0)  in cls_bpf_dump()
610 	if (prog->exts_integrated)  in cls_bpf_dump()
614 	if (prog->gen_flags &&  in cls_bpf_dump()
615 	    nla_put_u32(skb, TCA_BPF_FLAGS_GEN, prog->gen_flags))  in cls_bpf_dump()
620 	if (tcf_exts_dump_stats(skb, &prog->exts) < 0)  in cls_bpf_dump()
623 	return skb->len;  in cls_bpf_dump()
627 	return -1;  in cls_bpf_dump()
635 	tc_cls_bind_class(classid, cl, q, &prog->res, base);  in cls_bpf_bind_class()
638 static void cls_bpf_walk(struct tcf_proto *tp, struct tcf_walker *arg,  in cls_bpf_walk()  argument
641 	struct cls_bpf_head *head = rtnl_dereference(tp->root);  in cls_bpf_walk()
644 	list_for_each_entry(prog, &head->plist, link) {  in cls_bpf_walk()
645 		if (!tc_cls_stats_dump(tp, arg, prog))  in cls_bpf_walk()
650 static int cls_bpf_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb,  in cls_bpf_reoffload()  argument
653 	struct cls_bpf_head *head = rtnl_dereference(tp->root);  in cls_bpf_reoffload()
654 	struct tcf_block *block = tp->chain->block;  in cls_bpf_reoffload()
659 	list_for_each_entry(prog, &head->plist, link) {  in cls_bpf_reoffload()
660 		if (tc_skip_hw(prog->gen_flags))  in cls_bpf_reoffload()
663 		tc_cls_common_offload_init(&cls_bpf.common, tp, prog->gen_flags,  in cls_bpf_reoffload()
666 		cls_bpf.exts = &prog->exts;  in cls_bpf_reoffload()
667 		cls_bpf.prog = add ? prog->filter : NULL;  in cls_bpf_reoffload()
668 		cls_bpf.oldprog = add ? NULL : prog->filter;  in cls_bpf_reoffload()
669 		cls_bpf.name = prog->bpf_name;  in cls_bpf_reoffload()
670 		cls_bpf.exts_integrated = prog->exts_integrated;  in cls_bpf_reoffload()
672 		err = tc_setup_cb_reoffload(block, tp, add, cb, TC_SETUP_CLSBPF,  in cls_bpf_reoffload()
673 					    &cls_bpf, cb_priv, &prog->gen_flags,  in cls_bpf_reoffload()
674 					    &prog->in_hw_count);  in cls_bpf_reoffload()