xref: /linux/net/sched/cls_bpf.c (revision b85d45947951d23cb22d90caecf4c1eb81342c96) 
1 /*
2  * Berkeley Packet Filter based traffic classifier
3  *
4  * Might be used to classify traffic through flexible, user-defined and
5  * possibly JIT-ed BPF filters for traffic control as an alternative to
6  * ematches.
7  *
8  * (C) 2013 Daniel Borkmann <dborkman@redhat.com>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License version 2 as
12  * published by the Free Software Foundation.
13  */
14 
15 #include <linux/module.h>
16 #include <linux/types.h>
17 #include <linux/skbuff.h>
18 #include <linux/filter.h>
19 #include <linux/bpf.h>
20 
21 #include <net/rtnetlink.h>
22 #include <net/pkt_cls.h>
23 #include <net/sock.h>
24 
25 MODULE_LICENSE("GPL");
26 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
27 MODULE_DESCRIPTION("TC BPF based classifier");
28 
29 #define CLS_BPF_NAME_LEN	256
30 
31 struct cls_bpf_head {
32 	struct list_head plist;
33 	u32 hgen;
34 	struct rcu_head rcu;
35 };
36 
37 struct cls_bpf_prog {
38 	struct bpf_prog *filter;
39 	struct list_head link;
40 	struct tcf_result res;
41 	struct tcf_exts exts;
42 	u32 handle;
43 	union {
44 		u32 bpf_fd;
45 		u16 bpf_num_ops;
46 	};
47 	struct sock_filter *bpf_ops;
48 	const char *bpf_name;
49 	struct tcf_proto *tp;
50 	struct rcu_head rcu;
51 };
52 
53 static const struct nla_policy bpf_policy[TCA_BPF_MAX + 1] = {
54 	[TCA_BPF_CLASSID]	= { .type = NLA_U32 },
55 	[TCA_BPF_FD]		= { .type = NLA_U32 },
56 	[TCA_BPF_NAME]		= { .type = NLA_NUL_STRING, .len = CLS_BPF_NAME_LEN },
57 	[TCA_BPF_OPS_LEN]	= { .type = NLA_U16 },
58 	[TCA_BPF_OPS]		= { .type = NLA_BINARY,
59 				    .len = sizeof(struct sock_filter) * BPF_MAXINSNS },
60 };
61 
62 static int cls_bpf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
63 			    struct tcf_result *res)
64 {
65 	struct cls_bpf_head *head = rcu_dereference_bh(tp->root);
66 	struct cls_bpf_prog *prog;
67 #ifdef CONFIG_NET_CLS_ACT
68 	bool at_ingress = G_TC_AT(skb->tc_verd) & AT_INGRESS;
69 #else
70 	bool at_ingress = false;
71 #endif
72 	int ret = -1;
73 
74 	if (unlikely(!skb_mac_header_was_set(skb)))
75 		return -1;
76 
77 	/* Needed here for accessing maps. */
78 	rcu_read_lock();
79 	list_for_each_entry_rcu(prog, &head->plist, link) {
80 		int filter_res;
81 
82 		if (at_ingress) {
83 			/* It is safe to push/pull even if skb_shared() */
84 			__skb_push(skb, skb->mac_len);
85 			filter_res = BPF_PROG_RUN(prog->filter, skb);
86 			__skb_pull(skb, skb->mac_len);
87 		} else {
88 			filter_res = BPF_PROG_RUN(prog->filter, skb);
89 		}
90 
91 		if (filter_res == 0)
92 			continue;
93 
94 		*res = prog->res;
95 		if (filter_res != -1)
96 			res->classid = filter_res;
97 
98 		ret = tcf_exts_exec(skb, &prog->exts, res);
99 		if (ret < 0)
100 			continue;
101 
102 		break;
103 	}
104 	rcu_read_unlock();
105 
106 	return ret;
107 }
108 
109 static bool cls_bpf_is_ebpf(const struct cls_bpf_prog *prog)
110 {
111 	return !prog->bpf_ops;
112 }
113 
114 static int cls_bpf_init(struct tcf_proto *tp)
115 {
116 	struct cls_bpf_head *head;
117 
118 	head = kzalloc(sizeof(*head), GFP_KERNEL);
119 	if (head == NULL)
120 		return -ENOBUFS;
121 
122 	INIT_LIST_HEAD_RCU(&head->plist);
123 	rcu_assign_pointer(tp->root, head);
124 
125 	return 0;
126 }
127 
128 static void cls_bpf_delete_prog(struct tcf_proto *tp, struct cls_bpf_prog *prog)
129 {
130 	tcf_exts_destroy(&prog->exts);
131 
132 	if (cls_bpf_is_ebpf(prog))
133 		bpf_prog_put(prog->filter);
134 	else
135 		bpf_prog_destroy(prog->filter);
136 
137 	kfree(prog->bpf_name);
138 	kfree(prog->bpf_ops);
139 	kfree(prog);
140 }
141 
142 static void __cls_bpf_delete_prog(struct rcu_head *rcu)
143 {
144 	struct cls_bpf_prog *prog = container_of(rcu, struct cls_bpf_prog, rcu);
145 
146 	cls_bpf_delete_prog(prog->tp, prog);
147 }
148 
149 static int cls_bpf_delete(struct tcf_proto *tp, unsigned long arg)
150 {
151 	struct cls_bpf_prog *prog = (struct cls_bpf_prog *) arg;
152 
153 	list_del_rcu(&prog->link);
154 	tcf_unbind_filter(tp, &prog->res);
155 	call_rcu(&prog->rcu, __cls_bpf_delete_prog);
156 
157 	return 0;
158 }
159 
160 static bool cls_bpf_destroy(struct tcf_proto *tp, bool force)
161 {
162 	struct cls_bpf_head *head = rtnl_dereference(tp->root);
163 	struct cls_bpf_prog *prog, *tmp;
164 
165 	if (!force && !list_empty(&head->plist))
166 		return false;
167 
168 	list_for_each_entry_safe(prog, tmp, &head->plist, link) {
169 		list_del_rcu(&prog->link);
170 		tcf_unbind_filter(tp, &prog->res);
171 		call_rcu(&prog->rcu, __cls_bpf_delete_prog);
172 	}
173 
174 	RCU_INIT_POINTER(tp->root, NULL);
175 	kfree_rcu(head, rcu);
176 	return true;
177 }
178 
179 static unsigned long cls_bpf_get(struct tcf_proto *tp, u32 handle)
180 {
181 	struct cls_bpf_head *head = rtnl_dereference(tp->root);
182 	struct cls_bpf_prog *prog;
183 	unsigned long ret = 0UL;
184 
185 	if (head == NULL)
186 		return 0UL;
187 
188 	list_for_each_entry(prog, &head->plist, link) {
189 		if (prog->handle == handle) {
190 			ret = (unsigned long) prog;
191 			break;
192 		}
193 	}
194 
195 	return ret;
196 }
197 
198 static int cls_bpf_prog_from_ops(struct nlattr **tb,
199 				 struct cls_bpf_prog *prog, u32 classid)
200 {
201 	struct sock_filter *bpf_ops;
202 	struct sock_fprog_kern fprog_tmp;
203 	struct bpf_prog *fp;
204 	u16 bpf_size, bpf_num_ops;
205 	int ret;
206 
207 	bpf_num_ops = nla_get_u16(tb[TCA_BPF_OPS_LEN]);
208 	if (bpf_num_ops > BPF_MAXINSNS || bpf_num_ops == 0)
209 		return -EINVAL;
210 
211 	bpf_size = bpf_num_ops * sizeof(*bpf_ops);
212 	if (bpf_size != nla_len(tb[TCA_BPF_OPS]))
213 		return -EINVAL;
214 
215 	bpf_ops = kzalloc(bpf_size, GFP_KERNEL);
216 	if (bpf_ops == NULL)
217 		return -ENOMEM;
218 
219 	memcpy(bpf_ops, nla_data(tb[TCA_BPF_OPS]), bpf_size);
220 
221 	fprog_tmp.len = bpf_num_ops;
222 	fprog_tmp.filter = bpf_ops;
223 
224 	ret = bpf_prog_create(&fp, &fprog_tmp);
225 	if (ret < 0) {
226 		kfree(bpf_ops);
227 		return ret;
228 	}
229 
230 	prog->bpf_ops = bpf_ops;
231 	prog->bpf_num_ops = bpf_num_ops;
232 	prog->bpf_name = NULL;
233 
234 	prog->filter = fp;
235 	prog->res.classid = classid;
236 
237 	return 0;
238 }
239 
240 static int cls_bpf_prog_from_efd(struct nlattr **tb,
241 				 struct cls_bpf_prog *prog, u32 classid)
242 {
243 	struct bpf_prog *fp;
244 	char *name = NULL;
245 	u32 bpf_fd;
246 
247 	bpf_fd = nla_get_u32(tb[TCA_BPF_FD]);
248 
249 	fp = bpf_prog_get(bpf_fd);
250 	if (IS_ERR(fp))
251 		return PTR_ERR(fp);
252 
253 	if (fp->type != BPF_PROG_TYPE_SCHED_CLS) {
254 		bpf_prog_put(fp);
255 		return -EINVAL;
256 	}
257 
258 	if (tb[TCA_BPF_NAME]) {
259 		name = kmemdup(nla_data(tb[TCA_BPF_NAME]),
260 			       nla_len(tb[TCA_BPF_NAME]),
261 			       GFP_KERNEL);
262 		if (!name) {
263 			bpf_prog_put(fp);
264 			return -ENOMEM;
265 		}
266 	}
267 
268 	prog->bpf_ops = NULL;
269 	prog->bpf_fd = bpf_fd;
270 	prog->bpf_name = name;
271 
272 	prog->filter = fp;
273 	prog->res.classid = classid;
274 
275 	return 0;
276 }
277 
278 static int cls_bpf_modify_existing(struct net *net, struct tcf_proto *tp,
279 				   struct cls_bpf_prog *prog,
280 				   unsigned long base, struct nlattr **tb,
281 				   struct nlattr *est, bool ovr)
282 {
283 	struct tcf_exts exts;
284 	bool is_bpf, is_ebpf;
285 	u32 classid;
286 	int ret;
287 
288 	is_bpf = tb[TCA_BPF_OPS_LEN] && tb[TCA_BPF_OPS];
289 	is_ebpf = tb[TCA_BPF_FD];
290 
291 	if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf) ||
292 	    !tb[TCA_BPF_CLASSID])
293 		return -EINVAL;
294 
295 	tcf_exts_init(&exts, TCA_BPF_ACT, TCA_BPF_POLICE);
296 	ret = tcf_exts_validate(net, tp, tb, est, &exts, ovr);
297 	if (ret < 0)
298 		return ret;
299 
300 	classid = nla_get_u32(tb[TCA_BPF_CLASSID]);
301 
302 	ret = is_bpf ? cls_bpf_prog_from_ops(tb, prog, classid) :
303 		       cls_bpf_prog_from_efd(tb, prog, classid);
304 	if (ret < 0) {
305 		tcf_exts_destroy(&exts);
306 		return ret;
307 	}
308 
309 	tcf_bind_filter(tp, &prog->res, base);
310 	tcf_exts_change(tp, &prog->exts, &exts);
311 
312 	return 0;
313 }
314 
315 static u32 cls_bpf_grab_new_handle(struct tcf_proto *tp,
316 				   struct cls_bpf_head *head)
317 {
318 	unsigned int i = 0x80000000;
319 	u32 handle;
320 
321 	do {
322 		if (++head->hgen == 0x7FFFFFFF)
323 			head->hgen = 1;
324 	} while (--i > 0 && cls_bpf_get(tp, head->hgen));
325 
326 	if (unlikely(i == 0)) {
327 		pr_err("Insufficient number of handles\n");
328 		handle = 0;
329 	} else {
330 		handle = head->hgen;
331 	}
332 
333 	return handle;
334 }
335 
336 static int cls_bpf_change(struct net *net, struct sk_buff *in_skb,
337 			  struct tcf_proto *tp, unsigned long base,
338 			  u32 handle, struct nlattr **tca,
339 			  unsigned long *arg, bool ovr)
340 {
341 	struct cls_bpf_head *head = rtnl_dereference(tp->root);
342 	struct cls_bpf_prog *oldprog = (struct cls_bpf_prog *) *arg;
343 	struct nlattr *tb[TCA_BPF_MAX + 1];
344 	struct cls_bpf_prog *prog;
345 	int ret;
346 
347 	if (tca[TCA_OPTIONS] == NULL)
348 		return -EINVAL;
349 
350 	ret = nla_parse_nested(tb, TCA_BPF_MAX, tca[TCA_OPTIONS], bpf_policy);
351 	if (ret < 0)
352 		return ret;
353 
354 	prog = kzalloc(sizeof(*prog), GFP_KERNEL);
355 	if (!prog)
356 		return -ENOBUFS;
357 
358 	tcf_exts_init(&prog->exts, TCA_BPF_ACT, TCA_BPF_POLICE);
359 
360 	if (oldprog) {
361 		if (handle && oldprog->handle != handle) {
362 			ret = -EINVAL;
363 			goto errout;
364 		}
365 	}
366 
367 	if (handle == 0)
368 		prog->handle = cls_bpf_grab_new_handle(tp, head);
369 	else
370 		prog->handle = handle;
371 	if (prog->handle == 0) {
372 		ret = -EINVAL;
373 		goto errout;
374 	}
375 
376 	ret = cls_bpf_modify_existing(net, tp, prog, base, tb, tca[TCA_RATE], ovr);
377 	if (ret < 0)
378 		goto errout;
379 
380 	if (oldprog) {
381 		list_replace_rcu(&oldprog->link, &prog->link);
382 		tcf_unbind_filter(tp, &oldprog->res);
383 		call_rcu(&oldprog->rcu, __cls_bpf_delete_prog);
384 	} else {
385 		list_add_rcu(&prog->link, &head->plist);
386 	}
387 
388 	*arg = (unsigned long) prog;
389 	return 0;
390 errout:
391 	kfree(prog);
392 
393 	return ret;
394 }
395 
396 static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog *prog,
397 				 struct sk_buff *skb)
398 {
399 	struct nlattr *nla;
400 
401 	if (nla_put_u16(skb, TCA_BPF_OPS_LEN, prog->bpf_num_ops))
402 		return -EMSGSIZE;
403 
404 	nla = nla_reserve(skb, TCA_BPF_OPS, prog->bpf_num_ops *
405 			  sizeof(struct sock_filter));
406 	if (nla == NULL)
407 		return -EMSGSIZE;
408 
409 	memcpy(nla_data(nla), prog->bpf_ops, nla_len(nla));
410 
411 	return 0;
412 }
413 
414 static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog *prog,
415 				  struct sk_buff *skb)
416 {
417 	if (nla_put_u32(skb, TCA_BPF_FD, prog->bpf_fd))
418 		return -EMSGSIZE;
419 
420 	if (prog->bpf_name &&
421 	    nla_put_string(skb, TCA_BPF_NAME, prog->bpf_name))
422 		return -EMSGSIZE;
423 
424 	return 0;
425 }
426 
427 static int cls_bpf_dump(struct net *net, struct tcf_proto *tp, unsigned long fh,
428 			struct sk_buff *skb, struct tcmsg *tm)
429 {
430 	struct cls_bpf_prog *prog = (struct cls_bpf_prog *) fh;
431 	struct nlattr *nest;
432 	int ret;
433 
434 	if (prog == NULL)
435 		return skb->len;
436 
437 	tm->tcm_handle = prog->handle;
438 
439 	nest = nla_nest_start(skb, TCA_OPTIONS);
440 	if (nest == NULL)
441 		goto nla_put_failure;
442 
443 	if (nla_put_u32(skb, TCA_BPF_CLASSID, prog->res.classid))
444 		goto nla_put_failure;
445 
446 	if (cls_bpf_is_ebpf(prog))
447 		ret = cls_bpf_dump_ebpf_info(prog, skb);
448 	else
449 		ret = cls_bpf_dump_bpf_info(prog, skb);
450 	if (ret)
451 		goto nla_put_failure;
452 
453 	if (tcf_exts_dump(skb, &prog->exts) < 0)
454 		goto nla_put_failure;
455 
456 	nla_nest_end(skb, nest);
457 
458 	if (tcf_exts_dump_stats(skb, &prog->exts) < 0)
459 		goto nla_put_failure;
460 
461 	return skb->len;
462 
463 nla_put_failure:
464 	nla_nest_cancel(skb, nest);
465 	return -1;
466 }
467 
468 static void cls_bpf_walk(struct tcf_proto *tp, struct tcf_walker *arg)
469 {
470 	struct cls_bpf_head *head = rtnl_dereference(tp->root);
471 	struct cls_bpf_prog *prog;
472 
473 	list_for_each_entry(prog, &head->plist, link) {
474 		if (arg->count < arg->skip)
475 			goto skip;
476 		if (arg->fn(tp, (unsigned long) prog, arg) < 0) {
477 			arg->stop = 1;
478 			break;
479 		}
480 skip:
481 		arg->count++;
482 	}
483 }
484 
485 static struct tcf_proto_ops cls_bpf_ops __read_mostly = {
486 	.kind		=	"bpf",
487 	.owner		=	THIS_MODULE,
488 	.classify	=	cls_bpf_classify,
489 	.init		=	cls_bpf_init,
490 	.destroy	=	cls_bpf_destroy,
491 	.get		=	cls_bpf_get,
492 	.change		=	cls_bpf_change,
493 	.delete		=	cls_bpf_delete,
494 	.walk		=	cls_bpf_walk,
495 	.dump		=	cls_bpf_dump,
496 };
497 
498 static int __init cls_bpf_init_mod(void)
499 {
500 	return register_tcf_proto_ops(&cls_bpf_ops);
501 }
502 
503 static void __exit cls_bpf_exit_mod(void)
504 {
505 	unregister_tcf_proto_ops(&cls_bpf_ops);
506 }
507 
508 module_init(cls_bpf_init_mod);
509 module_exit(cls_bpf_exit_mod);
510