xref: /linux/net/sched/cls_flower.c (revision 32786fdc9506aeba98278c1844d4bfb766863832)
1 /*
2  * net/sched/cls_flower.c		Flower classifier
3  *
4  * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  */
11 
12 #include <linux/kernel.h>
13 #include <linux/init.h>
14 #include <linux/module.h>
15 #include <linux/rhashtable.h>
16 #include <linux/workqueue.h>
17 
18 #include <linux/if_ether.h>
19 #include <linux/in6.h>
20 #include <linux/ip.h>
21 
22 #include <net/sch_generic.h>
23 #include <net/pkt_cls.h>
24 #include <net/ip.h>
25 #include <net/flow_dissector.h>
26 
27 #include <net/dst.h>
28 #include <net/dst_metadata.h>
29 
30 struct fl_flow_key {
31 	int	indev_ifindex;
32 	struct flow_dissector_key_control control;
33 	struct flow_dissector_key_control enc_control;
34 	struct flow_dissector_key_basic basic;
35 	struct flow_dissector_key_eth_addrs eth;
36 	struct flow_dissector_key_vlan vlan;
37 	union {
38 		struct flow_dissector_key_ipv4_addrs ipv4;
39 		struct flow_dissector_key_ipv6_addrs ipv6;
40 	};
41 	struct flow_dissector_key_ports tp;
42 	struct flow_dissector_key_icmp icmp;
43 	struct flow_dissector_key_keyid enc_key_id;
44 	union {
45 		struct flow_dissector_key_ipv4_addrs enc_ipv4;
46 		struct flow_dissector_key_ipv6_addrs enc_ipv6;
47 	};
48 	struct flow_dissector_key_ports enc_tp;
49 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
50 
51 struct fl_flow_mask_range {
52 	unsigned short int start;
53 	unsigned short int end;
54 };
55 
56 struct fl_flow_mask {
57 	struct fl_flow_key key;
58 	struct fl_flow_mask_range range;
59 	struct rcu_head	rcu;
60 };
61 
62 struct cls_fl_head {
63 	struct rhashtable ht;
64 	struct fl_flow_mask mask;
65 	struct flow_dissector dissector;
66 	u32 hgen;
67 	bool mask_assigned;
68 	struct list_head filters;
69 	struct rhashtable_params ht_params;
70 	union {
71 		struct work_struct work;
72 		struct rcu_head	rcu;
73 	};
74 };
75 
76 struct cls_fl_filter {
77 	struct rhash_head ht_node;
78 	struct fl_flow_key mkey;
79 	struct tcf_exts exts;
80 	struct tcf_result res;
81 	struct fl_flow_key key;
82 	struct list_head list;
83 	u32 handle;
84 	u32 flags;
85 	struct rcu_head	rcu;
86 	struct tc_to_netdev tc;
87 	struct net_device *hw_dev;
88 };
89 
90 static unsigned short int fl_mask_range(const struct fl_flow_mask *mask)
91 {
92 	return mask->range.end - mask->range.start;
93 }
94 
95 static void fl_mask_update_range(struct fl_flow_mask *mask)
96 {
97 	const u8 *bytes = (const u8 *) &mask->key;
98 	size_t size = sizeof(mask->key);
99 	size_t i, first = 0, last = size - 1;
100 
101 	for (i = 0; i < sizeof(mask->key); i++) {
102 		if (bytes[i]) {
103 			if (!first && i)
104 				first = i;
105 			last = i;
106 		}
107 	}
108 	mask->range.start = rounddown(first, sizeof(long));
109 	mask->range.end = roundup(last + 1, sizeof(long));
110 }
111 
112 static void *fl_key_get_start(struct fl_flow_key *key,
113 			      const struct fl_flow_mask *mask)
114 {
115 	return (u8 *) key + mask->range.start;
116 }
117 
118 static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key,
119 			      struct fl_flow_mask *mask)
120 {
121 	const long *lkey = fl_key_get_start(key, mask);
122 	const long *lmask = fl_key_get_start(&mask->key, mask);
123 	long *lmkey = fl_key_get_start(mkey, mask);
124 	int i;
125 
126 	for (i = 0; i < fl_mask_range(mask); i += sizeof(long))
127 		*lmkey++ = *lkey++ & *lmask++;
128 }
129 
130 static void fl_clear_masked_range(struct fl_flow_key *key,
131 				  struct fl_flow_mask *mask)
132 {
133 	memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask));
134 }
135 
136 static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
137 		       struct tcf_result *res)
138 {
139 	struct cls_fl_head *head = rcu_dereference_bh(tp->root);
140 	struct cls_fl_filter *f;
141 	struct fl_flow_key skb_key;
142 	struct fl_flow_key skb_mkey;
143 	struct ip_tunnel_info *info;
144 
145 	if (!atomic_read(&head->ht.nelems))
146 		return -1;
147 
148 	fl_clear_masked_range(&skb_key, &head->mask);
149 
150 	info = skb_tunnel_info(skb);
151 	if (info) {
152 		struct ip_tunnel_key *key = &info->key;
153 
154 		switch (ip_tunnel_info_af(info)) {
155 		case AF_INET:
156 			skb_key.enc_ipv4.src = key->u.ipv4.src;
157 			skb_key.enc_ipv4.dst = key->u.ipv4.dst;
158 			break;
159 		case AF_INET6:
160 			skb_key.enc_ipv6.src = key->u.ipv6.src;
161 			skb_key.enc_ipv6.dst = key->u.ipv6.dst;
162 			break;
163 		}
164 
165 		skb_key.enc_key_id.keyid = tunnel_id_to_key32(key->tun_id);
166 		skb_key.enc_tp.src = key->tp_src;
167 		skb_key.enc_tp.dst = key->tp_dst;
168 	}
169 
170 	skb_key.indev_ifindex = skb->skb_iif;
171 	/* skb_flow_dissect() does not set n_proto in case an unknown protocol,
172 	 * so do it rather here.
173 	 */
174 	skb_key.basic.n_proto = skb->protocol;
175 	skb_flow_dissect(skb, &head->dissector, &skb_key, 0);
176 
177 	fl_set_masked_key(&skb_mkey, &skb_key, &head->mask);
178 
179 	f = rhashtable_lookup_fast(&head->ht,
180 				   fl_key_get_start(&skb_mkey, &head->mask),
181 				   head->ht_params);
182 	if (f && !tc_skip_sw(f->flags)) {
183 		*res = f->res;
184 		return tcf_exts_exec(skb, &f->exts, res);
185 	}
186 	return -1;
187 }
188 
189 static int fl_init(struct tcf_proto *tp)
190 {
191 	struct cls_fl_head *head;
192 
193 	head = kzalloc(sizeof(*head), GFP_KERNEL);
194 	if (!head)
195 		return -ENOBUFS;
196 
197 	INIT_LIST_HEAD_RCU(&head->filters);
198 	rcu_assign_pointer(tp->root, head);
199 
200 	return 0;
201 }
202 
203 static void fl_destroy_filter(struct rcu_head *head)
204 {
205 	struct cls_fl_filter *f = container_of(head, struct cls_fl_filter, rcu);
206 
207 	tcf_exts_destroy(&f->exts);
208 	kfree(f);
209 }
210 
211 static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f)
212 {
213 	struct tc_cls_flower_offload offload = {0};
214 	struct net_device *dev = f->hw_dev;
215 	struct tc_to_netdev *tc = &f->tc;
216 
217 	if (!tc_can_offload(dev, tp))
218 		return;
219 
220 	offload.command = TC_CLSFLOWER_DESTROY;
221 	offload.cookie = (unsigned long)f;
222 
223 	tc->type = TC_SETUP_CLSFLOWER;
224 	tc->cls_flower = &offload;
225 
226 	dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, tc);
227 }
228 
229 static int fl_hw_replace_filter(struct tcf_proto *tp,
230 				struct flow_dissector *dissector,
231 				struct fl_flow_key *mask,
232 				struct cls_fl_filter *f)
233 {
234 	struct net_device *dev = tp->q->dev_queue->dev;
235 	struct tc_cls_flower_offload offload = {0};
236 	struct tc_to_netdev *tc = &f->tc;
237 	int err;
238 
239 	if (!tc_can_offload(dev, tp)) {
240 		if (tcf_exts_get_dev(dev, &f->exts, &f->hw_dev) ||
241 		    (f->hw_dev && !tc_can_offload(f->hw_dev, tp))) {
242 			f->hw_dev = dev;
243 			return tc_skip_sw(f->flags) ? -EINVAL : 0;
244 		}
245 		dev = f->hw_dev;
246 		tc->egress_dev = true;
247 	} else {
248 		f->hw_dev = dev;
249 	}
250 
251 	offload.command = TC_CLSFLOWER_REPLACE;
252 	offload.cookie = (unsigned long)f;
253 	offload.dissector = dissector;
254 	offload.mask = mask;
255 	offload.key = &f->mkey;
256 	offload.exts = &f->exts;
257 
258 	tc->type = TC_SETUP_CLSFLOWER;
259 	tc->cls_flower = &offload;
260 
261 	err = dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol,
262 					    tc);
263 
264 	if (tc_skip_sw(f->flags))
265 		return err;
266 	return 0;
267 }
268 
269 static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f)
270 {
271 	struct tc_cls_flower_offload offload = {0};
272 	struct net_device *dev = f->hw_dev;
273 	struct tc_to_netdev *tc = &f->tc;
274 
275 	if (!tc_can_offload(dev, tp))
276 		return;
277 
278 	offload.command = TC_CLSFLOWER_STATS;
279 	offload.cookie = (unsigned long)f;
280 	offload.exts = &f->exts;
281 
282 	tc->type = TC_SETUP_CLSFLOWER;
283 	tc->cls_flower = &offload;
284 
285 	dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, tc);
286 }
287 
288 static void __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f)
289 {
290 	list_del_rcu(&f->list);
291 	if (!tc_skip_hw(f->flags))
292 		fl_hw_destroy_filter(tp, f);
293 	tcf_unbind_filter(tp, &f->res);
294 	call_rcu(&f->rcu, fl_destroy_filter);
295 }
296 
297 static void fl_destroy_sleepable(struct work_struct *work)
298 {
299 	struct cls_fl_head *head = container_of(work, struct cls_fl_head,
300 						work);
301 	if (head->mask_assigned)
302 		rhashtable_destroy(&head->ht);
303 	kfree(head);
304 	module_put(THIS_MODULE);
305 }
306 
307 static void fl_destroy_rcu(struct rcu_head *rcu)
308 {
309 	struct cls_fl_head *head = container_of(rcu, struct cls_fl_head, rcu);
310 
311 	INIT_WORK(&head->work, fl_destroy_sleepable);
312 	schedule_work(&head->work);
313 }
314 
315 static bool fl_destroy(struct tcf_proto *tp, bool force)
316 {
317 	struct cls_fl_head *head = rtnl_dereference(tp->root);
318 	struct cls_fl_filter *f, *next;
319 
320 	if (!force && !list_empty(&head->filters))
321 		return false;
322 
323 	list_for_each_entry_safe(f, next, &head->filters, list)
324 		__fl_delete(tp, f);
325 
326 	__module_get(THIS_MODULE);
327 	call_rcu(&head->rcu, fl_destroy_rcu);
328 
329 	return true;
330 }
331 
332 static unsigned long fl_get(struct tcf_proto *tp, u32 handle)
333 {
334 	struct cls_fl_head *head = rtnl_dereference(tp->root);
335 	struct cls_fl_filter *f;
336 
337 	list_for_each_entry(f, &head->filters, list)
338 		if (f->handle == handle)
339 			return (unsigned long) f;
340 	return 0;
341 }
342 
343 static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
344 	[TCA_FLOWER_UNSPEC]		= { .type = NLA_UNSPEC },
345 	[TCA_FLOWER_CLASSID]		= { .type = NLA_U32 },
346 	[TCA_FLOWER_INDEV]		= { .type = NLA_STRING,
347 					    .len = IFNAMSIZ },
348 	[TCA_FLOWER_KEY_ETH_DST]	= { .len = ETH_ALEN },
349 	[TCA_FLOWER_KEY_ETH_DST_MASK]	= { .len = ETH_ALEN },
350 	[TCA_FLOWER_KEY_ETH_SRC]	= { .len = ETH_ALEN },
351 	[TCA_FLOWER_KEY_ETH_SRC_MASK]	= { .len = ETH_ALEN },
352 	[TCA_FLOWER_KEY_ETH_TYPE]	= { .type = NLA_U16 },
353 	[TCA_FLOWER_KEY_IP_PROTO]	= { .type = NLA_U8 },
354 	[TCA_FLOWER_KEY_IPV4_SRC]	= { .type = NLA_U32 },
355 	[TCA_FLOWER_KEY_IPV4_SRC_MASK]	= { .type = NLA_U32 },
356 	[TCA_FLOWER_KEY_IPV4_DST]	= { .type = NLA_U32 },
357 	[TCA_FLOWER_KEY_IPV4_DST_MASK]	= { .type = NLA_U32 },
358 	[TCA_FLOWER_KEY_IPV6_SRC]	= { .len = sizeof(struct in6_addr) },
359 	[TCA_FLOWER_KEY_IPV6_SRC_MASK]	= { .len = sizeof(struct in6_addr) },
360 	[TCA_FLOWER_KEY_IPV6_DST]	= { .len = sizeof(struct in6_addr) },
361 	[TCA_FLOWER_KEY_IPV6_DST_MASK]	= { .len = sizeof(struct in6_addr) },
362 	[TCA_FLOWER_KEY_TCP_SRC]	= { .type = NLA_U16 },
363 	[TCA_FLOWER_KEY_TCP_DST]	= { .type = NLA_U16 },
364 	[TCA_FLOWER_KEY_UDP_SRC]	= { .type = NLA_U16 },
365 	[TCA_FLOWER_KEY_UDP_DST]	= { .type = NLA_U16 },
366 	[TCA_FLOWER_KEY_VLAN_ID]	= { .type = NLA_U16 },
367 	[TCA_FLOWER_KEY_VLAN_PRIO]	= { .type = NLA_U8 },
368 	[TCA_FLOWER_KEY_VLAN_ETH_TYPE]	= { .type = NLA_U16 },
369 	[TCA_FLOWER_KEY_ENC_KEY_ID]	= { .type = NLA_U32 },
370 	[TCA_FLOWER_KEY_ENC_IPV4_SRC]	= { .type = NLA_U32 },
371 	[TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 },
372 	[TCA_FLOWER_KEY_ENC_IPV4_DST]	= { .type = NLA_U32 },
373 	[TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 },
374 	[TCA_FLOWER_KEY_ENC_IPV6_SRC]	= { .len = sizeof(struct in6_addr) },
375 	[TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
376 	[TCA_FLOWER_KEY_ENC_IPV6_DST]	= { .len = sizeof(struct in6_addr) },
377 	[TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
378 	[TCA_FLOWER_KEY_TCP_SRC_MASK]	= { .type = NLA_U16 },
379 	[TCA_FLOWER_KEY_TCP_DST_MASK]	= { .type = NLA_U16 },
380 	[TCA_FLOWER_KEY_UDP_SRC_MASK]	= { .type = NLA_U16 },
381 	[TCA_FLOWER_KEY_UDP_DST_MASK]	= { .type = NLA_U16 },
382 	[TCA_FLOWER_KEY_SCTP_SRC_MASK]	= { .type = NLA_U16 },
383 	[TCA_FLOWER_KEY_SCTP_DST_MASK]	= { .type = NLA_U16 },
384 	[TCA_FLOWER_KEY_SCTP_SRC]	= { .type = NLA_U16 },
385 	[TCA_FLOWER_KEY_SCTP_DST]	= { .type = NLA_U16 },
386 	[TCA_FLOWER_KEY_ENC_UDP_SRC_PORT]	= { .type = NLA_U16 },
387 	[TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK]	= { .type = NLA_U16 },
388 	[TCA_FLOWER_KEY_ENC_UDP_DST_PORT]	= { .type = NLA_U16 },
389 	[TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK]	= { .type = NLA_U16 },
390 	[TCA_FLOWER_KEY_FLAGS]		= { .type = NLA_U32 },
391 	[TCA_FLOWER_KEY_FLAGS_MASK]	= { .type = NLA_U32 },
392 	[TCA_FLOWER_KEY_ICMPV4_TYPE]	= { .type = NLA_U8 },
393 	[TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 },
394 	[TCA_FLOWER_KEY_ICMPV4_CODE]	= { .type = NLA_U8 },
395 	[TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 },
396 	[TCA_FLOWER_KEY_ICMPV6_TYPE]	= { .type = NLA_U8 },
397 	[TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 },
398 	[TCA_FLOWER_KEY_ICMPV6_CODE]	= { .type = NLA_U8 },
399 	[TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 },
400 };
401 
402 static void fl_set_key_val(struct nlattr **tb,
403 			   void *val, int val_type,
404 			   void *mask, int mask_type, int len)
405 {
406 	if (!tb[val_type])
407 		return;
408 	memcpy(val, nla_data(tb[val_type]), len);
409 	if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type])
410 		memset(mask, 0xff, len);
411 	else
412 		memcpy(mask, nla_data(tb[mask_type]), len);
413 }
414 
415 static void fl_set_key_vlan(struct nlattr **tb,
416 			    struct flow_dissector_key_vlan *key_val,
417 			    struct flow_dissector_key_vlan *key_mask)
418 {
419 #define VLAN_PRIORITY_MASK	0x7
420 
421 	if (tb[TCA_FLOWER_KEY_VLAN_ID]) {
422 		key_val->vlan_id =
423 			nla_get_u16(tb[TCA_FLOWER_KEY_VLAN_ID]) & VLAN_VID_MASK;
424 		key_mask->vlan_id = VLAN_VID_MASK;
425 	}
426 	if (tb[TCA_FLOWER_KEY_VLAN_PRIO]) {
427 		key_val->vlan_priority =
428 			nla_get_u8(tb[TCA_FLOWER_KEY_VLAN_PRIO]) &
429 			VLAN_PRIORITY_MASK;
430 		key_mask->vlan_priority = VLAN_PRIORITY_MASK;
431 	}
432 }
433 
434 static void fl_set_key_flag(u32 flower_key, u32 flower_mask,
435 			    u32 *dissector_key, u32 *dissector_mask,
436 			    u32 flower_flag_bit, u32 dissector_flag_bit)
437 {
438 	if (flower_mask & flower_flag_bit) {
439 		*dissector_mask |= dissector_flag_bit;
440 		if (flower_key & flower_flag_bit)
441 			*dissector_key |= dissector_flag_bit;
442 	}
443 }
444 
445 static void fl_set_key_flags(struct nlattr **tb,
446 			     u32 *flags_key, u32 *flags_mask)
447 {
448 	u32 key, mask;
449 
450 	if (!tb[TCA_FLOWER_KEY_FLAGS])
451 		return;
452 
453 	key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS]));
454 
455 	if (!tb[TCA_FLOWER_KEY_FLAGS_MASK])
456 		mask = ~0;
457 	else
458 		mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK]));
459 
460 	*flags_key  = 0;
461 	*flags_mask = 0;
462 
463 	fl_set_key_flag(key, mask, flags_key, flags_mask,
464 			TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
465 }
466 
467 static int fl_set_key(struct net *net, struct nlattr **tb,
468 		      struct fl_flow_key *key, struct fl_flow_key *mask)
469 {
470 	__be16 ethertype;
471 #ifdef CONFIG_NET_CLS_IND
472 	if (tb[TCA_FLOWER_INDEV]) {
473 		int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV]);
474 		if (err < 0)
475 			return err;
476 		key->indev_ifindex = err;
477 		mask->indev_ifindex = 0xffffffff;
478 	}
479 #endif
480 
481 	fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
482 		       mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
483 		       sizeof(key->eth.dst));
484 	fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
485 		       mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
486 		       sizeof(key->eth.src));
487 
488 	if (tb[TCA_FLOWER_KEY_ETH_TYPE]) {
489 		ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]);
490 
491 		if (ethertype == htons(ETH_P_8021Q)) {
492 			fl_set_key_vlan(tb, &key->vlan, &mask->vlan);
493 			fl_set_key_val(tb, &key->basic.n_proto,
494 				       TCA_FLOWER_KEY_VLAN_ETH_TYPE,
495 				       &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
496 				       sizeof(key->basic.n_proto));
497 		} else {
498 			key->basic.n_proto = ethertype;
499 			mask->basic.n_proto = cpu_to_be16(~0);
500 		}
501 	}
502 
503 	if (key->basic.n_proto == htons(ETH_P_IP) ||
504 	    key->basic.n_proto == htons(ETH_P_IPV6)) {
505 		fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
506 			       &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
507 			       sizeof(key->basic.ip_proto));
508 	}
509 
510 	if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) {
511 		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
512 		mask->control.addr_type = ~0;
513 		fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
514 			       &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
515 			       sizeof(key->ipv4.src));
516 		fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
517 			       &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
518 			       sizeof(key->ipv4.dst));
519 	} else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) {
520 		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
521 		mask->control.addr_type = ~0;
522 		fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
523 			       &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
524 			       sizeof(key->ipv6.src));
525 		fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
526 			       &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
527 			       sizeof(key->ipv6.dst));
528 	}
529 
530 	if (key->basic.ip_proto == IPPROTO_TCP) {
531 		fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
532 			       &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
533 			       sizeof(key->tp.src));
534 		fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
535 			       &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
536 			       sizeof(key->tp.dst));
537 	} else if (key->basic.ip_proto == IPPROTO_UDP) {
538 		fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
539 			       &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
540 			       sizeof(key->tp.src));
541 		fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
542 			       &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
543 			       sizeof(key->tp.dst));
544 	} else if (key->basic.ip_proto == IPPROTO_SCTP) {
545 		fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
546 			       &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
547 			       sizeof(key->tp.src));
548 		fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
549 			       &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
550 			       sizeof(key->tp.dst));
551 	} else if (key->basic.n_proto == htons(ETH_P_IP) &&
552 		   key->basic.ip_proto == IPPROTO_ICMP) {
553 		fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE,
554 			       &mask->icmp.type,
555 			       TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
556 			       sizeof(key->icmp.type));
557 		fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE,
558 			       &mask->icmp.code,
559 			       TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
560 			       sizeof(key->icmp.code));
561 	} else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
562 		   key->basic.ip_proto == IPPROTO_ICMPV6) {
563 		fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE,
564 			       &mask->icmp.type,
565 			       TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
566 			       sizeof(key->icmp.type));
567 		fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE,
568 			       &mask->icmp.code,
569 			       TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
570 			       sizeof(key->icmp.code));
571 	}
572 
573 	if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] ||
574 	    tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) {
575 		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
576 		mask->enc_control.addr_type = ~0;
577 		fl_set_key_val(tb, &key->enc_ipv4.src,
578 			       TCA_FLOWER_KEY_ENC_IPV4_SRC,
579 			       &mask->enc_ipv4.src,
580 			       TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
581 			       sizeof(key->enc_ipv4.src));
582 		fl_set_key_val(tb, &key->enc_ipv4.dst,
583 			       TCA_FLOWER_KEY_ENC_IPV4_DST,
584 			       &mask->enc_ipv4.dst,
585 			       TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
586 			       sizeof(key->enc_ipv4.dst));
587 	}
588 
589 	if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] ||
590 	    tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) {
591 		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
592 		mask->enc_control.addr_type = ~0;
593 		fl_set_key_val(tb, &key->enc_ipv6.src,
594 			       TCA_FLOWER_KEY_ENC_IPV6_SRC,
595 			       &mask->enc_ipv6.src,
596 			       TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
597 			       sizeof(key->enc_ipv6.src));
598 		fl_set_key_val(tb, &key->enc_ipv6.dst,
599 			       TCA_FLOWER_KEY_ENC_IPV6_DST,
600 			       &mask->enc_ipv6.dst,
601 			       TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
602 			       sizeof(key->enc_ipv6.dst));
603 	}
604 
605 	fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID,
606 		       &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC,
607 		       sizeof(key->enc_key_id.keyid));
608 
609 	fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
610 		       &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
611 		       sizeof(key->enc_tp.src));
612 
613 	fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
614 		       &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
615 		       sizeof(key->enc_tp.dst));
616 
617 	fl_set_key_flags(tb, &key->control.flags, &mask->control.flags);
618 
619 	return 0;
620 }
621 
622 static bool fl_mask_eq(struct fl_flow_mask *mask1,
623 		       struct fl_flow_mask *mask2)
624 {
625 	const long *lmask1 = fl_key_get_start(&mask1->key, mask1);
626 	const long *lmask2 = fl_key_get_start(&mask2->key, mask2);
627 
628 	return !memcmp(&mask1->range, &mask2->range, sizeof(mask1->range)) &&
629 	       !memcmp(lmask1, lmask2, fl_mask_range(mask1));
630 }
631 
632 static const struct rhashtable_params fl_ht_params = {
633 	.key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */
634 	.head_offset = offsetof(struct cls_fl_filter, ht_node),
635 	.automatic_shrinking = true,
636 };
637 
638 static int fl_init_hashtable(struct cls_fl_head *head,
639 			     struct fl_flow_mask *mask)
640 {
641 	head->ht_params = fl_ht_params;
642 	head->ht_params.key_len = fl_mask_range(mask);
643 	head->ht_params.key_offset += mask->range.start;
644 
645 	return rhashtable_init(&head->ht, &head->ht_params);
646 }
647 
648 #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member)
649 #define FL_KEY_MEMBER_SIZE(member) (sizeof(((struct fl_flow_key *) 0)->member))
650 
651 #define FL_KEY_IS_MASKED(mask, member)						\
652 	memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member),		\
653 		   0, FL_KEY_MEMBER_SIZE(member))				\
654 
655 #define FL_KEY_SET(keys, cnt, id, member)					\
656 	do {									\
657 		keys[cnt].key_id = id;						\
658 		keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member);		\
659 		cnt++;								\
660 	} while(0);
661 
662 #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member)			\
663 	do {									\
664 		if (FL_KEY_IS_MASKED(mask, member))				\
665 			FL_KEY_SET(keys, cnt, id, member);			\
666 	} while(0);
667 
668 static void fl_init_dissector(struct cls_fl_head *head,
669 			      struct fl_flow_mask *mask)
670 {
671 	struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
672 	size_t cnt = 0;
673 
674 	FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
675 	FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
676 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
677 			     FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
678 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
679 			     FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
680 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
681 			     FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
682 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
683 			     FLOW_DISSECTOR_KEY_PORTS, tp);
684 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
685 			     FLOW_DISSECTOR_KEY_ICMP, icmp);
686 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
687 			     FLOW_DISSECTOR_KEY_VLAN, vlan);
688 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
689 			     FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
690 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
691 			     FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4);
692 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
693 			     FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6);
694 	if (FL_KEY_IS_MASKED(&mask->key, enc_ipv4) ||
695 	    FL_KEY_IS_MASKED(&mask->key, enc_ipv6))
696 		FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL,
697 			   enc_control);
698 	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
699 			     FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp);
700 
701 	skb_flow_dissector_init(&head->dissector, keys, cnt);
702 }
703 
704 static int fl_check_assign_mask(struct cls_fl_head *head,
705 				struct fl_flow_mask *mask)
706 {
707 	int err;
708 
709 	if (head->mask_assigned) {
710 		if (!fl_mask_eq(&head->mask, mask))
711 			return -EINVAL;
712 		else
713 			return 0;
714 	}
715 
716 	/* Mask is not assigned yet. So assign it and init hashtable
717 	 * according to that.
718 	 */
719 	err = fl_init_hashtable(head, mask);
720 	if (err)
721 		return err;
722 	memcpy(&head->mask, mask, sizeof(head->mask));
723 	head->mask_assigned = true;
724 
725 	fl_init_dissector(head, mask);
726 
727 	return 0;
728 }
729 
730 static int fl_set_parms(struct net *net, struct tcf_proto *tp,
731 			struct cls_fl_filter *f, struct fl_flow_mask *mask,
732 			unsigned long base, struct nlattr **tb,
733 			struct nlattr *est, bool ovr)
734 {
735 	struct tcf_exts e;
736 	int err;
737 
738 	err = tcf_exts_init(&e, TCA_FLOWER_ACT, 0);
739 	if (err < 0)
740 		return err;
741 	err = tcf_exts_validate(net, tp, tb, est, &e, ovr);
742 	if (err < 0)
743 		goto errout;
744 
745 	if (tb[TCA_FLOWER_CLASSID]) {
746 		f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]);
747 		tcf_bind_filter(tp, &f->res, base);
748 	}
749 
750 	err = fl_set_key(net, tb, &f->key, &mask->key);
751 	if (err)
752 		goto errout;
753 
754 	fl_mask_update_range(mask);
755 	fl_set_masked_key(&f->mkey, &f->key, mask);
756 
757 	tcf_exts_change(tp, &f->exts, &e);
758 
759 	return 0;
760 errout:
761 	tcf_exts_destroy(&e);
762 	return err;
763 }
764 
765 static u32 fl_grab_new_handle(struct tcf_proto *tp,
766 			      struct cls_fl_head *head)
767 {
768 	unsigned int i = 0x80000000;
769 	u32 handle;
770 
771 	do {
772 		if (++head->hgen == 0x7FFFFFFF)
773 			head->hgen = 1;
774 	} while (--i > 0 && fl_get(tp, head->hgen));
775 
776 	if (unlikely(i == 0)) {
777 		pr_err("Insufficient number of handles\n");
778 		handle = 0;
779 	} else {
780 		handle = head->hgen;
781 	}
782 
783 	return handle;
784 }
785 
786 static int fl_change(struct net *net, struct sk_buff *in_skb,
787 		     struct tcf_proto *tp, unsigned long base,
788 		     u32 handle, struct nlattr **tca,
789 		     unsigned long *arg, bool ovr)
790 {
791 	struct cls_fl_head *head = rtnl_dereference(tp->root);
792 	struct cls_fl_filter *fold = (struct cls_fl_filter *) *arg;
793 	struct cls_fl_filter *fnew;
794 	struct nlattr *tb[TCA_FLOWER_MAX + 1];
795 	struct fl_flow_mask mask = {};
796 	int err;
797 
798 	if (!tca[TCA_OPTIONS])
799 		return -EINVAL;
800 
801 	err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS], fl_policy);
802 	if (err < 0)
803 		return err;
804 
805 	if (fold && handle && fold->handle != handle)
806 		return -EINVAL;
807 
808 	fnew = kzalloc(sizeof(*fnew), GFP_KERNEL);
809 	if (!fnew)
810 		return -ENOBUFS;
811 
812 	err = tcf_exts_init(&fnew->exts, TCA_FLOWER_ACT, 0);
813 	if (err < 0)
814 		goto errout;
815 
816 	if (!handle) {
817 		handle = fl_grab_new_handle(tp, head);
818 		if (!handle) {
819 			err = -EINVAL;
820 			goto errout;
821 		}
822 	}
823 	fnew->handle = handle;
824 
825 	if (tb[TCA_FLOWER_FLAGS]) {
826 		fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]);
827 
828 		if (!tc_flags_valid(fnew->flags)) {
829 			err = -EINVAL;
830 			goto errout;
831 		}
832 	}
833 
834 	err = fl_set_parms(net, tp, fnew, &mask, base, tb, tca[TCA_RATE], ovr);
835 	if (err)
836 		goto errout;
837 
838 	err = fl_check_assign_mask(head, &mask);
839 	if (err)
840 		goto errout;
841 
842 	if (!tc_skip_sw(fnew->flags)) {
843 		err = rhashtable_insert_fast(&head->ht, &fnew->ht_node,
844 					     head->ht_params);
845 		if (err)
846 			goto errout;
847 	}
848 
849 	if (!tc_skip_hw(fnew->flags)) {
850 		err = fl_hw_replace_filter(tp,
851 					   &head->dissector,
852 					   &mask.key,
853 					   fnew);
854 		if (err)
855 			goto errout;
856 	}
857 
858 	if (fold) {
859 		if (!tc_skip_sw(fold->flags))
860 			rhashtable_remove_fast(&head->ht, &fold->ht_node,
861 					       head->ht_params);
862 		if (!tc_skip_hw(fold->flags))
863 			fl_hw_destroy_filter(tp, fold);
864 	}
865 
866 	*arg = (unsigned long) fnew;
867 
868 	if (fold) {
869 		list_replace_rcu(&fold->list, &fnew->list);
870 		tcf_unbind_filter(tp, &fold->res);
871 		call_rcu(&fold->rcu, fl_destroy_filter);
872 	} else {
873 		list_add_tail_rcu(&fnew->list, &head->filters);
874 	}
875 
876 	return 0;
877 
878 errout:
879 	tcf_exts_destroy(&fnew->exts);
880 	kfree(fnew);
881 	return err;
882 }
883 
884 static int fl_delete(struct tcf_proto *tp, unsigned long arg)
885 {
886 	struct cls_fl_head *head = rtnl_dereference(tp->root);
887 	struct cls_fl_filter *f = (struct cls_fl_filter *) arg;
888 
889 	if (!tc_skip_sw(f->flags))
890 		rhashtable_remove_fast(&head->ht, &f->ht_node,
891 				       head->ht_params);
892 	__fl_delete(tp, f);
893 	return 0;
894 }
895 
896 static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg)
897 {
898 	struct cls_fl_head *head = rtnl_dereference(tp->root);
899 	struct cls_fl_filter *f;
900 
901 	list_for_each_entry_rcu(f, &head->filters, list) {
902 		if (arg->count < arg->skip)
903 			goto skip;
904 		if (arg->fn(tp, (unsigned long) f, arg) < 0) {
905 			arg->stop = 1;
906 			break;
907 		}
908 skip:
909 		arg->count++;
910 	}
911 }
912 
913 static int fl_dump_key_val(struct sk_buff *skb,
914 			   void *val, int val_type,
915 			   void *mask, int mask_type, int len)
916 {
917 	int err;
918 
919 	if (!memchr_inv(mask, 0, len))
920 		return 0;
921 	err = nla_put(skb, val_type, len, val);
922 	if (err)
923 		return err;
924 	if (mask_type != TCA_FLOWER_UNSPEC) {
925 		err = nla_put(skb, mask_type, len, mask);
926 		if (err)
927 			return err;
928 	}
929 	return 0;
930 }
931 
932 static int fl_dump_key_vlan(struct sk_buff *skb,
933 			    struct flow_dissector_key_vlan *vlan_key,
934 			    struct flow_dissector_key_vlan *vlan_mask)
935 {
936 	int err;
937 
938 	if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask)))
939 		return 0;
940 	if (vlan_mask->vlan_id) {
941 		err = nla_put_u16(skb, TCA_FLOWER_KEY_VLAN_ID,
942 				  vlan_key->vlan_id);
943 		if (err)
944 			return err;
945 	}
946 	if (vlan_mask->vlan_priority) {
947 		err = nla_put_u8(skb, TCA_FLOWER_KEY_VLAN_PRIO,
948 				 vlan_key->vlan_priority);
949 		if (err)
950 			return err;
951 	}
952 	return 0;
953 }
954 
955 static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask,
956 			    u32 *flower_key, u32 *flower_mask,
957 			    u32 flower_flag_bit, u32 dissector_flag_bit)
958 {
959 	if (dissector_mask & dissector_flag_bit) {
960 		*flower_mask |= flower_flag_bit;
961 		if (dissector_key & dissector_flag_bit)
962 			*flower_key |= flower_flag_bit;
963 	}
964 }
965 
966 static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask)
967 {
968 	u32 key, mask;
969 	__be32 _key, _mask;
970 	int err;
971 
972 	if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask)))
973 		return 0;
974 
975 	key = 0;
976 	mask = 0;
977 
978 	fl_get_key_flag(flags_key, flags_mask, &key, &mask,
979 			TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
980 
981 	_key = cpu_to_be32(key);
982 	_mask = cpu_to_be32(mask);
983 
984 	err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key);
985 	if (err)
986 		return err;
987 
988 	return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask);
989 }
990 
991 static int fl_dump(struct net *net, struct tcf_proto *tp, unsigned long fh,
992 		   struct sk_buff *skb, struct tcmsg *t)
993 {
994 	struct cls_fl_head *head = rtnl_dereference(tp->root);
995 	struct cls_fl_filter *f = (struct cls_fl_filter *) fh;
996 	struct nlattr *nest;
997 	struct fl_flow_key *key, *mask;
998 
999 	if (!f)
1000 		return skb->len;
1001 
1002 	t->tcm_handle = f->handle;
1003 
1004 	nest = nla_nest_start(skb, TCA_OPTIONS);
1005 	if (!nest)
1006 		goto nla_put_failure;
1007 
1008 	if (f->res.classid &&
1009 	    nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid))
1010 		goto nla_put_failure;
1011 
1012 	key = &f->key;
1013 	mask = &head->mask.key;
1014 
1015 	if (mask->indev_ifindex) {
1016 		struct net_device *dev;
1017 
1018 		dev = __dev_get_by_index(net, key->indev_ifindex);
1019 		if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name))
1020 			goto nla_put_failure;
1021 	}
1022 
1023 	if (!tc_skip_hw(f->flags))
1024 		fl_hw_update_stats(tp, f);
1025 
1026 	if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
1027 			    mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
1028 			    sizeof(key->eth.dst)) ||
1029 	    fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
1030 			    mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
1031 			    sizeof(key->eth.src)) ||
1032 	    fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE,
1033 			    &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
1034 			    sizeof(key->basic.n_proto)))
1035 		goto nla_put_failure;
1036 
1037 	if (fl_dump_key_vlan(skb, &key->vlan, &mask->vlan))
1038 		goto nla_put_failure;
1039 
1040 	if ((key->basic.n_proto == htons(ETH_P_IP) ||
1041 	     key->basic.n_proto == htons(ETH_P_IPV6)) &&
1042 	    fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
1043 			    &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
1044 			    sizeof(key->basic.ip_proto)))
1045 		goto nla_put_failure;
1046 
1047 	if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
1048 	    (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
1049 			     &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
1050 			     sizeof(key->ipv4.src)) ||
1051 	     fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
1052 			     &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
1053 			     sizeof(key->ipv4.dst))))
1054 		goto nla_put_failure;
1055 	else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
1056 		 (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
1057 				  &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
1058 				  sizeof(key->ipv6.src)) ||
1059 		  fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
1060 				  &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
1061 				  sizeof(key->ipv6.dst))))
1062 		goto nla_put_failure;
1063 
1064 	if (key->basic.ip_proto == IPPROTO_TCP &&
1065 	    (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
1066 			     &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
1067 			     sizeof(key->tp.src)) ||
1068 	     fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
1069 			     &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
1070 			     sizeof(key->tp.dst))))
1071 		goto nla_put_failure;
1072 	else if (key->basic.ip_proto == IPPROTO_UDP &&
1073 		 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
1074 				  &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
1075 				  sizeof(key->tp.src)) ||
1076 		  fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
1077 				  &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
1078 				  sizeof(key->tp.dst))))
1079 		goto nla_put_failure;
1080 	else if (key->basic.ip_proto == IPPROTO_SCTP &&
1081 		 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
1082 				  &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
1083 				  sizeof(key->tp.src)) ||
1084 		  fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
1085 				  &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
1086 				  sizeof(key->tp.dst))))
1087 		goto nla_put_failure;
1088 	else if (key->basic.n_proto == htons(ETH_P_IP) &&
1089 		 key->basic.ip_proto == IPPROTO_ICMP &&
1090 		 (fl_dump_key_val(skb, &key->icmp.type,
1091 				  TCA_FLOWER_KEY_ICMPV4_TYPE, &mask->icmp.type,
1092 				  TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
1093 				  sizeof(key->icmp.type)) ||
1094 		  fl_dump_key_val(skb, &key->icmp.code,
1095 				  TCA_FLOWER_KEY_ICMPV4_CODE, &mask->icmp.code,
1096 				  TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
1097 				  sizeof(key->icmp.code))))
1098 		goto nla_put_failure;
1099 	else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
1100 		 key->basic.ip_proto == IPPROTO_ICMPV6 &&
1101 		 (fl_dump_key_val(skb, &key->icmp.type,
1102 				  TCA_FLOWER_KEY_ICMPV6_TYPE, &mask->icmp.type,
1103 				  TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
1104 				  sizeof(key->icmp.type)) ||
1105 		  fl_dump_key_val(skb, &key->icmp.code,
1106 				  TCA_FLOWER_KEY_ICMPV6_CODE, &mask->icmp.code,
1107 				  TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
1108 				  sizeof(key->icmp.code))))
1109 		goto nla_put_failure;
1110 
1111 	if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
1112 	    (fl_dump_key_val(skb, &key->enc_ipv4.src,
1113 			    TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src,
1114 			    TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
1115 			    sizeof(key->enc_ipv4.src)) ||
1116 	     fl_dump_key_val(skb, &key->enc_ipv4.dst,
1117 			     TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst,
1118 			     TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
1119 			     sizeof(key->enc_ipv4.dst))))
1120 		goto nla_put_failure;
1121 	else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
1122 		 (fl_dump_key_val(skb, &key->enc_ipv6.src,
1123 			    TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src,
1124 			    TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
1125 			    sizeof(key->enc_ipv6.src)) ||
1126 		 fl_dump_key_val(skb, &key->enc_ipv6.dst,
1127 				 TCA_FLOWER_KEY_ENC_IPV6_DST,
1128 				 &mask->enc_ipv6.dst,
1129 				 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
1130 			    sizeof(key->enc_ipv6.dst))))
1131 		goto nla_put_failure;
1132 
1133 	if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID,
1134 			    &mask->enc_key_id, TCA_FLOWER_UNSPEC,
1135 			    sizeof(key->enc_key_id)) ||
1136 	    fl_dump_key_val(skb, &key->enc_tp.src,
1137 			    TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
1138 			    &mask->enc_tp.src,
1139 			    TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
1140 			    sizeof(key->enc_tp.src)) ||
1141 	    fl_dump_key_val(skb, &key->enc_tp.dst,
1142 			    TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
1143 			    &mask->enc_tp.dst,
1144 			    TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
1145 			    sizeof(key->enc_tp.dst)))
1146 		goto nla_put_failure;
1147 
1148 	if (fl_dump_key_flags(skb, key->control.flags, mask->control.flags))
1149 		goto nla_put_failure;
1150 
1151 	nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags);
1152 
1153 	if (tcf_exts_dump(skb, &f->exts))
1154 		goto nla_put_failure;
1155 
1156 	nla_nest_end(skb, nest);
1157 
1158 	if (tcf_exts_dump_stats(skb, &f->exts) < 0)
1159 		goto nla_put_failure;
1160 
1161 	return skb->len;
1162 
1163 nla_put_failure:
1164 	nla_nest_cancel(skb, nest);
1165 	return -1;
1166 }
1167 
1168 static struct tcf_proto_ops cls_fl_ops __read_mostly = {
1169 	.kind		= "flower",
1170 	.classify	= fl_classify,
1171 	.init		= fl_init,
1172 	.destroy	= fl_destroy,
1173 	.get		= fl_get,
1174 	.change		= fl_change,
1175 	.delete		= fl_delete,
1176 	.walk		= fl_walk,
1177 	.dump		= fl_dump,
1178 	.owner		= THIS_MODULE,
1179 };
1180 
1181 static int __init cls_fl_init(void)
1182 {
1183 	return register_tcf_proto_ops(&cls_fl_ops);
1184 }
1185 
1186 static void __exit cls_fl_exit(void)
1187 {
1188 	unregister_tcf_proto_ops(&cls_fl_ops);
1189 }
1190 
1191 module_init(cls_fl_init);
1192 module_exit(cls_fl_exit);
1193 
1194 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
1195 MODULE_DESCRIPTION("Flower classifier");
1196 MODULE_LICENSE("GPL v2");
1197