1 #include <linux/kernel.h>
2 #include <linux/init.h>
3 #include <linux/module.h>
4 #include <linux/netfilter.h>
5 #include <linux/rhashtable.h>
6 #include <linux/netdevice.h>
7 #include <linux/tc_act/tc_csum.h>
8 #include <net/flow_offload.h>
9 #include <net/ip_tunnels.h>
10 #include <net/netfilter/nf_flow_table.h>
11 #include <net/netfilter/nf_tables.h>
12 #include <net/netfilter/nf_conntrack.h>
13 #include <net/netfilter/nf_conntrack_acct.h>
14 #include <net/netfilter/nf_conntrack_core.h>
15 #include <net/netfilter/nf_conntrack_tuple.h>
16
17 static struct workqueue_struct *nf_flow_offload_add_wq;
18 static struct workqueue_struct *nf_flow_offload_del_wq;
19 static struct workqueue_struct *nf_flow_offload_stats_wq;
20
21 struct flow_offload_work {
22 struct list_head list;
23 enum flow_cls_command cmd;
24 struct nf_flowtable *flowtable;
25 struct flow_offload *flow;
26 struct work_struct work;
27 };
28
29 #define NF_FLOW_DISSECTOR(__match, __type, __field) \
30 (__match)->dissector.offset[__type] = \
31 offsetof(struct nf_flow_key, __field)
32
nf_flow_rule_lwt_match(struct nf_flow_match * match,struct ip_tunnel_info * tun_info)33 static void nf_flow_rule_lwt_match(struct nf_flow_match *match,
34 struct ip_tunnel_info *tun_info)
35 {
36 struct nf_flow_key *mask = &match->mask;
37 struct nf_flow_key *key = &match->key;
38 unsigned long long enc_keys;
39
40 if (!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX))
41 return;
42
43 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_CONTROL, enc_control);
44 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
45 key->enc_key_id.keyid = tunnel_id_to_key32(tun_info->key.tun_id);
46 mask->enc_key_id.keyid = 0xffffffff;
47 enc_keys = BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
48 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL);
49
50 if (ip_tunnel_info_af(tun_info) == AF_INET) {
51 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
52 enc_ipv4);
53 key->enc_ipv4.src = tun_info->key.u.ipv4.dst;
54 key->enc_ipv4.dst = tun_info->key.u.ipv4.src;
55 if (key->enc_ipv4.src)
56 mask->enc_ipv4.src = 0xffffffff;
57 if (key->enc_ipv4.dst)
58 mask->enc_ipv4.dst = 0xffffffff;
59 enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS);
60 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
61 } else {
62 memcpy(&key->enc_ipv6.src, &tun_info->key.u.ipv6.dst,
63 sizeof(struct in6_addr));
64 memcpy(&key->enc_ipv6.dst, &tun_info->key.u.ipv6.src,
65 sizeof(struct in6_addr));
66 if (memcmp(&key->enc_ipv6.src, &in6addr_any,
67 sizeof(struct in6_addr)))
68 memset(&mask->enc_ipv6.src, 0xff,
69 sizeof(struct in6_addr));
70 if (memcmp(&key->enc_ipv6.dst, &in6addr_any,
71 sizeof(struct in6_addr)))
72 memset(&mask->enc_ipv6.dst, 0xff,
73 sizeof(struct in6_addr));
74 enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS);
75 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
76 }
77
78 match->dissector.used_keys |= enc_keys;
79 }
80
nf_flow_rule_vlan_match(struct flow_dissector_key_vlan * key,struct flow_dissector_key_vlan * mask,u16 vlan_id,__be16 proto)81 static void nf_flow_rule_vlan_match(struct flow_dissector_key_vlan *key,
82 struct flow_dissector_key_vlan *mask,
83 u16 vlan_id, __be16 proto)
84 {
85 key->vlan_id = vlan_id;
86 mask->vlan_id = VLAN_VID_MASK;
87 key->vlan_tpid = proto;
88 mask->vlan_tpid = 0xffff;
89 }
90
nf_flow_rule_match(struct nf_flow_match * match,const struct flow_offload_tuple * tuple,struct dst_entry * other_dst)91 static int nf_flow_rule_match(struct nf_flow_match *match,
92 const struct flow_offload_tuple *tuple,
93 struct dst_entry *other_dst)
94 {
95 struct nf_flow_key *mask = &match->mask;
96 struct nf_flow_key *key = &match->key;
97 struct ip_tunnel_info *tun_info;
98 bool vlan_encap = false;
99
100 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_META, meta);
101 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CONTROL, control);
102 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_BASIC, basic);
103 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
104 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
105 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_TCP, tcp);
106 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_PORTS, tp);
107
108 if (other_dst && other_dst->lwtstate) {
109 tun_info = lwt_tun_info(other_dst->lwtstate);
110 nf_flow_rule_lwt_match(match, tun_info);
111 }
112
113 if (tuple->xmit_type == FLOW_OFFLOAD_XMIT_TC)
114 key->meta.ingress_ifindex = tuple->tc.iifidx;
115 else
116 key->meta.ingress_ifindex = tuple->iifidx;
117
118 mask->meta.ingress_ifindex = 0xffffffff;
119
120 if (tuple->encap_num > 0 && !(tuple->in_vlan_ingress & BIT(0)) &&
121 tuple->encap[0].proto == htons(ETH_P_8021Q)) {
122 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, vlan);
123 nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
124 tuple->encap[0].id,
125 tuple->encap[0].proto);
126 vlan_encap = true;
127 }
128
129 if (tuple->encap_num > 1 && !(tuple->in_vlan_ingress & BIT(1)) &&
130 tuple->encap[1].proto == htons(ETH_P_8021Q)) {
131 if (vlan_encap) {
132 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CVLAN,
133 cvlan);
134 nf_flow_rule_vlan_match(&key->cvlan, &mask->cvlan,
135 tuple->encap[1].id,
136 tuple->encap[1].proto);
137 } else {
138 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN,
139 vlan);
140 nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
141 tuple->encap[1].id,
142 tuple->encap[1].proto);
143 }
144 }
145
146 switch (tuple->l3proto) {
147 case AF_INET:
148 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
149 key->basic.n_proto = htons(ETH_P_IP);
150 key->ipv4.src = tuple->src_v4.s_addr;
151 mask->ipv4.src = 0xffffffff;
152 key->ipv4.dst = tuple->dst_v4.s_addr;
153 mask->ipv4.dst = 0xffffffff;
154 break;
155 case AF_INET6:
156 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
157 key->basic.n_proto = htons(ETH_P_IPV6);
158 key->ipv6.src = tuple->src_v6;
159 memset(&mask->ipv6.src, 0xff, sizeof(mask->ipv6.src));
160 key->ipv6.dst = tuple->dst_v6;
161 memset(&mask->ipv6.dst, 0xff, sizeof(mask->ipv6.dst));
162 break;
163 default:
164 return -EOPNOTSUPP;
165 }
166 mask->control.addr_type = 0xffff;
167 match->dissector.used_keys |= BIT_ULL(key->control.addr_type);
168 mask->basic.n_proto = 0xffff;
169
170 switch (tuple->l4proto) {
171 case IPPROTO_TCP:
172 key->tcp.flags = 0;
173 mask->tcp.flags = cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16);
174 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_TCP);
175 break;
176 case IPPROTO_UDP:
177 case IPPROTO_GRE:
178 break;
179 default:
180 return -EOPNOTSUPP;
181 }
182
183 key->basic.ip_proto = tuple->l4proto;
184 mask->basic.ip_proto = 0xff;
185
186 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_META) |
187 BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
188 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC);
189
190 switch (tuple->l4proto) {
191 case IPPROTO_TCP:
192 case IPPROTO_UDP:
193 key->tp.src = tuple->src_port;
194 mask->tp.src = 0xffff;
195 key->tp.dst = tuple->dst_port;
196 mask->tp.dst = 0xffff;
197
198 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_PORTS);
199 break;
200 }
201
202 return 0;
203 }
204
flow_offload_mangle(struct flow_action_entry * entry,enum flow_action_mangle_base htype,u32 offset,const __be32 * value,const __be32 * mask)205 static void flow_offload_mangle(struct flow_action_entry *entry,
206 enum flow_action_mangle_base htype, u32 offset,
207 const __be32 *value, const __be32 *mask)
208 {
209 entry->id = FLOW_ACTION_MANGLE;
210 entry->mangle.htype = htype;
211 entry->mangle.offset = offset;
212 memcpy(&entry->mangle.mask, mask, sizeof(u32));
213 memcpy(&entry->mangle.val, value, sizeof(u32));
214 }
215
216 static inline struct flow_action_entry *
flow_action_entry_next(struct nf_flow_rule * flow_rule)217 flow_action_entry_next(struct nf_flow_rule *flow_rule)
218 {
219 int i = flow_rule->rule->action.num_entries++;
220
221 return &flow_rule->rule->action.entries[i];
222 }
223
flow_offload_eth_src(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)224 static int flow_offload_eth_src(struct net *net,
225 const struct flow_offload *flow,
226 enum flow_offload_tuple_dir dir,
227 struct nf_flow_rule *flow_rule)
228 {
229 struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
230 struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
231 const struct flow_offload_tuple *other_tuple, *this_tuple;
232 struct net_device *dev = NULL;
233 const unsigned char *addr;
234 u32 mask, val;
235 u16 val16;
236
237 this_tuple = &flow->tuplehash[dir].tuple;
238
239 switch (this_tuple->xmit_type) {
240 case FLOW_OFFLOAD_XMIT_DIRECT:
241 addr = this_tuple->out.h_source;
242 break;
243 case FLOW_OFFLOAD_XMIT_NEIGH:
244 other_tuple = &flow->tuplehash[!dir].tuple;
245 dev = dev_get_by_index(net, other_tuple->iifidx);
246 if (!dev)
247 return -ENOENT;
248
249 addr = dev->dev_addr;
250 break;
251 default:
252 return -EOPNOTSUPP;
253 }
254
255 mask = ~0xffff0000;
256 memcpy(&val16, addr, 2);
257 val = val16 << 16;
258 flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
259 &val, &mask);
260
261 mask = ~0xffffffff;
262 memcpy(&val, addr + 2, 4);
263 flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 8,
264 &val, &mask);
265
266 dev_put(dev);
267
268 return 0;
269 }
270
flow_offload_eth_dst(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)271 static int flow_offload_eth_dst(struct net *net,
272 const struct flow_offload *flow,
273 enum flow_offload_tuple_dir dir,
274 struct nf_flow_rule *flow_rule)
275 {
276 struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
277 struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
278 const struct flow_offload_tuple *other_tuple, *this_tuple;
279 const struct dst_entry *dst_cache;
280 unsigned char ha[ETH_ALEN];
281 struct neighbour *n;
282 const void *daddr;
283 u32 mask, val;
284 u8 nud_state;
285 u16 val16;
286
287 this_tuple = &flow->tuplehash[dir].tuple;
288
289 switch (this_tuple->xmit_type) {
290 case FLOW_OFFLOAD_XMIT_DIRECT:
291 ether_addr_copy(ha, this_tuple->out.h_dest);
292 break;
293 case FLOW_OFFLOAD_XMIT_NEIGH:
294 other_tuple = &flow->tuplehash[!dir].tuple;
295 daddr = &other_tuple->src_v4;
296 dst_cache = this_tuple->dst_cache;
297 n = dst_neigh_lookup(dst_cache, daddr);
298 if (!n)
299 return -ENOENT;
300
301 read_lock_bh(&n->lock);
302 nud_state = n->nud_state;
303 ether_addr_copy(ha, n->ha);
304 read_unlock_bh(&n->lock);
305 neigh_release(n);
306
307 if (!(nud_state & NUD_VALID))
308 return -ENOENT;
309 break;
310 default:
311 return -EOPNOTSUPP;
312 }
313
314 mask = ~0xffffffff;
315 memcpy(&val, ha, 4);
316 flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 0,
317 &val, &mask);
318
319 mask = ~0x0000ffff;
320 memcpy(&val16, ha + 4, 2);
321 val = val16;
322 flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
323 &val, &mask);
324
325 return 0;
326 }
327
flow_offload_ipv4_snat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)328 static void flow_offload_ipv4_snat(struct net *net,
329 const struct flow_offload *flow,
330 enum flow_offload_tuple_dir dir,
331 struct nf_flow_rule *flow_rule)
332 {
333 struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
334 u32 mask = ~htonl(0xffffffff);
335 __be32 addr;
336 u32 offset;
337
338 switch (dir) {
339 case FLOW_OFFLOAD_DIR_ORIGINAL:
340 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr;
341 offset = offsetof(struct iphdr, saddr);
342 break;
343 case FLOW_OFFLOAD_DIR_REPLY:
344 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
345 offset = offsetof(struct iphdr, daddr);
346 break;
347 default:
348 return;
349 }
350
351 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
352 &addr, &mask);
353 }
354
flow_offload_ipv4_dnat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)355 static void flow_offload_ipv4_dnat(struct net *net,
356 const struct flow_offload *flow,
357 enum flow_offload_tuple_dir dir,
358 struct nf_flow_rule *flow_rule)
359 {
360 struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
361 u32 mask = ~htonl(0xffffffff);
362 __be32 addr;
363 u32 offset;
364
365 switch (dir) {
366 case FLOW_OFFLOAD_DIR_ORIGINAL:
367 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr;
368 offset = offsetof(struct iphdr, daddr);
369 break;
370 case FLOW_OFFLOAD_DIR_REPLY:
371 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
372 offset = offsetof(struct iphdr, saddr);
373 break;
374 default:
375 return;
376 }
377
378 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
379 &addr, &mask);
380 }
381
flow_offload_ipv6_mangle(struct nf_flow_rule * flow_rule,unsigned int offset,const __be32 * addr,const __be32 * mask)382 static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule,
383 unsigned int offset,
384 const __be32 *addr, const __be32 *mask)
385 {
386 struct flow_action_entry *entry;
387 int i;
388
389 for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i++) {
390 entry = flow_action_entry_next(flow_rule);
391 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6,
392 offset + i * sizeof(u32), &addr[i], mask);
393 }
394 }
395
flow_offload_ipv6_snat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)396 static void flow_offload_ipv6_snat(struct net *net,
397 const struct flow_offload *flow,
398 enum flow_offload_tuple_dir dir,
399 struct nf_flow_rule *flow_rule)
400 {
401 u32 mask = ~htonl(0xffffffff);
402 const __be32 *addr;
403 u32 offset;
404
405 switch (dir) {
406 case FLOW_OFFLOAD_DIR_ORIGINAL:
407 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6.s6_addr32;
408 offset = offsetof(struct ipv6hdr, saddr);
409 break;
410 case FLOW_OFFLOAD_DIR_REPLY:
411 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6.s6_addr32;
412 offset = offsetof(struct ipv6hdr, daddr);
413 break;
414 default:
415 return;
416 }
417
418 flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
419 }
420
flow_offload_ipv6_dnat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)421 static void flow_offload_ipv6_dnat(struct net *net,
422 const struct flow_offload *flow,
423 enum flow_offload_tuple_dir dir,
424 struct nf_flow_rule *flow_rule)
425 {
426 u32 mask = ~htonl(0xffffffff);
427 const __be32 *addr;
428 u32 offset;
429
430 switch (dir) {
431 case FLOW_OFFLOAD_DIR_ORIGINAL:
432 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6.s6_addr32;
433 offset = offsetof(struct ipv6hdr, daddr);
434 break;
435 case FLOW_OFFLOAD_DIR_REPLY:
436 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6.s6_addr32;
437 offset = offsetof(struct ipv6hdr, saddr);
438 break;
439 default:
440 return;
441 }
442
443 flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
444 }
445
flow_offload_l4proto(const struct flow_offload * flow)446 static int flow_offload_l4proto(const struct flow_offload *flow)
447 {
448 u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
449 u8 type = 0;
450
451 switch (protonum) {
452 case IPPROTO_TCP:
453 type = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
454 break;
455 case IPPROTO_UDP:
456 type = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
457 break;
458 default:
459 break;
460 }
461
462 return type;
463 }
464
flow_offload_port_snat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)465 static void flow_offload_port_snat(struct net *net,
466 const struct flow_offload *flow,
467 enum flow_offload_tuple_dir dir,
468 struct nf_flow_rule *flow_rule)
469 {
470 struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
471 u32 mask, port;
472 u32 offset;
473
474 switch (dir) {
475 case FLOW_OFFLOAD_DIR_ORIGINAL:
476 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port);
477 offset = 0; /* offsetof(struct tcphdr, source); */
478 port = htonl(port << 16);
479 mask = ~htonl(0xffff0000);
480 break;
481 case FLOW_OFFLOAD_DIR_REPLY:
482 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port);
483 offset = 0; /* offsetof(struct tcphdr, dest); */
484 port = htonl(port);
485 mask = ~htonl(0xffff);
486 break;
487 default:
488 return;
489 }
490
491 flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
492 &port, &mask);
493 }
494
flow_offload_port_dnat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)495 static void flow_offload_port_dnat(struct net *net,
496 const struct flow_offload *flow,
497 enum flow_offload_tuple_dir dir,
498 struct nf_flow_rule *flow_rule)
499 {
500 struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
501 u32 mask, port;
502 u32 offset;
503
504 switch (dir) {
505 case FLOW_OFFLOAD_DIR_ORIGINAL:
506 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port);
507 offset = 0; /* offsetof(struct tcphdr, dest); */
508 port = htonl(port);
509 mask = ~htonl(0xffff);
510 break;
511 case FLOW_OFFLOAD_DIR_REPLY:
512 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port);
513 offset = 0; /* offsetof(struct tcphdr, source); */
514 port = htonl(port << 16);
515 mask = ~htonl(0xffff0000);
516 break;
517 default:
518 return;
519 }
520
521 flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
522 &port, &mask);
523 }
524
flow_offload_ipv4_checksum(struct net * net,const struct flow_offload * flow,struct nf_flow_rule * flow_rule)525 static void flow_offload_ipv4_checksum(struct net *net,
526 const struct flow_offload *flow,
527 struct nf_flow_rule *flow_rule)
528 {
529 u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
530 struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
531
532 entry->id = FLOW_ACTION_CSUM;
533 entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR;
534
535 switch (protonum) {
536 case IPPROTO_TCP:
537 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_TCP;
538 break;
539 case IPPROTO_UDP:
540 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP;
541 break;
542 }
543 }
544
flow_offload_redirect(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)545 static void flow_offload_redirect(struct net *net,
546 const struct flow_offload *flow,
547 enum flow_offload_tuple_dir dir,
548 struct nf_flow_rule *flow_rule)
549 {
550 const struct flow_offload_tuple *this_tuple, *other_tuple;
551 struct flow_action_entry *entry;
552 struct net_device *dev;
553 int ifindex;
554
555 this_tuple = &flow->tuplehash[dir].tuple;
556 switch (this_tuple->xmit_type) {
557 case FLOW_OFFLOAD_XMIT_DIRECT:
558 this_tuple = &flow->tuplehash[dir].tuple;
559 ifindex = this_tuple->out.ifidx;
560 break;
561 case FLOW_OFFLOAD_XMIT_NEIGH:
562 other_tuple = &flow->tuplehash[!dir].tuple;
563 ifindex = other_tuple->iifidx;
564 break;
565 default:
566 return;
567 }
568
569 dev = dev_get_by_index(net, ifindex);
570 if (!dev)
571 return;
572
573 entry = flow_action_entry_next(flow_rule);
574 entry->id = FLOW_ACTION_REDIRECT;
575 entry->dev = dev;
576 }
577
flow_offload_encap_tunnel(const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)578 static void flow_offload_encap_tunnel(const struct flow_offload *flow,
579 enum flow_offload_tuple_dir dir,
580 struct nf_flow_rule *flow_rule)
581 {
582 const struct flow_offload_tuple *this_tuple;
583 struct flow_action_entry *entry;
584 struct dst_entry *dst;
585
586 this_tuple = &flow->tuplehash[dir].tuple;
587 if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
588 return;
589
590 dst = this_tuple->dst_cache;
591 if (dst && dst->lwtstate) {
592 struct ip_tunnel_info *tun_info;
593
594 tun_info = lwt_tun_info(dst->lwtstate);
595 if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
596 entry = flow_action_entry_next(flow_rule);
597 entry->id = FLOW_ACTION_TUNNEL_ENCAP;
598 entry->tunnel = tun_info;
599 }
600 }
601 }
602
flow_offload_decap_tunnel(const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)603 static void flow_offload_decap_tunnel(const struct flow_offload *flow,
604 enum flow_offload_tuple_dir dir,
605 struct nf_flow_rule *flow_rule)
606 {
607 const struct flow_offload_tuple *other_tuple;
608 struct flow_action_entry *entry;
609 struct dst_entry *dst;
610
611 other_tuple = &flow->tuplehash[!dir].tuple;
612 if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
613 return;
614
615 dst = other_tuple->dst_cache;
616 if (dst && dst->lwtstate) {
617 struct ip_tunnel_info *tun_info;
618
619 tun_info = lwt_tun_info(dst->lwtstate);
620 if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
621 entry = flow_action_entry_next(flow_rule);
622 entry->id = FLOW_ACTION_TUNNEL_DECAP;
623 }
624 }
625 }
626
627 static int
nf_flow_rule_route_common(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)628 nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow,
629 enum flow_offload_tuple_dir dir,
630 struct nf_flow_rule *flow_rule)
631 {
632 const struct flow_offload_tuple *other_tuple;
633 const struct flow_offload_tuple *tuple;
634 int i;
635
636 flow_offload_decap_tunnel(flow, dir, flow_rule);
637 flow_offload_encap_tunnel(flow, dir, flow_rule);
638
639 if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 ||
640 flow_offload_eth_dst(net, flow, dir, flow_rule) < 0)
641 return -1;
642
643 tuple = &flow->tuplehash[dir].tuple;
644
645 for (i = 0; i < tuple->encap_num; i++) {
646 struct flow_action_entry *entry;
647
648 if (tuple->in_vlan_ingress & BIT(i))
649 continue;
650
651 if (tuple->encap[i].proto == htons(ETH_P_8021Q)) {
652 entry = flow_action_entry_next(flow_rule);
653 entry->id = FLOW_ACTION_VLAN_POP;
654 }
655 }
656
657 other_tuple = &flow->tuplehash[!dir].tuple;
658
659 for (i = 0; i < other_tuple->encap_num; i++) {
660 struct flow_action_entry *entry;
661
662 if (other_tuple->in_vlan_ingress & BIT(i))
663 continue;
664
665 entry = flow_action_entry_next(flow_rule);
666
667 switch (other_tuple->encap[i].proto) {
668 case htons(ETH_P_PPP_SES):
669 entry->id = FLOW_ACTION_PPPOE_PUSH;
670 entry->pppoe.sid = other_tuple->encap[i].id;
671 break;
672 case htons(ETH_P_8021Q):
673 entry->id = FLOW_ACTION_VLAN_PUSH;
674 entry->vlan.vid = other_tuple->encap[i].id;
675 entry->vlan.proto = other_tuple->encap[i].proto;
676 break;
677 }
678 }
679
680 return 0;
681 }
682
nf_flow_rule_route_ipv4(struct net * net,struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)683 int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow,
684 enum flow_offload_tuple_dir dir,
685 struct nf_flow_rule *flow_rule)
686 {
687 if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
688 return -1;
689
690 if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
691 flow_offload_ipv4_snat(net, flow, dir, flow_rule);
692 flow_offload_port_snat(net, flow, dir, flow_rule);
693 }
694 if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
695 flow_offload_ipv4_dnat(net, flow, dir, flow_rule);
696 flow_offload_port_dnat(net, flow, dir, flow_rule);
697 }
698 if (test_bit(NF_FLOW_SNAT, &flow->flags) ||
699 test_bit(NF_FLOW_DNAT, &flow->flags))
700 flow_offload_ipv4_checksum(net, flow, flow_rule);
701
702 flow_offload_redirect(net, flow, dir, flow_rule);
703
704 return 0;
705 }
706 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv4);
707
nf_flow_rule_route_ipv6(struct net * net,struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)708 int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow,
709 enum flow_offload_tuple_dir dir,
710 struct nf_flow_rule *flow_rule)
711 {
712 if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
713 return -1;
714
715 if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
716 flow_offload_ipv6_snat(net, flow, dir, flow_rule);
717 flow_offload_port_snat(net, flow, dir, flow_rule);
718 }
719 if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
720 flow_offload_ipv6_dnat(net, flow, dir, flow_rule);
721 flow_offload_port_dnat(net, flow, dir, flow_rule);
722 }
723
724 flow_offload_redirect(net, flow, dir, flow_rule);
725
726 return 0;
727 }
728 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6);
729
730 #define NF_FLOW_RULE_ACTION_MAX 16
731
732 static struct nf_flow_rule *
nf_flow_offload_rule_alloc(struct net * net,const struct flow_offload_work * offload,enum flow_offload_tuple_dir dir)733 nf_flow_offload_rule_alloc(struct net *net,
734 const struct flow_offload_work *offload,
735 enum flow_offload_tuple_dir dir)
736 {
737 const struct nf_flowtable *flowtable = offload->flowtable;
738 const struct flow_offload_tuple *tuple, *other_tuple;
739 struct flow_offload *flow = offload->flow;
740 struct dst_entry *other_dst = NULL;
741 struct nf_flow_rule *flow_rule;
742 int err = -ENOMEM;
743
744 flow_rule = kzalloc_obj(*flow_rule);
745 if (!flow_rule)
746 goto err_flow;
747
748 flow_rule->rule = flow_rule_alloc(NF_FLOW_RULE_ACTION_MAX);
749 if (!flow_rule->rule)
750 goto err_flow_rule;
751
752 flow_rule->rule->match.dissector = &flow_rule->match.dissector;
753 flow_rule->rule->match.mask = &flow_rule->match.mask;
754 flow_rule->rule->match.key = &flow_rule->match.key;
755
756 tuple = &flow->tuplehash[dir].tuple;
757 other_tuple = &flow->tuplehash[!dir].tuple;
758 if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH)
759 other_dst = other_tuple->dst_cache;
760
761 err = nf_flow_rule_match(&flow_rule->match, tuple, other_dst);
762 if (err < 0)
763 goto err_flow_match;
764
765 flow_rule->rule->action.num_entries = 0;
766 if (flowtable->type->action(net, flow, dir, flow_rule) < 0)
767 goto err_flow_match;
768
769 return flow_rule;
770
771 err_flow_match:
772 kfree(flow_rule->rule);
773 err_flow_rule:
774 kfree(flow_rule);
775 err_flow:
776 return NULL;
777 }
778
__nf_flow_offload_destroy(struct nf_flow_rule * flow_rule)779 static void __nf_flow_offload_destroy(struct nf_flow_rule *flow_rule)
780 {
781 struct flow_action_entry *entry;
782 int i;
783
784 for (i = 0; i < flow_rule->rule->action.num_entries; i++) {
785 entry = &flow_rule->rule->action.entries[i];
786 if (entry->id != FLOW_ACTION_REDIRECT)
787 continue;
788
789 dev_put(entry->dev);
790 }
791 kfree(flow_rule->rule);
792 kfree(flow_rule);
793 }
794
nf_flow_offload_destroy(struct nf_flow_rule * flow_rule[])795 static void nf_flow_offload_destroy(struct nf_flow_rule *flow_rule[])
796 {
797 int i;
798
799 for (i = 0; i < FLOW_OFFLOAD_DIR_MAX; i++)
800 __nf_flow_offload_destroy(flow_rule[i]);
801 }
802
nf_flow_offload_alloc(const struct flow_offload_work * offload,struct nf_flow_rule * flow_rule[])803 static int nf_flow_offload_alloc(const struct flow_offload_work *offload,
804 struct nf_flow_rule *flow_rule[])
805 {
806 struct net *net = read_pnet(&offload->flowtable->net);
807
808 flow_rule[0] = nf_flow_offload_rule_alloc(net, offload,
809 FLOW_OFFLOAD_DIR_ORIGINAL);
810 if (!flow_rule[0])
811 return -ENOMEM;
812
813 flow_rule[1] = nf_flow_offload_rule_alloc(net, offload,
814 FLOW_OFFLOAD_DIR_REPLY);
815 if (!flow_rule[1]) {
816 __nf_flow_offload_destroy(flow_rule[0]);
817 return -ENOMEM;
818 }
819
820 return 0;
821 }
822
nf_flow_offload_init(struct flow_cls_offload * cls_flow,__be16 proto,int priority,enum flow_cls_command cmd,const struct flow_offload_tuple * tuple,struct netlink_ext_ack * extack)823 static void nf_flow_offload_init(struct flow_cls_offload *cls_flow,
824 __be16 proto, int priority,
825 enum flow_cls_command cmd,
826 const struct flow_offload_tuple *tuple,
827 struct netlink_ext_ack *extack)
828 {
829 cls_flow->common.protocol = proto;
830 cls_flow->common.prio = priority;
831 cls_flow->common.extack = extack;
832 cls_flow->command = cmd;
833 cls_flow->cookie = (unsigned long)tuple;
834 }
835
nf_flow_offload_tuple(struct nf_flowtable * flowtable,struct flow_offload * flow,struct nf_flow_rule * flow_rule,enum flow_offload_tuple_dir dir,int priority,int cmd,struct flow_stats * stats,struct list_head * block_cb_list)836 static int nf_flow_offload_tuple(struct nf_flowtable *flowtable,
837 struct flow_offload *flow,
838 struct nf_flow_rule *flow_rule,
839 enum flow_offload_tuple_dir dir,
840 int priority, int cmd,
841 struct flow_stats *stats,
842 struct list_head *block_cb_list)
843 {
844 struct flow_cls_offload cls_flow = {};
845 struct netlink_ext_ack extack = {};
846 struct flow_block_cb *block_cb;
847 __be16 proto = ETH_P_ALL;
848 int err, i = 0;
849
850 nf_flow_offload_init(&cls_flow, proto, priority, cmd,
851 &flow->tuplehash[dir].tuple, &extack);
852 if (cmd == FLOW_CLS_REPLACE)
853 cls_flow.rule = flow_rule->rule;
854
855 down_read(&flowtable->flow_block_lock);
856 list_for_each_entry(block_cb, block_cb_list, list) {
857 err = block_cb->cb(TC_SETUP_CLSFLOWER, &cls_flow,
858 block_cb->cb_priv);
859 if (err < 0)
860 continue;
861
862 i++;
863 }
864 up_read(&flowtable->flow_block_lock);
865
866 if (cmd == FLOW_CLS_STATS)
867 memcpy(stats, &cls_flow.stats, sizeof(*stats));
868
869 return i;
870 }
871
flow_offload_tuple_add(struct flow_offload_work * offload,struct nf_flow_rule * flow_rule,enum flow_offload_tuple_dir dir)872 static int flow_offload_tuple_add(struct flow_offload_work *offload,
873 struct nf_flow_rule *flow_rule,
874 enum flow_offload_tuple_dir dir)
875 {
876 return nf_flow_offload_tuple(offload->flowtable, offload->flow,
877 flow_rule, dir,
878 offload->flowtable->priority,
879 FLOW_CLS_REPLACE, NULL,
880 &offload->flowtable->flow_block.cb_list);
881 }
882
flow_offload_tuple_del(struct flow_offload_work * offload,enum flow_offload_tuple_dir dir)883 static void flow_offload_tuple_del(struct flow_offload_work *offload,
884 enum flow_offload_tuple_dir dir)
885 {
886 nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
887 offload->flowtable->priority,
888 FLOW_CLS_DESTROY, NULL,
889 &offload->flowtable->flow_block.cb_list);
890 }
891
flow_offload_rule_add(struct flow_offload_work * offload,struct nf_flow_rule * flow_rule[])892 static int flow_offload_rule_add(struct flow_offload_work *offload,
893 struct nf_flow_rule *flow_rule[])
894 {
895 int ok_count = 0;
896
897 ok_count += flow_offload_tuple_add(offload, flow_rule[0],
898 FLOW_OFFLOAD_DIR_ORIGINAL);
899 if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
900 ok_count += flow_offload_tuple_add(offload, flow_rule[1],
901 FLOW_OFFLOAD_DIR_REPLY);
902 if (ok_count == 0)
903 return -ENOENT;
904
905 return 0;
906 }
907
flow_offload_work_add(struct flow_offload_work * offload)908 static void flow_offload_work_add(struct flow_offload_work *offload)
909 {
910 struct nf_flow_rule *flow_rule[FLOW_OFFLOAD_DIR_MAX];
911 int err;
912
913 err = nf_flow_offload_alloc(offload, flow_rule);
914 if (err < 0)
915 return;
916
917 err = flow_offload_rule_add(offload, flow_rule);
918 if (err < 0)
919 goto out;
920
921 set_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
922
923 out:
924 nf_flow_offload_destroy(flow_rule);
925 }
926
flow_offload_work_del(struct flow_offload_work * offload)927 static void flow_offload_work_del(struct flow_offload_work *offload)
928 {
929 clear_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
930 flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL);
931 if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
932 flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY);
933 set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags);
934 }
935
flow_offload_tuple_stats(struct flow_offload_work * offload,enum flow_offload_tuple_dir dir,struct flow_stats * stats)936 static void flow_offload_tuple_stats(struct flow_offload_work *offload,
937 enum flow_offload_tuple_dir dir,
938 struct flow_stats *stats)
939 {
940 nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
941 offload->flowtable->priority,
942 FLOW_CLS_STATS, stats,
943 &offload->flowtable->flow_block.cb_list);
944 }
945
flow_offload_work_stats(struct flow_offload_work * offload)946 static void flow_offload_work_stats(struct flow_offload_work *offload)
947 {
948 struct flow_stats stats[FLOW_OFFLOAD_DIR_MAX] = {};
949 u64 lastused;
950
951 flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_ORIGINAL, &stats[0]);
952 if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
953 flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_REPLY,
954 &stats[1]);
955
956 lastused = max_t(u64, stats[0].lastused, stats[1].lastused);
957 offload->flow->timeout = max_t(u64, offload->flow->timeout,
958 lastused + flow_offload_get_timeout(offload->flow));
959
960 if (offload->flowtable->flags & NF_FLOWTABLE_COUNTER) {
961 if (stats[0].pkts)
962 nf_ct_acct_add(offload->flow->ct,
963 FLOW_OFFLOAD_DIR_ORIGINAL,
964 stats[0].pkts, stats[0].bytes);
965 if (stats[1].pkts)
966 nf_ct_acct_add(offload->flow->ct,
967 FLOW_OFFLOAD_DIR_REPLY,
968 stats[1].pkts, stats[1].bytes);
969 }
970 }
971
flow_offload_work_handler(struct work_struct * work)972 static void flow_offload_work_handler(struct work_struct *work)
973 {
974 struct flow_offload_work *offload;
975 struct net *net;
976
977 offload = container_of(work, struct flow_offload_work, work);
978 net = read_pnet(&offload->flowtable->net);
979 switch (offload->cmd) {
980 case FLOW_CLS_REPLACE:
981 flow_offload_work_add(offload);
982 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_add);
983 break;
984 case FLOW_CLS_DESTROY:
985 flow_offload_work_del(offload);
986 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_del);
987 break;
988 case FLOW_CLS_STATS:
989 flow_offload_work_stats(offload);
990 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_stats);
991 break;
992 default:
993 WARN_ON_ONCE(1);
994 }
995
996 clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags);
997 kfree(offload);
998 }
999
flow_offload_queue_work(struct flow_offload_work * offload)1000 static void flow_offload_queue_work(struct flow_offload_work *offload)
1001 {
1002 struct net *net = read_pnet(&offload->flowtable->net);
1003
1004 if (offload->cmd == FLOW_CLS_REPLACE) {
1005 NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_add);
1006 queue_work(nf_flow_offload_add_wq, &offload->work);
1007 } else if (offload->cmd == FLOW_CLS_DESTROY) {
1008 NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_del);
1009 queue_work(nf_flow_offload_del_wq, &offload->work);
1010 } else {
1011 NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_stats);
1012 queue_work(nf_flow_offload_stats_wq, &offload->work);
1013 }
1014 }
1015
1016 static struct flow_offload_work *
nf_flow_offload_work_alloc(struct nf_flowtable * flowtable,struct flow_offload * flow,unsigned int cmd)1017 nf_flow_offload_work_alloc(struct nf_flowtable *flowtable,
1018 struct flow_offload *flow, unsigned int cmd)
1019 {
1020 struct flow_offload_work *offload;
1021
1022 if (test_and_set_bit(NF_FLOW_HW_PENDING, &flow->flags))
1023 return NULL;
1024
1025 offload = kmalloc_obj(struct flow_offload_work, GFP_ATOMIC);
1026 if (!offload) {
1027 clear_bit(NF_FLOW_HW_PENDING, &flow->flags);
1028 return NULL;
1029 }
1030
1031 offload->cmd = cmd;
1032 offload->flow = flow;
1033 offload->flowtable = flowtable;
1034 INIT_WORK(&offload->work, flow_offload_work_handler);
1035
1036 return offload;
1037 }
1038
1039
nf_flow_offload_add(struct nf_flowtable * flowtable,struct flow_offload * flow)1040 void nf_flow_offload_add(struct nf_flowtable *flowtable,
1041 struct flow_offload *flow)
1042 {
1043 struct flow_offload_work *offload;
1044
1045 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_REPLACE);
1046 if (!offload)
1047 return;
1048
1049 flow_offload_queue_work(offload);
1050 }
1051
nf_flow_offload_del(struct nf_flowtable * flowtable,struct flow_offload * flow)1052 void nf_flow_offload_del(struct nf_flowtable *flowtable,
1053 struct flow_offload *flow)
1054 {
1055 struct flow_offload_work *offload;
1056
1057 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_DESTROY);
1058 if (!offload)
1059 return;
1060
1061 set_bit(NF_FLOW_HW_DYING, &flow->flags);
1062 flow_offload_queue_work(offload);
1063 }
1064
nf_flow_offload_stats(struct nf_flowtable * flowtable,struct flow_offload * flow)1065 void nf_flow_offload_stats(struct nf_flowtable *flowtable,
1066 struct flow_offload *flow)
1067 {
1068 struct flow_offload_work *offload;
1069 __s32 delta;
1070
1071 delta = nf_flow_timeout_delta(flow->timeout);
1072 if ((delta >= (9 * flow_offload_get_timeout(flow)) / 10))
1073 return;
1074
1075 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_STATS);
1076 if (!offload)
1077 return;
1078
1079 flow_offload_queue_work(offload);
1080 }
1081
nf_flow_table_offload_flush_cleanup(struct nf_flowtable * flowtable)1082 void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable)
1083 {
1084 if (nf_flowtable_hw_offload(flowtable)) {
1085 flush_workqueue(nf_flow_offload_del_wq);
1086 nf_flow_table_gc_run(flowtable);
1087 }
1088 }
1089
nf_flow_table_offload_flush(struct nf_flowtable * flowtable)1090 void nf_flow_table_offload_flush(struct nf_flowtable *flowtable)
1091 {
1092 if (nf_flowtable_hw_offload(flowtable)) {
1093 flush_workqueue(nf_flow_offload_add_wq);
1094 flush_workqueue(nf_flow_offload_del_wq);
1095 flush_workqueue(nf_flow_offload_stats_wq);
1096 }
1097 }
1098
nf_flow_table_block_setup(struct nf_flowtable * flowtable,struct flow_block_offload * bo,enum flow_block_command cmd)1099 static int nf_flow_table_block_setup(struct nf_flowtable *flowtable,
1100 struct flow_block_offload *bo,
1101 enum flow_block_command cmd)
1102 {
1103 struct flow_block_cb *block_cb, *next;
1104 int err = 0;
1105
1106 down_write(&flowtable->flow_block_lock);
1107 switch (cmd) {
1108 case FLOW_BLOCK_BIND:
1109 list_splice(&bo->cb_list, &flowtable->flow_block.cb_list);
1110 break;
1111 case FLOW_BLOCK_UNBIND:
1112 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1113 list_del(&block_cb->list);
1114 flow_block_cb_free(block_cb);
1115 }
1116 break;
1117 default:
1118 WARN_ON_ONCE(1);
1119 err = -EOPNOTSUPP;
1120 }
1121 up_write(&flowtable->flow_block_lock);
1122
1123 return err;
1124 }
1125
nf_flow_table_block_offload_init(struct flow_block_offload * bo,struct net * net,enum flow_block_command cmd,struct nf_flowtable * flowtable,struct netlink_ext_ack * extack)1126 static void nf_flow_table_block_offload_init(struct flow_block_offload *bo,
1127 struct net *net,
1128 enum flow_block_command cmd,
1129 struct nf_flowtable *flowtable,
1130 struct netlink_ext_ack *extack)
1131 {
1132 memset(bo, 0, sizeof(*bo));
1133 bo->net = net;
1134 bo->block = &flowtable->flow_block;
1135 bo->command = cmd;
1136 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
1137 bo->extack = extack;
1138 bo->cb_list_head = &flowtable->flow_block.cb_list;
1139 INIT_LIST_HEAD(&bo->cb_list);
1140 }
1141
nf_flow_table_indr_cleanup(struct flow_block_cb * block_cb)1142 static void nf_flow_table_indr_cleanup(struct flow_block_cb *block_cb)
1143 {
1144 struct nf_flowtable *flowtable = block_cb->indr.data;
1145 struct net_device *dev = block_cb->indr.dev;
1146
1147 nf_flow_table_gc_cleanup(flowtable, dev);
1148 down_write(&flowtable->flow_block_lock);
1149 list_del(&block_cb->list);
1150 list_del(&block_cb->driver_list);
1151 flow_block_cb_free(block_cb);
1152 up_write(&flowtable->flow_block_lock);
1153 }
1154
nf_flow_table_indr_offload_cmd(struct flow_block_offload * bo,struct nf_flowtable * flowtable,struct net_device * dev,enum flow_block_command cmd,struct netlink_ext_ack * extack)1155 static int nf_flow_table_indr_offload_cmd(struct flow_block_offload *bo,
1156 struct nf_flowtable *flowtable,
1157 struct net_device *dev,
1158 enum flow_block_command cmd,
1159 struct netlink_ext_ack *extack)
1160 {
1161 nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
1162 extack);
1163
1164 return flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_FT, flowtable, bo,
1165 nf_flow_table_indr_cleanup);
1166 }
1167
nf_flow_table_offload_cmd(struct flow_block_offload * bo,struct nf_flowtable * flowtable,struct net_device * dev,enum flow_block_command cmd,struct netlink_ext_ack * extack)1168 static int nf_flow_table_offload_cmd(struct flow_block_offload *bo,
1169 struct nf_flowtable *flowtable,
1170 struct net_device *dev,
1171 enum flow_block_command cmd,
1172 struct netlink_ext_ack *extack)
1173 {
1174 int err;
1175
1176 nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
1177 extack);
1178 down_write(&flowtable->flow_block_lock);
1179 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_FT, bo);
1180 up_write(&flowtable->flow_block_lock);
1181 if (err < 0)
1182 return err;
1183
1184 return 0;
1185 }
1186
nf_flow_table_offload_setup(struct nf_flowtable * flowtable,struct net_device * dev,enum flow_block_command cmd)1187 int nf_flow_table_offload_setup(struct nf_flowtable *flowtable,
1188 struct net_device *dev,
1189 enum flow_block_command cmd)
1190 {
1191 struct netlink_ext_ack extack = {};
1192 struct flow_block_offload bo;
1193 int err;
1194
1195 if (!nf_flowtable_hw_offload(flowtable))
1196 return nf_flow_offload_xdp_setup(flowtable, dev, cmd);
1197
1198 if (dev->netdev_ops->ndo_setup_tc)
1199 err = nf_flow_table_offload_cmd(&bo, flowtable, dev, cmd,
1200 &extack);
1201 else
1202 err = nf_flow_table_indr_offload_cmd(&bo, flowtable, dev, cmd,
1203 &extack);
1204 if (err < 0)
1205 return err;
1206
1207 return nf_flow_table_block_setup(flowtable, &bo, cmd);
1208 }
1209 EXPORT_SYMBOL_GPL(nf_flow_table_offload_setup);
1210
nf_flow_table_offload_init(void)1211 int nf_flow_table_offload_init(void)
1212 {
1213 nf_flow_offload_add_wq = alloc_workqueue("nf_ft_offload_add",
1214 WQ_UNBOUND | WQ_SYSFS, 0);
1215 if (!nf_flow_offload_add_wq)
1216 return -ENOMEM;
1217
1218 nf_flow_offload_del_wq = alloc_workqueue("nf_ft_offload_del",
1219 WQ_UNBOUND | WQ_SYSFS, 0);
1220 if (!nf_flow_offload_del_wq)
1221 goto err_del_wq;
1222
1223 nf_flow_offload_stats_wq = alloc_workqueue("nf_ft_offload_stats",
1224 WQ_UNBOUND | WQ_SYSFS, 0);
1225 if (!nf_flow_offload_stats_wq)
1226 goto err_stats_wq;
1227
1228 return 0;
1229
1230 err_stats_wq:
1231 destroy_workqueue(nf_flow_offload_del_wq);
1232 err_del_wq:
1233 destroy_workqueue(nf_flow_offload_add_wq);
1234 return -ENOMEM;
1235 }
1236
nf_flow_table_offload_exit(void)1237 void nf_flow_table_offload_exit(void)
1238 {
1239 destroy_workqueue(nf_flow_offload_add_wq);
1240 destroy_workqueue(nf_flow_offload_del_wq);
1241 destroy_workqueue(nf_flow_offload_stats_wq);
1242 }
1243