1 #include <linux/kernel.h> 2 #include <linux/init.h> 3 #include <linux/module.h> 4 #include <linux/netfilter.h> 5 #include <linux/rhashtable.h> 6 #include <linux/netdevice.h> 7 #include <linux/tc_act/tc_csum.h> 8 #include <net/flow_offload.h> 9 #include <net/ip_tunnels.h> 10 #include <net/netfilter/nf_flow_table.h> 11 #include <net/netfilter/nf_tables.h> 12 #include <net/netfilter/nf_conntrack.h> 13 #include <net/netfilter/nf_conntrack_acct.h> 14 #include <net/netfilter/nf_conntrack_core.h> 15 #include <net/netfilter/nf_conntrack_tuple.h> 16 17 static struct workqueue_struct *nf_flow_offload_add_wq; 18 static struct workqueue_struct *nf_flow_offload_del_wq; 19 static struct workqueue_struct *nf_flow_offload_stats_wq; 20 21 struct flow_offload_work { 22 struct list_head list; 23 enum flow_cls_command cmd; 24 struct nf_flowtable *flowtable; 25 struct flow_offload *flow; 26 struct work_struct work; 27 }; 28 29 #define NF_FLOW_DISSECTOR(__match, __type, __field) \ 30 (__match)->dissector.offset[__type] = \ 31 offsetof(struct nf_flow_key, __field) 32 33 static void nf_flow_rule_lwt_match(struct nf_flow_match *match, 34 struct ip_tunnel_info *tun_info) 35 { 36 struct nf_flow_key *mask = &match->mask; 37 struct nf_flow_key *key = &match->key; 38 unsigned long long enc_keys; 39 40 if (!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX)) 41 return; 42 43 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_CONTROL, enc_control); 44 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id); 45 key->enc_key_id.keyid = tunnel_id_to_key32(tun_info->key.tun_id); 46 mask->enc_key_id.keyid = 0xffffffff; 47 enc_keys = BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | 48 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL); 49 50 if (ip_tunnel_info_af(tun_info) == AF_INET) { 51 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, 52 enc_ipv4); 53 key->enc_ipv4.src = tun_info->key.u.ipv4.dst; 54 key->enc_ipv4.dst = tun_info->key.u.ipv4.src; 55 if (key->enc_ipv4.src) 56 mask->enc_ipv4.src = 0xffffffff; 57 if (key->enc_ipv4.dst) 58 mask->enc_ipv4.dst = 0xffffffff; 59 enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS); 60 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 61 } else { 62 memcpy(&key->enc_ipv6.src, &tun_info->key.u.ipv6.dst, 63 sizeof(struct in6_addr)); 64 memcpy(&key->enc_ipv6.dst, &tun_info->key.u.ipv6.src, 65 sizeof(struct in6_addr)); 66 if (memcmp(&key->enc_ipv6.src, &in6addr_any, 67 sizeof(struct in6_addr))) 68 memset(&mask->enc_ipv6.src, 0xff, 69 sizeof(struct in6_addr)); 70 if (memcmp(&key->enc_ipv6.dst, &in6addr_any, 71 sizeof(struct in6_addr))) 72 memset(&mask->enc_ipv6.dst, 0xff, 73 sizeof(struct in6_addr)); 74 enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS); 75 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 76 } 77 78 match->dissector.used_keys |= enc_keys; 79 } 80 81 static void nf_flow_rule_vlan_match(struct flow_dissector_key_vlan *key, 82 struct flow_dissector_key_vlan *mask, 83 u16 vlan_id, __be16 proto) 84 { 85 key->vlan_id = vlan_id; 86 mask->vlan_id = VLAN_VID_MASK; 87 key->vlan_tpid = proto; 88 mask->vlan_tpid = 0xffff; 89 } 90 91 static int nf_flow_rule_match(struct nf_flow_match *match, 92 const struct flow_offload_tuple *tuple, 93 struct dst_entry *other_dst) 94 { 95 struct nf_flow_key *mask = &match->mask; 96 struct nf_flow_key *key = &match->key; 97 struct ip_tunnel_info *tun_info; 98 bool vlan_encap = false; 99 100 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_META, meta); 101 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CONTROL, control); 102 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_BASIC, basic); 103 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4); 104 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6); 105 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_TCP, tcp); 106 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_PORTS, tp); 107 108 if (other_dst && other_dst->lwtstate) { 109 tun_info = lwt_tun_info(other_dst->lwtstate); 110 nf_flow_rule_lwt_match(match, tun_info); 111 } 112 113 if (tuple->xmit_type == FLOW_OFFLOAD_XMIT_TC) 114 key->meta.ingress_ifindex = tuple->tc.iifidx; 115 else 116 key->meta.ingress_ifindex = tuple->iifidx; 117 118 mask->meta.ingress_ifindex = 0xffffffff; 119 120 if (tuple->encap_num > 0 && !(tuple->in_vlan_ingress & BIT(0)) && 121 tuple->encap[0].proto == htons(ETH_P_8021Q)) { 122 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, vlan); 123 nf_flow_rule_vlan_match(&key->vlan, &mask->vlan, 124 tuple->encap[0].id, 125 tuple->encap[0].proto); 126 vlan_encap = true; 127 } 128 129 if (tuple->encap_num > 1 && !(tuple->in_vlan_ingress & BIT(1)) && 130 tuple->encap[1].proto == htons(ETH_P_8021Q)) { 131 if (vlan_encap) { 132 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CVLAN, 133 cvlan); 134 nf_flow_rule_vlan_match(&key->cvlan, &mask->cvlan, 135 tuple->encap[1].id, 136 tuple->encap[1].proto); 137 } else { 138 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, 139 vlan); 140 nf_flow_rule_vlan_match(&key->vlan, &mask->vlan, 141 tuple->encap[1].id, 142 tuple->encap[1].proto); 143 } 144 } 145 146 switch (tuple->l3proto) { 147 case AF_INET: 148 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 149 key->basic.n_proto = htons(ETH_P_IP); 150 key->ipv4.src = tuple->src_v4.s_addr; 151 mask->ipv4.src = 0xffffffff; 152 key->ipv4.dst = tuple->dst_v4.s_addr; 153 mask->ipv4.dst = 0xffffffff; 154 break; 155 case AF_INET6: 156 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 157 key->basic.n_proto = htons(ETH_P_IPV6); 158 key->ipv6.src = tuple->src_v6; 159 memset(&mask->ipv6.src, 0xff, sizeof(mask->ipv6.src)); 160 key->ipv6.dst = tuple->dst_v6; 161 memset(&mask->ipv6.dst, 0xff, sizeof(mask->ipv6.dst)); 162 break; 163 default: 164 return -EOPNOTSUPP; 165 } 166 mask->control.addr_type = 0xffff; 167 match->dissector.used_keys |= BIT_ULL(key->control.addr_type); 168 mask->basic.n_proto = 0xffff; 169 170 switch (tuple->l4proto) { 171 case IPPROTO_TCP: 172 key->tcp.flags = 0; 173 mask->tcp.flags = cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16); 174 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_TCP); 175 break; 176 case IPPROTO_UDP: 177 case IPPROTO_GRE: 178 break; 179 default: 180 return -EOPNOTSUPP; 181 } 182 183 key->basic.ip_proto = tuple->l4proto; 184 mask->basic.ip_proto = 0xff; 185 186 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_META) | 187 BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | 188 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC); 189 190 switch (tuple->l4proto) { 191 case IPPROTO_TCP: 192 case IPPROTO_UDP: 193 key->tp.src = tuple->src_port; 194 mask->tp.src = 0xffff; 195 key->tp.dst = tuple->dst_port; 196 mask->tp.dst = 0xffff; 197 198 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_PORTS); 199 break; 200 } 201 202 return 0; 203 } 204 205 static void flow_offload_mangle(struct flow_action_entry *entry, 206 enum flow_action_mangle_base htype, u32 offset, 207 const __be32 *value, const __be32 *mask) 208 { 209 entry->id = FLOW_ACTION_MANGLE; 210 entry->mangle.htype = htype; 211 entry->mangle.offset = offset; 212 memcpy(&entry->mangle.mask, mask, sizeof(u32)); 213 memcpy(&entry->mangle.val, value, sizeof(u32)); 214 } 215 216 static inline struct flow_action_entry * 217 flow_action_entry_next(struct nf_flow_rule *flow_rule) 218 { 219 int i = flow_rule->rule->action.num_entries++; 220 221 return &flow_rule->rule->action.entries[i]; 222 } 223 224 static int flow_offload_eth_src(struct net *net, 225 const struct flow_offload *flow, 226 enum flow_offload_tuple_dir dir, 227 struct nf_flow_rule *flow_rule) 228 { 229 struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule); 230 struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule); 231 const struct flow_offload_tuple *other_tuple, *this_tuple; 232 struct net_device *dev = NULL; 233 const unsigned char *addr; 234 u32 mask, val; 235 u16 val16; 236 237 this_tuple = &flow->tuplehash[dir].tuple; 238 239 switch (this_tuple->xmit_type) { 240 case FLOW_OFFLOAD_XMIT_DIRECT: 241 addr = this_tuple->out.h_source; 242 break; 243 case FLOW_OFFLOAD_XMIT_NEIGH: 244 other_tuple = &flow->tuplehash[!dir].tuple; 245 dev = dev_get_by_index(net, other_tuple->iifidx); 246 if (!dev) 247 return -ENOENT; 248 249 addr = dev->dev_addr; 250 break; 251 default: 252 return -EOPNOTSUPP; 253 } 254 255 mask = ~0xffff0000; 256 memcpy(&val16, addr, 2); 257 val = val16 << 16; 258 flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4, 259 &val, &mask); 260 261 mask = ~0xffffffff; 262 memcpy(&val, addr + 2, 4); 263 flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 8, 264 &val, &mask); 265 266 dev_put(dev); 267 268 return 0; 269 } 270 271 static int flow_offload_eth_dst(struct net *net, 272 const struct flow_offload *flow, 273 enum flow_offload_tuple_dir dir, 274 struct nf_flow_rule *flow_rule) 275 { 276 struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule); 277 struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule); 278 const struct flow_offload_tuple *other_tuple, *this_tuple; 279 const struct dst_entry *dst_cache; 280 unsigned char ha[ETH_ALEN]; 281 struct neighbour *n; 282 const void *daddr; 283 u32 mask, val; 284 u8 nud_state; 285 u16 val16; 286 287 this_tuple = &flow->tuplehash[dir].tuple; 288 289 switch (this_tuple->xmit_type) { 290 case FLOW_OFFLOAD_XMIT_DIRECT: 291 ether_addr_copy(ha, this_tuple->out.h_dest); 292 break; 293 case FLOW_OFFLOAD_XMIT_NEIGH: 294 other_tuple = &flow->tuplehash[!dir].tuple; 295 daddr = &other_tuple->src_v4; 296 dst_cache = this_tuple->dst_cache; 297 n = dst_neigh_lookup(dst_cache, daddr); 298 if (!n) 299 return -ENOENT; 300 301 read_lock_bh(&n->lock); 302 nud_state = n->nud_state; 303 ether_addr_copy(ha, n->ha); 304 read_unlock_bh(&n->lock); 305 neigh_release(n); 306 307 if (!(nud_state & NUD_VALID)) 308 return -ENOENT; 309 break; 310 default: 311 return -EOPNOTSUPP; 312 } 313 314 mask = ~0xffffffff; 315 memcpy(&val, ha, 4); 316 flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 0, 317 &val, &mask); 318 319 mask = ~0x0000ffff; 320 memcpy(&val16, ha + 4, 2); 321 val = val16; 322 flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4, 323 &val, &mask); 324 325 return 0; 326 } 327 328 static void flow_offload_ipv4_snat(struct net *net, 329 const struct flow_offload *flow, 330 enum flow_offload_tuple_dir dir, 331 struct nf_flow_rule *flow_rule) 332 { 333 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 334 u32 mask = ~htonl(0xffffffff); 335 __be32 addr; 336 u32 offset; 337 338 switch (dir) { 339 case FLOW_OFFLOAD_DIR_ORIGINAL: 340 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; 341 offset = offsetof(struct iphdr, saddr); 342 break; 343 case FLOW_OFFLOAD_DIR_REPLY: 344 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr; 345 offset = offsetof(struct iphdr, daddr); 346 break; 347 default: 348 return; 349 } 350 351 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, 352 &addr, &mask); 353 } 354 355 static void flow_offload_ipv4_dnat(struct net *net, 356 const struct flow_offload *flow, 357 enum flow_offload_tuple_dir dir, 358 struct nf_flow_rule *flow_rule) 359 { 360 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 361 u32 mask = ~htonl(0xffffffff); 362 __be32 addr; 363 u32 offset; 364 365 switch (dir) { 366 case FLOW_OFFLOAD_DIR_ORIGINAL: 367 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; 368 offset = offsetof(struct iphdr, daddr); 369 break; 370 case FLOW_OFFLOAD_DIR_REPLY: 371 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr; 372 offset = offsetof(struct iphdr, saddr); 373 break; 374 default: 375 return; 376 } 377 378 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, 379 &addr, &mask); 380 } 381 382 static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule, 383 unsigned int offset, 384 const __be32 *addr, const __be32 *mask) 385 { 386 struct flow_action_entry *entry; 387 int i; 388 389 for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i++) { 390 entry = flow_action_entry_next(flow_rule); 391 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6, 392 offset + i * sizeof(u32), &addr[i], mask); 393 } 394 } 395 396 static void flow_offload_ipv6_snat(struct net *net, 397 const struct flow_offload *flow, 398 enum flow_offload_tuple_dir dir, 399 struct nf_flow_rule *flow_rule) 400 { 401 u32 mask = ~htonl(0xffffffff); 402 const __be32 *addr; 403 u32 offset; 404 405 switch (dir) { 406 case FLOW_OFFLOAD_DIR_ORIGINAL: 407 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6.s6_addr32; 408 offset = offsetof(struct ipv6hdr, saddr); 409 break; 410 case FLOW_OFFLOAD_DIR_REPLY: 411 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6.s6_addr32; 412 offset = offsetof(struct ipv6hdr, daddr); 413 break; 414 default: 415 return; 416 } 417 418 flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); 419 } 420 421 static void flow_offload_ipv6_dnat(struct net *net, 422 const struct flow_offload *flow, 423 enum flow_offload_tuple_dir dir, 424 struct nf_flow_rule *flow_rule) 425 { 426 u32 mask = ~htonl(0xffffffff); 427 const __be32 *addr; 428 u32 offset; 429 430 switch (dir) { 431 case FLOW_OFFLOAD_DIR_ORIGINAL: 432 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6.s6_addr32; 433 offset = offsetof(struct ipv6hdr, daddr); 434 break; 435 case FLOW_OFFLOAD_DIR_REPLY: 436 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6.s6_addr32; 437 offset = offsetof(struct ipv6hdr, saddr); 438 break; 439 default: 440 return; 441 } 442 443 flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); 444 } 445 446 static int flow_offload_l4proto(const struct flow_offload *flow) 447 { 448 u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto; 449 u8 type = 0; 450 451 switch (protonum) { 452 case IPPROTO_TCP: 453 type = FLOW_ACT_MANGLE_HDR_TYPE_TCP; 454 break; 455 case IPPROTO_UDP: 456 type = FLOW_ACT_MANGLE_HDR_TYPE_UDP; 457 break; 458 default: 459 break; 460 } 461 462 return type; 463 } 464 465 static void flow_offload_port_snat(struct net *net, 466 const struct flow_offload *flow, 467 enum flow_offload_tuple_dir dir, 468 struct nf_flow_rule *flow_rule) 469 { 470 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 471 u32 mask, port; 472 u32 offset; 473 474 switch (dir) { 475 case FLOW_OFFLOAD_DIR_ORIGINAL: 476 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port); 477 offset = 0; /* offsetof(struct tcphdr, source); */ 478 port = htonl(port << 16); 479 mask = ~htonl(0xffff0000); 480 break; 481 case FLOW_OFFLOAD_DIR_REPLY: 482 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port); 483 offset = 0; /* offsetof(struct tcphdr, dest); */ 484 port = htonl(port); 485 mask = ~htonl(0xffff); 486 break; 487 default: 488 return; 489 } 490 491 flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, 492 &port, &mask); 493 } 494 495 static void flow_offload_port_dnat(struct net *net, 496 const struct flow_offload *flow, 497 enum flow_offload_tuple_dir dir, 498 struct nf_flow_rule *flow_rule) 499 { 500 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 501 u32 mask, port; 502 u32 offset; 503 504 switch (dir) { 505 case FLOW_OFFLOAD_DIR_ORIGINAL: 506 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port); 507 offset = 0; /* offsetof(struct tcphdr, dest); */ 508 port = htonl(port); 509 mask = ~htonl(0xffff); 510 break; 511 case FLOW_OFFLOAD_DIR_REPLY: 512 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port); 513 offset = 0; /* offsetof(struct tcphdr, source); */ 514 port = htonl(port << 16); 515 mask = ~htonl(0xffff0000); 516 break; 517 default: 518 return; 519 } 520 521 flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, 522 &port, &mask); 523 } 524 525 static void flow_offload_ipv4_checksum(struct net *net, 526 const struct flow_offload *flow, 527 struct nf_flow_rule *flow_rule) 528 { 529 u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto; 530 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 531 532 entry->id = FLOW_ACTION_CSUM; 533 entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR; 534 535 switch (protonum) { 536 case IPPROTO_TCP: 537 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_TCP; 538 break; 539 case IPPROTO_UDP: 540 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP; 541 break; 542 } 543 } 544 545 static void flow_offload_redirect(struct net *net, 546 const struct flow_offload *flow, 547 enum flow_offload_tuple_dir dir, 548 struct nf_flow_rule *flow_rule) 549 { 550 const struct flow_offload_tuple *this_tuple, *other_tuple; 551 struct flow_action_entry *entry; 552 struct net_device *dev; 553 int ifindex; 554 555 this_tuple = &flow->tuplehash[dir].tuple; 556 switch (this_tuple->xmit_type) { 557 case FLOW_OFFLOAD_XMIT_DIRECT: 558 this_tuple = &flow->tuplehash[dir].tuple; 559 ifindex = this_tuple->out.ifidx; 560 break; 561 case FLOW_OFFLOAD_XMIT_NEIGH: 562 other_tuple = &flow->tuplehash[!dir].tuple; 563 ifindex = other_tuple->iifidx; 564 break; 565 default: 566 return; 567 } 568 569 dev = dev_get_by_index(net, ifindex); 570 if (!dev) 571 return; 572 573 entry = flow_action_entry_next(flow_rule); 574 entry->id = FLOW_ACTION_REDIRECT; 575 entry->dev = dev; 576 } 577 578 static void flow_offload_encap_tunnel(const struct flow_offload *flow, 579 enum flow_offload_tuple_dir dir, 580 struct nf_flow_rule *flow_rule) 581 { 582 const struct flow_offload_tuple *this_tuple; 583 struct flow_action_entry *entry; 584 struct dst_entry *dst; 585 586 this_tuple = &flow->tuplehash[dir].tuple; 587 if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) 588 return; 589 590 dst = this_tuple->dst_cache; 591 if (dst && dst->lwtstate) { 592 struct ip_tunnel_info *tun_info; 593 594 tun_info = lwt_tun_info(dst->lwtstate); 595 if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { 596 entry = flow_action_entry_next(flow_rule); 597 entry->id = FLOW_ACTION_TUNNEL_ENCAP; 598 entry->tunnel = tun_info; 599 } 600 } 601 } 602 603 static void flow_offload_decap_tunnel(const struct flow_offload *flow, 604 enum flow_offload_tuple_dir dir, 605 struct nf_flow_rule *flow_rule) 606 { 607 const struct flow_offload_tuple *other_tuple; 608 struct flow_action_entry *entry; 609 struct dst_entry *dst; 610 611 other_tuple = &flow->tuplehash[!dir].tuple; 612 if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) 613 return; 614 615 dst = other_tuple->dst_cache; 616 if (dst && dst->lwtstate) { 617 struct ip_tunnel_info *tun_info; 618 619 tun_info = lwt_tun_info(dst->lwtstate); 620 if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { 621 entry = flow_action_entry_next(flow_rule); 622 entry->id = FLOW_ACTION_TUNNEL_DECAP; 623 } 624 } 625 } 626 627 static int 628 nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow, 629 enum flow_offload_tuple_dir dir, 630 struct nf_flow_rule *flow_rule) 631 { 632 const struct flow_offload_tuple *other_tuple; 633 const struct flow_offload_tuple *tuple; 634 int i; 635 636 flow_offload_decap_tunnel(flow, dir, flow_rule); 637 flow_offload_encap_tunnel(flow, dir, flow_rule); 638 639 if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 || 640 flow_offload_eth_dst(net, flow, dir, flow_rule) < 0) 641 return -1; 642 643 tuple = &flow->tuplehash[dir].tuple; 644 645 for (i = 0; i < tuple->encap_num; i++) { 646 struct flow_action_entry *entry; 647 648 if (tuple->in_vlan_ingress & BIT(i)) 649 continue; 650 651 if (tuple->encap[i].proto == htons(ETH_P_8021Q)) { 652 entry = flow_action_entry_next(flow_rule); 653 entry->id = FLOW_ACTION_VLAN_POP; 654 } 655 } 656 657 other_tuple = &flow->tuplehash[!dir].tuple; 658 659 for (i = 0; i < other_tuple->encap_num; i++) { 660 struct flow_action_entry *entry; 661 662 if (other_tuple->in_vlan_ingress & BIT(i)) 663 continue; 664 665 entry = flow_action_entry_next(flow_rule); 666 667 switch (other_tuple->encap[i].proto) { 668 case htons(ETH_P_PPP_SES): 669 entry->id = FLOW_ACTION_PPPOE_PUSH; 670 entry->pppoe.sid = other_tuple->encap[i].id; 671 break; 672 case htons(ETH_P_8021Q): 673 entry->id = FLOW_ACTION_VLAN_PUSH; 674 entry->vlan.vid = other_tuple->encap[i].id; 675 entry->vlan.proto = other_tuple->encap[i].proto; 676 break; 677 } 678 } 679 680 return 0; 681 } 682 683 int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow, 684 enum flow_offload_tuple_dir dir, 685 struct nf_flow_rule *flow_rule) 686 { 687 if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0) 688 return -1; 689 690 if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 691 flow_offload_ipv4_snat(net, flow, dir, flow_rule); 692 flow_offload_port_snat(net, flow, dir, flow_rule); 693 } 694 if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 695 flow_offload_ipv4_dnat(net, flow, dir, flow_rule); 696 flow_offload_port_dnat(net, flow, dir, flow_rule); 697 } 698 if (test_bit(NF_FLOW_SNAT, &flow->flags) || 699 test_bit(NF_FLOW_DNAT, &flow->flags)) 700 flow_offload_ipv4_checksum(net, flow, flow_rule); 701 702 flow_offload_redirect(net, flow, dir, flow_rule); 703 704 return 0; 705 } 706 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv4); 707 708 int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow, 709 enum flow_offload_tuple_dir dir, 710 struct nf_flow_rule *flow_rule) 711 { 712 if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0) 713 return -1; 714 715 if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 716 flow_offload_ipv6_snat(net, flow, dir, flow_rule); 717 flow_offload_port_snat(net, flow, dir, flow_rule); 718 } 719 if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 720 flow_offload_ipv6_dnat(net, flow, dir, flow_rule); 721 flow_offload_port_dnat(net, flow, dir, flow_rule); 722 } 723 724 flow_offload_redirect(net, flow, dir, flow_rule); 725 726 return 0; 727 } 728 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6); 729 730 #define NF_FLOW_RULE_ACTION_MAX 16 731 732 static struct nf_flow_rule * 733 nf_flow_offload_rule_alloc(struct net *net, 734 const struct flow_offload_work *offload, 735 enum flow_offload_tuple_dir dir) 736 { 737 const struct nf_flowtable *flowtable = offload->flowtable; 738 const struct flow_offload_tuple *tuple, *other_tuple; 739 struct flow_offload *flow = offload->flow; 740 struct dst_entry *other_dst = NULL; 741 struct nf_flow_rule *flow_rule; 742 int err = -ENOMEM; 743 744 flow_rule = kzalloc(sizeof(*flow_rule), GFP_KERNEL); 745 if (!flow_rule) 746 goto err_flow; 747 748 flow_rule->rule = flow_rule_alloc(NF_FLOW_RULE_ACTION_MAX); 749 if (!flow_rule->rule) 750 goto err_flow_rule; 751 752 flow_rule->rule->match.dissector = &flow_rule->match.dissector; 753 flow_rule->rule->match.mask = &flow_rule->match.mask; 754 flow_rule->rule->match.key = &flow_rule->match.key; 755 756 tuple = &flow->tuplehash[dir].tuple; 757 other_tuple = &flow->tuplehash[!dir].tuple; 758 if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH) 759 other_dst = other_tuple->dst_cache; 760 761 err = nf_flow_rule_match(&flow_rule->match, tuple, other_dst); 762 if (err < 0) 763 goto err_flow_match; 764 765 flow_rule->rule->action.num_entries = 0; 766 if (flowtable->type->action(net, flow, dir, flow_rule) < 0) 767 goto err_flow_match; 768 769 return flow_rule; 770 771 err_flow_match: 772 kfree(flow_rule->rule); 773 err_flow_rule: 774 kfree(flow_rule); 775 err_flow: 776 return NULL; 777 } 778 779 static void __nf_flow_offload_destroy(struct nf_flow_rule *flow_rule) 780 { 781 struct flow_action_entry *entry; 782 int i; 783 784 for (i = 0; i < flow_rule->rule->action.num_entries; i++) { 785 entry = &flow_rule->rule->action.entries[i]; 786 if (entry->id != FLOW_ACTION_REDIRECT) 787 continue; 788 789 dev_put(entry->dev); 790 } 791 kfree(flow_rule->rule); 792 kfree(flow_rule); 793 } 794 795 static void nf_flow_offload_destroy(struct nf_flow_rule *flow_rule[]) 796 { 797 int i; 798 799 for (i = 0; i < FLOW_OFFLOAD_DIR_MAX; i++) 800 __nf_flow_offload_destroy(flow_rule[i]); 801 } 802 803 static int nf_flow_offload_alloc(const struct flow_offload_work *offload, 804 struct nf_flow_rule *flow_rule[]) 805 { 806 struct net *net = read_pnet(&offload->flowtable->net); 807 808 flow_rule[0] = nf_flow_offload_rule_alloc(net, offload, 809 FLOW_OFFLOAD_DIR_ORIGINAL); 810 if (!flow_rule[0]) 811 return -ENOMEM; 812 813 flow_rule[1] = nf_flow_offload_rule_alloc(net, offload, 814 FLOW_OFFLOAD_DIR_REPLY); 815 if (!flow_rule[1]) { 816 __nf_flow_offload_destroy(flow_rule[0]); 817 return -ENOMEM; 818 } 819 820 return 0; 821 } 822 823 static void nf_flow_offload_init(struct flow_cls_offload *cls_flow, 824 __be16 proto, int priority, 825 enum flow_cls_command cmd, 826 const struct flow_offload_tuple *tuple, 827 struct netlink_ext_ack *extack) 828 { 829 cls_flow->common.protocol = proto; 830 cls_flow->common.prio = priority; 831 cls_flow->common.extack = extack; 832 cls_flow->command = cmd; 833 cls_flow->cookie = (unsigned long)tuple; 834 } 835 836 static int nf_flow_offload_tuple(struct nf_flowtable *flowtable, 837 struct flow_offload *flow, 838 struct nf_flow_rule *flow_rule, 839 enum flow_offload_tuple_dir dir, 840 int priority, int cmd, 841 struct flow_stats *stats, 842 struct list_head *block_cb_list) 843 { 844 struct flow_cls_offload cls_flow = {}; 845 struct netlink_ext_ack extack = {}; 846 struct flow_block_cb *block_cb; 847 __be16 proto = ETH_P_ALL; 848 int err, i = 0; 849 850 nf_flow_offload_init(&cls_flow, proto, priority, cmd, 851 &flow->tuplehash[dir].tuple, &extack); 852 if (cmd == FLOW_CLS_REPLACE) 853 cls_flow.rule = flow_rule->rule; 854 855 down_read(&flowtable->flow_block_lock); 856 list_for_each_entry(block_cb, block_cb_list, list) { 857 err = block_cb->cb(TC_SETUP_CLSFLOWER, &cls_flow, 858 block_cb->cb_priv); 859 if (err < 0) 860 continue; 861 862 i++; 863 } 864 up_read(&flowtable->flow_block_lock); 865 866 if (cmd == FLOW_CLS_STATS) 867 memcpy(stats, &cls_flow.stats, sizeof(*stats)); 868 869 return i; 870 } 871 872 static int flow_offload_tuple_add(struct flow_offload_work *offload, 873 struct nf_flow_rule *flow_rule, 874 enum flow_offload_tuple_dir dir) 875 { 876 return nf_flow_offload_tuple(offload->flowtable, offload->flow, 877 flow_rule, dir, 878 offload->flowtable->priority, 879 FLOW_CLS_REPLACE, NULL, 880 &offload->flowtable->flow_block.cb_list); 881 } 882 883 static void flow_offload_tuple_del(struct flow_offload_work *offload, 884 enum flow_offload_tuple_dir dir) 885 { 886 nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir, 887 offload->flowtable->priority, 888 FLOW_CLS_DESTROY, NULL, 889 &offload->flowtable->flow_block.cb_list); 890 } 891 892 static int flow_offload_rule_add(struct flow_offload_work *offload, 893 struct nf_flow_rule *flow_rule[]) 894 { 895 int ok_count = 0; 896 897 ok_count += flow_offload_tuple_add(offload, flow_rule[0], 898 FLOW_OFFLOAD_DIR_ORIGINAL); 899 if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags)) 900 ok_count += flow_offload_tuple_add(offload, flow_rule[1], 901 FLOW_OFFLOAD_DIR_REPLY); 902 if (ok_count == 0) 903 return -ENOENT; 904 905 return 0; 906 } 907 908 static void flow_offload_work_add(struct flow_offload_work *offload) 909 { 910 struct nf_flow_rule *flow_rule[FLOW_OFFLOAD_DIR_MAX]; 911 int err; 912 913 err = nf_flow_offload_alloc(offload, flow_rule); 914 if (err < 0) 915 return; 916 917 err = flow_offload_rule_add(offload, flow_rule); 918 if (err < 0) 919 goto out; 920 921 set_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status); 922 923 out: 924 nf_flow_offload_destroy(flow_rule); 925 } 926 927 static void flow_offload_work_del(struct flow_offload_work *offload) 928 { 929 clear_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status); 930 flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL); 931 if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags)) 932 flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY); 933 set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags); 934 } 935 936 static void flow_offload_tuple_stats(struct flow_offload_work *offload, 937 enum flow_offload_tuple_dir dir, 938 struct flow_stats *stats) 939 { 940 nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir, 941 offload->flowtable->priority, 942 FLOW_CLS_STATS, stats, 943 &offload->flowtable->flow_block.cb_list); 944 } 945 946 static void flow_offload_work_stats(struct flow_offload_work *offload) 947 { 948 struct flow_stats stats[FLOW_OFFLOAD_DIR_MAX] = {}; 949 u64 lastused; 950 951 flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_ORIGINAL, &stats[0]); 952 if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags)) 953 flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_REPLY, 954 &stats[1]); 955 956 lastused = max_t(u64, stats[0].lastused, stats[1].lastused); 957 offload->flow->timeout = max_t(u64, offload->flow->timeout, 958 lastused + flow_offload_get_timeout(offload->flow)); 959 960 if (offload->flowtable->flags & NF_FLOWTABLE_COUNTER) { 961 if (stats[0].pkts) 962 nf_ct_acct_add(offload->flow->ct, 963 FLOW_OFFLOAD_DIR_ORIGINAL, 964 stats[0].pkts, stats[0].bytes); 965 if (stats[1].pkts) 966 nf_ct_acct_add(offload->flow->ct, 967 FLOW_OFFLOAD_DIR_REPLY, 968 stats[1].pkts, stats[1].bytes); 969 } 970 } 971 972 static void flow_offload_work_handler(struct work_struct *work) 973 { 974 struct flow_offload_work *offload; 975 struct net *net; 976 977 offload = container_of(work, struct flow_offload_work, work); 978 net = read_pnet(&offload->flowtable->net); 979 switch (offload->cmd) { 980 case FLOW_CLS_REPLACE: 981 flow_offload_work_add(offload); 982 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_add); 983 break; 984 case FLOW_CLS_DESTROY: 985 flow_offload_work_del(offload); 986 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_del); 987 break; 988 case FLOW_CLS_STATS: 989 flow_offload_work_stats(offload); 990 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_stats); 991 break; 992 default: 993 WARN_ON_ONCE(1); 994 } 995 996 clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags); 997 kfree(offload); 998 } 999 1000 static void flow_offload_queue_work(struct flow_offload_work *offload) 1001 { 1002 struct net *net = read_pnet(&offload->flowtable->net); 1003 1004 if (offload->cmd == FLOW_CLS_REPLACE) { 1005 NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_add); 1006 queue_work(nf_flow_offload_add_wq, &offload->work); 1007 } else if (offload->cmd == FLOW_CLS_DESTROY) { 1008 NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_del); 1009 queue_work(nf_flow_offload_del_wq, &offload->work); 1010 } else { 1011 NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_stats); 1012 queue_work(nf_flow_offload_stats_wq, &offload->work); 1013 } 1014 } 1015 1016 static struct flow_offload_work * 1017 nf_flow_offload_work_alloc(struct nf_flowtable *flowtable, 1018 struct flow_offload *flow, unsigned int cmd) 1019 { 1020 struct flow_offload_work *offload; 1021 1022 if (test_and_set_bit(NF_FLOW_HW_PENDING, &flow->flags)) 1023 return NULL; 1024 1025 offload = kmalloc(sizeof(struct flow_offload_work), GFP_ATOMIC); 1026 if (!offload) { 1027 clear_bit(NF_FLOW_HW_PENDING, &flow->flags); 1028 return NULL; 1029 } 1030 1031 offload->cmd = cmd; 1032 offload->flow = flow; 1033 offload->flowtable = flowtable; 1034 INIT_WORK(&offload->work, flow_offload_work_handler); 1035 1036 return offload; 1037 } 1038 1039 1040 void nf_flow_offload_add(struct nf_flowtable *flowtable, 1041 struct flow_offload *flow) 1042 { 1043 struct flow_offload_work *offload; 1044 1045 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_REPLACE); 1046 if (!offload) 1047 return; 1048 1049 flow_offload_queue_work(offload); 1050 } 1051 1052 void nf_flow_offload_del(struct nf_flowtable *flowtable, 1053 struct flow_offload *flow) 1054 { 1055 struct flow_offload_work *offload; 1056 1057 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_DESTROY); 1058 if (!offload) 1059 return; 1060 1061 set_bit(NF_FLOW_HW_DYING, &flow->flags); 1062 flow_offload_queue_work(offload); 1063 } 1064 1065 void nf_flow_offload_stats(struct nf_flowtable *flowtable, 1066 struct flow_offload *flow) 1067 { 1068 struct flow_offload_work *offload; 1069 __s32 delta; 1070 1071 delta = nf_flow_timeout_delta(flow->timeout); 1072 if ((delta >= (9 * flow_offload_get_timeout(flow)) / 10)) 1073 return; 1074 1075 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_STATS); 1076 if (!offload) 1077 return; 1078 1079 flow_offload_queue_work(offload); 1080 } 1081 1082 void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable) 1083 { 1084 if (nf_flowtable_hw_offload(flowtable)) { 1085 flush_workqueue(nf_flow_offload_del_wq); 1086 nf_flow_table_gc_run(flowtable); 1087 } 1088 } 1089 1090 void nf_flow_table_offload_flush(struct nf_flowtable *flowtable) 1091 { 1092 if (nf_flowtable_hw_offload(flowtable)) { 1093 flush_workqueue(nf_flow_offload_add_wq); 1094 flush_workqueue(nf_flow_offload_del_wq); 1095 flush_workqueue(nf_flow_offload_stats_wq); 1096 } 1097 } 1098 1099 static int nf_flow_table_block_setup(struct nf_flowtable *flowtable, 1100 struct flow_block_offload *bo, 1101 enum flow_block_command cmd) 1102 { 1103 struct flow_block_cb *block_cb, *next; 1104 int err = 0; 1105 1106 down_write(&flowtable->flow_block_lock); 1107 switch (cmd) { 1108 case FLOW_BLOCK_BIND: 1109 list_splice(&bo->cb_list, &flowtable->flow_block.cb_list); 1110 break; 1111 case FLOW_BLOCK_UNBIND: 1112 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { 1113 list_del(&block_cb->list); 1114 flow_block_cb_free(block_cb); 1115 } 1116 break; 1117 default: 1118 WARN_ON_ONCE(1); 1119 err = -EOPNOTSUPP; 1120 } 1121 up_write(&flowtable->flow_block_lock); 1122 1123 return err; 1124 } 1125 1126 static void nf_flow_table_block_offload_init(struct flow_block_offload *bo, 1127 struct net *net, 1128 enum flow_block_command cmd, 1129 struct nf_flowtable *flowtable, 1130 struct netlink_ext_ack *extack) 1131 { 1132 memset(bo, 0, sizeof(*bo)); 1133 bo->net = net; 1134 bo->block = &flowtable->flow_block; 1135 bo->command = cmd; 1136 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; 1137 bo->extack = extack; 1138 bo->cb_list_head = &flowtable->flow_block.cb_list; 1139 INIT_LIST_HEAD(&bo->cb_list); 1140 } 1141 1142 static void nf_flow_table_indr_cleanup(struct flow_block_cb *block_cb) 1143 { 1144 struct nf_flowtable *flowtable = block_cb->indr.data; 1145 struct net_device *dev = block_cb->indr.dev; 1146 1147 nf_flow_table_gc_cleanup(flowtable, dev); 1148 down_write(&flowtable->flow_block_lock); 1149 list_del(&block_cb->list); 1150 list_del(&block_cb->driver_list); 1151 flow_block_cb_free(block_cb); 1152 up_write(&flowtable->flow_block_lock); 1153 } 1154 1155 static int nf_flow_table_indr_offload_cmd(struct flow_block_offload *bo, 1156 struct nf_flowtable *flowtable, 1157 struct net_device *dev, 1158 enum flow_block_command cmd, 1159 struct netlink_ext_ack *extack) 1160 { 1161 nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable, 1162 extack); 1163 1164 return flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_FT, flowtable, bo, 1165 nf_flow_table_indr_cleanup); 1166 } 1167 1168 static int nf_flow_table_offload_cmd(struct flow_block_offload *bo, 1169 struct nf_flowtable *flowtable, 1170 struct net_device *dev, 1171 enum flow_block_command cmd, 1172 struct netlink_ext_ack *extack) 1173 { 1174 int err; 1175 1176 nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable, 1177 extack); 1178 down_write(&flowtable->flow_block_lock); 1179 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_FT, bo); 1180 up_write(&flowtable->flow_block_lock); 1181 if (err < 0) 1182 return err; 1183 1184 return 0; 1185 } 1186 1187 int nf_flow_table_offload_setup(struct nf_flowtable *flowtable, 1188 struct net_device *dev, 1189 enum flow_block_command cmd) 1190 { 1191 struct netlink_ext_ack extack = {}; 1192 struct flow_block_offload bo; 1193 int err; 1194 1195 if (!nf_flowtable_hw_offload(flowtable)) 1196 return nf_flow_offload_xdp_setup(flowtable, dev, cmd); 1197 1198 if (dev->netdev_ops->ndo_setup_tc) 1199 err = nf_flow_table_offload_cmd(&bo, flowtable, dev, cmd, 1200 &extack); 1201 else 1202 err = nf_flow_table_indr_offload_cmd(&bo, flowtable, dev, cmd, 1203 &extack); 1204 if (err < 0) 1205 return err; 1206 1207 return nf_flow_table_block_setup(flowtable, &bo, cmd); 1208 } 1209 EXPORT_SYMBOL_GPL(nf_flow_table_offload_setup); 1210 1211 int nf_flow_table_offload_init(void) 1212 { 1213 nf_flow_offload_add_wq = alloc_workqueue("nf_ft_offload_add", 1214 WQ_UNBOUND | WQ_SYSFS, 0); 1215 if (!nf_flow_offload_add_wq) 1216 return -ENOMEM; 1217 1218 nf_flow_offload_del_wq = alloc_workqueue("nf_ft_offload_del", 1219 WQ_UNBOUND | WQ_SYSFS, 0); 1220 if (!nf_flow_offload_del_wq) 1221 goto err_del_wq; 1222 1223 nf_flow_offload_stats_wq = alloc_workqueue("nf_ft_offload_stats", 1224 WQ_UNBOUND | WQ_SYSFS, 0); 1225 if (!nf_flow_offload_stats_wq) 1226 goto err_stats_wq; 1227 1228 return 0; 1229 1230 err_stats_wq: 1231 destroy_workqueue(nf_flow_offload_del_wq); 1232 err_del_wq: 1233 destroy_workqueue(nf_flow_offload_add_wq); 1234 return -ENOMEM; 1235 } 1236 1237 void nf_flow_table_offload_exit(void) 1238 { 1239 destroy_workqueue(nf_flow_offload_add_wq); 1240 destroy_workqueue(nf_flow_offload_del_wq); 1241 destroy_workqueue(nf_flow_offload_stats_wq); 1242 } 1243