1 #include <linux/kernel.h> 2 #include <linux/skbuff.h> 3 #include <linux/export.h> 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip.h> 8 #include <net/ipv6.h> 9 #include <net/gre.h> 10 #include <net/pptp.h> 11 #include <linux/igmp.h> 12 #include <linux/icmp.h> 13 #include <linux/sctp.h> 14 #include <linux/dccp.h> 15 #include <linux/if_tunnel.h> 16 #include <linux/if_pppox.h> 17 #include <linux/ppp_defs.h> 18 #include <linux/stddef.h> 19 #include <linux/if_ether.h> 20 #include <linux/mpls.h> 21 #include <net/flow_dissector.h> 22 #include <scsi/fc/fc_fcoe.h> 23 24 static void dissector_set_key(struct flow_dissector *flow_dissector, 25 enum flow_dissector_key_id key_id) 26 { 27 flow_dissector->used_keys |= (1 << key_id); 28 } 29 30 void skb_flow_dissector_init(struct flow_dissector *flow_dissector, 31 const struct flow_dissector_key *key, 32 unsigned int key_count) 33 { 34 unsigned int i; 35 36 memset(flow_dissector, 0, sizeof(*flow_dissector)); 37 38 for (i = 0; i < key_count; i++, key++) { 39 /* User should make sure that every key target offset is withing 40 * boundaries of unsigned short. 41 */ 42 BUG_ON(key->offset > USHRT_MAX); 43 BUG_ON(dissector_uses_key(flow_dissector, 44 key->key_id)); 45 46 dissector_set_key(flow_dissector, key->key_id); 47 flow_dissector->offset[key->key_id] = key->offset; 48 } 49 50 /* Ensure that the dissector always includes control and basic key. 51 * That way we are able to avoid handling lack of these in fast path. 52 */ 53 BUG_ON(!dissector_uses_key(flow_dissector, 54 FLOW_DISSECTOR_KEY_CONTROL)); 55 BUG_ON(!dissector_uses_key(flow_dissector, 56 FLOW_DISSECTOR_KEY_BASIC)); 57 } 58 EXPORT_SYMBOL(skb_flow_dissector_init); 59 60 /** 61 * __skb_flow_get_ports - extract the upper layer ports and return them 62 * @skb: sk_buff to extract the ports from 63 * @thoff: transport header offset 64 * @ip_proto: protocol for which to get port offset 65 * @data: raw buffer pointer to the packet, if NULL use skb->data 66 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 67 * 68 * The function will try to retrieve the ports at offset thoff + poff where poff 69 * is the protocol port offset returned from proto_ports_offset 70 */ 71 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, 72 void *data, int hlen) 73 { 74 int poff = proto_ports_offset(ip_proto); 75 76 if (!data) { 77 data = skb->data; 78 hlen = skb_headlen(skb); 79 } 80 81 if (poff >= 0) { 82 __be32 *ports, _ports; 83 84 ports = __skb_header_pointer(skb, thoff + poff, 85 sizeof(_ports), data, hlen, &_ports); 86 if (ports) 87 return *ports; 88 } 89 90 return 0; 91 } 92 EXPORT_SYMBOL(__skb_flow_get_ports); 93 94 /** 95 * __skb_flow_dissect - extract the flow_keys struct and return it 96 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified 97 * @flow_dissector: list of keys to dissect 98 * @target_container: target structure to put dissected values into 99 * @data: raw buffer pointer to the packet, if NULL use skb->data 100 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol 101 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb) 102 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 103 * 104 * The function will try to retrieve individual keys into target specified 105 * by flow_dissector from either the skbuff or a raw buffer specified by the 106 * rest parameters. 107 * 108 * Caller must take care of zeroing target container memory. 109 */ 110 bool __skb_flow_dissect(const struct sk_buff *skb, 111 struct flow_dissector *flow_dissector, 112 void *target_container, 113 void *data, __be16 proto, int nhoff, int hlen, 114 unsigned int flags) 115 { 116 struct flow_dissector_key_control *key_control; 117 struct flow_dissector_key_basic *key_basic; 118 struct flow_dissector_key_addrs *key_addrs; 119 struct flow_dissector_key_ports *key_ports; 120 struct flow_dissector_key_tags *key_tags; 121 struct flow_dissector_key_vlan *key_vlan; 122 struct flow_dissector_key_keyid *key_keyid; 123 bool skip_vlan = false; 124 u8 ip_proto = 0; 125 bool ret = false; 126 127 if (!data) { 128 data = skb->data; 129 proto = skb_vlan_tag_present(skb) ? 130 skb->vlan_proto : skb->protocol; 131 nhoff = skb_network_offset(skb); 132 hlen = skb_headlen(skb); 133 } 134 135 /* It is ensured by skb_flow_dissector_init() that control key will 136 * be always present. 137 */ 138 key_control = skb_flow_dissector_target(flow_dissector, 139 FLOW_DISSECTOR_KEY_CONTROL, 140 target_container); 141 142 /* It is ensured by skb_flow_dissector_init() that basic key will 143 * be always present. 144 */ 145 key_basic = skb_flow_dissector_target(flow_dissector, 146 FLOW_DISSECTOR_KEY_BASIC, 147 target_container); 148 149 if (dissector_uses_key(flow_dissector, 150 FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 151 struct ethhdr *eth = eth_hdr(skb); 152 struct flow_dissector_key_eth_addrs *key_eth_addrs; 153 154 key_eth_addrs = skb_flow_dissector_target(flow_dissector, 155 FLOW_DISSECTOR_KEY_ETH_ADDRS, 156 target_container); 157 memcpy(key_eth_addrs, ð->h_dest, sizeof(*key_eth_addrs)); 158 } 159 160 again: 161 switch (proto) { 162 case htons(ETH_P_IP): { 163 const struct iphdr *iph; 164 struct iphdr _iph; 165 ip: 166 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 167 if (!iph || iph->ihl < 5) 168 goto out_bad; 169 nhoff += iph->ihl * 4; 170 171 ip_proto = iph->protocol; 172 173 if (dissector_uses_key(flow_dissector, 174 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 175 key_addrs = skb_flow_dissector_target(flow_dissector, 176 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 177 target_container); 178 179 memcpy(&key_addrs->v4addrs, &iph->saddr, 180 sizeof(key_addrs->v4addrs)); 181 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 182 } 183 184 if (ip_is_fragment(iph)) { 185 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 186 187 if (iph->frag_off & htons(IP_OFFSET)) { 188 goto out_good; 189 } else { 190 key_control->flags |= FLOW_DIS_FIRST_FRAG; 191 if (!(flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) 192 goto out_good; 193 } 194 } 195 196 if (flags & FLOW_DISSECTOR_F_STOP_AT_L3) 197 goto out_good; 198 199 break; 200 } 201 case htons(ETH_P_IPV6): { 202 const struct ipv6hdr *iph; 203 struct ipv6hdr _iph; 204 205 ipv6: 206 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 207 if (!iph) 208 goto out_bad; 209 210 ip_proto = iph->nexthdr; 211 nhoff += sizeof(struct ipv6hdr); 212 213 if (dissector_uses_key(flow_dissector, 214 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 215 key_addrs = skb_flow_dissector_target(flow_dissector, 216 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 217 target_container); 218 219 memcpy(&key_addrs->v6addrs, &iph->saddr, 220 sizeof(key_addrs->v6addrs)); 221 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 222 } 223 224 if ((dissector_uses_key(flow_dissector, 225 FLOW_DISSECTOR_KEY_FLOW_LABEL) || 226 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) && 227 ip6_flowlabel(iph)) { 228 __be32 flow_label = ip6_flowlabel(iph); 229 230 if (dissector_uses_key(flow_dissector, 231 FLOW_DISSECTOR_KEY_FLOW_LABEL)) { 232 key_tags = skb_flow_dissector_target(flow_dissector, 233 FLOW_DISSECTOR_KEY_FLOW_LABEL, 234 target_container); 235 key_tags->flow_label = ntohl(flow_label); 236 } 237 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) 238 goto out_good; 239 } 240 241 if (flags & FLOW_DISSECTOR_F_STOP_AT_L3) 242 goto out_good; 243 244 break; 245 } 246 case htons(ETH_P_8021AD): 247 case htons(ETH_P_8021Q): { 248 const struct vlan_hdr *vlan; 249 250 if (skb_vlan_tag_present(skb)) 251 proto = skb->protocol; 252 253 if (!skb_vlan_tag_present(skb) || 254 proto == cpu_to_be16(ETH_P_8021Q) || 255 proto == cpu_to_be16(ETH_P_8021AD)) { 256 struct vlan_hdr _vlan; 257 258 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), 259 data, hlen, &_vlan); 260 if (!vlan) 261 goto out_bad; 262 proto = vlan->h_vlan_encapsulated_proto; 263 nhoff += sizeof(*vlan); 264 if (skip_vlan) 265 goto again; 266 } 267 268 skip_vlan = true; 269 if (dissector_uses_key(flow_dissector, 270 FLOW_DISSECTOR_KEY_VLAN)) { 271 key_vlan = skb_flow_dissector_target(flow_dissector, 272 FLOW_DISSECTOR_KEY_VLAN, 273 target_container); 274 275 if (skb_vlan_tag_present(skb)) { 276 key_vlan->vlan_id = skb_vlan_tag_get_id(skb); 277 key_vlan->vlan_priority = 278 (skb_vlan_tag_get_prio(skb) >> VLAN_PRIO_SHIFT); 279 } else { 280 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) & 281 VLAN_VID_MASK; 282 key_vlan->vlan_priority = 283 (ntohs(vlan->h_vlan_TCI) & 284 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 285 } 286 } 287 288 goto again; 289 } 290 case htons(ETH_P_PPP_SES): { 291 struct { 292 struct pppoe_hdr hdr; 293 __be16 proto; 294 } *hdr, _hdr; 295 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 296 if (!hdr) 297 goto out_bad; 298 proto = hdr->proto; 299 nhoff += PPPOE_SES_HLEN; 300 switch (proto) { 301 case htons(PPP_IP): 302 goto ip; 303 case htons(PPP_IPV6): 304 goto ipv6; 305 default: 306 goto out_bad; 307 } 308 } 309 case htons(ETH_P_TIPC): { 310 struct { 311 __be32 pre[3]; 312 __be32 srcnode; 313 } *hdr, _hdr; 314 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 315 if (!hdr) 316 goto out_bad; 317 318 if (dissector_uses_key(flow_dissector, 319 FLOW_DISSECTOR_KEY_TIPC_ADDRS)) { 320 key_addrs = skb_flow_dissector_target(flow_dissector, 321 FLOW_DISSECTOR_KEY_TIPC_ADDRS, 322 target_container); 323 key_addrs->tipcaddrs.srcnode = hdr->srcnode; 324 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC_ADDRS; 325 } 326 goto out_good; 327 } 328 329 case htons(ETH_P_MPLS_UC): 330 case htons(ETH_P_MPLS_MC): { 331 struct mpls_label *hdr, _hdr[2]; 332 mpls: 333 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, 334 hlen, &_hdr); 335 if (!hdr) 336 goto out_bad; 337 338 if ((ntohl(hdr[0].entry) & MPLS_LS_LABEL_MASK) >> 339 MPLS_LS_LABEL_SHIFT == MPLS_LABEL_ENTROPY) { 340 if (dissector_uses_key(flow_dissector, 341 FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) { 342 key_keyid = skb_flow_dissector_target(flow_dissector, 343 FLOW_DISSECTOR_KEY_MPLS_ENTROPY, 344 target_container); 345 key_keyid->keyid = hdr[1].entry & 346 htonl(MPLS_LS_LABEL_MASK); 347 } 348 349 goto out_good; 350 } 351 352 goto out_good; 353 } 354 355 case htons(ETH_P_FCOE): 356 if ((hlen - nhoff) < FCOE_HEADER_LEN) 357 goto out_bad; 358 359 nhoff += FCOE_HEADER_LEN; 360 goto out_good; 361 default: 362 goto out_bad; 363 } 364 365 ip_proto_again: 366 switch (ip_proto) { 367 case IPPROTO_GRE: { 368 struct gre_base_hdr *hdr, _hdr; 369 u16 gre_ver; 370 int offset = 0; 371 372 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 373 if (!hdr) 374 goto out_bad; 375 376 /* Only look inside GRE without routing */ 377 if (hdr->flags & GRE_ROUTING) 378 break; 379 380 /* Only look inside GRE for version 0 and 1 */ 381 gre_ver = ntohs(hdr->flags & GRE_VERSION); 382 if (gre_ver > 1) 383 break; 384 385 proto = hdr->protocol; 386 if (gre_ver) { 387 /* Version1 must be PPTP, and check the flags */ 388 if (!(proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY))) 389 break; 390 } 391 392 offset += sizeof(struct gre_base_hdr); 393 394 if (hdr->flags & GRE_CSUM) 395 offset += sizeof(((struct gre_full_hdr *)0)->csum) + 396 sizeof(((struct gre_full_hdr *)0)->reserved1); 397 398 if (hdr->flags & GRE_KEY) { 399 const __be32 *keyid; 400 __be32 _keyid; 401 402 keyid = __skb_header_pointer(skb, nhoff + offset, sizeof(_keyid), 403 data, hlen, &_keyid); 404 if (!keyid) 405 goto out_bad; 406 407 if (dissector_uses_key(flow_dissector, 408 FLOW_DISSECTOR_KEY_GRE_KEYID)) { 409 key_keyid = skb_flow_dissector_target(flow_dissector, 410 FLOW_DISSECTOR_KEY_GRE_KEYID, 411 target_container); 412 if (gre_ver == 0) 413 key_keyid->keyid = *keyid; 414 else 415 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK; 416 } 417 offset += sizeof(((struct gre_full_hdr *)0)->key); 418 } 419 420 if (hdr->flags & GRE_SEQ) 421 offset += sizeof(((struct pptp_gre_header *)0)->seq); 422 423 if (gre_ver == 0) { 424 if (proto == htons(ETH_P_TEB)) { 425 const struct ethhdr *eth; 426 struct ethhdr _eth; 427 428 eth = __skb_header_pointer(skb, nhoff + offset, 429 sizeof(_eth), 430 data, hlen, &_eth); 431 if (!eth) 432 goto out_bad; 433 proto = eth->h_proto; 434 offset += sizeof(*eth); 435 436 /* Cap headers that we access via pointers at the 437 * end of the Ethernet header as our maximum alignment 438 * at that point is only 2 bytes. 439 */ 440 if (NET_IP_ALIGN) 441 hlen = (nhoff + offset); 442 } 443 } else { /* version 1, must be PPTP */ 444 u8 _ppp_hdr[PPP_HDRLEN]; 445 u8 *ppp_hdr; 446 447 if (hdr->flags & GRE_ACK) 448 offset += sizeof(((struct pptp_gre_header *)0)->ack); 449 450 ppp_hdr = skb_header_pointer(skb, nhoff + offset, 451 sizeof(_ppp_hdr), _ppp_hdr); 452 if (!ppp_hdr) 453 goto out_bad; 454 455 switch (PPP_PROTOCOL(ppp_hdr)) { 456 case PPP_IP: 457 proto = htons(ETH_P_IP); 458 break; 459 case PPP_IPV6: 460 proto = htons(ETH_P_IPV6); 461 break; 462 default: 463 /* Could probably catch some more like MPLS */ 464 break; 465 } 466 467 offset += PPP_HDRLEN; 468 } 469 470 nhoff += offset; 471 key_control->flags |= FLOW_DIS_ENCAPSULATION; 472 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 473 goto out_good; 474 475 goto again; 476 } 477 case NEXTHDR_HOP: 478 case NEXTHDR_ROUTING: 479 case NEXTHDR_DEST: { 480 u8 _opthdr[2], *opthdr; 481 482 if (proto != htons(ETH_P_IPV6)) 483 break; 484 485 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr), 486 data, hlen, &_opthdr); 487 if (!opthdr) 488 goto out_bad; 489 490 ip_proto = opthdr[0]; 491 nhoff += (opthdr[1] + 1) << 3; 492 493 goto ip_proto_again; 494 } 495 case NEXTHDR_FRAGMENT: { 496 struct frag_hdr _fh, *fh; 497 498 if (proto != htons(ETH_P_IPV6)) 499 break; 500 501 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh), 502 data, hlen, &_fh); 503 504 if (!fh) 505 goto out_bad; 506 507 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 508 509 nhoff += sizeof(_fh); 510 ip_proto = fh->nexthdr; 511 512 if (!(fh->frag_off & htons(IP6_OFFSET))) { 513 key_control->flags |= FLOW_DIS_FIRST_FRAG; 514 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) 515 goto ip_proto_again; 516 } 517 goto out_good; 518 } 519 case IPPROTO_IPIP: 520 proto = htons(ETH_P_IP); 521 522 key_control->flags |= FLOW_DIS_ENCAPSULATION; 523 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 524 goto out_good; 525 526 goto ip; 527 case IPPROTO_IPV6: 528 proto = htons(ETH_P_IPV6); 529 530 key_control->flags |= FLOW_DIS_ENCAPSULATION; 531 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 532 goto out_good; 533 534 goto ipv6; 535 case IPPROTO_MPLS: 536 proto = htons(ETH_P_MPLS_UC); 537 goto mpls; 538 default: 539 break; 540 } 541 542 if (dissector_uses_key(flow_dissector, 543 FLOW_DISSECTOR_KEY_PORTS)) { 544 key_ports = skb_flow_dissector_target(flow_dissector, 545 FLOW_DISSECTOR_KEY_PORTS, 546 target_container); 547 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto, 548 data, hlen); 549 } 550 551 out_good: 552 ret = true; 553 554 out_bad: 555 key_basic->n_proto = proto; 556 key_basic->ip_proto = ip_proto; 557 key_control->thoff = (u16)nhoff; 558 559 return ret; 560 } 561 EXPORT_SYMBOL(__skb_flow_dissect); 562 563 static u32 hashrnd __read_mostly; 564 static __always_inline void __flow_hash_secret_init(void) 565 { 566 net_get_random_once(&hashrnd, sizeof(hashrnd)); 567 } 568 569 static __always_inline u32 __flow_hash_words(const u32 *words, u32 length, 570 u32 keyval) 571 { 572 return jhash2(words, length, keyval); 573 } 574 575 static inline const u32 *flow_keys_hash_start(const struct flow_keys *flow) 576 { 577 const void *p = flow; 578 579 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32)); 580 return (const u32 *)(p + FLOW_KEYS_HASH_OFFSET); 581 } 582 583 static inline size_t flow_keys_hash_length(const struct flow_keys *flow) 584 { 585 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs); 586 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32)); 587 BUILD_BUG_ON(offsetof(typeof(*flow), addrs) != 588 sizeof(*flow) - sizeof(flow->addrs)); 589 590 switch (flow->control.addr_type) { 591 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 592 diff -= sizeof(flow->addrs.v4addrs); 593 break; 594 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 595 diff -= sizeof(flow->addrs.v6addrs); 596 break; 597 case FLOW_DISSECTOR_KEY_TIPC_ADDRS: 598 diff -= sizeof(flow->addrs.tipcaddrs); 599 break; 600 } 601 return (sizeof(*flow) - diff) / sizeof(u32); 602 } 603 604 __be32 flow_get_u32_src(const struct flow_keys *flow) 605 { 606 switch (flow->control.addr_type) { 607 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 608 return flow->addrs.v4addrs.src; 609 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 610 return (__force __be32)ipv6_addr_hash( 611 &flow->addrs.v6addrs.src); 612 case FLOW_DISSECTOR_KEY_TIPC_ADDRS: 613 return flow->addrs.tipcaddrs.srcnode; 614 default: 615 return 0; 616 } 617 } 618 EXPORT_SYMBOL(flow_get_u32_src); 619 620 __be32 flow_get_u32_dst(const struct flow_keys *flow) 621 { 622 switch (flow->control.addr_type) { 623 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 624 return flow->addrs.v4addrs.dst; 625 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 626 return (__force __be32)ipv6_addr_hash( 627 &flow->addrs.v6addrs.dst); 628 default: 629 return 0; 630 } 631 } 632 EXPORT_SYMBOL(flow_get_u32_dst); 633 634 static inline void __flow_hash_consistentify(struct flow_keys *keys) 635 { 636 int addr_diff, i; 637 638 switch (keys->control.addr_type) { 639 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 640 addr_diff = (__force u32)keys->addrs.v4addrs.dst - 641 (__force u32)keys->addrs.v4addrs.src; 642 if ((addr_diff < 0) || 643 (addr_diff == 0 && 644 ((__force u16)keys->ports.dst < 645 (__force u16)keys->ports.src))) { 646 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst); 647 swap(keys->ports.src, keys->ports.dst); 648 } 649 break; 650 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 651 addr_diff = memcmp(&keys->addrs.v6addrs.dst, 652 &keys->addrs.v6addrs.src, 653 sizeof(keys->addrs.v6addrs.dst)); 654 if ((addr_diff < 0) || 655 (addr_diff == 0 && 656 ((__force u16)keys->ports.dst < 657 (__force u16)keys->ports.src))) { 658 for (i = 0; i < 4; i++) 659 swap(keys->addrs.v6addrs.src.s6_addr32[i], 660 keys->addrs.v6addrs.dst.s6_addr32[i]); 661 swap(keys->ports.src, keys->ports.dst); 662 } 663 break; 664 } 665 } 666 667 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) 668 { 669 u32 hash; 670 671 __flow_hash_consistentify(keys); 672 673 hash = __flow_hash_words(flow_keys_hash_start(keys), 674 flow_keys_hash_length(keys), keyval); 675 if (!hash) 676 hash = 1; 677 678 return hash; 679 } 680 681 u32 flow_hash_from_keys(struct flow_keys *keys) 682 { 683 __flow_hash_secret_init(); 684 return __flow_hash_from_keys(keys, hashrnd); 685 } 686 EXPORT_SYMBOL(flow_hash_from_keys); 687 688 static inline u32 ___skb_get_hash(const struct sk_buff *skb, 689 struct flow_keys *keys, u32 keyval) 690 { 691 skb_flow_dissect_flow_keys(skb, keys, 692 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 693 694 return __flow_hash_from_keys(keys, keyval); 695 } 696 697 struct _flow_keys_digest_data { 698 __be16 n_proto; 699 u8 ip_proto; 700 u8 padding; 701 __be32 ports; 702 __be32 src; 703 __be32 dst; 704 }; 705 706 void make_flow_keys_digest(struct flow_keys_digest *digest, 707 const struct flow_keys *flow) 708 { 709 struct _flow_keys_digest_data *data = 710 (struct _flow_keys_digest_data *)digest; 711 712 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest)); 713 714 memset(digest, 0, sizeof(*digest)); 715 716 data->n_proto = flow->basic.n_proto; 717 data->ip_proto = flow->basic.ip_proto; 718 data->ports = flow->ports.ports; 719 data->src = flow->addrs.v4addrs.src; 720 data->dst = flow->addrs.v4addrs.dst; 721 } 722 EXPORT_SYMBOL(make_flow_keys_digest); 723 724 static struct flow_dissector flow_keys_dissector_symmetric __read_mostly; 725 726 u32 __skb_get_hash_symmetric(struct sk_buff *skb) 727 { 728 struct flow_keys keys; 729 730 __flow_hash_secret_init(); 731 732 memset(&keys, 0, sizeof(keys)); 733 __skb_flow_dissect(skb, &flow_keys_dissector_symmetric, &keys, 734 NULL, 0, 0, 0, 735 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 736 737 return __flow_hash_from_keys(&keys, hashrnd); 738 } 739 EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric); 740 741 /** 742 * __skb_get_hash: calculate a flow hash 743 * @skb: sk_buff to calculate flow hash from 744 * 745 * This function calculates a flow hash based on src/dst addresses 746 * and src/dst port numbers. Sets hash in skb to non-zero hash value 747 * on success, zero indicates no valid hash. Also, sets l4_hash in skb 748 * if hash is a canonical 4-tuple hash over transport ports. 749 */ 750 void __skb_get_hash(struct sk_buff *skb) 751 { 752 struct flow_keys keys; 753 u32 hash; 754 755 __flow_hash_secret_init(); 756 757 hash = ___skb_get_hash(skb, &keys, hashrnd); 758 759 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys)); 760 } 761 EXPORT_SYMBOL(__skb_get_hash); 762 763 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb) 764 { 765 struct flow_keys keys; 766 767 return ___skb_get_hash(skb, &keys, perturb); 768 } 769 EXPORT_SYMBOL(skb_get_hash_perturb); 770 771 __u32 __skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6) 772 { 773 struct flow_keys keys; 774 775 memset(&keys, 0, sizeof(keys)); 776 777 memcpy(&keys.addrs.v6addrs.src, &fl6->saddr, 778 sizeof(keys.addrs.v6addrs.src)); 779 memcpy(&keys.addrs.v6addrs.dst, &fl6->daddr, 780 sizeof(keys.addrs.v6addrs.dst)); 781 keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 782 keys.ports.src = fl6->fl6_sport; 783 keys.ports.dst = fl6->fl6_dport; 784 keys.keyid.keyid = fl6->fl6_gre_key; 785 keys.tags.flow_label = (__force u32)fl6->flowlabel; 786 keys.basic.ip_proto = fl6->flowi6_proto; 787 788 __skb_set_sw_hash(skb, flow_hash_from_keys(&keys), 789 flow_keys_have_l4(&keys)); 790 791 return skb->hash; 792 } 793 EXPORT_SYMBOL(__skb_get_hash_flowi6); 794 795 __u32 __skb_get_hash_flowi4(struct sk_buff *skb, const struct flowi4 *fl4) 796 { 797 struct flow_keys keys; 798 799 memset(&keys, 0, sizeof(keys)); 800 801 keys.addrs.v4addrs.src = fl4->saddr; 802 keys.addrs.v4addrs.dst = fl4->daddr; 803 keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 804 keys.ports.src = fl4->fl4_sport; 805 keys.ports.dst = fl4->fl4_dport; 806 keys.keyid.keyid = fl4->fl4_gre_key; 807 keys.basic.ip_proto = fl4->flowi4_proto; 808 809 __skb_set_sw_hash(skb, flow_hash_from_keys(&keys), 810 flow_keys_have_l4(&keys)); 811 812 return skb->hash; 813 } 814 EXPORT_SYMBOL(__skb_get_hash_flowi4); 815 816 u32 __skb_get_poff(const struct sk_buff *skb, void *data, 817 const struct flow_keys *keys, int hlen) 818 { 819 u32 poff = keys->control.thoff; 820 821 /* skip L4 headers for fragments after the first */ 822 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) && 823 !(keys->control.flags & FLOW_DIS_FIRST_FRAG)) 824 return poff; 825 826 switch (keys->basic.ip_proto) { 827 case IPPROTO_TCP: { 828 /* access doff as u8 to avoid unaligned access */ 829 const u8 *doff; 830 u8 _doff; 831 832 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff), 833 data, hlen, &_doff); 834 if (!doff) 835 return poff; 836 837 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2); 838 break; 839 } 840 case IPPROTO_UDP: 841 case IPPROTO_UDPLITE: 842 poff += sizeof(struct udphdr); 843 break; 844 /* For the rest, we do not really care about header 845 * extensions at this point for now. 846 */ 847 case IPPROTO_ICMP: 848 poff += sizeof(struct icmphdr); 849 break; 850 case IPPROTO_ICMPV6: 851 poff += sizeof(struct icmp6hdr); 852 break; 853 case IPPROTO_IGMP: 854 poff += sizeof(struct igmphdr); 855 break; 856 case IPPROTO_DCCP: 857 poff += sizeof(struct dccp_hdr); 858 break; 859 case IPPROTO_SCTP: 860 poff += sizeof(struct sctphdr); 861 break; 862 } 863 864 return poff; 865 } 866 867 /** 868 * skb_get_poff - get the offset to the payload 869 * @skb: sk_buff to get the payload offset from 870 * 871 * The function will get the offset to the payload as far as it could 872 * be dissected. The main user is currently BPF, so that we can dynamically 873 * truncate packets without needing to push actual payload to the user 874 * space and can analyze headers only, instead. 875 */ 876 u32 skb_get_poff(const struct sk_buff *skb) 877 { 878 struct flow_keys keys; 879 880 if (!skb_flow_dissect_flow_keys(skb, &keys, 0)) 881 return 0; 882 883 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); 884 } 885 886 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys) 887 { 888 memset(keys, 0, sizeof(*keys)); 889 890 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr, 891 sizeof(keys->addrs.v6addrs.src)); 892 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr, 893 sizeof(keys->addrs.v6addrs.dst)); 894 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 895 keys->ports.src = fl6->fl6_sport; 896 keys->ports.dst = fl6->fl6_dport; 897 keys->keyid.keyid = fl6->fl6_gre_key; 898 keys->tags.flow_label = (__force u32)fl6->flowlabel; 899 keys->basic.ip_proto = fl6->flowi6_proto; 900 901 return flow_hash_from_keys(keys); 902 } 903 EXPORT_SYMBOL(__get_hash_from_flowi6); 904 905 __u32 __get_hash_from_flowi4(const struct flowi4 *fl4, struct flow_keys *keys) 906 { 907 memset(keys, 0, sizeof(*keys)); 908 909 keys->addrs.v4addrs.src = fl4->saddr; 910 keys->addrs.v4addrs.dst = fl4->daddr; 911 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 912 keys->ports.src = fl4->fl4_sport; 913 keys->ports.dst = fl4->fl4_dport; 914 keys->keyid.keyid = fl4->fl4_gre_key; 915 keys->basic.ip_proto = fl4->flowi4_proto; 916 917 return flow_hash_from_keys(keys); 918 } 919 EXPORT_SYMBOL(__get_hash_from_flowi4); 920 921 static const struct flow_dissector_key flow_keys_dissector_keys[] = { 922 { 923 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 924 .offset = offsetof(struct flow_keys, control), 925 }, 926 { 927 .key_id = FLOW_DISSECTOR_KEY_BASIC, 928 .offset = offsetof(struct flow_keys, basic), 929 }, 930 { 931 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 932 .offset = offsetof(struct flow_keys, addrs.v4addrs), 933 }, 934 { 935 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 936 .offset = offsetof(struct flow_keys, addrs.v6addrs), 937 }, 938 { 939 .key_id = FLOW_DISSECTOR_KEY_TIPC_ADDRS, 940 .offset = offsetof(struct flow_keys, addrs.tipcaddrs), 941 }, 942 { 943 .key_id = FLOW_DISSECTOR_KEY_PORTS, 944 .offset = offsetof(struct flow_keys, ports), 945 }, 946 { 947 .key_id = FLOW_DISSECTOR_KEY_VLAN, 948 .offset = offsetof(struct flow_keys, vlan), 949 }, 950 { 951 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL, 952 .offset = offsetof(struct flow_keys, tags), 953 }, 954 { 955 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID, 956 .offset = offsetof(struct flow_keys, keyid), 957 }, 958 }; 959 960 static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = { 961 { 962 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 963 .offset = offsetof(struct flow_keys, control), 964 }, 965 { 966 .key_id = FLOW_DISSECTOR_KEY_BASIC, 967 .offset = offsetof(struct flow_keys, basic), 968 }, 969 { 970 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 971 .offset = offsetof(struct flow_keys, addrs.v4addrs), 972 }, 973 { 974 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 975 .offset = offsetof(struct flow_keys, addrs.v6addrs), 976 }, 977 { 978 .key_id = FLOW_DISSECTOR_KEY_PORTS, 979 .offset = offsetof(struct flow_keys, ports), 980 }, 981 }; 982 983 static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = { 984 { 985 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 986 .offset = offsetof(struct flow_keys, control), 987 }, 988 { 989 .key_id = FLOW_DISSECTOR_KEY_BASIC, 990 .offset = offsetof(struct flow_keys, basic), 991 }, 992 }; 993 994 struct flow_dissector flow_keys_dissector __read_mostly; 995 EXPORT_SYMBOL(flow_keys_dissector); 996 997 struct flow_dissector flow_keys_buf_dissector __read_mostly; 998 999 static int __init init_default_flow_dissectors(void) 1000 { 1001 skb_flow_dissector_init(&flow_keys_dissector, 1002 flow_keys_dissector_keys, 1003 ARRAY_SIZE(flow_keys_dissector_keys)); 1004 skb_flow_dissector_init(&flow_keys_dissector_symmetric, 1005 flow_keys_dissector_symmetric_keys, 1006 ARRAY_SIZE(flow_keys_dissector_symmetric_keys)); 1007 skb_flow_dissector_init(&flow_keys_buf_dissector, 1008 flow_keys_buf_dissector_keys, 1009 ARRAY_SIZE(flow_keys_buf_dissector_keys)); 1010 return 0; 1011 } 1012 1013 late_initcall_sync(init_default_flow_dissectors); 1014