1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kernel.h> 3 #include <linux/skbuff.h> 4 #include <linux/export.h> 5 #include <linux/ip.h> 6 #include <linux/ipv6.h> 7 #include <linux/if_vlan.h> 8 #include <net/dsa.h> 9 #include <net/dst_metadata.h> 10 #include <net/ip.h> 11 #include <net/ipv6.h> 12 #include <net/gre.h> 13 #include <net/pptp.h> 14 #include <net/tipc.h> 15 #include <linux/igmp.h> 16 #include <linux/icmp.h> 17 #include <linux/sctp.h> 18 #include <linux/dccp.h> 19 #include <linux/if_tunnel.h> 20 #include <linux/if_pppox.h> 21 #include <linux/ppp_defs.h> 22 #include <linux/stddef.h> 23 #include <linux/if_ether.h> 24 #include <linux/mpls.h> 25 #include <linux/tcp.h> 26 #include <net/flow_dissector.h> 27 #include <scsi/fc/fc_fcoe.h> 28 #include <uapi/linux/batadv_packet.h> 29 #include <linux/bpf.h> 30 31 static DEFINE_MUTEX(flow_dissector_mutex); 32 33 static void dissector_set_key(struct flow_dissector *flow_dissector, 34 enum flow_dissector_key_id key_id) 35 { 36 flow_dissector->used_keys |= (1 << key_id); 37 } 38 39 void skb_flow_dissector_init(struct flow_dissector *flow_dissector, 40 const struct flow_dissector_key *key, 41 unsigned int key_count) 42 { 43 unsigned int i; 44 45 memset(flow_dissector, 0, sizeof(*flow_dissector)); 46 47 for (i = 0; i < key_count; i++, key++) { 48 /* User should make sure that every key target offset is withing 49 * boundaries of unsigned short. 50 */ 51 BUG_ON(key->offset > USHRT_MAX); 52 BUG_ON(dissector_uses_key(flow_dissector, 53 key->key_id)); 54 55 dissector_set_key(flow_dissector, key->key_id); 56 flow_dissector->offset[key->key_id] = key->offset; 57 } 58 59 /* Ensure that the dissector always includes control and basic key. 60 * That way we are able to avoid handling lack of these in fast path. 61 */ 62 BUG_ON(!dissector_uses_key(flow_dissector, 63 FLOW_DISSECTOR_KEY_CONTROL)); 64 BUG_ON(!dissector_uses_key(flow_dissector, 65 FLOW_DISSECTOR_KEY_BASIC)); 66 } 67 EXPORT_SYMBOL(skb_flow_dissector_init); 68 69 int skb_flow_dissector_prog_query(const union bpf_attr *attr, 70 union bpf_attr __user *uattr) 71 { 72 __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); 73 u32 prog_id, prog_cnt = 0, flags = 0; 74 struct bpf_prog *attached; 75 struct net *net; 76 77 if (attr->query.query_flags) 78 return -EINVAL; 79 80 net = get_net_ns_by_fd(attr->query.target_fd); 81 if (IS_ERR(net)) 82 return PTR_ERR(net); 83 84 rcu_read_lock(); 85 attached = rcu_dereference(net->flow_dissector_prog); 86 if (attached) { 87 prog_cnt = 1; 88 prog_id = attached->aux->id; 89 } 90 rcu_read_unlock(); 91 92 put_net(net); 93 94 if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) 95 return -EFAULT; 96 if (copy_to_user(&uattr->query.prog_cnt, &prog_cnt, sizeof(prog_cnt))) 97 return -EFAULT; 98 99 if (!attr->query.prog_cnt || !prog_ids || !prog_cnt) 100 return 0; 101 102 if (copy_to_user(prog_ids, &prog_id, sizeof(u32))) 103 return -EFAULT; 104 105 return 0; 106 } 107 108 int skb_flow_dissector_bpf_prog_attach(const union bpf_attr *attr, 109 struct bpf_prog *prog) 110 { 111 struct bpf_prog *attached; 112 struct net *net; 113 114 net = current->nsproxy->net_ns; 115 mutex_lock(&flow_dissector_mutex); 116 attached = rcu_dereference_protected(net->flow_dissector_prog, 117 lockdep_is_held(&flow_dissector_mutex)); 118 if (attached) { 119 /* Only one BPF program can be attached at a time */ 120 mutex_unlock(&flow_dissector_mutex); 121 return -EEXIST; 122 } 123 rcu_assign_pointer(net->flow_dissector_prog, prog); 124 mutex_unlock(&flow_dissector_mutex); 125 return 0; 126 } 127 128 int skb_flow_dissector_bpf_prog_detach(const union bpf_attr *attr) 129 { 130 struct bpf_prog *attached; 131 struct net *net; 132 133 net = current->nsproxy->net_ns; 134 mutex_lock(&flow_dissector_mutex); 135 attached = rcu_dereference_protected(net->flow_dissector_prog, 136 lockdep_is_held(&flow_dissector_mutex)); 137 if (!attached) { 138 mutex_unlock(&flow_dissector_mutex); 139 return -ENOENT; 140 } 141 bpf_prog_put(attached); 142 RCU_INIT_POINTER(net->flow_dissector_prog, NULL); 143 mutex_unlock(&flow_dissector_mutex); 144 return 0; 145 } 146 /** 147 * skb_flow_get_be16 - extract be16 entity 148 * @skb: sk_buff to extract from 149 * @poff: offset to extract at 150 * @data: raw buffer pointer to the packet 151 * @hlen: packet header length 152 * 153 * The function will try to retrieve a be32 entity at 154 * offset poff 155 */ 156 static __be16 skb_flow_get_be16(const struct sk_buff *skb, int poff, 157 void *data, int hlen) 158 { 159 __be16 *u, _u; 160 161 u = __skb_header_pointer(skb, poff, sizeof(_u), data, hlen, &_u); 162 if (u) 163 return *u; 164 165 return 0; 166 } 167 168 /** 169 * __skb_flow_get_ports - extract the upper layer ports and return them 170 * @skb: sk_buff to extract the ports from 171 * @thoff: transport header offset 172 * @ip_proto: protocol for which to get port offset 173 * @data: raw buffer pointer to the packet, if NULL use skb->data 174 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 175 * 176 * The function will try to retrieve the ports at offset thoff + poff where poff 177 * is the protocol port offset returned from proto_ports_offset 178 */ 179 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, 180 void *data, int hlen) 181 { 182 int poff = proto_ports_offset(ip_proto); 183 184 if (!data) { 185 data = skb->data; 186 hlen = skb_headlen(skb); 187 } 188 189 if (poff >= 0) { 190 __be32 *ports, _ports; 191 192 ports = __skb_header_pointer(skb, thoff + poff, 193 sizeof(_ports), data, hlen, &_ports); 194 if (ports) 195 return *ports; 196 } 197 198 return 0; 199 } 200 EXPORT_SYMBOL(__skb_flow_get_ports); 201 202 static void 203 skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type, 204 struct flow_dissector *flow_dissector, 205 void *target_container) 206 { 207 struct flow_dissector_key_control *ctrl; 208 209 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) 210 return; 211 212 ctrl = skb_flow_dissector_target(flow_dissector, 213 FLOW_DISSECTOR_KEY_ENC_CONTROL, 214 target_container); 215 ctrl->addr_type = type; 216 } 217 218 void 219 skb_flow_dissect_tunnel_info(const struct sk_buff *skb, 220 struct flow_dissector *flow_dissector, 221 void *target_container) 222 { 223 struct ip_tunnel_info *info; 224 struct ip_tunnel_key *key; 225 226 /* A quick check to see if there might be something to do. */ 227 if (!dissector_uses_key(flow_dissector, 228 FLOW_DISSECTOR_KEY_ENC_KEYID) && 229 !dissector_uses_key(flow_dissector, 230 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) && 231 !dissector_uses_key(flow_dissector, 232 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) && 233 !dissector_uses_key(flow_dissector, 234 FLOW_DISSECTOR_KEY_ENC_CONTROL) && 235 !dissector_uses_key(flow_dissector, 236 FLOW_DISSECTOR_KEY_ENC_PORTS) && 237 !dissector_uses_key(flow_dissector, 238 FLOW_DISSECTOR_KEY_ENC_IP) && 239 !dissector_uses_key(flow_dissector, 240 FLOW_DISSECTOR_KEY_ENC_OPTS)) 241 return; 242 243 info = skb_tunnel_info(skb); 244 if (!info) 245 return; 246 247 key = &info->key; 248 249 switch (ip_tunnel_info_af(info)) { 250 case AF_INET: 251 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV4_ADDRS, 252 flow_dissector, 253 target_container); 254 if (dissector_uses_key(flow_dissector, 255 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { 256 struct flow_dissector_key_ipv4_addrs *ipv4; 257 258 ipv4 = skb_flow_dissector_target(flow_dissector, 259 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, 260 target_container); 261 ipv4->src = key->u.ipv4.src; 262 ipv4->dst = key->u.ipv4.dst; 263 } 264 break; 265 case AF_INET6: 266 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV6_ADDRS, 267 flow_dissector, 268 target_container); 269 if (dissector_uses_key(flow_dissector, 270 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { 271 struct flow_dissector_key_ipv6_addrs *ipv6; 272 273 ipv6 = skb_flow_dissector_target(flow_dissector, 274 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, 275 target_container); 276 ipv6->src = key->u.ipv6.src; 277 ipv6->dst = key->u.ipv6.dst; 278 } 279 break; 280 } 281 282 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 283 struct flow_dissector_key_keyid *keyid; 284 285 keyid = skb_flow_dissector_target(flow_dissector, 286 FLOW_DISSECTOR_KEY_ENC_KEYID, 287 target_container); 288 keyid->keyid = tunnel_id_to_key32(key->tun_id); 289 } 290 291 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) { 292 struct flow_dissector_key_ports *tp; 293 294 tp = skb_flow_dissector_target(flow_dissector, 295 FLOW_DISSECTOR_KEY_ENC_PORTS, 296 target_container); 297 tp->src = key->tp_src; 298 tp->dst = key->tp_dst; 299 } 300 301 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_IP)) { 302 struct flow_dissector_key_ip *ip; 303 304 ip = skb_flow_dissector_target(flow_dissector, 305 FLOW_DISSECTOR_KEY_ENC_IP, 306 target_container); 307 ip->tos = key->tos; 308 ip->ttl = key->ttl; 309 } 310 311 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_OPTS)) { 312 struct flow_dissector_key_enc_opts *enc_opt; 313 314 enc_opt = skb_flow_dissector_target(flow_dissector, 315 FLOW_DISSECTOR_KEY_ENC_OPTS, 316 target_container); 317 318 if (info->options_len) { 319 enc_opt->len = info->options_len; 320 ip_tunnel_info_opts_get(enc_opt->data, info); 321 enc_opt->dst_opt_type = info->key.tun_flags & 322 TUNNEL_OPTIONS_PRESENT; 323 } 324 } 325 } 326 EXPORT_SYMBOL(skb_flow_dissect_tunnel_info); 327 328 static enum flow_dissect_ret 329 __skb_flow_dissect_mpls(const struct sk_buff *skb, 330 struct flow_dissector *flow_dissector, 331 void *target_container, void *data, int nhoff, int hlen) 332 { 333 struct flow_dissector_key_keyid *key_keyid; 334 struct mpls_label *hdr, _hdr[2]; 335 u32 entry, label; 336 337 if (!dissector_uses_key(flow_dissector, 338 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) && 339 !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) 340 return FLOW_DISSECT_RET_OUT_GOOD; 341 342 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, 343 hlen, &_hdr); 344 if (!hdr) 345 return FLOW_DISSECT_RET_OUT_BAD; 346 347 entry = ntohl(hdr[0].entry); 348 label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT; 349 350 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) { 351 struct flow_dissector_key_mpls *key_mpls; 352 353 key_mpls = skb_flow_dissector_target(flow_dissector, 354 FLOW_DISSECTOR_KEY_MPLS, 355 target_container); 356 key_mpls->mpls_label = label; 357 key_mpls->mpls_ttl = (entry & MPLS_LS_TTL_MASK) 358 >> MPLS_LS_TTL_SHIFT; 359 key_mpls->mpls_tc = (entry & MPLS_LS_TC_MASK) 360 >> MPLS_LS_TC_SHIFT; 361 key_mpls->mpls_bos = (entry & MPLS_LS_S_MASK) 362 >> MPLS_LS_S_SHIFT; 363 } 364 365 if (label == MPLS_LABEL_ENTROPY) { 366 key_keyid = skb_flow_dissector_target(flow_dissector, 367 FLOW_DISSECTOR_KEY_MPLS_ENTROPY, 368 target_container); 369 key_keyid->keyid = hdr[1].entry & htonl(MPLS_LS_LABEL_MASK); 370 } 371 return FLOW_DISSECT_RET_OUT_GOOD; 372 } 373 374 static enum flow_dissect_ret 375 __skb_flow_dissect_arp(const struct sk_buff *skb, 376 struct flow_dissector *flow_dissector, 377 void *target_container, void *data, int nhoff, int hlen) 378 { 379 struct flow_dissector_key_arp *key_arp; 380 struct { 381 unsigned char ar_sha[ETH_ALEN]; 382 unsigned char ar_sip[4]; 383 unsigned char ar_tha[ETH_ALEN]; 384 unsigned char ar_tip[4]; 385 } *arp_eth, _arp_eth; 386 const struct arphdr *arp; 387 struct arphdr _arp; 388 389 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP)) 390 return FLOW_DISSECT_RET_OUT_GOOD; 391 392 arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data, 393 hlen, &_arp); 394 if (!arp) 395 return FLOW_DISSECT_RET_OUT_BAD; 396 397 if (arp->ar_hrd != htons(ARPHRD_ETHER) || 398 arp->ar_pro != htons(ETH_P_IP) || 399 arp->ar_hln != ETH_ALEN || 400 arp->ar_pln != 4 || 401 (arp->ar_op != htons(ARPOP_REPLY) && 402 arp->ar_op != htons(ARPOP_REQUEST))) 403 return FLOW_DISSECT_RET_OUT_BAD; 404 405 arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp), 406 sizeof(_arp_eth), data, 407 hlen, &_arp_eth); 408 if (!arp_eth) 409 return FLOW_DISSECT_RET_OUT_BAD; 410 411 key_arp = skb_flow_dissector_target(flow_dissector, 412 FLOW_DISSECTOR_KEY_ARP, 413 target_container); 414 415 memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip)); 416 memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip)); 417 418 /* Only store the lower byte of the opcode; 419 * this covers ARPOP_REPLY and ARPOP_REQUEST. 420 */ 421 key_arp->op = ntohs(arp->ar_op) & 0xff; 422 423 ether_addr_copy(key_arp->sha, arp_eth->ar_sha); 424 ether_addr_copy(key_arp->tha, arp_eth->ar_tha); 425 426 return FLOW_DISSECT_RET_OUT_GOOD; 427 } 428 429 static enum flow_dissect_ret 430 __skb_flow_dissect_gre(const struct sk_buff *skb, 431 struct flow_dissector_key_control *key_control, 432 struct flow_dissector *flow_dissector, 433 void *target_container, void *data, 434 __be16 *p_proto, int *p_nhoff, int *p_hlen, 435 unsigned int flags) 436 { 437 struct flow_dissector_key_keyid *key_keyid; 438 struct gre_base_hdr *hdr, _hdr; 439 int offset = 0; 440 u16 gre_ver; 441 442 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), 443 data, *p_hlen, &_hdr); 444 if (!hdr) 445 return FLOW_DISSECT_RET_OUT_BAD; 446 447 /* Only look inside GRE without routing */ 448 if (hdr->flags & GRE_ROUTING) 449 return FLOW_DISSECT_RET_OUT_GOOD; 450 451 /* Only look inside GRE for version 0 and 1 */ 452 gre_ver = ntohs(hdr->flags & GRE_VERSION); 453 if (gre_ver > 1) 454 return FLOW_DISSECT_RET_OUT_GOOD; 455 456 *p_proto = hdr->protocol; 457 if (gre_ver) { 458 /* Version1 must be PPTP, and check the flags */ 459 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY))) 460 return FLOW_DISSECT_RET_OUT_GOOD; 461 } 462 463 offset += sizeof(struct gre_base_hdr); 464 465 if (hdr->flags & GRE_CSUM) 466 offset += FIELD_SIZEOF(struct gre_full_hdr, csum) + 467 FIELD_SIZEOF(struct gre_full_hdr, reserved1); 468 469 if (hdr->flags & GRE_KEY) { 470 const __be32 *keyid; 471 __be32 _keyid; 472 473 keyid = __skb_header_pointer(skb, *p_nhoff + offset, 474 sizeof(_keyid), 475 data, *p_hlen, &_keyid); 476 if (!keyid) 477 return FLOW_DISSECT_RET_OUT_BAD; 478 479 if (dissector_uses_key(flow_dissector, 480 FLOW_DISSECTOR_KEY_GRE_KEYID)) { 481 key_keyid = skb_flow_dissector_target(flow_dissector, 482 FLOW_DISSECTOR_KEY_GRE_KEYID, 483 target_container); 484 if (gre_ver == 0) 485 key_keyid->keyid = *keyid; 486 else 487 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK; 488 } 489 offset += FIELD_SIZEOF(struct gre_full_hdr, key); 490 } 491 492 if (hdr->flags & GRE_SEQ) 493 offset += FIELD_SIZEOF(struct pptp_gre_header, seq); 494 495 if (gre_ver == 0) { 496 if (*p_proto == htons(ETH_P_TEB)) { 497 const struct ethhdr *eth; 498 struct ethhdr _eth; 499 500 eth = __skb_header_pointer(skb, *p_nhoff + offset, 501 sizeof(_eth), 502 data, *p_hlen, &_eth); 503 if (!eth) 504 return FLOW_DISSECT_RET_OUT_BAD; 505 *p_proto = eth->h_proto; 506 offset += sizeof(*eth); 507 508 /* Cap headers that we access via pointers at the 509 * end of the Ethernet header as our maximum alignment 510 * at that point is only 2 bytes. 511 */ 512 if (NET_IP_ALIGN) 513 *p_hlen = *p_nhoff + offset; 514 } 515 } else { /* version 1, must be PPTP */ 516 u8 _ppp_hdr[PPP_HDRLEN]; 517 u8 *ppp_hdr; 518 519 if (hdr->flags & GRE_ACK) 520 offset += FIELD_SIZEOF(struct pptp_gre_header, ack); 521 522 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset, 523 sizeof(_ppp_hdr), 524 data, *p_hlen, _ppp_hdr); 525 if (!ppp_hdr) 526 return FLOW_DISSECT_RET_OUT_BAD; 527 528 switch (PPP_PROTOCOL(ppp_hdr)) { 529 case PPP_IP: 530 *p_proto = htons(ETH_P_IP); 531 break; 532 case PPP_IPV6: 533 *p_proto = htons(ETH_P_IPV6); 534 break; 535 default: 536 /* Could probably catch some more like MPLS */ 537 break; 538 } 539 540 offset += PPP_HDRLEN; 541 } 542 543 *p_nhoff += offset; 544 key_control->flags |= FLOW_DIS_ENCAPSULATION; 545 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 546 return FLOW_DISSECT_RET_OUT_GOOD; 547 548 return FLOW_DISSECT_RET_PROTO_AGAIN; 549 } 550 551 /** 552 * __skb_flow_dissect_batadv() - dissect batman-adv header 553 * @skb: sk_buff to with the batman-adv header 554 * @key_control: flow dissectors control key 555 * @data: raw buffer pointer to the packet, if NULL use skb->data 556 * @p_proto: pointer used to update the protocol to process next 557 * @p_nhoff: pointer used to update inner network header offset 558 * @hlen: packet header length 559 * @flags: any combination of FLOW_DISSECTOR_F_* 560 * 561 * ETH_P_BATMAN packets are tried to be dissected. Only 562 * &struct batadv_unicast packets are actually processed because they contain an 563 * inner ethernet header and are usually followed by actual network header. This 564 * allows the flow dissector to continue processing the packet. 565 * 566 * Return: FLOW_DISSECT_RET_PROTO_AGAIN when &struct batadv_unicast was found, 567 * FLOW_DISSECT_RET_OUT_GOOD when dissector should stop after encapsulation, 568 * otherwise FLOW_DISSECT_RET_OUT_BAD 569 */ 570 static enum flow_dissect_ret 571 __skb_flow_dissect_batadv(const struct sk_buff *skb, 572 struct flow_dissector_key_control *key_control, 573 void *data, __be16 *p_proto, int *p_nhoff, int hlen, 574 unsigned int flags) 575 { 576 struct { 577 struct batadv_unicast_packet batadv_unicast; 578 struct ethhdr eth; 579 } *hdr, _hdr; 580 581 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), data, hlen, 582 &_hdr); 583 if (!hdr) 584 return FLOW_DISSECT_RET_OUT_BAD; 585 586 if (hdr->batadv_unicast.version != BATADV_COMPAT_VERSION) 587 return FLOW_DISSECT_RET_OUT_BAD; 588 589 if (hdr->batadv_unicast.packet_type != BATADV_UNICAST) 590 return FLOW_DISSECT_RET_OUT_BAD; 591 592 *p_proto = hdr->eth.h_proto; 593 *p_nhoff += sizeof(*hdr); 594 595 key_control->flags |= FLOW_DIS_ENCAPSULATION; 596 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 597 return FLOW_DISSECT_RET_OUT_GOOD; 598 599 return FLOW_DISSECT_RET_PROTO_AGAIN; 600 } 601 602 static void 603 __skb_flow_dissect_tcp(const struct sk_buff *skb, 604 struct flow_dissector *flow_dissector, 605 void *target_container, void *data, int thoff, int hlen) 606 { 607 struct flow_dissector_key_tcp *key_tcp; 608 struct tcphdr *th, _th; 609 610 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP)) 611 return; 612 613 th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th); 614 if (!th) 615 return; 616 617 if (unlikely(__tcp_hdrlen(th) < sizeof(_th))) 618 return; 619 620 key_tcp = skb_flow_dissector_target(flow_dissector, 621 FLOW_DISSECTOR_KEY_TCP, 622 target_container); 623 key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF)); 624 } 625 626 static void 627 __skb_flow_dissect_ipv4(const struct sk_buff *skb, 628 struct flow_dissector *flow_dissector, 629 void *target_container, void *data, const struct iphdr *iph) 630 { 631 struct flow_dissector_key_ip *key_ip; 632 633 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP)) 634 return; 635 636 key_ip = skb_flow_dissector_target(flow_dissector, 637 FLOW_DISSECTOR_KEY_IP, 638 target_container); 639 key_ip->tos = iph->tos; 640 key_ip->ttl = iph->ttl; 641 } 642 643 static void 644 __skb_flow_dissect_ipv6(const struct sk_buff *skb, 645 struct flow_dissector *flow_dissector, 646 void *target_container, void *data, const struct ipv6hdr *iph) 647 { 648 struct flow_dissector_key_ip *key_ip; 649 650 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP)) 651 return; 652 653 key_ip = skb_flow_dissector_target(flow_dissector, 654 FLOW_DISSECTOR_KEY_IP, 655 target_container); 656 key_ip->tos = ipv6_get_dsfield(iph); 657 key_ip->ttl = iph->hop_limit; 658 } 659 660 /* Maximum number of protocol headers that can be parsed in 661 * __skb_flow_dissect 662 */ 663 #define MAX_FLOW_DISSECT_HDRS 15 664 665 static bool skb_flow_dissect_allowed(int *num_hdrs) 666 { 667 ++*num_hdrs; 668 669 return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS); 670 } 671 672 static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys, 673 struct flow_dissector *flow_dissector, 674 void *target_container) 675 { 676 struct flow_dissector_key_control *key_control; 677 struct flow_dissector_key_basic *key_basic; 678 struct flow_dissector_key_addrs *key_addrs; 679 struct flow_dissector_key_ports *key_ports; 680 681 key_control = skb_flow_dissector_target(flow_dissector, 682 FLOW_DISSECTOR_KEY_CONTROL, 683 target_container); 684 key_control->thoff = flow_keys->thoff; 685 if (flow_keys->is_frag) 686 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 687 if (flow_keys->is_first_frag) 688 key_control->flags |= FLOW_DIS_FIRST_FRAG; 689 if (flow_keys->is_encap) 690 key_control->flags |= FLOW_DIS_ENCAPSULATION; 691 692 key_basic = skb_flow_dissector_target(flow_dissector, 693 FLOW_DISSECTOR_KEY_BASIC, 694 target_container); 695 key_basic->n_proto = flow_keys->n_proto; 696 key_basic->ip_proto = flow_keys->ip_proto; 697 698 if (flow_keys->addr_proto == ETH_P_IP && 699 dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 700 key_addrs = skb_flow_dissector_target(flow_dissector, 701 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 702 target_container); 703 key_addrs->v4addrs.src = flow_keys->ipv4_src; 704 key_addrs->v4addrs.dst = flow_keys->ipv4_dst; 705 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 706 } else if (flow_keys->addr_proto == ETH_P_IPV6 && 707 dissector_uses_key(flow_dissector, 708 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 709 key_addrs = skb_flow_dissector_target(flow_dissector, 710 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 711 target_container); 712 memcpy(&key_addrs->v6addrs, &flow_keys->ipv6_src, 713 sizeof(key_addrs->v6addrs)); 714 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 715 } 716 717 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS)) { 718 key_ports = skb_flow_dissector_target(flow_dissector, 719 FLOW_DISSECTOR_KEY_PORTS, 720 target_container); 721 key_ports->src = flow_keys->sport; 722 key_ports->dst = flow_keys->dport; 723 } 724 } 725 726 bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, 727 __be16 proto, int nhoff, int hlen) 728 { 729 struct bpf_flow_keys *flow_keys = ctx->flow_keys; 730 u32 result; 731 732 /* Pass parameters to the BPF program */ 733 memset(flow_keys, 0, sizeof(*flow_keys)); 734 flow_keys->n_proto = proto; 735 flow_keys->nhoff = nhoff; 736 flow_keys->thoff = flow_keys->nhoff; 737 738 preempt_disable(); 739 result = BPF_PROG_RUN(prog, ctx); 740 preempt_enable(); 741 742 flow_keys->nhoff = clamp_t(u16, flow_keys->nhoff, nhoff, hlen); 743 flow_keys->thoff = clamp_t(u16, flow_keys->thoff, 744 flow_keys->nhoff, hlen); 745 746 return result == BPF_OK; 747 } 748 749 /** 750 * __skb_flow_dissect - extract the flow_keys struct and return it 751 * @net: associated network namespace, derived from @skb if NULL 752 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified 753 * @flow_dissector: list of keys to dissect 754 * @target_container: target structure to put dissected values into 755 * @data: raw buffer pointer to the packet, if NULL use skb->data 756 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol 757 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb) 758 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 759 * @flags: flags that control the dissection process, e.g. 760 * FLOW_DISSECTOR_F_STOP_AT_ENCAP. 761 * 762 * The function will try to retrieve individual keys into target specified 763 * by flow_dissector from either the skbuff or a raw buffer specified by the 764 * rest parameters. 765 * 766 * Caller must take care of zeroing target container memory. 767 */ 768 bool __skb_flow_dissect(const struct net *net, 769 const struct sk_buff *skb, 770 struct flow_dissector *flow_dissector, 771 void *target_container, 772 void *data, __be16 proto, int nhoff, int hlen, 773 unsigned int flags) 774 { 775 struct flow_dissector_key_control *key_control; 776 struct flow_dissector_key_basic *key_basic; 777 struct flow_dissector_key_addrs *key_addrs; 778 struct flow_dissector_key_ports *key_ports; 779 struct flow_dissector_key_icmp *key_icmp; 780 struct flow_dissector_key_tags *key_tags; 781 struct flow_dissector_key_vlan *key_vlan; 782 struct bpf_prog *attached = NULL; 783 enum flow_dissect_ret fdret; 784 enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX; 785 int num_hdrs = 0; 786 u8 ip_proto = 0; 787 bool ret; 788 789 if (!data) { 790 data = skb->data; 791 proto = skb_vlan_tag_present(skb) ? 792 skb->vlan_proto : skb->protocol; 793 nhoff = skb_network_offset(skb); 794 hlen = skb_headlen(skb); 795 #if IS_ENABLED(CONFIG_NET_DSA) 796 if (unlikely(skb->dev && netdev_uses_dsa(skb->dev))) { 797 const struct dsa_device_ops *ops; 798 int offset; 799 800 ops = skb->dev->dsa_ptr->tag_ops; 801 if (ops->flow_dissect && 802 !ops->flow_dissect(skb, &proto, &offset)) { 803 hlen -= offset; 804 nhoff += offset; 805 } 806 } 807 #endif 808 } 809 810 /* It is ensured by skb_flow_dissector_init() that control key will 811 * be always present. 812 */ 813 key_control = skb_flow_dissector_target(flow_dissector, 814 FLOW_DISSECTOR_KEY_CONTROL, 815 target_container); 816 817 /* It is ensured by skb_flow_dissector_init() that basic key will 818 * be always present. 819 */ 820 key_basic = skb_flow_dissector_target(flow_dissector, 821 FLOW_DISSECTOR_KEY_BASIC, 822 target_container); 823 824 if (skb) { 825 if (!net) { 826 if (skb->dev) 827 net = dev_net(skb->dev); 828 else if (skb->sk) 829 net = sock_net(skb->sk); 830 } 831 } 832 833 WARN_ON_ONCE(!net); 834 if (net) { 835 rcu_read_lock(); 836 attached = rcu_dereference(net->flow_dissector_prog); 837 838 if (attached) { 839 struct bpf_flow_keys flow_keys; 840 struct bpf_flow_dissector ctx = { 841 .flow_keys = &flow_keys, 842 .data = data, 843 .data_end = data + hlen, 844 }; 845 __be16 n_proto = proto; 846 847 if (skb) { 848 ctx.skb = skb; 849 /* we can't use 'proto' in the skb case 850 * because it might be set to skb->vlan_proto 851 * which has been pulled from the data 852 */ 853 n_proto = skb->protocol; 854 } 855 856 ret = bpf_flow_dissect(attached, &ctx, n_proto, nhoff, 857 hlen); 858 __skb_flow_bpf_to_target(&flow_keys, flow_dissector, 859 target_container); 860 rcu_read_unlock(); 861 return ret; 862 } 863 rcu_read_unlock(); 864 } 865 866 if (dissector_uses_key(flow_dissector, 867 FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 868 struct ethhdr *eth = eth_hdr(skb); 869 struct flow_dissector_key_eth_addrs *key_eth_addrs; 870 871 key_eth_addrs = skb_flow_dissector_target(flow_dissector, 872 FLOW_DISSECTOR_KEY_ETH_ADDRS, 873 target_container); 874 memcpy(key_eth_addrs, ð->h_dest, sizeof(*key_eth_addrs)); 875 } 876 877 proto_again: 878 fdret = FLOW_DISSECT_RET_CONTINUE; 879 880 switch (proto) { 881 case htons(ETH_P_IP): { 882 const struct iphdr *iph; 883 struct iphdr _iph; 884 885 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 886 if (!iph || iph->ihl < 5) { 887 fdret = FLOW_DISSECT_RET_OUT_BAD; 888 break; 889 } 890 891 nhoff += iph->ihl * 4; 892 893 ip_proto = iph->protocol; 894 895 if (dissector_uses_key(flow_dissector, 896 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 897 key_addrs = skb_flow_dissector_target(flow_dissector, 898 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 899 target_container); 900 901 memcpy(&key_addrs->v4addrs, &iph->saddr, 902 sizeof(key_addrs->v4addrs)); 903 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 904 } 905 906 if (ip_is_fragment(iph)) { 907 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 908 909 if (iph->frag_off & htons(IP_OFFSET)) { 910 fdret = FLOW_DISSECT_RET_OUT_GOOD; 911 break; 912 } else { 913 key_control->flags |= FLOW_DIS_FIRST_FRAG; 914 if (!(flags & 915 FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) { 916 fdret = FLOW_DISSECT_RET_OUT_GOOD; 917 break; 918 } 919 } 920 } 921 922 __skb_flow_dissect_ipv4(skb, flow_dissector, 923 target_container, data, iph); 924 925 break; 926 } 927 case htons(ETH_P_IPV6): { 928 const struct ipv6hdr *iph; 929 struct ipv6hdr _iph; 930 931 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 932 if (!iph) { 933 fdret = FLOW_DISSECT_RET_OUT_BAD; 934 break; 935 } 936 937 ip_proto = iph->nexthdr; 938 nhoff += sizeof(struct ipv6hdr); 939 940 if (dissector_uses_key(flow_dissector, 941 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 942 key_addrs = skb_flow_dissector_target(flow_dissector, 943 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 944 target_container); 945 946 memcpy(&key_addrs->v6addrs, &iph->saddr, 947 sizeof(key_addrs->v6addrs)); 948 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 949 } 950 951 if ((dissector_uses_key(flow_dissector, 952 FLOW_DISSECTOR_KEY_FLOW_LABEL) || 953 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) && 954 ip6_flowlabel(iph)) { 955 __be32 flow_label = ip6_flowlabel(iph); 956 957 if (dissector_uses_key(flow_dissector, 958 FLOW_DISSECTOR_KEY_FLOW_LABEL)) { 959 key_tags = skb_flow_dissector_target(flow_dissector, 960 FLOW_DISSECTOR_KEY_FLOW_LABEL, 961 target_container); 962 key_tags->flow_label = ntohl(flow_label); 963 } 964 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) { 965 fdret = FLOW_DISSECT_RET_OUT_GOOD; 966 break; 967 } 968 } 969 970 __skb_flow_dissect_ipv6(skb, flow_dissector, 971 target_container, data, iph); 972 973 break; 974 } 975 case htons(ETH_P_8021AD): 976 case htons(ETH_P_8021Q): { 977 const struct vlan_hdr *vlan = NULL; 978 struct vlan_hdr _vlan; 979 __be16 saved_vlan_tpid = proto; 980 981 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX && 982 skb && skb_vlan_tag_present(skb)) { 983 proto = skb->protocol; 984 } else { 985 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), 986 data, hlen, &_vlan); 987 if (!vlan) { 988 fdret = FLOW_DISSECT_RET_OUT_BAD; 989 break; 990 } 991 992 proto = vlan->h_vlan_encapsulated_proto; 993 nhoff += sizeof(*vlan); 994 } 995 996 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX) { 997 dissector_vlan = FLOW_DISSECTOR_KEY_VLAN; 998 } else if (dissector_vlan == FLOW_DISSECTOR_KEY_VLAN) { 999 dissector_vlan = FLOW_DISSECTOR_KEY_CVLAN; 1000 } else { 1001 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1002 break; 1003 } 1004 1005 if (dissector_uses_key(flow_dissector, dissector_vlan)) { 1006 key_vlan = skb_flow_dissector_target(flow_dissector, 1007 dissector_vlan, 1008 target_container); 1009 1010 if (!vlan) { 1011 key_vlan->vlan_id = skb_vlan_tag_get_id(skb); 1012 key_vlan->vlan_priority = skb_vlan_tag_get_prio(skb); 1013 } else { 1014 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) & 1015 VLAN_VID_MASK; 1016 key_vlan->vlan_priority = 1017 (ntohs(vlan->h_vlan_TCI) & 1018 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1019 } 1020 key_vlan->vlan_tpid = saved_vlan_tpid; 1021 } 1022 1023 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1024 break; 1025 } 1026 case htons(ETH_P_PPP_SES): { 1027 struct { 1028 struct pppoe_hdr hdr; 1029 __be16 proto; 1030 } *hdr, _hdr; 1031 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 1032 if (!hdr) { 1033 fdret = FLOW_DISSECT_RET_OUT_BAD; 1034 break; 1035 } 1036 1037 proto = hdr->proto; 1038 nhoff += PPPOE_SES_HLEN; 1039 switch (proto) { 1040 case htons(PPP_IP): 1041 proto = htons(ETH_P_IP); 1042 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1043 break; 1044 case htons(PPP_IPV6): 1045 proto = htons(ETH_P_IPV6); 1046 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1047 break; 1048 default: 1049 fdret = FLOW_DISSECT_RET_OUT_BAD; 1050 break; 1051 } 1052 break; 1053 } 1054 case htons(ETH_P_TIPC): { 1055 struct tipc_basic_hdr *hdr, _hdr; 1056 1057 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), 1058 data, hlen, &_hdr); 1059 if (!hdr) { 1060 fdret = FLOW_DISSECT_RET_OUT_BAD; 1061 break; 1062 } 1063 1064 if (dissector_uses_key(flow_dissector, 1065 FLOW_DISSECTOR_KEY_TIPC)) { 1066 key_addrs = skb_flow_dissector_target(flow_dissector, 1067 FLOW_DISSECTOR_KEY_TIPC, 1068 target_container); 1069 key_addrs->tipckey.key = tipc_hdr_rps_key(hdr); 1070 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC; 1071 } 1072 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1073 break; 1074 } 1075 1076 case htons(ETH_P_MPLS_UC): 1077 case htons(ETH_P_MPLS_MC): 1078 fdret = __skb_flow_dissect_mpls(skb, flow_dissector, 1079 target_container, data, 1080 nhoff, hlen); 1081 break; 1082 case htons(ETH_P_FCOE): 1083 if ((hlen - nhoff) < FCOE_HEADER_LEN) { 1084 fdret = FLOW_DISSECT_RET_OUT_BAD; 1085 break; 1086 } 1087 1088 nhoff += FCOE_HEADER_LEN; 1089 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1090 break; 1091 1092 case htons(ETH_P_ARP): 1093 case htons(ETH_P_RARP): 1094 fdret = __skb_flow_dissect_arp(skb, flow_dissector, 1095 target_container, data, 1096 nhoff, hlen); 1097 break; 1098 1099 case htons(ETH_P_BATMAN): 1100 fdret = __skb_flow_dissect_batadv(skb, key_control, data, 1101 &proto, &nhoff, hlen, flags); 1102 break; 1103 1104 default: 1105 fdret = FLOW_DISSECT_RET_OUT_BAD; 1106 break; 1107 } 1108 1109 /* Process result of proto processing */ 1110 switch (fdret) { 1111 case FLOW_DISSECT_RET_OUT_GOOD: 1112 goto out_good; 1113 case FLOW_DISSECT_RET_PROTO_AGAIN: 1114 if (skb_flow_dissect_allowed(&num_hdrs)) 1115 goto proto_again; 1116 goto out_good; 1117 case FLOW_DISSECT_RET_CONTINUE: 1118 case FLOW_DISSECT_RET_IPPROTO_AGAIN: 1119 break; 1120 case FLOW_DISSECT_RET_OUT_BAD: 1121 default: 1122 goto out_bad; 1123 } 1124 1125 ip_proto_again: 1126 fdret = FLOW_DISSECT_RET_CONTINUE; 1127 1128 switch (ip_proto) { 1129 case IPPROTO_GRE: 1130 fdret = __skb_flow_dissect_gre(skb, key_control, flow_dissector, 1131 target_container, data, 1132 &proto, &nhoff, &hlen, flags); 1133 break; 1134 1135 case NEXTHDR_HOP: 1136 case NEXTHDR_ROUTING: 1137 case NEXTHDR_DEST: { 1138 u8 _opthdr[2], *opthdr; 1139 1140 if (proto != htons(ETH_P_IPV6)) 1141 break; 1142 1143 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr), 1144 data, hlen, &_opthdr); 1145 if (!opthdr) { 1146 fdret = FLOW_DISSECT_RET_OUT_BAD; 1147 break; 1148 } 1149 1150 ip_proto = opthdr[0]; 1151 nhoff += (opthdr[1] + 1) << 3; 1152 1153 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN; 1154 break; 1155 } 1156 case NEXTHDR_FRAGMENT: { 1157 struct frag_hdr _fh, *fh; 1158 1159 if (proto != htons(ETH_P_IPV6)) 1160 break; 1161 1162 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh), 1163 data, hlen, &_fh); 1164 1165 if (!fh) { 1166 fdret = FLOW_DISSECT_RET_OUT_BAD; 1167 break; 1168 } 1169 1170 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 1171 1172 nhoff += sizeof(_fh); 1173 ip_proto = fh->nexthdr; 1174 1175 if (!(fh->frag_off & htons(IP6_OFFSET))) { 1176 key_control->flags |= FLOW_DIS_FIRST_FRAG; 1177 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) { 1178 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN; 1179 break; 1180 } 1181 } 1182 1183 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1184 break; 1185 } 1186 case IPPROTO_IPIP: 1187 proto = htons(ETH_P_IP); 1188 1189 key_control->flags |= FLOW_DIS_ENCAPSULATION; 1190 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) { 1191 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1192 break; 1193 } 1194 1195 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1196 break; 1197 1198 case IPPROTO_IPV6: 1199 proto = htons(ETH_P_IPV6); 1200 1201 key_control->flags |= FLOW_DIS_ENCAPSULATION; 1202 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) { 1203 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1204 break; 1205 } 1206 1207 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1208 break; 1209 1210 1211 case IPPROTO_MPLS: 1212 proto = htons(ETH_P_MPLS_UC); 1213 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1214 break; 1215 1216 case IPPROTO_TCP: 1217 __skb_flow_dissect_tcp(skb, flow_dissector, target_container, 1218 data, nhoff, hlen); 1219 break; 1220 1221 default: 1222 break; 1223 } 1224 1225 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS) && 1226 !(key_control->flags & FLOW_DIS_IS_FRAGMENT)) { 1227 key_ports = skb_flow_dissector_target(flow_dissector, 1228 FLOW_DISSECTOR_KEY_PORTS, 1229 target_container); 1230 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto, 1231 data, hlen); 1232 } 1233 1234 if (dissector_uses_key(flow_dissector, 1235 FLOW_DISSECTOR_KEY_ICMP)) { 1236 key_icmp = skb_flow_dissector_target(flow_dissector, 1237 FLOW_DISSECTOR_KEY_ICMP, 1238 target_container); 1239 key_icmp->icmp = skb_flow_get_be16(skb, nhoff, data, hlen); 1240 } 1241 1242 /* Process result of IP proto processing */ 1243 switch (fdret) { 1244 case FLOW_DISSECT_RET_PROTO_AGAIN: 1245 if (skb_flow_dissect_allowed(&num_hdrs)) 1246 goto proto_again; 1247 break; 1248 case FLOW_DISSECT_RET_IPPROTO_AGAIN: 1249 if (skb_flow_dissect_allowed(&num_hdrs)) 1250 goto ip_proto_again; 1251 break; 1252 case FLOW_DISSECT_RET_OUT_GOOD: 1253 case FLOW_DISSECT_RET_CONTINUE: 1254 break; 1255 case FLOW_DISSECT_RET_OUT_BAD: 1256 default: 1257 goto out_bad; 1258 } 1259 1260 out_good: 1261 ret = true; 1262 1263 out: 1264 key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen); 1265 key_basic->n_proto = proto; 1266 key_basic->ip_proto = ip_proto; 1267 1268 return ret; 1269 1270 out_bad: 1271 ret = false; 1272 goto out; 1273 } 1274 EXPORT_SYMBOL(__skb_flow_dissect); 1275 1276 static u32 hashrnd __read_mostly; 1277 static __always_inline void __flow_hash_secret_init(void) 1278 { 1279 net_get_random_once(&hashrnd, sizeof(hashrnd)); 1280 } 1281 1282 static __always_inline u32 __flow_hash_words(const u32 *words, u32 length, 1283 u32 keyval) 1284 { 1285 return jhash2(words, length, keyval); 1286 } 1287 1288 static inline const u32 *flow_keys_hash_start(const struct flow_keys *flow) 1289 { 1290 const void *p = flow; 1291 1292 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32)); 1293 return (const u32 *)(p + FLOW_KEYS_HASH_OFFSET); 1294 } 1295 1296 static inline size_t flow_keys_hash_length(const struct flow_keys *flow) 1297 { 1298 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs); 1299 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32)); 1300 BUILD_BUG_ON(offsetof(typeof(*flow), addrs) != 1301 sizeof(*flow) - sizeof(flow->addrs)); 1302 1303 switch (flow->control.addr_type) { 1304 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1305 diff -= sizeof(flow->addrs.v4addrs); 1306 break; 1307 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1308 diff -= sizeof(flow->addrs.v6addrs); 1309 break; 1310 case FLOW_DISSECTOR_KEY_TIPC: 1311 diff -= sizeof(flow->addrs.tipckey); 1312 break; 1313 } 1314 return (sizeof(*flow) - diff) / sizeof(u32); 1315 } 1316 1317 __be32 flow_get_u32_src(const struct flow_keys *flow) 1318 { 1319 switch (flow->control.addr_type) { 1320 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1321 return flow->addrs.v4addrs.src; 1322 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1323 return (__force __be32)ipv6_addr_hash( 1324 &flow->addrs.v6addrs.src); 1325 case FLOW_DISSECTOR_KEY_TIPC: 1326 return flow->addrs.tipckey.key; 1327 default: 1328 return 0; 1329 } 1330 } 1331 EXPORT_SYMBOL(flow_get_u32_src); 1332 1333 __be32 flow_get_u32_dst(const struct flow_keys *flow) 1334 { 1335 switch (flow->control.addr_type) { 1336 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1337 return flow->addrs.v4addrs.dst; 1338 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1339 return (__force __be32)ipv6_addr_hash( 1340 &flow->addrs.v6addrs.dst); 1341 default: 1342 return 0; 1343 } 1344 } 1345 EXPORT_SYMBOL(flow_get_u32_dst); 1346 1347 static inline void __flow_hash_consistentify(struct flow_keys *keys) 1348 { 1349 int addr_diff, i; 1350 1351 switch (keys->control.addr_type) { 1352 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1353 addr_diff = (__force u32)keys->addrs.v4addrs.dst - 1354 (__force u32)keys->addrs.v4addrs.src; 1355 if ((addr_diff < 0) || 1356 (addr_diff == 0 && 1357 ((__force u16)keys->ports.dst < 1358 (__force u16)keys->ports.src))) { 1359 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst); 1360 swap(keys->ports.src, keys->ports.dst); 1361 } 1362 break; 1363 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1364 addr_diff = memcmp(&keys->addrs.v6addrs.dst, 1365 &keys->addrs.v6addrs.src, 1366 sizeof(keys->addrs.v6addrs.dst)); 1367 if ((addr_diff < 0) || 1368 (addr_diff == 0 && 1369 ((__force u16)keys->ports.dst < 1370 (__force u16)keys->ports.src))) { 1371 for (i = 0; i < 4; i++) 1372 swap(keys->addrs.v6addrs.src.s6_addr32[i], 1373 keys->addrs.v6addrs.dst.s6_addr32[i]); 1374 swap(keys->ports.src, keys->ports.dst); 1375 } 1376 break; 1377 } 1378 } 1379 1380 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) 1381 { 1382 u32 hash; 1383 1384 __flow_hash_consistentify(keys); 1385 1386 hash = __flow_hash_words(flow_keys_hash_start(keys), 1387 flow_keys_hash_length(keys), keyval); 1388 if (!hash) 1389 hash = 1; 1390 1391 return hash; 1392 } 1393 1394 u32 flow_hash_from_keys(struct flow_keys *keys) 1395 { 1396 __flow_hash_secret_init(); 1397 return __flow_hash_from_keys(keys, hashrnd); 1398 } 1399 EXPORT_SYMBOL(flow_hash_from_keys); 1400 1401 static inline u32 ___skb_get_hash(const struct sk_buff *skb, 1402 struct flow_keys *keys, u32 keyval) 1403 { 1404 skb_flow_dissect_flow_keys(skb, keys, 1405 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 1406 1407 return __flow_hash_from_keys(keys, keyval); 1408 } 1409 1410 struct _flow_keys_digest_data { 1411 __be16 n_proto; 1412 u8 ip_proto; 1413 u8 padding; 1414 __be32 ports; 1415 __be32 src; 1416 __be32 dst; 1417 }; 1418 1419 void make_flow_keys_digest(struct flow_keys_digest *digest, 1420 const struct flow_keys *flow) 1421 { 1422 struct _flow_keys_digest_data *data = 1423 (struct _flow_keys_digest_data *)digest; 1424 1425 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest)); 1426 1427 memset(digest, 0, sizeof(*digest)); 1428 1429 data->n_proto = flow->basic.n_proto; 1430 data->ip_proto = flow->basic.ip_proto; 1431 data->ports = flow->ports.ports; 1432 data->src = flow->addrs.v4addrs.src; 1433 data->dst = flow->addrs.v4addrs.dst; 1434 } 1435 EXPORT_SYMBOL(make_flow_keys_digest); 1436 1437 static struct flow_dissector flow_keys_dissector_symmetric __read_mostly; 1438 1439 u32 __skb_get_hash_symmetric(const struct sk_buff *skb) 1440 { 1441 struct flow_keys keys; 1442 1443 __flow_hash_secret_init(); 1444 1445 memset(&keys, 0, sizeof(keys)); 1446 __skb_flow_dissect(NULL, skb, &flow_keys_dissector_symmetric, 1447 &keys, NULL, 0, 0, 0, 1448 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 1449 1450 return __flow_hash_from_keys(&keys, hashrnd); 1451 } 1452 EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric); 1453 1454 /** 1455 * __skb_get_hash: calculate a flow hash 1456 * @skb: sk_buff to calculate flow hash from 1457 * 1458 * This function calculates a flow hash based on src/dst addresses 1459 * and src/dst port numbers. Sets hash in skb to non-zero hash value 1460 * on success, zero indicates no valid hash. Also, sets l4_hash in skb 1461 * if hash is a canonical 4-tuple hash over transport ports. 1462 */ 1463 void __skb_get_hash(struct sk_buff *skb) 1464 { 1465 struct flow_keys keys; 1466 u32 hash; 1467 1468 __flow_hash_secret_init(); 1469 1470 hash = ___skb_get_hash(skb, &keys, hashrnd); 1471 1472 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys)); 1473 } 1474 EXPORT_SYMBOL(__skb_get_hash); 1475 1476 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb) 1477 { 1478 struct flow_keys keys; 1479 1480 return ___skb_get_hash(skb, &keys, perturb); 1481 } 1482 EXPORT_SYMBOL(skb_get_hash_perturb); 1483 1484 u32 __skb_get_poff(const struct sk_buff *skb, void *data, 1485 const struct flow_keys_basic *keys, int hlen) 1486 { 1487 u32 poff = keys->control.thoff; 1488 1489 /* skip L4 headers for fragments after the first */ 1490 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) && 1491 !(keys->control.flags & FLOW_DIS_FIRST_FRAG)) 1492 return poff; 1493 1494 switch (keys->basic.ip_proto) { 1495 case IPPROTO_TCP: { 1496 /* access doff as u8 to avoid unaligned access */ 1497 const u8 *doff; 1498 u8 _doff; 1499 1500 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff), 1501 data, hlen, &_doff); 1502 if (!doff) 1503 return poff; 1504 1505 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2); 1506 break; 1507 } 1508 case IPPROTO_UDP: 1509 case IPPROTO_UDPLITE: 1510 poff += sizeof(struct udphdr); 1511 break; 1512 /* For the rest, we do not really care about header 1513 * extensions at this point for now. 1514 */ 1515 case IPPROTO_ICMP: 1516 poff += sizeof(struct icmphdr); 1517 break; 1518 case IPPROTO_ICMPV6: 1519 poff += sizeof(struct icmp6hdr); 1520 break; 1521 case IPPROTO_IGMP: 1522 poff += sizeof(struct igmphdr); 1523 break; 1524 case IPPROTO_DCCP: 1525 poff += sizeof(struct dccp_hdr); 1526 break; 1527 case IPPROTO_SCTP: 1528 poff += sizeof(struct sctphdr); 1529 break; 1530 } 1531 1532 return poff; 1533 } 1534 1535 /** 1536 * skb_get_poff - get the offset to the payload 1537 * @skb: sk_buff to get the payload offset from 1538 * 1539 * The function will get the offset to the payload as far as it could 1540 * be dissected. The main user is currently BPF, so that we can dynamically 1541 * truncate packets without needing to push actual payload to the user 1542 * space and can analyze headers only, instead. 1543 */ 1544 u32 skb_get_poff(const struct sk_buff *skb) 1545 { 1546 struct flow_keys_basic keys; 1547 1548 if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys, 1549 NULL, 0, 0, 0, 0)) 1550 return 0; 1551 1552 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); 1553 } 1554 1555 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys) 1556 { 1557 memset(keys, 0, sizeof(*keys)); 1558 1559 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr, 1560 sizeof(keys->addrs.v6addrs.src)); 1561 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr, 1562 sizeof(keys->addrs.v6addrs.dst)); 1563 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1564 keys->ports.src = fl6->fl6_sport; 1565 keys->ports.dst = fl6->fl6_dport; 1566 keys->keyid.keyid = fl6->fl6_gre_key; 1567 keys->tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 1568 keys->basic.ip_proto = fl6->flowi6_proto; 1569 1570 return flow_hash_from_keys(keys); 1571 } 1572 EXPORT_SYMBOL(__get_hash_from_flowi6); 1573 1574 static const struct flow_dissector_key flow_keys_dissector_keys[] = { 1575 { 1576 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1577 .offset = offsetof(struct flow_keys, control), 1578 }, 1579 { 1580 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1581 .offset = offsetof(struct flow_keys, basic), 1582 }, 1583 { 1584 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1585 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1586 }, 1587 { 1588 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1589 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1590 }, 1591 { 1592 .key_id = FLOW_DISSECTOR_KEY_TIPC, 1593 .offset = offsetof(struct flow_keys, addrs.tipckey), 1594 }, 1595 { 1596 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1597 .offset = offsetof(struct flow_keys, ports), 1598 }, 1599 { 1600 .key_id = FLOW_DISSECTOR_KEY_VLAN, 1601 .offset = offsetof(struct flow_keys, vlan), 1602 }, 1603 { 1604 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL, 1605 .offset = offsetof(struct flow_keys, tags), 1606 }, 1607 { 1608 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID, 1609 .offset = offsetof(struct flow_keys, keyid), 1610 }, 1611 }; 1612 1613 static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = { 1614 { 1615 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1616 .offset = offsetof(struct flow_keys, control), 1617 }, 1618 { 1619 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1620 .offset = offsetof(struct flow_keys, basic), 1621 }, 1622 { 1623 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1624 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1625 }, 1626 { 1627 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1628 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1629 }, 1630 { 1631 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1632 .offset = offsetof(struct flow_keys, ports), 1633 }, 1634 }; 1635 1636 static const struct flow_dissector_key flow_keys_basic_dissector_keys[] = { 1637 { 1638 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1639 .offset = offsetof(struct flow_keys, control), 1640 }, 1641 { 1642 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1643 .offset = offsetof(struct flow_keys, basic), 1644 }, 1645 }; 1646 1647 struct flow_dissector flow_keys_dissector __read_mostly; 1648 EXPORT_SYMBOL(flow_keys_dissector); 1649 1650 struct flow_dissector flow_keys_basic_dissector __read_mostly; 1651 EXPORT_SYMBOL(flow_keys_basic_dissector); 1652 1653 static int __init init_default_flow_dissectors(void) 1654 { 1655 skb_flow_dissector_init(&flow_keys_dissector, 1656 flow_keys_dissector_keys, 1657 ARRAY_SIZE(flow_keys_dissector_keys)); 1658 skb_flow_dissector_init(&flow_keys_dissector_symmetric, 1659 flow_keys_dissector_symmetric_keys, 1660 ARRAY_SIZE(flow_keys_dissector_symmetric_keys)); 1661 skb_flow_dissector_init(&flow_keys_basic_dissector, 1662 flow_keys_basic_dissector_keys, 1663 ARRAY_SIZE(flow_keys_basic_dissector_keys)); 1664 return 0; 1665 } 1666 1667 core_initcall(init_default_flow_dissectors); 1668