1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2020 Facebook */ 3 4 #include <stddef.h> 5 #include <errno.h> 6 #include <stdbool.h> 7 #include <sys/types.h> 8 #include <sys/socket.h> 9 #include <linux/tcp.h> 10 #include <linux/socket.h> 11 #include <linux/bpf.h> 12 #include <linux/types.h> 13 #include <bpf/bpf_helpers.h> 14 #include <bpf/bpf_endian.h> 15 #define BPF_PROG_TEST_TCP_HDR_OPTIONS 16 #include "test_tcp_hdr_options.h" 17 18 #ifndef sizeof_field 19 #define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) 20 #endif 21 22 __u8 test_kind = TCPOPT_EXP; 23 __u16 test_magic = 0xeB9F; 24 __u32 inherit_cb_flags = 0; 25 26 struct bpf_test_option passive_synack_out = {}; 27 struct bpf_test_option passive_fin_out = {}; 28 29 struct bpf_test_option passive_estab_in = {}; 30 struct bpf_test_option passive_fin_in = {}; 31 32 struct bpf_test_option active_syn_out = {}; 33 struct bpf_test_option active_fin_out = {}; 34 35 struct bpf_test_option active_estab_in = {}; 36 struct bpf_test_option active_fin_in = {}; 37 38 struct { 39 __uint(type, BPF_MAP_TYPE_SK_STORAGE); 40 __uint(map_flags, BPF_F_NO_PREALLOC); 41 __type(key, int); 42 __type(value, struct hdr_stg); 43 } hdr_stg_map SEC(".maps"); 44 45 static bool skops_want_cookie(const struct bpf_sock_ops *skops) 46 { 47 return skops->args[0] == BPF_WRITE_HDR_TCP_SYNACK_COOKIE; 48 } 49 50 static bool skops_current_mss(const struct bpf_sock_ops *skops) 51 { 52 return skops->args[0] == BPF_WRITE_HDR_TCP_CURRENT_MSS; 53 } 54 55 static __u8 option_total_len(__u8 flags) 56 { 57 __u8 i, len = 1; /* +1 for flags */ 58 59 if (!flags) 60 return 0; 61 62 /* RESEND bit does not use a byte */ 63 for (i = OPTION_RESEND + 1; i < __NR_OPTION_FLAGS; i++) 64 len += !!TEST_OPTION_FLAGS(flags, i); 65 66 if (test_kind == TCPOPT_EXP) 67 return len + TCP_BPF_EXPOPT_BASE_LEN; 68 else 69 return len + 2; /* +1 kind, +1 kind-len */ 70 } 71 72 static void write_test_option(const struct bpf_test_option *test_opt, 73 __u8 *data) 74 { 75 __u8 offset = 0; 76 77 data[offset++] = test_opt->flags; 78 if (TEST_OPTION_FLAGS(test_opt->flags, OPTION_MAX_DELACK_MS)) 79 data[offset++] = test_opt->max_delack_ms; 80 81 if (TEST_OPTION_FLAGS(test_opt->flags, OPTION_RAND)) 82 data[offset++] = test_opt->rand; 83 } 84 85 static int store_option(struct bpf_sock_ops *skops, 86 const struct bpf_test_option *test_opt) 87 { 88 union { 89 struct tcp_exprm_opt exprm; 90 struct tcp_opt regular; 91 } write_opt; 92 int err; 93 94 if (test_kind == TCPOPT_EXP) { 95 write_opt.exprm.kind = TCPOPT_EXP; 96 write_opt.exprm.len = option_total_len(test_opt->flags); 97 write_opt.exprm.magic = __bpf_htons(test_magic); 98 write_opt.exprm.data32 = 0; 99 write_test_option(test_opt, write_opt.exprm.data); 100 err = bpf_store_hdr_opt(skops, &write_opt.exprm, 101 sizeof(write_opt.exprm), 0); 102 } else { 103 write_opt.regular.kind = test_kind; 104 write_opt.regular.len = option_total_len(test_opt->flags); 105 write_opt.regular.data32 = 0; 106 write_test_option(test_opt, write_opt.regular.data); 107 err = bpf_store_hdr_opt(skops, &write_opt.regular, 108 sizeof(write_opt.regular), 0); 109 } 110 111 if (err) 112 RET_CG_ERR(err); 113 114 return CG_OK; 115 } 116 117 static int parse_test_option(struct bpf_test_option *opt, const __u8 *start) 118 { 119 opt->flags = *start++; 120 121 if (TEST_OPTION_FLAGS(opt->flags, OPTION_MAX_DELACK_MS)) 122 opt->max_delack_ms = *start++; 123 124 if (TEST_OPTION_FLAGS(opt->flags, OPTION_RAND)) 125 opt->rand = *start++; 126 127 return 0; 128 } 129 130 static int load_option(struct bpf_sock_ops *skops, 131 struct bpf_test_option *test_opt, bool from_syn) 132 { 133 union { 134 struct tcp_exprm_opt exprm; 135 struct tcp_opt regular; 136 } search_opt; 137 int ret, load_flags = from_syn ? BPF_LOAD_HDR_OPT_TCP_SYN : 0; 138 139 if (test_kind == TCPOPT_EXP) { 140 search_opt.exprm.kind = TCPOPT_EXP; 141 search_opt.exprm.len = 4; 142 search_opt.exprm.magic = __bpf_htons(test_magic); 143 search_opt.exprm.data32 = 0; 144 ret = bpf_load_hdr_opt(skops, &search_opt.exprm, 145 sizeof(search_opt.exprm), load_flags); 146 if (ret < 0) 147 return ret; 148 return parse_test_option(test_opt, search_opt.exprm.data); 149 } else { 150 search_opt.regular.kind = test_kind; 151 search_opt.regular.len = 0; 152 search_opt.regular.data32 = 0; 153 ret = bpf_load_hdr_opt(skops, &search_opt.regular, 154 sizeof(search_opt.regular), load_flags); 155 if (ret < 0) 156 return ret; 157 return parse_test_option(test_opt, search_opt.regular.data); 158 } 159 } 160 161 static int synack_opt_len(struct bpf_sock_ops *skops) 162 { 163 struct bpf_test_option test_opt = {}; 164 __u8 optlen; 165 int err; 166 167 if (!passive_synack_out.flags) 168 return CG_OK; 169 170 err = load_option(skops, &test_opt, true); 171 172 /* bpf_test_option is not found */ 173 if (err == -ENOMSG) 174 return CG_OK; 175 176 if (err) 177 RET_CG_ERR(err); 178 179 optlen = option_total_len(passive_synack_out.flags); 180 if (optlen) { 181 err = bpf_reserve_hdr_opt(skops, optlen, 0); 182 if (err) 183 RET_CG_ERR(err); 184 } 185 186 return CG_OK; 187 } 188 189 static int write_synack_opt(struct bpf_sock_ops *skops) 190 { 191 struct bpf_test_option opt; 192 193 if (!passive_synack_out.flags) 194 /* We should not even be called since no header 195 * space has been reserved. 196 */ 197 RET_CG_ERR(0); 198 199 opt = passive_synack_out; 200 if (skops_want_cookie(skops)) 201 SET_OPTION_FLAGS(opt.flags, OPTION_RESEND); 202 203 return store_option(skops, &opt); 204 } 205 206 static int syn_opt_len(struct bpf_sock_ops *skops) 207 { 208 __u8 optlen; 209 int err; 210 211 if (!active_syn_out.flags) 212 return CG_OK; 213 214 optlen = option_total_len(active_syn_out.flags); 215 if (optlen) { 216 err = bpf_reserve_hdr_opt(skops, optlen, 0); 217 if (err) 218 RET_CG_ERR(err); 219 } 220 221 return CG_OK; 222 } 223 224 static int write_syn_opt(struct bpf_sock_ops *skops) 225 { 226 if (!active_syn_out.flags) 227 RET_CG_ERR(0); 228 229 return store_option(skops, &active_syn_out); 230 } 231 232 static int fin_opt_len(struct bpf_sock_ops *skops) 233 { 234 struct bpf_test_option *opt; 235 struct hdr_stg *hdr_stg; 236 __u8 optlen; 237 int err; 238 239 if (!skops->sk) 240 RET_CG_ERR(0); 241 242 hdr_stg = bpf_sk_storage_get(&hdr_stg_map, skops->sk, NULL, 0); 243 if (!hdr_stg) 244 RET_CG_ERR(0); 245 246 if (hdr_stg->active) 247 opt = &active_fin_out; 248 else 249 opt = &passive_fin_out; 250 251 optlen = option_total_len(opt->flags); 252 if (optlen) { 253 err = bpf_reserve_hdr_opt(skops, optlen, 0); 254 if (err) 255 RET_CG_ERR(err); 256 } 257 258 return CG_OK; 259 } 260 261 static int write_fin_opt(struct bpf_sock_ops *skops) 262 { 263 struct bpf_test_option *opt; 264 struct hdr_stg *hdr_stg; 265 266 if (!skops->sk) 267 RET_CG_ERR(0); 268 269 hdr_stg = bpf_sk_storage_get(&hdr_stg_map, skops->sk, NULL, 0); 270 if (!hdr_stg) 271 RET_CG_ERR(0); 272 273 if (hdr_stg->active) 274 opt = &active_fin_out; 275 else 276 opt = &passive_fin_out; 277 278 if (!opt->flags) 279 RET_CG_ERR(0); 280 281 return store_option(skops, opt); 282 } 283 284 static int resend_in_ack(struct bpf_sock_ops *skops) 285 { 286 struct hdr_stg *hdr_stg; 287 288 if (!skops->sk) 289 return -1; 290 291 hdr_stg = bpf_sk_storage_get(&hdr_stg_map, skops->sk, NULL, 0); 292 if (!hdr_stg) 293 return -1; 294 295 return !!hdr_stg->resend_syn; 296 } 297 298 static int nodata_opt_len(struct bpf_sock_ops *skops) 299 { 300 int resend; 301 302 resend = resend_in_ack(skops); 303 if (resend < 0) 304 RET_CG_ERR(0); 305 306 if (resend) 307 return syn_opt_len(skops); 308 309 return CG_OK; 310 } 311 312 static int write_nodata_opt(struct bpf_sock_ops *skops) 313 { 314 int resend; 315 316 resend = resend_in_ack(skops); 317 if (resend < 0) 318 RET_CG_ERR(0); 319 320 if (resend) 321 return write_syn_opt(skops); 322 323 return CG_OK; 324 } 325 326 static int data_opt_len(struct bpf_sock_ops *skops) 327 { 328 /* Same as the nodata version. Mostly to show 329 * an example usage on skops->skb_len. 330 */ 331 return nodata_opt_len(skops); 332 } 333 334 static int write_data_opt(struct bpf_sock_ops *skops) 335 { 336 return write_nodata_opt(skops); 337 } 338 339 static int current_mss_opt_len(struct bpf_sock_ops *skops) 340 { 341 /* Reserve maximum that may be needed */ 342 int err; 343 344 err = bpf_reserve_hdr_opt(skops, option_total_len(OPTION_MASK), 0); 345 if (err) 346 RET_CG_ERR(err); 347 348 return CG_OK; 349 } 350 351 static int handle_hdr_opt_len(struct bpf_sock_ops *skops) 352 { 353 __u8 tcp_flags = skops_tcp_flags(skops); 354 355 if ((tcp_flags & TCPHDR_SYNACK) == TCPHDR_SYNACK) 356 return synack_opt_len(skops); 357 358 if (tcp_flags & TCPHDR_SYN) 359 return syn_opt_len(skops); 360 361 if (tcp_flags & TCPHDR_FIN) 362 return fin_opt_len(skops); 363 364 if (skops_current_mss(skops)) 365 /* The kernel is calculating the MSS */ 366 return current_mss_opt_len(skops); 367 368 if (skops->skb_len) 369 return data_opt_len(skops); 370 371 return nodata_opt_len(skops); 372 } 373 374 static int handle_write_hdr_opt(struct bpf_sock_ops *skops) 375 { 376 __u8 tcp_flags = skops_tcp_flags(skops); 377 struct tcphdr *th; 378 379 if ((tcp_flags & TCPHDR_SYNACK) == TCPHDR_SYNACK) 380 return write_synack_opt(skops); 381 382 if (tcp_flags & TCPHDR_SYN) 383 return write_syn_opt(skops); 384 385 if (tcp_flags & TCPHDR_FIN) 386 return write_fin_opt(skops); 387 388 th = skops->skb_data; 389 if (th + 1 > skops->skb_data_end) 390 RET_CG_ERR(0); 391 392 if (skops->skb_len > tcp_hdrlen(th)) 393 return write_data_opt(skops); 394 395 return write_nodata_opt(skops); 396 } 397 398 static int set_delack_max(struct bpf_sock_ops *skops, __u8 max_delack_ms) 399 { 400 __u32 max_delack_us = max_delack_ms * 1000; 401 402 return bpf_setsockopt(skops, SOL_TCP, TCP_BPF_DELACK_MAX, 403 &max_delack_us, sizeof(max_delack_us)); 404 } 405 406 static int set_rto_min(struct bpf_sock_ops *skops, __u8 peer_max_delack_ms) 407 { 408 __u32 min_rto_us = peer_max_delack_ms * 1000; 409 410 return bpf_setsockopt(skops, SOL_TCP, TCP_BPF_RTO_MIN, &min_rto_us, 411 sizeof(min_rto_us)); 412 } 413 414 static int handle_active_estab(struct bpf_sock_ops *skops) 415 { 416 struct hdr_stg init_stg = { 417 .active = true, 418 }; 419 int err; 420 421 err = load_option(skops, &active_estab_in, false); 422 if (err && err != -ENOMSG) 423 RET_CG_ERR(err); 424 425 init_stg.resend_syn = TEST_OPTION_FLAGS(active_estab_in.flags, 426 OPTION_RESEND); 427 if (!skops->sk || !bpf_sk_storage_get(&hdr_stg_map, skops->sk, 428 &init_stg, 429 BPF_SK_STORAGE_GET_F_CREATE)) 430 RET_CG_ERR(0); 431 432 if (init_stg.resend_syn) 433 /* Don't clear the write_hdr cb now because 434 * the ACK may get lost and retransmit may 435 * be needed. 436 * 437 * PARSE_ALL_HDR cb flag is set to learn if this 438 * resend_syn option has received by the peer. 439 * 440 * The header option will be resent until a valid 441 * packet is received at handle_parse_hdr() 442 * and all hdr cb flags will be cleared in 443 * handle_parse_hdr(). 444 */ 445 set_parse_all_hdr_cb_flags(skops); 446 else if (!active_fin_out.flags) 447 /* No options will be written from now */ 448 clear_hdr_cb_flags(skops); 449 450 if (active_syn_out.max_delack_ms) { 451 err = set_delack_max(skops, active_syn_out.max_delack_ms); 452 if (err) 453 RET_CG_ERR(err); 454 } 455 456 if (active_estab_in.max_delack_ms) { 457 err = set_rto_min(skops, active_estab_in.max_delack_ms); 458 if (err) 459 RET_CG_ERR(err); 460 } 461 462 return CG_OK; 463 } 464 465 static int handle_passive_estab(struct bpf_sock_ops *skops) 466 { 467 struct hdr_stg init_stg = {}; 468 struct tcphdr *th; 469 int err; 470 471 inherit_cb_flags = skops->bpf_sock_ops_cb_flags; 472 473 err = load_option(skops, &passive_estab_in, true); 474 if (err == -ENOENT) { 475 /* saved_syn is not found. It was in syncookie mode. 476 * We have asked the active side to resend the options 477 * in ACK, so try to find the bpf_test_option from ACK now. 478 */ 479 err = load_option(skops, &passive_estab_in, false); 480 init_stg.syncookie = true; 481 } 482 483 /* ENOMSG: The bpf_test_option is not found which is fine. 484 * Bail out now for all other errors. 485 */ 486 if (err && err != -ENOMSG) 487 RET_CG_ERR(err); 488 489 th = skops->skb_data; 490 if (th + 1 > skops->skb_data_end) 491 RET_CG_ERR(0); 492 493 if (th->syn) { 494 /* Fastopen */ 495 496 /* Cannot clear cb_flags to stop write_hdr cb. 497 * synack is not sent yet for fast open. 498 * Even it was, the synack may need to be retransmitted. 499 * 500 * PARSE_ALL_HDR cb flag is set to learn 501 * if synack has reached the peer. 502 * All cb_flags will be cleared in handle_parse_hdr(). 503 */ 504 set_parse_all_hdr_cb_flags(skops); 505 init_stg.fastopen = true; 506 } else if (!passive_fin_out.flags) { 507 /* No options will be written from now */ 508 clear_hdr_cb_flags(skops); 509 } 510 511 if (!skops->sk || 512 !bpf_sk_storage_get(&hdr_stg_map, skops->sk, &init_stg, 513 BPF_SK_STORAGE_GET_F_CREATE)) 514 RET_CG_ERR(0); 515 516 if (passive_synack_out.max_delack_ms) { 517 err = set_delack_max(skops, passive_synack_out.max_delack_ms); 518 if (err) 519 RET_CG_ERR(err); 520 } 521 522 if (passive_estab_in.max_delack_ms) { 523 err = set_rto_min(skops, passive_estab_in.max_delack_ms); 524 if (err) 525 RET_CG_ERR(err); 526 } 527 528 return CG_OK; 529 } 530 531 static int handle_parse_hdr(struct bpf_sock_ops *skops) 532 { 533 struct hdr_stg *hdr_stg; 534 struct tcphdr *th; 535 536 if (!skops->sk) 537 RET_CG_ERR(0); 538 539 th = skops->skb_data; 540 if (th + 1 > skops->skb_data_end) 541 RET_CG_ERR(0); 542 543 hdr_stg = bpf_sk_storage_get(&hdr_stg_map, skops->sk, NULL, 0); 544 if (!hdr_stg) 545 RET_CG_ERR(0); 546 547 if (hdr_stg->resend_syn || hdr_stg->fastopen) 548 /* The PARSE_ALL_HDR cb flag was turned on 549 * to ensure that the previously written 550 * options have reached the peer. 551 * Those previously written option includes: 552 * - Active side: resend_syn in ACK during syncookie 553 * or 554 * - Passive side: SYNACK during fastopen 555 * 556 * A valid packet has been received here after 557 * the 3WHS, so the PARSE_ALL_HDR cb flag 558 * can be cleared now. 559 */ 560 clear_parse_all_hdr_cb_flags(skops); 561 562 if (hdr_stg->resend_syn && !active_fin_out.flags) 563 /* Active side resent the syn option in ACK 564 * because the server was in syncookie mode. 565 * A valid packet has been received, so 566 * clear header cb flags if there is no 567 * more option to send. 568 */ 569 clear_hdr_cb_flags(skops); 570 571 if (hdr_stg->fastopen && !passive_fin_out.flags) 572 /* Passive side was in fastopen. 573 * A valid packet has been received, so 574 * the SYNACK has reached the peer. 575 * Clear header cb flags if there is no more 576 * option to send. 577 */ 578 clear_hdr_cb_flags(skops); 579 580 if (th->fin) { 581 struct bpf_test_option *fin_opt; 582 int err; 583 584 if (hdr_stg->active) 585 fin_opt = &active_fin_in; 586 else 587 fin_opt = &passive_fin_in; 588 589 err = load_option(skops, fin_opt, false); 590 if (err && err != -ENOMSG) 591 RET_CG_ERR(err); 592 } 593 594 return CG_OK; 595 } 596 597 SEC("sockops") 598 int estab(struct bpf_sock_ops *skops) 599 { 600 int true_val = 1; 601 602 switch (skops->op) { 603 case BPF_SOCK_OPS_TCP_LISTEN_CB: 604 bpf_setsockopt(skops, SOL_TCP, TCP_SAVE_SYN, 605 &true_val, sizeof(true_val)); 606 set_hdr_cb_flags(skops, BPF_SOCK_OPS_STATE_CB_FLAG); 607 break; 608 case BPF_SOCK_OPS_TCP_CONNECT_CB: 609 set_hdr_cb_flags(skops, 0); 610 break; 611 case BPF_SOCK_OPS_PARSE_HDR_OPT_CB: 612 return handle_parse_hdr(skops); 613 case BPF_SOCK_OPS_HDR_OPT_LEN_CB: 614 return handle_hdr_opt_len(skops); 615 case BPF_SOCK_OPS_WRITE_HDR_OPT_CB: 616 return handle_write_hdr_opt(skops); 617 case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB: 618 return handle_passive_estab(skops); 619 case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB: 620 return handle_active_estab(skops); 621 } 622 623 return CG_OK; 624 } 625 626 char _license[] SEC("license") = "GPL"; 627