1# 2# SPDX-License-Identifier: BSD-2-Clause 3# 4# Copyright (c) 2018 Orange Business Services 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 1. Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# 2. Redistributions in binary form must reproduce the above copyright 12# notice, this list of conditions and the following disclaimer in the 13# documentation and/or other materials provided with the distribution. 14# 15# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25# SUCH DAMAGE. 26 27. $(atf_get_srcdir)/utils.subr 28 29common_dir=$(atf_get_srcdir)/../common 30 31atf_test_case "basic" "cleanup" 32basic_head() 33{ 34 atf_set descr 'Basic pfsync test' 35 atf_set require.user root 36} 37 38basic_body() 39{ 40 common_body 41} 42 43common_body() 44{ 45 defer=$1 46 pfsynct_init 47 48 epair_sync=$(vnet_mkepair) 49 epair_one=$(vnet_mkepair) 50 epair_two=$(vnet_mkepair) 51 52 vnet_mkjail one ${epair_one}a ${epair_sync}a 53 vnet_mkjail two ${epair_two}a ${epair_sync}b 54 55 # pfsync interface 56 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 57 jexec one ifconfig ${epair_one}a 198.51.100.1/24 up 58 jexec one ifconfig pfsync0 \ 59 syncdev ${epair_sync}a \ 60 maxupd 1 \ 61 $defer \ 62 up 63 jexec two ifconfig ${epair_two}a 198.51.100.2/24 up 64 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 65 jexec two ifconfig pfsync0 \ 66 syncdev ${epair_sync}b \ 67 maxupd 1 \ 68 $defer \ 69 up 70 71 # Enable pf! 72 jexec one pfctl -e 73 pft_set_rules one \ 74 "set skip on ${epair_sync}a" \ 75 "pass out keep state" 76 jexec two pfctl -e 77 pft_set_rules two \ 78 "set skip on ${epair_sync}b" \ 79 "pass out keep state" 80 81 hostid_one=$(jexec one pfctl -si -v | awk '/Hostid:/ { gsub(/0x/, "", $2); printf($2); }') 82 83 ifconfig ${epair_one}b 198.51.100.254/24 up 84 85 ping -c 1 -S 198.51.100.254 198.51.100.1 86 87 # Give pfsync time to do its thing 88 sleep 2 89 90 if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 91 grep 198.51.100.254 ; then 92 atf_fail "state not found on synced host" 93 fi 94 95 if ! jexec two pfctl -sc | grep ""${hostid_one}""; 96 then 97 jexec two pfctl -sc 98 atf_fail "HostID for host one not found on two" 99 fi 100} 101 102basic_cleanup() 103{ 104 pfsynct_cleanup 105} 106 107atf_test_case "basic_defer" "cleanup" 108basic_defer_head() 109{ 110 atf_set descr 'Basic defer mode pfsync test' 111 atf_set require.user root 112} 113 114basic_defer_body() 115{ 116 common_body defer 117} 118 119basic_defer_cleanup() 120{ 121 pfsynct_cleanup 122} 123 124atf_test_case "defer" "cleanup" 125defer_head() 126{ 127 atf_set descr 'Defer mode pfsync test' 128 atf_set require.user root 129 atf_set require.progs python3 scapy 130} 131 132defer_body() 133{ 134 pfsynct_init 135 136 epair_sync=$(vnet_mkepair) 137 epair_in=$(vnet_mkepair) 138 epair_out=$(vnet_mkepair) 139 140 vnet_mkjail alcatraz ${epair_sync}a ${epair_in}a ${epair_out}a 141 142 jexec alcatraz ifconfig ${epair_sync}a 192.0.2.1/24 up 143 jexec alcatraz ifconfig ${epair_out}a 198.51.100.1/24 up 144 jexec alcatraz ifconfig ${epair_in}a 203.0.113.1/24 up 145 jexec alcatraz arp -s 203.0.113.2 00:01:02:03:04:05 146 jexec alcatraz sysctl net.inet.ip.forwarding=1 147 148 # Set a long defer delay 149 jexec alcatraz sysctl net.pfsync.defer_delay=2500 150 151 jexec alcatraz ifconfig pfsync0 \ 152 syncdev ${epair_sync}a \ 153 maxupd 1 \ 154 defer \ 155 up 156 157 ifconfig ${epair_sync}b 192.0.2.2/24 up 158 ifconfig ${epair_out}b 198.51.100.2/24 up 159 ifconfig ${epair_in}b up 160 route add -net 203.0.113.0/24 198.51.100.1 161 162 # Enable pf 163 jexec alcatraz sysctl net.pf.filter_local=0 164 jexec alcatraz pfctl -e 165 pft_set_rules alcatraz \ 166 "set skip on ${epair_sync}a" \ 167 "pass keep state" 168 169 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 170 $(atf_get_srcdir)/pfsync_defer.py \ 171 --syncdev ${epair_sync}b \ 172 --indev ${epair_in}b \ 173 --outdev ${epair_out}b 174 175 # Now disable defer mode and expect failure. 176 jexec alcatraz ifconfig pfsync0 -defer 177 178 # Flush state 179 pft_set_rules alcatraz \ 180 "set skip on ${epair_sync}a" \ 181 "pass keep state" 182 183 atf_check -s exit:3 env PYTHONPATH=${common_dir} \ 184 $(atf_get_srcdir)/pfsync_defer.py \ 185 --syncdev ${epair_sync}b \ 186 --indev ${epair_in}b \ 187 --outdev ${epair_out}b 188} 189 190defer_cleanup() 191{ 192 pfsynct_cleanup 193} 194 195atf_test_case "bulk" "cleanup" 196bulk_head() 197{ 198 atf_set descr 'Test bulk updates' 199 atf_set require.user root 200} 201 202bulk_body() 203{ 204 pfsynct_init 205 206 epair_sync=$(vnet_mkepair) 207 epair_one=$(vnet_mkepair) 208 epair_two=$(vnet_mkepair) 209 210 vnet_mkjail one ${epair_one}a ${epair_sync}a 211 vnet_mkjail two ${epair_two}a ${epair_sync}b 212 213 # pfsync interface 214 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 215 jexec one ifconfig ${epair_one}a 198.51.100.1/24 up 216 jexec one ifconfig pfsync0 \ 217 syncdev ${epair_sync}a \ 218 maxupd 1\ 219 up 220 jexec two ifconfig ${epair_two}a 198.51.100.2/24 up 221 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 222 223 # Enable pf 224 jexec one pfctl -e 225 pft_set_rules one \ 226 "set skip on ${epair_sync}a" \ 227 "pass keep state" 228 jexec two pfctl -e 229 pft_set_rules two \ 230 "set skip on ${epair_sync}b" \ 231 "pass keep state" 232 233 ifconfig ${epair_one}b 198.51.100.254/24 up 234 235 # Create state prior to setting up pfsync 236 ping -c 1 -S 198.51.100.254 198.51.100.1 237 238 # Wait before setting up pfsync on two, so we don't accidentally catch 239 # the update anyway. 240 sleep 1 241 242 # Now set up pfsync in jail two 243 jexec two ifconfig pfsync0 \ 244 syncdev ${epair_sync}b \ 245 up 246 247 # Give pfsync time to do its thing 248 sleep 2 249 250 jexec two pfctl -s states 251 if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 252 grep 198.51.100.2 ; then 253 atf_fail "state not found on synced host" 254 fi 255} 256 257bulk_cleanup() 258{ 259 pfsynct_cleanup 260} 261 262atf_test_case "pbr" "cleanup" 263pbr_head() 264{ 265 atf_set descr 'route_to and reply_to directives test' 266 atf_set require.user root 267 atf_set timeout '600' 268} 269 270pbr_body() 271{ 272 pbr_common_body 273} 274 275pbr_cleanup() 276{ 277 pbr_common_cleanup 278} 279 280atf_test_case "pfsync_pbr" "cleanup" 281pfsync_pbr_head() 282{ 283 atf_set descr 'route_to and reply_to directives pfsync test' 284 atf_set require.user root 285 atf_set timeout '600' 286} 287 288pfsync_pbr_body() 289{ 290 pbr_common_body backup_promotion 291} 292 293pfsync_pbr_cleanup() 294{ 295 pbr_common_cleanup 296} 297 298pbr_common_body() 299{ 300 # + builds bellow topology and initiate a single ping session 301 # from client to server. 302 # + gw* forward traffic through pbr not fib lookups. 303 # + if backup_promotion arg is given, a carp failover event occurs 304 # during the ping session on both gateways. 305 # ┌──────┐ 306 # │client│ 307 # └───┬──┘ 308 # │ 309 # ┌───┴───┐ 310 # │bridge0│ 311 # └┬─────┬┘ 312 # │ │ 313 # ┌────────────────┴─┐ ┌─┴────────────────┐ 314 # │gw_route_to_master├─┤gw_route_to_backup│ 315 # └────────────────┬─┘ └─┬────────────────┘ 316 # │ │ 317 # ┌┴─────┴┐ 318 # │bridge1│ 319 # └┬─────┬┘ 320 # │ │ 321 # ┌────────────────┴─┐ ┌─┴────────────────┐ 322 # │gw_reply_to_master├─┤gw_reply_to_backup│ 323 # └────────────────┬─┘ └─┬────────────────┘ 324 # │ │ 325 # ┌┴─────┴┐ 326 # │bridge2│ 327 # └───┬───┘ 328 # │ 329 # ┌───┴──┐ 330 # │server│ 331 # └──────┘ 332 333 if ! kldstat -q -m carp 334 then 335 atf_skip "This test requires carp" 336 fi 337 pfsynct_init 338 vnet_init_bridge 339 340 bridge0=$(vnet_mkbridge) 341 bridge1=$(vnet_mkbridge) 342 bridge2=$(vnet_mkbridge) 343 344 epair_sync_gw_route_to=$(vnet_mkepair) 345 epair_sync_gw_reply_to=$(vnet_mkepair) 346 epair_client_bridge0=$(vnet_mkepair) 347 348 epair_gw_route_to_master_bridge0=$(vnet_mkepair) 349 epair_gw_route_to_backup_bridge0=$(vnet_mkepair) 350 epair_gw_route_to_master_bridge1=$(vnet_mkepair) 351 epair_gw_route_to_backup_bridge1=$(vnet_mkepair) 352 353 epair_gw_reply_to_master_bridge1=$(vnet_mkepair) 354 epair_gw_reply_to_backup_bridge1=$(vnet_mkepair) 355 epair_gw_reply_to_master_bridge2=$(vnet_mkepair) 356 epair_gw_reply_to_backup_bridge2=$(vnet_mkepair) 357 358 epair_server_bridge2=$(vnet_mkepair) 359 360 ifconfig ${bridge0} up 361 ifconfig ${epair_client_bridge0}b up 362 ifconfig ${epair_gw_route_to_master_bridge0}b up 363 ifconfig ${epair_gw_route_to_backup_bridge0}b up 364 ifconfig ${bridge0} \ 365 addm ${epair_client_bridge0}b \ 366 addm ${epair_gw_route_to_master_bridge0}b \ 367 addm ${epair_gw_route_to_backup_bridge0}b 368 369 ifconfig ${bridge1} up 370 ifconfig ${epair_gw_route_to_master_bridge1}b up 371 ifconfig ${epair_gw_route_to_backup_bridge1}b up 372 ifconfig ${epair_gw_reply_to_master_bridge1}b up 373 ifconfig ${epair_gw_reply_to_backup_bridge1}b up 374 ifconfig ${bridge1} \ 375 addm ${epair_gw_route_to_master_bridge1}b \ 376 addm ${epair_gw_route_to_backup_bridge1}b \ 377 addm ${epair_gw_reply_to_master_bridge1}b \ 378 addm ${epair_gw_reply_to_backup_bridge1}b 379 380 ifconfig ${bridge2} up 381 ifconfig ${epair_gw_reply_to_master_bridge2}b up 382 ifconfig ${epair_gw_reply_to_backup_bridge2}b up 383 ifconfig ${epair_server_bridge2}b up 384 ifconfig ${bridge2} \ 385 addm ${epair_gw_reply_to_master_bridge2}b \ 386 addm ${epair_gw_reply_to_backup_bridge2}b \ 387 addm ${epair_server_bridge2}b 388 389 vnet_mkjail client ${epair_client_bridge0}a 390 jexec client hostname client 391 vnet_mkjail gw_route_to_master \ 392 ${epair_gw_route_to_master_bridge0}a \ 393 ${epair_gw_route_to_master_bridge1}a \ 394 ${epair_sync_gw_route_to}a 395 jexec gw_route_to_master hostname gw_route_to_master 396 vnet_mkjail gw_route_to_backup \ 397 ${epair_gw_route_to_backup_bridge0}a \ 398 ${epair_gw_route_to_backup_bridge1}a \ 399 ${epair_sync_gw_route_to}b 400 jexec gw_route_to_backup hostname gw_route_to_backup 401 vnet_mkjail gw_reply_to_master \ 402 ${epair_gw_reply_to_master_bridge1}a \ 403 ${epair_gw_reply_to_master_bridge2}a \ 404 ${epair_sync_gw_reply_to}a 405 jexec gw_reply_to_master hostname gw_reply_to_master 406 vnet_mkjail gw_reply_to_backup \ 407 ${epair_gw_reply_to_backup_bridge1}a \ 408 ${epair_gw_reply_to_backup_bridge2}a \ 409 ${epair_sync_gw_reply_to}b 410 jexec gw_reply_to_backup hostname gw_reply_to_backup 411 vnet_mkjail server ${epair_server_bridge2}a 412 jexec server hostname server 413 414 jexec client ifconfig ${epair_client_bridge0}a inet 198.18.0.1/24 up 415 jexec client route add 198.18.2.0/24 198.18.0.10 416 417 jexec gw_route_to_master ifconfig ${epair_sync_gw_route_to}a \ 418 inet 198.19.10.1/24 up 419 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge0}a \ 420 inet 198.18.0.8/24 up 421 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge0}a \ 422 alias 198.18.0.10/32 vhid 10 pass 3WjvVVw7 advskew 50 423 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge1}a \ 424 inet 198.18.1.8/24 up 425 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge1}a \ 426 alias 198.18.1.10/32 vhid 11 pass 3WjvVVw7 advskew 50 427 jexec gw_route_to_master sysctl net.inet.ip.forwarding=1 428 jexec gw_route_to_master sysctl net.inet.carp.preempt=1 429 430 vnet_ifrename_jail gw_route_to_master ${epair_sync_gw_route_to}a if_pfsync 431 vnet_ifrename_jail gw_route_to_master ${epair_gw_route_to_master_bridge0}a if_br0 432 vnet_ifrename_jail gw_route_to_master ${epair_gw_route_to_master_bridge1}a if_br1 433 434 jexec gw_route_to_master ifconfig pfsync0 \ 435 syncpeer 198.19.10.2 \ 436 syncdev if_pfsync \ 437 maxupd 1 \ 438 up 439 pft_set_rules gw_route_to_master \ 440 "keep_state = 'tag auth_packet keep state'" \ 441 "set timeout { icmp.first 120, icmp.error 60 }" \ 442 "block log all" \ 443 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 444 "pass quick on { if_br0 if_br1 } proto carp keep state (no-sync)" \ 445 "block drop in quick to 224.0.0.18/32" \ 446 "pass out quick tagged auth_packet keep state" \ 447 "pass in quick log on if_br0 route-to (if_br1 198.18.1.20) proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 \$keep_state" 448 jexec gw_route_to_master pfctl -e 449 450 jexec gw_route_to_backup ifconfig ${epair_sync_gw_route_to}b \ 451 inet 198.19.10.2/24 up 452 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge0}a \ 453 inet 198.18.0.9/24 up 454 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge0}a \ 455 alias 198.18.0.10/32 vhid 10 pass 3WjvVVw7 advskew 100 456 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge1}a \ 457 inet 198.18.1.9/24 up 458 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge1}a \ 459 alias 198.18.1.10/32 vhid 11 pass 3WjvVVw7 advskew 100 460 jexec gw_route_to_backup sysctl net.inet.ip.forwarding=1 461 jexec gw_route_to_backup sysctl net.inet.carp.preempt=1 462 463 vnet_ifrename_jail gw_route_to_backup ${epair_sync_gw_route_to}b if_pfsync 464 vnet_ifrename_jail gw_route_to_backup ${epair_gw_route_to_backup_bridge0}a if_br0 465 vnet_ifrename_jail gw_route_to_backup ${epair_gw_route_to_backup_bridge1}a if_br1 466 467 jexec gw_route_to_backup ifconfig pfsync0 \ 468 syncpeer 198.19.10.1 \ 469 syncdev if_pfsync \ 470 up 471 pft_set_rules gw_route_to_backup \ 472 "keep_state = 'tag auth_packet keep state'" \ 473 "set timeout { icmp.first 120, icmp.error 60 }" \ 474 "block log all" \ 475 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 476 "pass quick on { if_br0 if_br1 } proto carp keep state (no-sync)" \ 477 "block drop in quick to 224.0.0.18/32" \ 478 "pass out quick tagged auth_packet keep state" \ 479 "pass in quick log on if_br0 route-to (if_br1 198.18.1.20) proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 \$keep_state" 480 jexec gw_route_to_backup pfctl -e 481 482 jexec gw_reply_to_master ifconfig ${epair_sync_gw_reply_to}a \ 483 inet 198.19.20.1/24 up 484 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge1}a \ 485 inet 198.18.1.18/24 up 486 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge1}a \ 487 alias 198.18.1.20/32 vhid 21 pass 3WjvVVw7 advskew 50 488 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge2}a \ 489 inet 198.18.2.18/24 up 490 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge2}a \ 491 alias 198.18.2.20/32 vhid 22 pass 3WjvVVw7 advskew 50 492 jexec gw_reply_to_master sysctl net.inet.ip.forwarding=1 493 jexec gw_reply_to_master sysctl net.inet.carp.preempt=1 494 495 vnet_ifrename_jail gw_reply_to_master ${epair_sync_gw_reply_to}a if_pfsync 496 vnet_ifrename_jail gw_reply_to_master ${epair_gw_reply_to_master_bridge1}a if_br1 497 vnet_ifrename_jail gw_reply_to_master ${epair_gw_reply_to_master_bridge2}a if_br2 498 499 jexec gw_reply_to_master ifconfig pfsync0 \ 500 syncpeer 198.19.20.2 \ 501 syncdev if_pfsync \ 502 maxupd 1 \ 503 up 504 pft_set_rules gw_reply_to_master \ 505 "set timeout { icmp.first 120, icmp.error 60 }" \ 506 "block log all" \ 507 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 508 "pass quick on { if_br1 if_br2 } proto carp keep state (no-sync)" \ 509 "block drop in quick to 224.0.0.18/32" \ 510 "pass out quick on if_br2 reply-to (if_br1 198.18.1.10) tagged auth_packet_reply_to keep state" \ 511 "pass in quick log on if_br1 proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 tag auth_packet_reply_to keep state" 512 jexec gw_reply_to_master pfctl -e 513 514 jexec gw_reply_to_backup ifconfig ${epair_sync_gw_reply_to}b \ 515 inet 198.19.20.2/24 up 516 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge1}a \ 517 inet 198.18.1.19/24 up 518 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge1}a \ 519 alias 198.18.1.20/32 vhid 21 pass 3WjvVVw7 advskew 100 520 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge2}a \ 521 inet 198.18.2.19/24 up 522 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge2}a \ 523 alias 198.18.2.20/32 vhid 22 pass 3WjvVVw7 advskew 100 524 jexec gw_reply_to_backup sysctl net.inet.ip.forwarding=1 525 jexec gw_reply_to_backup sysctl net.inet.carp.preempt=1 526 527 vnet_ifrename_jail gw_reply_to_backup ${epair_sync_gw_reply_to}b if_pfsync 528 vnet_ifrename_jail gw_reply_to_backup ${epair_gw_reply_to_backup_bridge1}a if_br1 529 vnet_ifrename_jail gw_reply_to_backup ${epair_gw_reply_to_backup_bridge2}a if_br2 530 531 jexec gw_reply_to_backup ifconfig pfsync0 \ 532 syncpeer 198.19.20.1 \ 533 syncdev if_pfsync \ 534 up 535 pft_set_rules gw_reply_to_backup \ 536 "set timeout { icmp.first 120, icmp.error 60 }" \ 537 "block log all" \ 538 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 539 "pass quick on { if_br1 if_br2 } proto carp keep state (no-sync)" \ 540 "block drop in quick to 224.0.0.18/32" \ 541 "pass out quick on if_br2 reply-to (if_br1 198.18.1.10) tagged auth_packet_reply_to keep state" \ 542 "pass in quick log on if_br1 proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 tag auth_packet_reply_to keep state" 543 jexec gw_reply_to_backup pfctl -e 544 545 jexec server ifconfig ${epair_server_bridge2}a inet 198.18.2.1/24 up 546 jexec server route add 198.18.0.0/24 198.18.2.20 547 548 # Waiting for platform to settle 549 while ! jexec gw_route_to_backup ifconfig | grep 'carp: BACKUP' 550 do 551 sleep 1 552 done 553 while ! jexec gw_reply_to_backup ifconfig | grep 'carp: BACKUP' 554 do 555 sleep 1 556 done 557 while ! jexec client ping -c 10 198.18.2.1 | grep ', 0.0% packet loss' 558 do 559 sleep 1 560 done 561 562 # Checking cluster members pf.conf checksums match 563 gw_route_to_master_checksum=$(jexec gw_route_to_master pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 564 gw_route_to_backup_checksum=$(jexec gw_route_to_backup pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 565 gw_reply_to_master_checksum=$(jexec gw_reply_to_master pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 566 gw_reply_to_backup_checksum=$(jexec gw_reply_to_backup pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 567 if [ "$gw_route_to_master_checksum" != "$gw_route_to_backup_checksum" ] 568 then 569 atf_fail "gw_route_to cluster members pf.conf do not match each others" 570 fi 571 if [ "$gw_reply_to_master_checksum" != "$gw_reply_to_backup_checksum" ] 572 then 573 atf_fail "gw_reply_to cluster members pf.conf do not match each others" 574 fi 575 576 # Creating state entries 577 (jexec client ping -c 10 198.18.2.1 >ping.stdout) & 578 579 if [ "$1" = "backup_promotion" ] 580 then 581 sleep 1 582 jexec gw_route_to_backup ifconfig if_br0 vhid 10 advskew 0 583 jexec gw_route_to_backup ifconfig if_br1 vhid 11 advskew 0 584 jexec gw_reply_to_backup ifconfig if_br1 vhid 21 advskew 0 585 jexec gw_reply_to_backup ifconfig if_br2 vhid 22 advskew 0 586 fi 587 while ! grep -q -e 'packet loss' ping.stdout 588 do 589 sleep 1 590 done 591 592 atf_check -s exit:0 -e ignore -o ignore grep ', 0.0% packet loss' ping.stdout 593} 594 595pbr_common_cleanup() 596{ 597 pft_cleanup 598} 599 600atf_test_case "ipsec" "cleanup" 601ipsec_head() 602{ 603 atf_set descr 'Transport pfsync over IPSec' 604 atf_set require.user root 605} 606 607ipsec_body() 608{ 609 if ! sysctl -q kern.features.ipsec >/dev/null ; then 610 atf_skip "This test requires ipsec" 611 fi 612 613 # Run the common test, to set up pfsync 614 common_body 615 616 # But we want unicast pfsync 617 jexec one ifconfig pfsync0 syncpeer 192.0.2.2 618 jexec two ifconfig pfsync0 syncpeer 192.0.2.1 619 620 # Flush existing states 621 jexec one pfctl -Fs 622 jexec two pfctl -Fs 623 624 # Now define an ipsec policy to run over the epair_sync interfaces 625 echo "flush; 626 spdflush; 627 spdadd 192.0.2.1/32 192.0.2.2/32 any -P out ipsec esp/transport//require; 628 spdadd 192.0.2.2/32 192.0.2.1/32 any -P in ipsec esp/transport//require; 629 add 192.0.2.1 192.0.2.2 esp 0x1000 -E aes-gcm-16 \"12345678901234567890\"; 630 add 192.0.2.2 192.0.2.1 esp 0x1001 -E aes-gcm-16 \"12345678901234567890\";" \ 631 | jexec one setkey -c 632 633 echo "flush; 634 spdflush; 635 spdadd 192.0.2.2/32 192.0.2.1/32 any -P out ipsec esp/transport//require; 636 spdadd 192.0.2.1/32 192.0.2.2/32 any -P in ipsec esp/transport//require; 637 add 192.0.2.1 192.0.2.2 esp 0x1000 -E aes-gcm-16 \"12345678901234567891\"; 638 add 192.0.2.2 192.0.2.1 esp 0x1001 -E aes-gcm-16 \"12345678901234567891\";" \ 639 | jexec two setkey -c 640 641 # We've set incompatible keys, so pfsync will be broken. 642 ping -c 1 -S 198.51.100.254 198.51.100.1 643 644 # Give pfsync time to do its thing 645 sleep 2 646 647 if jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 648 grep 198.51.100.2 ; then 649 atf_fail "state synced although IPSec should have prevented it" 650 fi 651 652 # Flush existing states 653 jexec one pfctl -Fs 654 jexec two pfctl -Fs 655 656 # Fix the IPSec key to match 657 echo "flush; 658 spdflush; 659 spdadd 192.0.2.2/32 192.0.2.1/32 any -P out ipsec esp/transport//require; 660 spdadd 192.0.2.1/32 192.0.2.2/32 any -P in ipsec esp/transport//require; 661 add 192.0.2.1 192.0.2.2 esp 0x1000 -E aes-gcm-16 \"12345678901234567890\"; 662 add 192.0.2.2 192.0.2.1 esp 0x1001 -E aes-gcm-16 \"12345678901234567890\";" \ 663 | jexec two setkey -c 664 665 ping -c 1 -S 198.51.100.254 198.51.100.1 666 667 # Give pfsync time to do its thing 668 sleep 2 669 670 if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 671 grep 198.51.100.2 ; then 672 atf_fail "state not found on synced host" 673 fi 674} 675 676ipsec_cleanup() 677{ 678 pft_cleanup 679} 680 681atf_test_case "timeout" "cleanup" 682timeout_head() 683{ 684 atf_set descr 'Trigger pfsync_timeout()' 685 atf_set require.user root 686} 687 688timeout_body() 689{ 690 pft_init 691 692 vnet_mkjail one 693 694 jexec one ifconfig lo0 127.0.0.1/8 up 695 jexec one ifconfig lo0 inet6 ::1/128 up 696 697 pft_set_rules one \ 698 "pass all" 699 jexec one pfctl -e 700 jexec one ifconfig pfsync0 defer up 701 702 jexec one ping -c 1 ::1 703 jexec one ping -c 1 127.0.0.1 704 705 # Give pfsync_timeout() time to fire (a callout on a 1 second delay) 706 sleep 2 707} 708 709timeout_cleanup() 710{ 711 pft_cleanup 712} 713 714atf_test_case "basic_ipv6_unicast" "cleanup" 715basic_ipv6_unicast_head() 716{ 717 atf_set descr 'Basic pfsync test (IPv6)' 718 atf_set require.user root 719} 720 721basic_ipv6_unicast_body() 722{ 723 pfsynct_init 724 725 epair_sync=$(vnet_mkepair) 726 epair_one=$(vnet_mkepair) 727 epair_two=$(vnet_mkepair) 728 729 vnet_mkjail one ${epair_one}a ${epair_sync}a 730 vnet_mkjail two ${epair_two}a ${epair_sync}b 731 732 # pfsync interface 733 jexec one ifconfig ${epair_sync}a inet6 fd2c::1/64 no_dad up 734 jexec one ifconfig ${epair_one}a inet6 fd2b::1/64 no_dad up 735 jexec one ifconfig pfsync0 \ 736 syncdev ${epair_sync}a \ 737 syncpeer fd2c::2 \ 738 maxupd 1 \ 739 up 740 jexec two ifconfig ${epair_two}a inet6 fd2b::2/64 no_dad up 741 jexec two ifconfig ${epair_sync}b inet6 fd2c::2/64 no_dad up 742 jexec two ifconfig pfsync0 \ 743 syncdev ${epair_sync}b \ 744 syncpeer fd2c::1 \ 745 maxupd 1 \ 746 up 747 748 # Enable pf! 749 jexec one pfctl -e 750 pft_set_rules one \ 751 "block on ${epair_sync}a inet" \ 752 "pass out keep state" 753 jexec two pfctl -e 754 pft_set_rules two \ 755 "block on ${epair_sync}b inet" \ 756 "pass out keep state" 757 758 ifconfig ${epair_one}b inet6 fd2b::f0/64 no_dad up 759 760 ping6 -c 1 -S fd2b::f0 fd2b::1 761 762 # Give pfsync time to do its thing 763 sleep 2 764 765 if ! jexec two pfctl -s states | grep icmp | grep fd2b::1 | \ 766 grep fd2b::f0 ; then 767 atf_fail "state not found on synced host" 768 fi 769} 770 771basic_ipv6_unicast_cleanup() 772{ 773 pfsynct_cleanup 774} 775 776atf_test_case "basic_ipv6" "cleanup" 777basic_ipv6_head() 778{ 779 atf_set descr 'Basic pfsync test (IPv6)' 780 atf_set require.user root 781} 782 783basic_ipv6_body() 784{ 785 pfsynct_init 786 787 epair_sync=$(vnet_mkepair) 788 epair_one=$(vnet_mkepair) 789 epair_two=$(vnet_mkepair) 790 791 vnet_mkjail one ${epair_one}a ${epair_sync}a 792 vnet_mkjail two ${epair_two}a ${epair_sync}b 793 794 # pfsync interface 795 jexec one ifconfig ${epair_sync}a inet6 fd2c::1/64 no_dad up 796 jexec one ifconfig ${epair_one}a inet6 fd2b::1/64 no_dad up 797 jexec one ifconfig pfsync0 \ 798 syncdev ${epair_sync}a \ 799 syncpeer ff12::f0 \ 800 maxupd 1 \ 801 up 802 jexec two ifconfig ${epair_two}a inet6 fd2b::2/64 no_dad up 803 jexec two ifconfig ${epair_sync}b inet6 fd2c::2/64 no_dad up 804 jexec two ifconfig pfsync0 \ 805 syncdev ${epair_sync}b \ 806 syncpeer ff12::f0 \ 807 maxupd 1 \ 808 up 809 810 # Enable pf! 811 jexec one pfctl -e 812 pft_set_rules one \ 813 "block on ${epair_sync}a inet" \ 814 "pass out keep state" 815 jexec two pfctl -e 816 pft_set_rules two \ 817 "block on ${epair_sync}b inet" \ 818 "pass out keep state" 819 820 ifconfig ${epair_one}b inet6 fd2b::f0/64 no_dad up 821 822 ping6 -c 1 -S fd2b::f0 fd2b::1 823 824 # Give pfsync time to do its thing 825 sleep 2 826 827 if ! jexec two pfctl -s states | grep icmp | grep fd2b::1 | \ 828 grep fd2b::f0 ; then 829 atf_fail "state not found on synced host" 830 fi 831} 832 833basic_ipv6_cleanup() 834{ 835 pfsynct_cleanup 836} 837 838atf_test_case "rtable" "cleanup" 839rtable_head() 840{ 841 atf_set descr 'Test handling of invalid rtableid' 842 atf_set require.user root 843} 844 845rtable_body() 846{ 847 pfsynct_init 848 849 epair_sync=$(vnet_mkepair) 850 epair_one=$(vnet_mkepair) 851 epair_two=$(vnet_mkepair) 852 853 vnet_mkjail one ${epair_one}a ${epair_sync}a 854 vnet_mkjail two ${epair_two}a ${epair_sync}b 855 856 # pfsync interface 857 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 858 jexec one ifconfig ${epair_one}a 198.51.100.1/24 up 859 jexec one ifconfig pfsync0 \ 860 syncdev ${epair_sync}a \ 861 maxupd 1 \ 862 up 863 jexec two ifconfig ${epair_two}a 198.51.100.1/24 up 864 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 865 jexec two ifconfig pfsync0 \ 866 syncdev ${epair_sync}b \ 867 maxupd 1 \ 868 up 869 870 # Make life easy, give ${epair_two}a the same mac addrss as ${epair_one}a 871 mac=$(jexec one ifconfig ${epair_one}a | awk '/ether/ { print($2); }') 872 jexec two ifconfig ${epair_two}a ether ${mac} 873 874 # Enable pf! 875 jexec one /sbin/sysctl net.fibs=8 876 jexec one pfctl -e 877 pft_set_rules one \ 878 "set skip on ${epair_sync}a" \ 879 "pass rtable 3 keep state" 880 # No extra fibs in two 881 jexec two pfctl -e 882 pft_set_rules two \ 883 "set skip on ${epair_sync}b" \ 884 "pass keep state" 885 886 ifconfig ${epair_one}b 198.51.100.254/24 up 887 ifconfig ${epair_two}b 198.51.100.253/24 up 888 889 # Create a new state 890 env PYTHONPATH=${common_dir} \ 891 ${common_dir}/pft_ping.py \ 892 --sendif ${epair_one}b \ 893 --fromaddr 198.51.100.254 \ 894 --to 198.51.100.1 \ 895 --recvif ${epair_one}b 896 897 # Now 898 jexec one pfctl -ss -vv 899 sleep 2 900 901 # Now try to use that state on jail two 902 env PYTHONPATH=${common_dir} \ 903 ${common_dir}/pft_ping.py \ 904 --sendif ${epair_two}b \ 905 --fromaddr 198.51.100.254 \ 906 --to 198.51.100.1 \ 907 --recvif ${epair_two}b 908 909 echo one 910 jexec one pfctl -ss -vv 911 jexec one pfctl -sr -vv 912 echo two 913 jexec two pfctl -ss -vv 914 jexec two pfctl -sr -vv 915} 916 917rtable_cleanup() 918{ 919 pfsynct_cleanup 920} 921 922route_to_common_head() 923{ 924 # TODO: Extend setup_router_server_nat64 to create a 2nd router 925 926 pfsync_version=$1 927 shift 928 929 pfsynct_init 930 931 epair_sync=$(vnet_mkepair) 932 epair_one=$(vnet_mkepair) 933 epair_two=$(vnet_mkepair) 934 epair_out_one=$(vnet_mkepair) 935 epair_out_two=$(vnet_mkepair) 936 937 vnet_mkjail one ${epair_one}a ${epair_sync}a ${epair_out_one}a 938 vnet_mkjail two ${epair_two}a ${epair_sync}b ${epair_out_two}a 939 940 # pfsync interface 941 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 942 jexec one ifconfig ${epair_one}a 198.51.100.1/28 up 943 jexec one ifconfig ${epair_one}a inet6 2001:db8:4211::1/64 no_dad 944 jexec one ifconfig ${epair_one}a name inif 945 jexec one ifconfig ${epair_out_one}a 203.0.113.1/24 up 946 jexec one ifconfig ${epair_out_one}a inet6 2001:db8:4200::1/64 no_dad 947 jexec one ifconfig ${epair_out_one}a name outif 948 jexec one sysctl net.inet.ip.forwarding=1 949 jexec one sysctl net.inet6.ip6.forwarding=1 950 jexec one arp -s 203.0.113.254 00:01:02:00:00:04 951 jexec one ndp -s 2001:db8:4200::fe 00:01:02:00:00:06 952 jexec one ifconfig pfsync0 \ 953 syncdev ${epair_sync}a \ 954 maxupd 1 \ 955 version $pfsync_version \ 956 up 957 958 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 959 jexec two ifconfig ${epair_two}a 198.51.100.17/28 up 960 jexec two ifconfig ${epair_two}a inet6 2001:db8:4212::1/64 no_dad 961 jexec two ifconfig ${epair_two}a name inif 962 jexec two ifconfig ${epair_out_two}a 203.0.113.1/24 up 963 jexec two ifconfig ${epair_out_two}a inet6 2001:db8:4200::2/64 no_dad 964 jexec two ifconfig ${epair_out_two}a name outif 965 jexec two sysctl net.inet.ip.forwarding=1 966 jexec two sysctl net.inet6.ip6.forwarding=1 967 jexec two arp -s 203.0.113.254 00:01:02:00:00:04 968 jexec two ndp -s 2001:db8:4200::fe 00:01:02:00:00:06 969 jexec two ifconfig pfsync0 \ 970 syncdev ${epair_sync}b \ 971 maxupd 1 \ 972 version $pfsync_version \ 973 up 974 975 ifconfig ${epair_one}b 198.51.100.2/28 up 976 ifconfig ${epair_one}b inet6 2001:db8:4211::2/64 no_dad 977 ifconfig ${epair_two}b 198.51.100.18/28 up 978 ifconfig ${epair_two}b inet6 2001:db8:4212::2/64 no_dad 979 # Target is behind router "one" 980 route add -net 203.0.113.0/24 198.51.100.1 981 route add -inet6 -net 64:ff9b::/96 2001:db8:4211::1 982 983 ifconfig ${epair_two}b up 984 ifconfig ${epair_out_one}b up 985 ifconfig ${epair_out_two}b up 986} 987 988route_to_common_tail() 989{ 990 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 991 ${common_dir}/pft_ping.py \ 992 --sendif ${epair_one}b \ 993 --fromaddr 198.51.100.254 \ 994 --to 203.0.113.254 \ 995 --recvif ${epair_out_one}b 996 997 # Allow time for sync 998 sleep 2 999 1000 states_one=$(mktemp) 1001 states_two=$(mktemp) 1002 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1003 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1004} 1005 1006atf_test_case "route_to_1301_body" "cleanup" 1007route_to_1301_head() 1008{ 1009 atf_set descr 'Test route-to with pfsync version 13.1' 1010 atf_set require.user root 1011 atf_set require.progs python3 scapy 1012} 1013 1014route_to_1301_body() 1015{ 1016 route_to_common_head 1301 1017 1018 jexec one pfctl -e 1019 pft_set_rules one \ 1020 "set skip on ${epair_sync}a" \ 1021 "pass out route-to (outif 203.0.113.254)" 1022 1023 jexec two pfctl -e 1024 pft_set_rules two \ 1025 "set skip on ${epair_sync}b" \ 1026 "pass out route-to (outif 203.0.113.254)" 1027 1028 route_to_common_tail 1029 1030 # Sanity check 1031 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1032 atf_fail "State missing on router one" 1033 1034 # With identical ruleset the routing information is recovered from the matching rule. 1035 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif' $states_two || 1036 atf_fail "State missing on router two" 1037 1038 true 1039} 1040 1041route_to_1301_cleanup() 1042{ 1043 pfsynct_cleanup 1044} 1045 1046atf_test_case "route_to_1301_bad_ruleset" "cleanup" 1047route_to_1301_bad_ruleset_head() 1048{ 1049 atf_set descr 'Test route-to with pfsync version 13.1 and incompatible ruleset' 1050 atf_set require.user root 1051 atf_set require.progs python3 scapy 1052} 1053 1054route_to_1301_bad_ruleset_body() 1055{ 1056 route_to_common_head 1301 1057 1058 jexec one pfctl -e 1059 pft_set_rules one \ 1060 "set skip on ${epair_sync}a" \ 1061 "pass out route-to (outif 203.0.113.254)" 1062 1063 jexec two pfctl -e 1064 pft_set_rules two \ 1065 "set debug loud" \ 1066 "set skip on ${epair_sync}b" \ 1067 "pass out route-to (outif 203.0.113.254)" \ 1068 "pass out proto tcp" 1069 1070 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1071 ${common_dir}/pft_ping.py \ 1072 --sendif ${epair_one}b \ 1073 --fromaddr 198.51.100.254 \ 1074 --to 203.0.113.254 \ 1075 --recvif ${epair_out_one}b 1076 1077 route_to_common_tail 1078 1079 # Sanity check 1080 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1081 atf_fail "State missing on router one" 1082 1083 # Different ruleset on each router means the routing information recovery 1084 # from rule is impossible. The state is not synced. 1085 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*' $states_two && 1086 atf_fail "State present on router two" 1087 1088 true 1089} 1090 1091route_to_1301_bad_ruleset_cleanup() 1092{ 1093 pfsynct_cleanup 1094} 1095 1096atf_test_case "route_to_1301_bad_rpool" "cleanup" 1097route_to_1301_bad_rpool_head() 1098{ 1099 atf_set descr 'Test route-to with pfsync version 13.1 and different interface' 1100 atf_set require.user root 1101 atf_set require.progs python3 scapy 1102} 1103 1104route_to_1301_bad_rpool_body() 1105{ 1106 route_to_common_head 1301 1107 1108 jexec one pfctl -e 1109 pft_set_rules one \ 1110 "set skip on ${epair_sync}a" \ 1111 "pass out route-to { (outif 203.0.113.254) (outif 203.0.113.254) }" 1112 1113 jexec two pfctl -e 1114 pft_set_rules two \ 1115 "set skip on ${epair_sync}b" \ 1116 "pass out route-to { (outif 203.0.113.254) (outif 203.0.113.254) }" 1117 1118 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1119 ${common_dir}/pft_ping.py \ 1120 --sendif ${epair_one}b \ 1121 --fromaddr 198.51.100.254 \ 1122 --to 203.0.113.254 \ 1123 --recvif ${epair_out_one}b 1124 1125 route_to_common_tail 1126 1127 # Sanity check 1128 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1129 atf_fail "State missing on router one" 1130 1131 # The ruleset is identical but since the redirection pool contains multiple interfaces 1132 # pfsync will not attempt to recover the routing information from the rule. 1133 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*' $states_two && 1134 atf_fail "State present on router two" 1135 1136 true 1137} 1138 1139route_to_1301_bad_rpool_cleanup() 1140{ 1141 pfsynct_cleanup 1142} 1143 1144atf_test_case "route_to_1400_bad_ruleset" "cleanup" 1145route_to_1400_bad_ruleset_head() 1146{ 1147 atf_set descr 'Test route-to with pfsync version 14.0' 1148 atf_set require.user root 1149 atf_set require.progs python3 scapy 1150} 1151 1152route_to_1400_bad_ruleset_body() 1153{ 1154 route_to_common_head 1400 1155 1156 jexec one pfctl -e 1157 pft_set_rules one \ 1158 "set skip on ${epair_sync}a" \ 1159 "pass out route-to (outif 203.0.113.254)" 1160 1161 jexec two pfctl -e 1162 pft_set_rules two \ 1163 "set skip on ${epair_sync}b" 1164 1165 route_to_common_tail 1166 1167 # Sanity check 1168 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1169 atf_fail "State missing on router one" 1170 1171 # Even with a different ruleset FreeBSD 14 syncs the state just fine. 1172 # There's no recovery involved, the pfsync packet contains the routing information. 1173 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .* route-to: 203.0.113.254@outif' $states_two || 1174 atf_fail "State missing on router two" 1175 1176 true 1177} 1178 1179route_to_1400_bad_ruleset_cleanup() 1180{ 1181 pfsynct_cleanup 1182} 1183 1184atf_test_case "route_to_1400_bad_ifname" "cleanup" 1185route_to_1400_bad_ifname_head() 1186{ 1187 atf_set descr 'Test route-to with pfsync version 14.0' 1188 atf_set require.user root 1189 atf_set require.progs python3 scapy 1190} 1191 1192route_to_1400_bad_ifname_body() 1193{ 1194 route_to_common_head 1400 1195 1196 jexec one pfctl -e 1197 pft_set_rules one \ 1198 "set skip on ${epair_sync}a" \ 1199 "pass out route-to (outif 203.0.113.254)" 1200 1201 jexec two pfctl -e 1202 jexec two ifconfig outif name outif_new 1203 pft_set_rules two \ 1204 "set skip on ${epair_sync}b" \ 1205 "pass out route-to (outif_new 203.0.113.254)" 1206 1207 route_to_common_tail 1208 1209 # Sanity check 1210 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1211 atf_fail "State missing on router one" 1212 1213 # Since FreeBSD 14 never attempts recovery of missing routing information 1214 # a state synced to a router with a different interface name is dropped. 1215 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*' $states_two && 1216 atf_fail "State present on router two" 1217 1218 true 1219} 1220 1221route_to_1400_bad_ifname_cleanup() 1222{ 1223 pfsynct_cleanup 1224} 1225 1226atf_test_case "af_to_in_floating" "cleanup" 1227af_to_in_floating_head() 1228{ 1229 atf_set descr 'Test syncing of states created by inbound af-to rules with floating states' 1230 atf_set require.user root 1231 atf_set require.progs python3 scapy 1232} 1233 1234af_to_in_floating_body() 1235{ 1236 route_to_common_head 1500 1237 1238 jexec one pfctl -e 1239 pft_set_rules one \ 1240 "set state-policy floating" \ 1241 "set skip on ${epair_sync}a" \ 1242 "block" \ 1243 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1244 "pass in on inif to 64:ff9b::/96 af-to inet from (outif) keep state" 1245 1246 jexec two pfctl -e 1247 pft_set_rules two \ 1248 "set skip on ${epair_sync}b" \ 1249 "block" \ 1250 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" 1251 1252 # ptf_ping can't deal with nat64, this test will fail but generate states 1253 atf_check -s exit:1 env PYTHONPATH=${common_dir} \ 1254 ${common_dir}/pft_ping.py \ 1255 --sendif ${epair_one}b \ 1256 --fromaddr 2001:db8:4201::fe \ 1257 --to 64:ff9b::203.0.113.254 \ 1258 --recvif ${epair_out_one}b 1259 1260 # Allow time for sync 1261 sleep 2 1262 1263 states_one=$(mktemp) 1264 states_two=$(mktemp) 1265 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1266 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1267 1268 # Sanity check 1269 grep -qE 'all ipv6-icmp 203.0.113.1 \(2001:db8:4201::fe\) -> 203.0.113.254:8 \(64:ff9b::cb00:71fe) .* rule 3 .* origif: inif' $states_one || 1270 atf_fail "State missing on router one" 1271 1272 grep -qE 'all ipv6-icmp 203.0.113.1 \(2001:db8:4201::fe\) -> 203.0.113.254:8 \(64:ff9b::cb00:71fe) .* origif: inif' $states_two || 1273 atf_fail "State missing on router two" 1274} 1275 1276af_to_in_floating_cleanup() 1277{ 1278 pfsynct_cleanup 1279} 1280 1281atf_test_case "af_to_in_if_bound" "cleanup" 1282af_to_in_if_bound_head() 1283{ 1284 atf_set descr 'Test syncing of states created by inbound af-to rules with if-bound states' 1285 atf_set require.user root 1286 atf_set require.progs python3 scapy 1287} 1288 1289af_to_in_if_bound_body() 1290{ 1291 route_to_common_head 1500 1292 1293 jexec one pfctl -e 1294 pft_set_rules one \ 1295 "set state-policy if-bound" \ 1296 "set skip on ${epair_sync}a" \ 1297 "block" \ 1298 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1299 "pass in on inif to 64:ff9b::/96 af-to inet from (outif) keep state" 1300 1301 jexec two pfctl -e 1302 pft_set_rules two \ 1303 "set skip on ${epair_sync}b" \ 1304 "block" \ 1305 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" 1306 1307 # ptf_ping can't deal with nat64, this test will fail but generate states 1308 atf_check -s exit:1 env PYTHONPATH=${common_dir} \ 1309 ${common_dir}/pft_ping.py \ 1310 --sendif ${epair_one}b \ 1311 --fromaddr 2001:db8:4201::fe \ 1312 --to 64:ff9b::203.0.113.254 \ 1313 --recvif ${epair_out_one}b 1314 1315 # Allow time for sync 1316 sleep 2 1317 1318 states_one=$(mktemp) 1319 states_two=$(mktemp) 1320 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1321 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1322 1323 # Sanity check 1324 grep -qE 'outif ipv6-icmp 203.0.113.1 \(2001:db8:4201::fe\) -> 203.0.113.254:8 \(64:ff9b::cb00:71fe) .* rule 3 .* origif: inif' $states_one || 1325 atf_fail "State missing on router one" 1326 1327 grep -qE 'outif ipv6-icmp 203.0.113.1 \(2001:db8:4201::fe\) -> 203.0.113.254:8 \(64:ff9b::cb00:71fe) .* origif: inif' $states_two || 1328 atf_fail "State missing on router two" 1329} 1330 1331af_to_in_if_bound_cleanup() 1332{ 1333 pfsynct_cleanup 1334} 1335 1336atf_test_case "af_to_out_if_bound" "cleanup" 1337af_to_out_if_bound_head() 1338{ 1339 atf_set descr 'Test syncing of states created by outbound af-to rules with if-bound states' 1340 atf_set require.user root 1341 atf_set require.progs python3 scapy 1342} 1343 1344af_to_out_if_bound_body() 1345{ 1346 route_to_common_head 1500 1347 1348 jexec one route add -inet6 -net 64:ff9b::/96 -iface outif 1349 jexec one sysctl net.inet6.ip6.forwarding=1 1350 1351 jexec one pfctl -e 1352 pft_set_rules one \ 1353 "set state-policy if-bound" \ 1354 "set skip on ${epair_sync}a" \ 1355 "block" \ 1356 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1357 "pass in on inif to 64:ff9b::/96 keep state" \ 1358 "pass out on outif to 64:ff9b::/96 af-to inet from (outif) keep state" 1359 1360 jexec two pfctl -e 1361 pft_set_rules two \ 1362 "set skip on ${epair_sync}b" \ 1363 "block" \ 1364 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" 1365 1366 # ptf_ping can't deal with nat64, this test will fail but generate states 1367 atf_check -s exit:1 env PYTHONPATH=${common_dir} \ 1368 ${common_dir}/pft_ping.py \ 1369 --sendif ${epair_one}b \ 1370 --fromaddr 2001:db8:4201::fe \ 1371 --to 64:ff9b::203.0.113.254 \ 1372 --recvif ${epair_out_one}b 1373 1374 # Allow time for sync 1375 sleep 2 1376 1377 states_one=$(mktemp) 1378 states_two=$(mktemp) 1379 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1380 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1381 1382 # Sanity check 1383 # st->orig_kif is the same as st->kif, so st->orig_kif is not printed. 1384 for state_regexp in \ 1385 "inif ipv6-icmp 64:ff9b::cb00:71fe\[128\] <- 2001:db8:4201::fe .* rule 3 .* creatorid: [0-9a-f]+" \ 1386 "outif icmp 203.0.113.1 \(64:ff9b::cb00:71fe\[8\]\) -> 203.0.113.254:8 \(2001:db8:4201::fe\) .* rule 4 .* creatorid: [0-9a-f]+" \ 1387 ; do 1388 grep -qE "${state_regexp}" $states_one || atf_fail "State not found for '${state_regexp}'" 1389 done 1390 1391 for state_regexp in \ 1392 "inif ipv6-icmp 64:ff9b::cb00:71fe\[128\] <- 2001:db8:4201::fe .* creatorid: [0-9a-f]+" \ 1393 "outif icmp 203.0.113.1 \(64:ff9b::cb00:71fe\[8\]\) -> 203.0.113.254:8 \(2001:db8:4201::fe\) .* creatorid: [0-9a-f]+" \ 1394 ; do 1395 grep -qE "${state_regexp}" $states_two || atf_fail "State not found for '${state_regexp}'" 1396 done 1397} 1398 1399af_to_out_if_bound_cleanup() 1400{ 1401 pfsynct_cleanup 1402} 1403 1404atf_test_case "tag" "cleanup" 1405tag_head() 1406{ 1407 atf_set descr 'Test if the pf tag is synced' 1408 atf_set require.user root 1409 atf_set require.progs python3 scapy 1410} 1411 1412tag_body() 1413{ 1414 route_to_common_head 1500 1415 1416 jexec one pfctl -e 1417 pft_set_rules one \ 1418 "set skip on ${epair_sync}a" \ 1419 "block" \ 1420 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1421 "pass in on inif inet proto udp tag sometag keep state" \ 1422 "pass out on outif tagged sometag keep state (no-sync)" 1423 1424 jexec two pfctl -e 1425 pft_set_rules two \ 1426 "set debug loud" \ 1427 "set skip on ${epair_sync}b" \ 1428 "block" \ 1429 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1430 "block tagged othertag" \ 1431 "pass out on outif tagged sometag keep state (no-sync)" 1432 1433 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1434 ${common_dir}/pft_ping.py \ 1435 --ping-type=udp \ 1436 --sendif ${epair_one}b \ 1437 --fromaddr 198.51.100.254 \ 1438 --to 203.0.113.254 \ 1439 --recvif ${epair_out_one}b 1440 1441 # Allow time for sync 1442 sleep 2 1443 1444 # Force the next request to go through the 2nd router 1445 route change -net 203.0.113.0/24 198.51.100.17 1446 1447 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1448 ${common_dir}/pft_ping.py \ 1449 --ping-type=udp \ 1450 --sendif ${epair_two}b \ 1451 --fromaddr 198.51.100.254 \ 1452 --to 203.0.113.254 \ 1453 --recvif ${epair_out_two}b 1454} 1455 1456tag_cleanup() 1457{ 1458 pfsynct_cleanup 1459} 1460 1461atf_test_case "altq_queues" "cleanup" 1462altq_queues_head() 1463{ 1464 atf_set descr 'Test if the altq queues are synced' 1465 atf_set require.user root 1466 atf_set require.progs python3 scapy 1467} 1468 1469altq_queues_body() 1470{ 1471 route_to_common_head 1500 1472 altq_init 1473 is_altq_supported hfsc 1474 1475 jexec one pfctl -e 1476 pft_set_rules one \ 1477 "set skip on ${epair_sync}a" \ 1478 "altq on outif bandwidth 30000b hfsc queue { default other1 other2 }" \ 1479 "queue default hfsc(linkshare 10000b default)" \ 1480 "queue other1 hfsc(linkshare 10000b)" \ 1481 "queue other2 hfsc(linkshare 10000b)" \ 1482 "block" \ 1483 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1484 "pass in on inif inet proto udp queue other1 keep state" \ 1485 "pass out on outif inet proto udp keep state" 1486 1487 jexec two pfctl -e 1488 pft_set_rules two \ 1489 "set debug loud" \ 1490 "set skip on ${epair_sync}b" \ 1491 "altq on outif bandwidth 30000b hfsc queue { default other2 other1 }" \ 1492 "queue default hfsc(linkshare 10000b default)" \ 1493 "queue other2 hfsc(linkshare 10000b)" \ 1494 "queue other1 hfsc(linkshare 10000b)" \ 1495 "block" \ 1496 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1497 "pass out on outif inet proto udp keep state" 1498 1499 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1500 ${common_dir}/pft_ping.py \ 1501 --ping-type=udp \ 1502 --sendif ${epair_one}b \ 1503 --fromaddr 198.51.100.254 \ 1504 --to 203.0.113.254 \ 1505 --recvif ${epair_out_one}b 1506 1507 queues_one=$(mktemp) 1508 jexec one pfctl -qvsq | normalize_pfctl_s > $queues_one 1509 echo " === queues one === " 1510 cat $queues_one 1511 grep -qE 'queue other1 on outif .* pkts: 1 ' $queues_one || atf_fail 'Packets not sent through queue "other1"' 1512 1513 # Allow time for sync 1514 sleep 2 1515 1516 # Force the next request to go through the 2nd router 1517 route change -net 203.0.113.0/24 198.51.100.17 1518 1519 # Send a packet through router "two". It lacks the inbound rule 1520 # but the inbound state should have been pfsynced from router "one" 1521 # including altq queuing information. However the queues are created 1522 # on router "two" in different order and we only sync queue index, 1523 # so the packet ends up in a different queue. One must have identical 1524 # queue set on both routers! 1525 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1526 ${common_dir}/pft_ping.py \ 1527 --ping-type=udp \ 1528 --sendif ${epair_two}b \ 1529 --fromaddr 198.51.100.254 \ 1530 --to 203.0.113.254 \ 1531 --recvif ${epair_out_two}b 1532 1533 queues_two=$(mktemp) 1534 jexec two pfctl -qvsq | normalize_pfctl_s > $queues_two 1535 echo " === queues two === " 1536 cat $queues_two 1537 grep -qE 'queue other2 on outif .* pkts: 1 ' $queues_two || atf_fail 'Packets not sent through queue "other2"' 1538} 1539 1540altq_queues_cleanup() 1541{ 1542 # Interface detaching seems badly broken in altq. If interfaces are 1543 # destroyed when shutting down the vnet and then pf is unloaded, it will 1544 # cause a kernel crash. Work around the issue by first flushing the 1545 # pf rulesets 1546 jexec one pfctl -F all 1547 jexec two pfctl -F all 1548 pfsynct_cleanup 1549} 1550 1551atf_test_case "rt_af" "cleanup" 1552rt_af_head() 1553{ 1554 atf_set descr 'Test if the rt_af is synced' 1555 atf_set require.user root 1556 atf_set require.progs python3 scapy 1557} 1558 1559rt_af_body() 1560{ 1561 route_to_common_head 1500 1562 1563 jexec one pfctl -e 1564 pft_set_rules one \ 1565 "set skip on ${epair_sync}a" \ 1566 "block" \ 1567 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1568 "pass in on inif \ 1569 route-to (outif 203.0.113.254) prefer-ipv6-nexthop \ 1570 inet proto udp \ 1571 to 203.0.113.241 \ 1572 keep state" \ 1573 "pass in on inif \ 1574 route-to (outif 2001:db8:4200::fe) prefer-ipv6-nexthop \ 1575 inet proto udp \ 1576 to 203.0.113.242 \ 1577 keep state" \ 1578 "pass in on inif \ 1579 route-to (outif 2001:db8:4200::fe) prefer-ipv6-nexthop \ 1580 inet6 proto udp \ 1581 to 2001:db8:4200::f3 \ 1582 keep state" \ 1583 "pass out on outif inet proto udp keep state (no-sync)" \ 1584 "pass out on outif inet6 proto udp keep state (no-sync)" 1585 1586 jexec two pfctl -e 1587 pft_set_rules two \ 1588 "set debug loud" \ 1589 "set skip on ${epair_sync}b" \ 1590 "block" \ 1591 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1592 1593 # IPv4 packet over IPv4 gateway 1594 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1595 ${common_dir}/pft_ping.py \ 1596 --ping-type=udp \ 1597 --sendif ${epair_one}b \ 1598 --fromaddr 198.51.100.254 \ 1599 --to 203.0.113.241 \ 1600 --recvif ${epair_out_one}b 1601 1602 # FIXME: Routing IPv4 packets over IPv6 gateways with gateway added 1603 # with `ndp -s` causes the static NDP entry to become expired. 1604 # Pfsync tests don't use "servers" which can reply to ARP and NDP, 1605 # but such static entry for gateway and only check if a stateless 1606 # ICMP or UDP packet is forward through. 1607 # 1608 # IPv4 packert over IPv6 gateway 1609 #atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1610 # ${common_dir}/pft_ping.py \ 1611 # --ping-type=udp \ 1612 # --sendif ${epair_one}b \ 1613 # --fromaddr 198.51.100.254 \ 1614 # --to 203.0.113.242 \ 1615 # --recvif ${epair_out_one}b 1616 1617 # IPv6 packet over IPv6 gateway 1618 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1619 ${common_dir}/pft_ping.py \ 1620 --ping-type=udp \ 1621 --sendif ${epair_one}b \ 1622 --fromaddr 2001:db8:4211::fe \ 1623 --to 2001:db8:4200::f3 \ 1624 --recvif ${epair_out_one}b 1625 1626 sleep 5 # Wait for pfsync 1627 1628 states_one=$(mktemp) 1629 states_two=$(mktemp) 1630 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1631 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1632 1633 echo " === states one === " 1634 cat $states_one 1635 echo " === states two === " 1636 cat $states_two 1637 1638 for state_regexp in \ 1639 "all udp 203.0.113.241:9 <- 198.51.100.254 .* route-to: 203.0.113.254@outif origif: inif" \ 1640 "all udp 2001:db8:4200::f3\[9\] <- 2001:db8:4211::fe .* route-to: 2001:db8:4200::fe@outif origif: inif" \ 1641 ; do 1642 grep -qE "${state_regexp}" $states_two || atf_fail "State not found for '${state_regexp}' on router two" 1643 done 1644} 1645 1646rt_af_cleanup() 1647{ 1648 jexec one pfctl -qvvsr 1649 jexec one pfctl -qvvss 1650 jexec one arp -an 1651 jexec one ndp -an 1652 pfsynct_cleanup 1653} 1654 1655atf_init_test_cases() 1656{ 1657 atf_add_test_case "basic" 1658 atf_add_test_case "basic_defer" 1659 atf_add_test_case "defer" 1660 atf_add_test_case "bulk" 1661 atf_add_test_case "pbr" 1662 atf_add_test_case "pfsync_pbr" 1663 atf_add_test_case "ipsec" 1664 atf_add_test_case "timeout" 1665 atf_add_test_case "basic_ipv6_unicast" 1666 atf_add_test_case "basic_ipv6" 1667 atf_add_test_case "rtable" 1668 atf_add_test_case "route_to_1301" 1669 atf_add_test_case "route_to_1301_bad_ruleset" 1670 atf_add_test_case "route_to_1301_bad_rpool" 1671 atf_add_test_case "route_to_1400_bad_ruleset" 1672 atf_add_test_case "route_to_1400_bad_ifname" 1673 atf_add_test_case "af_to_in_floating" 1674 atf_add_test_case "af_to_in_if_bound" 1675 atf_add_test_case "af_to_out_if_bound" 1676 atf_add_test_case "tag" 1677 atf_add_test_case "altq_queues" 1678 atf_add_test_case "rt_af" 1679} 1680