1# 2# SPDX-License-Identifier: BSD-2-Clause 3# 4# Copyright (c) 2018 Orange Business Services 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 1. Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# 2. Redistributions in binary form must reproduce the above copyright 12# notice, this list of conditions and the following disclaimer in the 13# documentation and/or other materials provided with the distribution. 14# 15# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25# SUCH DAMAGE. 26 27. $(atf_get_srcdir)/utils.subr 28 29common_dir=$(atf_get_srcdir)/../common 30 31atf_test_case "basic" "cleanup" 32basic_head() 33{ 34 atf_set descr 'Basic pfsync test' 35 atf_set require.user root 36} 37 38basic_body() 39{ 40 common_body 41} 42 43common_body() 44{ 45 defer=$1 46 pfsynct_init 47 48 epair_sync=$(vnet_mkepair) 49 epair_one=$(vnet_mkepair) 50 epair_two=$(vnet_mkepair) 51 52 vnet_mkjail one ${epair_one}a ${epair_sync}a 53 vnet_mkjail two ${epair_two}a ${epair_sync}b 54 55 # pfsync interface 56 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 57 jexec one ifconfig ${epair_one}a 198.51.100.1/24 up 58 jexec one ifconfig pfsync0 \ 59 syncdev ${epair_sync}a \ 60 maxupd 1 \ 61 $defer \ 62 up 63 jexec two ifconfig ${epair_two}a 198.51.100.2/24 up 64 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 65 jexec two ifconfig pfsync0 \ 66 syncdev ${epair_sync}b \ 67 maxupd 1 \ 68 $defer \ 69 up 70 71 # Enable pf! 72 jexec one pfctl -e 73 pft_set_rules one \ 74 "set skip on ${epair_sync}a" \ 75 "pass out keep state" 76 jexec two pfctl -e 77 pft_set_rules two \ 78 "set skip on ${epair_sync}b" \ 79 "pass out keep state" 80 81 hostid_one=$(jexec one pfctl -si -v | awk '/Hostid:/ { gsub(/0x/, "", $2); printf($2); }') 82 83 ifconfig ${epair_one}b 198.51.100.254/24 up 84 85 ping -c 1 -S 198.51.100.254 198.51.100.1 86 87 # Give pfsync time to do its thing 88 sleep 2 89 90 if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 91 grep 198.51.100.254 ; then 92 atf_fail "state not found on synced host" 93 fi 94 95 if ! jexec two pfctl -sc | grep ""${hostid_one}""; 96 then 97 jexec two pfctl -sc 98 atf_fail "HostID for host one not found on two" 99 fi 100} 101 102basic_cleanup() 103{ 104 pfsynct_cleanup 105} 106 107atf_test_case "basic_defer" "cleanup" 108basic_defer_head() 109{ 110 atf_set descr 'Basic defer mode pfsync test' 111 atf_set require.user root 112} 113 114basic_defer_body() 115{ 116 common_body defer 117} 118 119basic_defer_cleanup() 120{ 121 pfsynct_cleanup 122} 123 124atf_test_case "defer" "cleanup" 125defer_head() 126{ 127 atf_set descr 'Defer mode pfsync test' 128 atf_set require.user root 129 atf_set require.progs python3 scapy 130} 131 132defer_body() 133{ 134 pfsynct_init 135 136 epair_sync=$(vnet_mkepair) 137 epair_in=$(vnet_mkepair) 138 epair_out=$(vnet_mkepair) 139 140 vnet_mkjail alcatraz ${epair_sync}a ${epair_in}a ${epair_out}a 141 142 jexec alcatraz ifconfig ${epair_sync}a 192.0.2.1/24 up 143 jexec alcatraz ifconfig ${epair_out}a 198.51.100.1/24 up 144 jexec alcatraz ifconfig ${epair_in}a 203.0.113.1/24 up 145 jexec alcatraz arp -s 203.0.113.2 00:01:02:03:04:05 146 jexec alcatraz sysctl net.inet.ip.forwarding=1 147 148 # Set a long defer delay 149 jexec alcatraz sysctl net.pfsync.defer_delay=2500 150 151 jexec alcatraz ifconfig pfsync0 \ 152 syncdev ${epair_sync}a \ 153 maxupd 1 \ 154 defer \ 155 up 156 157 ifconfig ${epair_sync}b 192.0.2.2/24 up 158 ifconfig ${epair_out}b 198.51.100.2/24 up 159 ifconfig ${epair_in}b up 160 route add -net 203.0.113.0/24 198.51.100.1 161 162 # Enable pf 163 jexec alcatraz sysctl net.pf.filter_local=0 164 jexec alcatraz pfctl -e 165 pft_set_rules alcatraz \ 166 "set skip on ${epair_sync}a" \ 167 "pass keep state" 168 169 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 170 $(atf_get_srcdir)/pfsync_defer.py \ 171 --syncdev ${epair_sync}b \ 172 --indev ${epair_in}b \ 173 --outdev ${epair_out}b 174 175 # Now disable defer mode and expect failure. 176 jexec alcatraz ifconfig pfsync0 -defer 177 178 # Flush state 179 pft_set_rules alcatraz \ 180 "set skip on ${epair_sync}a" \ 181 "pass keep state" 182 183 atf_check -s exit:3 env PYTHONPATH=${common_dir} \ 184 $(atf_get_srcdir)/pfsync_defer.py \ 185 --syncdev ${epair_sync}b \ 186 --indev ${epair_in}b \ 187 --outdev ${epair_out}b 188} 189 190defer_cleanup() 191{ 192 pfsynct_cleanup 193} 194 195atf_test_case "bulk" "cleanup" 196bulk_head() 197{ 198 atf_set descr 'Test bulk updates' 199 atf_set require.user root 200} 201 202bulk_body() 203{ 204 pfsynct_init 205 206 epair_sync=$(vnet_mkepair) 207 epair_one=$(vnet_mkepair) 208 epair_two=$(vnet_mkepair) 209 210 vnet_mkjail one ${epair_one}a ${epair_sync}a 211 vnet_mkjail two ${epair_two}a ${epair_sync}b 212 213 # pfsync interface 214 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 215 jexec one ifconfig ${epair_sync}a mtu 9000 216 jexec one ifconfig ${epair_one}a 198.51.100.1/24 up 217 jexec one ifconfig pfsync0 \ 218 syncdev ${epair_sync}a \ 219 up mtu 9000 220 jexec two ifconfig ${epair_two}a 198.51.100.2/24 up 221 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 222 jexec two ifconfig ${epair_sync}b mtu 9000 223 224 # Enable pf 225 jexec one pfctl -e 226 pft_set_rules one \ 227 "set skip on ${epair_sync}a" \ 228 "pass keep state" 229 jexec two pfctl -e 230 pft_set_rules two \ 231 "set skip on ${epair_sync}b" \ 232 "pass keep state" 233 234 ifconfig ${epair_one}b 198.51.100.254/24 up 235 236 # Create states prior to setting up pfsync 237 ${common_dir}/pft_synflood.py \ 238 --sendif ${epair_one}b \ 239 --to 198.51.100.1 \ 240 --count 500 241 ping -c 1 -S 198.51.100.254 198.51.100.1 242 243 # Wait before setting up pfsync on two, so we don't accidentally catch 244 # the update anyway. 245 sleep 1 246 247 # Now set up pfsync in jail two 248 jexec two ifconfig pfsync0 \ 249 syncdev ${epair_sync}b \ 250 up 251 252 # Give pfsync time to do its thing 253 sleep 2 254 255 jexec two pfctl -s states 256 if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 257 grep 198.51.100.2 ; then 258 atf_fail "state not found on synced host" 259 fi 260} 261 262bulk_cleanup() 263{ 264 pfsynct_cleanup 265} 266 267atf_test_case "pbr" "cleanup" 268pbr_head() 269{ 270 atf_set descr 'route_to and reply_to directives test' 271 atf_set require.user root 272 atf_set timeout '600' 273} 274 275pbr_body() 276{ 277 pbr_common_body 278} 279 280pbr_cleanup() 281{ 282 pbr_common_cleanup 283} 284 285atf_test_case "pfsync_pbr" "cleanup" 286pfsync_pbr_head() 287{ 288 atf_set descr 'route_to and reply_to directives pfsync test' 289 atf_set require.user root 290 atf_set timeout '600' 291} 292 293pfsync_pbr_body() 294{ 295 pbr_common_body backup_promotion 296} 297 298pfsync_pbr_cleanup() 299{ 300 pbr_common_cleanup 301} 302 303pbr_common_body() 304{ 305 # + builds bellow topology and initiate a single ping session 306 # from client to server. 307 # + gw* forward traffic through pbr not fib lookups. 308 # + if backup_promotion arg is given, a carp failover event occurs 309 # during the ping session on both gateways. 310 # ┌──────┐ 311 # │client│ 312 # └───┬──┘ 313 # │ 314 # ┌───┴───┐ 315 # │bridge0│ 316 # └┬─────┬┘ 317 # │ │ 318 # ┌────────────────┴─┐ ┌─┴────────────────┐ 319 # │gw_route_to_master├─┤gw_route_to_backup│ 320 # └────────────────┬─┘ └─┬────────────────┘ 321 # │ │ 322 # ┌┴─────┴┐ 323 # │bridge1│ 324 # └┬─────┬┘ 325 # │ │ 326 # ┌────────────────┴─┐ ┌─┴────────────────┐ 327 # │gw_reply_to_master├─┤gw_reply_to_backup│ 328 # └────────────────┬─┘ └─┬────────────────┘ 329 # │ │ 330 # ┌┴─────┴┐ 331 # │bridge2│ 332 # └───┬───┘ 333 # │ 334 # ┌───┴──┐ 335 # │server│ 336 # └──────┘ 337 338 if ! kldstat -q -m carp 339 then 340 atf_skip "This test requires carp" 341 fi 342 pfsynct_init 343 vnet_init_bridge 344 345 bridge0=$(vnet_mkbridge) 346 bridge1=$(vnet_mkbridge) 347 bridge2=$(vnet_mkbridge) 348 349 epair_sync_gw_route_to=$(vnet_mkepair) 350 epair_sync_gw_reply_to=$(vnet_mkepair) 351 epair_client_bridge0=$(vnet_mkepair) 352 353 epair_gw_route_to_master_bridge0=$(vnet_mkepair) 354 epair_gw_route_to_backup_bridge0=$(vnet_mkepair) 355 epair_gw_route_to_master_bridge1=$(vnet_mkepair) 356 epair_gw_route_to_backup_bridge1=$(vnet_mkepair) 357 358 epair_gw_reply_to_master_bridge1=$(vnet_mkepair) 359 epair_gw_reply_to_backup_bridge1=$(vnet_mkepair) 360 epair_gw_reply_to_master_bridge2=$(vnet_mkepair) 361 epair_gw_reply_to_backup_bridge2=$(vnet_mkepair) 362 363 epair_server_bridge2=$(vnet_mkepair) 364 365 ifconfig ${bridge0} up 366 ifconfig ${epair_client_bridge0}b up 367 ifconfig ${epair_gw_route_to_master_bridge0}b up 368 ifconfig ${epair_gw_route_to_backup_bridge0}b up 369 ifconfig ${bridge0} \ 370 addm ${epair_client_bridge0}b \ 371 addm ${epair_gw_route_to_master_bridge0}b \ 372 addm ${epair_gw_route_to_backup_bridge0}b 373 374 ifconfig ${bridge1} up 375 ifconfig ${epair_gw_route_to_master_bridge1}b up 376 ifconfig ${epair_gw_route_to_backup_bridge1}b up 377 ifconfig ${epair_gw_reply_to_master_bridge1}b up 378 ifconfig ${epair_gw_reply_to_backup_bridge1}b up 379 ifconfig ${bridge1} \ 380 addm ${epair_gw_route_to_master_bridge1}b \ 381 addm ${epair_gw_route_to_backup_bridge1}b \ 382 addm ${epair_gw_reply_to_master_bridge1}b \ 383 addm ${epair_gw_reply_to_backup_bridge1}b 384 385 ifconfig ${bridge2} up 386 ifconfig ${epair_gw_reply_to_master_bridge2}b up 387 ifconfig ${epair_gw_reply_to_backup_bridge2}b up 388 ifconfig ${epair_server_bridge2}b up 389 ifconfig ${bridge2} \ 390 addm ${epair_gw_reply_to_master_bridge2}b \ 391 addm ${epair_gw_reply_to_backup_bridge2}b \ 392 addm ${epair_server_bridge2}b 393 394 vnet_mkjail client ${epair_client_bridge0}a 395 jexec client hostname client 396 vnet_mkjail gw_route_to_master \ 397 ${epair_gw_route_to_master_bridge0}a \ 398 ${epair_gw_route_to_master_bridge1}a \ 399 ${epair_sync_gw_route_to}a 400 jexec gw_route_to_master hostname gw_route_to_master 401 vnet_mkjail gw_route_to_backup \ 402 ${epair_gw_route_to_backup_bridge0}a \ 403 ${epair_gw_route_to_backup_bridge1}a \ 404 ${epair_sync_gw_route_to}b 405 jexec gw_route_to_backup hostname gw_route_to_backup 406 vnet_mkjail gw_reply_to_master \ 407 ${epair_gw_reply_to_master_bridge1}a \ 408 ${epair_gw_reply_to_master_bridge2}a \ 409 ${epair_sync_gw_reply_to}a 410 jexec gw_reply_to_master hostname gw_reply_to_master 411 vnet_mkjail gw_reply_to_backup \ 412 ${epair_gw_reply_to_backup_bridge1}a \ 413 ${epair_gw_reply_to_backup_bridge2}a \ 414 ${epair_sync_gw_reply_to}b 415 jexec gw_reply_to_backup hostname gw_reply_to_backup 416 vnet_mkjail server ${epair_server_bridge2}a 417 jexec server hostname server 418 419 jexec client ifconfig ${epair_client_bridge0}a inet 198.18.0.1/24 up 420 jexec client route add 198.18.2.0/24 198.18.0.10 421 422 jexec gw_route_to_master ifconfig ${epair_sync_gw_route_to}a \ 423 inet 198.19.10.1/24 up 424 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge0}a \ 425 inet 198.18.0.8/24 up 426 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge0}a \ 427 alias 198.18.0.10/32 vhid 10 pass 3WjvVVw7 advskew 50 428 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge1}a \ 429 inet 198.18.1.8/24 up 430 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge1}a \ 431 alias 198.18.1.10/32 vhid 11 pass 3WjvVVw7 advskew 50 432 jexec gw_route_to_master sysctl net.inet.ip.forwarding=1 433 jexec gw_route_to_master sysctl net.inet.carp.preempt=1 434 435 vnet_ifrename_jail gw_route_to_master ${epair_sync_gw_route_to}a if_pfsync 436 vnet_ifrename_jail gw_route_to_master ${epair_gw_route_to_master_bridge0}a if_br0 437 vnet_ifrename_jail gw_route_to_master ${epair_gw_route_to_master_bridge1}a if_br1 438 439 jexec gw_route_to_master ifconfig pfsync0 \ 440 syncpeer 198.19.10.2 \ 441 syncdev if_pfsync \ 442 maxupd 1 \ 443 up 444 pft_set_rules gw_route_to_master \ 445 "keep_state = 'tag auth_packet keep state'" \ 446 "set timeout { icmp.first 120, icmp.error 60 }" \ 447 "block log all" \ 448 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 449 "pass quick on { if_br0 if_br1 } proto carp keep state (no-sync)" \ 450 "block drop in quick to 224.0.0.18/32" \ 451 "pass out quick tagged auth_packet keep state" \ 452 "pass in quick log on if_br0 route-to (if_br1 198.18.1.20) proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 \$keep_state" 453 jexec gw_route_to_master pfctl -e 454 455 jexec gw_route_to_backup ifconfig ${epair_sync_gw_route_to}b \ 456 inet 198.19.10.2/24 up 457 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge0}a \ 458 inet 198.18.0.9/24 up 459 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge0}a \ 460 alias 198.18.0.10/32 vhid 10 pass 3WjvVVw7 advskew 100 461 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge1}a \ 462 inet 198.18.1.9/24 up 463 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge1}a \ 464 alias 198.18.1.10/32 vhid 11 pass 3WjvVVw7 advskew 100 465 jexec gw_route_to_backup sysctl net.inet.ip.forwarding=1 466 jexec gw_route_to_backup sysctl net.inet.carp.preempt=1 467 468 vnet_ifrename_jail gw_route_to_backup ${epair_sync_gw_route_to}b if_pfsync 469 vnet_ifrename_jail gw_route_to_backup ${epair_gw_route_to_backup_bridge0}a if_br0 470 vnet_ifrename_jail gw_route_to_backup ${epair_gw_route_to_backup_bridge1}a if_br1 471 472 jexec gw_route_to_backup ifconfig pfsync0 \ 473 syncpeer 198.19.10.1 \ 474 syncdev if_pfsync \ 475 up 476 pft_set_rules gw_route_to_backup \ 477 "keep_state = 'tag auth_packet keep state'" \ 478 "set timeout { icmp.first 120, icmp.error 60 }" \ 479 "block log all" \ 480 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 481 "pass quick on { if_br0 if_br1 } proto carp keep state (no-sync)" \ 482 "block drop in quick to 224.0.0.18/32" \ 483 "pass out quick tagged auth_packet keep state" \ 484 "pass in quick log on if_br0 route-to (if_br1 198.18.1.20) proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 \$keep_state" 485 jexec gw_route_to_backup pfctl -e 486 487 jexec gw_reply_to_master ifconfig ${epair_sync_gw_reply_to}a \ 488 inet 198.19.20.1/24 up 489 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge1}a \ 490 inet 198.18.1.18/24 up 491 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge1}a \ 492 alias 198.18.1.20/32 vhid 21 pass 3WjvVVw7 advskew 50 493 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge2}a \ 494 inet 198.18.2.18/24 up 495 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge2}a \ 496 alias 198.18.2.20/32 vhid 22 pass 3WjvVVw7 advskew 50 497 jexec gw_reply_to_master sysctl net.inet.ip.forwarding=1 498 jexec gw_reply_to_master sysctl net.inet.carp.preempt=1 499 500 vnet_ifrename_jail gw_reply_to_master ${epair_sync_gw_reply_to}a if_pfsync 501 vnet_ifrename_jail gw_reply_to_master ${epair_gw_reply_to_master_bridge1}a if_br1 502 vnet_ifrename_jail gw_reply_to_master ${epair_gw_reply_to_master_bridge2}a if_br2 503 504 jexec gw_reply_to_master ifconfig pfsync0 \ 505 syncpeer 198.19.20.2 \ 506 syncdev if_pfsync \ 507 maxupd 1 \ 508 up 509 pft_set_rules gw_reply_to_master \ 510 "set timeout { icmp.first 120, icmp.error 60 }" \ 511 "block log all" \ 512 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 513 "pass quick on { if_br1 if_br2 } proto carp keep state (no-sync)" \ 514 "block drop in quick to 224.0.0.18/32" \ 515 "pass out quick on if_br2 reply-to (if_br1 198.18.1.10) tagged auth_packet_reply_to keep state" \ 516 "pass in quick log on if_br1 proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 tag auth_packet_reply_to keep state" 517 jexec gw_reply_to_master pfctl -e 518 519 jexec gw_reply_to_backup ifconfig ${epair_sync_gw_reply_to}b \ 520 inet 198.19.20.2/24 up 521 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge1}a \ 522 inet 198.18.1.19/24 up 523 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge1}a \ 524 alias 198.18.1.20/32 vhid 21 pass 3WjvVVw7 advskew 100 525 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge2}a \ 526 inet 198.18.2.19/24 up 527 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge2}a \ 528 alias 198.18.2.20/32 vhid 22 pass 3WjvVVw7 advskew 100 529 jexec gw_reply_to_backup sysctl net.inet.ip.forwarding=1 530 jexec gw_reply_to_backup sysctl net.inet.carp.preempt=1 531 532 vnet_ifrename_jail gw_reply_to_backup ${epair_sync_gw_reply_to}b if_pfsync 533 vnet_ifrename_jail gw_reply_to_backup ${epair_gw_reply_to_backup_bridge1}a if_br1 534 vnet_ifrename_jail gw_reply_to_backup ${epair_gw_reply_to_backup_bridge2}a if_br2 535 536 jexec gw_reply_to_backup ifconfig pfsync0 \ 537 syncpeer 198.19.20.1 \ 538 syncdev if_pfsync \ 539 up 540 pft_set_rules gw_reply_to_backup \ 541 "set timeout { icmp.first 120, icmp.error 60 }" \ 542 "block log all" \ 543 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 544 "pass quick on { if_br1 if_br2 } proto carp keep state (no-sync)" \ 545 "block drop in quick to 224.0.0.18/32" \ 546 "pass out quick on if_br2 reply-to (if_br1 198.18.1.10) tagged auth_packet_reply_to keep state" \ 547 "pass in quick log on if_br1 proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 tag auth_packet_reply_to keep state" 548 jexec gw_reply_to_backup pfctl -e 549 550 jexec server ifconfig ${epair_server_bridge2}a inet 198.18.2.1/24 up 551 jexec server route add 198.18.0.0/24 198.18.2.20 552 553 # Waiting for platform to settle 554 while ! jexec gw_route_to_backup ifconfig | grep 'carp: BACKUP' 555 do 556 sleep 1 557 done 558 while ! jexec gw_reply_to_backup ifconfig | grep 'carp: BACKUP' 559 do 560 sleep 1 561 done 562 while ! jexec client ping -c 10 198.18.2.1 | grep ', 0.0% packet loss' 563 do 564 sleep 1 565 done 566 567 # Checking cluster members pf.conf checksums match 568 gw_route_to_master_checksum=$(jexec gw_route_to_master pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 569 gw_route_to_backup_checksum=$(jexec gw_route_to_backup pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 570 gw_reply_to_master_checksum=$(jexec gw_reply_to_master pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 571 gw_reply_to_backup_checksum=$(jexec gw_reply_to_backup pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 572 if [ "$gw_route_to_master_checksum" != "$gw_route_to_backup_checksum" ] 573 then 574 atf_fail "gw_route_to cluster members pf.conf do not match each others" 575 fi 576 if [ "$gw_reply_to_master_checksum" != "$gw_reply_to_backup_checksum" ] 577 then 578 atf_fail "gw_reply_to cluster members pf.conf do not match each others" 579 fi 580 581 # Creating state entries 582 (jexec client ping -c 10 198.18.2.1 >ping.stdout) & 583 584 if [ "$1" = "backup_promotion" ] 585 then 586 sleep 1 587 jexec gw_route_to_backup ifconfig if_br0 vhid 10 advskew 0 588 jexec gw_route_to_backup ifconfig if_br1 vhid 11 advskew 0 589 jexec gw_reply_to_backup ifconfig if_br1 vhid 21 advskew 0 590 jexec gw_reply_to_backup ifconfig if_br2 vhid 22 advskew 0 591 fi 592 while ! grep -q -e 'packet loss' ping.stdout 593 do 594 sleep 1 595 done 596 597 atf_check -s exit:0 -e ignore -o ignore grep ', 0.0% packet loss' ping.stdout 598} 599 600pbr_common_cleanup() 601{ 602 pft_cleanup 603} 604 605atf_test_case "ipsec" "cleanup" 606ipsec_head() 607{ 608 atf_set descr 'Transport pfsync over IPSec' 609 atf_set require.user root 610} 611 612ipsec_body() 613{ 614 if ! sysctl -q kern.features.ipsec >/dev/null ; then 615 atf_skip "This test requires ipsec" 616 fi 617 618 # Run the common test, to set up pfsync 619 common_body 620 621 # But we want unicast pfsync 622 jexec one ifconfig pfsync0 syncpeer 192.0.2.2 623 jexec two ifconfig pfsync0 syncpeer 192.0.2.1 624 625 # Flush existing states 626 jexec one pfctl -Fs 627 jexec two pfctl -Fs 628 629 # Now define an ipsec policy to run over the epair_sync interfaces 630 echo "flush; 631 spdflush; 632 spdadd 192.0.2.1/32 192.0.2.2/32 any -P out ipsec esp/transport//require; 633 spdadd 192.0.2.2/32 192.0.2.1/32 any -P in ipsec esp/transport//require; 634 add 192.0.2.1 192.0.2.2 esp 0x1000 -E aes-gcm-16 \"12345678901234567890\"; 635 add 192.0.2.2 192.0.2.1 esp 0x1001 -E aes-gcm-16 \"12345678901234567890\";" \ 636 | jexec one setkey -c 637 638 echo "flush; 639 spdflush; 640 spdadd 192.0.2.2/32 192.0.2.1/32 any -P out ipsec esp/transport//require; 641 spdadd 192.0.2.1/32 192.0.2.2/32 any -P in ipsec esp/transport//require; 642 add 192.0.2.1 192.0.2.2 esp 0x1000 -E aes-gcm-16 \"12345678901234567891\"; 643 add 192.0.2.2 192.0.2.1 esp 0x1001 -E aes-gcm-16 \"12345678901234567891\";" \ 644 | jexec two setkey -c 645 646 # We've set incompatible keys, so pfsync will be broken. 647 ping -c 1 -S 198.51.100.254 198.51.100.1 648 649 # Give pfsync time to do its thing 650 sleep 2 651 652 if jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 653 grep 198.51.100.2 ; then 654 atf_fail "state synced although IPSec should have prevented it" 655 fi 656 657 # Flush existing states 658 jexec one pfctl -Fs 659 jexec two pfctl -Fs 660 661 # Fix the IPSec key to match 662 echo "flush; 663 spdflush; 664 spdadd 192.0.2.2/32 192.0.2.1/32 any -P out ipsec esp/transport//require; 665 spdadd 192.0.2.1/32 192.0.2.2/32 any -P in ipsec esp/transport//require; 666 add 192.0.2.1 192.0.2.2 esp 0x1000 -E aes-gcm-16 \"12345678901234567890\"; 667 add 192.0.2.2 192.0.2.1 esp 0x1001 -E aes-gcm-16 \"12345678901234567890\";" \ 668 | jexec two setkey -c 669 670 ping -c 1 -S 198.51.100.254 198.51.100.1 671 672 # Give pfsync time to do its thing 673 sleep 2 674 675 if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 676 grep 198.51.100.2 ; then 677 atf_fail "state not found on synced host" 678 fi 679} 680 681ipsec_cleanup() 682{ 683 pft_cleanup 684} 685 686atf_test_case "timeout" "cleanup" 687timeout_head() 688{ 689 atf_set descr 'Trigger pfsync_timeout()' 690 atf_set require.user root 691} 692 693timeout_body() 694{ 695 pft_init 696 697 vnet_mkjail one 698 699 jexec one ifconfig lo0 127.0.0.1/8 up 700 jexec one ifconfig lo0 inet6 ::1/128 up 701 702 pft_set_rules one \ 703 "pass all" 704 jexec one pfctl -e 705 jexec one ifconfig pfsync0 defer up 706 707 jexec one ping -c 1 ::1 708 jexec one ping -c 1 127.0.0.1 709 710 # Give pfsync_timeout() time to fire (a callout on a 1 second delay) 711 sleep 2 712} 713 714timeout_cleanup() 715{ 716 pft_cleanup 717} 718 719atf_test_case "basic_ipv6_unicast" "cleanup" 720basic_ipv6_unicast_head() 721{ 722 atf_set descr 'Basic pfsync test (IPv6)' 723 atf_set require.user root 724} 725 726basic_ipv6_unicast_body() 727{ 728 pfsynct_init 729 730 epair_sync=$(vnet_mkepair) 731 epair_one=$(vnet_mkepair) 732 epair_two=$(vnet_mkepair) 733 734 vnet_mkjail one ${epair_one}a ${epair_sync}a 735 vnet_mkjail two ${epair_two}a ${epair_sync}b 736 737 # pfsync interface 738 jexec one ifconfig ${epair_sync}a inet6 fd2c::1/64 no_dad up 739 jexec one ifconfig ${epair_one}a inet6 fd2b::1/64 no_dad up 740 jexec one ifconfig pfsync0 \ 741 syncdev ${epair_sync}a \ 742 syncpeer fd2c::2 \ 743 maxupd 1 \ 744 up 745 jexec two ifconfig ${epair_two}a inet6 fd2b::2/64 no_dad up 746 jexec two ifconfig ${epair_sync}b inet6 fd2c::2/64 no_dad up 747 jexec two ifconfig pfsync0 \ 748 syncdev ${epair_sync}b \ 749 syncpeer fd2c::1 \ 750 maxupd 1 \ 751 up 752 753 # Enable pf! 754 jexec one pfctl -e 755 pft_set_rules one \ 756 "block on ${epair_sync}a inet" \ 757 "pass out keep state" 758 jexec two pfctl -e 759 pft_set_rules two \ 760 "block on ${epair_sync}b inet" \ 761 "pass out keep state" 762 763 ifconfig ${epair_one}b inet6 fd2b::f0/64 no_dad up 764 765 ping6 -c 1 -S fd2b::f0 fd2b::1 766 767 # Give pfsync time to do its thing 768 sleep 2 769 770 if ! jexec two pfctl -s states | grep icmp | grep fd2b::1 | \ 771 grep fd2b::f0 ; then 772 atf_fail "state not found on synced host" 773 fi 774} 775 776basic_ipv6_unicast_cleanup() 777{ 778 pfsynct_cleanup 779} 780 781atf_test_case "basic_ipv6" "cleanup" 782basic_ipv6_head() 783{ 784 atf_set descr 'Basic pfsync test (IPv6)' 785 atf_set require.user root 786} 787 788basic_ipv6_body() 789{ 790 pfsynct_init 791 792 epair_sync=$(vnet_mkepair) 793 epair_one=$(vnet_mkepair) 794 epair_two=$(vnet_mkepair) 795 796 vnet_mkjail one ${epair_one}a ${epair_sync}a 797 vnet_mkjail two ${epair_two}a ${epair_sync}b 798 799 # pfsync interface 800 jexec one ifconfig ${epair_sync}a inet6 fd2c::1/64 no_dad up 801 jexec one ifconfig ${epair_one}a inet6 fd2b::1/64 no_dad up 802 jexec one ifconfig pfsync0 \ 803 syncdev ${epair_sync}a \ 804 syncpeer ff12::f0 \ 805 maxupd 1 \ 806 up 807 jexec two ifconfig ${epair_two}a inet6 fd2b::2/64 no_dad up 808 jexec two ifconfig ${epair_sync}b inet6 fd2c::2/64 no_dad up 809 jexec two ifconfig pfsync0 \ 810 syncdev ${epair_sync}b \ 811 syncpeer ff12::f0 \ 812 maxupd 1 \ 813 up 814 815 # Enable pf! 816 jexec one pfctl -e 817 pft_set_rules one \ 818 "block on ${epair_sync}a inet" \ 819 "pass out keep state" 820 jexec two pfctl -e 821 pft_set_rules two \ 822 "block on ${epair_sync}b inet" \ 823 "pass out keep state" 824 825 ifconfig ${epair_one}b inet6 fd2b::f0/64 no_dad up 826 827 ping6 -c 1 -S fd2b::f0 fd2b::1 828 829 # Give pfsync time to do its thing 830 sleep 2 831 832 if ! jexec two pfctl -s states | grep icmp | grep fd2b::1 | \ 833 grep fd2b::f0 ; then 834 atf_fail "state not found on synced host" 835 fi 836} 837 838basic_ipv6_cleanup() 839{ 840 pfsynct_cleanup 841} 842 843atf_test_case "rtable" "cleanup" 844rtable_head() 845{ 846 atf_set descr 'Test handling of invalid rtableid' 847 atf_set require.user root 848} 849 850rtable_body() 851{ 852 pfsynct_init 853 854 epair_sync=$(vnet_mkepair) 855 epair_one=$(vnet_mkepair) 856 epair_two=$(vnet_mkepair) 857 858 vnet_mkjail one ${epair_one}a ${epair_sync}a 859 vnet_mkjail two ${epair_two}a ${epair_sync}b 860 861 # pfsync interface 862 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 863 jexec one ifconfig ${epair_one}a 198.51.100.1/24 up 864 jexec one ifconfig pfsync0 \ 865 syncdev ${epair_sync}a \ 866 maxupd 1 \ 867 up 868 jexec two ifconfig ${epair_two}a 198.51.100.1/24 up 869 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 870 jexec two ifconfig pfsync0 \ 871 syncdev ${epair_sync}b \ 872 maxupd 1 \ 873 up 874 875 # Make life easy, give ${epair_two}a the same mac addrss as ${epair_one}a 876 mac=$(jexec one ifconfig ${epair_one}a | awk '/ether/ { print($2); }') 877 jexec two ifconfig ${epair_two}a ether ${mac} 878 879 # Enable pf! 880 jexec one /sbin/sysctl net.fibs=8 881 jexec one pfctl -e 882 pft_set_rules one \ 883 "set skip on ${epair_sync}a" \ 884 "pass rtable 3 keep state" 885 # No extra fibs in two 886 jexec two pfctl -e 887 pft_set_rules two \ 888 "set skip on ${epair_sync}b" \ 889 "pass keep state" 890 891 ifconfig ${epair_one}b 198.51.100.254/24 up 892 ifconfig ${epair_two}b 198.51.100.253/24 up 893 894 # Create a new state 895 env PYTHONPATH=${common_dir} \ 896 ${common_dir}/pft_ping.py \ 897 --sendif ${epair_one}b \ 898 --fromaddr 198.51.100.254 \ 899 --to 198.51.100.1 \ 900 --recvif ${epair_one}b 901 902 # Now 903 jexec one pfctl -ss -vv 904 sleep 2 905 906 # Now try to use that state on jail two 907 env PYTHONPATH=${common_dir} \ 908 ${common_dir}/pft_ping.py \ 909 --sendif ${epair_two}b \ 910 --fromaddr 198.51.100.254 \ 911 --to 198.51.100.1 \ 912 --recvif ${epair_two}b 913 914 echo one 915 jexec one pfctl -ss -vv 916 jexec one pfctl -sr -vv 917 echo two 918 jexec two pfctl -ss -vv 919 jexec two pfctl -sr -vv 920} 921 922rtable_cleanup() 923{ 924 pfsynct_cleanup 925} 926 927route_to_common_head() 928{ 929 # TODO: Extend setup_router_server_nat64 to create a 2nd router 930 931 pfsync_version=$1 932 shift 933 934 pfsynct_init 935 936 epair_sync=$(vnet_mkepair) 937 epair_one=$(vnet_mkepair) 938 epair_two=$(vnet_mkepair) 939 epair_out_one=$(vnet_mkepair) 940 epair_out_two=$(vnet_mkepair) 941 942 vnet_mkjail one ${epair_one}a ${epair_sync}a ${epair_out_one}a 943 vnet_mkjail two ${epair_two}a ${epair_sync}b ${epair_out_two}a 944 945 # pfsync interface 946 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 947 jexec one ifconfig ${epair_one}a 198.51.100.1/28 up 948 jexec one ifconfig ${epair_one}a inet6 2001:db8:4211::1/64 no_dad 949 jexec one ifconfig ${epair_one}a name inif 950 jexec one ifconfig ${epair_out_one}a 203.0.113.1/24 up 951 jexec one ifconfig ${epair_out_one}a inet6 2001:db8:4200::1/64 no_dad 952 jexec one ifconfig ${epair_out_one}a name outif 953 jexec one sysctl net.inet.ip.forwarding=1 954 jexec one sysctl net.inet6.ip6.forwarding=1 955 jexec one arp -s 203.0.113.254 00:01:02:00:00:04 956 jexec one ndp -s 2001:db8:4200::fe 00:01:02:00:00:06 957 jexec one ifconfig pfsync0 \ 958 syncdev ${epair_sync}a \ 959 maxupd 1 \ 960 version $pfsync_version \ 961 up 962 963 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 964 jexec two ifconfig ${epair_two}a 198.51.100.17/28 up 965 jexec two ifconfig ${epair_two}a inet6 2001:db8:4212::1/64 no_dad 966 jexec two ifconfig ${epair_two}a name inif 967 jexec two ifconfig ${epair_out_two}a 203.0.113.1/24 up 968 jexec two ifconfig ${epair_out_two}a inet6 2001:db8:4200::2/64 no_dad 969 jexec two ifconfig ${epair_out_two}a name outif 970 jexec two sysctl net.inet.ip.forwarding=1 971 jexec two sysctl net.inet6.ip6.forwarding=1 972 jexec two arp -s 203.0.113.254 00:01:02:00:00:04 973 jexec two ndp -s 2001:db8:4200::fe 00:01:02:00:00:06 974 jexec two ifconfig pfsync0 \ 975 syncdev ${epair_sync}b \ 976 maxupd 1 \ 977 version $pfsync_version \ 978 up 979 980 ifconfig ${epair_one}b 198.51.100.2/28 up 981 ifconfig ${epair_one}b inet6 2001:db8:4211::2/64 no_dad 982 ifconfig ${epair_two}b 198.51.100.18/28 up 983 ifconfig ${epair_two}b inet6 2001:db8:4212::2/64 no_dad 984 # Target is behind router "one" 985 route add -net 203.0.113.0/24 198.51.100.1 986 route add -inet6 -net 64:ff9b::/96 2001:db8:4211::1 987 988 ifconfig ${epair_two}b up 989 ifconfig ${epair_out_one}b up 990 ifconfig ${epair_out_two}b up 991} 992 993route_to_common_tail() 994{ 995 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 996 ${common_dir}/pft_ping.py \ 997 --sendif ${epair_one}b \ 998 --fromaddr 198.51.100.254 \ 999 --to 203.0.113.254 \ 1000 --recvif ${epair_out_one}b 1001 1002 # Allow time for sync 1003 sleep 2 1004 1005 states_one=$(mktemp) 1006 states_two=$(mktemp) 1007 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1008 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1009} 1010 1011atf_test_case "route_to_1301_body" "cleanup" 1012route_to_1301_head() 1013{ 1014 atf_set descr 'Test route-to with pfsync version 13.1' 1015 atf_set require.user root 1016 atf_set require.progs python3 scapy 1017} 1018 1019route_to_1301_body() 1020{ 1021 route_to_common_head 1301 1022 1023 jexec one pfctl -e 1024 pft_set_rules one \ 1025 "set skip on ${epair_sync}a" \ 1026 "pass out route-to (outif 203.0.113.254)" 1027 1028 jexec two pfctl -e 1029 pft_set_rules two \ 1030 "set skip on ${epair_sync}b" \ 1031 "pass out route-to (outif 203.0.113.254)" 1032 1033 route_to_common_tail 1034 1035 # Sanity check 1036 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1037 atf_fail "State missing on router one" 1038 1039 # With identical ruleset the routing information is recovered from the matching rule. 1040 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif' $states_two || 1041 atf_fail "State missing on router two" 1042 1043 true 1044} 1045 1046route_to_1301_cleanup() 1047{ 1048 pfsynct_cleanup 1049} 1050 1051atf_test_case "route_to_1301_bad_ruleset" "cleanup" 1052route_to_1301_bad_ruleset_head() 1053{ 1054 atf_set descr 'Test route-to with pfsync version 13.1 and incompatible ruleset' 1055 atf_set require.user root 1056 atf_set require.progs python3 scapy 1057} 1058 1059route_to_1301_bad_ruleset_body() 1060{ 1061 route_to_common_head 1301 1062 1063 jexec one pfctl -e 1064 pft_set_rules one \ 1065 "set skip on ${epair_sync}a" \ 1066 "pass out route-to (outif 203.0.113.254)" 1067 1068 jexec two pfctl -e 1069 pft_set_rules two \ 1070 "set debug loud" \ 1071 "set skip on ${epair_sync}b" \ 1072 "pass out route-to (outif 203.0.113.254)" \ 1073 "pass out proto tcp" 1074 1075 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1076 ${common_dir}/pft_ping.py \ 1077 --sendif ${epair_one}b \ 1078 --fromaddr 198.51.100.254 \ 1079 --to 203.0.113.254 \ 1080 --recvif ${epair_out_one}b 1081 1082 route_to_common_tail 1083 1084 # Sanity check 1085 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1086 atf_fail "State missing on router one" 1087 1088 # Different ruleset on each router means the routing information recovery 1089 # from rule is impossible. The state is not synced. 1090 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*' $states_two && 1091 atf_fail "State present on router two" 1092 1093 true 1094} 1095 1096route_to_1301_bad_ruleset_cleanup() 1097{ 1098 pfsynct_cleanup 1099} 1100 1101atf_test_case "route_to_1301_bad_rpool" "cleanup" 1102route_to_1301_bad_rpool_head() 1103{ 1104 atf_set descr 'Test route-to with pfsync version 13.1 and different interface' 1105 atf_set require.user root 1106 atf_set require.progs python3 scapy 1107} 1108 1109route_to_1301_bad_rpool_body() 1110{ 1111 route_to_common_head 1301 1112 1113 jexec one pfctl -e 1114 pft_set_rules one \ 1115 "set skip on ${epair_sync}a" \ 1116 "pass out route-to { (outif 203.0.113.254) (outif 203.0.113.254) }" 1117 1118 jexec two pfctl -e 1119 pft_set_rules two \ 1120 "set skip on ${epair_sync}b" \ 1121 "pass out route-to { (outif 203.0.113.254) (outif 203.0.113.254) }" 1122 1123 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1124 ${common_dir}/pft_ping.py \ 1125 --sendif ${epair_one}b \ 1126 --fromaddr 198.51.100.254 \ 1127 --to 203.0.113.254 \ 1128 --recvif ${epair_out_one}b 1129 1130 route_to_common_tail 1131 1132 # Sanity check 1133 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1134 atf_fail "State missing on router one" 1135 1136 # The ruleset is identical but since the redirection pool contains multiple interfaces 1137 # pfsync will not attempt to recover the routing information from the rule. 1138 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*' $states_two && 1139 atf_fail "State present on router two" 1140 1141 true 1142} 1143 1144route_to_1301_bad_rpool_cleanup() 1145{ 1146 pfsynct_cleanup 1147} 1148 1149atf_test_case "route_to_1400_bad_ruleset" "cleanup" 1150route_to_1400_bad_ruleset_head() 1151{ 1152 atf_set descr 'Test route-to with pfsync version 14.0' 1153 atf_set require.user root 1154 atf_set require.progs python3 scapy 1155} 1156 1157route_to_1400_bad_ruleset_body() 1158{ 1159 route_to_common_head 1400 1160 1161 jexec one pfctl -e 1162 pft_set_rules one \ 1163 "set skip on ${epair_sync}a" \ 1164 "pass out route-to (outif 203.0.113.254)" 1165 1166 jexec two pfctl -e 1167 pft_set_rules two \ 1168 "set skip on ${epair_sync}b" 1169 1170 route_to_common_tail 1171 1172 # Sanity check 1173 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1174 atf_fail "State missing on router one" 1175 1176 # Even with a different ruleset FreeBSD 14 syncs the state just fine. 1177 # There's no recovery involved, the pfsync packet contains the routing information. 1178 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .* route-to: 203.0.113.254@outif' $states_two || 1179 atf_fail "State missing on router two" 1180 1181 true 1182} 1183 1184route_to_1400_bad_ruleset_cleanup() 1185{ 1186 pfsynct_cleanup 1187} 1188 1189atf_test_case "route_to_1400_bad_ifname" "cleanup" 1190route_to_1400_bad_ifname_head() 1191{ 1192 atf_set descr 'Test route-to with pfsync version 14.0' 1193 atf_set require.user root 1194 atf_set require.progs python3 scapy 1195} 1196 1197route_to_1400_bad_ifname_body() 1198{ 1199 route_to_common_head 1400 1200 1201 jexec one pfctl -e 1202 pft_set_rules one \ 1203 "set skip on ${epair_sync}a" \ 1204 "pass out route-to (outif 203.0.113.254)" 1205 1206 jexec two pfctl -e 1207 jexec two ifconfig outif name outif_new 1208 pft_set_rules two \ 1209 "set skip on ${epair_sync}b" \ 1210 "pass out route-to (outif_new 203.0.113.254)" 1211 1212 route_to_common_tail 1213 1214 # Sanity check 1215 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1216 atf_fail "State missing on router one" 1217 1218 # Since FreeBSD 14 never attempts recovery of missing routing information 1219 # a state synced to a router with a different interface name is dropped. 1220 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*' $states_two && 1221 atf_fail "State present on router two" 1222 1223 true 1224} 1225 1226route_to_1400_bad_ifname_cleanup() 1227{ 1228 pfsynct_cleanup 1229} 1230 1231atf_test_case "af_to_in_floating" "cleanup" 1232af_to_in_floating_head() 1233{ 1234 atf_set descr 'Test syncing of states created by inbound af-to rules with floating states' 1235 atf_set require.user root 1236 atf_set require.progs python3 scapy 1237} 1238 1239af_to_in_floating_body() 1240{ 1241 route_to_common_head 1500 1242 1243 jexec one pfctl -e 1244 pft_set_rules one \ 1245 "set state-policy floating" \ 1246 "set skip on ${epair_sync}a" \ 1247 "block" \ 1248 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1249 "pass in on inif to 64:ff9b::/96 af-to inet from (outif) keep state" 1250 1251 jexec two pfctl -e 1252 pft_set_rules two \ 1253 "set skip on ${epair_sync}b" \ 1254 "block" \ 1255 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" 1256 1257 # ptf_ping can't deal with nat64, this test will fail but generate states 1258 atf_check -s exit:1 env PYTHONPATH=${common_dir} \ 1259 ${common_dir}/pft_ping.py \ 1260 --sendif ${epair_one}b \ 1261 --fromaddr 2001:db8:4201::fe \ 1262 --to 64:ff9b::203.0.113.254 \ 1263 --recvif ${epair_out_one}b 1264 1265 # Allow time for sync 1266 sleep 2 1267 1268 states_one=$(mktemp) 1269 states_two=$(mktemp) 1270 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1271 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1272 1273 # Sanity check 1274 grep -qE 'all ipv6-icmp 203.0.113.1 \(2001:db8:4201::fe\) -> 203.0.113.254:8 \(64:ff9b::cb00:71fe) .* rule 3 .* origif: inif' $states_one || 1275 atf_fail "State missing on router one" 1276 1277 grep -qE 'all ipv6-icmp 203.0.113.1 \(2001:db8:4201::fe\) -> 203.0.113.254:8 \(64:ff9b::cb00:71fe) .* origif: inif' $states_two || 1278 atf_fail "State missing on router two" 1279} 1280 1281af_to_in_floating_cleanup() 1282{ 1283 pfsynct_cleanup 1284} 1285 1286atf_test_case "af_to_in_if_bound" "cleanup" 1287af_to_in_if_bound_head() 1288{ 1289 atf_set descr 'Test syncing of states created by inbound af-to rules with if-bound states' 1290 atf_set require.user root 1291 atf_set require.progs python3 scapy 1292} 1293 1294af_to_in_if_bound_body() 1295{ 1296 route_to_common_head 1500 1297 1298 jexec one pfctl -e 1299 pft_set_rules one \ 1300 "set state-policy if-bound" \ 1301 "set skip on ${epair_sync}a" \ 1302 "block" \ 1303 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1304 "pass in on inif to 64:ff9b::/96 af-to inet from (outif) keep state" 1305 1306 jexec two pfctl -e 1307 pft_set_rules two \ 1308 "set skip on ${epair_sync}b" \ 1309 "block" \ 1310 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" 1311 1312 # ptf_ping can't deal with nat64, this test will fail but generate states 1313 atf_check -s exit:1 env PYTHONPATH=${common_dir} \ 1314 ${common_dir}/pft_ping.py \ 1315 --sendif ${epair_one}b \ 1316 --fromaddr 2001:db8:4201::fe \ 1317 --to 64:ff9b::203.0.113.254 \ 1318 --recvif ${epair_out_one}b 1319 1320 # Allow time for sync 1321 sleep 2 1322 1323 states_one=$(mktemp) 1324 states_two=$(mktemp) 1325 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1326 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1327 1328 # Sanity check 1329 grep -qE 'outif ipv6-icmp 203.0.113.1 \(2001:db8:4201::fe\) -> 203.0.113.254:8 \(64:ff9b::cb00:71fe) .* rule 3 .* origif: inif' $states_one || 1330 atf_fail "State missing on router one" 1331 1332 grep -qE 'outif ipv6-icmp 203.0.113.1 \(2001:db8:4201::fe\) -> 203.0.113.254:8 \(64:ff9b::cb00:71fe) .* origif: inif' $states_two || 1333 atf_fail "State missing on router two" 1334} 1335 1336af_to_in_if_bound_cleanup() 1337{ 1338 pfsynct_cleanup 1339} 1340 1341atf_test_case "af_to_out_if_bound" "cleanup" 1342af_to_out_if_bound_head() 1343{ 1344 atf_set descr 'Test syncing of states created by outbound af-to rules with if-bound states' 1345 atf_set require.user root 1346 atf_set require.progs python3 scapy 1347} 1348 1349af_to_out_if_bound_body() 1350{ 1351 route_to_common_head 1500 1352 1353 jexec one route add -inet6 -net 64:ff9b::/96 -iface outif 1354 jexec one sysctl net.inet6.ip6.forwarding=1 1355 1356 jexec one pfctl -e 1357 pft_set_rules one \ 1358 "set state-policy if-bound" \ 1359 "set skip on ${epair_sync}a" \ 1360 "block" \ 1361 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1362 "pass in on inif to 64:ff9b::/96 keep state" \ 1363 "pass out on outif to 64:ff9b::/96 af-to inet from (outif) keep state" 1364 1365 jexec two pfctl -e 1366 pft_set_rules two \ 1367 "set skip on ${epair_sync}b" \ 1368 "block" \ 1369 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" 1370 1371 # ptf_ping can't deal with nat64, this test will fail but generate states 1372 atf_check -s exit:1 env PYTHONPATH=${common_dir} \ 1373 ${common_dir}/pft_ping.py \ 1374 --sendif ${epair_one}b \ 1375 --fromaddr 2001:db8:4201::fe \ 1376 --to 64:ff9b::203.0.113.254 \ 1377 --recvif ${epair_out_one}b 1378 1379 # Allow time for sync 1380 sleep 2 1381 1382 states_one=$(mktemp) 1383 states_two=$(mktemp) 1384 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1385 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1386 1387 # Sanity check 1388 # st->orig_kif is the same as st->kif, so st->orig_kif is not printed. 1389 for state_regexp in \ 1390 "inif ipv6-icmp 64:ff9b::cb00:71fe\[128\] <- 2001:db8:4201::fe .* rule 3 .* creatorid: [0-9a-f]+" \ 1391 "outif icmp 203.0.113.1 \(64:ff9b::cb00:71fe\[8\]\) -> 203.0.113.254:8 \(2001:db8:4201::fe\) .* rule 4 .* creatorid: [0-9a-f]+" \ 1392 ; do 1393 grep -qE "${state_regexp}" $states_one || atf_fail "State not found for '${state_regexp}'" 1394 done 1395 1396 for state_regexp in \ 1397 "inif ipv6-icmp 64:ff9b::cb00:71fe\[128\] <- 2001:db8:4201::fe .* creatorid: [0-9a-f]+" \ 1398 "outif icmp 203.0.113.1 \(64:ff9b::cb00:71fe\[8\]\) -> 203.0.113.254:8 \(2001:db8:4201::fe\) .* creatorid: [0-9a-f]+" \ 1399 ; do 1400 grep -qE "${state_regexp}" $states_two || atf_fail "State not found for '${state_regexp}'" 1401 done 1402} 1403 1404af_to_out_if_bound_cleanup() 1405{ 1406 pfsynct_cleanup 1407} 1408 1409atf_test_case "tag" "cleanup" 1410tag_head() 1411{ 1412 atf_set descr 'Test if the pf tag is synced' 1413 atf_set require.user root 1414 atf_set require.progs python3 scapy 1415} 1416 1417tag_body() 1418{ 1419 route_to_common_head 1500 1420 1421 jexec one pfctl -e 1422 pft_set_rules one \ 1423 "set skip on ${epair_sync}a" \ 1424 "block" \ 1425 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1426 "pass in on inif inet proto udp tag sometag keep state" \ 1427 "pass out on outif tagged sometag keep state (no-sync)" 1428 1429 jexec two pfctl -e 1430 pft_set_rules two \ 1431 "set debug loud" \ 1432 "set skip on ${epair_sync}b" \ 1433 "block" \ 1434 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1435 "block tagged othertag" \ 1436 "pass out on outif tagged sometag keep state (no-sync)" 1437 1438 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1439 ${common_dir}/pft_ping.py \ 1440 --ping-type=udp \ 1441 --sendif ${epair_one}b \ 1442 --fromaddr 198.51.100.254 \ 1443 --to 203.0.113.254 \ 1444 --recvif ${epair_out_one}b 1445 1446 # Allow time for sync 1447 sleep 2 1448 1449 # Force the next request to go through the 2nd router 1450 route change -net 203.0.113.0/24 198.51.100.17 1451 1452 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1453 ${common_dir}/pft_ping.py \ 1454 --ping-type=udp \ 1455 --sendif ${epair_two}b \ 1456 --fromaddr 198.51.100.254 \ 1457 --to 203.0.113.254 \ 1458 --recvif ${epair_out_two}b 1459} 1460 1461tag_cleanup() 1462{ 1463 pfsynct_cleanup 1464} 1465 1466atf_test_case "altq_queues" "cleanup" 1467altq_queues_head() 1468{ 1469 atf_set descr 'Test if the altq queues are synced' 1470 atf_set require.user root 1471 atf_set require.progs python3 scapy 1472} 1473 1474altq_queues_body() 1475{ 1476 route_to_common_head 1500 1477 altq_init 1478 is_altq_supported hfsc 1479 1480 jexec one pfctl -e 1481 pft_set_rules one \ 1482 "set skip on ${epair_sync}a" \ 1483 "altq on outif bandwidth 30000b hfsc queue { default other1 other2 }" \ 1484 "queue default hfsc(linkshare 10000b default)" \ 1485 "queue other1 hfsc(linkshare 10000b)" \ 1486 "queue other2 hfsc(linkshare 10000b)" \ 1487 "block" \ 1488 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1489 "pass in on inif inet proto udp queue other1 keep state" \ 1490 "pass out on outif inet proto udp keep state" 1491 1492 jexec two pfctl -e 1493 pft_set_rules two \ 1494 "set debug loud" \ 1495 "set skip on ${epair_sync}b" \ 1496 "altq on outif bandwidth 30000b hfsc queue { default other2 other1 }" \ 1497 "queue default hfsc(linkshare 10000b default)" \ 1498 "queue other2 hfsc(linkshare 10000b)" \ 1499 "queue other1 hfsc(linkshare 10000b)" \ 1500 "block" \ 1501 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1502 "pass out on outif inet proto udp keep state" 1503 1504 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1505 ${common_dir}/pft_ping.py \ 1506 --ping-type=udp \ 1507 --sendif ${epair_one}b \ 1508 --fromaddr 198.51.100.254 \ 1509 --to 203.0.113.254 \ 1510 --recvif ${epair_out_one}b 1511 1512 queues_one=$(mktemp) 1513 jexec one pfctl -qvsq | normalize_pfctl_s > $queues_one 1514 echo " === queues one === " 1515 cat $queues_one 1516 grep -qE 'queue other1 on outif .* pkts: 1 ' $queues_one || atf_fail 'Packets not sent through queue "other1"' 1517 1518 # Allow time for sync 1519 sleep 2 1520 1521 # Force the next request to go through the 2nd router 1522 route change -net 203.0.113.0/24 198.51.100.17 1523 1524 # Send a packet through router "two". It lacks the inbound rule 1525 # but the inbound state should have been pfsynced from router "one" 1526 # including altq queuing information. However the queues are created 1527 # on router "two" in different order and we only sync queue index, 1528 # so the packet ends up in a different queue. One must have identical 1529 # queue set on both routers! 1530 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1531 ${common_dir}/pft_ping.py \ 1532 --ping-type=udp \ 1533 --sendif ${epair_two}b \ 1534 --fromaddr 198.51.100.254 \ 1535 --to 203.0.113.254 \ 1536 --recvif ${epair_out_two}b 1537 1538 queues_two=$(mktemp) 1539 jexec two pfctl -qvsq | normalize_pfctl_s > $queues_two 1540 echo " === queues two === " 1541 cat $queues_two 1542 grep -qE 'queue other2 on outif .* pkts: 1 ' $queues_two || atf_fail 'Packets not sent through queue "other2"' 1543} 1544 1545altq_queues_cleanup() 1546{ 1547 # Interface detaching seems badly broken in altq. If interfaces are 1548 # destroyed when shutting down the vnet and then pf is unloaded, it will 1549 # cause a kernel crash. Work around the issue by first flushing the 1550 # pf rulesets 1551 jexec one pfctl -F all 1552 jexec two pfctl -F all 1553 pfsynct_cleanup 1554} 1555 1556atf_test_case "rt_af" "cleanup" 1557rt_af_head() 1558{ 1559 atf_set descr 'Test if the rt_af is synced' 1560 atf_set require.user root 1561 atf_set require.progs python3 scapy 1562} 1563 1564rt_af_body() 1565{ 1566 route_to_common_head 1500 1567 1568 jexec one pfctl -e 1569 pft_set_rules one \ 1570 "set skip on ${epair_sync}a" \ 1571 "block" \ 1572 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1573 "pass in on inif \ 1574 route-to (outif 203.0.113.254) prefer-ipv6-nexthop \ 1575 inet proto udp \ 1576 to 203.0.113.241 \ 1577 keep state" \ 1578 "pass in on inif \ 1579 route-to (outif 2001:db8:4200::fe) prefer-ipv6-nexthop \ 1580 inet proto udp \ 1581 to 203.0.113.242 \ 1582 keep state" \ 1583 "pass in on inif \ 1584 route-to (outif 2001:db8:4200::fe) prefer-ipv6-nexthop \ 1585 inet6 proto udp \ 1586 to 2001:db8:4200::f3 \ 1587 keep state" \ 1588 "pass out on outif inet proto udp keep state (no-sync)" \ 1589 "pass out on outif inet6 proto udp keep state (no-sync)" 1590 1591 jexec two pfctl -e 1592 pft_set_rules two \ 1593 "set debug loud" \ 1594 "set skip on ${epair_sync}b" \ 1595 "block" \ 1596 "pass inet6 proto icmp6 icmp6-type { neighbrsol, neighbradv } keep state (no-sync)" \ 1597 1598 # IPv4 packet over IPv4 gateway 1599 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1600 ${common_dir}/pft_ping.py \ 1601 --ping-type=udp \ 1602 --sendif ${epair_one}b \ 1603 --fromaddr 198.51.100.254 \ 1604 --to 203.0.113.241 \ 1605 --recvif ${epair_out_one}b 1606 1607 # FIXME: Routing IPv4 packets over IPv6 gateways with gateway added 1608 # with `ndp -s` causes the static NDP entry to become expired. 1609 # Pfsync tests don't use "servers" which can reply to ARP and NDP, 1610 # but such static entry for gateway and only check if a stateless 1611 # ICMP or UDP packet is forward through. 1612 # 1613 # IPv4 packert over IPv6 gateway 1614 #atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1615 # ${common_dir}/pft_ping.py \ 1616 # --ping-type=udp \ 1617 # --sendif ${epair_one}b \ 1618 # --fromaddr 198.51.100.254 \ 1619 # --to 203.0.113.242 \ 1620 # --recvif ${epair_out_one}b 1621 1622 # IPv6 packet over IPv6 gateway 1623 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1624 ${common_dir}/pft_ping.py \ 1625 --ping-type=udp \ 1626 --sendif ${epair_one}b \ 1627 --fromaddr 2001:db8:4211::fe \ 1628 --to 2001:db8:4200::f3 \ 1629 --recvif ${epair_out_one}b 1630 1631 sleep 5 # Wait for pfsync 1632 1633 states_one=$(mktemp) 1634 states_two=$(mktemp) 1635 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 1636 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 1637 1638 echo " === states one === " 1639 cat $states_one 1640 echo " === states two === " 1641 cat $states_two 1642 1643 for state_regexp in \ 1644 "all udp 203.0.113.241:9 <- 198.51.100.254 .* route-to: 203.0.113.254@outif origif: inif" \ 1645 "all udp 2001:db8:4200::f3\[9\] <- 2001:db8:4211::fe .* route-to: 2001:db8:4200::fe@outif origif: inif" \ 1646 ; do 1647 grep -qE "${state_regexp}" $states_two || atf_fail "State not found for '${state_regexp}' on router two" 1648 done 1649} 1650 1651rt_af_cleanup() 1652{ 1653 jexec one pfctl -qvvsr 1654 jexec one pfctl -qvvss 1655 jexec one arp -an 1656 jexec one ndp -an 1657 pfsynct_cleanup 1658} 1659 1660atf_init_test_cases() 1661{ 1662 atf_add_test_case "basic" 1663 atf_add_test_case "basic_defer" 1664 atf_add_test_case "defer" 1665 atf_add_test_case "bulk" 1666 atf_add_test_case "pbr" 1667 atf_add_test_case "pfsync_pbr" 1668 atf_add_test_case "ipsec" 1669 atf_add_test_case "timeout" 1670 atf_add_test_case "basic_ipv6_unicast" 1671 atf_add_test_case "basic_ipv6" 1672 atf_add_test_case "rtable" 1673 atf_add_test_case "route_to_1301" 1674 atf_add_test_case "route_to_1301_bad_ruleset" 1675 atf_add_test_case "route_to_1301_bad_rpool" 1676 atf_add_test_case "route_to_1400_bad_ruleset" 1677 atf_add_test_case "route_to_1400_bad_ifname" 1678 atf_add_test_case "af_to_in_floating" 1679 atf_add_test_case "af_to_in_if_bound" 1680 atf_add_test_case "af_to_out_if_bound" 1681 atf_add_test_case "tag" 1682 atf_add_test_case "altq_queues" 1683 atf_add_test_case "rt_af" 1684} 1685