1# 2# SPDX-License-Identifier: BSD-2-Clause 3# 4# Copyright (c) 2018 Orange Business Services 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 1. Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# 2. Redistributions in binary form must reproduce the above copyright 12# notice, this list of conditions and the following disclaimer in the 13# documentation and/or other materials provided with the distribution. 14# 15# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25# SUCH DAMAGE. 26 27. $(atf_get_srcdir)/utils.subr 28 29common_dir=$(atf_get_srcdir)/../common 30 31atf_test_case "basic" "cleanup" 32basic_head() 33{ 34 atf_set descr 'Basic pfsync test' 35 atf_set require.user root 36} 37 38basic_body() 39{ 40 common_body 41} 42 43common_body() 44{ 45 defer=$1 46 pfsynct_init 47 48 epair_sync=$(vnet_mkepair) 49 epair_one=$(vnet_mkepair) 50 epair_two=$(vnet_mkepair) 51 52 vnet_mkjail one ${epair_one}a ${epair_sync}a 53 vnet_mkjail two ${epair_two}a ${epair_sync}b 54 55 # pfsync interface 56 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 57 jexec one ifconfig ${epair_one}a 198.51.100.1/24 up 58 jexec one ifconfig pfsync0 \ 59 syncdev ${epair_sync}a \ 60 maxupd 1 \ 61 $defer \ 62 up 63 jexec two ifconfig ${epair_two}a 198.51.100.2/24 up 64 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 65 jexec two ifconfig pfsync0 \ 66 syncdev ${epair_sync}b \ 67 maxupd 1 \ 68 $defer \ 69 up 70 71 # Enable pf! 72 jexec one pfctl -e 73 pft_set_rules one \ 74 "set skip on ${epair_sync}a" \ 75 "pass out keep state" 76 jexec two pfctl -e 77 pft_set_rules two \ 78 "set skip on ${epair_sync}b" \ 79 "pass out keep state" 80 81 hostid_one=$(jexec one pfctl -si -v | awk '/Hostid:/ { gsub(/0x/, "", $2); printf($2); }') 82 83 ifconfig ${epair_one}b 198.51.100.254/24 up 84 85 ping -c 1 -S 198.51.100.254 198.51.100.1 86 87 # Give pfsync time to do its thing 88 sleep 2 89 90 if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 91 grep 198.51.100.254 ; then 92 atf_fail "state not found on synced host" 93 fi 94 95 if ! jexec two pfctl -sc | grep ""${hostid_one}""; 96 then 97 jexec two pfctl -sc 98 atf_fail "HostID for host one not found on two" 99 fi 100} 101 102basic_cleanup() 103{ 104 pfsynct_cleanup 105} 106 107atf_test_case "basic_defer" "cleanup" 108basic_defer_head() 109{ 110 atf_set descr 'Basic defer mode pfsync test' 111 atf_set require.user root 112} 113 114basic_defer_body() 115{ 116 common_body defer 117} 118 119basic_defer_cleanup() 120{ 121 pfsynct_cleanup 122} 123 124atf_test_case "defer" "cleanup" 125defer_head() 126{ 127 atf_set descr 'Defer mode pfsync test' 128 atf_set require.user root 129 atf_set require.progs scapy 130} 131 132defer_body() 133{ 134 pfsynct_init 135 136 epair_sync=$(vnet_mkepair) 137 epair_in=$(vnet_mkepair) 138 epair_out=$(vnet_mkepair) 139 140 vnet_mkjail alcatraz ${epair_sync}a ${epair_in}a ${epair_out}a 141 142 jexec alcatraz ifconfig ${epair_sync}a 192.0.2.1/24 up 143 jexec alcatraz ifconfig ${epair_out}a 198.51.100.1/24 up 144 jexec alcatraz ifconfig ${epair_in}a 203.0.113.1/24 up 145 jexec alcatraz arp -s 203.0.113.2 00:01:02:03:04:05 146 jexec alcatraz sysctl net.inet.ip.forwarding=1 147 148 # Set a long defer delay 149 jexec alcatraz sysctl net.pfsync.defer_delay=2500 150 151 jexec alcatraz ifconfig pfsync0 \ 152 syncdev ${epair_sync}a \ 153 maxupd 1 \ 154 defer \ 155 up 156 157 ifconfig ${epair_sync}b 192.0.2.2/24 up 158 ifconfig ${epair_out}b 198.51.100.2/24 up 159 ifconfig ${epair_in}b up 160 route add -net 203.0.113.0/24 198.51.100.1 161 162 # Enable pf 163 jexec alcatraz sysctl net.pf.filter_local=0 164 jexec alcatraz pfctl -e 165 pft_set_rules alcatraz \ 166 "set skip on ${epair_sync}a" \ 167 "pass keep state" 168 169 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 170 $(atf_get_srcdir)/pfsync_defer.py \ 171 --syncdev ${epair_sync}b \ 172 --indev ${epair_in}b \ 173 --outdev ${epair_out}b 174 175 # Now disable defer mode and expect failure. 176 jexec alcatraz ifconfig pfsync0 -defer 177 178 # Flush state 179 pft_set_rules alcatraz \ 180 "set skip on ${epair_sync}a" \ 181 "pass keep state" 182 183 atf_check -s exit:3 env PYTHONPATH=${common_dir} \ 184 $(atf_get_srcdir)/pfsync_defer.py \ 185 --syncdev ${epair_sync}b \ 186 --indev ${epair_in}b \ 187 --outdev ${epair_out}b 188} 189 190defer_cleanup() 191{ 192 pfsynct_cleanup 193} 194 195atf_test_case "bulk" "cleanup" 196bulk_head() 197{ 198 atf_set descr 'Test bulk updates' 199 atf_set require.user root 200} 201 202bulk_body() 203{ 204 pfsynct_init 205 206 epair_sync=$(vnet_mkepair) 207 epair_one=$(vnet_mkepair) 208 epair_two=$(vnet_mkepair) 209 210 vnet_mkjail one ${epair_one}a ${epair_sync}a 211 vnet_mkjail two ${epair_two}a ${epair_sync}b 212 213 # pfsync interface 214 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 215 jexec one ifconfig ${epair_one}a 198.51.100.1/24 up 216 jexec one ifconfig pfsync0 \ 217 syncdev ${epair_sync}a \ 218 maxupd 1\ 219 up 220 jexec two ifconfig ${epair_two}a 198.51.100.2/24 up 221 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 222 223 # Enable pf 224 jexec one pfctl -e 225 pft_set_rules one \ 226 "set skip on ${epair_sync}a" \ 227 "pass keep state" 228 jexec two pfctl -e 229 pft_set_rules two \ 230 "set skip on ${epair_sync}b" \ 231 "pass keep state" 232 233 ifconfig ${epair_one}b 198.51.100.254/24 up 234 235 # Create state prior to setting up pfsync 236 ping -c 1 -S 198.51.100.254 198.51.100.1 237 238 # Wait before setting up pfsync on two, so we don't accidentally catch 239 # the update anyway. 240 sleep 1 241 242 # Now set up pfsync in jail two 243 jexec two ifconfig pfsync0 \ 244 syncdev ${epair_sync}b \ 245 up 246 247 # Give pfsync time to do its thing 248 sleep 2 249 250 jexec two pfctl -s states 251 if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 252 grep 198.51.100.2 ; then 253 atf_fail "state not found on synced host" 254 fi 255} 256 257bulk_cleanup() 258{ 259 pfsynct_cleanup 260} 261 262atf_test_case "pbr" "cleanup" 263pbr_head() 264{ 265 atf_set descr 'route_to and reply_to directives test' 266 atf_set require.user root 267 atf_set timeout '600' 268} 269 270pbr_body() 271{ 272 pbr_common_body 273} 274 275pbr_cleanup() 276{ 277 pbr_common_cleanup 278} 279 280atf_test_case "pfsync_pbr" "cleanup" 281pfsync_pbr_head() 282{ 283 atf_set descr 'route_to and reply_to directives pfsync test' 284 atf_set require.user root 285 atf_set timeout '600' 286} 287 288pfsync_pbr_body() 289{ 290 pbr_common_body backup_promotion 291} 292 293pfsync_pbr_cleanup() 294{ 295 pbr_common_cleanup 296} 297 298pbr_common_body() 299{ 300 # + builds bellow topology and initiate a single ping session 301 # from client to server. 302 # + gw* forward traffic through pbr not fib lookups. 303 # + if backup_promotion arg is given, a carp failover event occurs 304 # during the ping session on both gateways. 305 # ┌──────┐ 306 # │client│ 307 # └───┬──┘ 308 # │ 309 # ┌───┴───┐ 310 # │bridge0│ 311 # └┬─────┬┘ 312 # │ │ 313 # ┌────────────────┴─┐ ┌─┴────────────────┐ 314 # │gw_route_to_master├─┤gw_route_to_backup│ 315 # └────────────────┬─┘ └─┬────────────────┘ 316 # │ │ 317 # ┌┴─────┴┐ 318 # │bridge1│ 319 # └┬─────┬┘ 320 # │ │ 321 # ┌────────────────┴─┐ ┌─┴────────────────┐ 322 # │gw_reply_to_master├─┤gw_reply_to_backup│ 323 # └────────────────┬─┘ └─┬────────────────┘ 324 # │ │ 325 # ┌┴─────┴┐ 326 # │bridge2│ 327 # └───┬───┘ 328 # │ 329 # ┌───┴──┐ 330 # │server│ 331 # └──────┘ 332 333 if ! kldstat -q -m carp 334 then 335 atf_skip "This test requires carp" 336 fi 337 pfsynct_init 338 vnet_init_bridge 339 340 bridge0=$(vnet_mkbridge) 341 bridge1=$(vnet_mkbridge) 342 bridge2=$(vnet_mkbridge) 343 344 epair_sync_gw_route_to=$(vnet_mkepair) 345 epair_sync_gw_reply_to=$(vnet_mkepair) 346 epair_client_bridge0=$(vnet_mkepair) 347 348 epair_gw_route_to_master_bridge0=$(vnet_mkepair) 349 epair_gw_route_to_backup_bridge0=$(vnet_mkepair) 350 epair_gw_route_to_master_bridge1=$(vnet_mkepair) 351 epair_gw_route_to_backup_bridge1=$(vnet_mkepair) 352 353 epair_gw_reply_to_master_bridge1=$(vnet_mkepair) 354 epair_gw_reply_to_backup_bridge1=$(vnet_mkepair) 355 epair_gw_reply_to_master_bridge2=$(vnet_mkepair) 356 epair_gw_reply_to_backup_bridge2=$(vnet_mkepair) 357 358 epair_server_bridge2=$(vnet_mkepair) 359 360 ifconfig ${bridge0} up 361 ifconfig ${epair_client_bridge0}b up 362 ifconfig ${epair_gw_route_to_master_bridge0}b up 363 ifconfig ${epair_gw_route_to_backup_bridge0}b up 364 ifconfig ${bridge0} \ 365 addm ${epair_client_bridge0}b \ 366 addm ${epair_gw_route_to_master_bridge0}b \ 367 addm ${epair_gw_route_to_backup_bridge0}b 368 369 ifconfig ${bridge1} up 370 ifconfig ${epair_gw_route_to_master_bridge1}b up 371 ifconfig ${epair_gw_route_to_backup_bridge1}b up 372 ifconfig ${epair_gw_reply_to_master_bridge1}b up 373 ifconfig ${epair_gw_reply_to_backup_bridge1}b up 374 ifconfig ${bridge1} \ 375 addm ${epair_gw_route_to_master_bridge1}b \ 376 addm ${epair_gw_route_to_backup_bridge1}b \ 377 addm ${epair_gw_reply_to_master_bridge1}b \ 378 addm ${epair_gw_reply_to_backup_bridge1}b 379 380 ifconfig ${bridge2} up 381 ifconfig ${epair_gw_reply_to_master_bridge2}b up 382 ifconfig ${epair_gw_reply_to_backup_bridge2}b up 383 ifconfig ${epair_server_bridge2}b up 384 ifconfig ${bridge2} \ 385 addm ${epair_gw_reply_to_master_bridge2}b \ 386 addm ${epair_gw_reply_to_backup_bridge2}b \ 387 addm ${epair_server_bridge2}b 388 389 vnet_mkjail client ${epair_client_bridge0}a 390 jexec client hostname client 391 vnet_mkjail gw_route_to_master \ 392 ${epair_gw_route_to_master_bridge0}a \ 393 ${epair_gw_route_to_master_bridge1}a \ 394 ${epair_sync_gw_route_to}a 395 jexec gw_route_to_master hostname gw_route_to_master 396 vnet_mkjail gw_route_to_backup \ 397 ${epair_gw_route_to_backup_bridge0}a \ 398 ${epair_gw_route_to_backup_bridge1}a \ 399 ${epair_sync_gw_route_to}b 400 jexec gw_route_to_backup hostname gw_route_to_backup 401 vnet_mkjail gw_reply_to_master \ 402 ${epair_gw_reply_to_master_bridge1}a \ 403 ${epair_gw_reply_to_master_bridge2}a \ 404 ${epair_sync_gw_reply_to}a 405 jexec gw_reply_to_master hostname gw_reply_to_master 406 vnet_mkjail gw_reply_to_backup \ 407 ${epair_gw_reply_to_backup_bridge1}a \ 408 ${epair_gw_reply_to_backup_bridge2}a \ 409 ${epair_sync_gw_reply_to}b 410 jexec gw_reply_to_backup hostname gw_reply_to_backup 411 vnet_mkjail server ${epair_server_bridge2}a 412 jexec server hostname server 413 414 jexec client ifconfig ${epair_client_bridge0}a inet 198.18.0.1/24 up 415 jexec client route add 198.18.2.0/24 198.18.0.10 416 417 jexec gw_route_to_master ifconfig ${epair_sync_gw_route_to}a \ 418 inet 198.19.10.1/24 up 419 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge0}a \ 420 inet 198.18.0.8/24 up 421 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge0}a \ 422 alias 198.18.0.10/32 vhid 10 pass 3WjvVVw7 advskew 50 423 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge1}a \ 424 inet 198.18.1.8/24 up 425 jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge1}a \ 426 alias 198.18.1.10/32 vhid 11 pass 3WjvVVw7 advskew 50 427 jexec gw_route_to_master sysctl net.inet.ip.forwarding=1 428 jexec gw_route_to_master sysctl net.inet.carp.preempt=1 429 430 vnet_ifrename_jail gw_route_to_master ${epair_sync_gw_route_to}a if_pfsync 431 vnet_ifrename_jail gw_route_to_master ${epair_gw_route_to_master_bridge0}a if_br0 432 vnet_ifrename_jail gw_route_to_master ${epair_gw_route_to_master_bridge1}a if_br1 433 434 jexec gw_route_to_master ifconfig pfsync0 \ 435 syncpeer 198.19.10.2 \ 436 syncdev if_pfsync \ 437 maxupd 1 \ 438 up 439 pft_set_rules gw_route_to_master \ 440 "keep_state = 'tag auth_packet keep state'" \ 441 "set timeout { icmp.first 120, icmp.error 60 }" \ 442 "block log all" \ 443 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 444 "pass quick on { if_br0 if_br1 } proto carp keep state (no-sync)" \ 445 "block drop in quick to 224.0.0.18/32" \ 446 "pass out quick tagged auth_packet keep state" \ 447 "pass in quick log on if_br0 route-to (if_br1 198.18.1.20) proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 \$keep_state" 448 jexec gw_route_to_master pfctl -e 449 450 jexec gw_route_to_backup ifconfig ${epair_sync_gw_route_to}b \ 451 inet 198.19.10.2/24 up 452 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge0}a \ 453 inet 198.18.0.9/24 up 454 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge0}a \ 455 alias 198.18.0.10/32 vhid 10 pass 3WjvVVw7 advskew 100 456 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge1}a \ 457 inet 198.18.1.9/24 up 458 jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge1}a \ 459 alias 198.18.1.10/32 vhid 11 pass 3WjvVVw7 advskew 100 460 jexec gw_route_to_backup sysctl net.inet.ip.forwarding=1 461 jexec gw_route_to_backup sysctl net.inet.carp.preempt=1 462 463 vnet_ifrename_jail gw_route_to_backup ${epair_sync_gw_route_to}b if_pfsync 464 vnet_ifrename_jail gw_route_to_backup ${epair_gw_route_to_backup_bridge0}a if_br0 465 vnet_ifrename_jail gw_route_to_backup ${epair_gw_route_to_backup_bridge1}a if_br1 466 467 jexec gw_route_to_backup ifconfig pfsync0 \ 468 syncpeer 198.19.10.1 \ 469 syncdev if_pfsync \ 470 up 471 pft_set_rules gw_route_to_backup \ 472 "keep_state = 'tag auth_packet keep state'" \ 473 "set timeout { icmp.first 120, icmp.error 60 }" \ 474 "block log all" \ 475 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 476 "pass quick on { if_br0 if_br1 } proto carp keep state (no-sync)" \ 477 "block drop in quick to 224.0.0.18/32" \ 478 "pass out quick tagged auth_packet keep state" \ 479 "pass in quick log on if_br0 route-to (if_br1 198.18.1.20) proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 \$keep_state" 480 jexec gw_route_to_backup pfctl -e 481 482 jexec gw_reply_to_master ifconfig ${epair_sync_gw_reply_to}a \ 483 inet 198.19.20.1/24 up 484 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge1}a \ 485 inet 198.18.1.18/24 up 486 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge1}a \ 487 alias 198.18.1.20/32 vhid 21 pass 3WjvVVw7 advskew 50 488 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge2}a \ 489 inet 198.18.2.18/24 up 490 jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge2}a \ 491 alias 198.18.2.20/32 vhid 22 pass 3WjvVVw7 advskew 50 492 jexec gw_reply_to_master sysctl net.inet.ip.forwarding=1 493 jexec gw_reply_to_master sysctl net.inet.carp.preempt=1 494 495 vnet_ifrename_jail gw_reply_to_master ${epair_sync_gw_reply_to}a if_pfsync 496 vnet_ifrename_jail gw_reply_to_master ${epair_gw_reply_to_master_bridge1}a if_br1 497 vnet_ifrename_jail gw_reply_to_master ${epair_gw_reply_to_master_bridge2}a if_br2 498 499 jexec gw_reply_to_master ifconfig pfsync0 \ 500 syncpeer 198.19.20.2 \ 501 syncdev if_pfsync \ 502 maxupd 1 \ 503 up 504 pft_set_rules gw_reply_to_master \ 505 "set timeout { icmp.first 120, icmp.error 60 }" \ 506 "block log all" \ 507 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 508 "pass quick on { if_br1 if_br2 } proto carp keep state (no-sync)" \ 509 "block drop in quick to 224.0.0.18/32" \ 510 "pass out quick on if_br2 reply-to (if_br1 198.18.1.10) tagged auth_packet_reply_to keep state" \ 511 "pass in quick log on if_br1 proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 tag auth_packet_reply_to keep state" 512 jexec gw_reply_to_master pfctl -e 513 514 jexec gw_reply_to_backup ifconfig ${epair_sync_gw_reply_to}b \ 515 inet 198.19.20.2/24 up 516 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge1}a \ 517 inet 198.18.1.19/24 up 518 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge1}a \ 519 alias 198.18.1.20/32 vhid 21 pass 3WjvVVw7 advskew 100 520 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge2}a \ 521 inet 198.18.2.19/24 up 522 jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge2}a \ 523 alias 198.18.2.20/32 vhid 22 pass 3WjvVVw7 advskew 100 524 jexec gw_reply_to_backup sysctl net.inet.ip.forwarding=1 525 jexec gw_reply_to_backup sysctl net.inet.carp.preempt=1 526 527 vnet_ifrename_jail gw_reply_to_backup ${epair_sync_gw_reply_to}b if_pfsync 528 vnet_ifrename_jail gw_reply_to_backup ${epair_gw_reply_to_backup_bridge1}a if_br1 529 vnet_ifrename_jail gw_reply_to_backup ${epair_gw_reply_to_backup_bridge2}a if_br2 530 531 jexec gw_reply_to_backup ifconfig pfsync0 \ 532 syncpeer 198.19.20.1 \ 533 syncdev if_pfsync \ 534 up 535 pft_set_rules gw_reply_to_backup \ 536 "set timeout { icmp.first 120, icmp.error 60 }" \ 537 "block log all" \ 538 "pass quick on if_pfsync proto pfsync keep state (no-sync)" \ 539 "pass quick on { if_br1 if_br2 } proto carp keep state (no-sync)" \ 540 "block drop in quick to 224.0.0.18/32" \ 541 "pass out quick on if_br2 reply-to (if_br1 198.18.1.10) tagged auth_packet_reply_to keep state" \ 542 "pass in quick log on if_br1 proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 tag auth_packet_reply_to keep state" 543 jexec gw_reply_to_backup pfctl -e 544 545 jexec server ifconfig ${epair_server_bridge2}a inet 198.18.2.1/24 up 546 jexec server route add 198.18.0.0/24 198.18.2.20 547 548 # Waiting for platform to settle 549 while ! jexec gw_route_to_backup ifconfig | grep 'carp: BACKUP' 550 do 551 sleep 1 552 done 553 while ! jexec gw_reply_to_backup ifconfig | grep 'carp: BACKUP' 554 do 555 sleep 1 556 done 557 while ! jexec client ping -c 10 198.18.2.1 | grep ', 0.0% packet loss' 558 do 559 sleep 1 560 done 561 562 # Checking cluster members pf.conf checksums match 563 gw_route_to_master_checksum=$(jexec gw_route_to_master pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 564 gw_route_to_backup_checksum=$(jexec gw_route_to_backup pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 565 gw_reply_to_master_checksum=$(jexec gw_reply_to_master pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 566 gw_reply_to_backup_checksum=$(jexec gw_reply_to_backup pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2) 567 if [ "$gw_route_to_master_checksum" != "$gw_route_to_backup_checksum" ] 568 then 569 atf_fail "gw_route_to cluster members pf.conf do not match each others" 570 fi 571 if [ "$gw_reply_to_master_checksum" != "$gw_reply_to_backup_checksum" ] 572 then 573 atf_fail "gw_reply_to cluster members pf.conf do not match each others" 574 fi 575 576 # Creating state entries 577 (jexec client ping -c 10 198.18.2.1 >ping.stdout) & 578 579 if [ "$1" = "backup_promotion" ] 580 then 581 sleep 1 582 jexec gw_route_to_backup ifconfig if_br0 vhid 10 advskew 0 583 jexec gw_route_to_backup ifconfig if_br1 vhid 11 advskew 0 584 jexec gw_reply_to_backup ifconfig if_br1 vhid 21 advskew 0 585 jexec gw_reply_to_backup ifconfig if_br2 vhid 22 advskew 0 586 fi 587 while ! grep -q -e 'packet loss' ping.stdout 588 do 589 sleep 1 590 done 591 592 atf_check -s exit:0 -e ignore -o ignore grep ', 0.0% packet loss' ping.stdout 593} 594 595pbr_common_cleanup() 596{ 597 pft_cleanup 598} 599 600atf_test_case "ipsec" "cleanup" 601ipsec_head() 602{ 603 atf_set descr 'Transport pfsync over IPSec' 604 atf_set require.user root 605} 606 607ipsec_body() 608{ 609 if ! sysctl -q kern.features.ipsec >/dev/null ; then 610 atf_skip "This test requires ipsec" 611 fi 612 613 # Run the common test, to set up pfsync 614 common_body 615 616 # But we want unicast pfsync 617 jexec one ifconfig pfsync0 syncpeer 192.0.2.2 618 jexec two ifconfig pfsync0 syncpeer 192.0.2.1 619 620 # Flush existing states 621 jexec one pfctl -Fs 622 jexec two pfctl -Fs 623 624 # Now define an ipsec policy to run over the epair_sync interfaces 625 echo "flush; 626 spdflush; 627 spdadd 192.0.2.1/32 192.0.2.2/32 any -P out ipsec esp/transport//require; 628 spdadd 192.0.2.2/32 192.0.2.1/32 any -P in ipsec esp/transport//require; 629 add 192.0.2.1 192.0.2.2 esp 0x1000 -E aes-gcm-16 \"12345678901234567890\"; 630 add 192.0.2.2 192.0.2.1 esp 0x1001 -E aes-gcm-16 \"12345678901234567890\";" \ 631 | jexec one setkey -c 632 633 echo "flush; 634 spdflush; 635 spdadd 192.0.2.2/32 192.0.2.1/32 any -P out ipsec esp/transport//require; 636 spdadd 192.0.2.1/32 192.0.2.2/32 any -P in ipsec esp/transport//require; 637 add 192.0.2.1 192.0.2.2 esp 0x1000 -E aes-gcm-16 \"12345678901234567891\"; 638 add 192.0.2.2 192.0.2.1 esp 0x1001 -E aes-gcm-16 \"12345678901234567891\";" \ 639 | jexec two setkey -c 640 641 # We've set incompatible keys, so pfsync will be broken. 642 ping -c 1 -S 198.51.100.254 198.51.100.1 643 644 # Give pfsync time to do its thing 645 sleep 2 646 647 if jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 648 grep 198.51.100.2 ; then 649 atf_fail "state synced although IPSec should have prevented it" 650 fi 651 652 # Flush existing states 653 jexec one pfctl -Fs 654 jexec two pfctl -Fs 655 656 # Fix the IPSec key to match 657 echo "flush; 658 spdflush; 659 spdadd 192.0.2.2/32 192.0.2.1/32 any -P out ipsec esp/transport//require; 660 spdadd 192.0.2.1/32 192.0.2.2/32 any -P in ipsec esp/transport//require; 661 add 192.0.2.1 192.0.2.2 esp 0x1000 -E aes-gcm-16 \"12345678901234567890\"; 662 add 192.0.2.2 192.0.2.1 esp 0x1001 -E aes-gcm-16 \"12345678901234567890\";" \ 663 | jexec two setkey -c 664 665 ping -c 1 -S 198.51.100.254 198.51.100.1 666 667 # Give pfsync time to do its thing 668 sleep 2 669 670 if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \ 671 grep 198.51.100.2 ; then 672 atf_fail "state not found on synced host" 673 fi 674} 675 676ipsec_cleanup() 677{ 678 pft_cleanup 679} 680 681atf_test_case "timeout" "cleanup" 682timeout_head() 683{ 684 atf_set descr 'Trigger pfsync_timeout()' 685 atf_set require.user root 686} 687 688timeout_body() 689{ 690 pft_init 691 692 vnet_mkjail one 693 694 jexec one ifconfig lo0 127.0.0.1/8 up 695 jexec one ifconfig lo0 inet6 ::1/128 up 696 697 pft_set_rules one \ 698 "pass all" 699 jexec one pfctl -e 700 jexec one ifconfig pfsync0 defer up 701 702 jexec one ping -c 1 ::1 703 jexec one ping -c 1 127.0.0.1 704 705 # Give pfsync_timeout() time to fire (a callout on a 1 second delay) 706 sleep 2 707} 708 709timeout_cleanup() 710{ 711 pft_cleanup 712} 713 714atf_test_case "basic_ipv6_unicast" "cleanup" 715basic_ipv6_unicast_head() 716{ 717 atf_set descr 'Basic pfsync test (IPv6)' 718 atf_set require.user root 719} 720 721basic_ipv6_unicast_body() 722{ 723 pfsynct_init 724 725 epair_sync=$(vnet_mkepair) 726 epair_one=$(vnet_mkepair) 727 epair_two=$(vnet_mkepair) 728 729 vnet_mkjail one ${epair_one}a ${epair_sync}a 730 vnet_mkjail two ${epair_two}a ${epair_sync}b 731 732 # pfsync interface 733 jexec one ifconfig ${epair_sync}a inet6 fd2c::1/64 no_dad up 734 jexec one ifconfig ${epair_one}a inet6 fd2b::1/64 no_dad up 735 jexec one ifconfig pfsync0 \ 736 syncdev ${epair_sync}a \ 737 syncpeer fd2c::2 \ 738 maxupd 1 \ 739 up 740 jexec two ifconfig ${epair_two}a inet6 fd2b::2/64 no_dad up 741 jexec two ifconfig ${epair_sync}b inet6 fd2c::2/64 no_dad up 742 jexec two ifconfig pfsync0 \ 743 syncdev ${epair_sync}b \ 744 syncpeer fd2c::1 \ 745 maxupd 1 \ 746 up 747 748 # Enable pf! 749 jexec one pfctl -e 750 pft_set_rules one \ 751 "block on ${epair_sync}a inet" \ 752 "pass out keep state" 753 jexec two pfctl -e 754 pft_set_rules two \ 755 "block on ${epair_sync}b inet" \ 756 "pass out keep state" 757 758 ifconfig ${epair_one}b inet6 fd2b::f0/64 no_dad up 759 760 ping6 -c 1 -S fd2b::f0 fd2b::1 761 762 # Give pfsync time to do its thing 763 sleep 2 764 765 if ! jexec two pfctl -s states | grep icmp | grep fd2b::1 | \ 766 grep fd2b::f0 ; then 767 atf_fail "state not found on synced host" 768 fi 769} 770 771basic_ipv6_unicast_cleanup() 772{ 773 pfsynct_cleanup 774} 775 776atf_test_case "basic_ipv6" "cleanup" 777basic_ipv6_head() 778{ 779 atf_set descr 'Basic pfsync test (IPv6)' 780 atf_set require.user root 781} 782 783basic_ipv6_body() 784{ 785 pfsynct_init 786 787 epair_sync=$(vnet_mkepair) 788 epair_one=$(vnet_mkepair) 789 epair_two=$(vnet_mkepair) 790 791 vnet_mkjail one ${epair_one}a ${epair_sync}a 792 vnet_mkjail two ${epair_two}a ${epair_sync}b 793 794 # pfsync interface 795 jexec one ifconfig ${epair_sync}a inet6 fd2c::1/64 no_dad up 796 jexec one ifconfig ${epair_one}a inet6 fd2b::1/64 no_dad up 797 jexec one ifconfig pfsync0 \ 798 syncdev ${epair_sync}a \ 799 syncpeer ff12::f0 \ 800 maxupd 1 \ 801 up 802 jexec two ifconfig ${epair_two}a inet6 fd2b::2/64 no_dad up 803 jexec two ifconfig ${epair_sync}b inet6 fd2c::2/64 no_dad up 804 jexec two ifconfig pfsync0 \ 805 syncdev ${epair_sync}b \ 806 syncpeer ff12::f0 \ 807 maxupd 1 \ 808 up 809 810 # Enable pf! 811 jexec one pfctl -e 812 pft_set_rules one \ 813 "block on ${epair_sync}a inet" \ 814 "pass out keep state" 815 jexec two pfctl -e 816 pft_set_rules two \ 817 "block on ${epair_sync}b inet" \ 818 "pass out keep state" 819 820 ifconfig ${epair_one}b inet6 fd2b::f0/64 no_dad up 821 822 ping6 -c 1 -S fd2b::f0 fd2b::1 823 824 # Give pfsync time to do its thing 825 sleep 2 826 827 if ! jexec two pfctl -s states | grep icmp | grep fd2b::1 | \ 828 grep fd2b::f0 ; then 829 atf_fail "state not found on synced host" 830 fi 831} 832 833basic_ipv6_cleanup() 834{ 835 pfsynct_cleanup 836} 837 838route_to_common_head() 839{ 840 pfsync_version=$1 841 shift 842 843 pfsynct_init 844 845 epair_sync=$(vnet_mkepair) 846 epair_one=$(vnet_mkepair) 847 epair_two=$(vnet_mkepair) 848 epair_out_one=$(vnet_mkepair) 849 epair_out_two=$(vnet_mkepair) 850 851 vnet_mkjail one ${epair_one}a ${epair_sync}a ${epair_out_one}a 852 vnet_mkjail two ${epair_two}a ${epair_sync}b ${epair_out_two}a 853 854 # pfsync interface 855 jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up 856 jexec one ifconfig ${epair_one}a 198.51.100.1/24 up 857 jexec one ifconfig ${epair_out_one}a 203.0.113.1/24 up 858 jexec one ifconfig ${epair_out_one}a name outif 859 jexec one sysctl net.inet.ip.forwarding=1 860 jexec one arp -s 203.0.113.254 00:01:02:03:04:05 861 jexec one ifconfig pfsync0 \ 862 syncdev ${epair_sync}a \ 863 maxupd 1 \ 864 version $pfsync_version \ 865 up 866 867 jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up 868 jexec two ifconfig ${epair_two}a 198.51.100.2/24 up 869 jexec two ifconfig ${epair_out_two}a 203.0.113.2/24 up 870 jexec two ifconfig ${epair_out_two}a name outif 871 jexec two sysctl net.inet.ip.forwarding=1 872 jexec two arp -s 203.0.113.254 00:01:02:03:04:05 873 jexec two ifconfig pfsync0 \ 874 syncdev ${epair_sync}b \ 875 maxupd 1 \ 876 version $pfsync_version \ 877 up 878 879 ifconfig ${epair_one}b 198.51.100.254/24 up 880 ifconfig ${epair_two}b 198.51.100.253/24 up 881 route add -net 203.0.113.0/24 198.51.100.1 882 ifconfig ${epair_two}b up 883 ifconfig ${epair_out_one}b up 884 ifconfig ${epair_out_two}b up 885} 886 887route_to_common_tail() 888{ 889 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 890 ${common_dir}/pft_ping.py \ 891 --sendif ${epair_one}b \ 892 --fromaddr 198.51.100.254 \ 893 --to 203.0.113.254 \ 894 --recvif ${epair_out_one}b 895 896 # Allow time for sync 897 sleep 2 898 899 states_one=$(mktemp) 900 states_two=$(mktemp) 901 jexec one pfctl -qvvss | normalize_pfctl_s > $states_one 902 jexec two pfctl -qvvss | normalize_pfctl_s > $states_two 903} 904 905atf_test_case "route_to_1301_body" "cleanup" 906route_to_1301_head() 907{ 908 atf_set descr 'Test route-to with pfsync version 13.1' 909 atf_set require.user root 910 atf_set require.progs scapy 911} 912 913route_to_1301_body() 914{ 915 route_to_common_head 1301 916 917 jexec one pfctl -e 918 pft_set_rules one \ 919 "set skip on ${epair_sync}a" \ 920 "pass out route-to (outif 203.0.113.254)" 921 922 jexec two pfctl -e 923 pft_set_rules two \ 924 "set skip on ${epair_sync}b" \ 925 "pass out route-to (outif 203.0.113.254)" 926 927 route_to_common_tail 928 929 # Sanity check 930 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 931 atf_fail "State missing on router one" 932 933 # With identical ruleset the routing information is recovered from the matching rule. 934 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif' $states_two || 935 atf_fail "State missing on router two" 936 937 true 938} 939 940route_to_1301_cleanup() 941{ 942 pfsynct_cleanup 943} 944 945atf_test_case "route_to_1301_bad_ruleset" "cleanup" 946route_to_1301_bad_ruleset_head() 947{ 948 atf_set descr 'Test route-to with pfsync version 13.1 and incompatible ruleset' 949 atf_set require.user root 950 atf_set require.progs scapy 951} 952 953route_to_1301_bad_ruleset_body() 954{ 955 route_to_common_head 1301 956 957 jexec one pfctl -e 958 pft_set_rules one \ 959 "set skip on ${epair_sync}a" \ 960 "pass out route-to (outif 203.0.113.254)" 961 962 jexec two pfctl -e 963 pft_set_rules two \ 964 "set debug loud" \ 965 "set skip on ${epair_sync}b" \ 966 "pass out route-to (outif 203.0.113.254)" \ 967 "pass out proto tcp" 968 969 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 970 ${common_dir}/pft_ping.py \ 971 --sendif ${epair_one}b \ 972 --fromaddr 198.51.100.254 \ 973 --to 203.0.113.254 \ 974 --recvif ${epair_out_one}b 975 976 route_to_common_tail 977 978 # Sanity check 979 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 980 atf_fail "State missing on router one" 981 982 # Different ruleset on each router means the routing information recovery 983 # from rule is impossible. The state is not synced. 984 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*' $states_two && 985 atf_fail "State present on router two" 986 987 true 988} 989 990route_to_1301_bad_ruleset_cleanup() 991{ 992 pfsynct_cleanup 993} 994 995atf_test_case "route_to_1301_bad_rpool" "cleanup" 996route_to_1301_bad_rpool_head() 997{ 998 atf_set descr 'Test route-to with pfsync version 13.1 and different interface' 999 atf_set require.user root 1000 atf_set require.progs scapy 1001} 1002 1003route_to_1301_bad_rpool_body() 1004{ 1005 route_to_common_head 1301 1006 1007 jexec one pfctl -e 1008 pft_set_rules one \ 1009 "set skip on ${epair_sync}a" \ 1010 "pass out route-to { (outif 203.0.113.254) (outif 203.0.113.254) }" 1011 1012 jexec two pfctl -e 1013 pft_set_rules two \ 1014 "set skip on ${epair_sync}b" \ 1015 "pass out route-to { (outif 203.0.113.254) (outif 203.0.113.254) }" 1016 1017 atf_check -s exit:0 env PYTHONPATH=${common_dir} \ 1018 ${common_dir}/pft_ping.py \ 1019 --sendif ${epair_one}b \ 1020 --fromaddr 198.51.100.254 \ 1021 --to 203.0.113.254 \ 1022 --recvif ${epair_out_one}b 1023 1024 route_to_common_tail 1025 1026 # Sanity check 1027 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1028 atf_fail "State missing on router one" 1029 1030 # The ruleset is identical but since the redirection pool contains multiple interfaces 1031 # pfsync will not attempt to recover the routing information from the rule. 1032 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*' $states_two && 1033 atf_fail "State present on router two" 1034 1035 true 1036} 1037 1038route_to_1301_bad_rpool_cleanup() 1039{ 1040 pfsynct_cleanup 1041} 1042 1043atf_test_case "route_to_1400_bad_ruleset" "cleanup" 1044route_to_1400_bad_ruleset_head() 1045{ 1046 atf_set descr 'Test route-to with pfsync version 14.0' 1047 atf_set require.user root 1048 atf_set require.progs scapy 1049} 1050 1051route_to_1400_bad_ruleset_body() 1052{ 1053 route_to_common_head 1400 1054 1055 jexec one pfctl -e 1056 pft_set_rules one \ 1057 "set skip on ${epair_sync}a" \ 1058 "pass out route-to (outif 203.0.113.254)" 1059 1060 jexec two pfctl -e 1061 pft_set_rules two \ 1062 "set skip on ${epair_sync}b" 1063 1064 route_to_common_tail 1065 1066 # Sanity check 1067 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1068 atf_fail "State missing on router one" 1069 1070 # Even with a different ruleset FreeBSD 14 syncs the state just fine. 1071 # There's no recovery involved, the pfsync packet contains the routing information. 1072 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .* route-to: 203.0.113.254@outif' $states_two || 1073 atf_fail "State missing on router two" 1074 1075 true 1076} 1077 1078route_to_1400_bad_ruleset_cleanup() 1079{ 1080 pfsynct_cleanup 1081} 1082 1083atf_test_case "route_to_1400_bad_ifname" "cleanup" 1084route_to_1400_bad_ifname_head() 1085{ 1086 atf_set descr 'Test route-to with pfsync version 14.0' 1087 atf_set require.user root 1088 atf_set require.progs scapy 1089} 1090 1091route_to_1400_bad_ifname_body() 1092{ 1093 route_to_common_head 1400 1094 1095 jexec one pfctl -e 1096 pft_set_rules one \ 1097 "set skip on ${epair_sync}a" \ 1098 "pass out route-to (outif 203.0.113.254)" 1099 1100 jexec two pfctl -e 1101 jexec two ifconfig outif name outif_new 1102 pft_set_rules two \ 1103 "set skip on ${epair_sync}b" \ 1104 "pass out route-to (outif_new 203.0.113.254)" 1105 1106 route_to_common_tail 1107 1108 # Sanity check 1109 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*, rule 0 .* route-to: 203.0.113.254@outif origif: outif' $states_one || 1110 atf_fail "State missing on router one" 1111 1112 # Since FreeBSD 14 never attempts recovery of missing routing information 1113 # a state synced to a router with a different interface name is dropped. 1114 grep -qE 'all icmp 198.51.100.254 -> 203.0.113.254:8 .*' $states_two && 1115 atf_fail "State present on router two" 1116 1117 true 1118} 1119 1120route_to_1400_bad_ifname_cleanup() 1121{ 1122 pfsynct_cleanup 1123} 1124 1125atf_init_test_cases() 1126{ 1127 atf_add_test_case "basic" 1128 atf_add_test_case "basic_defer" 1129 atf_add_test_case "defer" 1130 atf_add_test_case "bulk" 1131 atf_add_test_case "pbr" 1132 atf_add_test_case "pfsync_pbr" 1133 atf_add_test_case "ipsec" 1134 atf_add_test_case "timeout" 1135 atf_add_test_case "basic_ipv6_unicast" 1136 atf_add_test_case "basic_ipv6" 1137 atf_add_test_case "route_to_1301" 1138 atf_add_test_case "route_to_1301_bad_ruleset" 1139 atf_add_test_case "route_to_1301_bad_rpool" 1140 atf_add_test_case "route_to_1400_bad_ruleset" 1141 atf_add_test_case "route_to_1400_bad_ifname" 1142} 1143