xref: /freebsd/tests/sys/netpfil/pf/pfsync.sh (revision 924226fba12cc9a228c73b956e1b7fa24c60b055)
1# $FreeBSD$
2#
3# SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4#
5# Copyright (c) 2018 Orange Business Services
6#
7# Redistribution and use in source and binary forms, with or without
8# modification, are permitted provided that the following conditions
9# are met:
10# 1. Redistributions of source code must retain the above copyright
11#    notice, this list of conditions and the following disclaimer.
12# 2. Redistributions in binary form must reproduce the above copyright
13#    notice, this list of conditions and the following disclaimer in the
14#    documentation and/or other materials provided with the distribution.
15#
16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26# SUCH DAMAGE.
27
28. $(atf_get_srcdir)/utils.subr
29
30common_dir=$(atf_get_srcdir)/../common
31
32atf_test_case "basic" "cleanup"
33basic_head()
34{
35	atf_set descr 'Basic pfsync test'
36	atf_set require.user root
37}
38
39basic_body()
40{
41	common_body
42}
43
44common_body()
45{
46	defer=$1
47	pfsynct_init
48
49	epair_sync=$(vnet_mkepair)
50	epair_one=$(vnet_mkepair)
51	epair_two=$(vnet_mkepair)
52
53	vnet_mkjail one ${epair_one}a ${epair_sync}a
54	vnet_mkjail two ${epair_two}a ${epair_sync}b
55
56	# pfsync interface
57	jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up
58	jexec one ifconfig ${epair_one}a 198.51.100.1/24 up
59	jexec one ifconfig pfsync0 \
60		syncdev ${epair_sync}a \
61		maxupd 1 \
62		$defer \
63		up
64	jexec two ifconfig ${epair_two}a 198.51.100.2/24 up
65	jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up
66	jexec two ifconfig pfsync0 \
67		syncdev ${epair_sync}b \
68		maxupd 1 \
69		$defer \
70		up
71
72	# Enable pf!
73	jexec one pfctl -e
74	pft_set_rules one \
75		"set skip on ${epair_sync}a" \
76		"pass keep state"
77	jexec two pfctl -e
78	pft_set_rules two \
79		"set skip on ${epair_sync}b" \
80		"pass keep state"
81
82	ifconfig ${epair_one}b 198.51.100.254/24 up
83
84	ping -c 1 -S 198.51.100.254 198.51.100.1
85
86	# Give pfsync time to do its thing
87	sleep 2
88
89	if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \
90	    grep 198.51.100.2 ; then
91		atf_fail "state not found on synced host"
92	fi
93}
94
95basic_cleanup()
96{
97	pfsynct_cleanup
98}
99
100atf_test_case "basic_defer" "cleanup"
101basic_defer_head()
102{
103	atf_set descr 'Basic defer mode pfsync test'
104	atf_set require.user root
105}
106
107basic_defer_body()
108{
109	common_body defer
110}
111
112basic_defer_cleanup()
113{
114	pfsynct_cleanup
115}
116
117atf_test_case "defer" "cleanup"
118defer_head()
119{
120	atf_set descr 'Defer mode pfsync test'
121	atf_set require.user root
122}
123
124defer_body()
125{
126	pfsynct_init
127
128	if [ "$(atf_config_get ci false)" = "true" ]; then
129		atf_skip "Skip know failing test (likely related to https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=260460)"
130	fi
131
132	epair_sync=$(vnet_mkepair)
133	epair_in=$(vnet_mkepair)
134	epair_out=$(vnet_mkepair)
135
136	vnet_mkjail alcatraz ${epair_sync}a ${epair_in}a ${epair_out}a
137
138	jexec alcatraz ifconfig ${epair_sync}a 192.0.2.1/24 up
139	jexec alcatraz ifconfig ${epair_out}a 198.51.100.1/24 up
140	jexec alcatraz ifconfig ${epair_in}a 203.0.113.1/24 up
141	jexec alcatraz arp -s 203.0.113.2 00:01:02:03:04:05
142	jexec alcatraz sysctl net.inet.ip.forwarding=1
143
144	jexec alcatraz ifconfig pfsync0 \
145		syncdev ${epair_sync}a \
146		maxupd 1 \
147		defer \
148		up
149
150	ifconfig ${epair_sync}b 192.0.2.2/24 up
151	ifconfig ${epair_out}b 198.51.100.2/24 up
152	ifconfig ${epair_in}b up
153	route add -net 203.0.113.0/24 198.51.100.1
154
155	# Enable pf
156	jexec alcatraz pfctl -e
157	pft_set_rules alcatraz \
158		"set skip on ${epair_sync}a" \
159		"pass keep state"
160
161	atf_check -s exit:0 env PYTHONPATH=${common_dir} \
162		$(atf_get_srcdir)/pfsync_defer.py \
163		--syncdev ${epair_sync}b \
164		--indev ${epair_in}b \
165		--outdev ${epair_out}b
166
167	# Now disable defer mode and expect failure.
168	jexec alcatraz ifconfig pfsync0 -defer
169
170	# Flush state
171	pft_set_rules alcatraz \
172		"set skip on ${epair_sync}a" \
173		"pass keep state"
174
175	atf_check -s exit:1 env PYTHONPATH=${common_dir} \
176		$(atf_get_srcdir)/pfsync_defer.py \
177		--syncdev ${epair_sync}b \
178		--indev ${epair_in}b \
179		--outdev ${epair_out}b
180}
181
182defer_cleanup()
183{
184	pfsynct_cleanup
185}
186
187atf_test_case "bulk" "cleanup"
188bulk_head()
189{
190	atf_set descr 'Test bulk updates'
191	atf_set require.user root
192}
193
194bulk_body()
195{
196	pfsynct_init
197
198	epair_sync=$(vnet_mkepair)
199	epair_one=$(vnet_mkepair)
200	epair_two=$(vnet_mkepair)
201
202	vnet_mkjail one ${epair_one}a ${epair_sync}a
203	vnet_mkjail two ${epair_two}a ${epair_sync}b
204
205	# pfsync interface
206	jexec one ifconfig ${epair_sync}a 192.0.2.1/24 up
207	jexec one ifconfig ${epair_one}a 198.51.100.1/24 up
208	jexec one ifconfig pfsync0 \
209		syncdev ${epair_sync}a \
210		maxupd 1\
211		up
212	jexec two ifconfig ${epair_two}a 198.51.100.2/24 up
213	jexec two ifconfig ${epair_sync}b 192.0.2.2/24 up
214
215	# Enable pf
216	jexec one pfctl -e
217	pft_set_rules one \
218		"set skip on ${epair_sync}a" \
219		"pass keep state"
220	jexec two pfctl -e
221	pft_set_rules two \
222		"set skip on ${epair_sync}b" \
223		"pass keep state"
224
225	ifconfig ${epair_one}b 198.51.100.254/24 up
226
227	# Create state prior to setting up pfsync
228	ping -c 1 -S 198.51.100.254 198.51.100.1
229
230	# Wait before setting up pfsync on two, so we don't accidentally catch
231	# the update anyway.
232	sleep 1
233
234	# Now set up pfsync in jail two
235	jexec two ifconfig pfsync0 \
236		syncdev ${epair_sync}b \
237		up
238
239	# Give pfsync time to do its thing
240	sleep 2
241
242	jexec two pfctl -s states
243	if ! jexec two pfctl -s states | grep icmp | grep 198.51.100.1 | \
244	    grep 198.51.100.2 ; then
245		atf_fail "state not found on synced host"
246	fi
247}
248
249bulk_cleanup()
250{
251	pfsynct_cleanup
252}
253
254atf_test_case "pbr" "cleanup"
255pbr_head()
256{
257	atf_set descr 'route_to and reply_to directives test'
258	atf_set require.user root
259	atf_set timeout '600'
260}
261
262pbr_body()
263{
264	pbr_common_body
265}
266
267pbr_cleanup()
268{
269	pbr_common_cleanup
270}
271
272atf_test_case "pfsync_pbr" "cleanup"
273pfsync_pbr_head()
274{
275	atf_set descr 'route_to and reply_to directives pfsync test'
276	atf_set require.user root
277	atf_set timeout '600'
278}
279
280pfsync_pbr_body()
281{
282	pbr_common_body backup_promotion
283}
284
285pfsync_pbr_cleanup()
286{
287	pbr_common_cleanup
288}
289
290pbr_common_body()
291{
292	# + builds bellow topology and initiate a single ping session
293	#   from client to server.
294	# + gw* forward traffic through pbr not fib lookups.
295	# + if backup_promotion arg is given, a carp failover event occurs
296	#   during the ping session on both gateways.
297	#                   ┌──────┐
298	#                   │client│
299	#                   └───┬──┘
300	#                       │
301	#                   ┌───┴───┐
302	#                   │bridge0│
303	#                   └┬─────┬┘
304	#                    │     │
305	#   ┌────────────────┴─┐ ┌─┴────────────────┐
306	#   │gw_route_to_master├─┤gw_route_to_backup│
307	#   └────────────────┬─┘ └─┬────────────────┘
308	#                    │     │
309	#                   ┌┴─────┴┐
310	#                   │bridge1│
311	#                   └┬─────┬┘
312	#                    │     │
313	#   ┌────────────────┴─┐ ┌─┴────────────────┐
314	#   │gw_reply_to_master├─┤gw_reply_to_backup│
315	#   └────────────────┬─┘ └─┬────────────────┘
316	#                    │     │
317	#                   ┌┴─────┴┐
318	#                   │bridge2│
319	#                   └───┬───┘
320	#                       │
321	#                   ┌───┴──┐
322	#                   │server│
323	#                   └──────┘
324
325	if ! kldstat -q -m carp
326	then
327		atf_skip "This test requires carp"
328	fi
329	pfsynct_init
330
331	bridge0=$(vnet_mkbridge)
332	bridge1=$(vnet_mkbridge)
333	bridge2=$(vnet_mkbridge)
334
335	epair_sync_gw_route_to=$(vnet_mkepair)
336	epair_sync_gw_reply_to=$(vnet_mkepair)
337	epair_client_bridge0=$(vnet_mkepair)
338
339	epair_gw_route_to_master_bridge0=$(vnet_mkepair)
340	epair_gw_route_to_backup_bridge0=$(vnet_mkepair)
341	epair_gw_route_to_master_bridge1=$(vnet_mkepair)
342	epair_gw_route_to_backup_bridge1=$(vnet_mkepair)
343
344	epair_gw_reply_to_master_bridge1=$(vnet_mkepair)
345	epair_gw_reply_to_backup_bridge1=$(vnet_mkepair)
346	epair_gw_reply_to_master_bridge2=$(vnet_mkepair)
347	epair_gw_reply_to_backup_bridge2=$(vnet_mkepair)
348
349	epair_server_bridge2=$(vnet_mkepair)
350
351	ifconfig ${bridge0} up
352	ifconfig ${epair_client_bridge0}b up
353	ifconfig ${epair_gw_route_to_master_bridge0}b up
354	ifconfig ${epair_gw_route_to_backup_bridge0}b up
355	ifconfig ${bridge0} \
356		addm ${epair_client_bridge0}b \
357		addm ${epair_gw_route_to_master_bridge0}b \
358		addm ${epair_gw_route_to_backup_bridge0}b
359
360	ifconfig ${bridge1} up
361	ifconfig ${epair_gw_route_to_master_bridge1}b up
362	ifconfig ${epair_gw_route_to_backup_bridge1}b up
363	ifconfig ${epair_gw_reply_to_master_bridge1}b up
364	ifconfig ${epair_gw_reply_to_backup_bridge1}b up
365	ifconfig ${bridge1} \
366		addm ${epair_gw_route_to_master_bridge1}b \
367		addm ${epair_gw_route_to_backup_bridge1}b \
368		addm ${epair_gw_reply_to_master_bridge1}b \
369		addm ${epair_gw_reply_to_backup_bridge1}b
370
371	ifconfig ${bridge2} up
372	ifconfig ${epair_gw_reply_to_master_bridge2}b up
373	ifconfig ${epair_gw_reply_to_backup_bridge2}b up
374	ifconfig ${epair_server_bridge2}b up
375	ifconfig ${bridge2} \
376		addm ${epair_gw_reply_to_master_bridge2}b \
377		addm ${epair_gw_reply_to_backup_bridge2}b \
378		addm ${epair_server_bridge2}b
379
380	vnet_mkjail client ${epair_client_bridge0}a
381	jexec client hostname client
382	vnet_mkjail gw_route_to_master \
383		${epair_gw_route_to_master_bridge0}a \
384		${epair_gw_route_to_master_bridge1}a \
385		${epair_sync_gw_route_to}a
386	jexec gw_route_to_master hostname gw_route_to_master
387	vnet_mkjail gw_route_to_backup \
388		${epair_gw_route_to_backup_bridge0}a \
389		${epair_gw_route_to_backup_bridge1}a \
390		${epair_sync_gw_route_to}b
391	jexec gw_route_to_backup hostname gw_route_to_backup
392	vnet_mkjail gw_reply_to_master \
393		${epair_gw_reply_to_master_bridge1}a \
394		${epair_gw_reply_to_master_bridge2}a \
395		${epair_sync_gw_reply_to}a
396	jexec gw_reply_to_master hostname gw_reply_to_master
397	vnet_mkjail gw_reply_to_backup \
398		${epair_gw_reply_to_backup_bridge1}a \
399		${epair_gw_reply_to_backup_bridge2}a \
400		${epair_sync_gw_reply_to}b
401	jexec gw_reply_to_backup hostname gw_reply_to_backup
402	vnet_mkjail server ${epair_server_bridge2}a
403	jexec server hostname server
404
405	jexec client ifconfig ${epair_client_bridge0}a inet 198.18.0.1/24 up
406	jexec client route add 198.18.2.0/24 198.18.0.10
407
408	jexec gw_route_to_master ifconfig ${epair_sync_gw_route_to}a \
409		inet 198.19.10.1/24 up
410	jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge0}a \
411		inet 198.18.0.8/24 up
412	jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge0}a \
413		alias 198.18.0.10/32 vhid 10 pass 3WjvVVw7 advskew 50
414	jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge1}a \
415		inet 198.18.1.8/24 up
416	jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge1}a \
417		alias 198.18.1.10/32 vhid 11 pass 3WjvVVw7 advskew 50
418	jexec gw_route_to_master sysctl net.inet.ip.forwarding=1
419	jexec gw_route_to_master sysctl net.inet.carp.preempt=1
420	jexec gw_route_to_master ifconfig ${epair_sync_gw_route_to}a name if_pfsync
421	sed -i '' -e 's/'${epair_sync_gw_route_to}'a/if_pfsync/g' created_interfaces.lst
422	jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge0}a name if_br0
423	sed -i '' -e 's/'${epair_gw_route_to_master_bridge0}'a/if_br0/g' created_interfaces.lst
424	jexec gw_route_to_master ifconfig ${epair_gw_route_to_master_bridge1}a name if_br1
425	sed -i '' -e 's/'${epair_gw_route_to_master_bridge1}'a/if_br1/g' created_interfaces.lst
426	jexec gw_route_to_master ifconfig pfsync0 \
427		syncpeer 198.19.10.2 \
428		syncdev if_pfsync \
429		maxupd 1 \
430		up
431	pft_set_rules gw_route_to_master \
432		"keep_state = 'tag auth_packet keep state'" \
433		"set timeout { icmp.first 120, icmp.error 60 }" \
434		"block log all" \
435		"pass quick on if_pfsync proto pfsync keep state (no-sync)" \
436		"pass quick on { if_br0 if_br1 } proto carp keep state (no-sync)" \
437		"block drop in quick to 224.0.0.18/32" \
438		"pass out quick tagged auth_packet keep state" \
439		"pass in quick log on if_br0 route-to (if_br1 198.18.1.20) proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 \$keep_state"
440	jexec gw_route_to_master pfctl -e
441
442	jexec gw_route_to_backup ifconfig ${epair_sync_gw_route_to}b \
443		inet 198.19.10.2/24 up
444	jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge0}a \
445		inet 198.18.0.9/24 up
446	jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge0}a \
447		alias 198.18.0.10/32 vhid 10 pass 3WjvVVw7 advskew 100
448	jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge1}a \
449		inet 198.18.1.9/24 up
450	jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge1}a \
451		alias 198.18.1.10/32 vhid 11 pass 3WjvVVw7 advskew 100
452	jexec gw_route_to_backup sysctl net.inet.ip.forwarding=1
453	jexec gw_route_to_backup sysctl net.inet.carp.preempt=1
454	jexec gw_route_to_backup ifconfig ${epair_sync_gw_route_to}b name if_pfsync
455	sed -i '' -e 's/'${epair_sync_gw_route_to}'b/if_pfsync/g' created_interfaces.lst
456	jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge0}a name if_br0
457	sed -i '' -e 's/'${epair_gw_route_to_backup_bridge0}'a/if_br0/g' created_interfaces.lst
458	jexec gw_route_to_backup ifconfig ${epair_gw_route_to_backup_bridge1}a name if_br1
459	sed -i '' -e 's/'${epair_gw_route_to_backup_bridge1}'a/if_br1/g' created_interfaces.lst
460	jexec gw_route_to_backup ifconfig pfsync0 \
461		syncpeer 198.19.10.1 \
462		syncdev if_pfsync \
463		up
464	pft_set_rules gw_route_to_backup \
465		"keep_state = 'tag auth_packet keep state'" \
466		"set timeout { icmp.first 120, icmp.error 60 }" \
467		"block log all" \
468		"pass quick on if_pfsync proto pfsync keep state (no-sync)" \
469		"pass quick on { if_br0 if_br1 } proto carp keep state (no-sync)" \
470		"block drop in quick to 224.0.0.18/32" \
471		"pass out quick tagged auth_packet keep state" \
472		"pass in quick log on if_br0 route-to (if_br1 198.18.1.20) proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 \$keep_state"
473	jexec gw_route_to_backup pfctl -e
474
475	jexec gw_reply_to_master ifconfig ${epair_sync_gw_reply_to}a \
476		inet 198.19.20.1/24 up
477	jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge1}a \
478		inet 198.18.1.18/24 up
479	jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge1}a \
480		alias 198.18.1.20/32 vhid 21 pass 3WjvVVw7 advskew 50
481	jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge2}a \
482		inet 198.18.2.18/24 up
483	jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge2}a \
484		alias 198.18.2.20/32 vhid 22 pass 3WjvVVw7 advskew 50
485	jexec gw_reply_to_master sysctl net.inet.ip.forwarding=1
486	jexec gw_reply_to_master sysctl net.inet.carp.preempt=1
487	jexec gw_reply_to_master ifconfig ${epair_sync_gw_reply_to}a name if_pfsync
488	sed -i '' -e 's/'${epair_sync_gw_reply_to}'a/if_pfsync/g' created_interfaces.lst
489	jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge1}a name if_br1
490	sed -i '' -e 's/'${epair_gw_reply_to_master_bridge1}'a/if_br1/g' created_interfaces.lst
491	jexec gw_reply_to_master ifconfig ${epair_gw_reply_to_master_bridge2}a name if_br2
492	sed -i '' -e 's/'${epair_gw_reply_to_master_bridge2}'a/if_br2/g' created_interfaces.lst
493	jexec gw_reply_to_master ifconfig pfsync0 \
494		syncpeer 198.19.20.2 \
495		syncdev if_pfsync \
496		maxupd 1 \
497		up
498	pft_set_rules gw_reply_to_master \
499		"set timeout { icmp.first 120, icmp.error 60 }" \
500		"block log all" \
501		"pass quick on if_pfsync proto pfsync keep state (no-sync)" \
502		"pass quick on { if_br1 if_br2 } proto carp keep state (no-sync)" \
503		"block drop in quick to 224.0.0.18/32" \
504		"pass out quick on if_br2 reply-to (if_br1 198.18.1.10) tagged auth_packet_reply_to keep state" \
505		"pass in quick log on if_br1 proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 tag auth_packet_reply_to keep state"
506	jexec gw_reply_to_master pfctl -e
507
508	jexec gw_reply_to_backup ifconfig ${epair_sync_gw_reply_to}b \
509		inet 198.19.20.2/24 up
510	jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge1}a \
511		inet 198.18.1.19/24 up
512	jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge1}a \
513		alias 198.18.1.20/32 vhid 21 pass 3WjvVVw7 advskew 100
514	jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge2}a \
515		inet 198.18.2.19/24 up
516	jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge2}a \
517		alias 198.18.2.20/32 vhid 22 pass 3WjvVVw7 advskew 100
518	jexec gw_reply_to_backup sysctl net.inet.ip.forwarding=1
519	jexec gw_reply_to_backup sysctl net.inet.carp.preempt=1
520	jexec gw_reply_to_backup ifconfig ${epair_sync_gw_reply_to}b name if_pfsync
521	sed -i '' -e 's/'${epair_sync_gw_reply_to}'b/if_pfsync/g' created_interfaces.lst
522	jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge1}a name if_br1
523	sed -i '' -e 's/'${epair_gw_reply_to_backup_bridge1}'a/if_br1/g' created_interfaces.lst
524	jexec gw_reply_to_backup ifconfig ${epair_gw_reply_to_backup_bridge2}a name if_br2
525	sed -i '' -e 's/'${epair_gw_reply_to_backup_bridge2}'a/if_br2/g' created_interfaces.lst
526	jexec gw_reply_to_backup ifconfig pfsync0 \
527		syncpeer 198.19.20.1 \
528		syncdev if_pfsync \
529		up
530	pft_set_rules gw_reply_to_backup \
531		"set timeout { icmp.first 120, icmp.error 60 }" \
532		"block log all" \
533		"pass quick on if_pfsync proto pfsync keep state (no-sync)" \
534		"pass quick on { if_br1 if_br2 } proto carp keep state (no-sync)" \
535		"block drop in quick to 224.0.0.18/32" \
536		"pass out quick on if_br2 reply-to (if_br1 198.18.1.10) tagged auth_packet_reply_to keep state" \
537		"pass in quick log on if_br1 proto { icmp udp tcp } from 198.18.0.0/24 to 198.18.2.0/24 tag auth_packet_reply_to keep state"
538	jexec gw_reply_to_backup pfctl -e
539
540	jexec server ifconfig ${epair_server_bridge2}a inet 198.18.2.1/24 up
541	jexec server route add 198.18.0.0/24 198.18.2.20
542
543	# Waiting for platform to settle
544	while ! jexec gw_route_to_backup ifconfig | grep 'carp: BACKUP'
545	do
546		sleep 1
547	done
548	while ! jexec gw_reply_to_backup ifconfig | grep 'carp: BACKUP'
549	do
550		sleep 1
551	done
552	while ! jexec client ping -c 10 198.18.2.1 | grep ', 0.0% packet loss'
553	do
554		sleep 1
555	done
556
557	# Checking cluster members pf.conf checksums match
558	gw_route_to_master_checksum=$(jexec gw_route_to_master pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2)
559	gw_route_to_backup_checksum=$(jexec gw_route_to_backup pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2)
560	gw_reply_to_master_checksum=$(jexec gw_reply_to_master pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2)
561	gw_reply_to_backup_checksum=$(jexec gw_reply_to_backup pfctl -si -v | grep 'Checksum:' | cut -d ' ' -f 2)
562	if [ "$gw_route_to_master_checksum" != "$gw_route_to_backup_checksum" ]
563	then
564		atf_fail "gw_route_to cluster members pf.conf do not match each others"
565	fi
566	if [ "$gw_reply_to_master_checksum" != "$gw_reply_to_backup_checksum" ]
567	then
568		atf_fail "gw_reply_to cluster members pf.conf do not match each others"
569	fi
570
571	# Creating state entries
572	(jexec client ping -c 10 198.18.2.1 >ping.stdout) &
573
574	if [ "$1" = "backup_promotion" ]
575	then
576		sleep 1
577		jexec gw_route_to_backup ifconfig if_br0 vhid 10 advskew 0
578		jexec gw_route_to_backup ifconfig if_br1 vhid 11 advskew 0
579		jexec gw_reply_to_backup ifconfig if_br1 vhid 21 advskew 0
580		jexec gw_reply_to_backup ifconfig if_br2 vhid 22 advskew 0
581	fi
582	while ! grep -q -e 'packet loss' ping.stdout
583	do
584		sleep 1
585	done
586
587	# As cleanup is long and may lead to a timeout,
588	# it's run directly into the body part.
589	# (as cleanup timeout is not settable)
590	jail -r \
591		client \
592		gw_route_to_master \
593		gw_route_to_backup \
594		gw_reply_to_master \
595		gw_reply_to_backup \
596		server
597	for ifname in $(grep -E -e 'if_' -e 'epair.*a' -e 'bridge' created_interfaces.lst)
598	do
599		ifconfig $ifname >/dev/null 2>&1 && ifconfig $ifname destroy
600	done
601
602	atf_check -s exit:0 -e ignore -o ignore grep ', 0.0% packet loss' ping.stdout
603}
604
605pbr_common_cleanup()
606{
607	for jailname in client gw_route_to_master gw_route_to_backup gw_reply_to_master gw_reply_to_backup server
608	do
609		if $(jls | grep -q $jailname); then
610			jail -r $jailname
611		else
612			echo "$jailname already cleaned"
613		fi
614	done
615	for ifname in $(grep -E -e 'if_' -e 'epair.*a' -e 'bridge' created_interfaces.lst)
616	do
617		ifconfig $ifname >/dev/null 2>&1
618		if [ "$?" -eq "0" ]; then
619			ifconfig $ifname destroy
620		else
621			echo "$ifname already destroyed"
622		fi
623	done
624}
625
626atf_init_test_cases()
627{
628	atf_add_test_case "basic"
629	atf_add_test_case "basic_defer"
630	atf_add_test_case "defer"
631	atf_add_test_case "bulk"
632	atf_add_test_case "pbr"
633	atf_add_test_case "pfsync_pbr"
634}
635