xref: /freebsd/tests/ci/Makefile (revision 3068d706eabe99f930fb01d3cbfd74ff1f0eb5a2) 
1# SPDX-License-Identifier: BSD-2-Clause
2#
3# Copyright (c) 2024 The FreeBSD Foundation
4#
5# This software was developed by Cybermancer Infosec <bofh@FreeBSD.org>
6# under sponsorship from the FreeBSD Foundation.
7#
8# Makefile for CI testing.
9#
10# User-driven targets:
11#  ci: Run CI tests
12#  ci-smoke: Run smoke tests which is simply booting the image
13#  ci-full: Run full tests
14#
15# Variables affecting the build process:
16#  TARGET/TARGET_ARCH: architecture of built release (default: same as build host)
17#  KERNELCONF: kernel configuration to use
18#  USE_QEMU: Use QEMU for testing rather than bhyve
19#
20
21WORLDDIR?=	${.CURDIR}/../..
22RELEASEDIR=	${WORLDDIR}/release
23MAKECONF?=	/dev/null
24SRCCONF?=	/dev/null
25_MEMORY!=sysctl -n hw.physmem 2>/dev/null
26PARALLEL_JOBS!=sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null
27TOTAL_MEMORY!=expr ${_MEMORY} / 1073741824
28KERNCONF?=	GENERIC
29LOCALBASE?=	/usr/local
30EXTRA_MAKE_FLAGS?=
31
32.if !defined(TARGET) || empty(TARGET)
33TARGET=		${MACHINE}
34.endif
35.if !defined(TARGET_ARCH) || empty(TARGET_ARCH)
36.  if ${TARGET} == ${MACHINE}
37TARGET_ARCH=	${MACHINE_ARCH}
38.  else
39TARGET_ARCH=	${TARGET}
40.  endif
41.endif
42IMAKE=		${MAKE} TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH}
43
44.if defined(CROSS_TOOLCHAIN) || !empty(CROSS_TOOLCHAIN)
45CROSS_TOOLCHAIN_PARAM=	"CROSS_TOOLCHAIN=${CROSS_TOOLCHAIN}"
46.endif
47
48# Define OSRELEASE by using newvers.sh
49.if !defined(OSRELEASE) || empty(OSRELEASE)
50.  for _V in TYPE BRANCH REVISION
51.    if !defined(${_V}) || empty(${_V})
52${_V}!=	eval $$(awk '/^${_V}=/{print}' ${.CURDIR}/../../sys/conf/newvers.sh); echo $$${_V}
53.    endif
54.  endfor
55.  for _V in ${TARGET_ARCH}
56.    if !empty(TARGET:M${_V})
57OSRELEASE=	${TYPE}-${REVISION}-${BRANCH}-${TARGET}
58VOLUME_LABEL=	${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-]/_/g}_${TARGET}
59.    else
60OSRELEASE=	${TYPE}-${REVISION}-${BRANCH}-${TARGET}-${TARGET_ARCH}
61VOLUME_LABEL=	${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-]/_/g}_${TARGET_ARCH}
62.    endif
63.  endfor
64.endif
65
66.if exists(${.CURDIR}/tools/ci.conf) && !defined(CICONF)
67CICONF?=	${.CURDIR}/tools/ci.conf
68.endif
69SWAPSIZE?=	1g
70VMFS?=		ufs
71FORMAT=		raw
72CIIMAGE=	ci-${OSRELEASE}-${GITREV}-${KERNCONF}.${FORMAT}
73CIDISK?=	${.OBJDIR}/${CIIMAGE}
74VMSIZE?=	6g
75CITYPE?=	full
76KYUA_TEST_FILTERS?=
77META_TAR!=mktemp /tmp/meta.XXXXXX
78META_DIR!=mktemp -d /tmp/meta.XXXXXX
79META_DIROUT!=mktemp -d /tmp/meta.XXXXXX
80DISC_CAM!=truncate -s 128m /tmp/disk-cam
81EXTRA_DISK_NUM?=5
82DISK_NUMBERS!=jot - 1 ${EXTRA_DISK_NUM}
83BHYVE_EXTRA_DISK_PARAM?=
84BHYVE_EXTRA_DISK_PARAM+=-s 4:0,ahci-hd,/tmp/disk-cam
85.for i in ${DISK_NUMBERS}
86NEW_DISK!=truncate -s 128m /tmp/disk${i}
87BHYVE_EXTRA_DISK_PARAM+=-s $$((${i} + 4)):0,virtio-blk,/tmp/disk${i}
88CLEANFILES+=/tmp/disk${i}
89.endfor
90TEST_VM_NAME=	ci-${OSRELEASE}-${GITREV}-${KERNCONF}
91.if ${TOTAL_MEMORY} >= 16
92VM_MEM=8
93.elif ${TOTAL_MEMORY} >=4
94VM_MEM=${TOTAL_MEMORY}
95.else
96echo "Please increase the memory to at least 4GB"
97exit 0
98.endif
99VM_MEM_SIZE?=${VM_MEM}g
100TIMEOUT_MS?=5400000
101TIMEOUT=$$((${TIMEOUT_MS} / 1000))
102TIMEOUT_EXPECT=$$((${TIMEOUT} - 60))
103TIMEOUT_VM=$$((${TIMEOUT_EXPECT} - 120))
104.if exists(${.CURDIR}/Makefile.${TARGET_ARCH})
105.	include "${.CURDIR}/Makefile.${TARGET_ARCH}"
106.endif
107.if ${TARGET_ARCH} != ${MACHINE_ARCH}
108.  if ( ${TARGET_ARCH} != "i386" ) || ( ${MACHINE_ARCH} != "amd64" )
109QEMUSTATIC=/usr/local/bin/qemu-${QEMU_ARCH}-static
110QEMUTGT=portinstall-qemu
111.  endif
112.endif
113QEMUTGT?=
114QEMU_DEVICES?=
115QEMU_EXTRA_PARAM?=
116QEMU_MACHINE?=virt
117QEMUBIN=/usr/local/bin/qemu-system-${QEMU_ARCH}
118.if ${PARALLEL_JOBS} >= ${QEMU_MAX_CPU_COUNT}
119QEMU_CPU_COUNT=${QEMU_MAX_CPU_COUNT}
120.else
121QEMU_CPU_COUNT=${PARALLEL_JOBS}
122.endif
123.if ${VM_MEM} >= ${QEMU_MAX_MEM_SIZE}
124VM_MEM_SIZE=${QEMU_MAX_MEM_SIZE}g
125.else
126VM_MEM_SIZE=${VM_MEM}g
127.endif
128VMGUEST!=sysctl -n kern.vm_guest 2>/dev/null || true
129.if ${VMGUEST} != "none"
130USE_QEMU?=1
131.endif
132KLDFILEMONISLOADED!=kldload -q -n filemon 2>/dev/null && echo "1" || echo "0"
133.if ${KLDFILEMONISLOADED} == "1"
134METAMODE?=-DWITH_META_MODE
135.endif
136
137CLEANFILES+=	${.OBJDIR}/${CIIMAGE} ${.OBJDIR}/ci.img ${META_TAR}
138IMAGEDIR=	${.OBJDIR}/ci-buildimage
139CLEANDIRS+=	${IMAGEDIR}
140
141portinstall: portinstall-pkg portinstall-qemu portinstall-expect portinstall-${TARGET_ARCH:tl} .PHONY
142
143portinstall-pkg: .PHONY
144.if !exists(/usr/local/sbin/pkg-static)
145	env ASSUME_ALWAYS_YES=yes pkg bootstrap
146.endif
147
148portinstall-qemu: portinstall-pkg .PHONY
149.if !exists(/usr/local/bin/qemu-${QEMU_ARCH}-static)
150	env ASSUME_ALWAYS_YES=yes pkg install emulators/qemu-user-static
151.endif
152.if !exists(${QEMUBIN})
153	env ASSUME_ALWAYS_YES=yes pkg install emulators/qemu@nox11
154.endif
155
156portinstall-expect: portinstall-pkg .PHONY
157.if !exists(/usr/local/bin/expect)
158	env ASSUME_ALWAYS_YES=yes pkg install lang/expect
159.endif
160
161beforeclean: .PHONY
162	chflags -R noschg ${IMAGEDIR}
163
164.include <bsd.obj.mk>
165clean: beforeclean .PHONY
166
167cleandir: beforeclean .PHONY
168
169ci-buildworld: .PHONY
170	@echo "Building world for ${TARGET_ARCH}"
171	${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \
172		${CROSS_TOOLCHAIN_PARAM} __MAKE_CONF=${MAKECONF} SRCCONF=${SRCCONF} \
173		${EXTRA_MAKE_FLAGS} buildworld > ${.CURDIR}/_.${TARGET_ARCH}.${.TARGET} 2>&1 || \
174		(echo "${.TARGET} failed, check _.${TARGET_ARCH}.${.TARGET} for details" ; false)
175
176
177ci-buildkernel: ci-buildworld-${TARGET_ARCH:tl} .PHONY
178	@echo "Building kernel for ${TARGET_ARCH}"
179	${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \
180		${CROSS_TOOLCHAIN_PARAM} __MAKE_CONF=${MAKECONF} SRCCONF=${SRCCONF} \
181		${EXTRA_MAKE_FLAGS} KERNCONF=${KERNCONF} \
182		buildkernel > ${.CURDIR}/_.${TARGET_ARCH}.${.TARGET} 2>&1 || \
183		(echo "${.TARGET} failed, check _.${TARGET_ARCH}.${.TARGET} for details" ; false)
184
185ci-buildimage: ${QEMUTGT} ci-buildkernel-${TARGET_ARCH:tl} .PHONY
186	@echo "Building ci image for ${TARGET_ARCH}"
187	mkdir -p ${.OBJDIR}/${.TARGET}
188	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} SWAPSIZE=${SWAPSIZE} \
189		QEMUSTATIC=${QEMUSTATIC} CITYPE=${CITYPE} \
190		${RELEASEDIR}/scripts/mk-vmimage.sh \
191		-C ${RELEASEDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} -F ${VMFS} \
192		-i ${.OBJDIR}/ci.img -s ${VMSIZE} -f ${FORMAT} \
193		-S ${WORLDDIR} -o ${.OBJDIR}/${CIIMAGE} -c ${CICONF} \
194		> ${.CURDIR}/_.${TARGET_ARCH}.${.TARGET} 2>&1 || \
195		(echo "${.TARGET} failed, check _.${TARGET_ARCH}.${.TARGET} for details" ; false)
196	touch ${.TARGET}
197
198ci-set-smoke-var: .PHONY
199CITYPE=smoke
200
201ci-set-full-var: .PHONY
202CITYPE=full
203
204ci-create-meta: .PHONY
205	truncate -s 512M ${META_TAR}
206	tar rvf ${META_TAR} -C ${META_DIR} .
207
208ci-extract-meta: .PHONY
209	tar xfv ${META_TAR} -C ${META_DIROUT}
210	rm -rf ${META_TAR} ${META_DIR}
211	@echo "Extracted kyua reports to ${META_DIROUT}"
212
213ci-runtest: ci-buildimage-${TARGET_ARCH:tl} portinstall .PHONY
214.if ${MACHINE} == "amd64" && ( ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" ) && ( !defined(USE_QEMU) || empty(USE_QEMU) )
215	/usr/sbin/bhyvectl --vm=${TEST_VM_NAME} --destroy || true
216	expect -c "set timeout ${TIMEOUT_EXPECT}; \
217		spawn /usr/bin/timeout -k 5s 30s /usr/sbin/bhyveload \
218		-c stdio -m ${VM_MEM_SIZE} -d ${CIDISK} ${TEST_VM_NAME}; \
219		expect { eof }; \
220		exit [lindex [wait] 3]"
221	expect -c "set timeout ${TIMEOUT_EXPECT}; \
222		spawn /usr/bin/timeout -k 60 ${TIMEOUT_VM} /usr/sbin/bhyve \
223		-c ${PARALLEL_JOBS} -m ${VM_MEM_SIZE} -A -H -P \
224		-s 0:0,hostbridge \
225		-s 1:0,lpc \
226		-s 2:0,virtio-blk,${CIDISK} \
227		-s 3:0,virtio-blk,${META_TAR} \
228		${BHYVE_EXTRA_DISK_PARAM} \
229		-l com1,stdio \
230		${TEST_VM_NAME}; \
231		expect { eof }"
232	/usr/sbin/bhyvectl --vm=${TEST_VM_NAME} --destroy
233.else
234	timeout -k 60 ${TIMEOUT_VM} ${QEMUBIN} \
235		-machine ${QEMU_MACHINE} \
236		-smp ${QEMU_CPU_COUNT} \
237		-m ${VM_MEM_SIZE} \
238		-nographic \
239		-no-reboot \
240		${QEMU_EXTRA_PARAM} \
241		-device virtio-blk,drive=hd0 \
242		-device virtio-blk,drive=hd1 \
243		-blockdev driver=raw,node-name=hd0,file.driver=file,file.filename=${CIDISK} \
244		-blockdev driver=raw,node-name=hd1,file.driver=file,file.filename=${META_TAR} \
245		${QEMU_DEVICES}
246.endif
247
248ci-checktarget: .PHONY
249.if ${TARGET_ARCH} != "aarch64" && \
250	${TARGET_ARCH} != "amd64" && \
251	${TARGET_ARCH} != "armv7" && \
252	${TARGET_ARCH} != "powerpc64" && \
253	${TARGET_ARCH} != "powerpc64le" && \
254	${TARGET_ARCH} != "riscv64"
255	@false
256.ERROR:
257	@echo "Error: ${TARGET_ARCH} is not supported on ${TYPE} ${REVISION} ${BRANCH}"
258.endif
259
260ci-smoke: ci-set-smoke-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY
261
262ci-full: ci-set-full-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .WAIT ci-extract-meta .PHONY
263
264ci: ci-${CITYPE:tl} .PHONY
265
266.include "${RELEASEDIR}/Makefile.inc1"
267