xref: /freebsd/tests/ci/Makefile (revision 02376be32d6bcccc0c722c08779bd71e70516599) 
1# SPDX-License-Identifier: BSD-2-Clause
2#
3# Copyright (c) 2024 The FreeBSD Foundation
4#
5# This software was developed by Cybermancer Infosec <bofh@FreeBSD.org>
6# under sponsorship from the FreeBSD Foundation.
7#
8# Makefile for CI testing.
9#
10# User-driven targets:
11#  ci: Run CI tests
12#  ci-smoke: Run smoke tests which is simply booting the image
13#  ci-full: Run full tests
14#
15# Variables affecting the build process:
16#  TARGET/TARGET_ARCH: architecture of built release (default: same as build host)
17#  KERNELCONF: kernel configuration to use
18#  USE_QEMU: Use QEMU for testing rather than bhyve
19#
20
21WORLDDIR?=	${.CURDIR:H:H}
22RELEASEDIR=	${WORLDDIR}/release
23MAKECONF?=	/dev/null
24SRCCONF?=	/dev/null
25_MEMORY!=sysctl -n hw.physmem 2>/dev/null
26PARALLEL_JOBS!=sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null
27TOTAL_MEMORY!=expr ${_MEMORY} / 1073741824
28KERNCONF?=	GENERIC
29LOCALBASE?=	/usr/local
30EXTRA_MAKE_FLAGS?=
31
32.if !defined(TARGET) || empty(TARGET)
33TARGET=		${MACHINE}
34.endif
35.if !defined(TARGET_ARCH) || empty(TARGET_ARCH)
36.  if ${TARGET} == ${MACHINE}
37TARGET_ARCH=	${MACHINE_ARCH}
38.  else
39TARGET_ARCH=	${TARGET}
40.  endif
41.endif
42IMAKE=		${MAKE} TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH}
43
44.if !defined(CIENV) || empty(CIENV)
45CIENV=		local
46LOG_TARGET=	> ${.CURDIR}/_.${TARGET_ARCH}.${.TARGET} 2>&1 || (echo "${.TARGET} failed, check _.${TARGET_ARCH}.${.TARGET} for details" ; false)
47.endif
48
49.if defined(CROSS_TOOLCHAIN) || !empty(CROSS_TOOLCHAIN)
50CROSS_TOOLCHAIN_PARAM=	"CROSS_TOOLCHAIN=${CROSS_TOOLCHAIN}"
51.endif
52
53# Define OSRELEASE by using newvers.sh
54.if !defined(OSRELEASE) || empty(OSRELEASE)
55.  for _V in TYPE BRANCH REVISION
56.    if !defined(${_V}) || empty(${_V})
57${_V}!=	eval $$(awk '/^${_V}=/{print}' ${.CURDIR}/../../sys/conf/newvers.sh); echo $$${_V}
58.    endif
59.  endfor
60.  for _V in ${TARGET_ARCH}
61.    if !empty(TARGET:M${_V})
62OSRELEASE=	${TYPE}-${REVISION}-${BRANCH:C/[.-\/]/_/g}-${TARGET}
63VOLUME_LABEL=	${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-\/]/_/g}_${TARGET}
64.    else
65OSRELEASE=	${TYPE}-${REVISION}-${BRANCH:C/[.-\/]/_/g}-${TARGET}-${TARGET_ARCH}
66VOLUME_LABEL=	${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-\/]/_/g}_${TARGET_ARCH}
67.    endif
68.  endfor
69.endif
70
71.if exists(${.CURDIR}/tools/ci.conf) && !defined(CICONF)
72CICONF?=	${.CURDIR}/tools/ci.conf
73.endif
74SWAPSIZE?=	1g
75VMFS?=		ufs
76FORMAT=		raw
77CIIMAGE=	ci-${OSRELEASE}-${GITREV}-${KERNCONF}.${FORMAT}
78CIDISK?=	${.OBJDIR}/${CIIMAGE}
79VMSIZE?=	6g
80CITYPE?=	full
81KYUA_TEST_FILTERS?=
82META_TAR!=mktemp /tmp/meta.XXXXXX
83META_DIR!=mktemp -d /tmp/meta.XXXXXX
84META_DIROUT!=mktemp -d /tmp/meta.XXXXXX
85ENV_FILE=	${META_DIR}/ci-${OSRELEASE}-${GITREV}-${KERNCONF}.env
86DISC_CAM!=truncate -s 128m /tmp/disk-cam
87EXTRA_DISK_NUM?=5
88DISK_NUMBERS!=jot - 1 ${EXTRA_DISK_NUM}
89BHYVE_EXTRA_DISK_PARAM?=
90BHYVE_EXTRA_DISK_PARAM+=-s 4:0,ahci-hd,/tmp/disk-cam
91.for i in ${DISK_NUMBERS}
92NEW_DISK!=truncate -s 128m /tmp/disk${i}
93BHYVE_EXTRA_DISK_PARAM+=-s $$((${i} + 4)):0,virtio-blk,/tmp/disk${i}
94CLEANFILES+=/tmp/disk${i}
95.endfor
96TEST_VM_NAME=	ci-${OSRELEASE}-${GITREV}-${KERNCONF}
97.if ${TOTAL_MEMORY} >= 16
98VM_MEM=8
99.elif ${TOTAL_MEMORY} >=4
100VM_MEM=${TOTAL_MEMORY}
101.else
102echo "Please increase the memory to at least 4GB"
103exit 0
104.endif
105VM_MEM_SIZE?=${VM_MEM}g
106TIMEOUT_MS?=5400000
107TIMEOUT=$$((${TIMEOUT_MS} / 1000))
108TIMEOUT_EXPECT=$$((${TIMEOUT} - 60))
109TIMEOUT_VM=$$((${TIMEOUT_EXPECT} - 120))
110.if exists(${.CURDIR}/Makefile.${TARGET_ARCH})
111.	include "${.CURDIR}/Makefile.${TARGET_ARCH}"
112.endif
113.if ${TARGET_ARCH} != ${MACHINE_ARCH}
114.  if ( ${TARGET_ARCH} != "i386" ) || ( ${MACHINE_ARCH} != "amd64" )
115QEMUSTATIC=/usr/local/bin/qemu-${QEMU_ARCH}-static
116QEMUTGT=portinstall-qemu
117.  endif
118.endif
119QEMUTGT?=
120QEMU_DEVICES?=
121QEMU_EXTRA_PARAM?=
122QEMU_MACHINE?=virt
123QEMUBIN=/usr/local/bin/qemu-system-${QEMU_ARCH}
124.if ${PARALLEL_JOBS} >= ${QEMU_MAX_CPU_COUNT}
125QEMU_CPU_COUNT=${QEMU_MAX_CPU_COUNT}
126.else
127QEMU_CPU_COUNT=${PARALLEL_JOBS}
128.endif
129.if ${VM_MEM} >= ${QEMU_MAX_MEM_SIZE}
130VM_MEM_SIZE=${QEMU_MAX_MEM_SIZE}g
131.else
132VM_MEM_SIZE=${VM_MEM}g
133.endif
134VMGUEST!=sysctl -n kern.vm_guest 2>/dev/null || true
135.if ${VMGUEST} != "none"
136USE_QEMU?=1
137.endif
138KLDFILEMONISLOADED!=kldload -q -n filemon 2>/dev/null && echo "1" || echo "0"
139.if ${KLDFILEMONISLOADED} == "1"
140METAMODE?=-DWITH_META_MODE
141.endif
142
143CLEANFILES+=	${.OBJDIR}/${CIIMAGE} ${.OBJDIR}/ci.img ${META_TAR}
144IMAGEDIR=	${.OBJDIR}/ci-buildimage
145CLEANDIRS+=	${IMAGEDIR}
146
147portinstall: portinstall-pkg portinstall-qemu portinstall-expect portinstall-${TARGET_ARCH:tl} .PHONY
148
149portinstall-pkg: .PHONY
150.if !exists(/usr/local/sbin/pkg-static)
151	env ASSUME_ALWAYS_YES=yes pkg bootstrap
152.endif
153
154portinstall-qemu: portinstall-pkg .PHONY
155.if !exists(/usr/local/bin/qemu-${QEMU_ARCH}-static)
156	env ASSUME_ALWAYS_YES=yes pkg install emulators/qemu-user-static
157.endif
158.if !exists(${QEMUBIN})
159	env ASSUME_ALWAYS_YES=yes pkg install emulators/qemu@nox11
160.endif
161
162portinstall-expect: portinstall-pkg .PHONY
163.if !exists(/usr/local/bin/expect)
164	env ASSUME_ALWAYS_YES=yes pkg install lang/expect
165.endif
166
167beforeclean: .PHONY
168	chflags -R noschg ${IMAGEDIR}
169
170.include <bsd.obj.mk>
171clean: beforeclean .PHONY
172
173cleandir: beforeclean .PHONY
174
175ci-buildworld: .PHONY
176	@echo "Building world for ${TARGET_ARCH}"
177	${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \
178		${CROSS_TOOLCHAIN_PARAM} __MAKE_CONF=${MAKECONF} SRCCONF=${SRCCONF} \
179		${EXTRA_MAKE_FLAGS} buildworld ${LOG_TARGET}
180
181ci-buildkernel: ci-buildworld-${TARGET_ARCH:tl} .PHONY
182	@echo "Building kernel for ${TARGET_ARCH}"
183	${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \
184		${CROSS_TOOLCHAIN_PARAM} __MAKE_CONF=${MAKECONF} SRCCONF=${SRCCONF} \
185		${EXTRA_MAKE_FLAGS} KERNCONF=${KERNCONF} \
186		buildkernel ${LOG_TARGET}
187
188ci-buildimage: ${QEMUTGT} ci-buildkernel-${TARGET_ARCH:tl} .PHONY
189	@echo "Building ci image for ${TARGET_ARCH}"
190	mkdir -p ${.OBJDIR}/${.TARGET}
191	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} SWAPSIZE=${SWAPSIZE} \
192		QEMUSTATIC=${QEMUSTATIC} CITYPE=${CITYPE} \
193		${RELEASEDIR}/scripts/mk-vmimage.sh \
194		-C ${RELEASEDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} -F ${VMFS} \
195		-i ${.OBJDIR}/ci.img -s ${VMSIZE} -f ${FORMAT} \
196		-S ${WORLDDIR} -o ${.OBJDIR}/${CIIMAGE} -c ${CICONF} ${LOG_TARGET}
197	touch ${.TARGET}
198
199ci-set-smoke-var: .PHONY
200CITYPE=smoke
201
202ci-set-full-var: .PHONY
203CITYPE=full
204
205ci-get-env: .PHONY
206	@echo "TARGET=${TARGET}" > ${ENV_FILE}
207	@echo "TARGET_ARCH=${TARGET_ARCH}" >> ${ENV_FILE}
208	@echo "KERNCONF=${KERNCONF}" >> ${ENV_FILE}
209	@echo "OSRELEASE=${OSRELEASE}" >> ${ENV_FILE}
210	@echo "CIIMAGE=${CIIMAGE}" >> ${ENV_FILE}
211	@echo "CIDISK=${CIDISK}" >> ${ENV_FILE}
212	@echo "VMSIZE=${VMSIZE}" >> ${ENV_FILE}
213	@echo "VM_MEM_SIZE=${VM_MEM_SIZE}" >> ${ENV_FILE}
214	@echo "TIMEOUT=${TIMEOUT}s" >> ${ENV_FILE}
215	@echo "CITYPE=${CITYPE}" >> ${ENV_FILE}
216	@echo "KYUA_TEST_FILTERS='${KYUA_TEST_FILTERS}'" >> ${ENV_FILE}
217	@echo "META_MODE='${METAMODE}'" >> ${ENV_FILE}
218	@echo "USE_QEMU='${USE_QEMU}'" >> ${ENV_FILE}
219	@echo "GITREV='${GITREV}'" >> ${ENV_FILE}
220	@echo "GITBRANCH='${GITBRANCH}'" >> ${ENV_FILE}
221	@echo "GITCOUNT='${GITCOUNT}'" >> ${ENV_FILE}
222	@echo "BUILDDATE='${BUILDDATE}'" >> ${ENV_FILE}
223	@echo "CC='${CC}'" >> ${ENV_FILE}
224	@echo "CFLAGS='${CFLAGS}'" >> ${ENV_FILE}
225	@echo "CPP='${CPP}'" >> ${ENV_FILE}
226	@echo "CXX='${CXX}'" >> ${ENV_FILE}
227	@echo "CXXFLAGS='${CXXFLAGS}'" >> ${ENV_FILE}
228	@echo "COMPILER_FEATURES='${COMPILER_FEATURES}'" >> ${ENV_FILE}
229	@echo "COMPILER_FREEBSD_VERSION='${COMPILER_FREEBSD_VERSION}'" >> ${ENV_FILE}
230	@echo "COMPILER_TYPE='${COMPILER_TYPE}'" >> ${ENV_FILE}
231	@echo "COMPILER_VERSION='${COMPILER_VERSION}'" >> ${ENV_FILE}
232	@echo "HOST_CC='${HOST_CC}'" >> ${ENV_FILE}
233	@echo "HOST_MACHINE='${HOST_MACHINE}'" >> ${ENV_FILE}
234	@echo "HOST_OS='${HOST_OS}'" >> ${ENV_FILE}
235	@echo "HOST_OSMAJOR='${HOST_OSMAJOR}'" >> ${ENV_FILE}
236	@echo "HOST_OSTYPE='${HOST_OSTYPE}'" >> ${ENV_FILE}
237	@echo "LD='${LD}'" >> ${ENV_FILE}
238	@echo "LDFLAGS='${LDFLAGS}'" >> ${ENV_FILE}
239	@echo "MACHINE='${MACHINE}'" >> ${ENV_FILE}
240	@echo "MACHINE_ABI='${MACHINE_ABI}'" >> ${ENV_FILE}
241	@echo "MACHINE_ARCH='${MACHINE_ARCH}'" >> ${ENV_FILE}
242	@echo "MACHINE_CPU='${MACHINE_CPU}'" >> ${ENV_FILE}
243	@echo "MACHINE_CPUARCH='${MACHINE_CPUARCH}'" >> ${ENV_FILE}
244
245ci-create-meta: ci-get-env .PHONY
246	truncate -s 512M ${META_TAR}
247	tar rvf ${META_TAR} -C ${META_DIR} .
248
249ci-extract-meta: .PHONY
250	tar xfv ${META_TAR} -C ${META_DIROUT}
251	rm -rf ${META_TAR} ${META_DIR}
252	@echo "Extracted kyua reports to ${META_DIROUT}"
253
254ci-runtest: ci-buildimage-${TARGET_ARCH:tl} portinstall .PHONY
255.if ${MACHINE} == "amd64" && ( ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" ) && ( !defined(USE_QEMU) || empty(USE_QEMU) )
256	/usr/sbin/bhyvectl --vm=${TEST_VM_NAME} --destroy || true
257	expect -c "set timeout ${TIMEOUT_EXPECT}; \
258		spawn /usr/bin/timeout -k 5s 30s /usr/sbin/bhyveload \
259		-c stdio -m ${VM_MEM_SIZE} -d ${CIDISK} ${TEST_VM_NAME}; \
260		expect { eof }; \
261		exit [lindex [wait] 3]"
262	expect -c "set timeout ${TIMEOUT_EXPECT}; \
263		spawn /usr/bin/timeout -k 60 ${TIMEOUT_VM} /usr/sbin/bhyve \
264		-c ${PARALLEL_JOBS} -m ${VM_MEM_SIZE} -A -H -P \
265		-s 0:0,hostbridge \
266		-s 1:0,lpc \
267		-s 2:0,virtio-blk,${CIDISK} \
268		-s 3:0,virtio-blk,${META_TAR} \
269		${BHYVE_EXTRA_DISK_PARAM} \
270		-l com1,stdio \
271		${TEST_VM_NAME}; \
272		expect { eof }"
273	/usr/sbin/bhyvectl --vm=${TEST_VM_NAME} --destroy
274.else
275	timeout -k 60 ${TIMEOUT_VM} ${QEMUBIN} \
276		-machine ${QEMU_MACHINE} \
277		-smp ${QEMU_CPU_COUNT} \
278		-m ${VM_MEM_SIZE} \
279		-nographic \
280		-no-reboot \
281		${QEMU_EXTRA_PARAM} \
282		-device virtio-blk,drive=hd0 \
283		-device virtio-blk,drive=hd1 \
284		-blockdev driver=raw,node-name=hd0,file.driver=file,file.filename=${CIDISK} \
285		-blockdev driver=raw,node-name=hd1,file.driver=file,file.filename=${META_TAR} \
286		${QEMU_DEVICES}
287.endif
288
289ci-checktarget: .PHONY
290.if ${TARGET_ARCH} != "aarch64" && \
291	${TARGET_ARCH} != "amd64" && \
292	${TARGET_ARCH} != "armv7" && \
293	${TARGET_ARCH} != "powerpc64" && \
294	${TARGET_ARCH} != "powerpc64le" && \
295	${TARGET_ARCH} != "riscv64"
296	@false
297.ERROR:
298	@echo "Error: ${TARGET_ARCH} is not supported on ${TYPE} ${REVISION} ${BRANCH}"
299.endif
300
301ci-smoke: ci-set-smoke-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY
302
303ci-full: ci-set-full-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .WAIT ci-extract-meta .PHONY
304
305ci: ci-${CITYPE:tl} .PHONY
306
307.include "${RELEASEDIR}/Makefile.inc1"
308