xref: /freebsd/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh (revision 61145dc2b94f12f6a47344fb9aac702321880e43)
1#!/usr/bin/env bash
2
3######################################################################
4# 4) configure and build openzfs modules.  This is run on the VMs.
5#
6# Usage:
7#
8#       qemu-4-build-vm.sh OS [--enable-debug][--dkms][--poweroff]
9#           [--release][--repo][--tarball]
10#
11# OS:           OS name like 'fedora41'
12# --enable-debug:  Build RPMs with '--enable-debug' (for testing)
13# --dkms:       Build DKMS RPMs as well
14# --poweroff:   Power-off the VM after building
15# --release     Build zfs-release*.rpm as well
16# --repo        After building everything, copy RPMs into /tmp/repo
17#               in the ZFS RPM repository file structure.  Also
18#               copy tarballs if they were built.
19# --tarball:    Also build a tarball of ZFS source
20######################################################################
21
22ENABLE_DEBUG=""
23DKMS=""
24POWEROFF=""
25RELEASE=""
26REPO=""
27TARBALL=""
28while [[ $# -gt 0 ]]; do
29  case $1 in
30    --enable-debug)
31      ENABLE_DEBUG=1
32      shift
33      ;;
34    --dkms)
35      DKMS=1
36      shift
37      ;;
38    --poweroff)
39      POWEROFF=1
40      shift
41      ;;
42    --release)
43      RELEASE=1
44      shift
45      ;;
46    --repo)
47      REPO=1
48      shift
49      ;;
50    --tarball)
51      TARBALL=1
52      shift
53      ;;
54    *)
55      OS=$1
56      shift
57      ;;
58  esac
59done
60
61set -eu
62
63function run() {
64  LOG="/var/tmp/build-stderr.txt"
65  echo "****************************************************"
66  echo "$(date) ($*)"
67  echo "****************************************************"
68  ($@ || echo $? > /tmp/rv) 3>&1 1>&2 2>&3 | stdbuf -eL -oL tee -a $LOG
69  if [ -f /tmp/rv ]; then
70    RV=$(cat /tmp/rv)
71    echo "****************************************************"
72    echo "exit with value=$RV ($*)"
73    echo "****************************************************"
74    echo 1 > /var/tmp/build-exitcode.txt
75    exit $RV
76  fi
77}
78
79# Look at the RPMs in the current directory and copy/move them to
80# /tmp/repo, using the directory structure we use for the ZFS RPM repos.
81#
82# For example:
83# /tmp/repo/epel-testing/9.5
84# /tmp/repo/epel-testing/9.5/SRPMS
85# /tmp/repo/epel-testing/9.5/SRPMS/zfs-2.3.99-1.el9.src.rpm
86# /tmp/repo/epel-testing/9.5/SRPMS/zfs-kmod-2.3.99-1.el9.src.rpm
87# /tmp/repo/epel-testing/9.5/kmod
88# /tmp/repo/epel-testing/9.5/kmod/x86_64
89# /tmp/repo/epel-testing/9.5/kmod/x86_64/debug
90# /tmp/repo/epel-testing/9.5/kmod/x86_64/debug/kmod-zfs-debuginfo-2.3.99-1.el9.x86_64.rpm
91# /tmp/repo/epel-testing/9.5/kmod/x86_64/debug/libnvpair3-debuginfo-2.3.99-1.el9.x86_64.rpm
92# /tmp/repo/epel-testing/9.5/kmod/x86_64/debug/libuutil3-debuginfo-2.3.99-1.el9.x86_64.rpm
93# ...
94function copy_rpms_to_repo {
95  # Pick a RPM to query. It doesn't matter which one - we just want to extract
96  # the 'Build Host' value from it.
97  rpm=$(ls zfs-*.rpm | head -n 1)
98
99  # Get zfs version '2.2.99'
100  zfs_ver=$(rpm -qpi $rpm | awk '/Version/{print $3}')
101
102  # Get "2.1" or "2.2"
103  zfs_major=$(echo $zfs_ver | grep -Eo [0-9]+\.[0-9]+)
104
105  # Get 'almalinux9.5' or 'fedora41' type string
106  build_host=$(rpm -qpi $rpm | awk '/Build Host/{print $4}')
107
108  # Get '9.5' or '41' OS version
109  os_ver=$(echo $build_host | grep -Eo '[0-9\.]+$')
110
111  # Our ZFS version and OS name will determine which repo the RPMs
112  # will go in (regular or testing).  Fedora always gets the newest
113  # releases, and Alma gets the older releases.
114  case $build_host in
115  almalinux*)
116    case $zfs_major in
117    2.2)
118      d="epel"
119      ;;
120    *)
121      d="epel-testing"
122      ;;
123    esac
124    ;;
125  fedora*)
126    d="fedora"
127    ;;
128  esac
129
130  prefix=/tmp/repo
131  dst="$prefix/$d/$os_ver"
132
133  # Special case: move zfs-release*.rpm out of the way first (if we built them).
134  # This will make filtering the other RPMs easier.
135  mkdir -p $dst
136  mv zfs-release*.rpm $dst || true
137
138  # Copy source RPMs
139  mkdir -p $dst/SRPMS
140  cp $(ls *.src.rpm) $dst/SRPMS/
141
142  if [[ "$build_host" =~ "almalinux" ]] ; then
143    # Copy kmods+userspace
144    mkdir -p $dst/kmod/x86_64/debug
145    cp $(ls *.rpm | grep -Ev 'src.rpm|dkms|debuginfo') $dst/kmod/x86_64
146    cp *debuginfo*.rpm $dst/kmod/x86_64/debug
147  fi
148
149  if [ -n "$DKMS" ] ; then
150    # Copy dkms+userspace
151    mkdir -p $dst/x86_64
152    cp $(ls *.rpm | grep -Ev 'src.rpm|kmod|debuginfo') $dst/x86_64
153  fi
154
155  # Copy debug
156  mkdir -p $dst/x86_64/debug
157  cp $(ls *debuginfo*.rpm | grep -v kmod) $dst/x86_64/debug
158}
159
160function freebsd() {
161  extra="${1:-}"
162
163  export MAKE="gmake"
164  echo "##[group]Autogen.sh"
165  run ./autogen.sh
166  echo "##[endgroup]"
167
168  echo "##[group]Configure"
169  run ./configure \
170    --prefix=/usr/local \
171    --with-libintl-prefix=/usr/local \
172    --enable-pyzfs \
173    --enable-debuginfo $extra
174  echo "##[endgroup]"
175
176  echo "##[group]Build"
177  run gmake -j$(sysctl -n hw.ncpu)
178  echo "##[endgroup]"
179
180  echo "##[group]Install"
181  run sudo gmake install
182  echo "##[endgroup]"
183}
184
185function linux() {
186  extra="${1:-}"
187
188  echo "##[group]Autogen.sh"
189  run ./autogen.sh
190  echo "##[endgroup]"
191
192  echo "##[group]Configure"
193  run ./configure \
194    --prefix=/usr \
195    --enable-pyzfs \
196    --enable-debuginfo $extra
197  echo "##[endgroup]"
198
199  echo "##[group]Build"
200  run make -j$(nproc)
201  echo "##[endgroup]"
202
203  echo "##[group]Install"
204  run sudo make install
205  echo "##[endgroup]"
206}
207
208function rpm_build_and_install() {
209  extra="${1:-}"
210
211  # Build RPMs with XZ compression by default (since gzip decompression is slow)
212  echo "%_binary_payload w7.xzdio" >> ~/.rpmmacros
213
214  echo "##[group]Autogen.sh"
215  run ./autogen.sh
216  echo "##[endgroup]"
217
218  echo "##[group]Configure"
219  run ./configure --enable-debuginfo $extra
220  echo "##[endgroup]"
221
222  echo "##[group]Build"
223  run make pkg-kmod pkg-utils
224  echo "##[endgroup]"
225
226  if [ -n "$DKMS" ] ; then
227    echo "##[group]DKMS"
228    make rpm-dkms
229    echo "##[endgroup]"
230  fi
231
232  if [ -n "$REPO" ] ; then
233    echo "Skipping install since we're only building RPMs and nothing else"
234  else
235    echo "##[group]Install"
236    run sudo dnf -y --nobest install $(ls *.rpm | grep -Ev 'dkms|src.rpm')
237    echo "##[endgroup]"
238  fi
239
240  # Optionally build the zfs-release.*.rpm
241  if [ -n "$RELEASE" ] ; then
242    echo "##[group]Release"
243    pushd ~
244    sudo dnf -y install rpm-build || true
245    # Check out a sparse copy of zfsonlinux.github.com.git so we don't get
246    # all the binaries.  We just need a few kilobytes of files to build RPMs.
247    git clone --depth 1 --no-checkout \
248      https://github.com/zfsonlinux/zfsonlinux.github.com.git
249
250    cd zfsonlinux.github.com
251    git sparse-checkout set zfs-release
252    git checkout
253    cd zfs-release
254
255    mkdir -p ~/rpmbuild/{BUILDROOT,SPECS,RPMS,SRPMS,SOURCES,BUILD}
256    cp RPM-GPG-KEY-openzfs* *.repo ~/rpmbuild/SOURCES
257    cp zfs-release.spec ~/rpmbuild/SPECS/
258    rpmbuild -ba ~/rpmbuild/SPECS/zfs-release.spec
259
260    # ZFS release RPMs are built.  Copy them to the ~/zfs directory just to
261    # keep all the RPMs in the same place.
262    cp ~/rpmbuild/RPMS/noarch/*.rpm ~/zfs
263    cp ~/rpmbuild/SRPMS/*.rpm ~/zfs
264
265    popd
266    rm -fr ~/rpmbuild
267    echo "##[endgroup]"
268  fi
269
270  if [ -n "$REPO" ] ; then
271    echo "##[group]Repo"
272    copy_rpms_to_repo
273    echo "##[endgroup]"
274  fi
275}
276
277function deb_build_and_install() {
278  extra="${1:-}"
279
280  echo "##[group]Autogen.sh"
281  run ./autogen.sh
282  echo "##[endgroup]"
283
284  echo "##[group]Configure"
285  run ./configure \
286    --prefix=/usr \
287    --enable-pyzfs \
288    --enable-debuginfo $extra
289  echo "##[endgroup]"
290
291  echo "##[group]Build"
292  run make native-deb-kmod native-deb-utils
293  echo "##[endgroup]"
294
295  echo "##[group]Install"
296  # Do kmod install.  Note that when you build the native debs, the
297  # packages themselves are placed in parent directory '../' rather than
298  # in the source directory like the rpms are.
299  run sudo apt-get -y install $(find ../ | grep -E '\.deb$' \
300    | grep -Ev 'dkms|dracut')
301  echo "##[endgroup]"
302}
303
304function build_tarball {
305  if [ -n "$REPO" ] ; then
306    ./autogen.sh
307    ./configure --with-config=srpm
308    make dist
309    mkdir -p /tmp/repo/releases
310    # The tarball name is based off of 'Version' field in the META file.
311    mv *.tar.gz /tmp/repo/releases/
312  fi
313}
314
315# Debug: show kernel cmdline
316if [ -f /proc/cmdline ] ; then
317  cat /proc/cmdline || true
318fi
319
320# Set our hostname to our OS name and version number.  Specifically, we set the
321# major and minor number so that when we query the Build Host field in the RPMs
322# we build, we can see what specific version of Fedora/Almalinux we were using
323# to build them.  This is helpful for matching up KMOD versions.
324#
325# Examples:
326#
327# rhel8.10
328# almalinux9.5
329# fedora40
330source /etc/os-release
331sudo hostname "$ID$VERSION_ID"
332
333# save some sysinfo
334uname -a > /var/tmp/uname.txt
335
336cd $HOME/zfs
337export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"
338
339extra=""
340if [ -n "$ENABLE_DEBUG" ] ; then
341  extra="--enable-debug"
342fi
343
344# build
345case "$OS" in
346  freebsd*)
347    freebsd "$extra"
348    ;;
349  alma*|centos*)
350    rpm_build_and_install "--with-spec=redhat $extra"
351    ;;
352  fedora*)
353    rpm_build_and_install "$extra"
354
355    # Historically, we've always built the release tarballs on Fedora, since
356    # there was one instance long ago where we built them on CentOS 7, and they
357    # didn't work correctly for everyone.
358    if [ -n "$TARBALL" ] ; then
359        build_tarball
360    fi
361    ;;
362  debian*|ubuntu*)
363    deb_build_and_install "$extra"
364    ;;
365  *)
366    linux "$extra"
367    ;;
368esac
369
370
371# building the zfs module was ok
372echo 0 > /var/tmp/build-exitcode.txt
373
374# reset cloud-init configuration and poweroff
375if [ -n "$POWEROFF" ] ; then
376        sudo cloud-init clean --logs
377        sync && sleep 2 && sudo poweroff &
378fi
379exit 0
380