xref: /freebsd/contrib/pam-krb5/m4/krb5.m4 (revision df21a004be237a1dccd03c7b47254625eea62fa9)
1dnl Find the compiler and linker flags for Kerberos.
2dnl
3dnl Finds the compiler and linker flags for linking with Kerberos libraries.
4dnl Provides the --with-krb5, --with-krb5-include, and --with-krb5-lib
5dnl configure options to specify non-standard paths to the Kerberos libraries.
6dnl Uses krb5-config where available unless reduced dependencies is requested
7dnl or --with-krb5-include or --with-krb5-lib are given.
8dnl
9dnl Provides the macro RRA_LIB_KRB5 and sets the substitution variables
10dnl KRB5_CPPFLAGS, KRB5_LDFLAGS, and KRB5_LIBS.  Also provides
11dnl RRA_LIB_KRB5_SWITCH to set CPPFLAGS, LDFLAGS, and LIBS to include the
12dnl Kerberos libraries, saving the current values first, and
13dnl RRA_LIB_KRB5_RESTORE to restore those settings to before the last
14dnl RRA_LIB_KRB5_SWITCH.  HAVE_KRB5 will always be defined if RRA_LIB_KRB5 is
15dnl used.
16dnl
17dnl If KRB5_CPPFLAGS, KRB5_LDFLAGS, or KRB5_LIBS are set before calling these
18dnl macros, their values will be added to whatever the macros discover.
19dnl
20dnl KRB5_CPPFLAGS_WARNINGS will be set to the same value as KRB5_CPPFLAGS but
21dnl with any occurrences of -I changed to -isystem.  This may be useful to
22dnl suppress warnings from the Kerberos header files when building with and
23dnl aggressive warning flags.  Be aware that this change will change the
24dnl compiler header file search order as well.
25dnl
26dnl Provides the RRA_LIB_KRB5_OPTIONAL macro, which should be used if Kerberos
27dnl support is optional.  In this case, Kerberos libraries are mandatory if
28dnl --with-krb5 is given, and will not be probed for if --without-krb5 is
29dnl given.  Otherwise, they'll be probed for but will not be required.
30dnl Defines HAVE_KRB5 and sets rra_use_KRB5 to true if the libraries are
31dnl found.  The substitution variables will always be set, but they will be
32dnl empty unless Kerberos libraries are found and the user did not disable
33dnl Kerberos support.
34dnl
35dnl Sets the Automake conditional KRB5_USES_COM_ERR saying whether we use
36dnl com_err, since if we're also linking with AFS libraries, we may have to
37dnl change library ordering in that case.
38dnl
39dnl Depends on RRA_KRB5_CONFIG, RRA_ENABLE_REDUCED_DEPENDS, and
40dnl RRA_SET_LDFLAGS.
41dnl
42dnl Also provides RRA_FUNC_KRB5_GET_INIT_CREDS_OPT_FREE_ARGS, which checks
43dnl whether krb5_get_init_creds_opt_free takes one argument or two.  Defines
44dnl HAVE_KRB5_GET_INIT_CREDS_OPT_FREE_2_ARGS if it takes two arguments.
45dnl
46dnl Also provides RRA_INCLUDES_KRB5, which are the headers to include when
47dnl probing the Kerberos library properties.
48dnl
49dnl The canonical version of this file is maintained in the rra-c-util
50dnl package, available at <https://www.eyrie.org/~eagle/software/rra-c-util/>.
51dnl
52dnl Written by Russ Allbery <eagle@eyrie.org>
53dnl Copyright 2018, 2020-2021 Russ Allbery <eagle@eyrie.org>
54dnl Copyright 2005-2011, 2013-2014
55dnl     The Board of Trustees of the Leland Stanford Junior University
56dnl
57dnl This file is free software; the authors give unlimited permission to copy
58dnl and/or distribute it, with or without modifications, as long as this
59dnl notice is preserved.
60dnl
61dnl SPDX-License-Identifier: FSFULLR
62
63dnl Headers to include when probing for Kerberos library properties.
64AC_DEFUN([RRA_INCLUDES_KRB5], [[
65#if HAVE_KRB5_H
66# include <krb5.h>
67#elif HAVE_KERBEROSV5_KRB5_H
68# include <kerberosv5/krb5.h>
69#else
70# include <krb5/krb5.h>
71#endif
72]])
73
74dnl Save the current CPPFLAGS, LDFLAGS, and LIBS settings and switch to
75dnl versions that include the Kerberos flags.  Used as a wrapper, with
76dnl RRA_LIB_KRB5_RESTORE, around tests.
77AC_DEFUN([RRA_LIB_KRB5_SWITCH],
78[rra_krb5_save_CPPFLAGS="$CPPFLAGS"
79 rra_krb5_save_LDFLAGS="$LDFLAGS"
80 rra_krb5_save_LIBS="$LIBS"
81 CPPFLAGS="$KRB5_CPPFLAGS $CPPFLAGS"
82 LDFLAGS="$KRB5_LDFLAGS $LDFLAGS"
83 LIBS="$KRB5_LIBS $LIBS"])
84
85dnl Restore CPPFLAGS, LDFLAGS, and LIBS to their previous values (before
86dnl RRA_LIB_KRB5_SWITCH was called).
87AC_DEFUN([RRA_LIB_KRB5_RESTORE],
88[CPPFLAGS="$rra_krb5_save_CPPFLAGS"
89 LDFLAGS="$rra_krb5_save_LDFLAGS"
90 LIBS="$rra_krb5_save_LIBS"])
91
92dnl Set KRB5_CPPFLAGS and KRB5_LDFLAGS based on rra_krb5_root,
93dnl rra_krb5_libdir, and rra_krb5_includedir.
94AC_DEFUN([_RRA_LIB_KRB5_PATHS],
95[AS_IF([test x"$rra_krb5_libdir" != x],
96    [KRB5_LDFLAGS="-L$rra_krb5_libdir"],
97    [AS_IF([test x"$rra_krb5_root" != x],
98        [RRA_SET_LDFLAGS([KRB5_LDFLAGS], [$rra_krb5_root])])])
99 AS_IF([test x"$rra_krb5_includedir" != x],
100    [KRB5_CPPFLAGS="-I$rra_krb5_includedir"],
101    [AS_IF([test x"$rra_krb5_root" != x],
102        [AS_IF([test x"$rra_krb5_root" != x/usr],
103            [KRB5_CPPFLAGS="-I${rra_krb5_root}/include"])])])])
104
105dnl Check for a header using a file existence check rather than using
106dnl AC_CHECK_HEADERS.  This is used if there were arguments to configure
107dnl specifying the Kerberos header path, since we may have one header in the
108dnl default include path and another under our explicitly-configured Kerberos
109dnl location.  The second argument is run if the header was found.
110AC_DEFUN([_RRA_LIB_KRB5_CHECK_HEADER],
111[AC_MSG_CHECKING([for $1])
112 AS_IF([test -f "${rra_krb5_incroot}/$1"],
113    [AC_DEFINE_UNQUOTED(AS_TR_CPP([HAVE_$1]), [1],
114        [Define to 1 if you have the <$1> header file.])
115     AC_MSG_RESULT([yes])
116     $2],
117    [AC_MSG_RESULT([no])])])
118
119dnl Check for the com_err header.  Internal helper macro since we need
120dnl to do the same checks in multiple places.
121AC_DEFUN([_RRA_LIB_KRB5_CHECK_HEADER_COM_ERR],
122[AS_IF([test x"$rra_krb5_incroot" = x],
123    [AC_CHECK_HEADERS([et/com_err.h kerberosv5/com_err.h])],
124        [_RRA_LIB_KRB5_CHECK_HEADER([et/com_err.h])
125         _RRA_LIB_KRB5_CHECK_HEADER([kerberosv5/com_err.h])])])
126
127dnl Check for the main Kerberos header.  Internal helper macro since we need
128dnl to do the same checks in multiple places.  The first argument is run if
129dnl some header was found, and the second if no header was found.
130dnl header could not be found.
131AC_DEFUN([_RRA_LIB_KRB5_CHECK_HEADER_KRB5],
132[rra_krb5_found_header=
133 AS_IF([test x"$rra_krb5_incroot" = x],
134     [AC_CHECK_HEADERS([krb5.h kerberosv5/krb5.h krb5/krb5.h],
135         [rra_krb5_found_header=true])],
136     [_RRA_LIB_KRB5_CHECK_HEADER([krb5.h],
137         [rra_krb5_found_header=true])
138      _RRA_LIB_KRB5_CHECK_HEADER([kerberosv5/krb5.h],
139         [rra_krb5_found_header=true])
140      _RRA_LIB_KRB5_CHECK_HEADER([krb5/krb5.h],
141         [rra_krb5_found_header=true])])
142 AS_IF([test x"$rra_krb5_found_header" = xtrue], [$1], [$2])])
143
144dnl Does the appropriate library checks for reduced-dependency Kerberos
145dnl linkage.  The single argument, if true, says to fail if Kerberos could not
146dnl be found.
147AC_DEFUN([_RRA_LIB_KRB5_REDUCED],
148[RRA_LIB_KRB5_SWITCH
149 AC_CHECK_LIB([krb5], [krb5_init_context],
150    [KRB5_LIBS="-lkrb5"
151     LIBS="$KRB5_LIBS $LIBS"
152     AC_CHECK_FUNCS([krb5_get_error_message],
153        [AC_CHECK_FUNCS([krb5_free_error_message])],
154        [AC_CHECK_FUNCS([krb5_get_error_string], [],
155            [AC_CHECK_FUNCS([krb5_get_err_txt], [],
156                [AC_CHECK_LIB([ksvc], [krb5_svc_get_msg],
157                    [KRB5_LIBS="$KRB5_LIBS -lksvc"
158                     AC_DEFINE([HAVE_KRB5_SVC_GET_MSG], [1])
159                     AC_CHECK_HEADERS([ibm_svc/krb5_svc.h], [], [],
160                        [RRA_INCLUDES_KRB5])],
161                    [AC_CHECK_LIB([com_err], [com_err],
162                        [KRB5_LIBS="$KRB5_LIBS -lcom_err"],
163                        [AS_IF([test x"$1" = xtrue],
164                            [AC_MSG_ERROR([cannot find usable com_err library])],
165                            [KRB5_LIBS=""])])
166                     _RRA_LIB_KRB5_CHECK_HEADER_COM_ERR])])])])
167     _RRA_LIB_KRB5_CHECK_HEADER_KRB5([],
168        [KRB5_CPPFLAGS=
169         KRB5_LIBS=
170         AS_IF([test x"$1" = xtrue],
171            [AC_MSG_ERROR([cannot find usable Kerberos header])])])],
172    [AS_IF([test x"$1" = xtrue],
173        [AC_MSG_ERROR([cannot find usable Kerberos library])])])
174 RRA_LIB_KRB5_RESTORE])
175
176dnl Does the appropriate library checks for Kerberos linkage when we don't
177dnl have krb5-config or reduced dependencies.  The single argument, if true,
178dnl says to fail if Kerberos could not be found.
179AC_DEFUN([_RRA_LIB_KRB5_MANUAL],
180[RRA_LIB_KRB5_SWITCH
181 rra_krb5_extra=
182 LIBS=
183 AC_SEARCH_LIBS([res_search], [resolv], [],
184    [AC_SEARCH_LIBS([__res_search], [resolv])])
185 AC_SEARCH_LIBS([gethostbyname], [nsl])
186 AC_SEARCH_LIBS([socket], [socket], [],
187    [AC_CHECK_LIB([nsl], [socket], [LIBS="-lnsl -lsocket $LIBS"], [],
188        [-lsocket])])
189 AC_SEARCH_LIBS([crypt], [crypt])
190 AC_SEARCH_LIBS([roken_concat], [roken])
191 rra_krb5_extra="$LIBS"
192 LIBS="$rra_krb5_save_LIBS"
193 AC_CHECK_LIB([krb5], [krb5_init_context],
194    [KRB5_LIBS="-lkrb5 -lasn1 -lcom_err -lcrypto $rra_krb5_extra"],
195    [AC_CHECK_LIB([krb5support], [krb5int_getspecific],
196        [rra_krb5_extra="-lkrb5support $rra_krb5_extra"],
197        [AC_CHECK_LIB([pthreads], [pthread_setspecific],
198            [rra_krb5_pthread="-lpthreads"],
199            [AC_CHECK_LIB([pthread], [pthread_setspecific],
200                [rra_krb5_pthread="-lpthread"])])
201         AC_CHECK_LIB([krb5support], [krb5int_setspecific],
202            [rra_krb5_extra="-lkrb5support $rra_krb5_extra $rra_krb5_pthread"],
203            [], [$rra_krb5_pthread $rra_krb5_extra])],
204        [$rra_krb5_extra])
205     AC_CHECK_LIB([com_err], [error_message],
206        [rra_krb5_extra="-lcom_err $rra_krb5_extra"], [], [$rra_krb5_extra])
207     AC_CHECK_LIB([ksvc], [krb5_svc_get_msg],
208        [rra_krb5_extra="-lksvc $rra_krb5_extra"], [], [$rra_krb5_extra])
209     AC_CHECK_LIB([k5crypto], [krb5int_hash_md5],
210        [rra_krb5_extra="-lk5crypto $rra_krb5_extra"], [], [$rra_krb5_extra])
211     AC_CHECK_LIB([k5profile], [profile_get_values],
212        [rra_krb5_extra="-lk5profile $rra_krb5_extra"], [], [$rra_krb5_extra])
213     AC_CHECK_LIB([krb5], [krb5_cc_default],
214        [KRB5_LIBS="-lkrb5 $rra_krb5_extra"],
215        [AS_IF([test x"$1" = xtrue],
216            [AC_MSG_ERROR([cannot find usable Kerberos library])])],
217        [$rra_krb5_extra])],
218    [-lasn1 -lcom_err -lcrypto $rra_krb5_extra])
219 LIBS="$KRB5_LIBS $LIBS"
220 AC_CHECK_FUNCS([krb5_get_error_message],
221     [AC_CHECK_FUNCS([krb5_free_error_message])],
222     [AC_CHECK_FUNCS([krb5_get_error_string], [],
223         [AC_CHECK_FUNCS([krb5_get_err_txt], [],
224             [AC_CHECK_FUNCS([krb5_svc_get_msg],
225                 [AC_CHECK_HEADERS([ibm_svc/krb5_svc.h], [], [],
226                     [RRA_INCLUDES_KRB5])],
227                 [_RRA_LIB_KRB5_CHECK_HEADER_COM_ERR])])])])
228 _RRA_LIB_KRB5_CHECK_HEADER_KRB5([],
229    [KRB5_CPPFLAGS=
230     KRB5_LIBS=
231     AS_IF([test x"$1" = xtrue],
232        [AC_MSG_ERROR([cannot find usable Kerberos header])])])
233 RRA_LIB_KRB5_RESTORE])
234
235dnl Sanity-check the results of krb5-config and be sure we can really link a
236dnl Kerberos program.  If that fails, clear KRB5_CPPFLAGS and KRB5_LIBS so
237dnl that we know we don't have usable flags and fall back on the manual
238dnl check.
239AC_DEFUN([_RRA_LIB_KRB5_CHECK],
240[RRA_LIB_KRB5_SWITCH
241 AC_CHECK_FUNC([krb5_init_context],
242    [_RRA_LIB_KRB5_CHECK_HEADER_KRB5([RRA_LIB_KRB5_RESTORE],
243        [RRA_LIB_KRB5_RESTORE
244         KRB5_CPPFLAGS=
245         KRB5_LIBS=
246         _RRA_LIB_KRB5_PATHS
247         _RRA_LIB_KRB5_MANUAL([$1])])],
248    [RRA_LIB_KRB5_RESTORE
249     KRB5_CPPFLAGS=
250     KRB5_LIBS=
251     _RRA_LIB_KRB5_PATHS
252     _RRA_LIB_KRB5_MANUAL([$1])])])
253
254dnl Determine Kerberos compiler and linker flags from krb5-config.  Does the
255dnl additional probing we need to do to uncover error handling features, and
256dnl falls back on the manual checks.
257AC_DEFUN([_RRA_LIB_KRB5_CONFIG],
258[RRA_KRB5_CONFIG([${rra_krb5_root}], [krb5], [KRB5],
259    [_RRA_LIB_KRB5_CHECK([$1])
260     RRA_LIB_KRB5_SWITCH
261     AC_CHECK_FUNCS([krb5_get_error_message],
262         [AC_CHECK_FUNCS([krb5_free_error_message])],
263         [AC_CHECK_FUNCS([krb5_get_error_string], [],
264             [AC_CHECK_FUNCS([krb5_get_err_txt], [],
265                 [AC_CHECK_FUNCS([krb5_svc_get_msg],
266                     [AC_CHECK_HEADERS([ibm_svc/krb5_svc.h], [], [],
267                         [RRA_INCLUDES_KRB5])],
268                     [_RRA_LIB_KRB5_CHECK_HEADER_COM_ERR])])])])
269     RRA_LIB_KRB5_RESTORE],
270    [_RRA_LIB_KRB5_PATHS
271     _RRA_LIB_KRB5_MANUAL([$1])])])
272
273dnl The core of the library checking, shared between RRA_LIB_KRB5 and
274dnl RRA_LIB_KRB5_OPTIONAL.  The single argument, if "true", says to fail if
275dnl Kerberos could not be found.  Set up rra_krb5_incroot for later header
276dnl checking.
277AC_DEFUN([_RRA_LIB_KRB5_INTERNAL],
278[AC_REQUIRE([RRA_ENABLE_REDUCED_DEPENDS])
279 rra_krb5_incroot=
280 AC_SUBST([KRB5_CPPFLAGS])
281 AC_SUBST([KRB5_CPPFLAGS_WARNINGS])
282 AC_SUBST([KRB5_LDFLAGS])
283 AC_SUBST([KRB5_LIBS])
284 AS_IF([test x"$rra_krb5_includedir" != x],
285    [rra_krb5_incroot="$rra_krb5_includedir"],
286    [AS_IF([test x"$rra_krb5_root" != x],
287        [rra_krb5_incroot="${rra_krb5_root}/include"])])
288 AS_IF([test x"$rra_reduced_depends" = xtrue],
289    [_RRA_LIB_KRB5_PATHS
290     _RRA_LIB_KRB5_REDUCED([$1])],
291    [AS_IF([test x"$rra_krb5_includedir" = x && test x"$rra_krb5_libdir" = x],
292        [_RRA_LIB_KRB5_CONFIG([$1])],
293        [_RRA_LIB_KRB5_PATHS
294         _RRA_LIB_KRB5_MANUAL([$1])])])
295 rra_krb5_uses_com_err=false
296 AS_CASE([$KRB5_LIBS], [*-lcom_err*], [rra_krb5_uses_com_err=true])
297 AM_CONDITIONAL([KRB5_USES_COM_ERR],
298    [test x"$rra_krb5_uses_com_err" = xtrue])
299 KRB5_CPPFLAGS_WARNINGS=`AS_ECHO(["$KRB5_CPPFLAGS"]) | sed 's/-I/-isystem /g'`])
300
301dnl The main macro for packages with mandatory Kerberos support.
302AC_DEFUN([RRA_LIB_KRB5],
303[rra_krb5_root=
304 rra_krb5_libdir=
305 rra_krb5_includedir=
306 rra_use_KRB5=true
307
308 AC_ARG_WITH([krb5],
309    [AS_HELP_STRING([--with-krb5=DIR],
310        [Location of Kerberos headers and libraries])],
311    [AS_IF([test x"$withval" != xyes && test x"$withval" != xno],
312        [rra_krb5_root="$withval"])])
313 AC_ARG_WITH([krb5-include],
314    [AS_HELP_STRING([--with-krb5-include=DIR],
315        [Location of Kerberos headers])],
316    [AS_IF([test x"$withval" != xyes && test x"$withval" != xno],
317        [rra_krb5_includedir="$withval"])])
318 AC_ARG_WITH([krb5-lib],
319    [AS_HELP_STRING([--with-krb5-lib=DIR],
320        [Location of Kerberos libraries])],
321    [AS_IF([test x"$withval" != xyes && test x"$withval" != xno],
322        [rra_krb5_libdir="$withval"])])
323 _RRA_LIB_KRB5_INTERNAL([true])
324 AC_DEFINE([HAVE_KRB5], 1, [Define to enable Kerberos features.])])
325
326dnl The main macro for packages with optional Kerberos support.
327AC_DEFUN([RRA_LIB_KRB5_OPTIONAL],
328[rra_krb5_root=
329 rra_krb5_libdir=
330 rra_krb5_includedir=
331 rra_use_KRB5=
332
333 AC_ARG_WITH([krb5],
334    [AS_HELP_STRING([--with-krb5@<:@=DIR@:>@],
335        [Location of Kerberos headers and libraries])],
336    [AS_IF([test x"$withval" = xno],
337        [rra_use_KRB5=false],
338        [AS_IF([test x"$withval" != xyes], [rra_krb5_root="$withval"])
339         rra_use_KRB5=true])])
340 AC_ARG_WITH([krb5-include],
341    [AS_HELP_STRING([--with-krb5-include=DIR],
342        [Location of Kerberos headers])],
343    [AS_IF([test x"$withval" != xyes && test x"$withval" != xno],
344        [rra_krb5_includedir="$withval"])])
345 AC_ARG_WITH([krb5-lib],
346    [AS_HELP_STRING([--with-krb5-lib=DIR],
347        [Location of Kerberos libraries])],
348    [AS_IF([test x"$withval" != xyes && test x"$withval" != xno],
349        [rra_krb5_libdir="$withval"])])
350
351 AS_IF([test x"$rra_use_KRB5" != xfalse],
352     [AS_IF([test x"$rra_use_KRB5" = xtrue],
353         [_RRA_LIB_KRB5_INTERNAL([true])],
354         [_RRA_LIB_KRB5_INTERNAL([false])])],
355     [AM_CONDITIONAL([KRB5_USES_COM_ERR], [false])])
356 AS_IF([test x"$KRB5_LIBS" != x],
357    [rra_use_KRB5=true
358     AC_DEFINE([HAVE_KRB5], 1, [Define to enable Kerberos features.])])])
359
360dnl Source used by RRA_FUNC_KRB5_GET_INIT_CREDS_OPT_FREE_ARGS.
361AC_DEFUN([_RRA_FUNC_KRB5_OPT_FREE_ARGS_SOURCE], [RRA_INCLUDES_KRB5] [[
362int
363main(void)
364{
365    krb5_get_init_creds_opt *opts;
366    krb5_context c;
367    krb5_get_init_creds_opt_free(c, opts);
368}
369]])
370
371dnl Check whether krb5_get_init_creds_opt_free takes one argument or two.
372dnl Early Heimdal used to take a single argument.  Defines
373dnl HAVE_KRB5_GET_INIT_CREDS_OPT_FREE_2_ARGS if it takes two arguments.
374dnl
375dnl Should be called with RRA_LIB_KRB5_SWITCH active.
376AC_DEFUN([RRA_FUNC_KRB5_GET_INIT_CREDS_OPT_FREE_ARGS],
377[AC_CACHE_CHECK([if krb5_get_init_creds_opt_free takes two arguments],
378    [rra_cv_func_krb5_get_init_creds_opt_free_args],
379    [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_RRA_FUNC_KRB5_OPT_FREE_ARGS_SOURCE])],
380        [rra_cv_func_krb5_get_init_creds_opt_free_args=yes],
381        [rra_cv_func_krb5_get_init_creds_opt_free_args=no])])
382 AS_IF([test $rra_cv_func_krb5_get_init_creds_opt_free_args = yes],
383    [AC_DEFINE([HAVE_KRB5_GET_INIT_CREDS_OPT_FREE_2_ARGS], 1,
384        [Define if krb5_get_init_creds_opt_free takes two arguments.])])])
385