xref: /titanic_41/usr/src/cmd/sendmail/src/tls.c (revision e9af4bc0b1cc30cea75d6ad4aa2fde97d985e9be)
17c478bd9Sstevel@tonic-gate /*
2*e9af4bc0SJohn Beck  * Copyright (c) 2000-2006, 2008, 2009 Sendmail, Inc. and its suppliers.
37c478bd9Sstevel@tonic-gate  *	All rights reserved.
47c478bd9Sstevel@tonic-gate  *
57c478bd9Sstevel@tonic-gate  * By using this file, you agree to the terms and conditions set
67c478bd9Sstevel@tonic-gate  * forth in the LICENSE file which can be found at the top level of
77c478bd9Sstevel@tonic-gate  * the sendmail distribution.
87c478bd9Sstevel@tonic-gate  *
97c478bd9Sstevel@tonic-gate  */
107c478bd9Sstevel@tonic-gate 
117c478bd9Sstevel@tonic-gate #include <sendmail.h>
127c478bd9Sstevel@tonic-gate 
13*e9af4bc0SJohn Beck SM_RCSID("@(#)$Id: tls.c,v 8.114 2009/08/10 15:11:09 ca Exp $")
147c478bd9Sstevel@tonic-gate 
157c478bd9Sstevel@tonic-gate #if STARTTLS
167c478bd9Sstevel@tonic-gate #  include <openssl/err.h>
177c478bd9Sstevel@tonic-gate #  include <openssl/bio.h>
187c478bd9Sstevel@tonic-gate #  include <openssl/pem.h>
197c478bd9Sstevel@tonic-gate #  ifndef HASURANDOMDEV
207c478bd9Sstevel@tonic-gate #   include <openssl/rand.h>
217c478bd9Sstevel@tonic-gate #  endif /* ! HASURANDOMDEV */
227c478bd9Sstevel@tonic-gate # if !TLS_NO_RSA
237c478bd9Sstevel@tonic-gate static RSA *rsa_tmp = NULL;	/* temporary RSA key */
247c478bd9Sstevel@tonic-gate static RSA *tmp_rsa_key __P((SSL *, int, int));
257c478bd9Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
267c478bd9Sstevel@tonic-gate #  if !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x00907000L
277c478bd9Sstevel@tonic-gate static int	tls_verify_cb __P((X509_STORE_CTX *));
287c478bd9Sstevel@tonic-gate #  else /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
297c478bd9Sstevel@tonic-gate static int	tls_verify_cb __P((X509_STORE_CTX *, void *));
307c478bd9Sstevel@tonic-gate #  endif /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
317c478bd9Sstevel@tonic-gate 
327c478bd9Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
337c478bd9Sstevel@tonic-gate static int x509_verify_cb __P((int, X509_STORE_CTX *));
347c478bd9Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
357c478bd9Sstevel@tonic-gate 
367c478bd9Sstevel@tonic-gate # if !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x00907000L
377c478bd9Sstevel@tonic-gate #  define CONST097
387c478bd9Sstevel@tonic-gate # else /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
397c478bd9Sstevel@tonic-gate #  define CONST097 const
407c478bd9Sstevel@tonic-gate # endif /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
417c478bd9Sstevel@tonic-gate static void	apps_ssl_info_cb __P((CONST097 SSL *, int , int));
427c478bd9Sstevel@tonic-gate static bool	tls_ok_f __P((char *, char *, int));
437c478bd9Sstevel@tonic-gate static bool	tls_safe_f __P((char *, long, bool));
447c478bd9Sstevel@tonic-gate static int	tls_verify_log __P((int, X509_STORE_CTX *, char *));
457c478bd9Sstevel@tonic-gate 
467c478bd9Sstevel@tonic-gate # if !NO_DH
477c478bd9Sstevel@tonic-gate static DH *get_dh512 __P((void));
487c478bd9Sstevel@tonic-gate 
497c478bd9Sstevel@tonic-gate static unsigned char dh512_p[] =
507c478bd9Sstevel@tonic-gate {
517c478bd9Sstevel@tonic-gate 	0xDA,0x58,0x3C,0x16,0xD9,0x85,0x22,0x89,0xD0,0xE4,0xAF,0x75,
527c478bd9Sstevel@tonic-gate 	0x6F,0x4C,0xCA,0x92,0xDD,0x4B,0xE5,0x33,0xB8,0x04,0xFB,0x0F,
537c478bd9Sstevel@tonic-gate 	0xED,0x94,0xEF,0x9C,0x8A,0x44,0x03,0xED,0x57,0x46,0x50,0xD3,
547c478bd9Sstevel@tonic-gate 	0x69,0x99,0xDB,0x29,0xD7,0x76,0x27,0x6B,0xA2,0xD3,0xD4,0x12,
557c478bd9Sstevel@tonic-gate 	0xE2,0x18,0xF4,0xDD,0x1E,0x08,0x4C,0xF6,0xD8,0x00,0x3E,0x7C,
567c478bd9Sstevel@tonic-gate 	0x47,0x74,0xE8,0x33
577c478bd9Sstevel@tonic-gate };
587c478bd9Sstevel@tonic-gate static unsigned char dh512_g[] =
597c478bd9Sstevel@tonic-gate {
607c478bd9Sstevel@tonic-gate 	0x02
617c478bd9Sstevel@tonic-gate };
627c478bd9Sstevel@tonic-gate 
637c478bd9Sstevel@tonic-gate static DH *
647c478bd9Sstevel@tonic-gate get_dh512()
657c478bd9Sstevel@tonic-gate {
667c478bd9Sstevel@tonic-gate 	DH *dh = NULL;
677c478bd9Sstevel@tonic-gate 
687c478bd9Sstevel@tonic-gate 	if ((dh = DH_new()) == NULL)
697c478bd9Sstevel@tonic-gate 		return NULL;
707c478bd9Sstevel@tonic-gate 	dh->p = BN_bin2bn(dh512_p, sizeof(dh512_p), NULL);
717c478bd9Sstevel@tonic-gate 	dh->g = BN_bin2bn(dh512_g, sizeof(dh512_g), NULL);
727c478bd9Sstevel@tonic-gate 	if ((dh->p == NULL) || (dh->g == NULL))
737c478bd9Sstevel@tonic-gate 		return NULL;
747c478bd9Sstevel@tonic-gate 	return dh;
757c478bd9Sstevel@tonic-gate }
767c478bd9Sstevel@tonic-gate # endif /* !NO_DH */
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate 
797c478bd9Sstevel@tonic-gate /*
807c478bd9Sstevel@tonic-gate **  TLS_RAND_INIT -- initialize STARTTLS random generator
817c478bd9Sstevel@tonic-gate **
827c478bd9Sstevel@tonic-gate **	Parameters:
837c478bd9Sstevel@tonic-gate **		randfile -- name of file with random data
847c478bd9Sstevel@tonic-gate **		logl -- loglevel
857c478bd9Sstevel@tonic-gate **
867c478bd9Sstevel@tonic-gate **	Returns:
877c478bd9Sstevel@tonic-gate **		success/failure
887c478bd9Sstevel@tonic-gate **
897c478bd9Sstevel@tonic-gate **	Side Effects:
907c478bd9Sstevel@tonic-gate **		initializes PRNG for tls library.
917c478bd9Sstevel@tonic-gate */
927c478bd9Sstevel@tonic-gate 
937c478bd9Sstevel@tonic-gate # define MIN_RAND_BYTES	128	/* 1024 bits */
947c478bd9Sstevel@tonic-gate 
957c478bd9Sstevel@tonic-gate # define RF_OK		0	/* randfile OK */
967c478bd9Sstevel@tonic-gate # define RF_MISS	1	/* randfile == NULL || *randfile == '\0' */
977c478bd9Sstevel@tonic-gate # define RF_UNKNOWN	2	/* unknown prefix for randfile */
987c478bd9Sstevel@tonic-gate 
997c478bd9Sstevel@tonic-gate # define RI_NONE	0	/* no init yet */
1007c478bd9Sstevel@tonic-gate # define RI_SUCCESS	1	/* init was successful */
1017c478bd9Sstevel@tonic-gate # define RI_FAIL	2	/* init failed */
1027c478bd9Sstevel@tonic-gate 
1037c478bd9Sstevel@tonic-gate static bool	tls_rand_init __P((char *, int));
1047c478bd9Sstevel@tonic-gate 
1057c478bd9Sstevel@tonic-gate static bool
1067c478bd9Sstevel@tonic-gate tls_rand_init(randfile, logl)
1077c478bd9Sstevel@tonic-gate 	char *randfile;
1087c478bd9Sstevel@tonic-gate 	int logl;
1097c478bd9Sstevel@tonic-gate {
1107c478bd9Sstevel@tonic-gate # ifndef HASURANDOMDEV
1117c478bd9Sstevel@tonic-gate 	/* not required if /dev/urandom exists, OpenSSL does it internally */
1127c478bd9Sstevel@tonic-gate 
1137c478bd9Sstevel@tonic-gate 	bool ok;
1147c478bd9Sstevel@tonic-gate 	int randdef;
1157c478bd9Sstevel@tonic-gate 	static int done = RI_NONE;
1167c478bd9Sstevel@tonic-gate 
1177c478bd9Sstevel@tonic-gate 	/*
1187c478bd9Sstevel@tonic-gate 	**  initialize PRNG
1197c478bd9Sstevel@tonic-gate 	*/
1207c478bd9Sstevel@tonic-gate 
1217c478bd9Sstevel@tonic-gate 	/* did we try this before? if yes: return old value */
1227c478bd9Sstevel@tonic-gate 	if (done != RI_NONE)
1237c478bd9Sstevel@tonic-gate 		return done == RI_SUCCESS;
1247c478bd9Sstevel@tonic-gate 
1257c478bd9Sstevel@tonic-gate 	/* set default values */
1267c478bd9Sstevel@tonic-gate 	ok = false;
1277c478bd9Sstevel@tonic-gate 	done = RI_FAIL;
1287c478bd9Sstevel@tonic-gate 	randdef = (randfile == NULL || *randfile == '\0') ? RF_MISS : RF_OK;
1297c478bd9Sstevel@tonic-gate #   if EGD
1307c478bd9Sstevel@tonic-gate 	if (randdef == RF_OK && sm_strncasecmp(randfile, "egd:", 4) == 0)
1317c478bd9Sstevel@tonic-gate 	{
1327c478bd9Sstevel@tonic-gate 		randfile += 4;
1337c478bd9Sstevel@tonic-gate 		if (RAND_egd(randfile) < 0)
1347c478bd9Sstevel@tonic-gate 		{
1357c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
1367c478bd9Sstevel@tonic-gate 				  "STARTTLS: RAND_egd(%s) failed: random number generator not seeded",
1377c478bd9Sstevel@tonic-gate 				   randfile);
1387c478bd9Sstevel@tonic-gate 		}
1397c478bd9Sstevel@tonic-gate 		else
1407c478bd9Sstevel@tonic-gate 			ok = true;
1417c478bd9Sstevel@tonic-gate 	}
1427c478bd9Sstevel@tonic-gate 	else
1437c478bd9Sstevel@tonic-gate #   endif /* EGD */
1447c478bd9Sstevel@tonic-gate 	if (randdef == RF_OK && sm_strncasecmp(randfile, "file:", 5) == 0)
1457c478bd9Sstevel@tonic-gate 	{
1467c478bd9Sstevel@tonic-gate 		int fd;
1477c478bd9Sstevel@tonic-gate 		long sff;
1487c478bd9Sstevel@tonic-gate 		struct stat st;
1497c478bd9Sstevel@tonic-gate 
1507c478bd9Sstevel@tonic-gate 		randfile += 5;
1517c478bd9Sstevel@tonic-gate 		sff = SFF_SAFEDIRPATH | SFF_NOWLINK
1527c478bd9Sstevel@tonic-gate 		      | SFF_NOGWFILES | SFF_NOWWFILES
1537c478bd9Sstevel@tonic-gate 		      | SFF_NOGRFILES | SFF_NOWRFILES
1547c478bd9Sstevel@tonic-gate 		      | SFF_MUSTOWN | SFF_ROOTOK | SFF_OPENASROOT;
1557c478bd9Sstevel@tonic-gate 		if (DontLockReadFiles)
1567c478bd9Sstevel@tonic-gate 			sff |= SFF_NOLOCK;
1577c478bd9Sstevel@tonic-gate 		if ((fd = safeopen(randfile, O_RDONLY, 0, sff)) >= 0)
1587c478bd9Sstevel@tonic-gate 		{
1597c478bd9Sstevel@tonic-gate 			if (fstat(fd, &st) < 0)
1607c478bd9Sstevel@tonic-gate 			{
1617c478bd9Sstevel@tonic-gate 				if (LogLevel > logl)
1627c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_ERR, NOQID,
1637c478bd9Sstevel@tonic-gate 						  "STARTTLS: can't fstat(%s)",
1647c478bd9Sstevel@tonic-gate 						  randfile);
1657c478bd9Sstevel@tonic-gate 			}
1667c478bd9Sstevel@tonic-gate 			else
1677c478bd9Sstevel@tonic-gate 			{
1687c478bd9Sstevel@tonic-gate 				bool use, problem;
1697c478bd9Sstevel@tonic-gate 
1707c478bd9Sstevel@tonic-gate 				use = true;
1717c478bd9Sstevel@tonic-gate 				problem = false;
1727c478bd9Sstevel@tonic-gate 
1737c478bd9Sstevel@tonic-gate 				/* max. age of file: 10 minutes */
1747c478bd9Sstevel@tonic-gate 				if (st.st_mtime + 600 < curtime())
1757c478bd9Sstevel@tonic-gate 				{
1767c478bd9Sstevel@tonic-gate 					use = bitnset(DBS_INSUFFICIENTENTROPY,
1777c478bd9Sstevel@tonic-gate 						      DontBlameSendmail);
1787c478bd9Sstevel@tonic-gate 					problem = true;
1797c478bd9Sstevel@tonic-gate 					if (LogLevel > logl)
1807c478bd9Sstevel@tonic-gate 						sm_syslog(LOG_ERR, NOQID,
1817c478bd9Sstevel@tonic-gate 							  "STARTTLS: RandFile %s too old: %s",
1827c478bd9Sstevel@tonic-gate 							  randfile,
1837c478bd9Sstevel@tonic-gate 							  use ? "unsafe" :
1847c478bd9Sstevel@tonic-gate 								"unusable");
1857c478bd9Sstevel@tonic-gate 				}
1867c478bd9Sstevel@tonic-gate 				if (use && st.st_size < MIN_RAND_BYTES)
1877c478bd9Sstevel@tonic-gate 				{
1887c478bd9Sstevel@tonic-gate 					use = bitnset(DBS_INSUFFICIENTENTROPY,
1897c478bd9Sstevel@tonic-gate 						      DontBlameSendmail);
1907c478bd9Sstevel@tonic-gate 					problem = true;
1917c478bd9Sstevel@tonic-gate 					if (LogLevel > logl)
1927c478bd9Sstevel@tonic-gate 						sm_syslog(LOG_ERR, NOQID,
1937c478bd9Sstevel@tonic-gate 							  "STARTTLS: size(%s) < %d: %s",
1947c478bd9Sstevel@tonic-gate 							  randfile,
1957c478bd9Sstevel@tonic-gate 							  MIN_RAND_BYTES,
1967c478bd9Sstevel@tonic-gate 							  use ? "unsafe" :
1977c478bd9Sstevel@tonic-gate 								"unusable");
1987c478bd9Sstevel@tonic-gate 				}
1997c478bd9Sstevel@tonic-gate 				if (use)
2007c478bd9Sstevel@tonic-gate 					ok = RAND_load_file(randfile, -1) >=
2017c478bd9Sstevel@tonic-gate 					     MIN_RAND_BYTES;
2027c478bd9Sstevel@tonic-gate 				if (use && !ok)
2037c478bd9Sstevel@tonic-gate 				{
2047c478bd9Sstevel@tonic-gate 					if (LogLevel > logl)
2057c478bd9Sstevel@tonic-gate 						sm_syslog(LOG_WARNING, NOQID,
2067c478bd9Sstevel@tonic-gate 							  "STARTTLS: RAND_load_file(%s) failed: random number generator not seeded",
2077c478bd9Sstevel@tonic-gate 							  randfile);
2087c478bd9Sstevel@tonic-gate 				}
2097c478bd9Sstevel@tonic-gate 				if (problem)
2107c478bd9Sstevel@tonic-gate 					ok = false;
2117c478bd9Sstevel@tonic-gate 			}
2127c478bd9Sstevel@tonic-gate 			if (ok || bitnset(DBS_INSUFFICIENTENTROPY,
2137c478bd9Sstevel@tonic-gate 					  DontBlameSendmail))
2147c478bd9Sstevel@tonic-gate 			{
2157c478bd9Sstevel@tonic-gate 				/* add this even if fstat() failed */
216058561cbSjbeck 				RAND_seed((void *) &st, sizeof(st));
2177c478bd9Sstevel@tonic-gate 			}
2187c478bd9Sstevel@tonic-gate 			(void) close(fd);
2197c478bd9Sstevel@tonic-gate 		}
2207c478bd9Sstevel@tonic-gate 		else
2217c478bd9Sstevel@tonic-gate 		{
2227c478bd9Sstevel@tonic-gate 			if (LogLevel > logl)
2237c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
2247c478bd9Sstevel@tonic-gate 					  "STARTTLS: Warning: safeopen(%s) failed",
2257c478bd9Sstevel@tonic-gate 					  randfile);
2267c478bd9Sstevel@tonic-gate 		}
2277c478bd9Sstevel@tonic-gate 	}
2287c478bd9Sstevel@tonic-gate 	else if (randdef == RF_OK)
2297c478bd9Sstevel@tonic-gate 	{
2307c478bd9Sstevel@tonic-gate 		if (LogLevel > logl)
2317c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
2327c478bd9Sstevel@tonic-gate 				  "STARTTLS: Error: no proper random file definition %s",
2337c478bd9Sstevel@tonic-gate 				  randfile);
2347c478bd9Sstevel@tonic-gate 		randdef = RF_UNKNOWN;
2357c478bd9Sstevel@tonic-gate 	}
2367c478bd9Sstevel@tonic-gate 	if (randdef == RF_MISS)
2377c478bd9Sstevel@tonic-gate 	{
2387c478bd9Sstevel@tonic-gate 		if (LogLevel > logl)
2397c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
2407c478bd9Sstevel@tonic-gate 				  "STARTTLS: Error: missing random file definition");
2417c478bd9Sstevel@tonic-gate 	}
2427c478bd9Sstevel@tonic-gate 	if (!ok && bitnset(DBS_INSUFFICIENTENTROPY, DontBlameSendmail))
2437c478bd9Sstevel@tonic-gate 	{
2447c478bd9Sstevel@tonic-gate 		int i;
2457c478bd9Sstevel@tonic-gate 		long r;
2467c478bd9Sstevel@tonic-gate 		unsigned char buf[MIN_RAND_BYTES];
2477c478bd9Sstevel@tonic-gate 
2487c478bd9Sstevel@tonic-gate 		/* assert((MIN_RAND_BYTES % sizeof(long)) == 0); */
2497c478bd9Sstevel@tonic-gate 		for (i = 0; i <= sizeof(buf) - sizeof(long); i += sizeof(long))
2507c478bd9Sstevel@tonic-gate 		{
2517c478bd9Sstevel@tonic-gate 			r = get_random();
2527c478bd9Sstevel@tonic-gate 			(void) memcpy(buf + i, (void *) &r, sizeof(long));
2537c478bd9Sstevel@tonic-gate 		}
254058561cbSjbeck 		RAND_seed(buf, sizeof(buf));
2557c478bd9Sstevel@tonic-gate 		if (LogLevel > logl)
2567c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
2577c478bd9Sstevel@tonic-gate 				  "STARTTLS: Warning: random number generator not properly seeded");
2587c478bd9Sstevel@tonic-gate 		ok = true;
2597c478bd9Sstevel@tonic-gate 	}
2607c478bd9Sstevel@tonic-gate 	done = ok ? RI_SUCCESS : RI_FAIL;
2617c478bd9Sstevel@tonic-gate 	return ok;
2627c478bd9Sstevel@tonic-gate # else /* ! HASURANDOMDEV */
2637c478bd9Sstevel@tonic-gate 	return true;
2647c478bd9Sstevel@tonic-gate # endif /* ! HASURANDOMDEV */
2657c478bd9Sstevel@tonic-gate }
2667c478bd9Sstevel@tonic-gate /*
2677c478bd9Sstevel@tonic-gate **  INIT_TLS_LIBRARY -- Calls functions which setup TLS library for global use.
2687c478bd9Sstevel@tonic-gate **
2697c478bd9Sstevel@tonic-gate **	Parameters:
2707c478bd9Sstevel@tonic-gate **		none.
2717c478bd9Sstevel@tonic-gate **
2727c478bd9Sstevel@tonic-gate **	Returns:
2737c478bd9Sstevel@tonic-gate **		succeeded?
2747c478bd9Sstevel@tonic-gate */
2757c478bd9Sstevel@tonic-gate 
2767c478bd9Sstevel@tonic-gate bool
2777c478bd9Sstevel@tonic-gate init_tls_library()
2787c478bd9Sstevel@tonic-gate {
2797c478bd9Sstevel@tonic-gate 	/* basic TLS initialization, ignore result for now */
2807c478bd9Sstevel@tonic-gate 	SSL_library_init();
2817c478bd9Sstevel@tonic-gate 	SSL_load_error_strings();
2827c478bd9Sstevel@tonic-gate # if 0
2837c478bd9Sstevel@tonic-gate 	/* this is currently a macro for SSL_library_init */
2847c478bd9Sstevel@tonic-gate 	SSLeay_add_ssl_algorithms();
2857c478bd9Sstevel@tonic-gate # endif /* 0 */
2867c478bd9Sstevel@tonic-gate 
2877c478bd9Sstevel@tonic-gate 	return tls_rand_init(RandFile, 7);
2887c478bd9Sstevel@tonic-gate }
2897c478bd9Sstevel@tonic-gate /*
2907c478bd9Sstevel@tonic-gate **  TLS_SET_VERIFY -- request client certificate?
2917c478bd9Sstevel@tonic-gate **
2927c478bd9Sstevel@tonic-gate **	Parameters:
2937c478bd9Sstevel@tonic-gate **		ctx -- TLS context
2947c478bd9Sstevel@tonic-gate **		ssl -- TLS structure
2957c478bd9Sstevel@tonic-gate **		vrfy -- require certificate?
2967c478bd9Sstevel@tonic-gate **
2977c478bd9Sstevel@tonic-gate **	Returns:
2987c478bd9Sstevel@tonic-gate **		none.
2997c478bd9Sstevel@tonic-gate **
3007c478bd9Sstevel@tonic-gate **	Side Effects:
3017c478bd9Sstevel@tonic-gate **		Sets verification state for TLS
3027c478bd9Sstevel@tonic-gate **
3037c478bd9Sstevel@tonic-gate # if TLS_VRFY_PER_CTX
3047c478bd9Sstevel@tonic-gate **	Notice:
3057c478bd9Sstevel@tonic-gate **		This is per TLS context, not per TLS structure;
3067c478bd9Sstevel@tonic-gate **		the former is global, the latter per connection.
3077c478bd9Sstevel@tonic-gate **		It would be nice to do this per connection, but this
3087c478bd9Sstevel@tonic-gate **		doesn't work in the current TLS libraries :-(
3097c478bd9Sstevel@tonic-gate # endif * TLS_VRFY_PER_CTX *
3107c478bd9Sstevel@tonic-gate */
3117c478bd9Sstevel@tonic-gate 
3127c478bd9Sstevel@tonic-gate void
3137c478bd9Sstevel@tonic-gate tls_set_verify(ctx, ssl, vrfy)
3147c478bd9Sstevel@tonic-gate 	SSL_CTX *ctx;
3157c478bd9Sstevel@tonic-gate 	SSL *ssl;
3167c478bd9Sstevel@tonic-gate 	bool vrfy;
3177c478bd9Sstevel@tonic-gate {
3187c478bd9Sstevel@tonic-gate # if !TLS_VRFY_PER_CTX
3197c478bd9Sstevel@tonic-gate 	SSL_set_verify(ssl, vrfy ? SSL_VERIFY_PEER : SSL_VERIFY_NONE, NULL);
3207c478bd9Sstevel@tonic-gate # else /* !TLS_VRFY_PER_CTX */
3217c478bd9Sstevel@tonic-gate 	SSL_CTX_set_verify(ctx, vrfy ? SSL_VERIFY_PEER : SSL_VERIFY_NONE,
3227c478bd9Sstevel@tonic-gate 			NULL);
3237c478bd9Sstevel@tonic-gate # endif /* !TLS_VRFY_PER_CTX */
3247c478bd9Sstevel@tonic-gate }
3257c478bd9Sstevel@tonic-gate 
3267c478bd9Sstevel@tonic-gate /*
3277c478bd9Sstevel@tonic-gate **  status in initialization
3287c478bd9Sstevel@tonic-gate **  these flags keep track of the status of the initialization
3297c478bd9Sstevel@tonic-gate **  i.e., whether a file exists (_EX) and whether it can be used (_OK)
3307c478bd9Sstevel@tonic-gate **  [due to permissions]
3317c478bd9Sstevel@tonic-gate */
3327c478bd9Sstevel@tonic-gate 
3337c478bd9Sstevel@tonic-gate # define TLS_S_NONE	0x00000000	/* none yet */
3347c478bd9Sstevel@tonic-gate # define TLS_S_CERT_EX	0x00000001	/* cert file exists */
3357c478bd9Sstevel@tonic-gate # define TLS_S_CERT_OK	0x00000002	/* cert file is ok */
3367c478bd9Sstevel@tonic-gate # define TLS_S_KEY_EX	0x00000004	/* key file exists */
3377c478bd9Sstevel@tonic-gate # define TLS_S_KEY_OK	0x00000008	/* key file is ok */
3387c478bd9Sstevel@tonic-gate # define TLS_S_CERTP_EX	0x00000010	/* CA cert path exists */
3397c478bd9Sstevel@tonic-gate # define TLS_S_CERTP_OK	0x00000020	/* CA cert path is ok */
3407c478bd9Sstevel@tonic-gate # define TLS_S_CERTF_EX	0x00000040	/* CA cert file exists */
3417c478bd9Sstevel@tonic-gate # define TLS_S_CERTF_OK	0x00000080	/* CA cert file is ok */
3427c478bd9Sstevel@tonic-gate # define TLS_S_CRLF_EX	0x00000100	/* CRL file exists */
3437c478bd9Sstevel@tonic-gate # define TLS_S_CRLF_OK	0x00000200	/* CRL file is ok */
3447c478bd9Sstevel@tonic-gate 
3457c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
3467c478bd9Sstevel@tonic-gate #  define TLS_S_CERT2_EX	0x00001000	/* 2nd cert file exists */
3477c478bd9Sstevel@tonic-gate #  define TLS_S_CERT2_OK	0x00002000	/* 2nd cert file is ok */
3487c478bd9Sstevel@tonic-gate #  define TLS_S_KEY2_EX	0x00004000	/* 2nd key file exists */
3497c478bd9Sstevel@tonic-gate #  define TLS_S_KEY2_OK	0x00008000	/* 2nd key file is ok */
3507c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
3517c478bd9Sstevel@tonic-gate 
3527c478bd9Sstevel@tonic-gate # define TLS_S_DH_OK	0x00200000	/* DH cert is ok */
3537c478bd9Sstevel@tonic-gate # define TLS_S_DHPAR_EX	0x00400000	/* DH param file exists */
3547c478bd9Sstevel@tonic-gate # define TLS_S_DHPAR_OK	0x00800000	/* DH param file is ok to use */
3557c478bd9Sstevel@tonic-gate 
3567c478bd9Sstevel@tonic-gate /* Type of variable */
3577c478bd9Sstevel@tonic-gate # define TLS_T_OTHER	0
3587c478bd9Sstevel@tonic-gate # define TLS_T_SRV	1
3597c478bd9Sstevel@tonic-gate # define TLS_T_CLT	2
3607c478bd9Sstevel@tonic-gate 
3617c478bd9Sstevel@tonic-gate /*
3627c478bd9Sstevel@tonic-gate **  TLS_OK_F -- can var be an absolute filename?
3637c478bd9Sstevel@tonic-gate **
3647c478bd9Sstevel@tonic-gate **	Parameters:
3657c478bd9Sstevel@tonic-gate **		var -- filename
3667c478bd9Sstevel@tonic-gate **		fn -- what is the filename used for?
3677c478bd9Sstevel@tonic-gate **		type -- type of variable
3687c478bd9Sstevel@tonic-gate **
3697c478bd9Sstevel@tonic-gate **	Returns:
3707c478bd9Sstevel@tonic-gate **		ok?
3717c478bd9Sstevel@tonic-gate */
3727c478bd9Sstevel@tonic-gate 
3737c478bd9Sstevel@tonic-gate static bool
3747c478bd9Sstevel@tonic-gate tls_ok_f(var, fn, type)
3757c478bd9Sstevel@tonic-gate 	char *var;
3767c478bd9Sstevel@tonic-gate 	char *fn;
3777c478bd9Sstevel@tonic-gate 	int type;
3787c478bd9Sstevel@tonic-gate {
3797c478bd9Sstevel@tonic-gate 	/* must be absolute pathname */
3807c478bd9Sstevel@tonic-gate 	if (var != NULL && *var == '/')
3817c478bd9Sstevel@tonic-gate 		return true;
3827c478bd9Sstevel@tonic-gate 	if (LogLevel > 12)
3837c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_WARNING, NOQID, "STARTTLS: %s%s missing",
3847c478bd9Sstevel@tonic-gate 			  type == TLS_T_SRV ? "Server" :
3857c478bd9Sstevel@tonic-gate 			  (type == TLS_T_CLT ? "Client" : ""), fn);
3867c478bd9Sstevel@tonic-gate 	return false;
3877c478bd9Sstevel@tonic-gate }
3887c478bd9Sstevel@tonic-gate /*
3897c478bd9Sstevel@tonic-gate **  TLS_SAFE_F -- is a file safe to use?
3907c478bd9Sstevel@tonic-gate **
3917c478bd9Sstevel@tonic-gate **	Parameters:
3927c478bd9Sstevel@tonic-gate **		var -- filename
3937c478bd9Sstevel@tonic-gate **		sff -- flags for safefile()
3947c478bd9Sstevel@tonic-gate **		srv -- server side?
3957c478bd9Sstevel@tonic-gate **
3967c478bd9Sstevel@tonic-gate **	Returns:
3977c478bd9Sstevel@tonic-gate **		ok?
3987c478bd9Sstevel@tonic-gate */
3997c478bd9Sstevel@tonic-gate 
4007c478bd9Sstevel@tonic-gate static bool
4017c478bd9Sstevel@tonic-gate tls_safe_f(var, sff, srv)
4027c478bd9Sstevel@tonic-gate 	char *var;
4037c478bd9Sstevel@tonic-gate 	long sff;
4047c478bd9Sstevel@tonic-gate 	bool srv;
4057c478bd9Sstevel@tonic-gate {
4067c478bd9Sstevel@tonic-gate 	int ret;
4077c478bd9Sstevel@tonic-gate 
4087c478bd9Sstevel@tonic-gate 	if ((ret = safefile(var, RunAsUid, RunAsGid, RunAsUserName, sff,
4097c478bd9Sstevel@tonic-gate 			    S_IRUSR, NULL)) == 0)
4107c478bd9Sstevel@tonic-gate 		return true;
4117c478bd9Sstevel@tonic-gate 	if (LogLevel > 7)
4127c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_WARNING, NOQID, "STARTTLS=%s: file %s unsafe: %s",
4137c478bd9Sstevel@tonic-gate 			  srv ? "server" : "client", var, sm_errstring(ret));
4147c478bd9Sstevel@tonic-gate 	return false;
4157c478bd9Sstevel@tonic-gate }
4167c478bd9Sstevel@tonic-gate 
4177c478bd9Sstevel@tonic-gate /*
4187c478bd9Sstevel@tonic-gate **  TLS_OK_F -- macro to simplify calls to tls_ok_f
4197c478bd9Sstevel@tonic-gate **
4207c478bd9Sstevel@tonic-gate **	Parameters:
4217c478bd9Sstevel@tonic-gate **		var -- filename
4227c478bd9Sstevel@tonic-gate **		fn -- what is the filename used for?
4237c478bd9Sstevel@tonic-gate **		req -- is the file required?
4247c478bd9Sstevel@tonic-gate **		st -- status bit to set if ok
4257c478bd9Sstevel@tonic-gate **		type -- type of variable
4267c478bd9Sstevel@tonic-gate **
4277c478bd9Sstevel@tonic-gate **	Side Effects:
4287c478bd9Sstevel@tonic-gate **		uses r, ok; may change ok and status.
4297c478bd9Sstevel@tonic-gate **
4307c478bd9Sstevel@tonic-gate */
4317c478bd9Sstevel@tonic-gate 
4327c478bd9Sstevel@tonic-gate # define TLS_OK_F(var, fn, req, st, type) if (ok) \
4337c478bd9Sstevel@tonic-gate 	{ \
4347c478bd9Sstevel@tonic-gate 		r = tls_ok_f(var, fn, type); \
4357c478bd9Sstevel@tonic-gate 		if (r) \
4367c478bd9Sstevel@tonic-gate 			status |= st; \
4377c478bd9Sstevel@tonic-gate 		else if (req) \
4387c478bd9Sstevel@tonic-gate 			ok = false; \
4397c478bd9Sstevel@tonic-gate 	}
4407c478bd9Sstevel@tonic-gate 
4417c478bd9Sstevel@tonic-gate /*
4427c478bd9Sstevel@tonic-gate **  TLS_UNR -- macro to return whether a file should be unreadable
4437c478bd9Sstevel@tonic-gate **
4447c478bd9Sstevel@tonic-gate **	Parameters:
4457c478bd9Sstevel@tonic-gate **		bit -- flag to test
4467c478bd9Sstevel@tonic-gate **		req -- flags
4477c478bd9Sstevel@tonic-gate **
4487c478bd9Sstevel@tonic-gate **	Returns:
4497c478bd9Sstevel@tonic-gate **		0/SFF_NORFILES
4507c478bd9Sstevel@tonic-gate */
4517c478bd9Sstevel@tonic-gate # define TLS_UNR(bit, req)	(bitset(bit, req) ? SFF_NORFILES : 0)
4527c478bd9Sstevel@tonic-gate # define TLS_OUNR(bit, req)	(bitset(bit, req) ? SFF_NOWRFILES : 0)
4537c478bd9Sstevel@tonic-gate # define TLS_KEYSFF(req)	\
4547c478bd9Sstevel@tonic-gate 	(bitnset(DBS_GROUPREADABLEKEYFILE, DontBlameSendmail) ?	\
4557c478bd9Sstevel@tonic-gate 		TLS_OUNR(TLS_I_KEY_OUNR, req) :			\
4567c478bd9Sstevel@tonic-gate 		TLS_UNR(TLS_I_KEY_UNR, req))
4577c478bd9Sstevel@tonic-gate 
4587c478bd9Sstevel@tonic-gate /*
4597c478bd9Sstevel@tonic-gate **  TLS_SAFE_F -- macro to simplify calls to tls_safe_f
4607c478bd9Sstevel@tonic-gate **
4617c478bd9Sstevel@tonic-gate **	Parameters:
4627c478bd9Sstevel@tonic-gate **		var -- filename
4637c478bd9Sstevel@tonic-gate **		sff -- flags for safefile()
4647c478bd9Sstevel@tonic-gate **		req -- is the file required?
4657c478bd9Sstevel@tonic-gate **		ex -- does the file exist?
4667c478bd9Sstevel@tonic-gate **		st -- status bit to set if ok
4677c478bd9Sstevel@tonic-gate **		srv -- server side?
4687c478bd9Sstevel@tonic-gate **
4697c478bd9Sstevel@tonic-gate **	Side Effects:
4707c478bd9Sstevel@tonic-gate **		uses r, ok, ex; may change ok and status.
4717c478bd9Sstevel@tonic-gate **
4727c478bd9Sstevel@tonic-gate */
4737c478bd9Sstevel@tonic-gate 
4747c478bd9Sstevel@tonic-gate # define TLS_SAFE_F(var, sff, req, ex, st, srv) if (ex && ok) \
4757c478bd9Sstevel@tonic-gate 	{ \
4767c478bd9Sstevel@tonic-gate 		r = tls_safe_f(var, sff, srv); \
4777c478bd9Sstevel@tonic-gate 		if (r) \
4787c478bd9Sstevel@tonic-gate 			status |= st;	\
4797c478bd9Sstevel@tonic-gate 		else if (req) \
4807c478bd9Sstevel@tonic-gate 			ok = false;	\
4817c478bd9Sstevel@tonic-gate 	}
4827c478bd9Sstevel@tonic-gate 
4837c478bd9Sstevel@tonic-gate /*
4847c478bd9Sstevel@tonic-gate **  INITTLS -- initialize TLS
4857c478bd9Sstevel@tonic-gate **
4867c478bd9Sstevel@tonic-gate **	Parameters:
4877c478bd9Sstevel@tonic-gate **		ctx -- pointer to context
4887c478bd9Sstevel@tonic-gate **		req -- requirements for initialization (see sendmail.h)
489*e9af4bc0SJohn Beck **		options -- options
4907c478bd9Sstevel@tonic-gate **		srv -- server side?
4917c478bd9Sstevel@tonic-gate **		certfile -- filename of certificate
4927c478bd9Sstevel@tonic-gate **		keyfile -- filename of private key
4937c478bd9Sstevel@tonic-gate **		cacertpath -- path to CAs
4947c478bd9Sstevel@tonic-gate **		cacertfile -- file with CA(s)
4957c478bd9Sstevel@tonic-gate **		dhparam -- parameters for DH
4967c478bd9Sstevel@tonic-gate **
4977c478bd9Sstevel@tonic-gate **	Returns:
4987c478bd9Sstevel@tonic-gate **		succeeded?
4997c478bd9Sstevel@tonic-gate */
5007c478bd9Sstevel@tonic-gate 
501445f2479Sjbeck /*
502445f2479Sjbeck **  The session_id_context identifies the service that created a session.
503445f2479Sjbeck **  This information is used to distinguish between multiple TLS-based
504445f2479Sjbeck **  servers running on the same server. We use the name of the mail system.
505445f2479Sjbeck **  Note: the session cache is not persistent.
506445f2479Sjbeck */
507445f2479Sjbeck 
508445f2479Sjbeck static char server_session_id_context[] = "sendmail8";
509445f2479Sjbeck 
5103ee0e492Sjbeck /* 0.9.8a and b have a problem with SSL_OP_TLS_BLOCK_PADDING_BUG */
5113ee0e492Sjbeck #if (OPENSSL_VERSION_NUMBER >= 0x0090800fL)
5123ee0e492Sjbeck # define SM_SSL_OP_TLS_BLOCK_PADDING_BUG	1
5133ee0e492Sjbeck #else
5143ee0e492Sjbeck # define SM_SSL_OP_TLS_BLOCK_PADDING_BUG	0
5153ee0e492Sjbeck #endif
5163ee0e492Sjbeck 
5177c478bd9Sstevel@tonic-gate bool
518*e9af4bc0SJohn Beck inittls(ctx, req, options, srv, certfile, keyfile, cacertpath, cacertfile, dhparam)
5197c478bd9Sstevel@tonic-gate 	SSL_CTX **ctx;
5207c478bd9Sstevel@tonic-gate 	unsigned long req;
521*e9af4bc0SJohn Beck 	long options;
5227c478bd9Sstevel@tonic-gate 	bool srv;
5237c478bd9Sstevel@tonic-gate 	char *certfile, *keyfile, *cacertpath, *cacertfile, *dhparam;
5247c478bd9Sstevel@tonic-gate {
5257c478bd9Sstevel@tonic-gate # if !NO_DH
5267c478bd9Sstevel@tonic-gate 	static DH *dh = NULL;
5277c478bd9Sstevel@tonic-gate # endif /* !NO_DH */
5287c478bd9Sstevel@tonic-gate 	int r;
5297c478bd9Sstevel@tonic-gate 	bool ok;
530*e9af4bc0SJohn Beck 	long sff, status;
5317c478bd9Sstevel@tonic-gate 	char *who;
5327c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
5337c478bd9Sstevel@tonic-gate 	char *cf2, *kf2;
5347c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
5357c478bd9Sstevel@tonic-gate #  if SM_CONF_SHM
5367c478bd9Sstevel@tonic-gate 	extern int ShmId;
5377c478bd9Sstevel@tonic-gate #  endif /* SM_CONF_SHM */
5387c478bd9Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
5397c478bd9Sstevel@tonic-gate 	BIO *crl_file;
5407c478bd9Sstevel@tonic-gate 	X509_CRL *crl;
5417c478bd9Sstevel@tonic-gate 	X509_STORE *store;
5427c478bd9Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
5433ee0e492Sjbeck #if SM_SSL_OP_TLS_BLOCK_PADDING_BUG
5443ee0e492Sjbeck 	long rt_version;
5453ee0e492Sjbeck 	STACK_OF(SSL_COMP) *comp_methods;
5463ee0e492Sjbeck #endif
5477c478bd9Sstevel@tonic-gate 
5487c478bd9Sstevel@tonic-gate 	status = TLS_S_NONE;
5497c478bd9Sstevel@tonic-gate 	who = srv ? "server" : "client";
5507c478bd9Sstevel@tonic-gate 	if (ctx == NULL)
5513ee0e492Sjbeck 	{
5527c478bd9Sstevel@tonic-gate 		syserr("STARTTLS=%s, inittls: ctx == NULL", who);
5533ee0e492Sjbeck 		/* NOTREACHED */
5543ee0e492Sjbeck 		SM_ASSERT(ctx != NULL);
5553ee0e492Sjbeck 	}
5567c478bd9Sstevel@tonic-gate 
5577c478bd9Sstevel@tonic-gate 	/* already initialized? (we could re-init...) */
5587c478bd9Sstevel@tonic-gate 	if (*ctx != NULL)
5597c478bd9Sstevel@tonic-gate 		return true;
5607c478bd9Sstevel@tonic-gate 	ok = true;
5617c478bd9Sstevel@tonic-gate 
5627c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
5637c478bd9Sstevel@tonic-gate 	/*
5647c478bd9Sstevel@tonic-gate 	**  look for a second filename: it must be separated by a ','
5657c478bd9Sstevel@tonic-gate 	**  no blanks allowed (they won't be skipped).
5667c478bd9Sstevel@tonic-gate 	**  we change a global variable here! this change will be undone
5677c478bd9Sstevel@tonic-gate 	**  before return from the function but only if it returns true.
5687c478bd9Sstevel@tonic-gate 	**  this isn't a problem since in a failure case this function
5697c478bd9Sstevel@tonic-gate 	**  won't be called again with the same (overwritten) values.
5707c478bd9Sstevel@tonic-gate 	**  otherwise each return must be replaced with a goto endinittls.
5717c478bd9Sstevel@tonic-gate 	*/
5727c478bd9Sstevel@tonic-gate 
5737c478bd9Sstevel@tonic-gate 	cf2 = NULL;
5747c478bd9Sstevel@tonic-gate 	kf2 = NULL;
5757c478bd9Sstevel@tonic-gate 	if (certfile != NULL && (cf2 = strchr(certfile, ',')) != NULL)
5767c478bd9Sstevel@tonic-gate 	{
5777c478bd9Sstevel@tonic-gate 		*cf2++ = '\0';
5787c478bd9Sstevel@tonic-gate 		if (keyfile != NULL && (kf2 = strchr(keyfile, ',')) != NULL)
5797c478bd9Sstevel@tonic-gate 			*kf2++ = '\0';
5807c478bd9Sstevel@tonic-gate 	}
5817c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
5827c478bd9Sstevel@tonic-gate 
5837c478bd9Sstevel@tonic-gate 	/*
5847c478bd9Sstevel@tonic-gate 	**  Check whether files/paths are defined
5857c478bd9Sstevel@tonic-gate 	*/
5867c478bd9Sstevel@tonic-gate 
5877c478bd9Sstevel@tonic-gate 	TLS_OK_F(certfile, "CertFile", bitset(TLS_I_CERT_EX, req),
5887c478bd9Sstevel@tonic-gate 		 TLS_S_CERT_EX, srv ? TLS_T_SRV : TLS_T_CLT);
5897c478bd9Sstevel@tonic-gate 	TLS_OK_F(keyfile, "KeyFile", bitset(TLS_I_KEY_EX, req),
5907c478bd9Sstevel@tonic-gate 		 TLS_S_KEY_EX, srv ? TLS_T_SRV : TLS_T_CLT);
5917c478bd9Sstevel@tonic-gate 	TLS_OK_F(cacertpath, "CACertPath", bitset(TLS_I_CERTP_EX, req),
5927c478bd9Sstevel@tonic-gate 		 TLS_S_CERTP_EX, TLS_T_OTHER);
5937c478bd9Sstevel@tonic-gate 	TLS_OK_F(cacertfile, "CACertFile", bitset(TLS_I_CERTF_EX, req),
5947c478bd9Sstevel@tonic-gate 		 TLS_S_CERTF_EX, TLS_T_OTHER);
5957c478bd9Sstevel@tonic-gate 
5967c478bd9Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
5977c478bd9Sstevel@tonic-gate 	TLS_OK_F(CRLFile, "CRLFile", bitset(TLS_I_CRLF_EX, req),
5987c478bd9Sstevel@tonic-gate 		 TLS_S_CRLF_EX, TLS_T_OTHER);
5997c478bd9Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
6007c478bd9Sstevel@tonic-gate 
6017c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
6027c478bd9Sstevel@tonic-gate 	/*
6037c478bd9Sstevel@tonic-gate 	**  if the second file is specified it must exist
6047c478bd9Sstevel@tonic-gate 	**  XXX: it is possible here to define only one of those files
6057c478bd9Sstevel@tonic-gate 	*/
6067c478bd9Sstevel@tonic-gate 
6077c478bd9Sstevel@tonic-gate 	if (cf2 != NULL)
6087c478bd9Sstevel@tonic-gate 	{
6097c478bd9Sstevel@tonic-gate 		TLS_OK_F(cf2, "CertFile", bitset(TLS_I_CERT_EX, req),
6107c478bd9Sstevel@tonic-gate 			 TLS_S_CERT2_EX, srv ? TLS_T_SRV : TLS_T_CLT);
6117c478bd9Sstevel@tonic-gate 	}
6127c478bd9Sstevel@tonic-gate 	if (kf2 != NULL)
6137c478bd9Sstevel@tonic-gate 	{
6147c478bd9Sstevel@tonic-gate 		TLS_OK_F(kf2, "KeyFile", bitset(TLS_I_KEY_EX, req),
6157c478bd9Sstevel@tonic-gate 			 TLS_S_KEY2_EX, srv ? TLS_T_SRV : TLS_T_CLT);
6167c478bd9Sstevel@tonic-gate 	}
6177c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
6187c478bd9Sstevel@tonic-gate 
6197c478bd9Sstevel@tonic-gate 	/*
6207c478bd9Sstevel@tonic-gate 	**  valid values for dhparam are (only the first char is checked)
6217c478bd9Sstevel@tonic-gate 	**  none	no parameters: don't use DH
6227c478bd9Sstevel@tonic-gate 	**  512		generate 512 bit parameters (fixed)
6237c478bd9Sstevel@tonic-gate 	**  1024	generate 1024 bit parameters
6247c478bd9Sstevel@tonic-gate 	**  /file/name	read parameters from /file/name
6257c478bd9Sstevel@tonic-gate 	**  default is: 1024 for server, 512 for client (OK? XXX)
6267c478bd9Sstevel@tonic-gate 	*/
6277c478bd9Sstevel@tonic-gate 
6287c478bd9Sstevel@tonic-gate 	if (bitset(TLS_I_TRY_DH, req))
6297c478bd9Sstevel@tonic-gate 	{
6307c478bd9Sstevel@tonic-gate 		if (dhparam != NULL)
6317c478bd9Sstevel@tonic-gate 		{
6327c478bd9Sstevel@tonic-gate 			char c = *dhparam;
6337c478bd9Sstevel@tonic-gate 
6347c478bd9Sstevel@tonic-gate 			if (c == '1')
6357c478bd9Sstevel@tonic-gate 				req |= TLS_I_DH1024;
6367c478bd9Sstevel@tonic-gate 			else if (c == '5')
6377c478bd9Sstevel@tonic-gate 				req |= TLS_I_DH512;
6387c478bd9Sstevel@tonic-gate 			else if (c != 'n' && c != 'N' && c != '/')
6397c478bd9Sstevel@tonic-gate 			{
6407c478bd9Sstevel@tonic-gate 				if (LogLevel > 12)
6417c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
6427c478bd9Sstevel@tonic-gate 						  "STARTTLS=%s, error: illegal value '%s' for DHParam",
6437c478bd9Sstevel@tonic-gate 						  who, dhparam);
6447c478bd9Sstevel@tonic-gate 				dhparam = NULL;
6457c478bd9Sstevel@tonic-gate 			}
6467c478bd9Sstevel@tonic-gate 		}
6477c478bd9Sstevel@tonic-gate 		if (dhparam == NULL)
648*e9af4bc0SJohn Beck 		{
6497c478bd9Sstevel@tonic-gate 			dhparam = srv ? "1" : "5";
650*e9af4bc0SJohn Beck 			req |= (srv ? TLS_I_DH1024 : TLS_I_DH512);
651*e9af4bc0SJohn Beck 		}
6527c478bd9Sstevel@tonic-gate 		else if (*dhparam == '/')
6537c478bd9Sstevel@tonic-gate 		{
6547c478bd9Sstevel@tonic-gate 			TLS_OK_F(dhparam, "DHParameters",
6557c478bd9Sstevel@tonic-gate 				 bitset(TLS_I_DHPAR_EX, req),
6567c478bd9Sstevel@tonic-gate 				 TLS_S_DHPAR_EX, TLS_T_OTHER);
6577c478bd9Sstevel@tonic-gate 		}
6587c478bd9Sstevel@tonic-gate 	}
6597c478bd9Sstevel@tonic-gate 	if (!ok)
6607c478bd9Sstevel@tonic-gate 		return ok;
6617c478bd9Sstevel@tonic-gate 
6627c478bd9Sstevel@tonic-gate 	/* certfile etc. must be "safe". */
6637c478bd9Sstevel@tonic-gate 	sff = SFF_REGONLY | SFF_SAFEDIRPATH | SFF_NOWLINK
6647c478bd9Sstevel@tonic-gate 	     | SFF_NOGWFILES | SFF_NOWWFILES
6657c478bd9Sstevel@tonic-gate 	     | SFF_MUSTOWN | SFF_ROOTOK | SFF_OPENASROOT;
6667c478bd9Sstevel@tonic-gate 	if (DontLockReadFiles)
6677c478bd9Sstevel@tonic-gate 		sff |= SFF_NOLOCK;
6687c478bd9Sstevel@tonic-gate 
6697c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(certfile, sff | TLS_UNR(TLS_I_CERT_UNR, req),
6707c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_CERT_EX, req),
6717c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_CERT_EX, status), TLS_S_CERT_OK, srv);
6727c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(keyfile, sff | TLS_KEYSFF(req),
6737c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_KEY_EX, req),
6747c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_KEY_EX, status), TLS_S_KEY_OK, srv);
6757c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(cacertfile, sff | TLS_UNR(TLS_I_CERTF_UNR, req),
6767c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_CERTF_EX, req),
6777c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_CERTF_EX, status), TLS_S_CERTF_OK, srv);
6787c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(dhparam, sff | TLS_UNR(TLS_I_DHPAR_UNR, req),
6797c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_DHPAR_EX, req),
6807c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_DHPAR_EX, status), TLS_S_DHPAR_OK, srv);
6817c478bd9Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
6827c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(CRLFile, sff | TLS_UNR(TLS_I_CRLF_UNR, req),
6837c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_CRLF_EX, req),
6847c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_CRLF_EX, status), TLS_S_CRLF_OK, srv);
6857c478bd9Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
6867c478bd9Sstevel@tonic-gate 	if (!ok)
6877c478bd9Sstevel@tonic-gate 		return ok;
6887c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
6897c478bd9Sstevel@tonic-gate 	if (cf2 != NULL)
6907c478bd9Sstevel@tonic-gate 	{
6917c478bd9Sstevel@tonic-gate 		TLS_SAFE_F(cf2, sff | TLS_UNR(TLS_I_CERT_UNR, req),
6927c478bd9Sstevel@tonic-gate 			   bitset(TLS_I_CERT_EX, req),
6937c478bd9Sstevel@tonic-gate 			   bitset(TLS_S_CERT2_EX, status), TLS_S_CERT2_OK, srv);
6947c478bd9Sstevel@tonic-gate 	}
6957c478bd9Sstevel@tonic-gate 	if (kf2 != NULL)
6967c478bd9Sstevel@tonic-gate 	{
6977c478bd9Sstevel@tonic-gate 		TLS_SAFE_F(kf2, sff | TLS_KEYSFF(req),
6987c478bd9Sstevel@tonic-gate 			   bitset(TLS_I_KEY_EX, req),
6997c478bd9Sstevel@tonic-gate 			   bitset(TLS_S_KEY2_EX, status), TLS_S_KEY2_OK, srv);
7007c478bd9Sstevel@tonic-gate 	}
7017c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
7027c478bd9Sstevel@tonic-gate 
7037c478bd9Sstevel@tonic-gate 	/* create a method and a new context */
7047c478bd9Sstevel@tonic-gate 	if ((*ctx = SSL_CTX_new(srv ? SSLv23_server_method() :
7057c478bd9Sstevel@tonic-gate 				      SSLv23_client_method())) == NULL)
7067c478bd9Sstevel@tonic-gate 	{
7077c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
7087c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
7097c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_new(SSLv23_%s_method()) failed",
7107c478bd9Sstevel@tonic-gate 				  who, who);
7117c478bd9Sstevel@tonic-gate 		if (LogLevel > 9)
7127c478bd9Sstevel@tonic-gate 			tlslogerr(who);
7137c478bd9Sstevel@tonic-gate 		return false;
7147c478bd9Sstevel@tonic-gate 	}
7157c478bd9Sstevel@tonic-gate 
7167c478bd9Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
7177c478bd9Sstevel@tonic-gate 	if (CRLFile != NULL)
7187c478bd9Sstevel@tonic-gate 	{
7197c478bd9Sstevel@tonic-gate 		/* get a pointer to the current certificate validation store */
7207c478bd9Sstevel@tonic-gate 		store = SSL_CTX_get_cert_store(*ctx);	/* does not fail */
7217c478bd9Sstevel@tonic-gate 		crl_file = BIO_new(BIO_s_file_internal());
7227c478bd9Sstevel@tonic-gate 		if (crl_file != NULL)
7237c478bd9Sstevel@tonic-gate 		{
7247c478bd9Sstevel@tonic-gate 			if (BIO_read_filename(crl_file, CRLFile) >= 0)
7257c478bd9Sstevel@tonic-gate 			{
7267c478bd9Sstevel@tonic-gate 				crl = PEM_read_bio_X509_CRL(crl_file, NULL,
7277c478bd9Sstevel@tonic-gate 							NULL, NULL);
7287c478bd9Sstevel@tonic-gate 				BIO_free(crl_file);
7297c478bd9Sstevel@tonic-gate 				X509_STORE_add_crl(store, crl);
7307c478bd9Sstevel@tonic-gate 				X509_CRL_free(crl);
7317c478bd9Sstevel@tonic-gate 				X509_STORE_set_flags(store,
7327c478bd9Sstevel@tonic-gate 					X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL);
7337c478bd9Sstevel@tonic-gate 				X509_STORE_set_verify_cb_func(store,
7347c478bd9Sstevel@tonic-gate 						x509_verify_cb);
7357c478bd9Sstevel@tonic-gate 			}
7367c478bd9Sstevel@tonic-gate 			else
7377c478bd9Sstevel@tonic-gate 			{
7387c478bd9Sstevel@tonic-gate 				if (LogLevel > 9)
7397c478bd9Sstevel@tonic-gate 				{
7407c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
7417c478bd9Sstevel@tonic-gate 						  "STARTTLS=%s, error: PEM_read_bio_X509_CRL(%s)=failed",
7427c478bd9Sstevel@tonic-gate 						  who, CRLFile);
7437c478bd9Sstevel@tonic-gate 				}
7447c478bd9Sstevel@tonic-gate 
7457c478bd9Sstevel@tonic-gate 				/* avoid memory leaks */
7467c478bd9Sstevel@tonic-gate 				BIO_free(crl_file);
7477c478bd9Sstevel@tonic-gate 				return false;
7487c478bd9Sstevel@tonic-gate 			}
7497c478bd9Sstevel@tonic-gate 
7507c478bd9Sstevel@tonic-gate 		}
7517c478bd9Sstevel@tonic-gate 		else if (LogLevel > 9)
7527c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
7537c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: BIO_new=failed", who);
7547c478bd9Sstevel@tonic-gate 	}
755058561cbSjbeck 	else
756058561cbSjbeck 		store = NULL;
7577c478bd9Sstevel@tonic-gate #  if _FFR_CRLPATH
758058561cbSjbeck 	if (CRLPath != NULL && store != NULL)
7597c478bd9Sstevel@tonic-gate 	{
7607c478bd9Sstevel@tonic-gate 		X509_LOOKUP *lookup;
7617c478bd9Sstevel@tonic-gate 
7627c478bd9Sstevel@tonic-gate 		lookup = X509_STORE_add_lookup(store, X509_LOOKUP_hash_dir());
7637c478bd9Sstevel@tonic-gate 		if (lookup == NULL)
7647c478bd9Sstevel@tonic-gate 		{
7657c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
7667c478bd9Sstevel@tonic-gate 			{
7677c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
7687c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, error: X509_STORE_add_lookup(hash)=failed",
7697c478bd9Sstevel@tonic-gate 					  who, CRLFile);
7707c478bd9Sstevel@tonic-gate 			}
7717c478bd9Sstevel@tonic-gate 			return false;
7727c478bd9Sstevel@tonic-gate 		}
7737c478bd9Sstevel@tonic-gate 		X509_LOOKUP_add_dir(lookup, CRLPath, X509_FILETYPE_PEM);
7747c478bd9Sstevel@tonic-gate 		X509_STORE_set_flags(store,
7757c478bd9Sstevel@tonic-gate 			X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL);
7767c478bd9Sstevel@tonic-gate 	}
7777c478bd9Sstevel@tonic-gate #  endif /* _FFR_CRLPATH */
7787c478bd9Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
7797c478bd9Sstevel@tonic-gate 
7807c478bd9Sstevel@tonic-gate # if TLS_NO_RSA
7817c478bd9Sstevel@tonic-gate 	/* turn off backward compatibility, required for no-rsa */
7827c478bd9Sstevel@tonic-gate 	SSL_CTX_set_options(*ctx, SSL_OP_NO_SSLv2);
7837c478bd9Sstevel@tonic-gate # endif /* TLS_NO_RSA */
7847c478bd9Sstevel@tonic-gate 
7857c478bd9Sstevel@tonic-gate 
7867c478bd9Sstevel@tonic-gate # if !TLS_NO_RSA
7877c478bd9Sstevel@tonic-gate 	/*
7887c478bd9Sstevel@tonic-gate 	**  Create a temporary RSA key
7897c478bd9Sstevel@tonic-gate 	**  XXX  Maybe we shouldn't create this always (even though it
7907c478bd9Sstevel@tonic-gate 	**  is only at startup).
7917c478bd9Sstevel@tonic-gate 	**  It is a time-consuming operation and it is not always necessary.
7927c478bd9Sstevel@tonic-gate 	**  maybe we should do it only on demand...
7937c478bd9Sstevel@tonic-gate 	*/
7947c478bd9Sstevel@tonic-gate 
7957c478bd9Sstevel@tonic-gate 	if (bitset(TLS_I_RSA_TMP, req)
7967c478bd9Sstevel@tonic-gate #   if SM_CONF_SHM
7977c478bd9Sstevel@tonic-gate 	    && ShmId != SM_SHM_NO_ID &&
7987c478bd9Sstevel@tonic-gate 	    (rsa_tmp = RSA_generate_key(RSA_KEYLENGTH, RSA_F4, NULL,
7997c478bd9Sstevel@tonic-gate 					NULL)) == NULL
8007c478bd9Sstevel@tonic-gate #   else /* SM_CONF_SHM */
8017c478bd9Sstevel@tonic-gate 	    && 0	/* no shared memory: no need to generate key now */
8027c478bd9Sstevel@tonic-gate #   endif /* SM_CONF_SHM */
8037c478bd9Sstevel@tonic-gate 	   )
8047c478bd9Sstevel@tonic-gate 	{
8057c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
8067c478bd9Sstevel@tonic-gate 		{
8077c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8087c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: RSA_generate_key failed",
8097c478bd9Sstevel@tonic-gate 				  who);
8107c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
8117c478bd9Sstevel@tonic-gate 				tlslogerr(who);
8127c478bd9Sstevel@tonic-gate 		}
8137c478bd9Sstevel@tonic-gate 		return false;
8147c478bd9Sstevel@tonic-gate 	}
8157c478bd9Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
8167c478bd9Sstevel@tonic-gate 
8177c478bd9Sstevel@tonic-gate 	/*
8187c478bd9Sstevel@tonic-gate 	**  load private key
8197c478bd9Sstevel@tonic-gate 	**  XXX change this for DSA-only version
8207c478bd9Sstevel@tonic-gate 	*/
8217c478bd9Sstevel@tonic-gate 
8227c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_KEY_OK, status) &&
8237c478bd9Sstevel@tonic-gate 	    SSL_CTX_use_PrivateKey_file(*ctx, keyfile,
8247c478bd9Sstevel@tonic-gate 					 SSL_FILETYPE_PEM) <= 0)
8257c478bd9Sstevel@tonic-gate 	{
8267c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
8277c478bd9Sstevel@tonic-gate 		{
8287c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8297c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_PrivateKey_file(%s) failed",
8307c478bd9Sstevel@tonic-gate 				  who, keyfile);
8317c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
8327c478bd9Sstevel@tonic-gate 				tlslogerr(who);
8337c478bd9Sstevel@tonic-gate 		}
8347c478bd9Sstevel@tonic-gate 		if (bitset(TLS_I_USE_KEY, req))
8357c478bd9Sstevel@tonic-gate 			return false;
8367c478bd9Sstevel@tonic-gate 	}
8377c478bd9Sstevel@tonic-gate 
8387c478bd9Sstevel@tonic-gate 	/* get the certificate file */
8397c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_CERT_OK, status) &&
8407c478bd9Sstevel@tonic-gate 	    SSL_CTX_use_certificate_file(*ctx, certfile,
8417c478bd9Sstevel@tonic-gate 					 SSL_FILETYPE_PEM) <= 0)
8427c478bd9Sstevel@tonic-gate 	{
8437c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
8447c478bd9Sstevel@tonic-gate 		{
8457c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8467c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_certificate_file(%s) failed",
8477c478bd9Sstevel@tonic-gate 				  who, certfile);
8487c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
8497c478bd9Sstevel@tonic-gate 				tlslogerr(who);
8507c478bd9Sstevel@tonic-gate 		}
8517c478bd9Sstevel@tonic-gate 		if (bitset(TLS_I_USE_CERT, req))
8527c478bd9Sstevel@tonic-gate 			return false;
8537c478bd9Sstevel@tonic-gate 	}
8547c478bd9Sstevel@tonic-gate 
8557c478bd9Sstevel@tonic-gate 	/* check the private key */
8567c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_KEY_OK, status) &&
8577c478bd9Sstevel@tonic-gate 	    (r = SSL_CTX_check_private_key(*ctx)) <= 0)
8587c478bd9Sstevel@tonic-gate 	{
8597c478bd9Sstevel@tonic-gate 		/* Private key does not match the certificate public key */
8607c478bd9Sstevel@tonic-gate 		if (LogLevel > 5)
8617c478bd9Sstevel@tonic-gate 		{
8627c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8637c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_check_private_key failed(%s): %d",
8647c478bd9Sstevel@tonic-gate 				  who, keyfile, r);
8657c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
8667c478bd9Sstevel@tonic-gate 				tlslogerr(who);
8677c478bd9Sstevel@tonic-gate 		}
8687c478bd9Sstevel@tonic-gate 		if (bitset(TLS_I_USE_KEY, req))
8697c478bd9Sstevel@tonic-gate 			return false;
8707c478bd9Sstevel@tonic-gate 	}
8717c478bd9Sstevel@tonic-gate 
8727c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
8737c478bd9Sstevel@tonic-gate 	/* XXX this code is pretty much duplicated from above! */
8747c478bd9Sstevel@tonic-gate 
8757c478bd9Sstevel@tonic-gate 	/* load private key */
8767c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_KEY2_OK, status) &&
8777c478bd9Sstevel@tonic-gate 	    SSL_CTX_use_PrivateKey_file(*ctx, kf2, SSL_FILETYPE_PEM) <= 0)
8787c478bd9Sstevel@tonic-gate 	{
8797c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
8807c478bd9Sstevel@tonic-gate 		{
8817c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8827c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_PrivateKey_file(%s) failed",
8837c478bd9Sstevel@tonic-gate 				  who, kf2);
8847c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
8857c478bd9Sstevel@tonic-gate 				tlslogerr(who);
8867c478bd9Sstevel@tonic-gate 		}
8877c478bd9Sstevel@tonic-gate 	}
8887c478bd9Sstevel@tonic-gate 
8897c478bd9Sstevel@tonic-gate 	/* get the certificate file */
8907c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_CERT2_OK, status) &&
8917c478bd9Sstevel@tonic-gate 	    SSL_CTX_use_certificate_file(*ctx, cf2, SSL_FILETYPE_PEM) <= 0)
8927c478bd9Sstevel@tonic-gate 	{
8937c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
8947c478bd9Sstevel@tonic-gate 		{
8957c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8967c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_certificate_file(%s) failed",
8977c478bd9Sstevel@tonic-gate 				  who, cf2);
8987c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
8997c478bd9Sstevel@tonic-gate 				tlslogerr(who);
9007c478bd9Sstevel@tonic-gate 		}
9017c478bd9Sstevel@tonic-gate 	}
9027c478bd9Sstevel@tonic-gate 
9037c478bd9Sstevel@tonic-gate 	/* also check the private key */
9047c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_KEY2_OK, status) &&
9057c478bd9Sstevel@tonic-gate 	    (r = SSL_CTX_check_private_key(*ctx)) <= 0)
9067c478bd9Sstevel@tonic-gate 	{
9077c478bd9Sstevel@tonic-gate 		/* Private key does not match the certificate public key */
9087c478bd9Sstevel@tonic-gate 		if (LogLevel > 5)
9097c478bd9Sstevel@tonic-gate 		{
9107c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
9117c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_check_private_key 2 failed: %d",
9127c478bd9Sstevel@tonic-gate 				  who, r);
9137c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
9147c478bd9Sstevel@tonic-gate 				tlslogerr(who);
9157c478bd9Sstevel@tonic-gate 		}
9167c478bd9Sstevel@tonic-gate 	}
9177c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
9187c478bd9Sstevel@tonic-gate 
9197c478bd9Sstevel@tonic-gate 	/* SSL_CTX_set_quiet_shutdown(*ctx, 1); violation of standard? */
9203ee0e492Sjbeck 
9213ee0e492Sjbeck #if SM_SSL_OP_TLS_BLOCK_PADDING_BUG
9223ee0e492Sjbeck 
9233ee0e492Sjbeck 	/*
9243ee0e492Sjbeck 	**  In OpenSSL 0.9.8[ab], enabling zlib compression breaks the
9253ee0e492Sjbeck 	**  padding bug work-around, leading to false positives and
9263ee0e492Sjbeck 	**  failed connections. We may not interoperate with systems
9273ee0e492Sjbeck 	**  with the bug, but this is better than breaking on all 0.9.8[ab]
9283ee0e492Sjbeck 	**  systems that have zlib support enabled.
9293ee0e492Sjbeck 	**  Note: this checks the runtime version of the library, not
9303ee0e492Sjbeck 	**  just the compile time version.
9313ee0e492Sjbeck 	*/
9323ee0e492Sjbeck 
9333ee0e492Sjbeck 	rt_version = SSLeay();
9343ee0e492Sjbeck 	if (rt_version >= 0x00908000L && rt_version <= 0x0090802fL)
9353ee0e492Sjbeck 	{
9363ee0e492Sjbeck 		comp_methods = SSL_COMP_get_compression_methods();
9373ee0e492Sjbeck 		if (comp_methods != NULL && sk_SSL_COMP_num(comp_methods) > 0)
9383ee0e492Sjbeck 			options &= ~SSL_OP_TLS_BLOCK_PADDING_BUG;
9393ee0e492Sjbeck 	}
9403ee0e492Sjbeck #endif
9413ee0e492Sjbeck 	SSL_CTX_set_options(*ctx, options);
9427c478bd9Sstevel@tonic-gate 
9437c478bd9Sstevel@tonic-gate # if !NO_DH
9447c478bd9Sstevel@tonic-gate 	/* Diffie-Hellman initialization */
9457c478bd9Sstevel@tonic-gate 	if (bitset(TLS_I_TRY_DH, req))
9467c478bd9Sstevel@tonic-gate 	{
9477c478bd9Sstevel@tonic-gate 		if (bitset(TLS_S_DHPAR_OK, status))
9487c478bd9Sstevel@tonic-gate 		{
9497c478bd9Sstevel@tonic-gate 			BIO *bio;
9507c478bd9Sstevel@tonic-gate 
9517c478bd9Sstevel@tonic-gate 			if ((bio = BIO_new_file(dhparam, "r")) != NULL)
9527c478bd9Sstevel@tonic-gate 			{
9537c478bd9Sstevel@tonic-gate 				dh = PEM_read_bio_DHparams(bio, NULL, NULL, NULL);
9547c478bd9Sstevel@tonic-gate 				BIO_free(bio);
9557c478bd9Sstevel@tonic-gate 				if (dh == NULL && LogLevel > 7)
9567c478bd9Sstevel@tonic-gate 				{
9577c478bd9Sstevel@tonic-gate 					unsigned long err;
9587c478bd9Sstevel@tonic-gate 
9597c478bd9Sstevel@tonic-gate 					err = ERR_get_error();
9607c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
9617c478bd9Sstevel@tonic-gate 						  "STARTTLS=%s, error: cannot read DH parameters(%s): %s",
9627c478bd9Sstevel@tonic-gate 						  who, dhparam,
9637c478bd9Sstevel@tonic-gate 						  ERR_error_string(err, NULL));
9647c478bd9Sstevel@tonic-gate 					if (LogLevel > 9)
9657c478bd9Sstevel@tonic-gate 						tlslogerr(who);
9667c478bd9Sstevel@tonic-gate 				}
9677c478bd9Sstevel@tonic-gate 			}
9687c478bd9Sstevel@tonic-gate 			else
9697c478bd9Sstevel@tonic-gate 			{
9707c478bd9Sstevel@tonic-gate 				if (LogLevel > 5)
9717c478bd9Sstevel@tonic-gate 				{
9727c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
9737c478bd9Sstevel@tonic-gate 						  "STARTTLS=%s, error: BIO_new_file(%s) failed",
9747c478bd9Sstevel@tonic-gate 						  who, dhparam);
9757c478bd9Sstevel@tonic-gate 					if (LogLevel > 9)
9767c478bd9Sstevel@tonic-gate 						tlslogerr(who);
9777c478bd9Sstevel@tonic-gate 				}
9787c478bd9Sstevel@tonic-gate 			}
9797c478bd9Sstevel@tonic-gate 		}
9807c478bd9Sstevel@tonic-gate 		if (dh == NULL && bitset(TLS_I_DH1024, req))
9817c478bd9Sstevel@tonic-gate 		{
9827c478bd9Sstevel@tonic-gate 			DSA *dsa;
9837c478bd9Sstevel@tonic-gate 
9847c478bd9Sstevel@tonic-gate 			/* this takes a while! (7-130s on a 450MHz AMD K6-2) */
9857c478bd9Sstevel@tonic-gate 			dsa = DSA_generate_parameters(1024, NULL, 0, NULL,
9867c478bd9Sstevel@tonic-gate 						      NULL, 0, NULL);
9877c478bd9Sstevel@tonic-gate 			dh = DSA_dup_DH(dsa);
9887c478bd9Sstevel@tonic-gate 			DSA_free(dsa);
9897c478bd9Sstevel@tonic-gate 		}
9907c478bd9Sstevel@tonic-gate 		else
9917c478bd9Sstevel@tonic-gate 		if (dh == NULL && bitset(TLS_I_DH512, req))
9927c478bd9Sstevel@tonic-gate 			dh = get_dh512();
9937c478bd9Sstevel@tonic-gate 
9947c478bd9Sstevel@tonic-gate 		if (dh == NULL)
9957c478bd9Sstevel@tonic-gate 		{
9967c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
9977c478bd9Sstevel@tonic-gate 			{
9987c478bd9Sstevel@tonic-gate 				unsigned long err;
9997c478bd9Sstevel@tonic-gate 
10007c478bd9Sstevel@tonic-gate 				err = ERR_get_error();
10017c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
10027c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, error: cannot read or set DH parameters(%s): %s",
10037c478bd9Sstevel@tonic-gate 					  who, dhparam,
10047c478bd9Sstevel@tonic-gate 					  ERR_error_string(err, NULL));
10057c478bd9Sstevel@tonic-gate 			}
10067c478bd9Sstevel@tonic-gate 			if (bitset(TLS_I_REQ_DH, req))
10077c478bd9Sstevel@tonic-gate 				return false;
10087c478bd9Sstevel@tonic-gate 		}
10097c478bd9Sstevel@tonic-gate 		else
10107c478bd9Sstevel@tonic-gate 		{
10117c478bd9Sstevel@tonic-gate 			SSL_CTX_set_tmp_dh(*ctx, dh);
10127c478bd9Sstevel@tonic-gate 
10137c478bd9Sstevel@tonic-gate 			/* important to avoid small subgroup attacks */
10147c478bd9Sstevel@tonic-gate 			SSL_CTX_set_options(*ctx, SSL_OP_SINGLE_DH_USE);
10157c478bd9Sstevel@tonic-gate 			if (LogLevel > 13)
10167c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_INFO, NOQID,
10177c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, Diffie-Hellman init, key=%d bit (%c)",
10187c478bd9Sstevel@tonic-gate 					  who, 8 * DH_size(dh), *dhparam);
10197c478bd9Sstevel@tonic-gate 			DH_free(dh);
10207c478bd9Sstevel@tonic-gate 		}
10217c478bd9Sstevel@tonic-gate 	}
10227c478bd9Sstevel@tonic-gate # endif /* !NO_DH */
10237c478bd9Sstevel@tonic-gate 
10247c478bd9Sstevel@tonic-gate 
10257c478bd9Sstevel@tonic-gate 	/* XXX do we need this cache here? */
10267c478bd9Sstevel@tonic-gate 	if (bitset(TLS_I_CACHE, req))
1027445f2479Sjbeck 	{
1028445f2479Sjbeck 		SSL_CTX_sess_set_cache_size(*ctx, 1);
1029445f2479Sjbeck 		SSL_CTX_set_timeout(*ctx, 1);
1030445f2479Sjbeck 		SSL_CTX_set_session_id_context(*ctx,
1031445f2479Sjbeck 			(void *) &server_session_id_context,
1032445f2479Sjbeck 			sizeof(server_session_id_context));
1033445f2479Sjbeck 		(void) SSL_CTX_set_session_cache_mode(*ctx,
1034445f2479Sjbeck 				SSL_SESS_CACHE_SERVER);
1035445f2479Sjbeck 	}
1036445f2479Sjbeck 	else
1037445f2479Sjbeck 	{
1038445f2479Sjbeck 		(void) SSL_CTX_set_session_cache_mode(*ctx,
1039445f2479Sjbeck 				SSL_SESS_CACHE_OFF);
1040445f2479Sjbeck 	}
10417c478bd9Sstevel@tonic-gate 
10427c478bd9Sstevel@tonic-gate 	/* load certificate locations and default CA paths */
10437c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_CERTP_EX, status) && bitset(TLS_S_CERTF_EX, status))
10447c478bd9Sstevel@tonic-gate 	{
10457c478bd9Sstevel@tonic-gate 		if ((r = SSL_CTX_load_verify_locations(*ctx, cacertfile,
10467c478bd9Sstevel@tonic-gate 						       cacertpath)) == 1)
10477c478bd9Sstevel@tonic-gate 		{
10487c478bd9Sstevel@tonic-gate # if !TLS_NO_RSA
10497c478bd9Sstevel@tonic-gate 			if (bitset(TLS_I_RSA_TMP, req))
10507c478bd9Sstevel@tonic-gate 				SSL_CTX_set_tmp_rsa_callback(*ctx, tmp_rsa_key);
10517c478bd9Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
10527c478bd9Sstevel@tonic-gate 
10537c478bd9Sstevel@tonic-gate 			/*
10547c478bd9Sstevel@tonic-gate 			**  We have to install our own verify callback:
10557c478bd9Sstevel@tonic-gate 			**  SSL_VERIFY_PEER requests a client cert but even
10567c478bd9Sstevel@tonic-gate 			**  though *FAIL_IF* isn't set, the connection
10577c478bd9Sstevel@tonic-gate 			**  will be aborted if the client presents a cert
10587c478bd9Sstevel@tonic-gate 			**  that is not "liked" (can't be verified?) by
10597c478bd9Sstevel@tonic-gate 			**  the TLS library :-(
10607c478bd9Sstevel@tonic-gate 			*/
10617c478bd9Sstevel@tonic-gate 
10627c478bd9Sstevel@tonic-gate 			/*
10637c478bd9Sstevel@tonic-gate 			**  XXX currently we could call tls_set_verify()
10647c478bd9Sstevel@tonic-gate 			**  but we hope that that function will later on
10657c478bd9Sstevel@tonic-gate 			**  only set the mode per connection.
10667c478bd9Sstevel@tonic-gate 			*/
10677c478bd9Sstevel@tonic-gate 			SSL_CTX_set_verify(*ctx,
10687c478bd9Sstevel@tonic-gate 				bitset(TLS_I_NO_VRFY, req) ? SSL_VERIFY_NONE
10697c478bd9Sstevel@tonic-gate 							   : SSL_VERIFY_PEER,
10707c478bd9Sstevel@tonic-gate 				NULL);
10717c478bd9Sstevel@tonic-gate 
10727c478bd9Sstevel@tonic-gate 			/* install verify callback */
10737c478bd9Sstevel@tonic-gate 			SSL_CTX_set_cert_verify_callback(*ctx, tls_verify_cb,
10747c478bd9Sstevel@tonic-gate 							 NULL);
10757c478bd9Sstevel@tonic-gate 			SSL_CTX_set_client_CA_list(*ctx,
10767c478bd9Sstevel@tonic-gate 				SSL_load_client_CA_file(cacertfile));
10777c478bd9Sstevel@tonic-gate 		}
10787c478bd9Sstevel@tonic-gate 		else
10797c478bd9Sstevel@tonic-gate 		{
10807c478bd9Sstevel@tonic-gate 			/*
10817c478bd9Sstevel@tonic-gate 			**  can't load CA data; do we care?
10827c478bd9Sstevel@tonic-gate 			**  the data is necessary to authenticate the client,
10837c478bd9Sstevel@tonic-gate 			**  which in turn would be necessary
10847c478bd9Sstevel@tonic-gate 			**  if we want to allow relaying based on it.
10857c478bd9Sstevel@tonic-gate 			*/
10867c478bd9Sstevel@tonic-gate 			if (LogLevel > 5)
10877c478bd9Sstevel@tonic-gate 			{
10887c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
10897c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, error: load verify locs %s, %s failed: %d",
10907c478bd9Sstevel@tonic-gate 					  who, cacertpath, cacertfile, r);
10917c478bd9Sstevel@tonic-gate 				if (LogLevel > 9)
10927c478bd9Sstevel@tonic-gate 					tlslogerr(who);
10937c478bd9Sstevel@tonic-gate 			}
10947c478bd9Sstevel@tonic-gate 			if (bitset(TLS_I_VRFY_LOC, req))
10957c478bd9Sstevel@tonic-gate 				return false;
10967c478bd9Sstevel@tonic-gate 		}
10977c478bd9Sstevel@tonic-gate 	}
10987c478bd9Sstevel@tonic-gate 
10997c478bd9Sstevel@tonic-gate 	/* XXX: make this dependent on an option? */
11007c478bd9Sstevel@tonic-gate 	if (tTd(96, 9))
11017c478bd9Sstevel@tonic-gate 		SSL_CTX_set_info_callback(*ctx, apps_ssl_info_cb);
11027c478bd9Sstevel@tonic-gate 
11037c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
11047c478bd9Sstevel@tonic-gate 	/* install our own cipher list */
11057c478bd9Sstevel@tonic-gate 	if (CipherList != NULL && *CipherList != '\0')
11067c478bd9Sstevel@tonic-gate 	{
11077c478bd9Sstevel@tonic-gate 		if (SSL_CTX_set_cipher_list(*ctx, CipherList) <= 0)
11087c478bd9Sstevel@tonic-gate 		{
11097c478bd9Sstevel@tonic-gate 			if (LogLevel > 7)
11107c478bd9Sstevel@tonic-gate 			{
11117c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
11127c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, error: SSL_CTX_set_cipher_list(%s) failed, list ignored",
11137c478bd9Sstevel@tonic-gate 					  who, CipherList);
11147c478bd9Sstevel@tonic-gate 
11157c478bd9Sstevel@tonic-gate 				if (LogLevel > 9)
11167c478bd9Sstevel@tonic-gate 					tlslogerr(who);
11177c478bd9Sstevel@tonic-gate 			}
11187c478bd9Sstevel@tonic-gate 			/* failure if setting to this list is required? */
11197c478bd9Sstevel@tonic-gate 		}
11207c478bd9Sstevel@tonic-gate 	}
11217c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
11227c478bd9Sstevel@tonic-gate 	if (LogLevel > 12)
11237c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_INFO, NOQID, "STARTTLS=%s, init=%d", who, ok);
11247c478bd9Sstevel@tonic-gate 
11257c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
11267c478bd9Sstevel@tonic-gate #  if 0
11277c478bd9Sstevel@tonic-gate 	/*
11287c478bd9Sstevel@tonic-gate 	**  this label is required if we want to have a "clean" exit
11297c478bd9Sstevel@tonic-gate 	**  see the comments above at the initialization of cf2
11307c478bd9Sstevel@tonic-gate 	*/
11317c478bd9Sstevel@tonic-gate 
11327c478bd9Sstevel@tonic-gate     endinittls:
11337c478bd9Sstevel@tonic-gate #  endif /* 0 */
11347c478bd9Sstevel@tonic-gate 
11357c478bd9Sstevel@tonic-gate 	/* undo damage to global variables */
11367c478bd9Sstevel@tonic-gate 	if (cf2 != NULL)
11377c478bd9Sstevel@tonic-gate 		*--cf2 = ',';
11387c478bd9Sstevel@tonic-gate 	if (kf2 != NULL)
11397c478bd9Sstevel@tonic-gate 		*--kf2 = ',';
11407c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
11417c478bd9Sstevel@tonic-gate 
11427c478bd9Sstevel@tonic-gate 	return ok;
11437c478bd9Sstevel@tonic-gate }
11447c478bd9Sstevel@tonic-gate /*
11457c478bd9Sstevel@tonic-gate **  TLS_GET_INFO -- get information about TLS connection
11467c478bd9Sstevel@tonic-gate **
11477c478bd9Sstevel@tonic-gate **	Parameters:
11487c478bd9Sstevel@tonic-gate **		ssl -- TLS connection structure
11497c478bd9Sstevel@tonic-gate **		srv -- server or client
11507c478bd9Sstevel@tonic-gate **		host -- hostname of other side
11517c478bd9Sstevel@tonic-gate **		mac -- macro storage
11527c478bd9Sstevel@tonic-gate **		certreq -- did we ask for a cert?
11537c478bd9Sstevel@tonic-gate **
11547c478bd9Sstevel@tonic-gate **	Returns:
11557c478bd9Sstevel@tonic-gate **		result of authentication.
11567c478bd9Sstevel@tonic-gate **
11577c478bd9Sstevel@tonic-gate **	Side Effects:
11587c478bd9Sstevel@tonic-gate **		sets macros: {cipher}, {tls_version}, {verify},
11597c478bd9Sstevel@tonic-gate **		{cipher_bits}, {alg_bits}, {cert}, {cert_subject},
11607c478bd9Sstevel@tonic-gate **		{cert_issuer}, {cn_subject}, {cn_issuer}
11617c478bd9Sstevel@tonic-gate */
11627c478bd9Sstevel@tonic-gate 
11637c478bd9Sstevel@tonic-gate int
11647c478bd9Sstevel@tonic-gate tls_get_info(ssl, srv, host, mac, certreq)
11657c478bd9Sstevel@tonic-gate 	SSL *ssl;
11667c478bd9Sstevel@tonic-gate 	bool srv;
11677c478bd9Sstevel@tonic-gate 	char *host;
11687c478bd9Sstevel@tonic-gate 	MACROS_T *mac;
11697c478bd9Sstevel@tonic-gate 	bool certreq;
11707c478bd9Sstevel@tonic-gate {
11717c478bd9Sstevel@tonic-gate 	SSL_CIPHER *c;
11727c478bd9Sstevel@tonic-gate 	int b, r;
11737c478bd9Sstevel@tonic-gate 	long verifyok;
11747c478bd9Sstevel@tonic-gate 	char *s, *who;
11757c478bd9Sstevel@tonic-gate 	char bitstr[16];
11767c478bd9Sstevel@tonic-gate 	X509 *cert;
11777c478bd9Sstevel@tonic-gate 
11787c478bd9Sstevel@tonic-gate 	c = SSL_get_current_cipher(ssl);
11797c478bd9Sstevel@tonic-gate 
11807c478bd9Sstevel@tonic-gate 	/* cast is just workaround for compiler warning */
11817c478bd9Sstevel@tonic-gate 	macdefine(mac, A_TEMP, macid("{cipher}"),
11827c478bd9Sstevel@tonic-gate 		  (char *) SSL_CIPHER_get_name(c));
11837c478bd9Sstevel@tonic-gate 	b = SSL_CIPHER_get_bits(c, &r);
1184058561cbSjbeck 	(void) sm_snprintf(bitstr, sizeof(bitstr), "%d", b);
11857c478bd9Sstevel@tonic-gate 	macdefine(mac, A_TEMP, macid("{cipher_bits}"), bitstr);
1186058561cbSjbeck 	(void) sm_snprintf(bitstr, sizeof(bitstr), "%d", r);
11877c478bd9Sstevel@tonic-gate 	macdefine(mac, A_TEMP, macid("{alg_bits}"), bitstr);
11887c478bd9Sstevel@tonic-gate 	s = SSL_CIPHER_get_version(c);
11897c478bd9Sstevel@tonic-gate 	if (s == NULL)
11907c478bd9Sstevel@tonic-gate 		s = "UNKNOWN";
11917c478bd9Sstevel@tonic-gate 	macdefine(mac, A_TEMP, macid("{tls_version}"), s);
11927c478bd9Sstevel@tonic-gate 
11937c478bd9Sstevel@tonic-gate 	who = srv ? "server" : "client";
11947c478bd9Sstevel@tonic-gate 	cert = SSL_get_peer_certificate(ssl);
11957c478bd9Sstevel@tonic-gate 	verifyok = SSL_get_verify_result(ssl);
11967c478bd9Sstevel@tonic-gate 	if (LogLevel > 14)
11977c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_INFO, NOQID,
11987c478bd9Sstevel@tonic-gate 			  "STARTTLS=%s, get_verify: %ld get_peer: 0x%lx",
11997c478bd9Sstevel@tonic-gate 			  who, verifyok, (unsigned long) cert);
12007c478bd9Sstevel@tonic-gate 	if (cert != NULL)
12017c478bd9Sstevel@tonic-gate 	{
12027c478bd9Sstevel@tonic-gate 		unsigned int n;
1203*e9af4bc0SJohn Beck 		X509_NAME *subj, *issuer;
12047c478bd9Sstevel@tonic-gate 		unsigned char md[EVP_MAX_MD_SIZE];
12057c478bd9Sstevel@tonic-gate 		char buf[MAXNAME];
12067c478bd9Sstevel@tonic-gate 
1207*e9af4bc0SJohn Beck 		subj = X509_get_subject_name(cert);
1208*e9af4bc0SJohn Beck 		issuer = X509_get_issuer_name(cert);
1209*e9af4bc0SJohn Beck 		X509_NAME_oneline(subj, buf, sizeof(buf));
12107c478bd9Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cert_subject}"),
12117c478bd9Sstevel@tonic-gate 			 xtextify(buf, "<>\")"));
1212*e9af4bc0SJohn Beck 		X509_NAME_oneline(issuer, buf, sizeof(buf));
12137c478bd9Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cert_issuer}"),
12147c478bd9Sstevel@tonic-gate 			 xtextify(buf, "<>\")"));
1215*e9af4bc0SJohn Beck 
1216*e9af4bc0SJohn Beck #define CHECK_X509_NAME(which)	\
1217*e9af4bc0SJohn Beck 	do {	\
1218*e9af4bc0SJohn Beck 		if (r == -1)	\
1219*e9af4bc0SJohn Beck 		{		\
1220*e9af4bc0SJohn Beck 			sm_strlcpy(buf, "BadCertificateUnknown", sizeof(buf)); \
1221*e9af4bc0SJohn Beck 			if (LogLevel > 7)	\
1222*e9af4bc0SJohn Beck 				sm_syslog(LOG_INFO, NOQID,	\
1223*e9af4bc0SJohn Beck 					"STARTTLS=%s, relay=%.100s, field=%s, status=failed to extract CN",	\
1224*e9af4bc0SJohn Beck 					who,	\
1225*e9af4bc0SJohn Beck 					host == NULL ? "local" : host,	\
1226*e9af4bc0SJohn Beck 					which);	\
1227*e9af4bc0SJohn Beck 		}		\
1228*e9af4bc0SJohn Beck 		else if ((size_t)r >= sizeof(buf) - 1)	\
1229*e9af4bc0SJohn Beck 		{		\
1230*e9af4bc0SJohn Beck 			sm_strlcpy(buf, "BadCertificateTooLong", sizeof(buf)); \
1231*e9af4bc0SJohn Beck 			if (LogLevel > 7)	\
1232*e9af4bc0SJohn Beck 				sm_syslog(LOG_INFO, NOQID,	\
1233*e9af4bc0SJohn Beck 					"STARTTLS=%s, relay=%.100s, field=%s, status=CN too long",	\
1234*e9af4bc0SJohn Beck 					who,	\
1235*e9af4bc0SJohn Beck 					host == NULL ? "local" : host,	\
1236*e9af4bc0SJohn Beck 					which);	\
1237*e9af4bc0SJohn Beck 		}		\
1238*e9af4bc0SJohn Beck 		else if ((size_t)r > strlen(buf))	\
1239*e9af4bc0SJohn Beck 		{		\
1240*e9af4bc0SJohn Beck 			sm_strlcpy(buf, "BadCertificateContainsNUL",	\
1241*e9af4bc0SJohn Beck 				sizeof(buf));	\
1242*e9af4bc0SJohn Beck 			if (LogLevel > 7)	\
1243*e9af4bc0SJohn Beck 				sm_syslog(LOG_INFO, NOQID,	\
1244*e9af4bc0SJohn Beck 					"STARTTLS=%s, relay=%.100s, field=%s, status=CN contains NUL",	\
1245*e9af4bc0SJohn Beck 					who,	\
1246*e9af4bc0SJohn Beck 					host == NULL ? "local" : host,	\
1247*e9af4bc0SJohn Beck 					which);	\
1248*e9af4bc0SJohn Beck 		}		\
1249*e9af4bc0SJohn Beck 	} while (0)
1250*e9af4bc0SJohn Beck 
1251*e9af4bc0SJohn Beck 		r = X509_NAME_get_text_by_NID(subj, NID_commonName, buf,
1252*e9af4bc0SJohn Beck 			sizeof buf);
1253*e9af4bc0SJohn Beck 		CHECK_X509_NAME("cn_subject");
12547c478bd9Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cn_subject}"),
12557c478bd9Sstevel@tonic-gate 			 xtextify(buf, "<>\")"));
1256*e9af4bc0SJohn Beck 		r = X509_NAME_get_text_by_NID(issuer, NID_commonName, buf,
1257*e9af4bc0SJohn Beck 			sizeof buf);
1258*e9af4bc0SJohn Beck 		CHECK_X509_NAME("cn_issuer");
12597c478bd9Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cn_issuer}"),
12607c478bd9Sstevel@tonic-gate 			 xtextify(buf, "<>\")"));
12617c478bd9Sstevel@tonic-gate 		n = 0;
12627c478bd9Sstevel@tonic-gate 		if (X509_digest(cert, EVP_md5(), md, &n) != 0 && n > 0)
12637c478bd9Sstevel@tonic-gate 		{
12647c478bd9Sstevel@tonic-gate 			char md5h[EVP_MAX_MD_SIZE * 3];
12657c478bd9Sstevel@tonic-gate 			static const char hexcodes[] = "0123456789ABCDEF";
12667c478bd9Sstevel@tonic-gate 
12677c478bd9Sstevel@tonic-gate 			SM_ASSERT((n * 3) + 2 < sizeof(md5h));
12687c478bd9Sstevel@tonic-gate 			for (r = 0; r < (int) n; r++)
12697c478bd9Sstevel@tonic-gate 			{
12707c478bd9Sstevel@tonic-gate 				md5h[r * 3] = hexcodes[(md[r] & 0xf0) >> 4];
12717c478bd9Sstevel@tonic-gate 				md5h[(r * 3) + 1] = hexcodes[(md[r] & 0x0f)];
12727c478bd9Sstevel@tonic-gate 				md5h[(r * 3) + 2] = ':';
12737c478bd9Sstevel@tonic-gate 			}
12747c478bd9Sstevel@tonic-gate 			md5h[(n * 3) - 1] = '\0';
12757c478bd9Sstevel@tonic-gate 			macdefine(mac, A_TEMP, macid("{cert_md5}"), md5h);
12767c478bd9Sstevel@tonic-gate 		}
12777c478bd9Sstevel@tonic-gate 		else
12787c478bd9Sstevel@tonic-gate 			macdefine(mac, A_TEMP, macid("{cert_md5}"), "");
12797c478bd9Sstevel@tonic-gate 	}
12807c478bd9Sstevel@tonic-gate 	else
12817c478bd9Sstevel@tonic-gate 	{
12827c478bd9Sstevel@tonic-gate 		macdefine(mac, A_PERM, macid("{cert_subject}"), "");
12837c478bd9Sstevel@tonic-gate 		macdefine(mac, A_PERM, macid("{cert_issuer}"), "");
12847c478bd9Sstevel@tonic-gate 		macdefine(mac, A_PERM, macid("{cn_subject}"), "");
12857c478bd9Sstevel@tonic-gate 		macdefine(mac, A_PERM, macid("{cn_issuer}"), "");
12867c478bd9Sstevel@tonic-gate 		macdefine(mac, A_TEMP, macid("{cert_md5}"), "");
12877c478bd9Sstevel@tonic-gate 	}
12887c478bd9Sstevel@tonic-gate 	switch (verifyok)
12897c478bd9Sstevel@tonic-gate 	{
12907c478bd9Sstevel@tonic-gate 	  case X509_V_OK:
12917c478bd9Sstevel@tonic-gate 		if (cert != NULL)
12927c478bd9Sstevel@tonic-gate 		{
12937c478bd9Sstevel@tonic-gate 			s = "OK";
12947c478bd9Sstevel@tonic-gate 			r = TLS_AUTH_OK;
12957c478bd9Sstevel@tonic-gate 		}
12967c478bd9Sstevel@tonic-gate 		else
12977c478bd9Sstevel@tonic-gate 		{
12987c478bd9Sstevel@tonic-gate 			s = certreq ? "NO" : "NOT",
12997c478bd9Sstevel@tonic-gate 			r = TLS_AUTH_NO;
13007c478bd9Sstevel@tonic-gate 		}
13017c478bd9Sstevel@tonic-gate 		break;
13027c478bd9Sstevel@tonic-gate 	  default:
13037c478bd9Sstevel@tonic-gate 		s = "FAIL";
13047c478bd9Sstevel@tonic-gate 		r = TLS_AUTH_FAIL;
13057c478bd9Sstevel@tonic-gate 		break;
13067c478bd9Sstevel@tonic-gate 	}
13077c478bd9Sstevel@tonic-gate 	macdefine(mac, A_PERM, macid("{verify}"), s);
13087c478bd9Sstevel@tonic-gate 	if (cert != NULL)
13097c478bd9Sstevel@tonic-gate 		X509_free(cert);
13107c478bd9Sstevel@tonic-gate 
13117c478bd9Sstevel@tonic-gate 	/* do some logging */
13127c478bd9Sstevel@tonic-gate 	if (LogLevel > 8)
13137c478bd9Sstevel@tonic-gate 	{
13147c478bd9Sstevel@tonic-gate 		char *vers, *s1, *s2, *cbits, *algbits;
13157c478bd9Sstevel@tonic-gate 
13167c478bd9Sstevel@tonic-gate 		vers = macget(mac, macid("{tls_version}"));
13177c478bd9Sstevel@tonic-gate 		cbits = macget(mac, macid("{cipher_bits}"));
13187c478bd9Sstevel@tonic-gate 		algbits = macget(mac, macid("{alg_bits}"));
13197c478bd9Sstevel@tonic-gate 		s1 = macget(mac, macid("{verify}"));
13207c478bd9Sstevel@tonic-gate 		s2 = macget(mac, macid("{cipher}"));
13217c478bd9Sstevel@tonic-gate 
13227c478bd9Sstevel@tonic-gate 		/* XXX: maybe cut off ident info? */
13237c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_INFO, NOQID,
13247c478bd9Sstevel@tonic-gate 			  "STARTTLS=%s, relay=%.100s, version=%.16s, verify=%.16s, cipher=%.64s, bits=%.6s/%.6s",
13257c478bd9Sstevel@tonic-gate 			  who,
13267c478bd9Sstevel@tonic-gate 			  host == NULL ? "local" : host,
13277c478bd9Sstevel@tonic-gate 			  vers, s1, s2, /* sm_snprintf() can deal with NULL */
13287c478bd9Sstevel@tonic-gate 			  algbits == NULL ? "0" : algbits,
13297c478bd9Sstevel@tonic-gate 			  cbits == NULL ? "0" : cbits);
13307c478bd9Sstevel@tonic-gate 		if (LogLevel > 11)
13317c478bd9Sstevel@tonic-gate 		{
13327c478bd9Sstevel@tonic-gate 			/*
13337c478bd9Sstevel@tonic-gate 			**  Maybe run xuntextify on the strings?
13347c478bd9Sstevel@tonic-gate 			**  That is easier to read but makes it maybe a bit
13357c478bd9Sstevel@tonic-gate 			**  more complicated to figure out the right values
13367c478bd9Sstevel@tonic-gate 			**  for the access map...
13377c478bd9Sstevel@tonic-gate 			*/
13387c478bd9Sstevel@tonic-gate 
13397c478bd9Sstevel@tonic-gate 			s1 = macget(mac, macid("{cert_subject}"));
13407c478bd9Sstevel@tonic-gate 			s2 = macget(mac, macid("{cert_issuer}"));
13417c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_INFO, NOQID,
13427c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, cert-subject=%.256s, cert-issuer=%.256s, verifymsg=%s",
13437c478bd9Sstevel@tonic-gate 				  who, s1, s2,
13447c478bd9Sstevel@tonic-gate 				  X509_verify_cert_error_string(verifyok));
13457c478bd9Sstevel@tonic-gate 		}
13467c478bd9Sstevel@tonic-gate 	}
13477c478bd9Sstevel@tonic-gate 	return r;
13487c478bd9Sstevel@tonic-gate }
13497c478bd9Sstevel@tonic-gate /*
13507c478bd9Sstevel@tonic-gate **  ENDTLS -- shutdown secure connection
13517c478bd9Sstevel@tonic-gate **
13527c478bd9Sstevel@tonic-gate **	Parameters:
13537c478bd9Sstevel@tonic-gate **		ssl -- SSL connection information.
13547c478bd9Sstevel@tonic-gate **		side -- server/client (for logging).
13557c478bd9Sstevel@tonic-gate **
13567c478bd9Sstevel@tonic-gate **	Returns:
13577c478bd9Sstevel@tonic-gate **		success? (EX_* code)
13587c478bd9Sstevel@tonic-gate */
13597c478bd9Sstevel@tonic-gate 
13607c478bd9Sstevel@tonic-gate int
13617c478bd9Sstevel@tonic-gate endtls(ssl, side)
13627c478bd9Sstevel@tonic-gate 	SSL *ssl;
13637c478bd9Sstevel@tonic-gate 	char *side;
13647c478bd9Sstevel@tonic-gate {
13657c478bd9Sstevel@tonic-gate 	int ret = EX_OK;
13667c478bd9Sstevel@tonic-gate 
13677c478bd9Sstevel@tonic-gate 	if (ssl != NULL)
13687c478bd9Sstevel@tonic-gate 	{
13697c478bd9Sstevel@tonic-gate 		int r;
13707c478bd9Sstevel@tonic-gate 
13717c478bd9Sstevel@tonic-gate 		if ((r = SSL_shutdown(ssl)) < 0)
13727c478bd9Sstevel@tonic-gate 		{
13737c478bd9Sstevel@tonic-gate 			if (LogLevel > 11)
13747c478bd9Sstevel@tonic-gate 			{
13757c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
13767c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, SSL_shutdown failed: %d",
13777c478bd9Sstevel@tonic-gate 					  side, r);
13787c478bd9Sstevel@tonic-gate 				tlslogerr(side);
13797c478bd9Sstevel@tonic-gate 			}
13807c478bd9Sstevel@tonic-gate 			ret = EX_SOFTWARE;
13817c478bd9Sstevel@tonic-gate 		}
13827c478bd9Sstevel@tonic-gate # if !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER > 0x0090602fL
13837c478bd9Sstevel@tonic-gate 
13847c478bd9Sstevel@tonic-gate 		/*
13857c478bd9Sstevel@tonic-gate 		**  Bug in OpenSSL (at least up to 0.9.6b):
13867c478bd9Sstevel@tonic-gate 		**  From: Lutz.Jaenicke@aet.TU-Cottbus.DE
13877c478bd9Sstevel@tonic-gate 		**  Message-ID: <20010723152244.A13122@serv01.aet.tu-cottbus.de>
13887c478bd9Sstevel@tonic-gate 		**  To: openssl-users@openssl.org
13897c478bd9Sstevel@tonic-gate 		**  Subject: Re: SSL_shutdown() woes (fwd)
13907c478bd9Sstevel@tonic-gate 		**
13917c478bd9Sstevel@tonic-gate 		**  The side sending the shutdown alert first will
13927c478bd9Sstevel@tonic-gate 		**  not care about the answer of the peer but will
13937c478bd9Sstevel@tonic-gate 		**  immediately return with a return value of "0"
13947c478bd9Sstevel@tonic-gate 		**  (ssl/s3_lib.c:ssl3_shutdown()). SSL_get_error will evaluate
13957c478bd9Sstevel@tonic-gate 		**  the value of "0" and as the shutdown alert of the peer was
13967c478bd9Sstevel@tonic-gate 		**  not received (actually, the program did not even wait for
13977c478bd9Sstevel@tonic-gate 		**  the answer), an SSL_ERROR_SYSCALL is flagged, because this
13987c478bd9Sstevel@tonic-gate 		**  is the default rule in case everything else does not apply.
13997c478bd9Sstevel@tonic-gate 		**
14007c478bd9Sstevel@tonic-gate 		**  For your server the problem is different, because it
14017c478bd9Sstevel@tonic-gate 		**  receives the shutdown first (setting SSL_RECEIVED_SHUTDOWN),
14027c478bd9Sstevel@tonic-gate 		**  then sends its response (SSL_SENT_SHUTDOWN), so for the
14037c478bd9Sstevel@tonic-gate 		**  server the shutdown was successfull.
14047c478bd9Sstevel@tonic-gate 		**
14057c478bd9Sstevel@tonic-gate 		**  As is by know, you would have to call SSL_shutdown() once
14067c478bd9Sstevel@tonic-gate 		**  and ignore an SSL_ERROR_SYSCALL returned. Then call
14077c478bd9Sstevel@tonic-gate 		**  SSL_shutdown() again to actually get the server's response.
14087c478bd9Sstevel@tonic-gate 		**
14097c478bd9Sstevel@tonic-gate 		**  In the last discussion, Bodo Moeller concluded that a
14107c478bd9Sstevel@tonic-gate 		**  rewrite of the shutdown code would be necessary, but
14117c478bd9Sstevel@tonic-gate 		**  probably with another API, as the change would not be
14127c478bd9Sstevel@tonic-gate 		**  compatible to the way it is now.  Things do not become
14137c478bd9Sstevel@tonic-gate 		**  easier as other programs do not follow the shutdown
14147c478bd9Sstevel@tonic-gate 		**  guidelines anyway, so that a lot error conditions and
14157c478bd9Sstevel@tonic-gate 		**  compitibility issues would have to be caught.
14167c478bd9Sstevel@tonic-gate 		**
14177c478bd9Sstevel@tonic-gate 		**  For now the recommondation is to ignore the error message.
14187c478bd9Sstevel@tonic-gate 		*/
14197c478bd9Sstevel@tonic-gate 
14207c478bd9Sstevel@tonic-gate 		else if (r == 0)
14217c478bd9Sstevel@tonic-gate 		{
14227c478bd9Sstevel@tonic-gate 			if (LogLevel > 15)
14237c478bd9Sstevel@tonic-gate 			{
14247c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
14257c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, SSL_shutdown not done",
14267c478bd9Sstevel@tonic-gate 					  side);
14277c478bd9Sstevel@tonic-gate 				tlslogerr(side);
14287c478bd9Sstevel@tonic-gate 			}
14297c478bd9Sstevel@tonic-gate 			ret = EX_SOFTWARE;
14307c478bd9Sstevel@tonic-gate 		}
14317c478bd9Sstevel@tonic-gate # endif /* !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER > 0x0090602fL */
14327c478bd9Sstevel@tonic-gate 		SSL_free(ssl);
14337c478bd9Sstevel@tonic-gate 		ssl = NULL;
14347c478bd9Sstevel@tonic-gate 	}
14357c478bd9Sstevel@tonic-gate 	return ret;
14367c478bd9Sstevel@tonic-gate }
14377c478bd9Sstevel@tonic-gate 
14387c478bd9Sstevel@tonic-gate # if !TLS_NO_RSA
14397c478bd9Sstevel@tonic-gate /*
14407c478bd9Sstevel@tonic-gate **  TMP_RSA_KEY -- return temporary RSA key
14417c478bd9Sstevel@tonic-gate **
14427c478bd9Sstevel@tonic-gate **	Parameters:
14437c478bd9Sstevel@tonic-gate **		s -- TLS connection structure
14447c478bd9Sstevel@tonic-gate **		export --
14457c478bd9Sstevel@tonic-gate **		keylength --
14467c478bd9Sstevel@tonic-gate **
14477c478bd9Sstevel@tonic-gate **	Returns:
14487c478bd9Sstevel@tonic-gate **		temporary RSA key.
14497c478bd9Sstevel@tonic-gate */
14507c478bd9Sstevel@tonic-gate 
14517c478bd9Sstevel@tonic-gate #   ifndef MAX_RSA_TMP_CNT
14527c478bd9Sstevel@tonic-gate #    define MAX_RSA_TMP_CNT	1000	/* XXX better value? */
14537c478bd9Sstevel@tonic-gate #   endif /* ! MAX_RSA_TMP_CNT */
14547c478bd9Sstevel@tonic-gate 
14557c478bd9Sstevel@tonic-gate /* ARGUSED0 */
14567c478bd9Sstevel@tonic-gate static RSA *
14577c478bd9Sstevel@tonic-gate tmp_rsa_key(s, export, keylength)
14587c478bd9Sstevel@tonic-gate 	SSL *s;
14597c478bd9Sstevel@tonic-gate 	int export;
14607c478bd9Sstevel@tonic-gate 	int keylength;
14617c478bd9Sstevel@tonic-gate {
14627c478bd9Sstevel@tonic-gate #   if SM_CONF_SHM
14637c478bd9Sstevel@tonic-gate 	extern int ShmId;
14647c478bd9Sstevel@tonic-gate 	extern int *PRSATmpCnt;
14657c478bd9Sstevel@tonic-gate 
14667c478bd9Sstevel@tonic-gate 	if (ShmId != SM_SHM_NO_ID && rsa_tmp != NULL &&
14677c478bd9Sstevel@tonic-gate 	    ++(*PRSATmpCnt) < MAX_RSA_TMP_CNT)
14687c478bd9Sstevel@tonic-gate 		return rsa_tmp;
14697c478bd9Sstevel@tonic-gate #   endif /* SM_CONF_SHM */
14707c478bd9Sstevel@tonic-gate 
14717c478bd9Sstevel@tonic-gate 	if (rsa_tmp != NULL)
14727c478bd9Sstevel@tonic-gate 		RSA_free(rsa_tmp);
14737c478bd9Sstevel@tonic-gate 	rsa_tmp = RSA_generate_key(RSA_KEYLENGTH, RSA_F4, NULL, NULL);
14747c478bd9Sstevel@tonic-gate 	if (rsa_tmp == NULL)
14757c478bd9Sstevel@tonic-gate 	{
14767c478bd9Sstevel@tonic-gate 		if (LogLevel > 0)
14777c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_ERR, NOQID,
14787c478bd9Sstevel@tonic-gate 				  "STARTTLS=server, tmp_rsa_key: RSA_generate_key failed!");
14797c478bd9Sstevel@tonic-gate 	}
14807c478bd9Sstevel@tonic-gate 	else
14817c478bd9Sstevel@tonic-gate 	{
14827c478bd9Sstevel@tonic-gate #   if SM_CONF_SHM
14837c478bd9Sstevel@tonic-gate #    if 0
14847c478bd9Sstevel@tonic-gate 		/*
14857c478bd9Sstevel@tonic-gate 		**  XXX we can't (yet) share the new key...
14867c478bd9Sstevel@tonic-gate 		**	The RSA structure contains pointers hence it can't be
14877c478bd9Sstevel@tonic-gate 		**	easily kept in shared memory.  It must be transformed
14887c478bd9Sstevel@tonic-gate 		**	into a continous memory region first, then stored,
14897c478bd9Sstevel@tonic-gate 		**	and later read out again (each time re-transformed).
14907c478bd9Sstevel@tonic-gate 		*/
14917c478bd9Sstevel@tonic-gate 
14927c478bd9Sstevel@tonic-gate 		if (ShmId != SM_SHM_NO_ID)
14937c478bd9Sstevel@tonic-gate 			*PRSATmpCnt = 0;
14947c478bd9Sstevel@tonic-gate #    endif /* 0 */
14957c478bd9Sstevel@tonic-gate #   endif /* SM_CONF_SHM */
14967c478bd9Sstevel@tonic-gate 		if (LogLevel > 9)
14977c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_ERR, NOQID,
14987c478bd9Sstevel@tonic-gate 				  "STARTTLS=server, tmp_rsa_key: new temp RSA key");
14997c478bd9Sstevel@tonic-gate 	}
15007c478bd9Sstevel@tonic-gate 	return rsa_tmp;
15017c478bd9Sstevel@tonic-gate }
15027c478bd9Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
15037c478bd9Sstevel@tonic-gate /*
15047c478bd9Sstevel@tonic-gate **  APPS_SSL_INFO_CB -- info callback for TLS connections
15057c478bd9Sstevel@tonic-gate **
15067c478bd9Sstevel@tonic-gate **	Parameters:
15077c478bd9Sstevel@tonic-gate **		s -- TLS connection structure
15087c478bd9Sstevel@tonic-gate **		where -- state in handshake
15097c478bd9Sstevel@tonic-gate **		ret -- return code of last operation
15107c478bd9Sstevel@tonic-gate **
15117c478bd9Sstevel@tonic-gate **	Returns:
15127c478bd9Sstevel@tonic-gate **		none.
15137c478bd9Sstevel@tonic-gate */
15147c478bd9Sstevel@tonic-gate 
15157c478bd9Sstevel@tonic-gate static void
15167c478bd9Sstevel@tonic-gate apps_ssl_info_cb(s, where, ret)
15177c478bd9Sstevel@tonic-gate 	CONST097 SSL *s;
15187c478bd9Sstevel@tonic-gate 	int where;
15197c478bd9Sstevel@tonic-gate 	int ret;
15207c478bd9Sstevel@tonic-gate {
15217c478bd9Sstevel@tonic-gate 	int w;
15227c478bd9Sstevel@tonic-gate 	char *str;
15237c478bd9Sstevel@tonic-gate 	BIO *bio_err = NULL;
15247c478bd9Sstevel@tonic-gate 
15257c478bd9Sstevel@tonic-gate 	if (LogLevel > 14)
15267c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_INFO, NOQID,
15277c478bd9Sstevel@tonic-gate 			  "STARTTLS: info_callback where=0x%x, ret=%d",
15287c478bd9Sstevel@tonic-gate 			  where, ret);
15297c478bd9Sstevel@tonic-gate 
15307c478bd9Sstevel@tonic-gate 	w = where & ~SSL_ST_MASK;
15317c478bd9Sstevel@tonic-gate 	if (bio_err == NULL)
15327c478bd9Sstevel@tonic-gate 		bio_err = BIO_new_fp(stderr, BIO_NOCLOSE);
15337c478bd9Sstevel@tonic-gate 
15347c478bd9Sstevel@tonic-gate 	if (bitset(SSL_ST_CONNECT, w))
15357c478bd9Sstevel@tonic-gate 		str = "SSL_connect";
15367c478bd9Sstevel@tonic-gate 	else if (bitset(SSL_ST_ACCEPT, w))
15377c478bd9Sstevel@tonic-gate 		str = "SSL_accept";
15387c478bd9Sstevel@tonic-gate 	else
15397c478bd9Sstevel@tonic-gate 		str = "undefined";
15407c478bd9Sstevel@tonic-gate 
15417c478bd9Sstevel@tonic-gate 	if (bitset(SSL_CB_LOOP, where))
15427c478bd9Sstevel@tonic-gate 	{
15437c478bd9Sstevel@tonic-gate 		if (LogLevel > 12)
15447c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_NOTICE, NOQID,
15457c478bd9Sstevel@tonic-gate 				"STARTTLS: %s:%s",
15467c478bd9Sstevel@tonic-gate 				str, SSL_state_string_long(s));
15477c478bd9Sstevel@tonic-gate 	}
15487c478bd9Sstevel@tonic-gate 	else if (bitset(SSL_CB_ALERT, where))
15497c478bd9Sstevel@tonic-gate 	{
15507c478bd9Sstevel@tonic-gate 		str = bitset(SSL_CB_READ, where) ? "read" : "write";
15517c478bd9Sstevel@tonic-gate 		if (LogLevel > 12)
15527c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_NOTICE, NOQID,
15537c478bd9Sstevel@tonic-gate 				"STARTTLS: SSL3 alert %s:%s:%s",
15547c478bd9Sstevel@tonic-gate 				str, SSL_alert_type_string_long(ret),
15557c478bd9Sstevel@tonic-gate 				SSL_alert_desc_string_long(ret));
15567c478bd9Sstevel@tonic-gate 	}
15577c478bd9Sstevel@tonic-gate 	else if (bitset(SSL_CB_EXIT, where))
15587c478bd9Sstevel@tonic-gate 	{
15597c478bd9Sstevel@tonic-gate 		if (ret == 0)
15607c478bd9Sstevel@tonic-gate 		{
15617c478bd9Sstevel@tonic-gate 			if (LogLevel > 7)
15627c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
15637c478bd9Sstevel@tonic-gate 					"STARTTLS: %s:failed in %s",
15647c478bd9Sstevel@tonic-gate 					str, SSL_state_string_long(s));
15657c478bd9Sstevel@tonic-gate 		}
15667c478bd9Sstevel@tonic-gate 		else if (ret < 0)
15677c478bd9Sstevel@tonic-gate 		{
15687c478bd9Sstevel@tonic-gate 			if (LogLevel > 7)
15697c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
15707c478bd9Sstevel@tonic-gate 					"STARTTLS: %s:error in %s",
15717c478bd9Sstevel@tonic-gate 					str, SSL_state_string_long(s));
15727c478bd9Sstevel@tonic-gate 		}
15737c478bd9Sstevel@tonic-gate 	}
15747c478bd9Sstevel@tonic-gate }
15757c478bd9Sstevel@tonic-gate /*
15767c478bd9Sstevel@tonic-gate **  TLS_VERIFY_LOG -- log verify error for TLS certificates
15777c478bd9Sstevel@tonic-gate **
15787c478bd9Sstevel@tonic-gate **	Parameters:
15797c478bd9Sstevel@tonic-gate **		ok -- verify ok?
15807c478bd9Sstevel@tonic-gate **		ctx -- x509 context
15817c478bd9Sstevel@tonic-gate **
15827c478bd9Sstevel@tonic-gate **	Returns:
15837c478bd9Sstevel@tonic-gate **		0 -- fatal error
15847c478bd9Sstevel@tonic-gate **		1 -- ok
15857c478bd9Sstevel@tonic-gate */
15867c478bd9Sstevel@tonic-gate 
15877c478bd9Sstevel@tonic-gate static int
15887c478bd9Sstevel@tonic-gate tls_verify_log(ok, ctx, name)
15897c478bd9Sstevel@tonic-gate 	int ok;
15907c478bd9Sstevel@tonic-gate 	X509_STORE_CTX *ctx;
15917c478bd9Sstevel@tonic-gate 	char *name;
15927c478bd9Sstevel@tonic-gate {
15937c478bd9Sstevel@tonic-gate 	SSL *ssl;
15947c478bd9Sstevel@tonic-gate 	X509 *cert;
15957c478bd9Sstevel@tonic-gate 	int reason, depth;
15967c478bd9Sstevel@tonic-gate 	char buf[512];
15977c478bd9Sstevel@tonic-gate 
15987c478bd9Sstevel@tonic-gate 	cert = X509_STORE_CTX_get_current_cert(ctx);
15997c478bd9Sstevel@tonic-gate 	reason = X509_STORE_CTX_get_error(ctx);
16007c478bd9Sstevel@tonic-gate 	depth = X509_STORE_CTX_get_error_depth(ctx);
16017c478bd9Sstevel@tonic-gate 	ssl = (SSL *) X509_STORE_CTX_get_ex_data(ctx,
16027c478bd9Sstevel@tonic-gate 			SSL_get_ex_data_X509_STORE_CTX_idx());
16037c478bd9Sstevel@tonic-gate 
16047c478bd9Sstevel@tonic-gate 	if (ssl == NULL)
16057c478bd9Sstevel@tonic-gate 	{
16067c478bd9Sstevel@tonic-gate 		/* internal error */
16077c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_ERR, NOQID,
16087c478bd9Sstevel@tonic-gate 			  "STARTTLS: internal error: tls_verify_cb: ssl == NULL");
16097c478bd9Sstevel@tonic-gate 		return 0;
16107c478bd9Sstevel@tonic-gate 	}
16117c478bd9Sstevel@tonic-gate 
1612058561cbSjbeck 	X509_NAME_oneline(X509_get_subject_name(cert), buf, sizeof(buf));
16137c478bd9Sstevel@tonic-gate 	sm_syslog(LOG_INFO, NOQID,
16147c478bd9Sstevel@tonic-gate 		  "STARTTLS: %s cert verify: depth=%d %s, state=%d, reason=%s",
16157c478bd9Sstevel@tonic-gate 		  name, depth, buf, ok, X509_verify_cert_error_string(reason));
16167c478bd9Sstevel@tonic-gate 	return 1;
16177c478bd9Sstevel@tonic-gate }
16187c478bd9Sstevel@tonic-gate 
16197c478bd9Sstevel@tonic-gate /*
16207c478bd9Sstevel@tonic-gate **  TLS_VERIFY_CB -- verify callback for TLS certificates
16217c478bd9Sstevel@tonic-gate **
16227c478bd9Sstevel@tonic-gate **	Parameters:
16237c478bd9Sstevel@tonic-gate **		ctx -- x509 context
16247c478bd9Sstevel@tonic-gate **
16257c478bd9Sstevel@tonic-gate **	Returns:
16267c478bd9Sstevel@tonic-gate **		accept connection?
16277c478bd9Sstevel@tonic-gate **		currently: always yes.
16287c478bd9Sstevel@tonic-gate */
16297c478bd9Sstevel@tonic-gate 
16307c478bd9Sstevel@tonic-gate static int
16317c478bd9Sstevel@tonic-gate #  if !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x00907000L
16327c478bd9Sstevel@tonic-gate tls_verify_cb(ctx)
16337c478bd9Sstevel@tonic-gate 	X509_STORE_CTX *ctx;
16347c478bd9Sstevel@tonic-gate #  else /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
16357c478bd9Sstevel@tonic-gate tls_verify_cb(ctx, unused)
16367c478bd9Sstevel@tonic-gate 	X509_STORE_CTX *ctx;
16377c478bd9Sstevel@tonic-gate 	void *unused;
16387c478bd9Sstevel@tonic-gate #  endif /* !defined() || OPENSSL_VERSION_NUMBER < 0x00907000L */
16397c478bd9Sstevel@tonic-gate {
16407c478bd9Sstevel@tonic-gate 	int ok;
16417c478bd9Sstevel@tonic-gate 
1642*e9af4bc0SJohn Beck 	/*
1643*e9af4bc0SJohn Beck 	**  man SSL_CTX_set_cert_verify_callback():
1644*e9af4bc0SJohn Beck 	**  callback should return 1 to indicate verification success
1645*e9af4bc0SJohn Beck 	**  and 0 to indicate verification failure.
1646*e9af4bc0SJohn Beck 	*/
1647*e9af4bc0SJohn Beck 
16487c478bd9Sstevel@tonic-gate 	ok = X509_verify_cert(ctx);
1649*e9af4bc0SJohn Beck 	if (ok <= 0)
16507c478bd9Sstevel@tonic-gate 	{
16517c478bd9Sstevel@tonic-gate 		if (LogLevel > 13)
16527c478bd9Sstevel@tonic-gate 			return tls_verify_log(ok, ctx, "TLS");
16537c478bd9Sstevel@tonic-gate 	}
1654*e9af4bc0SJohn Beck 	return 1;
16557c478bd9Sstevel@tonic-gate }
16567c478bd9Sstevel@tonic-gate /*
16577c478bd9Sstevel@tonic-gate **  TLSLOGERR -- log the errors from the TLS error stack
16587c478bd9Sstevel@tonic-gate **
16597c478bd9Sstevel@tonic-gate **	Parameters:
16607c478bd9Sstevel@tonic-gate **		who -- server/client (for logging).
16617c478bd9Sstevel@tonic-gate **
16627c478bd9Sstevel@tonic-gate **	Returns:
16637c478bd9Sstevel@tonic-gate **		none.
16647c478bd9Sstevel@tonic-gate */
16657c478bd9Sstevel@tonic-gate 
16667c478bd9Sstevel@tonic-gate void
16677c478bd9Sstevel@tonic-gate tlslogerr(who)
1668445f2479Sjbeck 	const char *who;
16697c478bd9Sstevel@tonic-gate {
16707c478bd9Sstevel@tonic-gate 	unsigned long l;
16717c478bd9Sstevel@tonic-gate 	int line, flags;
16727c478bd9Sstevel@tonic-gate 	unsigned long es;
16737c478bd9Sstevel@tonic-gate 	char *file, *data;
16747c478bd9Sstevel@tonic-gate 	char buf[256];
16757c478bd9Sstevel@tonic-gate #  define CP (const char **)
16767c478bd9Sstevel@tonic-gate 
16777c478bd9Sstevel@tonic-gate 	es = CRYPTO_thread_id();
16787c478bd9Sstevel@tonic-gate 	while ((l = ERR_get_error_line_data(CP &file, &line, CP &data, &flags))
16797c478bd9Sstevel@tonic-gate 		!= 0)
16807c478bd9Sstevel@tonic-gate 	{
16817c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_WARNING, NOQID,
16827c478bd9Sstevel@tonic-gate 			  "STARTTLS=%s: %lu:%s:%s:%d:%s", who, es,
16837c478bd9Sstevel@tonic-gate 			  ERR_error_string(l, buf),
16847c478bd9Sstevel@tonic-gate 			  file, line,
16857c478bd9Sstevel@tonic-gate 			  bitset(ERR_TXT_STRING, flags) ? data : "");
16867c478bd9Sstevel@tonic-gate 	}
16877c478bd9Sstevel@tonic-gate }
16887c478bd9Sstevel@tonic-gate 
16897c478bd9Sstevel@tonic-gate # if OPENSSL_VERSION_NUMBER > 0x00907000L
16907c478bd9Sstevel@tonic-gate /*
16917c478bd9Sstevel@tonic-gate **  X509_VERIFY_CB -- verify callback
16927c478bd9Sstevel@tonic-gate **
16937c478bd9Sstevel@tonic-gate **	Parameters:
16947c478bd9Sstevel@tonic-gate **		ctx -- x509 context
16957c478bd9Sstevel@tonic-gate **
16967c478bd9Sstevel@tonic-gate **	Returns:
16977c478bd9Sstevel@tonic-gate **		accept connection?
16987c478bd9Sstevel@tonic-gate **		currently: always yes.
16997c478bd9Sstevel@tonic-gate */
17007c478bd9Sstevel@tonic-gate 
17017c478bd9Sstevel@tonic-gate static int
17027c478bd9Sstevel@tonic-gate x509_verify_cb(ok, ctx)
17037c478bd9Sstevel@tonic-gate 	int ok;
17047c478bd9Sstevel@tonic-gate 	X509_STORE_CTX *ctx;
17057c478bd9Sstevel@tonic-gate {
17067c478bd9Sstevel@tonic-gate 	if (ok == 0)
17077c478bd9Sstevel@tonic-gate 	{
17087c478bd9Sstevel@tonic-gate 		if (LogLevel > 13)
17097c478bd9Sstevel@tonic-gate 			tls_verify_log(ok, ctx, "x509");
17107c478bd9Sstevel@tonic-gate 		if (ctx->error == X509_V_ERR_UNABLE_TO_GET_CRL)
17117c478bd9Sstevel@tonic-gate 		{
17127c478bd9Sstevel@tonic-gate 			ctx->error = 0;
17137c478bd9Sstevel@tonic-gate 			return 1;	/* override it */
17147c478bd9Sstevel@tonic-gate 		}
17157c478bd9Sstevel@tonic-gate 	}
17167c478bd9Sstevel@tonic-gate 	return ok;
17177c478bd9Sstevel@tonic-gate }
17187c478bd9Sstevel@tonic-gate # endif /* OPENSSL_VERSION_NUMBER > 0x00907000L */
17197c478bd9Sstevel@tonic-gate #endif /* STARTTLS */
1720