xref: /freebsd/contrib/sendmail/src/tls.h (revision d9a42747950146bf03cda7f6e25d219253f8a57a)
1 /*
2  * Copyright (c) 2015 Proofpoint, Inc. and its suppliers.
3  *	All rights reserved.
4  *
5  * By using this file, you agree to the terms and conditions set
6  * forth in the LICENSE file which can be found at the top level of
7  * the sendmail distribution.
8  */
9 
10 
11 #ifndef _TLS_H
12 # define _TLS_H 1
13 
14 #if STARTTLS
15 # include <openssl/ssl.h>
16 # if !TLS_NO_RSA
17 #  if _FFR_FIPSMODE
18 #   define RSA_KEYLENGTH	1024
19 #  else
20 #   define RSA_KEYLENGTH	512
21 #  endif
22 # endif /* !TLS_NO_RSA */
23 
24 # if OPENSSL_VERSION_NUMBER >= 0x10100000L && OPENSSL_VERSION_NUMBER < 0x20000000L
25 #  define TLS_version_num OpenSSL_version_num
26 # else
27 #  define TLS_version_num SSLeay
28 # endif
29 
30 #ifndef MTA_HAVE_TLSv1_3
31 /*
32 **  HACK: if openssl can disable TLSv1_3 then "assume" it supports all
33 **   related functions!
34 */
35 # ifdef SSL_OP_NO_TLSv1_3
36 #  define MTA_HAVE_TLSv1_3 1
37 # endif
38 #endif
39 
40 #ifdef _DEFINE
41 # define EXTERN
42 #else
43 # define EXTERN extern
44 #endif
45 
46 #if _FFR_TLS_EC && !defined(TLS_EC)
47 # define TLS_EC _FFR_TLS_EC
48 #endif
49 
50 #if DANE
51 extern int gettlsa __P((char *, char *, STAB **, unsigned long, unsigned int, unsigned int));
52 # define MAX_TLSA_RR	8
53 
54 # define DANE_VRFY_NONE	0	/* no TLSAs */
55 # define DANE_VRFY_OK		1	/* TLSA check was ok */
56 # define DANE_VRFY_FAIL	(-1)	/* TLSA check failed */
57 
58 /* return values for dane_tlsa_chk() */
59 # define TLSA_BOGUS	(-10)
60 # define TLSA_UNSUPP	(-1)
61 /* note: anything >= 0 is ok and refers to the hash algorithm */
62 # define TLSA_IS_KNOWN(r)	((r) >= 0)
63 # define TLSA_IS_VALID(r)	((r) >= TLSA_UNSUPP)
64 
65 struct dane_tlsa_S
66 {
67 	time_t		 dane_tlsa_exp;
68 	int		 dane_tlsa_n;
69 	int		 dane_tlsa_dnsrc;
70 	unsigned long	 dane_tlsa_flags;
71 	unsigned char	 dane_tlsa_usage[MAX_TLSA_RR];
72 	unsigned char	 dane_tlsa_selector[MAX_TLSA_RR];
73 	unsigned char	 dane_tlsa_digest[MAX_TLSA_RR];
74 	void		*dane_tlsa_rr[MAX_TLSA_RR];
75 	int		 dane_tlsa_len[MAX_TLSA_RR];
76 	char		*dane_tlsa_sni;
77 };
78 
79 # define TLSAFLNONE	0x00000000
80 /* Dane Mode */
81 # define TLSAFLALWAYS	0x00000001
82 # define TLSAFLSECURE	0x00000002
83 # define DANEMODE(fl)	((fl) & 0x3)
84 # define TLSAFLNOEXP	0x00000010	/* do not check expiration */
85 
86 # define TLSAFLADMX	0x00000100
87 # define TLSAFLADTLSA	0x00000200	/* currently unused */
88 
89 /* could be used to replace DNSRC */
90 # define TLSAFLTEMP	0x00001000
91 /* no TLSA? -- _n == 0 */
92 # define TLSAFLNOTLSA	0x00002000	/* currently unused */
93 
94 /*
95 **  Do not use this record, and do not look up new TLSA RRs because
96 **  the MX/host lookup was not secure.
97 **  XXX: to determine: interaction with DANE=always
98 */
99 
100 # define TLSAFLNOADMX	0x00010000
101 # define TLSAFLNOADTLSA	0x00020000	/* TLSA: no AD - for DANE=always? */
102 
103 # define TLSA_SET_FL(dane_tlsa, fl)	(dane_tlsa)->dane_tlsa_flags |= (fl)
104 # define TLSA_CLR_FL(dane_tlsa, fl)	(dane_tlsa)->dane_tlsa_flags &= ~(fl)
105 # define TLSA_IS_FL(dane_tlsa, fl)	((dane_tlsa)->dane_tlsa_flags & (fl))
106 # define TLSA_STORE_FL(fl)	((fl) >= TLSAFLTEMP)
107 
108 # define GETTLSA(host, pste, port)	gettlsa(host, NULL, pste, TLSAFLNONE, 0, port)
109 # define GETTLSANOX(host, pste, port)	gettlsa(host, NULL, pste, TLSAFLNOEXP, 0, port)
110 
111 /* values for DANE option and dane_vrfy_chk */
112 # define DANE_NEVER	TLSAFLNONE
113 # define DANE_ALWAYS	TLSAFLALWAYS		/* NOT documented, testing... */
114 # define DANE_SECURE	TLSAFLSECURE
115 # define CHK_DANE(dane)	((dane) != DANE_NEVER)
116 
117 /* temp fails? others? */
118 # define TLSA_RR_TEMPFAIL(dane_tlsa) (((dane_tlsa) != NULL) && (dane_tlsa)->dane_tlsa_dnsrc == TRY_AGAIN)
119 
120 #endif /* DANE */
121 
122 /*
123 **  TLS
124 */
125 
126 /* what to do in the TLS initialization */
127 #define TLS_I_NONE	0x00000000	/* no requirements... */
128 #define TLS_I_CERT_EX	0x00000001	/* cert must exist */
129 #define TLS_I_CERT_UNR	0x00000002	/* cert must be g/o unreadable */
130 #define TLS_I_KEY_EX	0x00000004	/* key must exist */
131 #define TLS_I_KEY_UNR	0x00000008	/* key must be g/o unreadable */
132 #define TLS_I_CERTP_EX	0x00000010	/* CA cert path must exist */
133 #define TLS_I_CERTP_UNR	0x00000020	/* CA cert path must be g/o unreadable */
134 #define TLS_I_CERTF_EX	0x00000040	/* CA cert file must exist */
135 #define TLS_I_CERTF_UNR	0x00000080	/* CA cert file must be g/o unreadable */
136 #define TLS_I_RSA_TMP	0x00000100	/* RSA TMP must be generated */
137 #define TLS_I_USE_KEY	0x00000200	/* private key must usable */
138 #define TLS_I_USE_CERT	0x00000400	/* certificate must be usable */
139 /*
140 not "read" anywhere
141 #define TLS_I_VRFY_PATH	0x00000800	* load verify path must succeed *
142 */
143 #define TLS_I_VRFY_LOC	0x00001000	/* load verify default must succeed */
144 #define TLS_I_CACHE	0x00002000	/* require cache */
145 #define TLS_I_TRY_DH	0x00004000	/* try DH certificate */
146 #define TLS_I_REQ_DH	0x00008000	/* require DH certificate */
147 #define TLS_I_DHPAR_EX	0x00010000	/* require DH parameters */
148 #define TLS_I_DHPAR_UNR	0x00020000	/* DH param. must be g/o unreadable */
149 #define TLS_I_DH512	0x00040000	/* generate 512bit DH param */
150 #define TLS_I_DH1024	0x00080000	/* generate 1024bit DH param */
151 #define TLS_I_DH2048	0x00100000	/* generate 2048bit DH param */
152 #define TLS_I_NO_VRFY	0x00200000	/* do not require authentication */
153 #define TLS_I_KEY_OUNR	0x00400000	/* Key must be other unreadable */
154 #define TLS_I_CRLF_EX	0x00800000	/* CRL file must exist */
155 #define TLS_I_CRLF_UNR	0x01000000	/* CRL file must be g/o unreadable */
156 #define TLS_I_DHFIXED	0x02000000	/* use fixed DH param */
157 
158 /* require server cert */
159 #define TLS_I_SRV_CERT	 (TLS_I_CERT_EX | TLS_I_KEY_EX | \
160 			  TLS_I_KEY_UNR | TLS_I_KEY_OUNR | \
161 			  TLS_I_CERTP_EX | TLS_I_CERTF_EX | \
162 			  TLS_I_USE_KEY | TLS_I_USE_CERT | TLS_I_CACHE)
163 
164 /* server requirements */
165 #define TLS_I_SRV	(TLS_I_SRV_CERT | TLS_I_RSA_TMP | /*TLS_I_VRFY_PATH|*/ \
166 			 TLS_I_VRFY_LOC | TLS_I_TRY_DH | TLS_I_CACHE)
167 
168 /* client requirements */
169 #define TLS_I_CLT	(TLS_I_KEY_UNR | TLS_I_KEY_OUNR)
170 
171 #define TLS_AUTH_OK	0
172 #define TLS_AUTH_NO	1
173 #define TLS_AUTH_FAIL	(-1)
174 
175 # ifndef TLS_VRFY_PER_CTX
176 #  define TLS_VRFY_PER_CTX 1
177 # endif
178 
179 #define SM_SSL_FREE(ssl)			\
180 	do {					\
181 		if (ssl != NULL)		\
182 		{				\
183 			SSL_free(ssl);		\
184 			ssl = NULL;		\
185 		}				\
186 	} while (0)
187 
188 /* functions */
189 extern int	endtls __P((SSL **, const char *));
190 extern int	get_tls_se_features __P((ENVELOPE *, SSL *, tlsi_ctx_T *, bool));
191 extern int	init_tls_library __P((bool _fipsmode));
192 extern bool	inittls __P((SSL_CTX **, unsigned long, unsigned long, bool, char *, char *, char *, char *, char *));
193 extern bool	initclttls __P((bool));
194 extern bool	initsrvtls __P((bool));
195 extern bool	load_certkey __P((SSL *, bool, char *, char *));
196 /* extern bool	load_crlpath __P((SSL_CTX *, bool , char *)); */
197 extern void	setclttls __P((bool));
198 extern int	tls_get_info __P((SSL *, bool, char *, MACROS_T *, bool));
199 extern void	tlslogerr __P((int, int, const char *));
200 extern void	tls_set_verify __P((SSL_CTX *, SSL *, bool));
201 # if DANE
202 extern int dane_tlsa_chk __P((const char *, int, const char *, bool));
203 extern int dane_tlsa_clr __P((dane_tlsa_P));
204 extern int dane_tlsa_free __P((dane_tlsa_P));
205 # endif
206 
207 EXTERN char	*CACertPath;	/* path to CA certificates (dir. with hashes) */
208 EXTERN char	*CACertFile;	/* file with CA certificate */
209 #if _FFR_CLIENTCA
210 EXTERN char	*CltCACertPath;	/* path to CA certificates (dir. with hashes) */
211 EXTERN char	*CltCACertFile;	/* file with CA certificate */
212 #endif
213 EXTERN char	*CltCertFile;	/* file with client certificate */
214 EXTERN char	*CltKeyFile;	/* file with client private key */
215 EXTERN char	*CipherList;	/* list of ciphers */
216 #if MTA_HAVE_TLSv1_3
217 EXTERN char	*CipherSuites;	/* cipher suites */
218 #endif
219 EXTERN char	*CertFingerprintAlgorithm;	/* name of fingerprint alg */
220 EXTERN const EVP_MD	*EVP_digest;	/* digest for cert fp */
221 EXTERN char	*DHParams;	/* file with DH parameters */
222 EXTERN char	*RandFile;	/* source of random data */
223 EXTERN char	*SrvCertFile;	/* file with server certificate */
224 EXTERN char	*SrvKeyFile;	/* file with server private key */
225 EXTERN char	*CRLFile;	/* file CRLs */
226 EXTERN char	*CRLPath;	/* path to CRLs (dir. with hashes) */
227 EXTERN unsigned long	TLS_Srv_Opts;	/* TLS server options */
228 EXTERN unsigned long	Srv_SSL_Options, Clt_SSL_Options; /* SSL options */
229 EXTERN bool	TLSFallbacktoClear;
230 
231 EXTERN char	*SSLEngine;
232 EXTERN char	*SSLEnginePath;
233 EXTERN bool	SSLEngineprefork;
234 
235 # if USE_OPENSSL_ENGINE
236 #define TLS_set_engine(id, prefork) SSL_set_engine(id)
237 # else
238 #  if !defined(OPENSSL_NO_ENGINE)
239 int TLS_set_engine __P((const char *, bool));
240 #  else
241 #define TLS_set_engine(id, prefork)	1
242 #  endif
243 # endif
244 
245 extern int	set_tls_rd_tmo __P((int));
246 extern int data2hex __P((unsigned char *, int, unsigned char *, int));
247 # if DANE
248 extern int pubkey_fp __P((X509 *, const char*, char **));
249 extern dane_tlsa_P dane_get_tlsa __P((dane_vrfy_ctx_P));
250 # endif
251 
252 #else /* STARTTLS */
253 # define set_tls_rd_tmo(rd_tmo)	0
254 #endif /* STARTTLS */
255 #undef EXTERN
256 #endif /* ! _TLS_H */
257