xref: /titanic_51/usr/src/lib/libpkg/common/security.c (revision c3a558e7c77127215b010652905be7916ec5a080)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 
28 /*
29  * Module: security.c
30  * Description:
31  *	Module for handling certificates and various
32  *	utilities to access their data.
33  */
34 
35 #include <stdio.h>
36 #include <string.h>
37 #include <errno.h>
38 #include <ctype.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <limits.h>
42 #include <pkgstrct.h>
43 #include <pkginfo.h>
44 #include <locale.h>
45 #include <libintl.h>
46 #include <unistd.h>
47 #include <stdlib.h>
48 
49 #include <openssl/bio.h>
50 #include <openssl/pkcs12.h>
51 #include <openssl/pkcs7.h>
52 #include <openssl/x509.h>
53 #include <openssl/err.h>
54 #include <openssl/ssl.h>
55 #include "pkgerr.h"
56 #include "pkglib.h"
57 #include "pkglibmsgs.h"
58 #include "pkglocale.h"
59 #include "p12lib.h"
60 
61 /* length of allowable passwords */
62 #define	MAX_PASSLEN		128
63 
64 /*
65  * Name:	init_security
66  * Description:	Initializes structures, libraries, for security operations
67  * Arguments:	none
68  * Returns:	0 if we couldn't initialize, non-zero otherwise
69  */
70 void
71 sec_init(void)
72 {
73 	OpenSSL_add_all_algorithms();
74 	SSL_load_error_strings();
75 	ERR_load_SUNW_strings();
76 	(void) SSL_library_init();
77 }
78 
79 /*
80  * get_cert_chain - Builds a chain of certificates, from a given
81  * user certificate to a trusted certificate.
82  *
83  * Arguments:
84  * err - Error object to add errors to
85  * cert - User cert to start with
86  * cas - Trusted certs to use as trust anchors
87  * chain - The resulting chain of certs (in the form of an
88  * ordered set) is placed here.
89  *
90  * Returns:
91  *   0 - Success - chain is stored in 'chain'.
92  * non-zero - Failure, errors recorded in err
93  */
94 int
95 get_cert_chain(PKG_ERR *err, X509 *cert, STACK_OF(X509) *clcerts,
96     STACK_OF(X509) *cas, STACK_OF(X509) **chain)
97 {
98 	X509_STORE_CTX	*store_ctx = NULL;
99 	X509_STORE 	*ca_store = NULL;
100 	X509		*ca_cert = NULL;
101 	int i;
102 	int ret = 0;
103 
104 	if ((ca_store = X509_STORE_new()) == NULL) {
105 		pkgerr_add(err, PKGERR_NOMEM,
106 		    gettext(ERR_MEM));
107 		ret = 1;
108 		goto cleanup;
109 	}
110 
111 	/* add all ca certs into the store */
112 	for (i = 0; i < sk_X509_num(cas); i++) {
113 		/* LINTED pointer cast may result in improper alignment */
114 		ca_cert = sk_X509_value(cas, i);
115 		if (X509_STORE_add_cert(ca_store, ca_cert) == 0) {
116 			pkgerr_add(err, PKGERR_NOMEM, gettext(ERR_MEM));
117 			ret = 1;
118 			goto cleanup;
119 		}
120 	}
121 
122 	/* initialize context object used during the chain resolution */
123 
124 	if ((store_ctx = X509_STORE_CTX_new()) == NULL) {
125 		pkgerr_add(err, PKGERR_NOMEM, gettext(ERR_MEM));
126 		ret = 1;
127 		goto cleanup;
128 	}
129 
130 	(void) X509_STORE_CTX_init(store_ctx, ca_store, cert, clcerts);
131 	/* attempt to verify the cert, which builds the cert chain */
132 	if (X509_verify_cert(store_ctx) <= 0) {
133 		pkgerr_add(err, PKGERR_CHAIN,
134 		    gettext(ERR_CERTCHAIN),
135 		    get_subject_display_name(cert),
136 		    X509_verify_cert_error_string(store_ctx->error));
137 		ret = 1;
138 		goto cleanup;
139 	}
140 	*chain = X509_STORE_CTX_get1_chain(store_ctx);
141 
142 cleanup:
143 	if (ca_store != NULL)
144 		(void) X509_STORE_free(ca_store);
145 	if (store_ctx != NULL) {
146 		(void) X509_STORE_CTX_cleanup(store_ctx);
147 		(void) X509_STORE_CTX_free(store_ctx);
148 	}
149 
150 	return (ret);
151 }
152 
153 /*
154  * Name:		get_subject_name
155  * Description:	Retrieves a name used for identifying a certificate's subject.
156  *
157  * Arguments:	cert - The certificate to get the name from
158  *
159  * Returns :	A static buffer containing the common name (CN) of the
160  * 		subject of the cert.
161  *
162  *		if the CN is not available, returns a string with the entire
163  * X509 distinguished name.
164  */
165 char
166 *get_subject_display_name(X509 *cert)
167 {
168 
169 	X509_NAME	*xname;
170 	static char	sname[ATTR_MAX];
171 
172 	xname = X509_get_subject_name(cert);
173 	if (X509_NAME_get_text_by_NID(xname,
174 	    NID_commonName, sname,
175 	    ATTR_MAX) <= 0) {
176 		(void) strncpy(sname,
177 		    X509_NAME_oneline(xname,
178 			NULL, 0), ATTR_MAX);
179 		sname[ATTR_MAX - 1] = '\0';
180 	}
181 	return (sname);
182 }
183 
184 /*
185  * Name:		get_display_name
186  * Description:	Retrieves a name used for identifying a certificate's issuer.
187  *
188  * Arguments:	cert - The certificate to get the name from
189  *
190  * Returns :	A static buffer containing the common name (CN)
191  *		of the issuer of the cert.
192  *
193  *		if the CN is not available, returns a string with the entire
194  *		X509 distinguished name.
195  */
196 char
197 *get_issuer_display_name(X509 *cert)
198 {
199 
200 	X509_NAME	*xname;
201 	static char	sname[ATTR_MAX];
202 
203 	xname = X509_get_issuer_name(cert);
204 	if (X509_NAME_get_text_by_NID(xname,
205 	    NID_commonName, sname,
206 	    ATTR_MAX) <= 0) {
207 		(void) strncpy(sname,
208 		    X509_NAME_oneline(xname,
209 			NULL, 0), ATTR_MAX);
210 		sname[ATTR_MAX - 1] = '\0';
211 	}
212 	return (sname);
213 }
214 
215 
216 /*
217  * Name:		get_serial_num
218  * Description:	Retrieves the serial number of an X509 cert
219  *
220  * Arguments:	cert - The certificate to get the data from
221  *
222  * Returns :	A static buffer containing the serial number
223  *		of the cert
224  *
225  *		if the SN is not available, returns NULL
226  */
227 char
228 *get_serial_num(X509 *cert)
229 {
230 	static char	 sn_str[ATTR_MAX];
231 	ASN1_INTEGER	*sn;
232 
233 	if ((sn = X509_get_serialNumber(cert)) != 0) {
234 		return (NULL);
235 	} else {
236 		(void) snprintf(sn_str, ATTR_MAX, "%ld",
237 		    ASN1_INTEGER_get(sn));
238 	}
239 
240 	return (sn_str);
241 }
242 
243 /*
244  * Name:		get_fingerprint
245  * Description:	Generates a fingerprint string given
246  *		a digest algorithm with which to calculate
247  *		the fingerprint
248  *
249  * Arguments:	cert - The certificate to get the data from
250  * Arguments:	alg - The algorithm to use to calculate the fingerprint
251  *
252  * Returns :	A static buffer containing the digest
253  *		NULL if cert is NULL, or digest cannot be calculated
254  */
255 char
256 *get_fingerprint(X509 *cert, const EVP_MD *alg)
257 {
258 	static char	 fp_str[ATTR_MAX];
259 	char		 tmp[ATTR_MAX] = "";
260 	unsigned int n;
261 	unsigned char md[EVP_MAX_MD_SIZE];
262 	int i;
263 
264 	if (!X509_digest(cert, alg, md, &n)) {
265 		return (NULL);
266 	}
267 
268 	/* start with empty string */
269 	fp_str[0] = '\0';
270 
271 	for (i = 0; i < (int)n; i++) {
272 		/* form a byte of the fingerprint */
273 		(void) snprintf(tmp, ATTR_MAX, "%02X:", md[i]);
274 		/* cat it onto the end of the result */
275 		(void) strlcat(fp_str, tmp, ATTR_MAX);
276 	}
277 
278 	/* nuke trailing ':' */
279 	fp_str[strlen(fp_str) - 1] = '\0';
280 
281 	return (fp_str);
282 }
283