xref: /titanic_41/usr/src/lib/libipsecutil/common/ipsec_libssl_setup.c (revision 5203bc321053fb87d7073c7640548fab73634793)
1 /*
2  * Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3  * All rights reserved.
4  *
5  * This package is an SSL implementation written
6  * by Eric Young (eay@cryptsoft.com).
7  * The implementation was written so as to conform with Netscapes SSL.
8  *
9  * This library is free for commercial and non-commercial use as long as
10  * the following conditions are aheared to.  The following conditions
11  * apply to all code found in this distribution, be it the RC4, RSA,
12  * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
13  * included with this distribution is covered by the same copyright terms
14  * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15  *
16  * Copyright remains Eric Young's, and as such any Copyright notices in
17  * the code are not to be removed.
18  * If this package is used in a product, Eric Young should be given attribution
19  * as the author of the parts of the library used.
20  * This can be in the form of a textual message at program startup or
21  * in documentation (online or textual) provided with the package.
22  *
23  * Redistribution and use in source and binary forms, with or without
24  * modification, are permitted provided that the following conditions
25  * are met:
26  * 1. Redistributions of source code must retain the copyright
27  *    notice, this list of conditions and the following disclaimer.
28  * 2. Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in the
30  *    documentation and/or other materials provided with the distribution.
31  * 3. All advertising materials mentioning features or use of this software
32  *    must display the following acknowledgement:
33  *    "This product includes cryptographic software written by
34  *     Eric Young (eay@cryptsoft.com)"
35  *    The word 'cryptographic' can be left out if the rouines from the library
36  *    being used are not cryptographic related :-).
37  * 4. If you include any Windows specific code (or a derivative thereof) from
38  *    the apps directory (application code) you must include an acknowledgement:
39  *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40  *
41  * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51  * SUCH DAMAGE.
52  *
53  * The licence and distribution terms for any publically available version or
54  * derivative of this code cannot be changed.  i.e. this code cannot simply be
55  * copied and put under another distribution licence
56  * [including the GNU Public Licence.]
57  */
58 
59 /*
60  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
61  * Use is subject to license terms.
62  */
63 
64 /*
65  * Thread setup portions of this code derived from
66  * OpenSSL 0.9.x file mt/mttest.c examples
67  */
68 
69 #include <stdio.h>
70 #include <stdlib.h>
71 #include <string.h>
72 #include <errno.h>
73 #include <libintl.h>
74 #include <synch.h>
75 #include <thread.h>
76 #include <dlfcn.h>
77 #include <openssl/lhash.h>
78 #include <openssl/crypto.h>
79 #include <openssl/ssl.h>
80 #include <openssl/err.h>
81 #include "ipsec_util.h"
82 
83 /* OpenSSL function pointers */
84 static X509_NAME *(*d2i_X509_NAME_fn)() = NULL;
85 static int (*X509_NAME_print_ex_fp_fn)() = NULL;
86 static char *(*ERR_get_error_fn)() = NULL;
87 static char *(*ERR_error_string_fn)() = NULL;
88 static void (*SSL_load_error_strings_fn)() = NULL;
89 static void (*ERR_free_strings_fn)() = NULL;
90 static void (*CRYPTO_set_locking_callback_fn)() = NULL;
91 static void (*CRYPTO_set_id_callback_fn)() = NULL;
92 static void (*X509_NAME_free_fn)() = NULL;
93 static int (*CRYPTO_num_locks_fn)() = NULL;
94 static void *(*OPENSSL_malloc_fn)() = NULL;
95 static void (*OPENSSL_free_fn)() = NULL;
96 
97 static void solaris_locking_callback(int, int, char *, int);
98 static unsigned long solaris_thread_id(void);
99 static boolean_t thread_setup(void);
100 /* LINTED E_STATIC_UNUSED */
101 static void thread_cleanup(void);
102 
103 mutex_t init_lock = DEFAULTMUTEX;
104 static mutex_t *lock_cs;
105 static long *lock_count;
106 
107 static boolean_t libssl_loaded = B_FALSE;
108 static boolean_t libcrypto_loaded = B_FALSE;
109 
110 void
111 libssl_load()
112 {
113 	void *dldesc;
114 
115 	(void) mutex_lock(&init_lock);
116 	if (libssl_loaded) {
117 		(void) mutex_unlock(&init_lock);
118 		return;
119 	}
120 
121 	dldesc = dlopen(LIBSSL, RTLD_LAZY);
122 	if (dldesc != NULL) {
123 		d2i_X509_NAME_fn = (X509_NAME*(*)())dlsym(dldesc,
124 		    "d2i_X509_NAME");
125 		if (d2i_X509_NAME_fn == NULL)
126 			goto libssl_err;
127 
128 		X509_NAME_print_ex_fp_fn = (int(*)())dlsym(dldesc,
129 		    "X509_NAME_print_ex_fp");
130 		if (X509_NAME_print_ex_fp_fn == NULL)
131 			goto libssl_err;
132 
133 		ERR_get_error_fn = (char *(*)())dlsym(dldesc,
134 		    "ERR_get_error");
135 		if (ERR_get_error_fn == NULL)
136 			goto libssl_err;
137 
138 		ERR_error_string_fn = (char *(*)())dlsym(dldesc,
139 		    "ERR_error_string");
140 		if (ERR_error_string_fn == NULL)
141 			goto libssl_err;
142 
143 		SSL_load_error_strings_fn = (void(*)())dlsym(dldesc,
144 		    "SSL_load_error_strings");
145 		if (SSL_load_error_strings_fn == NULL)
146 			goto libssl_err;
147 
148 		ERR_free_strings_fn = (void(*)())dlsym(dldesc,
149 		    "ERR_free_strings");
150 		if (ERR_free_strings_fn == NULL)
151 			goto libssl_err;
152 
153 		CRYPTO_set_locking_callback_fn = (void(*)())dlsym(dldesc,
154 		    "CRYPTO_set_locking_callback");
155 		if (CRYPTO_set_locking_callback_fn == NULL)
156 			goto libssl_err;
157 
158 		CRYPTO_set_id_callback_fn = (void(*)())dlsym(dldesc,
159 		    "CRYPTO_set_id_callback");
160 		if (CRYPTO_set_id_callback_fn == NULL)
161 			goto libssl_err;
162 
163 		X509_NAME_free_fn = (void(*)())dlsym(dldesc,
164 		    "X509_NAME_free");
165 		if (X509_NAME_free_fn == NULL)
166 			goto libssl_err;
167 
168 		if (thread_setup() == B_FALSE)
169 			goto libssl_err;
170 
171 		libssl_loaded = B_TRUE;
172 	}
173 	(void) mutex_unlock(&init_lock);
174 	return;
175 libssl_err:
176 	(void) dlclose(dldesc);
177 	(void) mutex_unlock(&init_lock);
178 }
179 
180 void
181 libcrypto_load()
182 {
183 	void *dldesc;
184 
185 	(void) mutex_lock(&init_lock);
186 	if (libcrypto_loaded) {
187 		(void) mutex_unlock(&init_lock);
188 		return;
189 	}
190 
191 	dldesc = dlopen(LIBCRYPTO, RTLD_LAZY);
192 	if (dldesc != NULL) {
193 		CRYPTO_num_locks_fn = (int(*)())dlsym(dldesc,
194 		    "CRYPTO_num_locks");
195 		if (CRYPTO_num_locks_fn == NULL)
196 			goto libcrypto_err;
197 
198 		/*
199 		 * OPENSSL_free is really a macro, so we
200 		 * need to reference the actual symbol,
201 		 * which is CRYPTO_free.
202 		 */
203 		OPENSSL_free_fn = (void(*)())dlsym(dldesc,
204 		    "CRYPTO_free");
205 		if (OPENSSL_free_fn == NULL)
206 			goto libcrypto_err;
207 
208 		/*
209 		 * OPENSSL_malloc is really a macro, so we
210 		 * need to reference the actual symbol,
211 		 * which is CRYPTO_malloc.
212 		 */
213 		OPENSSL_malloc_fn = (void *(*)())dlsym(dldesc,
214 		    "CRYPTO_malloc");
215 		if (OPENSSL_malloc_fn == NULL)
216 			goto libcrypto_err;
217 
218 		libcrypto_loaded = B_TRUE;
219 	}
220 	(void) mutex_unlock(&init_lock);
221 	return;
222 libcrypto_err:
223 	(void) dlclose(dldesc);
224 	(void) mutex_unlock(&init_lock);
225 }
226 
227 static boolean_t
228 thread_setup(void)
229 {
230 	int i;
231 
232 	if ((lock_cs = OPENSSL_malloc_fn(CRYPTO_num_locks_fn() *
233 	    sizeof (mutex_t))) == NULL)
234 		return (B_FALSE);
235 	if ((lock_count = OPENSSL_malloc_fn(CRYPTO_num_locks_fn() *
236 	    sizeof (long))) == NULL) {
237 		OPENSSL_free_fn(lock_cs);
238 		return (B_FALSE);
239 	}
240 
241 	for (i = 0; i < CRYPTO_num_locks_fn(); i++) {
242 		lock_count[i] = 0;
243 		(void) mutex_init(&(lock_cs[i]), USYNC_THREAD, NULL);
244 	}
245 
246 	CRYPTO_set_id_callback_fn((unsigned long (*)())solaris_thread_id);
247 	CRYPTO_set_locking_callback_fn((void (*)())solaris_locking_callback);
248 	return (B_TRUE);
249 }
250 
251 static void
252 thread_cleanup(void)
253 {
254 	int i;
255 
256 	(void) mutex_lock(&init_lock);
257 	CRYPTO_set_locking_callback_fn(NULL);
258 	CRYPTO_set_id_callback_fn(NULL);
259 	for (i = 0; i < CRYPTO_num_locks_fn(); i++)
260 		(void) mutex_destroy(&(lock_cs[i]));
261 	OPENSSL_free_fn(lock_cs);
262 	OPENSSL_free_fn(lock_count);
263 	(void) mutex_unlock(&init_lock);
264 }
265 
266 /* ARGSUSED */
267 static void
268 solaris_locking_callback(int mode, int type, char *file, int line)
269 {
270 	if (mode & CRYPTO_LOCK) {
271 		(void) mutex_lock(&(lock_cs[type]));
272 		lock_count[type]++;
273 	} else {
274 		(void) mutex_unlock(&(lock_cs[type]));
275 	}
276 }
277 
278 static unsigned long
279 solaris_thread_id(void)
280 {
281 	unsigned long ret;
282 
283 	ret = (unsigned long)thr_self();
284 	return (ret);
285 }
286 
287 void
288 print_asn1_name(FILE *file, const unsigned char *buf, long buflen)
289 {
290 	libcrypto_load();
291 	if (libcrypto_loaded)
292 		libssl_load();
293 
294 	if (libssl_loaded && libcrypto_loaded) {
295 		X509_NAME *x509name = NULL;
296 		const unsigned char *p;
297 
298 		/* Make an effort to decode the ASN1 encoded name */
299 		SSL_load_error_strings_fn();
300 
301 		/*
302 		 * Temporary variable is mandatory per d2i_X509(3). Upcoming
303 		 * call to d2i_X509_NAME_fn() will change the 'p' pointer.
304 		 */
305 		p = buf;
306 
307 		x509name = d2i_X509_NAME_fn(NULL, &p, buflen);
308 		if (x509name != NULL) {
309 			(void) X509_NAME_print_ex_fp_fn(file, x509name, 0,
310 			    (ASN1_STRFLGS_RFC2253 | ASN1_STRFLGS_ESC_QUOTE |
311 			    XN_FLAG_SEP_CPLUS_SPC | XN_FLAG_FN_SN));
312 			X509_NAME_free_fn(x509name);
313 			(void) fprintf(file, "\n");
314 		} else {
315 			char errbuf[80];
316 
317 			(void) fprintf(file, "\n# %s\n",
318 			    ERR_error_string_fn(ERR_get_error_fn(), errbuf));
319 			(void) fprintf(file, dgettext(TEXT_DOMAIN,
320 			    "<cannot interpret>\n"));
321 		}
322 		ERR_free_strings_fn();
323 	} else {
324 		(void) fprintf(file, dgettext(TEXT_DOMAIN, "<cannot print>\n"));
325 	}
326 }
327