xref: /titanic_41/usr/src/lib/libresolv2/common/dst/hmac_link.c (revision bb294766735789e8f3edfcbac1edb83dfb9ee53b)
1 /*
2  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
3  * Use is subject to license terms.
4  */
5 
6 
7 #ifdef HMAC_MD5
8 #ifndef LINT
9 static const char rcsid[] = "$Header: /proj/cvs/prod/libbind/dst/hmac_link.c,v 1.8 2007/09/24 17:18:25 each Exp $";
10 #endif
11 /*
12  * Portions Copyright (c) 1995-1998 by Trusted Information Systems, Inc.
13  *
14  * Permission to use, copy modify, and distribute this software for any
15  * purpose with or without fee is hereby granted, provided that the above
16  * copyright notice and this permission notice appear in all copies.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS" AND TRUSTED INFORMATION SYSTEMS
19  * DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL
21  * TRUSTED INFORMATION SYSTEMS BE LIABLE FOR ANY SPECIAL, DIRECT,
22  * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
23  * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
24  * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
25  * WITH THE USE OR PERFORMANCE OF THE SOFTWARE.
26  */
27 
28 /*%
29  * This file contains an implementation of the HMAC-MD5 algorithm.
30  */
31 #include "port_before.h"
32 
33 #include <stdio.h>
34 #include <unistd.h>
35 #include <stdlib.h>
36 #include <string.h>
37 #include <memory.h>
38 #include <sys/param.h>
39 #include <sys/time.h>
40 #include <netinet/in.h>
41 #include <arpa/nameser.h>
42 #include <resolv.h>
43 
44 #include "dst_internal.h"
45 
46 #ifdef USE_MD5
47 # ifndef HAVE_MD5
48 #  include "md5.h"
49 # else
50 #  ifdef SOLARIS2
51 #   include <sys/md5.h>
52 #  endif
53 # endif
54 # ifndef _MD5_H_
55 #  define _MD5_H_ 1	/*%< make sure we do not include rsaref md5.h file */
56 # endif
57 #endif
58 
59 #include "port_after.h"
60 
61 
62 #define HMAC_LEN	64
63 #define HMAC_IPAD	0x36
64 #define HMAC_OPAD	0x5c
65 #define MD5_LEN		16
66 
67 
68 typedef struct hmackey {
69 	u_char hk_ipad[64], hk_opad[64];
70 } HMAC_Key;
71 
72 
73 /**************************************************************************
74  * dst_hmac_md5_sign
75  *     Call HMAC signing functions to sign a block of data.
76  *     There are three steps to signing, INIT (initialize structures),
77  *     UPDATE (hash (more) data), FINAL (generate a signature).  This
78  *     routine performs one or more of these steps.
79  * Parameters
80  *     mode	SIG_MODE_INIT, SIG_MODE_UPDATE and/or SIG_MODE_FINAL.
81  *     priv_key    key to use for signing.
82  *     context   the context to be used in this digest
83  *     data	data to be signed.
84  *     len	 length in bytes of data.
85  *     signature   location to store signature.
86  *     sig_len     size of the signature location
87  * returns
88  *	N  Success on SIG_MODE_FINAL = returns signature length in bytes
89  *	0  Success on SIG_MODE_INIT  and UPDATE
90  *	 <0  Failure
91  */
92 
93 static int
94 dst_hmac_md5_sign(const int mode, DST_KEY *d_key, void **context,
95 		  const u_char *data, const int len,
96 		  u_char *signature, const int sig_len)
97 {
98 	HMAC_Key *key;
99 	int sign_len = 0;
100 	MD5_CTX *ctx = NULL;
101 
102 	if (d_key == NULL || d_key->dk_KEY_struct == NULL)
103 		return (-1);
104 
105 	if (mode & SIG_MODE_INIT)
106 		ctx = (MD5_CTX *) malloc(sizeof(*ctx));
107 	else if (context)
108 		ctx = (MD5_CTX *) *context;
109 	if (ctx == NULL)
110 		return (-1);
111 
112 	key = (HMAC_Key *) d_key->dk_KEY_struct;
113 
114 	if (mode & SIG_MODE_INIT) {
115 		MD5Init(ctx);
116 		MD5Update(ctx, key->hk_ipad, HMAC_LEN);
117 	}
118 
119 	if ((mode & SIG_MODE_UPDATE) && (data && len > 0))
120 		MD5Update(ctx, data, len);
121 
122 	if (mode & SIG_MODE_FINAL) {
123 		if (signature == NULL || sig_len < MD5_LEN)
124 			return (SIGN_FINAL_FAILURE);
125 		MD5Final(signature, ctx);
126 
127 		/* perform outer MD5 */
128 		MD5Init(ctx);
129 		MD5Update(ctx, key->hk_opad, HMAC_LEN);
130 		MD5Update(ctx, signature, MD5_LEN);
131 		MD5Final(signature, ctx);
132 		sign_len = MD5_LEN;
133 		SAFE_FREE(ctx);
134 	}
135 	else {
136 		if (context == NULL)
137 			return (-1);
138 		*context = (void *) ctx;
139 	}
140 	return (sign_len);
141 }
142 
143 
144 /**************************************************************************
145  * dst_hmac_md5_verify()
146  *     Calls HMAC verification routines.  There are three steps to
147  *     verification, INIT (initialize structures), UPDATE (hash (more) data),
148  *     FINAL (generate a signature).  This routine performs one or more of
149  *     these steps.
150  * Parameters
151  *     mode	SIG_MODE_INIT, SIG_MODE_UPDATE and/or SIG_MODE_FINAL.
152  *     dkey	key to use for verify.
153  *     data	data signed.
154  *     len	 length in bytes of data.
155  *     signature   signature.
156  *     sig_len     length in bytes of signature.
157  * returns
158  *     0  Success
159  *    <0  Failure
160  */
161 
162 static int
163 dst_hmac_md5_verify(const int mode, DST_KEY *d_key, void **context,
164 		const u_char *data, const int len,
165 		const u_char *signature, const int sig_len)
166 {
167 	HMAC_Key *key;
168 	MD5_CTX *ctx = NULL;
169 
170 	if (d_key == NULL || d_key->dk_KEY_struct == NULL)
171 		return (-1);
172 
173 	if (mode & SIG_MODE_INIT)
174 		ctx = (MD5_CTX *) malloc(sizeof(*ctx));
175 	else if (context)
176 		ctx = (MD5_CTX *) *context;
177 	if (ctx == NULL)
178 		return (-1);
179 
180 	key = (HMAC_Key *) d_key->dk_KEY_struct;
181 	if (mode & SIG_MODE_INIT) {
182 		MD5Init(ctx);
183 		MD5Update(ctx, key->hk_ipad, HMAC_LEN);
184 	}
185 	if ((mode & SIG_MODE_UPDATE) && (data && len > 0))
186 		MD5Update(ctx, data, len);
187 
188 	if (mode & SIG_MODE_FINAL) {
189 		u_char digest[MD5_LEN];
190 		if (signature == NULL || key == NULL || sig_len != MD5_LEN)
191 			return (VERIFY_FINAL_FAILURE);
192 		MD5Final(digest, ctx);
193 
194 		/* perform outer MD5 */
195 		MD5Init(ctx);
196 		MD5Update(ctx, key->hk_opad, HMAC_LEN);
197 		MD5Update(ctx, digest, MD5_LEN);
198 		MD5Final(digest, ctx);
199 
200 		SAFE_FREE(ctx);
201 		if (memcmp(digest, signature, MD5_LEN) != 0)
202 			return (VERIFY_FINAL_FAILURE);
203 	}
204 	else {
205 		if (context == NULL)
206 			return (-1);
207 		*context = (void *) ctx;
208 	}
209 	return (0);
210 }
211 
212 
213 /**************************************************************************
214  * dst_buffer_to_hmac_md5
215  *     Converts key from raw data to an HMAC Key
216  *     This function gets in a pointer to the data
217  * Parameters
218  *     hkey	the HMAC key to be filled in
219  *     key	the key in raw format
220  *     keylen	the length of the key
221  * Return
222  *	0	Success
223  *	<0	Failure
224  */
225 static int
226 dst_buffer_to_hmac_md5(DST_KEY *dkey, const u_char *key, const int keylen)
227 {
228 	int i;
229 	HMAC_Key *hkey = NULL;
230 	MD5_CTX ctx;
231 	int local_keylen = keylen;
232 	u_char tk[MD5_LEN];
233 
234 	if (dkey == NULL || key == NULL || keylen < 0)
235 		return (-1);
236 
237 	if ((hkey = (HMAC_Key *) malloc(sizeof(HMAC_Key))) == NULL)
238 		  return (-2);
239 
240 	memset(hkey->hk_ipad, 0, sizeof(hkey->hk_ipad));
241 	memset(hkey->hk_opad, 0, sizeof(hkey->hk_opad));
242 
243 	/* if key is longer than HMAC_LEN bytes reset it to key=MD5(key) */
244 	if (keylen > HMAC_LEN) {
245 		MD5Init(&ctx);
246 		MD5Update(&ctx, key, keylen);
247 		MD5Final(tk, &ctx);
248 		memset((void *) &ctx, 0, sizeof(ctx));
249 		key = tk;
250 		local_keylen = MD5_LEN;
251 	}
252 	/* start out by storing key in pads */
253 	memcpy(hkey->hk_ipad, key, local_keylen);
254 	memcpy(hkey->hk_opad, key, local_keylen);
255 
256 	/* XOR key with hk_ipad and opad values */
257 	for (i = 0; i < HMAC_LEN; i++) {
258 		hkey->hk_ipad[i] ^= HMAC_IPAD;
259 		hkey->hk_opad[i] ^= HMAC_OPAD;
260 	}
261 	dkey->dk_key_size = local_keylen;
262 	dkey->dk_KEY_struct = (void *) hkey;
263 	return (1);
264 }
265 
266 
267 /**************************************************************************
268  *  dst_hmac_md5_key_to_file_format
269  *	Encodes an HMAC Key into the portable file format.
270  *  Parameters
271  *	hkey      HMAC KEY structure
272  *	buff      output buffer
273  *	buff_len  size of output buffer
274  *  Return
275  *	0  Failure - null input hkey
276  *     -1  Failure - not enough space in output area
277  *	N  Success - Length of data returned in buff
278  */
279 
280 static int
281 dst_hmac_md5_key_to_file_format(const DST_KEY *dkey, char *buff,
282 			        const int buff_len)
283 {
284 	char *bp;
285 	int len, i, key_len;
286 	u_char key[HMAC_LEN];
287 	HMAC_Key *hkey;
288 
289 	if (dkey == NULL || dkey->dk_KEY_struct == NULL)
290 		return (0);
291 	/*
292 	 * Using snprintf() would be so much simpler here.
293 	 */
294 	if (buff == NULL ||
295 	    buff_len <= (int)(strlen(key_file_fmt_str) +
296 			      strlen(KEY_FILE_FORMAT) + 4))
297 		return (-1);	/*%< no OR not enough space in output area */
298 	hkey = (HMAC_Key *) dkey->dk_KEY_struct;
299 	memset(buff, 0, buff_len);	/*%< just in case */
300 	/* write file header */
301 	sprintf(buff, key_file_fmt_str, KEY_FILE_FORMAT, KEY_HMAC_MD5, "HMAC");
302 
303 	bp = buff + strlen(buff);
304 
305 	memset(key, 0, HMAC_LEN);
306 	for (i = 0; i < HMAC_LEN; i++)
307 		key[i] = hkey->hk_ipad[i] ^ HMAC_IPAD;
308 	for (i = HMAC_LEN - 1; i >= 0; i--)
309 		if (key[i] != 0)
310 			break;
311 	key_len = i + 1;
312 
313 	if (buff_len - (bp - buff) < 6)
314 		return (-1);
315 	strcat(bp, "Key: ");
316 	bp += strlen("Key: ");
317 
318 	len = b64_ntop(key, key_len, bp, buff_len - (bp - buff));
319 	if (len < 0)
320 		return (-1);
321 	bp += len;
322 	if (buff_len - (bp - buff) < 2)
323 		return (-1);
324 	*(bp++) = '\n';
325 	*bp = '\0';
326 
327 	return (bp - buff);
328 }
329 
330 
331 /**************************************************************************
332  * dst_hmac_md5_key_from_file_format
333  *     Converts contents of a key file into an HMAC key.
334  * Parameters
335  *     hkey    structure to put key into
336  *     buff       buffer containing the encoded key
337  *     buff_len   the length of the buffer
338  * Return
339  *     n >= 0 Foot print of the key converted
340  *     n <  0 Error in conversion
341  */
342 
343 static int
344 dst_hmac_md5_key_from_file_format(DST_KEY *dkey, const char *buff,
345 			      const int buff_len)
346 {
347 	const char *p = buff, *eol;
348 	u_char key[HMAC_LEN+1];	/* b64_pton needs more than 64 bytes do decode
349 				 * it should probably be fixed rather than doing
350 				 * this
351 				 */
352 	u_char *tmp;
353 	int key_len, len;
354 
355 	if (dkey == NULL)
356 		return (-2);
357 	if (buff == NULL || buff_len < 0)
358 		return (-1);
359 
360 	memset(key, 0, sizeof(key));
361 
362 	if (!dst_s_verify_str(&p, "Key: "))
363 		return (-3);
364 
365 	eol = strchr(p, '\n');
366 	if (eol == NULL)
367 		return (-4);
368 	len = eol - p;
369 	tmp = malloc(len + 2);
370 	if (tmp == NULL)
371 		return (-5);
372 	memcpy(tmp, p, len);
373 	*(tmp + len) = 0x0;
374 	key_len = b64_pton((char *)tmp, key, HMAC_LEN+1);	/*%< see above */
375 	SAFE_FREE2(tmp, len + 2);
376 
377 	if (dst_buffer_to_hmac_md5(dkey, key, key_len) < 0) {
378 		return (-6);
379 	}
380 	return (0);
381 }
382 
383 /*%
384  * dst_hmac_md5_to_dns_key()
385  *         function to extract hmac key from DST_KEY structure
386  * intput:
387  *      in_key:  HMAC-MD5 key
388  * output:
389  *	out_str: buffer to write ot
390  *      out_len: size of output buffer
391  * returns:
392  *      number of bytes written to output buffer
393  */
394 static int
395 dst_hmac_md5_to_dns_key(const DST_KEY *in_key, u_char *out_str,
396 			const int out_len)
397 {
398 
399 	HMAC_Key *hkey;
400 	int i;
401 
402 	if (in_key == NULL || in_key->dk_KEY_struct == NULL ||
403 	    out_len <= in_key->dk_key_size || out_str == NULL)
404 		return (-1);
405 
406 	hkey = (HMAC_Key *) in_key->dk_KEY_struct;
407 	for (i = 0; i < in_key->dk_key_size; i++)
408 		out_str[i] = hkey->hk_ipad[i] ^ HMAC_IPAD;
409 	return (i);
410 }
411 
412 /**************************************************************************
413  *  dst_hmac_md5_compare_keys
414  *	Compare two keys for equality.
415  *  Return
416  *	0	  The keys are equal
417  *	NON-ZERO   The keys are not equal
418  */
419 
420 static int
421 dst_hmac_md5_compare_keys(const DST_KEY *key1, const DST_KEY *key2)
422 {
423 	HMAC_Key *hkey1 = (HMAC_Key *) key1->dk_KEY_struct;
424 	HMAC_Key *hkey2 = (HMAC_Key *) key2->dk_KEY_struct;
425 	return memcmp(hkey1->hk_ipad, hkey2->hk_ipad, HMAC_LEN);
426 }
427 
428 /**************************************************************************
429  * dst_hmac_md5_free_key_structure
430  *     Frees all (none) dynamically allocated structures in hkey
431  */
432 
433 static void *
434 dst_hmac_md5_free_key_structure(void *key)
435 {
436 	HMAC_Key *hkey = key;
437 	SAFE_FREE(hkey);
438 	return (NULL);
439 }
440 
441 
442 /***************************************************************************
443  * dst_hmac_md5_generate_key
444  *     Creates a HMAC key of size size with a maximum size of 63 bytes
445  *     generating a HMAC key larger than 63 bytes makes no sense as that key
446  *     is digested before use.
447  */
448 
449 static int
450 dst_hmac_md5_generate_key(DST_KEY *key, const int nothing)
451 {
452 	(void)key;
453 	(void)nothing;
454 	return (-1);
455 }
456 
457 /*%
458  * dst_hmac_md5_init()  Function to answer set up function pointers for HMAC
459  *	   related functions
460  */
461 int
462 dst_hmac_md5_init()
463 {
464 	if (dst_t_func[KEY_HMAC_MD5] != NULL)
465 		return (1);
466 	dst_t_func[KEY_HMAC_MD5] = malloc(sizeof(struct dst_func));
467 	if (dst_t_func[KEY_HMAC_MD5] == NULL)
468 		return (0);
469 	memset(dst_t_func[KEY_HMAC_MD5], 0, sizeof(struct dst_func));
470 	dst_t_func[KEY_HMAC_MD5]->sign = dst_hmac_md5_sign;
471 	dst_t_func[KEY_HMAC_MD5]->verify = dst_hmac_md5_verify;
472 	dst_t_func[KEY_HMAC_MD5]->compare = dst_hmac_md5_compare_keys;
473 	dst_t_func[KEY_HMAC_MD5]->generate = dst_hmac_md5_generate_key;
474 	dst_t_func[KEY_HMAC_MD5]->destroy = dst_hmac_md5_free_key_structure;
475 	dst_t_func[KEY_HMAC_MD5]->to_dns_key = dst_hmac_md5_to_dns_key;
476 	dst_t_func[KEY_HMAC_MD5]->from_dns_key = dst_buffer_to_hmac_md5;
477 	dst_t_func[KEY_HMAC_MD5]->to_file_fmt = dst_hmac_md5_key_to_file_format;
478 	dst_t_func[KEY_HMAC_MD5]->from_file_fmt = dst_hmac_md5_key_from_file_format;
479 	return (1);
480 }
481 
482 #else
483 #define	dst_hmac_md5_init	__dst_hmac_md5_init
484 
485 int
486 dst_hmac_md5_init(){
487 	return (0);
488 }
489 #endif
490 
491 /*! \file */
492