xref: /illumos-gate/usr/src/lib/libnsl/key/xcrypt.c (revision 694c35faa87b858ecdadfe4fc592615f4eefbb07)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 
23 /*
24  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
25  * Use is subject to license terms.
26  */
27 
28 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
29 /* All Rights Reserved */
30 
31 /*
32  * Portions of this source code were derived from Berkeley 4.3 BSD
33  * under license from the Regents of the University of California.
34  */
35 
36 /*
37  * Hex encryption/decryption and utility routines
38  */
39 
40 #include "mt.h"
41 #include <stdio.h>
42 #include <stdlib.h>
43 #include <sys/types.h>
44 #include <rpc/rpc.h>
45 #include <rpc/key_prot.h>   /* for KEYCHECKSUMSIZE */
46 #include <rpc/des_crypt.h>
47 #include <string.h>
48 #include <rpcsvc/nis_dhext.h>
49 #include <md5.h>
50 
51 #define	MD5HEXSIZE	32
52 
53 extern int bin2hex(int len, unsigned char *binnum, char *hexnum);
54 extern int hex2bin(int len, char *hexnum, char *binnum);
55 static char hex[];	/* forward */
56 static char hexval();
57 
58 int passwd2des(char *, char *);
59 static int weak_DES_key(des_block);
60 
61 /*
62  * For export control reasons, we want to limit the maximum size of
63  * data that can be encrypted or decrypted.  We limit this to 1024
64  * bits of key data, which amounts to 128 bytes.
65  *
66  * For the extended DH project, we have increased it to
67  * 144 bytes (128key + 16checksum) to accomadate all the 128 bytes
68  * being used by the new 1024bit keys plus 16 bytes MD5 checksum.
69  * We discussed this with Sun's export control office and lawyers
70  * and we have reason to believe this is ok for export.
71  */
72 #define	MAX_KEY_CRYPT_LEN	144
73 
74 /*
75  * Encrypt a secret key given passwd
76  * The secret key is passed and returned in hex notation.
77  * Its length must be a multiple of 16 hex digits (64 bits).
78  */
79 int
xencrypt(secret,passwd)80 xencrypt(secret, passwd)
81 	char *secret;
82 	char *passwd;
83 {
84 	char key[8];
85 	char ivec[8];
86 	char *buf;
87 	int err;
88 	int len;
89 
90 	len = (int)strlen(secret) / 2;
91 	if (len > MAX_KEY_CRYPT_LEN)
92 		return (0);
93 	buf = malloc((unsigned)len);
94 	(void) hex2bin(len, secret, buf);
95 	(void) passwd2des(passwd, key);
96 	(void) memset(ivec, 0, 8);
97 
98 	err = cbc_crypt(key, buf, len, DES_ENCRYPT | DES_HW, ivec);
99 	if (DES_FAILED(err)) {
100 		free(buf);
101 		return (0);
102 	}
103 	(void) bin2hex(len, (unsigned char *) buf, secret);
104 	free(buf);
105 	return (1);
106 }
107 
108 /*
109  * Decrypt secret key using passwd
110  * The secret key is passed and returned in hex notation.
111  * Once again, the length is a multiple of 16 hex digits
112  */
113 int
xdecrypt(secret,passwd)114 xdecrypt(secret, passwd)
115 	char *secret;
116 	char *passwd;
117 {
118 	char key[8];
119 	char ivec[8];
120 	char *buf;
121 	int err;
122 	int len;
123 
124 	len = (int)strlen(secret) / 2;
125 	if (len > MAX_KEY_CRYPT_LEN)
126 		return (0);
127 	buf = malloc((unsigned)len);
128 
129 	(void) hex2bin(len, secret, buf);
130 	(void) passwd2des(passwd, key);
131 	(void) memset(ivec, 0, 8);
132 
133 	err = cbc_crypt(key, buf, len, DES_DECRYPT | DES_HW, ivec);
134 	if (DES_FAILED(err)) {
135 		free(buf);
136 		return (0);
137 	}
138 	(void) bin2hex(len, (unsigned char *) buf, secret);
139 	free(buf);
140 	return (1);
141 }
142 
143 /*
144  * Turn password into DES key
145  */
146 int
passwd2des(pw,key)147 passwd2des(pw, key)
148 	char *pw;
149 	char *key;
150 {
151 	int i;
152 
153 	(void) memset(key, 0, 8);
154 	for (i = 0; *pw; i = (i+1) % 8) {
155 		key[i] ^= *pw++ << 1;
156 	}
157 	des_setparity(key);
158 	return (1);
159 }
160 
161 
162 /*
163  * Hex to binary conversion
164  */
165 int
hex2bin(len,hexnum,binnum)166 hex2bin(len, hexnum, binnum)
167 	int len;
168 	char *hexnum;
169 	char *binnum;
170 {
171 	int i;
172 
173 	for (i = 0; i < len; i++) {
174 		*binnum++ = 16 * hexval(hexnum[2 * i]) +
175 					hexval(hexnum[2 * i + 1]);
176 	}
177 	return (1);
178 }
179 
180 /*
181  * Binary to hex conversion
182  */
183 int
bin2hex(len,binnum,hexnum)184 bin2hex(len, binnum, hexnum)
185 	int len;
186 	unsigned char *binnum;
187 	char *hexnum;
188 {
189 	int i;
190 	unsigned val;
191 
192 	for (i = 0; i < len; i++) {
193 		val = binnum[i];
194 		hexnum[i*2] = hex[val >> 4];
195 		hexnum[i*2+1] = hex[val & 0xf];
196 	}
197 	hexnum[len*2] = 0;
198 	return (1);
199 }
200 
201 static char hex[16] = {
202 	'0', '1', '2', '3', '4', '5', '6', '7',
203 	'8', '9', 'a', 'b', 'c', 'd', 'e', 'f',
204 };
205 
206 static char
hexval(c)207 hexval(c)
208 	char c;
209 {
210 	if (c >= '0' && c <= '9') {
211 		return (c - '0');
212 	} else if (c >= 'a' && c <= 'z') {
213 		return (c - 'a' + 10);
214 	} else if (c >= 'A' && c <= 'Z') {
215 		return (c - 'A' + 10);
216 	} else {
217 		return (-1);
218 	}
219 }
220 
221 /*
222  * Generic key length/algorithm version of xencrypt().
223  *
224  * Encrypt a secret key given passwd.
225  * The secret key is passed in hex notation.
226  * Arg encrypted_secret will be set to point to the encrypted
227  * secret key (NUL term, hex notation).
228  *
229  * Its length must be a multiple of 16 hex digits (64 bits).
230  *
231  * For 192-0 (AUTH_DES), then encrypt using the same method as xencrypt().
232  *
233  * If arg do_chksum is TRUE, append the checksum before the encrypt.
234  * For 192-0, the checksum is done the same as in xencrypt().  For
235  * bigger keys, MD5 is used.
236  *
237  * Arg netname can be NULL for 192-0.
238  */
239 int
xencrypt_g(char * secret,keylen_t keylen,algtype_t algtype,const char * passwd,const char netname[],char ** encrypted_secret,bool_t do_chksum)240 xencrypt_g(
241 	char *secret,			/* in  */
242 	keylen_t keylen,		/* in  */
243 	algtype_t algtype,		/* in  */
244 	const char *passwd,		/* in  */
245 	const char netname[],  		/* in  */
246 	char **encrypted_secret,	/* out */
247 	bool_t do_chksum)		/* in  */
248 {
249 	des_block key;
250 	char ivec[8];
251 	char *binkeybuf;
252 	int err;
253 	const int classic_des = keylen == 192 && algtype == 0;
254 	const int hexkeybytes = BITS2NIBBLES(keylen);
255 	const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
256 	const int binkeybytes = do_chksum ? keylen/8 + keychecksumsize/2 :
257 		keylen/8;
258 	const int bufsize = do_chksum ? hexkeybytes + keychecksumsize + 1 :
259 		hexkeybytes + 1;
260 	char *hexkeybuf;
261 
262 	if (!secret || !keylen || !passwd || !encrypted_secret)
263 		return (0);
264 
265 	if ((hexkeybuf = malloc(bufsize)) == 0)
266 		return (0);
267 
268 	(void) memcpy(hexkeybuf, secret, hexkeybytes);
269 	if (do_chksum)
270 		if (classic_des) {
271 			(void) memcpy(hexkeybuf + hexkeybytes, secret,
272 					keychecksumsize);
273 		} else {
274 			MD5_CTX md5_ctx;
275 			char md5hexbuf[MD5HEXSIZE + 1] = {0};
276 			uint8_t digest[MD5HEXSIZE/2];
277 
278 			MD5Init(&md5_ctx);
279 			MD5Update(&md5_ctx, (unsigned char *)hexkeybuf,
280 					hexkeybytes);
281 			MD5Final(digest, &md5_ctx);
282 
283 			/* convert md5 binary digest to hex */
284 			(void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
285 
286 			/* append the hex md5 string to the end of the key */
287 			(void) memcpy(hexkeybuf + hexkeybytes,
288 					(void *)md5hexbuf, MD5HEXSIZE);
289 		}
290 	hexkeybuf[bufsize - 1] = 0;
291 
292 	if (binkeybytes > MAX_KEY_CRYPT_LEN) {
293 		free(hexkeybuf);
294 		return (0);
295 	}
296 	if ((binkeybuf = malloc((unsigned)binkeybytes)) == 0) {
297 		free(hexkeybuf);
298 		return (0);
299 	}
300 
301 	(void) hex2bin(binkeybytes, hexkeybuf, binkeybuf);
302 	if (classic_des)
303 		(void) passwd2des((char *)passwd, key.c);
304 	else
305 		if (netname)
306 			(void) passwd2des_g(passwd, netname,
307 					(int)strlen(netname), &key, FALSE);
308 		else {
309 			free(hexkeybuf);
310 			return (0);
311 		}
312 
313 	(void) memset(ivec, 0, 8);
314 
315 	err = cbc_crypt(key.c, binkeybuf, binkeybytes, DES_ENCRYPT | DES_HW,
316 			ivec);
317 	if (DES_FAILED(err)) {
318 		free(hexkeybuf);
319 		free(binkeybuf);
320 		return (0);
321 	}
322 	(void) bin2hex(binkeybytes, (unsigned char *) binkeybuf, hexkeybuf);
323 	free(binkeybuf);
324 	*encrypted_secret = hexkeybuf;
325 	return (1);
326 }
327 
328 /*
329  * Generic key len and alg type for version of xdecrypt.
330  *
331  * Decrypt secret key using passwd.  The decrypted secret key
332  * *overwrites* the supplied encrypted secret key.
333  * The secret key is passed and returned in hex notation.
334  * Once again, the length is a multiple of 16 hex digits.
335  *
336  * If 'do_chksum' is TRUE, the 'secret' buffer is assumed to contain
337  * a checksum calculated by a call to xencrypt_g().
338  *
339  * If keylen is 192 and algtype is 0, then decrypt the same way
340  * as xdecrypt().
341  *
342  * Arg netname can be NULL for 192-0.
343  */
344 int
xdecrypt_g(char * secret,int keylen,int algtype,const char * passwd,const char netname[],bool_t do_chksum)345 xdecrypt_g(
346 	char *secret,		/* out  */
347 	int keylen,		/* in  */
348 	int algtype,		/* in  */
349 	const char *passwd,	/* in  */
350 	const char netname[],	/* in  */
351 	bool_t do_chksum)	/* in  */
352 {
353 	des_block key;
354 	char ivec[8];
355 	char *buf;
356 	int err;
357 	int len;
358 	const int classic_des = keylen == 192 && algtype == 0;
359 	const int hexkeybytes = BITS2NIBBLES(keylen);
360 	const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
361 
362 	len = (int)strlen(secret) / 2;
363 	if (len > MAX_KEY_CRYPT_LEN)
364 		return (0);
365 	if ((buf = malloc((unsigned)len)) == 0)
366 		return (0);
367 
368 	(void) hex2bin(len, secret, buf);
369 	if (classic_des)
370 		(void) passwd2des((char *)passwd, key.c);
371 	else
372 		if (netname)
373 			(void) passwd2des_g(passwd, netname,
374 					(int)strlen(netname), &key, FALSE);
375 		else {
376 			free(buf);
377 			return (0);
378 		}
379 	(void) memset(ivec, 0, 8);
380 
381 	err = cbc_crypt(key.c, buf, len, DES_DECRYPT | DES_HW, ivec);
382 	if (DES_FAILED(err)) {
383 		free(buf);
384 		return (0);
385 	}
386 	(void) bin2hex(len, (unsigned char *) buf, secret);
387 	free(buf);
388 
389 	if (do_chksum)
390 		if (classic_des) {
391 			if (memcmp(secret, &(secret[hexkeybytes]),
392 					keychecksumsize) != 0) {
393 				secret[0] = 0;
394 				return (0);
395 			}
396 		} else {
397 			MD5_CTX md5_ctx;
398 			char md5hexbuf[MD5HEXSIZE + 1] = {0};
399 			uint8_t digest[MD5HEXSIZE/2];
400 
401 			MD5Init(&md5_ctx);
402 			MD5Update(&md5_ctx, (unsigned char *)secret,
403 					hexkeybytes);
404 			MD5Final(digest, &md5_ctx);
405 
406 			/* convert md5 binary digest to hex */
407 			(void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
408 
409 			/* does the digest match the appended one? */
410 			if (memcmp(&(secret[hexkeybytes]),
411 					md5hexbuf, MD5HEXSIZE) != 0) {
412 				secret[0] = 0;
413 				return (0);
414 			}
415 		}
416 
417 	secret[hexkeybytes] = '\0';
418 
419 	return (1);
420 }
421 
422 
423 /*
424  * Modified version of passwd2des(). passwd2des_g() uses the Kerberos
425  * RFC 1510 algorithm to generate a DES key from a user password
426  * and mix-in string. The mix-in is expected to be the netname.
427  * This function to be used only for extended Diffie-Hellman keys.
428  *
429  * If altarg is TRUE, reverse the concat of passwd and mix-in.
430  */
431 int
passwd2des_g(const char * pw,const char * mixin,int len,des_block * key,bool_t altalg)432 passwd2des_g(
433 	const char *pw,
434 	const char *mixin,
435 	int len,
436 	des_block *key, /* out */
437 	bool_t altalg)
438 {
439 
440 	int  i, j, incr = 1;
441 	des_block ivec, tkey;
442 	char *text;
443 	int  plen, tlen;
444 
445 	(void) memset(tkey.c, 0, 8);
446 	(void) memset(ivec.c, 0, 8);
447 
448 
449 /*
450  * Concatentate the password and the mix-in string, fan-fold and XOR them
451  * to the required eight byte initial DES key. Since passwords can be
452  * expected to use mostly seven bit ASCII, left shift the password one
453  * bit in order to preserve as much key space as possible.
454  */
455 
456 #define	KEYLEN sizeof (tkey.c)
457 	plen = strlen(pw);
458 	tlen = ((plen + len + (KEYLEN-1))/KEYLEN)*KEYLEN;
459 	if ((text = malloc(tlen)) == NULL) {
460 		return (0);
461 	}
462 
463 	(void) memset(text, 0, tlen);
464 
465 	if (!altalg) {
466 
467 /*
468  * Concatenate the password and the mix-in string, fan-fold and XOR them
469  * to the required eight byte initial DES key. Since passwords can be
470  * expected to use mostly seven bit ASCII, left shift the password one
471  * bit in order to preserve as much key space as possible.
472  */
473 		(void) memcpy(text, pw, plen);
474 		(void) memcpy(&text[plen], mixin, len);
475 
476 		for (i = 0, j = 0; pw[j]; j++) {
477 			tkey.c[i] ^= pw[j] << 1;
478 			i += incr;
479 			if (i == 8) {
480 				i = 7;
481 				incr = -incr;
482 			} else if (i == -1) {
483 				i = 0;
484 				incr = -incr;
485 			}
486 		}
487 
488 		for (j = 0; j < len; j++) {
489 			tkey.c[i] ^= mixin[j];
490 			i += incr;
491 			if (i == 8) {
492 				i = 7;
493 				incr = -incr;
494 			} else if (i == -1) {
495 				i = 0;
496 				incr = -incr;
497 			}
498 		}
499 	} else {  /* use alternative algorithm */
500 		(void) memcpy(text, mixin, len);
501 		(void) memcpy(&text[len], pw, plen);
502 
503 		for (i = 0, j = 0; j < len; j++) {
504 			tkey.c[i] ^= mixin[j];
505 			i += incr;
506 			if (i == 8) {
507 				i = 7;
508 				incr = -incr;
509 			} else if (i == -1) {
510 				i = 0;
511 				incr = -incr;
512 			}
513 		}
514 
515 		for (j = 0; pw[j]; j++) {
516 			tkey.c[i] ^= pw[j] << 1;
517 			i += incr;
518 			if (i == 8) {
519 				i = 7;
520 				incr = -incr;
521 			} else if (i == -1) {
522 				i = 0;
523 				incr = -incr;
524 			}
525 		}
526 	}
527 	des_setparity_g(&tkey);
528 
529 	/*
530 	 * Use the temporary key to produce a DES CBC checksum for the text
531 	 * string; cbc_crypt returns the checksum in the ivec.
532 	 */
533 	(void) cbc_crypt(tkey.c, text, tlen, DES_ENCRYPT|DES_HW, ivec.c);
534 	des_setparity_g(&ivec);
535 	free(text);
536 
537 	if (weak_DES_key(ivec)) {
538 		ivec.c[7] ^= 0xf0;
539 		/*
540 		 *  XORing with 0xf0 preserves parity, so no need to check
541 		 *  that again.
542 		 */
543 	}
544 
545 	(void) memcpy((*key).c, ivec.c, sizeof (ivec.c));
546 
547 	return (1);
548 
549 }
550 
551 struct DESkey {
552 	uint32_t h1;
553 	uint32_t h2;
554 };
555 
556 /*
557  * Weak and semiweak keys from "Applied Cryptography", second edition,
558  * by Bruce Schneier, Wiley 1996.
559  */
560 static struct DESkey weakDESkeys[] = {
561 	/* Weak keys */
562 	{0x01010101, 0x01010101},
563 	{0x1f1f1f1f, 0x1f1f1f1f},
564 	{0xe0e0e0e0, 0xe0e0e0e0},
565 	{0xfefefefe, 0xfefefefe},
566 	/* Semiweak keys */
567 	{0x01fe01fe, 0x01fe01fe},
568 	{0x1fe01fe0, 0x0ef10ef1},
569 	{0x01e001e0, 0x01f101f1},
570 	{0x1ffe1ffe, 0x0efe0efe},
571 	{0x011f011f, 0x010e010e},
572 	{0xe0fee0fe, 0xf1fef1fe},
573 	{0xfe01fe01, 0xfe01fe01},
574 	{0xe01fe01f, 0xf10ef10e},
575 	{0xe001e001, 0xf101f101},
576 	{0xfe1ffe1f, 0xfe0efe0e},
577 	{0x1f011f01, 0x0e010e01},
578 	{0xfee0fee0, 0xfef1fef1}
579 };
580 
581 static int
weak_DES_key(des_block db)582 weak_DES_key(des_block db)
583 {
584 	int i;
585 
586 	for (i = 0; i < sizeof (weakDESkeys)/sizeof (struct DESkey); i++) {
587 		if (weakDESkeys[i].h1 == db.key.high &&
588 			weakDESkeys[i].h2 == db.key.low)
589 			return (1);
590 	}
591 
592 	return (0);
593 }
594