17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only
67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance
77c478bd9Sstevel@tonic-gate * with the License.
87c478bd9Sstevel@tonic-gate *
97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
127c478bd9Sstevel@tonic-gate * and limitations under the License.
137c478bd9Sstevel@tonic-gate *
147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
197c478bd9Sstevel@tonic-gate *
207c478bd9Sstevel@tonic-gate * CDDL HEADER END
217c478bd9Sstevel@tonic-gate */
2261961e0fSrobinson
237c478bd9Sstevel@tonic-gate /*
24*e8031f0aSraf * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
257c478bd9Sstevel@tonic-gate * Use is subject to license terms.
267c478bd9Sstevel@tonic-gate */
277c478bd9Sstevel@tonic-gate
287c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
297c478bd9Sstevel@tonic-gate /* All Rights Reserved */
307c478bd9Sstevel@tonic-gate
317c478bd9Sstevel@tonic-gate /*
327c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD
337c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California.
347c478bd9Sstevel@tonic-gate */
357c478bd9Sstevel@tonic-gate
367c478bd9Sstevel@tonic-gate /*
37*e8031f0aSraf * Hex encryption/decryption and utility routines
387c478bd9Sstevel@tonic-gate */
397c478bd9Sstevel@tonic-gate
40*e8031f0aSraf #include "mt.h"
417c478bd9Sstevel@tonic-gate #include <stdio.h>
427c478bd9Sstevel@tonic-gate #include <stdlib.h>
437c478bd9Sstevel@tonic-gate #include <sys/types.h>
447c478bd9Sstevel@tonic-gate #include <rpc/rpc.h>
457c478bd9Sstevel@tonic-gate #include <rpc/key_prot.h> /* for KEYCHECKSUMSIZE */
467c478bd9Sstevel@tonic-gate #include <rpc/des_crypt.h>
477c478bd9Sstevel@tonic-gate #include <string.h>
487c478bd9Sstevel@tonic-gate #include <rpcsvc/nis_dhext.h>
497c478bd9Sstevel@tonic-gate #include <md5.h>
507c478bd9Sstevel@tonic-gate
517c478bd9Sstevel@tonic-gate #define MD5HEXSIZE 32
527c478bd9Sstevel@tonic-gate
537c478bd9Sstevel@tonic-gate extern int bin2hex(int len, unsigned char *binnum, char *hexnum);
547c478bd9Sstevel@tonic-gate extern int hex2bin(int len, char *hexnum, char *binnum);
557c478bd9Sstevel@tonic-gate static char hex[]; /* forward */
567c478bd9Sstevel@tonic-gate static char hexval();
577c478bd9Sstevel@tonic-gate
587c478bd9Sstevel@tonic-gate int passwd2des(char *, char *);
597c478bd9Sstevel@tonic-gate static int weak_DES_key(des_block);
607c478bd9Sstevel@tonic-gate
617c478bd9Sstevel@tonic-gate /*
627c478bd9Sstevel@tonic-gate * For export control reasons, we want to limit the maximum size of
637c478bd9Sstevel@tonic-gate * data that can be encrypted or decrypted. We limit this to 1024
647c478bd9Sstevel@tonic-gate * bits of key data, which amounts to 128 bytes.
657c478bd9Sstevel@tonic-gate *
667c478bd9Sstevel@tonic-gate * For the extended DH project, we have increased it to
677c478bd9Sstevel@tonic-gate * 144 bytes (128key + 16checksum) to accomadate all the 128 bytes
687c478bd9Sstevel@tonic-gate * being used by the new 1024bit keys plus 16 bytes MD5 checksum.
697c478bd9Sstevel@tonic-gate * We discussed this with Sun's export control office and lawyers
707c478bd9Sstevel@tonic-gate * and we have reason to believe this is ok for export.
717c478bd9Sstevel@tonic-gate */
727c478bd9Sstevel@tonic-gate #define MAX_KEY_CRYPT_LEN 144
737c478bd9Sstevel@tonic-gate
747c478bd9Sstevel@tonic-gate /*
757c478bd9Sstevel@tonic-gate * Encrypt a secret key given passwd
767c478bd9Sstevel@tonic-gate * The secret key is passed and returned in hex notation.
777c478bd9Sstevel@tonic-gate * Its length must be a multiple of 16 hex digits (64 bits).
787c478bd9Sstevel@tonic-gate */
797c478bd9Sstevel@tonic-gate int
xencrypt(secret,passwd)807c478bd9Sstevel@tonic-gate xencrypt(secret, passwd)
817c478bd9Sstevel@tonic-gate char *secret;
827c478bd9Sstevel@tonic-gate char *passwd;
837c478bd9Sstevel@tonic-gate {
847c478bd9Sstevel@tonic-gate char key[8];
857c478bd9Sstevel@tonic-gate char ivec[8];
867c478bd9Sstevel@tonic-gate char *buf;
877c478bd9Sstevel@tonic-gate int err;
887c478bd9Sstevel@tonic-gate int len;
897c478bd9Sstevel@tonic-gate
907c478bd9Sstevel@tonic-gate len = (int)strlen(secret) / 2;
9161961e0fSrobinson if (len > MAX_KEY_CRYPT_LEN)
927c478bd9Sstevel@tonic-gate return (0);
937c478bd9Sstevel@tonic-gate buf = malloc((unsigned)len);
947c478bd9Sstevel@tonic-gate (void) hex2bin(len, secret, buf);
957c478bd9Sstevel@tonic-gate (void) passwd2des(passwd, key);
967c478bd9Sstevel@tonic-gate (void) memset(ivec, 0, 8);
977c478bd9Sstevel@tonic-gate
987c478bd9Sstevel@tonic-gate err = cbc_crypt(key, buf, len, DES_ENCRYPT | DES_HW, ivec);
997c478bd9Sstevel@tonic-gate if (DES_FAILED(err)) {
1007c478bd9Sstevel@tonic-gate free(buf);
1017c478bd9Sstevel@tonic-gate return (0);
1027c478bd9Sstevel@tonic-gate }
1037c478bd9Sstevel@tonic-gate (void) bin2hex(len, (unsigned char *) buf, secret);
1047c478bd9Sstevel@tonic-gate free(buf);
1057c478bd9Sstevel@tonic-gate return (1);
1067c478bd9Sstevel@tonic-gate }
1077c478bd9Sstevel@tonic-gate
1087c478bd9Sstevel@tonic-gate /*
1097c478bd9Sstevel@tonic-gate * Decrypt secret key using passwd
1107c478bd9Sstevel@tonic-gate * The secret key is passed and returned in hex notation.
1117c478bd9Sstevel@tonic-gate * Once again, the length is a multiple of 16 hex digits
1127c478bd9Sstevel@tonic-gate */
1137c478bd9Sstevel@tonic-gate int
xdecrypt(secret,passwd)1147c478bd9Sstevel@tonic-gate xdecrypt(secret, passwd)
1157c478bd9Sstevel@tonic-gate char *secret;
1167c478bd9Sstevel@tonic-gate char *passwd;
1177c478bd9Sstevel@tonic-gate {
1187c478bd9Sstevel@tonic-gate char key[8];
1197c478bd9Sstevel@tonic-gate char ivec[8];
1207c478bd9Sstevel@tonic-gate char *buf;
1217c478bd9Sstevel@tonic-gate int err;
1227c478bd9Sstevel@tonic-gate int len;
1237c478bd9Sstevel@tonic-gate
1247c478bd9Sstevel@tonic-gate len = (int)strlen(secret) / 2;
12561961e0fSrobinson if (len > MAX_KEY_CRYPT_LEN)
1267c478bd9Sstevel@tonic-gate return (0);
1277c478bd9Sstevel@tonic-gate buf = malloc((unsigned)len);
1287c478bd9Sstevel@tonic-gate
1297c478bd9Sstevel@tonic-gate (void) hex2bin(len, secret, buf);
1307c478bd9Sstevel@tonic-gate (void) passwd2des(passwd, key);
1317c478bd9Sstevel@tonic-gate (void) memset(ivec, 0, 8);
1327c478bd9Sstevel@tonic-gate
1337c478bd9Sstevel@tonic-gate err = cbc_crypt(key, buf, len, DES_DECRYPT | DES_HW, ivec);
1347c478bd9Sstevel@tonic-gate if (DES_FAILED(err)) {
1357c478bd9Sstevel@tonic-gate free(buf);
1367c478bd9Sstevel@tonic-gate return (0);
1377c478bd9Sstevel@tonic-gate }
1387c478bd9Sstevel@tonic-gate (void) bin2hex(len, (unsigned char *) buf, secret);
1397c478bd9Sstevel@tonic-gate free(buf);
1407c478bd9Sstevel@tonic-gate return (1);
1417c478bd9Sstevel@tonic-gate }
1427c478bd9Sstevel@tonic-gate
1437c478bd9Sstevel@tonic-gate /*
1447c478bd9Sstevel@tonic-gate * Turn password into DES key
1457c478bd9Sstevel@tonic-gate */
1467c478bd9Sstevel@tonic-gate int
passwd2des(pw,key)1477c478bd9Sstevel@tonic-gate passwd2des(pw, key)
1487c478bd9Sstevel@tonic-gate char *pw;
1497c478bd9Sstevel@tonic-gate char *key;
1507c478bd9Sstevel@tonic-gate {
1517c478bd9Sstevel@tonic-gate int i;
1527c478bd9Sstevel@tonic-gate
1537c478bd9Sstevel@tonic-gate (void) memset(key, 0, 8);
1547c478bd9Sstevel@tonic-gate for (i = 0; *pw; i = (i+1) % 8) {
1557c478bd9Sstevel@tonic-gate key[i] ^= *pw++ << 1;
1567c478bd9Sstevel@tonic-gate }
1577c478bd9Sstevel@tonic-gate des_setparity(key);
1587c478bd9Sstevel@tonic-gate return (1);
1597c478bd9Sstevel@tonic-gate }
1607c478bd9Sstevel@tonic-gate
1617c478bd9Sstevel@tonic-gate
1627c478bd9Sstevel@tonic-gate /*
1637c478bd9Sstevel@tonic-gate * Hex to binary conversion
1647c478bd9Sstevel@tonic-gate */
1657c478bd9Sstevel@tonic-gate int
hex2bin(len,hexnum,binnum)1667c478bd9Sstevel@tonic-gate hex2bin(len, hexnum, binnum)
1677c478bd9Sstevel@tonic-gate int len;
1687c478bd9Sstevel@tonic-gate char *hexnum;
1697c478bd9Sstevel@tonic-gate char *binnum;
1707c478bd9Sstevel@tonic-gate {
1717c478bd9Sstevel@tonic-gate int i;
1727c478bd9Sstevel@tonic-gate
1737c478bd9Sstevel@tonic-gate for (i = 0; i < len; i++) {
1747c478bd9Sstevel@tonic-gate *binnum++ = 16 * hexval(hexnum[2 * i]) +
1757c478bd9Sstevel@tonic-gate hexval(hexnum[2 * i + 1]);
1767c478bd9Sstevel@tonic-gate }
1777c478bd9Sstevel@tonic-gate return (1);
1787c478bd9Sstevel@tonic-gate }
1797c478bd9Sstevel@tonic-gate
1807c478bd9Sstevel@tonic-gate /*
1817c478bd9Sstevel@tonic-gate * Binary to hex conversion
1827c478bd9Sstevel@tonic-gate */
1837c478bd9Sstevel@tonic-gate int
bin2hex(len,binnum,hexnum)1847c478bd9Sstevel@tonic-gate bin2hex(len, binnum, hexnum)
1857c478bd9Sstevel@tonic-gate int len;
1867c478bd9Sstevel@tonic-gate unsigned char *binnum;
1877c478bd9Sstevel@tonic-gate char *hexnum;
1887c478bd9Sstevel@tonic-gate {
1897c478bd9Sstevel@tonic-gate int i;
1907c478bd9Sstevel@tonic-gate unsigned val;
1917c478bd9Sstevel@tonic-gate
1927c478bd9Sstevel@tonic-gate for (i = 0; i < len; i++) {
1937c478bd9Sstevel@tonic-gate val = binnum[i];
1947c478bd9Sstevel@tonic-gate hexnum[i*2] = hex[val >> 4];
1957c478bd9Sstevel@tonic-gate hexnum[i*2+1] = hex[val & 0xf];
1967c478bd9Sstevel@tonic-gate }
1977c478bd9Sstevel@tonic-gate hexnum[len*2] = 0;
1987c478bd9Sstevel@tonic-gate return (1);
1997c478bd9Sstevel@tonic-gate }
2007c478bd9Sstevel@tonic-gate
2017c478bd9Sstevel@tonic-gate static char hex[16] = {
2027c478bd9Sstevel@tonic-gate '0', '1', '2', '3', '4', '5', '6', '7',
2037c478bd9Sstevel@tonic-gate '8', '9', 'a', 'b', 'c', 'd', 'e', 'f',
2047c478bd9Sstevel@tonic-gate };
2057c478bd9Sstevel@tonic-gate
2067c478bd9Sstevel@tonic-gate static char
hexval(c)2077c478bd9Sstevel@tonic-gate hexval(c)
2087c478bd9Sstevel@tonic-gate char c;
2097c478bd9Sstevel@tonic-gate {
2107c478bd9Sstevel@tonic-gate if (c >= '0' && c <= '9') {
2117c478bd9Sstevel@tonic-gate return (c - '0');
2127c478bd9Sstevel@tonic-gate } else if (c >= 'a' && c <= 'z') {
2137c478bd9Sstevel@tonic-gate return (c - 'a' + 10);
2147c478bd9Sstevel@tonic-gate } else if (c >= 'A' && c <= 'Z') {
2157c478bd9Sstevel@tonic-gate return (c - 'A' + 10);
2167c478bd9Sstevel@tonic-gate } else {
2177c478bd9Sstevel@tonic-gate return (-1);
2187c478bd9Sstevel@tonic-gate }
2197c478bd9Sstevel@tonic-gate }
2207c478bd9Sstevel@tonic-gate
2217c478bd9Sstevel@tonic-gate /*
2227c478bd9Sstevel@tonic-gate * Generic key length/algorithm version of xencrypt().
2237c478bd9Sstevel@tonic-gate *
2247c478bd9Sstevel@tonic-gate * Encrypt a secret key given passwd.
2257c478bd9Sstevel@tonic-gate * The secret key is passed in hex notation.
2267c478bd9Sstevel@tonic-gate * Arg encrypted_secret will be set to point to the encrypted
2277c478bd9Sstevel@tonic-gate * secret key (NUL term, hex notation).
2287c478bd9Sstevel@tonic-gate *
2297c478bd9Sstevel@tonic-gate * Its length must be a multiple of 16 hex digits (64 bits).
2307c478bd9Sstevel@tonic-gate *
2317c478bd9Sstevel@tonic-gate * For 192-0 (AUTH_DES), then encrypt using the same method as xencrypt().
2327c478bd9Sstevel@tonic-gate *
2337c478bd9Sstevel@tonic-gate * If arg do_chksum is TRUE, append the checksum before the encrypt.
2347c478bd9Sstevel@tonic-gate * For 192-0, the checksum is done the same as in xencrypt(). For
2357c478bd9Sstevel@tonic-gate * bigger keys, MD5 is used.
2367c478bd9Sstevel@tonic-gate *
2377c478bd9Sstevel@tonic-gate * Arg netname can be NULL for 192-0.
2387c478bd9Sstevel@tonic-gate */
2397c478bd9Sstevel@tonic-gate int
xencrypt_g(char * secret,keylen_t keylen,algtype_t algtype,const char * passwd,const char netname[],char ** encrypted_secret,bool_t do_chksum)2407c478bd9Sstevel@tonic-gate xencrypt_g(
2417c478bd9Sstevel@tonic-gate char *secret, /* in */
2427c478bd9Sstevel@tonic-gate keylen_t keylen, /* in */
2437c478bd9Sstevel@tonic-gate algtype_t algtype, /* in */
2447c478bd9Sstevel@tonic-gate const char *passwd, /* in */
2457c478bd9Sstevel@tonic-gate const char netname[], /* in */
2467c478bd9Sstevel@tonic-gate char **encrypted_secret, /* out */
2477c478bd9Sstevel@tonic-gate bool_t do_chksum) /* in */
2487c478bd9Sstevel@tonic-gate {
2497c478bd9Sstevel@tonic-gate des_block key;
2507c478bd9Sstevel@tonic-gate char ivec[8];
2517c478bd9Sstevel@tonic-gate char *binkeybuf;
2527c478bd9Sstevel@tonic-gate int err;
2537c478bd9Sstevel@tonic-gate const int classic_des = keylen == 192 && algtype == 0;
2547c478bd9Sstevel@tonic-gate const int hexkeybytes = BITS2NIBBLES(keylen);
2557c478bd9Sstevel@tonic-gate const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
2567c478bd9Sstevel@tonic-gate const int binkeybytes = do_chksum ? keylen/8 + keychecksumsize/2 :
2577c478bd9Sstevel@tonic-gate keylen/8;
2587c478bd9Sstevel@tonic-gate const int bufsize = do_chksum ? hexkeybytes + keychecksumsize + 1 :
2597c478bd9Sstevel@tonic-gate hexkeybytes + 1;
2607c478bd9Sstevel@tonic-gate char *hexkeybuf;
2617c478bd9Sstevel@tonic-gate
26261961e0fSrobinson if (!secret || !keylen || !passwd || !encrypted_secret)
2637c478bd9Sstevel@tonic-gate return (0);
2647c478bd9Sstevel@tonic-gate
2657c478bd9Sstevel@tonic-gate if ((hexkeybuf = malloc(bufsize)) == 0)
2667c478bd9Sstevel@tonic-gate return (0);
2677c478bd9Sstevel@tonic-gate
2687c478bd9Sstevel@tonic-gate (void) memcpy(hexkeybuf, secret, hexkeybytes);
2697c478bd9Sstevel@tonic-gate if (do_chksum)
2707c478bd9Sstevel@tonic-gate if (classic_des) {
2717c478bd9Sstevel@tonic-gate (void) memcpy(hexkeybuf + hexkeybytes, secret,
2727c478bd9Sstevel@tonic-gate keychecksumsize);
2737c478bd9Sstevel@tonic-gate } else {
2747c478bd9Sstevel@tonic-gate MD5_CTX md5_ctx;
2757c478bd9Sstevel@tonic-gate char md5hexbuf[MD5HEXSIZE + 1] = {0};
2767c478bd9Sstevel@tonic-gate uint8_t digest[MD5HEXSIZE/2];
2777c478bd9Sstevel@tonic-gate
2787c478bd9Sstevel@tonic-gate MD5Init(&md5_ctx);
2797c478bd9Sstevel@tonic-gate MD5Update(&md5_ctx, (unsigned char *)hexkeybuf,
2807c478bd9Sstevel@tonic-gate hexkeybytes);
2817c478bd9Sstevel@tonic-gate MD5Final(digest, &md5_ctx);
2827c478bd9Sstevel@tonic-gate
2837c478bd9Sstevel@tonic-gate /* convert md5 binary digest to hex */
2847c478bd9Sstevel@tonic-gate (void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
2857c478bd9Sstevel@tonic-gate
2867c478bd9Sstevel@tonic-gate /* append the hex md5 string to the end of the key */
2877c478bd9Sstevel@tonic-gate (void) memcpy(hexkeybuf + hexkeybytes,
2887c478bd9Sstevel@tonic-gate (void *)md5hexbuf, MD5HEXSIZE);
2897c478bd9Sstevel@tonic-gate }
2907c478bd9Sstevel@tonic-gate hexkeybuf[bufsize - 1] = 0;
2917c478bd9Sstevel@tonic-gate
2927c478bd9Sstevel@tonic-gate if (binkeybytes > MAX_KEY_CRYPT_LEN) {
2937c478bd9Sstevel@tonic-gate free(hexkeybuf);
2947c478bd9Sstevel@tonic-gate return (0);
2957c478bd9Sstevel@tonic-gate }
2967c478bd9Sstevel@tonic-gate if ((binkeybuf = malloc((unsigned)binkeybytes)) == 0) {
2977c478bd9Sstevel@tonic-gate free(hexkeybuf);
2987c478bd9Sstevel@tonic-gate return (0);
2997c478bd9Sstevel@tonic-gate }
3007c478bd9Sstevel@tonic-gate
3017c478bd9Sstevel@tonic-gate (void) hex2bin(binkeybytes, hexkeybuf, binkeybuf);
3027c478bd9Sstevel@tonic-gate if (classic_des)
3037c478bd9Sstevel@tonic-gate (void) passwd2des((char *)passwd, key.c);
3047c478bd9Sstevel@tonic-gate else
3057c478bd9Sstevel@tonic-gate if (netname)
3067c478bd9Sstevel@tonic-gate (void) passwd2des_g(passwd, netname,
3077c478bd9Sstevel@tonic-gate (int)strlen(netname), &key, FALSE);
3087c478bd9Sstevel@tonic-gate else {
3097c478bd9Sstevel@tonic-gate free(hexkeybuf);
3107c478bd9Sstevel@tonic-gate return (0);
3117c478bd9Sstevel@tonic-gate }
3127c478bd9Sstevel@tonic-gate
3137c478bd9Sstevel@tonic-gate (void) memset(ivec, 0, 8);
3147c478bd9Sstevel@tonic-gate
3157c478bd9Sstevel@tonic-gate err = cbc_crypt(key.c, binkeybuf, binkeybytes, DES_ENCRYPT | DES_HW,
3167c478bd9Sstevel@tonic-gate ivec);
3177c478bd9Sstevel@tonic-gate if (DES_FAILED(err)) {
3187c478bd9Sstevel@tonic-gate free(hexkeybuf);
3197c478bd9Sstevel@tonic-gate free(binkeybuf);
3207c478bd9Sstevel@tonic-gate return (0);
3217c478bd9Sstevel@tonic-gate }
3227c478bd9Sstevel@tonic-gate (void) bin2hex(binkeybytes, (unsigned char *) binkeybuf, hexkeybuf);
3237c478bd9Sstevel@tonic-gate free(binkeybuf);
3247c478bd9Sstevel@tonic-gate *encrypted_secret = hexkeybuf;
3257c478bd9Sstevel@tonic-gate return (1);
3267c478bd9Sstevel@tonic-gate }
3277c478bd9Sstevel@tonic-gate
3287c478bd9Sstevel@tonic-gate /*
3297c478bd9Sstevel@tonic-gate * Generic key len and alg type for version of xdecrypt.
3307c478bd9Sstevel@tonic-gate *
3317c478bd9Sstevel@tonic-gate * Decrypt secret key using passwd. The decrypted secret key
3327c478bd9Sstevel@tonic-gate * *overwrites* the supplied encrypted secret key.
3337c478bd9Sstevel@tonic-gate * The secret key is passed and returned in hex notation.
3347c478bd9Sstevel@tonic-gate * Once again, the length is a multiple of 16 hex digits.
3357c478bd9Sstevel@tonic-gate *
3367c478bd9Sstevel@tonic-gate * If 'do_chksum' is TRUE, the 'secret' buffer is assumed to contain
3377c478bd9Sstevel@tonic-gate * a checksum calculated by a call to xencrypt_g().
3387c478bd9Sstevel@tonic-gate *
3397c478bd9Sstevel@tonic-gate * If keylen is 192 and algtype is 0, then decrypt the same way
3407c478bd9Sstevel@tonic-gate * as xdecrypt().
3417c478bd9Sstevel@tonic-gate *
3427c478bd9Sstevel@tonic-gate * Arg netname can be NULL for 192-0.
3437c478bd9Sstevel@tonic-gate */
3447c478bd9Sstevel@tonic-gate int
xdecrypt_g(char * secret,int keylen,int algtype,const char * passwd,const char netname[],bool_t do_chksum)3457c478bd9Sstevel@tonic-gate xdecrypt_g(
3467c478bd9Sstevel@tonic-gate char *secret, /* out */
3477c478bd9Sstevel@tonic-gate int keylen, /* in */
3487c478bd9Sstevel@tonic-gate int algtype, /* in */
3497c478bd9Sstevel@tonic-gate const char *passwd, /* in */
3507c478bd9Sstevel@tonic-gate const char netname[], /* in */
3517c478bd9Sstevel@tonic-gate bool_t do_chksum) /* in */
3527c478bd9Sstevel@tonic-gate {
3537c478bd9Sstevel@tonic-gate des_block key;
3547c478bd9Sstevel@tonic-gate char ivec[8];
3557c478bd9Sstevel@tonic-gate char *buf;
3567c478bd9Sstevel@tonic-gate int err;
3577c478bd9Sstevel@tonic-gate int len;
3587c478bd9Sstevel@tonic-gate const int classic_des = keylen == 192 && algtype == 0;
3597c478bd9Sstevel@tonic-gate const int hexkeybytes = BITS2NIBBLES(keylen);
3607c478bd9Sstevel@tonic-gate const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
3617c478bd9Sstevel@tonic-gate
3627c478bd9Sstevel@tonic-gate len = (int)strlen(secret) / 2;
36361961e0fSrobinson if (len > MAX_KEY_CRYPT_LEN)
3647c478bd9Sstevel@tonic-gate return (0);
3657c478bd9Sstevel@tonic-gate if ((buf = malloc((unsigned)len)) == 0)
3667c478bd9Sstevel@tonic-gate return (0);
3677c478bd9Sstevel@tonic-gate
3687c478bd9Sstevel@tonic-gate (void) hex2bin(len, secret, buf);
3697c478bd9Sstevel@tonic-gate if (classic_des)
3707c478bd9Sstevel@tonic-gate (void) passwd2des((char *)passwd, key.c);
3717c478bd9Sstevel@tonic-gate else
3727c478bd9Sstevel@tonic-gate if (netname)
3737c478bd9Sstevel@tonic-gate (void) passwd2des_g(passwd, netname,
3747c478bd9Sstevel@tonic-gate (int)strlen(netname), &key, FALSE);
3757c478bd9Sstevel@tonic-gate else {
3767c478bd9Sstevel@tonic-gate free(buf);
3777c478bd9Sstevel@tonic-gate return (0);
3787c478bd9Sstevel@tonic-gate }
3797c478bd9Sstevel@tonic-gate (void) memset(ivec, 0, 8);
3807c478bd9Sstevel@tonic-gate
3817c478bd9Sstevel@tonic-gate err = cbc_crypt(key.c, buf, len, DES_DECRYPT | DES_HW, ivec);
3827c478bd9Sstevel@tonic-gate if (DES_FAILED(err)) {
3837c478bd9Sstevel@tonic-gate free(buf);
3847c478bd9Sstevel@tonic-gate return (0);
3857c478bd9Sstevel@tonic-gate }
3867c478bd9Sstevel@tonic-gate (void) bin2hex(len, (unsigned char *) buf, secret);
3877c478bd9Sstevel@tonic-gate free(buf);
3887c478bd9Sstevel@tonic-gate
3897c478bd9Sstevel@tonic-gate if (do_chksum)
3907c478bd9Sstevel@tonic-gate if (classic_des) {
3917c478bd9Sstevel@tonic-gate if (memcmp(secret, &(secret[hexkeybytes]),
3927c478bd9Sstevel@tonic-gate keychecksumsize) != 0) {
3937c478bd9Sstevel@tonic-gate secret[0] = 0;
3947c478bd9Sstevel@tonic-gate return (0);
3957c478bd9Sstevel@tonic-gate }
3967c478bd9Sstevel@tonic-gate } else {
3977c478bd9Sstevel@tonic-gate MD5_CTX md5_ctx;
3987c478bd9Sstevel@tonic-gate char md5hexbuf[MD5HEXSIZE + 1] = {0};
3997c478bd9Sstevel@tonic-gate uint8_t digest[MD5HEXSIZE/2];
4007c478bd9Sstevel@tonic-gate
4017c478bd9Sstevel@tonic-gate MD5Init(&md5_ctx);
4027c478bd9Sstevel@tonic-gate MD5Update(&md5_ctx, (unsigned char *)secret,
4037c478bd9Sstevel@tonic-gate hexkeybytes);
4047c478bd9Sstevel@tonic-gate MD5Final(digest, &md5_ctx);
4057c478bd9Sstevel@tonic-gate
4067c478bd9Sstevel@tonic-gate /* convert md5 binary digest to hex */
4077c478bd9Sstevel@tonic-gate (void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
4087c478bd9Sstevel@tonic-gate
4097c478bd9Sstevel@tonic-gate /* does the digest match the appended one? */
4107c478bd9Sstevel@tonic-gate if (memcmp(&(secret[hexkeybytes]),
41161961e0fSrobinson md5hexbuf, MD5HEXSIZE) != 0) {
4127c478bd9Sstevel@tonic-gate secret[0] = 0;
4137c478bd9Sstevel@tonic-gate return (0);
4147c478bd9Sstevel@tonic-gate }
4157c478bd9Sstevel@tonic-gate }
4167c478bd9Sstevel@tonic-gate
4177c478bd9Sstevel@tonic-gate secret[hexkeybytes] = '\0';
4187c478bd9Sstevel@tonic-gate
4197c478bd9Sstevel@tonic-gate return (1);
4207c478bd9Sstevel@tonic-gate }
4217c478bd9Sstevel@tonic-gate
4227c478bd9Sstevel@tonic-gate
4237c478bd9Sstevel@tonic-gate /*
4247c478bd9Sstevel@tonic-gate * Modified version of passwd2des(). passwd2des_g() uses the Kerberos
4257c478bd9Sstevel@tonic-gate * RFC 1510 algorithm to generate a DES key from a user password
4267c478bd9Sstevel@tonic-gate * and mix-in string. The mix-in is expected to be the netname.
4277c478bd9Sstevel@tonic-gate * This function to be used only for extended Diffie-Hellman keys.
4287c478bd9Sstevel@tonic-gate *
4297c478bd9Sstevel@tonic-gate * If altarg is TRUE, reverse the concat of passwd and mix-in.
4307c478bd9Sstevel@tonic-gate */
4317c478bd9Sstevel@tonic-gate int
passwd2des_g(const char * pw,const char * mixin,int len,des_block * key,bool_t altalg)4327c478bd9Sstevel@tonic-gate passwd2des_g(
4337c478bd9Sstevel@tonic-gate const char *pw,
4347c478bd9Sstevel@tonic-gate const char *mixin,
4357c478bd9Sstevel@tonic-gate int len,
4367c478bd9Sstevel@tonic-gate des_block *key, /* out */
4377c478bd9Sstevel@tonic-gate bool_t altalg)
4387c478bd9Sstevel@tonic-gate {
4397c478bd9Sstevel@tonic-gate
4407c478bd9Sstevel@tonic-gate int i, j, incr = 1;
4417c478bd9Sstevel@tonic-gate des_block ivec, tkey;
4427c478bd9Sstevel@tonic-gate char *text;
4437c478bd9Sstevel@tonic-gate int plen, tlen;
4447c478bd9Sstevel@tonic-gate
4457c478bd9Sstevel@tonic-gate (void) memset(tkey.c, 0, 8);
4467c478bd9Sstevel@tonic-gate (void) memset(ivec.c, 0, 8);
4477c478bd9Sstevel@tonic-gate
4487c478bd9Sstevel@tonic-gate
4497c478bd9Sstevel@tonic-gate /*
4507c478bd9Sstevel@tonic-gate * Concatentate the password and the mix-in string, fan-fold and XOR them
4517c478bd9Sstevel@tonic-gate * to the required eight byte initial DES key. Since passwords can be
4527c478bd9Sstevel@tonic-gate * expected to use mostly seven bit ASCII, left shift the password one
4537c478bd9Sstevel@tonic-gate * bit in order to preserve as much key space as possible.
4547c478bd9Sstevel@tonic-gate */
4557c478bd9Sstevel@tonic-gate
4567c478bd9Sstevel@tonic-gate #define KEYLEN sizeof (tkey.c)
4577c478bd9Sstevel@tonic-gate plen = strlen(pw);
4587c478bd9Sstevel@tonic-gate tlen = ((plen + len + (KEYLEN-1))/KEYLEN)*KEYLEN;
45961961e0fSrobinson if ((text = malloc(tlen)) == NULL) {
4607c478bd9Sstevel@tonic-gate return (0);
4617c478bd9Sstevel@tonic-gate }
4627c478bd9Sstevel@tonic-gate
4637c478bd9Sstevel@tonic-gate (void) memset(text, 0, tlen);
4647c478bd9Sstevel@tonic-gate
4657c478bd9Sstevel@tonic-gate if (!altalg) {
4667c478bd9Sstevel@tonic-gate
4677c478bd9Sstevel@tonic-gate /*
4687c478bd9Sstevel@tonic-gate * Concatenate the password and the mix-in string, fan-fold and XOR them
4697c478bd9Sstevel@tonic-gate * to the required eight byte initial DES key. Since passwords can be
4707c478bd9Sstevel@tonic-gate * expected to use mostly seven bit ASCII, left shift the password one
4717c478bd9Sstevel@tonic-gate * bit in order to preserve as much key space as possible.
4727c478bd9Sstevel@tonic-gate */
4737c478bd9Sstevel@tonic-gate (void) memcpy(text, pw, plen);
4747c478bd9Sstevel@tonic-gate (void) memcpy(&text[plen], mixin, len);
4757c478bd9Sstevel@tonic-gate
4767c478bd9Sstevel@tonic-gate for (i = 0, j = 0; pw[j]; j++) {
4777c478bd9Sstevel@tonic-gate tkey.c[i] ^= pw[j] << 1;
4787c478bd9Sstevel@tonic-gate i += incr;
4797c478bd9Sstevel@tonic-gate if (i == 8) {
4807c478bd9Sstevel@tonic-gate i = 7;
4817c478bd9Sstevel@tonic-gate incr = -incr;
4827c478bd9Sstevel@tonic-gate } else if (i == -1) {
4837c478bd9Sstevel@tonic-gate i = 0;
4847c478bd9Sstevel@tonic-gate incr = -incr;
4857c478bd9Sstevel@tonic-gate }
4867c478bd9Sstevel@tonic-gate }
4877c478bd9Sstevel@tonic-gate
4887c478bd9Sstevel@tonic-gate for (j = 0; j < len; j++) {
4897c478bd9Sstevel@tonic-gate tkey.c[i] ^= mixin[j];
4907c478bd9Sstevel@tonic-gate i += incr;
4917c478bd9Sstevel@tonic-gate if (i == 8) {
4927c478bd9Sstevel@tonic-gate i = 7;
4937c478bd9Sstevel@tonic-gate incr = -incr;
4947c478bd9Sstevel@tonic-gate } else if (i == -1) {
4957c478bd9Sstevel@tonic-gate i = 0;
4967c478bd9Sstevel@tonic-gate incr = -incr;
4977c478bd9Sstevel@tonic-gate }
4987c478bd9Sstevel@tonic-gate }
4997c478bd9Sstevel@tonic-gate } else { /* use alternative algorithm */
5007c478bd9Sstevel@tonic-gate (void) memcpy(text, mixin, len);
5017c478bd9Sstevel@tonic-gate (void) memcpy(&text[len], pw, plen);
5027c478bd9Sstevel@tonic-gate
5037c478bd9Sstevel@tonic-gate for (i = 0, j = 0; j < len; j++) {
5047c478bd9Sstevel@tonic-gate tkey.c[i] ^= mixin[j];
5057c478bd9Sstevel@tonic-gate i += incr;
5067c478bd9Sstevel@tonic-gate if (i == 8) {
5077c478bd9Sstevel@tonic-gate i = 7;
5087c478bd9Sstevel@tonic-gate incr = -incr;
5097c478bd9Sstevel@tonic-gate } else if (i == -1) {
5107c478bd9Sstevel@tonic-gate i = 0;
5117c478bd9Sstevel@tonic-gate incr = -incr;
5127c478bd9Sstevel@tonic-gate }
5137c478bd9Sstevel@tonic-gate }
5147c478bd9Sstevel@tonic-gate
5157c478bd9Sstevel@tonic-gate for (j = 0; pw[j]; j++) {
5167c478bd9Sstevel@tonic-gate tkey.c[i] ^= pw[j] << 1;
5177c478bd9Sstevel@tonic-gate i += incr;
5187c478bd9Sstevel@tonic-gate if (i == 8) {
5197c478bd9Sstevel@tonic-gate i = 7;
5207c478bd9Sstevel@tonic-gate incr = -incr;
5217c478bd9Sstevel@tonic-gate } else if (i == -1) {
5227c478bd9Sstevel@tonic-gate i = 0;
5237c478bd9Sstevel@tonic-gate incr = -incr;
5247c478bd9Sstevel@tonic-gate }
5257c478bd9Sstevel@tonic-gate }
5267c478bd9Sstevel@tonic-gate }
5277c478bd9Sstevel@tonic-gate des_setparity_g(&tkey);
5287c478bd9Sstevel@tonic-gate
5297c478bd9Sstevel@tonic-gate /*
5307c478bd9Sstevel@tonic-gate * Use the temporary key to produce a DES CBC checksum for the text
5317c478bd9Sstevel@tonic-gate * string; cbc_crypt returns the checksum in the ivec.
5327c478bd9Sstevel@tonic-gate */
5337c478bd9Sstevel@tonic-gate (void) cbc_crypt(tkey.c, text, tlen, DES_ENCRYPT|DES_HW, ivec.c);
5347c478bd9Sstevel@tonic-gate des_setparity_g(&ivec);
5357c478bd9Sstevel@tonic-gate free(text);
5367c478bd9Sstevel@tonic-gate
5377c478bd9Sstevel@tonic-gate if (weak_DES_key(ivec)) {
5387c478bd9Sstevel@tonic-gate ivec.c[7] ^= 0xf0;
5397c478bd9Sstevel@tonic-gate /*
5407c478bd9Sstevel@tonic-gate * XORing with 0xf0 preserves parity, so no need to check
5417c478bd9Sstevel@tonic-gate * that again.
5427c478bd9Sstevel@tonic-gate */
5437c478bd9Sstevel@tonic-gate }
5447c478bd9Sstevel@tonic-gate
5457c478bd9Sstevel@tonic-gate (void) memcpy((*key).c, ivec.c, sizeof (ivec.c));
5467c478bd9Sstevel@tonic-gate
5477c478bd9Sstevel@tonic-gate return (1);
5487c478bd9Sstevel@tonic-gate
5497c478bd9Sstevel@tonic-gate }
5507c478bd9Sstevel@tonic-gate
5517c478bd9Sstevel@tonic-gate struct DESkey {
5527c478bd9Sstevel@tonic-gate uint32_t h1;
5537c478bd9Sstevel@tonic-gate uint32_t h2;
5547c478bd9Sstevel@tonic-gate };
5557c478bd9Sstevel@tonic-gate
5567c478bd9Sstevel@tonic-gate /*
5577c478bd9Sstevel@tonic-gate * Weak and semiweak keys from "Applied Cryptography", second edition,
5587c478bd9Sstevel@tonic-gate * by Bruce Schneier, Wiley 1996.
5597c478bd9Sstevel@tonic-gate */
5607c478bd9Sstevel@tonic-gate static struct DESkey weakDESkeys[] = {
5617c478bd9Sstevel@tonic-gate /* Weak keys */
5627c478bd9Sstevel@tonic-gate {0x01010101, 0x01010101},
5637c478bd9Sstevel@tonic-gate {0x1f1f1f1f, 0x1f1f1f1f},
5647c478bd9Sstevel@tonic-gate {0xe0e0e0e0, 0xe0e0e0e0},
5657c478bd9Sstevel@tonic-gate {0xfefefefe, 0xfefefefe},
5667c478bd9Sstevel@tonic-gate /* Semiweak keys */
5677c478bd9Sstevel@tonic-gate {0x01fe01fe, 0x01fe01fe},
5687c478bd9Sstevel@tonic-gate {0x1fe01fe0, 0x0ef10ef1},
5697c478bd9Sstevel@tonic-gate {0x01e001e0, 0x01f101f1},
5707c478bd9Sstevel@tonic-gate {0x1ffe1ffe, 0x0efe0efe},
5717c478bd9Sstevel@tonic-gate {0x011f011f, 0x010e010e},
5727c478bd9Sstevel@tonic-gate {0xe0fee0fe, 0xf1fef1fe},
5737c478bd9Sstevel@tonic-gate {0xfe01fe01, 0xfe01fe01},
5747c478bd9Sstevel@tonic-gate {0xe01fe01f, 0xf10ef10e},
5757c478bd9Sstevel@tonic-gate {0xe001e001, 0xf101f101},
5767c478bd9Sstevel@tonic-gate {0xfe1ffe1f, 0xfe0efe0e},
5777c478bd9Sstevel@tonic-gate {0x1f011f01, 0x0e010e01},
5787c478bd9Sstevel@tonic-gate {0xfee0fee0, 0xfef1fef1}
5797c478bd9Sstevel@tonic-gate };
5807c478bd9Sstevel@tonic-gate
5817c478bd9Sstevel@tonic-gate static int
weak_DES_key(des_block db)5827c478bd9Sstevel@tonic-gate weak_DES_key(des_block db)
5837c478bd9Sstevel@tonic-gate {
5847c478bd9Sstevel@tonic-gate int i;
5857c478bd9Sstevel@tonic-gate
5867c478bd9Sstevel@tonic-gate for (i = 0; i < sizeof (weakDESkeys)/sizeof (struct DESkey); i++) {
5877c478bd9Sstevel@tonic-gate if (weakDESkeys[i].h1 == db.key.high &&
58861961e0fSrobinson weakDESkeys[i].h2 == db.key.low)
5897c478bd9Sstevel@tonic-gate return (1);
5907c478bd9Sstevel@tonic-gate }
5917c478bd9Sstevel@tonic-gate
5927c478bd9Sstevel@tonic-gate return (0);
5937c478bd9Sstevel@tonic-gate }
594