xref: /illumos-gate/usr/src/lib/gss_mechs/mech_krb5/crypto/keyhash_provider/k5_md5des.c (revision 004388ebfdfe2ed7dfd2d153a876dfcc22d2c006)
1 /*
2  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
3  * Use is subject to license terms.
4  */
5 
6 #pragma ident	"%Z%%M%	%I%	%E% SMI"
7 
8 /*
9  * Copyright (C) 1998 by the FundsXpress, INC.
10  *
11  * All rights reserved.
12  *
13  * Export of this software from the United States of America may require
14  * a specific license from the United States Government.  It is the
15  * responsibility of any person or organization contemplating export to
16  * obtain such a license before exporting.
17  *
18  * WITHIN THAT CONSTRAINT, permission to use, copy, modify, and
19  * distribute this software and its documentation for any purpose and
20  * without fee is hereby granted, provided that the above copyright
21  * notice appear in all copies and that both that copyright notice and
22  * this permission notice appear in supporting documentation, and that
23  * the name of FundsXpress. not be used in advertising or publicity pertaining
24  * to distribution of the software without specific, written prior
25  * permission.  FundsXpress makes no representations about the suitability of
26  * this software for any purpose.  It is provided "as is" without express
27  * or implied warranty.
28  *
29  * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
30  * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
31  * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
32  */
33 
34 #include <k5-int.h>
35 #include <des_int.h>
36 #include <keyhash_provider.h>
37 
38 #define CONFLENGTH 8
39 
40 /* Force acceptance of krb5-beta5 md5des checksum for now. */
41 #define KRB5_MD5DES_BETA5_COMPAT
42 
43 /* des-cbc(xorkey, conf | rsa-md5(conf | data)) */
44 
45 /* this could be done in terms of the md5 and des providers, but
46    that's less efficient, and there's no need for this to be generic */
47 
48 /*ARGSUSED*/
49 static krb5_error_code
50 k5_md5des_hash(krb5_context context,
51 	krb5_const krb5_keyblock *key,
52 	krb5_keyusage usage,
53 	krb5_const krb5_data *ivec,
54 	krb5_const krb5_data *input, krb5_data *output)
55 {
56     krb5_error_code ret = 0;
57     krb5_data data;
58     unsigned char conf[CONFLENGTH];
59     krb5_keyblock xorkey;
60     int i;
61     CK_MECHANISM mechanism;
62     CK_RV rv;
63     CK_ULONG hashlen = MD5_CKSUM_LENGTH;
64 
65     if (key->length != 8)
66 	return(KRB5_BAD_KEYSIZE);
67     if (ivec)
68 	return(KRB5_CRYPTO_INTERNAL);
69     if (output->length != (CONFLENGTH+MD5_CKSUM_LENGTH))
70 	return(KRB5_CRYPTO_INTERNAL);
71 
72     /* create the confouder */
73 
74     data.length = CONFLENGTH;
75     data.data = (char *) conf;
76     if ((ret = krb5_c_random_make_octets(context, &data)))
77 	return(ret);
78 
79     xorkey.magic = key->magic;
80     xorkey.enctype = key->enctype;
81     xorkey.length = key->length;
82     xorkey.contents = (krb5_octet *)malloc(key->length);
83     if (xorkey.contents == NULL)
84 	return(KRB5_CRYPTO_INTERNAL);
85 
86     (void) memcpy(xorkey.contents, key->contents, xorkey.length);
87 
88     for (i=0; i<xorkey.length; i++)
89 	xorkey.contents[i] ^= 0xf0;
90 
91     if (!mit_des_check_key_parity(xorkey.contents)) {
92 	ret = KRB5DES_BAD_KEYPAR;
93 	goto cleanup;
94     }
95 
96     if (mit_des_is_weak_key(xorkey.contents)) {
97 	ret = KRB5DES_WEAK_KEY;
98 	goto cleanup;
99     }
100 
101     /* hash the confounder, then the input data */
102     mechanism.mechanism = CKM_MD5;
103     mechanism.pParameter = NULL_PTR;
104     mechanism.ulParameterLen = 0;
105 
106     if ((rv = C_DigestInit(krb_ctx_hSession(context), &mechanism)) != CKR_OK) {
107 	KRB5_LOG(KRB5_ERR, "C_DigestInit failed in k5_md5des_hash: "
108 	"rv = 0x%x.", rv);
109 	ret = PKCS_ERR;
110 	goto cleanup;
111     }
112 
113     if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
114 	(CK_BYTE_PTR)conf, (CK_ULONG)sizeof(conf))) != CKR_OK) {
115 	KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_hash: "
116 	    "rv = 0x%x", rv);
117 	ret = PKCS_ERR;
118 	goto cleanup;
119     }
120 
121     if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
122 	(CK_BYTE_PTR)input->data, (CK_ULONG)input->length)) != CKR_OK) {
123 	KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_hash: "
124 	    "rv = 0x%x", rv);
125 	return(PKCS_ERR);
126     }
127 
128     if ((rv = C_DigestFinal(krb_ctx_hSession(context),
129 	(CK_BYTE_PTR)(output->data + CONFLENGTH),
130 	(CK_ULONG_PTR)&hashlen)) != CKR_OK) {
131 	KRB5_LOG(KRB5_ERR, "C_DigestFinal failed in k5_md5des_hash: "
132 	    "rv = 0x%x", rv);
133 	ret = PKCS_ERR;
134 	goto cleanup;
135     }
136 
137     /* construct the buffer to be encrypted */
138 
139     (void) memcpy(output->data, conf, CONFLENGTH);
140 
141     /* encrypt it, in place.  this has a return value, but it's
142        always zero.  */
143 
144     ret = mit_des_cbc_encrypt(context,
145 	(krb5_pointer) output->data,
146 	(krb5_pointer) output->data, output->length,
147 	&xorkey, (unsigned char*) mit_des_zeroblock, 1);
148 
149 cleanup:
150     free(xorkey.contents);
151     return(ret);
152 }
153 
154 /*ARGSUSED*/
155 static krb5_error_code
156 k5_md5des_verify(krb5_context context,
157 	krb5_const krb5_keyblock *key,
158 	krb5_keyusage usage,
159 	krb5_const krb5_data *ivec,
160 	krb5_const krb5_data *input,
161 	krb5_const krb5_data *hash,
162 	krb5_boolean *valid)
163 {
164     krb5_error_code ret = 0;
165     unsigned char plaintext[CONFLENGTH+MD5_CKSUM_LENGTH];
166     unsigned char digest[MD5_CKSUM_LENGTH];
167     krb5_keyblock xorkey;
168     int i;
169     int compathash = 0;
170     CK_MECHANISM mechanism;
171     CK_RV rv;
172     CK_ULONG hashlen = MD5_CKSUM_LENGTH;
173 
174     if (key->length != 8)
175 	return(KRB5_BAD_KEYSIZE);
176     if (ivec)
177 	return(KRB5_CRYPTO_INTERNAL);
178     if (hash->length != (CONFLENGTH + MD5_CKSUM_LENGTH)) {
179 #ifdef KRB5_MD5DES_BETA5_COMPAT
180 	if (hash->length != MD5_CKSUM_LENGTH)
181 	    return(KRB5_CRYPTO_INTERNAL);
182 	else
183 	    compathash = 1;
184 #else
185 	return(KRB5_CRYPTO_INTERNAL);
186 #endif
187     }
188 
189     /* create and the encryption key */
190     xorkey.magic = key->magic;
191     xorkey.enctype = key->enctype;
192     xorkey.length = key->length;
193     xorkey.contents = (krb5_octet *)malloc(key->length);
194     if (xorkey.contents == NULL)
195 	return(KRB5_CRYPTO_INTERNAL);
196 
197     (void) memcpy(xorkey.contents, key->contents, xorkey.length);
198     if (!compathash) {
199         for (i=0; i<xorkey.length; i++)
200 	    xorkey.contents[i] ^= 0xf0;
201     }
202 
203     if (!mit_des_check_key_parity(xorkey.contents)) {
204 	ret = KRB5DES_BAD_KEYPAR;
205 	goto cleanup;
206     }
207 
208     if (mit_des_is_weak_key(xorkey.contents)) {
209 	ret = KRB5DES_WEAK_KEY;
210 	goto cleanup;
211     }
212 
213     /* decrypt it.  this has a return value, but it's always zero.  */
214     if (!compathash) {
215 	ret = mit_des_cbc_encrypt(context,
216 		(krb5_pointer) hash->data,
217 		(krb5_pointer) plaintext, hash->length,
218 		&xorkey, (unsigned char*) mit_des_zeroblock, 0);
219     } else {
220 	ret = mit_des_cbc_encrypt(context,
221 		(krb5_pointer) hash->data,
222 		(krb5_pointer) plaintext, hash->length,
223 		&xorkey, xorkey.contents, 0);
224     }
225     if (ret) goto cleanup;
226 
227     /* hash the confounder, then the input data */
228     mechanism.mechanism = CKM_MD5;
229     mechanism.pParameter = NULL_PTR;
230     mechanism.ulParameterLen = 0;
231 
232     if ((rv = C_DigestInit(krb_ctx_hSession(context), &mechanism)) != CKR_OK) {
233 	KRB5_LOG(KRB5_ERR, "C_DigestInit failed in k5_md5des_verify: "
234 	"rv = 0x%x.", rv);
235 	ret = PKCS_ERR;
236 	goto cleanup;
237     }
238 
239     if (!compathash) {
240 	if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
241 	    (CK_BYTE_PTR)plaintext, (CK_ULONG)CONFLENGTH)) != CKR_OK) {
242 	    KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_verify: "
243 		"rv = 0x%x", rv);
244 	    ret = PKCS_ERR;
245 	    goto cleanup;
246 	}
247     }
248     if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
249 	(CK_BYTE_PTR)input->data, (CK_ULONG)input->length)) != CKR_OK) {
250 	KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_verify: "
251 	    "rv = 0x%x", rv);
252 	ret = PKCS_ERR;
253 	goto cleanup;
254     }
255     if ((rv = C_DigestFinal(krb_ctx_hSession(context),
256 	(CK_BYTE_PTR)digest, (CK_ULONG_PTR)&hashlen)) != CKR_OK) {
257 	KRB5_LOG(KRB5_ERR, "C_DigestFinal failed in k5_md5des_verify: "
258 	    "rv = 0x%x", rv);
259 	ret = PKCS_ERR;
260 	goto cleanup;
261     }
262 
263     /* compare the decrypted hash to the computed one */
264 
265     if (!compathash) {
266 	*valid = (memcmp(plaintext+CONFLENGTH, digest, sizeof(digest)) == 0);
267     } else {
268 	*valid = (memcmp(plaintext, digest, sizeof(digest)) == 0);
269     }
270     (void) memset(plaintext, 0, sizeof(plaintext));
271 
272 cleanup:
273     free(xorkey.contents);
274     return(ret);
275 }
276 
277 const struct krb5_keyhash_provider krb5_keyhash_md5des = {
278     CONFLENGTH + MD5_CKSUM_LENGTH,
279     k5_md5des_hash,
280     k5_md5des_verify
281 };
282