xref: /illumos-gate/usr/src/lib/gss_mechs/mech_krb5/crypto/keyhash_provider/k5_md5des.c (revision 440a8a36792bdf9ef51639066aab0b7771ffcab8)
1 /*
2  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
3  * Use is subject to license terms.
4  */
5 
6 
7 /*
8  * Copyright (C) 1998 by the FundsXpress, INC.
9  *
10  * All rights reserved.
11  *
12  * Export of this software from the United States of America may require
13  * a specific license from the United States Government.  It is the
14  * responsibility of any person or organization contemplating export to
15  * obtain such a license before exporting.
16  *
17  * WITHIN THAT CONSTRAINT, permission to use, copy, modify, and
18  * distribute this software and its documentation for any purpose and
19  * without fee is hereby granted, provided that the above copyright
20  * notice appear in all copies and that both that copyright notice and
21  * this permission notice appear in supporting documentation, and that
22  * the name of FundsXpress. not be used in advertising or publicity pertaining
23  * to distribution of the software without specific, written prior
24  * permission.  FundsXpress makes no representations about the suitability of
25  * this software for any purpose.  It is provided "as is" without express
26  * or implied warranty.
27  *
28  * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
29  * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
30  * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
31  */
32 
33 #include "k5-int.h"
34 #include "des_int.h"
35 #include "keyhash_provider.h"
36 
37 #define CONFLENGTH 8
38 
39 /* Force acceptance of krb5-beta5 md5des checksum for now. */
40 #define KRB5_MD5DES_BETA5_COMPAT
41 
42 /* des-cbc(xorkey, conf | rsa-md5(conf | data)) */
43 
44 /* this could be done in terms of the md5 and des providers, but
45    that's less efficient, and there's no need for this to be generic */
46 
47 /*ARGSUSED*/
48 static krb5_error_code
49 k5_md5des_hash(krb5_context context, krb5_const krb5_keyblock *key,
50 	       krb5_keyusage usage, const krb5_data *ivec,
51 	       const krb5_data *input, krb5_data *output)
52 {
53     krb5_error_code ret = 0;
54     krb5_data data;
55     unsigned char conf[CONFLENGTH];
56     krb5_keyblock xorkey;
57     int i;
58     CK_MECHANISM mechanism;
59     CK_RV rv;
60     CK_ULONG hashlen = MD5_CKSUM_LENGTH;
61 
62     if (key->length != 8)
63 	return(KRB5_BAD_KEYSIZE);
64     if (ivec)
65 	return(KRB5_CRYPTO_INTERNAL);
66     if (output->length != (CONFLENGTH+MD5_CKSUM_LENGTH))
67 	return(KRB5_CRYPTO_INTERNAL);
68 
69     /* create the confouder */
70 
71     data.length = CONFLENGTH;
72     data.data = (char *) conf;
73     if ((ret = krb5_c_random_make_octets(context, &data)))
74 	return(ret);
75 
76     xorkey.magic = key->magic;
77     xorkey.enctype = key->enctype;
78     xorkey.length = key->length;
79     xorkey.contents = (krb5_octet *)malloc(key->length);
80     if (xorkey.contents == NULL)
81 	return(KRB5_CRYPTO_INTERNAL);
82 
83     (void) memcpy(xorkey.contents, key->contents, xorkey.length);
84 
85     for (i=0; i<xorkey.length; i++)
86 	xorkey.contents[i] ^= 0xf0;
87 
88     if (!mit_des_check_key_parity(xorkey.contents)) {
89 	ret = KRB5DES_BAD_KEYPAR;
90 	goto cleanup;
91     }
92 
93     if (mit_des_is_weak_key(xorkey.contents)) {
94 	ret = KRB5DES_WEAK_KEY;
95 	goto cleanup;
96     }
97 
98     /* hash the confounder, then the input data */
99     mechanism.mechanism = CKM_MD5;
100     mechanism.pParameter = NULL_PTR;
101     mechanism.ulParameterLen = 0;
102 
103     if ((rv = C_DigestInit(krb_ctx_hSession(context), &mechanism)) != CKR_OK) {
104 	KRB5_LOG(KRB5_ERR, "C_DigestInit failed in k5_md5des_hash: "
105 	"rv = 0x%x.", rv);
106 	ret = PKCS_ERR;
107 	goto cleanup;
108     }
109 
110     if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
111 	(CK_BYTE_PTR)conf, (CK_ULONG)sizeof(conf))) != CKR_OK) {
112 	KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_hash: "
113 	    "rv = 0x%x", rv);
114 	ret = PKCS_ERR;
115 	goto cleanup;
116     }
117 
118     if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
119 	(CK_BYTE_PTR)input->data, (CK_ULONG)input->length)) != CKR_OK) {
120 	KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_hash: "
121 	    "rv = 0x%x", rv);
122 	return(PKCS_ERR);
123     }
124 
125     if ((rv = C_DigestFinal(krb_ctx_hSession(context),
126 	(CK_BYTE_PTR)(output->data + CONFLENGTH),
127 	(CK_ULONG_PTR)&hashlen)) != CKR_OK) {
128 	KRB5_LOG(KRB5_ERR, "C_DigestFinal failed in k5_md5des_hash: "
129 	    "rv = 0x%x", rv);
130 	ret = PKCS_ERR;
131 	goto cleanup;
132     }
133 
134     /* construct the buffer to be encrypted */
135 
136     (void) memcpy(output->data, conf, CONFLENGTH);
137 
138     /* encrypt it, in place.  this has a return value, but it's
139        always zero.  */
140 
141     ret = mit_des_cbc_encrypt(context,
142 	(krb5_pointer) output->data,
143 	(krb5_pointer) output->data, output->length,
144 	&xorkey, (unsigned char*) mit_des_zeroblock, 1);
145 
146 cleanup:
147     free(xorkey.contents);
148     return(ret);
149 }
150 
151 /*ARGSUSED*/
152 static krb5_error_code
153 k5_md5des_verify(krb5_context context,
154 	krb5_const krb5_keyblock *key,
155 	krb5_keyusage usage,
156 	krb5_const krb5_data *ivec,
157 	krb5_const krb5_data *input,
158 	krb5_const krb5_data *hash,
159 	krb5_boolean *valid)
160 {
161     krb5_error_code ret = 0;
162     unsigned char plaintext[CONFLENGTH+MD5_CKSUM_LENGTH];
163     unsigned char digest[MD5_CKSUM_LENGTH];
164     krb5_keyblock xorkey;
165     int i;
166     int compathash = 0;
167     CK_MECHANISM mechanism;
168     CK_RV rv;
169     CK_ULONG hashlen = MD5_CKSUM_LENGTH;
170 
171     if (key->length != 8)
172 	return(KRB5_BAD_KEYSIZE);
173     if (ivec)
174 	return(KRB5_CRYPTO_INTERNAL);
175     if (hash->length != (CONFLENGTH + MD5_CKSUM_LENGTH)) {
176 #ifdef KRB5_MD5DES_BETA5_COMPAT
177 	if (hash->length != MD5_CKSUM_LENGTH)
178 	    return(KRB5_CRYPTO_INTERNAL);
179 	else
180 	    compathash = 1;
181 #else
182 	return(KRB5_CRYPTO_INTERNAL);
183 #endif
184     }
185 
186     /* create and the encryption key */
187     xorkey.magic = key->magic;
188     xorkey.enctype = key->enctype;
189     xorkey.length = key->length;
190     xorkey.contents = (krb5_octet *)malloc(key->length);
191     if (xorkey.contents == NULL)
192 	return(KRB5_CRYPTO_INTERNAL);
193 
194     (void) memcpy(xorkey.contents, key->contents, xorkey.length);
195     if (!compathash) {
196         for (i=0; i<xorkey.length; i++)
197 	    xorkey.contents[i] ^= 0xf0;
198     }
199 
200     if (!mit_des_check_key_parity(xorkey.contents)) {
201 	ret = KRB5DES_BAD_KEYPAR;
202 	goto cleanup;
203     }
204 
205     if (mit_des_is_weak_key(xorkey.contents)) {
206 	ret = KRB5DES_WEAK_KEY;
207 	goto cleanup;
208     }
209 
210     /* decrypt it.  this has a return value, but it's always zero.  */
211     if (!compathash) {
212 	ret = mit_des_cbc_encrypt(context,
213 		(krb5_pointer) hash->data,
214 		(krb5_pointer) plaintext, hash->length,
215 		&xorkey, (unsigned char*) mit_des_zeroblock, 0);
216     } else {
217 	ret = mit_des_cbc_encrypt(context,
218 		(krb5_pointer) hash->data,
219 		(krb5_pointer) plaintext, hash->length,
220 		&xorkey, xorkey.contents, 0);
221     }
222     if (ret) goto cleanup;
223 
224     /* hash the confounder, then the input data */
225     mechanism.mechanism = CKM_MD5;
226     mechanism.pParameter = NULL_PTR;
227     mechanism.ulParameterLen = 0;
228 
229     if ((rv = C_DigestInit(krb_ctx_hSession(context), &mechanism)) != CKR_OK) {
230 	KRB5_LOG(KRB5_ERR, "C_DigestInit failed in k5_md5des_verify: "
231 	"rv = 0x%x.", rv);
232 	ret = PKCS_ERR;
233 	goto cleanup;
234     }
235 
236     if (!compathash) {
237 	if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
238 	    (CK_BYTE_PTR)plaintext, (CK_ULONG)CONFLENGTH)) != CKR_OK) {
239 	    KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_verify: "
240 		"rv = 0x%x", rv);
241 	    ret = PKCS_ERR;
242 	    goto cleanup;
243 	}
244     }
245     if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
246 	(CK_BYTE_PTR)input->data, (CK_ULONG)input->length)) != CKR_OK) {
247 	KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_verify: "
248 	    "rv = 0x%x", rv);
249 	ret = PKCS_ERR;
250 	goto cleanup;
251     }
252     if ((rv = C_DigestFinal(krb_ctx_hSession(context),
253 	(CK_BYTE_PTR)digest, (CK_ULONG_PTR)&hashlen)) != CKR_OK) {
254 	KRB5_LOG(KRB5_ERR, "C_DigestFinal failed in k5_md5des_verify: "
255 	    "rv = 0x%x", rv);
256 	ret = PKCS_ERR;
257 	goto cleanup;
258     }
259 
260     /* compare the decrypted hash to the computed one */
261 
262     if (!compathash) {
263 	*valid = (memcmp(plaintext+CONFLENGTH, digest, sizeof(digest)) == 0);
264     } else {
265 	*valid = (memcmp(plaintext, digest, sizeof(digest)) == 0);
266     }
267     (void) memset(plaintext, 0, sizeof(plaintext));
268 
269 cleanup:
270     free(xorkey.contents);
271     return(ret);
272 }
273 
274 const struct krb5_keyhash_provider krb5int_keyhash_md5des = {
275     CONFLENGTH + MD5_CKSUM_LENGTH,
276     k5_md5des_hash,
277     k5_md5des_verify
278 };
279