xref: /illumos-gate/usr/src/lib/gss_mechs/mech_dh/backend/mech/crypto.c (revision 694c35faa87b858ecdadfe4fc592615f4eefbb07)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  *	crypto.c
24  *
25  *	Copyright (c) 1997, by Sun Microsystems, Inc.
26  *	All rights reserved.
27  *
28  */
29 
30 #include <sys/note.h>
31 #include "dh_gssapi.h"
32 #include "crypto.h"
33 
34 /* Release the storage for a signature */
35 void
__free_signature(dh_signature_t sig)36 __free_signature(dh_signature_t sig)
37 {
38 	Free(sig->dh_signature_val);
39 	sig->dh_signature_val = NULL;
40 	sig->dh_signature_len = 0;
41 }
42 
43 /* Release the storage for a gss_buffer */
44 void
__dh_release_buffer(gss_buffer_t b)45 __dh_release_buffer(gss_buffer_t b)
46 {
47 	Free(b->value);
48 	b->length = 0;
49 	b->value = NULL;
50 }
51 
52 typedef struct cipher_entry {
53 	cipher_proc cipher;	/* Routine to en/decrypt with */
54 	unsigned int pad;	/* Padding need for the routine */
55 } cipher_entry, *cipher_t;
56 
57 typedef struct verifer_entry {
58 	verifier_proc msg;	/* Routine to calculate the check sum */
59 	unsigned int size;	/* Size of check sum */
60 	cipher_t signer;	/* Cipher entry to sign the check sum */
61 } verifier_entry, *verifier_t;
62 
63 typedef struct QOP_entry {
64 	int export_level;	/* Not currentlyt used */
65 	verifier_t verifier;	/* Verifier entry to use for integrity */
66 } QOP_entry;
67 
68 /*
69  * Return the length produced by using cipher entry c given the supplied len
70  */
71 static unsigned int
cipher_pad(cipher_t c,unsigned int len)72 cipher_pad(cipher_t c, unsigned int len)
73 {
74 	unsigned int pad;
75 
76 	pad = c ? c->pad : 1;
77 
78 	return (((len + pad - 1)/pad)*pad);
79 }
80 
81 
82 /*
83  * Des [en/de]crypt buffer, buf of length, len for each key provided using
84  * an CBC initialization vector ivec.
85  * If the mode is encrypt we will use the following pattern if the number
86  * of keys is odd
87  * encrypt(buf, k[0]), decrypt(buf, k[1]), encrypt(buf, k[2])
88  *	decrypt(buf, k[4]) ... encrypt(buf, k[keynum - 1])
89  * If we have an even number of keys and additional encryption will be
90  * done with the first key, i.e., ecrypt(buf, k[0]);
91  * In each [en/de]cription above we will used the passed in CBC initialization
92  * vector. The new initialization vector will be the vector return from the
93  * last encryption.
94  *
95  * In the decryption case we reverse the proccess. Note in this case
96  * the return ivec will be from the first decryption.
97  */
98 
99 static int
__desN_crypt(des_block keys[],int keynum,char * buf,unsigned int len,unsigned int mode,char * ivec)100 __desN_crypt(des_block keys[], int keynum, char *buf, unsigned int len,
101     unsigned int mode, char *ivec)
102 {
103 	/* Get the direction of ciphering */
104 	unsigned int m = mode & (DES_ENCRYPT | DES_DECRYPT);
105 	/* Get the remaining flags from mode */
106 	unsigned int flags = mode & ~(DES_ENCRYPT | DES_DECRYPT);
107 	des_block svec, dvec;
108 	int i, j, stat;
109 
110 	/* Do we have at least one key */
111 	if (keynum < 1)
112 		return (DESERR_BADPARAM);
113 
114 	/* Save the passed in ivec */
115 	memcpy(svec.c, ivec, sizeof (des_block));
116 
117 	/* For  each key do the appropriate cipher */
118 	for (i = 0; i < keynum; i++) {
119 		j = (mode & DES_DECRYPT) ? keynum - 1 - i : i;
120 		stat = cbc_crypt(keys[j].c, buf, len, m | flags, ivec);
121 		if (mode & DES_DECRYPT && i == 0)
122 			memcpy(dvec.c, ivec, sizeof (des_block));
123 
124 		if (DES_FAILED(stat))
125 			return (stat);
126 
127 		m = (m == DES_ENCRYPT ? DES_DECRYPT : DES_ENCRYPT);
128 
129 		if ((mode & DES_DECRYPT) || i != keynum - 1 || i%2)
130 			memcpy(ivec, svec.c, sizeof (des_block));
131 	}
132 
133 	/*
134 	 * If we have an even number of keys then do an extra round of
135 	 * [en/de]cryption with the first key.
136 	 */
137 	if (keynum % 2 == 0)
138 		stat = cbc_crypt(keys[0].c, buf, len, mode, ivec);
139 
140 	/* If were decrypting ivec is set from first decryption */
141 	if (mode & DES_DECRYPT)
142 		memcpy(ivec, dvec.c, sizeof (des_block));
143 
144 	return (stat);
145 }
146 
147 
148 /*
149  * DesN crypt packaged for use as a cipher entry
150  */
151 static OM_uint32
__dh_desN_crypt(gss_buffer_t buf,dh_key_set_t keys,cipher_mode_t cipher_mode)152 __dh_desN_crypt(gss_buffer_t buf, dh_key_set_t keys, cipher_mode_t cipher_mode)
153 {
154 	int stat = DESERR_BADPARAM;
155 	int encrypt_flag = (cipher_mode == ENCIPHER);
156 	unsigned mode = (encrypt_flag ? DES_ENCRYPT : DES_DECRYPT) | DES_HW;
157 	des_block ivec;
158 
159 	if (keys->dh_key_set_len < 1)
160 		return (DH_BADARG_FAILURE);
161 
162 	/*
163 	 * We all ways start of with ivec set to zeros. There is no
164 	 * good way to maintain ivecs since packets could be out of sequence
165 	 * duplicated or worst of all lost. Under these conditions the
166 	 * higher level protocol would have to some how resync the ivecs
167 	 * on both sides and start again. Theres no mechanism for this in
168 	 * GSS.
169 	 */
170 	memset(&ivec, 0, sizeof (ivec));
171 
172 	/* Do the encryption/decryption */
173 	stat = __desN_crypt(keys->dh_key_set_val, keys->dh_key_set_len,
174 			    (char *)buf->value, buf->length, mode, ivec.c);
175 
176 	if (DES_FAILED(stat))
177 		return (DH_CIPHER_FAILURE);
178 
179 	return (DH_SUCCESS);
180 }
181 
182 /*
183  * Package up plain des cbc crypt for use as a cipher entry.
184  */
185 static OM_uint32
__dh_des_crypt(gss_buffer_t buf,dh_key_set_t keys,cipher_mode_t cipher_mode)186 __dh_des_crypt(gss_buffer_t buf, dh_key_set_t keys, cipher_mode_t cipher_mode)
187 {
188 	int stat = DESERR_BADPARAM;
189 	int encrypt_flag = (cipher_mode == ENCIPHER);
190 	unsigned mode = (encrypt_flag ? DES_ENCRYPT : DES_DECRYPT) | DES_HW;
191 	des_block ivec;
192 
193 	if (keys->dh_key_set_len < 1)
194 		return (DH_BADARG_FAILURE);
195 
196 	/*  Set the ivec to zeros and then cbc crypt the result */
197 	memset(&ivec, 0, sizeof (ivec));
198 	stat = cbc_crypt(keys->dh_key_set_val[0].c, (char *)buf->value,
199 			buf->length, mode, ivec.c);
200 
201 	if (DES_FAILED(stat))
202 		return (DH_CIPHER_FAILURE);
203 
204 	return (DH_SUCCESS);
205 }
206 
207 /*
208  * MD5_verifier: This is a verifier routine suitable for use in a
209  * verifier entry. It calculates the MD5 check sum over an optional
210  * msg and a token. It signs it using the supplied cipher_proc and stores
211  * the result in signature.
212  *
213  * Note signature should already be allocated and be large enough to
214  * hold the signature after its been encrypted. If keys is null, then
215  * we will just return the unencrypted check sum.
216  */
217 static OM_uint32
MD5_verifier(gss_buffer_t tok,gss_buffer_t msg,cipher_proc signer,dh_key_set_t keys,dh_signature_t signature)218 MD5_verifier(gss_buffer_t tok, /* The buffer to sign */
219 	    gss_buffer_t msg, /* Optional buffer to include */
220 	    cipher_proc signer, /* Routine to encrypt the integrity check */
221 	    dh_key_set_t keys, /* Optiona keys to be used with the above */
222 	    dh_signature_t signature /* The resulting MIC */)
223 {
224 	MD5_CTX md5_ctx;	/* MD5 context */
225 	gss_buffer_desc buf;	/* GSS buffer to hold keys for cipher routine */
226 
227 	/* Initialize the MD5 context */
228 	MD5Init(&md5_ctx);
229 	/* If we have a message to digest, digest it */
230 	if (msg)
231 	    MD5Update(&md5_ctx, (unsigned char *)msg->value, msg->length);
232 	/* Digest the supplied token */
233 	MD5Update(&md5_ctx, (unsigned char *)tok->value, tok->length);
234 	/* Finalize the sum. The MD5 context contains the digets */
235 	MD5Final(&md5_ctx);
236 
237 	/* Copy the digest to the signature */
238 	memcpy(signature->dh_signature_val, (void *)md5_ctx.digest, 16);
239 
240 	buf.length = signature->dh_signature_len;
241 	buf.value = signature->dh_signature_val;
242 
243 	/* If we have keys encrypt it */
244 	if (keys != NULL)
245 		return (signer(&buf, keys, ENCIPHER));
246 
247 	return (DH_SUCCESS);
248 }
249 
250 /* Cipher table */
251 static
252 cipher_entry cipher_tab[] = {
253 	{ NULL, 1},
254 	{ __dh_desN_crypt, 8},
255 	{ __dh_des_crypt, 8}
256 };
257 
258 
259 #define	__NO_CRYPT	&cipher_tab[0]
260 #define	__DES_N_CRYPT	&cipher_tab[1]
261 #define	__DES_CRYPT	&cipher_tab[2]
262 
263 /* Verifier table */
264 static
265 verifier_entry verifier_tab[] = {
266 	{ MD5_verifier, 16, __DES_N_CRYPT },
267 	{ MD5_verifier, 16, __DES_CRYPT }
268 };
269 
270 /* QOP table */
271 static
272 QOP_entry QOP_table[] = {
273 	{ 0, &verifier_tab[0] },
274 	{ 0, &verifier_tab[1] }
275 };
276 
277 #define	QOP_ENTRIES (sizeof (QOP_table) / sizeof (QOP_entry))
278 
279 /*
280  * __dh_is_valid_QOP: Return true if qop is valid entry into the QOP
281  * table, else return false.
282  */
283 bool_t
__dh_is_valid_QOP(dh_qop_t qop)284 __dh_is_valid_QOP(dh_qop_t qop)
285 {
286 	bool_t is_valid = FALSE;
287 
288 	is_valid = qop < QOP_ENTRIES;
289 
290 	return (is_valid);
291 }
292 
293 /*
294  * __alloc_sig: Allocate a signature for a given QOP. This takes into
295  * account the size of the signature after padding for the encryption
296  * routine.
297  */
298 OM_uint32
__alloc_sig(dh_qop_t qop,dh_signature_t sig)299 __alloc_sig(dh_qop_t qop, dh_signature_t sig)
300 {
301 	OM_uint32 stat = DH_VERIFIER_FAILURE;
302 	verifier_entry *v;
303 
304 	/* Check that the QOP is valid */
305 	if (!__dh_is_valid_QOP(qop))
306 		return (DH_UNKNOWN_QOP);
307 
308 	/* Get the verifier entry from the QOP entry */
309 	v = QOP_table[qop].verifier;
310 
311 	/* Calulate the length needed for the signature */
312 	sig->dh_signature_len = cipher_pad(v->signer, v->size);
313 
314 	/* Allocate the signature */
315 	sig->dh_signature_val = (void*)New(char, sig->dh_signature_len);
316 	if (sig->dh_signature_val == NULL) {
317 		sig->dh_signature_len = 0;
318 		return (DH_NOMEM_FAILURE);
319 	}
320 
321 	stat = DH_SUCCESS;
322 
323 	return (stat);
324 }
325 
326 /*
327  * __get_sig_size: Return the total size needed for a signature given a QOP.
328  */
329 OM_uint32
__get_sig_size(dh_qop_t qop,unsigned int * size)330 __get_sig_size(dh_qop_t qop, unsigned int *size)
331 {
332 	/* Check for valid QOP */
333 	if (__dh_is_valid_QOP(qop)) {
334 		/* Get the verifier entry */
335 		verifier_t v = QOP_table[qop].verifier;
336 
337 		/* Return the size include the padding needed for encryption */
338 		*size = v ? cipher_pad(v->signer, v->size) : 0;
339 
340 		return (DH_SUCCESS);
341 	}
342 	*size = 0;
343 
344 	return (DH_UNKNOWN_QOP);
345 }
346 
347 /*
348  * __mk_sig: Generate a signature using a given qop over a token of a
349  * given length and an optional message. We use the supplied keys to
350  * encrypt the check sum if they are available. The output is place
351  * in a preallocate signature, that was allocated using __alloc_sig.
352  */
353 OM_uint32
__mk_sig(dh_qop_t qop,char * tok,long len,gss_buffer_t mesg,dh_key_set_t keys,dh_signature_t sig)354 __mk_sig(dh_qop_t qop, /* The QOP to use */
355 	char *tok, /* The token to sign */
356 	long len, /* The tokens length */
357 	gss_buffer_t mesg,	/* An optional message to be included */
358 	dh_key_set_t keys, /* The optional encryption keys */
359 	dh_signature_t sig /* The resulting MIC */)
360 {
361 	OM_uint32 stat = DH_VERIFIER_FAILURE;
362 
363 
364 	verifier_entry *v;	/* Verifier entry */
365 	gss_buffer_desc buf;	/* Buffer to package tok */
366 
367 	/* Make sure the QOP is valid */
368 	if (!__dh_is_valid_QOP(qop))
369 		return (DH_UNKNOWN_QOP);
370 
371 	/* Grab the verifier entry for the qop */
372 	v = QOP_table[qop].verifier;
373 
374 	/* Package the token for use in a verifier_proc */
375 	buf.length = len;
376 	buf.value = tok;
377 
378 	/*
379 	 * Calculate the signature using the supplied keys. If keys
380 	 * is null, the the v->signer->cipher routine will not be called
381 	 * and sig will not be encrypted.
382 	 */
383 	stat = (*v->msg)(&buf, mesg, v->signer->cipher, keys, sig);
384 
385 	return (stat);
386 }
387 
388 /*
389  * __verify_sig: Verify that the supplied signature, sig, is the same
390  * as the token verifier
391  */
392 OM_uint32
__verify_sig(dh_token_t token,dh_qop_t qop,dh_key_set_t keys,dh_signature_t sig)393 __verify_sig(dh_token_t token, /* The token to be verified */
394 	    dh_qop_t qop, /* The QOP to use */
395 	    dh_key_set_t keys, /* The context session keys */
396 	    dh_signature_t sig /* The signature from the serialized token */)
397 {
398 	OM_uint32 stat = DH_VERIFIER_FAILURE;
399 
400 	cipher_proc cipher;	/* cipher routine to use */
401 	gss_buffer_desc buf;	/* Packaging for sig */
402 
403 	/* Check the QOP */
404 	if (!__dh_is_valid_QOP(qop))
405 		return (DH_UNKNOWN_QOP);
406 
407 	/* Package up the supplied signature */
408 	buf.length = sig->dh_signature_len;
409 	buf.value = sig->dh_signature_val;
410 
411 	/* Get the cipher proc to use from the verifier entry for qop */
412 	cipher = QOP_table[qop].verifier->signer->cipher;
413 
414 	/* Encrypt the check sum using the supplied set of keys */
415 	if ((stat = (*cipher)(&buf, keys, ENCIPHER)) != DH_SUCCESS)
416 		return (stat);
417 
418 	/* Compare the signatures */
419 	if (__cmpsig(sig, &token->verifier))
420 		return (DH_SUCCESS);
421 
422 	stat = DH_VERIFIER_MISMATCH;
423 
424 	return (stat);
425 }
426 
427 /*
428  * __cmpsig: Return true if two signatures are the same, else false.
429  */
430 bool_t
__cmpsig(dh_signature_t s1,dh_signature_t s2)431 __cmpsig(dh_signature_t s1, dh_signature_t s2)
432 {
433 	return (s1->dh_signature_len == s2->dh_signature_len &&
434 	    memcmp(s1->dh_signature_val,
435 		s2->dh_signature_val, s1->dh_signature_len) == 0);
436 }
437 
438 /*
439  * wrap_msg_body: Wrap the message pointed to be in into a
440  * message pointed to by out that has ben padded out by pad bytes.
441  *
442  * The output message looks like:
443  * out->length = total length of out->value including any padding
444  * out->value points to memory as follows:
445  * +------------+-------------------------+---------|
446  * | in->length | in->value               | XDR PAD |
447  * +------------+-------------------------+---------|
448  *    4 bytes      in->length bytes         0 - 3
449  */
450 static OM_uint32
wrap_msg_body(gss_buffer_t in,gss_buffer_t out)451 wrap_msg_body(gss_buffer_t in, gss_buffer_t out)
452 {
453 	XDR xdrs;			/* xdrs to wrap with */
454 	unsigned int len, out_len;	/* length  */
455 	size_t size;
456 
457 	out->length = 0;
458 	out->value = 0;
459 
460 	/* Make sure the address of len points to a 32 bit word */
461 	len = (unsigned int)in->length;
462 	if (len != in->length)
463 		return (DH_ENCODE_FAILURE);
464 
465 	size = ((in->length + sizeof (OM_uint32) + 3)/4) * 4;
466 	out_len = size;
467 	if (out_len != size)
468 		return (DH_ENCODE_FAILURE);
469 
470 	/* Allocate the output buffer and set the length */
471 	if ((out->value = (void *)New(char, len)) == NULL)
472 		return (DH_NOMEM_FAILURE);
473 	out->length = out_len;
474 
475 
476 	/* Create xdr stream to wrap into */
477 	xdrmem_create(&xdrs, out->value, out->length, XDR_ENCODE);
478 
479 	/* Wrap the bytes in value */
480 	if (!xdr_bytes(&xdrs, (char **)&in->value, &len, len)) {
481 		__dh_release_buffer(out);
482 		return (DH_ENCODE_FAILURE);
483 	}
484 
485 	return (DH_SUCCESS);
486 }
487 
488 /*
489  * __QOPSeal: Wrap the input message placing the output in output given
490  * a valid QOP. If confidentialiy is requested it is ignored. We can't
491  * support privacy. The return flag will always be zero.
492  */
493 OM_uint32
__QOPSeal(dh_qop_t qop,gss_buffer_t input,int conf_req,dh_key_set_t keys,gss_buffer_t output,int * conf_ret)494 __QOPSeal(dh_qop_t qop, /* The QOP to use */
495 	gss_buffer_t input, /* The buffer to wrap */
496 	int conf_req, /* Do we want privacy ? */
497 	dh_key_set_t keys, /* The session keys */
498 	gss_buffer_t output, /* The wraped message */
499 	int *conf_ret /* Did we encrypt it? */)
500 {
501 _NOTE(ARGUNUSED(conf_req,keys))
502 	OM_uint32 stat = DH_CIPHER_FAILURE;
503 
504 	*conf_ret = FALSE;	/* No encryption allowed */
505 
506 	/* Check for valid QOP */
507 	if (!__dh_is_valid_QOP(qop))
508 		return (DH_UNKNOWN_QOP);
509 
510 	/* Wrap the message */
511 	if ((stat = wrap_msg_body(input, output))
512 	    != DH_SUCCESS)
513 		return (stat);
514 
515 	return (stat);
516 }
517 
518 /*
519  * unwrap_msg_body: Unwrap the message, that was wrapped from above
520  */
521 static OM_uint32
unwrap_msg_body(gss_buffer_t in,gss_buffer_t out)522 unwrap_msg_body(gss_buffer_t in, gss_buffer_t out)
523 {
524 	XDR xdrs;
525 	unsigned int len;	/* sizeof (len) == 32bits */
526 
527 	/* Create an xdr stream to on wrap in */
528 	xdrmem_create(&xdrs, in->value, in->length, XDR_DECODE);
529 
530 	/* Unwrap the input into out->value */
531 	if (!xdr_bytes(&xdrs, (char **)&out->value, &len, in->length))
532 		return (DH_DECODE_FAILURE);
533 
534 	/* set the length */
535 	out->length = len;
536 
537 	return (DH_SUCCESS);
538 }
539 
540 /*
541  * __QOPUnSeal: Unwrap the input message into output using the supplied QOP.
542  * Note it is the callers responsibility to release the allocated output
543  * buffer. If conf_req is true we return DH_CIPHER_FAILURE since we don't
544  * support privacy.
545  */
546 OM_uint32
__QOPUnSeal(dh_qop_t qop,gss_buffer_t input,int conf_req,dh_key_set_t keys,gss_buffer_t output)547 __QOPUnSeal(dh_qop_t qop, /* The QOP to use */
548 	    gss_buffer_t input, /* The message to unwrap */
549 	    int conf_req, /* Is the message encrypted */
550 	    dh_key_set_t keys, /* The session keys to decrypt if conf_req */
551 	    gss_buffer_t output /* The unwraped message */)
552 {
553 _NOTE(ARGUNUSED(keys))
554 	OM_uint32 stat = DH_CIPHER_FAILURE;
555 
556 	/* Check that the qop is valid */
557 	if (!__dh_is_valid_QOP(qop))
558 		return (DH_UNKNOWN_QOP);
559 
560 	/* Set output to sane values */
561 	output->length = 0;
562 	output->value = NULL;
563 
564 	/* Fail if this is privacy */
565 	if (conf_req)
566 		return (DH_CIPHER_FAILURE);
567 
568 	/* Unwrap the input into the output, return the status */
569 	stat = unwrap_msg_body(input, output);
570 
571 	return (stat);
572 }
573