xref: /freebsd/crypto/heimdal/appl/telnet/libtelnet/enc_des.c (revision b52b9d56d4e96089873a75f9e29062eec19fabba)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *	This product includes software developed by the University of
16  *	California, Berkeley and its contributors.
17  * 4. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33 
34 #include <config.h>
35 
36 RCSID("$Id: enc_des.c,v 1.20 2001/08/29 00:45:19 assar Exp $");
37 
38 #if	defined(AUTHENTICATION) && defined(ENCRYPTION) && defined(DES_ENCRYPTION)
39 #include <arpa/telnet.h>
40 #include <stdio.h>
41 #ifdef	__STDC__
42 #include <stdlib.h>
43 #include <string.h>
44 #endif
45 #include <roken.h>
46 #ifdef SOCKS
47 #include <socks.h>
48 #endif
49 
50 #include "encrypt.h"
51 #include "misc-proto.h"
52 
53 #ifdef HAVE_OPENSSL
54 #include <openssl/des.h>
55 #else
56 #include <des.h>
57 #endif
58 
59 extern int encrypt_debug_mode;
60 
61 #define	CFB	0
62 #define	OFB	1
63 
64 #define	NO_SEND_IV	1
65 #define	NO_RECV_IV	2
66 #define	NO_KEYID	4
67 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
68 #define	SUCCESS		0
69 #define	FAILED		-1
70 
71 
72 struct stinfo {
73   des_cblock	str_output;
74   des_cblock	str_feed;
75   des_cblock	str_iv;
76   des_cblock	str_ikey;
77   des_key_schedule str_sched;
78   int		str_index;
79   int		str_flagshift;
80 };
81 
82 struct fb {
83 	des_cblock krbdes_key;
84 	des_key_schedule krbdes_sched;
85 	des_cblock temp_feed;
86 	unsigned char fb_feed[64];
87 	int need_start;
88 	int state[2];
89 	int keyid[2];
90 	int once;
91 	struct stinfo streams[2];
92 };
93 
94 static struct fb fb[2];
95 
96 struct keyidlist {
97 	char	*keyid;
98 	int	keyidlen;
99 	char	*key;
100 	int	keylen;
101 	int	flags;
102 } keyidlist [] = {
103 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
104 	{ 0, 0, 0, 0, 0 }
105 };
106 
107 #define	KEYFLAG_MASK	03
108 
109 #define	KEYFLAG_NOINIT	00
110 #define	KEYFLAG_INIT	01
111 #define	KEYFLAG_OK	02
112 #define	KEYFLAG_BAD	03
113 
114 #define	KEYFLAG_SHIFT	2
115 
116 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
117 
118 #define	FB64_IV		1
119 #define	FB64_IV_OK	2
120 #define	FB64_IV_BAD	3
121 
122 
123 void fb64_stream_iv (des_cblock, struct stinfo *);
124 void fb64_init (struct fb *);
125 static int fb64_start (struct fb *, int, int);
126 int fb64_is (unsigned char *, int, struct fb *);
127 int fb64_reply (unsigned char *, int, struct fb *);
128 static void fb64_session (Session_Key *, int, struct fb *);
129 void fb64_stream_key (des_cblock, struct stinfo *);
130 int fb64_keyid (int, unsigned char *, int *, struct fb *);
131 void fb64_printsub(unsigned char *, int ,
132 		   unsigned char *, int , char *);
133 
134 void cfb64_init(int server)
135 {
136 	fb64_init(&fb[CFB]);
137 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
138 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
139 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
140 }
141 
142 
143 void ofb64_init(int server)
144 {
145 	fb64_init(&fb[OFB]);
146 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
147 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
148 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
149 }
150 
151 void fb64_init(struct fb *fbp)
152 {
153 	memset(fbp,0, sizeof(*fbp));
154 	fbp->state[0] = fbp->state[1] = FAILED;
155 	fbp->fb_feed[0] = IAC;
156 	fbp->fb_feed[1] = SB;
157 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
158 	fbp->fb_feed[3] = ENCRYPT_IS;
159 }
160 
161 /*
162  * Returns:
163  *	-1: some error.  Negotiation is done, encryption not ready.
164  *	 0: Successful, initial negotiation all done.
165  *	 1: successful, negotiation not done yet.
166  *	 2: Not yet.  Other things (like getting the key from
167  *	    Kerberos) have to happen before we can continue.
168  */
169 int cfb64_start(int dir, int server)
170 {
171 	return(fb64_start(&fb[CFB], dir, server));
172 }
173 
174 int ofb64_start(int dir, int server)
175 {
176 	return(fb64_start(&fb[OFB], dir, server));
177 }
178 
179 static int fb64_start(struct fb *fbp, int dir, int server)
180 {
181 	int x;
182 	unsigned char *p;
183 	int state;
184 
185 	switch (dir) {
186 	case DIR_DECRYPT:
187 		/*
188 		 * This is simply a request to have the other side
189 		 * start output (our input).  He will negotiate an
190 		 * IV so we need not look for it.
191 		 */
192 		state = fbp->state[dir-1];
193 		if (state == FAILED)
194 			state = IN_PROGRESS;
195 		break;
196 
197 	case DIR_ENCRYPT:
198 		state = fbp->state[dir-1];
199 		if (state == FAILED)
200 			state = IN_PROGRESS;
201 		else if ((state & NO_SEND_IV) == 0) {
202 			break;
203 		}
204 
205 		if (!VALIDKEY(fbp->krbdes_key)) {
206 		        fbp->need_start = 1;
207 			break;
208 		}
209 
210 		state &= ~NO_SEND_IV;
211 		state |= NO_RECV_IV;
212 		if (encrypt_debug_mode)
213 			printf("Creating new feed\r\n");
214 		/*
215 		 * Create a random feed and send it over.
216 		 */
217 #ifndef OLD_DES_RANDOM_KEY
218 		des_new_random_key(&fbp->temp_feed);
219 #else
220 		/*
221 		 * From des_cryp.man "If the des_check_key flag is non-zero,
222 		 *  des_set_key will check that the key passed is
223 		 *  of odd parity and is not a week or semi-weak key."
224 		 */
225 		do {
226 			des_random_key(fbp->temp_feed);
227 			des_set_odd_parity(fbp->temp_feed);
228 		} while (des_is_weak_key(fbp->temp_feed));
229 #endif
230 		des_ecb_encrypt(&fbp->temp_feed,
231 				&fbp->temp_feed,
232 				fbp->krbdes_sched, 1);
233 		p = fbp->fb_feed + 3;
234 		*p++ = ENCRYPT_IS;
235 		p++;
236 		*p++ = FB64_IV;
237 		for (x = 0; x < sizeof(des_cblock); ++x) {
238 			if ((*p++ = fbp->temp_feed[x]) == IAC)
239 				*p++ = IAC;
240 		}
241 		*p++ = IAC;
242 		*p++ = SE;
243 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
244 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
245 		break;
246 	default:
247 		return(FAILED);
248 	}
249 	return(fbp->state[dir-1] = state);
250 }
251 
252 /*
253  * Returns:
254  *	-1: some error.  Negotiation is done, encryption not ready.
255  *	 0: Successful, initial negotiation all done.
256  *	 1: successful, negotiation not done yet.
257  */
258 
259 int cfb64_is(unsigned char *data, int cnt)
260 {
261 	return(fb64_is(data, cnt, &fb[CFB]));
262 }
263 
264 int ofb64_is(unsigned char *data, int cnt)
265 {
266 	return(fb64_is(data, cnt, &fb[OFB]));
267 }
268 
269 
270 int fb64_is(unsigned char *data, int cnt, struct fb *fbp)
271 {
272 	unsigned char *p;
273 	int state = fbp->state[DIR_DECRYPT-1];
274 
275 	if (cnt-- < 1)
276 		goto failure;
277 
278 	switch (*data++) {
279 	case FB64_IV:
280 		if (cnt != sizeof(des_cblock)) {
281 			if (encrypt_debug_mode)
282 				printf("CFB64: initial vector failed on size\r\n");
283 			state = FAILED;
284 			goto failure;
285 		}
286 
287 		if (encrypt_debug_mode)
288 			printf("CFB64: initial vector received\r\n");
289 
290 		if (encrypt_debug_mode)
291 			printf("Initializing Decrypt stream\r\n");
292 
293 		fb64_stream_iv(data, &fbp->streams[DIR_DECRYPT-1]);
294 
295 		p = fbp->fb_feed + 3;
296 		*p++ = ENCRYPT_REPLY;
297 		p++;
298 		*p++ = FB64_IV_OK;
299 		*p++ = IAC;
300 		*p++ = SE;
301 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
302 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
303 
304 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
305 		break;
306 
307 	default:
308 		if (encrypt_debug_mode) {
309 			printf("Unknown option type: %d\r\n", *(data-1));
310 			printd(data, cnt);
311 			printf("\r\n");
312 		}
313 		/* FALL THROUGH */
314 	failure:
315 		/*
316 		 * We failed.  Send an FB64_IV_BAD option
317 		 * to the other side so it will know that
318 		 * things failed.
319 		 */
320 		p = fbp->fb_feed + 3;
321 		*p++ = ENCRYPT_REPLY;
322 		p++;
323 		*p++ = FB64_IV_BAD;
324 		*p++ = IAC;
325 		*p++ = SE;
326 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
327 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
328 
329 		break;
330 	}
331 	return(fbp->state[DIR_DECRYPT-1] = state);
332 }
333 
334 /*
335  * Returns:
336  *	-1: some error.  Negotiation is done, encryption not ready.
337  *	 0: Successful, initial negotiation all done.
338  *	 1: successful, negotiation not done yet.
339  */
340 
341 int cfb64_reply(unsigned char *data, int cnt)
342 {
343 	return(fb64_reply(data, cnt, &fb[CFB]));
344 }
345 
346 int ofb64_reply(unsigned char *data, int cnt)
347 {
348 	return(fb64_reply(data, cnt, &fb[OFB]));
349 }
350 
351 
352 int fb64_reply(unsigned char *data, int cnt, struct fb *fbp)
353 {
354 	int state = fbp->state[DIR_ENCRYPT-1];
355 
356 	if (cnt-- < 1)
357 		goto failure;
358 
359 	switch (*data++) {
360 	case FB64_IV_OK:
361 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
362 		if (state == FAILED)
363 			state = IN_PROGRESS;
364 		state &= ~NO_RECV_IV;
365 		encrypt_send_keyid(DIR_ENCRYPT, (unsigned char *)"\0", 1, 1);
366 		break;
367 
368 	case FB64_IV_BAD:
369 		memset(fbp->temp_feed, 0, sizeof(des_cblock));
370 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
371 		state = FAILED;
372 		break;
373 
374 	default:
375 		if (encrypt_debug_mode) {
376 			printf("Unknown option type: %d\r\n", data[-1]);
377 			printd(data, cnt);
378 			printf("\r\n");
379 		}
380 		/* FALL THROUGH */
381 	failure:
382 		state = FAILED;
383 		break;
384 	}
385 	return(fbp->state[DIR_ENCRYPT-1] = state);
386 }
387 
388 void cfb64_session(Session_Key *key, int server)
389 {
390 	fb64_session(key, server, &fb[CFB]);
391 }
392 
393 void ofb64_session(Session_Key *key, int server)
394 {
395 	fb64_session(key, server, &fb[OFB]);
396 }
397 
398 static void fb64_session(Session_Key *key, int server, struct fb *fbp)
399 {
400 
401 	if (!key || key->type != SK_DES) {
402 		if (encrypt_debug_mode)
403 			printf("Can't set krbdes's session key (%d != %d)\r\n",
404 				key ? key->type : -1, SK_DES);
405 		return;
406 	}
407 	memcpy(fbp->krbdes_key, key->data, sizeof(des_cblock));
408 
409 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
410 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
411 
412 	if (fbp->once == 0) {
413 #if !defined(OLD_DES_RANDOM_KEY) && !defined(HAVE_OPENSSL)
414 		des_init_random_number_generator(&fbp->krbdes_key);
415 #endif
416 		fbp->once = 1;
417 	}
418 	des_key_sched(&fbp->krbdes_key, fbp->krbdes_sched);
419 	/*
420 	 * Now look to see if krbdes_start() was was waiting for
421 	 * the key to show up.  If so, go ahead an call it now
422 	 * that we have the key.
423 	 */
424 	if (fbp->need_start) {
425 		fbp->need_start = 0;
426 		fb64_start(fbp, DIR_ENCRYPT, server);
427 	}
428 }
429 
430 /*
431  * We only accept a keyid of 0.  If we get a keyid of
432  * 0, then mark the state as SUCCESS.
433  */
434 
435 int cfb64_keyid(int dir, unsigned char *kp, int *lenp)
436 {
437 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
438 }
439 
440 int ofb64_keyid(int dir, unsigned char *kp, int *lenp)
441 {
442 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
443 }
444 
445 int fb64_keyid(int dir, unsigned char *kp, int *lenp, struct fb *fbp)
446 {
447 	int state = fbp->state[dir-1];
448 
449 	if (*lenp != 1 || (*kp != '\0')) {
450 		*lenp = 0;
451 		return(state);
452 	}
453 
454 	if (state == FAILED)
455 		state = IN_PROGRESS;
456 
457 	state &= ~NO_KEYID;
458 
459 	return(fbp->state[dir-1] = state);
460 }
461 
462 void fb64_printsub(unsigned char *data, int cnt,
463 		   unsigned char *buf, int buflen, char *type)
464 {
465 	char lbuf[32];
466 	int i;
467 	char *cp;
468 
469 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
470 	buflen -= 1;
471 
472 	switch(data[2]) {
473 	case FB64_IV:
474 		snprintf(lbuf, sizeof(lbuf), "%s_IV", type);
475 		cp = lbuf;
476 		goto common;
477 
478 	case FB64_IV_OK:
479 		snprintf(lbuf, sizeof(lbuf), "%s_IV_OK", type);
480 		cp = lbuf;
481 		goto common;
482 
483 	case FB64_IV_BAD:
484 		snprintf(lbuf, sizeof(lbuf), "%s_IV_BAD", type);
485 		cp = lbuf;
486 		goto common;
487 
488 	default:
489 		snprintf(lbuf, sizeof(lbuf), " %d (unknown)", data[2]);
490 		cp = lbuf;
491 	common:
492 		for (; (buflen > 0) && (*buf = *cp++); buf++)
493 			buflen--;
494 		for (i = 3; i < cnt; i++) {
495 			snprintf(lbuf, sizeof(lbuf), " %d", data[i]);
496 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
497 				buflen--;
498 		}
499 		break;
500 	}
501 }
502 
503 void cfb64_printsub(unsigned char *data, int cnt,
504 		    unsigned char *buf, int buflen)
505 {
506 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
507 }
508 
509 void ofb64_printsub(unsigned char *data, int cnt,
510 		    unsigned char *buf, int buflen)
511 {
512 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
513 }
514 
515 void fb64_stream_iv(des_cblock seed, struct stinfo *stp)
516 {
517 
518 	memcpy(stp->str_iv, seed,sizeof(des_cblock));
519 	memcpy(stp->str_output, seed, sizeof(des_cblock));
520 
521 	des_key_sched(&stp->str_ikey, stp->str_sched);
522 
523 	stp->str_index = sizeof(des_cblock);
524 }
525 
526 void fb64_stream_key(des_cblock key, struct stinfo *stp)
527 {
528 	memcpy(stp->str_ikey, key, sizeof(des_cblock));
529 	des_key_sched((des_cblock*)key, stp->str_sched);
530 
531 	memcpy(stp->str_output, stp->str_iv, sizeof(des_cblock));
532 
533 	stp->str_index = sizeof(des_cblock);
534 }
535 
536 /*
537  * DES 64 bit Cipher Feedback
538  *
539  *     key --->+-----+
540  *          +->| DES |--+
541  *          |  +-----+  |
542  *	    |           v
543  *  INPUT --(--------->(+)+---> DATA
544  *          |             |
545  *	    +-------------+
546  *
547  *
548  * Given:
549  *	iV: Initial vector, 64 bits (8 bytes) long.
550  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
551  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
552  *
553  *	V0 = DES(iV, key)
554  *	On = Dn ^ Vn
555  *	V(n+1) = DES(On, key)
556  */
557 
558 void cfb64_encrypt(unsigned char *s, int c)
559 {
560 	struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
561 	int index;
562 
563 	index = stp->str_index;
564 	while (c-- > 0) {
565 		if (index == sizeof(des_cblock)) {
566 			des_cblock b;
567 			des_ecb_encrypt(&stp->str_output, &b,stp->str_sched, 1);
568 			memcpy(stp->str_feed, b, sizeof(des_cblock));
569 			index = 0;
570 		}
571 
572 		/* On encryption, we store (feed ^ data) which is cypher */
573 		*s = stp->str_output[index] = (stp->str_feed[index] ^ *s);
574 		s++;
575 		index++;
576 	}
577 	stp->str_index = index;
578 }
579 
580 int cfb64_decrypt(int data)
581 {
582 	struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
583 	int index;
584 
585 	if (data == -1) {
586 		/*
587 		 * Back up one byte.  It is assumed that we will
588 		 * never back up more than one byte.  If we do, this
589 		 * may or may not work.
590 		 */
591 		if (stp->str_index)
592 			--stp->str_index;
593 		return(0);
594 	}
595 
596 	index = stp->str_index++;
597 	if (index == sizeof(des_cblock)) {
598 		des_cblock b;
599 		des_ecb_encrypt(&stp->str_output,&b, stp->str_sched, 1);
600 		memcpy(stp->str_feed, b, sizeof(des_cblock));
601 		stp->str_index = 1;	/* Next time will be 1 */
602 		index = 0;		/* But now use 0 */
603 	}
604 
605 	/* On decryption we store (data) which is cypher. */
606 	stp->str_output[index] = data;
607 	return(data ^ stp->str_feed[index]);
608 }
609 
610 /*
611  * DES 64 bit Output Feedback
612  *
613  * key --->+-----+
614  *	+->| DES |--+
615  *	|  +-----+  |
616  *	+-----------+
617  *	            v
618  *  INPUT -------->(+) ----> DATA
619  *
620  * Given:
621  *	iV: Initial vector, 64 bits (8 bytes) long.
622  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
623  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
624  *
625  *	V0 = DES(iV, key)
626  *	V(n+1) = DES(Vn, key)
627  *	On = Dn ^ Vn
628  */
629 
630 void ofb64_encrypt(unsigned char *s, int c)
631 {
632 	struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
633 	int index;
634 
635 	index = stp->str_index;
636 	while (c-- > 0) {
637 		if (index == sizeof(des_cblock)) {
638 			des_cblock b;
639 			des_ecb_encrypt(&stp->str_feed,&b, stp->str_sched, 1);
640 			memcpy(stp->str_feed, b, sizeof(des_cblock));
641 			index = 0;
642 		}
643 		*s++ ^= stp->str_feed[index];
644 		index++;
645 	}
646 	stp->str_index = index;
647 }
648 
649 int ofb64_decrypt(int data)
650 {
651 	struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
652 	int index;
653 
654 	if (data == -1) {
655 		/*
656 		 * Back up one byte.  It is assumed that we will
657 		 * never back up more than one byte.  If we do, this
658 		 * may or may not work.
659 		 */
660 		if (stp->str_index)
661 			--stp->str_index;
662 		return(0);
663 	}
664 
665 	index = stp->str_index++;
666 	if (index == sizeof(des_cblock)) {
667 		des_cblock b;
668 		des_ecb_encrypt(&stp->str_feed,&b,stp->str_sched, 1);
669 		memcpy(stp->str_feed, b, sizeof(des_cblock));
670 		stp->str_index = 1;	/* Next time will be 1 */
671 		index = 0;		/* But now use 0 */
672 	}
673 
674 	return(data ^ stp->str_feed[index]);
675 }
676 #endif
677 
678