xref: /freebsd/crypto/heimdal/appl/telnet/libtelnet/enc_des.c (revision 5e9cd1ae3e10592ed70e7575551cba1bbab04d84)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *	This product includes software developed by the University of
16  *	California, Berkeley and its contributors.
17  * 4. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33 
34 #include <config.h>
35 
36 RCSID("$Id: enc_des.c,v 1.16 1998/07/09 23:16:23 assar Exp $");
37 
38 #if	defined(AUTHENTICATION) && defined(ENCRYPTION) && defined(DES_ENCRYPTION)
39 #include <arpa/telnet.h>
40 #include <stdio.h>
41 #ifdef	__STDC__
42 #include <stdlib.h>
43 #include <string.h>
44 #endif
45 #include <roken.h>
46 #ifdef SOCKS
47 #include <socks.h>
48 #endif
49 
50 #include "encrypt.h"
51 #include "misc-proto.h"
52 
53 #include <des.h>
54 
55 extern int encrypt_debug_mode;
56 
57 #define	CFB	0
58 #define	OFB	1
59 
60 #define	NO_SEND_IV	1
61 #define	NO_RECV_IV	2
62 #define	NO_KEYID	4
63 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
64 #define	SUCCESS		0
65 #define	FAILED		-1
66 
67 
68 struct stinfo {
69   des_cblock	str_output;
70   des_cblock	str_feed;
71   des_cblock	str_iv;
72   des_cblock	str_ikey;
73   des_key_schedule str_sched;
74   int		str_index;
75   int		str_flagshift;
76 };
77 
78 struct fb {
79 	des_cblock krbdes_key;
80 	des_key_schedule krbdes_sched;
81 	des_cblock temp_feed;
82 	unsigned char fb_feed[64];
83 	int need_start;
84 	int state[2];
85 	int keyid[2];
86 	int once;
87 	struct stinfo streams[2];
88 };
89 
90 static struct fb fb[2];
91 
92 struct keyidlist {
93 	char	*keyid;
94 	int	keyidlen;
95 	char	*key;
96 	int	keylen;
97 	int	flags;
98 } keyidlist [] = {
99 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
100 	{ 0, 0, 0, 0, 0 }
101 };
102 
103 #define	KEYFLAG_MASK	03
104 
105 #define	KEYFLAG_NOINIT	00
106 #define	KEYFLAG_INIT	01
107 #define	KEYFLAG_OK	02
108 #define	KEYFLAG_BAD	03
109 
110 #define	KEYFLAG_SHIFT	2
111 
112 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
113 
114 #define	FB64_IV		1
115 #define	FB64_IV_OK	2
116 #define	FB64_IV_BAD	3
117 
118 
119 void fb64_stream_iv (des_cblock, struct stinfo *);
120 void fb64_init (struct fb *);
121 static int fb64_start (struct fb *, int, int);
122 int fb64_is (unsigned char *, int, struct fb *);
123 int fb64_reply (unsigned char *, int, struct fb *);
124 static void fb64_session (Session_Key *, int, struct fb *);
125 void fb64_stream_key (des_cblock, struct stinfo *);
126 int fb64_keyid (int, unsigned char *, int *, struct fb *);
127 
128 void cfb64_init(int server)
129 {
130 	fb64_init(&fb[CFB]);
131 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
132 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
133 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
134 }
135 
136 
137 void ofb64_init(int server)
138 {
139 	fb64_init(&fb[OFB]);
140 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
141 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
142 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
143 }
144 
145 void fb64_init(struct fb *fbp)
146 {
147 	memset(fbp,0, sizeof(*fbp));
148 	fbp->state[0] = fbp->state[1] = FAILED;
149 	fbp->fb_feed[0] = IAC;
150 	fbp->fb_feed[1] = SB;
151 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
152 	fbp->fb_feed[3] = ENCRYPT_IS;
153 }
154 
155 /*
156  * Returns:
157  *	-1: some error.  Negotiation is done, encryption not ready.
158  *	 0: Successful, initial negotiation all done.
159  *	 1: successful, negotiation not done yet.
160  *	 2: Not yet.  Other things (like getting the key from
161  *	    Kerberos) have to happen before we can continue.
162  */
163 int cfb64_start(int dir, int server)
164 {
165 	return(fb64_start(&fb[CFB], dir, server));
166 }
167 
168 int ofb64_start(int dir, int server)
169 {
170 	return(fb64_start(&fb[OFB], dir, server));
171 }
172 
173 static int fb64_start(struct fb *fbp, int dir, int server)
174 {
175 	int x;
176 	unsigned char *p;
177 	int state;
178 
179 	switch (dir) {
180 	case DIR_DECRYPT:
181 		/*
182 		 * This is simply a request to have the other side
183 		 * start output (our input).  He will negotiate an
184 		 * IV so we need not look for it.
185 		 */
186 		state = fbp->state[dir-1];
187 		if (state == FAILED)
188 			state = IN_PROGRESS;
189 		break;
190 
191 	case DIR_ENCRYPT:
192 		state = fbp->state[dir-1];
193 		if (state == FAILED)
194 			state = IN_PROGRESS;
195 		else if ((state & NO_SEND_IV) == 0) {
196 			break;
197 		}
198 
199 		if (!VALIDKEY(fbp->krbdes_key)) {
200 		        fbp->need_start = 1;
201 			break;
202 		}
203 
204 		state &= ~NO_SEND_IV;
205 		state |= NO_RECV_IV;
206 		if (encrypt_debug_mode)
207 			printf("Creating new feed\r\n");
208 		/*
209 		 * Create a random feed and send it over.
210 		 */
211 #ifndef OLD_DES_RANDOM_KEY
212 		des_new_random_key(&fbp->temp_feed);
213 #else
214 		/*
215 		 * From des_cryp.man "If the des_check_key flag is non-zero,
216 		 *  des_set_key will check that the key passed is
217 		 *  of odd parity and is not a week or semi-weak key."
218 		 */
219 		do {
220 			des_random_key(fbp->temp_feed);
221 			des_set_odd_parity(fbp->temp_feed);
222 		} while (des_is_weak_key(fbp->temp_feed));
223 #endif
224 		des_ecb_encrypt(&fbp->temp_feed,
225 				&fbp->temp_feed,
226 				fbp->krbdes_sched, 1);
227 		p = fbp->fb_feed + 3;
228 		*p++ = ENCRYPT_IS;
229 		p++;
230 		*p++ = FB64_IV;
231 		for (x = 0; x < sizeof(des_cblock); ++x) {
232 			if ((*p++ = fbp->temp_feed[x]) == IAC)
233 				*p++ = IAC;
234 		}
235 		*p++ = IAC;
236 		*p++ = SE;
237 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
238 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
239 		break;
240 	default:
241 		return(FAILED);
242 	}
243 	return(fbp->state[dir-1] = state);
244 }
245 
246 /*
247  * Returns:
248  *	-1: some error.  Negotiation is done, encryption not ready.
249  *	 0: Successful, initial negotiation all done.
250  *	 1: successful, negotiation not done yet.
251  */
252 
253 int cfb64_is(unsigned char *data, int cnt)
254 {
255 	return(fb64_is(data, cnt, &fb[CFB]));
256 }
257 
258 int ofb64_is(unsigned char *data, int cnt)
259 {
260 	return(fb64_is(data, cnt, &fb[OFB]));
261 }
262 
263 
264 int fb64_is(unsigned char *data, int cnt, struct fb *fbp)
265 {
266 	unsigned char *p;
267 	int state = fbp->state[DIR_DECRYPT-1];
268 
269 	if (cnt-- < 1)
270 		goto failure;
271 
272 	switch (*data++) {
273 	case FB64_IV:
274 		if (cnt != sizeof(des_cblock)) {
275 			if (encrypt_debug_mode)
276 				printf("CFB64: initial vector failed on size\r\n");
277 			state = FAILED;
278 			goto failure;
279 		}
280 
281 		if (encrypt_debug_mode)
282 			printf("CFB64: initial vector received\r\n");
283 
284 		if (encrypt_debug_mode)
285 			printf("Initializing Decrypt stream\r\n");
286 
287 		fb64_stream_iv(data, &fbp->streams[DIR_DECRYPT-1]);
288 
289 		p = fbp->fb_feed + 3;
290 		*p++ = ENCRYPT_REPLY;
291 		p++;
292 		*p++ = FB64_IV_OK;
293 		*p++ = IAC;
294 		*p++ = SE;
295 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
296 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
297 
298 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
299 		break;
300 
301 	default:
302 		if (encrypt_debug_mode) {
303 			printf("Unknown option type: %d\r\n", *(data-1));
304 			printd(data, cnt);
305 			printf("\r\n");
306 		}
307 		/* FALL THROUGH */
308 	failure:
309 		/*
310 		 * We failed.  Send an FB64_IV_BAD option
311 		 * to the other side so it will know that
312 		 * things failed.
313 		 */
314 		p = fbp->fb_feed + 3;
315 		*p++ = ENCRYPT_REPLY;
316 		p++;
317 		*p++ = FB64_IV_BAD;
318 		*p++ = IAC;
319 		*p++ = SE;
320 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
321 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
322 
323 		break;
324 	}
325 	return(fbp->state[DIR_DECRYPT-1] = state);
326 }
327 
328 /*
329  * Returns:
330  *	-1: some error.  Negotiation is done, encryption not ready.
331  *	 0: Successful, initial negotiation all done.
332  *	 1: successful, negotiation not done yet.
333  */
334 
335 int cfb64_reply(unsigned char *data, int cnt)
336 {
337 	return(fb64_reply(data, cnt, &fb[CFB]));
338 }
339 
340 int ofb64_reply(unsigned char *data, int cnt)
341 {
342 	return(fb64_reply(data, cnt, &fb[OFB]));
343 }
344 
345 
346 int fb64_reply(unsigned char *data, int cnt, struct fb *fbp)
347 {
348 	int state = fbp->state[DIR_ENCRYPT-1];
349 
350 	if (cnt-- < 1)
351 		goto failure;
352 
353 	switch (*data++) {
354 	case FB64_IV_OK:
355 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
356 		if (state == FAILED)
357 			state = IN_PROGRESS;
358 		state &= ~NO_RECV_IV;
359 		encrypt_send_keyid(DIR_ENCRYPT, (unsigned char *)"\0", 1, 1);
360 		break;
361 
362 	case FB64_IV_BAD:
363 		memset(fbp->temp_feed, 0, sizeof(des_cblock));
364 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
365 		state = FAILED;
366 		break;
367 
368 	default:
369 		if (encrypt_debug_mode) {
370 			printf("Unknown option type: %d\r\n", data[-1]);
371 			printd(data, cnt);
372 			printf("\r\n");
373 		}
374 		/* FALL THROUGH */
375 	failure:
376 		state = FAILED;
377 		break;
378 	}
379 	return(fbp->state[DIR_ENCRYPT-1] = state);
380 }
381 
382 void cfb64_session(Session_Key *key, int server)
383 {
384 	fb64_session(key, server, &fb[CFB]);
385 }
386 
387 void ofb64_session(Session_Key *key, int server)
388 {
389 	fb64_session(key, server, &fb[OFB]);
390 }
391 
392 static void fb64_session(Session_Key *key, int server, struct fb *fbp)
393 {
394 
395 	if (!key || key->type != SK_DES) {
396 		if (encrypt_debug_mode)
397 			printf("Can't set krbdes's session key (%d != %d)\r\n",
398 				key ? key->type : -1, SK_DES);
399 		return;
400 	}
401 	memcpy(fbp->krbdes_key, key->data, sizeof(des_cblock));
402 
403 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
404 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
405 
406 	if (fbp->once == 0) {
407 #ifndef OLD_DES_RANDOM_KEY
408 		des_init_random_number_generator(&fbp->krbdes_key);
409 #endif
410 		fbp->once = 1;
411 	}
412 	des_key_sched(&fbp->krbdes_key, fbp->krbdes_sched);
413 	/*
414 	 * Now look to see if krbdes_start() was was waiting for
415 	 * the key to show up.  If so, go ahead an call it now
416 	 * that we have the key.
417 	 */
418 	if (fbp->need_start) {
419 		fbp->need_start = 0;
420 		fb64_start(fbp, DIR_ENCRYPT, server);
421 	}
422 }
423 
424 /*
425  * We only accept a keyid of 0.  If we get a keyid of
426  * 0, then mark the state as SUCCESS.
427  */
428 
429 int cfb64_keyid(int dir, unsigned char *kp, int *lenp)
430 {
431 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
432 }
433 
434 int ofb64_keyid(int dir, unsigned char *kp, int *lenp)
435 {
436 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
437 }
438 
439 int fb64_keyid(int dir, unsigned char *kp, int *lenp, struct fb *fbp)
440 {
441 	int state = fbp->state[dir-1];
442 
443 	if (*lenp != 1 || (*kp != '\0')) {
444 		*lenp = 0;
445 		return(state);
446 	}
447 
448 	if (state == FAILED)
449 		state = IN_PROGRESS;
450 
451 	state &= ~NO_KEYID;
452 
453 	return(fbp->state[dir-1] = state);
454 }
455 
456 void fb64_printsub(unsigned char *data, int cnt,
457 		   unsigned char *buf, int buflen, char *type)
458 {
459 	char lbuf[32];
460 	int i;
461 	char *cp;
462 
463 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
464 	buflen -= 1;
465 
466 	switch(data[2]) {
467 	case FB64_IV:
468 		snprintf(lbuf, sizeof(lbuf), "%s_IV", type);
469 		cp = lbuf;
470 		goto common;
471 
472 	case FB64_IV_OK:
473 		snprintf(lbuf, sizeof(lbuf), "%s_IV_OK", type);
474 		cp = lbuf;
475 		goto common;
476 
477 	case FB64_IV_BAD:
478 		snprintf(lbuf, sizeof(lbuf), "%s_IV_BAD", type);
479 		cp = lbuf;
480 		goto common;
481 
482 	default:
483 		snprintf(lbuf, sizeof(lbuf), " %d (unknown)", data[2]);
484 		cp = lbuf;
485 	common:
486 		for (; (buflen > 0) && (*buf = *cp++); buf++)
487 			buflen--;
488 		for (i = 3; i < cnt; i++) {
489 			snprintf(lbuf, sizeof(lbuf), " %d", data[i]);
490 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
491 				buflen--;
492 		}
493 		break;
494 	}
495 }
496 
497 void cfb64_printsub(unsigned char *data, int cnt,
498 		    unsigned char *buf, int buflen)
499 {
500 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
501 }
502 
503 void ofb64_printsub(unsigned char *data, int cnt,
504 		    unsigned char *buf, int buflen)
505 {
506 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
507 }
508 
509 void fb64_stream_iv(des_cblock seed, struct stinfo *stp)
510 {
511 
512 	memcpy(stp->str_iv, seed,sizeof(des_cblock));
513 	memcpy(stp->str_output, seed, sizeof(des_cblock));
514 
515 	des_key_sched(&stp->str_ikey, stp->str_sched);
516 
517 	stp->str_index = sizeof(des_cblock);
518 }
519 
520 void fb64_stream_key(des_cblock key, struct stinfo *stp)
521 {
522 	memcpy(stp->str_ikey, key, sizeof(des_cblock));
523 	des_key_sched((des_cblock*)key, stp->str_sched);
524 
525 	memcpy(stp->str_output, stp->str_iv, sizeof(des_cblock));
526 
527 	stp->str_index = sizeof(des_cblock);
528 }
529 
530 /*
531  * DES 64 bit Cipher Feedback
532  *
533  *     key --->+-----+
534  *          +->| DES |--+
535  *          |  +-----+  |
536  *	    |           v
537  *  INPUT --(--------->(+)+---> DATA
538  *          |             |
539  *	    +-------------+
540  *
541  *
542  * Given:
543  *	iV: Initial vector, 64 bits (8 bytes) long.
544  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
545  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
546  *
547  *	V0 = DES(iV, key)
548  *	On = Dn ^ Vn
549  *	V(n+1) = DES(On, key)
550  */
551 
552 void cfb64_encrypt(unsigned char *s, int c)
553 {
554 	struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
555 	int index;
556 
557 	index = stp->str_index;
558 	while (c-- > 0) {
559 		if (index == sizeof(des_cblock)) {
560 			des_cblock b;
561 			des_ecb_encrypt(&stp->str_output, &b,stp->str_sched, 1);
562 			memcpy(stp->str_feed, b, sizeof(des_cblock));
563 			index = 0;
564 		}
565 
566 		/* On encryption, we store (feed ^ data) which is cypher */
567 		*s = stp->str_output[index] = (stp->str_feed[index] ^ *s);
568 		s++;
569 		index++;
570 	}
571 	stp->str_index = index;
572 }
573 
574 int cfb64_decrypt(int data)
575 {
576 	struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
577 	int index;
578 
579 	if (data == -1) {
580 		/*
581 		 * Back up one byte.  It is assumed that we will
582 		 * never back up more than one byte.  If we do, this
583 		 * may or may not work.
584 		 */
585 		if (stp->str_index)
586 			--stp->str_index;
587 		return(0);
588 	}
589 
590 	index = stp->str_index++;
591 	if (index == sizeof(des_cblock)) {
592 		des_cblock b;
593 		des_ecb_encrypt(&stp->str_output,&b, stp->str_sched, 1);
594 		memcpy(stp->str_feed, b, sizeof(des_cblock));
595 		stp->str_index = 1;	/* Next time will be 1 */
596 		index = 0;		/* But now use 0 */
597 	}
598 
599 	/* On decryption we store (data) which is cypher. */
600 	stp->str_output[index] = data;
601 	return(data ^ stp->str_feed[index]);
602 }
603 
604 /*
605  * DES 64 bit Output Feedback
606  *
607  * key --->+-----+
608  *	+->| DES |--+
609  *	|  +-----+  |
610  *	+-----------+
611  *	            v
612  *  INPUT -------->(+) ----> DATA
613  *
614  * Given:
615  *	iV: Initial vector, 64 bits (8 bytes) long.
616  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
617  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
618  *
619  *	V0 = DES(iV, key)
620  *	V(n+1) = DES(Vn, key)
621  *	On = Dn ^ Vn
622  */
623 
624 void ofb64_encrypt(unsigned char *s, int c)
625 {
626 	struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
627 	int index;
628 
629 	index = stp->str_index;
630 	while (c-- > 0) {
631 		if (index == sizeof(des_cblock)) {
632 			des_cblock b;
633 			des_ecb_encrypt(&stp->str_feed,&b, stp->str_sched, 1);
634 			memcpy(stp->str_feed, b, sizeof(des_cblock));
635 			index = 0;
636 		}
637 		*s++ ^= stp->str_feed[index];
638 		index++;
639 	}
640 	stp->str_index = index;
641 }
642 
643 int ofb64_decrypt(int data)
644 {
645 	struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
646 	int index;
647 
648 	if (data == -1) {
649 		/*
650 		 * Back up one byte.  It is assumed that we will
651 		 * never back up more than one byte.  If we do, this
652 		 * may or may not work.
653 		 */
654 		if (stp->str_index)
655 			--stp->str_index;
656 		return(0);
657 	}
658 
659 	index = stp->str_index++;
660 	if (index == sizeof(des_cblock)) {
661 		des_cblock b;
662 		des_ecb_encrypt(&stp->str_feed,&b,stp->str_sched, 1);
663 		memcpy(stp->str_feed, b, sizeof(des_cblock));
664 		stp->str_index = 1;	/* Next time will be 1 */
665 		index = 0;		/* But now use 0 */
666 	}
667 
668 	return(data ^ stp->str_feed[index]);
669 }
670 #endif
671 
672