xref: /freebsd/crypto/heimdal/appl/telnet/libtelnet/enc_des.c (revision 9336e0699bda8a301cd2bfa37106b6ec5e32012e)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *	This product includes software developed by the University of
16  *	California, Berkeley and its contributors.
17  * 4. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33 
34 #include <config.h>
35 
36 RCSID("$Id: enc_des.c,v 1.21 2002/09/10 20:03:47 joda Exp $");
37 
38 #if	defined(AUTHENTICATION) && defined(ENCRYPTION) && defined(DES_ENCRYPTION)
39 #include <arpa/telnet.h>
40 #include <stdio.h>
41 #ifdef	__STDC__
42 #include <stdlib.h>
43 #include <string.h>
44 #endif
45 #include <roken.h>
46 #ifdef SOCKS
47 #include <socks.h>
48 #endif
49 
50 #include "encrypt.h"
51 #include "misc-proto.h"
52 
53 #include "crypto-headers.h"
54 
55 extern int encrypt_debug_mode;
56 
57 #define	CFB	0
58 #define	OFB	1
59 
60 #define	NO_SEND_IV	1
61 #define	NO_RECV_IV	2
62 #define	NO_KEYID	4
63 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
64 #define	SUCCESS		0
65 #define	FAILED		-1
66 
67 
68 struct stinfo {
69   des_cblock	str_output;
70   des_cblock	str_feed;
71   des_cblock	str_iv;
72   des_cblock	str_ikey;
73   des_key_schedule str_sched;
74   int		str_index;
75   int		str_flagshift;
76 };
77 
78 struct fb {
79 	des_cblock krbdes_key;
80 	des_key_schedule krbdes_sched;
81 	des_cblock temp_feed;
82 	unsigned char fb_feed[64];
83 	int need_start;
84 	int state[2];
85 	int keyid[2];
86 	int once;
87 	struct stinfo streams[2];
88 };
89 
90 static struct fb fb[2];
91 
92 struct keyidlist {
93 	char	*keyid;
94 	int	keyidlen;
95 	char	*key;
96 	int	keylen;
97 	int	flags;
98 } keyidlist [] = {
99 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
100 	{ 0, 0, 0, 0, 0 }
101 };
102 
103 #define	KEYFLAG_MASK	03
104 
105 #define	KEYFLAG_NOINIT	00
106 #define	KEYFLAG_INIT	01
107 #define	KEYFLAG_OK	02
108 #define	KEYFLAG_BAD	03
109 
110 #define	KEYFLAG_SHIFT	2
111 
112 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
113 
114 #define	FB64_IV		1
115 #define	FB64_IV_OK	2
116 #define	FB64_IV_BAD	3
117 
118 
119 void fb64_stream_iv (des_cblock, struct stinfo *);
120 void fb64_init (struct fb *);
121 static int fb64_start (struct fb *, int, int);
122 int fb64_is (unsigned char *, int, struct fb *);
123 int fb64_reply (unsigned char *, int, struct fb *);
124 static void fb64_session (Session_Key *, int, struct fb *);
125 void fb64_stream_key (des_cblock, struct stinfo *);
126 int fb64_keyid (int, unsigned char *, int *, struct fb *);
127 void fb64_printsub(unsigned char *, int ,
128 		   unsigned char *, int , char *);
129 
130 void cfb64_init(int server)
131 {
132 	fb64_init(&fb[CFB]);
133 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
134 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
135 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
136 }
137 
138 
139 void ofb64_init(int server)
140 {
141 	fb64_init(&fb[OFB]);
142 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
143 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
144 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
145 }
146 
147 void fb64_init(struct fb *fbp)
148 {
149 	memset(fbp,0, sizeof(*fbp));
150 	fbp->state[0] = fbp->state[1] = FAILED;
151 	fbp->fb_feed[0] = IAC;
152 	fbp->fb_feed[1] = SB;
153 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
154 	fbp->fb_feed[3] = ENCRYPT_IS;
155 }
156 
157 /*
158  * Returns:
159  *	-1: some error.  Negotiation is done, encryption not ready.
160  *	 0: Successful, initial negotiation all done.
161  *	 1: successful, negotiation not done yet.
162  *	 2: Not yet.  Other things (like getting the key from
163  *	    Kerberos) have to happen before we can continue.
164  */
165 int cfb64_start(int dir, int server)
166 {
167 	return(fb64_start(&fb[CFB], dir, server));
168 }
169 
170 int ofb64_start(int dir, int server)
171 {
172 	return(fb64_start(&fb[OFB], dir, server));
173 }
174 
175 static int fb64_start(struct fb *fbp, int dir, int server)
176 {
177 	int x;
178 	unsigned char *p;
179 	int state;
180 
181 	switch (dir) {
182 	case DIR_DECRYPT:
183 		/*
184 		 * This is simply a request to have the other side
185 		 * start output (our input).  He will negotiate an
186 		 * IV so we need not look for it.
187 		 */
188 		state = fbp->state[dir-1];
189 		if (state == FAILED)
190 			state = IN_PROGRESS;
191 		break;
192 
193 	case DIR_ENCRYPT:
194 		state = fbp->state[dir-1];
195 		if (state == FAILED)
196 			state = IN_PROGRESS;
197 		else if ((state & NO_SEND_IV) == 0) {
198 			break;
199 		}
200 
201 		if (!VALIDKEY(fbp->krbdes_key)) {
202 		        fbp->need_start = 1;
203 			break;
204 		}
205 
206 		state &= ~NO_SEND_IV;
207 		state |= NO_RECV_IV;
208 		if (encrypt_debug_mode)
209 			printf("Creating new feed\r\n");
210 		/*
211 		 * Create a random feed and send it over.
212 		 */
213 #ifndef OLD_DES_RANDOM_KEY
214 		des_new_random_key(&fbp->temp_feed);
215 #else
216 		/*
217 		 * From des_cryp.man "If the des_check_key flag is non-zero,
218 		 *  des_set_key will check that the key passed is
219 		 *  of odd parity and is not a week or semi-weak key."
220 		 */
221 		do {
222 			des_random_key(fbp->temp_feed);
223 			des_set_odd_parity(fbp->temp_feed);
224 		} while (des_is_weak_key(fbp->temp_feed));
225 #endif
226 		des_ecb_encrypt(&fbp->temp_feed,
227 				&fbp->temp_feed,
228 				fbp->krbdes_sched, 1);
229 		p = fbp->fb_feed + 3;
230 		*p++ = ENCRYPT_IS;
231 		p++;
232 		*p++ = FB64_IV;
233 		for (x = 0; x < sizeof(des_cblock); ++x) {
234 			if ((*p++ = fbp->temp_feed[x]) == IAC)
235 				*p++ = IAC;
236 		}
237 		*p++ = IAC;
238 		*p++ = SE;
239 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
240 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
241 		break;
242 	default:
243 		return(FAILED);
244 	}
245 	return(fbp->state[dir-1] = state);
246 }
247 
248 /*
249  * Returns:
250  *	-1: some error.  Negotiation is done, encryption not ready.
251  *	 0: Successful, initial negotiation all done.
252  *	 1: successful, negotiation not done yet.
253  */
254 
255 int cfb64_is(unsigned char *data, int cnt)
256 {
257 	return(fb64_is(data, cnt, &fb[CFB]));
258 }
259 
260 int ofb64_is(unsigned char *data, int cnt)
261 {
262 	return(fb64_is(data, cnt, &fb[OFB]));
263 }
264 
265 
266 int fb64_is(unsigned char *data, int cnt, struct fb *fbp)
267 {
268 	unsigned char *p;
269 	int state = fbp->state[DIR_DECRYPT-1];
270 
271 	if (cnt-- < 1)
272 		goto failure;
273 
274 	switch (*data++) {
275 	case FB64_IV:
276 		if (cnt != sizeof(des_cblock)) {
277 			if (encrypt_debug_mode)
278 				printf("CFB64: initial vector failed on size\r\n");
279 			state = FAILED;
280 			goto failure;
281 		}
282 
283 		if (encrypt_debug_mode)
284 			printf("CFB64: initial vector received\r\n");
285 
286 		if (encrypt_debug_mode)
287 			printf("Initializing Decrypt stream\r\n");
288 
289 		fb64_stream_iv(data, &fbp->streams[DIR_DECRYPT-1]);
290 
291 		p = fbp->fb_feed + 3;
292 		*p++ = ENCRYPT_REPLY;
293 		p++;
294 		*p++ = FB64_IV_OK;
295 		*p++ = IAC;
296 		*p++ = SE;
297 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
298 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
299 
300 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
301 		break;
302 
303 	default:
304 		if (encrypt_debug_mode) {
305 			printf("Unknown option type: %d\r\n", *(data-1));
306 			printd(data, cnt);
307 			printf("\r\n");
308 		}
309 		/* FALL THROUGH */
310 	failure:
311 		/*
312 		 * We failed.  Send an FB64_IV_BAD option
313 		 * to the other side so it will know that
314 		 * things failed.
315 		 */
316 		p = fbp->fb_feed + 3;
317 		*p++ = ENCRYPT_REPLY;
318 		p++;
319 		*p++ = FB64_IV_BAD;
320 		*p++ = IAC;
321 		*p++ = SE;
322 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
323 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
324 
325 		break;
326 	}
327 	return(fbp->state[DIR_DECRYPT-1] = state);
328 }
329 
330 /*
331  * Returns:
332  *	-1: some error.  Negotiation is done, encryption not ready.
333  *	 0: Successful, initial negotiation all done.
334  *	 1: successful, negotiation not done yet.
335  */
336 
337 int cfb64_reply(unsigned char *data, int cnt)
338 {
339 	return(fb64_reply(data, cnt, &fb[CFB]));
340 }
341 
342 int ofb64_reply(unsigned char *data, int cnt)
343 {
344 	return(fb64_reply(data, cnt, &fb[OFB]));
345 }
346 
347 
348 int fb64_reply(unsigned char *data, int cnt, struct fb *fbp)
349 {
350 	int state = fbp->state[DIR_ENCRYPT-1];
351 
352 	if (cnt-- < 1)
353 		goto failure;
354 
355 	switch (*data++) {
356 	case FB64_IV_OK:
357 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
358 		if (state == FAILED)
359 			state = IN_PROGRESS;
360 		state &= ~NO_RECV_IV;
361 		encrypt_send_keyid(DIR_ENCRYPT, (unsigned char *)"\0", 1, 1);
362 		break;
363 
364 	case FB64_IV_BAD:
365 		memset(fbp->temp_feed, 0, sizeof(des_cblock));
366 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
367 		state = FAILED;
368 		break;
369 
370 	default:
371 		if (encrypt_debug_mode) {
372 			printf("Unknown option type: %d\r\n", data[-1]);
373 			printd(data, cnt);
374 			printf("\r\n");
375 		}
376 		/* FALL THROUGH */
377 	failure:
378 		state = FAILED;
379 		break;
380 	}
381 	return(fbp->state[DIR_ENCRYPT-1] = state);
382 }
383 
384 void cfb64_session(Session_Key *key, int server)
385 {
386 	fb64_session(key, server, &fb[CFB]);
387 }
388 
389 void ofb64_session(Session_Key *key, int server)
390 {
391 	fb64_session(key, server, &fb[OFB]);
392 }
393 
394 static void fb64_session(Session_Key *key, int server, struct fb *fbp)
395 {
396 
397 	if (!key || key->type != SK_DES) {
398 		if (encrypt_debug_mode)
399 			printf("Can't set krbdes's session key (%d != %d)\r\n",
400 				key ? key->type : -1, SK_DES);
401 		return;
402 	}
403 	memcpy(fbp->krbdes_key, key->data, sizeof(des_cblock));
404 
405 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
406 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
407 
408 	if (fbp->once == 0) {
409 #if !defined(OLD_DES_RANDOM_KEY) && !defined(HAVE_OPENSSL)
410 		des_init_random_number_generator(&fbp->krbdes_key);
411 #endif
412 		fbp->once = 1;
413 	}
414 	des_key_sched(&fbp->krbdes_key, fbp->krbdes_sched);
415 	/*
416 	 * Now look to see if krbdes_start() was was waiting for
417 	 * the key to show up.  If so, go ahead an call it now
418 	 * that we have the key.
419 	 */
420 	if (fbp->need_start) {
421 		fbp->need_start = 0;
422 		fb64_start(fbp, DIR_ENCRYPT, server);
423 	}
424 }
425 
426 /*
427  * We only accept a keyid of 0.  If we get a keyid of
428  * 0, then mark the state as SUCCESS.
429  */
430 
431 int cfb64_keyid(int dir, unsigned char *kp, int *lenp)
432 {
433 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
434 }
435 
436 int ofb64_keyid(int dir, unsigned char *kp, int *lenp)
437 {
438 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
439 }
440 
441 int fb64_keyid(int dir, unsigned char *kp, int *lenp, struct fb *fbp)
442 {
443 	int state = fbp->state[dir-1];
444 
445 	if (*lenp != 1 || (*kp != '\0')) {
446 		*lenp = 0;
447 		return(state);
448 	}
449 
450 	if (state == FAILED)
451 		state = IN_PROGRESS;
452 
453 	state &= ~NO_KEYID;
454 
455 	return(fbp->state[dir-1] = state);
456 }
457 
458 void fb64_printsub(unsigned char *data, int cnt,
459 		   unsigned char *buf, int buflen, char *type)
460 {
461 	char lbuf[32];
462 	int i;
463 	char *cp;
464 
465 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
466 	buflen -= 1;
467 
468 	switch(data[2]) {
469 	case FB64_IV:
470 		snprintf(lbuf, sizeof(lbuf), "%s_IV", type);
471 		cp = lbuf;
472 		goto common;
473 
474 	case FB64_IV_OK:
475 		snprintf(lbuf, sizeof(lbuf), "%s_IV_OK", type);
476 		cp = lbuf;
477 		goto common;
478 
479 	case FB64_IV_BAD:
480 		snprintf(lbuf, sizeof(lbuf), "%s_IV_BAD", type);
481 		cp = lbuf;
482 		goto common;
483 
484 	default:
485 		snprintf(lbuf, sizeof(lbuf), " %d (unknown)", data[2]);
486 		cp = lbuf;
487 	common:
488 		for (; (buflen > 0) && (*buf = *cp++); buf++)
489 			buflen--;
490 		for (i = 3; i < cnt; i++) {
491 			snprintf(lbuf, sizeof(lbuf), " %d", data[i]);
492 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
493 				buflen--;
494 		}
495 		break;
496 	}
497 }
498 
499 void cfb64_printsub(unsigned char *data, int cnt,
500 		    unsigned char *buf, int buflen)
501 {
502 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
503 }
504 
505 void ofb64_printsub(unsigned char *data, int cnt,
506 		    unsigned char *buf, int buflen)
507 {
508 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
509 }
510 
511 void fb64_stream_iv(des_cblock seed, struct stinfo *stp)
512 {
513 
514 	memcpy(stp->str_iv, seed,sizeof(des_cblock));
515 	memcpy(stp->str_output, seed, sizeof(des_cblock));
516 
517 	des_key_sched(&stp->str_ikey, stp->str_sched);
518 
519 	stp->str_index = sizeof(des_cblock);
520 }
521 
522 void fb64_stream_key(des_cblock key, struct stinfo *stp)
523 {
524 	memcpy(stp->str_ikey, key, sizeof(des_cblock));
525 	des_key_sched((des_cblock*)key, stp->str_sched);
526 
527 	memcpy(stp->str_output, stp->str_iv, sizeof(des_cblock));
528 
529 	stp->str_index = sizeof(des_cblock);
530 }
531 
532 /*
533  * DES 64 bit Cipher Feedback
534  *
535  *     key --->+-----+
536  *          +->| DES |--+
537  *          |  +-----+  |
538  *	    |           v
539  *  INPUT --(--------->(+)+---> DATA
540  *          |             |
541  *	    +-------------+
542  *
543  *
544  * Given:
545  *	iV: Initial vector, 64 bits (8 bytes) long.
546  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
547  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
548  *
549  *	V0 = DES(iV, key)
550  *	On = Dn ^ Vn
551  *	V(n+1) = DES(On, key)
552  */
553 
554 void cfb64_encrypt(unsigned char *s, int c)
555 {
556 	struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
557 	int index;
558 
559 	index = stp->str_index;
560 	while (c-- > 0) {
561 		if (index == sizeof(des_cblock)) {
562 			des_cblock b;
563 			des_ecb_encrypt(&stp->str_output, &b,stp->str_sched, 1);
564 			memcpy(stp->str_feed, b, sizeof(des_cblock));
565 			index = 0;
566 		}
567 
568 		/* On encryption, we store (feed ^ data) which is cypher */
569 		*s = stp->str_output[index] = (stp->str_feed[index] ^ *s);
570 		s++;
571 		index++;
572 	}
573 	stp->str_index = index;
574 }
575 
576 int cfb64_decrypt(int data)
577 {
578 	struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
579 	int index;
580 
581 	if (data == -1) {
582 		/*
583 		 * Back up one byte.  It is assumed that we will
584 		 * never back up more than one byte.  If we do, this
585 		 * may or may not work.
586 		 */
587 		if (stp->str_index)
588 			--stp->str_index;
589 		return(0);
590 	}
591 
592 	index = stp->str_index++;
593 	if (index == sizeof(des_cblock)) {
594 		des_cblock b;
595 		des_ecb_encrypt(&stp->str_output,&b, stp->str_sched, 1);
596 		memcpy(stp->str_feed, b, sizeof(des_cblock));
597 		stp->str_index = 1;	/* Next time will be 1 */
598 		index = 0;		/* But now use 0 */
599 	}
600 
601 	/* On decryption we store (data) which is cypher. */
602 	stp->str_output[index] = data;
603 	return(data ^ stp->str_feed[index]);
604 }
605 
606 /*
607  * DES 64 bit Output Feedback
608  *
609  * key --->+-----+
610  *	+->| DES |--+
611  *	|  +-----+  |
612  *	+-----------+
613  *	            v
614  *  INPUT -------->(+) ----> DATA
615  *
616  * Given:
617  *	iV: Initial vector, 64 bits (8 bytes) long.
618  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
619  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
620  *
621  *	V0 = DES(iV, key)
622  *	V(n+1) = DES(Vn, key)
623  *	On = Dn ^ Vn
624  */
625 
626 void ofb64_encrypt(unsigned char *s, int c)
627 {
628 	struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
629 	int index;
630 
631 	index = stp->str_index;
632 	while (c-- > 0) {
633 		if (index == sizeof(des_cblock)) {
634 			des_cblock b;
635 			des_ecb_encrypt(&stp->str_feed,&b, stp->str_sched, 1);
636 			memcpy(stp->str_feed, b, sizeof(des_cblock));
637 			index = 0;
638 		}
639 		*s++ ^= stp->str_feed[index];
640 		index++;
641 	}
642 	stp->str_index = index;
643 }
644 
645 int ofb64_decrypt(int data)
646 {
647 	struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
648 	int index;
649 
650 	if (data == -1) {
651 		/*
652 		 * Back up one byte.  It is assumed that we will
653 		 * never back up more than one byte.  If we do, this
654 		 * may or may not work.
655 		 */
656 		if (stp->str_index)
657 			--stp->str_index;
658 		return(0);
659 	}
660 
661 	index = stp->str_index++;
662 	if (index == sizeof(des_cblock)) {
663 		des_cblock b;
664 		des_ecb_encrypt(&stp->str_feed,&b,stp->str_sched, 1);
665 		memcpy(stp->str_feed, b, sizeof(des_cblock));
666 		stp->str_index = 1;	/* Next time will be 1 */
667 		index = 0;		/* But now use 0 */
668 	}
669 
670 	return(data ^ stp->str_feed[index]);
671 }
672 #endif
673 
674