xref: /freebsd/crypto/heimdal/appl/telnet/libtelnet/enc_des.c (revision af23369a6deaaeb612ab266eb88b8bb8d560c322)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *	This product includes software developed by the University of
16  *	California, Berkeley and its contributors.
17  * 4. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33 
34 #include <config.h>
35 
36 RCSID("$Id$");
37 
38 #if	defined(AUTHENTICATION) && defined(ENCRYPTION) && defined(DES_ENCRYPTION)
39 #include <arpa/telnet.h>
40 #include <stdio.h>
41 #ifdef	__STDC__
42 #include <stdlib.h>
43 #include <string.h>
44 #endif
45 #include <roken.h>
46 #ifdef SOCKS
47 #include <socks.h>
48 #endif
49 
50 #include "encrypt.h"
51 #include "misc-proto.h"
52 
53 #include "crypto-headers.h"
54 
55 extern int encrypt_debug_mode;
56 
57 #define	CFB	0
58 #define	OFB	1
59 
60 #define	NO_SEND_IV	1
61 #define	NO_RECV_IV	2
62 #define	NO_KEYID	4
63 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
64 #define	SUCCESS		0
65 #define	FAILED		-1
66 
67 
68 struct stinfo {
69   DES_cblock	str_output;
70   DES_cblock	str_feed;
71   DES_cblock	str_iv;
72   DES_cblock	str_ikey;
73   DES_key_schedule str_sched;
74   int		str_index;
75   int		str_flagshift;
76 };
77 
78 struct fb {
79 	DES_cblock krbdes_key;
80 	DES_key_schedule krbdes_sched;
81 	DES_cblock temp_feed;
82 	unsigned char fb_feed[64];
83 	int need_start;
84 	int state[2];
85 	int keyid[2];
86 	struct stinfo streams[2];
87 };
88 
89 static struct fb fb[2];
90 
91 struct keyidlist {
92 	char	*keyid;
93 	int	keyidlen;
94 	char	*key;
95 	int	keylen;
96 	int	flags;
97 } keyidlist [] = {
98 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
99 	{ 0, 0, 0, 0, 0 }
100 };
101 
102 #define	KEYFLAG_MASK	03
103 
104 #define	KEYFLAG_NOINIT	00
105 #define	KEYFLAG_INIT	01
106 #define	KEYFLAG_OK	02
107 #define	KEYFLAG_BAD	03
108 
109 #define	KEYFLAG_SHIFT	2
110 
111 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
112 
113 #define	FB64_IV		1
114 #define	FB64_IV_OK	2
115 #define	FB64_IV_BAD	3
116 
117 
118 void fb64_stream_iv (DES_cblock, struct stinfo *);
119 void fb64_init (struct fb *);
120 static int fb64_start (struct fb *, int, int);
121 int fb64_is (unsigned char *, int, struct fb *);
122 int fb64_reply (unsigned char *, int, struct fb *);
123 static void fb64_session (Session_Key *, int, struct fb *);
124 void fb64_stream_key (DES_cblock, struct stinfo *);
125 int fb64_keyid (int, unsigned char *, int *, struct fb *);
126 void fb64_printsub(unsigned char *, size_t ,
127 		   unsigned char *, size_t , char *);
128 
129 void cfb64_init(int server)
130 {
131 	fb64_init(&fb[CFB]);
132 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
133 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
134 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
135 }
136 
137 
138 void ofb64_init(int server)
139 {
140 	fb64_init(&fb[OFB]);
141 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
142 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
143 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
144 }
145 
146 void fb64_init(struct fb *fbp)
147 {
148 	memset(fbp,0, sizeof(*fbp));
149 	fbp->state[0] = fbp->state[1] = FAILED;
150 	fbp->fb_feed[0] = IAC;
151 	fbp->fb_feed[1] = SB;
152 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
153 	fbp->fb_feed[3] = ENCRYPT_IS;
154 }
155 
156 /*
157  * Returns:
158  *	-1: some error.  Negotiation is done, encryption not ready.
159  *	 0: Successful, initial negotiation all done.
160  *	 1: successful, negotiation not done yet.
161  *	 2: Not yet.  Other things (like getting the key from
162  *	    Kerberos) have to happen before we can continue.
163  */
164 int cfb64_start(int dir, int server)
165 {
166 	return(fb64_start(&fb[CFB], dir, server));
167 }
168 
169 int ofb64_start(int dir, int server)
170 {
171 	return(fb64_start(&fb[OFB], dir, server));
172 }
173 
174 static int fb64_start(struct fb *fbp, int dir, int server)
175 {
176 	int x;
177 	unsigned char *p;
178 	int state;
179 
180 	switch (dir) {
181 	case DIR_DECRYPT:
182 		/*
183 		 * This is simply a request to have the other side
184 		 * start output (our input).  He will negotiate an
185 		 * IV so we need not look for it.
186 		 */
187 		state = fbp->state[dir-1];
188 		if (state == FAILED)
189 			state = IN_PROGRESS;
190 		break;
191 
192 	case DIR_ENCRYPT:
193 		state = fbp->state[dir-1];
194 		if (state == FAILED)
195 			state = IN_PROGRESS;
196 		else if ((state & NO_SEND_IV) == 0) {
197 			break;
198 		}
199 
200 		if (!VALIDKEY(fbp->krbdes_key)) {
201 		        fbp->need_start = 1;
202 			break;
203 		}
204 
205 		state &= ~NO_SEND_IV;
206 		state |= NO_RECV_IV;
207 		if (encrypt_debug_mode)
208 			printf("Creating new feed\r\n");
209 		/*
210 		 * Create a random feed and send it over.
211 		 */
212 		do {
213 		    if (RAND_bytes(fbp->temp_feed,
214 				   sizeof(*fbp->temp_feed)) != 1)
215 			abort();
216 		    DES_set_odd_parity(&fbp->temp_feed);
217 		} while(DES_is_weak_key(&fbp->temp_feed));
218 
219 		p = fbp->fb_feed + 3;
220 		*p++ = ENCRYPT_IS;
221 		p++;
222 		*p++ = FB64_IV;
223 		for (x = 0; x < sizeof(DES_cblock); ++x) {
224 			if ((*p++ = fbp->temp_feed[x]) == IAC)
225 				*p++ = IAC;
226 		}
227 		*p++ = IAC;
228 		*p++ = SE;
229 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
230 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
231 		break;
232 	default:
233 		return(FAILED);
234 	}
235 	return(fbp->state[dir-1] = state);
236 }
237 
238 /*
239  * Returns:
240  *	-1: some error.  Negotiation is done, encryption not ready.
241  *	 0: Successful, initial negotiation all done.
242  *	 1: successful, negotiation not done yet.
243  */
244 
245 int cfb64_is(unsigned char *data, int cnt)
246 {
247 	return(fb64_is(data, cnt, &fb[CFB]));
248 }
249 
250 int ofb64_is(unsigned char *data, int cnt)
251 {
252 	return(fb64_is(data, cnt, &fb[OFB]));
253 }
254 
255 
256 int fb64_is(unsigned char *data, int cnt, struct fb *fbp)
257 {
258 	unsigned char *p;
259 	int state = fbp->state[DIR_DECRYPT-1];
260 
261 	if (cnt-- < 1)
262 		goto failure;
263 
264 	switch (*data++) {
265 	case FB64_IV:
266 		if (cnt != sizeof(DES_cblock)) {
267 			if (encrypt_debug_mode)
268 				printf("CFB64: initial vector failed on size\r\n");
269 			state = FAILED;
270 			goto failure;
271 		}
272 
273 		if (encrypt_debug_mode)
274 			printf("CFB64: initial vector received\r\n");
275 
276 		if (encrypt_debug_mode)
277 			printf("Initializing Decrypt stream\r\n");
278 
279 		fb64_stream_iv(data, &fbp->streams[DIR_DECRYPT-1]);
280 
281 		p = fbp->fb_feed + 3;
282 		*p++ = ENCRYPT_REPLY;
283 		p++;
284 		*p++ = FB64_IV_OK;
285 		*p++ = IAC;
286 		*p++ = SE;
287 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
288 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
289 
290 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
291 		break;
292 
293 	default:
294 		if (encrypt_debug_mode) {
295 			printf("Unknown option type: %d\r\n", *(data-1));
296 			printd(data, cnt);
297 			printf("\r\n");
298 		}
299 		/* FALL THROUGH */
300 	failure:
301 		/*
302 		 * We failed.  Send an FB64_IV_BAD option
303 		 * to the other side so it will know that
304 		 * things failed.
305 		 */
306 		p = fbp->fb_feed + 3;
307 		*p++ = ENCRYPT_REPLY;
308 		p++;
309 		*p++ = FB64_IV_BAD;
310 		*p++ = IAC;
311 		*p++ = SE;
312 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
313 		telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
314 
315 		break;
316 	}
317 	return(fbp->state[DIR_DECRYPT-1] = state);
318 }
319 
320 /*
321  * Returns:
322  *	-1: some error.  Negotiation is done, encryption not ready.
323  *	 0: Successful, initial negotiation all done.
324  *	 1: successful, negotiation not done yet.
325  */
326 
327 int cfb64_reply(unsigned char *data, int cnt)
328 {
329 	return(fb64_reply(data, cnt, &fb[CFB]));
330 }
331 
332 int ofb64_reply(unsigned char *data, int cnt)
333 {
334 	return(fb64_reply(data, cnt, &fb[OFB]));
335 }
336 
337 
338 int fb64_reply(unsigned char *data, int cnt, struct fb *fbp)
339 {
340 	int state = fbp->state[DIR_ENCRYPT-1];
341 
342 	if (cnt-- < 1)
343 		goto failure;
344 
345 	switch (*data++) {
346 	case FB64_IV_OK:
347 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
348 		if (state == FAILED)
349 			state = IN_PROGRESS;
350 		state &= ~NO_RECV_IV;
351 		encrypt_send_keyid(DIR_ENCRYPT, (unsigned char *)"\0", 1, 1);
352 		break;
353 
354 	case FB64_IV_BAD:
355 		memset(fbp->temp_feed, 0, sizeof(DES_cblock));
356 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
357 		state = FAILED;
358 		break;
359 
360 	default:
361 		if (encrypt_debug_mode) {
362 			printf("Unknown option type: %d\r\n", data[-1]);
363 			printd(data, cnt);
364 			printf("\r\n");
365 		}
366 		/* FALL THROUGH */
367 	failure:
368 		state = FAILED;
369 		break;
370 	}
371 	return(fbp->state[DIR_ENCRYPT-1] = state);
372 }
373 
374 void cfb64_session(Session_Key *key, int server)
375 {
376 	fb64_session(key, server, &fb[CFB]);
377 }
378 
379 void ofb64_session(Session_Key *key, int server)
380 {
381 	fb64_session(key, server, &fb[OFB]);
382 }
383 
384 static void fb64_session(Session_Key *key, int server, struct fb *fbp)
385 {
386 
387 	if (!key || key->type != SK_DES) {
388 		if (encrypt_debug_mode)
389 			printf("Can't set krbdes's session key (%d != %d)\r\n",
390 				key ? key->type : -1, SK_DES);
391 		return;
392 	}
393 	memcpy(fbp->krbdes_key, key->data, sizeof(DES_cblock));
394 
395 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
396 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
397 
398 	RAND_seed(key->data, key->length);
399 
400 	DES_set_key_checked((DES_cblock *)&fbp->krbdes_key,
401 			    &fbp->krbdes_sched);
402 	/*
403 	 * Now look to see if krbdes_start() was waiting for the key to
404 	 * show up.  If so, go ahead an call it now that we have the key.
405 	 */
406 	if (fbp->need_start) {
407 		fbp->need_start = 0;
408 		fb64_start(fbp, DIR_ENCRYPT, server);
409 	}
410 }
411 
412 /*
413  * We only accept a keyid of 0.  If we get a keyid of
414  * 0, then mark the state as SUCCESS.
415  */
416 
417 int cfb64_keyid(int dir, unsigned char *kp, int *lenp)
418 {
419 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
420 }
421 
422 int ofb64_keyid(int dir, unsigned char *kp, int *lenp)
423 {
424 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
425 }
426 
427 int fb64_keyid(int dir, unsigned char *kp, int *lenp, struct fb *fbp)
428 {
429 	int state = fbp->state[dir-1];
430 
431 	if (*lenp != 1 || (*kp != '\0')) {
432 		*lenp = 0;
433 		return(state);
434 	}
435 
436 	if (state == FAILED)
437 		state = IN_PROGRESS;
438 
439 	state &= ~NO_KEYID;
440 
441 	return(fbp->state[dir-1] = state);
442 }
443 
444 void fb64_printsub(unsigned char *data, size_t cnt,
445 		   unsigned char *buf, size_t buflen, char *type)
446 {
447 	char lbuf[32];
448 	int i;
449 	char *cp;
450 
451 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
452 	buflen -= 1;
453 
454 	switch(data[2]) {
455 	case FB64_IV:
456 		snprintf(lbuf, sizeof(lbuf), "%s_IV", type);
457 		cp = lbuf;
458 		goto common;
459 
460 	case FB64_IV_OK:
461 		snprintf(lbuf, sizeof(lbuf), "%s_IV_OK", type);
462 		cp = lbuf;
463 		goto common;
464 
465 	case FB64_IV_BAD:
466 		snprintf(lbuf, sizeof(lbuf), "%s_IV_BAD", type);
467 		cp = lbuf;
468 		goto common;
469 
470 	default:
471 		snprintf(lbuf, sizeof(lbuf), " %d (unknown)", data[2]);
472 		cp = lbuf;
473 	common:
474 		for (; (buflen > 0) && (*buf = *cp++); buf++)
475 			buflen--;
476 		for (i = 3; i < cnt; i++) {
477 			snprintf(lbuf, sizeof(lbuf), " %d", data[i]);
478 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
479 				buflen--;
480 		}
481 		break;
482 	}
483 }
484 
485 void cfb64_printsub(unsigned char *data, size_t cnt,
486 		    unsigned char *buf, size_t buflen)
487 {
488 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
489 }
490 
491 void ofb64_printsub(unsigned char *data, size_t cnt,
492 		    unsigned char *buf, size_t buflen)
493 {
494 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
495 }
496 
497 void fb64_stream_iv(DES_cblock seed, struct stinfo *stp)
498 {
499 
500 	memcpy(stp->str_iv, seed,sizeof(DES_cblock));
501 	memcpy(stp->str_output, seed, sizeof(DES_cblock));
502 
503 	DES_set_key_checked(&stp->str_ikey, &stp->str_sched);
504 
505 	stp->str_index = sizeof(DES_cblock);
506 }
507 
508 void fb64_stream_key(DES_cblock key, struct stinfo *stp)
509 {
510 	memcpy(stp->str_ikey, key, sizeof(DES_cblock));
511 	DES_set_key_checked((DES_cblock*)key, &stp->str_sched);
512 
513 	memcpy(stp->str_output, stp->str_iv, sizeof(DES_cblock));
514 
515 	stp->str_index = sizeof(DES_cblock);
516 }
517 
518 /*
519  * DES 64 bit Cipher Feedback
520  *
521  *     key --->+-----+
522  *          +->| DES |--+
523  *          |  +-----+  |
524  *	    |           v
525  *  INPUT --(--------->(+)+---> DATA
526  *          |             |
527  *	    +-------------+
528  *
529  *
530  * Given:
531  *	iV: Initial vector, 64 bits (8 bytes) long.
532  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
533  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
534  *
535  *	V0 = DES(iV, key)
536  *	On = Dn ^ Vn
537  *	V(n+1) = DES(On, key)
538  */
539 
540 void cfb64_encrypt(unsigned char *s, int c)
541 {
542 	struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
543 	int index;
544 
545 	index = stp->str_index;
546 	while (c-- > 0) {
547 		if (index == sizeof(DES_cblock)) {
548 			DES_cblock b;
549 			DES_ecb_encrypt(&stp->str_output, &b,&stp->str_sched, 1);
550 			memcpy(stp->str_feed, b, sizeof(DES_cblock));
551 			index = 0;
552 		}
553 
554 		/* On encryption, we store (feed ^ data) which is cypher */
555 		*s = stp->str_output[index] = (stp->str_feed[index] ^ *s);
556 		s++;
557 		index++;
558 	}
559 	stp->str_index = index;
560 }
561 
562 int cfb64_decrypt(int data)
563 {
564 	struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
565 	int index;
566 
567 	if (data == -1) {
568 		/*
569 		 * Back up one byte.  It is assumed that we will
570 		 * never back up more than one byte.  If we do, this
571 		 * may or may not work.
572 		 */
573 		if (stp->str_index)
574 			--stp->str_index;
575 		return(0);
576 	}
577 
578 	index = stp->str_index++;
579 	if (index == sizeof(DES_cblock)) {
580 		DES_cblock b;
581 		DES_ecb_encrypt(&stp->str_output,&b, &stp->str_sched, 1);
582 		memcpy(stp->str_feed, b, sizeof(DES_cblock));
583 		stp->str_index = 1;	/* Next time will be 1 */
584 		index = 0;		/* But now use 0 */
585 	}
586 
587 	/* On decryption we store (data) which is cypher. */
588 	stp->str_output[index] = data;
589 	return(data ^ stp->str_feed[index]);
590 }
591 
592 /*
593  * DES 64 bit Output Feedback
594  *
595  * key --->+-----+
596  *	+->| DES |--+
597  *	|  +-----+  |
598  *	+-----------+
599  *	            v
600  *  INPUT -------->(+) ----> DATA
601  *
602  * Given:
603  *	iV: Initial vector, 64 bits (8 bytes) long.
604  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
605  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
606  *
607  *	V0 = DES(iV, key)
608  *	V(n+1) = DES(Vn, key)
609  *	On = Dn ^ Vn
610  */
611 
612 void ofb64_encrypt(unsigned char *s, int c)
613 {
614 	struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
615 	int index;
616 
617 	index = stp->str_index;
618 	while (c-- > 0) {
619 		if (index == sizeof(DES_cblock)) {
620 			DES_cblock b;
621 			DES_ecb_encrypt(&stp->str_feed,&b, &stp->str_sched, 1);
622 			memcpy(stp->str_feed, b, sizeof(DES_cblock));
623 			index = 0;
624 		}
625 		*s++ ^= stp->str_feed[index];
626 		index++;
627 	}
628 	stp->str_index = index;
629 }
630 
631 int ofb64_decrypt(int data)
632 {
633 	struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
634 	int index;
635 
636 	if (data == -1) {
637 		/*
638 		 * Back up one byte.  It is assumed that we will
639 		 * never back up more than one byte.  If we do, this
640 		 * may or may not work.
641 		 */
642 		if (stp->str_index)
643 			--stp->str_index;
644 		return(0);
645 	}
646 
647 	index = stp->str_index++;
648 	if (index == sizeof(DES_cblock)) {
649 		DES_cblock b;
650 		DES_ecb_encrypt(&stp->str_feed,&b,&stp->str_sched, 1);
651 		memcpy(stp->str_feed, b, sizeof(DES_cblock));
652 		stp->str_index = 1;	/* Next time will be 1 */
653 		index = 0;		/* But now use 0 */
654 	}
655 
656 	return(data ^ stp->str_feed[index]);
657 }
658 #endif
659 
660