xref: /freebsd/contrib/telnet/libtelnet/enc_des.c (revision 8fa113e5fc65fe6abc757f0089f477a87ee4d185)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *	This product includes software developed by the University of
16  *	California, Berkeley and its contributors.
17  * 4. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  *
33  * $FreeBSD$
34  */
35 
36 #include <sys/cdefs.h>
37 
38 __FBSDID("$FreeBSD$");
39 
40 #ifndef lint
41 static const char sccsid[] = "@(#)enc_des.c	8.3 (Berkeley) 5/30/95";
42 #endif /* not lint */
43 
44 #ifdef	ENCRYPTION
45 # ifdef	AUTHENTICATION
46 #include <arpa/telnet.h>
47 #include <openssl/des.h>
48 #include <stdio.h>
49 #include <stdlib.h>
50 #include <string.h>
51 
52 #include "encrypt.h"
53 #include "key-proto.h"
54 #include "misc-proto.h"
55 
56 extern int encrypt_debug_mode;
57 void des_set_random_generator_seed(des_cblock *); /* XXX */
58 
59 #define	CFB	0
60 #define	OFB	1
61 
62 #define	NO_SEND_IV	1
63 #define	NO_RECV_IV	2
64 #define	NO_KEYID	4
65 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
66 #define	SUCCESS		0
67 #define	FAILED		-1
68 
69 
70 struct fb {
71 	Block krbdes_key;
72 	Schedule krbdes_sched;
73 	Block temp_feed;
74 	unsigned char fb_feed[64];
75 	int need_start;
76 	int state[2];
77 	int keyid[2];
78 	int once;
79 	struct stinfo {
80 		Block		str_output;
81 		Block		str_feed;
82 		Block		str_iv;
83 		Block		str_ikey;
84 		Schedule	str_sched;
85 		int		str_index;
86 		int		str_flagshift;
87 	} streams[2];
88 };
89 
90 static struct fb fb[2];
91 
92 struct keyidlist {
93 	const char *keyid;
94 	int	keyidlen;
95 	char	*key;
96 	int	keylen;
97 	int	flags;
98 } keyidlist [] = {
99 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
100 	{ 0, 0, 0, 0, 0 }
101 };
102 
103 #define	KEYFLAG_MASK	03
104 
105 #define	KEYFLAG_NOINIT	00
106 #define	KEYFLAG_INIT	01
107 #define	KEYFLAG_OK	02
108 #define	KEYFLAG_BAD	03
109 
110 #define	KEYFLAG_SHIFT	2
111 
112 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
113 
114 #define	FB64_IV		1
115 #define	FB64_IV_OK	2
116 #define	FB64_IV_BAD	3
117 
118 
119 void fb64_stream_iv P((Block, struct stinfo *));
120 void fb64_init P((struct fb *));
121 static int fb64_start P((struct fb *, int, int));
122 int fb64_is P((unsigned char *, int, struct fb *));
123 int fb64_reply P((unsigned char *, int, struct fb *));
124 static void fb64_session P((Session_Key *, int, struct fb *));
125 void fb64_stream_key P((Block, struct stinfo *));
126 int fb64_keyid P((int, unsigned char *, int *, struct fb *));
127 
128 void
129 cfb64_init(int server __unused)
130 {
131 	fb64_init(&fb[CFB]);
132 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
133 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
134 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
135 }
136 
137 void
138 ofb64_init(int server __unused)
139 {
140 	fb64_init(&fb[OFB]);
141 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
142 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
143 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
144 }
145 
146 void
147 fb64_init(struct fb *fbp)
148 {
149 	memset((void *)fbp, 0, sizeof(*fbp));
150 	fbp->state[0] = fbp->state[1] = FAILED;
151 	fbp->fb_feed[0] = IAC;
152 	fbp->fb_feed[1] = SB;
153 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
154 	fbp->fb_feed[3] = ENCRYPT_IS;
155 }
156 
157 /*
158  * Returns:
159  *	-1: some error.  Negotiation is done, encryption not ready.
160  *	 0: Successful, initial negotiation all done.
161  *	 1: successful, negotiation not done yet.
162  *	 2: Not yet.  Other things (like getting the key from
163  *	    Kerberos) have to happen before we can continue.
164  */
165 int
166 cfb64_start(int dir, int server)
167 {
168 	return(fb64_start(&fb[CFB], dir, server));
169 }
170 
171 int
172 ofb64_start(int dir, int server)
173 {
174 	return(fb64_start(&fb[OFB], dir, server));
175 }
176 
177 static int
178 fb64_start(struct fb *fbp, int dir, int server __unused)
179 {
180 	size_t x;
181 	unsigned char *p;
182 	int state;
183 
184 	switch (dir) {
185 	case DIR_DECRYPT:
186 		/*
187 		 * This is simply a request to have the other side
188 		 * start output (our input).  He will negotiate an
189 		 * IV so we need not look for it.
190 		 */
191 		state = fbp->state[dir-1];
192 		if (state == FAILED)
193 			state = IN_PROGRESS;
194 		break;
195 
196 	case DIR_ENCRYPT:
197 		state = fbp->state[dir-1];
198 		if (state == FAILED)
199 			state = IN_PROGRESS;
200 		else if ((state & NO_SEND_IV) == 0)
201 			break;
202 
203 		if (!VALIDKEY(fbp->krbdes_key)) {
204 			fbp->need_start = 1;
205 			break;
206 		}
207 		state &= ~NO_SEND_IV;
208 		state |= NO_RECV_IV;
209 		if (encrypt_debug_mode)
210 			printf("Creating new feed\r\n");
211 		/*
212 		 * Create a random feed and send it over.
213 		 */
214 		des_new_random_key((Block *)fbp->temp_feed);
215 		des_ecb_encrypt((Block *)fbp->temp_feed, (Block *)fbp->temp_feed,
216 				fbp->krbdes_sched, 1);
217 		p = fbp->fb_feed + 3;
218 		*p++ = ENCRYPT_IS;
219 		p++;
220 		*p++ = FB64_IV;
221 		for (x = 0; x < sizeof(Block); ++x) {
222 			if ((*p++ = fbp->temp_feed[x]) == IAC)
223 				*p++ = IAC;
224 		}
225 		*p++ = IAC;
226 		*p++ = SE;
227 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
228 		net_write(fbp->fb_feed, p - fbp->fb_feed);
229 		break;
230 	default:
231 		return(FAILED);
232 	}
233 	return(fbp->state[dir-1] = state);
234 }
235 
236 /*
237  * Returns:
238  *	-1: some error.  Negotiation is done, encryption not ready.
239  *	 0: Successful, initial negotiation all done.
240  *	 1: successful, negotiation not done yet.
241  */
242 int
243 cfb64_is(unsigned char *data, int cnt)
244 {
245 	return(fb64_is(data, cnt, &fb[CFB]));
246 }
247 
248 int
249 ofb64_is(unsigned char *data, int cnt)
250 {
251 	return(fb64_is(data, cnt, &fb[OFB]));
252 }
253 
254 int
255 fb64_is(unsigned char *data, int cnt, struct fb *fbp)
256 {
257 	unsigned char *p;
258 	int state = fbp->state[DIR_DECRYPT-1];
259 
260 	if (cnt-- < 1)
261 		goto failure;
262 
263 	switch (*data++) {
264 	case FB64_IV:
265 		if (cnt != sizeof(Block)) {
266 			if (encrypt_debug_mode)
267 				printf("CFB64: initial vector failed on size\r\n");
268 			state = FAILED;
269 			goto failure;
270 		}
271 
272 		if (encrypt_debug_mode)
273 			printf("CFB64: initial vector received\r\n");
274 
275 		if (encrypt_debug_mode)
276 			printf("Initializing Decrypt stream\r\n");
277 
278 		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
279 
280 		p = fbp->fb_feed + 3;
281 		*p++ = ENCRYPT_REPLY;
282 		p++;
283 		*p++ = FB64_IV_OK;
284 		*p++ = IAC;
285 		*p++ = SE;
286 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
287 		net_write(fbp->fb_feed, p - fbp->fb_feed);
288 
289 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
290 		break;
291 
292 	default:
293 		if (encrypt_debug_mode) {
294 			printf("Unknown option type: %d\r\n", *(data-1));
295 			printd(data, cnt);
296 			printf("\r\n");
297 		}
298 		/* FALL THROUGH */
299 	failure:
300 		/*
301 		 * We failed.  Send an FB64_IV_BAD option
302 		 * to the other side so it will know that
303 		 * things failed.
304 		 */
305 		p = fbp->fb_feed + 3;
306 		*p++ = ENCRYPT_REPLY;
307 		p++;
308 		*p++ = FB64_IV_BAD;
309 		*p++ = IAC;
310 		*p++ = SE;
311 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
312 		net_write(fbp->fb_feed, p - fbp->fb_feed);
313 
314 		break;
315 	}
316 	return(fbp->state[DIR_DECRYPT-1] = state);
317 }
318 
319 /*
320  * Returns:
321  *	-1: some error.  Negotiation is done, encryption not ready.
322  *	 0: Successful, initial negotiation all done.
323  *	 1: successful, negotiation not done yet.
324  */
325 int
326 cfb64_reply(unsigned char *data, int cnt)
327 {
328 	return(fb64_reply(data, cnt, &fb[CFB]));
329 }
330 
331 int
332 ofb64_reply(unsigned char *data, int cnt)
333 {
334 	return(fb64_reply(data, cnt, &fb[OFB]));
335 }
336 
337 int
338 fb64_reply(unsigned char *data, int cnt, struct fb *fbp)
339 {
340 	int state = fbp->state[DIR_ENCRYPT-1];
341 
342 	if (cnt-- < 1)
343 		goto failure;
344 
345 	switch (*data++) {
346 	case FB64_IV_OK:
347 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
348 		if (state == FAILED)
349 			state = IN_PROGRESS;
350 		state &= ~NO_RECV_IV;
351 		encrypt_send_keyid(DIR_ENCRYPT, "\0", 1, 1);
352 		break;
353 
354 	case FB64_IV_BAD:
355 		memset(fbp->temp_feed, 0, sizeof(Block));
356 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
357 		state = FAILED;
358 		break;
359 
360 	default:
361 		if (encrypt_debug_mode) {
362 			printf("Unknown option type: %d\r\n", data[-1]);
363 			printd(data, cnt);
364 			printf("\r\n");
365 		}
366 		/* FALL THROUGH */
367 	failure:
368 		state = FAILED;
369 		break;
370 	}
371 	return(fbp->state[DIR_ENCRYPT-1] = state);
372 }
373 
374 void
375 cfb64_session(Session_Key *key, int server)
376 {
377 	fb64_session(key, server, &fb[CFB]);
378 }
379 
380 void
381 ofb64_session(Session_Key *key, int server)
382 {
383 	fb64_session(key, server, &fb[OFB]);
384 }
385 
386 static void
387 fb64_session(Session_Key *key, int server, struct fb *fbp)
388 {
389 	if (!key || key->type != SK_DES) {
390 		if (encrypt_debug_mode)
391 			printf("Can't set krbdes's session key (%d != %d)\r\n",
392 				key ? key->type : -1, SK_DES);
393 		return;
394 	}
395 	memmove((void *)fbp->krbdes_key, (void *)key->data, sizeof(Block));
396 
397 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
398 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
399 
400 	if (fbp->once == 0) {
401 		des_set_random_generator_seed((Block *)fbp->krbdes_key);
402 		fbp->once = 1;
403 	}
404 	des_key_sched((Block *)fbp->krbdes_key, fbp->krbdes_sched);
405 	/*
406 	 * Now look to see if krbdes_start() was was waiting for
407 	 * the key to show up.  If so, go ahead an call it now
408 	 * that we have the key.
409 	 */
410 	if (fbp->need_start) {
411 		fbp->need_start = 0;
412 		fb64_start(fbp, DIR_ENCRYPT, server);
413 	}
414 }
415 
416 /*
417  * We only accept a keyid of 0.  If we get a keyid of
418  * 0, then mark the state as SUCCESS.
419  */
420 int
421 cfb64_keyid(int dir, unsigned char *kp, int *lenp)
422 {
423 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
424 }
425 
426 int
427 ofb64_keyid(int dir, unsigned char *kp, int *lenp)
428 {
429 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
430 }
431 
432 int
433 fb64_keyid(int dir, unsigned char *kp, int *lenp, struct fb *fbp)
434 {
435 	int state = fbp->state[dir-1];
436 
437 	if (*lenp != 1 || (*kp != '\0')) {
438 		*lenp = 0;
439 		return(state);
440 	}
441 
442 	if (state == FAILED)
443 		state = IN_PROGRESS;
444 
445 	state &= ~NO_KEYID;
446 
447 	return(fbp->state[dir-1] = state);
448 }
449 
450 void
451 fb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen, const char *type)
452 {
453 	char lbuf[32];
454 	int i;
455 	char *cp;
456 
457 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
458 	buflen -= 1;
459 
460 	switch(data[2]) {
461 	case FB64_IV:
462 		sprintf(lbuf, "%s_IV", type);
463 		cp = lbuf;
464 		goto common;
465 
466 	case FB64_IV_OK:
467 		sprintf(lbuf, "%s_IV_OK", type);
468 		cp = lbuf;
469 		goto common;
470 
471 	case FB64_IV_BAD:
472 		sprintf(lbuf, "%s_IV_BAD", type);
473 		cp = lbuf;
474 		goto common;
475 
476 	default:
477 		sprintf(lbuf, " %d (unknown)", data[2]);
478 		cp = lbuf;
479 	common:
480 		for (; (buflen > 0) && (*buf = *cp++); buf++)
481 			buflen--;
482 		for (i = 3; i < cnt; i++) {
483 			sprintf(lbuf, " %d", data[i]);
484 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
485 				buflen--;
486 		}
487 		break;
488 	}
489 }
490 
491 void
492 cfb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
493 {
494 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
495 }
496 
497 void
498 ofb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
499 {
500 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
501 }
502 
503 void
504 fb64_stream_iv(Block seed, struct stinfo *stp)
505 {
506 
507 	memmove((void *)stp->str_iv, (void *)seed, sizeof(Block));
508 	memmove((void *)stp->str_output, (void *)seed, sizeof(Block));
509 
510 	des_key_sched((Block *)stp->str_ikey, stp->str_sched);
511 
512 	stp->str_index = sizeof(Block);
513 }
514 
515 void
516 fb64_stream_key(Block key, struct stinfo *stp)
517 {
518 	memmove((void *)stp->str_ikey, (void *)key, sizeof(Block));
519 	des_key_sched((Block *)key, stp->str_sched);
520 
521 	memmove((void *)stp->str_output, (void *)stp->str_iv, sizeof(Block));
522 
523 	stp->str_index = sizeof(Block);
524 }
525 
526 /*
527  * DES 64 bit Cipher Feedback
528  *
529  *     key --->+-----+
530  *          +->| DES |--+
531  *          |  +-----+  |
532  *	    |           v
533  *  INPUT --(--------->(+)+---> DATA
534  *          |             |
535  *	    +-------------+
536  *
537  *
538  * Given:
539  *	iV: Initial vector, 64 bits (8 bytes) long.
540  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
541  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
542  *
543  *	V0 = DES(iV, key)
544  *	On = Dn ^ Vn
545  *	V(n+1) = DES(On, key)
546  */
547 
548 void
549 cfb64_encrypt(unsigned char *s, int c)
550 {
551 	struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
552 	int idx;
553 
554 	idx = stp->str_index;
555 	while (c-- > 0) {
556 		if (idx == sizeof(Block)) {
557 			Block b;
558 			des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
559 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
560 			idx = 0;
561 		}
562 
563 		/* On encryption, we store (feed ^ data) which is cypher */
564 		*s = stp->str_output[idx] = (stp->str_feed[idx] ^ *s);
565 		s++;
566 		idx++;
567 	}
568 	stp->str_index = idx;
569 }
570 
571 int
572 cfb64_decrypt(int data)
573 {
574 	struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
575 	int idx;
576 
577 	if (data == -1) {
578 		/*
579 		 * Back up one byte.  It is assumed that we will
580 		 * never back up more than one byte.  If we do, this
581 		 * may or may not work.
582 		 */
583 		if (stp->str_index)
584 			--stp->str_index;
585 		return(0);
586 	}
587 
588 	idx = stp->str_index++;
589 	if (idx == sizeof(Block)) {
590 		Block b;
591 		des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
592 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
593 		stp->str_index = 1;	/* Next time will be 1 */
594 		idx = 0;		/* But now use 0 */
595 	}
596 
597 	/* On decryption we store (data) which is cypher. */
598 	stp->str_output[idx] = data;
599 	return(data ^ stp->str_feed[idx]);
600 }
601 
602 /*
603  * DES 64 bit Output Feedback
604  *
605  * key --->+-----+
606  *	+->| DES |--+
607  *	|  +-----+  |
608  *	+-----------+
609  *	            v
610  *  INPUT -------->(+) ----> DATA
611  *
612  * Given:
613  *	iV: Initial vector, 64 bits (8 bytes) long.
614  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
615  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
616  *
617  *	V0 = DES(iV, key)
618  *	V(n+1) = DES(Vn, key)
619  *	On = Dn ^ Vn
620  */
621 void
622 ofb64_encrypt(unsigned char *s, int c)
623 {
624 	struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
625 	int idx;
626 
627 	idx = stp->str_index;
628 	while (c-- > 0) {
629 		if (idx == sizeof(Block)) {
630 			Block b;
631 			des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
632 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
633 			idx = 0;
634 		}
635 		*s++ ^= stp->str_feed[idx];
636 		idx++;
637 	}
638 	stp->str_index = idx;
639 }
640 
641 int
642 ofb64_decrypt(int data)
643 {
644 	struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
645 	int idx;
646 
647 	if (data == -1) {
648 		/*
649 		 * Back up one byte.  It is assumed that we will
650 		 * never back up more than one byte.  If we do, this
651 		 * may or may not work.
652 		 */
653 		if (stp->str_index)
654 			--stp->str_index;
655 		return(0);
656 	}
657 
658 	idx = stp->str_index++;
659 	if (idx == sizeof(Block)) {
660 		Block b;
661 		des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
662 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
663 		stp->str_index = 1;	/* Next time will be 1 */
664 		idx = 0;		/* But now use 0 */
665 	}
666 
667 	return(data ^ stp->str_feed[idx]);
668 }
669 # endif	/* AUTHENTICATION */
670 #endif	/* ENCRYPTION */
671