xref: /freebsd/contrib/telnet/libtelnet/enc_des.c (revision af23369a6deaaeb612ab266eb88b8bb8d560c322)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. Neither the name of the University nor the names of its contributors
14  *    may be used to endorse or promote products derived from this software
15  *    without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #if 0
31 #ifndef lint
32 static const char sccsid[] = "@(#)enc_des.c	8.3 (Berkeley) 5/30/95";
33 #endif /* not lint */
34 #endif
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
37 
38 #ifdef	ENCRYPTION
39 # ifdef	AUTHENTICATION
40 #include <arpa/telnet.h>
41 #include <openssl/des.h>
42 #include <stdio.h>
43 #include <stdlib.h>
44 #include <string.h>
45 
46 #include "encrypt.h"
47 #include "key-proto.h"
48 #include "misc-proto.h"
49 
50 extern int encrypt_debug_mode;
51 
52 #define	CFB	0
53 #define	OFB	1
54 
55 #define	NO_SEND_IV	1
56 #define	NO_RECV_IV	2
57 #define	NO_KEYID	4
58 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
59 #define	SUCCESS		0
60 #define	FAILED		-1
61 
62 
63 struct fb {
64 	Block krbdes_key;
65 	Schedule krbdes_sched;
66 	Block temp_feed;
67 	unsigned char fb_feed[64];
68 	int need_start;
69 	int state[2];
70 	int keyid[2];
71 	struct stinfo {
72 		Block		str_output;
73 		Block		str_feed;
74 		Block		str_iv;
75 		Block		str_ikey;
76 		Schedule	str_sched;
77 		int		str_index;
78 		int		str_flagshift;
79 	} streams[2];
80 };
81 
82 static struct fb fb[2];
83 
84 struct keyidlist {
85 	const char *keyid;
86 	int	keyidlen;
87 	char	*key;
88 	int	keylen;
89 	int	flags;
90 } keyidlist [] = {
91 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
92 	{ 0, 0, 0, 0, 0 }
93 };
94 
95 #define	KEYFLAG_MASK	03
96 
97 #define	KEYFLAG_NOINIT	00
98 #define	KEYFLAG_INIT	01
99 #define	KEYFLAG_OK	02
100 #define	KEYFLAG_BAD	03
101 
102 #define	KEYFLAG_SHIFT	2
103 
104 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
105 
106 #define	FB64_IV		1
107 #define	FB64_IV_OK	2
108 #define	FB64_IV_BAD	3
109 
110 
111 void fb64_stream_iv(Block, struct stinfo *);
112 void fb64_init(struct fb *);
113 static int fb64_start(struct fb *, int, int);
114 int fb64_is(unsigned char *, int, struct fb *);
115 int fb64_reply(unsigned char *, int, struct fb *);
116 static void fb64_session(Session_Key *, int, struct fb *);
117 void fb64_stream_key(Block, struct stinfo *);
118 int fb64_keyid(int, unsigned char *, int *, struct fb *);
119 
120 void
121 cfb64_init(int server __unused)
122 {
123 	fb64_init(&fb[CFB]);
124 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
125 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
126 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
127 }
128 
129 void
130 ofb64_init(int server __unused)
131 {
132 	fb64_init(&fb[OFB]);
133 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
134 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
135 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
136 }
137 
138 void
139 fb64_init(struct fb *fbp)
140 {
141 	memset((void *)fbp, 0, sizeof(*fbp));
142 	fbp->state[0] = fbp->state[1] = FAILED;
143 	fbp->fb_feed[0] = IAC;
144 	fbp->fb_feed[1] = SB;
145 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
146 	fbp->fb_feed[3] = ENCRYPT_IS;
147 }
148 
149 /*
150  * Returns:
151  *	-1: some error.  Negotiation is done, encryption not ready.
152  *	 0: Successful, initial negotiation all done.
153  *	 1: successful, negotiation not done yet.
154  *	 2: Not yet.  Other things (like getting the key from
155  *	    Kerberos) have to happen before we can continue.
156  */
157 int
158 cfb64_start(int dir, int server)
159 {
160 	return(fb64_start(&fb[CFB], dir, server));
161 }
162 
163 int
164 ofb64_start(int dir, int server)
165 {
166 	return(fb64_start(&fb[OFB], dir, server));
167 }
168 
169 static int
170 fb64_start(struct fb *fbp, int dir, int server __unused)
171 {
172 	size_t x;
173 	unsigned char *p;
174 	int state;
175 
176 	switch (dir) {
177 	case DIR_DECRYPT:
178 		/*
179 		 * This is simply a request to have the other side
180 		 * start output (our input).  He will negotiate an
181 		 * IV so we need not look for it.
182 		 */
183 		state = fbp->state[dir-1];
184 		if (state == FAILED)
185 			state = IN_PROGRESS;
186 		break;
187 
188 	case DIR_ENCRYPT:
189 		state = fbp->state[dir-1];
190 		if (state == FAILED)
191 			state = IN_PROGRESS;
192 		else if ((state & NO_SEND_IV) == 0)
193 			break;
194 
195 		if (!VALIDKEY(fbp->krbdes_key)) {
196 			fbp->need_start = 1;
197 			break;
198 		}
199 		state &= ~NO_SEND_IV;
200 		state |= NO_RECV_IV;
201 		if (encrypt_debug_mode)
202 			printf("Creating new feed\r\n");
203 		/*
204 		 * Create a random feed and send it over.
205 		 */
206 		DES_random_key((Block *)fbp->temp_feed);
207 		DES_ecb_encrypt((Block *)fbp->temp_feed, (Block *)fbp->temp_feed,
208 				&fbp->krbdes_sched, 1);
209 		p = fbp->fb_feed + 3;
210 		*p++ = ENCRYPT_IS;
211 		p++;
212 		*p++ = FB64_IV;
213 		for (x = 0; x < sizeof(Block); ++x) {
214 			if ((*p++ = fbp->temp_feed[x]) == IAC)
215 				*p++ = IAC;
216 		}
217 		*p++ = IAC;
218 		*p++ = SE;
219 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
220 		net_write(fbp->fb_feed, p - fbp->fb_feed);
221 		break;
222 	default:
223 		return(FAILED);
224 	}
225 	return(fbp->state[dir-1] = state);
226 }
227 
228 /*
229  * Returns:
230  *	-1: some error.  Negotiation is done, encryption not ready.
231  *	 0: Successful, initial negotiation all done.
232  *	 1: successful, negotiation not done yet.
233  */
234 int
235 cfb64_is(unsigned char *data, int cnt)
236 {
237 	return(fb64_is(data, cnt, &fb[CFB]));
238 }
239 
240 int
241 ofb64_is(unsigned char *data, int cnt)
242 {
243 	return(fb64_is(data, cnt, &fb[OFB]));
244 }
245 
246 int
247 fb64_is(unsigned char *data, int cnt, struct fb *fbp)
248 {
249 	unsigned char *p;
250 	int state = fbp->state[DIR_DECRYPT-1];
251 
252 	if (cnt-- < 1)
253 		goto failure;
254 
255 	switch (*data++) {
256 	case FB64_IV:
257 		if (cnt != sizeof(Block)) {
258 			if (encrypt_debug_mode)
259 				printf("CFB64: initial vector failed on size\r\n");
260 			state = FAILED;
261 			goto failure;
262 		}
263 
264 		if (encrypt_debug_mode)
265 			printf("CFB64: initial vector received\r\n");
266 
267 		if (encrypt_debug_mode)
268 			printf("Initializing Decrypt stream\r\n");
269 
270 		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
271 
272 		p = fbp->fb_feed + 3;
273 		*p++ = ENCRYPT_REPLY;
274 		p++;
275 		*p++ = FB64_IV_OK;
276 		*p++ = IAC;
277 		*p++ = SE;
278 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
279 		net_write(fbp->fb_feed, p - fbp->fb_feed);
280 
281 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
282 		break;
283 
284 	default:
285 		if (encrypt_debug_mode) {
286 			printf("Unknown option type: %d\r\n", *(data-1));
287 			printd(data, cnt);
288 			printf("\r\n");
289 		}
290 		/* FALL THROUGH */
291 	failure:
292 		/*
293 		 * We failed.  Send an FB64_IV_BAD option
294 		 * to the other side so it will know that
295 		 * things failed.
296 		 */
297 		p = fbp->fb_feed + 3;
298 		*p++ = ENCRYPT_REPLY;
299 		p++;
300 		*p++ = FB64_IV_BAD;
301 		*p++ = IAC;
302 		*p++ = SE;
303 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
304 		net_write(fbp->fb_feed, p - fbp->fb_feed);
305 
306 		break;
307 	}
308 	return(fbp->state[DIR_DECRYPT-1] = state);
309 }
310 
311 /*
312  * Returns:
313  *	-1: some error.  Negotiation is done, encryption not ready.
314  *	 0: Successful, initial negotiation all done.
315  *	 1: successful, negotiation not done yet.
316  */
317 int
318 cfb64_reply(unsigned char *data, int cnt)
319 {
320 	return(fb64_reply(data, cnt, &fb[CFB]));
321 }
322 
323 int
324 ofb64_reply(unsigned char *data, int cnt)
325 {
326 	return(fb64_reply(data, cnt, &fb[OFB]));
327 }
328 
329 int
330 fb64_reply(unsigned char *data, int cnt, struct fb *fbp)
331 {
332 	int state = fbp->state[DIR_ENCRYPT-1];
333 
334 	if (cnt-- < 1)
335 		goto failure;
336 
337 	switch (*data++) {
338 	case FB64_IV_OK:
339 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
340 		if (state == FAILED)
341 			state = IN_PROGRESS;
342 		state &= ~NO_RECV_IV;
343 		encrypt_send_keyid(DIR_ENCRYPT, "\0", 1, 1);
344 		break;
345 
346 	case FB64_IV_BAD:
347 		memset(fbp->temp_feed, 0, sizeof(Block));
348 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
349 		state = FAILED;
350 		break;
351 
352 	default:
353 		if (encrypt_debug_mode) {
354 			printf("Unknown option type: %d\r\n", data[-1]);
355 			printd(data, cnt);
356 			printf("\r\n");
357 		}
358 		/* FALL THROUGH */
359 	failure:
360 		state = FAILED;
361 		break;
362 	}
363 	return(fbp->state[DIR_ENCRYPT-1] = state);
364 }
365 
366 void
367 cfb64_session(Session_Key *key, int server)
368 {
369 	fb64_session(key, server, &fb[CFB]);
370 }
371 
372 void
373 ofb64_session(Session_Key *key, int server)
374 {
375 	fb64_session(key, server, &fb[OFB]);
376 }
377 
378 static void
379 fb64_session(Session_Key *key, int server, struct fb *fbp)
380 {
381 	if (!key || key->type != SK_DES) {
382 		if (encrypt_debug_mode)
383 			printf("Can't set krbdes's session key (%d != %d)\r\n",
384 				key ? key->type : -1, SK_DES);
385 		return;
386 	}
387 	memmove((void *)fbp->krbdes_key, (void *)key->data, sizeof(Block));
388 
389 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
390 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
391 
392 	DES_key_sched((Block *)fbp->krbdes_key, &fbp->krbdes_sched);
393 	/*
394 	 * Now look to see if krbdes_start() was was waiting for
395 	 * the key to show up.  If so, go ahead an call it now
396 	 * that we have the key.
397 	 */
398 	if (fbp->need_start) {
399 		fbp->need_start = 0;
400 		fb64_start(fbp, DIR_ENCRYPT, server);
401 	}
402 }
403 
404 /*
405  * We only accept a keyid of 0.  If we get a keyid of
406  * 0, then mark the state as SUCCESS.
407  */
408 int
409 cfb64_keyid(int dir, unsigned char *kp, int *lenp)
410 {
411 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
412 }
413 
414 int
415 ofb64_keyid(int dir, unsigned char *kp, int *lenp)
416 {
417 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
418 }
419 
420 int
421 fb64_keyid(int dir, unsigned char *kp, int *lenp, struct fb *fbp)
422 {
423 	int state = fbp->state[dir-1];
424 
425 	if (*lenp != 1 || (*kp != '\0')) {
426 		*lenp = 0;
427 		return(state);
428 	}
429 
430 	if (state == FAILED)
431 		state = IN_PROGRESS;
432 
433 	state &= ~NO_KEYID;
434 
435 	return(fbp->state[dir-1] = state);
436 }
437 
438 void
439 fb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen, const char *type)
440 {
441 	char lbuf[32];
442 	int i;
443 	char *cp;
444 
445 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
446 	buflen -= 1;
447 
448 	switch(data[2]) {
449 	case FB64_IV:
450 		sprintf(lbuf, "%s_IV", type);
451 		cp = lbuf;
452 		goto common;
453 
454 	case FB64_IV_OK:
455 		sprintf(lbuf, "%s_IV_OK", type);
456 		cp = lbuf;
457 		goto common;
458 
459 	case FB64_IV_BAD:
460 		sprintf(lbuf, "%s_IV_BAD", type);
461 		cp = lbuf;
462 		goto common;
463 
464 	default:
465 		sprintf(lbuf, " %d (unknown)", data[2]);
466 		cp = lbuf;
467 	common:
468 		for (; (buflen > 0) && (*buf = *cp++); buf++)
469 			buflen--;
470 		for (i = 3; i < cnt; i++) {
471 			sprintf(lbuf, " %d", data[i]);
472 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
473 				buflen--;
474 		}
475 		break;
476 	}
477 }
478 
479 void
480 cfb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
481 {
482 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
483 }
484 
485 void
486 ofb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
487 {
488 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
489 }
490 
491 void
492 fb64_stream_iv(Block seed, struct stinfo *stp)
493 {
494 
495 	memmove((void *)stp->str_iv, (void *)seed, sizeof(Block));
496 	memmove((void *)stp->str_output, (void *)seed, sizeof(Block));
497 
498 	DES_key_sched((Block *)stp->str_ikey, &stp->str_sched);
499 
500 	stp->str_index = sizeof(Block);
501 }
502 
503 void
504 fb64_stream_key(Block key, struct stinfo *stp)
505 {
506 	memmove((void *)stp->str_ikey, (void *)key, sizeof(Block));
507 	DES_key_sched((Block *)key, &stp->str_sched);
508 
509 	memmove((void *)stp->str_output, (void *)stp->str_iv, sizeof(Block));
510 
511 	stp->str_index = sizeof(Block);
512 }
513 
514 /*
515  * DES 64 bit Cipher Feedback
516  *
517  *     key --->+-----+
518  *          +->| DES |--+
519  *          |  +-----+  |
520  *	    |           v
521  *  INPUT --(--------->(+)+---> DATA
522  *          |             |
523  *	    +-------------+
524  *
525  *
526  * Given:
527  *	iV: Initial vector, 64 bits (8 bytes) long.
528  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
529  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
530  *
531  *	V0 = DES(iV, key)
532  *	On = Dn ^ Vn
533  *	V(n+1) = DES(On, key)
534  */
535 
536 void
537 cfb64_encrypt(unsigned char *s, int c)
538 {
539 	struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
540 	int idx;
541 
542 	idx = stp->str_index;
543 	while (c-- > 0) {
544 		if (idx == sizeof(Block)) {
545 			Block b;
546 			DES_ecb_encrypt((Block *)stp->str_output, (Block *)b, &stp->str_sched, 1);
547 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
548 			idx = 0;
549 		}
550 
551 		/* On encryption, we store (feed ^ data) which is cypher */
552 		*s = stp->str_output[idx] = (stp->str_feed[idx] ^ *s);
553 		s++;
554 		idx++;
555 	}
556 	stp->str_index = idx;
557 }
558 
559 int
560 cfb64_decrypt(int data)
561 {
562 	struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
563 	int idx;
564 
565 	if (data == -1) {
566 		/*
567 		 * Back up one byte.  It is assumed that we will
568 		 * never back up more than one byte.  If we do, this
569 		 * may or may not work.
570 		 */
571 		if (stp->str_index)
572 			--stp->str_index;
573 		return(0);
574 	}
575 
576 	idx = stp->str_index++;
577 	if (idx == sizeof(Block)) {
578 		Block b;
579 		DES_ecb_encrypt((Block *)stp->str_output, (Block *)b, &stp->str_sched, 1);
580 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
581 		stp->str_index = 1;	/* Next time will be 1 */
582 		idx = 0;		/* But now use 0 */
583 	}
584 
585 	/* On decryption we store (data) which is cypher. */
586 	stp->str_output[idx] = data;
587 	return(data ^ stp->str_feed[idx]);
588 }
589 
590 /*
591  * DES 64 bit Output Feedback
592  *
593  * key --->+-----+
594  *	+->| DES |--+
595  *	|  +-----+  |
596  *	+-----------+
597  *	            v
598  *  INPUT -------->(+) ----> DATA
599  *
600  * Given:
601  *	iV: Initial vector, 64 bits (8 bytes) long.
602  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
603  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
604  *
605  *	V0 = DES(iV, key)
606  *	V(n+1) = DES(Vn, key)
607  *	On = Dn ^ Vn
608  */
609 void
610 ofb64_encrypt(unsigned char *s, int c)
611 {
612 	struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
613 	int idx;
614 
615 	idx = stp->str_index;
616 	while (c-- > 0) {
617 		if (idx == sizeof(Block)) {
618 			Block b;
619 			DES_ecb_encrypt((Block *)stp->str_feed, (Block *)b, &stp->str_sched, 1);
620 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
621 			idx = 0;
622 		}
623 		*s++ ^= stp->str_feed[idx];
624 		idx++;
625 	}
626 	stp->str_index = idx;
627 }
628 
629 int
630 ofb64_decrypt(int data)
631 {
632 	struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
633 	int idx;
634 
635 	if (data == -1) {
636 		/*
637 		 * Back up one byte.  It is assumed that we will
638 		 * never back up more than one byte.  If we do, this
639 		 * may or may not work.
640 		 */
641 		if (stp->str_index)
642 			--stp->str_index;
643 		return(0);
644 	}
645 
646 	idx = stp->str_index++;
647 	if (idx == sizeof(Block)) {
648 		Block b;
649 		DES_ecb_encrypt((Block *)stp->str_feed, (Block *)b, &stp->str_sched, 1);
650 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
651 		stp->str_index = 1;	/* Next time will be 1 */
652 		idx = 0;		/* But now use 0 */
653 	}
654 
655 	return(data ^ stp->str_feed[idx]);
656 }
657 # endif	/* AUTHENTICATION */
658 #endif	/* ENCRYPTION */
659