xref: /freebsd/contrib/telnet/libtelnet/enc_des.c (revision a03411e84728e9b267056fd31c7d1d9d1dc1b01e)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. Neither the name of the University nor the names of its contributors
14  *    may be used to endorse or promote products derived from this software
15  *    without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #if 0
31 #ifndef lint
32 static const char sccsid[] = "@(#)enc_des.c	8.3 (Berkeley) 5/30/95";
33 #endif /* not lint */
34 #endif
35 
36 #ifdef	ENCRYPTION
37 # ifdef	AUTHENTICATION
38 #include <arpa/telnet.h>
39 #include <openssl/des.h>
40 #include <stdio.h>
41 #include <stdlib.h>
42 #include <string.h>
43 
44 #include "encrypt.h"
45 #include "key-proto.h"
46 #include "misc-proto.h"
47 
48 extern int encrypt_debug_mode;
49 
50 #define	CFB	0
51 #define	OFB	1
52 
53 #define	NO_SEND_IV	1
54 #define	NO_RECV_IV	2
55 #define	NO_KEYID	4
56 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
57 #define	SUCCESS		0
58 #define	FAILED		-1
59 
60 
61 struct fb {
62 	Block krbdes_key;
63 	Schedule krbdes_sched;
64 	Block temp_feed;
65 	unsigned char fb_feed[64];
66 	int need_start;
67 	int state[2];
68 	int keyid[2];
69 	struct stinfo {
70 		Block		str_output;
71 		Block		str_feed;
72 		Block		str_iv;
73 		Block		str_ikey;
74 		Schedule	str_sched;
75 		int		str_index;
76 		int		str_flagshift;
77 	} streams[2];
78 };
79 
80 static struct fb fb[2];
81 
82 struct keyidlist {
83 	const char *keyid;
84 	int	keyidlen;
85 	char	*key;
86 	int	keylen;
87 	int	flags;
88 } keyidlist [] = {
89 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
90 	{ 0, 0, 0, 0, 0 }
91 };
92 
93 #define	KEYFLAG_MASK	03
94 
95 #define	KEYFLAG_NOINIT	00
96 #define	KEYFLAG_INIT	01
97 #define	KEYFLAG_OK	02
98 #define	KEYFLAG_BAD	03
99 
100 #define	KEYFLAG_SHIFT	2
101 
102 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
103 
104 #define	FB64_IV		1
105 #define	FB64_IV_OK	2
106 #define	FB64_IV_BAD	3
107 
108 
109 void fb64_stream_iv(Block, struct stinfo *);
110 void fb64_init(struct fb *);
111 static int fb64_start(struct fb *, int, int);
112 int fb64_is(unsigned char *, int, struct fb *);
113 int fb64_reply(unsigned char *, int, struct fb *);
114 static void fb64_session(Session_Key *, int, struct fb *);
115 void fb64_stream_key(Block, struct stinfo *);
116 int fb64_keyid(int, unsigned char *, int *, struct fb *);
117 
118 void
119 cfb64_init(int server __unused)
120 {
121 	fb64_init(&fb[CFB]);
122 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
123 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
124 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
125 }
126 
127 void
128 ofb64_init(int server __unused)
129 {
130 	fb64_init(&fb[OFB]);
131 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
132 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
133 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
134 }
135 
136 void
137 fb64_init(struct fb *fbp)
138 {
139 	memset((void *)fbp, 0, sizeof(*fbp));
140 	fbp->state[0] = fbp->state[1] = FAILED;
141 	fbp->fb_feed[0] = IAC;
142 	fbp->fb_feed[1] = SB;
143 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
144 	fbp->fb_feed[3] = ENCRYPT_IS;
145 }
146 
147 /*
148  * Returns:
149  *	-1: some error.  Negotiation is done, encryption not ready.
150  *	 0: Successful, initial negotiation all done.
151  *	 1: successful, negotiation not done yet.
152  *	 2: Not yet.  Other things (like getting the key from
153  *	    Kerberos) have to happen before we can continue.
154  */
155 int
156 cfb64_start(int dir, int server)
157 {
158 	return(fb64_start(&fb[CFB], dir, server));
159 }
160 
161 int
162 ofb64_start(int dir, int server)
163 {
164 	return(fb64_start(&fb[OFB], dir, server));
165 }
166 
167 static int
168 fb64_start(struct fb *fbp, int dir, int server __unused)
169 {
170 	size_t x;
171 	unsigned char *p;
172 	int state;
173 
174 	switch (dir) {
175 	case DIR_DECRYPT:
176 		/*
177 		 * This is simply a request to have the other side
178 		 * start output (our input).  He will negotiate an
179 		 * IV so we need not look for it.
180 		 */
181 		state = fbp->state[dir-1];
182 		if (state == FAILED)
183 			state = IN_PROGRESS;
184 		break;
185 
186 	case DIR_ENCRYPT:
187 		state = fbp->state[dir-1];
188 		if (state == FAILED)
189 			state = IN_PROGRESS;
190 		else if ((state & NO_SEND_IV) == 0)
191 			break;
192 
193 		if (!VALIDKEY(fbp->krbdes_key)) {
194 			fbp->need_start = 1;
195 			break;
196 		}
197 		state &= ~NO_SEND_IV;
198 		state |= NO_RECV_IV;
199 		if (encrypt_debug_mode)
200 			printf("Creating new feed\r\n");
201 		/*
202 		 * Create a random feed and send it over.
203 		 */
204 		DES_random_key((Block *)fbp->temp_feed);
205 		DES_ecb_encrypt((Block *)fbp->temp_feed, (Block *)fbp->temp_feed,
206 				&fbp->krbdes_sched, 1);
207 		p = fbp->fb_feed + 3;
208 		*p++ = ENCRYPT_IS;
209 		p++;
210 		*p++ = FB64_IV;
211 		for (x = 0; x < sizeof(Block); ++x) {
212 			if ((*p++ = fbp->temp_feed[x]) == IAC)
213 				*p++ = IAC;
214 		}
215 		*p++ = IAC;
216 		*p++ = SE;
217 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
218 		net_write(fbp->fb_feed, p - fbp->fb_feed);
219 		break;
220 	default:
221 		return(FAILED);
222 	}
223 	return(fbp->state[dir-1] = state);
224 }
225 
226 /*
227  * Returns:
228  *	-1: some error.  Negotiation is done, encryption not ready.
229  *	 0: Successful, initial negotiation all done.
230  *	 1: successful, negotiation not done yet.
231  */
232 int
233 cfb64_is(unsigned char *data, int cnt)
234 {
235 	return(fb64_is(data, cnt, &fb[CFB]));
236 }
237 
238 int
239 ofb64_is(unsigned char *data, int cnt)
240 {
241 	return(fb64_is(data, cnt, &fb[OFB]));
242 }
243 
244 int
245 fb64_is(unsigned char *data, int cnt, struct fb *fbp)
246 {
247 	unsigned char *p;
248 	int state = fbp->state[DIR_DECRYPT-1];
249 
250 	if (cnt-- < 1)
251 		goto failure;
252 
253 	switch (*data++) {
254 	case FB64_IV:
255 		if (cnt != sizeof(Block)) {
256 			if (encrypt_debug_mode)
257 				printf("CFB64: initial vector failed on size\r\n");
258 			state = FAILED;
259 			goto failure;
260 		}
261 
262 		if (encrypt_debug_mode)
263 			printf("CFB64: initial vector received\r\n");
264 
265 		if (encrypt_debug_mode)
266 			printf("Initializing Decrypt stream\r\n");
267 
268 		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
269 
270 		p = fbp->fb_feed + 3;
271 		*p++ = ENCRYPT_REPLY;
272 		p++;
273 		*p++ = FB64_IV_OK;
274 		*p++ = IAC;
275 		*p++ = SE;
276 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
277 		net_write(fbp->fb_feed, p - fbp->fb_feed);
278 
279 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
280 		break;
281 
282 	default:
283 		if (encrypt_debug_mode) {
284 			printf("Unknown option type: %d\r\n", *(data-1));
285 			printd(data, cnt);
286 			printf("\r\n");
287 		}
288 		/* FALL THROUGH */
289 	failure:
290 		/*
291 		 * We failed.  Send an FB64_IV_BAD option
292 		 * to the other side so it will know that
293 		 * things failed.
294 		 */
295 		p = fbp->fb_feed + 3;
296 		*p++ = ENCRYPT_REPLY;
297 		p++;
298 		*p++ = FB64_IV_BAD;
299 		*p++ = IAC;
300 		*p++ = SE;
301 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
302 		net_write(fbp->fb_feed, p - fbp->fb_feed);
303 
304 		break;
305 	}
306 	return(fbp->state[DIR_DECRYPT-1] = state);
307 }
308 
309 /*
310  * Returns:
311  *	-1: some error.  Negotiation is done, encryption not ready.
312  *	 0: Successful, initial negotiation all done.
313  *	 1: successful, negotiation not done yet.
314  */
315 int
316 cfb64_reply(unsigned char *data, int cnt)
317 {
318 	return(fb64_reply(data, cnt, &fb[CFB]));
319 }
320 
321 int
322 ofb64_reply(unsigned char *data, int cnt)
323 {
324 	return(fb64_reply(data, cnt, &fb[OFB]));
325 }
326 
327 int
328 fb64_reply(unsigned char *data, int cnt, struct fb *fbp)
329 {
330 	int state = fbp->state[DIR_ENCRYPT-1];
331 
332 	if (cnt-- < 1)
333 		goto failure;
334 
335 	switch (*data++) {
336 	case FB64_IV_OK:
337 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
338 		if (state == FAILED)
339 			state = IN_PROGRESS;
340 		state &= ~NO_RECV_IV;
341 		encrypt_send_keyid(DIR_ENCRYPT, "\0", 1, 1);
342 		break;
343 
344 	case FB64_IV_BAD:
345 		memset(fbp->temp_feed, 0, sizeof(Block));
346 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
347 		state = FAILED;
348 		break;
349 
350 	default:
351 		if (encrypt_debug_mode) {
352 			printf("Unknown option type: %d\r\n", data[-1]);
353 			printd(data, cnt);
354 			printf("\r\n");
355 		}
356 		/* FALL THROUGH */
357 	failure:
358 		state = FAILED;
359 		break;
360 	}
361 	return(fbp->state[DIR_ENCRYPT-1] = state);
362 }
363 
364 void
365 cfb64_session(Session_Key *key, int server)
366 {
367 	fb64_session(key, server, &fb[CFB]);
368 }
369 
370 void
371 ofb64_session(Session_Key *key, int server)
372 {
373 	fb64_session(key, server, &fb[OFB]);
374 }
375 
376 static void
377 fb64_session(Session_Key *key, int server, struct fb *fbp)
378 {
379 	if (!key || key->type != SK_DES) {
380 		if (encrypt_debug_mode)
381 			printf("Can't set krbdes's session key (%d != %d)\r\n",
382 				key ? key->type : -1, SK_DES);
383 		return;
384 	}
385 	memmove((void *)fbp->krbdes_key, (void *)key->data, sizeof(Block));
386 
387 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
388 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
389 
390 	DES_key_sched((Block *)fbp->krbdes_key, &fbp->krbdes_sched);
391 	/*
392 	 * Now look to see if krbdes_start() was was waiting for
393 	 * the key to show up.  If so, go ahead an call it now
394 	 * that we have the key.
395 	 */
396 	if (fbp->need_start) {
397 		fbp->need_start = 0;
398 		fb64_start(fbp, DIR_ENCRYPT, server);
399 	}
400 }
401 
402 /*
403  * We only accept a keyid of 0.  If we get a keyid of
404  * 0, then mark the state as SUCCESS.
405  */
406 int
407 cfb64_keyid(int dir, unsigned char *kp, int *lenp)
408 {
409 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
410 }
411 
412 int
413 ofb64_keyid(int dir, unsigned char *kp, int *lenp)
414 {
415 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
416 }
417 
418 int
419 fb64_keyid(int dir, unsigned char *kp, int *lenp, struct fb *fbp)
420 {
421 	int state = fbp->state[dir-1];
422 
423 	if (*lenp != 1 || (*kp != '\0')) {
424 		*lenp = 0;
425 		return(state);
426 	}
427 
428 	if (state == FAILED)
429 		state = IN_PROGRESS;
430 
431 	state &= ~NO_KEYID;
432 
433 	return(fbp->state[dir-1] = state);
434 }
435 
436 void
437 fb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen, const char *type)
438 {
439 	char lbuf[32];
440 	int i;
441 	char *cp;
442 
443 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
444 	buflen -= 1;
445 
446 	switch(data[2]) {
447 	case FB64_IV:
448 		sprintf(lbuf, "%s_IV", type);
449 		cp = lbuf;
450 		goto common;
451 
452 	case FB64_IV_OK:
453 		sprintf(lbuf, "%s_IV_OK", type);
454 		cp = lbuf;
455 		goto common;
456 
457 	case FB64_IV_BAD:
458 		sprintf(lbuf, "%s_IV_BAD", type);
459 		cp = lbuf;
460 		goto common;
461 
462 	default:
463 		sprintf(lbuf, " %d (unknown)", data[2]);
464 		cp = lbuf;
465 	common:
466 		for (; (buflen > 0) && (*buf = *cp++); buf++)
467 			buflen--;
468 		for (i = 3; i < cnt; i++) {
469 			sprintf(lbuf, " %d", data[i]);
470 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
471 				buflen--;
472 		}
473 		break;
474 	}
475 }
476 
477 void
478 cfb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
479 {
480 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
481 }
482 
483 void
484 ofb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
485 {
486 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
487 }
488 
489 void
490 fb64_stream_iv(Block seed, struct stinfo *stp)
491 {
492 
493 	memmove((void *)stp->str_iv, (void *)seed, sizeof(Block));
494 	memmove((void *)stp->str_output, (void *)seed, sizeof(Block));
495 
496 	DES_key_sched((Block *)stp->str_ikey, &stp->str_sched);
497 
498 	stp->str_index = sizeof(Block);
499 }
500 
501 void
502 fb64_stream_key(Block key, struct stinfo *stp)
503 {
504 	memmove((void *)stp->str_ikey, (void *)key, sizeof(Block));
505 	DES_key_sched((Block *)key, &stp->str_sched);
506 
507 	memmove((void *)stp->str_output, (void *)stp->str_iv, sizeof(Block));
508 
509 	stp->str_index = sizeof(Block);
510 }
511 
512 /*
513  * DES 64 bit Cipher Feedback
514  *
515  *     key --->+-----+
516  *          +->| DES |--+
517  *          |  +-----+  |
518  *	    |           v
519  *  INPUT --(--------->(+)+---> DATA
520  *          |             |
521  *	    +-------------+
522  *
523  *
524  * Given:
525  *	iV: Initial vector, 64 bits (8 bytes) long.
526  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
527  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
528  *
529  *	V0 = DES(iV, key)
530  *	On = Dn ^ Vn
531  *	V(n+1) = DES(On, key)
532  */
533 
534 void
535 cfb64_encrypt(unsigned char *s, int c)
536 {
537 	struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
538 	int idx;
539 
540 	idx = stp->str_index;
541 	while (c-- > 0) {
542 		if (idx == sizeof(Block)) {
543 			Block b;
544 			DES_ecb_encrypt((Block *)stp->str_output, (Block *)b, &stp->str_sched, 1);
545 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
546 			idx = 0;
547 		}
548 
549 		/* On encryption, we store (feed ^ data) which is cypher */
550 		*s = stp->str_output[idx] = (stp->str_feed[idx] ^ *s);
551 		s++;
552 		idx++;
553 	}
554 	stp->str_index = idx;
555 }
556 
557 int
558 cfb64_decrypt(int data)
559 {
560 	struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
561 	int idx;
562 
563 	if (data == -1) {
564 		/*
565 		 * Back up one byte.  It is assumed that we will
566 		 * never back up more than one byte.  If we do, this
567 		 * may or may not work.
568 		 */
569 		if (stp->str_index)
570 			--stp->str_index;
571 		return(0);
572 	}
573 
574 	idx = stp->str_index++;
575 	if (idx == sizeof(Block)) {
576 		Block b;
577 		DES_ecb_encrypt((Block *)stp->str_output, (Block *)b, &stp->str_sched, 1);
578 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
579 		stp->str_index = 1;	/* Next time will be 1 */
580 		idx = 0;		/* But now use 0 */
581 	}
582 
583 	/* On decryption we store (data) which is cypher. */
584 	stp->str_output[idx] = data;
585 	return(data ^ stp->str_feed[idx]);
586 }
587 
588 /*
589  * DES 64 bit Output Feedback
590  *
591  * key --->+-----+
592  *	+->| DES |--+
593  *	|  +-----+  |
594  *	+-----------+
595  *	            v
596  *  INPUT -------->(+) ----> DATA
597  *
598  * Given:
599  *	iV: Initial vector, 64 bits (8 bytes) long.
600  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
601  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
602  *
603  *	V0 = DES(iV, key)
604  *	V(n+1) = DES(Vn, key)
605  *	On = Dn ^ Vn
606  */
607 void
608 ofb64_encrypt(unsigned char *s, int c)
609 {
610 	struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
611 	int idx;
612 
613 	idx = stp->str_index;
614 	while (c-- > 0) {
615 		if (idx == sizeof(Block)) {
616 			Block b;
617 			DES_ecb_encrypt((Block *)stp->str_feed, (Block *)b, &stp->str_sched, 1);
618 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
619 			idx = 0;
620 		}
621 		*s++ ^= stp->str_feed[idx];
622 		idx++;
623 	}
624 	stp->str_index = idx;
625 }
626 
627 int
628 ofb64_decrypt(int data)
629 {
630 	struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
631 	int idx;
632 
633 	if (data == -1) {
634 		/*
635 		 * Back up one byte.  It is assumed that we will
636 		 * never back up more than one byte.  If we do, this
637 		 * may or may not work.
638 		 */
639 		if (stp->str_index)
640 			--stp->str_index;
641 		return(0);
642 	}
643 
644 	idx = stp->str_index++;
645 	if (idx == sizeof(Block)) {
646 		Block b;
647 		DES_ecb_encrypt((Block *)stp->str_feed, (Block *)b, &stp->str_sched, 1);
648 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
649 		stp->str_index = 1;	/* Next time will be 1 */
650 		idx = 0;		/* But now use 0 */
651 	}
652 
653 	return(data ^ stp->str_feed[idx]);
654 }
655 # endif	/* AUTHENTICATION */
656 #endif	/* ENCRYPTION */
657