xref: /freebsd/usr.sbin/bhyve/rfb.c (revision cbc6f7e941e42639a0314cd121b06493cce8e0e6)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2015 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5  * Copyright (c) 2015 Leon Dang
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/param.h>
31 #ifndef WITHOUT_CAPSICUM
32 #include <sys/capsicum.h>
33 #endif
34 #include <sys/endian.h>
35 #include <sys/socket.h>
36 #include <sys/select.h>
37 #include <sys/time.h>
38 #include <sys/un.h>
39 #include <arpa/inet.h>
40 #include <stdatomic.h>
41 #include <machine/cpufunc.h>
42 #include <machine/specialreg.h>
43 #include <netinet/in.h>
44 #include <netdb.h>
45 
46 #include <assert.h>
47 #ifndef WITHOUT_CAPSICUM
48 #include <capsicum_helpers.h>
49 #endif
50 #include <err.h>
51 #include <errno.h>
52 #include <pthread.h>
53 #include <pthread_np.h>
54 #include <signal.h>
55 #include <stdbool.h>
56 #include <stdlib.h>
57 #include <stdio.h>
58 #include <string.h>
59 #include <sysexits.h>
60 #include <unistd.h>
61 
62 #include <zlib.h>
63 
64 #include "bhyvegc.h"
65 #include "debug.h"
66 #include "console.h"
67 #include "config.h"
68 #include "rfb.h"
69 #include "sockstream.h"
70 
71 #ifndef NO_OPENSSL
72 #include <openssl/des.h>
73 #endif
74 
75 /* Delays in microseconds */
76 #define	CFD_SEL_DELAY	10000
77 #define	SCREEN_REFRESH_DELAY	33300	/* 30Hz */
78 #define	SCREEN_POLL_DELAY	(SCREEN_REFRESH_DELAY / 2)
79 
80 static int rfb_debug = 0;
81 #define	DPRINTF(params) if (rfb_debug) PRINTLN params
82 #define	WPRINTF(params) PRINTLN params
83 
84 #define VERSION_LENGTH	12
85 #define AUTH_LENGTH	16
86 #define PASSWD_LENGTH	8
87 
88 /* Protocol versions */
89 #define CVERS_3_3	'3'
90 #define CVERS_3_7	'7'
91 #define CVERS_3_8	'8'
92 
93 /* Client-to-server msg types */
94 #define CS_SET_PIXEL_FORMAT	0
95 #define CS_SET_ENCODINGS	2
96 #define CS_UPDATE_MSG		3
97 #define CS_KEY_EVENT		4
98 #define CS_POINTER_EVENT	5
99 #define CS_CUT_TEXT		6
100 #define CS_MSG_CLIENT_QEMU	255
101 
102 #define SECURITY_TYPE_NONE	1
103 #define SECURITY_TYPE_VNC_AUTH	2
104 
105 #define AUTH_FAILED_UNAUTH	1
106 #define AUTH_FAILED_ERROR	2
107 
108 struct pixfmt {
109 	bool		adjust_pixels;
110 	uint8_t		red_shift;
111 	uint8_t		green_shift;
112 	uint8_t		blue_shift;
113 };
114 
115 struct rfb_softc {
116 	int		sfd;
117 	pthread_t	tid;
118 
119 	int		cfd;
120 
121 	int		width, height;
122 
123 	const char	*password;
124 
125 	bool		enc_raw_ok;
126 	bool		enc_zlib_ok;
127 	bool		enc_resize_ok;
128 	bool		enc_extkeyevent_ok;
129 
130 	bool		enc_extkeyevent_send;
131 
132 	z_stream	zstream;
133 	uint8_t		*zbuf;
134 	int		zbuflen;
135 
136 	int		conn_wait;
137 	int		wrcount;
138 
139 	atomic_bool	sending;
140 	atomic_bool	pending;
141 	atomic_bool	update_all;
142 	atomic_bool	input_detected;
143 	atomic_bool	update_pixfmt;
144 
145 	pthread_mutex_t mtx;
146 	pthread_mutex_t pixfmt_mtx;
147 	pthread_cond_t  cond;
148 
149 	int		hw_crc;
150 	uint32_t	*crc;		/* WxH crc cells */
151 	uint32_t	*crc_tmp;	/* buffer to store single crc row */
152 	int		crc_width, crc_height;
153 
154 	struct pixfmt	pixfmt;		/* owned by the write thread */
155 	struct pixfmt	new_pixfmt;	/* managed with pixfmt_mtx */
156 	uint32_t	*pixrow;
157 	char		*fbname;
158 	int		fbnamelen;
159 };
160 
161 struct rfb_pixfmt {
162 	uint8_t		bpp;
163 	uint8_t		depth;
164 	uint8_t		bigendian;
165 	uint8_t		truecolor;
166 	uint16_t	red_max;
167 	uint16_t	green_max;
168 	uint16_t	blue_max;
169 	uint8_t		red_shift;
170 	uint8_t		green_shift;
171 	uint8_t		blue_shift;
172 	uint8_t		pad[3];
173 };
174 
175 struct rfb_srvr_info {
176 	uint16_t		width;
177 	uint16_t		height;
178 	struct rfb_pixfmt	pixfmt;
179 	uint32_t		namelen;
180 };
181 
182 struct rfb_pixfmt_msg {
183 	uint8_t			type;
184 	uint8_t			pad[3];
185 	struct rfb_pixfmt	pixfmt;
186 };
187 
188 #define	RFB_ENCODING_RAW		0
189 #define	RFB_ENCODING_ZLIB		6
190 #define	RFB_ENCODING_RESIZE		-223
191 #define	RFB_ENCODING_EXT_KEYEVENT	-258
192 
193 #define	RFB_CLIENTMSG_EXT_KEYEVENT	0
194 
195 #define	RFB_MAX_WIDTH			2000
196 #define	RFB_MAX_HEIGHT			1200
197 #define	RFB_ZLIB_BUFSZ			RFB_MAX_WIDTH*RFB_MAX_HEIGHT*4
198 
199 #define PIXEL_RED_SHIFT		16
200 #define PIXEL_GREEN_SHIFT	8
201 #define PIXEL_BLUE_SHIFT	0
202 
203 /* percentage changes to screen before sending the entire screen */
204 #define	RFB_SEND_ALL_THRESH		25
205 
206 struct rfb_enc_msg {
207 	uint8_t		type;
208 	uint8_t		pad;
209 	uint16_t	numencs;
210 };
211 
212 struct rfb_updt_msg {
213 	uint8_t		type;
214 	uint8_t		incremental;
215 	uint16_t	x;
216 	uint16_t	y;
217 	uint16_t	width;
218 	uint16_t	height;
219 };
220 
221 struct rfb_key_msg {
222 	uint8_t		type;
223 	uint8_t		down;
224 	uint16_t	pad;
225 	uint32_t	sym;
226 };
227 
228 struct rfb_client_msg {
229 	uint8_t		type;
230 	uint8_t		subtype;
231 };
232 
233 struct rfb_extended_key_msg {
234 	uint8_t		type;
235 	uint8_t		subtype;
236 	uint16_t	down;
237 	uint32_t	sym;
238 	uint32_t	code;
239 };
240 
241 struct rfb_ptr_msg {
242 	uint8_t		type;
243 	uint8_t		button;
244 	uint16_t	x;
245 	uint16_t	y;
246 };
247 
248 struct rfb_srvr_updt_msg {
249 	uint8_t		type;
250 	uint8_t		pad;
251 	uint16_t	numrects;
252 };
253 
254 struct rfb_srvr_rect_hdr {
255 	uint16_t	x;
256 	uint16_t	y;
257 	uint16_t	width;
258 	uint16_t	height;
259 	uint32_t	encoding;
260 };
261 
262 struct rfb_cuttext_msg {
263 	uint8_t		type;
264 	uint8_t		padding[3];
265 	uint32_t	length;
266 };
267 
268 static void
rfb_send_server_init_msg(struct rfb_softc * rc,int cfd)269 rfb_send_server_init_msg(struct rfb_softc *rc, int cfd)
270 {
271 	struct bhyvegc_image *gc_image;
272 	struct rfb_srvr_info sinfo;
273 
274 	gc_image = console_get_image();
275 
276 	sinfo.width = htons(gc_image->width);
277 	sinfo.height = htons(gc_image->height);
278 	sinfo.pixfmt.bpp = 32;
279 	sinfo.pixfmt.depth = 32;
280 	sinfo.pixfmt.bigendian = 0;
281 	sinfo.pixfmt.truecolor = 1;
282 	sinfo.pixfmt.red_max = htons(255);
283 	sinfo.pixfmt.green_max = htons(255);
284 	sinfo.pixfmt.blue_max = htons(255);
285 	sinfo.pixfmt.red_shift = PIXEL_RED_SHIFT;
286 	sinfo.pixfmt.green_shift = PIXEL_GREEN_SHIFT;
287 	sinfo.pixfmt.blue_shift = PIXEL_BLUE_SHIFT;
288 	sinfo.pixfmt.pad[0] = 0;
289 	sinfo.pixfmt.pad[1] = 0;
290 	sinfo.pixfmt.pad[2] = 0;
291 	sinfo.namelen = htonl(rc->fbnamelen);
292 	(void)stream_write(cfd, &sinfo, sizeof(sinfo));
293 	(void)stream_write(cfd, rc->fbname, rc->fbnamelen);
294 }
295 
296 static void
rfb_send_resize_update_msg(struct rfb_softc * rc,int cfd)297 rfb_send_resize_update_msg(struct rfb_softc *rc, int cfd)
298 {
299 	struct rfb_srvr_updt_msg supdt_msg;
300 	struct rfb_srvr_rect_hdr srect_hdr;
301 
302 	/* Number of rectangles: 1 */
303 	supdt_msg.type = 0;
304 	supdt_msg.pad = 0;
305 	supdt_msg.numrects = htons(1);
306 	stream_write(cfd, &supdt_msg, sizeof(struct rfb_srvr_updt_msg));
307 
308 	/* Rectangle header */
309 	srect_hdr.x = htons(0);
310 	srect_hdr.y = htons(0);
311 	srect_hdr.width = htons(rc->width);
312 	srect_hdr.height = htons(rc->height);
313 	srect_hdr.encoding = htonl(RFB_ENCODING_RESIZE);
314 	stream_write(cfd, &srect_hdr, sizeof(struct rfb_srvr_rect_hdr));
315 }
316 
317 static void
rfb_send_extended_keyevent_update_msg(struct rfb_softc * rc,int cfd)318 rfb_send_extended_keyevent_update_msg(struct rfb_softc *rc, int cfd)
319 {
320 	struct rfb_srvr_updt_msg supdt_msg;
321 	struct rfb_srvr_rect_hdr srect_hdr;
322 
323 	/* Number of rectangles: 1 */
324 	supdt_msg.type = 0;
325 	supdt_msg.pad = 0;
326 	supdt_msg.numrects = htons(1);
327 	stream_write(cfd, &supdt_msg, sizeof(struct rfb_srvr_updt_msg));
328 
329 	/* Rectangle header */
330 	srect_hdr.x = htons(0);
331 	srect_hdr.y = htons(0);
332 	srect_hdr.width = htons(rc->width);
333 	srect_hdr.height = htons(rc->height);
334 	srect_hdr.encoding = htonl(RFB_ENCODING_EXT_KEYEVENT);
335 	stream_write(cfd, &srect_hdr, sizeof(struct rfb_srvr_rect_hdr));
336 }
337 
338 static void
rfb_recv_set_pixfmt_msg(struct rfb_softc * rc __unused,int cfd)339 rfb_recv_set_pixfmt_msg(struct rfb_softc *rc __unused, int cfd)
340 {
341 	struct rfb_pixfmt_msg pixfmt_msg;
342 	uint8_t red_shift, green_shift, blue_shift;
343 	uint16_t red_max, green_max, blue_max;
344 	bool adjust_pixels = true;
345 
346 	(void)stream_read(cfd, (uint8_t *)&pixfmt_msg + 1,
347 	    sizeof(pixfmt_msg) - 1);
348 
349 	/*
350 	 * The framebuffer is fixed at 32 bit and orders the colors
351 	 * as RGB bytes. However, some VNC clients request a different
352 	 * ordering. We will still require the same bit depth and size
353 	 * but allow the colors to be shifted when sent to the client.
354 	 */
355 	if (pixfmt_msg.pixfmt.bpp != 32 || pixfmt_msg.pixfmt.truecolor != 1) {
356 		WPRINTF(("rfb: pixfmt unsupported bitdepth bpp: %d "
357 			 "truecolor: %d",
358 			 pixfmt_msg.pixfmt.bpp, pixfmt_msg.pixfmt.truecolor));
359 		return;
360 	}
361 
362 	red_max = ntohs(pixfmt_msg.pixfmt.red_max);
363 	green_max = ntohs(pixfmt_msg.pixfmt.green_max);
364 	blue_max = ntohs(pixfmt_msg.pixfmt.blue_max);
365 
366 	/* Check for valid max values */
367 	if (red_max != 255 || green_max != 255 || blue_max != 255) {
368 		WPRINTF(("rfb: pixfmt unsupported max values "
369 			 "r: %d g: %d b: %d",
370 			 red_max, green_max, blue_max));
371 		return;
372 	}
373 
374 	red_shift = pixfmt_msg.pixfmt.red_shift;
375 	green_shift = pixfmt_msg.pixfmt.green_shift;
376 	blue_shift = pixfmt_msg.pixfmt.blue_shift;
377 
378 	/* Check shifts are 8 bit aligned */
379 	if ((red_shift & 0x7) != 0 ||
380 	    (green_shift & 0x7) != 0 ||
381 	    (blue_shift & 0x7) != 0) {
382 		WPRINTF(("rfb: pixfmt unsupported shift values "
383 			 "r: %d g: %d b: %d",
384 			 red_shift, green_shift, blue_shift));
385 		return;
386 	}
387 
388 	if (red_shift == PIXEL_RED_SHIFT &&
389 	    green_shift == PIXEL_GREEN_SHIFT &&
390 	    blue_shift == PIXEL_BLUE_SHIFT) {
391 		adjust_pixels = false;
392 	}
393 
394 	pthread_mutex_lock(&rc->pixfmt_mtx);
395 	rc->new_pixfmt.red_shift = red_shift;
396 	rc->new_pixfmt.green_shift = green_shift;
397 	rc->new_pixfmt.blue_shift = blue_shift;
398 	rc->new_pixfmt.adjust_pixels = adjust_pixels;
399 	pthread_mutex_unlock(&rc->pixfmt_mtx);
400 
401 	/* Notify the write thread to update */
402 	rc->update_pixfmt = true;
403 }
404 
405 static void
rfb_recv_set_encodings_msg(struct rfb_softc * rc,int cfd)406 rfb_recv_set_encodings_msg(struct rfb_softc *rc, int cfd)
407 {
408 	struct rfb_enc_msg enc_msg;
409 	int i;
410 	uint32_t encoding;
411 
412 	(void)stream_read(cfd, (uint8_t *)&enc_msg + 1, sizeof(enc_msg) - 1);
413 
414 	for (i = 0; i < htons(enc_msg.numencs); i++) {
415 		(void)stream_read(cfd, &encoding, sizeof(encoding));
416 		switch (htonl(encoding)) {
417 		case RFB_ENCODING_RAW:
418 			rc->enc_raw_ok = true;
419 			break;
420 		case RFB_ENCODING_ZLIB:
421 			if (!rc->enc_zlib_ok) {
422 				deflateInit(&rc->zstream, Z_BEST_SPEED);
423 				rc->enc_zlib_ok = true;
424 			}
425 			break;
426 		case RFB_ENCODING_RESIZE:
427 			rc->enc_resize_ok = true;
428 			break;
429 		case RFB_ENCODING_EXT_KEYEVENT:
430 			rc->enc_extkeyevent_ok = true;
431 			break;
432 		}
433 	}
434 }
435 
436 /*
437  * Calculate CRC32 using SSE4.2; Intel or AMD Bulldozer+ CPUs only
438  */
439 static __inline uint32_t
fast_crc32(void * buf,int len,uint32_t crcval)440 fast_crc32(void *buf, int len, uint32_t crcval)
441 {
442 	uint32_t q = len / sizeof(uint32_t);
443 	uint32_t *p = (uint32_t *)buf;
444 
445 	while (q--) {
446 		asm volatile (
447 			".byte 0xf2, 0xf, 0x38, 0xf1, 0xf1;"
448 			:"=S" (crcval)
449 			:"0" (crcval), "c" (*p)
450 		);
451 		p++;
452 	}
453 
454 	return (crcval);
455 }
456 
457 static int
rfb_send_update_header(struct rfb_softc * rc __unused,int cfd,int numrects)458 rfb_send_update_header(struct rfb_softc *rc __unused, int cfd, int numrects)
459 {
460 	struct rfb_srvr_updt_msg supdt_msg;
461 
462 	supdt_msg.type = 0;
463 	supdt_msg.pad = 0;
464 	supdt_msg.numrects = htons(numrects);
465 
466 	return stream_write(cfd, &supdt_msg,
467 	    sizeof(struct rfb_srvr_updt_msg));
468 }
469 
470 static uint32_t *
rfb_adjust_pixels(struct rfb_softc * rc,uint32_t * gcptr,int width)471 rfb_adjust_pixels(struct rfb_softc *rc, uint32_t *gcptr, int width)
472 {
473 	uint32_t *pixelp;
474 	uint32_t red, green, blue;
475 	int i;
476 
477 	/* If no pixel adjustment needed, send in server format */
478 	if (!rc->pixfmt.adjust_pixels) {
479 		return (gcptr);
480 	}
481 
482 	for (i = 0, pixelp = rc->pixrow; i < width; i++, pixelp++, gcptr++) {
483 		red = (*gcptr >> 16) & 0xFF;
484 		green = (*gcptr >> 8) & 0xFF;
485 		blue = (*gcptr & 0xFF);
486 		*pixelp = (red << rc->pixfmt.red_shift) |
487 			  (green << rc->pixfmt.green_shift) |
488 			  (blue << rc->pixfmt.blue_shift);
489 	}
490 
491 	return (rc->pixrow);
492 }
493 
494 static int
rfb_send_rect(struct rfb_softc * rc,int cfd,struct bhyvegc_image * gc,int x,int y,int w,int h)495 rfb_send_rect(struct rfb_softc *rc, int cfd, struct bhyvegc_image *gc,
496               int x, int y, int w, int h)
497 {
498 	struct rfb_srvr_rect_hdr srect_hdr;
499 	unsigned long zlen;
500 	ssize_t nwrite, total;
501 	int err, width;
502 	uint32_t *p, *pixelp;
503 	uint8_t *zbufp;
504 
505 	/*
506 	 * Send a single rectangle of the given x, y, w h dimensions.
507 	 */
508 
509 	/* Rectangle header */
510 	srect_hdr.x = htons(x);
511 	srect_hdr.y = htons(y);
512 	srect_hdr.width = htons(w);
513 	srect_hdr.height = htons(h);
514 
515 	width = w;
516 	h = y + h;
517 	w *= sizeof(uint32_t);
518 	if (rc->enc_zlib_ok) {
519 		zbufp = rc->zbuf;
520 		rc->zstream.total_in = 0;
521 		rc->zstream.total_out = 0;
522 		for (p = &gc->data[y * gc->width + x]; y < h; y++) {
523 			pixelp = rfb_adjust_pixels(rc, p, width);
524 			rc->zstream.next_in = (Bytef *)pixelp;
525 			rc->zstream.avail_in = w;
526 			rc->zstream.next_out = (Bytef *)zbufp;
527 			rc->zstream.avail_out = RFB_ZLIB_BUFSZ + 16 -
528 			                        rc->zstream.total_out;
529 			rc->zstream.data_type = Z_BINARY;
530 
531 			/* Compress with zlib */
532 			err = deflate(&rc->zstream, Z_SYNC_FLUSH);
533 			if (err != Z_OK) {
534 				WPRINTF(("zlib[rect] deflate err: %d", err));
535 				rc->enc_zlib_ok = false;
536 				deflateEnd(&rc->zstream);
537 				goto doraw;
538 			}
539 			zbufp = rc->zbuf + rc->zstream.total_out;
540 			p += gc->width;
541 		}
542 		srect_hdr.encoding = htonl(RFB_ENCODING_ZLIB);
543 		nwrite = stream_write(cfd, &srect_hdr,
544 		                      sizeof(struct rfb_srvr_rect_hdr));
545 		if (nwrite <= 0)
546 			return (nwrite);
547 
548 		zlen = htonl(rc->zstream.total_out);
549 		nwrite = stream_write(cfd, &zlen, sizeof(uint32_t));
550 		if (nwrite <= 0)
551 			return (nwrite);
552 		return (stream_write(cfd, rc->zbuf, rc->zstream.total_out));
553 	}
554 
555 doraw:
556 
557 	total = 0;
558 	zbufp = rc->zbuf;
559 	for (p = &gc->data[y * gc->width + x]; y < h; y++) {
560 		pixelp = rfb_adjust_pixels(rc, p, width);
561 		memcpy(zbufp, pixelp, w);
562 		zbufp += w;
563 		total += w;
564 		p += gc->width;
565 	}
566 
567 	srect_hdr.encoding = htonl(RFB_ENCODING_RAW);
568 	nwrite = stream_write(cfd, &srect_hdr,
569 	                      sizeof(struct rfb_srvr_rect_hdr));
570 	if (nwrite <= 0)
571 		return (nwrite);
572 
573 	total = stream_write(cfd, rc->zbuf, total);
574 
575 	return (total);
576 }
577 
578 static int
rfb_send_all(struct rfb_softc * rc,int cfd,struct bhyvegc_image * gc)579 rfb_send_all(struct rfb_softc *rc, int cfd, struct bhyvegc_image *gc)
580 {
581 	struct rfb_srvr_updt_msg supdt_msg;
582         struct rfb_srvr_rect_hdr srect_hdr;
583 	ssize_t nwrite;
584 	unsigned long zlen;
585 	int err;
586 
587 	/*
588 	 * Send the whole thing
589 	 */
590 
591 	/* Number of rectangles: 1 */
592 	supdt_msg.type = 0;
593 	supdt_msg.pad = 0;
594 	supdt_msg.numrects = htons(1);
595 	nwrite = stream_write(cfd, &supdt_msg,
596 	                      sizeof(struct rfb_srvr_updt_msg));
597 	if (nwrite <= 0)
598 		return (nwrite);
599 
600 	if (rc->pixfmt.adjust_pixels) {
601 		return (rfb_send_rect(rc, cfd, gc, 0, 0,
602 				gc->width, gc->height));
603 	}
604 
605 	/* Rectangle header */
606 	srect_hdr.x = 0;
607 	srect_hdr.y = 0;
608 	srect_hdr.width = htons(gc->width);
609 	srect_hdr.height = htons(gc->height);
610 	if (rc->enc_zlib_ok) {
611 		rc->zstream.next_in = (Bytef *)gc->data;
612 		rc->zstream.avail_in = gc->width * gc->height *
613 		                   sizeof(uint32_t);
614 		rc->zstream.next_out = (Bytef *)rc->zbuf;
615 		rc->zstream.avail_out = RFB_ZLIB_BUFSZ + 16;
616 		rc->zstream.data_type = Z_BINARY;
617 
618 		rc->zstream.total_in = 0;
619 		rc->zstream.total_out = 0;
620 
621 		/* Compress with zlib */
622 		err = deflate(&rc->zstream, Z_SYNC_FLUSH);
623 		if (err != Z_OK) {
624 			WPRINTF(("zlib deflate err: %d", err));
625 			rc->enc_zlib_ok = false;
626 			deflateEnd(&rc->zstream);
627 			goto doraw;
628 		}
629 
630 		srect_hdr.encoding = htonl(RFB_ENCODING_ZLIB);
631 		nwrite = stream_write(cfd, &srect_hdr,
632 		                      sizeof(struct rfb_srvr_rect_hdr));
633 		if (nwrite <= 0)
634 			return (nwrite);
635 
636 		zlen = htonl(rc->zstream.total_out);
637 		nwrite = stream_write(cfd, &zlen, sizeof(uint32_t));
638 		if (nwrite <= 0)
639 			return (nwrite);
640 		return (stream_write(cfd, rc->zbuf, rc->zstream.total_out));
641 	}
642 
643 doraw:
644 	srect_hdr.encoding = htonl(RFB_ENCODING_RAW);
645 	nwrite = stream_write(cfd, &srect_hdr,
646 	                      sizeof(struct rfb_srvr_rect_hdr));
647 	if (nwrite <= 0)
648 		return (nwrite);
649 
650 	nwrite = stream_write(cfd, gc->data,
651 	               gc->width * gc->height * sizeof(uint32_t));
652 
653 	return (nwrite);
654 }
655 
656 #define	PIX_PER_CELL	32
657 #define	PIXCELL_SHIFT	5
658 #define	PIXCELL_MASK	0x1F
659 
660 static void
rfb_set_pixel_adjustment(struct rfb_softc * rc)661 rfb_set_pixel_adjustment(struct rfb_softc *rc)
662 {
663 	pthread_mutex_lock(&rc->pixfmt_mtx);
664 	rc->pixfmt = rc->new_pixfmt;
665 	pthread_mutex_unlock(&rc->pixfmt_mtx);
666 }
667 
668 static int
rfb_send_screen(struct rfb_softc * rc,int cfd)669 rfb_send_screen(struct rfb_softc *rc, int cfd)
670 {
671 	struct bhyvegc_image *gc_image;
672 	ssize_t nwrite;
673 	int x, y;
674 	int celly, cellwidth;
675 	int xcells, ycells;
676 	int w, h;
677 	uint32_t *p;
678 	int rem_x, rem_y;   /* remainder for resolutions not x32 pixels ratio */
679 	int retval;
680 	uint32_t *crc_p, *orig_crc;
681 	int changes;
682 	bool expected;
683 
684 	/* Return if another thread sending */
685 	expected = false;
686 	if (atomic_compare_exchange_strong(&rc->sending, &expected, true) == false)
687 		return (1);
688 
689 	retval = 1;
690 
691 	/* Updates require a preceding update request */
692 	if (atomic_exchange(&rc->pending, false) == false)
693 		goto done;
694 
695 	if (atomic_exchange(&rc->update_pixfmt, false) == true) {
696 		rfb_set_pixel_adjustment(rc);
697 	}
698 
699 	console_refresh();
700 	gc_image = console_get_image();
701 
702 	/* Clear old CRC values when the size changes */
703 	if (rc->crc_width != gc_image->width ||
704 	    rc->crc_height != gc_image->height) {
705 		memset(rc->crc, 0, sizeof(uint32_t) *
706 		    howmany(RFB_MAX_WIDTH, PIX_PER_CELL) *
707 		    howmany(RFB_MAX_HEIGHT, PIX_PER_CELL));
708 		rc->crc_width = gc_image->width;
709 		rc->crc_height = gc_image->height;
710 	}
711 
712        /* A size update counts as an update in itself */
713        if (rc->width != gc_image->width ||
714            rc->height != gc_image->height) {
715                rc->width = gc_image->width;
716                rc->height = gc_image->height;
717                if (rc->enc_resize_ok) {
718                        rfb_send_resize_update_msg(rc, cfd);
719 		       rc->update_all = true;
720                        goto done;
721                }
722        }
723 
724        if (atomic_exchange(&rc->update_all, false) == true) {
725 	       retval = rfb_send_all(rc, cfd, gc_image);
726 	       goto done;
727        }
728 
729 	/*
730 	 * Calculate the checksum for each 32x32 cell. Send each that
731 	 * has changed since the last scan.
732 	 */
733 
734 	w = rc->crc_width;
735 	h = rc->crc_height;
736 	xcells = howmany(rc->crc_width, PIX_PER_CELL);
737 	ycells = howmany(rc->crc_height, PIX_PER_CELL);
738 
739 	rem_x = w & PIXCELL_MASK;
740 
741 	rem_y = h & PIXCELL_MASK;
742 	if (!rem_y)
743 		rem_y = PIX_PER_CELL;
744 
745 	p = gc_image->data;
746 
747 	/*
748 	 * Go through all cells and calculate crc. If significant number
749 	 * of changes, then send entire screen.
750 	 * crc_tmp is dual purpose: to store the new crc and to flag as
751 	 * a cell that has changed.
752 	 */
753 	crc_p = rc->crc_tmp - xcells;
754 	orig_crc = rc->crc - xcells;
755 	changes = 0;
756 	memset(rc->crc_tmp, 0, sizeof(uint32_t) * xcells * ycells);
757 	for (y = 0; y < h; y++) {
758 		if ((y & PIXCELL_MASK) == 0) {
759 			crc_p += xcells;
760 			orig_crc += xcells;
761 		}
762 
763 		for (x = 0; x < xcells; x++) {
764 			if (x == (xcells - 1) && rem_x > 0)
765 				cellwidth = rem_x;
766 			else
767 				cellwidth = PIX_PER_CELL;
768 
769 			if (rc->hw_crc)
770 				crc_p[x] = fast_crc32(p,
771 				             cellwidth * sizeof(uint32_t),
772 				             crc_p[x]);
773 			else
774 				crc_p[x] = (uint32_t)crc32(crc_p[x],
775 				             (Bytef *)p,
776 				             cellwidth * sizeof(uint32_t));
777 
778 			p += cellwidth;
779 
780 			/* check for crc delta if last row in cell */
781 			if ((y & PIXCELL_MASK) == PIXCELL_MASK || y == (h-1)) {
782 				if (orig_crc[x] != crc_p[x]) {
783 					orig_crc[x] = crc_p[x];
784 					crc_p[x] = 1;
785 					changes++;
786 				} else {
787 					crc_p[x] = 0;
788 				}
789 			}
790 		}
791 	}
792 
793        /*
794 	* We only send the update if there are changes.
795 	* Restore the pending flag since it was unconditionally cleared
796 	* above.
797 	*/
798 	if (!changes) {
799 		rc->pending = true;
800 		goto done;
801 	}
802 
803 	/* If number of changes is > THRESH percent, send the whole screen */
804 	if (((changes * 100) / (xcells * ycells)) >= RFB_SEND_ALL_THRESH) {
805 		retval = rfb_send_all(rc, cfd, gc_image);
806 		goto done;
807 	}
808 
809 	rfb_send_update_header(rc, cfd, changes);
810 
811 	/* Go through all cells, and send only changed ones */
812 	crc_p = rc->crc_tmp;
813 	for (y = 0; y < h; y += PIX_PER_CELL) {
814 		/* previous cell's row */
815 		celly = (y >> PIXCELL_SHIFT);
816 
817 		/* Delta check crc to previous set */
818 		for (x = 0; x < xcells; x++) {
819 			if (*crc_p++ == 0)
820 				continue;
821 
822 			if (x == (xcells - 1) && rem_x > 0)
823 				cellwidth = rem_x;
824 			else
825 				cellwidth = PIX_PER_CELL;
826 			nwrite = rfb_send_rect(rc, cfd,
827 				gc_image,
828 				x * PIX_PER_CELL,
829 				celly * PIX_PER_CELL,
830 			        cellwidth,
831 				y + PIX_PER_CELL >= h ? rem_y : PIX_PER_CELL);
832 			if (nwrite <= 0) {
833 				retval = nwrite;
834 				goto done;
835 			}
836 		}
837 	}
838 
839 done:
840 	rc->sending = false;
841 
842 	return (retval);
843 }
844 
845 
846 static void
rfb_recv_update_msg(struct rfb_softc * rc,int cfd)847 rfb_recv_update_msg(struct rfb_softc *rc, int cfd)
848 {
849 	struct rfb_updt_msg updt_msg;
850 
851 	(void)stream_read(cfd, (uint8_t *)&updt_msg + 1 , sizeof(updt_msg) - 1);
852 
853 	if (rc->enc_extkeyevent_ok && (!rc->enc_extkeyevent_send)) {
854 		rfb_send_extended_keyevent_update_msg(rc, cfd);
855 		rc->enc_extkeyevent_send = true;
856 	}
857 
858 	rc->pending = true;
859 	if (!updt_msg.incremental)
860 		rc->update_all = true;
861 }
862 
863 static void
rfb_recv_key_msg(struct rfb_softc * rc,int cfd)864 rfb_recv_key_msg(struct rfb_softc *rc, int cfd)
865 {
866 	struct rfb_key_msg key_msg;
867 
868 	(void)stream_read(cfd, (uint8_t *)&key_msg + 1, sizeof(key_msg) - 1);
869 
870 	console_key_event(key_msg.down, htonl(key_msg.sym), htonl(0));
871 	rc->input_detected = true;
872 }
873 
874 static void
rfb_recv_client_msg(struct rfb_softc * rc,int cfd)875 rfb_recv_client_msg(struct rfb_softc *rc, int cfd)
876 {
877 	struct rfb_client_msg client_msg;
878 	struct rfb_extended_key_msg extkey_msg;
879 
880 	(void)stream_read(cfd, (uint8_t *)&client_msg + 1,
881 	    sizeof(client_msg) - 1);
882 
883 	if (client_msg.subtype == RFB_CLIENTMSG_EXT_KEYEVENT) {
884 		(void)stream_read(cfd, (uint8_t *)&extkey_msg + 2,
885 		    sizeof(extkey_msg) - 2);
886 		console_key_event((int)extkey_msg.down, htonl(extkey_msg.sym), htonl(extkey_msg.code));
887 		rc->input_detected = true;
888 	}
889 }
890 
891 static void
rfb_recv_ptr_msg(struct rfb_softc * rc,int cfd)892 rfb_recv_ptr_msg(struct rfb_softc *rc, int cfd)
893 {
894 	struct rfb_ptr_msg ptr_msg;
895 
896 	(void)stream_read(cfd, (uint8_t *)&ptr_msg + 1, sizeof(ptr_msg) - 1);
897 
898 	console_ptr_event(ptr_msg.button, htons(ptr_msg.x), htons(ptr_msg.y));
899 	rc->input_detected = true;
900 }
901 
902 static void
rfb_recv_cuttext_msg(struct rfb_softc * rc __unused,int cfd)903 rfb_recv_cuttext_msg(struct rfb_softc *rc __unused, int cfd)
904 {
905 	struct rfb_cuttext_msg ct_msg;
906 	unsigned char buf[32];
907 	int len;
908 
909 	len = stream_read(cfd, (uint8_t *)&ct_msg + 1, sizeof(ct_msg) - 1);
910 	ct_msg.length = htonl(ct_msg.length);
911 	while (ct_msg.length > 0) {
912 		len = stream_read(cfd, buf, ct_msg.length > sizeof(buf) ?
913 			sizeof(buf) : ct_msg.length);
914 		ct_msg.length -= len;
915 	}
916 }
917 
918 static int64_t
timeval_delta(struct timeval * prev,struct timeval * now)919 timeval_delta(struct timeval *prev, struct timeval *now)
920 {
921 	int64_t n1, n2;
922 	n1 = now->tv_sec * 1000000 + now->tv_usec;
923 	n2 = prev->tv_sec * 1000000 + prev->tv_usec;
924 	return (n1 - n2);
925 }
926 
927 static void *
rfb_wr_thr(void * arg)928 rfb_wr_thr(void *arg)
929 {
930 	struct rfb_softc *rc;
931 	fd_set rfds;
932 	struct timeval tv;
933 	struct timeval prev_tv;
934 	int64_t tdiff;
935 	int cfd;
936 	int err;
937 
938 	rc = arg;
939 	cfd = rc->cfd;
940 
941 	prev_tv.tv_sec = 0;
942 	prev_tv.tv_usec = 0;
943 	while (rc->cfd >= 0) {
944 		FD_ZERO(&rfds);
945 		FD_SET(cfd, &rfds);
946 		tv.tv_sec = 0;
947 		tv.tv_usec = CFD_SEL_DELAY;
948 
949 		err = select(cfd+1, &rfds, NULL, NULL, &tv);
950 		if (err < 0)
951 			return (NULL);
952 
953 		/* Determine if its time to push screen; ~24hz */
954 		gettimeofday(&tv, NULL);
955 		tdiff = timeval_delta(&prev_tv, &tv);
956 		if (tdiff >= SCREEN_POLL_DELAY) {
957 			bool input;
958 			prev_tv.tv_sec = tv.tv_sec;
959 			prev_tv.tv_usec = tv.tv_usec;
960 			input = atomic_exchange(&rc->input_detected, false);
961 			/*
962 			 * Refresh the screen on every second trip through the loop,
963 			 * or if keyboard/mouse input has been detected.
964 			 */
965 			if ((++rc->wrcount & 1) || input) {
966 				if (rfb_send_screen(rc, cfd) <= 0) {
967 					return (NULL);
968 				}
969 			}
970 		} else {
971 			/* sleep */
972 			usleep(SCREEN_POLL_DELAY - tdiff);
973 		}
974 	}
975 
976 	return (NULL);
977 }
978 
979 static void
rfb_handle(struct rfb_softc * rc,int cfd)980 rfb_handle(struct rfb_softc *rc, int cfd)
981 {
982 	const char *vbuf = "RFB 003.008\n";
983 	unsigned char buf[80];
984 	unsigned const char *message;
985 
986 #ifndef NO_OPENSSL
987 	unsigned char challenge[AUTH_LENGTH];
988 	unsigned char keystr[PASSWD_LENGTH];
989 	unsigned char crypt_expected[AUTH_LENGTH];
990 
991 	DES_key_schedule ks;
992 	int i;
993 #endif
994 	uint8_t client_ver;
995 	uint8_t auth_type;
996 	pthread_t tid;
997 	uint32_t sres = 0;
998 	int len;
999 	int perror = 1;
1000 
1001 	rc->cfd = cfd;
1002 
1003 	/* 1a. Send server version */
1004 	stream_write(cfd, vbuf, strlen(vbuf));
1005 
1006 	/* 1b. Read client version */
1007 	len = stream_read(cfd, buf, VERSION_LENGTH);
1008 	if (len != VERSION_LENGTH ||
1009 	    strncmp(vbuf, buf, VERSION_LENGTH - 2) != 0) {
1010 		goto done;
1011 	}
1012 
1013 	client_ver = buf[VERSION_LENGTH - 2];
1014 	if (client_ver != CVERS_3_8 && client_ver != CVERS_3_7) {
1015 		/* only recognize 3.3, 3.7 & 3.8. Others dflt to 3.3 */
1016 		client_ver = CVERS_3_3;
1017 	}
1018 
1019 	/* 2a. Send security type */
1020 	buf[0] = 1;
1021 
1022 	/* In versions 3.7 & 3.8, it's 2-way handshake */
1023 	/* For version 3.3, server says what the authentication type must be */
1024 #ifndef NO_OPENSSL
1025 	if (rc->password) {
1026 		auth_type = SECURITY_TYPE_VNC_AUTH;
1027 	} else {
1028 		auth_type = SECURITY_TYPE_NONE;
1029 	}
1030 #else
1031 	auth_type = SECURITY_TYPE_NONE;
1032 #endif
1033 
1034 	switch (client_ver) {
1035 	case CVERS_3_7:
1036 	case CVERS_3_8:
1037 		buf[0] = 1;
1038 		buf[1] = auth_type;
1039 		stream_write(cfd, buf, 2);
1040 
1041 		/* 2b. Read agreed security type */
1042 		len = stream_read(cfd, buf, 1);
1043 		if (buf[0] != auth_type) {
1044 			/* deny */
1045 			sres = htonl(1);
1046 			message = "Auth failed: authentication type mismatch";
1047 			goto report_and_done;
1048 		}
1049 		break;
1050 	case CVERS_3_3:
1051 	default:
1052 		be32enc(buf, auth_type);
1053 		stream_write(cfd, buf, 4);
1054 		break;
1055 	}
1056 
1057 	/* 2c. Do VNC authentication */
1058 	switch (auth_type) {
1059 	case SECURITY_TYPE_NONE:
1060 		break;
1061 	case SECURITY_TYPE_VNC_AUTH:
1062 		/*
1063 		 * The client encrypts the challenge with DES, using a password
1064 		 * supplied by the user as the key.
1065 		 * To form the key, the password is truncated to
1066 		 * eight characters, or padded with null bytes on the right.
1067 		 * The client then sends the resulting 16-bytes response.
1068 		 */
1069 #ifndef NO_OPENSSL
1070 		strncpy(keystr, rc->password, PASSWD_LENGTH);
1071 
1072 		/* VNC clients encrypts the challenge with all the bit fields
1073 		 * in each byte of the password mirrored.
1074 		 * Here we flip each byte of the keystr.
1075 		 */
1076 		for (i = 0; i < PASSWD_LENGTH; i++) {
1077 			keystr[i] = (keystr[i] & 0xF0) >> 4
1078 				  | (keystr[i] & 0x0F) << 4;
1079 			keystr[i] = (keystr[i] & 0xCC) >> 2
1080 				  | (keystr[i] & 0x33) << 2;
1081 			keystr[i] = (keystr[i] & 0xAA) >> 1
1082 				  | (keystr[i] & 0x55) << 1;
1083 		}
1084 
1085 		/* Initialize a 16-byte random challenge */
1086 		arc4random_buf(challenge, sizeof(challenge));
1087 		stream_write(cfd, challenge, AUTH_LENGTH);
1088 
1089 		/* Receive the 16-byte challenge response */
1090 		stream_read(cfd, buf, AUTH_LENGTH);
1091 
1092 		memcpy(crypt_expected, challenge, AUTH_LENGTH);
1093 
1094 		/* Encrypt the Challenge with DES */
1095 		DES_set_key((const_DES_cblock *)keystr, &ks);
1096 		DES_ecb_encrypt((const_DES_cblock *)challenge,
1097 				(const_DES_cblock *)crypt_expected,
1098 				&ks, DES_ENCRYPT);
1099 		DES_ecb_encrypt((const_DES_cblock *)(challenge + PASSWD_LENGTH),
1100 				(const_DES_cblock *)(crypt_expected +
1101 				PASSWD_LENGTH),
1102 				&ks, DES_ENCRYPT);
1103 
1104 		if (memcmp(crypt_expected, buf, AUTH_LENGTH) != 0) {
1105 			message = "Auth Failed: Invalid Password.";
1106 			sres = htonl(1);
1107 		} else {
1108 			sres = 0;
1109 		}
1110 #else
1111 		sres = htonl(1);
1112 		WPRINTF(("Auth not supported, no OpenSSL in your system"));
1113 #endif
1114 
1115 		break;
1116 	}
1117 
1118 	switch (client_ver) {
1119 	case CVERS_3_7:
1120 	case CVERS_3_8:
1121 report_and_done:
1122 		/* 2d. Write back a status */
1123 		stream_write(cfd, &sres, 4);
1124 
1125 		if (sres) {
1126 			/* 3.7 does not want string explaining cause */
1127 			if (client_ver == CVERS_3_8) {
1128 				be32enc(buf, strlen(message));
1129 				stream_write(cfd, buf, 4);
1130 				stream_write(cfd, message, strlen(message));
1131 			}
1132 			goto done;
1133 		}
1134 		break;
1135 	case CVERS_3_3:
1136 	default:
1137 		/* for VNC auth case send status */
1138 		if (auth_type == SECURITY_TYPE_VNC_AUTH) {
1139 			/* 2d. Write back a status */
1140 			stream_write(cfd, &sres, 4);
1141 		}
1142 		if (sres) {
1143 			goto done;
1144 		}
1145 		break;
1146 	}
1147 	/* 3a. Read client shared-flag byte */
1148 	len = stream_read(cfd, buf, 1);
1149 
1150 	/* 4a. Write server-init info */
1151 	rfb_send_server_init_msg(rc, cfd);
1152 
1153 	if (!rc->zbuf) {
1154 		rc->zbuf = malloc(RFB_ZLIB_BUFSZ + 16);
1155 		assert(rc->zbuf != NULL);
1156 	}
1157 
1158 	perror = pthread_create(&tid, NULL, rfb_wr_thr, rc);
1159 	if (perror == 0)
1160 		pthread_set_name_np(tid, "rfbout");
1161 
1162         /* Now read in client requests. 1st byte identifies type */
1163 	for (;;) {
1164 		len = read(cfd, buf, 1);
1165 		if (len <= 0) {
1166 			DPRINTF(("rfb client exiting"));
1167 			break;
1168 		}
1169 
1170 		switch (buf[0]) {
1171 		case CS_SET_PIXEL_FORMAT:
1172 			rfb_recv_set_pixfmt_msg(rc, cfd);
1173 			break;
1174 		case CS_SET_ENCODINGS:
1175 			rfb_recv_set_encodings_msg(rc, cfd);
1176 			break;
1177 		case CS_UPDATE_MSG:
1178 			rfb_recv_update_msg(rc, cfd);
1179 			break;
1180 		case CS_KEY_EVENT:
1181 			rfb_recv_key_msg(rc, cfd);
1182 			break;
1183 		case CS_POINTER_EVENT:
1184 			rfb_recv_ptr_msg(rc, cfd);
1185 			break;
1186 		case CS_CUT_TEXT:
1187 			rfb_recv_cuttext_msg(rc, cfd);
1188 			break;
1189 		case CS_MSG_CLIENT_QEMU:
1190 			rfb_recv_client_msg(rc, cfd);
1191 			break;
1192 		default:
1193 			WPRINTF(("rfb unknown cli-code %d!", buf[0] & 0xff));
1194 			goto done;
1195 		}
1196 	}
1197 done:
1198 	rc->cfd = -1;
1199 	if (perror == 0)
1200 		pthread_join(tid, NULL);
1201 	if (rc->enc_zlib_ok)
1202 		deflateEnd(&rc->zstream);
1203 }
1204 
1205 static void *
rfb_thr(void * arg)1206 rfb_thr(void *arg)
1207 {
1208 	struct rfb_softc *rc;
1209 	sigset_t set;
1210 
1211 	int cfd;
1212 
1213 	rc = arg;
1214 
1215 	sigemptyset(&set);
1216 	sigaddset(&set, SIGPIPE);
1217 	if (pthread_sigmask(SIG_BLOCK, &set, NULL) != 0) {
1218 		perror("pthread_sigmask");
1219 		return (NULL);
1220 	}
1221 
1222 	for (;;) {
1223 		rc->enc_raw_ok = false;
1224 		rc->enc_zlib_ok = false;
1225 		rc->enc_resize_ok = false;
1226 		rc->enc_extkeyevent_ok = false;
1227 
1228 		rc->enc_extkeyevent_send = false;
1229 
1230 		cfd = accept(rc->sfd, NULL, NULL);
1231 		if (rc->conn_wait) {
1232 			pthread_mutex_lock(&rc->mtx);
1233 			pthread_cond_signal(&rc->cond);
1234 			pthread_mutex_unlock(&rc->mtx);
1235 			rc->conn_wait = 0;
1236 		}
1237 		rfb_handle(rc, cfd);
1238 		close(cfd);
1239 	}
1240 
1241 	/* NOTREACHED */
1242 	return (NULL);
1243 }
1244 
1245 static int
sse42_supported(void)1246 sse42_supported(void)
1247 {
1248 	u_int cpu_registers[4], ecx;
1249 
1250 	do_cpuid(1, cpu_registers);
1251 
1252 	ecx = cpu_registers[2];
1253 
1254 	return ((ecx & CPUID2_SSE42) != 0);
1255 }
1256 
1257 int
rfb_init(sa_family_t family,const char * hostname,int port,int wait,const char * password)1258 rfb_init(sa_family_t family, const char *hostname, int port, int wait,
1259     const char *password)
1260 {
1261 	int e;
1262 	char servname[6];
1263 	struct rfb_softc *rc;
1264 	struct addrinfo *ai = NULL;
1265 	struct addrinfo hints;
1266 	struct sockaddr_un sun;
1267 	int on = 1;
1268 	int cnt;
1269 #ifndef WITHOUT_CAPSICUM
1270 	cap_rights_t rights;
1271 #endif
1272 
1273 	rc = calloc(1, sizeof(struct rfb_softc));
1274 
1275 	cnt = howmany(RFB_MAX_WIDTH, PIX_PER_CELL) *
1276 	    howmany(RFB_MAX_HEIGHT, PIX_PER_CELL);
1277 	rc->crc = calloc(cnt, sizeof(uint32_t));
1278 	rc->crc_tmp = calloc(cnt, sizeof(uint32_t));
1279 	rc->crc_width = RFB_MAX_WIDTH;
1280 	rc->crc_height = RFB_MAX_HEIGHT;
1281 	rc->sfd = -1;
1282 
1283 	rc->password = password;
1284 
1285 	rc->fbnamelen = asprintf(&rc->fbname, "bhyve:%s",
1286 	    get_config_value("name"));
1287 	if (rc->fbnamelen < 0) {
1288 		EPRINTLN("rfb: failed to allocate memory for VNC title");
1289 		goto error;
1290 	}
1291 
1292 	rc->pixrow = malloc(RFB_MAX_WIDTH * sizeof(uint32_t));
1293 	if (rc->pixrow == NULL) {
1294 		EPRINTLN("rfb: failed to allocate memory for pixrow buffer");
1295 		goto error;
1296 	}
1297 
1298 	snprintf(servname, sizeof(servname), "%d", port ? port : 5900);
1299 
1300 	if (!hostname || strlen(hostname) == 0)
1301 #if defined(INET)
1302 		hostname = "127.0.0.1";
1303 #elif defined(INET6)
1304 		hostname = "[::1]";
1305 #endif
1306 
1307 	if (family == AF_UNIX) {
1308 		memset(&sun, 0, sizeof(sun));
1309 		sun.sun_family = AF_UNIX;
1310 		if (strlcpy(sun.sun_path, hostname, sizeof(sun.sun_path)) >=
1311 		    sizeof(sun.sun_path)) {
1312 			EPRINTLN("rfb: socket path too long");
1313 			goto error;
1314 		}
1315 		rc->sfd = socket(AF_UNIX, SOCK_STREAM, 0);
1316 	} else {
1317 		memset(&hints, 0, sizeof(hints));
1318 		hints.ai_socktype = SOCK_STREAM;
1319 		hints.ai_family = family;
1320 		hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV | AI_PASSIVE;
1321 
1322 		if ((e = getaddrinfo(hostname, servname, &hints, &ai)) != 0) {
1323 			EPRINTLN("getaddrinfo: %s", gai_strerror(e));
1324 			goto error;
1325 		}
1326 		rc->sfd = socket(ai->ai_family, ai->ai_socktype, 0);
1327 	}
1328 
1329 	if (rc->sfd < 0) {
1330 		perror("socket");
1331 		goto error;
1332 	}
1333 
1334 	/* No effect for UNIX domain sockets. */
1335 	setsockopt(rc->sfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
1336 
1337 	if (family == AF_UNIX) {
1338 		unlink(hostname);
1339 		e = bind(rc->sfd, (struct sockaddr *)&sun, SUN_LEN(&sun));
1340 	} else
1341 		e = bind(rc->sfd, ai->ai_addr, ai->ai_addrlen);
1342 	if (e < 0) {
1343 		perror("bind");
1344 		goto error;
1345 	}
1346 
1347 	if (listen(rc->sfd, 1) < 0) {
1348 		perror("listen");
1349 		goto error;
1350 	}
1351 
1352 #ifndef WITHOUT_CAPSICUM
1353 	cap_rights_init(&rights, CAP_ACCEPT, CAP_EVENT, CAP_READ, CAP_WRITE);
1354 	if (caph_rights_limit(rc->sfd, &rights) == -1)
1355 		errx(EX_OSERR, "Unable to apply rights for sandbox");
1356 #endif
1357 
1358 	rc->hw_crc = sse42_supported();
1359 
1360 	rc->conn_wait = wait;
1361 	if (wait) {
1362 		pthread_mutex_init(&rc->mtx, NULL);
1363 		pthread_cond_init(&rc->cond, NULL);
1364 	}
1365 
1366 	pthread_mutex_init(&rc->pixfmt_mtx, NULL);
1367 	pthread_create(&rc->tid, NULL, rfb_thr, rc);
1368 	pthread_set_name_np(rc->tid, "rfb");
1369 
1370 	if (wait) {
1371 		DPRINTF(("Waiting for rfb client..."));
1372 		pthread_mutex_lock(&rc->mtx);
1373 		pthread_cond_wait(&rc->cond, &rc->mtx);
1374 		pthread_mutex_unlock(&rc->mtx);
1375 		DPRINTF(("rfb client connected"));
1376 	}
1377 
1378 	if (family != AF_UNIX)
1379 		freeaddrinfo(ai);
1380 	return (0);
1381 
1382  error:
1383 	if (rc->pixfmt_mtx)
1384 		pthread_mutex_destroy(&rc->pixfmt_mtx);
1385 	if (ai != NULL) {
1386 		assert(family != AF_UNIX);
1387 		freeaddrinfo(ai);
1388 	}
1389 	if (rc->sfd != -1)
1390 		close(rc->sfd);
1391 	free(rc->crc);
1392 	free(rc->crc_tmp);
1393 	free(rc->pixrow);
1394 	free(rc->fbname);
1395 	free(rc);
1396 	return (-1);
1397 }
1398