xref: /freebsd/usr.sbin/bhyve/snapshot.c (revision 927358dd98cb902160093e0dc0bac002d6b43858)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2016 Flavius Anton
5  * Copyright (c) 2016 Mihai Tiganus
6  * Copyright (c) 2016-2019 Mihai Carabas
7  * Copyright (c) 2017-2019 Darius Mihai
8  * Copyright (c) 2017-2019 Elena Mihailescu
9  * Copyright (c) 2018-2019 Sergiu Weisz
10  * All rights reserved.
11  * The bhyve-snapshot feature was developed under sponsorships
12  * from Matthew Grooms.
13  *
14  * Redistribution and use in source and binary forms, with or without
15  * modification, are permitted provided that the following conditions
16  * are met:
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in the
21  *    documentation and/or other materials provided with the distribution.
22  *
23  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
27  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33  * SUCH DAMAGE.
34  */
35 
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
38 
39 #include <sys/types.h>
40 #ifndef WITHOUT_CAPSICUM
41 #include <sys/capsicum.h>
42 #endif
43 #include <sys/mman.h>
44 #include <sys/socket.h>
45 #include <sys/stat.h>
46 #include <sys/time.h>
47 #include <sys/un.h>
48 
49 #include <machine/atomic.h>
50 #include <machine/segments.h>
51 
52 #ifndef WITHOUT_CAPSICUM
53 #include <capsicum_helpers.h>
54 #endif
55 #include <stdio.h>
56 #include <stdlib.h>
57 #include <string.h>
58 #include <err.h>
59 #include <errno.h>
60 #include <fcntl.h>
61 #include <libgen.h>
62 #include <signal.h>
63 #include <unistd.h>
64 #include <assert.h>
65 #include <errno.h>
66 #include <pthread.h>
67 #include <pthread_np.h>
68 #include <sysexits.h>
69 #include <stdbool.h>
70 #include <sys/ioctl.h>
71 
72 #include <machine/vmm.h>
73 #ifndef WITHOUT_CAPSICUM
74 #include <machine/vmm_dev.h>
75 #endif
76 #include <machine/vmm_snapshot.h>
77 #include <vmmapi.h>
78 
79 #include "bhyverun.h"
80 #include "acpi.h"
81 #include "atkbdc.h"
82 #include "debug.h"
83 #include "inout.h"
84 #include "ipc.h"
85 #include "fwctl.h"
86 #include "ioapic.h"
87 #include "mem.h"
88 #include "mevent.h"
89 #include "mptbl.h"
90 #include "pci_emul.h"
91 #include "pci_irq.h"
92 #include "pci_lpc.h"
93 #include "smbiostbl.h"
94 #include "snapshot.h"
95 #include "xmsr.h"
96 #include "spinup_ap.h"
97 #include "rtc.h"
98 
99 #include <libxo/xo.h>
100 #include <ucl.h>
101 
102 struct spinner_info {
103 	const size_t *crtval;
104 	const size_t maxval;
105 	const size_t total;
106 };
107 
108 extern int guest_ncpus;
109 
110 static struct winsize winsize;
111 static sig_t old_winch_handler;
112 
113 #define	KB		(1024UL)
114 #define	MB		(1024UL * KB)
115 #define	GB		(1024UL * MB)
116 
117 #define	SNAPSHOT_CHUNK	(4 * MB)
118 #define	PROG_BUF_SZ	(8192)
119 
120 #define	SNAPSHOT_BUFFER_SIZE (20 * MB)
121 
122 #define	JSON_STRUCT_ARR_KEY		"structs"
123 #define	JSON_DEV_ARR_KEY		"devices"
124 #define	JSON_BASIC_METADATA_KEY 	"basic metadata"
125 #define	JSON_SNAPSHOT_REQ_KEY		"snapshot_req"
126 #define	JSON_SIZE_KEY			"size"
127 #define	JSON_FILE_OFFSET_KEY		"file_offset"
128 
129 #define	JSON_NCPUS_KEY			"ncpus"
130 #define	JSON_VMNAME_KEY 		"vmname"
131 #define	JSON_MEMSIZE_KEY		"memsize"
132 #define	JSON_MEMFLAGS_KEY		"memflags"
133 
134 #define min(a,b)		\
135 ({				\
136  __typeof__ (a) _a = (a);	\
137  __typeof__ (b) _b = (b); 	\
138  _a < _b ? _a : _b;       	\
139  })
140 
141 static const struct vm_snapshot_dev_info snapshot_devs[] = {
142 	{ "atkbdc",	atkbdc_snapshot,	NULL,		NULL		},
143 	{ "virtio-net",	pci_snapshot,		pci_pause,	pci_resume	},
144 	{ "virtio-blk",	pci_snapshot,		pci_pause,	pci_resume	},
145 	{ "virtio-rnd",	pci_snapshot,		NULL,		NULL		},
146 	{ "lpc",	pci_snapshot,		NULL,		NULL		},
147 	{ "fbuf",	pci_snapshot,		NULL,		NULL		},
148 	{ "xhci",	pci_snapshot,		NULL,		NULL		},
149 	{ "e1000",	pci_snapshot,		NULL,		NULL		},
150 	{ "ahci",	pci_snapshot,		pci_pause,	pci_resume	},
151 	{ "ahci-hd",	pci_snapshot,		pci_pause,	pci_resume	},
152 	{ "ahci-cd",	pci_snapshot,		pci_pause,	pci_resume	},
153 };
154 
155 static const struct vm_snapshot_kern_info snapshot_kern_structs[] = {
156 	{ "vhpet",	STRUCT_VHPET	},
157 	{ "vm",		STRUCT_VM	},
158 	{ "vioapic",	STRUCT_VIOAPIC	},
159 	{ "vlapic",	STRUCT_VLAPIC	},
160 	{ "vmcx",	STRUCT_VMCX	},
161 	{ "vatpit",	STRUCT_VATPIT	},
162 	{ "vatpic",	STRUCT_VATPIC	},
163 	{ "vpmtmr",	STRUCT_VPMTMR	},
164 	{ "vrtc",	STRUCT_VRTC	},
165 };
166 
167 static cpuset_t vcpus_active, vcpus_suspended;
168 static pthread_mutex_t vcpu_lock;
169 static pthread_cond_t vcpus_idle, vcpus_can_run;
170 static bool checkpoint_active;
171 
172 /*
173  * TODO: Harden this function and all of its callers since 'base_str' is a user
174  * provided string.
175  */
176 static char *
177 strcat_extension(const char *base_str, const char *ext)
178 {
179 	char *res;
180 	size_t base_len, ext_len;
181 
182 	base_len = strnlen(base_str, NAME_MAX);
183 	ext_len = strnlen(ext, NAME_MAX);
184 
185 	if (base_len + ext_len > NAME_MAX) {
186 		fprintf(stderr, "Filename exceeds maximum length.\n");
187 		return (NULL);
188 	}
189 
190 	res = malloc(base_len + ext_len + 1);
191 	if (res == NULL) {
192 		perror("Failed to allocate memory.");
193 		return (NULL);
194 	}
195 
196 	memcpy(res, base_str, base_len);
197 	memcpy(res + base_len, ext, ext_len);
198 	res[base_len + ext_len] = 0;
199 
200 	return (res);
201 }
202 
203 void
204 destroy_restore_state(struct restore_state *rstate)
205 {
206 	if (rstate == NULL) {
207 		fprintf(stderr, "Attempting to destroy NULL restore struct.\n");
208 		return;
209 	}
210 
211 	if (rstate->kdata_map != MAP_FAILED)
212 		munmap(rstate->kdata_map, rstate->kdata_len);
213 
214 	if (rstate->kdata_fd > 0)
215 		close(rstate->kdata_fd);
216 	if (rstate->vmmem_fd > 0)
217 		close(rstate->vmmem_fd);
218 
219 	if (rstate->meta_root_obj != NULL)
220 		ucl_object_unref(rstate->meta_root_obj);
221 	if (rstate->meta_parser != NULL)
222 		ucl_parser_free(rstate->meta_parser);
223 }
224 
225 static int
226 load_vmmem_file(const char *filename, struct restore_state *rstate)
227 {
228 	struct stat sb;
229 	int err;
230 
231 	rstate->vmmem_fd = open(filename, O_RDONLY);
232 	if (rstate->vmmem_fd < 0) {
233 		perror("Failed to open restore file");
234 		return (-1);
235 	}
236 
237 	err = fstat(rstate->vmmem_fd, &sb);
238 	if (err < 0) {
239 		perror("Failed to stat restore file");
240 		goto err_load_vmmem;
241 	}
242 
243 	if (sb.st_size == 0) {
244 		fprintf(stderr, "Restore file is empty.\n");
245 		goto err_load_vmmem;
246 	}
247 
248 	rstate->vmmem_len = sb.st_size;
249 
250 	return (0);
251 
252 err_load_vmmem:
253 	if (rstate->vmmem_fd > 0)
254 		close(rstate->vmmem_fd);
255 	return (-1);
256 }
257 
258 static int
259 load_kdata_file(const char *filename, struct restore_state *rstate)
260 {
261 	struct stat sb;
262 	int err;
263 
264 	rstate->kdata_fd = open(filename, O_RDONLY);
265 	if (rstate->kdata_fd < 0) {
266 		perror("Failed to open kernel data file");
267 		return (-1);
268 	}
269 
270 	err = fstat(rstate->kdata_fd, &sb);
271 	if (err < 0) {
272 		perror("Failed to stat kernel data file");
273 		goto err_load_kdata;
274 	}
275 
276 	if (sb.st_size == 0) {
277 		fprintf(stderr, "Kernel data file is empty.\n");
278 		goto err_load_kdata;
279 	}
280 
281 	rstate->kdata_len = sb.st_size;
282 	rstate->kdata_map = mmap(NULL, rstate->kdata_len, PROT_READ,
283 				 MAP_SHARED, rstate->kdata_fd, 0);
284 	if (rstate->kdata_map == MAP_FAILED) {
285 		perror("Failed to map restore file");
286 		goto err_load_kdata;
287 	}
288 
289 	return (0);
290 
291 err_load_kdata:
292 	if (rstate->kdata_fd > 0)
293 		close(rstate->kdata_fd);
294 	return (-1);
295 }
296 
297 static int
298 load_metadata_file(const char *filename, struct restore_state *rstate)
299 {
300 	ucl_object_t *obj;
301 	struct ucl_parser *parser;
302 	int err;
303 
304 	parser = ucl_parser_new(UCL_PARSER_DEFAULT);
305 	if (parser == NULL) {
306 		fprintf(stderr, "Failed to initialize UCL parser.\n");
307 		err = -1;
308 		goto err_load_metadata;
309 	}
310 
311 	err = ucl_parser_add_file(parser, filename);
312 	if (err == 0) {
313 		fprintf(stderr, "Failed to parse metadata file: '%s'\n",
314 			filename);
315 		err = -1;
316 		goto err_load_metadata;
317 	}
318 
319 	obj = ucl_parser_get_object(parser);
320 	if (obj == NULL) {
321 		fprintf(stderr, "Failed to parse object.\n");
322 		err = -1;
323 		goto err_load_metadata;
324 	}
325 
326 	rstate->meta_parser = parser;
327 	rstate->meta_root_obj = (ucl_object_t *)obj;
328 
329 	return (0);
330 
331 err_load_metadata:
332 	if (parser != NULL)
333 		ucl_parser_free(parser);
334 	return (err);
335 }
336 
337 int
338 load_restore_file(const char *filename, struct restore_state *rstate)
339 {
340 	int err = 0;
341 	char *kdata_filename = NULL, *meta_filename = NULL;
342 
343 	assert(filename != NULL);
344 	assert(rstate != NULL);
345 
346 	memset(rstate, 0, sizeof(*rstate));
347 	rstate->kdata_map = MAP_FAILED;
348 
349 	err = load_vmmem_file(filename, rstate);
350 	if (err != 0) {
351 		fprintf(stderr, "Failed to load guest RAM file.\n");
352 		goto err_restore;
353 	}
354 
355 	kdata_filename = strcat_extension(filename, ".kern");
356 	if (kdata_filename == NULL) {
357 		fprintf(stderr, "Failed to construct kernel data filename.\n");
358 		goto err_restore;
359 	}
360 
361 	err = load_kdata_file(kdata_filename, rstate);
362 	if (err != 0) {
363 		fprintf(stderr, "Failed to load guest kernel data file.\n");
364 		goto err_restore;
365 	}
366 
367 	meta_filename = strcat_extension(filename, ".meta");
368 	if (meta_filename == NULL) {
369 		fprintf(stderr, "Failed to construct kernel metadata filename.\n");
370 		goto err_restore;
371 	}
372 
373 	err = load_metadata_file(meta_filename, rstate);
374 	if (err != 0) {
375 		fprintf(stderr, "Failed to load guest metadata file.\n");
376 		goto err_restore;
377 	}
378 
379 	return (0);
380 
381 err_restore:
382 	destroy_restore_state(rstate);
383 	if (kdata_filename != NULL)
384 		free(kdata_filename);
385 	if (meta_filename != NULL)
386 		free(meta_filename);
387 	return (-1);
388 }
389 
390 #define JSON_GET_INT_OR_RETURN(key, obj, result_ptr, ret)			\
391 do {										\
392 	const ucl_object_t *obj__;						\
393 	obj__ = ucl_object_lookup(obj, key);					\
394 	if (obj__ == NULL) {							\
395 		fprintf(stderr, "Missing key: '%s'", key);			\
396 		return (ret);							\
397 	}									\
398 	if (!ucl_object_toint_safe(obj__, result_ptr)) {			\
399 		fprintf(stderr, "Cannot convert '%s' value to int.", key);	\
400 		return (ret);							\
401 	}									\
402 } while(0)
403 
404 #define JSON_GET_STRING_OR_RETURN(key, obj, result_ptr, ret)			\
405 do {										\
406 	const ucl_object_t *obj__;						\
407 	obj__ = ucl_object_lookup(obj, key);					\
408 	if (obj__ == NULL) {							\
409 		fprintf(stderr, "Missing key: '%s'", key);			\
410 		return (ret);							\
411 	}									\
412 	if (!ucl_object_tostring_safe(obj__, result_ptr)) {			\
413 		fprintf(stderr, "Cannot convert '%s' value to string.", key);	\
414 		return (ret);							\
415 	}									\
416 } while(0)
417 
418 static void *
419 lookup_struct(enum snapshot_req struct_id, struct restore_state *rstate,
420 	      size_t *struct_size)
421 {
422 	const ucl_object_t *structs = NULL, *obj = NULL;
423 	ucl_object_iter_t it = NULL;
424 	int64_t snapshot_req, size, file_offset;
425 
426 	structs = ucl_object_lookup(rstate->meta_root_obj, JSON_STRUCT_ARR_KEY);
427 	if (structs == NULL) {
428 		fprintf(stderr, "Failed to find '%s' object.\n",
429 			JSON_STRUCT_ARR_KEY);
430 		return (NULL);
431 	}
432 
433 	if (ucl_object_type(structs) != UCL_ARRAY) {
434 		fprintf(stderr, "Object '%s' is not an array.\n",
435 		JSON_STRUCT_ARR_KEY);
436 		return (NULL);
437 	}
438 
439 	while ((obj = ucl_object_iterate(structs, &it, true)) != NULL) {
440 		snapshot_req = -1;
441 		JSON_GET_INT_OR_RETURN(JSON_SNAPSHOT_REQ_KEY, obj,
442 				       &snapshot_req, NULL);
443 		assert(snapshot_req >= 0);
444 		if ((enum snapshot_req) snapshot_req == struct_id) {
445 			JSON_GET_INT_OR_RETURN(JSON_SIZE_KEY, obj,
446 					       &size, NULL);
447 			assert(size >= 0);
448 
449 			JSON_GET_INT_OR_RETURN(JSON_FILE_OFFSET_KEY, obj,
450 					       &file_offset, NULL);
451 			assert(file_offset >= 0);
452 			assert((uint64_t)file_offset + size <=
453 			    rstate->kdata_len);
454 
455 			*struct_size = (size_t)size;
456 			return ((uint8_t *)rstate->kdata_map + file_offset);
457 		}
458 	}
459 
460 	return (NULL);
461 }
462 
463 static void *
464 lookup_check_dev(const char *dev_name, struct restore_state *rstate,
465 		 const ucl_object_t *obj, size_t *data_size)
466 {
467 	const char *snapshot_req;
468 	int64_t size, file_offset;
469 
470 	snapshot_req = NULL;
471 	JSON_GET_STRING_OR_RETURN(JSON_SNAPSHOT_REQ_KEY, obj,
472 				  &snapshot_req, NULL);
473 	assert(snapshot_req != NULL);
474 	if (!strcmp(snapshot_req, dev_name)) {
475 		JSON_GET_INT_OR_RETURN(JSON_SIZE_KEY, obj,
476 				       &size, NULL);
477 		assert(size >= 0);
478 
479 		JSON_GET_INT_OR_RETURN(JSON_FILE_OFFSET_KEY, obj,
480 				       &file_offset, NULL);
481 		assert(file_offset >= 0);
482 		assert((uint64_t)file_offset + size <= rstate->kdata_len);
483 
484 		*data_size = (size_t)size;
485 		return ((uint8_t *)rstate->kdata_map + file_offset);
486 	}
487 
488 	return (NULL);
489 }
490 
491 static void*
492 lookup_dev(const char *dev_name, struct restore_state *rstate,
493 	   size_t *data_size)
494 {
495 	const ucl_object_t *devs = NULL, *obj = NULL;
496 	ucl_object_iter_t it = NULL;
497 	void *ret;
498 
499 	devs = ucl_object_lookup(rstate->meta_root_obj, JSON_DEV_ARR_KEY);
500 	if (devs == NULL) {
501 		fprintf(stderr, "Failed to find '%s' object.\n",
502 			JSON_DEV_ARR_KEY);
503 		return (NULL);
504 	}
505 
506 	if (ucl_object_type(devs) != UCL_ARRAY) {
507 		fprintf(stderr, "Object '%s' is not an array.\n",
508 			JSON_DEV_ARR_KEY);
509 		return (NULL);
510 	}
511 
512 	while ((obj = ucl_object_iterate(devs, &it, true)) != NULL) {
513 		ret = lookup_check_dev(dev_name, rstate, obj, data_size);
514 		if (ret != NULL)
515 			return (ret);
516 	}
517 
518 	return (NULL);
519 }
520 
521 static const ucl_object_t *
522 lookup_basic_metadata_object(struct restore_state *rstate)
523 {
524 	const ucl_object_t *basic_meta_obj = NULL;
525 
526 	basic_meta_obj = ucl_object_lookup(rstate->meta_root_obj,
527 					   JSON_BASIC_METADATA_KEY);
528 	if (basic_meta_obj == NULL) {
529 		fprintf(stderr, "Failed to find '%s' object.\n",
530 			JSON_BASIC_METADATA_KEY);
531 		return (NULL);
532 	}
533 
534 	if (ucl_object_type(basic_meta_obj) != UCL_OBJECT) {
535 		fprintf(stderr, "Object '%s' is not a JSON object.\n",
536 		JSON_BASIC_METADATA_KEY);
537 		return (NULL);
538 	}
539 
540 	return (basic_meta_obj);
541 }
542 
543 const char *
544 lookup_vmname(struct restore_state *rstate)
545 {
546 	const char *vmname;
547 	const ucl_object_t *obj;
548 
549 	obj = lookup_basic_metadata_object(rstate);
550 	if (obj == NULL)
551 		return (NULL);
552 
553 	JSON_GET_STRING_OR_RETURN(JSON_VMNAME_KEY, obj, &vmname, NULL);
554 	return (vmname);
555 }
556 
557 int
558 lookup_memflags(struct restore_state *rstate)
559 {
560 	int64_t memflags;
561 	const ucl_object_t *obj;
562 
563 	obj = lookup_basic_metadata_object(rstate);
564 	if (obj == NULL)
565 		return (0);
566 
567 	JSON_GET_INT_OR_RETURN(JSON_MEMFLAGS_KEY, obj, &memflags, 0);
568 
569 	return ((int)memflags);
570 }
571 
572 size_t
573 lookup_memsize(struct restore_state *rstate)
574 {
575 	int64_t memsize;
576 	const ucl_object_t *obj;
577 
578 	obj = lookup_basic_metadata_object(rstate);
579 	if (obj == NULL)
580 		return (0);
581 
582 	JSON_GET_INT_OR_RETURN(JSON_MEMSIZE_KEY, obj, &memsize, 0);
583 	if (memsize < 0)
584 		memsize = 0;
585 
586 	return ((size_t)memsize);
587 }
588 
589 
590 int
591 lookup_guest_ncpus(struct restore_state *rstate)
592 {
593 	int64_t ncpus;
594 	const ucl_object_t *obj;
595 
596 	obj = lookup_basic_metadata_object(rstate);
597 	if (obj == NULL)
598 		return (0);
599 
600 	JSON_GET_INT_OR_RETURN(JSON_NCPUS_KEY, obj, &ncpus, 0);
601 	return ((int)ncpus);
602 }
603 
604 static void
605 winch_handler(int signal __unused)
606 {
607 #ifdef TIOCGWINSZ
608 	ioctl(STDOUT_FILENO, TIOCGWINSZ, &winsize);
609 #endif /* TIOCGWINSZ */
610 }
611 
612 static int
613 print_progress(size_t crtval, const size_t maxval)
614 {
615 	size_t rc;
616 	double crtval_gb, maxval_gb;
617 	size_t i, win_width, prog_start, prog_done, prog_end;
618 	int mval_len;
619 
620 	static char prog_buf[PROG_BUF_SZ];
621 	static const size_t len = sizeof(prog_buf);
622 
623 	static size_t div;
624 	static const char *div_str;
625 
626 	static char wip_bar[] = { '/', '-', '\\', '|' };
627 	static int wip_idx = 0;
628 
629 	if (maxval == 0) {
630 		printf("[0B / 0B]\r\n");
631 		return (0);
632 	}
633 
634 	if (crtval > maxval)
635 		crtval = maxval;
636 
637 	if (maxval > 10 * GB) {
638 		div = GB;
639 		div_str = "GiB";
640 	} else if (maxval > 10 * MB) {
641 		div = MB;
642 		div_str = "MiB";
643 	} else {
644 		div = KB;
645 		div_str = "KiB";
646 	}
647 
648 	crtval_gb = (double) crtval / div;
649 	maxval_gb = (double) maxval / div;
650 
651 	rc = snprintf(prog_buf, len, "%.03lf", maxval_gb);
652 	if (rc == len) {
653 		fprintf(stderr, "Maxval too big\n");
654 		return (-1);
655 	}
656 	mval_len = rc;
657 
658 	rc = snprintf(prog_buf, len, "\r[%*.03lf%s / %.03lf%s] |",
659 		mval_len, crtval_gb, div_str, maxval_gb, div_str);
660 
661 	if (rc == len) {
662 		fprintf(stderr, "Buffer too small to print progress\n");
663 		return (-1);
664 	}
665 
666 	win_width = min(winsize.ws_col, len);
667 	prog_start = rc;
668 
669 	if (prog_start < (win_width - 2)) {
670 		prog_end = win_width - prog_start - 2;
671 		prog_done = prog_end * (crtval_gb / maxval_gb);
672 
673 		for (i = prog_start; i < prog_start + prog_done; i++)
674 			prog_buf[i] = '#';
675 
676 		if (crtval != maxval) {
677 			prog_buf[i] = wip_bar[wip_idx];
678 			wip_idx = (wip_idx + 1) % sizeof(wip_bar);
679 			i++;
680 		} else {
681 			prog_buf[i++] = '#';
682 		}
683 
684 		for (; i < win_width - 2; i++)
685 			prog_buf[i] = '_';
686 
687 		prog_buf[win_width - 2] = '|';
688 	}
689 
690 	prog_buf[win_width - 1] = '\0';
691 	write(STDOUT_FILENO, prog_buf, win_width);
692 
693 	return (0);
694 }
695 
696 static void *
697 snapshot_spinner_cb(void *arg)
698 {
699 	int rc;
700 	size_t crtval, maxval, total;
701 	struct spinner_info *si;
702 	struct timespec ts;
703 
704 	si = arg;
705 	if (si == NULL)
706 		pthread_exit(NULL);
707 
708 	ts.tv_sec = 0;
709 	ts.tv_nsec = 50 * 1000 * 1000; /* 50 ms sleep time */
710 
711 	do {
712 		crtval = *si->crtval;
713 		maxval = si->maxval;
714 		total = si->total;
715 
716 		rc = print_progress(crtval, total);
717 		if (rc < 0) {
718 			fprintf(stderr, "Failed to parse progress\n");
719 			break;
720 		}
721 
722 		nanosleep(&ts, NULL);
723 	} while (crtval < maxval);
724 
725 	pthread_exit(NULL);
726 	return NULL;
727 }
728 
729 static int
730 vm_snapshot_mem_part(const int snapfd, const size_t foff, void *src,
731 		     const size_t len, const size_t totalmem, const bool op_wr)
732 {
733 	int rc;
734 	size_t part_done, todo, rem;
735 	ssize_t done;
736 	bool show_progress;
737 	pthread_t spinner_th;
738 	struct spinner_info *si;
739 
740 	if (lseek(snapfd, foff, SEEK_SET) < 0) {
741 		perror("Failed to change file offset");
742 		return (-1);
743 	}
744 
745 	show_progress = false;
746 	if (isatty(STDIN_FILENO) && (winsize.ws_col != 0))
747 		show_progress = true;
748 
749 	part_done = foff;
750 	rem = len;
751 
752 	if (show_progress) {
753 		si = &(struct spinner_info) {
754 			.crtval = &part_done,
755 			.maxval = foff + len,
756 			.total = totalmem
757 		};
758 
759 		rc = pthread_create(&spinner_th, 0, snapshot_spinner_cb, si);
760 		if (rc) {
761 			perror("Unable to create spinner thread");
762 			show_progress = false;
763 		}
764 	}
765 
766 	while (rem > 0) {
767 		if (show_progress)
768 			todo = min(SNAPSHOT_CHUNK, rem);
769 		else
770 			todo = rem;
771 
772 		if (op_wr)
773 			done = write(snapfd, src, todo);
774 		else
775 			done = read(snapfd, src, todo);
776 		if (done < 0) {
777 			perror("Failed to write in file");
778 			return (-1);
779 		}
780 
781 		src = (uint8_t *)src + done;
782 		part_done += done;
783 		rem -= done;
784 	}
785 
786 	if (show_progress) {
787 		rc = pthread_join(spinner_th, NULL);
788 		if (rc)
789 			perror("Unable to end spinner thread");
790 	}
791 
792 	return (0);
793 }
794 
795 static size_t
796 vm_snapshot_mem(struct vmctx *ctx, int snapfd, size_t memsz, const bool op_wr)
797 {
798 	int ret;
799 	size_t lowmem, highmem, totalmem;
800 	char *baseaddr;
801 
802 	ret = vm_get_guestmem_from_ctx(ctx, &baseaddr, &lowmem, &highmem);
803 	if (ret) {
804 		fprintf(stderr, "%s: unable to retrieve guest memory size\r\n",
805 			__func__);
806 		return (0);
807 	}
808 	totalmem = lowmem + highmem;
809 
810 	if ((op_wr == false) && (totalmem != memsz)) {
811 		fprintf(stderr, "%s: mem size mismatch: %ld vs %ld\r\n",
812 			__func__, totalmem, memsz);
813 		return (0);
814 	}
815 
816 	winsize.ws_col = 80;
817 #ifdef TIOCGWINSZ
818 	ioctl(STDOUT_FILENO, TIOCGWINSZ, &winsize);
819 #endif /* TIOCGWINSZ */
820 	old_winch_handler = signal(SIGWINCH, winch_handler);
821 
822 	ret = vm_snapshot_mem_part(snapfd, 0, baseaddr, lowmem,
823 		totalmem, op_wr);
824 	if (ret) {
825 		fprintf(stderr, "%s: Could not %s lowmem\r\n",
826 			__func__, op_wr ? "write" : "read");
827 		totalmem = 0;
828 		goto done;
829 	}
830 
831 	if (highmem == 0)
832 		goto done;
833 
834 	ret = vm_snapshot_mem_part(snapfd, lowmem, baseaddr + 4*GB,
835 		highmem, totalmem, op_wr);
836 	if (ret) {
837 		fprintf(stderr, "%s: Could not %s highmem\r\n",
838 		        __func__, op_wr ? "write" : "read");
839 		totalmem = 0;
840 		goto done;
841 	}
842 
843 done:
844 	printf("\r\n");
845 	signal(SIGWINCH, old_winch_handler);
846 
847 	return (totalmem);
848 }
849 
850 int
851 restore_vm_mem(struct vmctx *ctx, struct restore_state *rstate)
852 {
853 	size_t restored;
854 
855 	restored = vm_snapshot_mem(ctx, rstate->vmmem_fd, rstate->vmmem_len,
856 				   false);
857 
858 	if (restored != rstate->vmmem_len)
859 		return (-1);
860 
861 	return (0);
862 }
863 
864 static int
865 vm_restore_kern_struct(struct vmctx *ctx, struct restore_state *rstate,
866 		       const struct vm_snapshot_kern_info *info)
867 {
868 	void *struct_ptr;
869 	size_t struct_size;
870 	int ret;
871 	struct vm_snapshot_meta *meta;
872 
873 	struct_ptr = lookup_struct(info->req, rstate, &struct_size);
874 	if (struct_ptr == NULL) {
875 		fprintf(stderr, "%s: Failed to lookup struct %s\r\n",
876 			__func__, info->struct_name);
877 		ret = -1;
878 		goto done;
879 	}
880 
881 	if (struct_size == 0) {
882 		fprintf(stderr, "%s: Kernel struct size was 0 for: %s\r\n",
883 			__func__, info->struct_name);
884 		ret = -1;
885 		goto done;
886 	}
887 
888 	meta = &(struct vm_snapshot_meta) {
889 		.ctx = ctx,
890 		.dev_name = info->struct_name,
891 		.dev_req  = info->req,
892 
893 		.buffer.buf_start = struct_ptr,
894 		.buffer.buf_size = struct_size,
895 
896 		.buffer.buf = struct_ptr,
897 		.buffer.buf_rem = struct_size,
898 
899 		.op = VM_SNAPSHOT_RESTORE,
900 	};
901 
902 	ret = vm_snapshot_req(meta);
903 	if (ret != 0) {
904 		fprintf(stderr, "%s: Failed to restore struct: %s\r\n",
905 			__func__, info->struct_name);
906 		goto done;
907 	}
908 
909 done:
910 	return (ret);
911 }
912 
913 int
914 vm_restore_kern_structs(struct vmctx *ctx, struct restore_state *rstate)
915 {
916 	size_t i;
917 	int ret;
918 
919 	for (i = 0; i < nitems(snapshot_kern_structs); i++) {
920 		ret = vm_restore_kern_struct(ctx, rstate,
921 					     &snapshot_kern_structs[i]);
922 		if (ret != 0)
923 			return (ret);
924 	}
925 
926 	return (0);
927 }
928 
929 static int
930 vm_restore_user_dev(struct vmctx *ctx, struct restore_state *rstate,
931 		    const struct vm_snapshot_dev_info *info)
932 {
933 	void *dev_ptr;
934 	size_t dev_size;
935 	int ret;
936 	struct vm_snapshot_meta *meta;
937 
938 	dev_ptr = lookup_dev(info->dev_name, rstate, &dev_size);
939 	if (dev_ptr == NULL) {
940 		fprintf(stderr, "Failed to lookup dev: %s\r\n", info->dev_name);
941 		fprintf(stderr, "Continuing the restore/migration process\r\n");
942 		return (0);
943 	}
944 
945 	if (dev_size == 0) {
946 		fprintf(stderr, "%s: Device size is 0. "
947 			"Assuming %s is not used\r\n",
948 			__func__, info->dev_name);
949 		return (0);
950 	}
951 
952 	meta = &(struct vm_snapshot_meta) {
953 		.ctx = ctx,
954 		.dev_name = info->dev_name,
955 
956 		.buffer.buf_start = dev_ptr,
957 		.buffer.buf_size = dev_size,
958 
959 		.buffer.buf = dev_ptr,
960 		.buffer.buf_rem = dev_size,
961 
962 		.op = VM_SNAPSHOT_RESTORE,
963 	};
964 
965 	ret = (*info->snapshot_cb)(meta);
966 	if (ret != 0) {
967 		fprintf(stderr, "Failed to restore dev: %s\r\n",
968 			info->dev_name);
969 		return (-1);
970 	}
971 
972 	return (0);
973 }
974 
975 
976 int
977 vm_restore_user_devs(struct vmctx *ctx, struct restore_state *rstate)
978 {
979 	size_t i;
980 	int ret;
981 
982 	for (i = 0; i < nitems(snapshot_devs); i++) {
983 		ret = vm_restore_user_dev(ctx, rstate, &snapshot_devs[i]);
984 		if (ret != 0)
985 			return (ret);
986 	}
987 
988 	return 0;
989 }
990 
991 int
992 vm_pause_user_devs(void)
993 {
994 	const struct vm_snapshot_dev_info *info;
995 	size_t i;
996 	int ret;
997 
998 	for (i = 0; i < nitems(snapshot_devs); i++) {
999 		info = &snapshot_devs[i];
1000 		if (info->pause_cb == NULL)
1001 			continue;
1002 
1003 		ret = info->pause_cb(info->dev_name);
1004 		if (ret != 0)
1005 			return (ret);
1006 	}
1007 
1008 	return (0);
1009 }
1010 
1011 int
1012 vm_resume_user_devs(void)
1013 {
1014 	const struct vm_snapshot_dev_info *info;
1015 	size_t i;
1016 	int ret;
1017 
1018 	for (i = 0; i < nitems(snapshot_devs); i++) {
1019 		info = &snapshot_devs[i];
1020 		if (info->resume_cb == NULL)
1021 			continue;
1022 
1023 		ret = info->resume_cb(info->dev_name);
1024 		if (ret != 0)
1025 			return (ret);
1026 	}
1027 
1028 	return (0);
1029 }
1030 
1031 static int
1032 vm_snapshot_kern_struct(int data_fd, xo_handle_t *xop, const char *array_key,
1033 			struct vm_snapshot_meta *meta, off_t *offset)
1034 {
1035 	int ret;
1036 	size_t data_size;
1037 	ssize_t write_cnt;
1038 
1039 	ret = vm_snapshot_req(meta);
1040 	if (ret != 0) {
1041 		fprintf(stderr, "%s: Failed to snapshot struct %s\r\n",
1042 			__func__, meta->dev_name);
1043 		ret = -1;
1044 		goto done;
1045 	}
1046 
1047 	data_size = vm_get_snapshot_size(meta);
1048 
1049 	/* XXX-MJ no handling for short writes. */
1050 	write_cnt = write(data_fd, meta->buffer.buf_start, data_size);
1051 	if (write_cnt < 0 || (size_t)write_cnt != data_size) {
1052 		perror("Failed to write all snapshotted data.");
1053 		ret = -1;
1054 		goto done;
1055 	}
1056 
1057 	/* Write metadata. */
1058 	xo_open_instance_h(xop, array_key);
1059 	xo_emit_h(xop, "{:debug_name/%s}\n", meta->dev_name);
1060 	xo_emit_h(xop, "{:" JSON_SNAPSHOT_REQ_KEY "/%d}\n",
1061 		  meta->dev_req);
1062 	xo_emit_h(xop, "{:" JSON_SIZE_KEY "/%lu}\n", data_size);
1063 	xo_emit_h(xop, "{:" JSON_FILE_OFFSET_KEY "/%lu}\n", *offset);
1064 	xo_close_instance_h(xop, JSON_STRUCT_ARR_KEY);
1065 
1066 	*offset += data_size;
1067 
1068 done:
1069 	return (ret);
1070 }
1071 
1072 static int
1073 vm_snapshot_kern_structs(struct vmctx *ctx, int data_fd, xo_handle_t *xop)
1074 {
1075 	int ret, error;
1076 	size_t buf_size, i, offset;
1077 	char *buffer;
1078 	struct vm_snapshot_meta *meta;
1079 
1080 	error = 0;
1081 	offset = 0;
1082 	buf_size = SNAPSHOT_BUFFER_SIZE;
1083 
1084 	buffer = malloc(SNAPSHOT_BUFFER_SIZE * sizeof(char));
1085 	if (buffer == NULL) {
1086 		error = ENOMEM;
1087 		perror("Failed to allocate memory for snapshot buffer");
1088 		goto err_vm_snapshot_kern_data;
1089 	}
1090 
1091 	meta = &(struct vm_snapshot_meta) {
1092 		.ctx = ctx,
1093 
1094 		.buffer.buf_start = buffer,
1095 		.buffer.buf_size = buf_size,
1096 
1097 		.op = VM_SNAPSHOT_SAVE,
1098 	};
1099 
1100 	xo_open_list_h(xop, JSON_STRUCT_ARR_KEY);
1101 	for (i = 0; i < nitems(snapshot_kern_structs); i++) {
1102 		meta->dev_name = snapshot_kern_structs[i].struct_name;
1103 		meta->dev_req  = snapshot_kern_structs[i].req;
1104 
1105 		memset(meta->buffer.buf_start, 0, meta->buffer.buf_size);
1106 		meta->buffer.buf = meta->buffer.buf_start;
1107 		meta->buffer.buf_rem = meta->buffer.buf_size;
1108 
1109 		ret = vm_snapshot_kern_struct(data_fd, xop, JSON_DEV_ARR_KEY,
1110 					      meta, &offset);
1111 		if (ret != 0) {
1112 			error = -1;
1113 			goto err_vm_snapshot_kern_data;
1114 		}
1115 	}
1116 	xo_close_list_h(xop, JSON_STRUCT_ARR_KEY);
1117 
1118 err_vm_snapshot_kern_data:
1119 	if (buffer != NULL)
1120 		free(buffer);
1121 	return (error);
1122 }
1123 
1124 static int
1125 vm_snapshot_basic_metadata(struct vmctx *ctx, xo_handle_t *xop, size_t memsz)
1126 {
1127 
1128 	xo_open_container_h(xop, JSON_BASIC_METADATA_KEY);
1129 	xo_emit_h(xop, "{:" JSON_NCPUS_KEY "/%ld}\n", guest_ncpus);
1130 	xo_emit_h(xop, "{:" JSON_VMNAME_KEY "/%s}\n", vm_get_name(ctx));
1131 	xo_emit_h(xop, "{:" JSON_MEMSIZE_KEY "/%lu}\n", memsz);
1132 	xo_emit_h(xop, "{:" JSON_MEMFLAGS_KEY "/%d}\n", vm_get_memflags(ctx));
1133 	xo_close_container_h(xop, JSON_BASIC_METADATA_KEY);
1134 
1135 	return (0);
1136 }
1137 
1138 static int
1139 vm_snapshot_dev_write_data(int data_fd, xo_handle_t *xop, const char *array_key,
1140 			   struct vm_snapshot_meta *meta, off_t *offset)
1141 {
1142 	ssize_t ret;
1143 	size_t data_size;
1144 
1145 	data_size = vm_get_snapshot_size(meta);
1146 
1147 	/* XXX-MJ no handling for short writes. */
1148 	ret = write(data_fd, meta->buffer.buf_start, data_size);
1149 	if (ret < 0 || (size_t)ret != data_size) {
1150 		perror("Failed to write all snapshotted data.");
1151 		return (-1);
1152 	}
1153 
1154 	/* Write metadata. */
1155 	xo_open_instance_h(xop, array_key);
1156 	xo_emit_h(xop, "{:" JSON_SNAPSHOT_REQ_KEY "/%s}\n", meta->dev_name);
1157 	xo_emit_h(xop, "{:" JSON_SIZE_KEY "/%lu}\n", data_size);
1158 	xo_emit_h(xop, "{:" JSON_FILE_OFFSET_KEY "/%lu}\n", *offset);
1159 	xo_close_instance_h(xop, array_key);
1160 
1161 	*offset += data_size;
1162 
1163 	return (0);
1164 }
1165 
1166 static int
1167 vm_snapshot_user_dev(const struct vm_snapshot_dev_info *info,
1168 		     int data_fd, xo_handle_t *xop,
1169 		     struct vm_snapshot_meta *meta, off_t *offset)
1170 {
1171 	int ret;
1172 
1173 	ret = (*info->snapshot_cb)(meta);
1174 	if (ret != 0) {
1175 		fprintf(stderr, "Failed to snapshot %s; ret=%d\r\n",
1176 			meta->dev_name, ret);
1177 		return (ret);
1178 	}
1179 
1180 	ret = vm_snapshot_dev_write_data(data_fd, xop, JSON_DEV_ARR_KEY, meta,
1181 					 offset);
1182 	if (ret != 0)
1183 		return (ret);
1184 
1185 	return (0);
1186 }
1187 
1188 static int
1189 vm_snapshot_user_devs(struct vmctx *ctx, int data_fd, xo_handle_t *xop)
1190 {
1191 	int ret;
1192 	off_t offset;
1193 	void *buffer;
1194 	size_t buf_size, i;
1195 	struct vm_snapshot_meta *meta;
1196 
1197 	buf_size = SNAPSHOT_BUFFER_SIZE;
1198 
1199 	offset = lseek(data_fd, 0, SEEK_CUR);
1200 	if (offset < 0) {
1201 		perror("Failed to get data file current offset.");
1202 		return (-1);
1203 	}
1204 
1205 	buffer = malloc(buf_size);
1206 	if (buffer == NULL) {
1207 		perror("Failed to allocate memory for snapshot buffer");
1208 		ret = ENOSPC;
1209 		goto snapshot_err;
1210 	}
1211 
1212 	meta = &(struct vm_snapshot_meta) {
1213 		.ctx = ctx,
1214 
1215 		.buffer.buf_start = buffer,
1216 		.buffer.buf_size = buf_size,
1217 
1218 		.op = VM_SNAPSHOT_SAVE,
1219 	};
1220 
1221 	xo_open_list_h(xop, JSON_DEV_ARR_KEY);
1222 
1223 	/* Restore other devices that support this feature */
1224 	for (i = 0; i < nitems(snapshot_devs); i++) {
1225 		meta->dev_name = snapshot_devs[i].dev_name;
1226 
1227 		memset(meta->buffer.buf_start, 0, meta->buffer.buf_size);
1228 		meta->buffer.buf = meta->buffer.buf_start;
1229 		meta->buffer.buf_rem = meta->buffer.buf_size;
1230 
1231 		ret = vm_snapshot_user_dev(&snapshot_devs[i], data_fd, xop,
1232 					   meta, &offset);
1233 		if (ret != 0)
1234 			goto snapshot_err;
1235 	}
1236 
1237 	xo_close_list_h(xop, JSON_DEV_ARR_KEY);
1238 
1239 snapshot_err:
1240 	if (buffer != NULL)
1241 		free(buffer);
1242 	return (ret);
1243 }
1244 
1245 void
1246 checkpoint_cpu_add(int vcpu)
1247 {
1248 
1249 	pthread_mutex_lock(&vcpu_lock);
1250 	CPU_SET(vcpu, &vcpus_active);
1251 
1252 	if (checkpoint_active) {
1253 		CPU_SET(vcpu, &vcpus_suspended);
1254 		while (checkpoint_active)
1255 			pthread_cond_wait(&vcpus_can_run, &vcpu_lock);
1256 		CPU_CLR(vcpu, &vcpus_suspended);
1257 	}
1258 	pthread_mutex_unlock(&vcpu_lock);
1259 }
1260 
1261 /*
1262  * When a vCPU is suspended for any reason, it calls
1263  * checkpoint_cpu_suspend().  This records that the vCPU is idle.
1264  * Before returning from suspension, checkpoint_cpu_resume() is
1265  * called.  In suspend we note that the vCPU is idle.  In resume we
1266  * pause the vCPU thread until the checkpoint is complete.  The reason
1267  * for the two-step process is that vCPUs might already be stopped in
1268  * the debug server when a checkpoint is requested.  This approach
1269  * allows us to account for and handle those vCPUs.
1270  */
1271 void
1272 checkpoint_cpu_suspend(int vcpu)
1273 {
1274 
1275 	pthread_mutex_lock(&vcpu_lock);
1276 	CPU_SET(vcpu, &vcpus_suspended);
1277 	if (checkpoint_active && CPU_CMP(&vcpus_active, &vcpus_suspended) == 0)
1278 		pthread_cond_signal(&vcpus_idle);
1279 	pthread_mutex_unlock(&vcpu_lock);
1280 }
1281 
1282 void
1283 checkpoint_cpu_resume(int vcpu)
1284 {
1285 
1286 	pthread_mutex_lock(&vcpu_lock);
1287 	while (checkpoint_active)
1288 		pthread_cond_wait(&vcpus_can_run, &vcpu_lock);
1289 	CPU_CLR(vcpu, &vcpus_suspended);
1290 	pthread_mutex_unlock(&vcpu_lock);
1291 }
1292 
1293 static void
1294 vm_vcpu_pause(struct vmctx *ctx)
1295 {
1296 
1297 	pthread_mutex_lock(&vcpu_lock);
1298 	checkpoint_active = true;
1299 	vm_suspend_cpu(ctx, -1);
1300 	while (CPU_CMP(&vcpus_active, &vcpus_suspended) != 0)
1301 		pthread_cond_wait(&vcpus_idle, &vcpu_lock);
1302 	pthread_mutex_unlock(&vcpu_lock);
1303 }
1304 
1305 static void
1306 vm_vcpu_resume(struct vmctx *ctx)
1307 {
1308 
1309 	pthread_mutex_lock(&vcpu_lock);
1310 	checkpoint_active = false;
1311 	pthread_mutex_unlock(&vcpu_lock);
1312 	vm_resume_cpu(ctx, -1);
1313 	pthread_cond_broadcast(&vcpus_can_run);
1314 }
1315 
1316 static int
1317 vm_checkpoint(struct vmctx *ctx, const char *checkpoint_file, bool stop_vm)
1318 {
1319 	int fd_checkpoint = 0, kdata_fd = 0;
1320 	int ret = 0;
1321 	int error = 0;
1322 	size_t memsz;
1323 	xo_handle_t *xop = NULL;
1324 	char *meta_filename = NULL;
1325 	char *kdata_filename = NULL;
1326 	FILE *meta_file = NULL;
1327 
1328 	kdata_filename = strcat_extension(checkpoint_file, ".kern");
1329 	if (kdata_filename == NULL) {
1330 		fprintf(stderr, "Failed to construct kernel data filename.\n");
1331 		return (-1);
1332 	}
1333 
1334 	kdata_fd = open(kdata_filename, O_WRONLY | O_CREAT | O_TRUNC, 0700);
1335 	if (kdata_fd < 0) {
1336 		perror("Failed to open kernel data snapshot file.");
1337 		error = -1;
1338 		goto done;
1339 	}
1340 
1341 	fd_checkpoint = open(checkpoint_file, O_RDWR | O_CREAT | O_TRUNC, 0700);
1342 
1343 	if (fd_checkpoint < 0) {
1344 		perror("Failed to create checkpoint file");
1345 		error = -1;
1346 		goto done;
1347 	}
1348 
1349 	meta_filename = strcat_extension(checkpoint_file, ".meta");
1350 	if (meta_filename == NULL) {
1351 		fprintf(stderr, "Failed to construct vm metadata filename.\n");
1352 		goto done;
1353 	}
1354 
1355 	meta_file = fopen(meta_filename, "w");
1356 	if (meta_file == NULL) {
1357 		perror("Failed to open vm metadata snapshot file.");
1358 		goto done;
1359 	}
1360 
1361 	xop = xo_create_to_file(meta_file, XO_STYLE_JSON, XOF_PRETTY);
1362 	if (xop == NULL) {
1363 		perror("Failed to get libxo handle on metadata file.");
1364 		goto done;
1365 	}
1366 
1367 	vm_vcpu_pause(ctx);
1368 
1369 	ret = vm_pause_user_devs();
1370 	if (ret != 0) {
1371 		fprintf(stderr, "Could not pause devices\r\n");
1372 		error = ret;
1373 		goto done;
1374 	}
1375 
1376 	memsz = vm_snapshot_mem(ctx, fd_checkpoint, 0, true);
1377 	if (memsz == 0) {
1378 		perror("Could not write guest memory to file");
1379 		error = -1;
1380 		goto done;
1381 	}
1382 
1383 	ret = vm_snapshot_basic_metadata(ctx, xop, memsz);
1384 	if (ret != 0) {
1385 		fprintf(stderr, "Failed to snapshot vm basic metadata.\n");
1386 		error = -1;
1387 		goto done;
1388 	}
1389 
1390 
1391 	ret = vm_snapshot_kern_structs(ctx, kdata_fd, xop);
1392 	if (ret != 0) {
1393 		fprintf(stderr, "Failed to snapshot vm kernel data.\n");
1394 		error = -1;
1395 		goto done;
1396 	}
1397 
1398 	ret = vm_snapshot_user_devs(ctx, kdata_fd, xop);
1399 	if (ret != 0) {
1400 		fprintf(stderr, "Failed to snapshot device state.\n");
1401 		error = -1;
1402 		goto done;
1403 	}
1404 
1405 	xo_finish_h(xop);
1406 
1407 	if (stop_vm) {
1408 		vm_destroy(ctx);
1409 		exit(0);
1410 	}
1411 
1412 done:
1413 	ret = vm_resume_user_devs();
1414 	if (ret != 0)
1415 		fprintf(stderr, "Could not resume devices\r\n");
1416 	vm_vcpu_resume(ctx);
1417 	if (fd_checkpoint > 0)
1418 		close(fd_checkpoint);
1419 	if (meta_filename != NULL)
1420 		free(meta_filename);
1421 	if (kdata_filename != NULL)
1422 		free(kdata_filename);
1423 	if (xop != NULL)
1424 		xo_destroy(xop);
1425 	if (meta_file != NULL)
1426 		fclose(meta_file);
1427 	if (kdata_fd > 0)
1428 		close(kdata_fd);
1429 	return (error);
1430 }
1431 
1432 static int
1433 handle_message(struct vmctx *ctx, nvlist_t *nvl)
1434 {
1435 	const char *cmd;
1436 	struct ipc_command **ipc_cmd;
1437 
1438 	if (!nvlist_exists_string(nvl, "cmd"))
1439 		return (EINVAL);
1440 
1441 	cmd = nvlist_get_string(nvl, "cmd");
1442 	IPC_COMMAND_FOREACH(ipc_cmd, ipc_cmd_set) {
1443 		if (strcmp(cmd, (*ipc_cmd)->name) == 0)
1444 			return ((*ipc_cmd)->handler(ctx, nvl));
1445 	}
1446 
1447 	return (EOPNOTSUPP);
1448 }
1449 
1450 /*
1451  * Listen for commands from bhyvectl
1452  */
1453 void *
1454 checkpoint_thread(void *param)
1455 {
1456 	int fd;
1457 	struct checkpoint_thread_info *thread_info;
1458 	nvlist_t *nvl;
1459 
1460 	pthread_set_name_np(pthread_self(), "checkpoint thread");
1461 	thread_info = (struct checkpoint_thread_info *)param;
1462 
1463 	while ((fd = accept(thread_info->socket_fd, NULL, NULL)) != -1) {
1464 		nvl = nvlist_recv(fd, 0);
1465 		if (nvl != NULL)
1466 			handle_message(thread_info->ctx, nvl);
1467 		else
1468 			EPRINTLN("nvlist_recv() failed: %s", strerror(errno));
1469 
1470 		close(fd);
1471 		nvlist_destroy(nvl);
1472 	}
1473 
1474 	return (NULL);
1475 }
1476 
1477 static int
1478 vm_do_checkpoint(struct vmctx *ctx, const nvlist_t *nvl)
1479 {
1480 	int error;
1481 
1482 	if (!nvlist_exists_string(nvl, "filename") ||
1483 	    !nvlist_exists_bool(nvl, "suspend"))
1484 		error = EINVAL;
1485 	else
1486 		error = vm_checkpoint(ctx, nvlist_get_string(nvl, "filename"),
1487 		    nvlist_get_bool(nvl, "suspend"));
1488 
1489 	return (error);
1490 }
1491 IPC_COMMAND(ipc_cmd_set, checkpoint, vm_do_checkpoint);
1492 
1493 void
1494 init_snapshot(void)
1495 {
1496 	int err;
1497 
1498 	err = pthread_mutex_init(&vcpu_lock, NULL);
1499 	if (err != 0)
1500 		errc(1, err, "checkpoint mutex init");
1501 	err = pthread_cond_init(&vcpus_idle, NULL);
1502 	if (err != 0)
1503 		errc(1, err, "checkpoint cv init (vcpus_idle)");
1504 	err = pthread_cond_init(&vcpus_can_run, NULL);
1505 	if (err != 0)
1506 		errc(1, err, "checkpoint cv init (vcpus_can_run)");
1507 }
1508 
1509 /*
1510  * Create the listening socket for IPC with bhyvectl
1511  */
1512 int
1513 init_checkpoint_thread(struct vmctx *ctx)
1514 {
1515 	struct checkpoint_thread_info *checkpoint_info = NULL;
1516 	struct sockaddr_un addr;
1517 	int socket_fd;
1518 	pthread_t checkpoint_pthread;
1519 	int err;
1520 #ifndef WITHOUT_CAPSICUM
1521 	cap_rights_t rights;
1522 #endif
1523 
1524 	memset(&addr, 0, sizeof(addr));
1525 
1526 	socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
1527 	if (socket_fd < 0) {
1528 		EPRINTLN("Socket creation failed: %s", strerror(errno));
1529 		err = -1;
1530 		goto fail;
1531 	}
1532 
1533 	addr.sun_family = AF_UNIX;
1534 
1535 	snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s",
1536 		 BHYVE_RUN_DIR, vm_get_name(ctx));
1537 	addr.sun_len = SUN_LEN(&addr);
1538 	unlink(addr.sun_path);
1539 
1540 	if (bind(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
1541 		EPRINTLN("Failed to bind socket \"%s\": %s\n",
1542 		    addr.sun_path, strerror(errno));
1543 		err = -1;
1544 		goto fail;
1545 	}
1546 
1547 	if (listen(socket_fd, 10) < 0) {
1548 		EPRINTLN("ipc socket listen: %s\n", strerror(errno));
1549 		err = errno;
1550 		goto fail;
1551 	}
1552 
1553 #ifndef WITHOUT_CAPSICUM
1554 	cap_rights_init(&rights, CAP_ACCEPT, CAP_READ, CAP_RECV, CAP_WRITE,
1555 	    CAP_SEND, CAP_GETSOCKOPT);
1556 
1557 	if (caph_rights_limit(socket_fd, &rights) == -1)
1558 		errx(EX_OSERR, "Unable to apply rights for sandbox");
1559 #endif
1560 	checkpoint_info = calloc(1, sizeof(*checkpoint_info));
1561 	checkpoint_info->ctx = ctx;
1562 	checkpoint_info->socket_fd = socket_fd;
1563 
1564 	err = pthread_create(&checkpoint_pthread, NULL, checkpoint_thread,
1565 		checkpoint_info);
1566 	if (err != 0)
1567 		goto fail;
1568 
1569 	return (0);
1570 fail:
1571 	free(checkpoint_info);
1572 	if (socket_fd > 0)
1573 		close(socket_fd);
1574 	unlink(addr.sun_path);
1575 
1576 	return (err);
1577 }
1578 
1579 void
1580 vm_snapshot_buf_err(const char *bufname, const enum vm_snapshot_op op)
1581 {
1582 	const char *__op;
1583 
1584 	if (op == VM_SNAPSHOT_SAVE)
1585 		__op = "save";
1586 	else if (op == VM_SNAPSHOT_RESTORE)
1587 		__op = "restore";
1588 	else
1589 		__op = "unknown";
1590 
1591 	fprintf(stderr, "%s: snapshot-%s failed for %s\r\n",
1592 		__func__, __op, bufname);
1593 }
1594 
1595 int
1596 vm_snapshot_buf(void *data, size_t data_size, struct vm_snapshot_meta *meta)
1597 {
1598 	struct vm_snapshot_buffer *buffer;
1599 	int op;
1600 
1601 	buffer = &meta->buffer;
1602 	op = meta->op;
1603 
1604 	if (buffer->buf_rem < data_size) {
1605 		fprintf(stderr, "%s: buffer too small\r\n", __func__);
1606 		return (E2BIG);
1607 	}
1608 
1609 	if (op == VM_SNAPSHOT_SAVE)
1610 		memcpy(buffer->buf, data, data_size);
1611 	else if (op == VM_SNAPSHOT_RESTORE)
1612 		memcpy(data, buffer->buf, data_size);
1613 	else
1614 		return (EINVAL);
1615 
1616 	buffer->buf += data_size;
1617 	buffer->buf_rem -= data_size;
1618 
1619 	return (0);
1620 }
1621 
1622 size_t
1623 vm_get_snapshot_size(struct vm_snapshot_meta *meta)
1624 {
1625 	size_t length;
1626 	struct vm_snapshot_buffer *buffer;
1627 
1628 	buffer = &meta->buffer;
1629 
1630 	if (buffer->buf_size < buffer->buf_rem) {
1631 		fprintf(stderr, "%s: Invalid buffer: size = %zu, rem = %zu\r\n",
1632 			__func__, buffer->buf_size, buffer->buf_rem);
1633 		length = 0;
1634 	} else {
1635 		length = buffer->buf_size - buffer->buf_rem;
1636 	}
1637 
1638 	return (length);
1639 }
1640 
1641 int
1642 vm_snapshot_guest2host_addr(void **addrp, size_t len, bool restore_null,
1643 			    struct vm_snapshot_meta *meta)
1644 {
1645 	int ret;
1646 	vm_paddr_t gaddr;
1647 
1648 	if (meta->op == VM_SNAPSHOT_SAVE) {
1649 		gaddr = paddr_host2guest(meta->ctx, *addrp);
1650 		if (gaddr == (vm_paddr_t) -1) {
1651 			if (!restore_null ||
1652 			    (restore_null && (*addrp != NULL))) {
1653 				ret = EFAULT;
1654 				goto done;
1655 			}
1656 		}
1657 
1658 		SNAPSHOT_VAR_OR_LEAVE(gaddr, meta, ret, done);
1659 	} else if (meta->op == VM_SNAPSHOT_RESTORE) {
1660 		SNAPSHOT_VAR_OR_LEAVE(gaddr, meta, ret, done);
1661 		if (gaddr == (vm_paddr_t) -1) {
1662 			if (!restore_null) {
1663 				ret = EFAULT;
1664 				goto done;
1665 			}
1666 		}
1667 
1668 		*addrp = paddr_guest2host(meta->ctx, gaddr, len);
1669 	} else {
1670 		ret = EINVAL;
1671 	}
1672 
1673 done:
1674 	return (ret);
1675 }
1676 
1677 int
1678 vm_snapshot_buf_cmp(void *data, size_t data_size, struct vm_snapshot_meta *meta)
1679 {
1680 	struct vm_snapshot_buffer *buffer;
1681 	int op;
1682 	int ret;
1683 
1684 	buffer = &meta->buffer;
1685 	op = meta->op;
1686 
1687 	if (buffer->buf_rem < data_size) {
1688 		fprintf(stderr, "%s: buffer too small\r\n", __func__);
1689 		ret = E2BIG;
1690 		goto done;
1691 	}
1692 
1693 	if (op == VM_SNAPSHOT_SAVE) {
1694 		ret = 0;
1695 		memcpy(buffer->buf, data, data_size);
1696 	} else if (op == VM_SNAPSHOT_RESTORE) {
1697 		ret = memcmp(data, buffer->buf, data_size);
1698 	} else {
1699 		ret = EINVAL;
1700 		goto done;
1701 	}
1702 
1703 	buffer->buf += data_size;
1704 	buffer->buf_rem -= data_size;
1705 
1706 done:
1707 	return (ret);
1708 }
1709