1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * Copyright (c) 2026 Hans Rosenfeld
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/param.h>
31 #include <sys/cpuset.h>
32 #include <sys/errno.h>
33 #include <sys/mman.h>
34 #include <sys/nv.h>
35 #include <sys/socket.h>
36 #include <sys/sysctl.h>
37 #include <sys/un.h>
38
39 #include <stdio.h>
40 #include <stdlib.h>
41 #include <stdbool.h>
42 #include <string.h>
43 #include <unistd.h>
44 #include <libgen.h>
45 #include <libutil.h>
46 #include <fcntl.h>
47 #include <getopt.h>
48 #include <libutil.h>
49
50 #include <machine/cpufunc.h>
51 #include <machine/vmm.h>
52 #include <machine/vmm_dev.h>
53 #include <vmmapi.h>
54
55 #include "ipc.h"
56 #ifdef BHYVE_SNAPSHOT
57 #include "snapshot.h"
58 #endif
59
60 #include "bhyvectl.h"
61
62 #define MB (1UL << 20)
63 #define GB (1UL << 30)
64
65 static const char *progname;
66
67 static int get_stats, getcap, setcap, capval;
68 static int force_reset, force_poweroff;
69 static const char *capname;
70 static int create, destroy, get_memmap, get_memseg;
71 static int get_active_cpus, get_debug_cpus, get_suspended_cpus;
72 static uint64_t memsize;
73 static int run;
74 static int get_cpu_topology;
75 #ifdef BHYVE_SNAPSHOT
76 static int vm_suspend_opt;
77 #endif
78
79 static int get_all;
80
81 enum {
82 VMNAME = OPT_START, /* avoid collision with return values from getopt */
83 VCPU,
84 SET_MEM,
85 SET_CAP,
86 CAPNAME,
87 #ifdef BHYVE_SNAPSHOT
88 SET_CHECKPOINT_FILE,
89 SET_SUSPEND_FILE,
90 SET_RUNDIR,
91 #endif
92 OPT_LAST,
93 };
94
95 _Static_assert(OPT_LAST < OPT_START_MD,
96 "OPT_LAST must be less than OPT_START_MD");
97
98 static void
print_cpus(const char * banner,const cpuset_t * cpus)99 print_cpus(const char *banner, const cpuset_t *cpus)
100 {
101 int i, first;
102
103 first = 1;
104 printf("%s:\t", banner);
105 if (!CPU_EMPTY(cpus)) {
106 for (i = 0; i < CPU_SETSIZE; i++) {
107 if (CPU_ISSET(i, cpus)) {
108 printf("%s%d", first ? " " : ", ", i);
109 first = 0;
110 }
111 }
112 } else
113 printf(" (none)");
114 printf("\n");
115 }
116
117 static struct option *
setup_options(void)118 setup_options(void)
119 {
120 const struct option common_opts[] = {
121 { "vm", REQ_ARG, 0, VMNAME },
122 { "cpu", REQ_ARG, 0, VCPU },
123 { "set-mem", REQ_ARG, 0, SET_MEM },
124 { "capname", REQ_ARG, 0, CAPNAME },
125 { "setcap", REQ_ARG, 0, SET_CAP },
126 { "getcap", NO_ARG, &getcap, 1 },
127 { "get-stats", NO_ARG, &get_stats, 1 },
128 { "get-memmap", NO_ARG, &get_memmap, 1 },
129 { "get-memseg", NO_ARG, &get_memseg, 1 },
130 { "get-all", NO_ARG, &get_all, 1 },
131 { "run", NO_ARG, &run, 1 },
132 { "create", NO_ARG, &create, 1 },
133 { "destroy", NO_ARG, &destroy, 1 },
134 { "force-reset", NO_ARG, &force_reset, 1 },
135 { "force-poweroff", NO_ARG, &force_poweroff, 1 },
136 { "get-active-cpus", NO_ARG, &get_active_cpus, 1 },
137 { "get-debug-cpus", NO_ARG, &get_debug_cpus, 1 },
138 { "get-suspended-cpus", NO_ARG, &get_suspended_cpus, 1 },
139 { "get-cpu-topology", NO_ARG, &get_cpu_topology, 1 },
140 #ifdef BHYVE_SNAPSHOT
141 { "checkpoint", REQ_ARG, 0, SET_CHECKPOINT_FILE},
142 { "suspend", REQ_ARG, 0, SET_SUSPEND_FILE},
143 { "rundir", REQ_ARG, 0, SET_RUNDIR},
144 #endif
145 };
146
147 return (bhyvectl_opts(common_opts, nitems(common_opts)));
148 }
149
150 void
usage(const struct option * opts)151 usage(const struct option *opts)
152 {
153 static const char *set_desc[] = {
154 [VCPU] = "vcpu_number",
155 [SET_MEM] = "memory in units of MB",
156 [SET_CAP] = "0|1",
157 [CAPNAME] = "capname",
158 #ifdef BHYVE_SNAPSHOT
159 [SET_CHECKPOINT_FILE] = "filename",
160 [SET_SUSPEND_FILE] = "filename",
161 [SET_RUNDIR] = "path",
162 #endif
163 };
164 (void)fprintf(stderr, "Usage: %s --vm=<vmname>\n", progname);
165 for (const struct option *o = opts; o->name; o++) {
166 if (strcmp(o->name, "vm") == 0)
167 continue;
168 if (o->has_arg == REQ_ARG) {
169 (void)fprintf(stderr, " [--%s=<%s>]\n", o->name,
170 o->val >= OPT_START_MD ? bhyvectl_opt_desc(o->val) :
171 set_desc[o->val]);
172 } else {
173 (void)fprintf(stderr, " [--%s]\n", o->name);
174 }
175 }
176 exit(1);
177 }
178
179 static int
show_memmap(struct vmctx * ctx)180 show_memmap(struct vmctx *ctx)
181 {
182 char name[SPECNAMELEN + 1], numbuf[8];
183 vm_ooffset_t segoff;
184 vm_paddr_t gpa;
185 size_t maplen, seglen;
186 int error, flags, prot, segid, delim;
187
188 printf("Address Length Segment Offset ");
189 printf("Prot Flags\n");
190
191 gpa = 0;
192 while (1) {
193 error = vm_mmap_getnext(ctx, &gpa, &segid, &segoff, &maplen,
194 &prot, &flags);
195 if (error)
196 return (errno == ENOENT ? 0 : error);
197
198 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
199 if (error)
200 return (error);
201
202 printf("%-12lX", gpa);
203 humanize_number(numbuf, sizeof(numbuf), maplen, "B",
204 HN_AUTOSCALE, HN_NOSPACE);
205 printf("%-12s", numbuf);
206
207 printf("%-12s", name[0] ? name : "sysmem");
208 printf("%-12lX", segoff);
209 printf("%c%c%c ", prot & PROT_READ ? 'R' : '-',
210 prot & PROT_WRITE ? 'W' : '-',
211 prot & PROT_EXEC ? 'X' : '-');
212
213 delim = '\0';
214 if (flags & VM_MEMMAP_F_WIRED) {
215 printf("%cwired", delim);
216 delim = '/';
217 }
218 #ifdef __amd64__
219 if (flags & VM_MEMMAP_F_IOMMU) {
220 printf("%ciommu", delim);
221 delim = '/';
222 }
223 #endif
224 printf("\n");
225
226 gpa += maplen;
227 }
228 }
229
230 static int
show_memseg(struct vmctx * ctx)231 show_memseg(struct vmctx *ctx)
232 {
233 char name[SPECNAMELEN + 1], numbuf[8];
234 size_t seglen;
235 int error, segid;
236
237 printf("ID Length Name\n");
238
239 segid = 0;
240 while (1) {
241 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
242 if (error)
243 return (errno == EINVAL ? 0 : error);
244
245 if (seglen) {
246 printf("%-4d", segid);
247 humanize_number(numbuf, sizeof(numbuf), seglen, "B",
248 HN_AUTOSCALE, HN_NOSPACE);
249 printf("%-12s", numbuf);
250 printf("%s", name[0] ? name : "sysmem");
251 printf("\n");
252 }
253 segid++;
254 }
255 }
256
257 static int __unused
ipc_send_message(const char * vmname,nvlist_t * request,const char * rundir)258 ipc_send_message(const char *vmname, nvlist_t *request, const char *rundir)
259 {
260 int err = 0, socket_fd, ret;
261 struct sockaddr_un addr;
262 const char* errmsg;
263 nvlist_t *reply;
264
265 socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
266 if (socket_fd < 0) {
267 perror("Error creating bhyvectl socket");
268 err = errno;
269 goto done;
270 }
271
272 memset(&addr, 0, sizeof(struct sockaddr_un));
273 ret = snprintf(addr.sun_path, sizeof(addr.sun_path), "%s/%s",
274 rundir, vmname);
275 if ((ret < 0) || ((size_t)ret >= sizeof(addr.sun_path))) {
276 fprintf(stderr, "%s: error setting socket path (%d)",
277 __func__, ret);
278 err = 1;
279 goto done;
280 }
281
282 addr.sun_family = AF_UNIX;
283 addr.sun_len = SUN_LEN(&addr);
284
285 if (connect(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
286 perror("connect() failed");
287 err = errno;
288 goto done;
289 }
290
291 reply = nvlist_xfer(socket_fd, request, 0);
292 request = NULL;
293 if (reply == NULL) {
294 perror("nvlist_xfer() failed");
295 goto done;
296 }
297 if (nvlist_exists_string(reply, "error")) {
298 errmsg = nvlist_get_string(reply, "error");
299 fprintf(stderr, "%s: IPC command failed: %s\n", __func__, errmsg);
300 err = -1;
301 }
302 done:
303 if (request != NULL)
304 nvlist_destroy(request);
305
306 if (socket_fd >= 0)
307 close(socket_fd);
308 return (err);
309 }
310
311 #ifdef BHYVE_SNAPSHOT
312 static int
open_directory(const char * file)313 open_directory(const char *file)
314 {
315 char *path;
316 int fd;
317
318 if ((path = strdup(file)) == NULL)
319 return (-1);
320
321 dirname(path);
322 fd = open(path, O_DIRECTORY);
323 free(path);
324
325 return (fd);
326 }
327
328 static int
snapshot_request(const char * vmname,char * file,bool suspend,const char * rundir)329 snapshot_request(const char *vmname, char *file, bool suspend, const char *rundir)
330 {
331 nvlist_t *nvl;
332 int fd;
333
334 if ((fd = open_directory(file)) < 0)
335 return (errno);
336
337 nvl = nvlist_create(0);
338 nvlist_add_string(nvl, "cmd", "checkpoint");
339 nvlist_add_string(nvl, "filename", basename(file));
340 nvlist_add_bool(nvl, "suspend", suspend);
341 nvlist_move_descriptor(nvl, "fddir", fd);
342
343 return (ipc_send_message(vmname, nvl, rundir));
344 }
345 #endif
346
347 int
main(int argc,char * argv[])348 main(int argc, char *argv[])
349 {
350 char *vmname;
351 int action_opts, error, ch, vcpuid;
352 struct vm_run vmrun;
353 struct vmctx *ctx;
354 struct vcpu *vcpu;
355 cpuset_t cpus;
356 struct option *opts;
357 #ifdef BHYVE_SNAPSHOT
358 char *checkpoint_file = NULL;
359 #endif
360 const char *rundir = NULL;
361
362 opts = setup_options();
363
364 action_opts = 0;
365 vcpuid = 0;
366 vmname = NULL;
367 progname = basename(argv[0]);
368
369 while ((ch = getopt_long(argc, argv, "", opts, NULL)) != -1) {
370 if (ch >= OPT_START_MD) {
371 bhyvectl_handle_opt(opts, ch);
372 continue;
373 }
374
375 switch (ch) {
376 case 0:
377 break;
378 case VMNAME:
379 vmname = optarg;
380 break;
381 case VCPU:
382 vcpuid = atoi(optarg);
383 break;
384 case SET_MEM:
385 memsize = atoi(optarg) * MB;
386 memsize = roundup(memsize, 2 * MB);
387 break;
388 case SET_CAP:
389 capval = strtoul(optarg, NULL, 0);
390 setcap = 1;
391 break;
392 case CAPNAME:
393 capname = optarg;
394 break;
395 #ifdef BHYVE_SNAPSHOT
396 case SET_CHECKPOINT_FILE:
397 case SET_SUSPEND_FILE:
398 if (checkpoint_file != NULL)
399 usage(opts);
400
401 checkpoint_file = optarg;
402 vm_suspend_opt = (ch == SET_SUSPEND_FILE);
403 break;
404
405 case SET_RUNDIR:
406 rundir = optarg;
407 break;
408
409 #endif
410 default:
411 usage(opts);
412 }
413 }
414 argc -= optind;
415 argv += optind;
416
417 if (vmname == NULL)
418 usage(opts);
419
420 if (rundir == NULL)
421 rundir = BHYVE_RUN_DIR;
422
423 action_opts = create + destroy + force_reset + force_poweroff;
424 #ifdef BHYVE_SNAPSHOT
425 if (checkpoint_file)
426 action_opts++;
427 #endif
428
429 if (action_opts > 1) {
430 fprintf(stderr, "mutually exclusive actions specified\n");
431 exit(1);
432 }
433
434 ctx = vm_openf(vmname, create ? VMMAPI_OPEN_CREATE : 0);
435 if (ctx == NULL) {
436 fprintf(stderr,
437 "vm_open: %s could not be opened: %s\n",
438 vmname, strerror(errno));
439 exit(1);
440 }
441 vcpu = vm_vcpu_open(ctx, vcpuid);
442 if (vcpu == NULL) {
443 fprintf(stderr,
444 "vm_vcpu_open: %s vcpu %d could not be opened: %s\n",
445 vmname, vcpuid, strerror(errno));
446 exit(1);
447 }
448
449 error = 0;
450 if (!error && memsize)
451 error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
452
453 if (!error && (get_memseg || get_all))
454 error = show_memseg(ctx);
455
456 if (!error && (get_memmap || get_all))
457 error = show_memmap(ctx);
458
459 if (!error)
460 bhyvectl_md_main(ctx, vcpu, vcpuid, get_all);
461
462 if (!error && setcap) {
463 int captype;
464
465 captype = vm_capability_name2type(capname);
466 error = vm_set_capability(vcpu, captype, capval);
467 if (error != 0 && errno == ENOENT)
468 printf("Capability \"%s\" is not available\n", capname);
469 }
470
471 if (!error && (getcap || get_all)) {
472 int captype, val, getcaptype;
473
474 if (getcap && capname)
475 getcaptype = vm_capability_name2type(capname);
476 else
477 getcaptype = -1;
478
479 for (captype = 0; captype < VM_CAP_MAX; captype++) {
480 if (getcaptype >= 0 && captype != getcaptype)
481 continue;
482 error = vm_get_capability(vcpu, captype, &val);
483 if (error == 0) {
484 printf("Capability \"%s\" is %s on vcpu %d\n",
485 vm_capability_type2name(captype),
486 val ? "set" : "not set", vcpuid);
487 } else if (errno == ENOENT) {
488 error = 0;
489 printf("Capability \"%s\" is not available\n",
490 vm_capability_type2name(captype));
491 } else {
492 break;
493 }
494 }
495 }
496
497 if (!error && (get_active_cpus || get_all)) {
498 error = vm_active_cpus(ctx, &cpus);
499 if (!error)
500 print_cpus("active cpus", &cpus);
501 }
502
503 if (!error && (get_debug_cpus || get_all)) {
504 error = vm_debug_cpus(ctx, &cpus);
505 if (!error)
506 print_cpus("debug cpus", &cpus);
507 }
508
509 if (!error && (get_suspended_cpus || get_all)) {
510 error = vm_suspended_cpus(ctx, &cpus);
511 if (!error)
512 print_cpus("suspended cpus", &cpus);
513 }
514
515 if (!error && (get_stats || get_all)) {
516 int i, num_stats;
517 uint64_t *stats;
518 struct timeval tv;
519 const char *desc;
520
521 stats = vm_get_stats(vcpu, &tv, &num_stats);
522 if (stats != NULL) {
523 printf("vcpu%d stats:\n", vcpuid);
524 for (i = 0; i < num_stats; i++) {
525 desc = vm_get_stat_desc(ctx, i);
526 printf("%-40s\t%ld\n", desc, stats[i]);
527 }
528 }
529 }
530
531 if (!error && (get_cpu_topology || get_all)) {
532 uint16_t sockets, cores, threads, maxcpus;
533
534 vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
535 printf("cpu_topology:\tsockets=%hu, cores=%hu, threads=%hu, "
536 "maxcpus=%hu\n", sockets, cores, threads, maxcpus);
537 }
538
539 if (!error && run) {
540 struct vm_exit vmexit;
541 cpuset_t cpuset;
542
543 vmrun.vm_exit = &vmexit;
544 vmrun.cpuset = &cpuset;
545 vmrun.cpusetsize = sizeof(cpuset);
546 error = vm_run(vcpu, &vmrun);
547 if (error == 0)
548 bhyvectl_dump_vm_run_exitcode(&vmexit, vcpuid);
549 else
550 printf("vm_run error %d\n", error);
551 }
552
553 if (!error && force_reset)
554 error = vm_suspend(ctx, VM_SUSPEND_RESET);
555
556 if (!error && force_poweroff)
557 error = vm_suspend(ctx, VM_SUSPEND_POWEROFF);
558
559 #ifdef BHYVE_SNAPSHOT
560 if (!error && checkpoint_file)
561 error = snapshot_request(vmname, checkpoint_file,
562 vm_suspend_opt,
563 rundir);
564 #endif
565
566 if (error)
567 printf("errno = %d\n", errno);
568
569 if (!error && destroy)
570 vm_destroy(ctx);
571
572 free(opts);
573 exit(error);
574 }
575