1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * Copyright (c) 2026 Hans Rosenfeld
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/param.h>
31 #include <sys/cpuset.h>
32 #include <sys/errno.h>
33 #include <sys/mman.h>
34 #include <sys/nv.h>
35 #include <sys/socket.h>
36 #include <sys/sysctl.h>
37 #include <sys/un.h>
38
39 #include <stdio.h>
40 #include <stdlib.h>
41 #include <stdbool.h>
42 #include <string.h>
43 #include <unistd.h>
44 #include <libgen.h>
45 #include <libutil.h>
46 #include <fcntl.h>
47 #include <getopt.h>
48 #include <libutil.h>
49
50 #include <machine/cpufunc.h>
51 #include <machine/vmm.h>
52 #include <machine/vmm_dev.h>
53 #include <vmmapi.h>
54
55 #ifdef BHYVE_SNAPSHOT
56 #include "snapshot.h"
57 #endif
58
59 #include "bhyvectl.h"
60
61 #define MB (1UL << 20)
62 #define GB (1UL << 30)
63
64 static const char *progname;
65
66 static int get_stats, getcap, setcap, capval;
67 static int force_reset, force_poweroff;
68 static const char *capname;
69 static int create, destroy, get_memmap, get_memseg;
70 static int get_active_cpus, get_debug_cpus, get_suspended_cpus;
71 static uint64_t memsize;
72 static int run;
73 static int get_cpu_topology;
74 #ifdef BHYVE_SNAPSHOT
75 static int vm_suspend_opt;
76 #endif
77
78 static int get_all;
79
80 enum {
81 VMNAME = OPT_START, /* avoid collision with return values from getopt */
82 VCPU,
83 SET_MEM,
84 SET_CAP,
85 CAPNAME,
86 #ifdef BHYVE_SNAPSHOT
87 SET_CHECKPOINT_FILE,
88 SET_SUSPEND_FILE,
89 #endif
90 OPT_LAST,
91 };
92
93 _Static_assert(OPT_LAST < OPT_START_MD,
94 "OPT_LAST must be less than OPT_START_MD");
95
96 static void
print_cpus(const char * banner,const cpuset_t * cpus)97 print_cpus(const char *banner, const cpuset_t *cpus)
98 {
99 int i, first;
100
101 first = 1;
102 printf("%s:\t", banner);
103 if (!CPU_EMPTY(cpus)) {
104 for (i = 0; i < CPU_SETSIZE; i++) {
105 if (CPU_ISSET(i, cpus)) {
106 printf("%s%d", first ? " " : ", ", i);
107 first = 0;
108 }
109 }
110 } else
111 printf(" (none)");
112 printf("\n");
113 }
114
115 static struct option *
setup_options(void)116 setup_options(void)
117 {
118 const struct option common_opts[] = {
119 { "vm", REQ_ARG, 0, VMNAME },
120 { "cpu", REQ_ARG, 0, VCPU },
121 { "set-mem", REQ_ARG, 0, SET_MEM },
122 { "capname", REQ_ARG, 0, CAPNAME },
123 { "setcap", REQ_ARG, 0, SET_CAP },
124 { "getcap", NO_ARG, &getcap, 1 },
125 { "get-stats", NO_ARG, &get_stats, 1 },
126 { "get-memmap", NO_ARG, &get_memmap, 1 },
127 { "get-memseg", NO_ARG, &get_memseg, 1 },
128 { "get-all", NO_ARG, &get_all, 1 },
129 { "run", NO_ARG, &run, 1 },
130 { "create", NO_ARG, &create, 1 },
131 { "destroy", NO_ARG, &destroy, 1 },
132 { "force-reset", NO_ARG, &force_reset, 1 },
133 { "force-poweroff", NO_ARG, &force_poweroff, 1 },
134 { "get-active-cpus", NO_ARG, &get_active_cpus, 1 },
135 { "get-debug-cpus", NO_ARG, &get_debug_cpus, 1 },
136 { "get-suspended-cpus", NO_ARG, &get_suspended_cpus, 1 },
137 { "get-cpu-topology", NO_ARG, &get_cpu_topology, 1 },
138 #ifdef BHYVE_SNAPSHOT
139 { "checkpoint", REQ_ARG, 0, SET_CHECKPOINT_FILE},
140 { "suspend", REQ_ARG, 0, SET_SUSPEND_FILE},
141 #endif
142 };
143
144 return (bhyvectl_opts(common_opts, nitems(common_opts)));
145 }
146
147 void
usage(const struct option * opts)148 usage(const struct option *opts)
149 {
150 static const char *set_desc[] = {
151 [VCPU] = "vcpu_number",
152 [SET_MEM] = "memory in units of MB",
153 [SET_CAP] = "0|1",
154 [CAPNAME] = "capname",
155 #ifdef BHYVE_SNAPSHOT
156 [SET_CHECKPOINT_FILE] = "filename",
157 [SET_SUSPEND_FILE] = "filename",
158 #endif
159 };
160 (void)fprintf(stderr, "Usage: %s --vm=<vmname>\n", progname);
161 for (const struct option *o = opts; o->name; o++) {
162 if (strcmp(o->name, "vm") == 0)
163 continue;
164 if (o->has_arg == REQ_ARG) {
165 (void)fprintf(stderr, " [--%s=<%s>]\n", o->name,
166 o->val >= OPT_START_MD ? bhyvectl_opt_desc(o->val) :
167 set_desc[o->val]);
168 } else {
169 (void)fprintf(stderr, " [--%s]\n", o->name);
170 }
171 }
172 exit(1);
173 }
174
175 static int
show_memmap(struct vmctx * ctx)176 show_memmap(struct vmctx *ctx)
177 {
178 char name[SPECNAMELEN + 1], numbuf[8];
179 vm_ooffset_t segoff;
180 vm_paddr_t gpa;
181 size_t maplen, seglen;
182 int error, flags, prot, segid, delim;
183
184 printf("Address Length Segment Offset ");
185 printf("Prot Flags\n");
186
187 gpa = 0;
188 while (1) {
189 error = vm_mmap_getnext(ctx, &gpa, &segid, &segoff, &maplen,
190 &prot, &flags);
191 if (error)
192 return (errno == ENOENT ? 0 : error);
193
194 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
195 if (error)
196 return (error);
197
198 printf("%-12lX", gpa);
199 humanize_number(numbuf, sizeof(numbuf), maplen, "B",
200 HN_AUTOSCALE, HN_NOSPACE);
201 printf("%-12s", numbuf);
202
203 printf("%-12s", name[0] ? name : "sysmem");
204 printf("%-12lX", segoff);
205 printf("%c%c%c ", prot & PROT_READ ? 'R' : '-',
206 prot & PROT_WRITE ? 'W' : '-',
207 prot & PROT_EXEC ? 'X' : '-');
208
209 delim = '\0';
210 if (flags & VM_MEMMAP_F_WIRED) {
211 printf("%cwired", delim);
212 delim = '/';
213 }
214 #ifdef __amd64__
215 if (flags & VM_MEMMAP_F_IOMMU) {
216 printf("%ciommu", delim);
217 delim = '/';
218 }
219 #endif
220 printf("\n");
221
222 gpa += maplen;
223 }
224 }
225
226 static int
show_memseg(struct vmctx * ctx)227 show_memseg(struct vmctx *ctx)
228 {
229 char name[SPECNAMELEN + 1], numbuf[8];
230 size_t seglen;
231 int error, segid;
232
233 printf("ID Length Name\n");
234
235 segid = 0;
236 while (1) {
237 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
238 if (error)
239 return (errno == EINVAL ? 0 : error);
240
241 if (seglen) {
242 printf("%-4d", segid);
243 humanize_number(numbuf, sizeof(numbuf), seglen, "B",
244 HN_AUTOSCALE, HN_NOSPACE);
245 printf("%-12s", numbuf);
246 printf("%s", name[0] ? name : "sysmem");
247 printf("\n");
248 }
249 segid++;
250 }
251 }
252
253 #ifdef BHYVE_SNAPSHOT
254 static int
send_message(const char * vmname,nvlist_t * nvl)255 send_message(const char *vmname, nvlist_t *nvl)
256 {
257 struct sockaddr_un addr;
258 int err = 0, socket_fd;
259
260 socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
261 if (socket_fd < 0) {
262 perror("Error creating bhyvectl socket");
263 err = errno;
264 goto done;
265 }
266
267 memset(&addr, 0, sizeof(struct sockaddr_un));
268 snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s",
269 BHYVE_RUN_DIR, vmname);
270 addr.sun_family = AF_UNIX;
271 addr.sun_len = SUN_LEN(&addr);
272
273 if (connect(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
274 perror("connect() failed");
275 err = errno;
276 goto done;
277 }
278
279 if (nvlist_send(socket_fd, nvl) < 0) {
280 perror("nvlist_send() failed");
281 err = errno;
282 }
283 done:
284 nvlist_destroy(nvl);
285
286 if (socket_fd >= 0)
287 close(socket_fd);
288 return (err);
289 }
290
291 static int
open_directory(const char * file)292 open_directory(const char *file)
293 {
294 char *path;
295 int fd;
296
297 if ((path = strdup(file)) == NULL)
298 return (-1);
299
300 dirname(path);
301 fd = open(path, O_DIRECTORY);
302 free(path);
303
304 return (fd);
305 }
306
307 static int
snapshot_request(const char * vmname,char * file,bool suspend)308 snapshot_request(const char *vmname, char *file, bool suspend)
309 {
310 nvlist_t *nvl;
311 int fd;
312
313 if ((fd = open_directory(file)) < 0)
314 return (errno);
315
316 nvl = nvlist_create(0);
317 nvlist_add_string(nvl, "cmd", "checkpoint");
318 nvlist_add_string(nvl, "filename", basename(file));
319 nvlist_add_bool(nvl, "suspend", suspend);
320 nvlist_move_descriptor(nvl, "fddir", fd);
321
322 return (send_message(vmname, nvl));
323 }
324 #endif
325
326 int
main(int argc,char * argv[])327 main(int argc, char *argv[])
328 {
329 char *vmname;
330 int action_opts, error, ch, vcpuid;
331 struct vm_run vmrun;
332 struct vmctx *ctx;
333 struct vcpu *vcpu;
334 cpuset_t cpus;
335 struct option *opts;
336 #ifdef BHYVE_SNAPSHOT
337 char *checkpoint_file = NULL;
338 #endif
339
340 opts = setup_options();
341
342 action_opts = 0;
343 vcpuid = 0;
344 vmname = NULL;
345 progname = basename(argv[0]);
346
347 while ((ch = getopt_long(argc, argv, "", opts, NULL)) != -1) {
348 if (ch >= OPT_START_MD) {
349 bhyvectl_handle_opt(opts, ch);
350 continue;
351 }
352
353 switch (ch) {
354 case 0:
355 break;
356 case VMNAME:
357 vmname = optarg;
358 break;
359 case VCPU:
360 vcpuid = atoi(optarg);
361 break;
362 case SET_MEM:
363 memsize = atoi(optarg) * MB;
364 memsize = roundup(memsize, 2 * MB);
365 break;
366 case SET_CAP:
367 capval = strtoul(optarg, NULL, 0);
368 setcap = 1;
369 break;
370 case CAPNAME:
371 capname = optarg;
372 break;
373 #ifdef BHYVE_SNAPSHOT
374 case SET_CHECKPOINT_FILE:
375 case SET_SUSPEND_FILE:
376 if (checkpoint_file != NULL)
377 usage(opts);
378
379 checkpoint_file = optarg;
380 vm_suspend_opt = (ch == SET_SUSPEND_FILE);
381 break;
382 #endif
383 default:
384 usage(opts);
385 }
386 }
387 argc -= optind;
388 argv += optind;
389
390 if (vmname == NULL)
391 usage(opts);
392
393 action_opts = create + destroy + force_reset + force_poweroff;
394 #ifdef BHYVE_SNAPSHOT
395 if (checkpoint_file)
396 action_opts++;
397 #endif
398
399 if (action_opts > 1) {
400 fprintf(stderr, "mutually exclusive actions specified\n");
401 exit(1);
402 }
403
404 ctx = vm_openf(vmname, create ? VMMAPI_OPEN_CREATE : 0);
405 if (ctx == NULL) {
406 fprintf(stderr,
407 "vm_open: %s could not be opened: %s\n",
408 vmname, strerror(errno));
409 exit(1);
410 }
411 vcpu = vm_vcpu_open(ctx, vcpuid);
412 if (vcpu == NULL) {
413 fprintf(stderr,
414 "vm_vcpu_open: %s vcpu %d could not be opened: %s\n",
415 vmname, vcpuid, strerror(errno));
416 exit(1);
417 }
418
419 error = 0;
420 if (!error && memsize)
421 error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
422
423 if (!error && (get_memseg || get_all))
424 error = show_memseg(ctx);
425
426 if (!error && (get_memmap || get_all))
427 error = show_memmap(ctx);
428
429 if (!error)
430 bhyvectl_md_main(ctx, vcpu, vcpuid, get_all);
431
432 if (!error && setcap) {
433 int captype;
434
435 captype = vm_capability_name2type(capname);
436 error = vm_set_capability(vcpu, captype, capval);
437 if (error != 0 && errno == ENOENT)
438 printf("Capability \"%s\" is not available\n", capname);
439 }
440
441 if (!error && (getcap || get_all)) {
442 int captype, val, getcaptype;
443
444 if (getcap && capname)
445 getcaptype = vm_capability_name2type(capname);
446 else
447 getcaptype = -1;
448
449 for (captype = 0; captype < VM_CAP_MAX; captype++) {
450 if (getcaptype >= 0 && captype != getcaptype)
451 continue;
452 error = vm_get_capability(vcpu, captype, &val);
453 if (error == 0) {
454 printf("Capability \"%s\" is %s on vcpu %d\n",
455 vm_capability_type2name(captype),
456 val ? "set" : "not set", vcpuid);
457 } else if (errno == ENOENT) {
458 error = 0;
459 printf("Capability \"%s\" is not available\n",
460 vm_capability_type2name(captype));
461 } else {
462 break;
463 }
464 }
465 }
466
467 if (!error && (get_active_cpus || get_all)) {
468 error = vm_active_cpus(ctx, &cpus);
469 if (!error)
470 print_cpus("active cpus", &cpus);
471 }
472
473 if (!error && (get_debug_cpus || get_all)) {
474 error = vm_debug_cpus(ctx, &cpus);
475 if (!error)
476 print_cpus("debug cpus", &cpus);
477 }
478
479 if (!error && (get_suspended_cpus || get_all)) {
480 error = vm_suspended_cpus(ctx, &cpus);
481 if (!error)
482 print_cpus("suspended cpus", &cpus);
483 }
484
485 if (!error && (get_stats || get_all)) {
486 int i, num_stats;
487 uint64_t *stats;
488 struct timeval tv;
489 const char *desc;
490
491 stats = vm_get_stats(vcpu, &tv, &num_stats);
492 if (stats != NULL) {
493 printf("vcpu%d stats:\n", vcpuid);
494 for (i = 0; i < num_stats; i++) {
495 desc = vm_get_stat_desc(ctx, i);
496 printf("%-40s\t%ld\n", desc, stats[i]);
497 }
498 }
499 }
500
501 if (!error && (get_cpu_topology || get_all)) {
502 uint16_t sockets, cores, threads, maxcpus;
503
504 vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
505 printf("cpu_topology:\tsockets=%hu, cores=%hu, threads=%hu, "
506 "maxcpus=%hu\n", sockets, cores, threads, maxcpus);
507 }
508
509 if (!error && run) {
510 struct vm_exit vmexit;
511 cpuset_t cpuset;
512
513 vmrun.vm_exit = &vmexit;
514 vmrun.cpuset = &cpuset;
515 vmrun.cpusetsize = sizeof(cpuset);
516 error = vm_run(vcpu, &vmrun);
517 if (error == 0)
518 bhyvectl_dump_vm_run_exitcode(&vmexit, vcpuid);
519 else
520 printf("vm_run error %d\n", error);
521 }
522
523 if (!error && force_reset)
524 error = vm_suspend(ctx, VM_SUSPEND_RESET);
525
526 if (!error && force_poweroff)
527 error = vm_suspend(ctx, VM_SUSPEND_POWEROFF);
528
529 #ifdef BHYVE_SNAPSHOT
530 if (!error && checkpoint_file)
531 error = snapshot_request(vmname, checkpoint_file, vm_suspend_opt);
532 #endif
533
534 if (error)
535 printf("errno = %d\n", errno);
536
537 if (!error && destroy)
538 vm_destroy(ctx);
539
540 free(opts);
541 exit(error);
542 }
543