1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/param.h>
30 #include <sys/cpuset.h>
31 #include <sys/errno.h>
32 #include <sys/mman.h>
33 #include <sys/nv.h>
34 #include <sys/socket.h>
35 #include <sys/sysctl.h>
36 #include <sys/un.h>
37
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <stdbool.h>
41 #include <string.h>
42 #include <unistd.h>
43 #include <libgen.h>
44 #include <libutil.h>
45 #include <fcntl.h>
46 #include <getopt.h>
47 #include <libutil.h>
48
49 #include <machine/cpufunc.h>
50 #include <machine/vmm.h>
51 #include <machine/vmm_dev.h>
52 #include <vmmapi.h>
53
54 #ifdef BHYVE_SNAPSHOT
55 #include "snapshot.h"
56 #endif
57
58 #include "bhyvectl.h"
59
60 #define MB (1UL << 20)
61 #define GB (1UL << 30)
62
63 static const char *progname;
64
65 static int get_stats, getcap, setcap, capval;
66 static int force_reset, force_poweroff;
67 static const char *capname;
68 static int create, destroy, get_memmap, get_memseg;
69 static int get_active_cpus, get_debug_cpus, get_suspended_cpus;
70 static uint64_t memsize;
71 static int run;
72 static int get_cpu_topology;
73 #ifdef BHYVE_SNAPSHOT
74 static int vm_suspend_opt;
75 #endif
76
77 static int get_all;
78
79 enum {
80 VMNAME = OPT_START, /* avoid collision with return values from getopt */
81 VCPU,
82 SET_MEM,
83 SET_CAP,
84 CAPNAME,
85 #ifdef BHYVE_SNAPSHOT
86 SET_CHECKPOINT_FILE,
87 SET_SUSPEND_FILE,
88 #endif
89 OPT_LAST,
90 };
91
92 _Static_assert(OPT_LAST < OPT_START_MD,
93 "OPT_LAST must be less than OPT_START_MD");
94
95 static void
print_cpus(const char * banner,const cpuset_t * cpus)96 print_cpus(const char *banner, const cpuset_t *cpus)
97 {
98 int i, first;
99
100 first = 1;
101 printf("%s:\t", banner);
102 if (!CPU_EMPTY(cpus)) {
103 for (i = 0; i < CPU_SETSIZE; i++) {
104 if (CPU_ISSET(i, cpus)) {
105 printf("%s%d", first ? " " : ", ", i);
106 first = 0;
107 }
108 }
109 } else
110 printf(" (none)");
111 printf("\n");
112 }
113
114 static struct option *
setup_options(void)115 setup_options(void)
116 {
117 const struct option common_opts[] = {
118 { "vm", REQ_ARG, 0, VMNAME },
119 { "cpu", REQ_ARG, 0, VCPU },
120 { "set-mem", REQ_ARG, 0, SET_MEM },
121 { "capname", REQ_ARG, 0, CAPNAME },
122 { "setcap", REQ_ARG, 0, SET_CAP },
123 { "getcap", NO_ARG, &getcap, 1 },
124 { "get-stats", NO_ARG, &get_stats, 1 },
125 { "get-memmap", NO_ARG, &get_memmap, 1 },
126 { "get-memseg", NO_ARG, &get_memseg, 1 },
127 { "get-all", NO_ARG, &get_all, 1 },
128 { "run", NO_ARG, &run, 1 },
129 { "create", NO_ARG, &create, 1 },
130 { "destroy", NO_ARG, &destroy, 1 },
131 { "force-reset", NO_ARG, &force_reset, 1 },
132 { "force-poweroff", NO_ARG, &force_poweroff, 1 },
133 { "get-active-cpus", NO_ARG, &get_active_cpus, 1 },
134 { "get-debug-cpus", NO_ARG, &get_debug_cpus, 1 },
135 { "get-suspended-cpus", NO_ARG, &get_suspended_cpus, 1 },
136 { "get-cpu-topology", NO_ARG, &get_cpu_topology, 1 },
137 #ifdef BHYVE_SNAPSHOT
138 { "checkpoint", REQ_ARG, 0, SET_CHECKPOINT_FILE},
139 { "suspend", REQ_ARG, 0, SET_SUSPEND_FILE},
140 #endif
141 };
142
143 return (bhyvectl_opts(common_opts, nitems(common_opts)));
144 }
145
146 void
usage(const struct option * opts)147 usage(const struct option *opts)
148 {
149 static const char *set_desc[] = {
150 [VCPU] = "vcpu_number",
151 [SET_MEM] = "memory in units of MB",
152 [SET_CAP] = "0|1",
153 [CAPNAME] = "capname",
154 #ifdef BHYVE_SNAPSHOT
155 [SET_CHECKPOINT_FILE] = "filename",
156 [SET_SUSPEND_FILE] = "filename",
157 #endif
158 };
159 (void)fprintf(stderr, "Usage: %s --vm=<vmname>\n", progname);
160 for (const struct option *o = opts; o->name; o++) {
161 if (strcmp(o->name, "vm") == 0)
162 continue;
163 if (o->has_arg == REQ_ARG) {
164 (void)fprintf(stderr, " [--%s=<%s>]\n", o->name,
165 o->val >= OPT_START_MD ? bhyvectl_opt_desc(o->val) :
166 set_desc[o->val]);
167 } else {
168 (void)fprintf(stderr, " [--%s]\n", o->name);
169 }
170 }
171 exit(1);
172 }
173
174 static int
show_memmap(struct vmctx * ctx)175 show_memmap(struct vmctx *ctx)
176 {
177 char name[SPECNAMELEN + 1], numbuf[8];
178 vm_ooffset_t segoff;
179 vm_paddr_t gpa;
180 size_t maplen, seglen;
181 int error, flags, prot, segid, delim;
182
183 printf("Address Length Segment Offset ");
184 printf("Prot Flags\n");
185
186 gpa = 0;
187 while (1) {
188 error = vm_mmap_getnext(ctx, &gpa, &segid, &segoff, &maplen,
189 &prot, &flags);
190 if (error)
191 return (errno == ENOENT ? 0 : error);
192
193 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
194 if (error)
195 return (error);
196
197 printf("%-12lX", gpa);
198 humanize_number(numbuf, sizeof(numbuf), maplen, "B",
199 HN_AUTOSCALE, HN_NOSPACE);
200 printf("%-12s", numbuf);
201
202 printf("%-12s", name[0] ? name : "sysmem");
203 printf("%-12lX", segoff);
204 printf("%c%c%c ", prot & PROT_READ ? 'R' : '-',
205 prot & PROT_WRITE ? 'W' : '-',
206 prot & PROT_EXEC ? 'X' : '-');
207
208 delim = '\0';
209 if (flags & VM_MEMMAP_F_WIRED) {
210 printf("%cwired", delim);
211 delim = '/';
212 }
213 #ifdef __amd64__
214 if (flags & VM_MEMMAP_F_IOMMU) {
215 printf("%ciommu", delim);
216 delim = '/';
217 }
218 #endif
219 printf("\n");
220
221 gpa += maplen;
222 }
223 }
224
225 static int
show_memseg(struct vmctx * ctx)226 show_memseg(struct vmctx *ctx)
227 {
228 char name[SPECNAMELEN + 1], numbuf[8];
229 size_t seglen;
230 int error, segid;
231
232 printf("ID Length Name\n");
233
234 segid = 0;
235 while (1) {
236 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
237 if (error)
238 return (errno == EINVAL ? 0 : error);
239
240 if (seglen) {
241 printf("%-4d", segid);
242 humanize_number(numbuf, sizeof(numbuf), seglen, "B",
243 HN_AUTOSCALE, HN_NOSPACE);
244 printf("%-12s", numbuf);
245 printf("%s", name[0] ? name : "sysmem");
246 printf("\n");
247 }
248 segid++;
249 }
250 }
251
252 #ifdef BHYVE_SNAPSHOT
253 static int
send_message(const char * vmname,nvlist_t * nvl)254 send_message(const char *vmname, nvlist_t *nvl)
255 {
256 struct sockaddr_un addr;
257 int err = 0, socket_fd;
258
259 socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
260 if (socket_fd < 0) {
261 perror("Error creating bhyvectl socket");
262 err = errno;
263 goto done;
264 }
265
266 memset(&addr, 0, sizeof(struct sockaddr_un));
267 snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s",
268 BHYVE_RUN_DIR, vmname);
269 addr.sun_family = AF_UNIX;
270 addr.sun_len = SUN_LEN(&addr);
271
272 if (connect(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
273 perror("connect() failed");
274 err = errno;
275 goto done;
276 }
277
278 if (nvlist_send(socket_fd, nvl) < 0) {
279 perror("nvlist_send() failed");
280 err = errno;
281 }
282 done:
283 nvlist_destroy(nvl);
284
285 if (socket_fd >= 0)
286 close(socket_fd);
287 return (err);
288 }
289
290 static int
open_directory(const char * file)291 open_directory(const char *file)
292 {
293 char *path;
294 int fd;
295
296 if ((path = strdup(file)) == NULL)
297 return (-1);
298
299 dirname(path);
300 fd = open(path, O_DIRECTORY);
301 free(path);
302
303 return (fd);
304 }
305
306 static int
snapshot_request(const char * vmname,char * file,bool suspend)307 snapshot_request(const char *vmname, char *file, bool suspend)
308 {
309 nvlist_t *nvl;
310 int fd;
311
312 if ((fd = open_directory(file)) < 0)
313 return (errno);
314
315 nvl = nvlist_create(0);
316 nvlist_add_string(nvl, "cmd", "checkpoint");
317 nvlist_add_string(nvl, "filename", basename(file));
318 nvlist_add_bool(nvl, "suspend", suspend);
319 nvlist_move_descriptor(nvl, "fddir", fd);
320
321 return (send_message(vmname, nvl));
322 }
323 #endif
324
325 int
main(int argc,char * argv[])326 main(int argc, char *argv[])
327 {
328 char *vmname;
329 int action_opts, error, ch, vcpuid;
330 struct vm_run vmrun;
331 struct vmctx *ctx;
332 struct vcpu *vcpu;
333 cpuset_t cpus;
334 struct option *opts;
335 #ifdef BHYVE_SNAPSHOT
336 char *checkpoint_file = NULL;
337 #endif
338
339 opts = setup_options();
340
341 action_opts = 0;
342 vcpuid = 0;
343 vmname = NULL;
344 progname = basename(argv[0]);
345
346 while ((ch = getopt_long(argc, argv, "", opts, NULL)) != -1) {
347 if (ch >= OPT_START_MD) {
348 bhyvectl_handle_opt(opts, ch);
349 continue;
350 }
351
352 switch (ch) {
353 case 0:
354 break;
355 case VMNAME:
356 vmname = optarg;
357 break;
358 case VCPU:
359 vcpuid = atoi(optarg);
360 break;
361 case SET_MEM:
362 memsize = atoi(optarg) * MB;
363 memsize = roundup(memsize, 2 * MB);
364 break;
365 case SET_CAP:
366 capval = strtoul(optarg, NULL, 0);
367 setcap = 1;
368 break;
369 case CAPNAME:
370 capname = optarg;
371 break;
372 #ifdef BHYVE_SNAPSHOT
373 case SET_CHECKPOINT_FILE:
374 case SET_SUSPEND_FILE:
375 if (checkpoint_file != NULL)
376 usage(opts);
377
378 checkpoint_file = optarg;
379 vm_suspend_opt = (ch == SET_SUSPEND_FILE);
380 break;
381 #endif
382 default:
383 usage(opts);
384 }
385 }
386 argc -= optind;
387 argv += optind;
388
389 if (vmname == NULL)
390 usage(opts);
391
392 action_opts = create + destroy + force_reset + force_poweroff;
393 #ifdef BHYVE_SNAPSHOT
394 if (checkpoint_file)
395 action_opts++;
396 #endif
397
398 if (action_opts > 1) {
399 fprintf(stderr, "mutually exclusive actions specified\n");
400 exit(1);
401 }
402
403 ctx = vm_openf(vmname, create ? VMMAPI_OPEN_CREATE : 0);
404 if (ctx == NULL) {
405 fprintf(stderr,
406 "vm_open: %s could not be opened: %s\n",
407 vmname, strerror(errno));
408 exit(1);
409 }
410 vcpu = vm_vcpu_open(ctx, vcpuid);
411
412 error = 0;
413 if (!error && memsize)
414 error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
415
416 if (!error && (get_memseg || get_all))
417 error = show_memseg(ctx);
418
419 if (!error && (get_memmap || get_all))
420 error = show_memmap(ctx);
421
422 if (!error)
423 bhyvectl_md_main(ctx, vcpu, vcpuid, get_all);
424
425 if (!error && setcap) {
426 int captype;
427
428 captype = vm_capability_name2type(capname);
429 error = vm_set_capability(vcpu, captype, capval);
430 if (error != 0 && errno == ENOENT)
431 printf("Capability \"%s\" is not available\n", capname);
432 }
433
434 if (!error && (getcap || get_all)) {
435 int captype, val, getcaptype;
436
437 if (getcap && capname)
438 getcaptype = vm_capability_name2type(capname);
439 else
440 getcaptype = -1;
441
442 for (captype = 0; captype < VM_CAP_MAX; captype++) {
443 if (getcaptype >= 0 && captype != getcaptype)
444 continue;
445 error = vm_get_capability(vcpu, captype, &val);
446 if (error == 0) {
447 printf("Capability \"%s\" is %s on vcpu %d\n",
448 vm_capability_type2name(captype),
449 val ? "set" : "not set", vcpuid);
450 } else if (errno == ENOENT) {
451 error = 0;
452 printf("Capability \"%s\" is not available\n",
453 vm_capability_type2name(captype));
454 } else {
455 break;
456 }
457 }
458 }
459
460 if (!error && (get_active_cpus || get_all)) {
461 error = vm_active_cpus(ctx, &cpus);
462 if (!error)
463 print_cpus("active cpus", &cpus);
464 }
465
466 if (!error && (get_debug_cpus || get_all)) {
467 error = vm_debug_cpus(ctx, &cpus);
468 if (!error)
469 print_cpus("debug cpus", &cpus);
470 }
471
472 if (!error && (get_suspended_cpus || get_all)) {
473 error = vm_suspended_cpus(ctx, &cpus);
474 if (!error)
475 print_cpus("suspended cpus", &cpus);
476 }
477
478 if (!error && (get_stats || get_all)) {
479 int i, num_stats;
480 uint64_t *stats;
481 struct timeval tv;
482 const char *desc;
483
484 stats = vm_get_stats(vcpu, &tv, &num_stats);
485 if (stats != NULL) {
486 printf("vcpu%d stats:\n", vcpuid);
487 for (i = 0; i < num_stats; i++) {
488 desc = vm_get_stat_desc(ctx, i);
489 printf("%-40s\t%ld\n", desc, stats[i]);
490 }
491 }
492 }
493
494 if (!error && (get_cpu_topology || get_all)) {
495 uint16_t sockets, cores, threads, maxcpus;
496
497 vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
498 printf("cpu_topology:\tsockets=%hu, cores=%hu, threads=%hu, "
499 "maxcpus=%hu\n", sockets, cores, threads, maxcpus);
500 }
501
502 if (!error && run) {
503 struct vm_exit vmexit;
504 cpuset_t cpuset;
505
506 vmrun.vm_exit = &vmexit;
507 vmrun.cpuset = &cpuset;
508 vmrun.cpusetsize = sizeof(cpuset);
509 error = vm_run(vcpu, &vmrun);
510 if (error == 0)
511 bhyvectl_dump_vm_run_exitcode(&vmexit, vcpuid);
512 else
513 printf("vm_run error %d\n", error);
514 }
515
516 if (!error && force_reset)
517 error = vm_suspend(ctx, VM_SUSPEND_RESET);
518
519 if (!error && force_poweroff)
520 error = vm_suspend(ctx, VM_SUSPEND_POWEROFF);
521
522 #ifdef BHYVE_SNAPSHOT
523 if (!error && checkpoint_file)
524 error = snapshot_request(vmname, checkpoint_file, vm_suspend_opt);
525 #endif
526
527 if (error)
528 printf("errno = %d\n", errno);
529
530 if (!error && destroy)
531 vm_destroy(ctx);
532
533 free(opts);
534 exit(error);
535 }
536