1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/cpuset.h> 31 #include <sys/errno.h> 32 #include <sys/mman.h> 33 #include <sys/nv.h> 34 #include <sys/socket.h> 35 #include <sys/sysctl.h> 36 #include <sys/un.h> 37 38 #include <stdio.h> 39 #include <stdlib.h> 40 #include <stdbool.h> 41 #include <string.h> 42 #include <unistd.h> 43 #include <libgen.h> 44 #include <libutil.h> 45 #include <fcntl.h> 46 #include <getopt.h> 47 #include <libutil.h> 48 49 #include <machine/cpufunc.h> 50 #include <machine/vmm.h> 51 #include <machine/vmm_dev.h> 52 #include <vmmapi.h> 53 54 #ifdef BHYVE_SNAPSHOT 55 #include "snapshot.h" 56 #endif 57 58 #include "bhyvectl.h" 59 60 #define MB (1UL << 20) 61 #define GB (1UL << 30) 62 63 static const char *progname; 64 65 static int get_stats, getcap, setcap, capval; 66 static int force_reset, force_poweroff; 67 static const char *capname; 68 static int create, destroy, get_memmap, get_memseg; 69 static int get_active_cpus, get_debug_cpus, get_suspended_cpus; 70 static uint64_t memsize; 71 static int run; 72 static int get_cpu_topology; 73 #ifdef BHYVE_SNAPSHOT 74 static int vm_suspend_opt; 75 #endif 76 77 static int get_all; 78 79 enum { 80 VMNAME = OPT_START, /* avoid collision with return values from getopt */ 81 VCPU, 82 SET_MEM, 83 SET_CAP, 84 CAPNAME, 85 #ifdef BHYVE_SNAPSHOT 86 SET_CHECKPOINT_FILE, 87 SET_SUSPEND_FILE, 88 #endif 89 OPT_LAST, 90 }; 91 92 _Static_assert(OPT_LAST < OPT_START_MD, 93 "OPT_LAST must be less than OPT_START_MD"); 94 95 static void 96 print_cpus(const char *banner, const cpuset_t *cpus) 97 { 98 int i, first; 99 100 first = 1; 101 printf("%s:\t", banner); 102 if (!CPU_EMPTY(cpus)) { 103 for (i = 0; i < CPU_SETSIZE; i++) { 104 if (CPU_ISSET(i, cpus)) { 105 printf("%s%d", first ? " " : ", ", i); 106 first = 0; 107 } 108 } 109 } else 110 printf(" (none)"); 111 printf("\n"); 112 } 113 114 static struct option * 115 setup_options(void) 116 { 117 const struct option common_opts[] = { 118 { "vm", REQ_ARG, 0, VMNAME }, 119 { "cpu", REQ_ARG, 0, VCPU }, 120 { "set-mem", REQ_ARG, 0, SET_MEM }, 121 { "capname", REQ_ARG, 0, CAPNAME }, 122 { "setcap", REQ_ARG, 0, SET_CAP }, 123 { "getcap", NO_ARG, &getcap, 1 }, 124 { "get-stats", NO_ARG, &get_stats, 1 }, 125 { "get-memmap", NO_ARG, &get_memmap, 1 }, 126 { "get-memseg", NO_ARG, &get_memseg, 1 }, 127 { "get-all", NO_ARG, &get_all, 1 }, 128 { "run", NO_ARG, &run, 1 }, 129 { "create", NO_ARG, &create, 1 }, 130 { "destroy", NO_ARG, &destroy, 1 }, 131 { "force-reset", NO_ARG, &force_reset, 1 }, 132 { "force-poweroff", NO_ARG, &force_poweroff, 1 }, 133 { "get-active-cpus", NO_ARG, &get_active_cpus, 1 }, 134 { "get-debug-cpus", NO_ARG, &get_debug_cpus, 1 }, 135 { "get-suspended-cpus", NO_ARG, &get_suspended_cpus, 1 }, 136 { "get-cpu-topology", NO_ARG, &get_cpu_topology, 1 }, 137 #ifdef BHYVE_SNAPSHOT 138 { "checkpoint", REQ_ARG, 0, SET_CHECKPOINT_FILE}, 139 { "suspend", REQ_ARG, 0, SET_SUSPEND_FILE}, 140 #endif 141 }; 142 143 return (bhyvectl_opts(common_opts, nitems(common_opts))); 144 } 145 146 void 147 usage(const struct option *opts) 148 { 149 static const char *set_desc[] = { 150 [VCPU] = "vcpu_number", 151 [SET_MEM] = "memory in units of MB", 152 [SET_CAP] = "0|1", 153 [CAPNAME] = "capname", 154 #ifdef BHYVE_SNAPSHOT 155 [SET_CHECKPOINT_FILE] = "filename", 156 [SET_SUSPEND_FILE] = "filename", 157 #endif 158 }; 159 (void)fprintf(stderr, "Usage: %s --vm=<vmname>\n", progname); 160 for (const struct option *o = opts; o->name; o++) { 161 if (strcmp(o->name, "vm") == 0) 162 continue; 163 if (o->has_arg == REQ_ARG) { 164 (void)fprintf(stderr, " [--%s=<%s>]\n", o->name, 165 o->val >= OPT_START_MD ? bhyvectl_opt_desc(o->val) : 166 set_desc[o->val]); 167 } else { 168 (void)fprintf(stderr, " [--%s]\n", o->name); 169 } 170 } 171 exit(1); 172 } 173 174 static int 175 show_memmap(struct vmctx *ctx) 176 { 177 char name[SPECNAMELEN + 1], numbuf[8]; 178 vm_ooffset_t segoff; 179 vm_paddr_t gpa; 180 size_t maplen, seglen; 181 int error, flags, prot, segid, delim; 182 183 printf("Address Length Segment Offset "); 184 printf("Prot Flags\n"); 185 186 gpa = 0; 187 while (1) { 188 error = vm_mmap_getnext(ctx, &gpa, &segid, &segoff, &maplen, 189 &prot, &flags); 190 if (error) 191 return (errno == ENOENT ? 0 : error); 192 193 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name)); 194 if (error) 195 return (error); 196 197 printf("%-12lX", gpa); 198 humanize_number(numbuf, sizeof(numbuf), maplen, "B", 199 HN_AUTOSCALE, HN_NOSPACE); 200 printf("%-12s", numbuf); 201 202 printf("%-12s", name[0] ? name : "sysmem"); 203 printf("%-12lX", segoff); 204 printf("%c%c%c ", prot & PROT_READ ? 'R' : '-', 205 prot & PROT_WRITE ? 'W' : '-', 206 prot & PROT_EXEC ? 'X' : '-'); 207 208 delim = '\0'; 209 if (flags & VM_MEMMAP_F_WIRED) { 210 printf("%cwired", delim); 211 delim = '/'; 212 } 213 #ifdef __amd64__ 214 if (flags & VM_MEMMAP_F_IOMMU) { 215 printf("%ciommu", delim); 216 delim = '/'; 217 } 218 #endif 219 printf("\n"); 220 221 gpa += maplen; 222 } 223 } 224 225 static int 226 show_memseg(struct vmctx *ctx) 227 { 228 char name[SPECNAMELEN + 1], numbuf[8]; 229 size_t seglen; 230 int error, segid; 231 232 printf("ID Length Name\n"); 233 234 segid = 0; 235 while (1) { 236 error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name)); 237 if (error) 238 return (errno == EINVAL ? 0 : error); 239 240 if (seglen) { 241 printf("%-4d", segid); 242 humanize_number(numbuf, sizeof(numbuf), seglen, "B", 243 HN_AUTOSCALE, HN_NOSPACE); 244 printf("%-12s", numbuf); 245 printf("%s", name[0] ? name : "sysmem"); 246 printf("\n"); 247 } 248 segid++; 249 } 250 } 251 252 #ifdef BHYVE_SNAPSHOT 253 static int 254 send_message(const char *vmname, nvlist_t *nvl) 255 { 256 struct sockaddr_un addr; 257 int err = 0, socket_fd; 258 259 socket_fd = socket(PF_UNIX, SOCK_STREAM, 0); 260 if (socket_fd < 0) { 261 perror("Error creating bhyvectl socket"); 262 err = errno; 263 goto done; 264 } 265 266 memset(&addr, 0, sizeof(struct sockaddr_un)); 267 snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s", 268 BHYVE_RUN_DIR, vmname); 269 addr.sun_family = AF_UNIX; 270 addr.sun_len = SUN_LEN(&addr); 271 272 if (connect(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) { 273 perror("connect() failed"); 274 err = errno; 275 goto done; 276 } 277 278 if (nvlist_send(socket_fd, nvl) < 0) { 279 perror("nvlist_send() failed"); 280 err = errno; 281 } 282 done: 283 nvlist_destroy(nvl); 284 285 if (socket_fd >= 0) 286 close(socket_fd); 287 return (err); 288 } 289 290 static int 291 open_directory(const char *file) 292 { 293 char *path; 294 int fd; 295 296 if ((path = strdup(file)) == NULL) 297 return (-1); 298 299 dirname(path); 300 fd = open(path, O_DIRECTORY); 301 free(path); 302 303 return (fd); 304 } 305 306 static int 307 snapshot_request(const char *vmname, char *file, bool suspend) 308 { 309 nvlist_t *nvl; 310 int fd; 311 312 if ((fd = open_directory(file)) < 0) 313 return (errno); 314 315 nvl = nvlist_create(0); 316 nvlist_add_string(nvl, "cmd", "checkpoint"); 317 nvlist_add_string(nvl, "filename", basename(file)); 318 nvlist_add_bool(nvl, "suspend", suspend); 319 nvlist_move_descriptor(nvl, "fddir", fd); 320 321 return (send_message(vmname, nvl)); 322 } 323 #endif 324 325 int 326 main(int argc, char *argv[]) 327 { 328 char *vmname; 329 int error, ch, vcpuid; 330 struct vm_run vmrun; 331 struct vmctx *ctx; 332 struct vcpu *vcpu; 333 cpuset_t cpus; 334 struct option *opts; 335 #ifdef BHYVE_SNAPSHOT 336 char *checkpoint_file = NULL; 337 #endif 338 339 opts = setup_options(); 340 341 vcpuid = 0; 342 vmname = NULL; 343 progname = basename(argv[0]); 344 345 while ((ch = getopt_long(argc, argv, "", opts, NULL)) != -1) { 346 if (ch >= OPT_START_MD) { 347 bhyvectl_handle_opt(opts, ch); 348 continue; 349 } 350 351 switch (ch) { 352 case 0: 353 break; 354 case VMNAME: 355 vmname = optarg; 356 break; 357 case VCPU: 358 vcpuid = atoi(optarg); 359 break; 360 case SET_MEM: 361 memsize = atoi(optarg) * MB; 362 memsize = roundup(memsize, 2 * MB); 363 break; 364 case SET_CAP: 365 capval = strtoul(optarg, NULL, 0); 366 setcap = 1; 367 break; 368 case CAPNAME: 369 capname = optarg; 370 break; 371 #ifdef BHYVE_SNAPSHOT 372 case SET_CHECKPOINT_FILE: 373 case SET_SUSPEND_FILE: 374 if (checkpoint_file != NULL) 375 usage(opts); 376 377 checkpoint_file = optarg; 378 vm_suspend_opt = (ch == SET_SUSPEND_FILE); 379 break; 380 #endif 381 default: 382 usage(opts); 383 } 384 } 385 argc -= optind; 386 argv += optind; 387 388 if (vmname == NULL) 389 usage(opts); 390 391 error = 0; 392 393 if (!error && create) 394 error = vm_create(vmname); 395 396 if (!error) { 397 ctx = vm_open(vmname); 398 if (ctx == NULL) { 399 fprintf(stderr, 400 "vm_open: %s could not be opened: %s\n", 401 vmname, strerror(errno)); 402 exit(1); 403 } 404 vcpu = vm_vcpu_open(ctx, vcpuid); 405 } 406 407 if (!error && memsize) 408 error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL); 409 410 if (!error && (get_memseg || get_all)) 411 error = show_memseg(ctx); 412 413 if (!error && (get_memmap || get_all)) 414 error = show_memmap(ctx); 415 416 if (!error) 417 bhyvectl_md_main(ctx, vcpu, vcpuid, get_all); 418 419 if (!error && setcap) { 420 int captype; 421 422 captype = vm_capability_name2type(capname); 423 error = vm_set_capability(vcpu, captype, capval); 424 if (error != 0 && errno == ENOENT) 425 printf("Capability \"%s\" is not available\n", capname); 426 } 427 428 if (!error && (getcap || get_all)) { 429 int captype, val, getcaptype; 430 431 if (getcap && capname) 432 getcaptype = vm_capability_name2type(capname); 433 else 434 getcaptype = -1; 435 436 for (captype = 0; captype < VM_CAP_MAX; captype++) { 437 if (getcaptype >= 0 && captype != getcaptype) 438 continue; 439 error = vm_get_capability(vcpu, captype, &val); 440 if (error == 0) { 441 printf("Capability \"%s\" is %s on vcpu %d\n", 442 vm_capability_type2name(captype), 443 val ? "set" : "not set", vcpuid); 444 } else if (errno == ENOENT) { 445 error = 0; 446 printf("Capability \"%s\" is not available\n", 447 vm_capability_type2name(captype)); 448 } else { 449 break; 450 } 451 } 452 } 453 454 if (!error && (get_active_cpus || get_all)) { 455 error = vm_active_cpus(ctx, &cpus); 456 if (!error) 457 print_cpus("active cpus", &cpus); 458 } 459 460 if (!error && (get_debug_cpus || get_all)) { 461 error = vm_debug_cpus(ctx, &cpus); 462 if (!error) 463 print_cpus("debug cpus", &cpus); 464 } 465 466 if (!error && (get_suspended_cpus || get_all)) { 467 error = vm_suspended_cpus(ctx, &cpus); 468 if (!error) 469 print_cpus("suspended cpus", &cpus); 470 } 471 472 if (!error && (get_stats || get_all)) { 473 int i, num_stats; 474 uint64_t *stats; 475 struct timeval tv; 476 const char *desc; 477 478 stats = vm_get_stats(vcpu, &tv, &num_stats); 479 if (stats != NULL) { 480 printf("vcpu%d stats:\n", vcpuid); 481 for (i = 0; i < num_stats; i++) { 482 desc = vm_get_stat_desc(ctx, i); 483 printf("%-40s\t%ld\n", desc, stats[i]); 484 } 485 } 486 } 487 488 if (!error && (get_cpu_topology || get_all)) { 489 uint16_t sockets, cores, threads, maxcpus; 490 491 vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus); 492 printf("cpu_topology:\tsockets=%hu, cores=%hu, threads=%hu, " 493 "maxcpus=%hu\n", sockets, cores, threads, maxcpus); 494 } 495 496 if (!error && run) { 497 struct vm_exit vmexit; 498 cpuset_t cpuset; 499 500 vmrun.vm_exit = &vmexit; 501 vmrun.cpuset = &cpuset; 502 vmrun.cpusetsize = sizeof(cpuset); 503 error = vm_run(vcpu, &vmrun); 504 if (error == 0) 505 bhyvectl_dump_vm_run_exitcode(&vmexit, vcpuid); 506 else 507 printf("vm_run error %d\n", error); 508 } 509 510 if (!error && force_reset) 511 error = vm_suspend(ctx, VM_SUSPEND_RESET); 512 513 if (!error && force_poweroff) 514 error = vm_suspend(ctx, VM_SUSPEND_POWEROFF); 515 516 if (error) 517 printf("errno = %d\n", errno); 518 519 if (!error && destroy) 520 vm_destroy(ctx); 521 522 #ifdef BHYVE_SNAPSHOT 523 if (!error && checkpoint_file) 524 error = snapshot_request(vmname, checkpoint_file, vm_suspend_opt); 525 #endif 526 527 free(opts); 528 exit(error); 529 } 530