1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for VirtIO memory balloon devices. */ 30 31 #include <sys/cdefs.h> 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/endian.h> 36 #include <sys/kthread.h> 37 #include <sys/malloc.h> 38 #include <sys/module.h> 39 #include <sys/sglist.h> 40 #include <sys/sysctl.h> 41 #include <sys/lock.h> 42 #include <sys/mutex.h> 43 #include <sys/queue.h> 44 45 #include <vm/vm.h> 46 #include <vm/vm_page.h> 47 48 #include <machine/bus.h> 49 #include <machine/resource.h> 50 #include <sys/bus.h> 51 #include <sys/rman.h> 52 53 #include <dev/virtio/virtio.h> 54 #include <dev/virtio/virtqueue.h> 55 #include <dev/virtio/balloon/virtio_balloon.h> 56 57 #include "virtio_if.h" 58 59 struct vtballoon_softc { 60 device_t vtballoon_dev; 61 struct mtx vtballoon_mtx; 62 uint64_t vtballoon_features; 63 uint32_t vtballoon_flags; 64 #define VTBALLOON_FLAG_DETACH 0x01 65 66 struct virtqueue *vtballoon_inflate_vq; 67 struct virtqueue *vtballoon_deflate_vq; 68 69 uint32_t vtballoon_desired_npages; 70 uint32_t vtballoon_current_npages; 71 TAILQ_HEAD(,vm_page) vtballoon_pages; 72 73 struct thread *vtballoon_td; 74 uint32_t *vtballoon_page_frames; 75 int vtballoon_timeout; 76 }; 77 78 static struct virtio_feature_desc vtballoon_feature_desc[] = { 79 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" }, 80 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" }, 81 { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" }, 82 83 { 0, NULL } 84 }; 85 86 static int vtballoon_probe(device_t); 87 static int vtballoon_attach(device_t); 88 static int vtballoon_detach(device_t); 89 static int vtballoon_config_change(device_t); 90 91 static int vtballoon_negotiate_features(struct vtballoon_softc *); 92 static int vtballoon_setup_features(struct vtballoon_softc *); 93 static int vtballoon_alloc_virtqueues(struct vtballoon_softc *); 94 95 static void vtballoon_vq_intr(void *); 96 97 static void vtballoon_inflate(struct vtballoon_softc *, int); 98 static void vtballoon_deflate(struct vtballoon_softc *, int); 99 100 static void vtballoon_send_page_frames(struct vtballoon_softc *, 101 struct virtqueue *, int); 102 103 static void vtballoon_pop(struct vtballoon_softc *); 104 static void vtballoon_stop(struct vtballoon_softc *); 105 106 static vm_page_t 107 vtballoon_alloc_page(struct vtballoon_softc *); 108 static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t); 109 110 static int vtballoon_sleep(struct vtballoon_softc *); 111 static void vtballoon_thread(void *); 112 static void vtballoon_setup_sysctl(struct vtballoon_softc *); 113 114 #define vtballoon_modern(_sc) \ 115 (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0) 116 117 /* Features desired/implemented by this driver. */ 118 #define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST 119 120 /* Timeout between retries when the balloon needs inflating. */ 121 #define VTBALLOON_LOWMEM_TIMEOUT hz 122 123 /* 124 * Maximum number of pages we'll request to inflate or deflate 125 * the balloon in one virtqueue request. Both Linux and NetBSD 126 * have settled on 256, doing up to 1MB at a time. 127 */ 128 #define VTBALLOON_PAGES_PER_REQUEST 256 129 130 /* Must be able to fix all pages frames in one page (segment). */ 131 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE); 132 133 #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx 134 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \ 135 "VirtIO Balloon Lock", MTX_DEF) 136 #define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc))) 137 #define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc))) 138 #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc))) 139 140 static device_method_t vtballoon_methods[] = { 141 /* Device methods. */ 142 DEVMETHOD(device_probe, vtballoon_probe), 143 DEVMETHOD(device_attach, vtballoon_attach), 144 DEVMETHOD(device_detach, vtballoon_detach), 145 146 /* VirtIO methods. */ 147 DEVMETHOD(virtio_config_change, vtballoon_config_change), 148 149 DEVMETHOD_END 150 }; 151 152 static driver_t vtballoon_driver = { 153 "vtballoon", 154 vtballoon_methods, 155 sizeof(struct vtballoon_softc) 156 }; 157 158 VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver, 0, 0); 159 MODULE_VERSION(virtio_balloon, 1); 160 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1); 161 162 VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON, 163 "VirtIO Balloon Adapter"); 164 165 static int 166 vtballoon_probe(device_t dev) 167 { 168 return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon)); 169 } 170 171 static int 172 vtballoon_attach(device_t dev) 173 { 174 struct vtballoon_softc *sc; 175 int error; 176 177 sc = device_get_softc(dev); 178 sc->vtballoon_dev = dev; 179 virtio_set_feature_desc(dev, vtballoon_feature_desc); 180 181 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev)); 182 TAILQ_INIT(&sc->vtballoon_pages); 183 184 vtballoon_setup_sysctl(sc); 185 186 error = vtballoon_setup_features(sc); 187 if (error) { 188 device_printf(dev, "cannot setup features\n"); 189 goto fail; 190 } 191 192 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST * 193 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO); 194 if (sc->vtballoon_page_frames == NULL) { 195 error = ENOMEM; 196 device_printf(dev, 197 "cannot allocate page frame request array\n"); 198 goto fail; 199 } 200 201 error = vtballoon_alloc_virtqueues(sc); 202 if (error) { 203 device_printf(dev, "cannot allocate virtqueues\n"); 204 goto fail; 205 } 206 207 error = virtio_setup_intr(dev, INTR_TYPE_MISC); 208 if (error) { 209 device_printf(dev, "cannot setup virtqueue interrupts\n"); 210 goto fail; 211 } 212 213 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td, 214 0, 0, "virtio_balloon"); 215 if (error) { 216 device_printf(dev, "cannot create balloon kthread\n"); 217 goto fail; 218 } 219 220 virtqueue_enable_intr(sc->vtballoon_inflate_vq); 221 virtqueue_enable_intr(sc->vtballoon_deflate_vq); 222 223 fail: 224 if (error) 225 vtballoon_detach(dev); 226 227 return (error); 228 } 229 230 static int 231 vtballoon_detach(device_t dev) 232 { 233 struct vtballoon_softc *sc; 234 235 sc = device_get_softc(dev); 236 237 if (sc->vtballoon_td != NULL) { 238 VTBALLOON_LOCK(sc); 239 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH; 240 wakeup_one(sc); 241 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0); 242 VTBALLOON_UNLOCK(sc); 243 244 sc->vtballoon_td = NULL; 245 } 246 247 if (device_is_attached(dev)) { 248 vtballoon_pop(sc); 249 vtballoon_stop(sc); 250 } 251 252 if (sc->vtballoon_page_frames != NULL) { 253 free(sc->vtballoon_page_frames, M_DEVBUF); 254 sc->vtballoon_page_frames = NULL; 255 } 256 257 VTBALLOON_LOCK_DESTROY(sc); 258 259 return (0); 260 } 261 262 static int 263 vtballoon_config_change(device_t dev) 264 { 265 struct vtballoon_softc *sc; 266 267 sc = device_get_softc(dev); 268 269 VTBALLOON_LOCK(sc); 270 wakeup_one(sc); 271 VTBALLOON_UNLOCK(sc); 272 273 return (1); 274 } 275 276 static int 277 vtballoon_negotiate_features(struct vtballoon_softc *sc) 278 { 279 device_t dev; 280 uint64_t features; 281 282 dev = sc->vtballoon_dev; 283 features = VTBALLOON_FEATURES; 284 285 sc->vtballoon_features = virtio_negotiate_features(dev, features); 286 return (virtio_finalize_features(dev)); 287 } 288 289 static int 290 vtballoon_setup_features(struct vtballoon_softc *sc) 291 { 292 int error; 293 294 error = vtballoon_negotiate_features(sc); 295 if (error) 296 return (error); 297 298 return (0); 299 } 300 301 static int 302 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc) 303 { 304 device_t dev; 305 struct vq_alloc_info vq_info[2]; 306 int nvqs; 307 308 dev = sc->vtballoon_dev; 309 nvqs = 2; 310 311 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc, 312 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev)); 313 314 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc, 315 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev)); 316 317 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 318 } 319 320 static void 321 vtballoon_vq_intr(void *xsc) 322 { 323 struct vtballoon_softc *sc; 324 325 sc = xsc; 326 327 VTBALLOON_LOCK(sc); 328 wakeup_one(sc); 329 VTBALLOON_UNLOCK(sc); 330 } 331 332 static void 333 vtballoon_inflate(struct vtballoon_softc *sc, int npages) 334 { 335 struct virtqueue *vq; 336 vm_page_t m; 337 int i; 338 339 vq = sc->vtballoon_inflate_vq; 340 341 if (npages > VTBALLOON_PAGES_PER_REQUEST) 342 npages = VTBALLOON_PAGES_PER_REQUEST; 343 344 for (i = 0; i < npages; i++) { 345 if ((m = vtballoon_alloc_page(sc)) == NULL) { 346 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT; 347 break; 348 } 349 350 sc->vtballoon_page_frames[i] = 351 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 352 353 KASSERT(m->a.queue == PQ_NONE, 354 ("%s: allocated page %p on queue", __func__, m)); 355 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q); 356 } 357 358 if (i > 0) 359 vtballoon_send_page_frames(sc, vq, i); 360 } 361 362 static void 363 vtballoon_deflate(struct vtballoon_softc *sc, int npages) 364 { 365 TAILQ_HEAD(, vm_page) free_pages; 366 struct virtqueue *vq; 367 vm_page_t m; 368 int i; 369 370 vq = sc->vtballoon_deflate_vq; 371 TAILQ_INIT(&free_pages); 372 373 if (npages > VTBALLOON_PAGES_PER_REQUEST) 374 npages = VTBALLOON_PAGES_PER_REQUEST; 375 376 for (i = 0; i < npages; i++) { 377 m = TAILQ_FIRST(&sc->vtballoon_pages); 378 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__)); 379 380 sc->vtballoon_page_frames[i] = 381 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 382 383 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q); 384 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q); 385 } 386 387 if (i > 0) { 388 /* Always tell host first before freeing the pages. */ 389 vtballoon_send_page_frames(sc, vq, i); 390 391 while ((m = TAILQ_FIRST(&free_pages)) != NULL) { 392 TAILQ_REMOVE(&free_pages, m, plinks.q); 393 vtballoon_free_page(sc, m); 394 } 395 } 396 397 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) && 398 sc->vtballoon_current_npages == 0) || 399 (!TAILQ_EMPTY(&sc->vtballoon_pages) && 400 sc->vtballoon_current_npages != 0), 401 ("%s: bogus page count %d", __func__, 402 sc->vtballoon_current_npages)); 403 } 404 405 static void 406 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq, 407 int npages) 408 { 409 struct sglist sg; 410 struct sglist_seg segs[1]; 411 void *c; 412 int error __diagused; 413 414 sglist_init(&sg, 1, segs); 415 416 error = sglist_append(&sg, sc->vtballoon_page_frames, 417 npages * sizeof(uint32_t)); 418 KASSERT(error == 0, ("error adding page frames to sglist")); 419 420 error = virtqueue_enqueue(vq, vq, &sg, 1, 0); 421 KASSERT(error == 0, ("error enqueuing page frames to virtqueue")); 422 virtqueue_notify(vq); 423 424 /* 425 * Inflate and deflate operations are done synchronously. The 426 * interrupt handler will wake us up. 427 */ 428 VTBALLOON_LOCK(sc); 429 while ((c = virtqueue_dequeue(vq, NULL)) == NULL) 430 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0); 431 VTBALLOON_UNLOCK(sc); 432 433 KASSERT(c == vq, ("unexpected balloon operation response")); 434 } 435 436 static void 437 vtballoon_pop(struct vtballoon_softc *sc) 438 { 439 440 while (!TAILQ_EMPTY(&sc->vtballoon_pages)) 441 vtballoon_deflate(sc, sc->vtballoon_current_npages); 442 } 443 444 static void 445 vtballoon_stop(struct vtballoon_softc *sc) 446 { 447 448 virtqueue_disable_intr(sc->vtballoon_inflate_vq); 449 virtqueue_disable_intr(sc->vtballoon_deflate_vq); 450 451 virtio_stop(sc->vtballoon_dev); 452 } 453 454 static vm_page_t 455 vtballoon_alloc_page(struct vtballoon_softc *sc) 456 { 457 vm_page_t m; 458 459 m = vm_page_alloc_noobj(VM_ALLOC_NODUMP); 460 if (m != NULL) 461 sc->vtballoon_current_npages++; 462 463 return (m); 464 } 465 466 static void 467 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m) 468 { 469 470 vm_page_free(m); 471 sc->vtballoon_current_npages--; 472 } 473 474 static uint32_t 475 vtballoon_desired_size(struct vtballoon_softc *sc) 476 { 477 uint32_t desired; 478 479 desired = virtio_read_dev_config_4(sc->vtballoon_dev, 480 offsetof(struct virtio_balloon_config, num_pages)); 481 482 if (vtballoon_modern(sc)) 483 return (desired); 484 else 485 return (le32toh(desired)); 486 } 487 488 static void 489 vtballoon_update_size(struct vtballoon_softc *sc) 490 { 491 uint32_t npages; 492 493 npages = sc->vtballoon_current_npages; 494 if (!vtballoon_modern(sc)) 495 npages = htole32(npages); 496 497 virtio_write_dev_config_4(sc->vtballoon_dev, 498 offsetof(struct virtio_balloon_config, actual), npages); 499 } 500 501 static int 502 vtballoon_sleep(struct vtballoon_softc *sc) 503 { 504 int rc, timeout; 505 uint32_t current, desired; 506 507 rc = 0; 508 current = sc->vtballoon_current_npages; 509 510 VTBALLOON_LOCK(sc); 511 for (;;) { 512 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) { 513 rc = 1; 514 break; 515 } 516 517 desired = vtballoon_desired_size(sc); 518 sc->vtballoon_desired_npages = desired; 519 520 /* 521 * If given, use non-zero timeout on the first time through 522 * the loop. On subsequent times, timeout will be zero so 523 * we will reevaluate the desired size of the balloon and 524 * break out to retry if needed. 525 */ 526 timeout = sc->vtballoon_timeout; 527 sc->vtballoon_timeout = 0; 528 529 if (current > desired) 530 break; 531 if (current < desired && timeout == 0) 532 break; 533 534 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout); 535 } 536 VTBALLOON_UNLOCK(sc); 537 538 return (rc); 539 } 540 541 static void 542 vtballoon_thread(void *xsc) 543 { 544 struct vtballoon_softc *sc; 545 uint32_t current, desired; 546 547 sc = xsc; 548 549 for (;;) { 550 if (vtballoon_sleep(sc) != 0) 551 break; 552 553 current = sc->vtballoon_current_npages; 554 desired = sc->vtballoon_desired_npages; 555 556 if (desired != current) { 557 if (desired > current) 558 vtballoon_inflate(sc, desired - current); 559 else 560 vtballoon_deflate(sc, current - desired); 561 562 vtballoon_update_size(sc); 563 } 564 } 565 566 kthread_exit(); 567 } 568 569 static void 570 vtballoon_setup_sysctl(struct vtballoon_softc *sc) 571 { 572 device_t dev; 573 struct sysctl_ctx_list *ctx; 574 struct sysctl_oid *tree; 575 struct sysctl_oid_list *child; 576 577 dev = sc->vtballoon_dev; 578 ctx = device_get_sysctl_ctx(dev); 579 tree = device_get_sysctl_tree(dev); 580 child = SYSCTL_CHILDREN(tree); 581 582 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired", 583 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t), 584 "Desired balloon size in pages"); 585 586 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current", 587 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t), 588 "Current balloon size in pages"); 589 } 590