1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for VirtIO memory balloon devices. */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/endian.h> 35 #include <sys/kthread.h> 36 #include <sys/malloc.h> 37 #include <sys/module.h> 38 #include <sys/sglist.h> 39 #include <sys/sysctl.h> 40 #include <sys/lock.h> 41 #include <sys/mutex.h> 42 #include <sys/queue.h> 43 44 #include <vm/vm.h> 45 #include <vm/vm_page.h> 46 47 #include <machine/bus.h> 48 #include <machine/resource.h> 49 #include <sys/bus.h> 50 #include <sys/rman.h> 51 52 #include <dev/virtio/virtio.h> 53 #include <dev/virtio/virtqueue.h> 54 #include <dev/virtio/balloon/virtio_balloon.h> 55 56 #include "virtio_if.h" 57 58 struct vtballoon_softc { 59 device_t vtballoon_dev; 60 struct mtx vtballoon_mtx; 61 uint64_t vtballoon_features; 62 uint32_t vtballoon_flags; 63 #define VTBALLOON_FLAG_DETACH 0x01 64 65 struct virtqueue *vtballoon_inflate_vq; 66 struct virtqueue *vtballoon_deflate_vq; 67 68 uint32_t vtballoon_desired_npages; 69 uint32_t vtballoon_current_npages; 70 TAILQ_HEAD(,vm_page) vtballoon_pages; 71 72 struct thread *vtballoon_td; 73 uint32_t *vtballoon_page_frames; 74 int vtballoon_timeout; 75 }; 76 77 static struct virtio_feature_desc vtballoon_feature_desc[] = { 78 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" }, 79 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" }, 80 { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" }, 81 82 { 0, NULL } 83 }; 84 85 static int vtballoon_probe(device_t); 86 static int vtballoon_attach(device_t); 87 static int vtballoon_detach(device_t); 88 static int vtballoon_config_change(device_t); 89 90 static int vtballoon_negotiate_features(struct vtballoon_softc *); 91 static int vtballoon_setup_features(struct vtballoon_softc *); 92 static int vtballoon_alloc_virtqueues(struct vtballoon_softc *); 93 94 static void vtballoon_vq_intr(void *); 95 96 static void vtballoon_inflate(struct vtballoon_softc *, int); 97 static void vtballoon_deflate(struct vtballoon_softc *, int); 98 99 static void vtballoon_send_page_frames(struct vtballoon_softc *, 100 struct virtqueue *, int); 101 102 static void vtballoon_pop(struct vtballoon_softc *); 103 static void vtballoon_stop(struct vtballoon_softc *); 104 105 static vm_page_t 106 vtballoon_alloc_page(struct vtballoon_softc *); 107 static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t); 108 109 static int vtballoon_sleep(struct vtballoon_softc *); 110 static void vtballoon_thread(void *); 111 static void vtballoon_setup_sysctl(struct vtballoon_softc *); 112 113 #define vtballoon_modern(_sc) \ 114 (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0) 115 116 /* Features desired/implemented by this driver. */ 117 #define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST 118 119 /* Timeout between retries when the balloon needs inflating. */ 120 #define VTBALLOON_LOWMEM_TIMEOUT hz 121 122 /* 123 * Maximum number of pages we'll request to inflate or deflate 124 * the balloon in one virtqueue request. Both Linux and NetBSD 125 * have settled on 256, doing up to 1MB at a time. 126 */ 127 #define VTBALLOON_PAGES_PER_REQUEST 256 128 129 /* Must be able to fix all pages frames in one page (segment). */ 130 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE); 131 132 #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx 133 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \ 134 "VirtIO Balloon Lock", MTX_DEF) 135 #define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc))) 136 #define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc))) 137 #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc))) 138 139 static device_method_t vtballoon_methods[] = { 140 /* Device methods. */ 141 DEVMETHOD(device_probe, vtballoon_probe), 142 DEVMETHOD(device_attach, vtballoon_attach), 143 DEVMETHOD(device_detach, vtballoon_detach), 144 145 /* VirtIO methods. */ 146 DEVMETHOD(virtio_config_change, vtballoon_config_change), 147 148 DEVMETHOD_END 149 }; 150 151 static driver_t vtballoon_driver = { 152 "vtballoon", 153 vtballoon_methods, 154 sizeof(struct vtballoon_softc) 155 }; 156 157 VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver, 0, 0); 158 MODULE_VERSION(virtio_balloon, 1); 159 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1); 160 161 VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON, 162 "VirtIO Balloon Adapter"); 163 164 static int 165 vtballoon_probe(device_t dev) 166 { 167 return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon)); 168 } 169 170 static int 171 vtballoon_attach(device_t dev) 172 { 173 struct vtballoon_softc *sc; 174 int error; 175 176 sc = device_get_softc(dev); 177 sc->vtballoon_dev = dev; 178 virtio_set_feature_desc(dev, vtballoon_feature_desc); 179 180 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev)); 181 TAILQ_INIT(&sc->vtballoon_pages); 182 183 vtballoon_setup_sysctl(sc); 184 185 error = vtballoon_setup_features(sc); 186 if (error) { 187 device_printf(dev, "cannot setup features\n"); 188 goto fail; 189 } 190 191 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST * 192 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO); 193 if (sc->vtballoon_page_frames == NULL) { 194 error = ENOMEM; 195 device_printf(dev, 196 "cannot allocate page frame request array\n"); 197 goto fail; 198 } 199 200 error = vtballoon_alloc_virtqueues(sc); 201 if (error) { 202 device_printf(dev, "cannot allocate virtqueues\n"); 203 goto fail; 204 } 205 206 error = virtio_setup_intr(dev, INTR_TYPE_MISC); 207 if (error) { 208 device_printf(dev, "cannot setup virtqueue interrupts\n"); 209 goto fail; 210 } 211 212 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td, 213 0, 0, "virtio_balloon"); 214 if (error) { 215 device_printf(dev, "cannot create balloon kthread\n"); 216 goto fail; 217 } 218 219 virtqueue_enable_intr(sc->vtballoon_inflate_vq); 220 virtqueue_enable_intr(sc->vtballoon_deflate_vq); 221 222 fail: 223 if (error) 224 vtballoon_detach(dev); 225 226 return (error); 227 } 228 229 static int 230 vtballoon_detach(device_t dev) 231 { 232 struct vtballoon_softc *sc; 233 234 sc = device_get_softc(dev); 235 236 if (sc->vtballoon_td != NULL) { 237 VTBALLOON_LOCK(sc); 238 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH; 239 wakeup_one(sc); 240 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0); 241 VTBALLOON_UNLOCK(sc); 242 243 sc->vtballoon_td = NULL; 244 } 245 246 if (device_is_attached(dev)) { 247 vtballoon_pop(sc); 248 vtballoon_stop(sc); 249 } 250 251 if (sc->vtballoon_page_frames != NULL) { 252 free(sc->vtballoon_page_frames, M_DEVBUF); 253 sc->vtballoon_page_frames = NULL; 254 } 255 256 VTBALLOON_LOCK_DESTROY(sc); 257 258 return (0); 259 } 260 261 static int 262 vtballoon_config_change(device_t dev) 263 { 264 struct vtballoon_softc *sc; 265 266 sc = device_get_softc(dev); 267 268 VTBALLOON_LOCK(sc); 269 wakeup_one(sc); 270 VTBALLOON_UNLOCK(sc); 271 272 return (1); 273 } 274 275 static int 276 vtballoon_negotiate_features(struct vtballoon_softc *sc) 277 { 278 device_t dev; 279 uint64_t features; 280 281 dev = sc->vtballoon_dev; 282 features = VTBALLOON_FEATURES; 283 284 sc->vtballoon_features = virtio_negotiate_features(dev, features); 285 return (virtio_finalize_features(dev)); 286 } 287 288 static int 289 vtballoon_setup_features(struct vtballoon_softc *sc) 290 { 291 int error; 292 293 error = vtballoon_negotiate_features(sc); 294 if (error) 295 return (error); 296 297 return (0); 298 } 299 300 static int 301 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc) 302 { 303 device_t dev; 304 struct vq_alloc_info vq_info[2]; 305 int nvqs; 306 307 dev = sc->vtballoon_dev; 308 nvqs = 2; 309 310 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc, 311 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev)); 312 313 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc, 314 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev)); 315 316 return (virtio_alloc_virtqueues(dev, nvqs, vq_info)); 317 } 318 319 static void 320 vtballoon_vq_intr(void *xsc) 321 { 322 struct vtballoon_softc *sc; 323 324 sc = xsc; 325 326 VTBALLOON_LOCK(sc); 327 wakeup_one(sc); 328 VTBALLOON_UNLOCK(sc); 329 } 330 331 static void 332 vtballoon_inflate(struct vtballoon_softc *sc, int npages) 333 { 334 struct virtqueue *vq; 335 vm_page_t m; 336 int i; 337 338 vq = sc->vtballoon_inflate_vq; 339 340 if (npages > VTBALLOON_PAGES_PER_REQUEST) 341 npages = VTBALLOON_PAGES_PER_REQUEST; 342 343 for (i = 0; i < npages; i++) { 344 if ((m = vtballoon_alloc_page(sc)) == NULL) { 345 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT; 346 break; 347 } 348 349 sc->vtballoon_page_frames[i] = 350 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 351 352 KASSERT(m->a.queue == PQ_NONE, 353 ("%s: allocated page %p on queue", __func__, m)); 354 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q); 355 } 356 357 if (i > 0) 358 vtballoon_send_page_frames(sc, vq, i); 359 } 360 361 static void 362 vtballoon_deflate(struct vtballoon_softc *sc, int npages) 363 { 364 TAILQ_HEAD(, vm_page) free_pages; 365 struct virtqueue *vq; 366 vm_page_t m; 367 int i; 368 369 vq = sc->vtballoon_deflate_vq; 370 TAILQ_INIT(&free_pages); 371 372 if (npages > VTBALLOON_PAGES_PER_REQUEST) 373 npages = VTBALLOON_PAGES_PER_REQUEST; 374 375 for (i = 0; i < npages; i++) { 376 m = TAILQ_FIRST(&sc->vtballoon_pages); 377 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__)); 378 379 sc->vtballoon_page_frames[i] = 380 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 381 382 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q); 383 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q); 384 } 385 386 if (i > 0) { 387 /* Always tell host first before freeing the pages. */ 388 vtballoon_send_page_frames(sc, vq, i); 389 390 while ((m = TAILQ_FIRST(&free_pages)) != NULL) { 391 TAILQ_REMOVE(&free_pages, m, plinks.q); 392 vtballoon_free_page(sc, m); 393 } 394 } 395 396 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) && 397 sc->vtballoon_current_npages == 0) || 398 (!TAILQ_EMPTY(&sc->vtballoon_pages) && 399 sc->vtballoon_current_npages != 0), 400 ("%s: bogus page count %d", __func__, 401 sc->vtballoon_current_npages)); 402 } 403 404 static void 405 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq, 406 int npages) 407 { 408 struct sglist sg; 409 struct sglist_seg segs[1]; 410 void *c; 411 int error __diagused; 412 413 sglist_init(&sg, 1, segs); 414 415 error = sglist_append(&sg, sc->vtballoon_page_frames, 416 npages * sizeof(uint32_t)); 417 KASSERT(error == 0, ("error adding page frames to sglist")); 418 419 error = virtqueue_enqueue(vq, vq, &sg, 1, 0); 420 KASSERT(error == 0, ("error enqueuing page frames to virtqueue")); 421 virtqueue_notify(vq); 422 423 /* 424 * Inflate and deflate operations are done synchronously. The 425 * interrupt handler will wake us up. 426 */ 427 VTBALLOON_LOCK(sc); 428 while ((c = virtqueue_dequeue(vq, NULL)) == NULL) 429 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0); 430 VTBALLOON_UNLOCK(sc); 431 432 KASSERT(c == vq, ("unexpected balloon operation response")); 433 } 434 435 static void 436 vtballoon_pop(struct vtballoon_softc *sc) 437 { 438 439 while (!TAILQ_EMPTY(&sc->vtballoon_pages)) 440 vtballoon_deflate(sc, sc->vtballoon_current_npages); 441 } 442 443 static void 444 vtballoon_stop(struct vtballoon_softc *sc) 445 { 446 447 virtqueue_disable_intr(sc->vtballoon_inflate_vq); 448 virtqueue_disable_intr(sc->vtballoon_deflate_vq); 449 450 virtio_stop(sc->vtballoon_dev); 451 } 452 453 static vm_page_t 454 vtballoon_alloc_page(struct vtballoon_softc *sc) 455 { 456 vm_page_t m; 457 458 m = vm_page_alloc_noobj(VM_ALLOC_NODUMP); 459 if (m != NULL) 460 sc->vtballoon_current_npages++; 461 462 return (m); 463 } 464 465 static void 466 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m) 467 { 468 469 vm_page_free(m); 470 sc->vtballoon_current_npages--; 471 } 472 473 static uint32_t 474 vtballoon_desired_size(struct vtballoon_softc *sc) 475 { 476 uint32_t desired; 477 478 desired = virtio_read_dev_config_4(sc->vtballoon_dev, 479 offsetof(struct virtio_balloon_config, num_pages)); 480 481 if (vtballoon_modern(sc)) 482 return (desired); 483 else 484 return (le32toh(desired)); 485 } 486 487 static void 488 vtballoon_update_size(struct vtballoon_softc *sc) 489 { 490 uint32_t npages; 491 492 npages = sc->vtballoon_current_npages; 493 if (!vtballoon_modern(sc)) 494 npages = htole32(npages); 495 496 virtio_write_dev_config_4(sc->vtballoon_dev, 497 offsetof(struct virtio_balloon_config, actual), npages); 498 } 499 500 static int 501 vtballoon_sleep(struct vtballoon_softc *sc) 502 { 503 int rc, timeout; 504 uint32_t current, desired; 505 506 rc = 0; 507 current = sc->vtballoon_current_npages; 508 509 VTBALLOON_LOCK(sc); 510 for (;;) { 511 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) { 512 rc = 1; 513 break; 514 } 515 516 desired = vtballoon_desired_size(sc); 517 sc->vtballoon_desired_npages = desired; 518 519 /* 520 * If given, use non-zero timeout on the first time through 521 * the loop. On subsequent times, timeout will be zero so 522 * we will reevaluate the desired size of the balloon and 523 * break out to retry if needed. 524 */ 525 timeout = sc->vtballoon_timeout; 526 sc->vtballoon_timeout = 0; 527 528 if (current > desired) 529 break; 530 if (current < desired && timeout == 0) 531 break; 532 533 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout); 534 } 535 VTBALLOON_UNLOCK(sc); 536 537 return (rc); 538 } 539 540 static void 541 vtballoon_thread(void *xsc) 542 { 543 struct vtballoon_softc *sc; 544 uint32_t current, desired; 545 546 sc = xsc; 547 548 for (;;) { 549 if (vtballoon_sleep(sc) != 0) 550 break; 551 552 current = sc->vtballoon_current_npages; 553 desired = sc->vtballoon_desired_npages; 554 555 if (desired != current) { 556 if (desired > current) 557 vtballoon_inflate(sc, desired - current); 558 else 559 vtballoon_deflate(sc, current - desired); 560 561 vtballoon_update_size(sc); 562 } 563 } 564 565 kthread_exit(); 566 } 567 568 static void 569 vtballoon_setup_sysctl(struct vtballoon_softc *sc) 570 { 571 device_t dev; 572 struct sysctl_ctx_list *ctx; 573 struct sysctl_oid *tree; 574 struct sysctl_oid_list *child; 575 576 dev = sc->vtballoon_dev; 577 ctx = device_get_sysctl_ctx(dev); 578 tree = device_get_sysctl_tree(dev); 579 child = SYSCTL_CHILDREN(tree); 580 581 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired", 582 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t), 583 "Desired balloon size in pages"); 584 585 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current", 586 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t), 587 "Current balloon size in pages"); 588 } 589