1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for VirtIO memory balloon devices. */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/endian.h> 38 #include <sys/kthread.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/sglist.h> 42 #include <sys/sysctl.h> 43 #include <sys/lock.h> 44 #include <sys/mutex.h> 45 #include <sys/queue.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_page.h> 49 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 #include <sys/bus.h> 53 #include <sys/rman.h> 54 55 #include <dev/virtio/virtio.h> 56 #include <dev/virtio/virtqueue.h> 57 #include <dev/virtio/balloon/virtio_balloon.h> 58 59 #include "virtio_if.h" 60 61 struct vtballoon_softc { 62 device_t vtballoon_dev; 63 struct mtx vtballoon_mtx; 64 uint64_t vtballoon_features; 65 uint32_t vtballoon_flags; 66 #define VTBALLOON_FLAG_DETACH 0x01 67 68 struct virtqueue *vtballoon_inflate_vq; 69 struct virtqueue *vtballoon_deflate_vq; 70 71 uint32_t vtballoon_desired_npages; 72 uint32_t vtballoon_current_npages; 73 TAILQ_HEAD(,vm_page) vtballoon_pages; 74 75 struct thread *vtballoon_td; 76 uint32_t *vtballoon_page_frames; 77 int vtballoon_timeout; 78 }; 79 80 static struct virtio_feature_desc vtballoon_feature_desc[] = { 81 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" }, 82 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" }, 83 84 { 0, NULL } 85 }; 86 87 static int vtballoon_probe(device_t); 88 static int vtballoon_attach(device_t); 89 static int vtballoon_detach(device_t); 90 static int vtballoon_config_change(device_t); 91 92 static void vtballoon_negotiate_features(struct vtballoon_softc *); 93 static int vtballoon_alloc_virtqueues(struct vtballoon_softc *); 94 95 static void vtballoon_vq_intr(void *); 96 97 static void vtballoon_inflate(struct vtballoon_softc *, int); 98 static void vtballoon_deflate(struct vtballoon_softc *, int); 99 100 static void vtballoon_send_page_frames(struct vtballoon_softc *, 101 struct virtqueue *, int); 102 103 static void vtballoon_pop(struct vtballoon_softc *); 104 static void vtballoon_stop(struct vtballoon_softc *); 105 106 static vm_page_t 107 vtballoon_alloc_page(struct vtballoon_softc *); 108 static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t); 109 110 static int vtballoon_sleep(struct vtballoon_softc *); 111 static void vtballoon_thread(void *); 112 static void vtballoon_add_sysctl(struct vtballoon_softc *); 113 114 /* Features desired/implemented by this driver. */ 115 #define VTBALLOON_FEATURES 0 116 117 /* Timeout between retries when the balloon needs inflating. */ 118 #define VTBALLOON_LOWMEM_TIMEOUT hz 119 120 /* 121 * Maximum number of pages we'll request to inflate or deflate 122 * the balloon in one virtqueue request. Both Linux and NetBSD 123 * have settled on 256, doing up to 1MB at a time. 124 */ 125 #define VTBALLOON_PAGES_PER_REQUEST 256 126 127 /* Must be able to fix all pages frames in one page (segment). */ 128 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE); 129 130 #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx 131 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \ 132 "VirtIO Balloon Lock", MTX_DEF) 133 #define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc))) 134 #define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc))) 135 #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc))) 136 137 static device_method_t vtballoon_methods[] = { 138 /* Device methods. */ 139 DEVMETHOD(device_probe, vtballoon_probe), 140 DEVMETHOD(device_attach, vtballoon_attach), 141 DEVMETHOD(device_detach, vtballoon_detach), 142 143 /* VirtIO methods. */ 144 DEVMETHOD(virtio_config_change, vtballoon_config_change), 145 146 DEVMETHOD_END 147 }; 148 149 static driver_t vtballoon_driver = { 150 "vtballoon", 151 vtballoon_methods, 152 sizeof(struct vtballoon_softc) 153 }; 154 static devclass_t vtballoon_devclass; 155 156 DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver, 157 vtballoon_devclass, 0, 0); 158 MODULE_VERSION(virtio_balloon, 1); 159 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1); 160 161 static int 162 vtballoon_probe(device_t dev) 163 { 164 165 if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON) 166 return (ENXIO); 167 168 device_set_desc(dev, "VirtIO Balloon Adapter"); 169 170 return (BUS_PROBE_DEFAULT); 171 } 172 173 static int 174 vtballoon_attach(device_t dev) 175 { 176 struct vtballoon_softc *sc; 177 int error; 178 179 sc = device_get_softc(dev); 180 sc->vtballoon_dev = dev; 181 182 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev)); 183 TAILQ_INIT(&sc->vtballoon_pages); 184 185 vtballoon_add_sysctl(sc); 186 187 virtio_set_feature_desc(dev, vtballoon_feature_desc); 188 vtballoon_negotiate_features(sc); 189 190 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST * 191 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO); 192 if (sc->vtballoon_page_frames == NULL) { 193 error = ENOMEM; 194 device_printf(dev, 195 "cannot allocate page frame request array\n"); 196 goto fail; 197 } 198 199 error = vtballoon_alloc_virtqueues(sc); 200 if (error) { 201 device_printf(dev, "cannot allocate virtqueues\n"); 202 goto fail; 203 } 204 205 error = virtio_setup_intr(dev, INTR_TYPE_MISC); 206 if (error) { 207 device_printf(dev, "cannot setup virtqueue interrupts\n"); 208 goto fail; 209 } 210 211 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td, 212 0, 0, "virtio_balloon"); 213 if (error) { 214 device_printf(dev, "cannot create balloon kthread\n"); 215 goto fail; 216 } 217 218 virtqueue_enable_intr(sc->vtballoon_inflate_vq); 219 virtqueue_enable_intr(sc->vtballoon_deflate_vq); 220 221 fail: 222 if (error) 223 vtballoon_detach(dev); 224 225 return (error); 226 } 227 228 static int 229 vtballoon_detach(device_t dev) 230 { 231 struct vtballoon_softc *sc; 232 233 sc = device_get_softc(dev); 234 235 if (sc->vtballoon_td != NULL) { 236 VTBALLOON_LOCK(sc); 237 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH; 238 wakeup_one(sc); 239 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0); 240 VTBALLOON_UNLOCK(sc); 241 242 sc->vtballoon_td = NULL; 243 } 244 245 if (device_is_attached(dev)) { 246 vtballoon_pop(sc); 247 vtballoon_stop(sc); 248 } 249 250 if (sc->vtballoon_page_frames != NULL) { 251 free(sc->vtballoon_page_frames, M_DEVBUF); 252 sc->vtballoon_page_frames = NULL; 253 } 254 255 VTBALLOON_LOCK_DESTROY(sc); 256 257 return (0); 258 } 259 260 static int 261 vtballoon_config_change(device_t dev) 262 { 263 struct vtballoon_softc *sc; 264 265 sc = device_get_softc(dev); 266 267 VTBALLOON_LOCK(sc); 268 wakeup_one(sc); 269 VTBALLOON_UNLOCK(sc); 270 271 return (1); 272 } 273 274 static void 275 vtballoon_negotiate_features(struct vtballoon_softc *sc) 276 { 277 device_t dev; 278 uint64_t features; 279 280 dev = sc->vtballoon_dev; 281 features = virtio_negotiate_features(dev, VTBALLOON_FEATURES); 282 sc->vtballoon_features = features; 283 } 284 285 static int 286 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc) 287 { 288 device_t dev; 289 struct vq_alloc_info vq_info[2]; 290 int nvqs; 291 292 dev = sc->vtballoon_dev; 293 nvqs = 2; 294 295 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc, 296 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev)); 297 298 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc, 299 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev)); 300 301 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 302 } 303 304 static void 305 vtballoon_vq_intr(void *xsc) 306 { 307 struct vtballoon_softc *sc; 308 309 sc = xsc; 310 311 VTBALLOON_LOCK(sc); 312 wakeup_one(sc); 313 VTBALLOON_UNLOCK(sc); 314 } 315 316 static void 317 vtballoon_inflate(struct vtballoon_softc *sc, int npages) 318 { 319 struct virtqueue *vq; 320 vm_page_t m; 321 int i; 322 323 vq = sc->vtballoon_inflate_vq; 324 325 if (npages > VTBALLOON_PAGES_PER_REQUEST) 326 npages = VTBALLOON_PAGES_PER_REQUEST; 327 328 for (i = 0; i < npages; i++) { 329 if ((m = vtballoon_alloc_page(sc)) == NULL) { 330 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT; 331 break; 332 } 333 334 sc->vtballoon_page_frames[i] = 335 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 336 337 KASSERT(m->queue == PQ_NONE, 338 ("%s: allocated page %p on queue", __func__, m)); 339 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q); 340 } 341 342 if (i > 0) 343 vtballoon_send_page_frames(sc, vq, i); 344 } 345 346 static void 347 vtballoon_deflate(struct vtballoon_softc *sc, int npages) 348 { 349 TAILQ_HEAD(, vm_page) free_pages; 350 struct virtqueue *vq; 351 vm_page_t m; 352 int i; 353 354 vq = sc->vtballoon_deflate_vq; 355 TAILQ_INIT(&free_pages); 356 357 if (npages > VTBALLOON_PAGES_PER_REQUEST) 358 npages = VTBALLOON_PAGES_PER_REQUEST; 359 360 for (i = 0; i < npages; i++) { 361 m = TAILQ_FIRST(&sc->vtballoon_pages); 362 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__)); 363 364 sc->vtballoon_page_frames[i] = 365 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 366 367 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q); 368 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q); 369 } 370 371 if (i > 0) { 372 /* Always tell host first before freeing the pages. */ 373 vtballoon_send_page_frames(sc, vq, i); 374 375 while ((m = TAILQ_FIRST(&free_pages)) != NULL) { 376 TAILQ_REMOVE(&free_pages, m, plinks.q); 377 vtballoon_free_page(sc, m); 378 } 379 } 380 381 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) && 382 sc->vtballoon_current_npages == 0) || 383 (!TAILQ_EMPTY(&sc->vtballoon_pages) && 384 sc->vtballoon_current_npages != 0), 385 ("%s: bogus page count %d", __func__, 386 sc->vtballoon_current_npages)); 387 } 388 389 static void 390 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq, 391 int npages) 392 { 393 struct sglist sg; 394 struct sglist_seg segs[1]; 395 void *c; 396 int error; 397 398 sglist_init(&sg, 1, segs); 399 400 error = sglist_append(&sg, sc->vtballoon_page_frames, 401 npages * sizeof(uint32_t)); 402 KASSERT(error == 0, ("error adding page frames to sglist")); 403 404 error = virtqueue_enqueue(vq, vq, &sg, 1, 0); 405 KASSERT(error == 0, ("error enqueuing page frames to virtqueue")); 406 virtqueue_notify(vq); 407 408 /* 409 * Inflate and deflate operations are done synchronously. The 410 * interrupt handler will wake us up. 411 */ 412 VTBALLOON_LOCK(sc); 413 while ((c = virtqueue_dequeue(vq, NULL)) == NULL) 414 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0); 415 VTBALLOON_UNLOCK(sc); 416 417 KASSERT(c == vq, ("unexpected balloon operation response")); 418 } 419 420 static void 421 vtballoon_pop(struct vtballoon_softc *sc) 422 { 423 424 while (!TAILQ_EMPTY(&sc->vtballoon_pages)) 425 vtballoon_deflate(sc, sc->vtballoon_current_npages); 426 } 427 428 static void 429 vtballoon_stop(struct vtballoon_softc *sc) 430 { 431 432 virtqueue_disable_intr(sc->vtballoon_inflate_vq); 433 virtqueue_disable_intr(sc->vtballoon_deflate_vq); 434 435 virtio_stop(sc->vtballoon_dev); 436 } 437 438 static vm_page_t 439 vtballoon_alloc_page(struct vtballoon_softc *sc) 440 { 441 vm_page_t m; 442 443 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ); 444 if (m != NULL) 445 sc->vtballoon_current_npages++; 446 447 return (m); 448 } 449 450 static void 451 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m) 452 { 453 454 vm_page_free(m); 455 sc->vtballoon_current_npages--; 456 } 457 458 static uint32_t 459 vtballoon_desired_size(struct vtballoon_softc *sc) 460 { 461 uint32_t desired; 462 463 desired = virtio_read_dev_config_4(sc->vtballoon_dev, 464 offsetof(struct virtio_balloon_config, num_pages)); 465 466 return (le32toh(desired)); 467 } 468 469 static void 470 vtballoon_update_size(struct vtballoon_softc *sc) 471 { 472 473 virtio_write_dev_config_4(sc->vtballoon_dev, 474 offsetof(struct virtio_balloon_config, actual), 475 htole32(sc->vtballoon_current_npages)); 476 } 477 478 static int 479 vtballoon_sleep(struct vtballoon_softc *sc) 480 { 481 int rc, timeout; 482 uint32_t current, desired; 483 484 rc = 0; 485 current = sc->vtballoon_current_npages; 486 487 VTBALLOON_LOCK(sc); 488 for (;;) { 489 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) { 490 rc = 1; 491 break; 492 } 493 494 desired = vtballoon_desired_size(sc); 495 sc->vtballoon_desired_npages = desired; 496 497 /* 498 * If given, use non-zero timeout on the first time through 499 * the loop. On subsequent times, timeout will be zero so 500 * we will reevaluate the desired size of the balloon and 501 * break out to retry if needed. 502 */ 503 timeout = sc->vtballoon_timeout; 504 sc->vtballoon_timeout = 0; 505 506 if (current > desired) 507 break; 508 if (current < desired && timeout == 0) 509 break; 510 511 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout); 512 } 513 VTBALLOON_UNLOCK(sc); 514 515 return (rc); 516 } 517 518 static void 519 vtballoon_thread(void *xsc) 520 { 521 struct vtballoon_softc *sc; 522 uint32_t current, desired; 523 524 sc = xsc; 525 526 for (;;) { 527 if (vtballoon_sleep(sc) != 0) 528 break; 529 530 current = sc->vtballoon_current_npages; 531 desired = sc->vtballoon_desired_npages; 532 533 if (desired != current) { 534 if (desired > current) 535 vtballoon_inflate(sc, desired - current); 536 else 537 vtballoon_deflate(sc, current - desired); 538 539 vtballoon_update_size(sc); 540 } 541 } 542 543 kthread_exit(); 544 } 545 546 static void 547 vtballoon_add_sysctl(struct vtballoon_softc *sc) 548 { 549 device_t dev; 550 struct sysctl_ctx_list *ctx; 551 struct sysctl_oid *tree; 552 struct sysctl_oid_list *child; 553 554 dev = sc->vtballoon_dev; 555 ctx = device_get_sysctl_ctx(dev); 556 tree = device_get_sysctl_tree(dev); 557 child = SYSCTL_CHILDREN(tree); 558 559 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired", 560 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t), 561 "Desired balloon size in pages"); 562 563 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current", 564 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t), 565 "Current balloon size in pages"); 566 } 567