1 /*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 /* Driver for VirtIO memory balloon devices. */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/endian.h> 36 #include <sys/kthread.h> 37 #include <sys/malloc.h> 38 #include <sys/module.h> 39 #include <sys/sglist.h> 40 #include <sys/sysctl.h> 41 #include <sys/lock.h> 42 #include <sys/mutex.h> 43 #include <sys/queue.h> 44 45 #include <vm/vm.h> 46 #include <vm/vm_page.h> 47 48 #include <machine/bus.h> 49 #include <machine/resource.h> 50 #include <sys/bus.h> 51 #include <sys/rman.h> 52 53 #include <dev/virtio/virtio.h> 54 #include <dev/virtio/virtqueue.h> 55 #include <dev/virtio/balloon/virtio_balloon.h> 56 57 #include "virtio_if.h" 58 59 struct vtballoon_softc { 60 device_t vtballoon_dev; 61 struct mtx vtballoon_mtx; 62 uint64_t vtballoon_features; 63 uint32_t vtballoon_flags; 64 #define VTBALLOON_FLAG_DETACH 0x01 65 66 struct virtqueue *vtballoon_inflate_vq; 67 struct virtqueue *vtballoon_deflate_vq; 68 69 uint32_t vtballoon_desired_npages; 70 uint32_t vtballoon_current_npages; 71 TAILQ_HEAD(,vm_page) vtballoon_pages; 72 73 struct proc *vtballoon_kproc; 74 uint32_t *vtballoon_page_frames; 75 int vtballoon_timeout; 76 }; 77 78 static struct virtio_feature_desc vtballoon_feature_desc[] = { 79 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" }, 80 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" }, 81 82 { 0, NULL } 83 }; 84 85 static int vtballoon_probe(device_t); 86 static int vtballoon_attach(device_t); 87 static int vtballoon_detach(device_t); 88 static int vtballoon_config_change(device_t); 89 90 static void vtballoon_negotiate_features(struct vtballoon_softc *); 91 static int vtballoon_alloc_virtqueues(struct vtballoon_softc *); 92 93 static int vtballoon_vq_intr(void *); 94 95 static void vtballoon_inflate(struct vtballoon_softc *, int); 96 static void vtballoon_deflate(struct vtballoon_softc *, int); 97 98 static void vtballoon_send_page_frames(struct vtballoon_softc *, 99 struct virtqueue *, int); 100 101 static void vtballoon_pop(struct vtballoon_softc *); 102 static void vtballoon_stop(struct vtballoon_softc *); 103 104 static vm_page_t 105 vtballoon_alloc_page(struct vtballoon_softc *); 106 static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t); 107 108 static int vtballoon_sleep(struct vtballoon_softc *); 109 static void vtballoon_thread(void *); 110 static void vtballoon_add_sysctl(struct vtballoon_softc *); 111 112 /* Features desired/implemented by this driver. */ 113 #define VTBALLOON_FEATURES 0 114 115 /* Timeout between retries when the balloon needs inflating. */ 116 #define VTBALLOON_LOWMEM_TIMEOUT hz 117 118 /* 119 * Maximum number of pages we'll request to inflate or deflate 120 * the balloon in one virtqueue request. Both Linux and NetBSD 121 * have settled on 256, doing up to 1MB at a time. 122 */ 123 #define VTBALLOON_PAGES_PER_REQUEST 256 124 125 #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx 126 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \ 127 "VirtIO Balloon Lock", MTX_SPIN) 128 #define VTBALLOON_LOCK(_sc) mtx_lock_spin(VTBALLOON_MTX((_sc))) 129 #define VTBALLOON_UNLOCK(_sc) mtx_unlock_spin(VTBALLOON_MTX((_sc))) 130 #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc))) 131 132 static device_method_t vtballoon_methods[] = { 133 /* Device methods. */ 134 DEVMETHOD(device_probe, vtballoon_probe), 135 DEVMETHOD(device_attach, vtballoon_attach), 136 DEVMETHOD(device_detach, vtballoon_detach), 137 138 /* VirtIO methods. */ 139 DEVMETHOD(virtio_config_change, vtballoon_config_change), 140 141 { 0, 0 } 142 }; 143 144 static driver_t vtballoon_driver = { 145 "vtballoon", 146 vtballoon_methods, 147 sizeof(struct vtballoon_softc) 148 }; 149 static devclass_t vtballoon_devclass; 150 151 DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver, 152 vtballoon_devclass, 0, 0); 153 MODULE_VERSION(virtio_balloon, 1); 154 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1); 155 156 static int 157 vtballoon_probe(device_t dev) 158 { 159 160 if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON) 161 return (ENXIO); 162 163 device_set_desc(dev, "VirtIO Balloon Adapter"); 164 165 return (BUS_PROBE_DEFAULT); 166 } 167 168 static int 169 vtballoon_attach(device_t dev) 170 { 171 struct vtballoon_softc *sc; 172 int error; 173 174 sc = device_get_softc(dev); 175 sc->vtballoon_dev = dev; 176 177 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev)); 178 TAILQ_INIT(&sc->vtballoon_pages); 179 180 vtballoon_add_sysctl(sc); 181 182 virtio_set_feature_desc(dev, vtballoon_feature_desc); 183 vtballoon_negotiate_features(sc); 184 185 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST * 186 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO); 187 if (sc->vtballoon_page_frames == NULL) { 188 error = ENOMEM; 189 device_printf(dev, 190 "cannot allocate page frame request array\n"); 191 goto fail; 192 } 193 194 error = vtballoon_alloc_virtqueues(sc); 195 if (error) { 196 device_printf(dev, "cannot allocate virtqueues\n"); 197 goto fail; 198 } 199 200 error = virtio_setup_intr(dev, INTR_TYPE_MISC); 201 if (error) { 202 device_printf(dev, "cannot setup virtqueue interrupts\n"); 203 goto fail; 204 } 205 206 error = kproc_create(vtballoon_thread, sc, &sc->vtballoon_kproc, 207 0, 0, "virtio_balloon"); 208 if (error) { 209 device_printf(dev, "cannot create balloon kproc\n"); 210 goto fail; 211 } 212 213 virtqueue_enable_intr(sc->vtballoon_inflate_vq); 214 virtqueue_enable_intr(sc->vtballoon_deflate_vq); 215 216 fail: 217 if (error) 218 vtballoon_detach(dev); 219 220 return (error); 221 } 222 223 static int 224 vtballoon_detach(device_t dev) 225 { 226 struct vtballoon_softc *sc; 227 228 sc = device_get_softc(dev); 229 230 if (sc->vtballoon_kproc != NULL) { 231 VTBALLOON_LOCK(sc); 232 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH; 233 wakeup_one(sc); 234 msleep_spin(sc->vtballoon_kproc, VTBALLOON_MTX(sc), 235 "vtbdth", 0); 236 VTBALLOON_UNLOCK(sc); 237 238 sc->vtballoon_kproc = NULL; 239 } 240 241 if (device_is_attached(dev)) { 242 vtballoon_pop(sc); 243 vtballoon_stop(sc); 244 } 245 246 if (sc->vtballoon_page_frames != NULL) { 247 free(sc->vtballoon_page_frames, M_DEVBUF); 248 sc->vtballoon_page_frames = NULL; 249 } 250 251 VTBALLOON_LOCK_DESTROY(sc); 252 253 return (0); 254 } 255 256 static int 257 vtballoon_config_change(device_t dev) 258 { 259 struct vtballoon_softc *sc; 260 261 sc = device_get_softc(dev); 262 263 VTBALLOON_LOCK(sc); 264 wakeup_one(sc); 265 VTBALLOON_UNLOCK(sc); 266 267 return (1); 268 } 269 270 static void 271 vtballoon_negotiate_features(struct vtballoon_softc *sc) 272 { 273 device_t dev; 274 uint64_t features; 275 276 dev = sc->vtballoon_dev; 277 features = virtio_negotiate_features(dev, VTBALLOON_FEATURES); 278 sc->vtballoon_features = features; 279 } 280 281 static int 282 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc) 283 { 284 device_t dev; 285 struct vq_alloc_info vq_info[2]; 286 int nvqs; 287 288 dev = sc->vtballoon_dev; 289 nvqs = 2; 290 291 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc, 292 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev)); 293 294 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc, 295 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev)); 296 297 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 298 } 299 300 static int 301 vtballoon_vq_intr(void *xsc) 302 { 303 struct vtballoon_softc *sc; 304 305 sc = xsc; 306 307 VTBALLOON_LOCK(sc); 308 wakeup_one(sc); 309 VTBALLOON_UNLOCK(sc); 310 311 return (1); 312 } 313 314 static void 315 vtballoon_inflate(struct vtballoon_softc *sc, int npages) 316 { 317 struct virtqueue *vq; 318 vm_page_t m; 319 int i; 320 321 vq = sc->vtballoon_inflate_vq; 322 m = NULL; 323 324 if (npages > VTBALLOON_PAGES_PER_REQUEST) 325 npages = VTBALLOON_PAGES_PER_REQUEST; 326 KASSERT(npages > 0, ("balloon doesn't need inflating?")); 327 328 for (i = 0; i < npages; i++) { 329 if ((m = vtballoon_alloc_page(sc)) == NULL) 330 break; 331 332 sc->vtballoon_page_frames[i] = 333 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 334 335 KASSERT(m->queue == PQ_NONE, ("allocated page on queue")); 336 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq); 337 } 338 339 if (i > 0) 340 vtballoon_send_page_frames(sc, vq, i); 341 342 if (m == NULL) 343 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT; 344 } 345 346 static void 347 vtballoon_deflate(struct vtballoon_softc *sc, int npages) 348 { 349 TAILQ_HEAD(, vm_page) free_pages; 350 struct virtqueue *vq; 351 vm_page_t m; 352 int i; 353 354 vq = sc->vtballoon_deflate_vq; 355 TAILQ_INIT(&free_pages); 356 357 if (npages > VTBALLOON_PAGES_PER_REQUEST) 358 npages = VTBALLOON_PAGES_PER_REQUEST; 359 KASSERT(npages > 0, ("balloon doesn't need deflating?")); 360 361 for (i = 0; i < npages; i++) { 362 m = TAILQ_FIRST(&sc->vtballoon_pages); 363 KASSERT(m != NULL, ("no more pages to deflate")); 364 365 sc->vtballoon_page_frames[i] = 366 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 367 368 TAILQ_REMOVE(&sc->vtballoon_pages, m, pageq); 369 TAILQ_INSERT_TAIL(&free_pages, m, pageq); 370 } 371 372 if (i > 0) { 373 /* Always tell host first before freeing the pages. */ 374 vtballoon_send_page_frames(sc, vq, i); 375 376 while ((m = TAILQ_FIRST(&free_pages)) != NULL) { 377 TAILQ_REMOVE(&free_pages, m, pageq); 378 vtballoon_free_page(sc, m); 379 } 380 } 381 382 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) && 383 sc->vtballoon_current_npages == 0) || 384 (!TAILQ_EMPTY(&sc->vtballoon_pages) && 385 sc->vtballoon_current_npages != 0), ("balloon empty?")); 386 } 387 388 static void 389 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq, 390 int npages) 391 { 392 struct sglist sg; 393 struct sglist_seg segs[1]; 394 void *c; 395 int error; 396 397 sglist_init(&sg, 1, segs); 398 399 error = sglist_append(&sg, sc->vtballoon_page_frames, 400 npages * sizeof(uint32_t)); 401 KASSERT(error == 0, ("error adding page frames to sglist")); 402 403 error = virtqueue_enqueue(vq, vq, &sg, 1, 0); 404 KASSERT(error == 0, ("error enqueuing page frames to virtqueue")); 405 406 /* 407 * Inflate and deflate operations are done synchronously. The 408 * interrupt handler will wake us up. 409 */ 410 VTBALLOON_LOCK(sc); 411 virtqueue_notify(vq); 412 413 while ((c = virtqueue_dequeue(vq, NULL)) == NULL) 414 msleep_spin(sc, VTBALLOON_MTX(sc), "vtbspf", 0); 415 VTBALLOON_UNLOCK(sc); 416 417 KASSERT(c == vq, ("unexpected balloon operation response")); 418 } 419 420 static void 421 vtballoon_pop(struct vtballoon_softc *sc) 422 { 423 424 while (!TAILQ_EMPTY(&sc->vtballoon_pages)) 425 vtballoon_deflate(sc, sc->vtballoon_current_npages); 426 } 427 428 static void 429 vtballoon_stop(struct vtballoon_softc *sc) 430 { 431 432 virtqueue_disable_intr(sc->vtballoon_inflate_vq); 433 virtqueue_disable_intr(sc->vtballoon_deflate_vq); 434 435 virtio_stop(sc->vtballoon_dev); 436 } 437 438 static vm_page_t 439 vtballoon_alloc_page(struct vtballoon_softc *sc) 440 { 441 vm_page_t m; 442 443 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED | 444 VM_ALLOC_NOOBJ); 445 if (m != NULL) 446 sc->vtballoon_current_npages++; 447 448 return (m); 449 } 450 451 static void 452 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m) 453 { 454 455 vm_page_unwire(m, 0); 456 vm_page_free(m); 457 sc->vtballoon_current_npages--; 458 } 459 460 static uint32_t 461 vtballoon_desired_size(struct vtballoon_softc *sc) 462 { 463 uint32_t desired; 464 465 desired = virtio_read_dev_config_4(sc->vtballoon_dev, 466 offsetof(struct virtio_balloon_config, num_pages)); 467 468 return (le32toh(desired)); 469 } 470 471 static void 472 vtballoon_update_size(struct vtballoon_softc *sc) 473 { 474 475 virtio_write_dev_config_4(sc->vtballoon_dev, 476 offsetof(struct virtio_balloon_config, actual), 477 htole32(sc->vtballoon_current_npages)); 478 479 } 480 481 static int 482 vtballoon_sleep(struct vtballoon_softc *sc) 483 { 484 int rc, timeout; 485 uint32_t current, desired; 486 487 rc = 0; 488 current = sc->vtballoon_current_npages; 489 490 VTBALLOON_LOCK(sc); 491 for (;;) { 492 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) { 493 rc = 1; 494 break; 495 } 496 497 desired = vtballoon_desired_size(sc); 498 sc->vtballoon_desired_npages = desired; 499 500 /* 501 * If given, use non-zero timeout on the first time through 502 * the loop. On subsequent times, timeout will be zero so 503 * we will reevaluate the desired size of the balloon and 504 * break out to retry if needed. 505 */ 506 timeout = sc->vtballoon_timeout; 507 sc->vtballoon_timeout = 0; 508 509 if (current > desired) 510 break; 511 if (current < desired && timeout == 0) 512 break; 513 514 msleep_spin(sc, VTBALLOON_MTX(sc), "vtbslp", timeout); 515 } 516 VTBALLOON_UNLOCK(sc); 517 518 return (rc); 519 } 520 521 static void 522 vtballoon_thread(void *xsc) 523 { 524 struct vtballoon_softc *sc; 525 uint32_t current, desired; 526 527 sc = xsc; 528 529 for (;;) { 530 if (vtballoon_sleep(sc) != 0) 531 break; 532 533 current = sc->vtballoon_current_npages; 534 desired = sc->vtballoon_desired_npages; 535 536 if (desired != current) { 537 if (desired > current) 538 vtballoon_inflate(sc, desired - current); 539 else 540 vtballoon_deflate(sc, current - desired); 541 542 vtballoon_update_size(sc); 543 } 544 } 545 546 kproc_exit(0); 547 } 548 549 static void 550 vtballoon_add_sysctl(struct vtballoon_softc *sc) 551 { 552 device_t dev; 553 struct sysctl_ctx_list *ctx; 554 struct sysctl_oid *tree; 555 struct sysctl_oid_list *child; 556 557 dev = sc->vtballoon_dev; 558 ctx = device_get_sysctl_ctx(dev); 559 tree = device_get_sysctl_tree(dev); 560 child = SYSCTL_CHILDREN(tree); 561 562 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired", 563 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t), 564 "Desired balloon size in pages"); 565 566 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current", 567 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t), 568 "Current balloon size in pages"); 569 } 570