1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /* Driver for VirtIO memory balloon devices. */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/endian.h>
35 #include <sys/kthread.h>
36 #include <sys/malloc.h>
37 #include <sys/module.h>
38 #include <sys/sglist.h>
39 #include <sys/sysctl.h>
40 #include <sys/lock.h>
41 #include <sys/mutex.h>
42 #include <sys/queue.h>
43
44 #include <vm/vm.h>
45 #include <vm/vm_page.h>
46
47 #include <machine/bus.h>
48 #include <machine/resource.h>
49 #include <sys/bus.h>
50 #include <sys/rman.h>
51
52 #include <dev/virtio/virtio.h>
53 #include <dev/virtio/virtqueue.h>
54 #include <dev/virtio/balloon/virtio_balloon.h>
55
56 #include "virtio_if.h"
57
58 struct vtballoon_softc {
59 device_t vtballoon_dev;
60 struct mtx vtballoon_mtx;
61 uint64_t vtballoon_features;
62 uint32_t vtballoon_flags;
63 #define VTBALLOON_FLAG_DETACH 0x01
64
65 struct virtqueue *vtballoon_inflate_vq;
66 struct virtqueue *vtballoon_deflate_vq;
67
68 uint32_t vtballoon_desired_npages;
69 uint32_t vtballoon_current_npages;
70 TAILQ_HEAD(,vm_page) vtballoon_pages;
71
72 struct thread *vtballoon_td;
73 uint32_t *vtballoon_page_frames;
74 int vtballoon_timeout;
75 };
76
77 static struct virtio_feature_desc vtballoon_feature_desc[] = {
78 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" },
79 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" },
80 { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" },
81
82 { 0, NULL }
83 };
84
85 static int vtballoon_probe(device_t);
86 static int vtballoon_attach(device_t);
87 static int vtballoon_detach(device_t);
88 static int vtballoon_config_change(device_t);
89
90 static int vtballoon_negotiate_features(struct vtballoon_softc *);
91 static int vtballoon_setup_features(struct vtballoon_softc *);
92 static int vtballoon_alloc_virtqueues(struct vtballoon_softc *);
93
94 static void vtballoon_vq_intr(void *);
95
96 static void vtballoon_inflate(struct vtballoon_softc *, int);
97 static void vtballoon_deflate(struct vtballoon_softc *, int);
98
99 static void vtballoon_send_page_frames(struct vtballoon_softc *,
100 struct virtqueue *, int);
101
102 static void vtballoon_pop(struct vtballoon_softc *);
103 static void vtballoon_stop(struct vtballoon_softc *);
104
105 static vm_page_t
106 vtballoon_alloc_page(struct vtballoon_softc *);
107 static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
108
109 static int vtballoon_sleep(struct vtballoon_softc *);
110 static void vtballoon_thread(void *);
111 static void vtballoon_setup_sysctl(struct vtballoon_softc *);
112
113 #define vtballoon_modern(_sc) \
114 (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)
115
116 /* Features desired/implemented by this driver. */
117 #define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST
118
119 /* Timeout between retries when the balloon needs inflating. */
120 #define VTBALLOON_LOWMEM_TIMEOUT hz
121
122 /*
123 * Maximum number of pages we'll request to inflate or deflate
124 * the balloon in one virtqueue request. Both Linux and NetBSD
125 * have settled on 256, doing up to 1MB at a time.
126 */
127 #define VTBALLOON_PAGES_PER_REQUEST 256
128
129 /* Must be able to fix all pages frames in one page (segment). */
130 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
131
132 #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx
133 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \
134 "VirtIO Balloon Lock", MTX_DEF)
135 #define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc)))
136 #define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc)))
137 #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc)))
138
139 static device_method_t vtballoon_methods[] = {
140 /* Device methods. */
141 DEVMETHOD(device_probe, vtballoon_probe),
142 DEVMETHOD(device_attach, vtballoon_attach),
143 DEVMETHOD(device_detach, vtballoon_detach),
144
145 /* VirtIO methods. */
146 DEVMETHOD(virtio_config_change, vtballoon_config_change),
147
148 DEVMETHOD_END
149 };
150
151 static driver_t vtballoon_driver = {
152 "vtballoon",
153 vtballoon_methods,
154 sizeof(struct vtballoon_softc)
155 };
156
157 VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver, 0, 0);
158 MODULE_VERSION(virtio_balloon, 1);
159 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
160
161 VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON,
162 "VirtIO Balloon Adapter");
163
164 static int
vtballoon_probe(device_t dev)165 vtballoon_probe(device_t dev)
166 {
167 return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
168 }
169
170 static int
vtballoon_attach(device_t dev)171 vtballoon_attach(device_t dev)
172 {
173 struct vtballoon_softc *sc;
174 int error;
175
176 sc = device_get_softc(dev);
177 sc->vtballoon_dev = dev;
178 virtio_set_feature_desc(dev, vtballoon_feature_desc);
179
180 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
181 TAILQ_INIT(&sc->vtballoon_pages);
182
183 vtballoon_setup_sysctl(sc);
184
185 error = vtballoon_setup_features(sc);
186 if (error) {
187 device_printf(dev, "cannot setup features\n");
188 goto fail;
189 }
190
191 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
192 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
193 if (sc->vtballoon_page_frames == NULL) {
194 error = ENOMEM;
195 device_printf(dev,
196 "cannot allocate page frame request array\n");
197 goto fail;
198 }
199
200 error = vtballoon_alloc_virtqueues(sc);
201 if (error) {
202 device_printf(dev, "cannot allocate virtqueues\n");
203 goto fail;
204 }
205
206 error = virtio_setup_intr(dev, INTR_TYPE_MISC);
207 if (error) {
208 device_printf(dev, "cannot setup virtqueue interrupts\n");
209 goto fail;
210 }
211
212 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
213 0, 0, "virtio_balloon");
214 if (error) {
215 device_printf(dev, "cannot create balloon kthread\n");
216 goto fail;
217 }
218
219 virtqueue_enable_intr(sc->vtballoon_inflate_vq);
220 virtqueue_enable_intr(sc->vtballoon_deflate_vq);
221
222 fail:
223 if (error)
224 vtballoon_detach(dev);
225
226 return (error);
227 }
228
229 static int
vtballoon_detach(device_t dev)230 vtballoon_detach(device_t dev)
231 {
232 struct vtballoon_softc *sc;
233
234 sc = device_get_softc(dev);
235
236 if (sc->vtballoon_td != NULL) {
237 VTBALLOON_LOCK(sc);
238 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
239 wakeup_one(sc);
240 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
241 VTBALLOON_UNLOCK(sc);
242
243 sc->vtballoon_td = NULL;
244 }
245
246 if (device_is_attached(dev)) {
247 vtballoon_pop(sc);
248 vtballoon_stop(sc);
249 }
250
251 if (sc->vtballoon_page_frames != NULL) {
252 free(sc->vtballoon_page_frames, M_DEVBUF);
253 sc->vtballoon_page_frames = NULL;
254 }
255
256 VTBALLOON_LOCK_DESTROY(sc);
257
258 return (0);
259 }
260
261 static int
vtballoon_config_change(device_t dev)262 vtballoon_config_change(device_t dev)
263 {
264 struct vtballoon_softc *sc;
265
266 sc = device_get_softc(dev);
267
268 VTBALLOON_LOCK(sc);
269 wakeup_one(sc);
270 VTBALLOON_UNLOCK(sc);
271
272 return (1);
273 }
274
275 static int
vtballoon_negotiate_features(struct vtballoon_softc * sc)276 vtballoon_negotiate_features(struct vtballoon_softc *sc)
277 {
278 device_t dev;
279 uint64_t features;
280
281 dev = sc->vtballoon_dev;
282 features = VTBALLOON_FEATURES;
283
284 sc->vtballoon_features = virtio_negotiate_features(dev, features);
285 return (virtio_finalize_features(dev));
286 }
287
288 static int
vtballoon_setup_features(struct vtballoon_softc * sc)289 vtballoon_setup_features(struct vtballoon_softc *sc)
290 {
291 int error;
292
293 error = vtballoon_negotiate_features(sc);
294 if (error)
295 return (error);
296
297 return (0);
298 }
299
300 static int
vtballoon_alloc_virtqueues(struct vtballoon_softc * sc)301 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
302 {
303 device_t dev;
304 struct vq_alloc_info vq_info[2];
305 int nvqs;
306
307 dev = sc->vtballoon_dev;
308 nvqs = 2;
309
310 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
311 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
312
313 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
314 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
315
316 return (virtio_alloc_virtqueues(dev, nvqs, vq_info));
317 }
318
319 static void
vtballoon_vq_intr(void * xsc)320 vtballoon_vq_intr(void *xsc)
321 {
322 struct vtballoon_softc *sc;
323
324 sc = xsc;
325
326 VTBALLOON_LOCK(sc);
327 wakeup_one(sc);
328 VTBALLOON_UNLOCK(sc);
329 }
330
331 static void
vtballoon_inflate(struct vtballoon_softc * sc,int npages)332 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
333 {
334 struct virtqueue *vq;
335 vm_page_t m;
336 int i;
337
338 vq = sc->vtballoon_inflate_vq;
339
340 if (npages > VTBALLOON_PAGES_PER_REQUEST)
341 npages = VTBALLOON_PAGES_PER_REQUEST;
342
343 for (i = 0; i < npages; i++) {
344 if ((m = vtballoon_alloc_page(sc)) == NULL) {
345 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
346 break;
347 }
348
349 sc->vtballoon_page_frames[i] =
350 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
351
352 KASSERT(m->a.queue == PQ_NONE,
353 ("%s: allocated page %p on queue", __func__, m));
354 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
355 }
356
357 if (i > 0)
358 vtballoon_send_page_frames(sc, vq, i);
359 }
360
361 static void
vtballoon_deflate(struct vtballoon_softc * sc,int npages)362 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
363 {
364 TAILQ_HEAD(, vm_page) free_pages;
365 struct virtqueue *vq;
366 vm_page_t m;
367 int i;
368
369 vq = sc->vtballoon_deflate_vq;
370 TAILQ_INIT(&free_pages);
371
372 if (npages > VTBALLOON_PAGES_PER_REQUEST)
373 npages = VTBALLOON_PAGES_PER_REQUEST;
374
375 for (i = 0; i < npages; i++) {
376 m = TAILQ_FIRST(&sc->vtballoon_pages);
377 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
378
379 sc->vtballoon_page_frames[i] =
380 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
381
382 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
383 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
384 }
385
386 if (i > 0) {
387 /* Always tell host first before freeing the pages. */
388 vtballoon_send_page_frames(sc, vq, i);
389
390 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
391 TAILQ_REMOVE(&free_pages, m, plinks.q);
392 vtballoon_free_page(sc, m);
393 }
394 }
395
396 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
397 sc->vtballoon_current_npages == 0) ||
398 (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
399 sc->vtballoon_current_npages != 0),
400 ("%s: bogus page count %d", __func__,
401 sc->vtballoon_current_npages));
402 }
403
404 static void
vtballoon_send_page_frames(struct vtballoon_softc * sc,struct virtqueue * vq,int npages)405 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
406 int npages)
407 {
408 struct sglist sg;
409 struct sglist_seg segs[1];
410 void *c;
411 int error __diagused;
412
413 sglist_init(&sg, 1, segs);
414
415 error = sglist_append(&sg, sc->vtballoon_page_frames,
416 npages * sizeof(uint32_t));
417 KASSERT(error == 0, ("error adding page frames to sglist"));
418
419 error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
420 KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
421 virtqueue_notify(vq);
422
423 /*
424 * Inflate and deflate operations are done synchronously. The
425 * interrupt handler will wake us up.
426 */
427 VTBALLOON_LOCK(sc);
428 while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
429 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
430 VTBALLOON_UNLOCK(sc);
431
432 KASSERT(c == vq, ("unexpected balloon operation response"));
433 }
434
435 static void
vtballoon_pop(struct vtballoon_softc * sc)436 vtballoon_pop(struct vtballoon_softc *sc)
437 {
438
439 while (!TAILQ_EMPTY(&sc->vtballoon_pages))
440 vtballoon_deflate(sc, sc->vtballoon_current_npages);
441 }
442
443 static void
vtballoon_stop(struct vtballoon_softc * sc)444 vtballoon_stop(struct vtballoon_softc *sc)
445 {
446
447 virtqueue_disable_intr(sc->vtballoon_inflate_vq);
448 virtqueue_disable_intr(sc->vtballoon_deflate_vq);
449
450 virtio_stop(sc->vtballoon_dev);
451 }
452
453 static vm_page_t
vtballoon_alloc_page(struct vtballoon_softc * sc)454 vtballoon_alloc_page(struct vtballoon_softc *sc)
455 {
456 vm_page_t m;
457
458 m = vm_page_alloc_noobj(VM_ALLOC_NODUMP);
459 if (m != NULL)
460 sc->vtballoon_current_npages++;
461
462 return (m);
463 }
464
465 static void
vtballoon_free_page(struct vtballoon_softc * sc,vm_page_t m)466 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
467 {
468
469 vm_page_free(m);
470 sc->vtballoon_current_npages--;
471 }
472
473 static uint32_t
vtballoon_desired_size(struct vtballoon_softc * sc)474 vtballoon_desired_size(struct vtballoon_softc *sc)
475 {
476 uint32_t desired;
477
478 desired = virtio_read_dev_config_4(sc->vtballoon_dev,
479 offsetof(struct virtio_balloon_config, num_pages));
480
481 if (vtballoon_modern(sc))
482 return (desired);
483 else
484 return (le32toh(desired));
485 }
486
487 static void
vtballoon_update_size(struct vtballoon_softc * sc)488 vtballoon_update_size(struct vtballoon_softc *sc)
489 {
490 uint32_t npages;
491
492 npages = sc->vtballoon_current_npages;
493 if (!vtballoon_modern(sc))
494 npages = htole32(npages);
495
496 virtio_write_dev_config_4(sc->vtballoon_dev,
497 offsetof(struct virtio_balloon_config, actual), npages);
498 }
499
500 static int
vtballoon_sleep(struct vtballoon_softc * sc)501 vtballoon_sleep(struct vtballoon_softc *sc)
502 {
503 int rc, timeout;
504 uint32_t current, desired;
505
506 rc = 0;
507 current = sc->vtballoon_current_npages;
508
509 VTBALLOON_LOCK(sc);
510 for (;;) {
511 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
512 rc = 1;
513 break;
514 }
515
516 desired = vtballoon_desired_size(sc);
517 sc->vtballoon_desired_npages = desired;
518
519 /*
520 * If given, use non-zero timeout on the first time through
521 * the loop. On subsequent times, timeout will be zero so
522 * we will reevaluate the desired size of the balloon and
523 * break out to retry if needed.
524 */
525 timeout = sc->vtballoon_timeout;
526 sc->vtballoon_timeout = 0;
527
528 if (current > desired)
529 break;
530 if (current < desired && timeout == 0)
531 break;
532
533 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
534 }
535 VTBALLOON_UNLOCK(sc);
536
537 return (rc);
538 }
539
540 static void
vtballoon_thread(void * xsc)541 vtballoon_thread(void *xsc)
542 {
543 struct vtballoon_softc *sc;
544 uint32_t current, desired;
545
546 sc = xsc;
547
548 for (;;) {
549 if (vtballoon_sleep(sc) != 0)
550 break;
551
552 current = sc->vtballoon_current_npages;
553 desired = sc->vtballoon_desired_npages;
554
555 if (desired != current) {
556 if (desired > current)
557 vtballoon_inflate(sc, desired - current);
558 else
559 vtballoon_deflate(sc, current - desired);
560
561 vtballoon_update_size(sc);
562 }
563 }
564
565 kthread_exit();
566 }
567
568 static void
vtballoon_setup_sysctl(struct vtballoon_softc * sc)569 vtballoon_setup_sysctl(struct vtballoon_softc *sc)
570 {
571 device_t dev;
572 struct sysctl_ctx_list *ctx;
573 struct sysctl_oid *tree;
574 struct sysctl_oid_list *child;
575
576 dev = sc->vtballoon_dev;
577 ctx = device_get_sysctl_ctx(dev);
578 tree = device_get_sysctl_tree(dev);
579 child = SYSCTL_CHILDREN(tree);
580
581 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
582 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
583 "Desired balloon size in pages");
584
585 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
586 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
587 "Current balloon size in pages");
588 }
589