110b59a9bSPeter Grehan /*-
24d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause
3718cf2ccSPedro F. Giffuni *
4abd6790cSBryan Venteicher * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
510b59a9bSPeter Grehan * All rights reserved.
610b59a9bSPeter Grehan *
710b59a9bSPeter Grehan * Redistribution and use in source and binary forms, with or without
810b59a9bSPeter Grehan * modification, are permitted provided that the following conditions
910b59a9bSPeter Grehan * are met:
1010b59a9bSPeter Grehan * 1. Redistributions of source code must retain the above copyright
1110b59a9bSPeter Grehan * notice unmodified, this list of conditions, and the following
1210b59a9bSPeter Grehan * disclaimer.
1310b59a9bSPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright
1410b59a9bSPeter Grehan * notice, this list of conditions and the following disclaimer in the
1510b59a9bSPeter Grehan * documentation and/or other materials provided with the distribution.
1610b59a9bSPeter Grehan *
1710b59a9bSPeter Grehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
1810b59a9bSPeter Grehan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
1910b59a9bSPeter Grehan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
2010b59a9bSPeter Grehan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
2110b59a9bSPeter Grehan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
2210b59a9bSPeter Grehan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
2310b59a9bSPeter Grehan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
2410b59a9bSPeter Grehan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
2510b59a9bSPeter Grehan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
2610b59a9bSPeter Grehan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2710b59a9bSPeter Grehan */
2810b59a9bSPeter Grehan
2910b59a9bSPeter Grehan /* Driver for VirtIO memory balloon devices. */
3010b59a9bSPeter Grehan
3110b59a9bSPeter Grehan #include <sys/param.h>
3210b59a9bSPeter Grehan #include <sys/systm.h>
3310b59a9bSPeter Grehan #include <sys/kernel.h>
3410b59a9bSPeter Grehan #include <sys/endian.h>
3510b59a9bSPeter Grehan #include <sys/kthread.h>
3610b59a9bSPeter Grehan #include <sys/malloc.h>
3710b59a9bSPeter Grehan #include <sys/module.h>
3810b59a9bSPeter Grehan #include <sys/sglist.h>
3910b59a9bSPeter Grehan #include <sys/sysctl.h>
4010b59a9bSPeter Grehan #include <sys/lock.h>
4110b59a9bSPeter Grehan #include <sys/mutex.h>
4210b59a9bSPeter Grehan #include <sys/queue.h>
4310b59a9bSPeter Grehan
4410b59a9bSPeter Grehan #include <vm/vm.h>
4510b59a9bSPeter Grehan #include <vm/vm_page.h>
4610b59a9bSPeter Grehan
4710b59a9bSPeter Grehan #include <machine/bus.h>
4810b59a9bSPeter Grehan #include <machine/resource.h>
4910b59a9bSPeter Grehan #include <sys/bus.h>
5010b59a9bSPeter Grehan #include <sys/rman.h>
5110b59a9bSPeter Grehan
5210b59a9bSPeter Grehan #include <dev/virtio/virtio.h>
5310b59a9bSPeter Grehan #include <dev/virtio/virtqueue.h>
5410b59a9bSPeter Grehan #include <dev/virtio/balloon/virtio_balloon.h>
5510b59a9bSPeter Grehan
5610b59a9bSPeter Grehan #include "virtio_if.h"
5710b59a9bSPeter Grehan
5810b59a9bSPeter Grehan struct vtballoon_softc {
5910b59a9bSPeter Grehan device_t vtballoon_dev;
6010b59a9bSPeter Grehan struct mtx vtballoon_mtx;
6110b59a9bSPeter Grehan uint64_t vtballoon_features;
6210b59a9bSPeter Grehan uint32_t vtballoon_flags;
6310b59a9bSPeter Grehan #define VTBALLOON_FLAG_DETACH 0x01
6410b59a9bSPeter Grehan
6510b59a9bSPeter Grehan struct virtqueue *vtballoon_inflate_vq;
6610b59a9bSPeter Grehan struct virtqueue *vtballoon_deflate_vq;
6710b59a9bSPeter Grehan
6810b59a9bSPeter Grehan uint32_t vtballoon_desired_npages;
6910b59a9bSPeter Grehan uint32_t vtballoon_current_npages;
7010b59a9bSPeter Grehan TAILQ_HEAD(,vm_page) vtballoon_pages;
7110b59a9bSPeter Grehan
726f7e6082SBryan Venteicher struct thread *vtballoon_td;
7310b59a9bSPeter Grehan uint32_t *vtballoon_page_frames;
7410b59a9bSPeter Grehan int vtballoon_timeout;
7510b59a9bSPeter Grehan };
7610b59a9bSPeter Grehan
7710b59a9bSPeter Grehan static struct virtio_feature_desc vtballoon_feature_desc[] = {
7810b59a9bSPeter Grehan { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" },
7910b59a9bSPeter Grehan { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" },
80f7f9c266SBryan Venteicher { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" },
81f7f9c266SBryan Venteicher
8210b59a9bSPeter Grehan { 0, NULL }
8310b59a9bSPeter Grehan };
8410b59a9bSPeter Grehan
8510b59a9bSPeter Grehan static int vtballoon_probe(device_t);
8610b59a9bSPeter Grehan static int vtballoon_attach(device_t);
8710b59a9bSPeter Grehan static int vtballoon_detach(device_t);
8810b59a9bSPeter Grehan static int vtballoon_config_change(device_t);
8910b59a9bSPeter Grehan
90e6cc42f1SBryan Venteicher static int vtballoon_negotiate_features(struct vtballoon_softc *);
91e6cc42f1SBryan Venteicher static int vtballoon_setup_features(struct vtballoon_softc *);
9210b59a9bSPeter Grehan static int vtballoon_alloc_virtqueues(struct vtballoon_softc *);
9310b59a9bSPeter Grehan
946632efe4SBryan Venteicher static void vtballoon_vq_intr(void *);
9510b59a9bSPeter Grehan
9610b59a9bSPeter Grehan static void vtballoon_inflate(struct vtballoon_softc *, int);
9710b59a9bSPeter Grehan static void vtballoon_deflate(struct vtballoon_softc *, int);
9810b59a9bSPeter Grehan
9910b59a9bSPeter Grehan static void vtballoon_send_page_frames(struct vtballoon_softc *,
10010b59a9bSPeter Grehan struct virtqueue *, int);
10110b59a9bSPeter Grehan
10210b59a9bSPeter Grehan static void vtballoon_pop(struct vtballoon_softc *);
10310b59a9bSPeter Grehan static void vtballoon_stop(struct vtballoon_softc *);
10410b59a9bSPeter Grehan
10510b59a9bSPeter Grehan static vm_page_t
10610b59a9bSPeter Grehan vtballoon_alloc_page(struct vtballoon_softc *);
10710b59a9bSPeter Grehan static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
10810b59a9bSPeter Grehan
10910b59a9bSPeter Grehan static int vtballoon_sleep(struct vtballoon_softc *);
11010b59a9bSPeter Grehan static void vtballoon_thread(void *);
111e6cc42f1SBryan Venteicher static void vtballoon_setup_sysctl(struct vtballoon_softc *);
11210b59a9bSPeter Grehan
113f7f9c266SBryan Venteicher #define vtballoon_modern(_sc) \
114f7f9c266SBryan Venteicher (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)
115f7f9c266SBryan Venteicher
11610b59a9bSPeter Grehan /* Features desired/implemented by this driver. */
117f7f9c266SBryan Venteicher #define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST
11810b59a9bSPeter Grehan
11910b59a9bSPeter Grehan /* Timeout between retries when the balloon needs inflating. */
12010b59a9bSPeter Grehan #define VTBALLOON_LOWMEM_TIMEOUT hz
12110b59a9bSPeter Grehan
12210b59a9bSPeter Grehan /*
12310b59a9bSPeter Grehan * Maximum number of pages we'll request to inflate or deflate
12410b59a9bSPeter Grehan * the balloon in one virtqueue request. Both Linux and NetBSD
12510b59a9bSPeter Grehan * have settled on 256, doing up to 1MB at a time.
12610b59a9bSPeter Grehan */
12710b59a9bSPeter Grehan #define VTBALLOON_PAGES_PER_REQUEST 256
12810b59a9bSPeter Grehan
129b8a58707SPeter Grehan /* Must be able to fix all pages frames in one page (segment). */
130b8a58707SPeter Grehan CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
131b8a58707SPeter Grehan
13210b59a9bSPeter Grehan #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx
13310b59a9bSPeter Grehan #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \
1346f7e6082SBryan Venteicher "VirtIO Balloon Lock", MTX_DEF)
1356f7e6082SBryan Venteicher #define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc)))
1366f7e6082SBryan Venteicher #define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc)))
13710b59a9bSPeter Grehan #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc)))
13810b59a9bSPeter Grehan
13910b59a9bSPeter Grehan static device_method_t vtballoon_methods[] = {
14010b59a9bSPeter Grehan /* Device methods. */
14110b59a9bSPeter Grehan DEVMETHOD(device_probe, vtballoon_probe),
14210b59a9bSPeter Grehan DEVMETHOD(device_attach, vtballoon_attach),
14310b59a9bSPeter Grehan DEVMETHOD(device_detach, vtballoon_detach),
14410b59a9bSPeter Grehan
14510b59a9bSPeter Grehan /* VirtIO methods. */
14610b59a9bSPeter Grehan DEVMETHOD(virtio_config_change, vtballoon_config_change),
14710b59a9bSPeter Grehan
148b8a58707SPeter Grehan DEVMETHOD_END
14910b59a9bSPeter Grehan };
15010b59a9bSPeter Grehan
15110b59a9bSPeter Grehan static driver_t vtballoon_driver = {
15210b59a9bSPeter Grehan "vtballoon",
15310b59a9bSPeter Grehan vtballoon_methods,
15410b59a9bSPeter Grehan sizeof(struct vtballoon_softc)
15510b59a9bSPeter Grehan };
15610b59a9bSPeter Grehan
1575c4c96d3SJohn Baldwin VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver, 0, 0);
15810b59a9bSPeter Grehan MODULE_VERSION(virtio_balloon, 1);
15910b59a9bSPeter Grehan MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
16010b59a9bSPeter Grehan
161633218eeSJessica Clarke VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON,
1620f6040f0SConrad Meyer "VirtIO Balloon Adapter");
1630f6040f0SConrad Meyer
16410b59a9bSPeter Grehan static int
vtballoon_probe(device_t dev)16510b59a9bSPeter Grehan vtballoon_probe(device_t dev)
16610b59a9bSPeter Grehan {
1670f6040f0SConrad Meyer return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
16810b59a9bSPeter Grehan }
16910b59a9bSPeter Grehan
17010b59a9bSPeter Grehan static int
vtballoon_attach(device_t dev)17110b59a9bSPeter Grehan vtballoon_attach(device_t dev)
17210b59a9bSPeter Grehan {
17310b59a9bSPeter Grehan struct vtballoon_softc *sc;
17410b59a9bSPeter Grehan int error;
17510b59a9bSPeter Grehan
17610b59a9bSPeter Grehan sc = device_get_softc(dev);
17710b59a9bSPeter Grehan sc->vtballoon_dev = dev;
178e6cc42f1SBryan Venteicher virtio_set_feature_desc(dev, vtballoon_feature_desc);
17910b59a9bSPeter Grehan
18010b59a9bSPeter Grehan VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
18110b59a9bSPeter Grehan TAILQ_INIT(&sc->vtballoon_pages);
18210b59a9bSPeter Grehan
183e6cc42f1SBryan Venteicher vtballoon_setup_sysctl(sc);
18410b59a9bSPeter Grehan
185e6cc42f1SBryan Venteicher error = vtballoon_setup_features(sc);
186e6cc42f1SBryan Venteicher if (error) {
187e6cc42f1SBryan Venteicher device_printf(dev, "cannot setup features\n");
188e6cc42f1SBryan Venteicher goto fail;
189e6cc42f1SBryan Venteicher }
19010b59a9bSPeter Grehan
19110b59a9bSPeter Grehan sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
19210b59a9bSPeter Grehan sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
19310b59a9bSPeter Grehan if (sc->vtballoon_page_frames == NULL) {
19410b59a9bSPeter Grehan error = ENOMEM;
19510b59a9bSPeter Grehan device_printf(dev,
19610b59a9bSPeter Grehan "cannot allocate page frame request array\n");
19710b59a9bSPeter Grehan goto fail;
19810b59a9bSPeter Grehan }
19910b59a9bSPeter Grehan
20010b59a9bSPeter Grehan error = vtballoon_alloc_virtqueues(sc);
20110b59a9bSPeter Grehan if (error) {
20210b59a9bSPeter Grehan device_printf(dev, "cannot allocate virtqueues\n");
20310b59a9bSPeter Grehan goto fail;
20410b59a9bSPeter Grehan }
20510b59a9bSPeter Grehan
20610b59a9bSPeter Grehan error = virtio_setup_intr(dev, INTR_TYPE_MISC);
20710b59a9bSPeter Grehan if (error) {
20810b59a9bSPeter Grehan device_printf(dev, "cannot setup virtqueue interrupts\n");
20910b59a9bSPeter Grehan goto fail;
21010b59a9bSPeter Grehan }
21110b59a9bSPeter Grehan
2126f7e6082SBryan Venteicher error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
21310b59a9bSPeter Grehan 0, 0, "virtio_balloon");
21410b59a9bSPeter Grehan if (error) {
2156f7e6082SBryan Venteicher device_printf(dev, "cannot create balloon kthread\n");
21610b59a9bSPeter Grehan goto fail;
21710b59a9bSPeter Grehan }
21810b59a9bSPeter Grehan
21910b59a9bSPeter Grehan virtqueue_enable_intr(sc->vtballoon_inflate_vq);
22010b59a9bSPeter Grehan virtqueue_enable_intr(sc->vtballoon_deflate_vq);
22110b59a9bSPeter Grehan
22210b59a9bSPeter Grehan fail:
22310b59a9bSPeter Grehan if (error)
22410b59a9bSPeter Grehan vtballoon_detach(dev);
22510b59a9bSPeter Grehan
22610b59a9bSPeter Grehan return (error);
22710b59a9bSPeter Grehan }
22810b59a9bSPeter Grehan
22910b59a9bSPeter Grehan static int
vtballoon_detach(device_t dev)23010b59a9bSPeter Grehan vtballoon_detach(device_t dev)
23110b59a9bSPeter Grehan {
23210b59a9bSPeter Grehan struct vtballoon_softc *sc;
23310b59a9bSPeter Grehan
23410b59a9bSPeter Grehan sc = device_get_softc(dev);
23510b59a9bSPeter Grehan
2366f7e6082SBryan Venteicher if (sc->vtballoon_td != NULL) {
23710b59a9bSPeter Grehan VTBALLOON_LOCK(sc);
23810b59a9bSPeter Grehan sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
23910b59a9bSPeter Grehan wakeup_one(sc);
2406f7e6082SBryan Venteicher msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
24110b59a9bSPeter Grehan VTBALLOON_UNLOCK(sc);
24210b59a9bSPeter Grehan
2436f7e6082SBryan Venteicher sc->vtballoon_td = NULL;
24410b59a9bSPeter Grehan }
24510b59a9bSPeter Grehan
24610b59a9bSPeter Grehan if (device_is_attached(dev)) {
24710b59a9bSPeter Grehan vtballoon_pop(sc);
24810b59a9bSPeter Grehan vtballoon_stop(sc);
24910b59a9bSPeter Grehan }
25010b59a9bSPeter Grehan
25110b59a9bSPeter Grehan if (sc->vtballoon_page_frames != NULL) {
25210b59a9bSPeter Grehan free(sc->vtballoon_page_frames, M_DEVBUF);
25310b59a9bSPeter Grehan sc->vtballoon_page_frames = NULL;
25410b59a9bSPeter Grehan }
25510b59a9bSPeter Grehan
25610b59a9bSPeter Grehan VTBALLOON_LOCK_DESTROY(sc);
25710b59a9bSPeter Grehan
25810b59a9bSPeter Grehan return (0);
25910b59a9bSPeter Grehan }
26010b59a9bSPeter Grehan
26110b59a9bSPeter Grehan static int
vtballoon_config_change(device_t dev)26210b59a9bSPeter Grehan vtballoon_config_change(device_t dev)
26310b59a9bSPeter Grehan {
26410b59a9bSPeter Grehan struct vtballoon_softc *sc;
26510b59a9bSPeter Grehan
26610b59a9bSPeter Grehan sc = device_get_softc(dev);
26710b59a9bSPeter Grehan
26810b59a9bSPeter Grehan VTBALLOON_LOCK(sc);
26910b59a9bSPeter Grehan wakeup_one(sc);
27010b59a9bSPeter Grehan VTBALLOON_UNLOCK(sc);
27110b59a9bSPeter Grehan
27210b59a9bSPeter Grehan return (1);
27310b59a9bSPeter Grehan }
27410b59a9bSPeter Grehan
275e6cc42f1SBryan Venteicher static int
vtballoon_negotiate_features(struct vtballoon_softc * sc)27610b59a9bSPeter Grehan vtballoon_negotiate_features(struct vtballoon_softc *sc)
27710b59a9bSPeter Grehan {
27810b59a9bSPeter Grehan device_t dev;
27910b59a9bSPeter Grehan uint64_t features;
28010b59a9bSPeter Grehan
28110b59a9bSPeter Grehan dev = sc->vtballoon_dev;
282f7f9c266SBryan Venteicher features = VTBALLOON_FEATURES;
283f7f9c266SBryan Venteicher
284f7f9c266SBryan Venteicher sc->vtballoon_features = virtio_negotiate_features(dev, features);
285e6cc42f1SBryan Venteicher return (virtio_finalize_features(dev));
286e6cc42f1SBryan Venteicher }
287e6cc42f1SBryan Venteicher
288e6cc42f1SBryan Venteicher static int
vtballoon_setup_features(struct vtballoon_softc * sc)289e6cc42f1SBryan Venteicher vtballoon_setup_features(struct vtballoon_softc *sc)
290e6cc42f1SBryan Venteicher {
291e6cc42f1SBryan Venteicher int error;
292e6cc42f1SBryan Venteicher
293e6cc42f1SBryan Venteicher error = vtballoon_negotiate_features(sc);
294e6cc42f1SBryan Venteicher if (error)
295e6cc42f1SBryan Venteicher return (error);
296e6cc42f1SBryan Venteicher
297e6cc42f1SBryan Venteicher return (0);
29810b59a9bSPeter Grehan }
29910b59a9bSPeter Grehan
30010b59a9bSPeter Grehan static int
vtballoon_alloc_virtqueues(struct vtballoon_softc * sc)30110b59a9bSPeter Grehan vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
30210b59a9bSPeter Grehan {
30310b59a9bSPeter Grehan device_t dev;
30410b59a9bSPeter Grehan struct vq_alloc_info vq_info[2];
30510b59a9bSPeter Grehan int nvqs;
30610b59a9bSPeter Grehan
30710b59a9bSPeter Grehan dev = sc->vtballoon_dev;
30810b59a9bSPeter Grehan nvqs = 2;
30910b59a9bSPeter Grehan
31010b59a9bSPeter Grehan VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
31110b59a9bSPeter Grehan &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
31210b59a9bSPeter Grehan
31310b59a9bSPeter Grehan VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
31410b59a9bSPeter Grehan &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
31510b59a9bSPeter Grehan
316*180c0240SMina Galić return (virtio_alloc_virtqueues(dev, nvqs, vq_info));
31710b59a9bSPeter Grehan }
31810b59a9bSPeter Grehan
3196632efe4SBryan Venteicher static void
vtballoon_vq_intr(void * xsc)32010b59a9bSPeter Grehan vtballoon_vq_intr(void *xsc)
32110b59a9bSPeter Grehan {
32210b59a9bSPeter Grehan struct vtballoon_softc *sc;
32310b59a9bSPeter Grehan
32410b59a9bSPeter Grehan sc = xsc;
32510b59a9bSPeter Grehan
32610b59a9bSPeter Grehan VTBALLOON_LOCK(sc);
32710b59a9bSPeter Grehan wakeup_one(sc);
32810b59a9bSPeter Grehan VTBALLOON_UNLOCK(sc);
32910b59a9bSPeter Grehan }
33010b59a9bSPeter Grehan
33110b59a9bSPeter Grehan static void
vtballoon_inflate(struct vtballoon_softc * sc,int npages)33210b59a9bSPeter Grehan vtballoon_inflate(struct vtballoon_softc *sc, int npages)
33310b59a9bSPeter Grehan {
33410b59a9bSPeter Grehan struct virtqueue *vq;
33510b59a9bSPeter Grehan vm_page_t m;
33610b59a9bSPeter Grehan int i;
33710b59a9bSPeter Grehan
33810b59a9bSPeter Grehan vq = sc->vtballoon_inflate_vq;
33910b59a9bSPeter Grehan
34010b59a9bSPeter Grehan if (npages > VTBALLOON_PAGES_PER_REQUEST)
34110b59a9bSPeter Grehan npages = VTBALLOON_PAGES_PER_REQUEST;
34210b59a9bSPeter Grehan
34310b59a9bSPeter Grehan for (i = 0; i < npages; i++) {
3446f7e6082SBryan Venteicher if ((m = vtballoon_alloc_page(sc)) == NULL) {
3456f7e6082SBryan Venteicher sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
34610b59a9bSPeter Grehan break;
3476f7e6082SBryan Venteicher }
34810b59a9bSPeter Grehan
34910b59a9bSPeter Grehan sc->vtballoon_page_frames[i] =
35010b59a9bSPeter Grehan VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
35110b59a9bSPeter Grehan
3525cff1f4dSMark Johnston KASSERT(m->a.queue == PQ_NONE,
353e8bcf696SMark Johnston ("%s: allocated page %p on queue", __func__, m));
354c325e866SKonstantin Belousov TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
35510b59a9bSPeter Grehan }
35610b59a9bSPeter Grehan
35710b59a9bSPeter Grehan if (i > 0)
35810b59a9bSPeter Grehan vtballoon_send_page_frames(sc, vq, i);
35910b59a9bSPeter Grehan }
36010b59a9bSPeter Grehan
36110b59a9bSPeter Grehan static void
vtballoon_deflate(struct vtballoon_softc * sc,int npages)36210b59a9bSPeter Grehan vtballoon_deflate(struct vtballoon_softc *sc, int npages)
36310b59a9bSPeter Grehan {
36410b59a9bSPeter Grehan TAILQ_HEAD(, vm_page) free_pages;
36510b59a9bSPeter Grehan struct virtqueue *vq;
36610b59a9bSPeter Grehan vm_page_t m;
36710b59a9bSPeter Grehan int i;
36810b59a9bSPeter Grehan
36910b59a9bSPeter Grehan vq = sc->vtballoon_deflate_vq;
37010b59a9bSPeter Grehan TAILQ_INIT(&free_pages);
37110b59a9bSPeter Grehan
37210b59a9bSPeter Grehan if (npages > VTBALLOON_PAGES_PER_REQUEST)
37310b59a9bSPeter Grehan npages = VTBALLOON_PAGES_PER_REQUEST;
37410b59a9bSPeter Grehan
37510b59a9bSPeter Grehan for (i = 0; i < npages; i++) {
37610b59a9bSPeter Grehan m = TAILQ_FIRST(&sc->vtballoon_pages);
3776f7e6082SBryan Venteicher KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
37810b59a9bSPeter Grehan
37910b59a9bSPeter Grehan sc->vtballoon_page_frames[i] =
38010b59a9bSPeter Grehan VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
38110b59a9bSPeter Grehan
382c325e866SKonstantin Belousov TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
383c325e866SKonstantin Belousov TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
38410b59a9bSPeter Grehan }
38510b59a9bSPeter Grehan
38610b59a9bSPeter Grehan if (i > 0) {
38710b59a9bSPeter Grehan /* Always tell host first before freeing the pages. */
38810b59a9bSPeter Grehan vtballoon_send_page_frames(sc, vq, i);
38910b59a9bSPeter Grehan
39010b59a9bSPeter Grehan while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
391c325e866SKonstantin Belousov TAILQ_REMOVE(&free_pages, m, plinks.q);
39210b59a9bSPeter Grehan vtballoon_free_page(sc, m);
39310b59a9bSPeter Grehan }
39410b59a9bSPeter Grehan }
39510b59a9bSPeter Grehan
39610b59a9bSPeter Grehan KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
39710b59a9bSPeter Grehan sc->vtballoon_current_npages == 0) ||
39810b59a9bSPeter Grehan (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
3996f7e6082SBryan Venteicher sc->vtballoon_current_npages != 0),
4006f7e6082SBryan Venteicher ("%s: bogus page count %d", __func__,
4016f7e6082SBryan Venteicher sc->vtballoon_current_npages));
40210b59a9bSPeter Grehan }
40310b59a9bSPeter Grehan
40410b59a9bSPeter Grehan static void
vtballoon_send_page_frames(struct vtballoon_softc * sc,struct virtqueue * vq,int npages)40510b59a9bSPeter Grehan vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
40610b59a9bSPeter Grehan int npages)
40710b59a9bSPeter Grehan {
40810b59a9bSPeter Grehan struct sglist sg;
40910b59a9bSPeter Grehan struct sglist_seg segs[1];
41010b59a9bSPeter Grehan void *c;
411b25ddb78SJohn Baldwin int error __diagused;
41210b59a9bSPeter Grehan
41310b59a9bSPeter Grehan sglist_init(&sg, 1, segs);
41410b59a9bSPeter Grehan
41510b59a9bSPeter Grehan error = sglist_append(&sg, sc->vtballoon_page_frames,
41610b59a9bSPeter Grehan npages * sizeof(uint32_t));
41710b59a9bSPeter Grehan KASSERT(error == 0, ("error adding page frames to sglist"));
41810b59a9bSPeter Grehan
41910b59a9bSPeter Grehan error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
42010b59a9bSPeter Grehan KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
421b8a58707SPeter Grehan virtqueue_notify(vq);
42210b59a9bSPeter Grehan
42310b59a9bSPeter Grehan /*
42410b59a9bSPeter Grehan * Inflate and deflate operations are done synchronously. The
42510b59a9bSPeter Grehan * interrupt handler will wake us up.
42610b59a9bSPeter Grehan */
42710b59a9bSPeter Grehan VTBALLOON_LOCK(sc);
42810b59a9bSPeter Grehan while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
4296f7e6082SBryan Venteicher msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
43010b59a9bSPeter Grehan VTBALLOON_UNLOCK(sc);
43110b59a9bSPeter Grehan
43210b59a9bSPeter Grehan KASSERT(c == vq, ("unexpected balloon operation response"));
43310b59a9bSPeter Grehan }
43410b59a9bSPeter Grehan
43510b59a9bSPeter Grehan static void
vtballoon_pop(struct vtballoon_softc * sc)43610b59a9bSPeter Grehan vtballoon_pop(struct vtballoon_softc *sc)
43710b59a9bSPeter Grehan {
43810b59a9bSPeter Grehan
43910b59a9bSPeter Grehan while (!TAILQ_EMPTY(&sc->vtballoon_pages))
44010b59a9bSPeter Grehan vtballoon_deflate(sc, sc->vtballoon_current_npages);
44110b59a9bSPeter Grehan }
44210b59a9bSPeter Grehan
44310b59a9bSPeter Grehan static void
vtballoon_stop(struct vtballoon_softc * sc)44410b59a9bSPeter Grehan vtballoon_stop(struct vtballoon_softc *sc)
44510b59a9bSPeter Grehan {
44610b59a9bSPeter Grehan
44710b59a9bSPeter Grehan virtqueue_disable_intr(sc->vtballoon_inflate_vq);
44810b59a9bSPeter Grehan virtqueue_disable_intr(sc->vtballoon_deflate_vq);
44910b59a9bSPeter Grehan
45010b59a9bSPeter Grehan virtio_stop(sc->vtballoon_dev);
45110b59a9bSPeter Grehan }
45210b59a9bSPeter Grehan
45310b59a9bSPeter Grehan static vm_page_t
vtballoon_alloc_page(struct vtballoon_softc * sc)45410b59a9bSPeter Grehan vtballoon_alloc_page(struct vtballoon_softc *sc)
45510b59a9bSPeter Grehan {
45610b59a9bSPeter Grehan vm_page_t m;
45710b59a9bSPeter Grehan
458a4667e09SMark Johnston m = vm_page_alloc_noobj(VM_ALLOC_NODUMP);
45910b59a9bSPeter Grehan if (m != NULL)
46010b59a9bSPeter Grehan sc->vtballoon_current_npages++;
46110b59a9bSPeter Grehan
46210b59a9bSPeter Grehan return (m);
46310b59a9bSPeter Grehan }
46410b59a9bSPeter Grehan
46510b59a9bSPeter Grehan static void
vtballoon_free_page(struct vtballoon_softc * sc,vm_page_t m)46610b59a9bSPeter Grehan vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
46710b59a9bSPeter Grehan {
46810b59a9bSPeter Grehan
46910b59a9bSPeter Grehan vm_page_free(m);
47010b59a9bSPeter Grehan sc->vtballoon_current_npages--;
47110b59a9bSPeter Grehan }
47210b59a9bSPeter Grehan
47310b59a9bSPeter Grehan static uint32_t
vtballoon_desired_size(struct vtballoon_softc * sc)47410b59a9bSPeter Grehan vtballoon_desired_size(struct vtballoon_softc *sc)
47510b59a9bSPeter Grehan {
47610b59a9bSPeter Grehan uint32_t desired;
47710b59a9bSPeter Grehan
47810b59a9bSPeter Grehan desired = virtio_read_dev_config_4(sc->vtballoon_dev,
47910b59a9bSPeter Grehan offsetof(struct virtio_balloon_config, num_pages));
48010b59a9bSPeter Grehan
481f7f9c266SBryan Venteicher if (vtballoon_modern(sc))
482f7f9c266SBryan Venteicher return (desired);
483f7f9c266SBryan Venteicher else
48410b59a9bSPeter Grehan return (le32toh(desired));
48510b59a9bSPeter Grehan }
48610b59a9bSPeter Grehan
48710b59a9bSPeter Grehan static void
vtballoon_update_size(struct vtballoon_softc * sc)48810b59a9bSPeter Grehan vtballoon_update_size(struct vtballoon_softc *sc)
48910b59a9bSPeter Grehan {
490f7f9c266SBryan Venteicher uint32_t npages;
491f7f9c266SBryan Venteicher
492f7f9c266SBryan Venteicher npages = sc->vtballoon_current_npages;
493f7f9c266SBryan Venteicher if (!vtballoon_modern(sc))
494f7f9c266SBryan Venteicher npages = htole32(npages);
49510b59a9bSPeter Grehan
49610b59a9bSPeter Grehan virtio_write_dev_config_4(sc->vtballoon_dev,
497f7f9c266SBryan Venteicher offsetof(struct virtio_balloon_config, actual), npages);
49810b59a9bSPeter Grehan }
49910b59a9bSPeter Grehan
50010b59a9bSPeter Grehan static int
vtballoon_sleep(struct vtballoon_softc * sc)50110b59a9bSPeter Grehan vtballoon_sleep(struct vtballoon_softc *sc)
50210b59a9bSPeter Grehan {
50310b59a9bSPeter Grehan int rc, timeout;
50410b59a9bSPeter Grehan uint32_t current, desired;
50510b59a9bSPeter Grehan
50610b59a9bSPeter Grehan rc = 0;
50710b59a9bSPeter Grehan current = sc->vtballoon_current_npages;
50810b59a9bSPeter Grehan
50910b59a9bSPeter Grehan VTBALLOON_LOCK(sc);
51010b59a9bSPeter Grehan for (;;) {
51110b59a9bSPeter Grehan if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
51210b59a9bSPeter Grehan rc = 1;
51310b59a9bSPeter Grehan break;
51410b59a9bSPeter Grehan }
51510b59a9bSPeter Grehan
51610b59a9bSPeter Grehan desired = vtballoon_desired_size(sc);
51710b59a9bSPeter Grehan sc->vtballoon_desired_npages = desired;
51810b59a9bSPeter Grehan
51910b59a9bSPeter Grehan /*
52010b59a9bSPeter Grehan * If given, use non-zero timeout on the first time through
52110b59a9bSPeter Grehan * the loop. On subsequent times, timeout will be zero so
52210b59a9bSPeter Grehan * we will reevaluate the desired size of the balloon and
52310b59a9bSPeter Grehan * break out to retry if needed.
52410b59a9bSPeter Grehan */
52510b59a9bSPeter Grehan timeout = sc->vtballoon_timeout;
52610b59a9bSPeter Grehan sc->vtballoon_timeout = 0;
52710b59a9bSPeter Grehan
52810b59a9bSPeter Grehan if (current > desired)
52910b59a9bSPeter Grehan break;
53010b59a9bSPeter Grehan if (current < desired && timeout == 0)
53110b59a9bSPeter Grehan break;
53210b59a9bSPeter Grehan
5336f7e6082SBryan Venteicher msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
53410b59a9bSPeter Grehan }
53510b59a9bSPeter Grehan VTBALLOON_UNLOCK(sc);
53610b59a9bSPeter Grehan
53710b59a9bSPeter Grehan return (rc);
53810b59a9bSPeter Grehan }
53910b59a9bSPeter Grehan
54010b59a9bSPeter Grehan static void
vtballoon_thread(void * xsc)54110b59a9bSPeter Grehan vtballoon_thread(void *xsc)
54210b59a9bSPeter Grehan {
54310b59a9bSPeter Grehan struct vtballoon_softc *sc;
54410b59a9bSPeter Grehan uint32_t current, desired;
54510b59a9bSPeter Grehan
54610b59a9bSPeter Grehan sc = xsc;
54710b59a9bSPeter Grehan
54810b59a9bSPeter Grehan for (;;) {
54910b59a9bSPeter Grehan if (vtballoon_sleep(sc) != 0)
55010b59a9bSPeter Grehan break;
55110b59a9bSPeter Grehan
55210b59a9bSPeter Grehan current = sc->vtballoon_current_npages;
55310b59a9bSPeter Grehan desired = sc->vtballoon_desired_npages;
55410b59a9bSPeter Grehan
55510b59a9bSPeter Grehan if (desired != current) {
55610b59a9bSPeter Grehan if (desired > current)
55710b59a9bSPeter Grehan vtballoon_inflate(sc, desired - current);
55810b59a9bSPeter Grehan else
55910b59a9bSPeter Grehan vtballoon_deflate(sc, current - desired);
56010b59a9bSPeter Grehan
56110b59a9bSPeter Grehan vtballoon_update_size(sc);
56210b59a9bSPeter Grehan }
56310b59a9bSPeter Grehan }
56410b59a9bSPeter Grehan
5656f7e6082SBryan Venteicher kthread_exit();
56610b59a9bSPeter Grehan }
56710b59a9bSPeter Grehan
56810b59a9bSPeter Grehan static void
vtballoon_setup_sysctl(struct vtballoon_softc * sc)569e6cc42f1SBryan Venteicher vtballoon_setup_sysctl(struct vtballoon_softc *sc)
57010b59a9bSPeter Grehan {
57110b59a9bSPeter Grehan device_t dev;
57210b59a9bSPeter Grehan struct sysctl_ctx_list *ctx;
57310b59a9bSPeter Grehan struct sysctl_oid *tree;
57410b59a9bSPeter Grehan struct sysctl_oid_list *child;
57510b59a9bSPeter Grehan
57610b59a9bSPeter Grehan dev = sc->vtballoon_dev;
57710b59a9bSPeter Grehan ctx = device_get_sysctl_ctx(dev);
57810b59a9bSPeter Grehan tree = device_get_sysctl_tree(dev);
57910b59a9bSPeter Grehan child = SYSCTL_CHILDREN(tree);
58010b59a9bSPeter Grehan
58110b59a9bSPeter Grehan SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
58210b59a9bSPeter Grehan CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
58310b59a9bSPeter Grehan "Desired balloon size in pages");
58410b59a9bSPeter Grehan
58510b59a9bSPeter Grehan SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
58610b59a9bSPeter Grehan CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
58710b59a9bSPeter Grehan "Current balloon size in pages");
58810b59a9bSPeter Grehan }
589