1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* 32 * BERI virtio block backend driver 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bus.h> 38 #include <sys/kernel.h> 39 #include <sys/module.h> 40 #include <sys/rman.h> 41 #include <sys/conf.h> 42 #include <sys/stat.h> 43 #include <sys/endian.h> 44 #include <sys/disk.h> 45 #include <sys/vnode.h> 46 #include <sys/fcntl.h> 47 #include <sys/kthread.h> 48 #include <sys/buf.h> 49 #include <sys/mdioctl.h> 50 #include <sys/namei.h> 51 52 #include <machine/bus.h> 53 #include <machine/fdt.h> 54 #include <machine/cpu.h> 55 #include <machine/intr.h> 56 57 #include <dev/fdt/fdt_common.h> 58 #include <dev/ofw/openfirm.h> 59 #include <dev/ofw/ofw_bus.h> 60 #include <dev/ofw/ofw_bus_subr.h> 61 62 #include <dev/beri/virtio/virtio.h> 63 #include <dev/beri/virtio/virtio_mmio_platform.h> 64 #include <dev/altera/pio/pio.h> 65 #include <dev/virtio/mmio/virtio_mmio.h> 66 #include <dev/virtio/block/virtio_blk.h> 67 #include <dev/virtio/virtio_ids.h> 68 #include <dev/virtio/virtio_config.h> 69 #include <dev/virtio/virtio_ring.h> 70 71 #include "pio_if.h" 72 73 #define DPRINTF(fmt, ...) 74 75 /* We use indirect descriptors */ 76 #define NUM_DESCS 1 77 #define NUM_QUEUES 1 78 79 #define VTBLK_BLK_ID_BYTES 20 80 #define VTBLK_MAXSEGS 256 81 82 struct beri_vtblk_softc { 83 struct resource *res[1]; 84 bus_space_tag_t bst; 85 bus_space_handle_t bsh; 86 struct cdev *cdev; 87 device_t dev; 88 int opened; 89 device_t pio_recv; 90 device_t pio_send; 91 struct vqueue_info vs_queues[NUM_QUEUES]; 92 char ident[VTBLK_BLK_ID_BYTES]; 93 struct ucred *cred; 94 struct vnode *vnode; 95 struct thread *vtblk_ktd; 96 struct sx sc_mtx; 97 int beri_mem_offset; 98 struct md_ioctl *mdio; 99 struct virtio_blk_config *cfg; 100 }; 101 102 static struct resource_spec beri_spec[] = { 103 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 104 { -1, 0 } 105 }; 106 107 static int 108 vtblk_rdwr(struct beri_vtblk_softc *sc, struct iovec *iov, 109 int cnt, int offset, int operation, int iolen) 110 { 111 struct vnode *vp; 112 struct mount *mp; 113 struct uio auio; 114 int error; 115 116 bzero(&auio, sizeof(auio)); 117 118 vp = sc->vnode; 119 120 KASSERT(vp != NULL, ("file not opened")); 121 122 auio.uio_iov = iov; 123 auio.uio_iovcnt = cnt; 124 auio.uio_offset = offset; 125 auio.uio_segflg = UIO_SYSSPACE; 126 auio.uio_rw = operation; 127 auio.uio_resid = iolen; 128 auio.uio_td = curthread; 129 130 if (operation == 0) { 131 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 132 error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred); 133 VOP_UNLOCK(vp); 134 } else { 135 (void) vn_start_write(vp, &mp, V_WAIT); 136 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 137 error = VOP_WRITE(vp, &auio, IO_SYNC, sc->cred); 138 VOP_UNLOCK(vp); 139 vn_finished_write(mp); 140 } 141 142 return (error); 143 } 144 145 static void 146 vtblk_proc(struct beri_vtblk_softc *sc, struct vqueue_info *vq) 147 { 148 struct iovec iov[VTBLK_MAXSEGS + 2]; 149 uint16_t flags[VTBLK_MAXSEGS + 2]; 150 struct virtio_blk_outhdr *vbh; 151 struct iovec *tiov; 152 uint8_t *status; 153 off_t offset; 154 int iolen; 155 int type; 156 int i, n; 157 int err; 158 159 n = vq_getchain(sc->beri_mem_offset, vq, iov, 160 VTBLK_MAXSEGS + 2, flags); 161 KASSERT(n >= 2 && n <= VTBLK_MAXSEGS + 2, 162 ("wrong n value %d", n)); 163 164 tiov = getcopy(iov, n); 165 vbh = iov[0].iov_base; 166 167 status = iov[n-1].iov_base; 168 KASSERT(iov[n-1].iov_len == 1, 169 ("iov_len == %d", iov[n-1].iov_len)); 170 171 type = be32toh(vbh->type) & ~VIRTIO_BLK_T_BARRIER; 172 offset = be64toh(vbh->sector) * DEV_BSIZE; 173 174 iolen = 0; 175 for (i = 1; i < (n-1); i++) { 176 iolen += iov[i].iov_len; 177 } 178 179 switch (type) { 180 case VIRTIO_BLK_T_OUT: 181 case VIRTIO_BLK_T_IN: 182 err = vtblk_rdwr(sc, tiov + 1, i - 1, 183 offset, type, iolen); 184 break; 185 case VIRTIO_BLK_T_GET_ID: 186 /* Assume a single buffer */ 187 strncpy(iov[1].iov_base, sc->ident, 188 MIN(iov[1].iov_len, sizeof(sc->ident))); 189 err = 0; 190 break; 191 case VIRTIO_BLK_T_FLUSH: 192 /* Possible? */ 193 default: 194 err = -ENOSYS; 195 break; 196 } 197 198 if (err < 0) { 199 if (err == -ENOSYS) { 200 *status = VIRTIO_BLK_S_UNSUPP; 201 } else 202 *status = VIRTIO_BLK_S_IOERR; 203 } else 204 *status = VIRTIO_BLK_S_OK; 205 206 free(tiov, M_DEVBUF); 207 vq_relchain(vq, iov, n, 1); 208 } 209 210 static int 211 close_file(struct beri_vtblk_softc *sc, struct thread *td) 212 { 213 int error; 214 215 if (sc->vnode != NULL) { 216 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); 217 sc->vnode->v_vflag &= ~VV_MD; 218 VOP_UNLOCK(sc->vnode); 219 error = vn_close(sc->vnode, (FREAD|FWRITE), 220 sc->cred, td); 221 if (error != 0) 222 return (error); 223 sc->vnode = NULL; 224 } 225 226 if (sc->cred != NULL) 227 crfree(sc->cred); 228 229 return (0); 230 } 231 232 static int 233 open_file(struct beri_vtblk_softc *sc, struct thread *td) 234 { 235 struct nameidata nd; 236 struct vattr vattr; 237 int error; 238 int flags; 239 240 flags = (FREAD | FWRITE); 241 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->mdio->md_file); 242 error = vn_open(&nd, &flags, 0, NULL); 243 if (error != 0) 244 return (error); 245 NDFREE_PNBUF(&nd); 246 247 if (nd.ni_vp->v_type != VREG) { 248 return (EINVAL); 249 } 250 251 error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); 252 if (error != 0) 253 return (error); 254 255 if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { 256 vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); 257 if (VN_IS_DOOMED(nd.ni_vp)) { 258 return (1); 259 } 260 } 261 nd.ni_vp->v_vflag |= VV_MD; 262 VOP_UNLOCK(nd.ni_vp); 263 264 sc->vnode = nd.ni_vp; 265 sc->cred = crhold(td->td_ucred); 266 267 return (0); 268 } 269 270 static int 271 vtblk_notify(struct beri_vtblk_softc *sc) 272 { 273 struct vqueue_info *vq; 274 int queue; 275 int reg; 276 277 vq = &sc->vs_queues[0]; 278 if (!vq_ring_ready(vq)) 279 return (0); 280 281 if (!sc->opened) 282 return (0); 283 284 reg = READ2(sc, VIRTIO_MMIO_QUEUE_NOTIFY); 285 queue = be16toh(reg); 286 287 KASSERT(queue == 0, ("we support single queue only")); 288 289 /* Process new descriptors */ 290 vq = &sc->vs_queues[queue]; 291 vq->vq_save_used = be16toh(vq->vq_used->idx); 292 while (vq_has_descs(vq)) 293 vtblk_proc(sc, vq); 294 295 /* Interrupt the other side */ 296 if ((be16toh(vq->vq_avail->flags) & VRING_AVAIL_F_NO_INTERRUPT) == 0) { 297 reg = htobe32(VIRTIO_MMIO_INT_VRING); 298 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 299 PIO_SET(sc->pio_send, Q_INTR, 1); 300 } 301 302 return (0); 303 } 304 305 static int 306 vq_init(struct beri_vtblk_softc *sc) 307 { 308 struct vqueue_info *vq; 309 uint8_t *base; 310 int size; 311 int reg; 312 int pfn; 313 314 vq = &sc->vs_queues[0]; 315 vq->vq_qsize = NUM_DESCS; 316 317 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); 318 pfn = be32toh(reg); 319 vq->vq_pfn = pfn; 320 321 size = vring_size(vq->vq_qsize, VRING_ALIGN); 322 base = paddr_map(sc->beri_mem_offset, 323 (pfn << PAGE_SHIFT), size); 324 325 /* First pages are descriptors */ 326 vq->vq_desc = (struct vring_desc *)base; 327 base += vq->vq_qsize * sizeof(struct vring_desc); 328 329 /* Then avail ring */ 330 vq->vq_avail = (struct vring_avail *)base; 331 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); 332 333 /* Then it's rounded up to the next page */ 334 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); 335 336 /* And the last pages are the used ring */ 337 vq->vq_used = (struct vring_used *)base; 338 339 /* Mark queue as allocated, and start at 0 when we use it. */ 340 vq->vq_flags = VQ_ALLOC; 341 vq->vq_last_avail = 0; 342 343 return (0); 344 } 345 346 static void 347 vtblk_thread(void *arg) 348 { 349 struct beri_vtblk_softc *sc; 350 int err; 351 352 sc = arg; 353 354 sx_xlock(&sc->sc_mtx); 355 for (;;) { 356 err = msleep(sc, &sc->sc_mtx, PCATCH | PZERO, "prd", hz); 357 vtblk_notify(sc); 358 } 359 sx_xunlock(&sc->sc_mtx); 360 361 kthread_exit(); 362 } 363 364 static int 365 backend_info(struct beri_vtblk_softc *sc) 366 { 367 struct virtio_blk_config *cfg; 368 uint32_t *s; 369 int reg; 370 int i; 371 372 /* Specify that we provide block device */ 373 reg = htobe32(VIRTIO_ID_BLOCK); 374 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); 375 376 /* Queue size */ 377 reg = htobe32(NUM_DESCS); 378 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); 379 380 /* Our features */ 381 reg = htobe32(VIRTIO_RING_F_INDIRECT_DESC 382 | VIRTIO_BLK_F_BLK_SIZE 383 | VIRTIO_BLK_F_SEG_MAX); 384 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); 385 386 cfg = sc->cfg; 387 cfg->capacity = htobe64(sc->mdio->md_mediasize / DEV_BSIZE); 388 cfg->size_max = 0; /* not negotiated */ 389 cfg->seg_max = htobe32(VTBLK_MAXSEGS); 390 cfg->blk_size = htobe32(DEV_BSIZE); 391 392 s = (uint32_t *)cfg; 393 394 for (i = 0; i < sizeof(struct virtio_blk_config); i+=4) { 395 WRITE4(sc, VIRTIO_MMIO_CONFIG + i, *s); 396 s+=1; 397 } 398 399 strncpy(sc->ident, "Virtio block backend", sizeof(sc->ident)); 400 401 return (0); 402 } 403 404 static void 405 vtblk_intr(void *arg) 406 { 407 struct beri_vtblk_softc *sc; 408 int pending; 409 int reg; 410 411 sc = arg; 412 413 reg = PIO_READ(sc->pio_recv); 414 415 /* Ack */ 416 PIO_SET(sc->pio_recv, reg, 0); 417 418 pending = htobe32(reg); 419 420 if (pending & Q_PFN) { 421 vq_init(sc); 422 } 423 424 if (pending & Q_NOTIFY) { 425 wakeup(sc); 426 } 427 } 428 429 static int 430 beri_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 431 int flags, struct thread *td) 432 { 433 struct beri_vtblk_softc *sc; 434 int err; 435 436 sc = dev->si_drv1; 437 438 switch (cmd) { 439 case MDIOCATTACH: 440 /* take file as argument */ 441 if (sc->vnode != NULL) { 442 /* Already opened */ 443 return (1); 444 } 445 sc->mdio = (struct md_ioctl *)addr; 446 backend_info(sc); 447 DPRINTF("opening file, td 0x%08x\n", (int)td); 448 err = open_file(sc, td); 449 if (err) 450 return (err); 451 PIO_SETUP_IRQ(sc->pio_recv, vtblk_intr, sc); 452 sc->opened = 1; 453 break; 454 case MDIOCDETACH: 455 if (sc->vnode == NULL) { 456 /* File not opened */ 457 return (1); 458 } 459 sc->opened = 0; 460 DPRINTF("closing file, td 0x%08x\n", (int)td); 461 err = close_file(sc, td); 462 if (err) 463 return (err); 464 PIO_TEARDOWN_IRQ(sc->pio_recv); 465 break; 466 default: 467 break; 468 } 469 470 return (0); 471 } 472 473 static struct cdevsw beri_cdevsw = { 474 .d_version = D_VERSION, 475 .d_ioctl = beri_ioctl, 476 .d_name = "virtio block backend", 477 }; 478 479 static int 480 beri_vtblk_probe(device_t dev) 481 { 482 483 if (!ofw_bus_status_okay(dev)) 484 return (ENXIO); 485 486 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtblk")) 487 return (ENXIO); 488 489 device_set_desc(dev, "SRI-Cambridge BERI block"); 490 return (BUS_PROBE_DEFAULT); 491 } 492 493 static int 494 beri_vtblk_attach(device_t dev) 495 { 496 struct beri_vtblk_softc *sc; 497 int error; 498 499 sc = device_get_softc(dev); 500 sc->dev = dev; 501 502 if (bus_alloc_resources(dev, beri_spec, sc->res)) { 503 device_printf(dev, "could not allocate resources\n"); 504 return (ENXIO); 505 } 506 507 /* Memory interface */ 508 sc->bst = rman_get_bustag(sc->res[0]); 509 sc->bsh = rman_get_bushandle(sc->res[0]); 510 511 sc->cfg = malloc(sizeof(struct virtio_blk_config), 512 M_DEVBUF, M_NOWAIT|M_ZERO); 513 514 sx_init(&sc->sc_mtx, device_get_nameunit(sc->dev)); 515 516 error = kthread_add(vtblk_thread, sc, NULL, &sc->vtblk_ktd, 517 0, 0, "beri_virtio_block"); 518 if (error) { 519 device_printf(dev, "cannot create kthread\n"); 520 return (ENXIO); 521 } 522 523 if (setup_offset(dev, &sc->beri_mem_offset) != 0) 524 return (ENXIO); 525 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) 526 return (ENXIO); 527 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) 528 return (ENXIO); 529 530 sc->cdev = make_dev(&beri_cdevsw, 0, UID_ROOT, GID_WHEEL, 531 S_IRWXU, "beri_vtblk"); 532 if (sc->cdev == NULL) { 533 device_printf(dev, "Failed to create character device.\n"); 534 return (ENXIO); 535 } 536 537 sc->cdev->si_drv1 = sc; 538 return (0); 539 } 540 541 static device_method_t beri_vtblk_methods[] = { 542 DEVMETHOD(device_probe, beri_vtblk_probe), 543 DEVMETHOD(device_attach, beri_vtblk_attach), 544 { 0, 0 } 545 }; 546 547 static driver_t beri_vtblk_driver = { 548 "beri_vtblk", 549 beri_vtblk_methods, 550 sizeof(struct beri_vtblk_softc), 551 }; 552 553 DRIVER_MODULE(beri_vtblk, simplebus, beri_vtblk_driver, 0, 0); 554