1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/linker_set.h> 36 #include <sys/select.h> 37 #include <sys/uio.h> 38 #include <sys/ioctl.h> 39 #include <net/ethernet.h> 40 #include <net/if.h> /* IFNAMSIZ */ 41 42 #include <err.h> 43 #include <errno.h> 44 #include <fcntl.h> 45 #include <stdio.h> 46 #include <stdlib.h> 47 #include <stdint.h> 48 #include <string.h> 49 #include <strings.h> 50 #include <unistd.h> 51 #include <assert.h> 52 #include <pthread.h> 53 #include <pthread_np.h> 54 55 #include "bhyverun.h" 56 #include "pci_emul.h" 57 #include "mevent.h" 58 #include "virtio.h" 59 #include "net_utils.h" 60 #include "net_backends.h" 61 #include "iov.h" 62 63 #define VTNET_RINGSZ 1024 64 65 #define VTNET_MAXSEGS 256 66 67 #define VTNET_MAX_PKT_LEN (65536 + 64) 68 69 #define VTNET_S_HOSTCAPS \ 70 ( VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | \ 71 VIRTIO_F_NOTIFY_ON_EMPTY | VIRTIO_RING_F_INDIRECT_DESC) 72 73 /* 74 * PCI config-space "registers" 75 */ 76 struct virtio_net_config { 77 uint8_t mac[6]; 78 uint16_t status; 79 } __packed; 80 81 /* 82 * Queue definitions. 83 */ 84 #define VTNET_RXQ 0 85 #define VTNET_TXQ 1 86 #define VTNET_CTLQ 2 /* NB: not yet supported */ 87 88 #define VTNET_MAXQ 3 89 90 /* 91 * Debug printf 92 */ 93 static int pci_vtnet_debug; 94 #define DPRINTF(params) if (pci_vtnet_debug) printf params 95 #define WPRINTF(params) printf params 96 97 /* 98 * Per-device softc 99 */ 100 struct pci_vtnet_softc { 101 struct virtio_softc vsc_vs; 102 struct vqueue_info vsc_queues[VTNET_MAXQ - 1]; 103 pthread_mutex_t vsc_mtx; 104 105 net_backend_t *vsc_be; 106 107 int resetting; /* protected by tx_mtx */ 108 109 uint64_t vsc_features; /* negotiated features */ 110 111 pthread_mutex_t rx_mtx; 112 unsigned int rx_vhdrlen; 113 int rx_merge; /* merged rx bufs in use */ 114 115 pthread_t tx_tid; 116 pthread_mutex_t tx_mtx; 117 pthread_cond_t tx_cond; 118 int tx_in_progress; 119 120 struct virtio_net_config vsc_config; 121 struct virtio_consts vsc_consts; 122 }; 123 124 static void pci_vtnet_reset(void *); 125 /* static void pci_vtnet_notify(void *, struct vqueue_info *); */ 126 static int pci_vtnet_cfgread(void *, int, int, uint32_t *); 127 static int pci_vtnet_cfgwrite(void *, int, int, uint32_t); 128 static void pci_vtnet_neg_features(void *, uint64_t); 129 130 static struct virtio_consts vtnet_vi_consts = { 131 "vtnet", /* our name */ 132 VTNET_MAXQ - 1, /* we currently support 2 virtqueues */ 133 sizeof(struct virtio_net_config), /* config reg size */ 134 pci_vtnet_reset, /* reset */ 135 NULL, /* device-wide qnotify -- not used */ 136 pci_vtnet_cfgread, /* read PCI config */ 137 pci_vtnet_cfgwrite, /* write PCI config */ 138 pci_vtnet_neg_features, /* apply negotiated features */ 139 VTNET_S_HOSTCAPS, /* our capabilities */ 140 }; 141 142 static void 143 pci_vtnet_reset(void *vsc) 144 { 145 struct pci_vtnet_softc *sc = vsc; 146 147 DPRINTF(("vtnet: device reset requested !\n")); 148 149 /* Acquire the RX lock to block RX processing. */ 150 pthread_mutex_lock(&sc->rx_mtx); 151 152 /* Set sc->resetting and give a chance to the TX thread to stop. */ 153 pthread_mutex_lock(&sc->tx_mtx); 154 sc->resetting = 1; 155 while (sc->tx_in_progress) { 156 pthread_mutex_unlock(&sc->tx_mtx); 157 usleep(10000); 158 pthread_mutex_lock(&sc->tx_mtx); 159 } 160 161 sc->rx_merge = 1; 162 sc->rx_vhdrlen = sizeof(struct virtio_net_rxhdr); 163 164 /* 165 * Now reset rings, MSI-X vectors, and negotiated capabilities. 166 * Do that with the TX lock held, since we need to reset 167 * sc->resetting. 168 */ 169 vi_reset_dev(&sc->vsc_vs); 170 171 sc->resetting = 0; 172 pthread_mutex_unlock(&sc->tx_mtx); 173 pthread_mutex_unlock(&sc->rx_mtx); 174 } 175 176 struct virtio_mrg_rxbuf_info { 177 uint16_t idx; 178 uint16_t pad; 179 uint32_t len; 180 }; 181 182 static void 183 pci_vtnet_rx(struct pci_vtnet_softc *sc) 184 { 185 struct virtio_mrg_rxbuf_info info[VTNET_MAXSEGS]; 186 struct iovec iov[VTNET_MAXSEGS + 1]; 187 struct vqueue_info *vq; 188 uint32_t cur_iov_bytes; 189 struct iovec *cur_iov; 190 uint16_t cur_iov_len; 191 uint32_t ulen; 192 int n_chains; 193 int len; 194 195 vq = &sc->vsc_queues[VTNET_RXQ]; 196 for (;;) { 197 /* 198 * Get a descriptor chain to store the next ingress 199 * packet. In case of mergeable rx buffers, get as 200 * many chains as necessary in order to make room 201 * for a maximum sized LRO packet. 202 */ 203 cur_iov_bytes = 0; 204 cur_iov_len = 0; 205 cur_iov = iov; 206 n_chains = 0; 207 do { 208 int n = vq_getchain(vq, &info[n_chains].idx, cur_iov, 209 VTNET_MAXSEGS - cur_iov_len, NULL); 210 211 if (n == 0) { 212 /* 213 * No rx buffers. Enable RX kicks and double 214 * check. 215 */ 216 vq_kick_enable(vq); 217 if (!vq_has_descs(vq)) { 218 /* 219 * Still no buffers. Return the unused 220 * chains (if any), interrupt if needed 221 * (including for NOTIFY_ON_EMPTY), and 222 * disable the backend until the next 223 * kick. 224 */ 225 vq_retchains(vq, n_chains); 226 vq_endchains(vq, /*used_all_avail=*/1); 227 netbe_rx_disable(sc->vsc_be); 228 return; 229 } 230 231 /* More rx buffers found, so keep going. */ 232 vq_kick_disable(vq); 233 continue; 234 } 235 assert(n >= 1 && cur_iov_len + n <= VTNET_MAXSEGS); 236 cur_iov_len += n; 237 if (!sc->rx_merge) { 238 n_chains = 1; 239 break; 240 } 241 info[n_chains].len = (uint32_t)count_iov(cur_iov, n); 242 cur_iov_bytes += info[n_chains].len; 243 cur_iov += n; 244 n_chains++; 245 } while (cur_iov_bytes < VTNET_MAX_PKT_LEN && 246 cur_iov_len < VTNET_MAXSEGS); 247 248 len = netbe_recv(sc->vsc_be, iov, cur_iov_len); 249 250 if (len <= 0) { 251 /* 252 * No more packets (len == 0), or backend errored 253 * (err < 0). Return unused available buffers 254 * and stop. 255 */ 256 vq_retchains(vq, n_chains); 257 /* Interrupt if needed/appropriate and stop. */ 258 vq_endchains(vq, /*used_all_avail=*/0); 259 return; 260 } 261 262 ulen = (uint32_t)len; /* avoid too many casts below */ 263 264 /* Publish the used buffers to the guest. */ 265 if (!sc->rx_merge) { 266 vq_relchain(vq, info[0].idx, ulen); 267 } else { 268 struct virtio_net_rxhdr *hdr = iov[0].iov_base; 269 uint32_t iolen; 270 int i = 0; 271 272 assert(iov[0].iov_len >= sizeof(*hdr)); 273 274 do { 275 iolen = info[i].len; 276 if (iolen > ulen) { 277 iolen = ulen; 278 } 279 vq_relchain_prepare(vq, info[i].idx, iolen); 280 ulen -= iolen; 281 i++; 282 assert(i <= n_chains); 283 } while (ulen > 0); 284 285 hdr->vrh_bufs = i; 286 vq_relchain_publish(vq); 287 vq_retchains(vq, n_chains - i); 288 } 289 } 290 291 } 292 293 /* 294 * Called when there is read activity on the backend file descriptor. 295 * Each buffer posted by the guest is assumed to be able to contain 296 * an entire ethernet frame + rx header. 297 */ 298 static void 299 pci_vtnet_rx_callback(int fd, enum ev_type type, void *param) 300 { 301 struct pci_vtnet_softc *sc = param; 302 303 pthread_mutex_lock(&sc->rx_mtx); 304 pci_vtnet_rx(sc); 305 pthread_mutex_unlock(&sc->rx_mtx); 306 307 } 308 309 /* Called on RX kick. */ 310 static void 311 pci_vtnet_ping_rxq(void *vsc, struct vqueue_info *vq) 312 { 313 struct pci_vtnet_softc *sc = vsc; 314 315 /* 316 * A qnotify means that the rx process can now begin. 317 */ 318 pthread_mutex_lock(&sc->rx_mtx); 319 vq_kick_disable(vq); 320 netbe_rx_enable(sc->vsc_be); 321 pthread_mutex_unlock(&sc->rx_mtx); 322 } 323 324 /* TX virtqueue processing, called by the TX thread. */ 325 static void 326 pci_vtnet_proctx(struct pci_vtnet_softc *sc, struct vqueue_info *vq) 327 { 328 struct iovec iov[VTNET_MAXSEGS + 1]; 329 uint16_t idx; 330 ssize_t len; 331 int n; 332 333 /* 334 * Obtain chain of descriptors. The first descriptor also 335 * contains the virtio-net header. 336 */ 337 n = vq_getchain(vq, &idx, iov, VTNET_MAXSEGS, NULL); 338 assert(n >= 1 && n <= VTNET_MAXSEGS); 339 340 len = netbe_send(sc->vsc_be, iov, n); 341 342 /* chain is processed, release it and set len */ 343 vq_relchain(vq, idx, len > 0 ? len : 0); 344 } 345 346 /* Called on TX kick. */ 347 static void 348 pci_vtnet_ping_txq(void *vsc, struct vqueue_info *vq) 349 { 350 struct pci_vtnet_softc *sc = vsc; 351 352 /* 353 * Any ring entries to process? 354 */ 355 if (!vq_has_descs(vq)) 356 return; 357 358 /* Signal the tx thread for processing */ 359 pthread_mutex_lock(&sc->tx_mtx); 360 vq_kick_disable(vq); 361 if (sc->tx_in_progress == 0) 362 pthread_cond_signal(&sc->tx_cond); 363 pthread_mutex_unlock(&sc->tx_mtx); 364 } 365 366 /* 367 * Thread which will handle processing of TX desc 368 */ 369 static void * 370 pci_vtnet_tx_thread(void *param) 371 { 372 struct pci_vtnet_softc *sc = param; 373 struct vqueue_info *vq; 374 int error; 375 376 vq = &sc->vsc_queues[VTNET_TXQ]; 377 378 /* 379 * Let us wait till the tx queue pointers get initialised & 380 * first tx signaled 381 */ 382 pthread_mutex_lock(&sc->tx_mtx); 383 error = pthread_cond_wait(&sc->tx_cond, &sc->tx_mtx); 384 assert(error == 0); 385 386 for (;;) { 387 /* note - tx mutex is locked here */ 388 while (sc->resetting || !vq_has_descs(vq)) { 389 vq_kick_enable(vq); 390 if (!sc->resetting && vq_has_descs(vq)) 391 break; 392 393 sc->tx_in_progress = 0; 394 error = pthread_cond_wait(&sc->tx_cond, &sc->tx_mtx); 395 assert(error == 0); 396 } 397 vq_kick_disable(vq); 398 sc->tx_in_progress = 1; 399 pthread_mutex_unlock(&sc->tx_mtx); 400 401 do { 402 /* 403 * Run through entries, placing them into 404 * iovecs and sending when an end-of-packet 405 * is found 406 */ 407 pci_vtnet_proctx(sc, vq); 408 } while (vq_has_descs(vq)); 409 410 /* 411 * Generate an interrupt if needed. 412 */ 413 vq_endchains(vq, /*used_all_avail=*/1); 414 415 pthread_mutex_lock(&sc->tx_mtx); 416 } 417 } 418 419 #ifdef notyet 420 static void 421 pci_vtnet_ping_ctlq(void *vsc, struct vqueue_info *vq) 422 { 423 424 DPRINTF(("vtnet: control qnotify!\n\r")); 425 } 426 #endif 427 428 static int 429 pci_vtnet_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) 430 { 431 struct pci_vtnet_softc *sc; 432 char tname[MAXCOMLEN + 1]; 433 int mac_provided; 434 435 /* 436 * Allocate data structures for further virtio initializations. 437 * sc also contains a copy of vtnet_vi_consts, since capabilities 438 * change depending on the backend. 439 */ 440 sc = calloc(1, sizeof(struct pci_vtnet_softc)); 441 442 sc->vsc_consts = vtnet_vi_consts; 443 pthread_mutex_init(&sc->vsc_mtx, NULL); 444 445 sc->vsc_queues[VTNET_RXQ].vq_qsize = VTNET_RINGSZ; 446 sc->vsc_queues[VTNET_RXQ].vq_notify = pci_vtnet_ping_rxq; 447 sc->vsc_queues[VTNET_TXQ].vq_qsize = VTNET_RINGSZ; 448 sc->vsc_queues[VTNET_TXQ].vq_notify = pci_vtnet_ping_txq; 449 #ifdef notyet 450 sc->vsc_queues[VTNET_CTLQ].vq_qsize = VTNET_RINGSZ; 451 sc->vsc_queues[VTNET_CTLQ].vq_notify = pci_vtnet_ping_ctlq; 452 #endif 453 454 /* 455 * Attempt to open the backend device and read the MAC address 456 * if specified. 457 */ 458 mac_provided = 0; 459 if (opts != NULL) { 460 char *devname; 461 char *vtopts; 462 int err; 463 464 devname = vtopts = strdup(opts); 465 (void) strsep(&vtopts, ","); 466 467 if (vtopts != NULL) { 468 err = net_parsemac(vtopts, sc->vsc_config.mac); 469 if (err != 0) { 470 free(devname); 471 free(sc); 472 return (err); 473 } 474 mac_provided = 1; 475 } 476 477 err = netbe_init(&sc->vsc_be, devname, pci_vtnet_rx_callback, 478 sc); 479 free(devname); 480 if (err) { 481 free(sc); 482 return (err); 483 } 484 sc->vsc_consts.vc_hv_caps |= netbe_get_cap(sc->vsc_be); 485 } 486 487 if (!mac_provided) { 488 net_genmac(pi, sc->vsc_config.mac); 489 } 490 491 /* initialize config space */ 492 pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_NET); 493 pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR); 494 pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK); 495 pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_NET); 496 pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR); 497 498 /* Link is up if we managed to open backend device. */ 499 sc->vsc_config.status = (opts == NULL || sc->vsc_be); 500 501 vi_softc_linkup(&sc->vsc_vs, &sc->vsc_consts, sc, pi, sc->vsc_queues); 502 sc->vsc_vs.vs_mtx = &sc->vsc_mtx; 503 504 /* use BAR 1 to map MSI-X table and PBA, if we're using MSI-X */ 505 if (vi_intr_init(&sc->vsc_vs, 1, fbsdrun_virtio_msix())) { 506 free(sc); 507 return (1); 508 } 509 510 /* use BAR 0 to map config regs in IO space */ 511 vi_set_io_bar(&sc->vsc_vs, 0); 512 513 sc->resetting = 0; 514 515 sc->rx_merge = 1; 516 sc->rx_vhdrlen = sizeof(struct virtio_net_rxhdr); 517 pthread_mutex_init(&sc->rx_mtx, NULL); 518 519 /* 520 * Initialize tx semaphore & spawn TX processing thread. 521 * As of now, only one thread for TX desc processing is 522 * spawned. 523 */ 524 sc->tx_in_progress = 0; 525 pthread_mutex_init(&sc->tx_mtx, NULL); 526 pthread_cond_init(&sc->tx_cond, NULL); 527 pthread_create(&sc->tx_tid, NULL, pci_vtnet_tx_thread, (void *)sc); 528 snprintf(tname, sizeof(tname), "vtnet-%d:%d tx", pi->pi_slot, 529 pi->pi_func); 530 pthread_set_name_np(sc->tx_tid, tname); 531 532 return (0); 533 } 534 535 static int 536 pci_vtnet_cfgwrite(void *vsc, int offset, int size, uint32_t value) 537 { 538 struct pci_vtnet_softc *sc = vsc; 539 void *ptr; 540 541 if (offset < (int)sizeof(sc->vsc_config.mac)) { 542 assert(offset + size <= (int)sizeof(sc->vsc_config.mac)); 543 /* 544 * The driver is allowed to change the MAC address 545 */ 546 ptr = &sc->vsc_config.mac[offset]; 547 memcpy(ptr, &value, size); 548 } else { 549 /* silently ignore other writes */ 550 DPRINTF(("vtnet: write to readonly reg %d\n\r", offset)); 551 } 552 553 return (0); 554 } 555 556 static int 557 pci_vtnet_cfgread(void *vsc, int offset, int size, uint32_t *retval) 558 { 559 struct pci_vtnet_softc *sc = vsc; 560 void *ptr; 561 562 ptr = (uint8_t *)&sc->vsc_config + offset; 563 memcpy(retval, ptr, size); 564 return (0); 565 } 566 567 static void 568 pci_vtnet_neg_features(void *vsc, uint64_t negotiated_features) 569 { 570 struct pci_vtnet_softc *sc = vsc; 571 572 sc->vsc_features = negotiated_features; 573 574 if (!(negotiated_features & VIRTIO_NET_F_MRG_RXBUF)) { 575 sc->rx_merge = 0; 576 /* Without mergeable rx buffers, virtio-net header is 2 577 * bytes shorter than sizeof(struct virtio_net_rxhdr). */ 578 sc->rx_vhdrlen = sizeof(struct virtio_net_rxhdr) - 2; 579 } 580 581 /* Tell the backend to enable some capabilities it has advertised. */ 582 netbe_set_cap(sc->vsc_be, negotiated_features, sc->rx_vhdrlen); 583 } 584 585 static struct pci_devemu pci_de_vnet = { 586 .pe_emu = "virtio-net", 587 .pe_init = pci_vtnet_init, 588 .pe_barwrite = vi_pci_write, 589 .pe_barread = vi_pci_read 590 }; 591 PCI_EMUL_SET(pci_de_vnet); 592