1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2016 Jakub Klama <jceel@FreeBSD.org>. 5 * Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer 13 * in this position and unchanged. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/linker_set.h> 36 #include <sys/types.h> 37 #include <sys/uio.h> 38 #include <sys/time.h> 39 #include <sys/queue.h> 40 #include <sys/sbuf.h> 41 42 #include <errno.h> 43 #include <fcntl.h> 44 #include <stdio.h> 45 #include <stdlib.h> 46 #include <stdbool.h> 47 #include <string.h> 48 #include <unistd.h> 49 #include <assert.h> 50 #include <pthread.h> 51 #include <pthread_np.h> 52 53 #include <cam/scsi/scsi_all.h> 54 #include <cam/scsi/scsi_message.h> 55 #include <cam/ctl/ctl.h> 56 #include <cam/ctl/ctl_io.h> 57 #include <cam/ctl/ctl_backend.h> 58 #include <cam/ctl/ctl_ioctl.h> 59 #include <cam/ctl/ctl_util.h> 60 #include <cam/ctl/ctl_scsi_all.h> 61 #include <camlib.h> 62 63 #include "bhyverun.h" 64 #include "config.h" 65 #include "debug.h" 66 #include "pci_emul.h" 67 #include "virtio.h" 68 #include "iov.h" 69 70 #define VTSCSI_RINGSZ 64 71 #define VTSCSI_REQUESTQ 1 72 #define VTSCSI_THR_PER_Q 16 73 #define VTSCSI_MAXQ (VTSCSI_REQUESTQ + 2) 74 #define VTSCSI_MAXSEG 64 75 76 #define VTSCSI_IN_HEADER_LEN(_sc) \ 77 (sizeof(struct pci_vtscsi_req_cmd_rd) + _sc->vss_config.cdb_size) 78 79 #define VTSCSI_OUT_HEADER_LEN(_sc) \ 80 (sizeof(struct pci_vtscsi_req_cmd_wr) + _sc->vss_config.sense_size) 81 82 #define VIRTIO_SCSI_MAX_CHANNEL 0 83 #define VIRTIO_SCSI_MAX_TARGET 0 84 #define VIRTIO_SCSI_MAX_LUN 16383 85 86 #define VIRTIO_SCSI_F_INOUT (1 << 0) 87 #define VIRTIO_SCSI_F_HOTPLUG (1 << 1) 88 #define VIRTIO_SCSI_F_CHANGE (1 << 2) 89 90 static int pci_vtscsi_debug = 0; 91 #define WPRINTF(msg, params...) PRINTLN("virtio-scsi: " msg, ##params) 92 #define DPRINTF(msg, params...) if (pci_vtscsi_debug) WPRINTF(msg, ##params) 93 94 struct pci_vtscsi_config { 95 uint32_t num_queues; 96 uint32_t seg_max; 97 uint32_t max_sectors; 98 uint32_t cmd_per_lun; 99 uint32_t event_info_size; 100 uint32_t sense_size; 101 uint32_t cdb_size; 102 uint16_t max_channel; 103 uint16_t max_target; 104 uint32_t max_lun; 105 } __attribute__((packed)); 106 107 struct pci_vtscsi_queue { 108 struct pci_vtscsi_softc * vsq_sc; 109 struct vqueue_info * vsq_vq; 110 pthread_mutex_t vsq_mtx; 111 pthread_mutex_t vsq_qmtx; 112 pthread_cond_t vsq_cv; 113 STAILQ_HEAD(, pci_vtscsi_request) vsq_requests; 114 LIST_HEAD(, pci_vtscsi_worker) vsq_workers; 115 }; 116 117 struct pci_vtscsi_worker { 118 struct pci_vtscsi_queue * vsw_queue; 119 pthread_t vsw_thread; 120 bool vsw_exiting; 121 LIST_ENTRY(pci_vtscsi_worker) vsw_link; 122 }; 123 124 struct pci_vtscsi_request { 125 struct pci_vtscsi_queue * vsr_queue; 126 struct iovec vsr_iov_in[VTSCSI_MAXSEG]; 127 int vsr_niov_in; 128 struct iovec vsr_iov_out[VTSCSI_MAXSEG]; 129 int vsr_niov_out; 130 uint32_t vsr_idx; 131 STAILQ_ENTRY(pci_vtscsi_request) vsr_link; 132 }; 133 134 /* 135 * Per-device softc 136 */ 137 struct pci_vtscsi_softc { 138 struct virtio_softc vss_vs; 139 struct vqueue_info vss_vq[VTSCSI_MAXQ]; 140 struct pci_vtscsi_queue vss_queues[VTSCSI_REQUESTQ]; 141 pthread_mutex_t vss_mtx; 142 int vss_iid; 143 int vss_ctl_fd; 144 uint32_t vss_features; 145 struct pci_vtscsi_config vss_config; 146 }; 147 148 #define VIRTIO_SCSI_T_TMF 0 149 #define VIRTIO_SCSI_T_TMF_ABORT_TASK 0 150 #define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET 1 151 #define VIRTIO_SCSI_T_TMF_CLEAR_ACA 2 152 #define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET 3 153 #define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET 4 154 #define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET 5 155 #define VIRTIO_SCSI_T_TMF_QUERY_TASK 6 156 #define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET 7 157 158 /* command-specific response values */ 159 #define VIRTIO_SCSI_S_FUNCTION_COMPLETE 0 160 #define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED 10 161 #define VIRTIO_SCSI_S_FUNCTION_REJECTED 11 162 163 struct pci_vtscsi_ctrl_tmf { 164 uint32_t type; 165 uint32_t subtype; 166 uint8_t lun[8]; 167 uint64_t id; 168 uint8_t response; 169 } __attribute__((packed)); 170 171 #define VIRTIO_SCSI_T_AN_QUERY 1 172 #define VIRTIO_SCSI_EVT_ASYNC_OPERATIONAL_CHANGE 2 173 #define VIRTIO_SCSI_EVT_ASYNC_POWER_MGMT 4 174 #define VIRTIO_SCSI_EVT_ASYNC_EXTERNAL_REQUEST 8 175 #define VIRTIO_SCSI_EVT_ASYNC_MEDIA_CHANGE 16 176 #define VIRTIO_SCSI_EVT_ASYNC_MULTI_HOST 32 177 #define VIRTIO_SCSI_EVT_ASYNC_DEVICE_BUSY 64 178 179 struct pci_vtscsi_ctrl_an { 180 uint32_t type; 181 uint8_t lun[8]; 182 uint32_t event_requested; 183 uint32_t event_actual; 184 uint8_t response; 185 } __attribute__((packed)); 186 187 /* command-specific response values */ 188 #define VIRTIO_SCSI_S_OK 0 189 #define VIRTIO_SCSI_S_OVERRUN 1 190 #define VIRTIO_SCSI_S_ABORTED 2 191 #define VIRTIO_SCSI_S_BAD_TARGET 3 192 #define VIRTIO_SCSI_S_RESET 4 193 #define VIRTIO_SCSI_S_BUSY 5 194 #define VIRTIO_SCSI_S_TRANSPORT_FAILURE 6 195 #define VIRTIO_SCSI_S_TARGET_FAILURE 7 196 #define VIRTIO_SCSI_S_NEXUS_FAILURE 8 197 #define VIRTIO_SCSI_S_FAILURE 9 198 #define VIRTIO_SCSI_S_INCORRECT_LUN 12 199 200 /* task_attr */ 201 #define VIRTIO_SCSI_S_SIMPLE 0 202 #define VIRTIO_SCSI_S_ORDERED 1 203 #define VIRTIO_SCSI_S_HEAD 2 204 #define VIRTIO_SCSI_S_ACA 3 205 206 struct pci_vtscsi_event { 207 uint32_t event; 208 uint8_t lun[8]; 209 uint32_t reason; 210 } __attribute__((packed)); 211 212 struct pci_vtscsi_req_cmd_rd { 213 uint8_t lun[8]; 214 uint64_t id; 215 uint8_t task_attr; 216 uint8_t prio; 217 uint8_t crn; 218 uint8_t cdb[]; 219 } __attribute__((packed)); 220 221 struct pci_vtscsi_req_cmd_wr { 222 uint32_t sense_len; 223 uint32_t residual; 224 uint16_t status_qualifier; 225 uint8_t status; 226 uint8_t response; 227 uint8_t sense[]; 228 } __attribute__((packed)); 229 230 static void *pci_vtscsi_proc(void *); 231 static void pci_vtscsi_reset(void *); 232 static void pci_vtscsi_neg_features(void *, uint64_t); 233 static int pci_vtscsi_cfgread(void *, int, int, uint32_t *); 234 static int pci_vtscsi_cfgwrite(void *, int, int, uint32_t); 235 static inline int pci_vtscsi_get_lun(uint8_t *); 236 static int pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *, size_t); 237 static int pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *, 238 struct pci_vtscsi_ctrl_tmf *); 239 static int pci_vtscsi_an_handle(struct pci_vtscsi_softc *, 240 struct pci_vtscsi_ctrl_an *); 241 static int pci_vtscsi_request_handle(struct pci_vtscsi_queue *, struct iovec *, 242 int, struct iovec *, int); 243 static void pci_vtscsi_controlq_notify(void *, struct vqueue_info *); 244 static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *); 245 static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *); 246 static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, 247 struct pci_vtscsi_queue *, int); 248 static int pci_vtscsi_init(struct vmctx *, struct pci_devinst *, nvlist_t *); 249 250 static struct virtio_consts vtscsi_vi_consts = { 251 "vtscsi", /* our name */ 252 VTSCSI_MAXQ, /* we support 2+n virtqueues */ 253 sizeof(struct pci_vtscsi_config), /* config reg size */ 254 pci_vtscsi_reset, /* reset */ 255 NULL, /* device-wide qnotify */ 256 pci_vtscsi_cfgread, /* read virtio config */ 257 pci_vtscsi_cfgwrite, /* write virtio config */ 258 pci_vtscsi_neg_features, /* apply negotiated features */ 259 0, /* our capabilities */ 260 }; 261 262 static void * 263 pci_vtscsi_proc(void *arg) 264 { 265 struct pci_vtscsi_worker *worker = (struct pci_vtscsi_worker *)arg; 266 struct pci_vtscsi_queue *q = worker->vsw_queue; 267 struct pci_vtscsi_request *req; 268 int iolen; 269 270 for (;;) { 271 pthread_mutex_lock(&q->vsq_mtx); 272 273 while (STAILQ_EMPTY(&q->vsq_requests) 274 && !worker->vsw_exiting) 275 pthread_cond_wait(&q->vsq_cv, &q->vsq_mtx); 276 277 if (worker->vsw_exiting) 278 break; 279 280 req = STAILQ_FIRST(&q->vsq_requests); 281 STAILQ_REMOVE_HEAD(&q->vsq_requests, vsr_link); 282 283 pthread_mutex_unlock(&q->vsq_mtx); 284 iolen = pci_vtscsi_request_handle(q, req->vsr_iov_in, 285 req->vsr_niov_in, req->vsr_iov_out, req->vsr_niov_out); 286 287 pthread_mutex_lock(&q->vsq_qmtx); 288 vq_relchain(q->vsq_vq, req->vsr_idx, iolen); 289 vq_endchains(q->vsq_vq, 0); 290 pthread_mutex_unlock(&q->vsq_qmtx); 291 292 DPRINTF("request <idx=%d> completed", req->vsr_idx); 293 free(req); 294 } 295 296 pthread_mutex_unlock(&q->vsq_mtx); 297 return (NULL); 298 } 299 300 static void 301 pci_vtscsi_reset(void *vsc) 302 { 303 struct pci_vtscsi_softc *sc; 304 305 sc = vsc; 306 307 DPRINTF("device reset requested"); 308 vi_reset_dev(&sc->vss_vs); 309 310 /* initialize config structure */ 311 sc->vss_config = (struct pci_vtscsi_config){ 312 .num_queues = VTSCSI_REQUESTQ, 313 /* Leave room for the request and the response. */ 314 .seg_max = VTSCSI_MAXSEG - 2, 315 .max_sectors = 2, 316 .cmd_per_lun = 1, 317 .event_info_size = sizeof(struct pci_vtscsi_event), 318 .sense_size = 96, 319 .cdb_size = 32, 320 .max_channel = VIRTIO_SCSI_MAX_CHANNEL, 321 .max_target = VIRTIO_SCSI_MAX_TARGET, 322 .max_lun = VIRTIO_SCSI_MAX_LUN 323 }; 324 } 325 326 static void 327 pci_vtscsi_neg_features(void *vsc, uint64_t negotiated_features) 328 { 329 struct pci_vtscsi_softc *sc = vsc; 330 331 sc->vss_features = negotiated_features; 332 } 333 334 static int 335 pci_vtscsi_cfgread(void *vsc, int offset, int size, uint32_t *retval) 336 { 337 struct pci_vtscsi_softc *sc = vsc; 338 void *ptr; 339 340 ptr = (uint8_t *)&sc->vss_config + offset; 341 memcpy(retval, ptr, size); 342 return (0); 343 } 344 345 static int 346 pci_vtscsi_cfgwrite(void *vsc, int offset, int size, uint32_t val) 347 { 348 349 return (0); 350 } 351 352 static inline int 353 pci_vtscsi_get_lun(uint8_t *lun) 354 { 355 356 return (((lun[2] << 8) | lun[3]) & 0x3fff); 357 } 358 359 static int 360 pci_vtscsi_control_handle(struct pci_vtscsi_softc *sc, void *buf, 361 size_t bufsize) 362 { 363 struct pci_vtscsi_ctrl_tmf *tmf; 364 struct pci_vtscsi_ctrl_an *an; 365 uint32_t type; 366 367 if (bufsize < sizeof(uint32_t)) { 368 WPRINTF("ignoring truncated control request"); 369 return (0); 370 } 371 372 type = *(uint32_t *)buf; 373 374 if (type == VIRTIO_SCSI_T_TMF) { 375 if (bufsize != sizeof(*tmf)) { 376 WPRINTF("ignoring tmf request with size %zu", bufsize); 377 return (0); 378 } 379 tmf = (struct pci_vtscsi_ctrl_tmf *)buf; 380 return (pci_vtscsi_tmf_handle(sc, tmf)); 381 } 382 383 if (type == VIRTIO_SCSI_T_AN_QUERY) { 384 if (bufsize != sizeof(*an)) { 385 WPRINTF("ignoring AN request with size %zu", bufsize); 386 return (0); 387 } 388 an = (struct pci_vtscsi_ctrl_an *)buf; 389 return (pci_vtscsi_an_handle(sc, an)); 390 } 391 392 return (0); 393 } 394 395 static int 396 pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *sc, 397 struct pci_vtscsi_ctrl_tmf *tmf) 398 { 399 union ctl_io *io; 400 int err; 401 402 io = ctl_scsi_alloc_io(sc->vss_iid); 403 ctl_scsi_zero_io(io); 404 405 io->io_hdr.io_type = CTL_IO_TASK; 406 io->io_hdr.nexus.initid = sc->vss_iid; 407 io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(tmf->lun); 408 io->taskio.tag_type = CTL_TAG_SIMPLE; 409 io->taskio.tag_num = (uint32_t)tmf->id; 410 411 switch (tmf->subtype) { 412 case VIRTIO_SCSI_T_TMF_ABORT_TASK: 413 io->taskio.task_action = CTL_TASK_ABORT_TASK; 414 break; 415 416 case VIRTIO_SCSI_T_TMF_ABORT_TASK_SET: 417 io->taskio.task_action = CTL_TASK_ABORT_TASK_SET; 418 break; 419 420 case VIRTIO_SCSI_T_TMF_CLEAR_ACA: 421 io->taskio.task_action = CTL_TASK_CLEAR_ACA; 422 break; 423 424 case VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET: 425 io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET; 426 break; 427 428 case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET: 429 io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET; 430 break; 431 432 case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: 433 io->taskio.task_action = CTL_TASK_LUN_RESET; 434 break; 435 436 case VIRTIO_SCSI_T_TMF_QUERY_TASK: 437 io->taskio.task_action = CTL_TASK_QUERY_TASK; 438 break; 439 440 case VIRTIO_SCSI_T_TMF_QUERY_TASK_SET: 441 io->taskio.task_action = CTL_TASK_QUERY_TASK_SET; 442 break; 443 } 444 445 if (pci_vtscsi_debug) { 446 struct sbuf *sb = sbuf_new_auto(); 447 ctl_io_sbuf(io, sb); 448 sbuf_finish(sb); 449 DPRINTF("%s", sbuf_data(sb)); 450 sbuf_delete(sb); 451 } 452 453 err = ioctl(sc->vss_ctl_fd, CTL_IO, io); 454 if (err != 0) 455 WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); 456 457 tmf->response = io->taskio.task_status; 458 ctl_scsi_free_io(io); 459 return (1); 460 } 461 462 static int 463 pci_vtscsi_an_handle(struct pci_vtscsi_softc *sc, 464 struct pci_vtscsi_ctrl_an *an) 465 { 466 467 return (0); 468 } 469 470 static int 471 pci_vtscsi_request_handle(struct pci_vtscsi_queue *q, struct iovec *iov_in, 472 int niov_in, struct iovec *iov_out, int niov_out) 473 { 474 struct pci_vtscsi_softc *sc = q->vsq_sc; 475 struct pci_vtscsi_req_cmd_rd *cmd_rd = NULL; 476 struct pci_vtscsi_req_cmd_wr *cmd_wr; 477 struct iovec data_iov_in[VTSCSI_MAXSEG], data_iov_out[VTSCSI_MAXSEG]; 478 union ctl_io *io; 479 int data_niov_in, data_niov_out; 480 void *ext_data_ptr = NULL; 481 uint32_t ext_data_len = 0, ext_sg_entries = 0; 482 int err, nxferred; 483 484 if (count_iov(iov_out, niov_out) < VTSCSI_OUT_HEADER_LEN(sc)) { 485 WPRINTF("ignoring request with insufficient output"); 486 return (0); 487 } 488 if (count_iov(iov_in, niov_in) < VTSCSI_IN_HEADER_LEN(sc)) { 489 WPRINTF("ignoring request with incomplete header"); 490 return (0); 491 } 492 493 seek_iov(iov_in, niov_in, data_iov_in, &data_niov_in, 494 VTSCSI_IN_HEADER_LEN(sc)); 495 seek_iov(iov_out, niov_out, data_iov_out, &data_niov_out, 496 VTSCSI_OUT_HEADER_LEN(sc)); 497 498 truncate_iov(iov_in, &niov_in, VTSCSI_IN_HEADER_LEN(sc)); 499 truncate_iov(iov_out, &niov_out, VTSCSI_OUT_HEADER_LEN(sc)); 500 iov_to_buf(iov_in, niov_in, (void **)&cmd_rd); 501 502 cmd_wr = calloc(1, VTSCSI_OUT_HEADER_LEN(sc)); 503 io = ctl_scsi_alloc_io(sc->vss_iid); 504 ctl_scsi_zero_io(io); 505 506 io->io_hdr.nexus.initid = sc->vss_iid; 507 io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(cmd_rd->lun); 508 509 io->io_hdr.io_type = CTL_IO_SCSI; 510 511 if (data_niov_in > 0) { 512 ext_data_ptr = (void *)data_iov_in; 513 ext_sg_entries = data_niov_in; 514 ext_data_len = count_iov(data_iov_in, data_niov_in); 515 io->io_hdr.flags |= CTL_FLAG_DATA_OUT; 516 } else if (data_niov_out > 0) { 517 ext_data_ptr = (void *)data_iov_out; 518 ext_sg_entries = data_niov_out; 519 ext_data_len = count_iov(data_iov_out, data_niov_out); 520 io->io_hdr.flags |= CTL_FLAG_DATA_IN; 521 } 522 523 io->scsiio.sense_len = sc->vss_config.sense_size; 524 io->scsiio.tag_num = (uint32_t)cmd_rd->id; 525 switch (cmd_rd->task_attr) { 526 case VIRTIO_SCSI_S_ORDERED: 527 io->scsiio.tag_type = CTL_TAG_ORDERED; 528 break; 529 case VIRTIO_SCSI_S_HEAD: 530 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE; 531 break; 532 case VIRTIO_SCSI_S_ACA: 533 io->scsiio.tag_type = CTL_TAG_ACA; 534 break; 535 case VIRTIO_SCSI_S_SIMPLE: 536 default: 537 io->scsiio.tag_type = CTL_TAG_SIMPLE; 538 break; 539 } 540 io->scsiio.ext_sg_entries = ext_sg_entries; 541 io->scsiio.ext_data_ptr = ext_data_ptr; 542 io->scsiio.ext_data_len = ext_data_len; 543 io->scsiio.ext_data_filled = 0; 544 io->scsiio.cdb_len = sc->vss_config.cdb_size; 545 memcpy(io->scsiio.cdb, cmd_rd->cdb, sc->vss_config.cdb_size); 546 547 if (pci_vtscsi_debug) { 548 struct sbuf *sb = sbuf_new_auto(); 549 ctl_io_sbuf(io, sb); 550 sbuf_finish(sb); 551 DPRINTF("%s", sbuf_data(sb)); 552 sbuf_delete(sb); 553 } 554 555 err = ioctl(sc->vss_ctl_fd, CTL_IO, io); 556 if (err != 0) { 557 WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); 558 cmd_wr->response = VIRTIO_SCSI_S_FAILURE; 559 } else { 560 cmd_wr->sense_len = MIN(io->scsiio.sense_len, 561 sc->vss_config.sense_size); 562 cmd_wr->residual = io->scsiio.residual; 563 cmd_wr->status = io->scsiio.scsi_status; 564 cmd_wr->response = VIRTIO_SCSI_S_OK; 565 memcpy(&cmd_wr->sense, &io->scsiio.sense_data, 566 cmd_wr->sense_len); 567 } 568 569 buf_to_iov(cmd_wr, VTSCSI_OUT_HEADER_LEN(sc), iov_out, niov_out, 0); 570 nxferred = VTSCSI_OUT_HEADER_LEN(sc) + io->scsiio.ext_data_filled; 571 free(cmd_rd); 572 free(cmd_wr); 573 ctl_scsi_free_io(io); 574 return (nxferred); 575 } 576 577 static void 578 pci_vtscsi_controlq_notify(void *vsc, struct vqueue_info *vq) 579 { 580 struct pci_vtscsi_softc *sc; 581 struct iovec iov[VTSCSI_MAXSEG]; 582 struct vi_req req; 583 void *buf = NULL; 584 size_t bufsize; 585 int iolen, n; 586 587 sc = vsc; 588 589 while (vq_has_descs(vq)) { 590 n = vq_getchain(vq, iov, VTSCSI_MAXSEG, &req); 591 assert(n >= 1 && n <= VTSCSI_MAXSEG); 592 593 bufsize = iov_to_buf(iov, n, &buf); 594 iolen = pci_vtscsi_control_handle(sc, buf, bufsize); 595 buf_to_iov(buf + bufsize - iolen, iolen, iov, n, 596 bufsize - iolen); 597 598 /* 599 * Release this chain and handle more 600 */ 601 vq_relchain(vq, req.idx, iolen); 602 } 603 vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ 604 free(buf); 605 } 606 607 static void 608 pci_vtscsi_eventq_notify(void *vsc, struct vqueue_info *vq) 609 { 610 611 vq_kick_disable(vq); 612 } 613 614 static void 615 pci_vtscsi_requestq_notify(void *vsc, struct vqueue_info *vq) 616 { 617 struct pci_vtscsi_softc *sc; 618 struct pci_vtscsi_queue *q; 619 struct pci_vtscsi_request *req; 620 struct iovec iov[VTSCSI_MAXSEG]; 621 struct vi_req vireq; 622 int n; 623 624 sc = vsc; 625 q = &sc->vss_queues[vq->vq_num - 2]; 626 627 while (vq_has_descs(vq)) { 628 n = vq_getchain(vq, iov, VTSCSI_MAXSEG, &vireq); 629 assert(n >= 1 && n <= VTSCSI_MAXSEG); 630 631 req = calloc(1, sizeof(struct pci_vtscsi_request)); 632 req->vsr_idx = vireq.idx; 633 req->vsr_queue = q; 634 req->vsr_niov_in = vireq.readable; 635 req->vsr_niov_out = vireq.writable; 636 memcpy(req->vsr_iov_in, iov, 637 req->vsr_niov_in * sizeof(struct iovec)); 638 memcpy(req->vsr_iov_out, iov + vireq.readable, 639 req->vsr_niov_out * sizeof(struct iovec)); 640 641 pthread_mutex_lock(&q->vsq_mtx); 642 STAILQ_INSERT_TAIL(&q->vsq_requests, req, vsr_link); 643 pthread_cond_signal(&q->vsq_cv); 644 pthread_mutex_unlock(&q->vsq_mtx); 645 646 DPRINTF("request <idx=%d> enqueued", vireq.idx); 647 } 648 } 649 650 static int 651 pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc, 652 struct pci_vtscsi_queue *queue, int num) 653 { 654 struct pci_vtscsi_worker *worker; 655 char tname[MAXCOMLEN + 1]; 656 int i; 657 658 queue->vsq_sc = sc; 659 queue->vsq_vq = &sc->vss_vq[num + 2]; 660 661 pthread_mutex_init(&queue->vsq_mtx, NULL); 662 pthread_mutex_init(&queue->vsq_qmtx, NULL); 663 pthread_cond_init(&queue->vsq_cv, NULL); 664 STAILQ_INIT(&queue->vsq_requests); 665 LIST_INIT(&queue->vsq_workers); 666 667 for (i = 0; i < VTSCSI_THR_PER_Q; i++) { 668 worker = calloc(1, sizeof(struct pci_vtscsi_worker)); 669 worker->vsw_queue = queue; 670 671 pthread_create(&worker->vsw_thread, NULL, &pci_vtscsi_proc, 672 (void *)worker); 673 674 snprintf(tname, sizeof(tname), "vtscsi:%d-%d", num, i); 675 pthread_set_name_np(worker->vsw_thread, tname); 676 LIST_INSERT_HEAD(&queue->vsq_workers, worker, vsw_link); 677 } 678 679 return (0); 680 } 681 682 static int 683 pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts) 684 { 685 char *cp, *devname; 686 687 if (opts == NULL) 688 return (0); 689 690 cp = strchr(opts, ','); 691 if (cp == NULL) { 692 set_config_value_node(nvl, "dev", opts); 693 return (0); 694 } 695 devname = strndup(opts, cp - opts); 696 set_config_value_node(nvl, "dev", devname); 697 free(devname); 698 return (pci_parse_legacy_config(nvl, cp + 1)); 699 } 700 701 static int 702 pci_vtscsi_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl) 703 { 704 struct pci_vtscsi_softc *sc; 705 const char *devname, *value; 706 int i; 707 708 sc = calloc(1, sizeof(struct pci_vtscsi_softc)); 709 value = get_config_value_node(nvl, "iid"); 710 if (value != NULL) 711 sc->vss_iid = strtoul(value, NULL, 10); 712 713 devname = get_config_value_node(nvl, "dev"); 714 if (devname == NULL) 715 devname = "/dev/cam/ctl"; 716 sc->vss_ctl_fd = open(devname, O_RDWR); 717 if (sc->vss_ctl_fd < 0) { 718 WPRINTF("cannot open %s: %s", devname, strerror(errno)); 719 free(sc); 720 return (1); 721 } 722 723 pthread_mutex_init(&sc->vss_mtx, NULL); 724 725 vi_softc_linkup(&sc->vss_vs, &vtscsi_vi_consts, sc, pi, sc->vss_vq); 726 sc->vss_vs.vs_mtx = &sc->vss_mtx; 727 728 /* controlq */ 729 sc->vss_vq[0].vq_qsize = VTSCSI_RINGSZ; 730 sc->vss_vq[0].vq_notify = pci_vtscsi_controlq_notify; 731 732 /* eventq */ 733 sc->vss_vq[1].vq_qsize = VTSCSI_RINGSZ; 734 sc->vss_vq[1].vq_notify = pci_vtscsi_eventq_notify; 735 736 /* request queues */ 737 for (i = 2; i < VTSCSI_MAXQ; i++) { 738 sc->vss_vq[i].vq_qsize = VTSCSI_RINGSZ; 739 sc->vss_vq[i].vq_notify = pci_vtscsi_requestq_notify; 740 pci_vtscsi_init_queue(sc, &sc->vss_queues[i - 2], i - 2); 741 } 742 743 /* initialize config space */ 744 pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_SCSI); 745 pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR); 746 pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); 747 pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_SCSI); 748 pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR); 749 750 if (vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix())) 751 return (1); 752 vi_set_io_bar(&sc->vss_vs, 0); 753 754 return (0); 755 } 756 757 758 static const struct pci_devemu pci_de_vscsi = { 759 .pe_emu = "virtio-scsi", 760 .pe_init = pci_vtscsi_init, 761 .pe_legacy_config = pci_vtscsi_legacy_config, 762 .pe_barwrite = vi_pci_write, 763 .pe_barread = vi_pci_read 764 }; 765 PCI_EMUL_SET(pci_de_vscsi); 766