1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2016 Jakub Klama <jceel@FreeBSD.org>.
5 * Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>.
6 * Copyright (c) 2026 Hans Rosenfeld
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer
14 * in this position and unchanged.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/param.h>
33 #include <sys/linker_set.h>
34 #include <sys/types.h>
35 #include <sys/uio.h>
36 #include <sys/time.h>
37 #include <sys/queue.h>
38
39 #include <errno.h>
40 #include <fcntl.h>
41 #include <stdio.h>
42 #include <stdlib.h>
43 #include <stdbool.h>
44 #include <string.h>
45 #include <unistd.h>
46 #include <assert.h>
47 #include <pthread.h>
48 #include <pthread_np.h>
49
50 #include "bhyverun.h"
51 #include "config.h"
52 #include "debug.h"
53 #include "pci_emul.h"
54 #include "virtio.h"
55 #include "iov.h"
56
57 #include "pci_virtio_scsi.h"
58
59 enum pci_vtscsi_walk {
60 PCI_VTSCSI_WALK_CONTINUE = 0,
61 PCI_VTSCSI_WALK_STOP,
62 };
63
64 typedef enum pci_vtscsi_walk pci_vtscsi_walk_t;
65 typedef pci_vtscsi_walk_t pci_vtscsi_walk_request_queue_cb_t(
66 struct pci_vtscsi_queue *, struct pci_vtscsi_request *, void *);
67
68 static void pci_vtscsi_print_supported_backends(void);
69
70 static void *pci_vtscsi_proc(void *);
71 static void pci_vtscsi_reset(void *);
72 static void pci_vtscsi_neg_features(void *, uint64_t);
73 static int pci_vtscsi_cfgread(void *, int, int, uint32_t *);
74 static int pci_vtscsi_cfgwrite(void *, int, int, uint32_t);
75
76 static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_abort_task;
77 static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_abort_task_set;
78 static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_clear_aca;
79 static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_clear_task_set;
80 static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_i_t_nexus_reset;
81 static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_lun_reset;
82 static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_query_task;
83 static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_query_task_set;
84
85 static pci_vtscsi_walk_t pci_vtscsi_walk_request_queue(
86 struct pci_vtscsi_queue *, pci_vtscsi_walk_request_queue_cb_t *, void *);
87
88 static void pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *,
89 struct pci_vtscsi_ctrl_tmf *);
90 static void pci_vtscsi_an_handle(struct pci_vtscsi_softc *,
91 struct pci_vtscsi_ctrl_an *);
92 static void pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *,
93 size_t);
94
95 static struct pci_vtscsi_request *pci_vtscsi_alloc_request(
96 struct pci_vtscsi_softc *);
97 static void pci_vtscsi_free_request(struct pci_vtscsi_softc *,
98 struct pci_vtscsi_request *);
99 static struct pci_vtscsi_request *pci_vtscsi_get_request(
100 struct pci_vtscsi_req_queue *);
101 static void pci_vtscsi_put_request(struct pci_vtscsi_req_queue *,
102 struct pci_vtscsi_request *);
103 static void pci_vtscsi_queue_request(struct pci_vtscsi_softc *,
104 struct vqueue_info *);
105 static void pci_vtscsi_return_request(struct pci_vtscsi_queue *,
106 struct pci_vtscsi_request *, int);
107 static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *, int,
108 struct pci_vtscsi_request *);
109
110 static void pci_vtscsi_controlq_notify(void *, struct vqueue_info *);
111 static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *);
112 static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *);
113
114 static int pci_vtscsi_add_target_config(nvlist_t *, const char *, int);
115 static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *,
116 struct pci_vtscsi_queue *, int);
117 static void pci_vtscsi_destroy_queue(struct pci_vtscsi_softc *,
118 struct pci_vtscsi_queue *);
119 static int pci_vtscsi_init(struct pci_devinst *, nvlist_t *);
120
121 SET_DECLARE(pci_vtscsi_backend_set, struct pci_vtscsi_backend);
122
123 static struct virtio_consts vtscsi_vi_consts = {
124 .vc_name = "vtscsi",
125 .vc_nvq = VTSCSI_DEF_REQUESTQ + VIRTIO_SCSI_ADDL_Q,
126 .vc_cfgsize = sizeof(struct pci_vtscsi_config),
127 .vc_reset = pci_vtscsi_reset,
128 .vc_cfgread = pci_vtscsi_cfgread,
129 .vc_cfgwrite = pci_vtscsi_cfgwrite,
130 .vc_apply_features = pci_vtscsi_neg_features,
131 .vc_hv_caps = VIRTIO_RING_F_INDIRECT_DESC,
132 };
133
134 static const struct pci_vtscsi_config vtscsi_config = {
135 .num_queues = VTSCSI_DEF_REQUESTQ,
136 /* Leave room for the request and the response. */
137 .seg_max = VTSCSI_DEF_MAXSEG - VIRTIO_SCSI_HDR_SEG,
138 .max_sectors = 0,
139 .cmd_per_lun = 1,
140 .event_info_size = sizeof(struct pci_vtscsi_event),
141 .sense_size = 96,
142 .cdb_size = 32,
143 .max_channel = VIRTIO_SCSI_MAX_CHANNEL,
144 .max_target = VIRTIO_SCSI_MAX_TARGET,
145 .max_lun = VIRTIO_SCSI_MAX_LUN
146 };
147
148 int pci_vtscsi_debug = 0;
149
150
151 static void
pci_vtscsi_print_supported_backends(void)152 pci_vtscsi_print_supported_backends(void)
153 {
154 struct pci_vtscsi_backend **vbpp;
155
156 if (SET_COUNT(pci_vtscsi_backend_set) == 0) {
157 printf("No virtio-scsi backends available");
158 return;
159 }
160
161 SET_FOREACH(vbpp, pci_vtscsi_backend_set) {
162 struct pci_vtscsi_backend *vbp = *vbpp;
163 printf("%s\n", vbp->vsb_name);
164 }
165 }
166
167 static void *
pci_vtscsi_proc(void * arg)168 pci_vtscsi_proc(void *arg)
169 {
170 struct pci_vtscsi_worker *worker = (struct pci_vtscsi_worker *)arg;
171 struct pci_vtscsi_queue *q = worker->vsw_queue;
172 struct pci_vtscsi_softc *sc = q->vsq_sc;
173
174 for (;;) {
175 struct pci_vtscsi_request *req;
176 uint8_t target;
177 int iolen;
178 int fd;
179
180 pthread_mutex_lock(&q->vsq_rmtx);
181
182 while (STAILQ_EMPTY(&q->vsq_requests) && !worker->vsw_exiting)
183 pthread_cond_wait(&q->vsq_cv, &q->vsq_rmtx);
184
185 if (worker->vsw_exiting) {
186 pthread_mutex_unlock(&q->vsq_rmtx);
187 return (NULL);
188 }
189
190 req = pci_vtscsi_get_request(&q->vsq_requests);
191 pthread_mutex_unlock(&q->vsq_rmtx);
192
193 target = pci_vtscsi_get_target(sc, req->vsr_cmd_rd->lun);
194 fd = sc->vss_targets[target].vst_fd;
195
196 DPRINTF("I/O request tgt %u, lun %d, data_niov_in %zu, "
197 "data_niov_out %zu", target,
198 pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun),
199 req->vsr_data_niov_in, req->vsr_data_niov_out);
200
201 iolen = pci_vtscsi_request_handle(sc, fd, req);
202
203 pci_vtscsi_return_request(q, req, iolen);
204 }
205 }
206
207 static void
pci_vtscsi_reset(void * vsc)208 pci_vtscsi_reset(void *vsc)
209 {
210 struct pci_vtscsi_softc *sc;
211
212 sc = vsc;
213
214 DPRINTF("device reset requested");
215 vi_reset_dev(&sc->vss_vs);
216
217 /* initialize config structure */
218 sc->vss_config = sc->vss_default_config;
219
220 sc->vss_config.max_target = MAX(1, sc->vss_num_target) - 1;
221
222 sc->vss_backend->vsb_reset(sc);
223 }
224
225 static void
pci_vtscsi_neg_features(void * vsc,uint64_t negotiated_features)226 pci_vtscsi_neg_features(void *vsc, uint64_t negotiated_features)
227 {
228 struct pci_vtscsi_softc *sc = vsc;
229
230 sc->vss_features = negotiated_features;
231 }
232
233 static int
pci_vtscsi_cfgread(void * vsc,int offset,int size,uint32_t * retval)234 pci_vtscsi_cfgread(void *vsc, int offset, int size, uint32_t *retval)
235 {
236 struct pci_vtscsi_softc *sc = vsc;
237 void *ptr;
238
239 ptr = (uint8_t *)&sc->vss_config + offset;
240 memcpy(retval, ptr, size);
241 return (0);
242 }
243
244 static int
pci_vtscsi_cfgwrite(void * vsc __unused,int offset __unused,int size __unused,uint32_t val __unused)245 pci_vtscsi_cfgwrite(void *vsc __unused, int offset __unused, int size __unused,
246 uint32_t val __unused)
247 {
248 return (0);
249 }
250
251 /*
252 * ABORT TASK: Abort the specifed task queued for this LUN.
253 *
254 * We can stop once we have found the specified task queued for this LUN.
255 */
256 static pci_vtscsi_walk_t
pci_vtscsi_tmf_handle_abort_task(struct pci_vtscsi_queue * q,struct pci_vtscsi_request * req,void * arg)257 pci_vtscsi_tmf_handle_abort_task(struct pci_vtscsi_queue *q,
258 struct pci_vtscsi_request *req, void *arg)
259 {
260 struct pci_vtscsi_ctrl_tmf *tmf = arg;
261
262 assert(tmf->subtype == VIRTIO_SCSI_T_TMF_ABORT_TASK);
263
264 if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) !=
265 pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun))
266 return (PCI_VTSCSI_WALK_CONTINUE);
267
268 if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) !=
269 pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun))
270 return (PCI_VTSCSI_WALK_CONTINUE);
271
272 if (tmf->id != req->vsr_cmd_rd->id)
273 return (PCI_VTSCSI_WALK_CONTINUE);
274
275 req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED;
276 STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link);
277 pci_vtscsi_return_request(q, req, 0);
278
279 return (PCI_VTSCSI_WALK_STOP);
280 }
281
282 /*
283 * ABORT TASK SET: Abort all tasks queued for this LUN.
284 */
285 static pci_vtscsi_walk_t
pci_vtscsi_tmf_handle_abort_task_set(struct pci_vtscsi_queue * q,struct pci_vtscsi_request * req,void * arg)286 pci_vtscsi_tmf_handle_abort_task_set(struct pci_vtscsi_queue *q,
287 struct pci_vtscsi_request *req, void *arg)
288 {
289 struct pci_vtscsi_ctrl_tmf *tmf = arg;
290
291 assert(tmf->subtype == VIRTIO_SCSI_T_TMF_ABORT_TASK_SET);
292
293 if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) !=
294 pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun))
295 return (PCI_VTSCSI_WALK_CONTINUE);
296
297 if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) !=
298 pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun))
299 return (PCI_VTSCSI_WALK_CONTINUE);
300
301 req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED;
302 STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link);
303 pci_vtscsi_return_request(q, req, 0);
304
305 return (PCI_VTSCSI_WALK_CONTINUE);
306 }
307
308 /*
309 * CLEAR ACA: Clear ACA (auto contingent allegiance) state.
310 */
311 static pci_vtscsi_walk_t
pci_vtscsi_tmf_handle_clear_aca(struct pci_vtscsi_queue * q __unused,struct pci_vtscsi_request * req __unused,void * arg)312 pci_vtscsi_tmf_handle_clear_aca(struct pci_vtscsi_queue *q __unused,
313 struct pci_vtscsi_request *req __unused, void *arg)
314 {
315 struct pci_vtscsi_ctrl_tmf *tmf = arg;
316
317 assert(tmf->subtype == VIRTIO_SCSI_T_TMF_CLEAR_ACA);
318
319 /*
320 * We don't implement handling of NACA=1 in the CONTROL byte at all.
321 *
322 * Thus, we probably should start filtering NORMACA in INQUIRY and
323 * reject any command that sets NACA=1.
324 *
325 * In any case, there isn't anything we need to do with our queued
326 * requests, so stop right here.
327 */
328
329 return (PCI_VTSCSI_WALK_STOP);
330 }
331
332 /*
333 * CLEAR TASK SET: Clear all tasks queued for this LUN.
334 *
335 * All tasks in our queue were placed there by us, so there can be no other
336 * I_T nexus involved. Hence, this is handled the same as ABORT TASK SET.
337 */
338 static pci_vtscsi_walk_t
pci_vtscsi_tmf_handle_clear_task_set(struct pci_vtscsi_queue * q,struct pci_vtscsi_request * req,void * arg)339 pci_vtscsi_tmf_handle_clear_task_set(struct pci_vtscsi_queue *q,
340 struct pci_vtscsi_request *req, void *arg)
341 {
342 struct pci_vtscsi_ctrl_tmf *tmf = arg;
343
344 assert(tmf->subtype == VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET);
345
346 if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) !=
347 pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun))
348 return (PCI_VTSCSI_WALK_CONTINUE);
349
350 if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) !=
351 pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun))
352 return (PCI_VTSCSI_WALK_CONTINUE);
353
354 req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED;
355 STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link);
356 pci_vtscsi_return_request(q, req, 0);
357
358 return (PCI_VTSCSI_WALK_CONTINUE);
359 }
360
361 /*
362 * I_T NEXUS RESET: Abort all tasks queued for any LUN of this target.
363 */
364 static pci_vtscsi_walk_t
pci_vtscsi_tmf_handle_i_t_nexus_reset(struct pci_vtscsi_queue * q,struct pci_vtscsi_request * req,void * arg)365 pci_vtscsi_tmf_handle_i_t_nexus_reset(struct pci_vtscsi_queue *q,
366 struct pci_vtscsi_request *req, void *arg)
367 {
368 struct pci_vtscsi_ctrl_tmf *tmf = arg;
369
370 assert(tmf->subtype == VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET);
371
372 if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) !=
373 pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun))
374 return (PCI_VTSCSI_WALK_CONTINUE);
375
376 /*
377 * T10 "06-026r4 SAM-4 TASK ABORTED status clarifications" indicates
378 * that we should actually return ABORTED here, but other documents
379 * such as the VirtIO spec suggest RESET.
380 */
381 req->vsr_cmd_wr->response = VIRTIO_SCSI_S_RESET;
382 STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link);
383 pci_vtscsi_return_request(q, req, 0);
384
385 return (PCI_VTSCSI_WALK_CONTINUE);
386 }
387
388 /*
389 * LOGICAL UNIT RESET: Abort all tasks queued for this LUN.
390 */
391 static pci_vtscsi_walk_t
pci_vtscsi_tmf_handle_lun_reset(struct pci_vtscsi_queue * q,struct pci_vtscsi_request * req,void * arg)392 pci_vtscsi_tmf_handle_lun_reset(struct pci_vtscsi_queue *q,
393 struct pci_vtscsi_request *req, void *arg)
394 {
395 struct pci_vtscsi_ctrl_tmf *tmf = arg;
396
397 assert(tmf->subtype == VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET);
398
399 if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) !=
400 pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun))
401 return (PCI_VTSCSI_WALK_CONTINUE);
402
403 if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) !=
404 pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun))
405 return (PCI_VTSCSI_WALK_CONTINUE);
406
407 /*
408 * T10 "06-026r4 SAM-4 TASK ABORTED status clarifications" indicates
409 * that we should actually return ABORTED here, but other documents
410 * such as the VirtIO spec suggest RESET.
411 */
412 req->vsr_cmd_wr->response = VIRTIO_SCSI_S_RESET;
413 STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link);
414 pci_vtscsi_return_request(q, req, 0);
415
416 return (PCI_VTSCSI_WALK_CONTINUE);
417 }
418
419 /*
420 * QUERY TASK: Is the specified task present in this LUN?
421 *
422 * We can stop once we have found the specified task queued for this LUN.
423 *
424 * Note that this function may cause false negatives under the following
425 * rare circumstances:
426 * (1) the specified task is still in the virtqueue, not yet having been
427 * processed by pci_vtscsi_requestq_notify()
428 * (2) the specified task was actively being processed by a worker thread
429 * but not yet processed by the backend by the time the QUERY TASK
430 * request was handled by the backend
431 *
432 * While a false negative may be confusing for a guest OS looking for the
433 * state of an I/O request it sent, it is not considered a fatal error of
434 * any kind and is easy to recover from. Also, in both of the above cases,
435 * the QUERY TASK TMF request would need to overtake the I/O request in
436 * question, which can only happen if the TMF request is sent immediately
437 * after the I/O request. While it is technically perfectly fine for a
438 * guest to do so, any normal use of QUERY TASK would involve a certain
439 * delay before the TMF request is sent, giving the I/O request time to
440 * be processed.
441 */
442 static pci_vtscsi_walk_t
pci_vtscsi_tmf_handle_query_task(struct pci_vtscsi_queue * q,struct pci_vtscsi_request * req,void * arg)443 pci_vtscsi_tmf_handle_query_task(struct pci_vtscsi_queue *q,
444 struct pci_vtscsi_request *req, void *arg)
445 {
446 struct pci_vtscsi_ctrl_tmf *tmf = arg;
447
448 assert(tmf->subtype == VIRTIO_SCSI_T_TMF_QUERY_TASK);
449
450 if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) !=
451 pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun))
452 return (PCI_VTSCSI_WALK_CONTINUE);
453
454 if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) !=
455 pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun))
456 return (PCI_VTSCSI_WALK_CONTINUE);
457
458 if (tmf->id != req->vsr_cmd_rd->id)
459 return (PCI_VTSCSI_WALK_CONTINUE);
460
461 tmf->response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;
462 return (PCI_VTSCSI_WALK_STOP);
463 }
464
465 /*
466 * QUERY TASK SET: Are there any tasks present in this LUN?
467 *
468 * We can stop as soon as we've found at least one task queued for this LUN.
469 */
470 static pci_vtscsi_walk_t
pci_vtscsi_tmf_handle_query_task_set(struct pci_vtscsi_queue * q,struct pci_vtscsi_request * req,void * arg)471 pci_vtscsi_tmf_handle_query_task_set(struct pci_vtscsi_queue *q,
472 struct pci_vtscsi_request *req, void *arg)
473 {
474 struct pci_vtscsi_ctrl_tmf *tmf = arg;
475
476 assert(tmf->subtype == VIRTIO_SCSI_T_TMF_QUERY_TASK_SET);
477
478 if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) !=
479 pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun))
480 return (PCI_VTSCSI_WALK_CONTINUE);
481
482 if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) !=
483 pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun))
484 return (PCI_VTSCSI_WALK_CONTINUE);
485
486 tmf->response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;
487 return (PCI_VTSCSI_WALK_STOP);
488 }
489
490 static pci_vtscsi_walk_t
pci_vtscsi_walk_request_queue(struct pci_vtscsi_queue * q,pci_vtscsi_walk_request_queue_cb_t cb,void * arg)491 pci_vtscsi_walk_request_queue(struct pci_vtscsi_queue *q,
492 pci_vtscsi_walk_request_queue_cb_t cb, void *arg)
493 {
494 struct pci_vtscsi_request *req, *tmp;
495
496 STAILQ_FOREACH_SAFE(req, &q->vsq_requests, vsr_link, tmp) {
497 if (cb(q, req, arg) == PCI_VTSCSI_WALK_STOP)
498 return (PCI_VTSCSI_WALK_STOP);
499 }
500
501 return (PCI_VTSCSI_WALK_CONTINUE);
502 }
503
504 static pci_vtscsi_walk_request_queue_cb_t *const pci_vtscsi_tmf_handler_cb[] = {
505 pci_vtscsi_tmf_handle_abort_task,
506 pci_vtscsi_tmf_handle_abort_task_set,
507 pci_vtscsi_tmf_handle_clear_aca,
508 pci_vtscsi_tmf_handle_clear_task_set,
509 pci_vtscsi_tmf_handle_i_t_nexus_reset,
510 pci_vtscsi_tmf_handle_lun_reset,
511 pci_vtscsi_tmf_handle_query_task,
512 pci_vtscsi_tmf_handle_query_task_set
513 };
514
515 static void
pci_vtscsi_tmf_handle(struct pci_vtscsi_softc * sc,struct pci_vtscsi_ctrl_tmf * tmf)516 pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *sc,
517 struct pci_vtscsi_ctrl_tmf *tmf)
518 {
519 uint8_t target;
520 int fd;
521
522 if (tmf->subtype > VIRTIO_SCSI_T_TMF_MAX_FUNC) {
523 WPRINTF("pci_vtscsi_tmf_handle: invalid subtype %u",
524 tmf->subtype);
525 tmf->response = VIRTIO_SCSI_S_FUNCTION_REJECTED;
526 return;
527 }
528
529 if (pci_vtscsi_check_lun(sc, tmf->lun) == false) {
530 DPRINTF("TMF request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-"
531 "%.2hhx%.2hhx-%.2hhx%.2hhx", tmf->lun[0], tmf->lun[1],
532 tmf->lun[2], tmf->lun[3], tmf->lun[4], tmf->lun[5],
533 tmf->lun[6], tmf->lun[7]);
534
535 tmf->response = VIRTIO_SCSI_S_BAD_TARGET;
536 return;
537 }
538
539 target = pci_vtscsi_get_target(sc, tmf->lun);
540
541 fd = sc->vss_targets[target].vst_fd;
542
543 DPRINTF("TMF request tgt %d, lun %d, subtype %d, id %lu",
544 target, pci_vtscsi_get_lun(sc, tmf->lun), tmf->subtype, tmf->id);
545
546 /*
547 * Lock out all the worker threads from processing any waiting requests
548 * while we're processing the TMF request. This also effectively blocks
549 * pci_vtscsi_requestq_notify() from adding any new requests to the
550 * request queue. This in turn means we will miss any I/O requests which
551 * may still be in the virtqueue.
552 *
553 * This does not prevent any requests currently being processed by the
554 * backend from being completed and returned, which we must guarantee to
555 * adhere to the ordering requirements for any TMF function which aborts
556 * tasks.
557 */
558 for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) {
559 struct pci_vtscsi_queue *q = &sc->vss_queues[i];
560
561 pthread_mutex_lock(&q->vsq_rmtx);
562 }
563
564 /*
565 * The backend may set response to FAILURE for the TMF request.
566 *
567 * The default response of all TMF functions is FUNCTION COMPLETE if
568 * there was no error, regardless of whether it actually succeeded or
569 * not. The two notable exceptions are QUERY TASK and QUERY TASK SET,
570 * which will explicitly return FUNCTION SUCCEEDED if the specified
571 * task or any task was active in the target/LUN, respectively.
572 *
573 * Thus, we will call the backend first. Only if the response we get is
574 * FUNCTION COMPLETE we'll continue processing the TMF function on our
575 * queues. Note that there's a slim chance that we're racing against a
576 * worker thread that is actively processing an I/O request, which may
577 * lead to our TMF request being processed by the backend before the
578 * same I/O request, in which case it won't be on any queue either.
579 */
580 sc->vss_backend->vsb_tmf_hdl(sc, fd, tmf);
581
582 if (tmf->response != VIRTIO_SCSI_S_FUNCTION_COMPLETE) {
583 /*
584 * If this is either a FAILURE or FUNCTION REJECTED, we must
585 * not continue to process the TMF function on our queued
586 * requests.
587 *
588 * If it is FUNCTION SUCCEEDED, we do not need to process the
589 * TMF function on our queued requests.
590 *
591 * If it is anything else, log a warning, but handle it the
592 * same as above.
593 */
594 if (tmf->response != VIRTIO_SCSI_S_FAILURE &&
595 tmf->response != VIRTIO_SCSI_S_FUNCTION_REJECTED &&
596 tmf->response != VIRTIO_SCSI_S_FUNCTION_SUCCEEDED) {
597 WPRINTF("pci_vtscsi_tmf_hdl: unexpected response from "
598 "backend: %d", tmf->response);
599 }
600 } else {
601 pci_vtscsi_walk_t ret = PCI_VTSCSI_WALK_CONTINUE;
602 uint32_t i;
603
604 for (i = 0; i < sc->vss_config.num_queues; i++) {
605 struct pci_vtscsi_queue *q = &sc->vss_queues[i];
606
607 ret = pci_vtscsi_walk_request_queue(q,
608 pci_vtscsi_tmf_handler_cb[tmf->subtype], tmf);
609
610 if (ret == PCI_VTSCSI_WALK_STOP)
611 break;
612 }
613 }
614
615 /* Unlock the request queues before we return. */
616 for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) {
617 struct pci_vtscsi_queue *q = &sc->vss_queues[i];
618
619 pthread_mutex_unlock(&q->vsq_rmtx);
620 }
621 }
622
623 static void
pci_vtscsi_an_handle(struct pci_vtscsi_softc * sc,struct pci_vtscsi_ctrl_an * an)624 pci_vtscsi_an_handle(struct pci_vtscsi_softc *sc, struct pci_vtscsi_ctrl_an *an)
625 {
626 int target;
627 int fd;
628
629 if (pci_vtscsi_check_lun(sc, an->lun) == false) {
630 DPRINTF("AN request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-"
631 "%.2hhx%.2hhx-%.2hhx%.2hhx", an->lun[0], an->lun[1],
632 an->lun[2], an->lun[3], an->lun[4], an->lun[5], an->lun[6],
633 an->lun[7]);
634 an->response = VIRTIO_SCSI_S_BAD_TARGET;
635 return;
636 }
637
638 target = pci_vtscsi_get_target(sc, an->lun);
639
640 fd = sc->vss_targets[target].vst_fd;
641
642 DPRINTF("AN request tgt %d, lun %d, event requested %x",
643 target, pci_vtscsi_get_lun(sc, an->lun), an->event_requested);
644
645 sc->vss_backend->vsb_an_hdl(sc, fd, an);
646 }
647
648 static void
pci_vtscsi_control_handle(struct pci_vtscsi_softc * sc,void * buf,size_t bufsize)649 pci_vtscsi_control_handle(struct pci_vtscsi_softc *sc, void *buf,
650 size_t bufsize)
651 {
652 uint32_t type;
653
654 if (bufsize < sizeof(uint32_t)) {
655 WPRINTF("ignoring truncated control request");
656 return;
657 }
658
659 type = *(uint32_t *)buf;
660
661 if (type == VIRTIO_SCSI_T_TMF) {
662 if (bufsize != sizeof(struct pci_vtscsi_ctrl_tmf)) {
663 WPRINTF("ignoring TMF request with size %zu", bufsize);
664 return;
665 }
666
667 pci_vtscsi_tmf_handle(sc, buf);
668 } else if (type == VIRTIO_SCSI_T_AN_QUERY) {
669 if (bufsize != sizeof(struct pci_vtscsi_ctrl_an)) {
670 WPRINTF("ignoring AN request with size %zu", bufsize);
671 return;
672 }
673
674 pci_vtscsi_an_handle(sc, buf);
675 } else {
676 WPRINTF("ignoring unknown control request type = %u", type);
677 }
678 }
679
680 static struct pci_vtscsi_request *
pci_vtscsi_alloc_request(struct pci_vtscsi_softc * sc)681 pci_vtscsi_alloc_request(struct pci_vtscsi_softc *sc)
682 {
683 struct pci_vtscsi_request *req;
684
685 req = calloc(1, sizeof(struct pci_vtscsi_request));
686 if (req == NULL)
687 goto fail;
688
689 req->vsr_iov = calloc(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG +
690 SPLIT_IOV_ADDL_IOV, sizeof(struct iovec));
691 if (req->vsr_iov == NULL)
692 goto fail;
693
694 req->vsr_cmd_rd = calloc(1, VTSCSI_IN_HEADER_LEN(sc));
695 if (req->vsr_cmd_rd == NULL)
696 goto fail;
697 req->vsr_cmd_wr = calloc(1, VTSCSI_OUT_HEADER_LEN(sc));
698 if (req->vsr_cmd_wr == NULL)
699 goto fail;
700
701 req->vsr_backend = sc->vss_backend->vsb_req_alloc(sc);
702 if (req->vsr_backend == NULL)
703 goto fail;
704
705 return (req);
706
707 fail:
708 EPRINTLN("failed to allocate request: %s", strerror(errno));
709
710 if (req != NULL)
711 pci_vtscsi_free_request(sc, req);
712
713 return (NULL);
714 }
715
716 static void
pci_vtscsi_free_request(struct pci_vtscsi_softc * sc,struct pci_vtscsi_request * req)717 pci_vtscsi_free_request(struct pci_vtscsi_softc *sc,
718 struct pci_vtscsi_request *req)
719 {
720 if (req->vsr_backend != NULL)
721 sc->vss_backend->vsb_req_free(req->vsr_backend);
722 if (req->vsr_cmd_rd != NULL)
723 free(req->vsr_cmd_rd);
724 if (req->vsr_cmd_wr != NULL)
725 free(req->vsr_cmd_wr);
726 if (req->vsr_iov != NULL)
727 free(req->vsr_iov);
728
729 free(req);
730 }
731
732 static struct pci_vtscsi_request *
pci_vtscsi_get_request(struct pci_vtscsi_req_queue * req_queue)733 pci_vtscsi_get_request(struct pci_vtscsi_req_queue *req_queue)
734 {
735 struct pci_vtscsi_request *req;
736
737 assert(!STAILQ_EMPTY(req_queue));
738
739 req = STAILQ_FIRST(req_queue);
740 STAILQ_REMOVE_HEAD(req_queue, vsr_link);
741
742 return (req);
743 }
744
745 static void
pci_vtscsi_put_request(struct pci_vtscsi_req_queue * req_queue,struct pci_vtscsi_request * req)746 pci_vtscsi_put_request(struct pci_vtscsi_req_queue *req_queue,
747 struct pci_vtscsi_request *req)
748 {
749 STAILQ_INSERT_TAIL(req_queue, req, vsr_link);
750 }
751
752 static void
pci_vtscsi_queue_request(struct pci_vtscsi_softc * sc,struct vqueue_info * vq)753 pci_vtscsi_queue_request(struct pci_vtscsi_softc *sc, struct vqueue_info *vq)
754 {
755 struct pci_vtscsi_queue *q;
756 struct pci_vtscsi_request *req;
757 struct vi_req vireq;
758 size_t res __maybe_unused;
759 int n, numseg;
760
761 q = &sc->vss_queues[vq->vq_num - VIRTIO_SCSI_ADDL_Q];
762
763 pthread_mutex_lock(&q->vsq_fmtx);
764 req = pci_vtscsi_get_request(&q->vsq_free_requests);
765 assert(req != NULL);
766 pthread_mutex_unlock(&q->vsq_fmtx);
767
768 numseg = (int)(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG);
769
770 n = vq_getchain(vq, req->vsr_iov, numseg, &vireq);
771 assert(n >= 1 && n <= numseg);
772
773 req->vsr_idx = vireq.idx;
774 req->vsr_queue = q;
775 req->vsr_iov_in = &req->vsr_iov[0];
776 req->vsr_niov_in = vireq.readable;
777 req->vsr_iov_out = &req->vsr_iov[vireq.readable];
778 req->vsr_niov_out = vireq.writable;
779
780 /*
781 * Make sure we got at least enough space for the VirtIO-SCSI
782 * command headers. If not, return this request immediately.
783 */
784 if (check_iov_len(req->vsr_iov_out, req->vsr_niov_out,
785 VTSCSI_OUT_HEADER_LEN(q->vsq_sc)) == false) {
786 WPRINTF("ignoring request with insufficient output");
787 req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE;
788 pci_vtscsi_return_request(q, req, 1);
789 return;
790 }
791
792 if (check_iov_len(req->vsr_iov_in, req->vsr_niov_in,
793 VTSCSI_IN_HEADER_LEN(q->vsq_sc)) == false) {
794 WPRINTF("ignoring request with incomplete header");
795 req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE;
796 pci_vtscsi_return_request(q, req, 1);
797 return;
798 }
799
800 /*
801 * We have to split the iovec array into a header and data portion each
802 * for input and output.
803 *
804 * We need to start with the output section (at the end of iov) in case
805 * the iovec covering the final part of the output header needs to be
806 * split, in which case split_iov() will move all reamaining iovecs up
807 * by one to make room for a new iovec covering the first part of the
808 * output data portion.
809 */
810 req->vsr_data_iov_out = split_iov(req->vsr_iov_out, &req->vsr_niov_out,
811 VTSCSI_OUT_HEADER_LEN(q->vsq_sc), &req->vsr_data_niov_out);
812
813 /*
814 * Similarly, to not overwrite the first iovec of the output section,
815 * the 2nd call to split_iov() to split the input section must actually
816 * cover the entire iovec array (both input and the already split output
817 * sections).
818 */
819 req->vsr_niov_in += req->vsr_niov_out + req->vsr_data_niov_out;
820
821 req->vsr_data_iov_in = split_iov(req->vsr_iov_in, &req->vsr_niov_in,
822 VTSCSI_IN_HEADER_LEN(q->vsq_sc), &req->vsr_data_niov_in);
823
824 /*
825 * And of course we now have to adjust data_niov_in accordingly.
826 */
827 req->vsr_data_niov_in -= req->vsr_niov_out + req->vsr_data_niov_out;
828
829 /*
830 * iov_to_buf() realloc()s the buffer given as 3rd argument to the
831 * total size of all iovecs it will be copying. Since we've just
832 * truncated it in split_iov(), we know that the size will be
833 * VTSCSI_IN_HEADER_LEN(q->vsq_sc).
834 *
835 * Since we pre-allocated req->vsr_cmd_rd to this size, the realloc()
836 * should never fail.
837 *
838 * This will have to change if we begin allowing config space writes
839 * to change sense size.
840 */
841 res = iov_to_buf(req->vsr_iov_in, req->vsr_niov_in,
842 (void **)&req->vsr_cmd_rd);
843 assert(res == VTSCSI_IN_HEADER_LEN(q->vsq_sc));
844
845 /* Make sure this request addresses a valid LUN. */
846 if (pci_vtscsi_check_lun(sc, req->vsr_cmd_rd->lun) == false) {
847 DPRINTF("I/O request to invalid LUN "
848 "%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx",
849 req->vsr_cmd_rd->lun[0], req->vsr_cmd_rd->lun[1],
850 req->vsr_cmd_rd->lun[2], req->vsr_cmd_rd->lun[3],
851 req->vsr_cmd_rd->lun[4], req->vsr_cmd_rd->lun[5],
852 req->vsr_cmd_rd->lun[6], req->vsr_cmd_rd->lun[7]);
853 req->vsr_cmd_wr->response = VIRTIO_SCSI_S_BAD_TARGET;
854 pci_vtscsi_return_request(q, req, 1);
855 return;
856 }
857
858 pthread_mutex_lock(&q->vsq_rmtx);
859 pci_vtscsi_put_request(&q->vsq_requests, req);
860 pthread_cond_signal(&q->vsq_cv);
861 pthread_mutex_unlock(&q->vsq_rmtx);
862
863 DPRINTF("request <idx=%d> enqueued", vireq.idx);
864 }
865
866 static void
pci_vtscsi_return_request(struct pci_vtscsi_queue * q,struct pci_vtscsi_request * req,int iolen)867 pci_vtscsi_return_request(struct pci_vtscsi_queue *q,
868 struct pci_vtscsi_request *req, int iolen)
869 {
870 struct pci_vtscsi_softc *sc = q->vsq_sc;
871 void *iov = req->vsr_iov;
872 void *cmd_rd = req->vsr_cmd_rd;
873 void *cmd_wr = req->vsr_cmd_wr;
874 void *backend = req->vsr_backend;
875 int idx = req->vsr_idx;
876
877 DPRINTF("request <idx=%d> completed, response %d", idx,
878 req->vsr_cmd_wr->response);
879
880 iolen += buf_to_iov(cmd_wr, VTSCSI_OUT_HEADER_LEN(q->vsq_sc),
881 req->vsr_iov_out, req->vsr_niov_out);
882
883 sc->vss_backend->vsb_req_clear(backend);
884
885 memset(iov, 0, sizeof(struct iovec) * (sc->vss_config.seg_max +
886 VIRTIO_SCSI_HDR_SEG + SPLIT_IOV_ADDL_IOV));
887 memset(cmd_rd, 0, VTSCSI_IN_HEADER_LEN(q->vsq_sc));
888 memset(cmd_wr, 0, VTSCSI_OUT_HEADER_LEN(q->vsq_sc));
889 memset(req, 0, sizeof(struct pci_vtscsi_request));
890
891 req->vsr_iov = iov;
892 req->vsr_cmd_rd = cmd_rd;
893 req->vsr_cmd_wr = cmd_wr;
894 req->vsr_backend = backend;
895
896 pthread_mutex_lock(&q->vsq_fmtx);
897 pci_vtscsi_put_request(&q->vsq_free_requests, req);
898 pthread_mutex_unlock(&q->vsq_fmtx);
899
900 pthread_mutex_lock(&q->vsq_qmtx);
901 vq_relchain(q->vsq_vq, idx, iolen);
902 vq_endchains(q->vsq_vq, 0);
903 pthread_mutex_unlock(&q->vsq_qmtx);
904 }
905
906 static int
pci_vtscsi_request_handle(struct pci_vtscsi_softc * sc,int fd,struct pci_vtscsi_request * req)907 pci_vtscsi_request_handle(struct pci_vtscsi_softc *sc, int fd,
908 struct pci_vtscsi_request *req)
909 {
910 return (sc->vss_backend->vsb_req_hdl(sc, fd, req));
911 }
912
913 static void
pci_vtscsi_controlq_notify(void * vsc,struct vqueue_info * vq)914 pci_vtscsi_controlq_notify(void *vsc, struct vqueue_info *vq)
915 {
916 struct pci_vtscsi_softc *sc = vsc;
917 int numseg = (int)(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG);
918 struct iovec *iov = calloc(numseg, sizeof (struct iovec));
919 struct vi_req req;
920 void *buf = NULL;
921 size_t bufsize;
922 int n;
923
924 while (vq_has_descs(vq)) {
925 n = vq_getchain(vq, iov, numseg, &req);
926 assert(n >= 1 && n <= numseg);
927
928 bufsize = iov_to_buf(iov, n, &buf);
929 pci_vtscsi_control_handle(sc, buf, bufsize);
930 buf_to_iov((uint8_t *)buf, bufsize, iov, n);
931
932 /*
933 * Release this chain and handle more
934 */
935 vq_relchain(vq, req.idx, bufsize);
936 }
937 vq_endchains(vq, 1); /* Generate interrupt if appropriate. */
938 free(buf);
939 free(iov);
940 }
941
942 static void
pci_vtscsi_eventq_notify(void * vsc __unused,struct vqueue_info * vq)943 pci_vtscsi_eventq_notify(void *vsc __unused, struct vqueue_info *vq)
944 {
945 vq_kick_disable(vq);
946 }
947
948 static void
pci_vtscsi_requestq_notify(void * vsc,struct vqueue_info * vq)949 pci_vtscsi_requestq_notify(void *vsc, struct vqueue_info *vq)
950 {
951 while (vq_has_descs(vq)) {
952 pci_vtscsi_queue_request(vsc, vq);
953 }
954 }
955
956 static int
pci_vtscsi_init_queue(struct pci_vtscsi_softc * sc,struct pci_vtscsi_queue * queue,int num)957 pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc,
958 struct pci_vtscsi_queue *queue, int num)
959 {
960 struct pci_vtscsi_worker *workers;
961 char tname[MAXCOMLEN + 1];
962 uint32_t i;
963
964 queue->vsq_sc = sc;
965 queue->vsq_vq = &sc->vss_vq[num];
966
967 pthread_mutex_init(&queue->vsq_rmtx, NULL);
968 pthread_mutex_init(&queue->vsq_fmtx, NULL);
969 pthread_mutex_init(&queue->vsq_qmtx, NULL);
970 pthread_cond_init(&queue->vsq_cv, NULL);
971 STAILQ_INIT(&queue->vsq_requests);
972 STAILQ_INIT(&queue->vsq_free_requests);
973 LIST_INIT(&queue->vsq_workers);
974
975 for (i = 0; i < sc->vss_req_ringsz; i++) {
976 struct pci_vtscsi_request *req;
977
978 req = pci_vtscsi_alloc_request(sc);
979 if (req == NULL)
980 goto fail;
981
982 pci_vtscsi_put_request(&queue->vsq_free_requests, req);
983 }
984
985 workers = calloc(sc->vss_thr_per_q, sizeof(struct pci_vtscsi_worker));
986 if (workers == NULL)
987 goto fail;
988
989 for (i = 0; i < sc->vss_thr_per_q; i++) {
990 workers[i].vsw_queue = queue;
991
992 pthread_create(&workers[i].vsw_thread, NULL, &pci_vtscsi_proc,
993 (void *)&workers[i]);
994
995 snprintf(tname, sizeof(tname), "vtscsi:%d-%d", num, i);
996 pthread_set_name_np(workers[i].vsw_thread, tname);
997 LIST_INSERT_HEAD(&queue->vsq_workers, &workers[i], vsw_link);
998 }
999
1000 return (0);
1001
1002 fail:
1003 pci_vtscsi_destroy_queue(sc, queue);
1004
1005 return (-1);
1006 }
1007
1008 static void
pci_vtscsi_destroy_queue(struct pci_vtscsi_softc * sc,struct pci_vtscsi_queue * queue)1009 pci_vtscsi_destroy_queue(struct pci_vtscsi_softc *sc,
1010 struct pci_vtscsi_queue *queue)
1011 {
1012 if (queue->vsq_sc == NULL)
1013 return;
1014
1015 for (int i = sc->vss_req_ringsz; i > 0; i--) {
1016 struct pci_vtscsi_request *req;
1017
1018 if (STAILQ_EMPTY(&queue->vsq_free_requests))
1019 break;
1020
1021 req = pci_vtscsi_get_request(&queue->vsq_free_requests);
1022 pci_vtscsi_free_request(queue->vsq_sc, req);
1023 }
1024
1025 pthread_cond_destroy(&queue->vsq_cv);
1026 pthread_mutex_destroy(&queue->vsq_qmtx);
1027 pthread_mutex_destroy(&queue->vsq_fmtx);
1028 pthread_mutex_destroy(&queue->vsq_rmtx);
1029 }
1030
1031 /*
1032 * Create a target config node, return target id. If the target number isn't
1033 * given as part of the path argument, use last_id + 1.
1034 */
1035 static int
pci_vtscsi_add_target_config(nvlist_t * nvl,const char * path,int last_id)1036 pci_vtscsi_add_target_config(nvlist_t *nvl, const char *path, int last_id)
1037 {
1038 uint64_t target;
1039 char *id;
1040 char tmp[4];
1041
1042 if (path == NULL) {
1043 EPRINTLN("target path must be specified");
1044 return (-1);
1045 }
1046
1047 if (path[0] != '/' && (id = strchr(path, ':')) != NULL) {
1048 const char *errstr;
1049 int len = id - path;
1050
1051 id = strndup(path, len);
1052 if (id == NULL) {
1053 EPRINTLN("failed to get id string: %s",
1054 strerror(errno));
1055 return (-1);
1056 }
1057
1058 target = strtonumx(id, 0, VIRTIO_SCSI_MAX_TARGET, &errstr, 0);
1059 if (errstr != NULL) {
1060 EPRINTLN("invalid target %s: target ID is %s", id,
1061 errstr);
1062 free(id);
1063 return (-1);
1064 }
1065
1066 free(id);
1067 path += len + 1;
1068 } else {
1069 target = last_id + 1;
1070
1071 if (target > VIRTIO_SCSI_MAX_TARGET) {
1072 EPRINTLN("max target (%d) reached, can't add another",
1073 VIRTIO_SCSI_MAX_TARGET);
1074 return (-1);
1075 }
1076 }
1077
1078 snprintf(tmp, sizeof(tmp), "%lu", target);
1079
1080 if (get_config_value_node(nvl, tmp) != NULL) {
1081 EPRINTLN("cannot add '%s' as target %s: already exits as '%s'",
1082 path, tmp, get_config_value_node(nvl, tmp));
1083 return (-1);
1084 }
1085
1086 set_config_value_node(nvl, tmp, path);
1087
1088 return (target);
1089 }
1090
1091 /*
1092 * The following forms are accepted for legacy config options to configure a
1093 * single target:
1094 *
1095 * (0) -s B:D:F,virtio-scsi
1096 * (1) -s B:D:F,virtio-scsi,<dev>
1097 * (2) -s B:D:F,virtio-scsi,<dev>,<name=value>,...
1098 * (3) -s B:D:F,virtio-scsi,<name=value>,...
1099 * (4) -s B:D:F,virtio-scsi,<name=value>
1100 *
1101 * To configure multiple targets, the following form is accepted:
1102 * (5) -s B:D:F,virtio-scsi,[target=[id:]<dev>,...]
1103 */
1104 static int
pci_vtscsi_legacy_config(nvlist_t * nvl,const char * opts)1105 pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts)
1106 {
1107 int last_id = -1;
1108 char *config, *tofree, *name, *value;
1109 nvlist_t *targets;
1110 size_t n;
1111
1112 /* Make sure no one accidentally sets "dev" anymore. */
1113 (void) create_relative_config_node(nvl, "dev");
1114
1115 targets = create_relative_config_node(nvl, "target");
1116
1117 /* Legacy form (0) is handled in pci_vtscsi_init(). */
1118 if (opts == NULL)
1119 return (0);
1120
1121 if (strcmp("help", opts) == 0) {
1122 pci_vtscsi_print_supported_backends();
1123 exit(0);
1124 }
1125
1126 n = strcspn(opts, ",=");
1127
1128 /* Handle legacy form (1) and (2). */
1129 if (opts[n] == ',' || opts[n] == '\0') {
1130 char *tmp = strndup(opts, n);
1131
1132 last_id = pci_vtscsi_add_target_config(targets, tmp, last_id);
1133 free(tmp);
1134
1135 if (last_id < 0)
1136 return (-1);
1137
1138 opts += n;
1139 if (opts[0] == ',' && opts[1] != '\0')
1140 opts++;
1141 }
1142
1143 /* If this was form (1), we're done. */
1144 if (opts[0] == '\0')
1145 return (0);
1146
1147 /*
1148 * For form (2), (3), (4), and (5), parse the remaining options.
1149 *
1150 * Contrary to other options, multiple target=<dev> options create a new
1151 * target for each such option.
1152 *
1153 * For compatibility reasons we also accept dev=<dev> options for
1154 * targets.
1155 */
1156 config = tofree = strdup(opts);
1157 while ((name = strsep(&config, ",")) != NULL) {
1158 value = strchr(name, '=');
1159 if (value != NULL)
1160 *value++ = '\0';
1161
1162 if (strcmp(name, "dev") == 0 || strcmp(name, "target") == 0) {
1163 int new_id = pci_vtscsi_add_target_config(targets,
1164 value, last_id);
1165
1166 if (new_id < 0) {
1167 free(tofree);
1168 return (-1);
1169 }
1170
1171 if (new_id > last_id)
1172 last_id = new_id;
1173
1174 } else if (value != NULL) {
1175 set_config_value_node(nvl, name, value);
1176 } else {
1177 set_config_bool_node(nvl, name, true);
1178 }
1179 }
1180
1181 free(tofree);
1182 return (0);
1183 }
1184
1185 static int
pci_vtscsi_count_targets(const char * prefix __unused,const nvlist_t * parent __unused,const char * name,int type,void * arg)1186 pci_vtscsi_count_targets(const char *prefix __unused,
1187 const nvlist_t *parent __unused, const char *name, int type, void *arg)
1188 {
1189 struct pci_vtscsi_softc *sc = arg;
1190 const char *errstr;
1191 uint64_t target;
1192
1193 if (type != NV_TYPE_STRING) {
1194 EPRINTLN("invalid target \"%s\" type: not a string", name);
1195 errno = EINVAL;
1196 return (-1);
1197 }
1198
1199 target = strtonumx(name, 0, VIRTIO_SCSI_MAX_TARGET, &errstr, 0);
1200 if (errstr != NULL) {
1201 EPRINTLN("invalid target %s: target ID is %s", name, errstr);
1202 return (-1);
1203 }
1204
1205 if (target >= sc->vss_num_target)
1206 sc->vss_num_target = target + 1;
1207
1208 return (0);
1209 }
1210
1211 static int
pci_vtscsi_init_target(const char * prefix __unused,const nvlist_t * parent,const char * name,int type,void * arg)1212 pci_vtscsi_init_target(const char *prefix __unused, const nvlist_t *parent,
1213 const char *name, int type, void *arg)
1214 {
1215 struct pci_vtscsi_softc *sc = arg;
1216 const char *value;
1217 const char *errstr;
1218 uint64_t target;
1219 int ret;
1220
1221 assert(type == NV_TYPE_STRING);
1222
1223 /*
1224 * Get the numeric value of the target id from 'name'.
1225 */
1226 target = strtonumx(name, 0, sc->vss_num_target - 1, &errstr, 0);
1227 assert(errstr == NULL);
1228 sc->vss_targets[target].vst_target = target;
1229
1230 /*
1231 * 'value' contains the backend path. Call the backend to open it.
1232 */
1233 value = nvlist_get_string(parent, name);
1234 ret = sc->vss_backend->vsb_open(sc, value, target);
1235 if (ret != 0)
1236 EPRINTLN("cannot open target %lu at %s: %s", target, value,
1237 strerror(errno));
1238 return (ret);
1239 }
1240
1241 static int
pci_vtscsi_get_config_num(nvlist_t * nvl,const char * name,uint32_t lim_lo,uint32_t lim_hi,uint32_t * res)1242 pci_vtscsi_get_config_num(nvlist_t *nvl, const char *name, uint32_t lim_lo,
1243 uint32_t lim_hi, uint32_t *res)
1244 {
1245 const char *value;
1246 const char *errstr;
1247 long long val;
1248
1249 value = get_config_value_node(nvl, name);
1250 if (value == NULL)
1251 return (0);
1252
1253 val = strtonumx(value, lim_lo, lim_hi, &errstr, 0);
1254 if (errstr != NULL) {
1255 EPRINTLN("Invalid value for %s: %s", name, value);
1256 return (-1);
1257 }
1258
1259 *res = (uint32_t)val;
1260 return (0);
1261 }
1262
1263 static int
pci_vtscsi_init(struct pci_devinst * pi,nvlist_t * nvl)1264 pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl)
1265 {
1266 struct pci_vtscsi_softc *sc;
1267 struct pci_vtscsi_backend *backend, **vbpp;
1268 const char *value;
1269 uint32_t val;
1270 size_t i;
1271 int q, err;
1272
1273 sc = calloc(1, sizeof(struct pci_vtscsi_softc));
1274 if (sc == NULL)
1275 return (-1);
1276
1277 sc->vss_vi_consts = vtscsi_vi_consts;
1278 sc->vss_ctl_ringsz = VTSCSI_DEF_RINGSZ;
1279 sc->vss_evt_ringsz = VTSCSI_DEF_RINGSZ;
1280 sc->vss_req_ringsz = VTSCSI_DEF_RINGSZ;
1281 sc->vss_thr_per_q = VTSCSI_DEF_THR_PER_Q;
1282 sc->vss_default_config = vtscsi_config;
1283
1284 value = get_config_value_node(nvl, "bootindex");
1285 if (value != NULL) {
1286 if (pci_emul_add_boot_device(pi, atoi(value))) {
1287 EPRINTLN("Invalid bootindex %d", atoi(value));
1288 errno = EINVAL;
1289 goto fail;
1290 }
1291 }
1292
1293 val = vtscsi_config.seg_max;
1294 if (pci_vtscsi_get_config_num(nvl, "seg_max", VTSCSI_MIN_MAXSEG,
1295 VTSCSI_MAX_MAXSEG, &val) != 0)
1296 goto fail;
1297 sc->vss_default_config.seg_max = val;
1298
1299 val = vtscsi_config.num_queues;
1300 if (pci_vtscsi_get_config_num(nvl, "num_queues", VTSCSI_MIN_REQUESTQ,
1301 VTSCSI_MAX_REQUESTQ, &val) != 0)
1302 goto fail;
1303 sc->vss_default_config.num_queues = val;
1304
1305 /*
1306 * num_queues is only the number of request queues, but nvq must
1307 * account for the control and event queues.
1308 */
1309 sc->vss_vi_consts.vc_nvq = val + VIRTIO_SCSI_ADDL_Q;
1310
1311 /*
1312 * Allocate queues early, so that they're there for the call to
1313 * vi_softc_linkup().
1314 */
1315 sc->vss_vq = calloc(sc->vss_vi_consts.vc_nvq,
1316 sizeof(struct vqueue_info));
1317 if (sc->vss_vq == NULL) {
1318 EPRINTLN("can't allocate space for %d virtqueues",
1319 sc->vss_vi_consts.vc_nvq);
1320 goto fail;
1321 }
1322
1323 sc->vss_queues = calloc(sc->vss_default_config.num_queues,
1324 sizeof(struct pci_vtscsi_queue));
1325 if (sc->vss_queues == NULL) {
1326 EPRINTLN("can't allocate space for %d request queues",
1327 sc->vss_config.num_queues);
1328 goto fail;
1329 }
1330
1331 if (pci_vtscsi_get_config_num(nvl, "ctl_ringsz", VTSCSI_MIN_RINGSZ,
1332 VTSCSI_MAX_RINGSZ, &sc->vss_ctl_ringsz) != 0)
1333 goto fail;
1334
1335 if (pci_vtscsi_get_config_num(nvl, "evt_ringsz", VTSCSI_MIN_RINGSZ,
1336 VTSCSI_MAX_RINGSZ, &sc->vss_evt_ringsz) != 0)
1337 goto fail;
1338
1339 if (pci_vtscsi_get_config_num(nvl, "req_ringsz", VTSCSI_MIN_RINGSZ,
1340 VTSCSI_MAX_RINGSZ, &sc->vss_req_ringsz) != 0)
1341 goto fail;
1342
1343 if (pci_vtscsi_get_config_num(nvl, "thr_per_q", VTSCSI_MIN_THR_PER_Q,
1344 VTSCSI_MAX_THR_PER_Q, &sc->vss_thr_per_q) != 0)
1345 goto fail;
1346
1347 value = get_config_value_node(nvl, "backend");
1348 if (value == NULL) {
1349 if (SET_COUNT(pci_vtscsi_backend_set) == 0) {
1350 WPRINTF("No virtio-scsi backends available");
1351 errno = EINVAL;
1352 goto fail;
1353 }
1354 backend = SET_ITEM(pci_vtscsi_backend_set, 0);
1355 } else {
1356 backend = NULL;
1357 SET_FOREACH(vbpp, pci_vtscsi_backend_set) {
1358 if (strcasecmp(value, (*vbpp)->vsb_name) == 0) {
1359 backend = *vbpp;
1360 break;
1361 }
1362 }
1363 if (backend == NULL) {
1364 WPRINTF("No such virtio-scsi backend: %s", value);
1365 errno = EINVAL;
1366 goto fail;
1367 }
1368 }
1369
1370 err = backend->vsb_init(sc, backend, nvl);
1371 if (err != 0) {
1372 errno = EINVAL;
1373 goto fail;
1374 }
1375
1376 nvl = find_relative_config_node(nvl, "target");
1377 if (nvl != NULL) {
1378 err = walk_config_nodes("", nvl, sc, pci_vtscsi_count_targets);
1379 if (err != 0)
1380 goto fail;
1381 }
1382
1383 if (sc->vss_num_target > 0) {
1384 sc->vss_targets = malloc(sc->vss_num_target *
1385 sizeof(struct pci_vtscsi_target));
1386 if (sc->vss_targets == NULL) {
1387 EPRINTLN("can't allocate space for %lu targets",
1388 sc->vss_num_target);
1389 goto fail;
1390 }
1391
1392 memset(sc->vss_targets, -1, sc->vss_num_target *
1393 sizeof(struct pci_vtscsi_target));
1394
1395 err = walk_config_nodes("", nvl, sc, pci_vtscsi_init_target);
1396 if (err != 0)
1397 goto fail;
1398 }
1399
1400 /*
1401 * All targets should be open now and have a valid fd.
1402 */
1403 for (i = 0; i < sc->vss_num_target; i++)
1404 if (sc->vss_targets[i].vst_target == i)
1405 assert(sc->vss_targets[i].vst_fd > 0);
1406
1407 pthread_mutex_init(&sc->vss_mtx, NULL);
1408
1409 vi_softc_linkup(&sc->vss_vs, &sc->vss_vi_consts, sc, pi, sc->vss_vq);
1410 sc->vss_vs.vs_mtx = &sc->vss_mtx;
1411
1412 /*
1413 * Perform a "reset" before we set up our queues.
1414 *
1415 * This will write the default config into vss_config, which is used
1416 * by the rest of the driver to get the request header size. Note that
1417 * if we ever allow the guest to override sense size through config
1418 * space writes, pre-allocation of I/O requests will have to change
1419 * accordingly.
1420 */
1421 pthread_mutex_lock(&sc->vss_mtx);
1422 pci_vtscsi_reset(sc);
1423 pthread_mutex_unlock(&sc->vss_mtx);
1424
1425 /* virtqueue 0: control queue */
1426 sc->vss_vq[0].vq_qsize = sc->vss_ctl_ringsz;
1427 sc->vss_vq[0].vq_notify = pci_vtscsi_controlq_notify;
1428
1429 /* virtqueue 1: event queue */
1430 sc->vss_vq[1].vq_qsize = sc->vss_evt_ringsz;
1431 sc->vss_vq[1].vq_notify = pci_vtscsi_eventq_notify;
1432
1433 /* virtqueue 2-n: request queues */
1434 for (q = VIRTIO_SCSI_ADDL_Q; q < sc->vss_vi_consts.vc_nvq; q++) {
1435 int rq = q - VIRTIO_SCSI_ADDL_Q;
1436
1437 sc->vss_vq[q].vq_qsize = sc->vss_req_ringsz;
1438 sc->vss_vq[q].vq_notify = pci_vtscsi_requestq_notify;
1439
1440 err = pci_vtscsi_init_queue(sc, &sc->vss_queues[rq], q);
1441 if (err != 0)
1442 goto fail;
1443 }
1444
1445 /* initialize config space */
1446 pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_SCSI);
1447 pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
1448 pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
1449 pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_SCSI);
1450 pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
1451
1452 err = vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix());
1453 if (err != 0)
1454 goto fail;
1455
1456 vi_set_io_bar(&sc->vss_vs, 0);
1457
1458 return (0);
1459
1460 fail:
1461 if (sc->vss_queues != NULL) {
1462 for (q = VIRTIO_SCSI_ADDL_Q;
1463 q < sc->vss_vi_consts.vc_nvq;
1464 q++) {
1465 int rq = q - VIRTIO_SCSI_ADDL_Q;
1466
1467 pci_vtscsi_destroy_queue(sc, &sc->vss_queues[rq]);
1468 }
1469 }
1470
1471 pthread_mutex_destroy(&sc->vss_mtx);
1472
1473 for (i = 0; i < sc->vss_num_target; i++) {
1474 if (sc->vss_targets[i].vst_target == i &&
1475 sc->vss_targets[i].vst_fd >= 0) {
1476 close(sc->vss_targets[i].vst_fd);
1477 }
1478 }
1479
1480 free(sc->vss_targets);
1481 free(sc->vss_backend);
1482 free(sc->vss_queues);
1483 free(sc->vss_vq);
1484 free(sc);
1485 return (-1);
1486 }
1487
1488
1489 static const struct pci_devemu pci_de_vscsi = {
1490 .pe_emu = "virtio-scsi",
1491 .pe_init = pci_vtscsi_init,
1492 .pe_legacy_config = pci_vtscsi_legacy_config,
1493 .pe_barwrite = vi_pci_write,
1494 .pe_barread = vi_pci_read
1495 };
1496 PCI_EMUL_SET(pci_de_vscsi);
1497