1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1997, 1998, 2000 Justin T. Gibbs.
5 * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions, and the following disclaimer,
13 * without modification, immediately at the beginning of the file.
14 * 2. The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "opt_pass.h"
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/conf.h>
36 #include <sys/types.h>
37 #include <sys/bio.h>
38 #include <sys/bus.h>
39 #include <sys/devicestat.h>
40 #include <sys/errno.h>
41 #include <sys/fcntl.h>
42 #include <sys/malloc.h>
43 #include <sys/proc.h>
44 #include <sys/poll.h>
45 #include <sys/selinfo.h>
46 #include <sys/sdt.h>
47 #include <sys/sysent.h>
48 #include <sys/taskqueue.h>
49 #include <vm/uma.h>
50 #include <vm/vm.h>
51 #include <vm/vm_extern.h>
52
53 #include <machine/bus.h>
54
55 #include <cam/cam.h>
56 #include <cam/cam_ccb.h>
57 #include <cam/cam_periph.h>
58 #include <cam/cam_queue.h>
59 #include <cam/cam_xpt.h>
60 #include <cam/cam_xpt_periph.h>
61 #include <cam/cam_debug.h>
62 #include <cam/cam_compat.h>
63 #include <cam/cam_xpt_periph.h>
64
65 #include <cam/scsi/scsi_pass.h>
66
67 #define PERIPH_NAME "pass"
68
69 typedef enum {
70 PASS_FLAG_OPEN = 0x01,
71 PASS_FLAG_LOCKED = 0x02,
72 PASS_FLAG_INVALID = 0x04,
73 PASS_FLAG_INITIAL_PHYSPATH = 0x08,
74 PASS_FLAG_ZONE_INPROG = 0x10,
75 PASS_FLAG_ZONE_VALID = 0x20,
76 PASS_FLAG_UNMAPPED_CAPABLE = 0x40,
77 PASS_FLAG_ABANDONED_REF_SET = 0x80
78 } pass_flags;
79
80 typedef enum {
81 PASS_STATE_NORMAL
82 } pass_state;
83
84 typedef enum {
85 PASS_CCB_BUFFER_IO,
86 PASS_CCB_QUEUED_IO
87 } pass_ccb_types;
88
89 #define ccb_type ppriv_field0
90 #define ccb_ioreq ppriv_ptr1
91
92 /*
93 * The maximum number of memory segments we preallocate.
94 */
95 #define PASS_MAX_SEGS 16
96
97 typedef enum {
98 PASS_IO_NONE = 0x00,
99 PASS_IO_USER_SEG_MALLOC = 0x01,
100 PASS_IO_KERN_SEG_MALLOC = 0x02,
101 PASS_IO_ABANDONED = 0x04
102 } pass_io_flags;
103
104 struct pass_io_req {
105 union ccb ccb;
106 union ccb *alloced_ccb;
107 union ccb *user_ccb_ptr;
108 camq_entry user_periph_links;
109 ccb_ppriv_area user_periph_priv;
110 struct cam_periph_map_info mapinfo;
111 pass_io_flags flags;
112 ccb_flags data_flags;
113 int num_user_segs;
114 bus_dma_segment_t user_segs[PASS_MAX_SEGS];
115 int num_kern_segs;
116 bus_dma_segment_t kern_segs[PASS_MAX_SEGS];
117 bus_dma_segment_t *user_segptr;
118 bus_dma_segment_t *kern_segptr;
119 int num_bufs;
120 uint32_t dirs[CAM_PERIPH_MAXMAPS];
121 uint32_t lengths[CAM_PERIPH_MAXMAPS];
122 uint8_t *user_bufs[CAM_PERIPH_MAXMAPS];
123 uint8_t *kern_bufs[CAM_PERIPH_MAXMAPS];
124 struct bintime start_time;
125 TAILQ_ENTRY(pass_io_req) links;
126 };
127
128 struct pass_softc {
129 pass_state state;
130 pass_flags flags;
131 uint8_t pd_type;
132 int open_count;
133 u_int maxio;
134 struct devstat *device_stats;
135 struct cdev *dev;
136 struct cdev *alias_dev;
137 struct task add_physpath_task;
138 struct task shutdown_kqueue_task;
139 struct selinfo read_select;
140 TAILQ_HEAD(, pass_io_req) incoming_queue;
141 TAILQ_HEAD(, pass_io_req) active_queue;
142 TAILQ_HEAD(, pass_io_req) abandoned_queue;
143 TAILQ_HEAD(, pass_io_req) done_queue;
144 struct cam_periph *periph;
145 char zone_name[12];
146 char io_zone_name[12];
147 uma_zone_t pass_zone;
148 uma_zone_t pass_io_zone;
149 size_t io_zone_size;
150 };
151
152 static d_open_t passopen;
153 static d_close_t passclose;
154 static d_ioctl_t passioctl;
155 static d_ioctl_t passdoioctl;
156 static d_poll_t passpoll;
157 static d_kqfilter_t passkqfilter;
158 static void passreadfiltdetach(struct knote *kn);
159 static int passreadfilt(struct knote *kn, long hint);
160
161 static periph_init_t passinit;
162 static periph_ctor_t passregister;
163 static periph_oninv_t passoninvalidate;
164 static periph_dtor_t passcleanup;
165 static periph_start_t passstart;
166 static void pass_shutdown_kqueue(void *context, int pending);
167 static void pass_add_physpath(void *context, int pending);
168 static void passasync(void *callback_arg, uint32_t code,
169 struct cam_path *path, void *arg);
170 static void passdone(struct cam_periph *periph,
171 union ccb *done_ccb);
172 static int passcreatezone(struct cam_periph *periph);
173 static void passiocleanup(struct pass_softc *softc,
174 struct pass_io_req *io_req);
175 static int passcopysglist(struct cam_periph *periph,
176 struct pass_io_req *io_req,
177 ccb_flags direction);
178 static int passmemsetup(struct cam_periph *periph,
179 struct pass_io_req *io_req);
180 static int passmemdone(struct cam_periph *periph,
181 struct pass_io_req *io_req);
182 static int passerror(union ccb *ccb, uint32_t cam_flags,
183 uint32_t sense_flags);
184 static int passsendccb(struct cam_periph *periph, union ccb *ccb,
185 union ccb *inccb);
186 static void passflags(union ccb *ccb, uint32_t *cam_flags,
187 uint32_t *sense_flags);
188
189 static struct periph_driver passdriver =
190 {
191 passinit, PERIPH_NAME,
192 TAILQ_HEAD_INITIALIZER(passdriver.units), /* generation */ 0
193 };
194
195 PERIPHDRIVER_DECLARE(pass, passdriver);
196
197 static struct cdevsw pass_cdevsw = {
198 .d_version = D_VERSION,
199 .d_flags = D_TRACKCLOSE,
200 .d_open = passopen,
201 .d_close = passclose,
202 .d_ioctl = passioctl,
203 .d_poll = passpoll,
204 .d_kqfilter = passkqfilter,
205 .d_name = PERIPH_NAME,
206 };
207
208 static const struct filterops passread_filtops = {
209 .f_isfd = 1,
210 .f_detach = passreadfiltdetach,
211 .f_event = passreadfilt,
212 .f_copy = knote_triv_copy,
213 };
214
215 static MALLOC_DEFINE(M_SCSIPASS, "scsi_pass", "scsi passthrough buffers");
216
217 static void
passinit(void)218 passinit(void)
219 {
220 cam_status status;
221
222 /*
223 * Install a global async callback. This callback will
224 * receive async callbacks like "new device found".
225 */
226 status = xpt_register_async(AC_FOUND_DEVICE, passasync, NULL, NULL);
227
228 if (status != CAM_REQ_CMP) {
229 printf("pass: Failed to attach master async callback "
230 "due to status 0x%x!\n", status);
231 }
232
233 }
234
235 static void
passrejectios(struct cam_periph * periph)236 passrejectios(struct cam_periph *periph)
237 {
238 struct pass_io_req *io_req, *io_req2;
239 struct pass_softc *softc;
240
241 softc = (struct pass_softc *)periph->softc;
242
243 /*
244 * The user can no longer get status for I/O on the done queue, so
245 * clean up all outstanding I/O on the done queue.
246 */
247 TAILQ_FOREACH_SAFE(io_req, &softc->done_queue, links, io_req2) {
248 TAILQ_REMOVE(&softc->done_queue, io_req, links);
249 passiocleanup(softc, io_req);
250 uma_zfree(softc->pass_zone, io_req);
251 }
252
253 /*
254 * The underlying device is gone, so we can't issue these I/Os.
255 * The devfs node has been shut down, so we can't return status to
256 * the user. Free any I/O left on the incoming queue.
257 */
258 TAILQ_FOREACH_SAFE(io_req, &softc->incoming_queue, links, io_req2) {
259 TAILQ_REMOVE(&softc->incoming_queue, io_req, links);
260 passiocleanup(softc, io_req);
261 uma_zfree(softc->pass_zone, io_req);
262 }
263
264 /*
265 * Normally we would put I/Os on the abandoned queue and acquire a
266 * reference when we saw the final close. But, the device went
267 * away and devfs may have moved everything off to deadfs by the
268 * time the I/O done callback is called; as a result, we won't see
269 * any more closes. So, if we have any active I/Os, we need to put
270 * them on the abandoned queue. When the abandoned queue is empty,
271 * we'll release the remaining reference (see below) to the peripheral.
272 */
273 TAILQ_FOREACH_SAFE(io_req, &softc->active_queue, links, io_req2) {
274 TAILQ_REMOVE(&softc->active_queue, io_req, links);
275 io_req->flags |= PASS_IO_ABANDONED;
276 TAILQ_INSERT_TAIL(&softc->abandoned_queue, io_req, links);
277 }
278
279 /*
280 * If we put any I/O on the abandoned queue, acquire a reference.
281 */
282 if ((!TAILQ_EMPTY(&softc->abandoned_queue))
283 && ((softc->flags & PASS_FLAG_ABANDONED_REF_SET) == 0)) {
284 cam_periph_doacquire(periph);
285 softc->flags |= PASS_FLAG_ABANDONED_REF_SET;
286 }
287 }
288
289 static void
passdevgonecb(void * arg)290 passdevgonecb(void *arg)
291 {
292 struct cam_periph *periph;
293 struct mtx *mtx;
294 struct pass_softc *softc;
295 int i;
296
297 periph = (struct cam_periph *)arg;
298 mtx = cam_periph_mtx(periph);
299 mtx_lock(mtx);
300
301 softc = (struct pass_softc *)periph->softc;
302 KASSERT(softc->open_count >= 0, ("Negative open count %d",
303 softc->open_count));
304
305 /*
306 * When we get this callback, we will get no more close calls from
307 * devfs. So if we have any dangling opens, we need to release the
308 * reference held for that particular context.
309 */
310 for (i = 0; i < softc->open_count; i++)
311 cam_periph_release_locked(periph);
312
313 softc->open_count = 0;
314
315 /*
316 * Release the reference held for the device node, it is gone now.
317 * Accordingly, inform all queued I/Os of their fate.
318 */
319 cam_periph_release_locked(periph);
320 passrejectios(periph);
321
322 /*
323 * We reference the SIM lock directly here, instead of using
324 * cam_periph_unlock(). The reason is that the final call to
325 * cam_periph_release_locked() above could result in the periph
326 * getting freed. If that is the case, dereferencing the periph
327 * with a cam_periph_unlock() call would cause a page fault.
328 */
329 mtx_unlock(mtx);
330
331 /*
332 * We have to remove our kqueue context from a thread because it
333 * may sleep. It would be nice if we could get a callback from
334 * kqueue when it is done cleaning up resources.
335 */
336 taskqueue_enqueue(taskqueue_thread, &softc->shutdown_kqueue_task);
337 }
338
339 static void
passoninvalidate(struct cam_periph * periph)340 passoninvalidate(struct cam_periph *periph)
341 {
342 struct pass_softc *softc;
343
344 softc = (struct pass_softc *)periph->softc;
345
346 /*
347 * De-register any async callbacks.
348 */
349 xpt_register_async(0, passasync, periph, periph->path);
350
351 softc->flags |= PASS_FLAG_INVALID;
352
353 /*
354 * Tell devfs this device has gone away, and ask for a callback
355 * when it has cleaned up its state.
356 */
357 destroy_dev_sched_cb(softc->dev, passdevgonecb, periph);
358 }
359
360 static void
passcleanup(struct cam_periph * periph)361 passcleanup(struct cam_periph *periph)
362 {
363 struct pass_softc *softc;
364
365 softc = (struct pass_softc *)periph->softc;
366
367 cam_periph_assert(periph, MA_OWNED);
368 KASSERT(TAILQ_EMPTY(&softc->active_queue),
369 ("%s called when there are commands on the active queue!\n",
370 __func__));
371 KASSERT(TAILQ_EMPTY(&softc->abandoned_queue),
372 ("%s called when there are commands on the abandoned queue!\n",
373 __func__));
374 KASSERT(TAILQ_EMPTY(&softc->incoming_queue),
375 ("%s called when there are commands on the incoming queue!\n",
376 __func__));
377 KASSERT(TAILQ_EMPTY(&softc->done_queue),
378 ("%s called when there are commands on the done queue!\n",
379 __func__));
380
381 devstat_remove_entry(softc->device_stats);
382
383 cam_periph_unlock(periph);
384
385 /*
386 * We call taskqueue_drain() for the physpath task to make sure it
387 * is complete. We drop the lock because this can potentially
388 * sleep. XXX KDM that is bad. Need a way to get a callback when
389 * a taskqueue is drained.
390 *
391 * Note that we don't drain the kqueue shutdown task queue. This
392 * is because we hold a reference on the periph for kqueue, and
393 * release that reference from the kqueue shutdown task queue. So
394 * we cannot come into this routine unless we've released that
395 * reference. Also, because that could be the last reference, we
396 * could be called from the cam_periph_release() call in
397 * pass_shutdown_kqueue(). In that case, the taskqueue_drain()
398 * would deadlock. It would be preferable if we had a way to
399 * get a callback when a taskqueue is done.
400 */
401 taskqueue_drain(taskqueue_thread, &softc->add_physpath_task);
402
403 /*
404 * It should be safe to destroy the zones from here, because all
405 * of the references to this peripheral have been freed, and all
406 * I/O has been terminated and freed. We check the zones for NULL
407 * because they may not have been allocated yet if the device went
408 * away before any asynchronous I/O has been issued.
409 */
410 if (softc->pass_zone != NULL)
411 uma_zdestroy(softc->pass_zone);
412 if (softc->pass_io_zone != NULL)
413 uma_zdestroy(softc->pass_io_zone);
414
415 cam_periph_lock(periph);
416
417 free(softc, M_DEVBUF);
418 }
419
420 static void
pass_shutdown_kqueue(void * context,int pending)421 pass_shutdown_kqueue(void *context, int pending)
422 {
423 struct cam_periph *periph;
424 struct pass_softc *softc;
425
426 periph = context;
427 softc = periph->softc;
428
429 knlist_clear(&softc->read_select.si_note, /*is_locked*/ 0);
430 knlist_destroy(&softc->read_select.si_note);
431
432 /*
433 * Release the reference we held for kqueue.
434 */
435 cam_periph_release(periph);
436 }
437
438 static void
pass_add_physpath(void * context,int pending)439 pass_add_physpath(void *context, int pending)
440 {
441 struct cam_periph *periph;
442 struct pass_softc *softc;
443 struct mtx *mtx;
444 char *physpath;
445
446 /*
447 * If we have one, create a devfs alias for our
448 * physical path.
449 */
450 periph = context;
451 softc = periph->softc;
452 physpath = malloc(MAXPATHLEN, M_DEVBUF, M_WAITOK);
453 mtx = cam_periph_mtx(periph);
454 mtx_lock(mtx);
455
456 if (periph->flags & CAM_PERIPH_INVALID)
457 goto out;
458
459 if (xpt_getattr(physpath, MAXPATHLEN,
460 "GEOM::physpath", periph->path) == 0
461 && strlen(physpath) != 0) {
462 mtx_unlock(mtx);
463 make_dev_physpath_alias(MAKEDEV_WAITOK | MAKEDEV_CHECKNAME,
464 &softc->alias_dev, softc->dev,
465 softc->alias_dev, physpath);
466 mtx_lock(mtx);
467 }
468
469 out:
470 /*
471 * Now that we've made our alias, we no longer have to have a
472 * reference to the device.
473 */
474 if ((softc->flags & PASS_FLAG_INITIAL_PHYSPATH) == 0)
475 softc->flags |= PASS_FLAG_INITIAL_PHYSPATH;
476
477 /*
478 * We always acquire a reference to the periph before queueing this
479 * task queue function, so it won't go away before we run.
480 */
481 while (pending-- > 0)
482 cam_periph_release_locked(periph);
483 mtx_unlock(mtx);
484
485 free(physpath, M_DEVBUF);
486 }
487
488 static void
passasync(void * callback_arg,uint32_t code,struct cam_path * path,void * arg)489 passasync(void *callback_arg, uint32_t code,
490 struct cam_path *path, void *arg)
491 {
492 struct cam_periph *periph;
493
494 periph = (struct cam_periph *)callback_arg;
495
496 switch (code) {
497 case AC_FOUND_DEVICE:
498 {
499 struct ccb_getdev *cgd;
500 cam_status status;
501
502 cgd = (struct ccb_getdev *)arg;
503 if (cgd == NULL)
504 break;
505
506 /*
507 * Allocate a peripheral instance for
508 * this device and start the probe
509 * process.
510 */
511 status = cam_periph_alloc(passregister, passoninvalidate,
512 passcleanup, passstart, PERIPH_NAME,
513 CAM_PERIPH_BIO, path,
514 passasync, AC_FOUND_DEVICE, cgd);
515
516 if (status != CAM_REQ_CMP
517 && status != CAM_REQ_INPROG) {
518 const struct cam_status_entry *entry;
519
520 entry = cam_fetch_status_entry(status);
521
522 printf("passasync: Unable to attach new device "
523 "due to status %#x: %s\n", status, entry ?
524 entry->status_text : "Unknown");
525 }
526
527 break;
528 }
529 case AC_ADVINFO_CHANGED:
530 {
531 uintptr_t buftype;
532
533 buftype = (uintptr_t)arg;
534 if (buftype == CDAI_TYPE_PHYS_PATH) {
535 struct pass_softc *softc;
536
537 softc = (struct pass_softc *)periph->softc;
538 /*
539 * Acquire a reference to the periph before we
540 * start the taskqueue, so that we don't run into
541 * a situation where the periph goes away before
542 * the task queue has a chance to run.
543 */
544 if (cam_periph_acquire(periph) != 0)
545 break;
546
547 taskqueue_enqueue(taskqueue_thread,
548 &softc->add_physpath_task);
549 }
550 break;
551 }
552 default:
553 cam_periph_async(periph, code, path, arg);
554 break;
555 }
556 }
557
558 static cam_status
passregister(struct cam_periph * periph,void * arg)559 passregister(struct cam_periph *periph, void *arg)
560 {
561 struct pass_softc *softc;
562 struct ccb_getdev *cgd;
563 struct ccb_pathinq cpi;
564 struct make_dev_args args;
565 int error, no_tags;
566
567 cgd = (struct ccb_getdev *)arg;
568 if (cgd == NULL) {
569 printf("%s: no getdev CCB, can't register device\n", __func__);
570 return(CAM_REQ_CMP_ERR);
571 }
572
573 softc = (struct pass_softc *)malloc(sizeof(*softc),
574 M_DEVBUF, M_NOWAIT);
575
576 if (softc == NULL) {
577 printf("%s: Unable to probe new device. "
578 "Unable to allocate softc\n", __func__);
579 return(CAM_REQ_CMP_ERR);
580 }
581
582 bzero(softc, sizeof(*softc));
583 softc->state = PASS_STATE_NORMAL;
584 if (cgd->protocol == PROTO_SCSI || cgd->protocol == PROTO_ATAPI)
585 softc->pd_type = SID_TYPE(&cgd->inq_data);
586 else if (cgd->protocol == PROTO_SATAPM)
587 softc->pd_type = T_ENCLOSURE;
588 else
589 softc->pd_type = T_DIRECT;
590
591 periph->softc = softc;
592 softc->periph = periph;
593 TAILQ_INIT(&softc->incoming_queue);
594 TAILQ_INIT(&softc->active_queue);
595 TAILQ_INIT(&softc->abandoned_queue);
596 TAILQ_INIT(&softc->done_queue);
597 snprintf(softc->zone_name, sizeof(softc->zone_name), "%s%d",
598 periph->periph_name, periph->unit_number);
599 snprintf(softc->io_zone_name, sizeof(softc->io_zone_name), "%s%dIO",
600 periph->periph_name, periph->unit_number);
601 softc->io_zone_size = maxphys;
602 knlist_init_mtx(&softc->read_select.si_note, cam_periph_mtx(periph));
603
604 xpt_path_inq(&cpi, periph->path);
605
606 if (cpi.maxio == 0)
607 softc->maxio = DFLTPHYS; /* traditional default */
608 else if (cpi.maxio > maxphys)
609 softc->maxio = maxphys; /* for safety */
610 else
611 softc->maxio = cpi.maxio; /* real value */
612
613 if (cpi.hba_misc & PIM_UNMAPPED)
614 softc->flags |= PASS_FLAG_UNMAPPED_CAPABLE;
615
616 /*
617 * We pass in 0 for a blocksize, since we don't know what the blocksize
618 * of this device is, if it even has a blocksize.
619 *
620 * Note: no_tags is valid only for SCSI peripherals, but we don't do any
621 * devstat accounting for tags on any other transport. SCSI is the only
622 * transport that uses the tag_action (ata has only vestigial references
623 * to it, others ignore it entirely).
624 */
625 cam_periph_unlock(periph);
626 no_tags = (cgd->inq_data.flags & SID_CmdQue) == 0;
627 softc->device_stats = devstat_new_entry(PERIPH_NAME,
628 periph->unit_number, 0,
629 DEVSTAT_NO_BLOCKSIZE
630 | (no_tags ? DEVSTAT_NO_ORDERED_TAGS : 0),
631 softc->pd_type |
632 XPORT_DEVSTAT_TYPE(cpi.transport) |
633 DEVSTAT_TYPE_PASS,
634 DEVSTAT_PRIORITY_PASS);
635
636 /*
637 * Initialize the taskqueue handler for shutting down kqueue.
638 */
639 TASK_INIT(&softc->shutdown_kqueue_task, /*priority*/ 0,
640 pass_shutdown_kqueue, periph);
641
642 /*
643 * Acquire a reference to the periph that we can release once we've
644 * cleaned up the kqueue.
645 */
646 if (cam_periph_acquire(periph) != 0) {
647 xpt_print(periph->path, "%s: lost periph during "
648 "registration!\n", __func__);
649 cam_periph_lock(periph);
650 return (CAM_REQ_CMP_ERR);
651 }
652
653 /*
654 * Acquire a reference to the periph before we create the devfs
655 * instance for it. We'll release this reference once the devfs
656 * instance has been freed.
657 */
658 if (cam_periph_acquire(periph) != 0) {
659 xpt_print(periph->path, "%s: lost periph during "
660 "registration!\n", __func__);
661 cam_periph_lock(periph);
662 return (CAM_REQ_CMP_ERR);
663 }
664
665 /* Register the device */
666 make_dev_args_init(&args);
667 args.mda_devsw = &pass_cdevsw;
668 args.mda_unit = periph->unit_number;
669 args.mda_uid = UID_ROOT;
670 args.mda_gid = GID_OPERATOR;
671 args.mda_mode = 0600;
672 args.mda_si_drv1 = periph;
673 args.mda_flags = MAKEDEV_NOWAIT;
674 error = make_dev_s(&args, &softc->dev, "%s%d", periph->periph_name,
675 periph->unit_number);
676 if (error != 0) {
677 cam_periph_lock(periph);
678 cam_periph_release_locked(periph);
679 return (CAM_REQ_CMP_ERR);
680 }
681
682 /*
683 * Hold a reference to the periph before we create the physical
684 * path alias so it can't go away.
685 */
686 if (cam_periph_acquire(periph) != 0) {
687 xpt_print(periph->path, "%s: lost periph during "
688 "registration!\n", __func__);
689 cam_periph_lock(periph);
690 return (CAM_REQ_CMP_ERR);
691 }
692
693 cam_periph_lock(periph);
694
695 TASK_INIT(&softc->add_physpath_task, /*priority*/0,
696 pass_add_physpath, periph);
697
698 /*
699 * See if physical path information is already available.
700 */
701 taskqueue_enqueue(taskqueue_thread, &softc->add_physpath_task);
702
703 /*
704 * Add an async callback so that we get notified if
705 * this device goes away or its physical path
706 * (stored in the advanced info data of the EDT) has
707 * changed.
708 */
709 xpt_register_async(AC_LOST_DEVICE | AC_ADVINFO_CHANGED,
710 passasync, periph, periph->path);
711
712 if (bootverbose)
713 xpt_announce_periph(periph, NULL);
714
715 return(CAM_REQ_CMP);
716 }
717
718 static int
passopen(struct cdev * dev,int flags,int fmt,struct thread * td)719 passopen(struct cdev *dev, int flags, int fmt, struct thread *td)
720 {
721 struct cam_periph *periph;
722 struct pass_softc *softc;
723 int error;
724
725 periph = (struct cam_periph *)dev->si_drv1;
726 if (cam_periph_acquire(periph) != 0)
727 return (ENXIO);
728
729 cam_periph_lock(periph);
730
731 softc = (struct pass_softc *)periph->softc;
732
733 if (softc->flags & PASS_FLAG_INVALID) {
734 cam_periph_release_locked(periph);
735 cam_periph_unlock(periph);
736 return(ENXIO);
737 }
738
739 /*
740 * Don't allow access when we're running at a high securelevel.
741 */
742 error = securelevel_gt(td->td_ucred, 1);
743 if (error) {
744 cam_periph_release_locked(periph);
745 cam_periph_unlock(periph);
746 return(error);
747 }
748
749 /*
750 * Only allow read-write access.
751 */
752 if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0)) {
753 cam_periph_release_locked(periph);
754 cam_periph_unlock(periph);
755 return(EPERM);
756 }
757
758 /*
759 * We don't allow nonblocking access.
760 */
761 if ((flags & O_NONBLOCK) != 0) {
762 xpt_print(periph->path, "can't do nonblocking access\n");
763 cam_periph_release_locked(periph);
764 cam_periph_unlock(periph);
765 return(EINVAL);
766 }
767
768 softc->open_count++;
769
770 cam_periph_unlock(periph);
771
772 return (error);
773 }
774
775 static int
passclose(struct cdev * dev,int flag,int fmt,struct thread * td)776 passclose(struct cdev *dev, int flag, int fmt, struct thread *td)
777 {
778 struct cam_periph *periph;
779 struct pass_softc *softc;
780 struct mtx *mtx;
781
782 periph = (struct cam_periph *)dev->si_drv1;
783 mtx = cam_periph_mtx(periph);
784 mtx_lock(mtx);
785
786 softc = periph->softc;
787 softc->open_count--;
788
789 if (softc->open_count == 0) {
790 struct pass_io_req *io_req, *io_req2;
791
792 TAILQ_FOREACH_SAFE(io_req, &softc->done_queue, links, io_req2) {
793 TAILQ_REMOVE(&softc->done_queue, io_req, links);
794 passiocleanup(softc, io_req);
795 uma_zfree(softc->pass_zone, io_req);
796 }
797
798 TAILQ_FOREACH_SAFE(io_req, &softc->incoming_queue, links,
799 io_req2) {
800 TAILQ_REMOVE(&softc->incoming_queue, io_req, links);
801 passiocleanup(softc, io_req);
802 uma_zfree(softc->pass_zone, io_req);
803 }
804
805 /*
806 * If there are any active I/Os, we need to forcibly acquire a
807 * reference to the peripheral so that we don't go away
808 * before they complete. We'll release the reference when
809 * the abandoned queue is empty.
810 */
811 io_req = TAILQ_FIRST(&softc->active_queue);
812 if ((io_req != NULL)
813 && (softc->flags & PASS_FLAG_ABANDONED_REF_SET) == 0) {
814 cam_periph_doacquire(periph);
815 softc->flags |= PASS_FLAG_ABANDONED_REF_SET;
816 }
817
818 /*
819 * Since the I/O in the active queue is not under our
820 * control, just set a flag so that we can clean it up when
821 * it completes and put it on the abandoned queue. This
822 * will prevent our sending spurious completions in the
823 * event that the device is opened again before these I/Os
824 * complete.
825 */
826 TAILQ_FOREACH_SAFE(io_req, &softc->active_queue, links,
827 io_req2) {
828 TAILQ_REMOVE(&softc->active_queue, io_req, links);
829 io_req->flags |= PASS_IO_ABANDONED;
830 TAILQ_INSERT_TAIL(&softc->abandoned_queue, io_req,
831 links);
832 }
833 }
834
835 cam_periph_release_locked(periph);
836
837 /*
838 * We reference the lock directly here, instead of using
839 * cam_periph_unlock(). The reason is that the call to
840 * cam_periph_release_locked() above could result in the periph
841 * getting freed. If that is the case, dereferencing the periph
842 * with a cam_periph_unlock() call would cause a page fault.
843 *
844 * cam_periph_release() avoids this problem using the same method,
845 * but we're manually acquiring and dropping the lock here to
846 * protect the open count and avoid another lock acquisition and
847 * release.
848 */
849 mtx_unlock(mtx);
850
851 return (0);
852 }
853
854 static void
passstart(struct cam_periph * periph,union ccb * start_ccb)855 passstart(struct cam_periph *periph, union ccb *start_ccb)
856 {
857 struct pass_softc *softc;
858
859 softc = (struct pass_softc *)periph->softc;
860
861 switch (softc->state) {
862 case PASS_STATE_NORMAL: {
863 struct pass_io_req *io_req;
864
865 /*
866 * Check for any queued I/O requests that require an
867 * allocated slot.
868 */
869 io_req = TAILQ_FIRST(&softc->incoming_queue);
870 if (io_req == NULL) {
871 xpt_release_ccb(start_ccb);
872 break;
873 }
874 TAILQ_REMOVE(&softc->incoming_queue, io_req, links);
875 TAILQ_INSERT_TAIL(&softc->active_queue, io_req, links);
876 /*
877 * Merge the user's CCB into the allocated CCB.
878 */
879 xpt_merge_ccb(start_ccb, &io_req->ccb);
880 start_ccb->ccb_h.ccb_type = PASS_CCB_QUEUED_IO;
881 start_ccb->ccb_h.ccb_ioreq = io_req;
882 start_ccb->ccb_h.cbfcnp = passdone;
883 io_req->alloced_ccb = start_ccb;
884 binuptime(&io_req->start_time);
885 devstat_start_transaction(softc->device_stats,
886 &io_req->start_time);
887
888 xpt_action(start_ccb);
889
890 /*
891 * If we have any more I/O waiting, schedule ourselves again.
892 */
893 if (!TAILQ_EMPTY(&softc->incoming_queue))
894 xpt_schedule(periph, CAM_PRIORITY_NORMAL);
895 break;
896 }
897 default:
898 break;
899 }
900 }
901
902 static void
passdone(struct cam_periph * periph,union ccb * done_ccb)903 passdone(struct cam_periph *periph, union ccb *done_ccb)
904 {
905 struct pass_softc *softc;
906 struct ccb_hdr *hdr;
907
908 softc = (struct pass_softc *)periph->softc;
909
910 cam_periph_assert(periph, MA_OWNED);
911
912 hdr = &done_ccb->ccb_h;
913 switch (hdr->ccb_type) {
914 case PASS_CCB_QUEUED_IO: {
915 struct pass_io_req *io_req;
916
917 io_req = hdr->ccb_ioreq;
918 #if 0
919 xpt_print(periph->path, "%s: called for user CCB %p\n",
920 __func__, io_req->user_ccb_ptr);
921 #endif
922 if (((hdr->status & CAM_STATUS_MASK) != CAM_REQ_CMP) &&
923 ((io_req->flags & PASS_IO_ABANDONED) == 0)) {
924 int error;
925 uint32_t cam_flags, sense_flags;
926
927 passflags(done_ccb, &cam_flags, &sense_flags);
928 error = passerror(done_ccb, cam_flags, sense_flags);
929
930 if (error == ERESTART) {
931 KASSERT(((sense_flags & SF_NO_RETRY) == 0),
932 ("passerror returned ERESTART with no retry requested\n"));
933 return;
934 }
935 }
936
937 /*
938 * Copy the allocated CCB contents back to the malloced CCB
939 * so we can give status back to the user when he requests it.
940 */
941 bcopy(done_ccb, &io_req->ccb, sizeof(*done_ccb));
942
943 /*
944 * Log data/transaction completion with devstat(9).
945 */
946 switch (hdr->func_code) {
947 case XPT_SCSI_IO:
948 devstat_end_transaction(softc->device_stats,
949 done_ccb->csio.dxfer_len - done_ccb->csio.resid,
950 done_ccb->csio.tag_action & 0x3,
951 ((hdr->flags & CAM_DIR_MASK) ==
952 CAM_DIR_NONE) ? DEVSTAT_NO_DATA :
953 (hdr->flags & CAM_DIR_OUT) ?
954 DEVSTAT_WRITE : DEVSTAT_READ, NULL,
955 &io_req->start_time);
956 break;
957 case XPT_ATA_IO:
958 devstat_end_transaction(softc->device_stats,
959 done_ccb->ataio.dxfer_len - done_ccb->ataio.resid,
960 0, /* Not used in ATA */
961 ((hdr->flags & CAM_DIR_MASK) ==
962 CAM_DIR_NONE) ? DEVSTAT_NO_DATA :
963 (hdr->flags & CAM_DIR_OUT) ?
964 DEVSTAT_WRITE : DEVSTAT_READ, NULL,
965 &io_req->start_time);
966 break;
967 case XPT_SMP_IO:
968 /*
969 * XXX KDM this isn't quite right, but there isn't
970 * currently an easy way to represent a bidirectional
971 * transfer in devstat. The only way to do it
972 * and have the byte counts come out right would
973 * mean that we would have to record two
974 * transactions, one for the request and one for the
975 * response. For now, so that we report something,
976 * just treat the entire thing as a read.
977 */
978 devstat_end_transaction(softc->device_stats,
979 done_ccb->smpio.smp_request_len +
980 done_ccb->smpio.smp_response_len,
981 DEVSTAT_TAG_SIMPLE, DEVSTAT_READ, NULL,
982 &io_req->start_time);
983 break;
984 /* XXX XPT_NVME_IO and XPT_NVME_ADMIN need cases here for resid */
985 default:
986 devstat_end_transaction(softc->device_stats, 0,
987 DEVSTAT_TAG_NONE, DEVSTAT_NO_DATA, NULL,
988 &io_req->start_time);
989 break;
990 }
991
992 /*
993 * In the normal case, take the completed I/O off of the
994 * active queue and put it on the done queue. Notitfy the
995 * user that we have a completed I/O.
996 */
997 if ((io_req->flags & PASS_IO_ABANDONED) == 0) {
998 TAILQ_REMOVE(&softc->active_queue, io_req, links);
999 TAILQ_INSERT_TAIL(&softc->done_queue, io_req, links);
1000 selwakeuppri(&softc->read_select, PRIBIO);
1001 KNOTE_LOCKED(&softc->read_select.si_note, 0);
1002 } else {
1003 /*
1004 * In the case of an abandoned I/O (final close
1005 * without fetching the I/O), take it off of the
1006 * abandoned queue and free it.
1007 */
1008 TAILQ_REMOVE(&softc->abandoned_queue, io_req, links);
1009 passiocleanup(softc, io_req);
1010 uma_zfree(softc->pass_zone, io_req);
1011
1012 /*
1013 * Release the done_ccb here, since we may wind up
1014 * freeing the peripheral when we decrement the
1015 * reference count below.
1016 */
1017 xpt_release_ccb(done_ccb);
1018
1019 /*
1020 * If the abandoned queue is empty, we can release
1021 * our reference to the periph since we won't have
1022 * any more completions coming.
1023 */
1024 if ((TAILQ_EMPTY(&softc->abandoned_queue))
1025 && (softc->flags & PASS_FLAG_ABANDONED_REF_SET)) {
1026 softc->flags &= ~PASS_FLAG_ABANDONED_REF_SET;
1027 cam_periph_release_locked(periph);
1028 }
1029
1030 /*
1031 * We have already released the CCB, so we can
1032 * return.
1033 */
1034 return;
1035 }
1036 break;
1037 }
1038 }
1039 xpt_release_ccb(done_ccb);
1040 }
1041
1042 static int
passcreatezone(struct cam_periph * periph)1043 passcreatezone(struct cam_periph *periph)
1044 {
1045 struct pass_softc *softc;
1046 int error;
1047
1048 error = 0;
1049 softc = (struct pass_softc *)periph->softc;
1050
1051 cam_periph_assert(periph, MA_OWNED);
1052 KASSERT(((softc->flags & PASS_FLAG_ZONE_VALID) == 0),
1053 ("%s called when the pass(4) zone is valid!\n", __func__));
1054 KASSERT((softc->pass_zone == NULL),
1055 ("%s called when the pass(4) zone is allocated!\n", __func__));
1056
1057 if ((softc->flags & PASS_FLAG_ZONE_INPROG) == 0) {
1058 /*
1059 * We're the first context through, so we need to create
1060 * the pass(4) UMA zone for I/O requests.
1061 */
1062 softc->flags |= PASS_FLAG_ZONE_INPROG;
1063
1064 /*
1065 * uma_zcreate() does a blocking (M_WAITOK) allocation,
1066 * so we cannot hold a mutex while we call it.
1067 */
1068 cam_periph_unlock(periph);
1069
1070 softc->pass_zone = uma_zcreate(softc->zone_name,
1071 sizeof(struct pass_io_req), NULL, NULL, NULL, NULL,
1072 /*align*/ 0, /*flags*/ 0);
1073
1074 softc->pass_io_zone = uma_zcreate(softc->io_zone_name,
1075 softc->io_zone_size, NULL, NULL, NULL, NULL,
1076 /*align*/ 0, /*flags*/ 0);
1077
1078 cam_periph_lock(periph);
1079
1080 if ((softc->pass_zone == NULL)
1081 || (softc->pass_io_zone == NULL)) {
1082 if (softc->pass_zone == NULL)
1083 xpt_print(periph->path, "unable to allocate "
1084 "IO Req UMA zone\n");
1085 else
1086 xpt_print(periph->path, "unable to allocate "
1087 "IO UMA zone\n");
1088 softc->flags &= ~PASS_FLAG_ZONE_INPROG;
1089 goto bailout;
1090 }
1091
1092 /*
1093 * Set the flags appropriately and notify any other waiters.
1094 */
1095 softc->flags &= ~PASS_FLAG_ZONE_INPROG;
1096 softc->flags |= PASS_FLAG_ZONE_VALID;
1097 wakeup(&softc->pass_zone);
1098 } else {
1099 /*
1100 * In this case, the UMA zone has not yet been created, but
1101 * another context is in the process of creating it. We
1102 * need to sleep until the creation is either done or has
1103 * failed.
1104 */
1105 while ((softc->flags & PASS_FLAG_ZONE_INPROG)
1106 && ((softc->flags & PASS_FLAG_ZONE_VALID) == 0)) {
1107 error = msleep(&softc->pass_zone,
1108 cam_periph_mtx(periph), PRIBIO,
1109 "paszon", 0);
1110 if (error != 0)
1111 goto bailout;
1112 }
1113 /*
1114 * If the zone creation failed, no luck for the user.
1115 */
1116 if ((softc->flags & PASS_FLAG_ZONE_VALID) == 0){
1117 error = ENOMEM;
1118 goto bailout;
1119 }
1120 }
1121 bailout:
1122 return (error);
1123 }
1124
1125 static void
passiocleanup(struct pass_softc * softc,struct pass_io_req * io_req)1126 passiocleanup(struct pass_softc *softc, struct pass_io_req *io_req)
1127 {
1128 union ccb *ccb;
1129 struct ccb_hdr *hdr;
1130 uint8_t **data_ptrs[CAM_PERIPH_MAXMAPS];
1131 int i, numbufs;
1132
1133 ccb = &io_req->ccb;
1134 hdr = &ccb->ccb_h;
1135
1136 switch (hdr->func_code) {
1137 case XPT_DEV_MATCH:
1138 numbufs = min(io_req->num_bufs, 2);
1139
1140 if (numbufs == 1) {
1141 data_ptrs[0] = (uint8_t **)&ccb->cdm.matches;
1142 } else {
1143 data_ptrs[0] = (uint8_t **)&ccb->cdm.patterns;
1144 data_ptrs[1] = (uint8_t **)&ccb->cdm.matches;
1145 }
1146 break;
1147 case XPT_SCSI_IO:
1148 case XPT_CONT_TARGET_IO:
1149 data_ptrs[0] = &ccb->csio.data_ptr;
1150 numbufs = min(io_req->num_bufs, 1);
1151 break;
1152 case XPT_ATA_IO:
1153 data_ptrs[0] = &ccb->ataio.data_ptr;
1154 numbufs = min(io_req->num_bufs, 1);
1155 break;
1156 case XPT_SMP_IO:
1157 numbufs = min(io_req->num_bufs, 2);
1158 data_ptrs[0] = &ccb->smpio.smp_request;
1159 data_ptrs[1] = &ccb->smpio.smp_response;
1160 break;
1161 case XPT_DEV_ADVINFO:
1162 numbufs = min(io_req->num_bufs, 1);
1163 data_ptrs[0] = (uint8_t **)&ccb->cdai.buf;
1164 break;
1165 case XPT_NVME_IO:
1166 case XPT_NVME_ADMIN:
1167 data_ptrs[0] = &ccb->nvmeio.data_ptr;
1168 numbufs = min(io_req->num_bufs, 1);
1169 break;
1170 default:
1171 /* allow ourselves to be swapped once again */
1172 return;
1173 break; /* NOTREACHED */
1174 }
1175
1176 if (io_req->flags & PASS_IO_USER_SEG_MALLOC) {
1177 free(io_req->user_segptr, M_SCSIPASS);
1178 io_req->user_segptr = NULL;
1179 }
1180
1181 /*
1182 * We only want to free memory we malloced.
1183 */
1184 if (io_req->data_flags == CAM_DATA_VADDR) {
1185 for (i = 0; i < io_req->num_bufs; i++) {
1186 if (io_req->kern_bufs[i] == NULL)
1187 continue;
1188
1189 free(io_req->kern_bufs[i], M_SCSIPASS);
1190 io_req->kern_bufs[i] = NULL;
1191 }
1192 } else if (io_req->data_flags == CAM_DATA_SG) {
1193 for (i = 0; i < io_req->num_kern_segs; i++) {
1194 if ((uint8_t *)(uintptr_t)
1195 io_req->kern_segptr[i].ds_addr == NULL)
1196 continue;
1197
1198 uma_zfree(softc->pass_io_zone, (uint8_t *)(uintptr_t)
1199 io_req->kern_segptr[i].ds_addr);
1200 io_req->kern_segptr[i].ds_addr = 0;
1201 }
1202 }
1203
1204 if (io_req->flags & PASS_IO_KERN_SEG_MALLOC) {
1205 free(io_req->kern_segptr, M_SCSIPASS);
1206 io_req->kern_segptr = NULL;
1207 }
1208
1209 if (io_req->data_flags != CAM_DATA_PADDR) {
1210 for (i = 0; i < numbufs; i++) {
1211 /*
1212 * Restore the user's buffer pointers to their
1213 * previous values.
1214 */
1215 if (io_req->user_bufs[i] != NULL)
1216 *data_ptrs[i] = io_req->user_bufs[i];
1217 }
1218 }
1219
1220 }
1221
1222 static int
passcopysglist(struct cam_periph * periph,struct pass_io_req * io_req,ccb_flags direction)1223 passcopysglist(struct cam_periph *periph, struct pass_io_req *io_req,
1224 ccb_flags direction)
1225 {
1226 bus_size_t kern_watermark, user_watermark, len_to_copy;
1227 bus_dma_segment_t *user_sglist, *kern_sglist;
1228 int i, j, error;
1229
1230 error = 0;
1231 kern_watermark = 0;
1232 user_watermark = 0;
1233 len_to_copy = 0;
1234 user_sglist = io_req->user_segptr;
1235 kern_sglist = io_req->kern_segptr;
1236
1237 for (i = 0, j = 0; i < io_req->num_user_segs &&
1238 j < io_req->num_kern_segs;) {
1239 uint8_t *user_ptr, *kern_ptr;
1240
1241 len_to_copy = min(user_sglist[i].ds_len -user_watermark,
1242 kern_sglist[j].ds_len - kern_watermark);
1243
1244 user_ptr = (uint8_t *)(uintptr_t)user_sglist[i].ds_addr;
1245 user_ptr = user_ptr + user_watermark;
1246 kern_ptr = (uint8_t *)(uintptr_t)kern_sglist[j].ds_addr;
1247 kern_ptr = kern_ptr + kern_watermark;
1248
1249 user_watermark += len_to_copy;
1250 kern_watermark += len_to_copy;
1251
1252 if (direction == CAM_DIR_IN) {
1253 error = copyout(kern_ptr, user_ptr, len_to_copy);
1254 if (error != 0) {
1255 xpt_print(periph->path, "%s: copyout of %u "
1256 "bytes from %p to %p failed with "
1257 "error %d\n", __func__, len_to_copy,
1258 kern_ptr, user_ptr, error);
1259 goto bailout;
1260 }
1261 } else {
1262 error = copyin(user_ptr, kern_ptr, len_to_copy);
1263 if (error != 0) {
1264 xpt_print(periph->path, "%s: copyin of %u "
1265 "bytes from %p to %p failed with "
1266 "error %d\n", __func__, len_to_copy,
1267 user_ptr, kern_ptr, error);
1268 goto bailout;
1269 }
1270 }
1271
1272 if (user_sglist[i].ds_len == user_watermark) {
1273 i++;
1274 user_watermark = 0;
1275 }
1276
1277 if (kern_sglist[j].ds_len == kern_watermark) {
1278 j++;
1279 kern_watermark = 0;
1280 }
1281 }
1282
1283 bailout:
1284
1285 return (error);
1286 }
1287
1288 static int
passmemsetup(struct cam_periph * periph,struct pass_io_req * io_req)1289 passmemsetup(struct cam_periph *periph, struct pass_io_req *io_req)
1290 {
1291 union ccb *ccb;
1292 struct ccb_hdr *hdr;
1293 struct pass_softc *softc;
1294 int numbufs, i;
1295 uint8_t **data_ptrs[CAM_PERIPH_MAXMAPS];
1296 uint32_t lengths[CAM_PERIPH_MAXMAPS];
1297 uint32_t dirs[CAM_PERIPH_MAXMAPS];
1298 uint32_t num_segs;
1299 uint16_t *seg_cnt_ptr;
1300 size_t maxmap;
1301 int error;
1302
1303 cam_periph_assert(periph, MA_NOTOWNED);
1304
1305 softc = periph->softc;
1306
1307 error = 0;
1308 ccb = &io_req->ccb;
1309 hdr = &ccb->ccb_h;
1310 maxmap = 0;
1311 num_segs = 0;
1312 seg_cnt_ptr = NULL;
1313
1314 switch(hdr->func_code) {
1315 case XPT_DEV_MATCH:
1316 if (ccb->cdm.match_buf_len == 0) {
1317 printf("%s: invalid match buffer length 0\n", __func__);
1318 return(EINVAL);
1319 }
1320 if (ccb->cdm.pattern_buf_len > 0) {
1321 data_ptrs[0] = (uint8_t **)&ccb->cdm.patterns;
1322 lengths[0] = ccb->cdm.pattern_buf_len;
1323 dirs[0] = CAM_DIR_OUT;
1324 data_ptrs[1] = (uint8_t **)&ccb->cdm.matches;
1325 lengths[1] = ccb->cdm.match_buf_len;
1326 dirs[1] = CAM_DIR_IN;
1327 numbufs = 2;
1328 } else {
1329 data_ptrs[0] = (uint8_t **)&ccb->cdm.matches;
1330 lengths[0] = ccb->cdm.match_buf_len;
1331 dirs[0] = CAM_DIR_IN;
1332 numbufs = 1;
1333 }
1334 io_req->data_flags = CAM_DATA_VADDR;
1335 break;
1336 case XPT_SCSI_IO:
1337 case XPT_CONT_TARGET_IO:
1338 if ((hdr->flags & CAM_DIR_MASK) == CAM_DIR_NONE)
1339 return(0);
1340
1341 /*
1342 * The user shouldn't be able to supply a bio.
1343 */
1344 if ((hdr->flags & CAM_DATA_MASK) == CAM_DATA_BIO)
1345 return (EINVAL);
1346
1347 io_req->data_flags = hdr->flags & CAM_DATA_MASK;
1348
1349 data_ptrs[0] = &ccb->csio.data_ptr;
1350 lengths[0] = ccb->csio.dxfer_len;
1351 dirs[0] = hdr->flags & CAM_DIR_MASK;
1352 num_segs = ccb->csio.sglist_cnt;
1353 seg_cnt_ptr = &ccb->csio.sglist_cnt;
1354 numbufs = 1;
1355 maxmap = softc->maxio;
1356 break;
1357 case XPT_ATA_IO:
1358 if ((hdr->flags & CAM_DIR_MASK) == CAM_DIR_NONE)
1359 return(0);
1360
1361 /*
1362 * We only support a single virtual address for ATA I/O.
1363 */
1364 if ((hdr->flags & CAM_DATA_MASK) != CAM_DATA_VADDR)
1365 return (EINVAL);
1366
1367 io_req->data_flags = CAM_DATA_VADDR;
1368
1369 data_ptrs[0] = &ccb->ataio.data_ptr;
1370 lengths[0] = ccb->ataio.dxfer_len;
1371 dirs[0] = hdr->flags & CAM_DIR_MASK;
1372 numbufs = 1;
1373 maxmap = softc->maxio;
1374 break;
1375 case XPT_SMP_IO:
1376 io_req->data_flags = CAM_DATA_VADDR;
1377
1378 data_ptrs[0] = &ccb->smpio.smp_request;
1379 lengths[0] = ccb->smpio.smp_request_len;
1380 dirs[0] = CAM_DIR_OUT;
1381 data_ptrs[1] = &ccb->smpio.smp_response;
1382 lengths[1] = ccb->smpio.smp_response_len;
1383 dirs[1] = CAM_DIR_IN;
1384 numbufs = 2;
1385 maxmap = softc->maxio;
1386 break;
1387 case XPT_DEV_ADVINFO:
1388 if (ccb->cdai.bufsiz == 0)
1389 return (0);
1390
1391 io_req->data_flags = CAM_DATA_VADDR;
1392
1393 data_ptrs[0] = (uint8_t **)&ccb->cdai.buf;
1394 lengths[0] = ccb->cdai.bufsiz;
1395 dirs[0] = CAM_DIR_IN;
1396 numbufs = 1;
1397 break;
1398 case XPT_NVME_ADMIN:
1399 case XPT_NVME_IO:
1400 if ((hdr->flags & CAM_DIR_MASK) == CAM_DIR_NONE)
1401 return (0);
1402
1403 io_req->data_flags = hdr->flags & CAM_DATA_MASK;
1404
1405 data_ptrs[0] = &ccb->nvmeio.data_ptr;
1406 lengths[0] = ccb->nvmeio.dxfer_len;
1407 dirs[0] = hdr->flags & CAM_DIR_MASK;
1408 num_segs = ccb->nvmeio.sglist_cnt;
1409 seg_cnt_ptr = &ccb->nvmeio.sglist_cnt;
1410 numbufs = 1;
1411 maxmap = softc->maxio;
1412 break;
1413 default:
1414 return(EINVAL);
1415 break; /* NOTREACHED */
1416 }
1417
1418 io_req->num_bufs = numbufs;
1419
1420 /*
1421 * If there is a maximum, check to make sure that the user's
1422 * request fits within the limit. In general, we should only have
1423 * a maximum length for requests that go to hardware. Otherwise it
1424 * is whatever we're able to malloc.
1425 */
1426 for (i = 0; i < numbufs; i++) {
1427 io_req->user_bufs[i] = *data_ptrs[i];
1428 io_req->dirs[i] = dirs[i];
1429 io_req->lengths[i] = lengths[i];
1430
1431 if (maxmap == 0)
1432 continue;
1433
1434 if (lengths[i] <= maxmap)
1435 continue;
1436
1437 xpt_print(periph->path, "%s: data length %u > max allowed %u "
1438 "bytes\n", __func__, lengths[i], maxmap);
1439 error = EINVAL;
1440 goto bailout;
1441 }
1442
1443 switch (io_req->data_flags) {
1444 case CAM_DATA_VADDR:
1445 /* Map or copy the buffer into kernel address space */
1446 for (i = 0; i < numbufs; i++) {
1447 uint8_t *tmp_buf;
1448
1449 /*
1450 * If for some reason no length is specified, we
1451 * don't need to allocate anything.
1452 */
1453 if (io_req->lengths[i] == 0)
1454 continue;
1455
1456 tmp_buf = malloc(lengths[i], M_SCSIPASS,
1457 M_WAITOK | M_ZERO);
1458 io_req->kern_bufs[i] = tmp_buf;
1459 *data_ptrs[i] = tmp_buf;
1460
1461 #if 0
1462 xpt_print(periph->path, "%s: malloced %p len %u, user "
1463 "buffer %p, operation: %s\n", __func__,
1464 tmp_buf, lengths[i], io_req->user_bufs[i],
1465 (dirs[i] == CAM_DIR_IN) ? "read" : "write");
1466 #endif
1467 /*
1468 * We only need to copy in if the user is writing.
1469 */
1470 if (dirs[i] != CAM_DIR_OUT)
1471 continue;
1472
1473 error = copyin(io_req->user_bufs[i],
1474 io_req->kern_bufs[i], lengths[i]);
1475 if (error != 0) {
1476 xpt_print(periph->path, "%s: copy of user "
1477 "buffer from %p to %p failed with "
1478 "error %d\n", __func__,
1479 io_req->user_bufs[i],
1480 io_req->kern_bufs[i], error);
1481 goto bailout;
1482 }
1483 }
1484 break;
1485 case CAM_DATA_PADDR:
1486 #ifdef PASS_UNSAFE_PADDR
1487 /* Pass down the pointer as-is */
1488 break;
1489 #else
1490 /*
1491 * Physical addresses from userspace are not allowed.
1492 * They could be used for arbitrary DMA to/from host
1493 * memory on systems without an IOMMU.
1494 */
1495 error = EINVAL;
1496 goto bailout;
1497 #endif
1498 case CAM_DATA_SG: {
1499 size_t sg_length, size_to_go, alloc_size;
1500 uint32_t num_segs_needed;
1501
1502 /*
1503 * Copy the user S/G list in, and then copy in the
1504 * individual segments.
1505 */
1506 /*
1507 * We shouldn't see this, but check just in case.
1508 */
1509 if (numbufs != 1) {
1510 xpt_print(periph->path, "%s: cannot currently handle "
1511 "more than one S/G list per CCB\n", __func__);
1512 error = EINVAL;
1513 goto bailout;
1514 }
1515
1516 /*
1517 * We have to have at least one segment.
1518 */
1519 if (num_segs == 0) {
1520 xpt_print(periph->path, "%s: CAM_DATA_SG flag set, "
1521 "but sglist_cnt=0!\n", __func__);
1522 error = EINVAL;
1523 goto bailout;
1524 }
1525
1526 /*
1527 * Make sure the user specified the total length and didn't
1528 * just leave it to us to decode the S/G list.
1529 */
1530 if (lengths[0] == 0) {
1531 xpt_print(periph->path, "%s: no dxfer_len specified, "
1532 "but CAM_DATA_SG flag is set!\n", __func__);
1533 error = EINVAL;
1534 goto bailout;
1535 }
1536
1537 /*
1538 * We allocate buffers in io_zone_size increments for an
1539 * S/G list. This will generally be maxphys.
1540 */
1541 if (lengths[0] <= softc->io_zone_size)
1542 num_segs_needed = 1;
1543 else {
1544 num_segs_needed = lengths[0] / softc->io_zone_size;
1545 if ((lengths[0] % softc->io_zone_size) != 0)
1546 num_segs_needed++;
1547 }
1548
1549 /* Figure out the size of the S/G list */
1550 sg_length = num_segs * sizeof(bus_dma_segment_t);
1551 io_req->num_user_segs = num_segs;
1552 io_req->num_kern_segs = num_segs_needed;
1553
1554 /* Save the user's S/G list pointer for later restoration */
1555 io_req->user_bufs[0] = *data_ptrs[0];
1556
1557 /*
1558 * If we have enough segments allocated by default to handle
1559 * the length of the user's S/G list,
1560 */
1561 if (num_segs > PASS_MAX_SEGS) {
1562 io_req->user_segptr = malloc(sizeof(bus_dma_segment_t) *
1563 num_segs, M_SCSIPASS, M_WAITOK | M_ZERO);
1564 io_req->flags |= PASS_IO_USER_SEG_MALLOC;
1565 } else
1566 io_req->user_segptr = io_req->user_segs;
1567
1568 error = copyin(*data_ptrs[0], io_req->user_segptr, sg_length);
1569 if (error != 0) {
1570 xpt_print(periph->path, "%s: copy of user S/G list "
1571 "from %p to %p failed with error %d\n",
1572 __func__, *data_ptrs[0], io_req->user_segptr,
1573 error);
1574 goto bailout;
1575 }
1576
1577 if (num_segs_needed > PASS_MAX_SEGS) {
1578 io_req->kern_segptr = malloc(sizeof(bus_dma_segment_t) *
1579 num_segs_needed, M_SCSIPASS, M_WAITOK | M_ZERO);
1580 io_req->flags |= PASS_IO_KERN_SEG_MALLOC;
1581 } else {
1582 io_req->kern_segptr = io_req->kern_segs;
1583 }
1584
1585 /*
1586 * Allocate the kernel S/G list.
1587 */
1588 for (size_to_go = lengths[0], i = 0;
1589 size_to_go > 0 && i < num_segs_needed;
1590 i++, size_to_go -= alloc_size) {
1591 uint8_t *kern_ptr;
1592
1593 alloc_size = min(size_to_go, softc->io_zone_size);
1594 kern_ptr = uma_zalloc(softc->pass_io_zone, M_WAITOK);
1595 io_req->kern_segptr[i].ds_addr =
1596 (bus_addr_t)(uintptr_t)kern_ptr;
1597 io_req->kern_segptr[i].ds_len = alloc_size;
1598 }
1599 if (size_to_go > 0) {
1600 printf("%s: size_to_go = %zu, software error!\n",
1601 __func__, size_to_go);
1602 error = EINVAL;
1603 goto bailout;
1604 }
1605
1606 *data_ptrs[0] = (uint8_t *)io_req->kern_segptr;
1607 *seg_cnt_ptr = io_req->num_kern_segs;
1608
1609 /*
1610 * We only need to copy data here if the user is writing.
1611 */
1612 if (dirs[0] == CAM_DIR_OUT)
1613 error = passcopysglist(periph, io_req, dirs[0]);
1614 break;
1615 }
1616 case CAM_DATA_SG_PADDR: {
1617 #ifdef PASS_UNSAFE_PADDR
1618 size_t sg_length;
1619
1620 /*
1621 * We shouldn't see this, but check just in case.
1622 */
1623 if (numbufs != 1) {
1624 printf("%s: cannot currently handle more than one "
1625 "S/G list per CCB\n", __func__);
1626 error = EINVAL;
1627 goto bailout;
1628 }
1629
1630 /*
1631 * We have to have at least one segment.
1632 */
1633 if (num_segs == 0) {
1634 xpt_print(periph->path, "%s: CAM_DATA_SG_PADDR flag "
1635 "set, but sglist_cnt=0!\n", __func__);
1636 error = EINVAL;
1637 goto bailout;
1638 }
1639
1640 /*
1641 * Make sure the user specified the total length and didn't
1642 * just leave it to us to decode the S/G list.
1643 */
1644 if (lengths[0] == 0) {
1645 xpt_print(periph->path, "%s: no dxfer_len specified, "
1646 "but CAM_DATA_SG flag is set!\n", __func__);
1647 error = EINVAL;
1648 goto bailout;
1649 }
1650
1651 /* Figure out the size of the S/G list */
1652 sg_length = num_segs * sizeof(bus_dma_segment_t);
1653 io_req->num_user_segs = num_segs;
1654 io_req->num_kern_segs = io_req->num_user_segs;
1655
1656 /* Save the user's S/G list pointer for later restoration */
1657 io_req->user_bufs[0] = *data_ptrs[0];
1658
1659 if (num_segs > PASS_MAX_SEGS) {
1660 io_req->user_segptr = malloc(sizeof(bus_dma_segment_t) *
1661 num_segs, M_SCSIPASS, M_WAITOK | M_ZERO);
1662 io_req->flags |= PASS_IO_USER_SEG_MALLOC;
1663 } else
1664 io_req->user_segptr = io_req->user_segs;
1665
1666 io_req->kern_segptr = io_req->user_segptr;
1667
1668 error = copyin(*data_ptrs[0], io_req->user_segptr, sg_length);
1669 if (error != 0) {
1670 xpt_print(periph->path, "%s: copy of user S/G list "
1671 "from %p to %p failed with error %d\n",
1672 __func__, *data_ptrs[0], io_req->user_segptr,
1673 error);
1674 goto bailout;
1675 }
1676 break;
1677 #else
1678 /*
1679 * Physical addresses from userspace are not allowed.
1680 * They could be used for arbitrary DMA to/from host
1681 * memory on systems without an IOMMU.
1682 */
1683 error = EINVAL;
1684 goto bailout;
1685 #endif
1686 }
1687 default:
1688 case CAM_DATA_BIO:
1689 /*
1690 * A user shouldn't be attaching a bio to the CCB. It
1691 * isn't a user-accessible structure.
1692 */
1693 error = EINVAL;
1694 break;
1695 }
1696
1697 bailout:
1698 if (error != 0)
1699 passiocleanup(softc, io_req);
1700
1701 return (error);
1702 }
1703
1704 static int
passmemdone(struct cam_periph * periph,struct pass_io_req * io_req)1705 passmemdone(struct cam_periph *periph, struct pass_io_req *io_req)
1706 {
1707 struct pass_softc *softc;
1708 int error;
1709 int i;
1710
1711 error = 0;
1712 softc = (struct pass_softc *)periph->softc;
1713
1714 switch (io_req->data_flags) {
1715 case CAM_DATA_VADDR:
1716 /*
1717 * Copy back to the user buffer if this was a read.
1718 */
1719 for (i = 0; i < io_req->num_bufs; i++) {
1720 if (io_req->dirs[i] != CAM_DIR_IN)
1721 continue;
1722
1723 error = copyout(io_req->kern_bufs[i],
1724 io_req->user_bufs[i], io_req->lengths[i]);
1725 if (error != 0) {
1726 xpt_print(periph->path, "Unable to copy %u "
1727 "bytes from %p to user address %p\n",
1728 io_req->lengths[i],
1729 io_req->kern_bufs[i],
1730 io_req->user_bufs[i]);
1731 goto bailout;
1732 }
1733 }
1734 break;
1735 case CAM_DATA_PADDR:
1736 /* Do nothing. The pointer is a physical address already */
1737 break;
1738 case CAM_DATA_SG:
1739 /*
1740 * Copy back to the user buffer if this was a read.
1741 * Restore the user's S/G list buffer pointer.
1742 */
1743 if (io_req->dirs[0] == CAM_DIR_IN)
1744 error = passcopysglist(periph, io_req, io_req->dirs[0]);
1745 break;
1746 case CAM_DATA_SG_PADDR:
1747 /*
1748 * Restore the user's S/G list buffer pointer. No need to
1749 * copy.
1750 */
1751 break;
1752 default:
1753 case CAM_DATA_BIO:
1754 error = EINVAL;
1755 break;
1756 }
1757
1758 bailout:
1759 /*
1760 * Reset the user's pointers to their original values and free
1761 * allocated memory.
1762 */
1763 passiocleanup(softc, io_req);
1764
1765 return (error);
1766 }
1767
1768 static int
passioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flag,struct thread * td)1769 passioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
1770 {
1771 int error;
1772
1773 if ((error = passdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
1774 error = cam_compat_ioctl(dev, cmd, addr, flag, td, passdoioctl);
1775 }
1776 return (error);
1777 }
1778
1779 /*
1780 * Allowlist of CCB function codes that userland is permitted to send
1781 * through the pass(4) driver. This is a security boundary: anything
1782 * not on this list could allow userland to corrupt kernel state.
1783 * Notable exclusions:
1784 * - XPT_ABORT: payload contains a kernel CCB pointer that xpt_action
1785 * dereferences without validation.
1786 * - XPT_SASYNC_CB: payload contains a function pointer that gets
1787 * registered as a kernel callback.
1788 * - Target mode CCBs (XPT_EN_LUN, XPT_TARGET_IO, etc.): should only
1789 * be reachable through the target mode driver.
1790 */
1791 static int
pass_is_allowed_fc(xpt_opcode fc)1792 pass_is_allowed_fc(xpt_opcode fc)
1793 {
1794 switch (fc) {
1795 /* I/O operations */
1796 case XPT_SCSI_IO:
1797 case XPT_ATA_IO:
1798 case XPT_SMP_IO:
1799 case XPT_NVME_IO:
1800 case XPT_NVME_ADMIN:
1801 case XPT_MMC_IO:
1802 /* Device info / queries */
1803 case XPT_GDEV_TYPE:
1804 case XPT_GDEVLIST:
1805 case XPT_PATH_INQ:
1806 case XPT_GDEV_STATS:
1807 case XPT_PATH_STATS:
1808 case XPT_DEV_ADVINFO:
1809 case XPT_GET_TRAN_SETTINGS:
1810 case XPT_SET_TRAN_SETTINGS:
1811 case XPT_GET_SIM_KNOB:
1812 case XPT_SET_SIM_KNOB:
1813 case XPT_MMC_GET_TRAN_SETTINGS:
1814 case XPT_MMC_SET_TRAN_SETTINGS:
1815 case XPT_CALC_GEOMETRY:
1816 /* Misc safe operations */
1817 case XPT_NOOP:
1818 case XPT_REL_SIMQ:
1819 case XPT_RESET_DEV:
1820 case XPT_DEBUG:
1821 return (1);
1822 default:
1823 return (0);
1824 }
1825 }
1826
1827 static int
passdoioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flag,struct thread * td)1828 passdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
1829 {
1830 struct cam_periph *periph;
1831 struct pass_softc *softc;
1832 int error;
1833 uint32_t priority;
1834
1835 periph = (struct cam_periph *)dev->si_drv1;
1836 cam_periph_lock(periph);
1837 softc = (struct pass_softc *)periph->softc;
1838
1839 error = 0;
1840
1841 switch (cmd) {
1842 case CAMIOCOMMAND:
1843 {
1844 union ccb *inccb;
1845 union ccb *ccb;
1846 int ccb_malloced;
1847
1848 inccb = (union ccb *)addr;
1849 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
1850 if (inccb->ccb_h.func_code == XPT_SCSI_IO)
1851 inccb->csio.bio = NULL;
1852 #endif
1853
1854 if (inccb->ccb_h.flags & CAM_UNLOCKED) {
1855 error = EINVAL;
1856 break;
1857 }
1858
1859 /*
1860 * Only allow CCB function codes that are known to be safe
1861 * for userland to issue through the pass(4) driver.
1862 */
1863 if (!pass_is_allowed_fc(inccb->ccb_h.func_code)) {
1864 xpt_print(periph->path, "CCB function code %#x is "
1865 "not supported by the pass(4) driver\n",
1866 inccb->ccb_h.func_code);
1867 error = EINVAL;
1868 break;
1869 }
1870
1871 /* Compatibility for RL/priority-unaware code. */
1872 priority = inccb->ccb_h.pinfo.priority;
1873 if (priority <= CAM_PRIORITY_OOB)
1874 priority += CAM_PRIORITY_OOB + 1;
1875
1876 /*
1877 * Non-immediate CCBs need a CCB from the per-device pool
1878 * of CCBs, which is scheduled by the transport layer.
1879 * Immediate CCBs and user-supplied CCBs should just be
1880 * malloced.
1881 */
1882 if ((inccb->ccb_h.func_code & XPT_FC_QUEUED)
1883 && ((inccb->ccb_h.func_code & XPT_FC_USER_CCB) == 0)) {
1884 ccb = cam_periph_getccb(periph, priority);
1885 ccb_malloced = 0;
1886 } else {
1887 ccb = xpt_alloc_ccb_nowait();
1888
1889 if (ccb != NULL)
1890 xpt_setup_ccb(&ccb->ccb_h, periph->path,
1891 priority);
1892 ccb_malloced = 1;
1893 }
1894
1895 if (ccb == NULL) {
1896 xpt_print(periph->path, "unable to allocate CCB\n");
1897 error = ENOMEM;
1898 break;
1899 }
1900
1901 error = passsendccb(periph, ccb, inccb);
1902
1903 if (ccb_malloced)
1904 xpt_free_ccb(ccb);
1905 else
1906 xpt_release_ccb(ccb);
1907
1908 break;
1909 }
1910 case CAMIOQUEUE:
1911 {
1912 struct pass_io_req *io_req;
1913 union ccb **user_ccb, *ccb;
1914 xpt_opcode fc;
1915
1916 #ifdef COMPAT_FREEBSD32
1917 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
1918 error = ENOTTY;
1919 goto bailout;
1920 }
1921 #endif
1922 if ((softc->flags & PASS_FLAG_ZONE_VALID) == 0) {
1923 error = passcreatezone(periph);
1924 if (error != 0)
1925 goto bailout;
1926 }
1927
1928 /*
1929 * We're going to do a blocking allocation for this I/O
1930 * request, so we have to drop the lock.
1931 */
1932 cam_periph_unlock(periph);
1933
1934 io_req = uma_zalloc(softc->pass_zone, M_WAITOK | M_ZERO);
1935 ccb = &io_req->ccb;
1936 user_ccb = (union ccb **)addr;
1937
1938 /*
1939 * Unlike the CAMIOCOMMAND ioctl above, we only have a
1940 * pointer to the user's CCB, so we have to copy the whole
1941 * thing in to a buffer we have allocated (above) instead
1942 * of allowing the ioctl code to malloc a buffer and copy
1943 * it in.
1944 *
1945 * This is an advantage for this asynchronous interface,
1946 * since we don't want the memory to get freed while the
1947 * CCB is outstanding.
1948 */
1949 #if 0
1950 xpt_print(periph->path, "Copying user CCB %p to "
1951 "kernel address %p\n", *user_ccb, ccb);
1952 #endif
1953 error = copyin(*user_ccb, ccb, sizeof(*ccb));
1954 if (error != 0) {
1955 xpt_print(periph->path, "Copy of user CCB %p to "
1956 "kernel address %p failed with error %d\n",
1957 *user_ccb, ccb, error);
1958 goto camioqueue_error;
1959 }
1960 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
1961 if (ccb->ccb_h.func_code == XPT_SCSI_IO)
1962 ccb->csio.bio = NULL;
1963 #endif
1964
1965 if (ccb->ccb_h.flags & CAM_UNLOCKED) {
1966 error = EINVAL;
1967 goto camioqueue_error;
1968 }
1969
1970 if (ccb->ccb_h.flags & CAM_CDB_POINTER) {
1971 if (ccb->csio.cdb_len > IOCDBLEN) {
1972 error = EINVAL;
1973 goto camioqueue_error;
1974 }
1975 error = copyin(ccb->csio.cdb_io.cdb_ptr,
1976 ccb->csio.cdb_io.cdb_bytes, ccb->csio.cdb_len);
1977 if (error != 0)
1978 goto camioqueue_error;
1979 ccb->ccb_h.flags &= ~CAM_CDB_POINTER;
1980 }
1981
1982 /*
1983 * Only allow CCB function codes that are known to be safe
1984 * for userland to issue through the pass(4) driver.
1985 */
1986 if (!pass_is_allowed_fc(ccb->ccb_h.func_code)) {
1987 xpt_print(periph->path, "CCB function code %#x is "
1988 "not supported by the pass(4) driver\n",
1989 ccb->ccb_h.func_code);
1990 error = EINVAL;
1991 goto camioqueue_error;
1992 }
1993
1994 /*
1995 * Save the user's CCB pointer as well as his linked list
1996 * pointers and peripheral private area so that we can
1997 * restore these later.
1998 */
1999 io_req->user_ccb_ptr = *user_ccb;
2000 io_req->user_periph_links = ccb->ccb_h.periph_links;
2001 io_req->user_periph_priv = ccb->ccb_h.periph_priv;
2002
2003 /*
2004 * Now that we've saved the user's values, we can set our
2005 * own peripheral private entry.
2006 */
2007 ccb->ccb_h.ccb_ioreq = io_req;
2008
2009 /* Compatibility for RL/priority-unaware code. */
2010 priority = ccb->ccb_h.pinfo.priority;
2011 if (priority <= CAM_PRIORITY_OOB)
2012 priority += CAM_PRIORITY_OOB + 1;
2013
2014 /*
2015 * Setup fields in the CCB like the path and the priority.
2016 * The path in particular cannot be done in userland, since
2017 * it is a pointer to a kernel data structure.
2018 */
2019 xpt_setup_ccb_flags(&ccb->ccb_h, periph->path, priority,
2020 ccb->ccb_h.flags);
2021
2022 /*
2023 * Setup our done routine. There is no way for the user to
2024 * have a valid pointer here.
2025 */
2026 ccb->ccb_h.cbfcnp = passdone;
2027
2028 fc = ccb->ccb_h.func_code;
2029 /*
2030 * If this function code has memory that can be mapped in
2031 * or out, we need to call passmemsetup().
2032 */
2033 if ((fc == XPT_SCSI_IO) || (fc == XPT_ATA_IO)
2034 || (fc == XPT_SMP_IO) || (fc == XPT_DEV_MATCH)
2035 || (fc == XPT_DEV_ADVINFO)
2036 || (fc == XPT_NVME_ADMIN) || (fc == XPT_NVME_IO)) {
2037 error = passmemsetup(periph, io_req);
2038 if (error != 0)
2039 goto camioqueue_error;
2040 } else
2041 io_req->mapinfo.num_bufs_used = 0;
2042
2043 cam_periph_lock(periph);
2044
2045 /*
2046 * Everything goes on the incoming queue initially.
2047 */
2048 TAILQ_INSERT_TAIL(&softc->incoming_queue, io_req, links);
2049
2050 /*
2051 * If the CCB is queued, and is not a user CCB, then
2052 * we need to allocate a slot for it. Call xpt_schedule()
2053 * so that our start routine will get called when a CCB is
2054 * available.
2055 */
2056 if ((fc & XPT_FC_QUEUED)
2057 && ((fc & XPT_FC_USER_CCB) == 0)) {
2058 xpt_schedule(periph, priority);
2059 break;
2060 }
2061
2062 /*
2063 * At this point, the CCB in question is either an
2064 * immediate CCB (like XPT_DEV_ADVINFO) or it is a user CCB
2065 * and therefore should be malloced, not allocated via a slot.
2066 * Remove the CCB from the incoming queue and add it to the
2067 * active queue.
2068 */
2069 TAILQ_REMOVE(&softc->incoming_queue, io_req, links);
2070 TAILQ_INSERT_TAIL(&softc->active_queue, io_req, links);
2071
2072 xpt_action(ccb);
2073
2074 /*
2075 * If this is not a queued CCB (i.e. it is an immediate CCB),
2076 * then it is already done. We need to put it on the done
2077 * queue for the user to fetch.
2078 */
2079 if ((fc & XPT_FC_QUEUED) == 0) {
2080 TAILQ_REMOVE(&softc->active_queue, io_req, links);
2081 TAILQ_INSERT_TAIL(&softc->done_queue, io_req, links);
2082 }
2083 break;
2084
2085 camioqueue_error:
2086 uma_zfree(softc->pass_zone, io_req);
2087 cam_periph_lock(periph);
2088 break;
2089 }
2090 case CAMIOGET:
2091 {
2092 union ccb **user_ccb;
2093 struct pass_io_req *io_req;
2094 int old_error;
2095
2096 #ifdef COMPAT_FREEBSD32
2097 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
2098 error = ENOTTY;
2099 goto bailout;
2100 }
2101 #endif
2102 user_ccb = (union ccb **)addr;
2103 old_error = 0;
2104
2105 io_req = TAILQ_FIRST(&softc->done_queue);
2106 if (io_req == NULL) {
2107 error = ENOENT;
2108 break;
2109 }
2110
2111 /*
2112 * Remove the I/O from the done queue.
2113 */
2114 TAILQ_REMOVE(&softc->done_queue, io_req, links);
2115
2116 /*
2117 * We have to drop the lock during the copyout because the
2118 * copyout can result in VM faults that require sleeping.
2119 */
2120 cam_periph_unlock(periph);
2121
2122 /*
2123 * Do any needed copies (e.g. for reads) and revert the
2124 * pointers in the CCB back to the user's pointers.
2125 */
2126 error = passmemdone(periph, io_req);
2127
2128 old_error = error;
2129
2130 io_req->ccb.ccb_h.periph_links = io_req->user_periph_links;
2131 io_req->ccb.ccb_h.periph_priv = io_req->user_periph_priv;
2132
2133 #if 0
2134 xpt_print(periph->path, "Copying to user CCB %p from "
2135 "kernel address %p\n", *user_ccb, &io_req->ccb);
2136 #endif
2137
2138 error = copyout(&io_req->ccb, *user_ccb, sizeof(union ccb));
2139 if (error != 0) {
2140 xpt_print(periph->path, "Copy to user CCB %p from "
2141 "kernel address %p failed with error %d\n",
2142 *user_ccb, &io_req->ccb, error);
2143 }
2144
2145 /*
2146 * Prefer the first error we got back, and make sure we
2147 * don't overwrite bad status with good.
2148 */
2149 if (old_error != 0)
2150 error = old_error;
2151
2152 cam_periph_lock(periph);
2153
2154 /*
2155 * At this point, if there was an error, we could potentially
2156 * re-queue the I/O and try again. But why? The error
2157 * would almost certainly happen again. We might as well
2158 * not leak memory.
2159 */
2160 uma_zfree(softc->pass_zone, io_req);
2161 break;
2162 }
2163 default:
2164 error = cam_periph_ioctl(periph, cmd, addr, passerror);
2165 break;
2166 }
2167
2168 bailout:
2169 cam_periph_unlock(periph);
2170
2171 return(error);
2172 }
2173
2174 static int
passpoll(struct cdev * dev,int poll_events,struct thread * td)2175 passpoll(struct cdev *dev, int poll_events, struct thread *td)
2176 {
2177 struct cam_periph *periph;
2178 struct pass_softc *softc;
2179 int revents;
2180
2181 periph = (struct cam_periph *)dev->si_drv1;
2182 softc = (struct pass_softc *)periph->softc;
2183
2184 revents = poll_events & (POLLOUT | POLLWRNORM);
2185 if ((poll_events & (POLLIN | POLLRDNORM)) != 0) {
2186 cam_periph_lock(periph);
2187
2188 if (!TAILQ_EMPTY(&softc->done_queue)) {
2189 revents |= poll_events & (POLLIN | POLLRDNORM);
2190 }
2191 cam_periph_unlock(periph);
2192 if (revents == 0)
2193 selrecord(td, &softc->read_select);
2194 }
2195
2196 return (revents);
2197 }
2198
2199 static int
passkqfilter(struct cdev * dev,struct knote * kn)2200 passkqfilter(struct cdev *dev, struct knote *kn)
2201 {
2202 struct cam_periph *periph;
2203 struct pass_softc *softc;
2204
2205 periph = (struct cam_periph *)dev->si_drv1;
2206 softc = (struct pass_softc *)periph->softc;
2207
2208 kn->kn_hook = (caddr_t)periph;
2209 kn->kn_fop = &passread_filtops;
2210 knlist_add(&softc->read_select.si_note, kn, 0);
2211
2212 return (0);
2213 }
2214
2215 static void
passreadfiltdetach(struct knote * kn)2216 passreadfiltdetach(struct knote *kn)
2217 {
2218 struct cam_periph *periph;
2219 struct pass_softc *softc;
2220
2221 periph = (struct cam_periph *)kn->kn_hook;
2222 softc = (struct pass_softc *)periph->softc;
2223
2224 knlist_remove(&softc->read_select.si_note, kn, 0);
2225 }
2226
2227 static int
passreadfilt(struct knote * kn,long hint)2228 passreadfilt(struct knote *kn, long hint)
2229 {
2230 struct cam_periph *periph;
2231 struct pass_softc *softc;
2232 int retval;
2233
2234 periph = (struct cam_periph *)kn->kn_hook;
2235 softc = (struct pass_softc *)periph->softc;
2236
2237 cam_periph_assert(periph, MA_OWNED);
2238
2239 if (TAILQ_EMPTY(&softc->done_queue))
2240 retval = 0;
2241 else
2242 retval = 1;
2243
2244 return (retval);
2245 }
2246
2247 /*
2248 * Generally, "ccb" should be the CCB supplied by the kernel. "inccb"
2249 * should be the CCB that is copied in from the user.
2250 */
2251 static int
passsendccb(struct cam_periph * periph,union ccb * ccb,union ccb * inccb)2252 passsendccb(struct cam_periph *periph, union ccb *ccb, union ccb *inccb)
2253 {
2254 struct pass_softc *softc;
2255 struct cam_periph_map_info mapinfo;
2256 uint8_t *cmd;
2257 xpt_opcode fc;
2258 int error;
2259
2260 softc = (struct pass_softc *)periph->softc;
2261
2262 /*
2263 * There are some fields in the CCB header that need to be
2264 * preserved, the rest we get from the user.
2265 */
2266 xpt_merge_ccb(ccb, inccb);
2267
2268 if (ccb->ccb_h.flags & CAM_CDB_POINTER) {
2269 cmd = __builtin_alloca(ccb->csio.cdb_len);
2270 error = copyin(ccb->csio.cdb_io.cdb_ptr, cmd, ccb->csio.cdb_len);
2271 if (error)
2272 return (error);
2273 ccb->csio.cdb_io.cdb_ptr = cmd;
2274 }
2275
2276 /*
2277 * Let cam_periph_mapmem do a sanity check on the data pointer format.
2278 * Even if no data transfer is needed, it's a cheap check and it
2279 * simplifies the code.
2280 */
2281 fc = ccb->ccb_h.func_code;
2282 if ((fc == XPT_SCSI_IO) || (fc == XPT_ATA_IO) || (fc == XPT_SMP_IO)
2283 || (fc == XPT_DEV_MATCH) || (fc == XPT_DEV_ADVINFO) || (fc == XPT_MMC_IO)
2284 || (fc == XPT_NVME_ADMIN) || (fc == XPT_NVME_IO)) {
2285 bzero(&mapinfo, sizeof(mapinfo));
2286
2287 /*
2288 * cam_periph_mapmem calls into proc and vm functions that can
2289 * sleep as well as trigger I/O, so we can't hold the lock.
2290 * Dropping it here is reasonably safe.
2291 */
2292 cam_periph_unlock(periph);
2293 error = cam_periph_mapmem(ccb, &mapinfo, softc->maxio);
2294 cam_periph_lock(periph);
2295
2296 /*
2297 * cam_periph_mapmem returned an error, we can't continue.
2298 * Return the error to the user.
2299 */
2300 if (error)
2301 return(error);
2302 } else
2303 /* Ensure that the unmap call later on is a no-op. */
2304 mapinfo.num_bufs_used = 0;
2305
2306 /*
2307 * If the user wants us to perform any error recovery, then honor
2308 * that request. Otherwise, it's up to the user to perform any
2309 * error recovery.
2310 */
2311 {
2312 uint32_t cam_flags, sense_flags;
2313
2314 passflags(ccb, &cam_flags, &sense_flags);
2315 cam_periph_runccb(ccb, passerror, cam_flags,
2316 sense_flags, softc->device_stats);
2317 }
2318
2319 cam_periph_unlock(periph);
2320 error = cam_periph_unmapmem(ccb, &mapinfo);
2321 cam_periph_lock(periph);
2322
2323 ccb->ccb_h.cbfcnp = NULL;
2324 ccb->ccb_h.periph_priv = inccb->ccb_h.periph_priv;
2325 bcopy(ccb, inccb, sizeof(union ccb));
2326
2327 return (error);
2328 }
2329
2330 /*
2331 * Set the cam_flags and sense_flags based on whether or not the request wants
2332 * error recovery. In order to log errors via devctl, we need to do at least
2333 * minimal recovery. We do this by not retrying unit attention (we let the
2334 * requester do it, or not, if appropriate) and specifically asking for no
2335 * recovery, like we do during device probing.
2336 */
2337 static void
passflags(union ccb * ccb,uint32_t * cam_flags,uint32_t * sense_flags)2338 passflags(union ccb *ccb, uint32_t *cam_flags, uint32_t *sense_flags)
2339 {
2340 if ((ccb->ccb_h.flags & CAM_PASS_ERR_RECOVER) != 0) {
2341 *cam_flags = CAM_RETRY_SELTO;
2342 *sense_flags = SF_RETRY_UA | SF_NO_PRINT;
2343 } else {
2344 *cam_flags = 0;
2345 *sense_flags = SF_NO_RETRY | SF_NO_RECOVERY | SF_NO_PRINT;
2346 }
2347 }
2348
2349 static int
passerror(union ccb * ccb,uint32_t cam_flags,uint32_t sense_flags)2350 passerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
2351 {
2352
2353 return(cam_periph_error(ccb, cam_flags, sense_flags));
2354 }
2355