Lines Matching +full:blocking +full:- +full:io

1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
5 * Copyright (c) 2009-2011 Spectra Logic Corporation
7 * Copyright (c) 2014-2021 Alexander Motin <mav@FreeBSD.org>
41 * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $
118 #define PRIV(io) \ argument
119 ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND])
120 #define ARGS(io) \ argument
121 ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN])
122 #define DSM_RANGE(io) ((io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].integer) argument
202 * Per-I/O information.
205 union ctl_io *io; member
237 static int ctl_be_block_move_done(union ctl_io *io, bool samethr);
258 union ctl_io *io);
260 union ctl_io *io);
262 union ctl_io *io);
264 static int ctl_be_block_submit(union ctl_io *io);
281 static int ctl_be_block_config_write(union ctl_io *io);
282 static int ctl_be_block_config_read(union ctl_io *io);
311 sg->addr = uma_zalloc(softc->bufmin_zone, M_WAITOK); in ctl_alloc_seg()
315 sg->addr = uma_zalloc(softc->bufmax_zone, M_WAITOK); in ctl_alloc_seg()
317 sg->len = len; in ctl_alloc_seg()
324 if (sg->len <= CTLBLK_MIN_SEG) { in ctl_free_seg()
325 uma_zfree(softc->bufmin_zone, sg->addr); in ctl_free_seg()
327 KASSERT(sg->len <= CTLBLK_MAX_SEG, in ctl_free_seg()
328 ("Too large free %zu > %lu", sg->len, CTLBLK_MAX_SEG)); in ctl_free_seg()
329 uma_zfree(softc->bufmax_zone, sg->addr); in ctl_free_seg()
338 beio = uma_zalloc(softc->beio_zone, M_WAITOK | M_ZERO); in ctl_alloc_beio()
339 beio->softc = softc; in ctl_alloc_beio()
340 beio->refcnt = 1; in ctl_alloc_beio()
347 struct ctl_be_block_softc *softc = beio->softc; in ctl_real_free_beio()
350 for (i = 0; i < beio->num_segs; i++) { in ctl_real_free_beio()
351 ctl_free_seg(softc, &beio->sg_segs[i]); in ctl_real_free_beio()
354 if (beio->two_sglists) { in ctl_real_free_beio()
356 &beio->sg_segs[i + CTLBLK_HALF_SEGS]); in ctl_real_free_beio()
360 uma_zfree(softc->beio_zone, beio); in ctl_real_free_beio()
368 if (atomic_fetchadd_int(&beio->refcnt, diff) + diff == 0) in ctl_refcnt_beio()
376 ctl_refcnt_beio(beio, -1); in ctl_free_beio()
382 union ctl_io *io = beio->io; in ctl_complete_beio() local
384 if (beio->beio_cont != NULL) { in ctl_complete_beio()
385 beio->beio_cont(beio); in ctl_complete_beio()
388 ctl_data_submit_done(io); in ctl_complete_beio()
393 ctl_be_block_io_error(union ctl_io *io, int bio_cmd, uint16_t retry_count) in ctl_be_block_io_error() argument
395 switch (io->io_hdr.io_type) { in ctl_be_block_io_error()
399 ctl_set_internal_failure(&io->scsiio, in ctl_be_block_io_error()
403 ctl_set_medium_error(&io->scsiio, bio_cmd == BIO_READ); in ctl_be_block_io_error()
410 ctl_nvme_set_write_fault(&io->nvmeio); in ctl_be_block_io_error()
413 ctl_nvme_set_unrecoverable_read_error(&io->nvmeio); in ctl_be_block_io_error()
416 ctl_nvme_set_internal_error(&io->nvmeio); in ctl_be_block_io_error()
438 ctl_be_block_compare(union ctl_io *io) in ctl_be_block_compare() argument
444 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_compare()
446 for (i = 0; i < beio->num_segs; i++) { in ctl_be_block_compare()
447 res = cmp(beio->sg_segs[i].addr, in ctl_be_block_compare()
448 beio->sg_segs[i + CTLBLK_HALF_SEGS].addr, in ctl_be_block_compare()
449 beio->sg_segs[i].len); in ctl_be_block_compare()
451 if (res < beio->sg_segs[i].len) in ctl_be_block_compare()
454 if (i < beio->num_segs) { in ctl_be_block_compare()
455 ctl_io_set_compare_failure(io, off); in ctl_be_block_compare()
457 ctl_io_set_success(io); in ctl_be_block_compare()
461 ctl_be_block_move_done(union ctl_io *io, bool samethr) in ctl_be_block_move_done() argument
467 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_move_done()
470 ctl_add_kern_rel_offset(io, ctl_kern_data_len(io)); in ctl_be_block_move_done()
475 if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 && in ctl_be_block_move_done()
476 (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) { in ctl_be_block_move_done()
477 lbalen = ARGS(io); in ctl_be_block_move_done()
478 if (lbalen->flags & CTL_LLF_READ) { in ctl_be_block_move_done()
479 ctl_io_set_success(io); in ctl_be_block_move_done()
480 } else if (lbalen->flags & CTL_LLF_COMPARE) { in ctl_be_block_move_done()
482 ctl_be_block_compare(io); in ctl_be_block_move_done()
489 if ((beio->bio_cmd == BIO_READ) in ctl_be_block_move_done()
490 || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0) in ctl_be_block_move_done()
491 || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) { in ctl_be_block_move_done()
500 * execute the backend I/O. That is because we do blocking in ctl_be_block_move_done()
501 * memory allocations, and in the file backing case, blocking I/O. in ctl_be_block_move_done()
505 be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io); in ctl_be_block_move_done()
507 be_lun->dispatch(be_lun, beio); in ctl_be_block_move_done()
509 mtx_lock(&be_lun->queue_lock); in ctl_be_block_move_done()
510 STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links); in ctl_be_block_move_done()
511 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_move_done()
512 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); in ctl_be_block_move_done()
520 struct ctl_be_block_io *beio = bio->bio_caller1; in ctl_be_block_biodone()
521 struct ctl_be_block_lun *be_lun = beio->lun; in ctl_be_block_biodone()
522 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_biodone()
523 union ctl_io *io; in ctl_be_block_biodone() local
526 io = beio->io; in ctl_be_block_biodone()
530 error = bio->bio_error; in ctl_be_block_biodone()
531 mtx_lock(&be_lun->io_lock); in ctl_be_block_biodone()
533 (beio->first_error == 0 || in ctl_be_block_biodone()
534 bio->bio_offset < beio->first_error_offset)) { in ctl_be_block_biodone()
535 beio->first_error = error; in ctl_be_block_biodone()
536 beio->first_error_offset = bio->bio_offset; in ctl_be_block_biodone()
539 beio->num_bios_done++; in ctl_be_block_biodone()
551 if ((beio->send_complete == 0) in ctl_be_block_biodone()
552 || (beio->num_bios_done < beio->num_bios_sent)) { in ctl_be_block_biodone()
553 mtx_unlock(&be_lun->io_lock); in ctl_be_block_biodone()
561 devstat_end_transaction(beio->lun->disk_stats, beio->io_len, in ctl_be_block_biodone()
562 beio->ds_tag_type, beio->ds_trans_type, in ctl_be_block_biodone()
563 /*now*/ NULL, /*then*/&beio->ds_t0); in ctl_be_block_biodone()
564 mtx_unlock(&be_lun->io_lock); in ctl_be_block_biodone()
570 error = beio->first_error; in ctl_be_block_biodone()
573 ctl_io_set_invalid_opcode(io); in ctl_be_block_biodone()
575 ctl_io_set_space_alloc_fail(io); in ctl_be_block_biodone()
577 ctl_io_set_hw_write_protected(io); in ctl_be_block_biodone()
579 ctl_be_block_io_error(io, beio->bio_cmd, in ctl_be_block_biodone()
590 if ((beio->bio_cmd == BIO_WRITE) in ctl_be_block_biodone()
591 || (beio->bio_cmd == BIO_FLUSH) in ctl_be_block_biodone()
592 || (beio->bio_cmd == BIO_DELETE) in ctl_be_block_biodone()
593 || (ARGS(io)->flags & CTL_LLF_VERIFY)) { in ctl_be_block_biodone()
594 ctl_io_set_success(io); in ctl_be_block_biodone()
597 if ((ARGS(io)->flags & CTL_LLF_READ) && in ctl_be_block_biodone()
598 beio->beio_cont == NULL) { in ctl_be_block_biodone()
599 ctl_io_set_success(io); in ctl_be_block_biodone()
600 if (cbe_lun->serseq >= CTL_LUN_SERSEQ_SOFT) in ctl_be_block_biodone()
601 ctl_serseq_done(io); in ctl_be_block_biodone()
603 ctl_datamove(io); in ctl_be_block_biodone()
611 union ctl_io *io = beio->io; in ctl_be_block_flush_file() local
617 binuptime(&beio->ds_t0); in ctl_be_block_flush_file()
618 devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); in ctl_be_block_flush_file()
620 (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT); in ctl_be_block_flush_file()
622 vn_lock(be_lun->vn, vn_lktype_write(mountpoint, be_lun->vn) | in ctl_be_block_flush_file()
624 error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT, in ctl_be_block_flush_file()
626 VOP_UNLOCK(be_lun->vn); in ctl_be_block_flush_file()
630 mtx_lock(&be_lun->io_lock); in ctl_be_block_flush_file()
631 devstat_end_transaction(beio->lun->disk_stats, beio->io_len, in ctl_be_block_flush_file()
632 beio->ds_tag_type, beio->ds_trans_type, in ctl_be_block_flush_file()
633 /*now*/ NULL, /*then*/&beio->ds_t0); in ctl_be_block_flush_file()
634 mtx_unlock(&be_lun->io_lock); in ctl_be_block_flush_file()
637 ctl_io_set_success(io); in ctl_be_block_flush_file()
639 ctl_be_block_io_error(io, BIO_FLUSH, in ctl_be_block_flush_file()
655 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_dispatch_file()
657 union ctl_io *io; in ctl_be_block_dispatch_file() local
665 file_data = &be_lun->backend.file; in ctl_be_block_dispatch_file()
666 io = beio->io; in ctl_be_block_dispatch_file()
668 if (ARGS(io)->flags & CTL_LLF_DPO) in ctl_be_block_dispatch_file()
670 if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) in ctl_be_block_dispatch_file()
674 if (beio->bio_cmd == BIO_READ) { in ctl_be_block_dispatch_file()
681 xuio.uio_offset = beio->io_offset; in ctl_be_block_dispatch_file()
682 xuio.uio_resid = beio->io_len; in ctl_be_block_dispatch_file()
684 xuio.uio_iov = beio->xiovecs; in ctl_be_block_dispatch_file()
685 xuio.uio_iovcnt = beio->num_segs; in ctl_be_block_dispatch_file()
689 xiovec->iov_base = beio->sg_segs[i].addr; in ctl_be_block_dispatch_file()
690 xiovec->iov_len = beio->sg_segs[i].len; in ctl_be_block_dispatch_file()
693 binuptime(&beio->ds_t0); in ctl_be_block_dispatch_file()
694 devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); in ctl_be_block_dispatch_file()
696 if (beio->bio_cmd == BIO_READ) { in ctl_be_block_dispatch_file()
697 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); in ctl_be_block_dispatch_file()
699 if (beio->beio_cont == NULL && in ctl_be_block_dispatch_file()
700 cbe_lun->serseq == CTL_LUN_SERSEQ_SOFT) in ctl_be_block_dispatch_file()
701 ctl_serseq_done(io); in ctl_be_block_dispatch_file()
705 * ffs_rawread(). But that only works for single-segment in ctl_be_block_dispatch_file()
719 error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred); in ctl_be_block_dispatch_file()
721 VOP_UNLOCK(be_lun->vn); in ctl_be_block_dispatch_file()
728 s = beio->io_len - xuio.uio_resid; in ctl_be_block_dispatch_file()
729 for (i = 0; i < beio->num_segs; i++) { in ctl_be_block_dispatch_file()
730 if (s >= beio->sg_segs[i].len) { in ctl_be_block_dispatch_file()
731 s -= beio->sg_segs[i].len; in ctl_be_block_dispatch_file()
734 bzero((uint8_t *)beio->sg_segs[i].addr + s, in ctl_be_block_dispatch_file()
735 beio->sg_segs[i].len - s); in ctl_be_block_dispatch_file()
742 (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT); in ctl_be_block_dispatch_file()
743 vn_lock(be_lun->vn, vn_lktype_write(mountpoint, in ctl_be_block_dispatch_file()
744 be_lun->vn) | LK_RETRY); in ctl_be_block_dispatch_file()
761 error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred); in ctl_be_block_dispatch_file()
762 VOP_UNLOCK(be_lun->vn); in ctl_be_block_dispatch_file()
768 mtx_lock(&be_lun->io_lock); in ctl_be_block_dispatch_file()
769 devstat_end_transaction(beio->lun->disk_stats, beio->io_len, in ctl_be_block_dispatch_file()
770 beio->ds_tag_type, beio->ds_trans_type, in ctl_be_block_dispatch_file()
771 /*now*/ NULL, /*then*/&beio->ds_t0); in ctl_be_block_dispatch_file()
772 mtx_unlock(&be_lun->io_lock); in ctl_be_block_dispatch_file()
780 ctl_io_set_space_alloc_fail(io); in ctl_be_block_dispatch_file()
782 ctl_io_set_hw_write_protected(io); in ctl_be_block_dispatch_file()
784 ctl_be_block_io_error(io, beio->bio_cmd, 0); in ctl_be_block_dispatch_file()
794 if ((beio->bio_cmd == BIO_WRITE) || in ctl_be_block_dispatch_file()
795 (ARGS(io)->flags & CTL_LLF_VERIFY)) { in ctl_be_block_dispatch_file()
796 ctl_io_set_success(io); in ctl_be_block_dispatch_file()
799 if ((ARGS(io)->flags & CTL_LLF_READ) && in ctl_be_block_dispatch_file()
800 beio->beio_cont == NULL) { in ctl_be_block_dispatch_file()
801 ctl_io_set_success(io); in ctl_be_block_dispatch_file()
802 if (cbe_lun->serseq > CTL_LUN_SERSEQ_SOFT) in ctl_be_block_dispatch_file()
803 ctl_serseq_done(io); in ctl_be_block_dispatch_file()
805 ctl_datamove(io); in ctl_be_block_dispatch_file()
813 union ctl_io *io = beio->io; in ctl_be_block_gls_file() local
814 struct ctl_lba_len_flags *lbalen = ARGS(io); in ctl_be_block_gls_file()
821 CTL_IO_ASSERT(io, SCSI); in ctl_be_block_gls_file()
823 off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize; in ctl_be_block_gls_file()
824 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); in ctl_be_block_gls_file()
825 error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off, in ctl_be_block_gls_file()
826 0, curthread->td_ucred, curthread); in ctl_be_block_gls_file()
830 error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off, in ctl_be_block_gls_file()
831 0, curthread->td_ucred, curthread); in ctl_be_block_gls_file()
836 off = be_lun->size_bytes; in ctl_be_block_gls_file()
839 VOP_UNLOCK(be_lun->vn); in ctl_be_block_gls_file()
841 data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; in ctl_be_block_gls_file()
842 scsi_u64to8b(lbalen->lba, data->descr[0].addr); in ctl_be_block_gls_file()
843 scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize - in ctl_be_block_gls_file()
844 lbalen->lba), data->descr[0].length); in ctl_be_block_gls_file()
845 data->descr[0].status = status; in ctl_be_block_gls_file()
859 if (be_lun->vn == NULL) in ctl_be_block_getattr_file()
861 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); in ctl_be_block_getattr_file()
863 error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); in ctl_be_block_getattr_file()
865 val = vattr.va_bytes / be_lun->cbe_lun.blocksize; in ctl_be_block_getattr_file()
868 !VN_IS_DOOMED(be_lun->vn)) { in ctl_be_block_getattr_file()
869 error = VFS_STATFS(be_lun->vn->v_mount, &statfs); in ctl_be_block_getattr_file()
872 be_lun->cbe_lun.blocksize; in ctl_be_block_getattr_file()
874 VOP_UNLOCK(be_lun->vn); in ctl_be_block_getattr_file()
883 union ctl_io *io; in ctl_be_block_unmap_file() local
890 io = beio->io; in ctl_be_block_unmap_file()
891 file_data = &be_lun->backend.file; in ctl_be_block_unmap_file()
895 binuptime(&beio->ds_t0); in ctl_be_block_unmap_file()
896 devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); in ctl_be_block_unmap_file()
898 (void)vn_start_write(be_lun->vn, &mp, V_WAIT); in ctl_be_block_unmap_file()
899 vn_lock(be_lun->vn, vn_lktype_write(mp, be_lun->vn) | LK_RETRY); in ctl_be_block_unmap_file()
900 if (beio->io_offset == -1) { in ctl_be_block_unmap_file()
901 beio->io_len = 0; in ctl_be_block_unmap_file()
903 &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; in ctl_be_block_unmap_file()
904 buf = (struct scsi_unmap_desc *)ptrlen->ptr; in ctl_be_block_unmap_file()
905 end = buf + ptrlen->len / sizeof(*buf); in ctl_be_block_unmap_file()
907 off = (off_t)scsi_8btou64(buf->lba) * in ctl_be_block_unmap_file()
908 be_lun->cbe_lun.blocksize; in ctl_be_block_unmap_file()
909 len = (off_t)scsi_4btoul(buf->length) * in ctl_be_block_unmap_file()
910 be_lun->cbe_lun.blocksize; in ctl_be_block_unmap_file()
911 beio->io_len += len; in ctl_be_block_unmap_file()
912 error = vn_deallocate(be_lun->vn, &off, &len, in ctl_be_block_unmap_file()
913 0, IO_NOMACCHECK | IO_NODELOCKED, file_data->cred, in ctl_be_block_unmap_file()
920 off = beio->io_offset; in ctl_be_block_unmap_file()
921 len = beio->io_len; in ctl_be_block_unmap_file()
922 error = vn_deallocate(be_lun->vn, &off, &len, 0, in ctl_be_block_unmap_file()
923 IO_NOMACCHECK | IO_NODELOCKED, file_data->cred, NOCRED); in ctl_be_block_unmap_file()
925 VOP_UNLOCK(be_lun->vn); in ctl_be_block_unmap_file()
928 mtx_lock(&be_lun->io_lock); in ctl_be_block_unmap_file()
929 devstat_end_transaction(beio->lun->disk_stats, beio->io_len, in ctl_be_block_unmap_file()
930 beio->ds_tag_type, beio->ds_trans_type, in ctl_be_block_unmap_file()
931 /*now*/ NULL, /*then*/&beio->ds_t0); in ctl_be_block_unmap_file()
932 mtx_unlock(&be_lun->io_lock); in ctl_be_block_unmap_file()
940 ctl_io_set_success(io); in ctl_be_block_unmap_file()
944 ctl_io_set_space_alloc_fail(io); in ctl_be_block_unmap_file()
948 ctl_io_set_hw_write_protected(io); in ctl_be_block_unmap_file()
951 ctl_be_block_io_error(io, BIO_DELETE, 0); in ctl_be_block_unmap_file()
960 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_dispatch_zvol()
961 union ctl_io *io; in ctl_be_block_dispatch_zvol() local
970 io = beio->io; in ctl_be_block_dispatch_zvol()
972 if (ARGS(io)->flags & CTL_LLF_DPO) in ctl_be_block_dispatch_zvol()
974 if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) in ctl_be_block_dispatch_zvol()
978 if (beio->bio_cmd == BIO_READ) { in ctl_be_block_dispatch_zvol()
985 xuio.uio_offset = beio->io_offset; in ctl_be_block_dispatch_zvol()
986 xuio.uio_resid = beio->io_len; in ctl_be_block_dispatch_zvol()
988 xuio.uio_iov = beio->xiovecs; in ctl_be_block_dispatch_zvol()
989 xuio.uio_iovcnt = beio->num_segs; in ctl_be_block_dispatch_zvol()
993 xiovec->iov_base = beio->sg_segs[i].addr; in ctl_be_block_dispatch_zvol()
994 xiovec->iov_len = beio->sg_segs[i].len; in ctl_be_block_dispatch_zvol()
997 binuptime(&beio->ds_t0); in ctl_be_block_dispatch_zvol()
998 devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); in ctl_be_block_dispatch_zvol()
1000 csw = devvn_refthread(be_lun->vn, &dev, &ref); in ctl_be_block_dispatch_zvol()
1002 if (beio->bio_cmd == BIO_READ) { in ctl_be_block_dispatch_zvol()
1003 if (beio->beio_cont == NULL && in ctl_be_block_dispatch_zvol()
1004 cbe_lun->serseq == CTL_LUN_SERSEQ_SOFT) in ctl_be_block_dispatch_zvol()
1005 ctl_serseq_done(io); in ctl_be_block_dispatch_zvol()
1006 error = csw->d_read(dev, &xuio, flags); in ctl_be_block_dispatch_zvol()
1008 error = csw->d_write(dev, &xuio, flags); in ctl_be_block_dispatch_zvol()
1013 if (beio->bio_cmd == BIO_READ) in ctl_be_block_dispatch_zvol()
1018 mtx_lock(&be_lun->io_lock); in ctl_be_block_dispatch_zvol()
1019 devstat_end_transaction(beio->lun->disk_stats, beio->io_len, in ctl_be_block_dispatch_zvol()
1020 beio->ds_tag_type, beio->ds_trans_type, in ctl_be_block_dispatch_zvol()
1021 /*now*/ NULL, /*then*/&beio->ds_t0); in ctl_be_block_dispatch_zvol()
1022 mtx_unlock(&be_lun->io_lock); in ctl_be_block_dispatch_zvol()
1030 ctl_io_set_space_alloc_fail(io); in ctl_be_block_dispatch_zvol()
1032 ctl_io_set_hw_write_protected(io); in ctl_be_block_dispatch_zvol()
1034 ctl_be_block_io_error(io, beio->bio_cmd, 0); in ctl_be_block_dispatch_zvol()
1044 if ((beio->bio_cmd == BIO_WRITE) || in ctl_be_block_dispatch_zvol()
1045 (ARGS(io)->flags & CTL_LLF_VERIFY)) { in ctl_be_block_dispatch_zvol()
1046 ctl_io_set_success(io); in ctl_be_block_dispatch_zvol()
1049 if ((ARGS(io)->flags & CTL_LLF_READ) && in ctl_be_block_dispatch_zvol()
1050 beio->beio_cont == NULL) { in ctl_be_block_dispatch_zvol()
1051 ctl_io_set_success(io); in ctl_be_block_dispatch_zvol()
1052 if (cbe_lun->serseq > CTL_LUN_SERSEQ_SOFT) in ctl_be_block_dispatch_zvol()
1053 ctl_serseq_done(io); in ctl_be_block_dispatch_zvol()
1055 ctl_datamove(io); in ctl_be_block_dispatch_zvol()
1063 union ctl_io *io = beio->io; in ctl_be_block_gls_zvol() local
1066 struct ctl_lba_len_flags *lbalen = ARGS(io); in ctl_be_block_gls_zvol()
1073 CTL_IO_ASSERT(io, SCSI); in ctl_be_block_gls_zvol()
1075 csw = devvn_refthread(be_lun->vn, &dev, &ref); in ctl_be_block_gls_zvol()
1078 off = be_lun->size_bytes; in ctl_be_block_gls_zvol()
1081 off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize; in ctl_be_block_gls_zvol()
1082 error = csw->d_ioctl(dev, FIOSEEKHOLE, (caddr_t)&off, FREAD, in ctl_be_block_gls_zvol()
1087 error = csw->d_ioctl(dev, FIOSEEKDATA, (caddr_t)&off, FREAD, in ctl_be_block_gls_zvol()
1093 off = be_lun->size_bytes; in ctl_be_block_gls_zvol()
1099 data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; in ctl_be_block_gls_zvol()
1100 scsi_u64to8b(lbalen->lba, data->descr[0].addr); in ctl_be_block_gls_zvol()
1101 scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize - in ctl_be_block_gls_zvol()
1102 lbalen->lba), data->descr[0].length); in ctl_be_block_gls_zvol()
1103 data->descr[0].status = status; in ctl_be_block_gls_zvol()
1119 /* This can't fail, it's a blocking allocation. */ in ctl_be_block_flush_dev()
1122 bio->bio_cmd = BIO_FLUSH; in ctl_be_block_flush_dev()
1123 bio->bio_offset = 0; in ctl_be_block_flush_dev()
1124 bio->bio_data = 0; in ctl_be_block_flush_dev()
1125 bio->bio_done = ctl_be_block_biodone; in ctl_be_block_flush_dev()
1126 bio->bio_caller1 = beio; in ctl_be_block_flush_dev()
1127 bio->bio_pblkno = 0; in ctl_be_block_flush_dev()
1134 beio->num_bios_sent = 1; in ctl_be_block_flush_dev()
1135 beio->send_complete = 1; in ctl_be_block_flush_dev()
1137 binuptime(&beio->ds_t0); in ctl_be_block_flush_dev()
1138 devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); in ctl_be_block_flush_dev()
1140 csw = devvn_refthread(be_lun->vn, &dev, &ref); in ctl_be_block_flush_dev()
1142 bio->bio_dev = dev; in ctl_be_block_flush_dev()
1143 csw->d_strategy(bio); in ctl_be_block_flush_dev()
1146 bio->bio_error = ENXIO; in ctl_be_block_flush_dev()
1162 csw = devvn_refthread(be_lun->vn, &dev, &ref); in ctl_be_block_unmap_dev_range()
1163 maxlen = LONG_MAX - (LONG_MAX % be_lun->cbe_lun.blocksize); in ctl_be_block_unmap_dev_range()
1166 bio->bio_cmd = BIO_DELETE; in ctl_be_block_unmap_dev_range()
1167 bio->bio_dev = dev; in ctl_be_block_unmap_dev_range()
1168 bio->bio_offset = off; in ctl_be_block_unmap_dev_range()
1169 bio->bio_length = MIN(len, maxlen); in ctl_be_block_unmap_dev_range()
1170 bio->bio_data = 0; in ctl_be_block_unmap_dev_range()
1171 bio->bio_done = ctl_be_block_biodone; in ctl_be_block_unmap_dev_range()
1172 bio->bio_caller1 = beio; in ctl_be_block_unmap_dev_range()
1173 bio->bio_pblkno = off / be_lun->cbe_lun.blocksize; in ctl_be_block_unmap_dev_range()
1175 off += bio->bio_length; in ctl_be_block_unmap_dev_range()
1176 len -= bio->bio_length; in ctl_be_block_unmap_dev_range()
1178 mtx_lock(&be_lun->io_lock); in ctl_be_block_unmap_dev_range()
1179 beio->num_bios_sent++; in ctl_be_block_unmap_dev_range()
1181 beio->send_complete = 1; in ctl_be_block_unmap_dev_range()
1182 mtx_unlock(&be_lun->io_lock); in ctl_be_block_unmap_dev_range()
1185 csw->d_strategy(bio); in ctl_be_block_unmap_dev_range()
1187 bio->bio_error = ENXIO; in ctl_be_block_unmap_dev_range()
1199 union ctl_io *io; in ctl_be_block_unmap_dev() local
1204 io = beio->io; in ctl_be_block_unmap_dev()
1208 binuptime(&beio->ds_t0); in ctl_be_block_unmap_dev()
1209 devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); in ctl_be_block_unmap_dev()
1211 if (beio->io_offset == -1) { in ctl_be_block_unmap_dev()
1212 beio->io_len = 0; in ctl_be_block_unmap_dev()
1213 ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; in ctl_be_block_unmap_dev()
1214 buf = (struct scsi_unmap_desc *)ptrlen->ptr; in ctl_be_block_unmap_dev()
1215 end = buf + ptrlen->len / sizeof(*buf); in ctl_be_block_unmap_dev()
1217 len = (uint64_t)scsi_4btoul(buf->length) * in ctl_be_block_unmap_dev()
1218 be_lun->cbe_lun.blocksize; in ctl_be_block_unmap_dev()
1219 beio->io_len += len; in ctl_be_block_unmap_dev()
1221 scsi_8btou64(buf->lba) * be_lun->cbe_lun.blocksize, in ctl_be_block_unmap_dev()
1222 len, (end - buf < 2) ? TRUE : FALSE); in ctl_be_block_unmap_dev()
1226 beio->io_offset, beio->io_len, TRUE); in ctl_be_block_unmap_dev()
1241 csw = devvn_refthread(be_lun->vn, &dev, &ref); in ctl_be_block_dispatch_dev()
1248 max_iosize = dev->si_iosize_max; in ctl_be_block_dispatch_dev()
1254 cur_offset = beio->io_offset; in ctl_be_block_dispatch_dev()
1255 for (i = 0; i < beio->num_segs; i++) { in ctl_be_block_dispatch_dev()
1259 cur_size = beio->sg_segs[i].len; in ctl_be_block_dispatch_dev()
1260 cur_ptr = beio->sg_segs[i].addr; in ctl_be_block_dispatch_dev()
1263 /* This can't fail, it's a blocking allocation. */ in ctl_be_block_dispatch_dev()
1268 bio->bio_cmd = beio->bio_cmd; in ctl_be_block_dispatch_dev()
1269 bio->bio_dev = dev; in ctl_be_block_dispatch_dev()
1270 bio->bio_caller1 = beio; in ctl_be_block_dispatch_dev()
1271 bio->bio_length = min(cur_size, max_iosize); in ctl_be_block_dispatch_dev()
1272 bio->bio_offset = cur_offset; in ctl_be_block_dispatch_dev()
1273 bio->bio_data = cur_ptr; in ctl_be_block_dispatch_dev()
1274 bio->bio_done = ctl_be_block_biodone; in ctl_be_block_dispatch_dev()
1275 bio->bio_pblkno = cur_offset / be_lun->cbe_lun.blocksize; in ctl_be_block_dispatch_dev()
1277 cur_offset += bio->bio_length; in ctl_be_block_dispatch_dev()
1278 cur_ptr += bio->bio_length; in ctl_be_block_dispatch_dev()
1279 cur_size -= bio->bio_length; in ctl_be_block_dispatch_dev()
1282 beio->num_bios_sent++; in ctl_be_block_dispatch_dev()
1285 beio->send_complete = 1; in ctl_be_block_dispatch_dev()
1286 binuptime(&beio->ds_t0); in ctl_be_block_dispatch_dev()
1287 devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); in ctl_be_block_dispatch_dev()
1295 csw->d_strategy(bio); in ctl_be_block_dispatch_dev()
1297 bio->bio_error = ENXIO; in ctl_be_block_dispatch_dev()
1313 csw = devvn_refthread(be_lun->vn, &dev, &ref); in ctl_be_block_getattr_dev()
1318 if (csw->d_ioctl) { in ctl_be_block_getattr_dev()
1319 error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD, in ctl_be_block_getattr_dev()
1331 union ctl_io *io) in ctl_be_block_namespace_data() argument
1333 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_namespace_data()
1336 nsdata = (struct nvme_namespace_data *)io->nvmeio.kern_data_ptr; in ctl_be_block_namespace_data()
1338 nsdata->nsze = htole64(be_lun->size_blocks); in ctl_be_block_namespace_data()
1339 nsdata->ncap = nsdata->nsze; in ctl_be_block_namespace_data()
1340 nsdata->nuse = nsdata->nsze; in ctl_be_block_namespace_data()
1341 nsdata->nlbaf = 1 - 1; in ctl_be_block_namespace_data()
1342 nsdata->dlfeat = NVMEM(NVME_NS_DATA_DLFEAT_DWZ) | in ctl_be_block_namespace_data()
1344 nsdata->flbas = NVMEF(NVME_NS_DATA_FLBAS_FORMAT, 0); in ctl_be_block_namespace_data()
1345 nsdata->lbaf[0] = NVMEF(NVME_NS_DATA_LBAF_LBADS, in ctl_be_block_namespace_data()
1346 ffs(cbe_lun->blocksize) - 1); in ctl_be_block_namespace_data()
1349 ctl_config_read_done(io); in ctl_be_block_namespace_data()
1354 union ctl_io *io) in ctl_be_block_nvme_ids() argument
1356 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_nvme_ids()
1358 ctl_lun_nvme_ids(cbe_lun, io->nvmeio.kern_data_ptr); in ctl_be_block_nvme_ids()
1359 ctl_config_read_done(io); in ctl_be_block_nvme_ids()
1364 union ctl_io *io) in ctl_be_block_cw_dispatch_sync() argument
1366 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_cw_dispatch_sync()
1371 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_cw_dispatch_sync()
1372 lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; in ctl_be_block_cw_dispatch_sync()
1374 beio->io_len = lbalen->len * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_sync()
1375 beio->io_offset = lbalen->lba * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_sync()
1376 beio->io_arg = (lbalen->flags & SSC_IMMED) != 0; in ctl_be_block_cw_dispatch_sync()
1377 beio->bio_cmd = BIO_FLUSH; in ctl_be_block_cw_dispatch_sync()
1378 beio->ds_trans_type = DEVSTAT_NO_DATA; in ctl_be_block_cw_dispatch_sync()
1380 be_lun->lun_flush(be_lun, beio); in ctl_be_block_cw_dispatch_sync()
1386 union ctl_io *io; in ctl_be_block_cw_done_ws() local
1388 io = beio->io; in ctl_be_block_cw_done_ws()
1390 if ((io->io_hdr.flags & CTL_FLAG_ABORT) || in ctl_be_block_cw_done_ws()
1391 ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && in ctl_be_block_cw_done_ws()
1392 (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { in ctl_be_block_cw_done_ws()
1393 ctl_config_write_done(io); in ctl_be_block_cw_done_ws()
1397 ctl_be_block_config_write(io); in ctl_be_block_cw_done_ws()
1402 union ctl_io *io) in ctl_be_block_cw_dispatch_ws() argument
1404 struct ctl_be_block_softc *softc = be_lun->softc; in ctl_be_block_cw_dispatch_ws()
1405 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_cw_dispatch_ws()
1415 CTL_IO_ASSERT(io, SCSI); in ctl_be_block_cw_dispatch_ws()
1417 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_cw_dispatch_ws()
1418 lbalen = ARGS(io); in ctl_be_block_cw_dispatch_ws()
1420 if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) || in ctl_be_block_cw_dispatch_ws()
1421 (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) { in ctl_be_block_cw_dispatch_ws()
1423 ctl_set_invalid_field(&io->scsiio, in ctl_be_block_cw_dispatch_ws()
1429 ctl_config_write_done(io); in ctl_be_block_cw_dispatch_ws()
1433 if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) { in ctl_be_block_cw_dispatch_ws()
1434 beio->io_offset = lbalen->lba * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_ws()
1435 beio->io_len = (uint64_t)lbalen->len * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_ws()
1436 beio->bio_cmd = BIO_DELETE; in ctl_be_block_cw_dispatch_ws()
1437 beio->ds_trans_type = DEVSTAT_FREE; in ctl_be_block_cw_dispatch_ws()
1439 be_lun->unmap(be_lun, beio); in ctl_be_block_cw_dispatch_ws()
1443 beio->bio_cmd = BIO_WRITE; in ctl_be_block_cw_dispatch_ws()
1444 beio->ds_trans_type = DEVSTAT_WRITE; in ctl_be_block_cw_dispatch_ws()
1447 (uintmax_t)lbalen->lba, lbalen->len); in ctl_be_block_cw_dispatch_ws()
1449 pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp; in ctl_be_block_cw_dispatch_ws()
1450 if (be_lun->cbe_lun.pblockoff > 0) in ctl_be_block_cw_dispatch_ws()
1451 pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff; in ctl_be_block_cw_dispatch_ws()
1454 len_left = (uint64_t)lbalen->len * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_ws()
1460 if (pb > cbe_lun->blocksize) { in ctl_be_block_cw_dispatch_ws()
1461 adj = ((lbalen->lba + lba) * cbe_lun->blocksize + in ctl_be_block_cw_dispatch_ws()
1462 seglen - pbo) % pb; in ctl_be_block_cw_dispatch_ws()
1464 seglen -= adj; in ctl_be_block_cw_dispatch_ws()
1466 seglen -= seglen % cbe_lun->blocksize; in ctl_be_block_cw_dispatch_ws()
1468 seglen -= seglen % cbe_lun->blocksize; in ctl_be_block_cw_dispatch_ws()
1469 ctl_alloc_seg(softc, &beio->sg_segs[i], seglen); in ctl_be_block_cw_dispatch_ws()
1472 beio->sg_segs[i].addr, beio->sg_segs[i].len); in ctl_be_block_cw_dispatch_ws()
1474 beio->num_segs++; in ctl_be_block_cw_dispatch_ws()
1475 len_left -= seglen; in ctl_be_block_cw_dispatch_ws()
1477 buf = beio->sg_segs[i].addr; in ctl_be_block_cw_dispatch_ws()
1479 for (; buf < end; buf += cbe_lun->blocksize) { in ctl_be_block_cw_dispatch_ws()
1480 if (lbalen->flags & SWS_NDOB) { in ctl_be_block_cw_dispatch_ws()
1481 memset(buf, 0, cbe_lun->blocksize); in ctl_be_block_cw_dispatch_ws()
1483 memcpy(buf, io->scsiio.kern_data_ptr, in ctl_be_block_cw_dispatch_ws()
1484 cbe_lun->blocksize); in ctl_be_block_cw_dispatch_ws()
1486 if (lbalen->flags & SWS_LBDATA) in ctl_be_block_cw_dispatch_ws()
1487 scsi_ulto4b(lbalen->lba + lba, buf); in ctl_be_block_cw_dispatch_ws()
1492 beio->io_offset = lbalen->lba * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_ws()
1493 beio->io_len = lba * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_ws()
1497 lbalen->lba += lba; in ctl_be_block_cw_dispatch_ws()
1498 lbalen->len -= lba; in ctl_be_block_cw_dispatch_ws()
1499 beio->beio_cont = ctl_be_block_cw_done_ws; in ctl_be_block_cw_dispatch_ws()
1502 be_lun->dispatch(be_lun, beio); in ctl_be_block_cw_dispatch_ws()
1507 union ctl_io *io) in ctl_be_block_cw_dispatch_unmap() argument
1514 CTL_IO_ASSERT(io, SCSI); in ctl_be_block_cw_dispatch_unmap()
1516 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_cw_dispatch_unmap()
1517 ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; in ctl_be_block_cw_dispatch_unmap()
1519 if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) { in ctl_be_block_cw_dispatch_unmap()
1521 ctl_set_invalid_field(&io->scsiio, in ctl_be_block_cw_dispatch_unmap()
1527 ctl_config_write_done(io); in ctl_be_block_cw_dispatch_unmap()
1531 beio->io_len = 0; in ctl_be_block_cw_dispatch_unmap()
1532 beio->io_offset = -1; in ctl_be_block_cw_dispatch_unmap()
1533 beio->bio_cmd = BIO_DELETE; in ctl_be_block_cw_dispatch_unmap()
1534 beio->ds_trans_type = DEVSTAT_FREE; in ctl_be_block_cw_dispatch_unmap()
1536 be_lun->unmap(be_lun, beio); in ctl_be_block_cw_dispatch_unmap()
1541 union ctl_io *io) in ctl_be_block_cw_dispatch_flush() argument
1546 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_cw_dispatch_flush()
1548 beio->io_len = be_lun->size_bytes; in ctl_be_block_cw_dispatch_flush()
1549 beio->io_offset = 0; in ctl_be_block_cw_dispatch_flush()
1550 beio->io_arg = 1; in ctl_be_block_cw_dispatch_flush()
1551 beio->bio_cmd = BIO_FLUSH; in ctl_be_block_cw_dispatch_flush()
1552 beio->ds_trans_type = DEVSTAT_NO_DATA; in ctl_be_block_cw_dispatch_flush()
1554 be_lun->lun_flush(be_lun, beio); in ctl_be_block_cw_dispatch_flush()
1559 union ctl_io *io) in ctl_be_block_cw_dispatch_wu() argument
1561 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_cw_dispatch_wu()
1565 CTL_IO_ASSERT(io, NVME); in ctl_be_block_cw_dispatch_wu()
1567 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_cw_dispatch_wu()
1568 lbalen = ARGS(io); in ctl_be_block_cw_dispatch_wu()
1574 beio->io_offset = lbalen->lba * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wu()
1575 beio->io_len = (uint64_t)lbalen->len * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wu()
1576 beio->bio_cmd = BIO_DELETE; in ctl_be_block_cw_dispatch_wu()
1577 beio->ds_trans_type = DEVSTAT_FREE; in ctl_be_block_cw_dispatch_wu()
1579 be_lun->unmap(be_lun, beio); in ctl_be_block_cw_dispatch_wu()
1584 union ctl_io *io) in ctl_be_block_cw_dispatch_wz() argument
1586 struct ctl_be_block_softc *softc = be_lun->softc; in ctl_be_block_cw_dispatch_wz()
1587 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_cw_dispatch_wz()
1596 CTL_IO_ASSERT(io, NVME); in ctl_be_block_cw_dispatch_wz()
1598 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_cw_dispatch_wz()
1599 lbalen = ARGS(io); in ctl_be_block_cw_dispatch_wz()
1601 if ((le32toh(io->nvmeio.cmd.cdw12) & (1U << 25)) != 0 && in ctl_be_block_cw_dispatch_wz()
1602 be_lun->unmap != NULL) { in ctl_be_block_cw_dispatch_wz()
1603 beio->io_offset = lbalen->lba * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wz()
1604 beio->io_len = (uint64_t)lbalen->len * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wz()
1605 beio->bio_cmd = BIO_DELETE; in ctl_be_block_cw_dispatch_wz()
1606 beio->ds_trans_type = DEVSTAT_FREE; in ctl_be_block_cw_dispatch_wz()
1608 be_lun->unmap(be_lun, beio); in ctl_be_block_cw_dispatch_wz()
1612 beio->bio_cmd = BIO_WRITE; in ctl_be_block_cw_dispatch_wz()
1613 beio->ds_trans_type = DEVSTAT_WRITE; in ctl_be_block_cw_dispatch_wz()
1616 (uintmax_t)lbalen->lba, lbalen->len); in ctl_be_block_cw_dispatch_wz()
1618 pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp; in ctl_be_block_cw_dispatch_wz()
1619 if (be_lun->cbe_lun.pblockoff > 0) in ctl_be_block_cw_dispatch_wz()
1620 pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff; in ctl_be_block_cw_dispatch_wz()
1623 len_left = (uint64_t)lbalen->len * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wz()
1629 if (pb > cbe_lun->blocksize) { in ctl_be_block_cw_dispatch_wz()
1630 adj = ((lbalen->lba + lba) * cbe_lun->blocksize + in ctl_be_block_cw_dispatch_wz()
1631 seglen - pbo) % pb; in ctl_be_block_cw_dispatch_wz()
1633 seglen -= adj; in ctl_be_block_cw_dispatch_wz()
1635 seglen -= seglen % cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wz()
1637 seglen -= seglen % cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wz()
1638 ctl_alloc_seg(softc, &beio->sg_segs[i], seglen); in ctl_be_block_cw_dispatch_wz()
1641 beio->sg_segs[i].addr, beio->sg_segs[i].len); in ctl_be_block_cw_dispatch_wz()
1643 beio->num_segs++; in ctl_be_block_cw_dispatch_wz()
1644 len_left -= seglen; in ctl_be_block_cw_dispatch_wz()
1646 memset(beio->sg_segs[i].addr, 0, seglen); in ctl_be_block_cw_dispatch_wz()
1647 lba += seglen / cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wz()
1650 beio->io_offset = lbalen->lba * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wz()
1651 beio->io_len = lba * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_wz()
1655 lbalen->lba += lba; in ctl_be_block_cw_dispatch_wz()
1656 lbalen->len -= lba; in ctl_be_block_cw_dispatch_wz()
1657 beio->beio_cont = ctl_be_block_cw_done_ws; in ctl_be_block_cw_dispatch_wz()
1660 be_lun->dispatch(be_lun, beio); in ctl_be_block_cw_dispatch_wz()
1665 union ctl_io *io) in ctl_be_block_cw_dispatch_dsm() argument
1667 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_cw_dispatch_dsm()
1674 CTL_IO_ASSERT(io, NVME); in ctl_be_block_cw_dispatch_dsm()
1676 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_cw_dispatch_dsm()
1678 if (be_lun->unmap == NULL) { in ctl_be_block_cw_dispatch_dsm()
1680 ctl_nvme_set_success(&io->nvmeio); in ctl_be_block_cw_dispatch_dsm()
1681 ctl_config_write_done(io); in ctl_be_block_cw_dispatch_dsm()
1685 ranges = le32toh(io->nvmeio.cmd.cdw10) & 0xff; in ctl_be_block_cw_dispatch_dsm()
1686 r = (struct nvme_dsm_range *)io->nvmeio.kern_data_ptr; in ctl_be_block_cw_dispatch_dsm()
1689 for (i = DSM_RANGE(io); i < ranges; i++) { in ctl_be_block_cw_dispatch_dsm()
1697 ctl_nvme_set_success(&io->nvmeio); in ctl_be_block_cw_dispatch_dsm()
1698 ctl_config_write_done(io); in ctl_be_block_cw_dispatch_dsm()
1704 DSM_RANGE(io) = i + 1; in ctl_be_block_cw_dispatch_dsm()
1705 beio->beio_cont = ctl_be_block_cw_done_ws; in ctl_be_block_cw_dispatch_dsm()
1711 beio->io_offset = lba * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_dsm()
1712 beio->io_len = (uint64_t)num_blocks * cbe_lun->blocksize; in ctl_be_block_cw_dispatch_dsm()
1713 beio->bio_cmd = BIO_DELETE; in ctl_be_block_cw_dispatch_dsm()
1714 beio->ds_trans_type = DEVSTAT_FREE; in ctl_be_block_cw_dispatch_dsm()
1716 be_lun->unmap(be_lun, beio); in ctl_be_block_cw_dispatch_dsm()
1722 union ctl_io *io; in ctl_be_block_scsi_cr_done() local
1724 io = beio->io; in ctl_be_block_scsi_cr_done()
1726 ctl_config_read_done(io); in ctl_be_block_scsi_cr_done()
1731 union ctl_io *io) in ctl_be_block_scsi_cr_dispatch() argument
1738 softc = be_lun->softc; in ctl_be_block_scsi_cr_dispatch()
1740 beio->io = io; in ctl_be_block_scsi_cr_dispatch()
1741 beio->lun = be_lun; in ctl_be_block_scsi_cr_dispatch()
1742 beio->beio_cont = ctl_be_block_scsi_cr_done; in ctl_be_block_scsi_cr_dispatch()
1743 PRIV(io)->ptr = (void *)beio; in ctl_be_block_scsi_cr_dispatch()
1745 switch (io->scsiio.cdb[0]) { in ctl_be_block_scsi_cr_dispatch()
1747 beio->bio_cmd = -1; in ctl_be_block_scsi_cr_dispatch()
1748 beio->ds_trans_type = DEVSTAT_NO_DATA; in ctl_be_block_scsi_cr_dispatch()
1749 beio->ds_tag_type = DEVSTAT_TAG_ORDERED; in ctl_be_block_scsi_cr_dispatch()
1750 beio->io_len = 0; in ctl_be_block_scsi_cr_dispatch()
1751 if (be_lun->get_lba_status) in ctl_be_block_scsi_cr_dispatch()
1752 be_lun->get_lba_status(be_lun, beio); in ctl_be_block_scsi_cr_dispatch()
1757 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); in ctl_be_block_scsi_cr_dispatch()
1764 union ctl_io *io) in ctl_be_block_nvme_cr_dispatch() argument
1770 MPASS(io->nvmeio.cmd.opc == NVME_OPC_IDENTIFY); in ctl_be_block_nvme_cr_dispatch()
1772 cns = le32toh(io->nvmeio.cmd.cdw10) & 0xff; in ctl_be_block_nvme_cr_dispatch()
1775 ctl_be_block_namespace_data(be_lun, io); in ctl_be_block_nvme_cr_dispatch()
1778 ctl_be_block_nvme_ids(be_lun, io); in ctl_be_block_nvme_cr_dispatch()
1787 union ctl_io *io) in ctl_be_block_cr_dispatch() argument
1789 switch (io->io_hdr.io_type) { in ctl_be_block_cr_dispatch()
1791 ctl_be_block_scsi_cr_dispatch(be_lun, io); in ctl_be_block_cr_dispatch()
1794 ctl_be_block_nvme_cr_dispatch(be_lun, io); in ctl_be_block_cr_dispatch()
1804 union ctl_io *io; in ctl_be_block_cw_done() local
1806 io = beio->io; in ctl_be_block_cw_done()
1808 ctl_config_write_done(io); in ctl_be_block_cw_done()
1813 union ctl_io *io) in ctl_be_block_scsi_cw_dispatch() argument
1819 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_scsi_cw_dispatch()
1821 switch (io->scsiio.tag_type) { in ctl_be_block_scsi_cw_dispatch()
1823 beio->ds_tag_type = DEVSTAT_TAG_ORDERED; in ctl_be_block_scsi_cw_dispatch()
1826 beio->ds_tag_type = DEVSTAT_TAG_HEAD; in ctl_be_block_scsi_cw_dispatch()
1832 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; in ctl_be_block_scsi_cw_dispatch()
1836 switch (io->scsiio.cdb[0]) { in ctl_be_block_scsi_cw_dispatch()
1839 ctl_be_block_cw_dispatch_sync(be_lun, io); in ctl_be_block_scsi_cw_dispatch()
1843 ctl_be_block_cw_dispatch_ws(be_lun, io); in ctl_be_block_scsi_cw_dispatch()
1846 ctl_be_block_cw_dispatch_unmap(be_lun, io); in ctl_be_block_scsi_cw_dispatch()
1849 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); in ctl_be_block_scsi_cw_dispatch()
1856 union ctl_io *io) in ctl_be_block_nvme_cw_dispatch() argument
1862 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_nvme_cw_dispatch()
1863 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; in ctl_be_block_nvme_cw_dispatch()
1865 switch (io->nvmeio.cmd.opc) { in ctl_be_block_nvme_cw_dispatch()
1867 ctl_be_block_cw_dispatch_flush(be_lun, io); in ctl_be_block_nvme_cw_dispatch()
1870 ctl_be_block_cw_dispatch_wu(be_lun, io); in ctl_be_block_nvme_cw_dispatch()
1873 ctl_be_block_cw_dispatch_wz(be_lun, io); in ctl_be_block_nvme_cw_dispatch()
1876 ctl_be_block_cw_dispatch_dsm(be_lun, io); in ctl_be_block_nvme_cw_dispatch()
1885 union ctl_io *io) in ctl_be_block_cw_dispatch() argument
1890 softc = be_lun->softc; in ctl_be_block_cw_dispatch()
1892 beio->io = io; in ctl_be_block_cw_dispatch()
1893 beio->lun = be_lun; in ctl_be_block_cw_dispatch()
1894 beio->beio_cont = ctl_be_block_cw_done; in ctl_be_block_cw_dispatch()
1895 PRIV(io)->ptr = (void *)beio; in ctl_be_block_cw_dispatch()
1897 switch (io->io_hdr.io_type) { in ctl_be_block_cw_dispatch()
1899 ctl_be_block_scsi_cw_dispatch(be_lun, io); in ctl_be_block_cw_dispatch()
1902 ctl_be_block_nvme_cw_dispatch(be_lun, io); in ctl_be_block_cw_dispatch()
1918 union ctl_io *io; in ctl_be_block_next() local
1920 io = beio->io; in ctl_be_block_next()
1921 be_lun = beio->lun; in ctl_be_block_next()
1923 if ((io->io_hdr.flags & CTL_FLAG_ABORT) || in ctl_be_block_next()
1924 ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && in ctl_be_block_next()
1925 (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { in ctl_be_block_next()
1926 ctl_data_submit_done(io); in ctl_be_block_next()
1930 io->io_hdr.status &= ~CTL_STATUS_MASK; in ctl_be_block_next()
1931 io->io_hdr.status |= CTL_STATUS_NONE; in ctl_be_block_next()
1933 mtx_lock(&be_lun->queue_lock); in ctl_be_block_next()
1934 STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); in ctl_be_block_next()
1935 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_next()
1936 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); in ctl_be_block_next()
1941 union ctl_io *io) in ctl_be_block_dispatch() argument
1943 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_dispatch()
1951 softc = be_lun->softc; in ctl_be_block_dispatch()
1955 lbalen = ARGS(io); in ctl_be_block_dispatch()
1956 if (lbalen->flags & CTL_LLF_WRITE) { in ctl_be_block_dispatch()
1963 beio->io = io; in ctl_be_block_dispatch()
1964 beio->lun = be_lun; in ctl_be_block_dispatch()
1965 bptrlen = PRIV(io); in ctl_be_block_dispatch()
1966 bptrlen->ptr = (void *)beio; in ctl_be_block_dispatch()
1968 switch (io->io_hdr.io_type) { in ctl_be_block_dispatch()
1970 switch (io->scsiio.tag_type) { in ctl_be_block_dispatch()
1972 beio->ds_tag_type = DEVSTAT_TAG_ORDERED; in ctl_be_block_dispatch()
1975 beio->ds_tag_type = DEVSTAT_TAG_HEAD; in ctl_be_block_dispatch()
1981 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; in ctl_be_block_dispatch()
1986 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; in ctl_be_block_dispatch()
1992 if (lbalen->flags & CTL_LLF_WRITE) { in ctl_be_block_dispatch()
1993 beio->bio_cmd = BIO_WRITE; in ctl_be_block_dispatch()
1994 beio->ds_trans_type = DEVSTAT_WRITE; in ctl_be_block_dispatch()
1996 beio->bio_cmd = BIO_READ; in ctl_be_block_dispatch()
1997 beio->ds_trans_type = DEVSTAT_READ; in ctl_be_block_dispatch()
2001 (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", in ctl_be_block_dispatch()
2002 (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len); in ctl_be_block_dispatch()
2004 if (lbalen->flags & CTL_LLF_COMPARE) { in ctl_be_block_dispatch()
2005 beio->two_sglists = 1; in ctl_be_block_dispatch()
2008 lbas = MIN(lbalen->len - bptrlen->len, lbas / cbe_lun->blocksize); in ctl_be_block_dispatch()
2009 beio->io_offset = (lbalen->lba + bptrlen->len) * cbe_lun->blocksize; in ctl_be_block_dispatch()
2010 beio->io_len = lbas * cbe_lun->blocksize; in ctl_be_block_dispatch()
2011 bptrlen->len += lbas; in ctl_be_block_dispatch()
2013 for (i = 0, len_left = beio->io_len; len_left > 0; i++) { in ctl_be_block_dispatch()
2020 ctl_alloc_seg(softc, &beio->sg_segs[i], in ctl_be_block_dispatch()
2024 beio->sg_segs[i].addr, beio->sg_segs[i].len); in ctl_be_block_dispatch()
2027 if (beio->two_sglists) { in ctl_be_block_dispatch()
2029 &beio->sg_segs[i + CTLBLK_HALF_SEGS], in ctl_be_block_dispatch()
2030 beio->sg_segs[i].len); in ctl_be_block_dispatch()
2033 beio->num_segs++; in ctl_be_block_dispatch()
2034 len_left -= beio->sg_segs[i].len; in ctl_be_block_dispatch()
2036 if (bptrlen->len < lbalen->len) in ctl_be_block_dispatch()
2037 beio->beio_cont = ctl_be_block_next; in ctl_be_block_dispatch()
2038 ctl_set_be_move_done(io, ctl_be_block_move_done); in ctl_be_block_dispatch()
2040 if (beio->two_sglists) in ctl_be_block_dispatch()
2041 ctl_set_kern_data_ptr(io, &beio->sg_segs[CTLBLK_HALF_SEGS]); in ctl_be_block_dispatch()
2043 ctl_set_kern_data_ptr(io, beio->sg_segs); in ctl_be_block_dispatch()
2044 ctl_set_kern_data_len(io, beio->io_len); in ctl_be_block_dispatch()
2045 ctl_set_kern_sg_entries(io, beio->num_segs); in ctl_be_block_dispatch()
2046 ctl_set_kern_data_ref(io, ctl_refcnt_beio); in ctl_be_block_dispatch()
2047 ctl_set_kern_data_arg(io, beio); in ctl_be_block_dispatch()
2048 io->io_hdr.flags |= CTL_FLAG_ALLOCATED; in ctl_be_block_dispatch()
2055 if (beio->bio_cmd == BIO_READ) { in ctl_be_block_dispatch()
2057 be_lun->dispatch(be_lun, beio); in ctl_be_block_dispatch()
2060 ctl_datamove(io); in ctl_be_block_dispatch()
2068 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_worker()
2069 union ctl_io *io; in ctl_be_block_worker() local
2075 * CTL_LUN_FLAG_NO_MEDIA status here -- it is result of a race, in ctl_be_block_worker()
2079 mtx_lock(&be_lun->queue_lock); in ctl_be_block_worker()
2080 io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue); in ctl_be_block_worker()
2081 if (io != NULL) { in ctl_be_block_worker()
2083 STAILQ_REMOVE_HEAD(&be_lun->datamove_queue, links); in ctl_be_block_worker()
2084 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_worker()
2085 beio = (struct ctl_be_block_io *)PRIV(io)->ptr; in ctl_be_block_worker()
2086 if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { in ctl_be_block_worker()
2087 ctl_io_set_busy(io); in ctl_be_block_worker()
2091 be_lun->dispatch(be_lun, beio); in ctl_be_block_worker()
2094 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue); in ctl_be_block_worker()
2095 if (io != NULL) { in ctl_be_block_worker()
2097 STAILQ_REMOVE_HEAD(&be_lun->config_write_queue, links); in ctl_be_block_worker()
2098 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_worker()
2099 if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { in ctl_be_block_worker()
2100 ctl_io_set_busy(io); in ctl_be_block_worker()
2101 ctl_config_write_done(io); in ctl_be_block_worker()
2104 ctl_be_block_cw_dispatch(be_lun, io); in ctl_be_block_worker()
2107 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue); in ctl_be_block_worker()
2108 if (io != NULL) { in ctl_be_block_worker()
2110 STAILQ_REMOVE_HEAD(&be_lun->config_read_queue, links); in ctl_be_block_worker()
2111 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_worker()
2112 if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { in ctl_be_block_worker()
2113 ctl_io_set_busy(io); in ctl_be_block_worker()
2114 ctl_config_read_done(io); in ctl_be_block_worker()
2117 ctl_be_block_cr_dispatch(be_lun, io); in ctl_be_block_worker()
2120 io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue); in ctl_be_block_worker()
2121 if (io != NULL) { in ctl_be_block_worker()
2123 STAILQ_REMOVE_HEAD(&be_lun->input_queue, links); in ctl_be_block_worker()
2124 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_worker()
2125 if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) { in ctl_be_block_worker()
2126 ctl_io_set_busy(io); in ctl_be_block_worker()
2127 ctl_data_submit_done(io); in ctl_be_block_worker()
2130 ctl_be_block_dispatch(be_lun, io); in ctl_be_block_worker()
2138 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_worker()
2149 ctl_be_block_submit(union ctl_io *io) in ctl_be_block_submit() argument
2155 be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io); in ctl_be_block_submit()
2157 CTL_IO_ASSERT(io, SCSI, NVME); in ctl_be_block_submit()
2159 PRIV(io)->len = 0; in ctl_be_block_submit()
2161 mtx_lock(&be_lun->queue_lock); in ctl_be_block_submit()
2162 STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); in ctl_be_block_submit()
2163 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_submit()
2164 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); in ctl_be_block_submit()
2183 switch (lun_req->reqtype) { in ctl_be_block_ioctl()
2194 lun_req->status = CTL_LUN_ERROR; in ctl_be_block_ioctl()
2195 snprintf(lun_req->error_str, sizeof(lun_req->error_str), in ctl_be_block_ioctl()
2197 lun_req->reqtype); in ctl_be_block_ioctl()
2222 cbe_lun = &be_lun->cbe_lun; in ctl_be_block_open_file()
2223 file_data = &be_lun->backend.file; in ctl_be_block_open_file()
2224 params = &be_lun->params; in ctl_be_block_open_file()
2226 be_lun->dev_type = CTL_BE_BLOCK_FILE; in ctl_be_block_open_file()
2227 be_lun->dispatch = ctl_be_block_dispatch_file; in ctl_be_block_open_file()
2228 be_lun->lun_flush = ctl_be_block_flush_file; in ctl_be_block_open_file()
2229 be_lun->get_lba_status = ctl_be_block_gls_file; in ctl_be_block_open_file()
2230 be_lun->getattr = ctl_be_block_getattr_file; in ctl_be_block_open_file()
2231 be_lun->unmap = ctl_be_block_unmap_file; in ctl_be_block_open_file()
2232 cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP; in ctl_be_block_open_file()
2234 error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); in ctl_be_block_open_file()
2236 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open_file()
2238 be_lun->dev_path); in ctl_be_block_open_file()
2242 error = VOP_PATHCONF(be_lun->vn, _PC_DEALLOC_PRESENT, &pconf); in ctl_be_block_open_file()
2244 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open_file()
2246 be_lun->dev_path); in ctl_be_block_open_file()
2250 cbe_lun->flags |= CTL_LUN_FLAG_UNMAP; in ctl_be_block_open_file()
2252 file_data->cred = crhold(curthread->td_ucred); in ctl_be_block_open_file()
2253 if (params->lun_size_bytes != 0) in ctl_be_block_open_file()
2254 be_lun->size_bytes = params->lun_size_bytes; in ctl_be_block_open_file()
2256 be_lun->size_bytes = vattr.va_size; in ctl_be_block_open_file()
2262 * logical block size -- report it as physical block size. in ctl_be_block_open_file()
2264 if (params->blocksize_bytes != 0) in ctl_be_block_open_file()
2265 cbe_lun->blocksize = params->blocksize_bytes; in ctl_be_block_open_file()
2266 else if (cbe_lun->lun_type == T_CDROM) in ctl_be_block_open_file()
2267 cbe_lun->blocksize = 2048; in ctl_be_block_open_file()
2269 cbe_lun->blocksize = 512; in ctl_be_block_open_file()
2270 be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize; in ctl_be_block_open_file()
2271 cbe_lun->maxlba = (be_lun->size_blocks == 0) ? in ctl_be_block_open_file()
2272 0 : (be_lun->size_blocks - 1); in ctl_be_block_open_file()
2277 value = dnvlist_get_string(cbe_lun->options, "pblocksize", NULL); in ctl_be_block_open_file()
2280 value = dnvlist_get_string(cbe_lun->options, "pblockoffset", NULL); in ctl_be_block_open_file()
2283 pss = ps / cbe_lun->blocksize; in ctl_be_block_open_file()
2284 pos = po / cbe_lun->blocksize; in ctl_be_block_open_file()
2285 if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) && in ctl_be_block_open_file()
2286 ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) { in ctl_be_block_open_file()
2287 cbe_lun->pblockexp = fls(pss) - 1; in ctl_be_block_open_file()
2288 cbe_lun->pblockoff = (pss - pos) % pss; in ctl_be_block_open_file()
2291 value = dnvlist_get_string(cbe_lun->options, "ublocksize", NULL); in ctl_be_block_open_file()
2294 value = dnvlist_get_string(cbe_lun->options, "ublockoffset", NULL); in ctl_be_block_open_file()
2297 uss = us / cbe_lun->blocksize; in ctl_be_block_open_file()
2298 uos = uo / cbe_lun->blocksize; in ctl_be_block_open_file()
2299 if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) && in ctl_be_block_open_file()
2300 ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) { in ctl_be_block_open_file()
2301 cbe_lun->ublockexp = fls(uss) - 1; in ctl_be_block_open_file()
2302 cbe_lun->ublockoff = (uss - uos) % uss; in ctl_be_block_open_file()
2309 if (be_lun->size_bytes < cbe_lun->blocksize) { in ctl_be_block_open_file()
2311 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open_file()
2312 "file %s size %ju < block size %u", be_lun->dev_path, in ctl_be_block_open_file()
2313 (uintmax_t)be_lun->size_bytes, cbe_lun->blocksize); in ctl_be_block_open_file()
2316 cbe_lun->opttxferlen = CTLBLK_MAX_IO_SIZE / cbe_lun->blocksize; in ctl_be_block_open_file()
2323 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_open_dev()
2331 params = &be_lun->params; in ctl_be_block_open_dev()
2333 be_lun->dev_type = CTL_BE_BLOCK_DEV; in ctl_be_block_open_dev()
2334 csw = devvn_refthread(be_lun->vn, &dev, &ref); in ctl_be_block_open_dev()
2337 if (strcmp(csw->d_name, "zvol") == 0) { in ctl_be_block_open_dev()
2338 be_lun->dispatch = ctl_be_block_dispatch_zvol; in ctl_be_block_open_dev()
2339 be_lun->get_lba_status = ctl_be_block_gls_zvol; in ctl_be_block_open_dev()
2342 be_lun->dispatch = ctl_be_block_dispatch_dev; in ctl_be_block_open_dev()
2343 be_lun->get_lba_status = NULL; in ctl_be_block_open_dev()
2345 maxio = dev->si_iosize_max; in ctl_be_block_open_dev()
2351 be_lun->lun_flush = ctl_be_block_flush_dev; in ctl_be_block_open_dev()
2352 be_lun->getattr = ctl_be_block_getattr_dev; in ctl_be_block_open_dev()
2353 be_lun->unmap = ctl_be_block_unmap_dev; in ctl_be_block_open_dev()
2355 if (!csw->d_ioctl) { in ctl_be_block_open_dev()
2357 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open_dev()
2358 "no d_ioctl for device %s!", be_lun->dev_path); in ctl_be_block_open_dev()
2362 error = csw->d_ioctl(dev, DIOCGSECTORSIZE, (caddr_t)&tmp, FREAD, in ctl_be_block_open_dev()
2366 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open_dev()
2368 "on %s!", error, be_lun->dev_path); in ctl_be_block_open_dev()
2378 if ((params->blocksize_bytes != 0) && in ctl_be_block_open_dev()
2379 (params->blocksize_bytes >= tmp)) { in ctl_be_block_open_dev()
2380 if (params->blocksize_bytes % tmp == 0) { in ctl_be_block_open_dev()
2381 cbe_lun->blocksize = params->blocksize_bytes; in ctl_be_block_open_dev()
2384 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open_dev()
2387 params->blocksize_bytes, tmp); in ctl_be_block_open_dev()
2390 } else if (params->blocksize_bytes != 0) { in ctl_be_block_open_dev()
2392 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open_dev()
2394 "blocksize %u", params->blocksize_bytes, tmp); in ctl_be_block_open_dev()
2396 } else if (cbe_lun->lun_type == T_CDROM) in ctl_be_block_open_dev()
2397 cbe_lun->blocksize = MAX(tmp, 2048); in ctl_be_block_open_dev()
2399 cbe_lun->blocksize = tmp; in ctl_be_block_open_dev()
2401 error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&otmp, FREAD, in ctl_be_block_open_dev()
2405 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open_dev()
2408 be_lun->dev_path); in ctl_be_block_open_dev()
2412 if (params->lun_size_bytes != 0) { in ctl_be_block_open_dev()
2413 if (params->lun_size_bytes > otmp) { in ctl_be_block_open_dev()
2415 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open_dev()
2418 (uintmax_t)params->lun_size_bytes, in ctl_be_block_open_dev()
2423 be_lun->size_bytes = params->lun_size_bytes; in ctl_be_block_open_dev()
2425 be_lun->size_bytes = otmp; in ctl_be_block_open_dev()
2426 be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize; in ctl_be_block_open_dev()
2427 cbe_lun->maxlba = (be_lun->size_blocks == 0) ? in ctl_be_block_open_dev()
2428 0 : (be_lun->size_blocks - 1); in ctl_be_block_open_dev()
2430 error = csw->d_ioctl(dev, DIOCGSTRIPESIZE, (caddr_t)&ps, FREAD, in ctl_be_block_open_dev()
2435 error = csw->d_ioctl(dev, DIOCGSTRIPEOFFSET, (caddr_t)&po, in ctl_be_block_open_dev()
2443 value = dnvlist_get_string(cbe_lun->options, "pblocksize", NULL); in ctl_be_block_open_dev()
2446 value = dnvlist_get_string(cbe_lun->options, "pblockoffset", NULL); in ctl_be_block_open_dev()
2449 pss = ps / cbe_lun->blocksize; in ctl_be_block_open_dev()
2450 pos = po / cbe_lun->blocksize; in ctl_be_block_open_dev()
2451 if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) && in ctl_be_block_open_dev()
2452 ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) { in ctl_be_block_open_dev()
2453 cbe_lun->pblockexp = fls(pss) - 1; in ctl_be_block_open_dev()
2454 cbe_lun->pblockoff = (pss - pos) % pss; in ctl_be_block_open_dev()
2457 value = dnvlist_get_string(cbe_lun->options, "ublocksize", NULL); in ctl_be_block_open_dev()
2460 value = dnvlist_get_string(cbe_lun->options, "ublockoffset", NULL); in ctl_be_block_open_dev()
2463 uss = us / cbe_lun->blocksize; in ctl_be_block_open_dev()
2464 uos = uo / cbe_lun->blocksize; in ctl_be_block_open_dev()
2465 if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) && in ctl_be_block_open_dev()
2466 ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) { in ctl_be_block_open_dev()
2467 cbe_lun->ublockexp = fls(uss) - 1; in ctl_be_block_open_dev()
2468 cbe_lun->ublockoff = (uss - uos) % uss; in ctl_be_block_open_dev()
2471 cbe_lun->atomicblock = atomic / cbe_lun->blocksize; in ctl_be_block_open_dev()
2472 cbe_lun->opttxferlen = maxio / cbe_lun->blocksize; in ctl_be_block_open_dev()
2474 if (be_lun->dispatch == ctl_be_block_dispatch_zvol) { in ctl_be_block_open_dev()
2481 error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD, in ctl_be_block_open_dev()
2485 value = dnvlist_get_string(cbe_lun->options, "unmap", NULL); in ctl_be_block_open_dev()
2489 cbe_lun->flags |= CTL_LUN_FLAG_UNMAP; in ctl_be_block_open_dev()
2491 cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP; in ctl_be_block_open_dev()
2500 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_close()
2503 if (be_lun->vn) { in ctl_be_block_close()
2505 if ((cbe_lun->flags & CTL_LUN_FLAG_READONLY) == 0) in ctl_be_block_close()
2507 (void)vn_close(be_lun->vn, flags, NOCRED, curthread); in ctl_be_block_close()
2508 be_lun->vn = NULL; in ctl_be_block_close()
2510 switch (be_lun->dev_type) { in ctl_be_block_close()
2514 if (be_lun->backend.file.cred != NULL) { in ctl_be_block_close()
2515 crfree(be_lun->backend.file.cred); in ctl_be_block_close()
2516 be_lun->backend.file.cred = NULL; in ctl_be_block_close()
2522 panic("Unexpected backend type %d", be_lun->dev_type); in ctl_be_block_close()
2525 be_lun->dev_type = CTL_BE_BLOCK_NONE; in ctl_be_block_close()
2533 struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; in ctl_be_block_open()
2540 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open()
2546 value = dnvlist_get_string(cbe_lun->options, "file", NULL); in ctl_be_block_open()
2548 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open()
2552 free(be_lun->dev_path, M_CTLBLK); in ctl_be_block_open()
2553 be_lun->dev_path = strdup(value, M_CTLBLK); in ctl_be_block_open()
2556 value = dnvlist_get_string(cbe_lun->options, "readonly", NULL); in ctl_be_block_open()
2560 } else if (cbe_lun->lun_type == T_DIRECT) in ctl_be_block_open()
2564 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path); in ctl_be_block_open()
2577 if (be_lun->dev_path[0] != '/') { in ctl_be_block_open()
2581 be_lun->dev_path); in ctl_be_block_open()
2582 free(be_lun->dev_path, M_CTLBLK); in ctl_be_block_open()
2583 be_lun->dev_path = dev_name; in ctl_be_block_open()
2586 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open()
2587 "error opening %s: %d", be_lun->dev_path, error); in ctl_be_block_open()
2591 cbe_lun->flags &= ~CTL_LUN_FLAG_READONLY; in ctl_be_block_open()
2593 cbe_lun->flags |= CTL_LUN_FLAG_READONLY; in ctl_be_block_open()
2596 be_lun->vn = nd.ni_vp; in ctl_be_block_open()
2599 if (vn_isdisk_error(be_lun->vn, &error)) { in ctl_be_block_open()
2601 } else if (be_lun->vn->v_type == VREG) { in ctl_be_block_open()
2605 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_open()
2606 "%s is not a disk or plain file", be_lun->dev_path); in ctl_be_block_open()
2608 VOP_UNLOCK(be_lun->vn); in ctl_be_block_open()
2612 cbe_lun->serseq = CTL_LUN_SERSEQ_OFF; in ctl_be_block_open()
2613 if (be_lun->dispatch != ctl_be_block_dispatch_dev) in ctl_be_block_open()
2614 cbe_lun->serseq = CTL_LUN_SERSEQ_SOFT; in ctl_be_block_open()
2615 value = dnvlist_get_string(cbe_lun->options, "serseq", NULL); in ctl_be_block_open()
2617 cbe_lun->serseq = CTL_LUN_SERSEQ_ON; in ctl_be_block_open()
2619 cbe_lun->serseq = CTL_LUN_SERSEQ_READ; in ctl_be_block_open()
2621 cbe_lun->serseq = CTL_LUN_SERSEQ_SOFT; in ctl_be_block_open()
2623 cbe_lun->serseq = CTL_LUN_SERSEQ_OFF; in ctl_be_block_open()
2639 params = &req->reqdata.create; in ctl_be_block_create()
2641 req->status = CTL_LUN_OK; in ctl_be_block_create()
2644 cbe_lun = &be_lun->cbe_lun; in ctl_be_block_create()
2645 be_lun->params = req->reqdata.create; in ctl_be_block_create()
2646 be_lun->softc = softc; in ctl_be_block_create()
2647 STAILQ_INIT(&be_lun->input_queue); in ctl_be_block_create()
2648 STAILQ_INIT(&be_lun->config_read_queue); in ctl_be_block_create()
2649 STAILQ_INIT(&be_lun->config_write_queue); in ctl_be_block_create()
2650 STAILQ_INIT(&be_lun->datamove_queue); in ctl_be_block_create()
2651 mtx_init(&be_lun->io_lock, "ctlblock io", NULL, MTX_DEF); in ctl_be_block_create()
2652 mtx_init(&be_lun->queue_lock, "ctlblock queue", NULL, MTX_DEF); in ctl_be_block_create()
2653 cbe_lun->options = nvlist_clone(req->args_nvl); in ctl_be_block_create()
2655 if (params->flags & CTL_LUN_FLAG_DEV_TYPE) in ctl_be_block_create()
2656 cbe_lun->lun_type = params->device_type; in ctl_be_block_create()
2658 cbe_lun->lun_type = T_DIRECT; in ctl_be_block_create()
2659 be_lun->flags = 0; in ctl_be_block_create()
2660 cbe_lun->flags = 0; in ctl_be_block_create()
2661 value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL); in ctl_be_block_create()
2664 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; in ctl_be_block_create()
2665 } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF) in ctl_be_block_create()
2666 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; in ctl_be_block_create()
2668 if (cbe_lun->lun_type == T_DIRECT || in ctl_be_block_create()
2669 cbe_lun->lun_type == T_CDROM) { in ctl_be_block_create()
2670 be_lun->size_bytes = params->lun_size_bytes; in ctl_be_block_create()
2671 if (params->blocksize_bytes != 0) in ctl_be_block_create()
2672 cbe_lun->blocksize = params->blocksize_bytes; in ctl_be_block_create()
2673 else if (cbe_lun->lun_type == T_CDROM) in ctl_be_block_create()
2674 cbe_lun->blocksize = 2048; in ctl_be_block_create()
2676 cbe_lun->blocksize = 512; in ctl_be_block_create()
2677 be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize; in ctl_be_block_create()
2678 cbe_lun->maxlba = (be_lun->size_blocks == 0) ? in ctl_be_block_create()
2679 0 : (be_lun->size_blocks - 1); in ctl_be_block_create()
2681 if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) || in ctl_be_block_create()
2682 control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) { in ctl_be_block_create()
2686 req->status = CTL_LUN_WARNING; in ctl_be_block_create()
2694 value = dnvlist_get_string(cbe_lun->options, "num_threads", NULL); in ctl_be_block_create()
2704 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_create()
2712 if (be_lun->vn == NULL) in ctl_be_block_create()
2713 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; in ctl_be_block_create()
2715 params->lun_size_bytes = be_lun->size_bytes; in ctl_be_block_create()
2716 params->blocksize_bytes = cbe_lun->blocksize; in ctl_be_block_create()
2717 if (params->flags & CTL_LUN_FLAG_ID_REQ) { in ctl_be_block_create()
2718 cbe_lun->req_lun_id = params->req_lun_id; in ctl_be_block_create()
2719 cbe_lun->flags |= CTL_LUN_FLAG_ID_REQ; in ctl_be_block_create()
2721 cbe_lun->req_lun_id = 0; in ctl_be_block_create()
2723 cbe_lun->lun_shutdown = ctl_be_block_lun_shutdown; in ctl_be_block_create()
2724 cbe_lun->be = &ctl_be_block_driver; in ctl_be_block_create()
2726 if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) { in ctl_be_block_create()
2728 softc->num_luns); in ctl_be_block_create()
2729 strncpy((char *)cbe_lun->serial_num, tmpstr, in ctl_be_block_create()
2730 MIN(sizeof(cbe_lun->serial_num), sizeof(tmpstr))); in ctl_be_block_create()
2733 strncpy((char *)params->serial_num, tmpstr, in ctl_be_block_create()
2734 MIN(sizeof(params->serial_num), sizeof(tmpstr))); in ctl_be_block_create()
2736 strncpy((char *)cbe_lun->serial_num, params->serial_num, in ctl_be_block_create()
2737 MIN(sizeof(cbe_lun->serial_num), in ctl_be_block_create()
2738 sizeof(params->serial_num))); in ctl_be_block_create()
2740 if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) { in ctl_be_block_create()
2741 snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%04d", softc->num_luns); in ctl_be_block_create()
2742 strncpy((char *)cbe_lun->device_id, tmpstr, in ctl_be_block_create()
2743 MIN(sizeof(cbe_lun->device_id), sizeof(tmpstr))); in ctl_be_block_create()
2746 strncpy((char *)params->device_id, tmpstr, in ctl_be_block_create()
2747 MIN(sizeof(params->device_id), sizeof(tmpstr))); in ctl_be_block_create()
2749 strncpy((char *)cbe_lun->device_id, params->device_id, in ctl_be_block_create()
2750 MIN(sizeof(cbe_lun->device_id), in ctl_be_block_create()
2751 sizeof(params->device_id))); in ctl_be_block_create()
2754 TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun); in ctl_be_block_create()
2756 be_lun->io_taskqueue = taskqueue_create("ctlblocktq", M_WAITOK, in ctl_be_block_create()
2757 taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue); in ctl_be_block_create()
2759 if (be_lun->io_taskqueue == NULL) { in ctl_be_block_create()
2760 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_create()
2769 * vnode interface is designed to be a blocking interface. For the in ctl_be_block_create()
2779 retval = taskqueue_start_threads_in_proc(&be_lun->io_taskqueue, in ctl_be_block_create()
2782 /*proc*/control_softc->ctl_proc, in ctl_be_block_create()
2788 be_lun->num_threads = num_threads; in ctl_be_block_create()
2790 retval = ctl_add_lun(&be_lun->cbe_lun); in ctl_be_block_create()
2792 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_create()
2799 be_lun->disk_stats = devstat_new_entry("cbb", cbe_lun->lun_id, in ctl_be_block_create()
2800 cbe_lun->blocksize, in ctl_be_block_create()
2802 cbe_lun->lun_type in ctl_be_block_create()
2806 mtx_lock(&softc->lock); in ctl_be_block_create()
2807 softc->num_luns++; in ctl_be_block_create()
2808 SLIST_INSERT_HEAD(&softc->lun_list, be_lun, links); in ctl_be_block_create()
2809 mtx_unlock(&softc->lock); in ctl_be_block_create()
2811 params->req_lun_id = cbe_lun->lun_id; in ctl_be_block_create()
2816 req->status = CTL_LUN_ERROR; in ctl_be_block_create()
2818 if (be_lun->io_taskqueue != NULL) in ctl_be_block_create()
2819 taskqueue_free(be_lun->io_taskqueue); in ctl_be_block_create()
2821 if (be_lun->dev_path != NULL) in ctl_be_block_create()
2822 free(be_lun->dev_path, M_CTLBLK); in ctl_be_block_create()
2823 nvlist_destroy(cbe_lun->options); in ctl_be_block_create()
2824 mtx_destroy(&be_lun->queue_lock); in ctl_be_block_create()
2825 mtx_destroy(&be_lun->io_lock); in ctl_be_block_create()
2839 params = &req->reqdata.rm; in ctl_be_block_rm()
2841 sx_xlock(&softc->modify_lock); in ctl_be_block_rm()
2842 mtx_lock(&softc->lock); in ctl_be_block_rm()
2843 SLIST_FOREACH(be_lun, &softc->lun_list, links) { in ctl_be_block_rm()
2844 if (be_lun->cbe_lun.lun_id == params->lun_id) { in ctl_be_block_rm()
2845 SLIST_REMOVE(&softc->lun_list, be_lun, in ctl_be_block_rm()
2847 softc->num_luns--; in ctl_be_block_rm()
2851 mtx_unlock(&softc->lock); in ctl_be_block_rm()
2852 sx_xunlock(&softc->modify_lock); in ctl_be_block_rm()
2854 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_rm()
2856 params->lun_id); in ctl_be_block_rm()
2859 cbe_lun = &be_lun->cbe_lun; in ctl_be_block_rm()
2861 if (be_lun->vn != NULL) { in ctl_be_block_rm()
2862 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; in ctl_be_block_rm()
2864 taskqueue_drain_all(be_lun->io_taskqueue); in ctl_be_block_rm()
2868 mtx_lock(&softc->lock); in ctl_be_block_rm()
2869 be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; in ctl_be_block_rm()
2870 mtx_unlock(&softc->lock); in ctl_be_block_rm()
2874 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_rm()
2876 "LUN %d", retval, params->lun_id); in ctl_be_block_rm()
2877 mtx_lock(&softc->lock); in ctl_be_block_rm()
2878 be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; in ctl_be_block_rm()
2879 mtx_unlock(&softc->lock); in ctl_be_block_rm()
2883 mtx_lock(&softc->lock); in ctl_be_block_rm()
2884 while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { in ctl_be_block_rm()
2885 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblockrm", 0); in ctl_be_block_rm()
2889 be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; in ctl_be_block_rm()
2890 if (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) { in ctl_be_block_rm()
2891 mtx_unlock(&softc->lock); in ctl_be_block_rm()
2894 mtx_unlock(&softc->lock); in ctl_be_block_rm()
2898 req->status = CTL_LUN_OK; in ctl_be_block_rm()
2902 req->status = CTL_LUN_ERROR; in ctl_be_block_rm()
2916 params = &req->reqdata.modify; in ctl_be_block_modify()
2918 sx_xlock(&softc->modify_lock); in ctl_be_block_modify()
2919 mtx_lock(&softc->lock); in ctl_be_block_modify()
2920 SLIST_FOREACH(be_lun, &softc->lun_list, links) { in ctl_be_block_modify()
2921 if (be_lun->cbe_lun.lun_id == params->lun_id) in ctl_be_block_modify()
2924 mtx_unlock(&softc->lock); in ctl_be_block_modify()
2926 snprintf(req->error_str, sizeof(req->error_str), in ctl_be_block_modify()
2928 params->lun_id); in ctl_be_block_modify()
2931 cbe_lun = &be_lun->cbe_lun; in ctl_be_block_modify()
2933 if (params->lun_size_bytes != 0) in ctl_be_block_modify()
2934 be_lun->params.lun_size_bytes = params->lun_size_bytes; in ctl_be_block_modify()
2936 if (req->args_nvl != NULL) { in ctl_be_block_modify()
2937 nvlist_destroy(cbe_lun->options); in ctl_be_block_modify()
2938 cbe_lun->options = nvlist_clone(req->args_nvl); in ctl_be_block_modify()
2941 wasprim = (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY); in ctl_be_block_modify()
2942 value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL); in ctl_be_block_modify()
2945 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; in ctl_be_block_modify()
2947 cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY; in ctl_be_block_modify()
2948 } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF) in ctl_be_block_modify()
2949 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY; in ctl_be_block_modify()
2951 cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY; in ctl_be_block_modify()
2952 if (wasprim != (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)) { in ctl_be_block_modify()
2953 if (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) in ctl_be_block_modify()
2959 oldsize = be_lun->size_blocks; in ctl_be_block_modify()
2960 if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) || in ctl_be_block_modify()
2961 control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) { in ctl_be_block_modify()
2962 if (be_lun->vn == NULL) in ctl_be_block_modify()
2964 else if (vn_isdisk_error(be_lun->vn, &error)) in ctl_be_block_modify()
2966 else if (be_lun->vn->v_type == VREG) { in ctl_be_block_modify()
2967 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); in ctl_be_block_modify()
2969 VOP_UNLOCK(be_lun->vn); in ctl_be_block_modify()
2972 if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) && in ctl_be_block_modify()
2973 be_lun->vn != NULL) { in ctl_be_block_modify()
2974 cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA; in ctl_be_block_modify()
2976 } else if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) == 0 && in ctl_be_block_modify()
2977 be_lun->vn == NULL) { in ctl_be_block_modify()
2978 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; in ctl_be_block_modify()
2981 cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED; in ctl_be_block_modify()
2983 if (be_lun->vn != NULL) { in ctl_be_block_modify()
2984 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; in ctl_be_block_modify()
2986 taskqueue_drain_all(be_lun->io_taskqueue); in ctl_be_block_modify()
2991 if (be_lun->size_blocks != oldsize) in ctl_be_block_modify()
2995 params->lun_size_bytes = be_lun->size_bytes; in ctl_be_block_modify()
2997 sx_xunlock(&softc->modify_lock); in ctl_be_block_modify()
2998 req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK; in ctl_be_block_modify()
3002 sx_xunlock(&softc->modify_lock); in ctl_be_block_modify()
3003 req->status = CTL_LUN_ERROR; in ctl_be_block_modify()
3011 struct ctl_be_block_softc *softc = be_lun->softc; in ctl_be_block_lun_shutdown()
3013 taskqueue_drain_all(be_lun->io_taskqueue); in ctl_be_block_lun_shutdown()
3014 taskqueue_free(be_lun->io_taskqueue); in ctl_be_block_lun_shutdown()
3015 if (be_lun->disk_stats != NULL) in ctl_be_block_lun_shutdown()
3016 devstat_remove_entry(be_lun->disk_stats); in ctl_be_block_lun_shutdown()
3017 nvlist_destroy(be_lun->cbe_lun.options); in ctl_be_block_lun_shutdown()
3018 free(be_lun->dev_path, M_CTLBLK); in ctl_be_block_lun_shutdown()
3019 mtx_destroy(&be_lun->queue_lock); in ctl_be_block_lun_shutdown()
3020 mtx_destroy(&be_lun->io_lock); in ctl_be_block_lun_shutdown()
3022 mtx_lock(&softc->lock); in ctl_be_block_lun_shutdown()
3023 be_lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED; in ctl_be_block_lun_shutdown()
3024 if (be_lun->flags & CTL_BE_BLOCK_LUN_WAITING) in ctl_be_block_lun_shutdown()
3028 mtx_unlock(&softc->lock); in ctl_be_block_lun_shutdown()
3032 ctl_be_block_scsi_config_write(union ctl_io *io) in ctl_be_block_scsi_config_write() argument
3040 cbe_lun = CTL_BACKEND_LUN(io); in ctl_be_block_scsi_config_write()
3044 switch (io->scsiio.cdb[0]) { in ctl_be_block_scsi_config_write()
3058 mtx_lock(&be_lun->queue_lock); in ctl_be_block_scsi_config_write()
3059 STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr, in ctl_be_block_scsi_config_write()
3061 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_scsi_config_write()
3062 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); in ctl_be_block_scsi_config_write()
3068 cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb; in ctl_be_block_scsi_config_write()
3069 if ((cdb->how & SSS_PC_MASK) != 0) { in ctl_be_block_scsi_config_write()
3070 ctl_set_success(&io->scsiio); in ctl_be_block_scsi_config_write()
3071 ctl_config_write_done(io); in ctl_be_block_scsi_config_write()
3074 if (cdb->how & SSS_START) { in ctl_be_block_scsi_config_write()
3075 if ((cdb->how & SSS_LOEJ) && be_lun->vn == NULL) { in ctl_be_block_scsi_config_write()
3077 cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED; in ctl_be_block_scsi_config_write()
3079 cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA; in ctl_be_block_scsi_config_write()
3082 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; in ctl_be_block_scsi_config_write()
3089 if (cdb->how & SSS_LOEJ) { in ctl_be_block_scsi_config_write()
3090 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA; in ctl_be_block_scsi_config_write()
3091 cbe_lun->flags |= CTL_LUN_FLAG_EJECTED; in ctl_be_block_scsi_config_write()
3093 if (be_lun->vn != NULL) in ctl_be_block_scsi_config_write()
3098 ctl_set_success(&io->scsiio); in ctl_be_block_scsi_config_write()
3099 ctl_config_write_done(io); in ctl_be_block_scsi_config_write()
3103 ctl_set_success(&io->scsiio); in ctl_be_block_scsi_config_write()
3104 ctl_config_write_done(io); in ctl_be_block_scsi_config_write()
3107 ctl_set_invalid_opcode(&io->scsiio); in ctl_be_block_scsi_config_write()
3108 ctl_config_write_done(io); in ctl_be_block_scsi_config_write()
3117 ctl_be_block_nvme_config_write(union ctl_io *io) in ctl_be_block_nvme_config_write() argument
3123 be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io); in ctl_be_block_nvme_config_write()
3125 switch (io->nvmeio.cmd.opc) { in ctl_be_block_nvme_config_write()
3127 DSM_RANGE(io) = 0; in ctl_be_block_nvme_config_write()
3132 mtx_lock(&be_lun->queue_lock); in ctl_be_block_nvme_config_write()
3133 STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr, in ctl_be_block_nvme_config_write()
3135 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_nvme_config_write()
3136 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); in ctl_be_block_nvme_config_write()
3139 ctl_nvme_set_invalid_opcode(&io->nvmeio); in ctl_be_block_nvme_config_write()
3140 ctl_config_write_done(io); in ctl_be_block_nvme_config_write()
3147 ctl_be_block_config_write(union ctl_io *io) in ctl_be_block_config_write() argument
3149 switch (io->io_hdr.io_type) { in ctl_be_block_config_write()
3151 return (ctl_be_block_scsi_config_write(io)); in ctl_be_block_config_write()
3153 return (ctl_be_block_nvme_config_write(io)); in ctl_be_block_config_write()
3160 ctl_be_block_scsi_config_read(union ctl_io *io) in ctl_be_block_scsi_config_read() argument
3167 be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io); in ctl_be_block_scsi_config_read()
3169 switch (io->scsiio.cdb[0]) { in ctl_be_block_scsi_config_read()
3171 if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) { in ctl_be_block_scsi_config_read()
3172 mtx_lock(&be_lun->queue_lock); in ctl_be_block_scsi_config_read()
3173 STAILQ_INSERT_TAIL(&be_lun->config_read_queue, in ctl_be_block_scsi_config_read()
3174 &io->io_hdr, links); in ctl_be_block_scsi_config_read()
3175 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_scsi_config_read()
3176 taskqueue_enqueue(be_lun->io_taskqueue, in ctl_be_block_scsi_config_read()
3177 &be_lun->io_task); in ctl_be_block_scsi_config_read()
3181 ctl_set_invalid_field(&io->scsiio, in ctl_be_block_scsi_config_read()
3187 ctl_config_read_done(io); in ctl_be_block_scsi_config_read()
3191 ctl_set_invalid_opcode(&io->scsiio); in ctl_be_block_scsi_config_read()
3192 ctl_config_read_done(io); in ctl_be_block_scsi_config_read()
3201 ctl_be_block_nvme_config_read(union ctl_io *io) in ctl_be_block_nvme_config_read() argument
3207 be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io); in ctl_be_block_nvme_config_read()
3209 switch (io->nvmeio.cmd.opc) { in ctl_be_block_nvme_config_read()
3214 cns = le32toh(io->nvmeio.cmd.cdw10) & 0xff; in ctl_be_block_nvme_config_read()
3218 mtx_lock(&be_lun->queue_lock); in ctl_be_block_nvme_config_read()
3219 STAILQ_INSERT_TAIL(&be_lun->config_read_queue, in ctl_be_block_nvme_config_read()
3220 &io->io_hdr, links); in ctl_be_block_nvme_config_read()
3221 mtx_unlock(&be_lun->queue_lock); in ctl_be_block_nvme_config_read()
3222 taskqueue_enqueue(be_lun->io_taskqueue, in ctl_be_block_nvme_config_read()
3223 &be_lun->io_task); in ctl_be_block_nvme_config_read()
3226 ctl_nvme_set_invalid_field(&io->nvmeio); in ctl_be_block_nvme_config_read()
3227 ctl_config_read_done(io); in ctl_be_block_nvme_config_read()
3233 ctl_nvme_set_invalid_opcode(&io->nvmeio); in ctl_be_block_nvme_config_read()
3234 ctl_config_read_done(io); in ctl_be_block_nvme_config_read()
3241 ctl_be_block_config_read(union ctl_io *io) in ctl_be_block_config_read() argument
3243 switch (io->io_hdr.io_type) { in ctl_be_block_config_read()
3245 return (ctl_be_block_scsi_config_read(io)); in ctl_be_block_config_read()
3247 return (ctl_be_block_nvme_config_read(io)); in ctl_be_block_config_read()
3262 retval = sbuf_printf(sb, "%d", lun->num_threads); in ctl_be_block_lun_info()
3276 if (lun->getattr == NULL) in ctl_be_block_lun_attr()
3278 return (lun->getattr(lun, attrname)); in ctl_be_block_lun_attr()
3286 sx_init(&softc->modify_lock, "ctlblock modify"); in ctl_be_block_init()
3287 mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF); in ctl_be_block_init()
3288 softc->beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io), in ctl_be_block_init()
3290 softc->bufmin_zone = uma_zcreate("ctlblockmin", CTLBLK_MIN_SEG, in ctl_be_block_init()
3293 softc->bufmax_zone = uma_zcreate("ctlblockmax", CTLBLK_MAX_SEG, in ctl_be_block_init()
3295 SLIST_INIT(&softc->lun_list); in ctl_be_block_init()
3305 mtx_lock(&softc->lock); in ctl_be_block_shutdown()
3306 while ((lun = SLIST_FIRST(&softc->lun_list)) != NULL) { in ctl_be_block_shutdown()
3307 SLIST_REMOVE_HEAD(&softc->lun_list, links); in ctl_be_block_shutdown()
3308 softc->num_luns--; in ctl_be_block_shutdown()
3314 mtx_unlock(&softc->lock); in ctl_be_block_shutdown()
3315 ctl_remove_lun(&lun->cbe_lun); in ctl_be_block_shutdown()
3316 mtx_lock(&softc->lock); in ctl_be_block_shutdown()
3318 mtx_unlock(&softc->lock); in ctl_be_block_shutdown()
3319 uma_zdestroy(softc->bufmin_zone); in ctl_be_block_shutdown()
3321 uma_zdestroy(softc->bufmax_zone); in ctl_be_block_shutdown()
3322 uma_zdestroy(softc->beio_zone); in ctl_be_block_shutdown()
3323 mtx_destroy(&softc->lock); in ctl_be_block_shutdown()
3324 sx_destroy(&softc->modify_lock); in ctl_be_block_shutdown()