144 typedef struct bd bd_t;
153 struct bd {  struct
246 #define	CAN_FREESPACE(bd) \  argument
247 	(((bd)->d_ops.o_free_space == NULL) ? B_FALSE : B_TRUE)
354 	rv = ddi_soft_state_init(&bd_state, sizeof (struct bd), 2);  in _init()
401 	bd_t	*bd;  in bd_getinfo()  local
410 		bd = ddi_get_soft_state(bd_state, inst);  in bd_getinfo()
411 		if (bd == NULL) {  in bd_getinfo()
414 		*resultp = (void *)bd->d_dip;  in bd_getinfo()
465 bd_create_errstats(bd_t *bd, int inst, bd_drive_t *drive)  in bd_create_errstats()  argument
471 	if (bd->d_errstats != NULL)  in bd_create_errstats()
475 	    ddi_driver_name(bd->d_dip));  in bd_create_errstats()
477 	    ddi_driver_name(bd->d_dip), inst);  in bd_create_errstats()
479 	bd->d_errstats = kstat_create(ks_module, inst, ks_name, "device_error",  in bd_create_errstats()
482 	mutex_init(&bd->d_errmutex, NULL, MUTEX_DRIVER, NULL);  in bd_create_errstats()
483 	if (bd->d_errstats == NULL) {  in bd_create_errstats()
490 		bd->d_kerr = kmem_zalloc(sizeof (struct bd_errstats),  in bd_create_errstats()
493 		bd->d_errstats->ks_lock = &bd->d_errmutex;  in bd_create_errstats()
494 		bd->d_kerr = (struct bd_errstats *)bd->d_errstats->ks_data;  in bd_create_errstats()
497 	kstat_named_init(&bd->d_kerr->bd_softerrs,	"Soft Errors",  in bd_create_errstats()
499 	kstat_named_init(&bd->d_kerr->bd_harderrs,	"Hard Errors",  in bd_create_errstats()
501 	kstat_named_init(&bd->d_kerr->bd_transerrs,	"Transport Errors",  in bd_create_errstats()
505 		kstat_named_init(&bd->d_kerr->bd_model,	"Model",  in bd_create_errstats()
508 		kstat_named_init(&bd->d_kerr->bd_vid,	"Vendor",  in bd_create_errstats()
510 		kstat_named_init(&bd->d_kerr->bd_pid,	"Product",  in bd_create_errstats()
514 	kstat_named_init(&bd->d_kerr->bd_revision,	"Revision",  in bd_create_errstats()
516 	kstat_named_init(&bd->d_kerr->bd_serial,	"Serial No",  in bd_create_errstats()
518 	kstat_named_init(&bd->d_kerr->bd_capacity,	"Size",  in bd_create_errstats()
520 	kstat_named_init(&bd->d_kerr->bd_rq_media_err,	"Media Error",  in bd_create_errstats()
522 	kstat_named_init(&bd->d_kerr->bd_rq_ntrdy_err,	"Device Not Ready",  in bd_create_errstats()
524 	kstat_named_init(&bd->d_kerr->bd_rq_nodev_err,	"No Device",  in bd_create_errstats()
526 	kstat_named_init(&bd->d_kerr->bd_rq_recov_err,	"Recoverable",  in bd_create_errstats()
528 	kstat_named_init(&bd->d_kerr->bd_rq_illrq_err,	"Illegal Request",  in bd_create_errstats()
530 	kstat_named_init(&bd->d_kerr->bd_rq_pfa_err,  in bd_create_errstats()
533 	bd->d_errstats->ks_private = bd;  in bd_create_errstats()
535 	kstat_install(bd->d_errstats);  in bd_create_errstats()
536 	bd_init_errstats(bd, drive);  in bd_create_errstats()
540 bd_destroy_errstats(bd_t *bd)  in bd_destroy_errstats()  argument
542 	if (bd->d_errstats != NULL) {  in bd_destroy_errstats()
543 		bd_fini_errstats(bd);  in bd_destroy_errstats()
544 		kstat_delete(bd->d_errstats);  in bd_destroy_errstats()
545 		bd->d_errstats = NULL;  in bd_destroy_errstats()
547 		kmem_free(bd->d_kerr, sizeof (struct bd_errstats));  in bd_destroy_errstats()
548 		bd->d_kerr = NULL;  in bd_destroy_errstats()
549 		mutex_destroy(&bd->d_errmutex);  in bd_destroy_errstats()
586 bd_init_errstats(bd_t *bd, bd_drive_t *drive)  in bd_init_errstats()  argument
588 	struct bd_errstats	*est = bd->d_kerr;  in bd_init_errstats()
590 	mutex_enter(&bd->d_errmutex);  in bd_init_errstats()
608 	mutex_exit(&bd->d_errmutex);  in bd_init_errstats()
612 bd_fini_errstats(bd_t *bd)  in bd_fini_errstats()  argument
614 	struct bd_errstats	*est = bd->d_kerr;  in bd_fini_errstats()
616 	mutex_enter(&bd->d_errmutex);  in bd_fini_errstats()
624 	mutex_exit(&bd->d_errmutex);  in bd_fini_errstats()
628 bd_queues_free(bd_t *bd)  in bd_queues_free()  argument
632 	for (i = 0; i < bd->d_qcount; i++) {  in bd_queues_free()
633 		bd_queue_t *bq = &bd->d_queues[i];  in bd_queues_free()
640 	kmem_free(bd->d_queues, sizeof (*bd->d_queues) * bd->d_qcount);  in bd_queues_free()
648 	bd_t		*bd;  in bd_attach()  local
682 	bd = ddi_get_soft_state(bd_state, inst);  in bd_attach()
685 		bd->d_dma = *(hdl->h_dma);  in bd_attach()
686 		bd->d_dma.dma_attr_granular =  in bd_attach()
687 		    max(DEV_BSIZE, bd->d_dma.dma_attr_granular);  in bd_attach()
688 		bd->d_use_dma = B_TRUE;  in bd_attach()
690 		if (bd->d_maxxfer &&  in bd_attach()
691 		    (bd->d_maxxfer != bd->d_dma.dma_attr_maxxfer)) {  in bd_attach()
696 			bd->d_maxxfer = bd->d_dma.dma_attr_maxxfer;  in bd_attach()
698 			bd->d_maxxfer = bd->d_dma.dma_attr_maxxfer;  in bd_attach()
701 		bd->d_use_dma = B_FALSE;  in bd_attach()
702 		if (bd->d_maxxfer == 0) {  in bd_attach()
703 			bd->d_maxxfer = 1024 * 1024;  in bd_attach()
706 	bd->d_ops = hdl->h_ops;  in bd_attach()
707 	bd->d_private = hdl->h_private;  in bd_attach()
708 	bd->d_blkshift = DEV_BSHIFT;	/* 512 bytes, to start */  in bd_attach()
710 	if (bd->d_maxxfer % DEV_BSIZE) {  in bd_attach()
712 		bd->d_maxxfer &= ~(DEV_BSIZE - 1);  in bd_attach()
714 	if (bd->d_maxxfer < DEV_BSIZE) {  in bd_attach()
720 	bd->d_dip = dip;  in bd_attach()
721 	bd->d_handle = hdl;  in bd_attach()
722 	ddi_set_driver_private(dip, bd);  in bd_attach()
724 	mutex_init(&bd->d_ksmutex, NULL, MUTEX_DRIVER, NULL);  in bd_attach()
725 	mutex_init(&bd->d_ocmutex, NULL, MUTEX_DRIVER, NULL);  in bd_attach()
726 	mutex_init(&bd->d_statemutex, NULL, MUTEX_DRIVER, NULL);  in bd_attach()
727 	cv_init(&bd->d_statecv, NULL, CV_DRIVER, NULL);  in bd_attach()
728 	mutex_init(&bd->d_dle_mutex, NULL, MUTEX_DRIVER, NULL);  in bd_attach()
729 	bd->d_dle_state = 0;  in bd_attach()
731 	bd->d_cache = kmem_cache_create(kcache, sizeof (bd_xfer_impl_t), 8,  in bd_attach()
732 	    bd_xfer_ctor, bd_xfer_dtor, NULL, bd, NULL, 0);  in bd_attach()
734 	bd->d_ksp = kstat_create(ddi_driver_name(dip), inst, NULL, "disk",  in bd_attach()
736 	if (bd->d_ksp != NULL) {  in bd_attach()
737 		bd->d_ksp->ks_lock = &bd->d_ksmutex;  in bd_attach()
738 		kstat_install(bd->d_ksp);  in bd_attach()
739 		bd->d_kiop = bd->d_ksp->ks_data;  in bd_attach()
747 		bd->d_kiop = kmem_zalloc(sizeof (kstat_io_t), KM_SLEEP);  in bd_attach()
750 	cmlb_alloc_handle(&bd->d_cmlbh);  in bd_attach()
752 	bd->d_state = DKIO_NONE;  in bd_attach()
761 	bd->d_ops.o_drive_info(bd->d_private, &drive);  in bd_attach()
796 	bd->d_qcount = drive.d_qcount;  in bd_attach()
797 	bd->d_removable = drive.d_removable;  in bd_attach()
798 	bd->d_hotpluggable = drive.d_hotpluggable;  in bd_attach()
800 	if (drive.d_maxxfer && drive.d_maxxfer < bd->d_maxxfer)  in bd_attach()
801 		bd->d_maxxfer = drive.d_maxxfer;  in bd_attach()
803 	bd->d_free_align = drive.d_free_align;  in bd_attach()
804 	bd->d_max_free_seg = drive.d_max_free_seg;  in bd_attach()
805 	bd->d_max_free_blks = drive.d_max_free_blks;  in bd_attach()
806 	bd->d_max_free_seg_blks = drive.d_max_free_seg_blks;  in bd_attach()
809 	bd_create_errstats(bd, inst, &drive);  in bd_attach()
810 	bd_update_state(bd);  in bd_attach()
812 	bd->d_queues = kmem_alloc(sizeof (*bd->d_queues) * bd->d_qcount,  in bd_attach()
814 	for (i = 0; i < bd->d_qcount; i++) {  in bd_attach()
815 		bd_queue_t *bq = &bd->d_queues[i];  in bd_attach()
837 	    bd->d_removable, bd->d_hotpluggable, node_type,  in bd_attach()
838 	    CMLB_FAKE_LABEL_ONE_PARTITION, bd->d_cmlbh, 0);  in bd_attach()
843 	if (bd->d_ops.o_devid_init != NULL) {  in bd_attach()
844 		rv = bd->d_ops.o_devid_init(bd->d_private, dip, &bd->d_devid);  in bd_attach()
846 			if (ddi_devid_register(dip, bd->d_devid) !=  in bd_attach()
861 	if (bd->d_removable) {  in bd_attach()
865 	if (bd->d_hotpluggable) {  in bd_attach()
881 	(void) cmlb_validate(bd->d_cmlbh, 0, 0);  in bd_attach()
883 	hdl->h_bd = bd;  in bd_attach()
889 	bd_queues_free(bd);  in bd_attach()
890 	bd_destroy_errstats(bd);  in bd_attach()
893 	cmlb_free_handle(&bd->d_cmlbh);  in bd_attach()
895 	if (bd->d_ksp != NULL) {  in bd_attach()
896 		kstat_delete(bd->d_ksp);  in bd_attach()
897 		bd->d_ksp = NULL;  in bd_attach()
899 		kmem_free(bd->d_kiop, sizeof (kstat_io_t));  in bd_attach()
902 	kmem_cache_destroy(bd->d_cache);  in bd_attach()
903 	cv_destroy(&bd->d_statecv);  in bd_attach()
904 	mutex_destroy(&bd->d_statemutex);  in bd_attach()
905 	mutex_destroy(&bd->d_ocmutex);  in bd_attach()
906 	mutex_destroy(&bd->d_ksmutex);  in bd_attach()
907 	mutex_destroy(&bd->d_dle_mutex);  in bd_attach()
916 	bd_t		*bd;  in bd_detach()  local
918 	bd = ddi_get_driver_private(dip);  in bd_detach()
933 	if (bd->d_ksp != NULL) {  in bd_detach()
934 		kstat_delete(bd->d_ksp);  in bd_detach()
935 		bd->d_ksp = NULL;  in bd_detach()
937 		kmem_free(bd->d_kiop, sizeof (kstat_io_t));  in bd_detach()
940 	bd_destroy_errstats(bd);  in bd_detach()
941 	cmlb_detach(bd->d_cmlbh, 0);  in bd_detach()
942 	cmlb_free_handle(&bd->d_cmlbh);  in bd_detach()
943 	if (bd->d_devid)  in bd_detach()
944 		ddi_devid_free(bd->d_devid);  in bd_detach()
945 	kmem_cache_destroy(bd->d_cache);  in bd_detach()
946 	mutex_destroy(&bd->d_ksmutex);  in bd_detach()
947 	mutex_destroy(&bd->d_ocmutex);  in bd_detach()
948 	mutex_destroy(&bd->d_statemutex);  in bd_detach()
949 	cv_destroy(&bd->d_statecv);  in bd_detach()
950 	mutex_destroy(&bd->d_dle_mutex);  in bd_detach()
951 	bd_queues_free(bd);  in bd_detach()
960 	bd_t		*bd = arg;  in bd_xfer_ctor()  local
971 	xi->i_bd = bd;  in bd_xfer_ctor()
973 	if (bd->d_use_dma) {  in bd_xfer_ctor()
974 		if (ddi_dma_alloc_handle(bd->d_dip, &bd->d_dma, dcb, NULL,  in bd_xfer_ctor()
996 bd_xfer_alloc(bd_t *bd, struct buf *bp, int (*func)(void *, bd_xfer_t *),  in bd_xfer_alloc()  argument
1013 	xi = kmem_cache_alloc(bd->d_cache, kmflag);  in bd_xfer_alloc()
1023 	xi->i_blkno = bp->b_lblkno >> (bd->d_blkshift - DEV_BSHIFT);  in bd_xfer_alloc()
1036 		xi->i_func = bd->d_ops.o_read;  in bd_xfer_alloc()
1039 		xi->i_func = bd->d_ops.o_write;  in bd_xfer_alloc()
1042 	shift = bd->d_blkshift;  in bd_xfer_alloc()
1045 	if (!bd->d_use_dma) {  in bd_xfer_alloc()
1050 		    (bp->b_bcount + (bd->d_maxxfer - 1)) / bd->d_maxxfer;  in bd_xfer_alloc()
1052 		xi->i_len = min(bp->b_bcount, bd->d_maxxfer);  in bd_xfer_alloc()
1114 		kmem_cache_free(bd->d_cache, xi);  in bd_xfer_alloc()
1139 	bd_t		*bd;  in bd_open()  local
1163 	if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {  in bd_open()
1168 	mutex_enter(&bd->d_ocmutex);  in bd_open()
1173 	bd_update_state(bd);  in bd_open()
1175 	if (cmlb_validate(bd->d_cmlbh, 0, 0) != 0) {  in bd_open()
1182 	} else if (cmlb_partinfo(bd->d_cmlbh, part, &nblks, &lba,  in bd_open()
1205 	if ((flag & FWRITE) && bd->d_rdonly) {  in bd_open()
1210 	if ((bd->d_open_excl) & (mask)) {  in bd_open()
1215 		if (bd->d_open_lyr[part]) {  in bd_open()
1220 			if (bd->d_open_reg[i] & mask) {  in bd_open()
1228 		bd->d_open_lyr[part]++;  in bd_open()
1230 		bd->d_open_reg[otyp] |= mask;  in bd_open()
1233 		bd->d_open_excl |= mask;  in bd_open()
1238 	mutex_exit(&bd->d_ocmutex);  in bd_open()
1247 	bd_t		*bd;  in bd_close()  local
1264 	if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {  in bd_close()
1269 	mutex_enter(&bd->d_ocmutex);  in bd_close()
1270 	if (bd->d_open_excl & mask) {  in bd_close()
1271 		bd->d_open_excl &= ~mask;  in bd_close()
1274 		bd->d_open_lyr[part]--;  in bd_close()
1276 		bd->d_open_reg[otyp] &= ~mask;  in bd_close()
1279 		if (bd->d_open_lyr[part]) {  in bd_close()
1284 		if (bd->d_open_reg[i]) {  in bd_close()
1288 	mutex_exit(&bd->d_ocmutex);  in bd_close()
1291 		cmlb_invalidate(bd->d_cmlbh, 0);  in bd_close()
1305 	bd_t		*bd;  in bd_dump()  local
1318 	if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {  in bd_dump()
1322 	shift = bd->d_blkshift;  in bd_dump()
1329 	if (cmlb_partinfo(bd->d_cmlbh, part, &psize, &pstart, NULL, NULL,  in bd_dump()
1350 	xi = bd_xfer_alloc(bd, bp,  bd->d_ops.o_write, KM_NOSLEEP);  in bd_dump()
1358 	bd_submit(bd, xi);  in bd_dump()
1377 	bd_t	*bd;  in bd_minphys()  local
1380 	bd = ddi_get_soft_state(bd_state, inst);  in bd_minphys()
1386 	ASSERT(bd);  in bd_minphys()
1388 	if (bp->b_bcount > bd->d_maxxfer)  in bd_minphys()
1389 		bp->b_bcount = bd->d_maxxfer;  in bd_minphys()
1395 	bd_t		*bd;  in bd_check_uio()  local
1398 	if ((bd = ddi_get_soft_state(bd_state, BDINST(dev))) == NULL) {  in bd_check_uio()
1402 	shift = bd->d_blkshift;  in bd_check_uio()
1460 	bd_t		*bd;  in bd_strategy()  local
1476 	if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {  in bd_strategy()
1482 	if (cmlb_partinfo(bd->d_cmlbh, part, &p_nblks, &p_lba,  in bd_strategy()
1489 	shift = bd->d_blkshift;  in bd_strategy()
1510 	func = (bp->b_flags & B_READ) ? bd->d_ops.o_read : bd->d_ops.o_write;  in bd_strategy()
1512 	xi = bd_xfer_alloc(bd, bp, func, KM_NOSLEEP);  in bd_strategy()
1514 		xi = bd_xfer_alloc(bd, bp, func, KM_PUSHPAGE);  in bd_strategy()
1523 	bd_submit(bd, xi);  in bd_strategy()
1533 	bd_t		*bd;  in bd_ioctl()  local
1540 	if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {  in bd_ioctl()
1544 	rv = cmlb_ioctl(bd->d_cmlbh, dev, cmd, arg, flag, credp, rvalp, 0);  in bd_ioctl()
1558 		bd_update_state(bd);  in bd_ioctl()
1561 		minfo.dki_lbsize = (1U << bd->d_blkshift);  in bd_ioctl()
1562 		minfo.dki_capacity = bd->d_numblks;  in bd_ioctl()
1573 		bd_update_state(bd);  in bd_ioctl()
1576 		miext.dki_lbsize = (1U << bd->d_blkshift);  in bd_ioctl()
1577 		miext.dki_pbsize = (1U << bd->d_pblkshift);  in bd_ioctl()
1578 		miext.dki_capacity = bd->d_numblks;  in bd_ioctl()
1598 		cinfo.dki_cnum = ddi_get_instance(ddi_get_parent(bd->d_dip));  in bd_ioctl()
1600 		    "%s", ddi_driver_name(ddi_get_parent(bd->d_dip)));  in bd_ioctl()
1602 		    "%s", ddi_driver_name(bd->d_dip));  in bd_ioctl()
1606 		cinfo.dki_maxtransfer = bd->d_maxxfer / DEV_BSIZE;  in bd_ioctl()
1619 		i = bd->d_removable ? 1 : 0;  in bd_ioctl()
1627 		i = bd->d_hotpluggable ? 1 : 0;  in bd_ioctl()
1635 		i = bd->d_rdonly ? 1 : 0;  in bd_ioctl()
1643 		i = bd->d_ssd ? 1 : 0;  in bd_ioctl()
1654 		if ((rv = bd_check_state(bd, &state)) != 0) {  in bd_ioctl()
1668 		rv = bd_flush_write_cache(bd, dkc);  in bd_ioctl()
1678 		if (!CAN_FREESPACE(bd))  in bd_ioctl()
1699 		rv = bd_free_space(dev, bd, dfl);  in bd_ioctl()
1704 		boolean_t supported = CAN_FREESPACE(bd);  in bd_ioctl()
1725 	bd_t	*bd;  in bd_prop_op()  local
1727 	bd = ddi_get_soft_state(bd_state, ddi_get_instance(dip));  in bd_prop_op()
1728 	if (bd == NULL)  in bd_prop_op()
1732 	return (cmlb_prop_op(bd->d_cmlbh, dev, dip, prop_op, mod_flags, name,  in bd_prop_op()
1741 	bd_t		*bd;  in bd_tg_rdwr()  local
1754 	bd = ddi_get_soft_state(bd_state, ddi_get_instance(dip));  in bd_tg_rdwr()
1756 	if (P2PHASE(length, (1U << bd->d_blkshift)) != 0) {  in bd_tg_rdwr()
1768 		func = bd->d_ops.o_read;  in bd_tg_rdwr()
1772 		func = bd->d_ops.o_write;  in bd_tg_rdwr()
1781 	xi = bd_xfer_alloc(bd, bp, func, kmflag);  in bd_tg_rdwr()
1789 	bd_submit(bd, xi);  in bd_tg_rdwr()
1800 	bd_t		*bd;  in bd_tg_getinfo()  local
1803 	bd = ddi_get_soft_state(bd_state, ddi_get_instance(dip));  in bd_tg_getinfo()
1815 		bd_update_state(bd);  in bd_tg_getinfo()
1816 		*(diskaddr_t *)arg = bd->d_numblks;  in bd_tg_getinfo()
1820 		*(uint32_t *)arg = (1U << bd->d_blkshift);  in bd_tg_getinfo()
1831 		bd_update_state(bd);  in bd_tg_getinfo()
1833 		    bd->d_rdonly ? B_FALSE : B_TRUE;  in bd_tg_getinfo()
1834 		((tg_attribute_t *)arg)->media_is_solid_state = bd->d_ssd;  in bd_tg_getinfo()
1845 bd_sched(bd_t *bd, bd_queue_t *bq)  in bd_sched()  argument
1855 		mutex_enter(&bd->d_ksmutex);  in bd_sched()
1856 		kstat_waitq_to_runq(bd->d_kiop);  in bd_sched()
1857 		mutex_exit(&bd->d_ksmutex);  in bd_sched()
1870 		rv = xi->i_func(bd->d_private, &xi->i_public);  in bd_sched()
1876 			atomic_inc_32(&bd->d_kerr->bd_transerrs.value.ui32);  in bd_sched()
1880 			mutex_enter(&bd->d_ksmutex);  in bd_sched()
1881 			kstat_runq_exit(bd->d_kiop);  in bd_sched()
1882 			mutex_exit(&bd->d_ksmutex);  in bd_sched()
1896 bd_submit(bd_t *bd, bd_xfer_impl_t *xi)  in bd_submit()  argument
1898 	uint64_t	nv = atomic_inc_64_nv(&bd->d_io_counter);  in bd_submit()
1899 	unsigned	q = nv % bd->d_qcount;  in bd_submit()
1900 	bd_queue_t	*bq = &bd->d_queues[q];  in bd_submit()
1909 	mutex_enter(&bd->d_ksmutex);  in bd_submit()
1910 	kstat_waitq_enter(bd->d_kiop);  in bd_submit()
1911 	mutex_exit(&bd->d_ksmutex);  in bd_submit()
1915 	bd_sched(bd, bq);  in bd_submit()
1921 	bd_t		*bd = xi->i_bd;  in bd_runq_exit()  local
1928 	mutex_enter(&bd->d_ksmutex);  in bd_runq_exit()
1929 	kstat_runq_exit(bd->d_kiop);  in bd_runq_exit()
1930 	mutex_exit(&bd->d_ksmutex);  in bd_runq_exit()
1937 			atomic_inc_uint(&bd->d_kiop->reads);  in bd_runq_exit()
1938 			atomic_add_64((uint64_t *)&bd->d_kiop->nread,  in bd_runq_exit()
1941 			atomic_inc_uint(&bd->d_kiop->writes);  in bd_runq_exit()
1942 			atomic_add_64((uint64_t *)&bd->d_kiop->nwritten,  in bd_runq_exit()
1946 	bd_sched(bd, bq);  in bd_runq_exit()
1954 	bd_t *bd = arg;  in bd_dle_sysevent_task()  local
1955 	dev_info_t *dip = bd->d_dip;  in bd_dle_sysevent_task()
1958 	mutex_enter(&bd->d_dle_mutex);  in bd_dle_sysevent_task()
1959 	bd->d_dle_state &= ~BD_DLE_PENDING;  in bd_dle_sysevent_task()
1960 	bd->d_dle_state |= BD_DLE_RUNNING;  in bd_dle_sysevent_task()
1961 	mutex_exit(&bd->d_dle_mutex);  in bd_dle_sysevent_task()
1966 		mutex_enter(&bd->d_dle_mutex);  in bd_dle_sysevent_task()
1967 		bd->d_dle_state &= ~(BD_DLE_RUNNING|BD_DLE_PENDING);  in bd_dle_sysevent_task()
1968 		mutex_exit(&bd->d_dle_mutex);  in bd_dle_sysevent_task()
1997 		mutex_enter(&bd->d_dle_mutex);  in bd_dle_sysevent_task()
1998 		if ((bd->d_dle_state & BD_DLE_PENDING) == 0) {  in bd_dle_sysevent_task()
1999 			bd->d_dle_state &= ~BD_DLE_RUNNING;  in bd_dle_sysevent_task()
2000 			mutex_exit(&bd->d_dle_mutex);  in bd_dle_sysevent_task()
2003 		bd->d_dle_state &= ~BD_DLE_PENDING;  in bd_dle_sysevent_task()
2004 		mutex_exit(&bd->d_dle_mutex);  in bd_dle_sysevent_task()
2012 bd_update_state(bd_t *bd)  in bd_update_state()  argument
2020 	mutex_enter(&bd->d_statemutex);  in bd_update_state()
2021 	if (bd->d_ops.o_media_info(bd->d_private, &media) != 0) {  in bd_update_state()
2022 		bd->d_numblks = 0;  in bd_update_state()
2029 	    (P2PHASE(bd->d_maxxfer, media.m_blksize))) {  in bd_update_state()
2030 		dev_err(bd->d_dip, CE_WARN, "Invalid media block size (%d)",  in bd_update_state()
2036 		bd->d_numblks = 0;  in bd_update_state()
2040 	if (((1U << bd->d_blkshift) != media.m_blksize) ||  in bd_update_state()
2041 	    (bd->d_numblks != media.m_nblks)) {  in bd_update_state()
2046 	bd->d_blkshift = ddi_ffs(media.m_blksize) - 1;  in bd_update_state()
2047 	bd->d_pblkshift = bd->d_blkshift;  in bd_update_state()
2048 	bd->d_numblks = media.m_nblks;  in bd_update_state()
2049 	bd->d_rdonly = media.m_readonly;  in bd_update_state()
2050 	bd->d_ssd = media.m_solidstate;  in bd_update_state()
2060 		bd->d_pblkshift = ddi_ffs(media.m_pblksize) - 1;  in bd_update_state()
2063 	if (state != bd->d_state) {  in bd_update_state()
2064 		bd->d_state = state;  in bd_update_state()
2065 		cv_broadcast(&bd->d_statecv);  in bd_update_state()
2068 	mutex_exit(&bd->d_statemutex);  in bd_update_state()
2070 	bd->d_kerr->bd_capacity.value.ui64 = bd->d_numblks << bd->d_blkshift;  in bd_update_state()
2074 			(void) cmlb_validate(bd->d_cmlbh, 0, 0);  in bd_update_state()
2076 			mutex_enter(&bd->d_dle_mutex);  in bd_update_state()
2081 			if ((bd->d_dle_state & BD_DLE_PENDING) == 0) {  in bd_update_state()
2082 				if ((bd->d_dle_state & BD_DLE_RUNNING) == 0) {  in bd_update_state()
2084 					    bd_dle_sysevent_task, bd, 0,  in bd_update_state()
2085 					    &bd->d_dle_ent);  in bd_update_state()
2087 				bd->d_dle_state |= BD_DLE_PENDING;  in bd_update_state()
2089 			mutex_exit(&bd->d_dle_mutex);  in bd_update_state()
2091 			cmlb_invalidate(bd->d_cmlbh, 0);  in bd_update_state()
2097 bd_check_state(bd_t *bd, enum dkio_state *state)  in bd_check_state()  argument
2103 		bd_update_state(bd);  in bd_check_state()
2105 		mutex_enter(&bd->d_statemutex);  in bd_check_state()
2107 		if (bd->d_state != *state) {  in bd_check_state()
2108 			*state = bd->d_state;  in bd_check_state()
2109 			mutex_exit(&bd->d_statemutex);  in bd_check_state()
2114 		if (cv_reltimedwait_sig(&bd->d_statecv, &bd->d_statemutex,  in bd_check_state()
2116 			mutex_exit(&bd->d_statemutex);  in bd_check_state()
2120 		mutex_exit(&bd->d_statemutex);  in bd_check_state()
2138 bd_flush_write_cache(bd_t *bd, struct dk_callback *dkc)  in bd_flush_write_cache()  argument
2145 	if (bd->d_ops.o_sync_cache == NULL) {  in bd_flush_write_cache()
2154 	xi = bd_xfer_alloc(bd, bp, bd->d_ops.o_sync_cache, KM_SLEEP);  in bd_flush_write_cache()
2169 		bd_submit(bd, xi);  in bd_flush_write_cache()
2174 	bd_submit(bd, xi);  in bd_flush_write_cache()
2192 	bd_t		*bd = arg;  in bd_free_space_cb()  local
2203 	xi = bd_xfer_alloc(bd, bp, bd->d_ops.o_free_space, kmflag);  in bd_free_space_cb()
2208 		bd_submit(bd, xi);  in bd_free_space_cb()
2213 	bd_submit(bd, xi);  in bd_free_space_cb()
2223 bd_free_space(dev_t dev, bd_t *bd, dkioc_free_list_t *dfl)  in bd_free_space()  argument
2228 	const uint_t bshift = bd->d_blkshift;  in bd_free_space()
2231 		.dfi_align = bd->d_free_align << bshift,  in bd_free_space()
2232 		.dfi_max_bytes = bd->d_max_free_blks << bshift,  in bd_free_space()
2233 		.dfi_max_ext = bd->d_max_free_seg,  in bd_free_space()
2234 		.dfi_max_ext_bytes = bd->d_max_free_seg_blks << bshift,  in bd_free_space()
2237 	if (cmlb_partinfo(bd->d_cmlbh, part, &p_len, &p_offset, NULL,  in bd_free_space()
2257 	    bd, KM_SLEEP));  in bd_free_space()
2465 	bd_t		*bd = xi->i_bd;  in bd_xfer_done()  local
2470 		atomic_inc_32(&bd->d_kerr->bd_harderrs.value.ui32);  in bd_xfer_done()
2493 	if (bd->d_use_dma) {  in bd_xfer_done()
2501 		len = min(bp->b_bcount - xi->i_offset, bd->d_maxxfer);  in bd_xfer_done()
2519 	rv = xi->i_func(bd->d_private, &xi->i_public);  in bd_xfer_done()
2523 		atomic_inc_32(&bd->d_kerr->bd_transerrs.value.ui32);  in bd_xfer_done()
2536 	bd_t		*bd = xi->i_bd;  in bd_error()  local
2540 		atomic_inc_32(&bd->d_kerr->bd_rq_media_err.value.ui32);  in bd_error()
2543 		atomic_inc_32(&bd->d_kerr->bd_rq_ntrdy_err.value.ui32);  in bd_error()
2546 		atomic_inc_32(&bd->d_kerr->bd_rq_nodev_err.value.ui32);  in bd_error()
2549 		atomic_inc_32(&bd->d_kerr->bd_rq_recov_err.value.ui32);  in bd_error()
2552 		atomic_inc_32(&bd->d_kerr->bd_rq_illrq_err.value.ui32);  in bd_error()
2555 		atomic_inc_32(&bd->d_kerr->bd_rq_pfa_err.value.ui32);  in bd_error()
2566 	bd_t		*bd;  in bd_state_change()  local
2568 	if ((bd = hdl->h_bd) != NULL) {  in bd_state_change()
2569 		bd_update_state(bd);  in bd_state_change()