280 errorq_intr(caddr_t eqp)  in errorq_intr()  argument
282 	errorq_drain((errorq_t *)eqp);  in errorq_intr()
297 	errorq_t *eqp = kmem_alloc(sizeof (errorq_t), KM_SLEEP);  in errorq_create()  local
315 	    &ibc, NULL, errorq_intr, (caddr_t)eqp) != DDI_SUCCESS) {  in errorq_create()
318 		kmem_free(eqp, sizeof (errorq_t));  in errorq_create()
322 	if ((eqp->eq_ksp = kstat_create("unix", 0, name, "errorq",  in errorq_create()
329 		kmem_free(eqp, sizeof (errorq_t));  in errorq_create()
333 	bcopy(&errorq_kstat_template, &eqp->eq_kstat,  in errorq_create()
335 	eqp->eq_ksp->ks_data = &eqp->eq_kstat;  in errorq_create()
336 	eqp->eq_ksp->ks_private = eqp;  in errorq_create()
337 	kstat_install(eqp->eq_ksp);  in errorq_create()
339 	(void) strncpy(eqp->eq_name, name, ERRORQ_NAMELEN);  in errorq_create()
340 	eqp->eq_name[ERRORQ_NAMELEN] = '\0';  in errorq_create()
341 	eqp->eq_func = func;  in errorq_create()
342 	eqp->eq_private = private;  in errorq_create()
343 	eqp->eq_data = kmem_alloc(qlen * size, KM_SLEEP);  in errorq_create()
344 	eqp->eq_qlen = qlen;  in errorq_create()
345 	eqp->eq_size = size;  in errorq_create()
346 	eqp->eq_ipl = ipl;  in errorq_create()
347 	eqp->eq_flags = flags | ERRORQ_ACTIVE;  in errorq_create()
348 	eqp->eq_id = id;  in errorq_create()
349 	mutex_init(&eqp->eq_lock, NULL, MUTEX_DEFAULT, NULL);  in errorq_create()
350 	eqp->eq_elems = kmem_alloc(qlen * sizeof (errorq_elem_t), KM_SLEEP);  in errorq_create()
351 	eqp->eq_phead = NULL;  in errorq_create()
352 	eqp->eq_ptail = NULL;  in errorq_create()
353 	eqp->eq_pend = NULL;  in errorq_create()
354 	eqp->eq_dump = NULL;  in errorq_create()
355 	eqp->eq_bitmap = kmem_zalloc(BT_SIZEOFMAP(qlen), KM_SLEEP);  in errorq_create()
356 	eqp->eq_rotor = 0;  in errorq_create()
362 	for (eep = eqp->eq_elems, data = eqp->eq_data; qlen > 1; qlen--) {  in errorq_create()
380 	eqp->eq_next = errorq_list;  in errorq_create()
381 	errorq_list = eqp;  in errorq_create()
384 	return (eqp);  in errorq_create()
396 	errorq_t *eqp;  in errorq_nvcreate()  local
399 	eqp = errorq_create(name, func, private, qlen,  in errorq_nvcreate()
402 	if (eqp == NULL)  in errorq_nvcreate()
405 	mutex_enter(&eqp->eq_lock);  in errorq_nvcreate()
407 	for (eep = eqp->eq_elems; qlen != 0; eep++, qlen--) {  in errorq_nvcreate()
413 	mutex_exit(&eqp->eq_lock);  in errorq_nvcreate()
414 	return (eqp);  in errorq_nvcreate()
426 errorq_destroy(errorq_t *eqp)  in errorq_destroy()  argument
432 	ASSERT(eqp != NULL);  in errorq_destroy()
433 	eqp->eq_flags &= ~ERRORQ_ACTIVE;  in errorq_destroy()
434 	errorq_drain(eqp);  in errorq_destroy()
440 		if (p == eqp) {  in errorq_destroy()
450 	if (eqp->eq_flags & ERRORQ_NVLIST) {  in errorq_destroy()
451 		for (eep = eqp->eq_elems, i = 0; i < eqp->eq_qlen; i++, eep++) {  in errorq_destroy()
457 	mutex_destroy(&eqp->eq_lock);  in errorq_destroy()
458 	kstat_delete(eqp->eq_ksp);  in errorq_destroy()
460 	if (eqp->eq_id != NULL)  in errorq_destroy()
461 		ddi_remove_softintr(eqp->eq_id);  in errorq_destroy()
463 	kmem_free(eqp->eq_elems, eqp->eq_qlen * sizeof (errorq_elem_t));  in errorq_destroy()
464 	kmem_free(eqp->eq_bitmap, BT_SIZEOFMAP(eqp->eq_qlen));  in errorq_destroy()
465 	kmem_free(eqp->eq_data, eqp->eq_qlen * eqp->eq_size);  in errorq_destroy()
467 	kmem_free(eqp, sizeof (errorq_t));  in errorq_destroy()
522 errorq_dispatch(errorq_t *eqp, const void *data, size_t len, uint_t flag)  in errorq_dispatch()  argument
526 	if (eqp == NULL || !(eqp->eq_flags & ERRORQ_ACTIVE)) {  in errorq_dispatch()
534 		if ((i = errorq_availbit(eqp->eq_bitmap, eqp->eq_qlen,  in errorq_dispatch()
535 		    eqp->eq_rotor)) == -1) {  in errorq_dispatch()
536 			atomic_inc_64(&eqp->eq_kstat.eqk_dropped.value.ui64);  in errorq_dispatch()
539 		BT_ATOMIC_SET_EXCL(eqp->eq_bitmap, i, rval);  in errorq_dispatch()
541 			eqp->eq_rotor = i;  in errorq_dispatch()
542 			eep = &eqp->eq_elems[i];  in errorq_dispatch()
547 	ASSERT(len <= eqp->eq_size);  in errorq_dispatch()
548 	bcopy(data, eep->eqe_data, MIN(eqp->eq_size, len));  in errorq_dispatch()
550 	if (len < eqp->eq_size)  in errorq_dispatch()
551 		bzero((caddr_t)eep->eqe_data + len, eqp->eq_size - len);  in errorq_dispatch()
554 		old = eqp->eq_pend;  in errorq_dispatch()
558 		if (atomic_cas_ptr(&eqp->eq_pend, old, eep) == old)  in errorq_dispatch()
562 	atomic_inc_64(&eqp->eq_kstat.eqk_dispatched.value.ui64);  in errorq_dispatch()
564 	if (flag == ERRORQ_ASYNC && eqp->eq_id != NULL)  in errorq_dispatch()
565 		ddi_trigger_softintr(eqp->eq_id);  in errorq_dispatch()
585 errorq_drain(errorq_t *eqp)  in errorq_drain()  argument
589 	ASSERT(eqp != NULL);  in errorq_drain()
590 	mutex_enter(&eqp->eq_lock);  in errorq_drain()
604 	while ((eep = eqp->eq_pend) != NULL) {  in errorq_drain()
605 		eqp->eq_ptail = eep;  in errorq_drain()
608 		if (atomic_cas_ptr(&eqp->eq_pend, eep, NULL) == eep)  in errorq_drain()
617 		ASSERT(eqp->eq_ptail == NULL);  in errorq_drain()
618 		mutex_exit(&eqp->eq_lock);  in errorq_drain()
641 	eqp->eq_phead = eep;  in errorq_drain()
644 	eqp->eq_ptail = NULL;  in errorq_drain()
652 	dep = eqp->eq_dump;  in errorq_drain()
666 	while ((eep = eqp->eq_phead) != NULL) {  in errorq_drain()
667 		eqp->eq_func(eqp->eq_private, eep->eqe_data, eep);  in errorq_drain()
668 		eqp->eq_kstat.eqk_logged.value.ui64++;  in errorq_drain()
670 		eqp->eq_phead = eep->eqe_next;  in errorq_drain()
681 		if (panicstr && (eqp->eq_flags & ERRORQ_NVLIST)) {  in errorq_drain()
682 			if (eqp->eq_dump == NULL)  in errorq_drain()
683 				dep = eqp->eq_dump = eep;  in errorq_drain()
691 		BT_ATOMIC_CLEAR(eqp->eq_bitmap, eep - eqp->eq_elems);  in errorq_drain()
694 	mutex_exit(&eqp->eq_lock);  in errorq_drain()
707 	errorq_t *eqp;  in errorq_init()  local
714 	for (eqp = errorq_list; eqp != NULL; eqp = eqp->eq_next) {  in errorq_init()
716 		    (ddi_iblock_cookie_t)(uintptr_t)ipltospl(eqp->eq_ipl);  in errorq_init()
718 		if (eqp->eq_id != NULL)  in errorq_init()
722 		    errorq_intr, (caddr_t)eqp) != DDI_SUCCESS) {  in errorq_init()
724 			    "for queue %s", eqp->eq_ipl, eqp->eq_name);  in errorq_init()
727 		eqp->eq_id = id;  in errorq_init()
728 		errorq_drain(eqp);  in errorq_init()
744 	errorq_t *eqp;  in errorq_panic_drain()  local
749 	for (eqp = errorq_list; eqp != NULL; eqp = eqp->eq_next) {  in errorq_panic_drain()
750 		if ((eqp->eq_flags & (ERRORQ_VITAL | ERRORQ_NVLIST)) != what)  in errorq_panic_drain()
753 		loggedtmp = eqp->eq_kstat.eqk_logged.value.ui64;  in errorq_panic_drain()
765 		for (eep = eqp->eq_pend; eep != NULL; eep = eep->eqe_prev) {  in errorq_panic_drain()
766 			if (eep == eqp->eq_ptail) {  in errorq_panic_drain()
767 				ASSERT(eqp->eq_phead == NULL);  in errorq_panic_drain()
768 				eqp->eq_ptail = NULL;  in errorq_panic_drain()
781 		if (eqp->eq_phead == NULL && (eep = eqp->eq_ptail) != NULL) {  in errorq_panic_drain()
786 			eqp->eq_phead = eep;  in errorq_panic_drain()
787 			eqp->eq_ptail = NULL;  in errorq_panic_drain()
801 		for (eep = eqp->eq_phead; eep != NULL; eep = nep) {  in errorq_panic_drain()
802 			eqp->eq_func(eqp->eq_private, eep->eqe_data, eep);  in errorq_panic_drain()
803 			eqp->eq_kstat.eqk_logged.value.ui64++;  in errorq_panic_drain()
814 			if (eqp->eq_flags & ERRORQ_NVLIST) {  in errorq_panic_drain()
815 				if (eqp->eq_dump == NULL)  in errorq_panic_drain()
816 					dep = eqp->eq_dump = eep;  in errorq_panic_drain()
824 			BT_ATOMIC_CLEAR(eqp->eq_bitmap, eep - eqp->eq_elems);  in errorq_panic_drain()
833 		errorq_drain(eqp);  in errorq_panic_drain()
835 		logged += eqp->eq_kstat.eqk_logged.value.ui64 - loggedtmp;  in errorq_panic_drain()
867 errorq_reserve(errorq_t *eqp)  in errorq_reserve()  argument
871 	if (eqp == NULL || !(eqp->eq_flags & ERRORQ_ACTIVE)) {  in errorq_reserve()
879 		if ((i = errorq_availbit(eqp->eq_bitmap, eqp->eq_qlen,  in errorq_reserve()
880 		    eqp->eq_rotor)) == -1) {  in errorq_reserve()
881 			atomic_inc_64(&eqp->eq_kstat.eqk_dropped.value.ui64);  in errorq_reserve()
884 		BT_ATOMIC_SET_EXCL(eqp->eq_bitmap, i, rval);  in errorq_reserve()
886 			eqp->eq_rotor = i;  in errorq_reserve()
887 			eqep = &eqp->eq_elems[i];  in errorq_reserve()
892 	if (eqp->eq_flags & ERRORQ_NVLIST) {  in errorq_reserve()
898 	atomic_inc_64(&eqp->eq_kstat.eqk_reserved.value.ui64);  in errorq_reserve()
908 errorq_commit(errorq_t *eqp, errorq_elem_t *eqep, uint_t flag)  in errorq_commit()  argument
912 	if (eqep == NULL || !(eqp->eq_flags & ERRORQ_ACTIVE)) {  in errorq_commit()
913 		atomic_inc_64(&eqp->eq_kstat.eqk_commit_fail.value.ui64);  in errorq_commit()
918 		old = eqp->eq_pend;  in errorq_commit()
922 		if (atomic_cas_ptr(&eqp->eq_pend, old, eqep) == old)  in errorq_commit()
926 	atomic_inc_64(&eqp->eq_kstat.eqk_committed.value.ui64);  in errorq_commit()
928 	if (flag == ERRORQ_ASYNC && eqp->eq_id != NULL)  in errorq_commit()
929 		ddi_trigger_softintr(eqp->eq_id);  in errorq_commit()
937 errorq_cancel(errorq_t *eqp, errorq_elem_t *eqep)  in errorq_cancel()  argument
939 	if (eqep == NULL || !(eqp->eq_flags & ERRORQ_ACTIVE))  in errorq_cancel()
942 	BT_ATOMIC_CLEAR(eqp->eq_bitmap, eqep - eqp->eq_elems);  in errorq_cancel()
944 	atomic_inc_64(&eqp->eq_kstat.eqk_cancelled.value.ui64);  in errorq_cancel()
955 	errorq_t *eqp;  in errorq_dump()  local
960 	for (eqp = errorq_list; eqp != NULL; eqp = eqp->eq_next) {  in errorq_dump()
961 		if (!(eqp->eq_flags & ERRORQ_NVLIST) ||  in errorq_dump()
962 		    !(eqp->eq_flags & ERRORQ_ACTIVE))  in errorq_dump()
965 		for (eep = eqp->eq_dump; eep != NULL; eep = eep->eqe_dump) {  in errorq_dump()
977 				    eqp->eq_name, (void *)eep, len);  in errorq_dump()
986 				    eqp->eq_name, (void *)eep, err);  in errorq_dump()
1006 errorq_elem_nvl(errorq_t *eqp, const errorq_elem_t *eqep)  in errorq_elem_nvl()  argument
1010 	ASSERT(eqp->eq_flags & ERRORQ_ACTIVE && eqp->eq_flags & ERRORQ_NVLIST);  in errorq_elem_nvl()
1016 errorq_elem_nva(errorq_t *eqp, const errorq_elem_t *eqep)  in errorq_elem_nva()  argument
1020 	ASSERT(eqp->eq_flags & ERRORQ_ACTIVE && eqp->eq_flags & ERRORQ_NVLIST);  in errorq_elem_nva()
1029 errorq_elem_dup(errorq_t *eqp, const errorq_elem_t *eqep, errorq_elem_t **neqep)  in errorq_elem_dup()  argument
1031 	ASSERT(eqp->eq_flags & ERRORQ_ACTIVE);  in errorq_elem_dup()
1032 	ASSERT(!(eqp->eq_flags & ERRORQ_NVLIST));  in errorq_elem_dup()
1034 	if ((*neqep = errorq_reserve(eqp)) == NULL)  in errorq_elem_dup()
1037 	bcopy(eqep->eqe_data, (*neqep)->eqe_data, eqp->eq_size);  in errorq_elem_dup()