Lines Matching refs:sq
467 #define SQPUT_Q(sq, qp) \ argument
469 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
474 EQUIV(sq->sq_head, sq->sq_tail); \
476 ASSERT(sq == qp->q_syncq); \
478 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \
480 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
481 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
487 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \
489 EQUIV(sq->sq_head, sq->sq_nqueues); \
490 if (sq->sq_head == NULL) { \
491 sq->sq_head = sq->sq_tail = qp; \
492 sq->sq_flags |= SQ_MESSAGES; \
494 qp->q_sqprev = sq->sq_tail; \
495 sq->sq_tail->q_sqnext = qp; \
496 sq->sq_tail = qp; \
502 queue_t **qpp = &sq->sq_tail; \
514 sq->sq_head = qp; \
515 sq->sq_pri = sq->sq_head->q_spri; \
521 sq->sq_nqueues++; \
529 #define SQRM_Q(sq, qp) \ argument
531 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
533 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \
534 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \
536 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \
537 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \
538 ASSERT(sq->sq_nqueues != 0); \
541 sq->sq_head = qp->q_sqnext; \
548 sq->sq_tail = qp->q_sqprev; \
557 if (sq->sq_head != NULL) { \
558 sq->sq_pri = sq->sq_head->q_spri; \
560 sq->sq_flags &= ~SQ_MESSAGES; \
561 sq->sq_pri = 0; \
563 sq->sq_nqueues--; \
564 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \
565 (sq->sq_flags & SQ_QUEUED) == 0); \
595 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ argument
596 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
597 if ((sq)->sq_ciputctrl != NULL) { \
599 int nlocks = (sq)->sq_nciputctrl; \
600 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
601 ASSERT((sq)->sq_type & SQ_CIPUT); \
610 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ argument
611 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
612 if ((sq)->sq_ciputctrl != NULL) { \
614 int nlocks = (sq)->sq_nciputctrl; \
615 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
616 ASSERT((sq)->sq_type & SQ_CIPUT); \
690 syncq_t *sq = &qip->qu_syncq; in queue_constructor() local
722 sq->sq_head = NULL; in queue_constructor()
723 sq->sq_tail = NULL; in queue_constructor()
724 sq->sq_evhead = NULL; in queue_constructor()
725 sq->sq_evtail = NULL; in queue_constructor()
726 sq->sq_callbpend = NULL; in queue_constructor()
727 sq->sq_outer = NULL; in queue_constructor()
728 sq->sq_onext = NULL; in queue_constructor()
729 sq->sq_oprev = NULL; in queue_constructor()
730 sq->sq_next = NULL; in queue_constructor()
731 sq->sq_svcflags = 0; in queue_constructor()
732 sq->sq_servcount = 0; in queue_constructor()
733 sq->sq_needexcl = 0; in queue_constructor()
734 sq->sq_nqueues = 0; in queue_constructor()
735 sq->sq_pri = 0; in queue_constructor()
737 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); in queue_constructor()
738 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); in queue_constructor()
739 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); in queue_constructor()
751 syncq_t *sq = &qip->qu_syncq; in queue_destructor() local
766 mutex_destroy(&sq->sq_lock); in queue_destructor()
767 cv_destroy(&sq->sq_wait); in queue_destructor()
768 cv_destroy(&sq->sq_exitwait); in queue_destructor()
778 syncq_t *sq = buf; in syncq_constructor() local
782 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); in syncq_constructor()
783 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); in syncq_constructor()
784 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); in syncq_constructor()
793 syncq_t *sq = buf; in syncq_destructor() local
795 ASSERT(sq->sq_head == NULL); in syncq_destructor()
796 ASSERT(sq->sq_tail == NULL); in syncq_destructor()
797 ASSERT(sq->sq_evhead == NULL); in syncq_destructor()
798 ASSERT(sq->sq_evtail == NULL); in syncq_destructor()
799 ASSERT(sq->sq_callbpend == NULL); in syncq_destructor()
800 ASSERT(sq->sq_callbflags == 0); in syncq_destructor()
801 ASSERT(sq->sq_outer == NULL); in syncq_destructor()
802 ASSERT(sq->sq_onext == NULL); in syncq_destructor()
803 ASSERT(sq->sq_oprev == NULL); in syncq_destructor()
804 ASSERT(sq->sq_next == NULL); in syncq_destructor()
805 ASSERT(sq->sq_needexcl == 0); in syncq_destructor()
806 ASSERT(sq->sq_svcflags == 0); in syncq_destructor()
807 ASSERT(sq->sq_servcount == 0); in syncq_destructor()
808 ASSERT(sq->sq_nqueues == 0); in syncq_destructor()
809 ASSERT(sq->sq_pri == 0); in syncq_destructor()
810 ASSERT(sq->sq_count == 0); in syncq_destructor()
811 ASSERT(sq->sq_rmqcount == 0); in syncq_destructor()
812 ASSERT(sq->sq_cancelid == 0); in syncq_destructor()
813 ASSERT(sq->sq_ciputctrl == NULL); in syncq_destructor()
814 ASSERT(sq->sq_nciputctrl == 0); in syncq_destructor()
815 ASSERT(sq->sq_type == 0); in syncq_destructor()
816 ASSERT(sq->sq_flags == 0); in syncq_destructor()
818 mutex_destroy(&sq->sq_lock); in syncq_destructor()
819 cv_destroy(&sq->sq_wait); in syncq_destructor()
820 cv_destroy(&sq->sq_exitwait); in syncq_destructor()
1686 wait_syncq(syncq_t *sq) in wait_syncq() argument
1690 mutex_enter(SQLOCK(sq)); in wait_syncq()
1691 count = sq->sq_count; in wait_syncq()
1692 SQ_PUTLOCKS_ENTER(sq); in wait_syncq()
1693 SUM_SQ_PUTCOUNTS(sq, count); in wait_syncq()
1695 sq->sq_flags |= SQ_WANTWAKEUP; in wait_syncq()
1696 SQ_PUTLOCKS_EXIT(sq); in wait_syncq()
1697 cv_wait(&sq->sq_wait, SQLOCK(sq)); in wait_syncq()
1698 count = sq->sq_count; in wait_syncq()
1699 SQ_PUTLOCKS_ENTER(sq); in wait_syncq()
1700 SUM_SQ_PUTCOUNTS(sq, count); in wait_syncq()
1702 SQ_PUTLOCKS_EXIT(sq); in wait_syncq()
1703 mutex_exit(SQLOCK(sq)); in wait_syncq()
1713 syncq_t *sq = q->q_syncq; in wait_q_syncq() local
1715 mutex_enter(SQLOCK(sq)); in wait_q_syncq()
1717 sq->sq_flags |= SQ_WANTWAKEUP; in wait_q_syncq()
1718 cv_wait(&sq->sq_wait, SQLOCK(sq)); in wait_q_syncq()
1720 mutex_exit(SQLOCK(sq)); in wait_q_syncq()
2101 syncq_t *sq = wrq->q_syncq; in munlink() local
2103 mutex_enter(SQLOCK(sq)); in munlink()
2105 sq->sq_flags |= SQ_WANTWAKEUP; in munlink()
2106 cv_wait(&sq->sq_wait, SQLOCK(sq)); in munlink()
2108 mutex_exit(SQLOCK(sq)); in munlink()
2113 syncq_t *sq, *outer; in munlink() local
2139 sq = rq->q_syncq; in munlink()
2140 if ((outer = sq->sq_outer) != NULL) { in munlink()
2167 wait_syncq(sq); in munlink()
2179 syncq_t *sq = rq->q_syncq; in munlink() local
2181 mutex_enter(SQLOCK(sq)); in munlink()
2183 sq->sq_flags |= SQ_WANTWAKEUP; in munlink()
2184 cv_wait(&sq->sq_wait, SQLOCK(sq)); in munlink()
2186 mutex_exit(SQLOCK(sq)); in munlink()
2490 syncq_t *sq, *outer; in setq() local
2509 sq = rq->q_syncq; in setq()
2510 outer = sq->sq_outer; in setq()
2517 ASSERT(sq->sq_outer == NULL); in setq()
2518 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); in setq()
2520 if (sq != SQ(rq)) { in setq()
2522 free_syncq(sq); in setq()
2527 if (wq->q_syncq != NULL && wq->q_syncq != sq && in setq()
2559 sq = SQ(rq); in setq()
2560 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); in setq()
2561 ASSERT(sq->sq_outer == NULL); in setq()
2562 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); in setq()
2568 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); in setq()
2570 rq->q_syncq = wq->q_syncq = sq; in setq()
2571 sq->sq_type = sqtype; in setq()
2572 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); in setq()
2580 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); in setq()
2581 sq->sq_svcflags = 0; in setq()
2601 sq = new_syncq(); in setq()
2602 sq->sq_type = rq->q_syncq->sq_type; in setq()
2603 sq->sq_flags = rq->q_syncq->sq_flags; in setq()
2604 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && in setq()
2605 sq->sq_oprev == NULL); in setq()
2606 wq->q_syncq = sq; in setq()
2609 sq = dmp->dm_sq; in setq()
2615 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && in setq()
2616 sq->sq_oprev == NULL); in setq()
2617 rq->q_syncq = wq->q_syncq = sq; in setq()
2645 syncq_t *sq; in hold_dm() local
2663 sq = new_syncq(); in hold_dm()
2665 sq->sq_type = sqtype | SQ_PERMOD; in hold_dm()
2666 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; in hold_dm()
2669 sq->sq_onext = sq->sq_oprev = sq; in hold_dm()
2673 dmp->dm_sq = sq; in hold_dm()
2683 free_syncq(sq); in hold_dm()
3353 syncq_t *sq; in allocq() local
3359 sq = &qip->qu_syncq; in allocq()
3367 qp->q_syncq = sq; in allocq()
3383 wqp->q_syncq = sq; in allocq()
3392 sq->sq_count = 0; in allocq()
3393 sq->sq_rmqcount = 0; in allocq()
3394 sq->sq_flags = 0; in allocq()
3395 sq->sq_type = 0; in allocq()
3396 sq->sq_callbflags = 0; in allocq()
3397 sq->sq_cancelid = 0; in allocq()
3398 sq->sq_ciputctrl = NULL; in allocq()
3399 sq->sq_nciputctrl = 0; in allocq()
3400 sq->sq_needexcl = 0; in allocq()
3401 sq->sq_svcflags = 0; in allocq()
3415 syncq_t *sq, *outer; in freeq() local
3465 sq = SQ(qp); in freeq()
3466 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in freeq()
3467 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); in freeq()
3468 ASSERT(sq->sq_outer == NULL); in freeq()
3469 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); in freeq()
3470 ASSERT(sq->sq_callbpend == NULL); in freeq()
3471 ASSERT(sq->sq_needexcl == 0); in freeq()
3473 if (sq->sq_ciputctrl != NULL) { in freeq()
3474 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); in freeq()
3475 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, in freeq()
3476 sq->sq_nciputctrl, 0); in freeq()
3478 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); in freeq()
3479 sq->sq_ciputctrl = NULL; in freeq()
3480 sq->sq_nciputctrl = 0; in freeq()
3761 syncq_t *sq; in streams_sqbkgrnd_service() local
3782 DQ(sq, sqhead, sqtail, sq_next); in streams_sqbkgrnd_service()
3783 ASSERT(sq != NULL); in streams_sqbkgrnd_service()
3784 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); in streams_sqbkgrnd_service()
3786 syncq_service(sq); in streams_sqbkgrnd_service()
3798 wait_sq_svc(syncq_t *sq) in wait_sq_svc() argument
3800 mutex_enter(SQLOCK(sq)); in wait_sq_svc()
3801 sq->sq_svcflags |= SQ_DISABLED; in wait_sq_svc()
3802 if (sq->sq_svcflags & SQ_BGTHREAD) { in wait_sq_svc()
3807 ASSERT(sq->sq_servcount == 1); in wait_sq_svc()
3809 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); in wait_sq_svc()
3812 sq->sq_svcflags &= ~SQ_BGTHREAD; in wait_sq_svc()
3813 sq->sq_servcount = 0; in wait_sq_svc()
3818 while (sq->sq_servcount != 0) { in wait_sq_svc()
3819 sq->sq_flags |= SQ_WANTWAKEUP; in wait_sq_svc()
3820 cv_wait(&sq->sq_wait, SQLOCK(sq)); in wait_sq_svc()
3823 mutex_exit(SQLOCK(sq)); in wait_sq_svc()
3839 sqenable(syncq_t *sq) in sqenable() argument
3847 ASSERT(MUTEX_HELD(SQLOCK(sq))); in sqenable()
3849 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); in sqenable()
3850 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); in sqenable()
3856 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) in sqenable()
3867 if (sq->sq_servcount != 0) { in sqenable()
3868 if (((sq->sq_type & SQ_PERMOD) == 0) || in sqenable()
3869 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { in sqenable()
3875 sq->sq_tstamp = ddi_get_lbolt(); in sqenable()
3879 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, in sqenable()
3880 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); in sqenable()
3881 if (sq->sq_servid != NULL) { in sqenable()
3882 sq->sq_servcount++; in sqenable()
3891 if (sq->sq_servcount != 0) in sqenable()
3901 ENQUEUE(sq, sqhead, sqtail, sq_next); in sqenable()
3902 sq->sq_svcflags |= SQ_BGTHREAD; in sqenable()
3903 sq->sq_servcount = 1; in sqenable()
4622 syncq_t *sq = sql->sql_sq; in strlock() local
4625 mutex_enter(SQLOCK(sq)); in strlock()
4626 count = sq->sq_count; in strlock()
4627 ASSERT(sq->sq_rmqcount <= count); in strlock()
4628 SQ_PUTLOCKS_ENTER(sq); in strlock()
4629 SUM_SQ_PUTCOUNTS(sq, count); in strlock()
4630 if (count == sq->sq_rmqcount) in strlock()
4658 sq->sq_needexcl++; in strlock()
4659 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); in strlock()
4660 while (count > sq->sq_rmqcount) { in strlock()
4661 sq->sq_flags |= SQ_WANTWAKEUP; in strlock()
4662 SQ_PUTLOCKS_EXIT(sq); in strlock()
4663 cv_wait(&sq->sq_wait, SQLOCK(sq)); in strlock()
4664 count = sq->sq_count; in strlock()
4665 SQ_PUTLOCKS_ENTER(sq); in strlock()
4666 SUM_SQ_PUTCOUNTS(sq, count); in strlock()
4668 sq->sq_needexcl--; in strlock()
4669 if (sq->sq_needexcl == 0) in strlock()
4670 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in strlock()
4671 SQ_PUTLOCKS_EXIT(sq); in strlock()
4672 ASSERT(count == sq->sq_rmqcount); in strlock()
4673 mutex_exit(SQLOCK(sq)); in strlock()
4859 syncq_t *sq = qp->q_syncq; in removeq() local
4874 mutex_enter(SQLOCK(sq)); in removeq()
4875 if (sq->sq_flags & SQ_WANTWAKEUP) { in removeq()
4876 sq->sq_flags &= ~SQ_WANTWAKEUP; in removeq()
4877 cv_broadcast(&sq->sq_wait); in removeq()
4879 mutex_exit(SQLOCK(sq)); in removeq()
4881 if (wsq != sq) { in removeq()
4905 mutex_enter(SQLOCK(sq)); in removeq()
4906 sq->sq_rmqcount++; in removeq()
4907 if (sq->sq_flags & SQ_WANTWAKEUP) { in removeq()
4908 sq->sq_flags &= ~SQ_WANTWAKEUP; in removeq()
4909 cv_broadcast(&sq->sq_wait); in removeq()
4911 mutex_exit(SQLOCK(sq)); in removeq()
4991 mutex_enter(SQLOCK(sq)); in removeq()
4992 sq->sq_rmqcount--; in removeq()
4993 mutex_exit(SQLOCK(sq)); in removeq()
5009 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) in blocksq() argument
5013 mutex_enter(SQLOCK(sq)); in blocksq()
5022 count = sq->sq_count; in blocksq()
5023 SQ_PUTLOCKS_ENTER(sq); in blocksq()
5024 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); in blocksq()
5025 SUM_SQ_PUTCOUNTS(sq, count); in blocksq()
5027 sq->sq_needexcl++; in blocksq()
5028 ASSERT(sq->sq_needexcl != 0); /* wraparound */ in blocksq()
5030 while ((sq->sq_flags & flag) || in blocksq()
5032 sq->sq_flags |= SQ_WANTWAKEUP; in blocksq()
5034 SQ_PUTLOCKS_EXIT(sq); in blocksq()
5036 cv_wait(&sq->sq_wait, SQLOCK(sq)); in blocksq()
5038 count = sq->sq_count; in blocksq()
5039 SQ_PUTLOCKS_ENTER(sq); in blocksq()
5040 SUM_SQ_PUTCOUNTS(sq, count); in blocksq()
5043 sq->sq_needexcl--; in blocksq()
5044 sq->sq_flags |= flag; in blocksq()
5047 if (sq->sq_needexcl == 0) { in blocksq()
5048 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in blocksq()
5050 SQ_PUTLOCKS_EXIT(sq); in blocksq()
5051 } else if (sq->sq_needexcl == 0) { in blocksq()
5052 SQ_PUTCOUNT_SETFAST(sq); in blocksq()
5055 mutex_exit(SQLOCK(sq)); in blocksq()
5071 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) in unblocksq() argument
5075 mutex_enter(SQLOCK(sq)); in unblocksq()
5077 ASSERT(sq->sq_flags & resetflag); in unblocksq()
5078 flags = sq->sq_flags & ~resetflag; in unblocksq()
5079 sq->sq_flags = flags; in unblocksq()
5083 cv_broadcast(&sq->sq_wait); in unblocksq()
5085 sq->sq_flags = flags; in unblocksq()
5089 drain_syncq(sq); in unblocksq()
5094 mutex_exit(SQLOCK(sq)); in unblocksq()
5106 dropsq(syncq_t *sq, uint16_t resetflag) in dropsq() argument
5110 mutex_enter(SQLOCK(sq)); in dropsq()
5111 ASSERT(sq->sq_flags & resetflag); in dropsq()
5112 flags = sq->sq_flags & ~resetflag; in dropsq()
5115 cv_broadcast(&sq->sq_wait); in dropsq()
5117 sq->sq_flags = flags; in dropsq()
5118 mutex_exit(SQLOCK(sq)); in dropsq()
5131 emptysq(syncq_t *sq) in emptysq() argument
5135 mutex_enter(SQLOCK(sq)); in emptysq()
5136 flags = sq->sq_flags; in emptysq()
5142 if (sq->sq_count == 0) { in emptysq()
5144 drain_syncq(sq); in emptysq()
5147 sqenable(sq); in emptysq()
5149 mutex_exit(SQLOCK(sq)); in emptysq()
5380 entersq(syncq_t *sq, int entrypoint) in entersq() argument
5392 ASSERT(sq); in entersq()
5394 mutex_enter(SQLOCK(sq)); in entersq()
5395 flags = sq->sq_flags; in entersq()
5396 type = sq->sq_type; in entersq()
5399 count = sq->sq_count; in entersq()
5400 SQ_PUTLOCKS_ENTER(sq); in entersq()
5401 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); in entersq()
5402 SUM_SQ_PUTCOUNTS(sq, count); in entersq()
5403 sq->sq_needexcl++; in entersq()
5404 ASSERT(sq->sq_needexcl != 0); /* wraparound */ in entersq()
5414 sq->sq_flags = flags | SQ_WANTWAKEUP; in entersq()
5416 SQ_PUTLOCKS_EXIT(sq); in entersq()
5418 cv_wait(&sq->sq_wait, SQLOCK(sq)); in entersq()
5420 count = sq->sq_count; in entersq()
5421 SQ_PUTLOCKS_ENTER(sq); in entersq()
5422 SUM_SQ_PUTCOUNTS(sq, count); in entersq()
5424 flags = sq->sq_flags; in entersq()
5428 ASSERT(sq->sq_needexcl > 0); in entersq()
5429 sq->sq_needexcl--; in entersq()
5430 if (sq->sq_needexcl == 0) { in entersq()
5431 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in entersq()
5445 SQ_PUTLOCKS_EXIT(sq); in entersq()
5447 mutex_exit(SQLOCK(sq)); in entersq()
5448 outer_enter(sq->sq_outer, SQ_GOAWAY); in entersq()
5449 mutex_enter(SQLOCK(sq)); in entersq()
5450 flags = sq->sq_flags; in entersq()
5456 count = sq->sq_count; in entersq()
5465 SUMCHECK_SQ_PUTCOUNTS(sq, 0); in entersq()
5469 sq->sq_flags = flags | SQ_WANTWAKEUP; in entersq()
5470 cv_wait(&sq->sq_wait, SQLOCK(sq)); in entersq()
5471 count = sq->sq_count; in entersq()
5472 flags = sq->sq_flags; in entersq()
5476 sq->sq_count++; in entersq()
5477 ASSERT(sq->sq_count != 0); /* Wraparound */ in entersq()
5480 ASSERT(sq->sq_count == 1); in entersq()
5481 sq->sq_flags |= SQ_EXCL; in entersq()
5483 SQ_PUTLOCKS_EXIT(sq); in entersq()
5486 mutex_exit(SQLOCK(sq)); in entersq()
5499 leavesq(syncq_t *sq, int entrypoint) in leavesq() argument
5512 ASSERT(sq); in leavesq()
5514 mutex_enter(SQLOCK(sq)); in leavesq()
5515 flags = sq->sq_flags; in leavesq()
5516 type = sq->sq_type; in leavesq()
5521 cv_broadcast(&sq->sq_wait); in leavesq()
5525 cv_broadcast(&sq->sq_exitwait); in leavesq()
5533 ASSERT(sq->sq_count != 0); in leavesq()
5534 sq->sq_count--; in leavesq()
5536 sq->sq_flags = flags & ~SQ_EXCL; in leavesq()
5537 drain_syncq(sq); in leavesq()
5538 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in leavesq()
5542 outer_exit(sq->sq_outer); in leavesq()
5546 ASSERT(sq->sq_count != 0); in leavesq()
5547 sq->sq_count--; in leavesq()
5549 sq->sq_flags = flags & ~SQ_EXCL; in leavesq()
5550 mutex_exit(SQLOCK(sq)); in leavesq()
5553 if (!(sq->sq_type & c_outer)) in leavesq()
5554 outer_exit(sq->sq_outer); in leavesq()
5566 syncq_t *sq = qp->q_syncq; in claimq() local
5568 mutex_enter(SQLOCK(sq)); in claimq()
5569 sq->sq_count++; in claimq()
5570 ASSERT(sq->sq_count != 0); /* Wraparound */ in claimq()
5571 mutex_exit(SQLOCK(sq)); in claimq()
5583 syncq_t *sq = qp->q_syncq; in releaseq() local
5586 mutex_enter(SQLOCK(sq)); in releaseq()
5587 ASSERT(sq->sq_count > 0); in releaseq()
5588 sq->sq_count--; in releaseq()
5590 flags = sq->sq_flags; in releaseq()
5594 cv_broadcast(&sq->sq_wait); in releaseq()
5596 sq->sq_flags = flags; in releaseq()
5603 if (sq->sq_count == 0) { in releaseq()
5604 drain_syncq(sq); in releaseq()
5607 sqenable(sq); in releaseq()
5610 mutex_exit(SQLOCK(sq)); in releaseq()
5649 free_syncq(syncq_t *sq) in free_syncq() argument
5651 ASSERT(sq->sq_head == NULL); in free_syncq()
5652 ASSERT(sq->sq_outer == NULL); in free_syncq()
5653 ASSERT(sq->sq_callbpend == NULL); in free_syncq()
5654 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || in free_syncq()
5655 (sq->sq_onext == sq && sq->sq_oprev == sq)); in free_syncq()
5657 if (sq->sq_ciputctrl != NULL) { in free_syncq()
5658 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); in free_syncq()
5659 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, in free_syncq()
5660 sq->sq_nciputctrl, 0); in free_syncq()
5662 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); in free_syncq()
5665 sq->sq_tail = NULL; in free_syncq()
5666 sq->sq_evhead = NULL; in free_syncq()
5667 sq->sq_evtail = NULL; in free_syncq()
5668 sq->sq_ciputctrl = NULL; in free_syncq()
5669 sq->sq_nciputctrl = 0; in free_syncq()
5670 sq->sq_count = 0; in free_syncq()
5671 sq->sq_rmqcount = 0; in free_syncq()
5672 sq->sq_callbflags = 0; in free_syncq()
5673 sq->sq_cancelid = 0; in free_syncq()
5674 sq->sq_next = NULL; in free_syncq()
5675 sq->sq_needexcl = 0; in free_syncq()
5676 sq->sq_svcflags = 0; in free_syncq()
5677 sq->sq_nqueues = 0; in free_syncq()
5678 sq->sq_pri = 0; in free_syncq()
5679 sq->sq_onext = NULL; in free_syncq()
5680 sq->sq_oprev = NULL; in free_syncq()
5681 sq->sq_flags = 0; in free_syncq()
5682 sq->sq_type = 0; in free_syncq()
5683 sq->sq_servcount = 0; in free_syncq()
5685 kmem_cache_free(syncq_cache, sq); in free_syncq()
5727 syncq_t *sq; in outer_enter() local
5752 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { in outer_enter()
5753 mutex_enter(SQLOCK(sq)); in outer_enter()
5754 count = sq->sq_count; in outer_enter()
5755 SQ_PUTLOCKS_ENTER(sq); in outer_enter()
5756 sq->sq_flags |= SQ_WRITER; in outer_enter()
5757 SUM_SQ_PUTCOUNTS(sq, count); in outer_enter()
5760 SQ_PUTLOCKS_EXIT(sq); in outer_enter()
5761 mutex_exit(SQLOCK(sq)); in outer_enter()
5777 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { in outer_enter()
5778 mutex_enter(SQLOCK(sq)); in outer_enter()
5779 count = sq->sq_count; in outer_enter()
5780 SQ_PUTLOCKS_ENTER(sq); in outer_enter()
5781 SUM_SQ_PUTCOUNTS(sq, count); in outer_enter()
5783 sq->sq_flags |= SQ_WANTWAKEUP; in outer_enter()
5784 SQ_PUTLOCKS_EXIT(sq); in outer_enter()
5785 cv_wait(&sq->sq_wait, SQLOCK(sq)); in outer_enter()
5786 count = sq->sq_count; in outer_enter()
5787 SQ_PUTLOCKS_ENTER(sq); in outer_enter()
5788 SUM_SQ_PUTCOUNTS(sq, count); in outer_enter()
5790 SQ_PUTLOCKS_EXIT(sq); in outer_enter()
5791 mutex_exit(SQLOCK(sq)); in outer_enter()
5817 syncq_t *sq; in outer_exit() local
5848 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) in outer_exit()
5849 drain_needed += dropsq(sq, SQ_WRITER); in outer_exit()
5855 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) in outer_exit()
5856 emptysq(sq); in outer_exit()
5879 outer_insert(syncq_t *outer, syncq_t *sq) in outer_insert() argument
5883 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && in outer_insert()
5884 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ in outer_insert()
5891 mutex_enter(SQLOCK(sq)); in outer_insert()
5892 sq->sq_outer = outer; in outer_insert()
5893 outer->sq_onext->sq_oprev = sq; in outer_insert()
5894 sq->sq_onext = outer->sq_onext; in outer_insert()
5895 outer->sq_onext = sq; in outer_insert()
5896 sq->sq_oprev = outer; in outer_insert()
5897 mutex_exit(SQLOCK(sq)); in outer_insert()
5910 outer_remove(syncq_t *outer, syncq_t *sq) in outer_remove() argument
5914 ASSERT(sq->sq_outer == outer); in outer_remove()
5921 mutex_enter(SQLOCK(sq)); in outer_remove()
5922 sq->sq_outer = NULL; in outer_remove()
5923 sq->sq_onext->sq_oprev = sq->sq_oprev; in outer_remove()
5924 sq->sq_oprev->sq_onext = sq->sq_onext; in outer_remove()
5925 sq->sq_oprev = sq->sq_onext = NULL; in outer_remove()
5926 mutex_exit(SQLOCK(sq)); in outer_remove()
5979 syncq_t *osq, *sq, *outer; in qwriter_outer() local
6046 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { in qwriter_outer()
6048 uint_t maxcnt = (sq == osq) ? 1 : 0; in qwriter_outer()
6050 mutex_enter(SQLOCK(sq)); in qwriter_outer()
6051 count = sq->sq_count; in qwriter_outer()
6052 SQ_PUTLOCKS_ENTER(sq); in qwriter_outer()
6053 SUM_SQ_PUTCOUNTS(sq, count); in qwriter_outer()
6054 if (sq->sq_count > maxcnt) in qwriter_outer()
6056 sq->sq_flags |= SQ_WRITER; in qwriter_outer()
6057 SQ_PUTLOCKS_EXIT(sq); in qwriter_outer()
6058 mutex_exit(SQLOCK(sq)); in qwriter_outer()
6172 sq_run_events(syncq_t *sq) in sq_run_events() argument
6176 uint16_t flags = sq->sq_flags; in sq_run_events()
6179 ASSERT(MUTEX_HELD(SQLOCK(sq))); in sq_run_events()
6180 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && in sq_run_events()
6181 sq->sq_oprev == NULL) || in sq_run_events()
6182 (sq->sq_outer != NULL && sq->sq_onext != NULL && in sq_run_events()
6183 sq->sq_oprev != NULL)); in sq_run_events()
6186 ASSERT(sq->sq_count == 1); in sq_run_events()
6199 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { in sq_run_events()
6200 ASSERT(bp->b_queue->q_syncq == sq); in sq_run_events()
6201 ASSERT(sq->sq_flags & SQ_EVENTS); in sq_run_events()
6211 ASSERT(sq->sq_evhead == bp); in sq_run_events()
6212 sq->sq_evhead = bp->b_next; in sq_run_events()
6216 ASSERT(sq->sq_evtail == bp); in sq_run_events()
6217 sq->sq_evtail = NULL; in sq_run_events()
6218 sq->sq_flags &= ~SQ_EVENTS; in sq_run_events()
6223 mutex_exit(SQLOCK(sq)); in sq_run_events()
6227 mutex_enter(SQLOCK(sq)); in sq_run_events()
6231 flags = sq->sq_flags; in sq_run_events()
6234 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); in sq_run_events()
6235 ASSERT(!(sq->sq_flags & SQ_EVENTS)); in sq_run_events()
6239 cv_broadcast(&sq->sq_wait); in sq_run_events()
6243 cv_broadcast(&sq->sq_exitwait); in sq_run_events()
6245 sq->sq_flags = flags; in sq_run_events()
6256 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) in sqfill_events() argument
6260 ASSERT(MUTEX_HELD(SQLOCK(sq))); in sqfill_events()
6270 if (sq->sq_evhead == NULL) { in sqfill_events()
6271 sq->sq_evhead = sq->sq_evtail = mp; in sqfill_events()
6272 sq->sq_flags |= SQ_EVENTS; in sqfill_events()
6274 ASSERT(sq->sq_evtail != NULL); in sqfill_events()
6275 ASSERT(sq->sq_evtail->b_next == NULL); in sqfill_events()
6276 ASSERT(sq->sq_flags & SQ_EVENTS); in sqfill_events()
6277 sq->sq_evtail->b_next = mp; in sqfill_events()
6278 sq->sq_evtail = mp; in sqfill_events()
6291 count = sq->sq_count; in sqfill_events()
6292 SQ_PUTLOCKS_ENTER(sq); in sqfill_events()
6293 SUM_SQ_PUTCOUNTS(sq, count); in sqfill_events()
6307 SQ_PUTLOCKS_EXIT(sq); in sqfill_events()
6308 mutex_exit(SQLOCK(sq)); in sqfill_events()
6314 ASSERT((sq->sq_flags & SQ_EXCL) == 0); in sqfill_events()
6315 sq->sq_flags |= SQ_EXCL; in sqfill_events()
6316 ASSERT(sq->sq_count == 0); in sqfill_events()
6317 sq->sq_count++; in sqfill_events()
6318 SQ_PUTLOCKS_EXIT(sq); in sqfill_events()
6321 sq_run_events(sq); in sqfill_events()
6326 sq->sq_count--; in sqfill_events()
6333 sq->sq_flags &= ~SQ_EXCL; in sqfill_events()
6338 ASSERT(!(sq->sq_flags & SQ_EXCL)); in sqfill_events()
6347 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) in sqfill_events()
6348 drain_syncq(sq); in sqfill_events()
6350 mutex_exit(SQLOCK(sq)); in sqfill_events()
6370 drain_syncq(syncq_t *sq) in drain_syncq() argument
6374 uint16_t type = sq->sq_type; in drain_syncq()
6375 uint16_t flags = sq->sq_flags; in drain_syncq()
6376 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; in drain_syncq()
6379 "drain_syncq start:%p", sq); in drain_syncq()
6380 ASSERT(MUTEX_HELD(SQLOCK(sq))); in drain_syncq()
6381 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && in drain_syncq()
6382 sq->sq_oprev == NULL) || in drain_syncq()
6383 (sq->sq_outer != NULL && sq->sq_onext != NULL && in drain_syncq()
6384 sq->sq_oprev != NULL)); in drain_syncq()
6390 sq->sq_svcflags &= ~SQ_SERVICE; in drain_syncq()
6398 ASSERT(sq->sq_servcount != 0); in drain_syncq()
6399 sq->sq_servcount--; in drain_syncq()
6401 mutex_exit(SQLOCK(sq)); in drain_syncq()
6414 cv_broadcast(&sq->sq_wait); in drain_syncq()
6418 cv_broadcast(&sq->sq_exitwait); in drain_syncq()
6420 sq->sq_flags = flags; in drain_syncq()
6422 ASSERT(sq->sq_servcount != 0); in drain_syncq()
6423 sq->sq_servcount--; in drain_syncq()
6425 mutex_exit(SQLOCK(sq)); in drain_syncq()
6436 type = sq->sq_type; in drain_syncq()
6438 if (sq->sq_count > 1) { in drain_syncq()
6440 ASSERT(sq->sq_servcount != 0); in drain_syncq()
6441 sq->sq_servcount--; in drain_syncq()
6443 mutex_exit(SQLOCK(sq)); in drain_syncq()
6446 sq->sq_flags |= SQ_EXCL; in drain_syncq()
6460 sq->sq_count++; in drain_syncq()
6461 ASSERT(sq->sq_count != 0); /* wraparound */ in drain_syncq()
6463 while ((flags = sq->sq_flags) & SQ_QUEUED) { in drain_syncq()
6477 if (sq->sq_evhead != NULL) { in drain_syncq()
6478 ASSERT(sq->sq_flags & SQ_EVENTS); in drain_syncq()
6480 count = sq->sq_count; in drain_syncq()
6481 SQ_PUTLOCKS_ENTER(sq); in drain_syncq()
6482 SUM_SQ_PUTCOUNTS(sq, count); in drain_syncq()
6484 SQ_PUTLOCKS_EXIT(sq); in drain_syncq()
6489 sq->sq_flags = flags | SQ_EXCL; in drain_syncq()
6490 SQ_PUTLOCKS_EXIT(sq); in drain_syncq()
6495 sq_run_events(sq); in drain_syncq()
6503 ASSERT(sq->sq_flags & SQ_EXCL); in drain_syncq()
6504 sq->sq_flags &= ~SQ_EXCL; in drain_syncq()
6511 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); in drain_syncq()
6515 ASSERT(sq->sq_evhead == NULL); in drain_syncq()
6516 ASSERT(!(sq->sq_flags & SQ_EVENTS)); in drain_syncq()
6529 for (qp = sq->sq_head; in drain_syncq()
6549 mutex_exit(SQLOCK(sq)); in drain_syncq()
6551 qdrain_syncq(sq, qp); in drain_syncq()
6552 mutex_enter(SQLOCK(sq)); in drain_syncq()
6563 ASSERT(MUTEX_HELD(SQLOCK(sq))); in drain_syncq()
6564 flags = sq->sq_flags; in drain_syncq()
6579 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || in drain_syncq()
6580 (type & SQ_CI) || sq->sq_head->q_draining); in drain_syncq()
6590 cv_broadcast(&sq->sq_wait); in drain_syncq()
6594 cv_broadcast(&sq->sq_exitwait); in drain_syncq()
6596 sq->sq_flags = flags; in drain_syncq()
6598 ASSERT(sq->sq_count != 0); in drain_syncq()
6600 sq->sq_count--; in drain_syncq()
6603 ASSERT(sq->sq_servcount != 0); in drain_syncq()
6604 sq->sq_servcount--; in drain_syncq()
6607 mutex_exit(SQLOCK(sq)); in drain_syncq()
6610 "drain_syncq end:%p", sq); in drain_syncq()
6645 qdrain_syncq(syncq_t *sq, queue_t *q) in qdrain_syncq() argument
6653 "drain_syncq start:%p", sq); in qdrain_syncq()
6654 ASSERT(q->q_syncq == sq); in qdrain_syncq()
6656 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in qdrain_syncq()
6662 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); in qdrain_syncq()
6663 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); in qdrain_syncq()
6664 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); in qdrain_syncq()
6668 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && in qdrain_syncq()
6669 sq->sq_oprev == NULL) || in qdrain_syncq()
6670 (sq->sq_outer != NULL && sq->sq_onext != NULL && in qdrain_syncq()
6671 sq->sq_oprev != NULL)); in qdrain_syncq()
6673 count = sq->sq_count; in qdrain_syncq()
6681 SUM_SQ_PUTCOUNTS(sq, count); in qdrain_syncq()
6727 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { in qdrain_syncq()
6739 ASSERT(bp->b_queue->q_syncq == sq); in qdrain_syncq()
6794 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { in qdrain_syncq()
6795 mutex_enter(SQLOCK(sq)); in qdrain_syncq()
6796 sq->sq_flags &= ~SQ_EXCL; in qdrain_syncq()
6797 mutex_exit(SQLOCK(sq)); in qdrain_syncq()
6807 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); in qdrain_syncq()
6810 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in qdrain_syncq()
6815 mutex_enter(SQLOCK(sq)); in qdrain_syncq()
6817 SQRM_Q(sq, q); in qdrain_syncq()
6818 mutex_exit(SQLOCK(sq)); in qdrain_syncq()
6839 "drain_syncq end:%p", sq); in qdrain_syncq()
6858 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) in qfill_syncq() argument
6860 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in qfill_syncq()
6862 ASSERT(sq->sq_count > 0); in qfill_syncq()
6863 ASSERT(q->q_syncq == sq); in qfill_syncq()
6864 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && in qfill_syncq()
6865 sq->sq_oprev == NULL) || in qfill_syncq()
6866 (sq->sq_outer != NULL && sq->sq_onext != NULL && in qfill_syncq()
6867 sq->sq_oprev != NULL)); in qfill_syncq()
6885 ASSERT(q->q_syncq == sq); in qfill_syncq()
6892 mutex_enter(SQLOCK(sq)); in qfill_syncq()
6902 SQPUT_Q(sq, q); in qfill_syncq()
6909 ASSERT(sq->sq_tail != NULL); in qfill_syncq()
6910 if (sq->sq_tail == sq->sq_head) { in qfill_syncq()
6917 ASSERT(sq->sq_flags & SQ_QUEUED); in qfill_syncq()
6926 sq->sq_count--; in qfill_syncq()
6928 putnext_tail(sq, q, 0); in qfill_syncq()
6951 flush_syncq(syncq_t *sq, queue_t *qp) in flush_syncq() argument
6957 mutex_enter(SQLOCK(sq)); in flush_syncq()
6964 if (qp != NULL && sq->sq_evhead != NULL) { in flush_syncq()
6965 ASSERT(sq->sq_flags & SQ_EVENTS); in flush_syncq()
6968 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { in flush_syncq()
6978 if (bp == sq->sq_evtail) { in flush_syncq()
6980 sq->sq_evtail = mp_prev; in flush_syncq()
6983 sq->sq_evhead = mp_next; in flush_syncq()
6984 if (sq->sq_evhead == NULL) in flush_syncq()
6985 sq->sq_flags &= ~SQ_EVENTS; in flush_syncq()
7000 q = sq->sq_head; in flush_syncq()
7002 ASSERT(q->q_syncq == sq); in flush_syncq()
7016 mp_head->b_queue->q_syncq == sq); in flush_syncq()
7031 SQRM_Q(sq, q); in flush_syncq()
7040 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { in flush_syncq()
7041 sq->sq_flags &= ~SQ_WANTWAKEUP; in flush_syncq()
7042 cv_broadcast(&sq->sq_wait); in flush_syncq()
7045 mutex_exit(SQLOCK(sq)); in flush_syncq()
7057 mutex_enter(SQLOCK(sq)); in flush_syncq()
7062 q = sq->sq_head; in flush_syncq()
7067 ASSERT(MUTEX_HELD(SQLOCK(sq))); in flush_syncq()
7070 if (sq->sq_flags & SQ_WANTWAKEUP) { in flush_syncq()
7071 sq->sq_flags &= ~SQ_WANTWAKEUP; in flush_syncq()
7072 cv_broadcast(&sq->sq_wait); in flush_syncq()
7075 mutex_exit(SQLOCK(sq)); in flush_syncq()
7095 syncq_t *sq; in propagate_syncq() local
7106 sq = qp->q_syncq; in propagate_syncq()
7107 ASSERT(MUTEX_HELD(SQLOCK(sq))); in propagate_syncq()
7109 SQ_PUTLOCKS_HELD(sq); in propagate_syncq()
7115 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); in propagate_syncq()
7136 SQRM_Q(sq, qp); in propagate_syncq()
7200 if (sq->sq_evhead != NULL) { in propagate_syncq()
7201 ASSERT(sq->sq_flags & SQ_EVENTS); in propagate_syncq()
7203 for (bp = sq->sq_evhead; bp != NULL; bp = next) { in propagate_syncq()
7213 if (bp == sq->sq_evtail) { in propagate_syncq()
7215 sq->sq_evtail = prev; in propagate_syncq()
7218 sq->sq_evhead = next; in propagate_syncq()
7219 if (sq->sq_evhead == NULL) in propagate_syncq()
7220 sq->sq_flags &= ~SQ_EVENTS; in propagate_syncq()
7229 flags = sq->sq_flags; in propagate_syncq()
7234 cv_broadcast(&sq->sq_wait); in propagate_syncq()
7236 sq->sq_flags = flags; in propagate_syncq()
7254 syncq_t *sq = q->q_syncq; in qwriter_inner() local
7257 mutex_enter(SQLOCK(sq)); in qwriter_inner()
7258 count = sq->sq_count; in qwriter_inner()
7259 SQ_PUTLOCKS_ENTER(sq); in qwriter_inner()
7260 SUM_SQ_PUTCOUNTS(sq, count); in qwriter_inner()
7262 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); in qwriter_inner()
7270 sq->sq_flags |= SQ_EXCL; in qwriter_inner()
7271 SQ_PUTLOCKS_EXIT(sq); in qwriter_inner()
7272 mutex_exit(SQLOCK(sq)); in qwriter_inner()
7284 SQ_PUTLOCKS_EXIT(sq); in qwriter_inner()
7285 sqfill_events(sq, q, mp, func); in qwriter_inner()
7298 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) in callbparams_alloc() argument
7319 cbp->cbp_sq = sq; in callbparams_alloc()
7322 mutex_enter(SQLOCK(sq)); in callbparams_alloc()
7323 cbp->cbp_next = sq->sq_callbpend; in callbparams_alloc()
7324 sq->sq_callbpend = cbp; in callbparams_alloc()
7329 callbparams_free(syncq_t *sq, callbparams_t *cbp) in callbparams_free() argument
7333 ASSERT(MUTEX_HELD(SQLOCK(sq))); in callbparams_free()
7335 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { in callbparams_free()
7347 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) in callbparams_free_id() argument
7351 ASSERT(MUTEX_HELD(SQLOCK(sq))); in callbparams_free_id()
7353 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { in callbparams_free_id()
7374 syncq_t *sq; in qcallbwrapper() local
7379 sq = cbp->cbp_sq; in qcallbwrapper()
7380 mutex_enter(SQLOCK(sq)); in qcallbwrapper()
7381 type = sq->sq_type; in qcallbwrapper()
7383 count = sq->sq_count; in qcallbwrapper()
7384 SQ_PUTLOCKS_ENTER(sq); in qcallbwrapper()
7385 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); in qcallbwrapper()
7386 SUM_SQ_PUTCOUNTS(sq, count); in qcallbwrapper()
7387 sq->sq_needexcl++; in qcallbwrapper()
7388 ASSERT(sq->sq_needexcl != 0); /* wraparound */ in qcallbwrapper()
7394 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { in qcallbwrapper()
7395 if ((sq->sq_callbflags & cbp->cbp_flags) && in qcallbwrapper()
7396 (sq->sq_cancelid == cbp->cbp_id)) { in qcallbwrapper()
7398 sq->sq_callbflags |= SQ_CALLB_BYPASSED; in qcallbwrapper()
7399 callbparams_free(sq, cbp); in qcallbwrapper()
7401 ASSERT(sq->sq_needexcl > 0); in qcallbwrapper()
7402 sq->sq_needexcl--; in qcallbwrapper()
7403 if (sq->sq_needexcl == 0) { in qcallbwrapper()
7404 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in qcallbwrapper()
7406 SQ_PUTLOCKS_EXIT(sq); in qcallbwrapper()
7408 mutex_exit(SQLOCK(sq)); in qcallbwrapper()
7411 sq->sq_flags |= SQ_WANTWAKEUP; in qcallbwrapper()
7413 SQ_PUTLOCKS_EXIT(sq); in qcallbwrapper()
7415 cv_wait(&sq->sq_wait, SQLOCK(sq)); in qcallbwrapper()
7417 count = sq->sq_count; in qcallbwrapper()
7418 SQ_PUTLOCKS_ENTER(sq); in qcallbwrapper()
7419 SUM_SQ_PUTCOUNTS(sq, count); in qcallbwrapper()
7423 sq->sq_count++; in qcallbwrapper()
7424 ASSERT(sq->sq_count != 0); /* Wraparound */ in qcallbwrapper()
7427 sq->sq_flags |= SQ_EXCL; in qcallbwrapper()
7428 ASSERT(sq->sq_needexcl > 0); in qcallbwrapper()
7429 sq->sq_needexcl--; in qcallbwrapper()
7430 if (sq->sq_needexcl == 0) { in qcallbwrapper()
7431 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in qcallbwrapper()
7433 SQ_PUTLOCKS_EXIT(sq); in qcallbwrapper()
7436 mutex_exit(SQLOCK(sq)); in qcallbwrapper()
7444 mutex_enter(SQLOCK(sq)); in qcallbwrapper()
7445 callbparams_free(sq, cbp); in qcallbwrapper()
7446 mutex_exit(SQLOCK(sq)); in qcallbwrapper()
7447 leavesq(sq, SQ_CALLBACK); in qcallbwrapper()
7469 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) in putnext_tail() argument
7471 uint16_t flags = sq->sq_flags; in putnext_tail()
7473 ASSERT(MUTEX_HELD(SQLOCK(sq))); in putnext_tail()
7482 cv_broadcast(&sq->sq_wait); in putnext_tail()
7486 cv_broadcast(&sq->sq_exitwait); in putnext_tail()
7488 sq->sq_flags = flags; in putnext_tail()
7498 (sq->sq_svcflags & SQ_DISABLED)) { in putnext_tail()
7500 drain_syncq(sq); in putnext_tail()
7503 sqenable(sq); in putnext_tail()
7507 mutex_exit(SQLOCK(sq)); in putnext_tail()
7509 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); in putnext_tail()
7812 syncq_t *sq; in pass_wput() local
7814 sq = _RD(q)->q_syncq; in pass_wput()
7815 if (sq->sq_flags & SQ_BLOCKED) in pass_wput()
7816 unblocksq(sq, SQ_BLOCKED, 0); in pass_wput()
8281 syncq_service(syncq_t *sq) in syncq_service() argument
8284 mutex_enter(SQLOCK(sq)); in syncq_service()
8285 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); in syncq_service()
8286 ASSERT(sq->sq_servcount != 0); in syncq_service()
8287 ASSERT(sq->sq_next == NULL); in syncq_service()
8290 if (sq->sq_svcflags & SQ_BGTHREAD) in syncq_service()
8291 sq->sq_svcflags &= ~SQ_BGTHREAD; in syncq_service()
8294 sq->sq_svcflags |= SQ_SERVICE; in syncq_service()
8295 drain_syncq(sq); in syncq_service()