Lines Matching defs:sqp
137 static void squeue_worker(squeue_t *sqp);
138 static void squeue_polling_thread(squeue_t *sqp);
156 #define ENQUEUE_CHAIN(sqp, mp, tail, cnt) { \
160 ASSERT(MUTEX_HELD(&(sqp)->sq_lock)); \
162 if ((sqp)->sq_last != NULL) \
163 (sqp)->sq_last->b_next = (mp); \
165 (sqp)->sq_first = (mp); \
166 (sqp)->sq_last = (tail); \
167 (sqp)->sq_count += (cnt); \
168 ASSERT((sqp)->sq_count > 0); \
169 DTRACE_PROBE4(squeue__enqueuechain, squeue_t *, sqp, \
181 #define SQS_POLLING_ON(sqp, sq_poll_capable, rx_ring) { \
182 ASSERT(MUTEX_HELD(&(sqp)->sq_lock)); \
185 ASSERT(sqp->sq_state & SQS_POLL_CAPAB); \
186 if (!(sqp->sq_state & SQS_POLLING)) { \
188 sqp->sq_state |= SQS_POLLING; \
193 #define SQS_POLLING_OFF(sqp, sq_poll_capable, rx_ring) { \
194 ASSERT(MUTEX_HELD(&(sqp)->sq_lock)); \
197 ASSERT(sqp->sq_state & SQS_POLL_CAPAB); \
198 if (sqp->sq_state & SQS_POLLING) { \
199 sqp->sq_state &= ~SQS_POLLING; \
206 #define SQS_POLL_RING(sqp) { \
207 ASSERT(MUTEX_HELD(&(sqp)->sq_lock)); \
208 if (sqp->sq_state & SQS_POLLING) { \
209 ASSERT(sqp->sq_state & SQS_POLL_CAPAB); \
210 if (!(sqp->sq_state & SQS_GET_PKTS)) { \
211 sqp->sq_state |= SQS_GET_PKTS; \
212 cv_signal(&sqp->sq_poll_cv); \
218 #define SQUEUE_DBG_SET(sqp, mp, proc, connp, tag) { \
219 (sqp)->sq_curmp = (mp); \
220 (sqp)->sq_curproc = (proc); \
221 (sqp)->sq_connp = (connp); \
222 (mp)->b_tag = (sqp)->sq_tag = (tag); \
225 #define SQUEUE_DBG_CLEAR(sqp) { \
226 (sqp)->sq_curmp = NULL; \
227 (sqp)->sq_curproc = NULL; \
228 (sqp)->sq_connp = NULL; \
231 #define SQUEUE_DBG_SET(sqp, mp, proc, connp, tag)
232 #define SQUEUE_DBG_CLEAR(sqp)
249 squeue_t *sqp = kmem_cache_alloc(squeue_cache, KM_SLEEP);
251 bzero(sqp, sizeof (squeue_t));
252 sqp->sq_bind = PBIND_NONE;
253 sqp->sq_priority = pri;
254 sqp->sq_wait = MSEC_TO_TICK(wait);
255 sqp->sq_worker = thread_create(NULL, 0, squeue_worker,
256 sqp, 0, &p0, TS_RUN, pri);
258 sqp->sq_poll_thr = thread_create(NULL, 0, squeue_polling_thread,
259 sqp, 0, &p0, TS_RUN, pri);
261 sqp->sq_enter = squeue_enter;
262 sqp->sq_drain = squeue_drain;
264 return (sqp);
275 squeue_bind(squeue_t *sqp, processorid_t bind)
277 mutex_enter(&sqp->sq_lock);
278 ASSERT(sqp->sq_bind != PBIND_NONE || bind != PBIND_NONE);
281 if (sqp->sq_state & SQS_BOUND) {
282 if (sqp->sq_bind == bind) {
283 mutex_exit(&sqp->sq_lock);
286 thread_affinity_clear(sqp->sq_worker);
288 sqp->sq_state |= SQS_BOUND;
292 sqp->sq_bind = bind;
294 thread_affinity_set(sqp->sq_worker, sqp->sq_bind);
295 mutex_exit(&sqp->sq_lock);
299 squeue_unbind(squeue_t *sqp)
301 mutex_enter(&sqp->sq_lock);
302 if (!(sqp->sq_state & SQS_BOUND)) {
303 mutex_exit(&sqp->sq_lock);
307 sqp->sq_state &= ~SQS_BOUND;
308 thread_affinity_clear(sqp->sq_worker);
309 mutex_exit(&sqp->sq_lock);
313 squeue_worker_wakeup(squeue_t *sqp)
315 timeout_id_t tid = (sqp)->sq_tid;
317 ASSERT(MUTEX_HELD(&(sqp)->sq_lock));
319 if (sqp->sq_wait == 0) {
321 ASSERT(!(sqp->sq_state & SQS_TMO_PROG));
322 sqp->sq_awaken = ddi_get_lbolt();
323 cv_signal(&sqp->sq_worker_cv);
324 mutex_exit(&sqp->sq_lock);
337 clock_t waited = now - sqp->sq_awaken;
339 if (TICK_TO_MSEC(waited) >= sqp->sq_wait) {
344 sqp->sq_tid = 0;
345 sqp->sq_awaken = now;
346 cv_signal(&sqp->sq_worker_cv);
347 mutex_exit(&sqp->sq_lock);
351 mutex_exit(&sqp->sq_lock);
353 } else if (sqp->sq_state & SQS_TMO_PROG) {
354 mutex_exit(&sqp->sq_lock);
357 clock_t wait = sqp->sq_wait;
359 * Wait up to sqp->sq_wait ms for an
370 sqp->sq_state |= SQS_TMO_PROG;
371 mutex_exit(&sqp->sq_lock);
372 tid = timeout(squeue_fire, sqp, wait);
373 mutex_enter(&sqp->sq_lock);
375 if (((sqp->sq_state & (SQS_PROC|SQS_TMO_PROG)) ==
376 SQS_TMO_PROG) && (sqp->sq_tid == 0) &&
377 (sqp->sq_first != NULL)) {
378 sqp->sq_state &= ~SQS_TMO_PROG;
379 sqp->sq_tid = tid;
380 mutex_exit(&sqp->sq_lock);
383 if (sqp->sq_state & SQS_TMO_PROG) {
384 sqp->sq_state &= ~SQS_TMO_PROG;
385 mutex_exit(&sqp->sq_lock);
395 mutex_exit(&sqp->sq_lock);
400 ASSERT(MUTEX_NOT_HELD(&sqp->sq_lock));
404 * squeue_enter() - enter squeue sqp with mblk mp (which can be
425 squeue_enter(squeue_t *sqp, mblk_t *mp, mblk_t *tail, uint32_t cnt,
432 ASSERT(sqp != NULL);
436 ASSERT(MUTEX_NOT_HELD(&sqp->sq_lock));
439 mutex_enter(&sqp->sq_lock);
446 if (!(sqp->sq_state & SQS_PROC) && ((process_flag == SQ_PROCESS) ||
447 (process_flag == SQ_NODRAIN && sqp->sq_first == NULL))) {
453 if (sqp->sq_first == NULL && cnt == 1) {
457 sqp->sq_state |= (SQS_PROC|SQS_FAST);
458 sqp->sq_run = curthread;
459 mutex_exit(&sqp->sq_lock);
478 if (connp->conn_sqp == sqp) {
479 SQUEUE_DBG_SET(sqp, mp, proc, connp,
483 sqp, mblk_t *, mp, conn_t *, connp);
484 (*proc)(connp, mp, sqp, ira);
486 sqp, conn_t *, connp);
488 SQUEUE_DBG_CLEAR(sqp);
494 ASSERT(MUTEX_NOT_HELD(&sqp->sq_lock));
495 mutex_enter(&sqp->sq_lock);
496 sqp->sq_state &= ~(SQS_PROC|SQS_FAST);
497 sqp->sq_run = NULL;
498 if (sqp->sq_first == NULL ||
500 if (sqp->sq_first != NULL) {
501 squeue_worker_wakeup(sqp);
510 if (sqp->sq_state & SQS_WORKER_THR_CONTROL)
511 cv_signal(&sqp->sq_worker_cv);
512 mutex_exit(&sqp->sq_lock);
522 mutex_exit(&sqp->sq_lock);
543 ENQUEUE_CHAIN(sqp, mp, tail, cnt);
555 ASSERT(MUTEX_HELD(&sqp->sq_lock));
556 ASSERT(sqp->sq_first != NULL);
558 sqp->sq_run = curthread;
559 sqp->sq_drain(sqp, SQS_ENTER, now + squeue_drain_ns);
567 sqp->sq_run = NULL;
568 if (sqp->sq_state & SQS_WORKER_THR_CONTROL)
569 cv_signal(&sqp->sq_worker_cv);
570 mutex_exit(&sqp->sq_lock);
587 if (!(sqp->sq_state & SQS_REENTER) &&
588 (process_flag != SQ_FILL) && (sqp->sq_first == NULL) &&
589 (sqp->sq_run == curthread) && (cnt == 1) &&
591 sqp->sq_state |= SQS_REENTER;
592 mutex_exit(&sqp->sq_lock);
605 if (connp->conn_sqp == sqp) {
608 sqp, mblk_t *, mp, conn_t *, connp);
609 (*proc)(connp, mp, sqp, ira);
611 sqp, conn_t *, connp);
619 mutex_enter(&sqp->sq_lock);
620 sqp->sq_state &= ~SQS_REENTER;
621 mutex_exit(&sqp->sq_lock);
640 mutex_exit(&sqp->sq_lock);
659 ENQUEUE_CHAIN(sqp, mp, tail, cnt);
660 if (!(sqp->sq_state & SQS_PROC)) {
661 squeue_worker_wakeup(sqp);
668 if (sqp->sq_state & SQS_WORKER_THR_CONTROL)
669 cv_signal(&sqp->sq_worker_cv);
670 mutex_exit(&sqp->sq_lock);
682 squeue_t *sqp = arg;
685 mutex_enter(&sqp->sq_lock);
687 state = sqp->sq_state;
688 if (sqp->sq_tid == 0 && !(state & SQS_TMO_PROG)) {
689 mutex_exit(&sqp->sq_lock);
693 sqp->sq_tid = 0;
701 sqp->sq_state &= ~SQS_TMO_PROG;
704 sqp->sq_awaken = ddi_get_lbolt();
705 cv_signal(&sqp->sq_worker_cv);
707 mutex_exit(&sqp->sq_lock);
711 squeue_drain(squeue_t *sqp, uint_t proc_type, hrtime_t expire)
718 ill_rx_ring_t *sq_rx_ring = sqp->sq_rx_ring;
731 ASSERT(mutex_owned(&sqp->sq_lock));
732 sqp->sq_awaken = ddi_get_lbolt();
733 cv_signal(&sqp->sq_worker_cv);
738 sq_poll_capable = (sqp->sq_state & SQS_POLL_CAPAB) != 0;
740 ASSERT(mutex_owned(&sqp->sq_lock));
741 ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |
744 head = sqp->sq_first;
745 sqp->sq_first = NULL;
746 sqp->sq_last = NULL;
747 sqp->sq_count = 0;
749 if ((tid = sqp->sq_tid) != 0)
750 sqp->sq_tid = 0;
752 sqp->sq_state |= SQS_PROC | proc_type;
765 SQS_POLLING_ON(sqp, sq_poll_capable, sq_rx_ring);
766 mutex_exit(&sqp->sq_lock);
810 if (connp->conn_sqp == sqp) {
811 SQUEUE_DBG_SET(sqp, mp, proc, connp,
815 sqp, mblk_t *, mp, conn_t *, connp);
816 (*proc)(connp, mp, sqp, ira);
818 sqp, conn_t *, connp);
829 SQUEUE_DBG_CLEAR(sqp);
831 mutex_enter(&sqp->sq_lock);
842 if (sqp->sq_first != NULL) {
865 SQS_POLL_RING(sqp);
869 sqp->sq_awaken = ddi_get_lbolt();
870 cv_signal(&sqp->sq_worker_cv);
879 if (sqp->sq_state & SQS_GET_PKTS) {
880 ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |
882 sqp->sq_state &= ~proc_type;
892 if ((proc_type == SQS_WORKER) && (sqp->sq_state & SQS_POLLING)) {
912 ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |
914 SQS_POLL_RING(sqp);
915 sqp->sq_state &= ~proc_type;
926 ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |
928 SQS_POLLING_OFF(sqp, sq_poll_capable, sq_rx_ring);
929 sqp->sq_state &= ~(SQS_PROC | proc_type);
930 if (!did_wakeup && sqp->sq_first != NULL) {
931 squeue_worker_wakeup(sqp);
932 mutex_enter(&sqp->sq_lock);
939 (sqp->sq_state & SQS_WORKER_THR_CONTROL))
940 cv_signal(&sqp->sq_worker_cv);
965 squeue_poll_thr_control(squeue_t *sqp)
967 if (sqp->sq_state & SQS_POLL_THR_RESTART) {
969 ASSERT(sqp->sq_state & SQS_POLL_THR_QUIESCED);
970 sqp->sq_state &= ~(SQS_POLL_THR_QUIESCED |
972 sqp->sq_state |= SQS_POLL_CAPAB;
973 cv_signal(&sqp->sq_worker_cv);
977 if (sqp->sq_state & SQS_POLL_THR_QUIESCE) {
978 sqp->sq_state |= SQS_POLL_THR_QUIESCED;
979 sqp->sq_state &= ~SQS_POLL_THR_QUIESCE;
980 cv_signal(&sqp->sq_worker_cv);
1031 squeue_polling_thread(squeue_t *sqp)
1033 kmutex_t *lock = &sqp->sq_lock;
1034 kcondvar_t *async = &sqp->sq_poll_cv;
1054 ctl_state = sqp->sq_state & (SQS_POLL_THR_CONTROL |
1068 squeue_poll_thr_control(sqp);
1072 if (!(sqp->sq_state & SQS_POLL_CAPAB))
1075 ASSERT((sqp->sq_state &
1080 sq_rx_ring = sqp->sq_rx_ring;
1098 mp = ip_accept(sq_ill, sq_rx_ring, sqp, head,
1107 ENQUEUE_CHAIN(sqp, mp, tail, cnt);
1109 ASSERT((sqp->sq_state &
1113 if (sqp->sq_first != NULL && !(sqp->sq_state & SQS_WORKER)) {
1125 sqp->sq_run = curthread;
1126 sqp->sq_drain(sqp, SQS_POLL_PROC, now +
1128 sqp->sq_run = NULL;
1130 if (sqp->sq_first == NULL)
1140 sqp->sq_awaken = ddi_get_lbolt();
1154 sqp->sq_state |= SQS_PROC_HELD;
1155 sqp->sq_state &= ~SQS_GET_PKTS;
1156 cv_signal(&sqp->sq_worker_cv);
1157 } else if (sqp->sq_first == NULL &&
1158 !(sqp->sq_state & SQS_WORKER)) {
1167 sqp->sq_state &= ~(SQS_PROC|SQS_GET_PKTS);
1169 SQS_POLLING_OFF(sqp, B_TRUE, sq_rx_ring);
1176 if (sqp->sq_state & SQS_WORKER_THR_CONTROL)
1177 cv_signal(&sqp->sq_worker_cv);
1183 sqp->sq_state &= ~SQS_GET_PKTS;
1185 if (sqp->sq_state & SQS_POLL_THR_CONTROL) {
1190 squeue_poll_thr_control(sqp);
1202 squeue_worker_thr_control(squeue_t *sqp)
1207 ASSERT(MUTEX_HELD(&sqp->sq_lock));
1209 if (sqp->sq_state & SQS_POLL_RESTART) {
1211 ASSERT((sqp->sq_state & (SQS_PROC_HELD |
1218 sqp->sq_state &= ~SQS_POLL_QUIESCE_DONE;
1219 sqp->sq_state |= SQS_POLL_THR_RESTART;
1220 cv_signal(&sqp->sq_poll_cv);
1221 while (sqp->sq_state & SQS_POLL_THR_QUIESCED)
1222 cv_wait(&sqp->sq_worker_cv, &sqp->sq_lock);
1223 sqp->sq_state &= ~(SQS_POLL_RESTART | SQS_PROC |
1229 sqp->sq_state |= SQS_POLL_RESTART_DONE;
1230 cv_signal(&sqp->sq_ctrlop_done_cv);
1234 if (sqp->sq_state & SQS_PROC_HELD) {
1236 ASSERT(sqp->sq_state & SQS_PROC);
1244 sqp->sq_state |= (SQS_PROC | SQS_WORKER);
1247 if (!(sqp->sq_state & SQS_POLL_THR_QUIESCED)) {
1248 sqp->sq_state |= SQS_POLL_THR_QUIESCE;
1249 cv_signal(&sqp->sq_poll_cv);
1250 while (!(sqp->sq_state & SQS_POLL_THR_QUIESCED))
1251 cv_wait(&sqp->sq_worker_cv, &sqp->sq_lock);
1254 rx_ring = sqp->sq_rx_ring;
1260 mutex_exit(&sqp->sq_lock);
1262 mutex_enter(&sqp->sq_lock);
1264 SQS_POLLING_OFF(sqp, (sqp->sq_state & SQS_POLL_CAPAB) != 0,
1265 sqp->sq_rx_ring);
1266 sqp->sq_state &= ~(SQS_POLL_CAPAB | SQS_GET_PKTS | SQS_PROC_HELD);
1267 if (sqp->sq_state & SQS_POLL_CLEANUP) {
1275 ASSERT(!(sqp->sq_state & SQS_DEFAULT));
1277 sqp->sq_rx_ring = NULL;
1280 sqp->sq_state &= ~(SQS_POLL_CLEANUP | SQS_POLL_THR_QUIESCED |
1282 sqp->sq_ill = NULL;
1288 sqp->sq_state |= SQS_POLL_CLEANUP_DONE;
1290 sqp->sq_state &= ~SQS_POLL_QUIESCE;
1291 sqp->sq_state |= SQS_POLL_QUIESCE_DONE;
1298 cv_signal(&sqp->sq_ctrlop_done_cv);
1300 if (sqp->sq_state & SQS_POLL_CLEANUP_DONE) {
1301 cv_wait(&sqp->sq_worker_cv, &sqp->sq_lock);
1302 sqp->sq_state &= ~(SQS_PROC | SQS_WORKER);
1307 squeue_worker(squeue_t *sqp)
1309 kmutex_t *lock = &sqp->sq_lock;
1310 kcondvar_t *async = &sqp->sq_worker_cv;
1323 if (sqp->sq_state & SQS_PROC_HELD)
1331 if (!(sqp->sq_state & SQS_PROC) &&
1332 ((sqp->sq_state & SQS_WORKER_THR_CONTROL) ||
1333 (sqp->sq_first != NULL)))
1342 if ((sqp->sq_state & SQS_WORKER_THR_CONTROL) &&
1343 (sqp->sq_state & SQS_WORKER))
1350 if (sqp->sq_state & SQS_WORKER_THR_CONTROL) {
1351 squeue_worker_thr_control(sqp);
1354 ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |
1358 if (sqp->sq_state & SQS_PROC_HELD)
1359 sqp->sq_state &= ~SQS_PROC_HELD;
1362 sqp->sq_run = curthread;
1363 sqp->sq_drain(sqp, SQS_WORKER, now + squeue_drain_ns);
1364 sqp->sq_run = NULL;
1369 squeue_getprivate(squeue_t *sqp, sqprivate_t p)
1373 return (&sqp->sq_private[p]);
1381 squeue_t *sqp = connp->conn_sqp;
1387 mutex_enter(&sqp->sq_lock);
1388 sqp->sq_state |= SQS_PAUSE;
1402 while (sqp->sq_state & SQS_PAUSE) {
1403 cv_wait(&sqp->sq_synch_cv, &sqp->sq_lock);
1405 mutex_exit(&sqp->sq_lock);
1411 squeue_t *sqp;
1414 sqp = connp->conn_sqp;
1416 mutex_enter(&sqp->sq_lock);
1417 if (sqp->sq_first == NULL && !(sqp->sq_state & SQS_PROC)) {
1425 ASSERT(sqp->sq_run == NULL);
1427 sqp->sq_state |= SQS_PROC;
1428 sqp->sq_run = curthread;
1429 mutex_exit(&sqp->sq_lock);
1437 if (sqp != connp->conn_sqp) {
1438 mutex_enter(&sqp->sq_lock);
1439 sqp->sq_state &= ~SQS_PROC;
1440 sqp->sq_run = NULL;
1441 mutex_exit(&sqp->sq_lock);
1445 sqp->sq_curmp = NULL;
1446 sqp->sq_curproc = NULL;
1447 sqp->sq_connp = connp;
1456 mutex_exit(&sqp->sq_lock);
1470 ENQUEUE_CHAIN(sqp, mp, mp, 1);
1472 ASSERT(sqp->sq_run != curthread);
1476 cv_wait(&connp->conn_sq_cv, &sqp->sq_lock);
1477 mutex_exit(&sqp->sq_lock);
1489 squeue_t *sqp = connp->conn_sqp;
1491 mutex_enter(&sqp->sq_lock);
1492 if (sqp->sq_run == curthread) {
1493 ASSERT(sqp->sq_state & SQS_PROC);
1495 sqp->sq_state &= ~SQS_PROC;
1496 sqp->sq_run = NULL;
1499 if (sqp->sq_first == NULL) {
1500 mutex_exit(&sqp->sq_lock);
1511 sqp->sq_awaken = ddi_get_lbolt();
1512 cv_signal(&sqp->sq_worker_cv);
1513 mutex_exit(&sqp->sq_lock);
1521 ASSERT(sqp->sq_state & SQS_PAUSE);
1522 sqp->sq_state &= ~SQS_PAUSE;
1525 cv_signal(&sqp->sq_synch_cv);
1526 mutex_exit(&sqp->sq_lock);