1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 22 /* All Rights Reserved */ 23 24 25 /* 26 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 27 * Use is subject to license terms. 28 */ 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 #include <sys/types.h> 33 #include <sys/sysmacros.h> 34 #include <sys/param.h> 35 #include <sys/errno.h> 36 #include <sys/signal.h> 37 #include <sys/proc.h> 38 #include <sys/conf.h> 39 #include <sys/cred.h> 40 #include <sys/user.h> 41 #include <sys/vnode.h> 42 #include <sys/file.h> 43 #include <sys/session.h> 44 #include <sys/stream.h> 45 #include <sys/strsubr.h> 46 #include <sys/stropts.h> 47 #include <sys/poll.h> 48 #include <sys/systm.h> 49 #include <sys/cpuvar.h> 50 #include <sys/uio.h> 51 #include <sys/cmn_err.h> 52 #include <sys/priocntl.h> 53 #include <sys/procset.h> 54 #include <sys/vmem.h> 55 #include <sys/bitmap.h> 56 #include <sys/kmem.h> 57 #include <sys/siginfo.h> 58 #include <sys/vtrace.h> 59 #include <sys/callb.h> 60 #include <sys/debug.h> 61 #include <sys/modctl.h> 62 #include <sys/vmsystm.h> 63 #include <vm/page.h> 64 #include <sys/atomic.h> 65 #include <sys/suntpi.h> 66 #include <sys/strlog.h> 67 #include <sys/promif.h> 68 #include <sys/project.h> 69 #include <sys/vm.h> 70 #include <sys/taskq.h> 71 #include <sys/sunddi.h> 72 #include <sys/sunldi_impl.h> 73 #include <sys/strsun.h> 74 #include <sys/isa_defs.h> 75 #include <sys/multidata.h> 76 #include <sys/pattr.h> 77 #include <sys/strft.h> 78 #include <sys/fs/snode.h> 79 #include <sys/zone.h> 80 #include <sys/open.h> 81 #include <sys/sunldi.h> 82 #include <sys/sad.h> 83 #include <sys/netstack.h> 84 85 #define O_SAMESTR(q) (((q)->q_next) && \ 86 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) 87 88 /* 89 * WARNING: 90 * The variables and routines in this file are private, belonging 91 * to the STREAMS subsystem. These should not be used by modules 92 * or drivers. Compatibility will not be guaranteed. 93 */ 94 95 /* 96 * Id value used to distinguish between different multiplexor links. 97 */ 98 static int32_t lnk_id = 0; 99 100 #define STREAMS_LOPRI MINCLSYSPRI 101 static pri_t streams_lopri = STREAMS_LOPRI; 102 103 #define STRSTAT(x) (str_statistics.x.value.ui64++) 104 typedef struct str_stat { 105 kstat_named_t sqenables; 106 kstat_named_t stenables; 107 kstat_named_t syncqservice; 108 kstat_named_t freebs; 109 kstat_named_t qwr_outer; 110 kstat_named_t rservice; 111 kstat_named_t strwaits; 112 kstat_named_t taskqfails; 113 kstat_named_t bufcalls; 114 kstat_named_t qhelps; 115 kstat_named_t qremoved; 116 kstat_named_t sqremoved; 117 kstat_named_t bcwaits; 118 kstat_named_t sqtoomany; 119 } str_stat_t; 120 121 static str_stat_t str_statistics = { 122 { "sqenables", KSTAT_DATA_UINT64 }, 123 { "stenables", KSTAT_DATA_UINT64 }, 124 { "syncqservice", KSTAT_DATA_UINT64 }, 125 { "freebs", KSTAT_DATA_UINT64 }, 126 { "qwr_outer", KSTAT_DATA_UINT64 }, 127 { "rservice", KSTAT_DATA_UINT64 }, 128 { "strwaits", KSTAT_DATA_UINT64 }, 129 { "taskqfails", KSTAT_DATA_UINT64 }, 130 { "bufcalls", KSTAT_DATA_UINT64 }, 131 { "qhelps", KSTAT_DATA_UINT64 }, 132 { "qremoved", KSTAT_DATA_UINT64 }, 133 { "sqremoved", KSTAT_DATA_UINT64 }, 134 { "bcwaits", KSTAT_DATA_UINT64 }, 135 { "sqtoomany", KSTAT_DATA_UINT64 }, 136 }; 137 138 static kstat_t *str_kstat; 139 140 /* 141 * qrunflag was used previously to control background scheduling of queues. It 142 * is not used anymore, but kept here in case some module still wants to access 143 * it via qready() and setqsched macros. 144 */ 145 char qrunflag; /* Unused */ 146 147 /* 148 * Most of the streams scheduling is done via task queues. Task queues may fail 149 * for non-sleep dispatches, so there are two backup threads servicing failed 150 * requests for queues and syncqs. Both of these threads also service failed 151 * dispatches freebs requests. Queues are put in the list specified by `qhead' 152 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs 153 * requests are put into `freebs_list' which has no tail pointer. All three 154 * lists are protected by a single `service_queue' lock and use 155 * `services_to_run' condition variable for signaling background threads. Use of 156 * a single lock should not be a problem because it is only used under heavy 157 * loads when task queues start to fail and at that time it may be a good idea 158 * to throttle scheduling requests. 159 * 160 * NOTE: queues and syncqs should be scheduled by two separate threads because 161 * queue servicing may be blocked waiting for a syncq which may be also 162 * scheduled for background execution. This may create a deadlock when only one 163 * thread is used for both. 164 */ 165 166 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ 167 168 static kmutex_t service_queue; /* protects all of servicing vars */ 169 static kcondvar_t services_to_run; /* wake up background service thread */ 170 static kcondvar_t syncqs_to_run; /* wake up background service thread */ 171 172 /* 173 * List of queues scheduled for background processing dueue to lack of resources 174 * in the task queues. Protected by service_queue lock; 175 */ 176 static struct queue *qhead; 177 static struct queue *qtail; 178 179 /* 180 * Same list for syncqs 181 */ 182 static syncq_t *sqhead; 183 static syncq_t *sqtail; 184 185 static mblk_t *freebs_list; /* list of buffers to free */ 186 187 /* 188 * Backup threads for servicing queues and syncqs 189 */ 190 kthread_t *streams_qbkgrnd_thread; 191 kthread_t *streams_sqbkgrnd_thread; 192 193 /* 194 * Bufcalls related variables. 195 */ 196 struct bclist strbcalls; /* list of waiting bufcalls */ 197 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ 198 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ 199 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ 200 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ 201 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ 202 203 kmutex_t strresources; /* protects global resources */ 204 kmutex_t muxifier; /* single-threads multiplexor creation */ 205 206 static void *str_stack_init(netstackid_t stackid, netstack_t *ns); 207 static void str_stack_shutdown(netstackid_t stackid, void *arg); 208 static void str_stack_fini(netstackid_t stackid, void *arg); 209 210 extern void time_to_wait(clock_t *, clock_t); 211 212 /* 213 * run_queues is no longer used, but is kept in case some 3-d party 214 * module/driver decides to use it. 215 */ 216 int run_queues = 0; 217 218 /* 219 * sq_max_size is the depth of the syncq (in number of messages) before 220 * qfill_syncq() starts QFULL'ing destination queues. As its primary 221 * consumer - IP is no longer D_MTPERMOD, but there may be other 222 * modules/drivers depend on this syncq flow control, we prefer to 223 * choose a large number as the default value. For potential 224 * performance gain, this value is tunable in /etc/system. 225 */ 226 int sq_max_size = 10000; 227 228 /* 229 * the number of ciputctrl structures per syncq and stream we create when 230 * needed. 231 */ 232 int n_ciputctrl; 233 int max_n_ciputctrl = 16; 234 /* 235 * if n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. 236 */ 237 int min_n_ciputctrl = 2; 238 239 /* 240 * Per-driver/module syncqs 241 * ======================== 242 * 243 * For drivers/modules that use PERMOD or outer syncqs we keep a list of 244 * perdm structures, new entries being added (and new syncqs allocated) when 245 * setq() encounters a module/driver with a streamtab that it hasn't seen 246 * before. 247 * The reason for this mechanism is that some modules and drivers share a 248 * common streamtab and it is necessary for those modules and drivers to also 249 * share a common PERMOD syncq. 250 * 251 * perdm_list --> dm_str == streamtab_1 252 * dm_sq == syncq_1 253 * dm_ref 254 * dm_next --> dm_str == streamtab_2 255 * dm_sq == syncq_2 256 * dm_ref 257 * dm_next --> ... NULL 258 * 259 * The dm_ref field is incremented for each new driver/module that takes 260 * a reference to the perdm structure and hence shares the syncq. 261 * References are held in the fmodsw_impl_t structure for each STREAMS module 262 * or the dev_impl array (indexed by device major number) for each driver. 263 * 264 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL 265 * ^ ^ ^ ^ 266 * | ______________/ | | 267 * | / | | 268 * dev_impl: ...|x|y|... module A module B 269 * 270 * When a module/driver is unloaded the reference count is decremented and, 271 * when it falls to zero, the perdm structure is removed from the list and 272 * the syncq is freed (see rele_dm()). 273 */ 274 perdm_t *perdm_list = NULL; 275 static krwlock_t perdm_rwlock; 276 cdevsw_impl_t *devimpl; 277 278 extern struct qinit strdata; 279 extern struct qinit stwdata; 280 281 static void runservice(queue_t *); 282 static void streams_bufcall_service(void); 283 static void streams_qbkgrnd_service(void); 284 static void streams_sqbkgrnd_service(void); 285 static syncq_t *new_syncq(void); 286 static void free_syncq(syncq_t *); 287 static void outer_insert(syncq_t *, syncq_t *); 288 static void outer_remove(syncq_t *, syncq_t *); 289 static void write_now(syncq_t *); 290 static void clr_qfull(queue_t *); 291 static void enable_svc(queue_t *); 292 static void runbufcalls(void); 293 static void sqenable(syncq_t *); 294 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); 295 static void wait_q_syncq(queue_t *); 296 static void backenable_insertedq(queue_t *); 297 298 static void queue_service(queue_t *); 299 static void stream_service(stdata_t *); 300 static void syncq_service(syncq_t *); 301 static void qwriter_outer_service(syncq_t *); 302 static void mblk_free(mblk_t *); 303 #ifdef DEBUG 304 static int qprocsareon(queue_t *); 305 #endif 306 307 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); 308 static void reset_nfsrv_ptr(queue_t *, queue_t *); 309 310 static void sq_run_events(syncq_t *); 311 static int propagate_syncq(queue_t *); 312 313 static void blocksq(syncq_t *, ushort_t, int); 314 static void unblocksq(syncq_t *, ushort_t, int); 315 static int dropsq(syncq_t *, uint16_t); 316 static void emptysq(syncq_t *); 317 static sqlist_t *sqlist_alloc(struct stdata *, int); 318 static void sqlist_free(sqlist_t *); 319 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); 320 static void sqlist_insert(sqlist_t *, syncq_t *); 321 static void sqlist_insertall(sqlist_t *, queue_t *); 322 323 static void strsetuio(stdata_t *); 324 325 struct kmem_cache *stream_head_cache; 326 struct kmem_cache *queue_cache; 327 struct kmem_cache *syncq_cache; 328 struct kmem_cache *qband_cache; 329 struct kmem_cache *linkinfo_cache; 330 struct kmem_cache *ciputctrl_cache = NULL; 331 332 static linkinfo_t *linkinfo_list; 333 334 /* global esballoc throttling queue */ 335 static esb_queue_t system_esbq; 336 337 /* 338 * esballoc tunable parameters. 339 */ 340 int esbq_max_qlen = 0x16; /* throttled queue length */ 341 clock_t esbq_timeout = 0x8; /* timeout to process esb queue */ 342 343 /* 344 * routines to handle esballoc queuing. 345 */ 346 static void esballoc_process_queue(esb_queue_t *); 347 static void esballoc_enqueue_mblk(mblk_t *); 348 static void esballoc_timer(void *); 349 static void esballoc_set_timer(esb_queue_t *, clock_t); 350 static void esballoc_mblk_free(mblk_t *); 351 352 /* 353 * Qinit structure and Module_info structures 354 * for passthru read and write queues 355 */ 356 357 static void pass_wput(queue_t *, mblk_t *); 358 static queue_t *link_addpassthru(stdata_t *); 359 static void link_rempassthru(queue_t *); 360 361 struct module_info passthru_info = { 362 0, 363 "passthru", 364 0, 365 INFPSZ, 366 STRHIGH, 367 STRLOW 368 }; 369 370 struct qinit passthru_rinit = { 371 (int (*)())putnext, 372 NULL, 373 NULL, 374 NULL, 375 NULL, 376 &passthru_info, 377 NULL 378 }; 379 380 struct qinit passthru_winit = { 381 (int (*)()) pass_wput, 382 NULL, 383 NULL, 384 NULL, 385 NULL, 386 &passthru_info, 387 NULL 388 }; 389 390 /* 391 * Special form of assertion: verify that X implies Y i.e. when X is true Y 392 * should also be true. 393 */ 394 #define IMPLY(X, Y) ASSERT(!(X) || (Y)) 395 396 /* 397 * Logical equivalence. Verify that both X and Y are either TRUE or FALSE. 398 */ 399 #define EQUIV(X, Y) { IMPLY(X, Y); IMPLY(Y, X); } 400 401 /* 402 * Verify correctness of list head/tail pointers. 403 */ 404 #define LISTCHECK(head, tail, link) { \ 405 EQUIV(head, tail); \ 406 IMPLY(tail != NULL, tail->link == NULL); \ 407 } 408 409 /* 410 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail' 411 * using a `link' field. 412 */ 413 #define ENQUEUE(el, head, tail, link) { \ 414 ASSERT(el->link == NULL); \ 415 LISTCHECK(head, tail, link); \ 416 if (head == NULL) \ 417 head = el; \ 418 else \ 419 tail->link = el; \ 420 tail = el; \ 421 } 422 423 /* 424 * Dequeue the first element of the list denoted by `head' and `tail' pointers 425 * using a `link' field and put result into `el'. 426 */ 427 #define DQ(el, head, tail, link) { \ 428 LISTCHECK(head, tail, link); \ 429 el = head; \ 430 if (head != NULL) { \ 431 head = head->link; \ 432 if (head == NULL) \ 433 tail = NULL; \ 434 el->link = NULL; \ 435 } \ 436 } 437 438 /* 439 * Remove `el' from the list using `chase' and `curr' pointers and return result 440 * in `succeed'. 441 */ 442 #define RMQ(el, head, tail, link, chase, curr, succeed) { \ 443 LISTCHECK(head, tail, link); \ 444 chase = NULL; \ 445 succeed = 0; \ 446 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ 447 chase = curr; \ 448 if (curr != NULL) { \ 449 succeed = 1; \ 450 ASSERT(curr == el); \ 451 if (chase != NULL) \ 452 chase->link = curr->link; \ 453 else \ 454 head = curr->link; \ 455 curr->link = NULL; \ 456 if (curr == tail) \ 457 tail = chase; \ 458 } \ 459 LISTCHECK(head, tail, link); \ 460 } 461 462 /* Handling of delayed messages on the inner syncq. */ 463 464 /* 465 * DEBUG versions should use function versions (to simplify tracing) and 466 * non-DEBUG kernels should use macro versions. 467 */ 468 469 /* 470 * Put a queue on the syncq list of queues. 471 * Assumes SQLOCK held. 472 */ 473 #define SQPUT_Q(sq, qp) \ 474 { \ 475 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 476 if (!(qp->q_sqflags & Q_SQQUEUED)) { \ 477 /* The queue should not be linked anywhere */ \ 478 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ 479 /* Head and tail may only be NULL simultaneously */ \ 480 EQUIV(sq->sq_head, sq->sq_tail); \ 481 /* Queue may be only enqueyed on its syncq */ \ 482 ASSERT(sq == qp->q_syncq); \ 483 /* Check the correctness of SQ_MESSAGES flag */ \ 484 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ 485 /* Sanity check first/last elements of the list */ \ 486 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ 487 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ 488 /* \ 489 * Sanity check of priority field: empty queue should \ 490 * have zero priority \ 491 * and nqueues equal to zero. \ 492 */ \ 493 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ 494 /* Sanity check of sq_nqueues field */ \ 495 EQUIV(sq->sq_head, sq->sq_nqueues); \ 496 if (sq->sq_head == NULL) { \ 497 sq->sq_head = sq->sq_tail = qp; \ 498 sq->sq_flags |= SQ_MESSAGES; \ 499 } else if (qp->q_spri == 0) { \ 500 qp->q_sqprev = sq->sq_tail; \ 501 sq->sq_tail->q_sqnext = qp; \ 502 sq->sq_tail = qp; \ 503 } else { \ 504 /* \ 505 * Put this queue in priority order: higher \ 506 * priority gets closer to the head. \ 507 */ \ 508 queue_t **qpp = &sq->sq_tail; \ 509 queue_t *qnext = NULL; \ 510 \ 511 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ 512 qnext = *qpp; \ 513 qpp = &(*qpp)->q_sqprev; \ 514 } \ 515 qp->q_sqnext = qnext; \ 516 qp->q_sqprev = *qpp; \ 517 if (*qpp != NULL) { \ 518 (*qpp)->q_sqnext = qp; \ 519 } else { \ 520 sq->sq_head = qp; \ 521 sq->sq_pri = sq->sq_head->q_spri; \ 522 } \ 523 *qpp = qp; \ 524 } \ 525 qp->q_sqflags |= Q_SQQUEUED; \ 526 qp->q_sqtstamp = lbolt; \ 527 sq->sq_nqueues++; \ 528 } \ 529 } 530 531 /* 532 * Remove a queue from the syncq list 533 * Assumes SQLOCK held. 534 */ 535 #define SQRM_Q(sq, qp) \ 536 { \ 537 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 538 ASSERT(qp->q_sqflags & Q_SQQUEUED); \ 539 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ 540 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ 541 /* Check that the queue is actually in the list */ \ 542 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ 543 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ 544 ASSERT(sq->sq_nqueues != 0); \ 545 if (qp->q_sqprev == NULL) { \ 546 /* First queue on list, make head q_sqnext */ \ 547 sq->sq_head = qp->q_sqnext; \ 548 } else { \ 549 /* Make prev->next == next */ \ 550 qp->q_sqprev->q_sqnext = qp->q_sqnext; \ 551 } \ 552 if (qp->q_sqnext == NULL) { \ 553 /* Last queue on list, make tail sqprev */ \ 554 sq->sq_tail = qp->q_sqprev; \ 555 } else { \ 556 /* Make next->prev == prev */ \ 557 qp->q_sqnext->q_sqprev = qp->q_sqprev; \ 558 } \ 559 /* clear out references on this queue */ \ 560 qp->q_sqprev = qp->q_sqnext = NULL; \ 561 qp->q_sqflags &= ~Q_SQQUEUED; \ 562 /* If there is nothing queued, clear SQ_MESSAGES */ \ 563 if (sq->sq_head != NULL) { \ 564 sq->sq_pri = sq->sq_head->q_spri; \ 565 } else { \ 566 sq->sq_flags &= ~SQ_MESSAGES; \ 567 sq->sq_pri = 0; \ 568 } \ 569 sq->sq_nqueues--; \ 570 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ 571 (sq->sq_flags & SQ_QUEUED) == 0); \ 572 } 573 574 /* Hide the definition from the header file. */ 575 #ifdef SQPUT_MP 576 #undef SQPUT_MP 577 #endif 578 579 /* 580 * Put a message on the queue syncq. 581 * Assumes QLOCK held. 582 */ 583 #define SQPUT_MP(qp, mp) \ 584 { \ 585 ASSERT(MUTEX_HELD(QLOCK(qp))); \ 586 ASSERT(qp->q_sqhead == NULL || \ 587 (qp->q_sqtail != NULL && \ 588 qp->q_sqtail->b_next == NULL)); \ 589 qp->q_syncqmsgs++; \ 590 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ 591 if (qp->q_sqhead == NULL) { \ 592 qp->q_sqhead = qp->q_sqtail = mp; \ 593 } else { \ 594 qp->q_sqtail->b_next = mp; \ 595 qp->q_sqtail = mp; \ 596 } \ 597 ASSERT(qp->q_syncqmsgs > 0); \ 598 } 599 600 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ 601 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 602 if ((sq)->sq_ciputctrl != NULL) { \ 603 int i; \ 604 int nlocks = (sq)->sq_nciputctrl; \ 605 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 606 ASSERT((sq)->sq_type & SQ_CIPUT); \ 607 for (i = 0; i <= nlocks; i++) { \ 608 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 609 cip[i].ciputctrl_count |= SQ_FASTPUT; \ 610 } \ 611 } \ 612 } 613 614 615 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ 616 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 617 if ((sq)->sq_ciputctrl != NULL) { \ 618 int i; \ 619 int nlocks = (sq)->sq_nciputctrl; \ 620 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 621 ASSERT((sq)->sq_type & SQ_CIPUT); \ 622 for (i = 0; i <= nlocks; i++) { \ 623 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 624 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ 625 } \ 626 } \ 627 } 628 629 /* 630 * Run service procedures for all queues in the stream head. 631 */ 632 #define STR_SERVICE(stp, q) { \ 633 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ 634 while (stp->sd_qhead != NULL) { \ 635 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ 636 ASSERT(stp->sd_nqueues > 0); \ 637 stp->sd_nqueues--; \ 638 ASSERT(!(q->q_flag & QINSERVICE)); \ 639 mutex_exit(&stp->sd_qlock); \ 640 queue_service(q); \ 641 mutex_enter(&stp->sd_qlock); \ 642 } \ 643 ASSERT(stp->sd_nqueues == 0); \ 644 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ 645 } 646 647 /* 648 * constructor/destructor routines for the stream head cache 649 */ 650 /* ARGSUSED */ 651 static int 652 stream_head_constructor(void *buf, void *cdrarg, int kmflags) 653 { 654 stdata_t *stp = buf; 655 656 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); 657 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); 658 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); 659 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); 660 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); 661 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); 662 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); 663 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); 664 stp->sd_wrq = NULL; 665 666 return (0); 667 } 668 669 /* ARGSUSED */ 670 static void 671 stream_head_destructor(void *buf, void *cdrarg) 672 { 673 stdata_t *stp = buf; 674 675 mutex_destroy(&stp->sd_lock); 676 mutex_destroy(&stp->sd_reflock); 677 mutex_destroy(&stp->sd_qlock); 678 cv_destroy(&stp->sd_monitor); 679 cv_destroy(&stp->sd_iocmonitor); 680 cv_destroy(&stp->sd_refmonitor); 681 cv_destroy(&stp->sd_qcv); 682 cv_destroy(&stp->sd_zcopy_wait); 683 } 684 685 /* 686 * constructor/destructor routines for the queue cache 687 */ 688 /* ARGSUSED */ 689 static int 690 queue_constructor(void *buf, void *cdrarg, int kmflags) 691 { 692 queinfo_t *qip = buf; 693 queue_t *qp = &qip->qu_rqueue; 694 queue_t *wqp = &qip->qu_wqueue; 695 syncq_t *sq = &qip->qu_syncq; 696 697 qp->q_first = NULL; 698 qp->q_link = NULL; 699 qp->q_count = 0; 700 qp->q_mblkcnt = 0; 701 qp->q_sqhead = NULL; 702 qp->q_sqtail = NULL; 703 qp->q_sqnext = NULL; 704 qp->q_sqprev = NULL; 705 qp->q_sqflags = 0; 706 qp->q_rwcnt = 0; 707 qp->q_spri = 0; 708 709 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); 710 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); 711 712 wqp->q_first = NULL; 713 wqp->q_link = NULL; 714 wqp->q_count = 0; 715 wqp->q_mblkcnt = 0; 716 wqp->q_sqhead = NULL; 717 wqp->q_sqtail = NULL; 718 wqp->q_sqnext = NULL; 719 wqp->q_sqprev = NULL; 720 wqp->q_sqflags = 0; 721 wqp->q_rwcnt = 0; 722 wqp->q_spri = 0; 723 724 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); 725 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); 726 727 sq->sq_head = NULL; 728 sq->sq_tail = NULL; 729 sq->sq_evhead = NULL; 730 sq->sq_evtail = NULL; 731 sq->sq_callbpend = NULL; 732 sq->sq_outer = NULL; 733 sq->sq_onext = NULL; 734 sq->sq_oprev = NULL; 735 sq->sq_next = NULL; 736 sq->sq_svcflags = 0; 737 sq->sq_servcount = 0; 738 sq->sq_needexcl = 0; 739 sq->sq_nqueues = 0; 740 sq->sq_pri = 0; 741 742 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 743 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 744 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 745 746 return (0); 747 } 748 749 /* ARGSUSED */ 750 static void 751 queue_destructor(void *buf, void *cdrarg) 752 { 753 queinfo_t *qip = buf; 754 queue_t *qp = &qip->qu_rqueue; 755 queue_t *wqp = &qip->qu_wqueue; 756 syncq_t *sq = &qip->qu_syncq; 757 758 ASSERT(qp->q_sqhead == NULL); 759 ASSERT(wqp->q_sqhead == NULL); 760 ASSERT(qp->q_sqnext == NULL); 761 ASSERT(wqp->q_sqnext == NULL); 762 ASSERT(qp->q_rwcnt == 0); 763 ASSERT(wqp->q_rwcnt == 0); 764 765 mutex_destroy(&qp->q_lock); 766 cv_destroy(&qp->q_wait); 767 768 mutex_destroy(&wqp->q_lock); 769 cv_destroy(&wqp->q_wait); 770 771 mutex_destroy(&sq->sq_lock); 772 cv_destroy(&sq->sq_wait); 773 cv_destroy(&sq->sq_exitwait); 774 } 775 776 /* 777 * constructor/destructor routines for the syncq cache 778 */ 779 /* ARGSUSED */ 780 static int 781 syncq_constructor(void *buf, void *cdrarg, int kmflags) 782 { 783 syncq_t *sq = buf; 784 785 bzero(buf, sizeof (syncq_t)); 786 787 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 788 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 789 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 790 791 return (0); 792 } 793 794 /* ARGSUSED */ 795 static void 796 syncq_destructor(void *buf, void *cdrarg) 797 { 798 syncq_t *sq = buf; 799 800 ASSERT(sq->sq_head == NULL); 801 ASSERT(sq->sq_tail == NULL); 802 ASSERT(sq->sq_evhead == NULL); 803 ASSERT(sq->sq_evtail == NULL); 804 ASSERT(sq->sq_callbpend == NULL); 805 ASSERT(sq->sq_callbflags == 0); 806 ASSERT(sq->sq_outer == NULL); 807 ASSERT(sq->sq_onext == NULL); 808 ASSERT(sq->sq_oprev == NULL); 809 ASSERT(sq->sq_next == NULL); 810 ASSERT(sq->sq_needexcl == 0); 811 ASSERT(sq->sq_svcflags == 0); 812 ASSERT(sq->sq_servcount == 0); 813 ASSERT(sq->sq_nqueues == 0); 814 ASSERT(sq->sq_pri == 0); 815 ASSERT(sq->sq_count == 0); 816 ASSERT(sq->sq_rmqcount == 0); 817 ASSERT(sq->sq_cancelid == 0); 818 ASSERT(sq->sq_ciputctrl == NULL); 819 ASSERT(sq->sq_nciputctrl == 0); 820 ASSERT(sq->sq_type == 0); 821 ASSERT(sq->sq_flags == 0); 822 823 mutex_destroy(&sq->sq_lock); 824 cv_destroy(&sq->sq_wait); 825 cv_destroy(&sq->sq_exitwait); 826 } 827 828 /* ARGSUSED */ 829 static int 830 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) 831 { 832 ciputctrl_t *cip = buf; 833 int i; 834 835 for (i = 0; i < n_ciputctrl; i++) { 836 cip[i].ciputctrl_count = SQ_FASTPUT; 837 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); 838 } 839 840 return (0); 841 } 842 843 /* ARGSUSED */ 844 static void 845 ciputctrl_destructor(void *buf, void *cdrarg) 846 { 847 ciputctrl_t *cip = buf; 848 int i; 849 850 for (i = 0; i < n_ciputctrl; i++) { 851 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); 852 mutex_destroy(&cip[i].ciputctrl_lock); 853 } 854 } 855 856 /* 857 * Init routine run from main at boot time. 858 */ 859 void 860 strinit(void) 861 { 862 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 863 864 stream_head_cache = kmem_cache_create("stream_head_cache", 865 sizeof (stdata_t), 0, 866 stream_head_constructor, stream_head_destructor, NULL, 867 NULL, NULL, 0); 868 869 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, 870 queue_constructor, queue_destructor, NULL, NULL, NULL, 0); 871 872 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, 873 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); 874 875 qband_cache = kmem_cache_create("qband_cache", 876 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 877 878 linkinfo_cache = kmem_cache_create("linkinfo_cache", 879 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 880 881 n_ciputctrl = ncpus; 882 n_ciputctrl = 1 << highbit(n_ciputctrl - 1); 883 ASSERT(n_ciputctrl >= 1); 884 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); 885 if (n_ciputctrl >= min_n_ciputctrl) { 886 ciputctrl_cache = kmem_cache_create("ciputctrl_cache", 887 sizeof (ciputctrl_t) * n_ciputctrl, 888 sizeof (ciputctrl_t), ciputctrl_constructor, 889 ciputctrl_destructor, NULL, NULL, NULL, 0); 890 } 891 892 streams_taskq = system_taskq; 893 894 if (streams_taskq == NULL) 895 panic("strinit: no memory for streams taskq!"); 896 897 bc_bkgrnd_thread = thread_create(NULL, 0, 898 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); 899 900 streams_qbkgrnd_thread = thread_create(NULL, 0, 901 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 902 903 streams_sqbkgrnd_thread = thread_create(NULL, 0, 904 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 905 906 /* 907 * Create STREAMS kstats. 908 */ 909 str_kstat = kstat_create("streams", 0, "strstat", 910 "net", KSTAT_TYPE_NAMED, 911 sizeof (str_statistics) / sizeof (kstat_named_t), 912 KSTAT_FLAG_VIRTUAL); 913 914 if (str_kstat != NULL) { 915 str_kstat->ks_data = &str_statistics; 916 kstat_install(str_kstat); 917 } 918 919 /* 920 * TPI support routine initialisation. 921 */ 922 tpi_init(); 923 924 /* 925 * Handle to have autopush and persistent link information per 926 * zone. 927 * Note: uses shutdown hook instead of destroy hook so that the 928 * persistent links can be torn down before the destroy hooks 929 * in the TCP/IP stack are called. 930 */ 931 netstack_register(NS_STR, str_stack_init, str_stack_shutdown, 932 str_stack_fini); 933 } 934 935 void 936 str_sendsig(vnode_t *vp, int event, uchar_t band, int error) 937 { 938 struct stdata *stp; 939 940 ASSERT(vp->v_stream); 941 stp = vp->v_stream; 942 /* Have to hold sd_lock to prevent siglist from changing */ 943 mutex_enter(&stp->sd_lock); 944 if (stp->sd_sigflags & event) 945 strsendsig(stp->sd_siglist, event, band, error); 946 mutex_exit(&stp->sd_lock); 947 } 948 949 /* 950 * Send the "sevent" set of signals to a process. 951 * This might send more than one signal if the process is registered 952 * for multiple events. The caller should pass in an sevent that only 953 * includes the events for which the process has registered. 954 */ 955 static void 956 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, 957 uchar_t band, int error) 958 { 959 ASSERT(MUTEX_HELD(&proc->p_lock)); 960 961 info->si_band = 0; 962 info->si_errno = 0; 963 964 if (sevent & S_ERROR) { 965 sevent &= ~S_ERROR; 966 info->si_code = POLL_ERR; 967 info->si_errno = error; 968 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 969 "strsendsig:proc %p info %p", proc, info); 970 sigaddq(proc, NULL, info, KM_NOSLEEP); 971 info->si_errno = 0; 972 } 973 if (sevent & S_HANGUP) { 974 sevent &= ~S_HANGUP; 975 info->si_code = POLL_HUP; 976 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 977 "strsendsig:proc %p info %p", proc, info); 978 sigaddq(proc, NULL, info, KM_NOSLEEP); 979 } 980 if (sevent & S_HIPRI) { 981 sevent &= ~S_HIPRI; 982 info->si_code = POLL_PRI; 983 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 984 "strsendsig:proc %p info %p", proc, info); 985 sigaddq(proc, NULL, info, KM_NOSLEEP); 986 } 987 if (sevent & S_RDBAND) { 988 sevent &= ~S_RDBAND; 989 if (events & S_BANDURG) 990 sigtoproc(proc, NULL, SIGURG); 991 else 992 sigtoproc(proc, NULL, SIGPOLL); 993 } 994 if (sevent & S_WRBAND) { 995 sevent &= ~S_WRBAND; 996 sigtoproc(proc, NULL, SIGPOLL); 997 } 998 if (sevent & S_INPUT) { 999 sevent &= ~S_INPUT; 1000 info->si_code = POLL_IN; 1001 info->si_band = band; 1002 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1003 "strsendsig:proc %p info %p", proc, info); 1004 sigaddq(proc, NULL, info, KM_NOSLEEP); 1005 info->si_band = 0; 1006 } 1007 if (sevent & S_OUTPUT) { 1008 sevent &= ~S_OUTPUT; 1009 info->si_code = POLL_OUT; 1010 info->si_band = band; 1011 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1012 "strsendsig:proc %p info %p", proc, info); 1013 sigaddq(proc, NULL, info, KM_NOSLEEP); 1014 info->si_band = 0; 1015 } 1016 if (sevent & S_MSG) { 1017 sevent &= ~S_MSG; 1018 info->si_code = POLL_MSG; 1019 info->si_band = band; 1020 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1021 "strsendsig:proc %p info %p", proc, info); 1022 sigaddq(proc, NULL, info, KM_NOSLEEP); 1023 info->si_band = 0; 1024 } 1025 if (sevent & S_RDNORM) { 1026 sevent &= ~S_RDNORM; 1027 sigtoproc(proc, NULL, SIGPOLL); 1028 } 1029 if (sevent != 0) { 1030 panic("strsendsig: unknown event(s) %x", sevent); 1031 } 1032 } 1033 1034 /* 1035 * Send SIGPOLL/SIGURG signal to all processes and process groups 1036 * registered on the given signal list that want a signal for at 1037 * least one of the specified events. 1038 * 1039 * Must be called with exclusive access to siglist (caller holding sd_lock). 1040 * 1041 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding 1042 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure 1043 * while it is in the siglist. 1044 * 1045 * For performance reasons (MP scalability) the code drops pidlock 1046 * when sending signals to a single process. 1047 * When sending to a process group the code holds 1048 * pidlock to prevent the membership in the process group from changing 1049 * while walking the p_pglink list. 1050 */ 1051 void 1052 strsendsig(strsig_t *siglist, int event, uchar_t band, int error) 1053 { 1054 strsig_t *ssp; 1055 k_siginfo_t info; 1056 struct pid *pidp; 1057 proc_t *proc; 1058 1059 info.si_signo = SIGPOLL; 1060 info.si_errno = 0; 1061 for (ssp = siglist; ssp; ssp = ssp->ss_next) { 1062 int sevent; 1063 1064 sevent = ssp->ss_events & event; 1065 if (sevent == 0) 1066 continue; 1067 1068 if ((pidp = ssp->ss_pidp) == NULL) { 1069 /* pid was released but still on event list */ 1070 continue; 1071 } 1072 1073 1074 if (ssp->ss_pid > 0) { 1075 /* 1076 * XXX This unfortunately still generates 1077 * a signal when a fd is closed but 1078 * the proc is active. 1079 */ 1080 ASSERT(ssp->ss_pid == pidp->pid_id); 1081 1082 mutex_enter(&pidlock); 1083 proc = prfind_zone(pidp->pid_id, ALL_ZONES); 1084 if (proc == NULL) { 1085 mutex_exit(&pidlock); 1086 continue; 1087 } 1088 mutex_enter(&proc->p_lock); 1089 mutex_exit(&pidlock); 1090 dosendsig(proc, ssp->ss_events, sevent, &info, 1091 band, error); 1092 mutex_exit(&proc->p_lock); 1093 } else { 1094 /* 1095 * Send to process group. Hold pidlock across 1096 * calls to dosendsig(). 1097 */ 1098 pid_t pgrp = -ssp->ss_pid; 1099 1100 mutex_enter(&pidlock); 1101 proc = pgfind_zone(pgrp, ALL_ZONES); 1102 while (proc != NULL) { 1103 mutex_enter(&proc->p_lock); 1104 dosendsig(proc, ssp->ss_events, sevent, 1105 &info, band, error); 1106 mutex_exit(&proc->p_lock); 1107 proc = proc->p_pglink; 1108 } 1109 mutex_exit(&pidlock); 1110 } 1111 } 1112 } 1113 1114 /* 1115 * Attach a stream device or module. 1116 * qp is a read queue; the new queue goes in so its next 1117 * read ptr is the argument, and the write queue corresponding 1118 * to the argument points to this queue. Return 0 on success, 1119 * or a non-zero errno on failure. 1120 */ 1121 int 1122 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, 1123 boolean_t is_insert) 1124 { 1125 major_t major; 1126 cdevsw_impl_t *dp; 1127 struct streamtab *str; 1128 queue_t *rq; 1129 queue_t *wrq; 1130 uint32_t qflag; 1131 uint32_t sqtype; 1132 perdm_t *dmp; 1133 int error; 1134 int sflag; 1135 1136 rq = allocq(); 1137 wrq = _WR(rq); 1138 STREAM(rq) = STREAM(wrq) = STREAM(qp); 1139 1140 if (fp != NULL) { 1141 str = fp->f_str; 1142 qflag = fp->f_qflag; 1143 sqtype = fp->f_sqtype; 1144 dmp = fp->f_dmp; 1145 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 1146 sflag = MODOPEN; 1147 1148 /* 1149 * stash away a pointer to the module structure so we can 1150 * unref it in qdetach. 1151 */ 1152 rq->q_fp = fp; 1153 } else { 1154 ASSERT(!is_insert); 1155 1156 major = getmajor(*devp); 1157 dp = &devimpl[major]; 1158 1159 str = dp->d_str; 1160 ASSERT(str == STREAMSTAB(major)); 1161 1162 qflag = dp->d_qflag; 1163 ASSERT(qflag & QISDRV); 1164 sqtype = dp->d_sqtype; 1165 1166 /* create perdm_t if needed */ 1167 if (NEED_DM(dp->d_dmp, qflag)) 1168 dp->d_dmp = hold_dm(str, qflag, sqtype); 1169 1170 dmp = dp->d_dmp; 1171 sflag = 0; 1172 } 1173 1174 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, 1175 "qattach:qflag == %X(%X)", qflag, *devp); 1176 1177 /* setq might sleep in allocator - avoid holding locks. */ 1178 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); 1179 1180 /* 1181 * Before calling the module's open routine, set up the q_next 1182 * pointer for inserting a module in the middle of a stream. 1183 * 1184 * Note that we can always set _QINSERTING and set up q_next 1185 * pointer for both inserting and pushing a module. Then there 1186 * is no need for the is_insert parameter. In insertq(), called 1187 * by qprocson(), assume that q_next of the new module always points 1188 * to the correct queue and use it for insertion. Everything should 1189 * work out fine. But in the first release of _I_INSERT, we 1190 * distinguish between inserting and pushing to make sure that 1191 * pushing a module follows the same code path as before. 1192 */ 1193 if (is_insert) { 1194 rq->q_flag |= _QINSERTING; 1195 rq->q_next = qp; 1196 } 1197 1198 /* 1199 * If there is an outer perimeter get exclusive access during 1200 * the open procedure. Bump up the reference count on the queue. 1201 */ 1202 entersq(rq->q_syncq, SQ_OPENCLOSE); 1203 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); 1204 if (error != 0) 1205 goto failed; 1206 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1207 ASSERT(qprocsareon(rq)); 1208 return (0); 1209 1210 failed: 1211 rq->q_flag &= ~_QINSERTING; 1212 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) 1213 qprocsoff(rq); 1214 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1215 rq->q_next = wrq->q_next = NULL; 1216 qdetach(rq, 0, 0, crp, B_FALSE); 1217 return (error); 1218 } 1219 1220 /* 1221 * Handle second open of stream. For modules, set the 1222 * last argument to MODOPEN and do not pass any open flags. 1223 * Ignore dummydev since this is not the first open. 1224 */ 1225 int 1226 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) 1227 { 1228 int error; 1229 dev_t dummydev; 1230 queue_t *wqp = _WR(qp); 1231 1232 ASSERT(qp->q_flag & QREADR); 1233 entersq(qp->q_syncq, SQ_OPENCLOSE); 1234 1235 dummydev = *devp; 1236 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, 1237 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { 1238 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1239 mutex_enter(&STREAM(qp)->sd_lock); 1240 qp->q_stream->sd_flag |= STREOPENFAIL; 1241 mutex_exit(&STREAM(qp)->sd_lock); 1242 return (error); 1243 } 1244 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1245 1246 /* 1247 * successful open should have done qprocson() 1248 */ 1249 ASSERT(qprocsareon(_RD(qp))); 1250 return (0); 1251 } 1252 1253 /* 1254 * Detach a stream module or device. 1255 * If clmode == 1 then the module or driver was opened and its 1256 * close routine must be called. If clmode == 0, the module 1257 * or driver was never opened or the open failed, and so its close 1258 * should not be called. 1259 */ 1260 void 1261 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) 1262 { 1263 queue_t *wqp = _WR(qp); 1264 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); 1265 1266 if (STREAM_NEEDSERVICE(STREAM(qp))) 1267 stream_runservice(STREAM(qp)); 1268 1269 if (clmode) { 1270 /* 1271 * Make sure that all the messages on the write side syncq are 1272 * processed and nothing is left. Since we are closing, no new 1273 * messages may appear there. 1274 */ 1275 wait_q_syncq(wqp); 1276 1277 entersq(qp->q_syncq, SQ_OPENCLOSE); 1278 if (is_remove) { 1279 mutex_enter(QLOCK(qp)); 1280 qp->q_flag |= _QREMOVING; 1281 mutex_exit(QLOCK(qp)); 1282 } 1283 (*qp->q_qinfo->qi_qclose)(qp, flag, crp); 1284 /* 1285 * Check that qprocsoff() was actually called. 1286 */ 1287 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); 1288 1289 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1290 } else { 1291 disable_svc(qp); 1292 } 1293 1294 /* 1295 * Allow any threads blocked in entersq to proceed and discover 1296 * the QWCLOSE is set. 1297 * Note: This assumes that all users of entersq check QWCLOSE. 1298 * Currently runservice is the only entersq that can happen 1299 * after removeq has finished. 1300 * Removeq will have discarded all messages destined to the closing 1301 * pair of queues from the syncq. 1302 * NOTE: Calling a function inside an assert is unconventional. 1303 * However, it does not cause any problem since flush_syncq() does 1304 * not change any state except when it returns non-zero i.e. 1305 * when the assert will trigger. 1306 */ 1307 ASSERT(flush_syncq(qp->q_syncq, qp) == 0); 1308 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); 1309 ASSERT((qp->q_flag & QPERMOD) || 1310 ((qp->q_syncq->sq_head == NULL) && 1311 (wqp->q_syncq->sq_head == NULL))); 1312 1313 /* release any fmodsw_impl_t structure held on behalf of the queue */ 1314 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); 1315 if (qp->q_fp != NULL) 1316 fmodsw_rele(qp->q_fp); 1317 1318 /* freeq removes us from the outer perimeter if any */ 1319 freeq(qp); 1320 } 1321 1322 /* Prevent service procedures from being called */ 1323 void 1324 disable_svc(queue_t *qp) 1325 { 1326 queue_t *wqp = _WR(qp); 1327 1328 ASSERT(qp->q_flag & QREADR); 1329 mutex_enter(QLOCK(qp)); 1330 qp->q_flag |= QWCLOSE; 1331 mutex_exit(QLOCK(qp)); 1332 mutex_enter(QLOCK(wqp)); 1333 wqp->q_flag |= QWCLOSE; 1334 mutex_exit(QLOCK(wqp)); 1335 } 1336 1337 /* allow service procedures to be called again */ 1338 void 1339 enable_svc(queue_t *qp) 1340 { 1341 queue_t *wqp = _WR(qp); 1342 1343 ASSERT(qp->q_flag & QREADR); 1344 mutex_enter(QLOCK(qp)); 1345 qp->q_flag &= ~QWCLOSE; 1346 mutex_exit(QLOCK(qp)); 1347 mutex_enter(QLOCK(wqp)); 1348 wqp->q_flag &= ~QWCLOSE; 1349 mutex_exit(QLOCK(wqp)); 1350 } 1351 1352 /* 1353 * Remove queue from qhead/qtail if it is enabled. 1354 * Only reset QENAB if the queue was removed from the runlist. 1355 * A queue goes through 3 stages: 1356 * It is on the service list and QENAB is set. 1357 * It is removed from the service list but QENAB is still set. 1358 * QENAB gets changed to QINSERVICE. 1359 * QINSERVICE is reset (when the service procedure is done) 1360 * Thus we can not reset QENAB unless we actually removed it from the service 1361 * queue. 1362 */ 1363 void 1364 remove_runlist(queue_t *qp) 1365 { 1366 if (qp->q_flag & QENAB && qhead != NULL) { 1367 queue_t *q_chase; 1368 queue_t *q_curr; 1369 int removed; 1370 1371 mutex_enter(&service_queue); 1372 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); 1373 mutex_exit(&service_queue); 1374 if (removed) { 1375 STRSTAT(qremoved); 1376 qp->q_flag &= ~QENAB; 1377 } 1378 } 1379 } 1380 1381 1382 /* 1383 * wait for any pending service processing to complete. 1384 * The removal of queues from the runlist is not atomic with the 1385 * clearing of the QENABLED flag and setting the INSERVICE flag. 1386 * consequently it is possible for remove_runlist in strclose 1387 * to not find the queue on the runlist but for it to be QENABLED 1388 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED 1389 * as well as INSERVICE. 1390 */ 1391 void 1392 wait_svc(queue_t *qp) 1393 { 1394 queue_t *wqp = _WR(qp); 1395 1396 ASSERT(qp->q_flag & QREADR); 1397 1398 /* 1399 * Try to remove queues from qhead/qtail list. 1400 */ 1401 if (qhead != NULL) { 1402 remove_runlist(qp); 1403 remove_runlist(wqp); 1404 } 1405 /* 1406 * Wait till the syncqs associated with the queue 1407 * will dissapear from background processing list. 1408 * This only needs to be done for non-PERMOD perimeters since 1409 * for PERMOD perimeters the syncq may be shared and will only be freed 1410 * when the last module/driver is unloaded. 1411 * If for PERMOD perimeters queue was on the syncq list, removeq() 1412 * should call propagate_syncq() or drain_syncq() for it. Both of these 1413 * function remove the queue from its syncq list, so sqthread will not 1414 * try to access the queue. 1415 */ 1416 if (!(qp->q_flag & QPERMOD)) { 1417 syncq_t *rsq = qp->q_syncq; 1418 syncq_t *wsq = wqp->q_syncq; 1419 1420 /* 1421 * Disable rsq and wsq and wait for any background processing of 1422 * syncq to complete. 1423 */ 1424 wait_sq_svc(rsq); 1425 if (wsq != rsq) 1426 wait_sq_svc(wsq); 1427 } 1428 1429 mutex_enter(QLOCK(qp)); 1430 while (qp->q_flag & (QINSERVICE|QENAB)) 1431 cv_wait(&qp->q_wait, QLOCK(qp)); 1432 mutex_exit(QLOCK(qp)); 1433 mutex_enter(QLOCK(wqp)); 1434 while (wqp->q_flag & (QINSERVICE|QENAB)) 1435 cv_wait(&wqp->q_wait, QLOCK(wqp)); 1436 mutex_exit(QLOCK(wqp)); 1437 } 1438 1439 /* 1440 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'. 1441 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may 1442 * also be set, and is passed through to allocb_cred_wait(). 1443 * 1444 * Returns errno on failure, zero on success. 1445 */ 1446 int 1447 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) 1448 { 1449 mblk_t *tmp; 1450 ssize_t count; 1451 size_t n; 1452 int error = 0; 1453 1454 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || 1455 (flag & (U_TO_K | K_TO_K)) == K_TO_K); 1456 1457 if (bp->b_datap->db_type == M_IOCTL) { 1458 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1459 } else { 1460 ASSERT(bp->b_datap->db_type == M_COPYIN); 1461 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1462 } 1463 /* 1464 * strdoioctl validates ioc_count, so if this assert fails it 1465 * cannot be due to user error. 1466 */ 1467 ASSERT(count >= 0); 1468 1469 while (count > 0) { 1470 n = MIN(MAXIOCBSZ, count); 1471 if ((tmp = allocb_cred_wait(n, (flag & STR_NOSIG), &error, 1472 cr)) == NULL) { 1473 return (error); 1474 } 1475 error = strcopyin(arg, tmp->b_wptr, n, flag & (U_TO_K|K_TO_K)); 1476 if (error != 0) { 1477 freeb(tmp); 1478 return (error); 1479 } 1480 arg += n; 1481 DB_CPID(tmp) = curproc->p_pid; 1482 tmp->b_wptr += n; 1483 count -= n; 1484 bp = (bp->b_cont = tmp); 1485 } 1486 1487 return (0); 1488 } 1489 1490 /* 1491 * Copy ioctl data to user-land. Return non-zero errno on failure, 1492 * 0 for success. 1493 */ 1494 int 1495 getiocd(mblk_t *bp, char *arg, int copymode) 1496 { 1497 ssize_t count; 1498 size_t n; 1499 int error; 1500 1501 if (bp->b_datap->db_type == M_IOCACK) 1502 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1503 else { 1504 ASSERT(bp->b_datap->db_type == M_COPYOUT); 1505 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1506 } 1507 ASSERT(count >= 0); 1508 1509 for (bp = bp->b_cont; bp && count; 1510 count -= n, bp = bp->b_cont, arg += n) { 1511 n = MIN(count, bp->b_wptr - bp->b_rptr); 1512 error = strcopyout(bp->b_rptr, arg, n, copymode); 1513 if (error) 1514 return (error); 1515 } 1516 ASSERT(count == 0); 1517 return (0); 1518 } 1519 1520 /* 1521 * Allocate a linkinfo entry given the write queue of the 1522 * bottom module of the top stream and the write queue of the 1523 * stream head of the bottom stream. 1524 */ 1525 linkinfo_t * 1526 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) 1527 { 1528 linkinfo_t *linkp; 1529 1530 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); 1531 1532 linkp->li_lblk.l_qtop = qup; 1533 linkp->li_lblk.l_qbot = qdown; 1534 linkp->li_fpdown = fpdown; 1535 1536 mutex_enter(&strresources); 1537 linkp->li_next = linkinfo_list; 1538 linkp->li_prev = NULL; 1539 if (linkp->li_next) 1540 linkp->li_next->li_prev = linkp; 1541 linkinfo_list = linkp; 1542 linkp->li_lblk.l_index = ++lnk_id; 1543 ASSERT(lnk_id != 0); /* this should never wrap in practice */ 1544 mutex_exit(&strresources); 1545 1546 return (linkp); 1547 } 1548 1549 /* 1550 * Free a linkinfo entry. 1551 */ 1552 void 1553 lbfree(linkinfo_t *linkp) 1554 { 1555 mutex_enter(&strresources); 1556 if (linkp->li_next) 1557 linkp->li_next->li_prev = linkp->li_prev; 1558 if (linkp->li_prev) 1559 linkp->li_prev->li_next = linkp->li_next; 1560 else 1561 linkinfo_list = linkp->li_next; 1562 mutex_exit(&strresources); 1563 1564 kmem_cache_free(linkinfo_cache, linkp); 1565 } 1566 1567 /* 1568 * Check for a potential linking cycle. 1569 * Return 1 if a link will result in a cycle, 1570 * and 0 otherwise. 1571 */ 1572 int 1573 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss) 1574 { 1575 struct mux_node *np; 1576 struct mux_edge *ep; 1577 int i; 1578 major_t lomaj; 1579 major_t upmaj; 1580 /* 1581 * if the lower stream is a pipe/FIFO, return, since link 1582 * cycles can not happen on pipes/FIFOs 1583 */ 1584 if (lostp->sd_vnode->v_type == VFIFO) 1585 return (0); 1586 1587 for (i = 0; i < ss->ss_devcnt; i++) { 1588 np = &ss->ss_mux_nodes[i]; 1589 MUX_CLEAR(np); 1590 } 1591 lomaj = getmajor(lostp->sd_vnode->v_rdev); 1592 upmaj = getmajor(upstp->sd_vnode->v_rdev); 1593 np = &ss->ss_mux_nodes[lomaj]; 1594 for (;;) { 1595 if (!MUX_DIDVISIT(np)) { 1596 if (np->mn_imaj == upmaj) 1597 return (1); 1598 if (np->mn_outp == NULL) { 1599 MUX_VISIT(np); 1600 if (np->mn_originp == NULL) 1601 return (0); 1602 np = np->mn_originp; 1603 continue; 1604 } 1605 MUX_VISIT(np); 1606 np->mn_startp = np->mn_outp; 1607 } else { 1608 if (np->mn_startp == NULL) { 1609 if (np->mn_originp == NULL) 1610 return (0); 1611 else { 1612 np = np->mn_originp; 1613 continue; 1614 } 1615 } 1616 /* 1617 * If ep->me_nodep is a FIFO (me_nodep == NULL), 1618 * ignore the edge and move on. ep->me_nodep gets 1619 * set to NULL in mux_addedge() if it is a FIFO. 1620 * 1621 */ 1622 ep = np->mn_startp; 1623 np->mn_startp = ep->me_nextp; 1624 if (ep->me_nodep == NULL) 1625 continue; 1626 ep->me_nodep->mn_originp = np; 1627 np = ep->me_nodep; 1628 } 1629 } 1630 } 1631 1632 /* 1633 * Find linkinfo entry corresponding to the parameters. 1634 */ 1635 linkinfo_t * 1636 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss) 1637 { 1638 linkinfo_t *linkp; 1639 struct mux_edge *mep; 1640 struct mux_node *mnp; 1641 queue_t *qup; 1642 1643 mutex_enter(&strresources); 1644 if ((type & LINKTYPEMASK) == LINKNORMAL) { 1645 qup = getendq(stp->sd_wrq); 1646 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1647 if ((qup == linkp->li_lblk.l_qtop) && 1648 (!index || (index == linkp->li_lblk.l_index))) { 1649 mutex_exit(&strresources); 1650 return (linkp); 1651 } 1652 } 1653 } else { 1654 ASSERT((type & LINKTYPEMASK) == LINKPERSIST); 1655 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; 1656 mep = mnp->mn_outp; 1657 while (mep) { 1658 if ((index == 0) || (index == mep->me_muxid)) 1659 break; 1660 mep = mep->me_nextp; 1661 } 1662 if (!mep) { 1663 mutex_exit(&strresources); 1664 return (NULL); 1665 } 1666 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1667 if ((!linkp->li_lblk.l_qtop) && 1668 (mep->me_muxid == linkp->li_lblk.l_index)) { 1669 mutex_exit(&strresources); 1670 return (linkp); 1671 } 1672 } 1673 } 1674 mutex_exit(&strresources); 1675 return (NULL); 1676 } 1677 1678 /* 1679 * Given a queue ptr, follow the chain of q_next pointers until you reach the 1680 * last queue on the chain and return it. 1681 */ 1682 queue_t * 1683 getendq(queue_t *q) 1684 { 1685 ASSERT(q != NULL); 1686 while (_SAMESTR(q)) 1687 q = q->q_next; 1688 return (q); 1689 } 1690 1691 /* 1692 * wait for the syncq count to drop to zero. 1693 * sq could be either outer or inner. 1694 */ 1695 1696 static void 1697 wait_syncq(syncq_t *sq) 1698 { 1699 uint16_t count; 1700 1701 mutex_enter(SQLOCK(sq)); 1702 count = sq->sq_count; 1703 SQ_PUTLOCKS_ENTER(sq); 1704 SUM_SQ_PUTCOUNTS(sq, count); 1705 while (count != 0) { 1706 sq->sq_flags |= SQ_WANTWAKEUP; 1707 SQ_PUTLOCKS_EXIT(sq); 1708 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1709 count = sq->sq_count; 1710 SQ_PUTLOCKS_ENTER(sq); 1711 SUM_SQ_PUTCOUNTS(sq, count); 1712 } 1713 SQ_PUTLOCKS_EXIT(sq); 1714 mutex_exit(SQLOCK(sq)); 1715 } 1716 1717 /* 1718 * Wait while there are any messages for the queue in its syncq. 1719 */ 1720 static void 1721 wait_q_syncq(queue_t *q) 1722 { 1723 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1724 syncq_t *sq = q->q_syncq; 1725 1726 mutex_enter(SQLOCK(sq)); 1727 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1728 sq->sq_flags |= SQ_WANTWAKEUP; 1729 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1730 } 1731 mutex_exit(SQLOCK(sq)); 1732 } 1733 } 1734 1735 1736 int 1737 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, 1738 int lhlink) 1739 { 1740 struct stdata *stp; 1741 struct strioctl strioc; 1742 struct linkinfo *linkp; 1743 struct stdata *stpdown; 1744 struct streamtab *str; 1745 queue_t *passq; 1746 syncq_t *passyncq; 1747 queue_t *rq; 1748 cdevsw_impl_t *dp; 1749 uint32_t qflag; 1750 uint32_t sqtype; 1751 perdm_t *dmp; 1752 int error = 0; 1753 netstack_t *ns; 1754 str_stack_t *ss; 1755 1756 stp = vp->v_stream; 1757 TRACE_1(TR_FAC_STREAMS_FR, 1758 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); 1759 /* 1760 * Test for invalid upper stream 1761 */ 1762 if (stp->sd_flag & STRHUP) { 1763 return (ENXIO); 1764 } 1765 if (vp->v_type == VFIFO) { 1766 return (EINVAL); 1767 } 1768 if (stp->sd_strtab == NULL) { 1769 return (EINVAL); 1770 } 1771 if (!stp->sd_strtab->st_muxwinit) { 1772 return (EINVAL); 1773 } 1774 if (fpdown == NULL) { 1775 return (EBADF); 1776 } 1777 ns = netstack_find_by_cred(crp); 1778 ASSERT(ns != NULL); 1779 ss = ns->netstack_str; 1780 ASSERT(ss != NULL); 1781 1782 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) { 1783 netstack_rele(ss->ss_netstack); 1784 return (EINVAL); 1785 } 1786 mutex_enter(&muxifier); 1787 if (stp->sd_flag & STPLEX) { 1788 mutex_exit(&muxifier); 1789 netstack_rele(ss->ss_netstack); 1790 return (ENXIO); 1791 } 1792 1793 /* 1794 * Test for invalid lower stream. 1795 * The check for the v_type != VFIFO and having a major 1796 * number not >= devcnt is done to avoid problems with 1797 * adding mux_node entry past the end of mux_nodes[]. 1798 * For FIFO's we don't add an entry so this isn't a 1799 * problem. 1800 */ 1801 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || 1802 (stpdown == stp) || (stpdown->sd_flag & 1803 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || 1804 ((stpdown->sd_vnode->v_type != VFIFO) && 1805 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) || 1806 linkcycle(stp, stpdown, ss)) { 1807 mutex_exit(&muxifier); 1808 netstack_rele(ss->ss_netstack); 1809 return (EINVAL); 1810 } 1811 TRACE_1(TR_FAC_STREAMS_FR, 1812 TR_STPDOWN, "stpdown:%p", stpdown); 1813 rq = getendq(stp->sd_wrq); 1814 if (cmd == I_PLINK) 1815 rq = NULL; 1816 1817 linkp = alloclink(rq, stpdown->sd_wrq, fpdown); 1818 1819 strioc.ic_cmd = cmd; 1820 strioc.ic_timout = INFTIM; 1821 strioc.ic_len = sizeof (struct linkblk); 1822 strioc.ic_dp = (char *)&linkp->li_lblk; 1823 1824 /* 1825 * STRPLUMB protects plumbing changes and should be set before 1826 * link_addpassthru()/link_rempassthru() are called, so it is set here 1827 * and cleared in the end of mlink when passthru queue is removed. 1828 * Setting of STRPLUMB prevents reopens of the stream while passthru 1829 * queue is in-place (it is not a proper module and doesn't have open 1830 * entry point). 1831 * 1832 * STPLEX prevents any threads from entering the stream from above. It 1833 * can't be set before the call to link_addpassthru() because putnext 1834 * from below may cause stream head I/O routines to be called and these 1835 * routines assert that STPLEX is not set. After link_addpassthru() 1836 * nothing may come from below since the pass queue syncq is blocked. 1837 * Note also that STPLEX should be cleared before the call to 1838 * link_remmpassthru() since when messages start flowing to the stream 1839 * head (e.g. because of message propagation from the pass queue) stream 1840 * head I/O routines may be called with STPLEX flag set. 1841 * 1842 * When STPLEX is set, nothing may come into the stream from above and 1843 * it is safe to do a setq which will change stream head. So, the 1844 * correct sequence of actions is: 1845 * 1846 * 1) Set STRPLUMB 1847 * 2) Call link_addpassthru() 1848 * 3) Set STPLEX 1849 * 4) Call setq and update the stream state 1850 * 5) Clear STPLEX 1851 * 6) Call link_rempassthru() 1852 * 7) Clear STRPLUMB 1853 * 1854 * The same sequence applies to munlink() code. 1855 */ 1856 mutex_enter(&stpdown->sd_lock); 1857 stpdown->sd_flag |= STRPLUMB; 1858 mutex_exit(&stpdown->sd_lock); 1859 /* 1860 * Add passthru queue below lower mux. This will block 1861 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 1862 */ 1863 passq = link_addpassthru(stpdown); 1864 1865 mutex_enter(&stpdown->sd_lock); 1866 stpdown->sd_flag |= STPLEX; 1867 mutex_exit(&stpdown->sd_lock); 1868 1869 rq = _RD(stpdown->sd_wrq); 1870 /* 1871 * There may be messages in the streamhead's syncq due to messages 1872 * that arrived before link_addpassthru() was done. To avoid 1873 * background processing of the syncq happening simultaneous with 1874 * setq processing, we disable the streamhead syncq and wait until 1875 * existing background thread finishes working on it. 1876 */ 1877 wait_sq_svc(rq->q_syncq); 1878 passyncq = passq->q_syncq; 1879 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1880 blocksq(passyncq, SQ_BLOCKED, 0); 1881 1882 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 1883 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 1884 rq->q_ptr = _WR(rq)->q_ptr = NULL; 1885 1886 /* setq might sleep in allocator - avoid holding locks. */ 1887 /* Note: we are holding muxifier here. */ 1888 1889 str = stp->sd_strtab; 1890 dp = &devimpl[getmajor(vp->v_rdev)]; 1891 ASSERT(dp->d_str == str); 1892 1893 qflag = dp->d_qflag; 1894 sqtype = dp->d_sqtype; 1895 1896 /* create perdm_t if needed */ 1897 if (NEED_DM(dp->d_dmp, qflag)) 1898 dp->d_dmp = hold_dm(str, qflag, sqtype); 1899 1900 dmp = dp->d_dmp; 1901 1902 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, 1903 B_TRUE); 1904 1905 /* 1906 * XXX Remove any "odd" messages from the queue. 1907 * Keep only M_DATA, M_PROTO, M_PCPROTO. 1908 */ 1909 error = strdoioctl(stp, &strioc, FNATIVE, 1910 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 1911 if (error != 0) { 1912 lbfree(linkp); 1913 1914 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1915 blocksq(passyncq, SQ_BLOCKED, 0); 1916 /* 1917 * Restore the stream head queue and then remove 1918 * the passq. Turn off STPLEX before we turn on 1919 * the stream by removing the passq. 1920 */ 1921 rq->q_ptr = _WR(rq)->q_ptr = stpdown; 1922 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, 1923 B_TRUE); 1924 1925 mutex_enter(&stpdown->sd_lock); 1926 stpdown->sd_flag &= ~STPLEX; 1927 mutex_exit(&stpdown->sd_lock); 1928 1929 link_rempassthru(passq); 1930 1931 mutex_enter(&stpdown->sd_lock); 1932 stpdown->sd_flag &= ~STRPLUMB; 1933 /* Wakeup anyone waiting for STRPLUMB to clear. */ 1934 cv_broadcast(&stpdown->sd_monitor); 1935 mutex_exit(&stpdown->sd_lock); 1936 1937 mutex_exit(&muxifier); 1938 netstack_rele(ss->ss_netstack); 1939 return (error); 1940 } 1941 mutex_enter(&fpdown->f_tlock); 1942 fpdown->f_count++; 1943 mutex_exit(&fpdown->f_tlock); 1944 1945 /* 1946 * if we've made it here the linkage is all set up so we should also 1947 * set up the layered driver linkages 1948 */ 1949 1950 ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); 1951 if (cmd == I_LINK) { 1952 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); 1953 } else { 1954 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); 1955 } 1956 1957 link_rempassthru(passq); 1958 1959 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss); 1960 1961 /* 1962 * Mark the upper stream as having dependent links 1963 * so that strclose can clean it up. 1964 */ 1965 if (cmd == I_LINK) { 1966 mutex_enter(&stp->sd_lock); 1967 stp->sd_flag |= STRHASLINKS; 1968 mutex_exit(&stp->sd_lock); 1969 } 1970 /* 1971 * Wake up any other processes that may have been 1972 * waiting on the lower stream. These will all 1973 * error out. 1974 */ 1975 mutex_enter(&stpdown->sd_lock); 1976 /* The passthru module is removed so we may release STRPLUMB */ 1977 stpdown->sd_flag &= ~STRPLUMB; 1978 cv_broadcast(&rq->q_wait); 1979 cv_broadcast(&_WR(rq)->q_wait); 1980 cv_broadcast(&stpdown->sd_monitor); 1981 mutex_exit(&stpdown->sd_lock); 1982 mutex_exit(&muxifier); 1983 *rvalp = linkp->li_lblk.l_index; 1984 netstack_rele(ss->ss_netstack); 1985 return (0); 1986 } 1987 1988 int 1989 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) 1990 { 1991 int ret; 1992 struct file *fpdown; 1993 1994 fpdown = getf(arg); 1995 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); 1996 if (fpdown != NULL) 1997 releasef(arg); 1998 return (ret); 1999 } 2000 2001 /* 2002 * Unlink a multiplexor link. Stp is the controlling stream for the 2003 * link, and linkp points to the link's entry in the linkinfo list. 2004 * The muxifier lock must be held on entry and is dropped on exit. 2005 * 2006 * NOTE : Currently it is assumed that mux would process all the messages 2007 * sitting on it's queue before ACKing the UNLINK. It is the responsibility 2008 * of the mux to handle all the messages that arrive before UNLINK. 2009 * If the mux has to send down messages on its lower stream before 2010 * ACKing I_UNLINK, then it *should* know to handle messages even 2011 * after the UNLINK is acked (actually it should be able to handle till we 2012 * re-block the read side of the pass queue here). If the mux does not 2013 * open up the lower stream, any messages that arrive during UNLINK 2014 * will be put in the stream head. In the case of lower stream opening 2015 * up, some messages might land in the stream head depending on when 2016 * the message arrived and when the read side of the pass queue was 2017 * re-blocked. 2018 */ 2019 int 2020 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp, 2021 str_stack_t *ss) 2022 { 2023 struct strioctl strioc; 2024 struct stdata *stpdown; 2025 queue_t *rq, *wrq; 2026 queue_t *passq; 2027 syncq_t *passyncq; 2028 int error = 0; 2029 file_t *fpdown; 2030 2031 ASSERT(MUTEX_HELD(&muxifier)); 2032 2033 stpdown = linkp->li_fpdown->f_vnode->v_stream; 2034 2035 /* 2036 * See the comment in mlink() concerning STRPLUMB/STPLEX flags. 2037 */ 2038 mutex_enter(&stpdown->sd_lock); 2039 stpdown->sd_flag |= STRPLUMB; 2040 mutex_exit(&stpdown->sd_lock); 2041 2042 /* 2043 * Add passthru queue below lower mux. This will block 2044 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 2045 */ 2046 passq = link_addpassthru(stpdown); 2047 2048 if ((flag & LINKTYPEMASK) == LINKNORMAL) 2049 strioc.ic_cmd = I_UNLINK; 2050 else 2051 strioc.ic_cmd = I_PUNLINK; 2052 strioc.ic_timout = INFTIM; 2053 strioc.ic_len = sizeof (struct linkblk); 2054 strioc.ic_dp = (char *)&linkp->li_lblk; 2055 2056 error = strdoioctl(stp, &strioc, FNATIVE, 2057 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 2058 2059 /* 2060 * If there was an error and this is not called via strclose, 2061 * return to the user. Otherwise, pretend there was no error 2062 * and close the link. 2063 */ 2064 if (error) { 2065 if (flag & LINKCLOSE) { 2066 cmn_err(CE_WARN, "KERNEL: munlink: could not perform " 2067 "unlink ioctl, closing anyway (%d)\n", error); 2068 } else { 2069 link_rempassthru(passq); 2070 mutex_enter(&stpdown->sd_lock); 2071 stpdown->sd_flag &= ~STRPLUMB; 2072 cv_broadcast(&stpdown->sd_monitor); 2073 mutex_exit(&stpdown->sd_lock); 2074 mutex_exit(&muxifier); 2075 return (error); 2076 } 2077 } 2078 2079 mux_rmvedge(stp, linkp->li_lblk.l_index, ss); 2080 fpdown = linkp->li_fpdown; 2081 lbfree(linkp); 2082 2083 /* 2084 * We go ahead and drop muxifier here--it's a nasty global lock that 2085 * can slow others down. It's okay to since attempts to mlink() this 2086 * stream will be stopped because STPLEX is still set in the stdata 2087 * structure, and munlink() is stopped because mux_rmvedge() and 2088 * lbfree() have removed it from mux_nodes[] and linkinfo_list, 2089 * respectively. Note that we defer the closef() of fpdown until 2090 * after we drop muxifier since strclose() can call munlinkall(). 2091 */ 2092 mutex_exit(&muxifier); 2093 2094 wrq = stpdown->sd_wrq; 2095 rq = _RD(wrq); 2096 2097 /* 2098 * Get rid of outstanding service procedure runs, before we make 2099 * it a stream head, since a stream head doesn't have any service 2100 * procedure. 2101 */ 2102 disable_svc(rq); 2103 wait_svc(rq); 2104 2105 /* 2106 * Since we don't disable the syncq for QPERMOD, we wait for whatever 2107 * is queued up to be finished. mux should take care that nothing is 2108 * send down to this queue. We should do it now as we're going to block 2109 * passyncq if it was unblocked. 2110 */ 2111 if (wrq->q_flag & QPERMOD) { 2112 syncq_t *sq = wrq->q_syncq; 2113 2114 mutex_enter(SQLOCK(sq)); 2115 while (wrq->q_sqflags & Q_SQQUEUED) { 2116 sq->sq_flags |= SQ_WANTWAKEUP; 2117 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2118 } 2119 mutex_exit(SQLOCK(sq)); 2120 } 2121 passyncq = passq->q_syncq; 2122 if (!(passyncq->sq_flags & SQ_BLOCKED)) { 2123 2124 syncq_t *sq, *outer; 2125 2126 /* 2127 * Messages could be flowing from underneath. We will 2128 * block the read side of the passq. This would be 2129 * sufficient for QPAIR and QPERQ muxes to ensure 2130 * that no data is flowing up into this queue 2131 * and hence no thread active in this instance of 2132 * lower mux. But for QPERMOD and QMTOUTPERIM there 2133 * could be messages on the inner and outer/inner 2134 * syncqs respectively. We will wait for them to drain. 2135 * Because passq is blocked messages end up in the syncq 2136 * And qfill_syncq could possibly end up setting QFULL 2137 * which will access the rq->q_flag. Hence, we have to 2138 * acquire the QLOCK in setq. 2139 * 2140 * XXX Messages can also flow from top into this 2141 * queue though the unlink is over (Ex. some instance 2142 * in putnext() called from top that has still not 2143 * accessed this queue. And also putq(lowerq) ?). 2144 * Solution : How about blocking the l_qtop queue ? 2145 * Do we really care about such pure D_MP muxes ? 2146 */ 2147 2148 blocksq(passyncq, SQ_BLOCKED, 0); 2149 2150 sq = rq->q_syncq; 2151 if ((outer = sq->sq_outer) != NULL) { 2152 2153 /* 2154 * We have to just wait for the outer sq_count 2155 * drop to zero. As this does not prevent new 2156 * messages to enter the outer perimeter, this 2157 * is subject to starvation. 2158 * 2159 * NOTE :Because of blocksq above, messages could 2160 * be in the inner syncq only because of some 2161 * thread holding the outer perimeter exclusively. 2162 * Hence it would be sufficient to wait for the 2163 * exclusive holder of the outer perimeter to drain 2164 * the inner and outer syncqs. But we will not depend 2165 * on this feature and hence check the inner syncqs 2166 * separately. 2167 */ 2168 wait_syncq(outer); 2169 } 2170 2171 2172 /* 2173 * There could be messages destined for 2174 * this queue. Let the exclusive holder 2175 * drain it. 2176 */ 2177 2178 wait_syncq(sq); 2179 ASSERT((rq->q_flag & QPERMOD) || 2180 ((rq->q_syncq->sq_head == NULL) && 2181 (_WR(rq)->q_syncq->sq_head == NULL))); 2182 } 2183 2184 /* 2185 * We haven't taken care of QPERMOD case yet. QPERMOD is a special 2186 * case as we don't disable its syncq or remove it off the syncq 2187 * service list. 2188 */ 2189 if (rq->q_flag & QPERMOD) { 2190 syncq_t *sq = rq->q_syncq; 2191 2192 mutex_enter(SQLOCK(sq)); 2193 while (rq->q_sqflags & Q_SQQUEUED) { 2194 sq->sq_flags |= SQ_WANTWAKEUP; 2195 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2196 } 2197 mutex_exit(SQLOCK(sq)); 2198 } 2199 2200 /* 2201 * flush_syncq changes states only when there is some messages to 2202 * free. ie when it returns non-zero value to return. 2203 */ 2204 ASSERT(flush_syncq(rq->q_syncq, rq) == 0); 2205 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); 2206 2207 /* 2208 * No body else should know about this queue now. 2209 * If the mux did not process the messages before 2210 * acking the I_UNLINK, free them now. 2211 */ 2212 2213 flushq(rq, FLUSHALL); 2214 flushq(_WR(rq), FLUSHALL); 2215 2216 /* 2217 * Convert the mux lower queue into a stream head queue. 2218 * Turn off STPLEX before we turn on the stream by removing the passq. 2219 */ 2220 rq->q_ptr = wrq->q_ptr = stpdown; 2221 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); 2222 2223 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 2224 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 2225 2226 enable_svc(rq); 2227 2228 /* 2229 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still 2230 * needs to be set to prevent reopen() of the stream - such reopen may 2231 * try to call non-existent pass queue open routine and panic. 2232 */ 2233 mutex_enter(&stpdown->sd_lock); 2234 stpdown->sd_flag &= ~STPLEX; 2235 mutex_exit(&stpdown->sd_lock); 2236 2237 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || 2238 ((flag & LINKTYPEMASK) == LINKPERSIST)); 2239 2240 /* clean up the layered driver linkages */ 2241 if ((flag & LINKTYPEMASK) == LINKNORMAL) { 2242 ldi_munlink_fp(stp, fpdown, LINKNORMAL); 2243 } else { 2244 ldi_munlink_fp(stp, fpdown, LINKPERSIST); 2245 } 2246 2247 link_rempassthru(passq); 2248 2249 /* 2250 * Now all plumbing changes are finished and STRPLUMB is no 2251 * longer needed. 2252 */ 2253 mutex_enter(&stpdown->sd_lock); 2254 stpdown->sd_flag &= ~STRPLUMB; 2255 cv_broadcast(&stpdown->sd_monitor); 2256 mutex_exit(&stpdown->sd_lock); 2257 2258 (void) closef(fpdown); 2259 return (0); 2260 } 2261 2262 /* 2263 * Unlink all multiplexor links for which stp is the controlling stream. 2264 * Return 0, or a non-zero errno on failure. 2265 */ 2266 int 2267 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss) 2268 { 2269 linkinfo_t *linkp; 2270 int error = 0; 2271 2272 mutex_enter(&muxifier); 2273 while (linkp = findlinks(stp, 0, flag, ss)) { 2274 /* 2275 * munlink() releases the muxifier lock. 2276 */ 2277 if (error = munlink(stp, linkp, flag, crp, rvalp, ss)) 2278 return (error); 2279 mutex_enter(&muxifier); 2280 } 2281 mutex_exit(&muxifier); 2282 return (0); 2283 } 2284 2285 /* 2286 * A multiplexor link has been made. Add an 2287 * edge to the directed graph. 2288 */ 2289 void 2290 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss) 2291 { 2292 struct mux_node *np; 2293 struct mux_edge *ep; 2294 major_t upmaj; 2295 major_t lomaj; 2296 2297 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2298 lomaj = getmajor(lostp->sd_vnode->v_rdev); 2299 np = &ss->ss_mux_nodes[upmaj]; 2300 if (np->mn_outp) { 2301 ep = np->mn_outp; 2302 while (ep->me_nextp) 2303 ep = ep->me_nextp; 2304 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2305 ep = ep->me_nextp; 2306 } else { 2307 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2308 ep = np->mn_outp; 2309 } 2310 ep->me_nextp = NULL; 2311 ep->me_muxid = muxid; 2312 /* 2313 * Save the dev_t for the purposes of str_stack_shutdown. 2314 * str_stack_shutdown assumes that the device allows reopen, since 2315 * this dev_t is the one after any cloning by xx_open(). 2316 * Would prefer finding the dev_t from before any cloning, 2317 * but specfs doesn't retain that. 2318 */ 2319 ep->me_dev = upstp->sd_vnode->v_rdev; 2320 if (lostp->sd_vnode->v_type == VFIFO) 2321 ep->me_nodep = NULL; 2322 else 2323 ep->me_nodep = &ss->ss_mux_nodes[lomaj]; 2324 } 2325 2326 /* 2327 * A multiplexor link has been removed. Remove the 2328 * edge in the directed graph. 2329 */ 2330 void 2331 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss) 2332 { 2333 struct mux_node *np; 2334 struct mux_edge *ep; 2335 struct mux_edge *pep = NULL; 2336 major_t upmaj; 2337 2338 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2339 np = &ss->ss_mux_nodes[upmaj]; 2340 ASSERT(np->mn_outp != NULL); 2341 ep = np->mn_outp; 2342 while (ep) { 2343 if (ep->me_muxid == muxid) { 2344 if (pep) 2345 pep->me_nextp = ep->me_nextp; 2346 else 2347 np->mn_outp = ep->me_nextp; 2348 kmem_free(ep, sizeof (struct mux_edge)); 2349 return; 2350 } 2351 pep = ep; 2352 ep = ep->me_nextp; 2353 } 2354 ASSERT(0); /* should not reach here */ 2355 } 2356 2357 /* 2358 * Translate the device flags (from conf.h) to the corresponding 2359 * qflag and sq_flag (type) values. 2360 */ 2361 int 2362 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, 2363 uint32_t *sqtypep) 2364 { 2365 uint32_t qflag = 0; 2366 uint32_t sqtype = 0; 2367 2368 if (devflag & _D_OLD) 2369 goto bad; 2370 2371 /* Inner perimeter presence and scope */ 2372 switch (devflag & D_MTINNER_MASK) { 2373 case D_MP: 2374 qflag |= QMTSAFE; 2375 sqtype |= SQ_CI; 2376 break; 2377 case D_MTPERQ|D_MP: 2378 qflag |= QPERQ; 2379 break; 2380 case D_MTQPAIR|D_MP: 2381 qflag |= QPAIR; 2382 break; 2383 case D_MTPERMOD|D_MP: 2384 qflag |= QPERMOD; 2385 break; 2386 default: 2387 goto bad; 2388 } 2389 2390 /* Outer perimeter */ 2391 if (devflag & D_MTOUTPERIM) { 2392 switch (devflag & D_MTINNER_MASK) { 2393 case D_MP: 2394 case D_MTPERQ|D_MP: 2395 case D_MTQPAIR|D_MP: 2396 break; 2397 default: 2398 goto bad; 2399 } 2400 qflag |= QMTOUTPERIM; 2401 } 2402 2403 /* Inner perimeter modifiers */ 2404 if (devflag & D_MTINNER_MOD) { 2405 switch (devflag & D_MTINNER_MASK) { 2406 case D_MP: 2407 goto bad; 2408 default: 2409 break; 2410 } 2411 if (devflag & D_MTPUTSHARED) 2412 sqtype |= SQ_CIPUT; 2413 if (devflag & _D_MTOCSHARED) { 2414 /* 2415 * The code in putnext assumes that it has the 2416 * highest concurrency by not checking sq_count. 2417 * Thus _D_MTOCSHARED can only be supported when 2418 * D_MTPUTSHARED is set. 2419 */ 2420 if (!(devflag & D_MTPUTSHARED)) 2421 goto bad; 2422 sqtype |= SQ_CIOC; 2423 } 2424 if (devflag & _D_MTCBSHARED) { 2425 /* 2426 * The code in putnext assumes that it has the 2427 * highest concurrency by not checking sq_count. 2428 * Thus _D_MTCBSHARED can only be supported when 2429 * D_MTPUTSHARED is set. 2430 */ 2431 if (!(devflag & D_MTPUTSHARED)) 2432 goto bad; 2433 sqtype |= SQ_CICB; 2434 } 2435 if (devflag & _D_MTSVCSHARED) { 2436 /* 2437 * The code in putnext assumes that it has the 2438 * highest concurrency by not checking sq_count. 2439 * Thus _D_MTSVCSHARED can only be supported when 2440 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is 2441 * supported only for QPERMOD. 2442 */ 2443 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) 2444 goto bad; 2445 sqtype |= SQ_CISVC; 2446 } 2447 } 2448 2449 /* Default outer perimeter concurrency */ 2450 sqtype |= SQ_CO; 2451 2452 /* Outer perimeter modifiers */ 2453 if (devflag & D_MTOCEXCL) { 2454 if (!(devflag & D_MTOUTPERIM)) { 2455 /* No outer perimeter */ 2456 goto bad; 2457 } 2458 sqtype &= ~SQ_COOC; 2459 } 2460 2461 /* Synchronous Streams extended qinit structure */ 2462 if (devflag & D_SYNCSTR) 2463 qflag |= QSYNCSTR; 2464 2465 /* 2466 * Private flag used by a transport module to indicate 2467 * to sockfs that it supports direct-access mode without 2468 * having to go through STREAMS. 2469 */ 2470 if (devflag & _D_DIRECT) { 2471 /* Reject unless the module is fully-MT (no perimeter) */ 2472 if ((qflag & QMT_TYPEMASK) != QMTSAFE) 2473 goto bad; 2474 qflag |= _QDIRECT; 2475 } 2476 2477 *qflagp = qflag; 2478 *sqtypep = sqtype; 2479 return (0); 2480 2481 bad: 2482 cmn_err(CE_WARN, 2483 "stropen: bad MT flags (0x%x) in driver '%s'", 2484 (int)(qflag & D_MTSAFETY_MASK), 2485 stp->st_rdinit->qi_minfo->mi_idname); 2486 2487 return (EINVAL); 2488 } 2489 2490 /* 2491 * Set the interface values for a pair of queues (qinit structure, 2492 * packet sizes, water marks). 2493 * setq assumes that the caller does not have a claim (entersq or claimq) 2494 * on the queue. 2495 */ 2496 void 2497 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, 2498 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) 2499 { 2500 queue_t *wq; 2501 syncq_t *sq, *outer; 2502 2503 ASSERT(rq->q_flag & QREADR); 2504 ASSERT((qflag & QMT_TYPEMASK) != 0); 2505 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 2506 2507 wq = _WR(rq); 2508 rq->q_qinfo = rinit; 2509 rq->q_hiwat = rinit->qi_minfo->mi_hiwat; 2510 rq->q_lowat = rinit->qi_minfo->mi_lowat; 2511 rq->q_minpsz = rinit->qi_minfo->mi_minpsz; 2512 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; 2513 wq->q_qinfo = winit; 2514 wq->q_hiwat = winit->qi_minfo->mi_hiwat; 2515 wq->q_lowat = winit->qi_minfo->mi_lowat; 2516 wq->q_minpsz = winit->qi_minfo->mi_minpsz; 2517 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; 2518 2519 /* Remove old syncqs */ 2520 sq = rq->q_syncq; 2521 outer = sq->sq_outer; 2522 if (outer != NULL) { 2523 ASSERT(wq->q_syncq->sq_outer == outer); 2524 outer_remove(outer, rq->q_syncq); 2525 if (wq->q_syncq != rq->q_syncq) 2526 outer_remove(outer, wq->q_syncq); 2527 } 2528 ASSERT(sq->sq_outer == NULL); 2529 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2530 2531 if (sq != SQ(rq)) { 2532 if (!(rq->q_flag & QPERMOD)) 2533 free_syncq(sq); 2534 if (wq->q_syncq == rq->q_syncq) 2535 wq->q_syncq = NULL; 2536 rq->q_syncq = NULL; 2537 } 2538 if (wq->q_syncq != NULL && wq->q_syncq != sq && 2539 wq->q_syncq != SQ(rq)) { 2540 free_syncq(wq->q_syncq); 2541 wq->q_syncq = NULL; 2542 } 2543 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && 2544 rq->q_syncq->sq_tail == NULL)); 2545 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && 2546 wq->q_syncq->sq_tail == NULL)); 2547 2548 if (!(rq->q_flag & QPERMOD) && 2549 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { 2550 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2551 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, 2552 rq->q_syncq->sq_nciputctrl, 0); 2553 ASSERT(ciputctrl_cache != NULL); 2554 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); 2555 rq->q_syncq->sq_ciputctrl = NULL; 2556 rq->q_syncq->sq_nciputctrl = 0; 2557 } 2558 2559 if (!(wq->q_flag & QPERMOD) && 2560 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { 2561 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2562 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, 2563 wq->q_syncq->sq_nciputctrl, 0); 2564 ASSERT(ciputctrl_cache != NULL); 2565 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); 2566 wq->q_syncq->sq_ciputctrl = NULL; 2567 wq->q_syncq->sq_nciputctrl = 0; 2568 } 2569 2570 sq = SQ(rq); 2571 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 2572 ASSERT(sq->sq_outer == NULL); 2573 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2574 2575 /* 2576 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS 2577 * bits in sq_flag based on the sqtype. 2578 */ 2579 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); 2580 2581 rq->q_syncq = wq->q_syncq = sq; 2582 sq->sq_type = sqtype; 2583 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); 2584 2585 /* 2586 * We are making sq_svcflags zero, 2587 * resetting SQ_DISABLED in case it was set by 2588 * wait_svc() in the munlink path. 2589 * 2590 */ 2591 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); 2592 sq->sq_svcflags = 0; 2593 2594 /* 2595 * We need to acquire the lock here for the mlink and munlink case, 2596 * where canputnext, backenable, etc can access the q_flag. 2597 */ 2598 if (lock_needed) { 2599 mutex_enter(QLOCK(rq)); 2600 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2601 mutex_exit(QLOCK(rq)); 2602 mutex_enter(QLOCK(wq)); 2603 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2604 mutex_exit(QLOCK(wq)); 2605 } else { 2606 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2607 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2608 } 2609 2610 if (qflag & QPERQ) { 2611 /* Allocate a separate syncq for the write side */ 2612 sq = new_syncq(); 2613 sq->sq_type = rq->q_syncq->sq_type; 2614 sq->sq_flags = rq->q_syncq->sq_flags; 2615 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2616 sq->sq_oprev == NULL); 2617 wq->q_syncq = sq; 2618 } 2619 if (qflag & QPERMOD) { 2620 sq = dmp->dm_sq; 2621 2622 /* 2623 * Assert that we do have an inner perimeter syncq and that it 2624 * does not have an outer perimeter associated with it. 2625 */ 2626 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2627 sq->sq_oprev == NULL); 2628 rq->q_syncq = wq->q_syncq = sq; 2629 } 2630 if (qflag & QMTOUTPERIM) { 2631 outer = dmp->dm_sq; 2632 2633 ASSERT(outer->sq_outer == NULL); 2634 outer_insert(outer, rq->q_syncq); 2635 if (wq->q_syncq != rq->q_syncq) 2636 outer_insert(outer, wq->q_syncq); 2637 } 2638 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2639 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2640 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2641 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2642 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); 2643 2644 /* 2645 * Initialize struio() types. 2646 */ 2647 rq->q_struiot = 2648 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; 2649 wq->q_struiot = 2650 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; 2651 } 2652 2653 perdm_t * 2654 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) 2655 { 2656 syncq_t *sq; 2657 perdm_t **pp; 2658 perdm_t *p; 2659 perdm_t *dmp; 2660 2661 ASSERT(str != NULL); 2662 ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); 2663 2664 rw_enter(&perdm_rwlock, RW_READER); 2665 for (p = perdm_list; p != NULL; p = p->dm_next) { 2666 if (p->dm_str == str) { /* found one */ 2667 atomic_add_32(&(p->dm_ref), 1); 2668 rw_exit(&perdm_rwlock); 2669 return (p); 2670 } 2671 } 2672 rw_exit(&perdm_rwlock); 2673 2674 sq = new_syncq(); 2675 if (qflag & QPERMOD) { 2676 sq->sq_type = sqtype | SQ_PERMOD; 2677 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; 2678 } else { 2679 ASSERT(qflag & QMTOUTPERIM); 2680 sq->sq_onext = sq->sq_oprev = sq; 2681 } 2682 2683 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); 2684 dmp->dm_sq = sq; 2685 dmp->dm_str = str; 2686 dmp->dm_ref = 1; 2687 dmp->dm_next = NULL; 2688 2689 rw_enter(&perdm_rwlock, RW_WRITER); 2690 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { 2691 if (p->dm_str == str) { /* already present */ 2692 p->dm_ref++; 2693 rw_exit(&perdm_rwlock); 2694 free_syncq(sq); 2695 kmem_free(dmp, sizeof (perdm_t)); 2696 return (p); 2697 } 2698 } 2699 2700 *pp = dmp; 2701 rw_exit(&perdm_rwlock); 2702 return (dmp); 2703 } 2704 2705 void 2706 rele_dm(perdm_t *dmp) 2707 { 2708 perdm_t **pp; 2709 perdm_t *p; 2710 2711 rw_enter(&perdm_rwlock, RW_WRITER); 2712 ASSERT(dmp->dm_ref > 0); 2713 2714 if (--dmp->dm_ref > 0) { 2715 rw_exit(&perdm_rwlock); 2716 return; 2717 } 2718 2719 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) 2720 if (p == dmp) 2721 break; 2722 ASSERT(p == dmp); 2723 *pp = p->dm_next; 2724 rw_exit(&perdm_rwlock); 2725 2726 /* 2727 * Wait for any background processing that relies on the 2728 * syncq to complete before it is freed. 2729 */ 2730 wait_sq_svc(p->dm_sq); 2731 free_syncq(p->dm_sq); 2732 kmem_free(p, sizeof (perdm_t)); 2733 } 2734 2735 /* 2736 * Make a protocol message given control and data buffers. 2737 * n.b., this can block; be careful of what locks you hold when calling it. 2738 * 2739 * If sd_maxblk is less than *iosize this routine can fail part way through 2740 * (due to an allocation failure). In this case on return *iosize will contain 2741 * the amount that was consumed. Otherwise *iosize will not be modified 2742 * i.e. it will contain the amount that was consumed. 2743 */ 2744 int 2745 strmakemsg( 2746 struct strbuf *mctl, 2747 ssize_t *iosize, 2748 struct uio *uiop, 2749 stdata_t *stp, 2750 int32_t flag, 2751 mblk_t **mpp) 2752 { 2753 mblk_t *mpctl = NULL; 2754 mblk_t *mpdata = NULL; 2755 int error; 2756 2757 ASSERT(uiop != NULL); 2758 2759 *mpp = NULL; 2760 /* Create control part, if any */ 2761 if ((mctl != NULL) && (mctl->len >= 0)) { 2762 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); 2763 if (error) 2764 return (error); 2765 } 2766 /* Create data part, if any */ 2767 if (*iosize >= 0) { 2768 error = strmakedata(iosize, uiop, stp, flag, &mpdata); 2769 if (error) { 2770 freemsg(mpctl); 2771 return (error); 2772 } 2773 } 2774 if (mpctl != NULL) { 2775 if (mpdata != NULL) 2776 linkb(mpctl, mpdata); 2777 *mpp = mpctl; 2778 } else { 2779 *mpp = mpdata; 2780 } 2781 return (0); 2782 } 2783 2784 /* 2785 * Make the control part of a protocol message given a control buffer. 2786 * n.b., this can block; be careful of what locks you hold when calling it. 2787 */ 2788 int 2789 strmakectl( 2790 struct strbuf *mctl, 2791 int32_t flag, 2792 int32_t fflag, 2793 mblk_t **mpp) 2794 { 2795 mblk_t *bp = NULL; 2796 unsigned char msgtype; 2797 int error = 0; 2798 2799 *mpp = NULL; 2800 /* 2801 * Create control part of message, if any. 2802 */ 2803 if ((mctl != NULL) && (mctl->len >= 0)) { 2804 caddr_t base; 2805 int ctlcount; 2806 int allocsz; 2807 2808 if (flag & RS_HIPRI) 2809 msgtype = M_PCPROTO; 2810 else 2811 msgtype = M_PROTO; 2812 2813 ctlcount = mctl->len; 2814 base = mctl->buf; 2815 2816 /* 2817 * Give modules a better chance to reuse M_PROTO/M_PCPROTO 2818 * blocks by increasing the size to something more usable. 2819 */ 2820 allocsz = MAX(ctlcount, 64); 2821 2822 /* 2823 * Range checking has already been done; simply try 2824 * to allocate a message block for the ctl part. 2825 */ 2826 while (!(bp = allocb(allocsz, BPRI_MED))) { 2827 if (fflag & (FNDELAY|FNONBLOCK)) 2828 return (EAGAIN); 2829 if (error = strwaitbuf(allocsz, BPRI_MED)) 2830 return (error); 2831 } 2832 2833 bp->b_datap->db_type = msgtype; 2834 if (copyin(base, bp->b_wptr, ctlcount)) { 2835 freeb(bp); 2836 return (EFAULT); 2837 } 2838 bp->b_wptr += ctlcount; 2839 } 2840 *mpp = bp; 2841 return (0); 2842 } 2843 2844 /* 2845 * Make a protocol message given data buffers. 2846 * n.b., this can block; be careful of what locks you hold when calling it. 2847 * 2848 * If sd_maxblk is less than *iosize this routine can fail part way through 2849 * (due to an allocation failure). In this case on return *iosize will contain 2850 * the amount that was consumed. Otherwise *iosize will not be modified 2851 * i.e. it will contain the amount that was consumed. 2852 */ 2853 int 2854 strmakedata( 2855 ssize_t *iosize, 2856 struct uio *uiop, 2857 stdata_t *stp, 2858 int32_t flag, 2859 mblk_t **mpp) 2860 { 2861 mblk_t *mp = NULL; 2862 mblk_t *bp; 2863 int wroff = (int)stp->sd_wroff; 2864 int tail_len = (int)stp->sd_tail; 2865 int extra = wroff + tail_len; 2866 int error = 0; 2867 ssize_t maxblk; 2868 ssize_t count = *iosize; 2869 cred_t *cr = CRED(); 2870 2871 *mpp = NULL; 2872 if (count < 0) 2873 return (0); 2874 2875 maxblk = stp->sd_maxblk; 2876 if (maxblk == INFPSZ) 2877 maxblk = count; 2878 2879 /* 2880 * Create data part of message, if any. 2881 */ 2882 do { 2883 ssize_t size; 2884 dblk_t *dp; 2885 2886 ASSERT(uiop); 2887 2888 size = MIN(count, maxblk); 2889 2890 while ((bp = allocb_cred(size + extra, cr)) == NULL) { 2891 error = EAGAIN; 2892 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 2893 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) { 2894 if (count == *iosize) { 2895 freemsg(mp); 2896 return (error); 2897 } else { 2898 *iosize -= count; 2899 *mpp = mp; 2900 return (0); 2901 } 2902 } 2903 } 2904 dp = bp->b_datap; 2905 dp->db_cpid = curproc->p_pid; 2906 ASSERT(wroff <= dp->db_lim - bp->b_wptr); 2907 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; 2908 2909 if (flag & STRUIO_POSTPONE) { 2910 /* 2911 * Setup the stream uio portion of the 2912 * dblk for subsequent use by struioget(). 2913 */ 2914 dp->db_struioflag = STRUIO_SPEC; 2915 dp->db_cksumstart = 0; 2916 dp->db_cksumstuff = 0; 2917 dp->db_cksumend = size; 2918 *(long long *)dp->db_struioun.data = 0ll; 2919 bp->b_wptr += size; 2920 } else { 2921 if (stp->sd_copyflag & STRCOPYCACHED) 2922 uiop->uio_extflg |= UIO_COPY_CACHED; 2923 2924 if (size != 0) { 2925 error = uiomove(bp->b_wptr, size, UIO_WRITE, 2926 uiop); 2927 if (error != 0) { 2928 freeb(bp); 2929 freemsg(mp); 2930 return (error); 2931 } 2932 } 2933 bp->b_wptr += size; 2934 2935 if (stp->sd_wputdatafunc != NULL) { 2936 mblk_t *newbp; 2937 2938 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode, 2939 bp, NULL, NULL, NULL, NULL); 2940 if (newbp == NULL) { 2941 freeb(bp); 2942 freemsg(mp); 2943 return (ECOMM); 2944 } 2945 bp = newbp; 2946 } 2947 } 2948 2949 count -= size; 2950 2951 if (mp == NULL) 2952 mp = bp; 2953 else 2954 linkb(mp, bp); 2955 } while (count > 0); 2956 2957 *mpp = mp; 2958 return (0); 2959 } 2960 2961 /* 2962 * Wait for a buffer to become available. Return non-zero errno 2963 * if not able to wait, 0 if buffer is probably there. 2964 */ 2965 int 2966 strwaitbuf(size_t size, int pri) 2967 { 2968 bufcall_id_t id; 2969 2970 mutex_enter(&bcall_monitor); 2971 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, 2972 &ttoproc(curthread)->p_flag_cv)) == 0) { 2973 mutex_exit(&bcall_monitor); 2974 return (ENOSR); 2975 } 2976 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { 2977 unbufcall(id); 2978 mutex_exit(&bcall_monitor); 2979 return (EINTR); 2980 } 2981 unbufcall(id); 2982 mutex_exit(&bcall_monitor); 2983 return (0); 2984 } 2985 2986 /* 2987 * This function waits for a read or write event to happen on a stream. 2988 * fmode can specify FNDELAY and/or FNONBLOCK. 2989 * The timeout is in ms with -1 meaning infinite. 2990 * The flag values work as follows: 2991 * READWAIT Check for read side errors, send M_READ 2992 * GETWAIT Check for read side errors, no M_READ 2993 * WRITEWAIT Check for write side errors. 2994 * NOINTR Do not return error if nonblocking or timeout. 2995 * STR_NOERROR Ignore all errors except STPLEX. 2996 * STR_NOSIG Ignore/hold signals during the duration of the call. 2997 * STR_PEEK Pass through the strgeterr(). 2998 */ 2999 int 3000 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, 3001 int *done) 3002 { 3003 int slpflg, errs; 3004 int error; 3005 kcondvar_t *sleepon; 3006 mblk_t *mp; 3007 ssize_t *rd_count; 3008 clock_t rval; 3009 3010 ASSERT(MUTEX_HELD(&stp->sd_lock)); 3011 if ((flag & READWAIT) || (flag & GETWAIT)) { 3012 slpflg = RSLEEP; 3013 sleepon = &_RD(stp->sd_wrq)->q_wait; 3014 errs = STRDERR|STPLEX; 3015 } else { 3016 slpflg = WSLEEP; 3017 sleepon = &stp->sd_wrq->q_wait; 3018 errs = STWRERR|STRHUP|STPLEX; 3019 } 3020 if (flag & STR_NOERROR) 3021 errs = STPLEX; 3022 3023 if (stp->sd_wakeq & slpflg) { 3024 /* 3025 * A strwakeq() is pending, no need to sleep. 3026 */ 3027 stp->sd_wakeq &= ~slpflg; 3028 *done = 0; 3029 return (0); 3030 } 3031 3032 if (fmode & (FNDELAY|FNONBLOCK)) { 3033 if (!(flag & NOINTR)) 3034 error = EAGAIN; 3035 else 3036 error = 0; 3037 *done = 1; 3038 return (error); 3039 } 3040 3041 if (stp->sd_flag & errs) { 3042 /* 3043 * Check for errors before going to sleep since the 3044 * caller might not have checked this while holding 3045 * sd_lock. 3046 */ 3047 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3048 if (error != 0) { 3049 *done = 1; 3050 return (error); 3051 } 3052 } 3053 3054 /* 3055 * If any module downstream has requested read notification 3056 * by setting SNDMREAD flag using M_SETOPTS, send a message 3057 * down stream. 3058 */ 3059 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) { 3060 mutex_exit(&stp->sd_lock); 3061 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED, 3062 (flag & STR_NOSIG), &error))) { 3063 mutex_enter(&stp->sd_lock); 3064 *done = 1; 3065 return (error); 3066 } 3067 mp->b_datap->db_type = M_READ; 3068 rd_count = (ssize_t *)mp->b_wptr; 3069 *rd_count = count; 3070 mp->b_wptr += sizeof (ssize_t); 3071 /* 3072 * Send the number of bytes requested by the 3073 * read as the argument to M_READ. 3074 */ 3075 stream_willservice(stp); 3076 putnext(stp->sd_wrq, mp); 3077 stream_runservice(stp); 3078 mutex_enter(&stp->sd_lock); 3079 3080 /* 3081 * If any data arrived due to inline processing 3082 * of putnext(), don't sleep. 3083 */ 3084 if (_RD(stp->sd_wrq)->q_first != NULL) { 3085 *done = 0; 3086 return (0); 3087 } 3088 } 3089 3090 stp->sd_flag |= slpflg; 3091 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2, 3092 "strwaitq sleeps (2):%p, %X, %lX, %X, %p", 3093 stp, flag, count, fmode, done); 3094 3095 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG); 3096 if (rval > 0) { 3097 /* EMPTY */ 3098 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2, 3099 "strwaitq awakes(2):%X, %X, %X, %X, %X", 3100 stp, flag, count, fmode, done); 3101 } else if (rval == 0) { 3102 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2, 3103 "strwaitq interrupt #2:%p, %X, %lX, %X, %p", 3104 stp, flag, count, fmode, done); 3105 stp->sd_flag &= ~slpflg; 3106 cv_broadcast(sleepon); 3107 if (!(flag & NOINTR)) 3108 error = EINTR; 3109 else 3110 error = 0; 3111 *done = 1; 3112 return (error); 3113 } else { 3114 /* timeout */ 3115 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME, 3116 "strwaitq timeout:%p, %X, %lX, %X, %p", 3117 stp, flag, count, fmode, done); 3118 *done = 1; 3119 if (!(flag & NOINTR)) 3120 return (ETIME); 3121 else 3122 return (0); 3123 } 3124 /* 3125 * If the caller implements delayed errors (i.e. queued after data) 3126 * we can not check for errors here since data as well as an 3127 * error might have arrived at the stream head. We return to 3128 * have the caller check the read queue before checking for errors. 3129 */ 3130 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) { 3131 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3132 if (error != 0) { 3133 *done = 1; 3134 return (error); 3135 } 3136 } 3137 *done = 0; 3138 return (0); 3139 } 3140 3141 /* 3142 * Perform job control discipline access checks. 3143 * Return 0 for success and the errno for failure. 3144 */ 3145 3146 #define cantsend(p, t, sig) \ 3147 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig)) 3148 3149 int 3150 straccess(struct stdata *stp, enum jcaccess mode) 3151 { 3152 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */ 3153 kthread_t *t = curthread; 3154 proc_t *p = ttoproc(t); 3155 sess_t *sp; 3156 3157 ASSERT(mutex_owned(&stp->sd_lock)); 3158 3159 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO) 3160 return (0); 3161 3162 mutex_enter(&p->p_lock); /* protects p_pgidp */ 3163 3164 for (;;) { 3165 mutex_enter(&p->p_splock); /* protects p->p_sessp */ 3166 sp = p->p_sessp; 3167 mutex_enter(&sp->s_lock); /* protects sp->* */ 3168 3169 /* 3170 * If this is not the calling process's controlling terminal 3171 * or if the calling process is already in the foreground 3172 * then allow access. 3173 */ 3174 if (sp->s_dev != stp->sd_vnode->v_rdev || 3175 p->p_pgidp == stp->sd_pgidp) { 3176 mutex_exit(&sp->s_lock); 3177 mutex_exit(&p->p_splock); 3178 mutex_exit(&p->p_lock); 3179 return (0); 3180 } 3181 3182 /* 3183 * Check to see if controlling terminal has been deallocated. 3184 */ 3185 if (sp->s_vp == NULL) { 3186 if (!cantsend(p, t, SIGHUP)) 3187 sigtoproc(p, t, SIGHUP); 3188 mutex_exit(&sp->s_lock); 3189 mutex_exit(&p->p_splock); 3190 mutex_exit(&p->p_lock); 3191 return (EIO); 3192 } 3193 3194 mutex_exit(&sp->s_lock); 3195 mutex_exit(&p->p_splock); 3196 3197 if (mode == JCGETP) { 3198 mutex_exit(&p->p_lock); 3199 return (0); 3200 } 3201 3202 if (mode == JCREAD) { 3203 if (p->p_detached || cantsend(p, t, SIGTTIN)) { 3204 mutex_exit(&p->p_lock); 3205 return (EIO); 3206 } 3207 mutex_exit(&p->p_lock); 3208 mutex_exit(&stp->sd_lock); 3209 pgsignal(p->p_pgidp, SIGTTIN); 3210 mutex_enter(&stp->sd_lock); 3211 mutex_enter(&p->p_lock); 3212 } else { /* mode == JCWRITE or JCSETP */ 3213 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) || 3214 cantsend(p, t, SIGTTOU)) { 3215 mutex_exit(&p->p_lock); 3216 return (0); 3217 } 3218 if (p->p_detached) { 3219 mutex_exit(&p->p_lock); 3220 return (EIO); 3221 } 3222 mutex_exit(&p->p_lock); 3223 mutex_exit(&stp->sd_lock); 3224 pgsignal(p->p_pgidp, SIGTTOU); 3225 mutex_enter(&stp->sd_lock); 3226 mutex_enter(&p->p_lock); 3227 } 3228 3229 /* 3230 * We call cv_wait_sig_swap() to cause the appropriate 3231 * action for the jobcontrol signal to take place. 3232 * If the signal is being caught, we will take the 3233 * EINTR error return. Otherwise, the default action 3234 * of causing the process to stop will take place. 3235 * In this case, we rely on the periodic cv_broadcast() on 3236 * &lbolt_cv to wake us up to loop around and test again. 3237 * We can't get here if the signal is ignored or 3238 * if the current thread is blocking the signal. 3239 */ 3240 mutex_exit(&stp->sd_lock); 3241 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) { 3242 mutex_exit(&p->p_lock); 3243 mutex_enter(&stp->sd_lock); 3244 return (EINTR); 3245 } 3246 mutex_exit(&p->p_lock); 3247 mutex_enter(&stp->sd_lock); 3248 mutex_enter(&p->p_lock); 3249 } 3250 } 3251 3252 /* 3253 * Return size of message of block type (bp->b_datap->db_type) 3254 */ 3255 size_t 3256 xmsgsize(mblk_t *bp) 3257 { 3258 unsigned char type; 3259 size_t count = 0; 3260 3261 type = bp->b_datap->db_type; 3262 3263 for (; bp; bp = bp->b_cont) { 3264 if (type != bp->b_datap->db_type) 3265 break; 3266 ASSERT(bp->b_wptr >= bp->b_rptr); 3267 count += bp->b_wptr - bp->b_rptr; 3268 } 3269 return (count); 3270 } 3271 3272 /* 3273 * Allocate a stream head. 3274 */ 3275 struct stdata * 3276 shalloc(queue_t *qp) 3277 { 3278 stdata_t *stp; 3279 3280 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP); 3281 3282 stp->sd_wrq = _WR(qp); 3283 stp->sd_strtab = NULL; 3284 stp->sd_iocid = 0; 3285 stp->sd_mate = NULL; 3286 stp->sd_freezer = NULL; 3287 stp->sd_refcnt = 0; 3288 stp->sd_wakeq = 0; 3289 stp->sd_anchor = 0; 3290 stp->sd_struiowrq = NULL; 3291 stp->sd_struiordq = NULL; 3292 stp->sd_struiodnak = 0; 3293 stp->sd_struionak = NULL; 3294 #ifdef C2_AUDIT 3295 stp->sd_t_audit_data = NULL; 3296 #endif 3297 stp->sd_rput_opt = 0; 3298 stp->sd_wput_opt = 0; 3299 stp->sd_read_opt = 0; 3300 stp->sd_rprotofunc = strrput_proto; 3301 stp->sd_rmiscfunc = strrput_misc; 3302 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL; 3303 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL; 3304 stp->sd_ciputctrl = NULL; 3305 stp->sd_nciputctrl = 0; 3306 stp->sd_qhead = NULL; 3307 stp->sd_qtail = NULL; 3308 stp->sd_servid = NULL; 3309 stp->sd_nqueues = 0; 3310 stp->sd_svcflags = 0; 3311 stp->sd_copyflag = 0; 3312 3313 return (stp); 3314 } 3315 3316 /* 3317 * Free a stream head. 3318 */ 3319 void 3320 shfree(stdata_t *stp) 3321 { 3322 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock)); 3323 3324 stp->sd_wrq = NULL; 3325 3326 mutex_enter(&stp->sd_qlock); 3327 while (stp->sd_svcflags & STRS_SCHEDULED) { 3328 STRSTAT(strwaits); 3329 cv_wait(&stp->sd_qcv, &stp->sd_qlock); 3330 } 3331 mutex_exit(&stp->sd_qlock); 3332 3333 if (stp->sd_ciputctrl != NULL) { 3334 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1); 3335 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl, 3336 stp->sd_nciputctrl, 0); 3337 ASSERT(ciputctrl_cache != NULL); 3338 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl); 3339 stp->sd_ciputctrl = NULL; 3340 stp->sd_nciputctrl = 0; 3341 } 3342 ASSERT(stp->sd_qhead == NULL); 3343 ASSERT(stp->sd_qtail == NULL); 3344 ASSERT(stp->sd_nqueues == 0); 3345 kmem_cache_free(stream_head_cache, stp); 3346 } 3347 3348 /* 3349 * Allocate a pair of queues and a syncq for the pair 3350 */ 3351 queue_t * 3352 allocq(void) 3353 { 3354 queinfo_t *qip; 3355 queue_t *qp, *wqp; 3356 syncq_t *sq; 3357 3358 qip = kmem_cache_alloc(queue_cache, KM_SLEEP); 3359 3360 qp = &qip->qu_rqueue; 3361 wqp = &qip->qu_wqueue; 3362 sq = &qip->qu_syncq; 3363 3364 qp->q_last = NULL; 3365 qp->q_next = NULL; 3366 qp->q_ptr = NULL; 3367 qp->q_flag = QUSE | QREADR; 3368 qp->q_bandp = NULL; 3369 qp->q_stream = NULL; 3370 qp->q_syncq = sq; 3371 qp->q_nband = 0; 3372 qp->q_nfsrv = NULL; 3373 qp->q_draining = 0; 3374 qp->q_syncqmsgs = 0; 3375 qp->q_spri = 0; 3376 qp->q_qtstamp = 0; 3377 qp->q_sqtstamp = 0; 3378 qp->q_fp = NULL; 3379 3380 wqp->q_last = NULL; 3381 wqp->q_next = NULL; 3382 wqp->q_ptr = NULL; 3383 wqp->q_flag = QUSE; 3384 wqp->q_bandp = NULL; 3385 wqp->q_stream = NULL; 3386 wqp->q_syncq = sq; 3387 wqp->q_nband = 0; 3388 wqp->q_nfsrv = NULL; 3389 wqp->q_draining = 0; 3390 wqp->q_syncqmsgs = 0; 3391 wqp->q_qtstamp = 0; 3392 wqp->q_sqtstamp = 0; 3393 wqp->q_spri = 0; 3394 3395 sq->sq_count = 0; 3396 sq->sq_rmqcount = 0; 3397 sq->sq_flags = 0; 3398 sq->sq_type = 0; 3399 sq->sq_callbflags = 0; 3400 sq->sq_cancelid = 0; 3401 sq->sq_ciputctrl = NULL; 3402 sq->sq_nciputctrl = 0; 3403 sq->sq_needexcl = 0; 3404 sq->sq_svcflags = 0; 3405 3406 return (qp); 3407 } 3408 3409 /* 3410 * Free a pair of queues and the "attached" syncq. 3411 * Discard any messages left on the syncq(s), remove the syncq(s) from the 3412 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq. 3413 */ 3414 void 3415 freeq(queue_t *qp) 3416 { 3417 qband_t *qbp, *nqbp; 3418 syncq_t *sq, *outer; 3419 queue_t *wqp = _WR(qp); 3420 3421 ASSERT(qp->q_flag & QREADR); 3422 3423 /* 3424 * If a previously dispatched taskq job is scheduled to run 3425 * sync_service() or a service routine is scheduled for the 3426 * queues about to be freed, wait here until all service is 3427 * done on the queue and all associated queues and syncqs. 3428 */ 3429 wait_svc(qp); 3430 3431 (void) flush_syncq(qp->q_syncq, qp); 3432 (void) flush_syncq(wqp->q_syncq, wqp); 3433 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0); 3434 3435 /* 3436 * Flush the queues before q_next is set to NULL This is needed 3437 * in order to backenable any downstream queue before we go away. 3438 * Note: we are already removed from the stream so that the 3439 * backenabling will not cause any messages to be delivered to our 3440 * put procedures. 3441 */ 3442 flushq(qp, FLUSHALL); 3443 flushq(wqp, FLUSHALL); 3444 3445 /* Tidy up - removeq only does a half-remove from stream */ 3446 qp->q_next = wqp->q_next = NULL; 3447 ASSERT(!(qp->q_flag & QENAB)); 3448 ASSERT(!(wqp->q_flag & QENAB)); 3449 3450 outer = qp->q_syncq->sq_outer; 3451 if (outer != NULL) { 3452 outer_remove(outer, qp->q_syncq); 3453 if (wqp->q_syncq != qp->q_syncq) 3454 outer_remove(outer, wqp->q_syncq); 3455 } 3456 /* 3457 * Free any syncqs that are outside what allocq returned. 3458 */ 3459 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD)) 3460 free_syncq(qp->q_syncq); 3461 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp)) 3462 free_syncq(wqp->q_syncq); 3463 3464 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3465 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3466 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 3467 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp))); 3468 sq = SQ(qp); 3469 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 3470 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 3471 ASSERT(sq->sq_outer == NULL); 3472 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 3473 ASSERT(sq->sq_callbpend == NULL); 3474 ASSERT(sq->sq_needexcl == 0); 3475 3476 if (sq->sq_ciputctrl != NULL) { 3477 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 3478 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 3479 sq->sq_nciputctrl, 0); 3480 ASSERT(ciputctrl_cache != NULL); 3481 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 3482 sq->sq_ciputctrl = NULL; 3483 sq->sq_nciputctrl = 0; 3484 } 3485 3486 ASSERT(qp->q_first == NULL && wqp->q_first == NULL); 3487 ASSERT(qp->q_count == 0 && wqp->q_count == 0); 3488 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0); 3489 3490 qp->q_flag &= ~QUSE; 3491 wqp->q_flag &= ~QUSE; 3492 3493 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */ 3494 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */ 3495 3496 qbp = qp->q_bandp; 3497 while (qbp) { 3498 nqbp = qbp->qb_next; 3499 freeband(qbp); 3500 qbp = nqbp; 3501 } 3502 qbp = wqp->q_bandp; 3503 while (qbp) { 3504 nqbp = qbp->qb_next; 3505 freeband(qbp); 3506 qbp = nqbp; 3507 } 3508 kmem_cache_free(queue_cache, qp); 3509 } 3510 3511 /* 3512 * Allocate a qband structure. 3513 */ 3514 qband_t * 3515 allocband(void) 3516 { 3517 qband_t *qbp; 3518 3519 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP); 3520 if (qbp == NULL) 3521 return (NULL); 3522 3523 qbp->qb_next = NULL; 3524 qbp->qb_count = 0; 3525 qbp->qb_mblkcnt = 0; 3526 qbp->qb_first = NULL; 3527 qbp->qb_last = NULL; 3528 qbp->qb_flag = 0; 3529 3530 return (qbp); 3531 } 3532 3533 /* 3534 * Free a qband structure. 3535 */ 3536 void 3537 freeband(qband_t *qbp) 3538 { 3539 kmem_cache_free(qband_cache, qbp); 3540 } 3541 3542 /* 3543 * Just like putnextctl(9F), except that allocb_wait() is used. 3544 * 3545 * Consolidation Private, and of course only callable from the stream head or 3546 * routines that may block. 3547 */ 3548 int 3549 putnextctl_wait(queue_t *q, int type) 3550 { 3551 mblk_t *bp; 3552 int error; 3553 3554 if ((datamsg(type) && (type != M_DELAY)) || 3555 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL) 3556 return (0); 3557 3558 bp->b_datap->db_type = (unsigned char)type; 3559 putnext(q, bp); 3560 return (1); 3561 } 3562 3563 /* 3564 * run any possible bufcalls. 3565 */ 3566 void 3567 runbufcalls(void) 3568 { 3569 strbufcall_t *bcp; 3570 3571 mutex_enter(&bcall_monitor); 3572 mutex_enter(&strbcall_lock); 3573 3574 if (strbcalls.bc_head) { 3575 size_t count; 3576 int nevent; 3577 3578 /* 3579 * count how many events are on the list 3580 * now so we can check to avoid looping 3581 * in low memory situations 3582 */ 3583 nevent = 0; 3584 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next) 3585 nevent++; 3586 3587 /* 3588 * get estimate of available memory from kmem_avail(). 3589 * awake all bufcall functions waiting for 3590 * memory whose request could be satisfied 3591 * by 'count' memory and let 'em fight for it. 3592 */ 3593 count = kmem_avail(); 3594 while ((bcp = strbcalls.bc_head) != NULL && nevent) { 3595 STRSTAT(bufcalls); 3596 --nevent; 3597 if (bcp->bc_size <= count) { 3598 bcp->bc_executor = curthread; 3599 mutex_exit(&strbcall_lock); 3600 (*bcp->bc_func)(bcp->bc_arg); 3601 mutex_enter(&strbcall_lock); 3602 bcp->bc_executor = NULL; 3603 cv_broadcast(&bcall_cv); 3604 strbcalls.bc_head = bcp->bc_next; 3605 kmem_free(bcp, sizeof (strbufcall_t)); 3606 } else { 3607 /* 3608 * too big, try again later - note 3609 * that nevent was decremented above 3610 * so we won't retry this one on this 3611 * iteration of the loop 3612 */ 3613 if (bcp->bc_next != NULL) { 3614 strbcalls.bc_head = bcp->bc_next; 3615 bcp->bc_next = NULL; 3616 strbcalls.bc_tail->bc_next = bcp; 3617 strbcalls.bc_tail = bcp; 3618 } 3619 } 3620 } 3621 if (strbcalls.bc_head == NULL) 3622 strbcalls.bc_tail = NULL; 3623 } 3624 3625 mutex_exit(&strbcall_lock); 3626 mutex_exit(&bcall_monitor); 3627 } 3628 3629 3630 /* 3631 * actually run queue's service routine. 3632 */ 3633 static void 3634 runservice(queue_t *q) 3635 { 3636 qband_t *qbp; 3637 3638 ASSERT(q->q_qinfo->qi_srvp); 3639 again: 3640 entersq(q->q_syncq, SQ_SVC); 3641 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START, 3642 "runservice starts:%p", q); 3643 3644 if (!(q->q_flag & QWCLOSE)) 3645 (*q->q_qinfo->qi_srvp)(q); 3646 3647 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END, 3648 "runservice ends:(%p)", q); 3649 3650 leavesq(q->q_syncq, SQ_SVC); 3651 3652 mutex_enter(QLOCK(q)); 3653 if (q->q_flag & QENAB) { 3654 q->q_flag &= ~QENAB; 3655 mutex_exit(QLOCK(q)); 3656 goto again; 3657 } 3658 q->q_flag &= ~QINSERVICE; 3659 q->q_flag &= ~QBACK; 3660 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next) 3661 qbp->qb_flag &= ~QB_BACK; 3662 /* 3663 * Wakeup thread waiting for the service procedure 3664 * to be run (strclose and qdetach). 3665 */ 3666 cv_broadcast(&q->q_wait); 3667 3668 mutex_exit(QLOCK(q)); 3669 } 3670 3671 /* 3672 * Background processing of bufcalls. 3673 */ 3674 void 3675 streams_bufcall_service(void) 3676 { 3677 callb_cpr_t cprinfo; 3678 3679 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr, 3680 "streams_bufcall_service"); 3681 3682 mutex_enter(&strbcall_lock); 3683 3684 for (;;) { 3685 if (strbcalls.bc_head != NULL && kmem_avail() > 0) { 3686 mutex_exit(&strbcall_lock); 3687 runbufcalls(); 3688 mutex_enter(&strbcall_lock); 3689 } 3690 if (strbcalls.bc_head != NULL) { 3691 clock_t wt, tick; 3692 3693 STRSTAT(bcwaits); 3694 /* Wait for memory to become available */ 3695 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3696 tick = SEC_TO_TICK(60); 3697 time_to_wait(&wt, tick); 3698 (void) cv_timedwait(&memavail_cv, &strbcall_lock, wt); 3699 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3700 } 3701 3702 /* Wait for new work to arrive */ 3703 if (strbcalls.bc_head == NULL) { 3704 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3705 cv_wait(&strbcall_cv, &strbcall_lock); 3706 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3707 } 3708 } 3709 } 3710 3711 /* 3712 * Background processing of streams background tasks which failed 3713 * taskq_dispatch. 3714 */ 3715 static void 3716 streams_qbkgrnd_service(void) 3717 { 3718 callb_cpr_t cprinfo; 3719 queue_t *q; 3720 3721 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3722 "streams_bkgrnd_service"); 3723 3724 mutex_enter(&service_queue); 3725 3726 for (;;) { 3727 /* 3728 * Wait for work to arrive. 3729 */ 3730 while ((freebs_list == NULL) && (qhead == NULL)) { 3731 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3732 cv_wait(&services_to_run, &service_queue); 3733 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3734 } 3735 /* 3736 * Handle all pending freebs requests to free memory. 3737 */ 3738 while (freebs_list != NULL) { 3739 mblk_t *mp = freebs_list; 3740 freebs_list = mp->b_next; 3741 mutex_exit(&service_queue); 3742 mblk_free(mp); 3743 mutex_enter(&service_queue); 3744 } 3745 /* 3746 * Run pending queues. 3747 */ 3748 while (qhead != NULL) { 3749 DQ(q, qhead, qtail, q_link); 3750 ASSERT(q != NULL); 3751 mutex_exit(&service_queue); 3752 queue_service(q); 3753 mutex_enter(&service_queue); 3754 } 3755 ASSERT(qhead == NULL && qtail == NULL); 3756 } 3757 } 3758 3759 /* 3760 * Background processing of streams background tasks which failed 3761 * taskq_dispatch. 3762 */ 3763 static void 3764 streams_sqbkgrnd_service(void) 3765 { 3766 callb_cpr_t cprinfo; 3767 syncq_t *sq; 3768 3769 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3770 "streams_sqbkgrnd_service"); 3771 3772 mutex_enter(&service_queue); 3773 3774 for (;;) { 3775 /* 3776 * Wait for work to arrive. 3777 */ 3778 while (sqhead == NULL) { 3779 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3780 cv_wait(&syncqs_to_run, &service_queue); 3781 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3782 } 3783 3784 /* 3785 * Run pending syncqs. 3786 */ 3787 while (sqhead != NULL) { 3788 DQ(sq, sqhead, sqtail, sq_next); 3789 ASSERT(sq != NULL); 3790 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); 3791 mutex_exit(&service_queue); 3792 syncq_service(sq); 3793 mutex_enter(&service_queue); 3794 } 3795 } 3796 } 3797 3798 /* 3799 * Disable the syncq and wait for background syncq processing to complete. 3800 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the 3801 * list. 3802 */ 3803 void 3804 wait_sq_svc(syncq_t *sq) 3805 { 3806 mutex_enter(SQLOCK(sq)); 3807 sq->sq_svcflags |= SQ_DISABLED; 3808 if (sq->sq_svcflags & SQ_BGTHREAD) { 3809 syncq_t *sq_chase; 3810 syncq_t *sq_curr; 3811 int removed; 3812 3813 ASSERT(sq->sq_servcount == 1); 3814 mutex_enter(&service_queue); 3815 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); 3816 mutex_exit(&service_queue); 3817 if (removed) { 3818 sq->sq_svcflags &= ~SQ_BGTHREAD; 3819 sq->sq_servcount = 0; 3820 STRSTAT(sqremoved); 3821 goto done; 3822 } 3823 } 3824 while (sq->sq_servcount != 0) { 3825 sq->sq_flags |= SQ_WANTWAKEUP; 3826 cv_wait(&sq->sq_wait, SQLOCK(sq)); 3827 } 3828 done: 3829 mutex_exit(SQLOCK(sq)); 3830 } 3831 3832 /* 3833 * Put a syncq on the list of syncq's to be serviced by the sqthread. 3834 * Add the argument to the end of the sqhead list and set the flag 3835 * indicating this syncq has been enabled. If it has already been 3836 * enabled, don't do anything. 3837 * This routine assumes that SQLOCK is held. 3838 * NOTE that the lock order is to have the SQLOCK first, 3839 * so if the service_syncq lock is held, we need to release it 3840 * before aquiring the SQLOCK (mostly relevant for the background 3841 * thread, and this seems to be common among the STREAMS global locks). 3842 * Note the the sq_svcflags are protected by the SQLOCK. 3843 */ 3844 void 3845 sqenable(syncq_t *sq) 3846 { 3847 /* 3848 * This is probably not important except for where I believe it 3849 * is being called. At that point, it should be held (and it 3850 * is a pain to release it just for this routine, so don't do 3851 * it). 3852 */ 3853 ASSERT(MUTEX_HELD(SQLOCK(sq))); 3854 3855 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); 3856 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); 3857 3858 /* 3859 * Do not put on list if background thread is scheduled or 3860 * syncq is disabled. 3861 */ 3862 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) 3863 return; 3864 3865 /* 3866 * Check whether we should enable sq at all. 3867 * Non PERMOD syncqs may be drained by at most one thread. 3868 * PERMOD syncqs may be drained by several threads but we limit the 3869 * total amount to the lesser of 3870 * Number of queues on the squeue and 3871 * Number of CPUs. 3872 */ 3873 if (sq->sq_servcount != 0) { 3874 if (((sq->sq_type & SQ_PERMOD) == 0) || 3875 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { 3876 STRSTAT(sqtoomany); 3877 return; 3878 } 3879 } 3880 3881 sq->sq_tstamp = lbolt; 3882 STRSTAT(sqenables); 3883 3884 /* Attempt a taskq dispatch */ 3885 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, 3886 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); 3887 if (sq->sq_servid != NULL) { 3888 sq->sq_servcount++; 3889 return; 3890 } 3891 3892 /* 3893 * This taskq dispatch failed, but a previous one may have succeeded. 3894 * Don't try to schedule on the background thread whilst there is 3895 * outstanding taskq processing. 3896 */ 3897 if (sq->sq_servcount != 0) 3898 return; 3899 3900 /* 3901 * System is low on resources and can't perform a non-sleeping 3902 * dispatch. Schedule the syncq for a background thread and mark the 3903 * syncq to avoid any further taskq dispatch attempts. 3904 */ 3905 mutex_enter(&service_queue); 3906 STRSTAT(taskqfails); 3907 ENQUEUE(sq, sqhead, sqtail, sq_next); 3908 sq->sq_svcflags |= SQ_BGTHREAD; 3909 sq->sq_servcount = 1; 3910 cv_signal(&syncqs_to_run); 3911 mutex_exit(&service_queue); 3912 } 3913 3914 /* 3915 * Note: fifo_close() depends on the mblk_t on the queue being freed 3916 * asynchronously. The asynchronous freeing of messages breaks the 3917 * recursive call chain of fifo_close() while there are I_SENDFD type of 3918 * messages refering other file pointers on the queue. Then when 3919 * closing pipes it can avoid stack overflow in case of daisy-chained 3920 * pipes, and also avoid deadlock in case of fifonode_t pairs (which 3921 * share the same fifolock_t). 3922 */ 3923 3924 void 3925 freebs_enqueue(mblk_t *mp, dblk_t *dbp) 3926 { 3927 esb_queue_t *eqp = &system_esbq; 3928 3929 ASSERT(dbp->db_mblk == mp); 3930 3931 /* 3932 * Check data sanity. The dblock should have non-empty free function. 3933 * It is better to panic here then later when the dblock is freed 3934 * asynchronously when the context is lost. 3935 */ 3936 if (dbp->db_frtnp->free_func == NULL) { 3937 panic("freebs_enqueue: dblock %p has a NULL free callback", 3938 (void *)dbp); 3939 } 3940 3941 mutex_enter(&eqp->eq_lock); 3942 /* queue the new mblk on the esballoc queue */ 3943 if (eqp->eq_head == NULL) { 3944 eqp->eq_head = eqp->eq_tail = mp; 3945 } else { 3946 eqp->eq_tail->b_next = mp; 3947 eqp->eq_tail = mp; 3948 } 3949 eqp->eq_len++; 3950 3951 /* If we're the first thread to reach the threshold, process */ 3952 if (eqp->eq_len >= esbq_max_qlen && 3953 !(eqp->eq_flags & ESBQ_PROCESSING)) 3954 esballoc_process_queue(eqp); 3955 3956 esballoc_set_timer(eqp, esbq_timeout); 3957 mutex_exit(&eqp->eq_lock); 3958 } 3959 3960 static void 3961 esballoc_process_queue(esb_queue_t *eqp) 3962 { 3963 mblk_t *mp; 3964 3965 ASSERT(MUTEX_HELD(&eqp->eq_lock)); 3966 3967 eqp->eq_flags |= ESBQ_PROCESSING; 3968 3969 do { 3970 /* 3971 * Detach the message chain for processing. 3972 */ 3973 mp = eqp->eq_head; 3974 eqp->eq_tail->b_next = NULL; 3975 eqp->eq_head = eqp->eq_tail = NULL; 3976 eqp->eq_len = 0; 3977 mutex_exit(&eqp->eq_lock); 3978 3979 /* 3980 * Process the message chain. 3981 */ 3982 esballoc_enqueue_mblk(mp); 3983 mutex_enter(&eqp->eq_lock); 3984 } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0)); 3985 3986 eqp->eq_flags &= ~ESBQ_PROCESSING; 3987 } 3988 3989 /* 3990 * taskq callback routine to free esballoced mblk's 3991 */ 3992 static void 3993 esballoc_mblk_free(mblk_t *mp) 3994 { 3995 mblk_t *nextmp; 3996 3997 for (; mp != NULL; mp = nextmp) { 3998 nextmp = mp->b_next; 3999 mp->b_next = NULL; 4000 mblk_free(mp); 4001 } 4002 } 4003 4004 static void 4005 esballoc_enqueue_mblk(mblk_t *mp) 4006 { 4007 4008 if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp, 4009 TQ_NOSLEEP) == NULL) { 4010 mblk_t *first_mp = mp; 4011 /* 4012 * System is low on resources and can't perform a non-sleeping 4013 * dispatch. Schedule for a background thread. 4014 */ 4015 mutex_enter(&service_queue); 4016 STRSTAT(taskqfails); 4017 4018 while (mp->b_next != NULL) 4019 mp = mp->b_next; 4020 4021 mp->b_next = freebs_list; 4022 freebs_list = first_mp; 4023 cv_signal(&services_to_run); 4024 mutex_exit(&service_queue); 4025 } 4026 } 4027 4028 static void 4029 esballoc_timer(void *arg) 4030 { 4031 esb_queue_t *eqp = arg; 4032 4033 mutex_enter(&eqp->eq_lock); 4034 eqp->eq_flags &= ~ESBQ_TIMER; 4035 4036 if (!(eqp->eq_flags & ESBQ_PROCESSING) && 4037 eqp->eq_len > 0) 4038 esballoc_process_queue(eqp); 4039 4040 esballoc_set_timer(eqp, esbq_timeout); 4041 mutex_exit(&eqp->eq_lock); 4042 } 4043 4044 static void 4045 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout) 4046 { 4047 ASSERT(MUTEX_HELD(&eqp->eq_lock)); 4048 4049 if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) { 4050 (void) timeout(esballoc_timer, eqp, eq_timeout); 4051 eqp->eq_flags |= ESBQ_TIMER; 4052 } 4053 } 4054 4055 void 4056 esballoc_queue_init(void) 4057 { 4058 system_esbq.eq_len = 0; 4059 system_esbq.eq_head = system_esbq.eq_tail = NULL; 4060 system_esbq.eq_flags = 0; 4061 } 4062 4063 /* 4064 * Set the QBACK or QB_BACK flag in the given queue for 4065 * the given priority band. 4066 */ 4067 void 4068 setqback(queue_t *q, unsigned char pri) 4069 { 4070 int i; 4071 qband_t *qbp; 4072 qband_t **qbpp; 4073 4074 ASSERT(MUTEX_HELD(QLOCK(q))); 4075 if (pri != 0) { 4076 if (pri > q->q_nband) { 4077 qbpp = &q->q_bandp; 4078 while (*qbpp) 4079 qbpp = &(*qbpp)->qb_next; 4080 while (pri > q->q_nband) { 4081 if ((*qbpp = allocband()) == NULL) { 4082 cmn_err(CE_WARN, 4083 "setqback: can't allocate qband\n"); 4084 return; 4085 } 4086 (*qbpp)->qb_hiwat = q->q_hiwat; 4087 (*qbpp)->qb_lowat = q->q_lowat; 4088 q->q_nband++; 4089 qbpp = &(*qbpp)->qb_next; 4090 } 4091 } 4092 qbp = q->q_bandp; 4093 i = pri; 4094 while (--i) 4095 qbp = qbp->qb_next; 4096 qbp->qb_flag |= QB_BACK; 4097 } else { 4098 q->q_flag |= QBACK; 4099 } 4100 } 4101 4102 int 4103 strcopyin(void *from, void *to, size_t len, int copyflag) 4104 { 4105 if (copyflag & U_TO_K) { 4106 ASSERT((copyflag & K_TO_K) == 0); 4107 if (copyin(from, to, len)) 4108 return (EFAULT); 4109 } else { 4110 ASSERT(copyflag & K_TO_K); 4111 bcopy(from, to, len); 4112 } 4113 return (0); 4114 } 4115 4116 int 4117 strcopyout(void *from, void *to, size_t len, int copyflag) 4118 { 4119 if (copyflag & U_TO_K) { 4120 if (copyout(from, to, len)) 4121 return (EFAULT); 4122 } else { 4123 ASSERT(copyflag & K_TO_K); 4124 bcopy(from, to, len); 4125 } 4126 return (0); 4127 } 4128 4129 /* 4130 * strsignal_nolock() posts a signal to the process(es) at the stream head. 4131 * It assumes that the stream head lock is already held, whereas strsignal() 4132 * acquires the lock first. This routine was created because a few callers 4133 * release the stream head lock before calling only to re-acquire it after 4134 * it returns. 4135 */ 4136 void 4137 strsignal_nolock(stdata_t *stp, int sig, int32_t band) 4138 { 4139 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4140 switch (sig) { 4141 case SIGPOLL: 4142 if (stp->sd_sigflags & S_MSG) 4143 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 4144 break; 4145 4146 default: 4147 if (stp->sd_pgidp) { 4148 pgsignal(stp->sd_pgidp, sig); 4149 } 4150 break; 4151 } 4152 } 4153 4154 void 4155 strsignal(stdata_t *stp, int sig, int32_t band) 4156 { 4157 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG, 4158 "strsignal:%p, %X, %X", stp, sig, band); 4159 4160 mutex_enter(&stp->sd_lock); 4161 switch (sig) { 4162 case SIGPOLL: 4163 if (stp->sd_sigflags & S_MSG) 4164 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 4165 break; 4166 4167 default: 4168 if (stp->sd_pgidp) { 4169 pgsignal(stp->sd_pgidp, sig); 4170 } 4171 break; 4172 } 4173 mutex_exit(&stp->sd_lock); 4174 } 4175 4176 void 4177 strhup(stdata_t *stp) 4178 { 4179 ASSERT(mutex_owned(&stp->sd_lock)); 4180 pollwakeup(&stp->sd_pollist, POLLHUP); 4181 if (stp->sd_sigflags & S_HANGUP) 4182 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0); 4183 } 4184 4185 /* 4186 * Backenable the first queue upstream from `q' with a service procedure. 4187 */ 4188 void 4189 backenable(queue_t *q, uchar_t pri) 4190 { 4191 queue_t *nq; 4192 4193 /* 4194 * our presence might not prevent other modules in our own 4195 * stream from popping/pushing since the caller of getq might not 4196 * have a claim on the queue (some drivers do a getq on somebody 4197 * else's queue - they know that the queue itself is not going away 4198 * but the framework has to guarantee q_next in that stream.) 4199 */ 4200 claimstr(q); 4201 4202 /* find nearest back queue with service proc */ 4203 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) { 4204 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq)); 4205 } 4206 4207 if (nq) { 4208 kthread_t *freezer; 4209 /* 4210 * backenable can be called either with no locks held 4211 * or with the stream frozen (the latter occurs when a module 4212 * calls rmvq with the stream frozen.) If the stream is frozen 4213 * by the caller the caller will hold all qlocks in the stream. 4214 * Note that a frozen stream doesn't freeze a mated stream, 4215 * so we explicitly check for that. 4216 */ 4217 freezer = STREAM(q)->sd_freezer; 4218 if (freezer != curthread || STREAM(q) != STREAM(nq)) { 4219 mutex_enter(QLOCK(nq)); 4220 } 4221 #ifdef DEBUG 4222 else { 4223 ASSERT(frozenstr(q)); 4224 ASSERT(MUTEX_HELD(QLOCK(q))); 4225 ASSERT(MUTEX_HELD(QLOCK(nq))); 4226 } 4227 #endif 4228 setqback(nq, pri); 4229 qenable_locked(nq); 4230 if (freezer != curthread || STREAM(q) != STREAM(nq)) 4231 mutex_exit(QLOCK(nq)); 4232 } 4233 releasestr(q); 4234 } 4235 4236 /* 4237 * Return the appropriate errno when one of flags_to_check is set 4238 * in sd_flags. Uses the exported error routines if they are set. 4239 * Will return 0 if non error is set (or if the exported error routines 4240 * do not return an error). 4241 * 4242 * If there is both a read and write error to check we prefer the read error. 4243 * Also, give preference to recorded errno's over the error functions. 4244 * The flags that are handled are: 4245 * STPLEX return EINVAL 4246 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST) 4247 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST) 4248 * STRHUP return sd_werror 4249 * 4250 * If the caller indicates that the operation is a peek a nonpersistent error 4251 * is not cleared. 4252 */ 4253 int 4254 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek) 4255 { 4256 int32_t sd_flag = stp->sd_flag & flags_to_check; 4257 int error = 0; 4258 4259 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4260 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0); 4261 if (sd_flag & STPLEX) 4262 error = EINVAL; 4263 else if (sd_flag & STRDERR) { 4264 error = stp->sd_rerror; 4265 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) { 4266 /* 4267 * Read errors are non-persistent i.e. discarded once 4268 * returned to a non-peeking caller, 4269 */ 4270 stp->sd_rerror = 0; 4271 stp->sd_flag &= ~STRDERR; 4272 } 4273 if (error == 0 && stp->sd_rderrfunc != NULL) { 4274 int clearerr = 0; 4275 4276 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek, 4277 &clearerr); 4278 if (clearerr) { 4279 stp->sd_flag &= ~STRDERR; 4280 stp->sd_rderrfunc = NULL; 4281 } 4282 } 4283 } else if (sd_flag & STWRERR) { 4284 error = stp->sd_werror; 4285 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) { 4286 /* 4287 * Write errors are non-persistent i.e. discarded once 4288 * returned to a non-peeking caller, 4289 */ 4290 stp->sd_werror = 0; 4291 stp->sd_flag &= ~STWRERR; 4292 } 4293 if (error == 0 && stp->sd_wrerrfunc != NULL) { 4294 int clearerr = 0; 4295 4296 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek, 4297 &clearerr); 4298 if (clearerr) { 4299 stp->sd_flag &= ~STWRERR; 4300 stp->sd_wrerrfunc = NULL; 4301 } 4302 } 4303 } else if (sd_flag & STRHUP) { 4304 /* sd_werror set when STRHUP */ 4305 error = stp->sd_werror; 4306 } 4307 return (error); 4308 } 4309 4310 4311 /* 4312 * single-thread open/close/push/pop 4313 * for twisted streams also 4314 */ 4315 int 4316 strstartplumb(stdata_t *stp, int flag, int cmd) 4317 { 4318 int waited = 1; 4319 int error = 0; 4320 4321 if (STRMATED(stp)) { 4322 struct stdata *stmatep = stp->sd_mate; 4323 4324 STRLOCKMATES(stp); 4325 while (waited) { 4326 waited = 0; 4327 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4328 if ((cmd == I_POP) && 4329 (flag & (FNDELAY|FNONBLOCK))) { 4330 STRUNLOCKMATES(stp); 4331 return (EAGAIN); 4332 } 4333 waited = 1; 4334 mutex_exit(&stp->sd_lock); 4335 if (!cv_wait_sig(&stmatep->sd_monitor, 4336 &stmatep->sd_lock)) { 4337 mutex_exit(&stmatep->sd_lock); 4338 return (EINTR); 4339 } 4340 mutex_exit(&stmatep->sd_lock); 4341 STRLOCKMATES(stp); 4342 } 4343 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4344 if ((cmd == I_POP) && 4345 (flag & (FNDELAY|FNONBLOCK))) { 4346 STRUNLOCKMATES(stp); 4347 return (EAGAIN); 4348 } 4349 waited = 1; 4350 mutex_exit(&stmatep->sd_lock); 4351 if (!cv_wait_sig(&stp->sd_monitor, 4352 &stp->sd_lock)) { 4353 mutex_exit(&stp->sd_lock); 4354 return (EINTR); 4355 } 4356 mutex_exit(&stp->sd_lock); 4357 STRLOCKMATES(stp); 4358 } 4359 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4360 error = strgeterr(stp, 4361 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4362 if (error != 0) { 4363 STRUNLOCKMATES(stp); 4364 return (error); 4365 } 4366 } 4367 } 4368 stp->sd_flag |= STRPLUMB; 4369 STRUNLOCKMATES(stp); 4370 } else { 4371 mutex_enter(&stp->sd_lock); 4372 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4373 if (((cmd == I_POP) || (cmd == _I_REMOVE)) && 4374 (flag & (FNDELAY|FNONBLOCK))) { 4375 mutex_exit(&stp->sd_lock); 4376 return (EAGAIN); 4377 } 4378 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) { 4379 mutex_exit(&stp->sd_lock); 4380 return (EINTR); 4381 } 4382 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4383 error = strgeterr(stp, 4384 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4385 if (error != 0) { 4386 mutex_exit(&stp->sd_lock); 4387 return (error); 4388 } 4389 } 4390 } 4391 stp->sd_flag |= STRPLUMB; 4392 mutex_exit(&stp->sd_lock); 4393 } 4394 return (0); 4395 } 4396 4397 /* 4398 * Complete the plumbing operation associated with stream `stp'. 4399 */ 4400 void 4401 strendplumb(stdata_t *stp) 4402 { 4403 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4404 ASSERT(stp->sd_flag & STRPLUMB); 4405 stp->sd_flag &= ~STRPLUMB; 4406 cv_broadcast(&stp->sd_monitor); 4407 } 4408 4409 /* 4410 * This describes how the STREAMS framework handles synchronization 4411 * during open/push and close/pop. 4412 * The key interfaces for open and close are qprocson and qprocsoff, 4413 * respectively. While the close case in general is harder both open 4414 * have close have significant similarities. 4415 * 4416 * During close the STREAMS framework has to both ensure that there 4417 * are no stale references to the queue pair (and syncq) that 4418 * are being closed and also provide the guarantees that are documented 4419 * in qprocsoff(9F). 4420 * If there are stale references to the queue that is closing it can 4421 * result in kernel memory corruption or kernel panics. 4422 * 4423 * Note that is it up to the module/driver to ensure that it itself 4424 * does not have any stale references to the closing queues once its close 4425 * routine returns. This includes: 4426 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines 4427 * associated with the queues. For timeout and bufcall callbacks the 4428 * module/driver also has to ensure (or wait for) any callbacks that 4429 * are in progress. 4430 * - If the module/driver is using esballoc it has to ensure that any 4431 * esballoc free functions do not refer to a queue that has closed. 4432 * (Note that in general the close routine can not wait for the esballoc'ed 4433 * messages to be freed since that can cause a deadlock.) 4434 * - Cancelling any interrupts that refer to the closing queues and 4435 * also ensuring that there are no interrupts in progress that will 4436 * refer to the closing queues once the close routine returns. 4437 * - For multiplexors removing any driver global state that refers to 4438 * the closing queue and also ensuring that there are no threads in 4439 * the multiplexor that has picked up a queue pointer but not yet 4440 * finished using it. 4441 * 4442 * In addition, a driver/module can only reference the q_next pointer 4443 * in its open, close, put, or service procedures or in a 4444 * qtimeout/qbufcall callback procedure executing "on" the correct 4445 * stream. Thus it can not reference the q_next pointer in an interrupt 4446 * routine or a timeout, bufcall or esballoc callback routine. Likewise 4447 * it can not reference q_next of a different queue e.g. in a mux that 4448 * passes messages from one queues put/service procedure to another queue. 4449 * In all the cases when the driver/module can not access the q_next 4450 * field it must use the *next* versions e.g. canputnext instead of 4451 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...). 4452 * 4453 * 4454 * Assuming that the driver/module conforms to the above constraints 4455 * the STREAMS framework has to avoid stale references to q_next for all 4456 * the framework internal cases which include (but are not limited to): 4457 * - Threads in canput/canputnext/backenable and elsewhere that are 4458 * walking q_next. 4459 * - Messages on a syncq that have a reference to the queue through b_queue. 4460 * - Messages on an outer perimeter (syncq) that have a reference to the 4461 * queue through b_queue. 4462 * - Threads that use q_nfsrv (e.g. canput) to find a queue. 4463 * Note that only canput and bcanput use q_nfsrv without any locking. 4464 * 4465 * The STREAMS framework providing the qprocsoff(9F) guarantees means that 4466 * after qprocsoff returns, the framework has to ensure that no threads can 4467 * enter the put or service routines for the closing read or write-side queue. 4468 * In addition to preventing "direct" entry into the put procedures 4469 * the framework also has to prevent messages being drained from 4470 * the syncq or the outer perimeter. 4471 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only 4472 * mechanism to prevent qwriter(PERIM_OUTER) from running after 4473 * qprocsoff has returned. 4474 * Note that if a module/driver uses put(9F) on one of its own queues 4475 * it is up to the module/driver to ensure that the put() doesn't 4476 * get called when the queue is closing. 4477 * 4478 * 4479 * The framework aspects of the above "contract" is implemented by 4480 * qprocsoff, removeq, and strlock: 4481 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from 4482 * entering the service procedures. 4483 * - strlock acquires the sd_lock and sd_reflock to prevent putnext, 4484 * canputnext, backenable etc from dereferencing the q_next that will 4485 * soon change. 4486 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext 4487 * or other q_next walker that uses claimstr/releasestr to finish. 4488 * - optionally for every syncq in the stream strlock acquires all the 4489 * sq_lock's and waits for all sq_counts to drop to a value that indicates 4490 * that no thread executes in the put or service procedures and that no 4491 * thread is draining into the module/driver. This ensures that no 4492 * open, close, put, service, or qtimeout/qbufcall callback procedure is 4493 * currently executing hence no such thread can end up with the old stale 4494 * q_next value and no canput/backenable can have the old stale 4495 * q_nfsrv/q_next. 4496 * - qdetach (wait_svc) makes sure that any scheduled or running threads 4497 * have either finished or observed the QWCLOSE flag and gone away. 4498 */ 4499 4500 4501 /* 4502 * Get all the locks necessary to change q_next. 4503 * 4504 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the 4505 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that 4506 * the only threads inside the sqncq are threads currently calling removeq(). 4507 * Since threads calling removeq() are in the process of removing their queues 4508 * from the stream, we do not need to worry about them accessing a stale q_next 4509 * pointer and thus we do not need to wait for them to exit (in fact, waiting 4510 * for them can cause deadlock). 4511 * 4512 * This routine is subject to starvation since it does not set any flag to 4513 * prevent threads from entering a module in the stream(i.e. sq_count can 4514 * increase on some syncq while it is waiting on some other syncq.) 4515 * 4516 * Assumes that only one thread attempts to call strlock for a given 4517 * stream. If this is not the case the two threads would deadlock. 4518 * This assumption is guaranteed since strlock is only called by insertq 4519 * and removeq and streams plumbing changes are single-threaded for 4520 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags. 4521 * 4522 * For pipes, it is not difficult to atomically designate a pair of streams 4523 * to be mated. Once mated atomically by the framework the twisted pair remain 4524 * configured that way until dismantled atomically by the framework. 4525 * When plumbing takes place on a twisted stream it is necessary to ensure that 4526 * this operation is done exclusively on the twisted stream since two such 4527 * operations, each initiated on different ends of the pipe will deadlock 4528 * waiting for each other to complete. 4529 * 4530 * On entry, no locks should be held. 4531 * The locks acquired and held by strlock depends on a few factors. 4532 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired 4533 * and held on exit and all sq_count are at an acceptable level. 4534 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with 4535 * sd_refcnt being zero. 4536 */ 4537 4538 static void 4539 strlock(struct stdata *stp, sqlist_t *sqlist) 4540 { 4541 syncql_t *sql, *sql2; 4542 retry: 4543 /* 4544 * Wait for any claimstr to go away. 4545 */ 4546 if (STRMATED(stp)) { 4547 struct stdata *stp1, *stp2; 4548 4549 STRLOCKMATES(stp); 4550 /* 4551 * Note that the selection of locking order is not 4552 * important, just that they are always aquired in 4553 * the same order. To assure this, we choose this 4554 * order based on the value of the pointer, and since 4555 * the pointer will not change for the life of this 4556 * pair, we will always grab the locks in the same 4557 * order (and hence, prevent deadlocks). 4558 */ 4559 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) { 4560 stp1 = stp; 4561 stp2 = stp->sd_mate; 4562 } else { 4563 stp2 = stp; 4564 stp1 = stp->sd_mate; 4565 } 4566 mutex_enter(&stp1->sd_reflock); 4567 if (stp1->sd_refcnt > 0) { 4568 STRUNLOCKMATES(stp); 4569 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock); 4570 mutex_exit(&stp1->sd_reflock); 4571 goto retry; 4572 } 4573 mutex_enter(&stp2->sd_reflock); 4574 if (stp2->sd_refcnt > 0) { 4575 STRUNLOCKMATES(stp); 4576 mutex_exit(&stp1->sd_reflock); 4577 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock); 4578 mutex_exit(&stp2->sd_reflock); 4579 goto retry; 4580 } 4581 STREAM_PUTLOCKS_ENTER(stp1); 4582 STREAM_PUTLOCKS_ENTER(stp2); 4583 } else { 4584 mutex_enter(&stp->sd_lock); 4585 mutex_enter(&stp->sd_reflock); 4586 while (stp->sd_refcnt > 0) { 4587 mutex_exit(&stp->sd_lock); 4588 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock); 4589 if (mutex_tryenter(&stp->sd_lock) == 0) { 4590 mutex_exit(&stp->sd_reflock); 4591 mutex_enter(&stp->sd_lock); 4592 mutex_enter(&stp->sd_reflock); 4593 } 4594 } 4595 STREAM_PUTLOCKS_ENTER(stp); 4596 } 4597 4598 if (sqlist == NULL) 4599 return; 4600 4601 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4602 syncq_t *sq = sql->sql_sq; 4603 uint16_t count; 4604 4605 mutex_enter(SQLOCK(sq)); 4606 count = sq->sq_count; 4607 ASSERT(sq->sq_rmqcount <= count); 4608 SQ_PUTLOCKS_ENTER(sq); 4609 SUM_SQ_PUTCOUNTS(sq, count); 4610 if (count == sq->sq_rmqcount) 4611 continue; 4612 4613 /* Failed - drop all locks that we have acquired so far */ 4614 if (STRMATED(stp)) { 4615 STREAM_PUTLOCKS_EXIT(stp); 4616 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4617 STRUNLOCKMATES(stp); 4618 mutex_exit(&stp->sd_reflock); 4619 mutex_exit(&stp->sd_mate->sd_reflock); 4620 } else { 4621 STREAM_PUTLOCKS_EXIT(stp); 4622 mutex_exit(&stp->sd_lock); 4623 mutex_exit(&stp->sd_reflock); 4624 } 4625 for (sql2 = sqlist->sqlist_head; sql2 != sql; 4626 sql2 = sql2->sql_next) { 4627 SQ_PUTLOCKS_EXIT(sql2->sql_sq); 4628 mutex_exit(SQLOCK(sql2->sql_sq)); 4629 } 4630 4631 /* 4632 * The wait loop below may starve when there are many threads 4633 * claiming the syncq. This is especially a problem with permod 4634 * syncqs (IP). To lessen the impact of the problem we increment 4635 * sq_needexcl and clear fastbits so that putnexts will slow 4636 * down and call sqenable instead of draining right away. 4637 */ 4638 sq->sq_needexcl++; 4639 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4640 while (count > sq->sq_rmqcount) { 4641 sq->sq_flags |= SQ_WANTWAKEUP; 4642 SQ_PUTLOCKS_EXIT(sq); 4643 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4644 count = sq->sq_count; 4645 SQ_PUTLOCKS_ENTER(sq); 4646 SUM_SQ_PUTCOUNTS(sq, count); 4647 } 4648 sq->sq_needexcl--; 4649 if (sq->sq_needexcl == 0) 4650 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4651 SQ_PUTLOCKS_EXIT(sq); 4652 ASSERT(count == sq->sq_rmqcount); 4653 mutex_exit(SQLOCK(sq)); 4654 goto retry; 4655 } 4656 } 4657 4658 /* 4659 * Drop all the locks that strlock acquired. 4660 */ 4661 static void 4662 strunlock(struct stdata *stp, sqlist_t *sqlist) 4663 { 4664 syncql_t *sql; 4665 4666 if (STRMATED(stp)) { 4667 STREAM_PUTLOCKS_EXIT(stp); 4668 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4669 STRUNLOCKMATES(stp); 4670 mutex_exit(&stp->sd_reflock); 4671 mutex_exit(&stp->sd_mate->sd_reflock); 4672 } else { 4673 STREAM_PUTLOCKS_EXIT(stp); 4674 mutex_exit(&stp->sd_lock); 4675 mutex_exit(&stp->sd_reflock); 4676 } 4677 4678 if (sqlist == NULL) 4679 return; 4680 4681 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4682 SQ_PUTLOCKS_EXIT(sql->sql_sq); 4683 mutex_exit(SQLOCK(sql->sql_sq)); 4684 } 4685 } 4686 4687 /* 4688 * When the module has service procedure, we need check if the next 4689 * module which has service procedure is in flow control to trigger 4690 * the backenable. 4691 */ 4692 static void 4693 backenable_insertedq(queue_t *q) 4694 { 4695 qband_t *qbp; 4696 4697 claimstr(q); 4698 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) { 4699 if (q->q_next->q_nfsrv->q_flag & QWANTW) 4700 backenable(q, 0); 4701 4702 qbp = q->q_next->q_nfsrv->q_bandp; 4703 for (; qbp != NULL; qbp = qbp->qb_next) 4704 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL) 4705 backenable(q, qbp->qb_first->b_band); 4706 } 4707 releasestr(q); 4708 } 4709 4710 /* 4711 * Given two read queues, insert a new single one after another. 4712 * 4713 * This routine acquires all the necessary locks in order to change 4714 * q_next and related pointer using strlock(). 4715 * It depends on the stream head ensuring that there are no concurrent 4716 * insertq or removeq on the same stream. The stream head ensures this 4717 * using the flags STWOPEN, STRCLOSE, and STRPLUMB. 4718 * 4719 * Note that no syncq locks are held during the q_next change. This is 4720 * applied to all streams since, unlike removeq, there is no problem of stale 4721 * pointers when adding a module to the stream. Thus drivers/modules that do a 4722 * canput(rq->q_next) would never get a closed/freed queue pointer even if we 4723 * applied this optimization to all streams. 4724 */ 4725 void 4726 insertq(struct stdata *stp, queue_t *new) 4727 { 4728 queue_t *after; 4729 queue_t *wafter; 4730 queue_t *wnew = _WR(new); 4731 boolean_t have_fifo = B_FALSE; 4732 4733 if (new->q_flag & _QINSERTING) { 4734 ASSERT(stp->sd_vnode->v_type != VFIFO); 4735 after = new->q_next; 4736 wafter = _WR(new->q_next); 4737 } else { 4738 after = _RD(stp->sd_wrq); 4739 wafter = stp->sd_wrq; 4740 } 4741 4742 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ, 4743 "insertq:%p, %p", after, new); 4744 ASSERT(after->q_flag & QREADR); 4745 ASSERT(new->q_flag & QREADR); 4746 4747 strlock(stp, NULL); 4748 4749 /* Do we have a FIFO? */ 4750 if (wafter->q_next == after) { 4751 have_fifo = B_TRUE; 4752 wnew->q_next = new; 4753 } else { 4754 wnew->q_next = wafter->q_next; 4755 } 4756 new->q_next = after; 4757 4758 set_nfsrv_ptr(new, wnew, after, wafter); 4759 /* 4760 * set_nfsrv_ptr() needs to know if this is an insertion or not, 4761 * so only reset this flag after calling it. 4762 */ 4763 new->q_flag &= ~_QINSERTING; 4764 4765 if (have_fifo) { 4766 wafter->q_next = wnew; 4767 } else { 4768 if (wafter->q_next) 4769 _OTHERQ(wafter->q_next)->q_next = new; 4770 wafter->q_next = wnew; 4771 } 4772 4773 set_qend(new); 4774 /* The QEND flag might have to be updated for the upstream guy */ 4775 set_qend(after); 4776 4777 ASSERT(_SAMESTR(new) == O_SAMESTR(new)); 4778 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew)); 4779 ASSERT(_SAMESTR(after) == O_SAMESTR(after)); 4780 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter)); 4781 strsetuio(stp); 4782 4783 /* 4784 * If this was a module insertion, bump the push count. 4785 */ 4786 if (!(new->q_flag & QISDRV)) 4787 stp->sd_pushcnt++; 4788 4789 strunlock(stp, NULL); 4790 4791 /* check if the write Q needs backenable */ 4792 backenable_insertedq(wnew); 4793 4794 /* check if the read Q needs backenable */ 4795 backenable_insertedq(new); 4796 } 4797 4798 /* 4799 * Given a read queue, unlink it from any neighbors. 4800 * 4801 * This routine acquires all the necessary locks in order to 4802 * change q_next and related pointers and also guard against 4803 * stale references (e.g. through q_next) to the queue that 4804 * is being removed. It also plays part of the role in ensuring 4805 * that the module's/driver's put procedure doesn't get called 4806 * after qprocsoff returns. 4807 * 4808 * Removeq depends on the stream head ensuring that there are 4809 * no concurrent insertq or removeq on the same stream. The 4810 * stream head ensures this using the flags STWOPEN, STRCLOSE and 4811 * STRPLUMB. 4812 * 4813 * The set of locks needed to remove the queue is different in 4814 * different cases: 4815 * 4816 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after 4817 * waiting for the syncq reference count to drop to 0 indicating that no 4818 * non-close threads are present anywhere in the stream. This ensures that any 4819 * module/driver can reference q_next in its open, close, put, or service 4820 * procedures. 4821 * 4822 * The sq_rmqcount counter tracks the number of threads inside removeq(). 4823 * strlock() ensures that there is either no threads executing inside perimeter 4824 * or there is only a thread calling qprocsoff(). 4825 * 4826 * strlock() compares the value of sq_count with the number of threads inside 4827 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup 4828 * any threads waiting in strlock() when the sq_rmqcount increases. 4829 */ 4830 4831 void 4832 removeq(queue_t *qp) 4833 { 4834 queue_t *wqp = _WR(qp); 4835 struct stdata *stp = STREAM(qp); 4836 sqlist_t *sqlist = NULL; 4837 boolean_t isdriver; 4838 int moved; 4839 syncq_t *sq = qp->q_syncq; 4840 syncq_t *wsq = wqp->q_syncq; 4841 4842 ASSERT(stp); 4843 4844 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ, 4845 "removeq:%p %p", qp, wqp); 4846 ASSERT(qp->q_flag&QREADR); 4847 4848 /* 4849 * For queues using Synchronous streams, we must wait for all threads in 4850 * rwnext() to drain out before proceeding. 4851 */ 4852 if (qp->q_flag & QSYNCSTR) { 4853 /* First, we need wakeup any threads blocked in rwnext() */ 4854 mutex_enter(SQLOCK(sq)); 4855 if (sq->sq_flags & SQ_WANTWAKEUP) { 4856 sq->sq_flags &= ~SQ_WANTWAKEUP; 4857 cv_broadcast(&sq->sq_wait); 4858 } 4859 mutex_exit(SQLOCK(sq)); 4860 4861 if (wsq != sq) { 4862 mutex_enter(SQLOCK(wsq)); 4863 if (wsq->sq_flags & SQ_WANTWAKEUP) { 4864 wsq->sq_flags &= ~SQ_WANTWAKEUP; 4865 cv_broadcast(&wsq->sq_wait); 4866 } 4867 mutex_exit(SQLOCK(wsq)); 4868 } 4869 4870 mutex_enter(QLOCK(qp)); 4871 while (qp->q_rwcnt > 0) { 4872 qp->q_flag |= QWANTRMQSYNC; 4873 cv_wait(&qp->q_wait, QLOCK(qp)); 4874 } 4875 mutex_exit(QLOCK(qp)); 4876 4877 mutex_enter(QLOCK(wqp)); 4878 while (wqp->q_rwcnt > 0) { 4879 wqp->q_flag |= QWANTRMQSYNC; 4880 cv_wait(&wqp->q_wait, QLOCK(wqp)); 4881 } 4882 mutex_exit(QLOCK(wqp)); 4883 } 4884 4885 mutex_enter(SQLOCK(sq)); 4886 sq->sq_rmqcount++; 4887 if (sq->sq_flags & SQ_WANTWAKEUP) { 4888 sq->sq_flags &= ~SQ_WANTWAKEUP; 4889 cv_broadcast(&sq->sq_wait); 4890 } 4891 mutex_exit(SQLOCK(sq)); 4892 4893 isdriver = (qp->q_flag & QISDRV); 4894 4895 sqlist = sqlist_build(qp, stp, STRMATED(stp)); 4896 strlock(stp, sqlist); 4897 4898 reset_nfsrv_ptr(qp, wqp); 4899 4900 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp); 4901 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp); 4902 /* Do we have a FIFO? */ 4903 if (wqp->q_next == qp) { 4904 stp->sd_wrq->q_next = _RD(stp->sd_wrq); 4905 } else { 4906 if (wqp->q_next) 4907 backq(qp)->q_next = qp->q_next; 4908 if (qp->q_next) 4909 backq(wqp)->q_next = wqp->q_next; 4910 } 4911 4912 /* The QEND flag might have to be updated for the upstream guy */ 4913 if (qp->q_next) 4914 set_qend(qp->q_next); 4915 4916 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq)); 4917 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq))); 4918 4919 /* 4920 * Move any messages destined for the put procedures to the next 4921 * syncq in line. Otherwise free them. 4922 */ 4923 moved = 0; 4924 /* 4925 * Quick check to see whether there are any messages or events. 4926 */ 4927 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS)) 4928 moved += propagate_syncq(qp); 4929 if (wqp->q_syncqmsgs != 0 || 4930 (wqp->q_syncq->sq_flags & SQ_EVENTS)) 4931 moved += propagate_syncq(wqp); 4932 4933 strsetuio(stp); 4934 4935 /* 4936 * If this was a module removal, decrement the push count. 4937 */ 4938 if (!isdriver) 4939 stp->sd_pushcnt--; 4940 4941 strunlock(stp, sqlist); 4942 sqlist_free(sqlist); 4943 4944 /* 4945 * Make sure any messages that were propagated are drained. 4946 * Also clear any QFULL bit caused by messages that were propagated. 4947 */ 4948 4949 if (qp->q_next != NULL) { 4950 clr_qfull(qp); 4951 /* 4952 * For the driver calling qprocsoff, propagate_syncq 4953 * frees all the messages instead of putting it in 4954 * the stream head 4955 */ 4956 if (!isdriver && (moved > 0)) 4957 emptysq(qp->q_next->q_syncq); 4958 } 4959 if (wqp->q_next != NULL) { 4960 clr_qfull(wqp); 4961 /* 4962 * We come here for any pop of a module except for the 4963 * case of driver being removed. We don't call emptysq 4964 * if we did not move any messages. This will avoid holding 4965 * PERMOD syncq locks in emptysq 4966 */ 4967 if (moved > 0) 4968 emptysq(wqp->q_next->q_syncq); 4969 } 4970 4971 mutex_enter(SQLOCK(sq)); 4972 sq->sq_rmqcount--; 4973 mutex_exit(SQLOCK(sq)); 4974 } 4975 4976 /* 4977 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or 4978 * SQ_WRITER) on a syncq. 4979 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the 4980 * sync queue and waits until sq_count reaches maxcnt. 4981 * 4982 * if maxcnt is -1 there's no need to grab sq_putlocks since the caller 4983 * does not care about putnext threads that are in the middle of calling put 4984 * entry points. 4985 * 4986 * This routine is used for both inner and outer syncqs. 4987 */ 4988 static void 4989 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) 4990 { 4991 uint16_t count = 0; 4992 4993 mutex_enter(SQLOCK(sq)); 4994 /* 4995 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset. 4996 * SQ_FROZEN will be set if there is a frozen stream that has a 4997 * queue which also refers to this "shared" syncq. 4998 * SQ_BLOCKED will be set if there is "off" queue which also 4999 * refers to this "shared" syncq. 5000 */ 5001 if (maxcnt != -1) { 5002 count = sq->sq_count; 5003 SQ_PUTLOCKS_ENTER(sq); 5004 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5005 SUM_SQ_PUTCOUNTS(sq, count); 5006 } 5007 sq->sq_needexcl++; 5008 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5009 5010 while ((sq->sq_flags & flag) || 5011 (maxcnt != -1 && count > (unsigned)maxcnt)) { 5012 sq->sq_flags |= SQ_WANTWAKEUP; 5013 if (maxcnt != -1) { 5014 SQ_PUTLOCKS_EXIT(sq); 5015 } 5016 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5017 if (maxcnt != -1) { 5018 count = sq->sq_count; 5019 SQ_PUTLOCKS_ENTER(sq); 5020 SUM_SQ_PUTCOUNTS(sq, count); 5021 } 5022 } 5023 sq->sq_needexcl--; 5024 sq->sq_flags |= flag; 5025 ASSERT(maxcnt == -1 || count == maxcnt); 5026 if (maxcnt != -1) { 5027 if (sq->sq_needexcl == 0) { 5028 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5029 } 5030 SQ_PUTLOCKS_EXIT(sq); 5031 } else if (sq->sq_needexcl == 0) { 5032 SQ_PUTCOUNT_SETFAST(sq); 5033 } 5034 5035 mutex_exit(SQLOCK(sq)); 5036 } 5037 5038 /* 5039 * Reset a flag that was set with blocksq. 5040 * 5041 * Can not use this routine to reset SQ_WRITER. 5042 * 5043 * If "isouter" is set then the syncq is assumed to be an outer perimeter 5044 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread 5045 * to handle the queued qwriter operations. 5046 * 5047 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5048 * sq_putlocks are used. 5049 */ 5050 static void 5051 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) 5052 { 5053 uint16_t flags; 5054 5055 mutex_enter(SQLOCK(sq)); 5056 ASSERT(resetflag != SQ_WRITER); 5057 ASSERT(sq->sq_flags & resetflag); 5058 flags = sq->sq_flags & ~resetflag; 5059 sq->sq_flags = flags; 5060 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) { 5061 if (flags & SQ_WANTWAKEUP) { 5062 flags &= ~SQ_WANTWAKEUP; 5063 cv_broadcast(&sq->sq_wait); 5064 } 5065 sq->sq_flags = flags; 5066 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5067 if (!isouter) { 5068 /* drain_syncq drops SQLOCK */ 5069 drain_syncq(sq); 5070 return; 5071 } 5072 } 5073 } 5074 mutex_exit(SQLOCK(sq)); 5075 } 5076 5077 /* 5078 * Reset a flag that was set with blocksq. 5079 * Does not drain the syncq. Use emptysq() for that. 5080 * Returns 1 if SQ_QUEUED is set. Otherwise 0. 5081 * 5082 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5083 * sq_putlocks are used. 5084 */ 5085 static int 5086 dropsq(syncq_t *sq, uint16_t resetflag) 5087 { 5088 uint16_t flags; 5089 5090 mutex_enter(SQLOCK(sq)); 5091 ASSERT(sq->sq_flags & resetflag); 5092 flags = sq->sq_flags & ~resetflag; 5093 if (flags & SQ_WANTWAKEUP) { 5094 flags &= ~SQ_WANTWAKEUP; 5095 cv_broadcast(&sq->sq_wait); 5096 } 5097 sq->sq_flags = flags; 5098 mutex_exit(SQLOCK(sq)); 5099 if (flags & SQ_QUEUED) 5100 return (1); 5101 return (0); 5102 } 5103 5104 /* 5105 * Empty all the messages on a syncq. 5106 * 5107 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5108 * sq_putlocks are used. 5109 */ 5110 static void 5111 emptysq(syncq_t *sq) 5112 { 5113 uint16_t flags; 5114 5115 mutex_enter(SQLOCK(sq)); 5116 flags = sq->sq_flags; 5117 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5118 /* 5119 * To prevent potential recursive invocation of drain_syncq we 5120 * do not call drain_syncq if count is non-zero. 5121 */ 5122 if (sq->sq_count == 0) { 5123 /* drain_syncq() drops SQLOCK */ 5124 drain_syncq(sq); 5125 return; 5126 } else 5127 sqenable(sq); 5128 } 5129 mutex_exit(SQLOCK(sq)); 5130 } 5131 5132 /* 5133 * Ordered insert while removing duplicates. 5134 */ 5135 static void 5136 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp) 5137 { 5138 syncql_t *sqlp, **prev_sqlpp, *new_sqlp; 5139 5140 prev_sqlpp = &sqlist->sqlist_head; 5141 while ((sqlp = *prev_sqlpp) != NULL) { 5142 if (sqlp->sql_sq >= sqp) { 5143 if (sqlp->sql_sq == sqp) /* duplicate */ 5144 return; 5145 break; 5146 } 5147 prev_sqlpp = &sqlp->sql_next; 5148 } 5149 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++]; 5150 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size); 5151 new_sqlp->sql_next = sqlp; 5152 new_sqlp->sql_sq = sqp; 5153 *prev_sqlpp = new_sqlp; 5154 } 5155 5156 /* 5157 * Walk the write side queues until we hit either the driver 5158 * or a twist in the stream (_SAMESTR will return false in both 5159 * these cases) then turn around and walk the read side queues 5160 * back up to the stream head. 5161 */ 5162 static void 5163 sqlist_insertall(sqlist_t *sqlist, queue_t *q) 5164 { 5165 while (q != NULL) { 5166 sqlist_insert(sqlist, q->q_syncq); 5167 5168 if (_SAMESTR(q)) 5169 q = q->q_next; 5170 else if (!(q->q_flag & QREADR)) 5171 q = _RD(q); 5172 else 5173 q = NULL; 5174 } 5175 } 5176 5177 /* 5178 * Allocate and build a list of all syncqs in a stream and the syncq(s) 5179 * associated with the "q" parameter. The resulting list is sorted in a 5180 * canonical order and is free of duplicates. 5181 * Assumes the passed queue is a _RD(q). 5182 */ 5183 static sqlist_t * 5184 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist) 5185 { 5186 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP); 5187 5188 /* 5189 * start with the current queue/qpair 5190 */ 5191 ASSERT(q->q_flag & QREADR); 5192 5193 sqlist_insert(sqlist, q->q_syncq); 5194 sqlist_insert(sqlist, _WR(q)->q_syncq); 5195 5196 sqlist_insertall(sqlist, stp->sd_wrq); 5197 if (do_twist) 5198 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq); 5199 5200 return (sqlist); 5201 } 5202 5203 static sqlist_t * 5204 sqlist_alloc(struct stdata *stp, int kmflag) 5205 { 5206 size_t sqlist_size; 5207 sqlist_t *sqlist; 5208 5209 /* 5210 * Allocate 2 syncql_t's for each pushed module. Note that 5211 * the sqlist_t structure already has 4 syncql_t's built in: 5212 * 2 for the stream head, and 2 for the driver/other stream head. 5213 */ 5214 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt + 5215 sizeof (sqlist_t); 5216 if (STRMATED(stp)) 5217 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt; 5218 sqlist = kmem_alloc(sqlist_size, kmflag); 5219 5220 sqlist->sqlist_head = NULL; 5221 sqlist->sqlist_size = sqlist_size; 5222 sqlist->sqlist_index = 0; 5223 5224 return (sqlist); 5225 } 5226 5227 /* 5228 * Free the list created by sqlist_alloc() 5229 */ 5230 static void 5231 sqlist_free(sqlist_t *sqlist) 5232 { 5233 kmem_free(sqlist, sqlist->sqlist_size); 5234 } 5235 5236 /* 5237 * Prevent any new entries into any syncq in this stream. 5238 * Used by freezestr. 5239 */ 5240 void 5241 strblock(queue_t *q) 5242 { 5243 struct stdata *stp; 5244 syncql_t *sql; 5245 sqlist_t *sqlist; 5246 5247 q = _RD(q); 5248 5249 stp = STREAM(q); 5250 ASSERT(stp != NULL); 5251 5252 /* 5253 * Get a sorted list with all the duplicates removed containing 5254 * all the syncqs referenced by this stream. 5255 */ 5256 sqlist = sqlist_build(q, stp, B_FALSE); 5257 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5258 blocksq(sql->sql_sq, SQ_FROZEN, -1); 5259 sqlist_free(sqlist); 5260 } 5261 5262 /* 5263 * Release the block on new entries into this stream 5264 */ 5265 void 5266 strunblock(queue_t *q) 5267 { 5268 struct stdata *stp; 5269 syncql_t *sql; 5270 sqlist_t *sqlist; 5271 int drain_needed; 5272 5273 q = _RD(q); 5274 5275 /* 5276 * Get a sorted list with all the duplicates removed containing 5277 * all the syncqs referenced by this stream. 5278 * Have to drop the SQ_FROZEN flag on all the syncqs before 5279 * starting to drain them; otherwise the draining might 5280 * cause a freezestr in some module on the stream (which 5281 * would deadlock.) 5282 */ 5283 stp = STREAM(q); 5284 ASSERT(stp != NULL); 5285 sqlist = sqlist_build(q, stp, B_FALSE); 5286 drain_needed = 0; 5287 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5288 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN); 5289 if (drain_needed) { 5290 for (sql = sqlist->sqlist_head; sql != NULL; 5291 sql = sql->sql_next) 5292 emptysq(sql->sql_sq); 5293 } 5294 sqlist_free(sqlist); 5295 } 5296 5297 #ifdef DEBUG 5298 static int 5299 qprocsareon(queue_t *rq) 5300 { 5301 if (rq->q_next == NULL) 5302 return (0); 5303 return (_WR(rq->q_next)->q_next == _WR(rq)); 5304 } 5305 5306 int 5307 qclaimed(queue_t *q) 5308 { 5309 uint_t count; 5310 5311 count = q->q_syncq->sq_count; 5312 SUM_SQ_PUTCOUNTS(q->q_syncq, count); 5313 return (count != 0); 5314 } 5315 5316 /* 5317 * Check if anyone has frozen this stream with freezestr 5318 */ 5319 int 5320 frozenstr(queue_t *q) 5321 { 5322 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0); 5323 } 5324 #endif /* DEBUG */ 5325 5326 /* 5327 * Enter a queue. 5328 * Obsoleted interface. Should not be used. 5329 */ 5330 void 5331 enterq(queue_t *q) 5332 { 5333 entersq(q->q_syncq, SQ_CALLBACK); 5334 } 5335 5336 void 5337 leaveq(queue_t *q) 5338 { 5339 leavesq(q->q_syncq, SQ_CALLBACK); 5340 } 5341 5342 /* 5343 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits 5344 * to check. 5345 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter 5346 * calls and the running of open, close and service procedures. 5347 * 5348 * if c_inner bit is set no need to grab sq_putlocks since we don't care 5349 * if other threads have entered or are entering put entry point. 5350 * 5351 * if c_inner bit is set it might have been posible to use 5352 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize 5353 * open/close path for IP) but since the count may need to be decremented in 5354 * qwait() we wouldn't know which counter to decrement. Currently counter is 5355 * selected by current cpu_seqid and current CPU can change at any moment. XXX 5356 * in the future we might use curthread id bits to select the counter and this 5357 * would stay constant across routine calls. 5358 */ 5359 void 5360 entersq(syncq_t *sq, int entrypoint) 5361 { 5362 uint16_t count = 0; 5363 uint16_t flags; 5364 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 5365 uint16_t type; 5366 uint_t c_inner = entrypoint & SQ_CI; 5367 uint_t c_outer = entrypoint & SQ_CO; 5368 5369 /* 5370 * Increment ref count to keep closes out of this queue. 5371 */ 5372 ASSERT(sq); 5373 ASSERT(c_inner && c_outer); 5374 mutex_enter(SQLOCK(sq)); 5375 flags = sq->sq_flags; 5376 type = sq->sq_type; 5377 if (!(type & c_inner)) { 5378 /* Make sure all putcounts now use slowlock. */ 5379 count = sq->sq_count; 5380 SQ_PUTLOCKS_ENTER(sq); 5381 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5382 SUM_SQ_PUTCOUNTS(sq, count); 5383 sq->sq_needexcl++; 5384 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5385 waitflags |= SQ_MESSAGES; 5386 } 5387 /* 5388 * Wait until we can enter the inner perimeter. 5389 * If we want exclusive access we wait until sq_count is 0. 5390 * We have to do this before entering the outer perimeter in order 5391 * to preserve put/close message ordering. 5392 */ 5393 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) { 5394 sq->sq_flags = flags | SQ_WANTWAKEUP; 5395 if (!(type & c_inner)) { 5396 SQ_PUTLOCKS_EXIT(sq); 5397 } 5398 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5399 if (!(type & c_inner)) { 5400 count = sq->sq_count; 5401 SQ_PUTLOCKS_ENTER(sq); 5402 SUM_SQ_PUTCOUNTS(sq, count); 5403 } 5404 flags = sq->sq_flags; 5405 } 5406 5407 if (!(type & c_inner)) { 5408 ASSERT(sq->sq_needexcl > 0); 5409 sq->sq_needexcl--; 5410 if (sq->sq_needexcl == 0) { 5411 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5412 } 5413 } 5414 5415 /* Check if we need to enter the outer perimeter */ 5416 if (!(type & c_outer)) { 5417 /* 5418 * We have to enter the outer perimeter exclusively before 5419 * we can increment sq_count to avoid deadlock. This implies 5420 * that we have to re-check sq_flags and sq_count. 5421 * 5422 * is it possible to have c_inner set when c_outer is not set? 5423 */ 5424 if (!(type & c_inner)) { 5425 SQ_PUTLOCKS_EXIT(sq); 5426 } 5427 mutex_exit(SQLOCK(sq)); 5428 outer_enter(sq->sq_outer, SQ_GOAWAY); 5429 mutex_enter(SQLOCK(sq)); 5430 flags = sq->sq_flags; 5431 /* 5432 * there should be no need to recheck sq_putcounts 5433 * because outer_enter() has already waited for them to clear 5434 * after setting SQ_WRITER. 5435 */ 5436 count = sq->sq_count; 5437 #ifdef DEBUG 5438 /* 5439 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead 5440 * of doing an ASSERT internally. Others should do 5441 * something like 5442 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0); 5443 * without the need to #ifdef DEBUG it. 5444 */ 5445 SUMCHECK_SQ_PUTCOUNTS(sq, 0); 5446 #endif 5447 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) || 5448 (!(type & c_inner) && count != 0)) { 5449 sq->sq_flags = flags | SQ_WANTWAKEUP; 5450 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5451 count = sq->sq_count; 5452 flags = sq->sq_flags; 5453 } 5454 } 5455 5456 sq->sq_count++; 5457 ASSERT(sq->sq_count != 0); /* Wraparound */ 5458 if (!(type & c_inner)) { 5459 /* Exclusive entry */ 5460 ASSERT(sq->sq_count == 1); 5461 sq->sq_flags |= SQ_EXCL; 5462 if (type & c_outer) { 5463 SQ_PUTLOCKS_EXIT(sq); 5464 } 5465 } 5466 mutex_exit(SQLOCK(sq)); 5467 } 5468 5469 /* 5470 * leave a syncq. announce to framework that closes may proceed. 5471 * c_inner and c_outer specifies which concurrency bits 5472 * to check. 5473 * 5474 * must never be called from driver or module put entry point. 5475 * 5476 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5477 * sq_putlocks are used. 5478 */ 5479 void 5480 leavesq(syncq_t *sq, int entrypoint) 5481 { 5482 uint16_t flags; 5483 uint16_t type; 5484 uint_t c_outer = entrypoint & SQ_CO; 5485 #ifdef DEBUG 5486 uint_t c_inner = entrypoint & SQ_CI; 5487 #endif 5488 5489 /* 5490 * decrement ref count, drain the syncq if possible, and wake up 5491 * any waiting close. 5492 */ 5493 ASSERT(sq); 5494 ASSERT(c_inner && c_outer); 5495 mutex_enter(SQLOCK(sq)); 5496 flags = sq->sq_flags; 5497 type = sq->sq_type; 5498 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) { 5499 5500 if (flags & SQ_WANTWAKEUP) { 5501 flags &= ~SQ_WANTWAKEUP; 5502 cv_broadcast(&sq->sq_wait); 5503 } 5504 if (flags & SQ_WANTEXWAKEUP) { 5505 flags &= ~SQ_WANTEXWAKEUP; 5506 cv_broadcast(&sq->sq_exitwait); 5507 } 5508 5509 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) { 5510 /* 5511 * The syncq needs to be drained. "Exit" the syncq 5512 * before calling drain_syncq. 5513 */ 5514 ASSERT(sq->sq_count != 0); 5515 sq->sq_count--; 5516 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5517 sq->sq_flags = flags & ~SQ_EXCL; 5518 drain_syncq(sq); 5519 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 5520 /* Check if we need to exit the outer perimeter */ 5521 /* XXX will this ever be true? */ 5522 if (!(type & c_outer)) 5523 outer_exit(sq->sq_outer); 5524 return; 5525 } 5526 } 5527 ASSERT(sq->sq_count != 0); 5528 sq->sq_count--; 5529 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5530 sq->sq_flags = flags & ~SQ_EXCL; 5531 mutex_exit(SQLOCK(sq)); 5532 5533 /* Check if we need to exit the outer perimeter */ 5534 if (!(sq->sq_type & c_outer)) 5535 outer_exit(sq->sq_outer); 5536 } 5537 5538 /* 5539 * Prevent q_next from changing in this stream by incrementing sq_count. 5540 * 5541 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5542 * sq_putlocks are used. 5543 */ 5544 void 5545 claimq(queue_t *qp) 5546 { 5547 syncq_t *sq = qp->q_syncq; 5548 5549 mutex_enter(SQLOCK(sq)); 5550 sq->sq_count++; 5551 ASSERT(sq->sq_count != 0); /* Wraparound */ 5552 mutex_exit(SQLOCK(sq)); 5553 } 5554 5555 /* 5556 * Undo claimq. 5557 * 5558 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5559 * sq_putlocks are used. 5560 */ 5561 void 5562 releaseq(queue_t *qp) 5563 { 5564 syncq_t *sq = qp->q_syncq; 5565 uint16_t flags; 5566 5567 mutex_enter(SQLOCK(sq)); 5568 ASSERT(sq->sq_count > 0); 5569 sq->sq_count--; 5570 5571 flags = sq->sq_flags; 5572 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) { 5573 if (flags & SQ_WANTWAKEUP) { 5574 flags &= ~SQ_WANTWAKEUP; 5575 cv_broadcast(&sq->sq_wait); 5576 } 5577 sq->sq_flags = flags; 5578 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5579 /* 5580 * To prevent potential recursive invocation of 5581 * drain_syncq we do not call drain_syncq if count is 5582 * non-zero. 5583 */ 5584 if (sq->sq_count == 0) { 5585 drain_syncq(sq); 5586 return; 5587 } else 5588 sqenable(sq); 5589 } 5590 } 5591 mutex_exit(SQLOCK(sq)); 5592 } 5593 5594 /* 5595 * Prevent q_next from changing in this stream by incrementing sd_refcnt. 5596 */ 5597 void 5598 claimstr(queue_t *qp) 5599 { 5600 struct stdata *stp = STREAM(qp); 5601 5602 mutex_enter(&stp->sd_reflock); 5603 stp->sd_refcnt++; 5604 ASSERT(stp->sd_refcnt != 0); /* Wraparound */ 5605 mutex_exit(&stp->sd_reflock); 5606 } 5607 5608 /* 5609 * Undo claimstr. 5610 */ 5611 void 5612 releasestr(queue_t *qp) 5613 { 5614 struct stdata *stp = STREAM(qp); 5615 5616 mutex_enter(&stp->sd_reflock); 5617 ASSERT(stp->sd_refcnt != 0); 5618 if (--stp->sd_refcnt == 0) 5619 cv_broadcast(&stp->sd_refmonitor); 5620 mutex_exit(&stp->sd_reflock); 5621 } 5622 5623 static syncq_t * 5624 new_syncq(void) 5625 { 5626 return (kmem_cache_alloc(syncq_cache, KM_SLEEP)); 5627 } 5628 5629 static void 5630 free_syncq(syncq_t *sq) 5631 { 5632 ASSERT(sq->sq_head == NULL); 5633 ASSERT(sq->sq_outer == NULL); 5634 ASSERT(sq->sq_callbpend == NULL); 5635 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || 5636 (sq->sq_onext == sq && sq->sq_oprev == sq)); 5637 5638 if (sq->sq_ciputctrl != NULL) { 5639 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 5640 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 5641 sq->sq_nciputctrl, 0); 5642 ASSERT(ciputctrl_cache != NULL); 5643 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 5644 } 5645 5646 sq->sq_tail = NULL; 5647 sq->sq_evhead = NULL; 5648 sq->sq_evtail = NULL; 5649 sq->sq_ciputctrl = NULL; 5650 sq->sq_nciputctrl = 0; 5651 sq->sq_count = 0; 5652 sq->sq_rmqcount = 0; 5653 sq->sq_callbflags = 0; 5654 sq->sq_cancelid = 0; 5655 sq->sq_next = NULL; 5656 sq->sq_needexcl = 0; 5657 sq->sq_svcflags = 0; 5658 sq->sq_nqueues = 0; 5659 sq->sq_pri = 0; 5660 sq->sq_onext = NULL; 5661 sq->sq_oprev = NULL; 5662 sq->sq_flags = 0; 5663 sq->sq_type = 0; 5664 sq->sq_servcount = 0; 5665 5666 kmem_cache_free(syncq_cache, sq); 5667 } 5668 5669 /* Outer perimeter code */ 5670 5671 /* 5672 * The outer syncq uses the fields and flags in the syncq slightly 5673 * differently from the inner syncqs. 5674 * sq_count Incremented when there are pending or running 5675 * writers at the outer perimeter to prevent the set of 5676 * inner syncqs that belong to the outer perimeter from 5677 * changing. 5678 * sq_head/tail List of deferred qwriter(OUTER) operations. 5679 * 5680 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while 5681 * inner syncqs are added to or removed from the 5682 * outer perimeter. 5683 * SQ_QUEUED sq_head/tail has messages or eventsqueued. 5684 * 5685 * SQ_WRITER A thread is currently traversing all the inner syncqs 5686 * setting the SQ_WRITER flag. 5687 */ 5688 5689 /* 5690 * Get write access at the outer perimeter. 5691 * Note that read access is done by entersq, putnext, and put by simply 5692 * incrementing sq_count in the inner syncq. 5693 * 5694 * Waits until "flags" is no longer set in the outer to prevent multiple 5695 * threads from having write access at the same time. SQ_WRITER has to be part 5696 * of "flags". 5697 * 5698 * Increases sq_count on the outer syncq to keep away outer_insert/remove 5699 * until the outer_exit is finished. 5700 * 5701 * outer_enter is vulnerable to starvation since it does not prevent new 5702 * threads from entering the inner syncqs while it is waiting for sq_count to 5703 * go to zero. 5704 */ 5705 void 5706 outer_enter(syncq_t *outer, uint16_t flags) 5707 { 5708 syncq_t *sq; 5709 int wait_needed; 5710 uint16_t count; 5711 5712 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5713 outer->sq_oprev != NULL); 5714 ASSERT(flags & SQ_WRITER); 5715 5716 retry: 5717 mutex_enter(SQLOCK(outer)); 5718 while (outer->sq_flags & flags) { 5719 outer->sq_flags |= SQ_WANTWAKEUP; 5720 cv_wait(&outer->sq_wait, SQLOCK(outer)); 5721 } 5722 5723 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5724 outer->sq_flags |= SQ_WRITER; 5725 outer->sq_count++; 5726 ASSERT(outer->sq_count != 0); /* wraparound */ 5727 wait_needed = 0; 5728 /* 5729 * Set SQ_WRITER on all the inner syncqs while holding 5730 * the SQLOCK on the outer syncq. This ensures that the changing 5731 * of SQ_WRITER is atomic under the outer SQLOCK. 5732 */ 5733 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5734 mutex_enter(SQLOCK(sq)); 5735 count = sq->sq_count; 5736 SQ_PUTLOCKS_ENTER(sq); 5737 sq->sq_flags |= SQ_WRITER; 5738 SUM_SQ_PUTCOUNTS(sq, count); 5739 if (count != 0) 5740 wait_needed = 1; 5741 SQ_PUTLOCKS_EXIT(sq); 5742 mutex_exit(SQLOCK(sq)); 5743 } 5744 mutex_exit(SQLOCK(outer)); 5745 5746 /* 5747 * Get everybody out of the syncqs sequentially. 5748 * Note that we don't actually need to aqiure the PUTLOCKS, since 5749 * we have already cleared the fastbit, and set QWRITER. By 5750 * definition, the count can not increase since putnext will 5751 * take the slowlock path (and the purpose of aquiring the 5752 * putlocks was to make sure it didn't increase while we were 5753 * waiting). 5754 * 5755 * Note that we still aquire the PUTLOCKS to be safe. 5756 */ 5757 if (wait_needed) { 5758 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5759 mutex_enter(SQLOCK(sq)); 5760 count = sq->sq_count; 5761 SQ_PUTLOCKS_ENTER(sq); 5762 SUM_SQ_PUTCOUNTS(sq, count); 5763 while (count != 0) { 5764 sq->sq_flags |= SQ_WANTWAKEUP; 5765 SQ_PUTLOCKS_EXIT(sq); 5766 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5767 count = sq->sq_count; 5768 SQ_PUTLOCKS_ENTER(sq); 5769 SUM_SQ_PUTCOUNTS(sq, count); 5770 } 5771 SQ_PUTLOCKS_EXIT(sq); 5772 mutex_exit(SQLOCK(sq)); 5773 } 5774 /* 5775 * Verify that none of the flags got set while we 5776 * were waiting for the sq_counts to drop. 5777 * If this happens we exit and retry entering the 5778 * outer perimeter. 5779 */ 5780 mutex_enter(SQLOCK(outer)); 5781 if (outer->sq_flags & (flags & ~SQ_WRITER)) { 5782 mutex_exit(SQLOCK(outer)); 5783 outer_exit(outer); 5784 goto retry; 5785 } 5786 mutex_exit(SQLOCK(outer)); 5787 } 5788 } 5789 5790 /* 5791 * Drop the write access at the outer perimeter. 5792 * Read access is dropped implicitly (by putnext, put, and leavesq) by 5793 * decrementing sq_count. 5794 */ 5795 void 5796 outer_exit(syncq_t *outer) 5797 { 5798 syncq_t *sq; 5799 int drain_needed; 5800 uint16_t flags; 5801 5802 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5803 outer->sq_oprev != NULL); 5804 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer))); 5805 5806 /* 5807 * Atomically (from the perspective of threads calling become_writer) 5808 * drop the write access at the outer perimeter by holding 5809 * SQLOCK(outer) across all the dropsq calls and the resetting of 5810 * SQ_WRITER. 5811 * This defines a locking order between the outer perimeter 5812 * SQLOCK and the inner perimeter SQLOCKs. 5813 */ 5814 mutex_enter(SQLOCK(outer)); 5815 flags = outer->sq_flags; 5816 ASSERT(outer->sq_flags & SQ_WRITER); 5817 if (flags & SQ_QUEUED) { 5818 write_now(outer); 5819 flags = outer->sq_flags; 5820 } 5821 5822 /* 5823 * sq_onext is stable since sq_count has not yet been decreased. 5824 * Reset the SQ_WRITER flags in all syncqs. 5825 * After dropping SQ_WRITER on the outer syncq we empty all the 5826 * inner syncqs. 5827 */ 5828 drain_needed = 0; 5829 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5830 drain_needed += dropsq(sq, SQ_WRITER); 5831 ASSERT(!(outer->sq_flags & SQ_QUEUED)); 5832 flags &= ~SQ_WRITER; 5833 if (drain_needed) { 5834 outer->sq_flags = flags; 5835 mutex_exit(SQLOCK(outer)); 5836 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5837 emptysq(sq); 5838 mutex_enter(SQLOCK(outer)); 5839 flags = outer->sq_flags; 5840 } 5841 if (flags & SQ_WANTWAKEUP) { 5842 flags &= ~SQ_WANTWAKEUP; 5843 cv_broadcast(&outer->sq_wait); 5844 } 5845 outer->sq_flags = flags; 5846 ASSERT(outer->sq_count > 0); 5847 outer->sq_count--; 5848 mutex_exit(SQLOCK(outer)); 5849 } 5850 5851 /* 5852 * Add another syncq to an outer perimeter. 5853 * Block out all other access to the outer perimeter while it is being 5854 * changed using blocksq. 5855 * Assumes that the caller has *not* done an outer_enter. 5856 * 5857 * Vulnerable to starvation in blocksq. 5858 */ 5859 static void 5860 outer_insert(syncq_t *outer, syncq_t *sq) 5861 { 5862 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5863 outer->sq_oprev != NULL); 5864 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 5865 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ 5866 5867 /* Get exclusive access to the outer perimeter list */ 5868 blocksq(outer, SQ_BLOCKED, 0); 5869 ASSERT(outer->sq_flags & SQ_BLOCKED); 5870 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5871 5872 mutex_enter(SQLOCK(sq)); 5873 sq->sq_outer = outer; 5874 outer->sq_onext->sq_oprev = sq; 5875 sq->sq_onext = outer->sq_onext; 5876 outer->sq_onext = sq; 5877 sq->sq_oprev = outer; 5878 mutex_exit(SQLOCK(sq)); 5879 unblocksq(outer, SQ_BLOCKED, 1); 5880 } 5881 5882 /* 5883 * Remove a syncq from an outer perimeter. 5884 * Block out all other access to the outer perimeter while it is being 5885 * changed using blocksq. 5886 * Assumes that the caller has *not* done an outer_enter. 5887 * 5888 * Vulnerable to starvation in blocksq. 5889 */ 5890 static void 5891 outer_remove(syncq_t *outer, syncq_t *sq) 5892 { 5893 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5894 outer->sq_oprev != NULL); 5895 ASSERT(sq->sq_outer == outer); 5896 5897 /* Get exclusive access to the outer perimeter list */ 5898 blocksq(outer, SQ_BLOCKED, 0); 5899 ASSERT(outer->sq_flags & SQ_BLOCKED); 5900 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5901 5902 mutex_enter(SQLOCK(sq)); 5903 sq->sq_outer = NULL; 5904 sq->sq_onext->sq_oprev = sq->sq_oprev; 5905 sq->sq_oprev->sq_onext = sq->sq_onext; 5906 sq->sq_oprev = sq->sq_onext = NULL; 5907 mutex_exit(SQLOCK(sq)); 5908 unblocksq(outer, SQ_BLOCKED, 1); 5909 } 5910 5911 /* 5912 * Queue a deferred qwriter(OUTER) callback for this outer perimeter. 5913 * If this is the first callback for this outer perimeter then add 5914 * this outer perimeter to the list of outer perimeters that 5915 * the qwriter_outer_thread will process. 5916 * 5917 * Increments sq_count in the outer syncq to prevent the membership 5918 * of the outer perimeter (in terms of inner syncqs) to change while 5919 * the callback is pending. 5920 */ 5921 static void 5922 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp) 5923 { 5924 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5925 5926 mp->b_prev = (mblk_t *)func; 5927 mp->b_queue = q; 5928 mp->b_next = NULL; 5929 outer->sq_count++; /* Decremented when dequeued */ 5930 ASSERT(outer->sq_count != 0); /* Wraparound */ 5931 if (outer->sq_evhead == NULL) { 5932 /* First message. */ 5933 outer->sq_evhead = outer->sq_evtail = mp; 5934 outer->sq_flags |= SQ_EVENTS; 5935 mutex_exit(SQLOCK(outer)); 5936 STRSTAT(qwr_outer); 5937 (void) taskq_dispatch(streams_taskq, 5938 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP); 5939 } else { 5940 ASSERT(outer->sq_flags & SQ_EVENTS); 5941 outer->sq_evtail->b_next = mp; 5942 outer->sq_evtail = mp; 5943 mutex_exit(SQLOCK(outer)); 5944 } 5945 } 5946 5947 /* 5948 * Try and upgrade to write access at the outer perimeter. If this can 5949 * not be done without blocking then queue the callback to be done 5950 * by the qwriter_outer_thread. 5951 * 5952 * This routine can only be called from put or service procedures plus 5953 * asynchronous callback routines that have properly entered to 5954 * queue (with entersq.) Thus qwriter(OUTER) assumes the caller has one claim 5955 * on the syncq associated with q. 5956 */ 5957 void 5958 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)()) 5959 { 5960 syncq_t *osq, *sq, *outer; 5961 int failed; 5962 uint16_t flags; 5963 5964 osq = q->q_syncq; 5965 outer = osq->sq_outer; 5966 if (outer == NULL) 5967 panic("qwriter(PERIM_OUTER): no outer perimeter"); 5968 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5969 outer->sq_oprev != NULL); 5970 5971 mutex_enter(SQLOCK(outer)); 5972 flags = outer->sq_flags; 5973 /* 5974 * If some thread is traversing sq_next, or if we are blocked by 5975 * outer_insert or outer_remove, or if the we already have queued 5976 * callbacks, then queue this callback for later processing. 5977 * 5978 * Also queue the qwriter for an interrupt thread in order 5979 * to reduce the time spent running at high IPL. 5980 * to identify there are events. 5981 */ 5982 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) { 5983 /* 5984 * Queue the become_writer request. 5985 * The queueing is atomic under SQLOCK(outer) in order 5986 * to synchronize with outer_exit. 5987 * queue_writer will drop the outer SQLOCK 5988 */ 5989 if (flags & SQ_BLOCKED) { 5990 /* Must set SQ_WRITER on inner perimeter */ 5991 mutex_enter(SQLOCK(osq)); 5992 osq->sq_flags |= SQ_WRITER; 5993 mutex_exit(SQLOCK(osq)); 5994 } else { 5995 if (!(flags & SQ_WRITER)) { 5996 /* 5997 * The outer could have been SQ_BLOCKED thus 5998 * SQ_WRITER might not be set on the inner. 5999 */ 6000 mutex_enter(SQLOCK(osq)); 6001 osq->sq_flags |= SQ_WRITER; 6002 mutex_exit(SQLOCK(osq)); 6003 } 6004 ASSERT(osq->sq_flags & SQ_WRITER); 6005 } 6006 queue_writer(outer, func, q, mp); 6007 return; 6008 } 6009 /* 6010 * We are half-way to exclusive access to the outer perimeter. 6011 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove 6012 * while the inner syncqs are traversed. 6013 */ 6014 outer->sq_count++; 6015 ASSERT(outer->sq_count != 0); /* wraparound */ 6016 flags |= SQ_WRITER; 6017 /* 6018 * Check if we can run the function immediately. Mark all 6019 * syncqs with the writer flag to prevent new entries into 6020 * put and service procedures. 6021 * 6022 * Set SQ_WRITER on all the inner syncqs while holding 6023 * the SQLOCK on the outer syncq. This ensures that the changing 6024 * of SQ_WRITER is atomic under the outer SQLOCK. 6025 */ 6026 failed = 0; 6027 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 6028 uint16_t count; 6029 uint_t maxcnt = (sq == osq) ? 1 : 0; 6030 6031 mutex_enter(SQLOCK(sq)); 6032 count = sq->sq_count; 6033 SQ_PUTLOCKS_ENTER(sq); 6034 SUM_SQ_PUTCOUNTS(sq, count); 6035 if (sq->sq_count > maxcnt) 6036 failed = 1; 6037 sq->sq_flags |= SQ_WRITER; 6038 SQ_PUTLOCKS_EXIT(sq); 6039 mutex_exit(SQLOCK(sq)); 6040 } 6041 if (failed) { 6042 /* 6043 * Some other thread has a read claim on the outer perimeter. 6044 * Queue the callback for deferred processing. 6045 * 6046 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER 6047 * so that other qwriter(OUTER) calls will queue their 6048 * callbacks as well. queue_writer increments sq_count so we 6049 * decrement to compensate for the our increment. 6050 * 6051 * Dropping SQ_WRITER enables the writer thread to work 6052 * on this outer perimeter. 6053 */ 6054 outer->sq_flags = flags; 6055 queue_writer(outer, func, q, mp); 6056 /* queue_writer dropper the lock */ 6057 mutex_enter(SQLOCK(outer)); 6058 ASSERT(outer->sq_count > 0); 6059 outer->sq_count--; 6060 ASSERT(outer->sq_flags & SQ_WRITER); 6061 flags = outer->sq_flags; 6062 flags &= ~SQ_WRITER; 6063 if (flags & SQ_WANTWAKEUP) { 6064 flags &= ~SQ_WANTWAKEUP; 6065 cv_broadcast(&outer->sq_wait); 6066 } 6067 outer->sq_flags = flags; 6068 mutex_exit(SQLOCK(outer)); 6069 return; 6070 } else { 6071 outer->sq_flags = flags; 6072 mutex_exit(SQLOCK(outer)); 6073 } 6074 6075 /* Can run it immediately */ 6076 (*func)(q, mp); 6077 6078 outer_exit(outer); 6079 } 6080 6081 /* 6082 * Dequeue all writer callbacks from the outer perimeter and run them. 6083 */ 6084 static void 6085 write_now(syncq_t *outer) 6086 { 6087 mblk_t *mp; 6088 queue_t *q; 6089 void (*func)(); 6090 6091 ASSERT(MUTEX_HELD(SQLOCK(outer))); 6092 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 6093 outer->sq_oprev != NULL); 6094 while ((mp = outer->sq_evhead) != NULL) { 6095 /* 6096 * queues cannot be placed on the queuelist on the outer 6097 * perimiter. 6098 */ 6099 ASSERT(!(outer->sq_flags & SQ_MESSAGES)); 6100 ASSERT((outer->sq_flags & SQ_EVENTS)); 6101 6102 outer->sq_evhead = mp->b_next; 6103 if (outer->sq_evhead == NULL) { 6104 outer->sq_evtail = NULL; 6105 outer->sq_flags &= ~SQ_EVENTS; 6106 } 6107 ASSERT(outer->sq_count != 0); 6108 outer->sq_count--; /* Incremented when enqueued. */ 6109 mutex_exit(SQLOCK(outer)); 6110 /* 6111 * Drop the message if the queue is closing. 6112 * Make sure that the queue is "claimed" when the callback 6113 * is run in order to satisfy various ASSERTs. 6114 */ 6115 q = mp->b_queue; 6116 func = (void (*)())mp->b_prev; 6117 ASSERT(func != NULL); 6118 mp->b_next = mp->b_prev = NULL; 6119 if (q->q_flag & QWCLOSE) { 6120 freemsg(mp); 6121 } else { 6122 claimq(q); 6123 (*func)(q, mp); 6124 releaseq(q); 6125 } 6126 mutex_enter(SQLOCK(outer)); 6127 } 6128 ASSERT(MUTEX_HELD(SQLOCK(outer))); 6129 } 6130 6131 /* 6132 * The list of messages on the inner syncq is effectively hashed 6133 * by destination queue. These destination queues are doubly 6134 * linked lists (hopefully) in priority order. Messages are then 6135 * put on the queue referenced by the q_sqhead/q_sqtail elements. 6136 * Additional messages are linked together by the b_next/b_prev 6137 * elements in the mblk, with (similar to putq()) the first message 6138 * having a NULL b_prev and the last message having a NULL b_next. 6139 * 6140 * Events, such as qwriter callbacks, are put onto a list in FIFO 6141 * order referenced by sq_evhead, and sq_evtail. This is a singly 6142 * linked list, and messages here MUST be processed in the order queued. 6143 */ 6144 6145 /* 6146 * Run the events on the syncq event list (sq_evhead). 6147 * Assumes there is only one claim on the syncq, it is 6148 * already exclusive (SQ_EXCL set), and the SQLOCK held. 6149 * Messages here are processed in order, with the SQ_EXCL bit 6150 * held all the way through till the last message is processed. 6151 */ 6152 void 6153 sq_run_events(syncq_t *sq) 6154 { 6155 mblk_t *bp; 6156 queue_t *qp; 6157 uint16_t flags = sq->sq_flags; 6158 void (*func)(); 6159 6160 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6161 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6162 sq->sq_oprev == NULL) || 6163 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6164 sq->sq_oprev != NULL)); 6165 6166 ASSERT(flags & SQ_EXCL); 6167 ASSERT(sq->sq_count == 1); 6168 6169 /* 6170 * We need to process all of the events on this list. It 6171 * is possible that new events will be added while we are 6172 * away processing a callback, so on every loop, we start 6173 * back at the beginning of the list. 6174 */ 6175 /* 6176 * We have to reaccess sq_evhead since there is a 6177 * possibility of a new entry while we were running 6178 * the callback. 6179 */ 6180 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { 6181 ASSERT(bp->b_queue->q_syncq == sq); 6182 ASSERT(sq->sq_flags & SQ_EVENTS); 6183 6184 qp = bp->b_queue; 6185 func = (void (*)())bp->b_prev; 6186 ASSERT(func != NULL); 6187 6188 /* 6189 * Messages from the event queue must be taken off in 6190 * FIFO order. 6191 */ 6192 ASSERT(sq->sq_evhead == bp); 6193 sq->sq_evhead = bp->b_next; 6194 6195 if (bp->b_next == NULL) { 6196 /* Deleting last */ 6197 ASSERT(sq->sq_evtail == bp); 6198 sq->sq_evtail = NULL; 6199 sq->sq_flags &= ~SQ_EVENTS; 6200 } 6201 bp->b_prev = bp->b_next = NULL; 6202 ASSERT(bp->b_datap->db_ref != 0); 6203 6204 mutex_exit(SQLOCK(sq)); 6205 6206 (*func)(qp, bp); 6207 6208 mutex_enter(SQLOCK(sq)); 6209 /* 6210 * re-read the flags, since they could have changed. 6211 */ 6212 flags = sq->sq_flags; 6213 ASSERT(flags & SQ_EXCL); 6214 } 6215 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); 6216 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6217 6218 if (flags & SQ_WANTWAKEUP) { 6219 flags &= ~SQ_WANTWAKEUP; 6220 cv_broadcast(&sq->sq_wait); 6221 } 6222 if (flags & SQ_WANTEXWAKEUP) { 6223 flags &= ~SQ_WANTEXWAKEUP; 6224 cv_broadcast(&sq->sq_exitwait); 6225 } 6226 sq->sq_flags = flags; 6227 } 6228 6229 /* 6230 * Put messages on the event list. 6231 * If we can go exclusive now, do so and process the event list, otherwise 6232 * let the last claim service this list (or wake the sqthread). 6233 * This procedure assumes SQLOCK is held. To run the event list, it 6234 * must be called with no claims. 6235 */ 6236 static void 6237 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) 6238 { 6239 uint16_t count; 6240 6241 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6242 ASSERT(func != NULL); 6243 6244 /* 6245 * This is a callback. Add it to the list of callbacks 6246 * and see about upgrading. 6247 */ 6248 mp->b_prev = (mblk_t *)func; 6249 mp->b_queue = q; 6250 mp->b_next = NULL; 6251 if (sq->sq_evhead == NULL) { 6252 sq->sq_evhead = sq->sq_evtail = mp; 6253 sq->sq_flags |= SQ_EVENTS; 6254 } else { 6255 ASSERT(sq->sq_evtail != NULL); 6256 ASSERT(sq->sq_evtail->b_next == NULL); 6257 ASSERT(sq->sq_flags & SQ_EVENTS); 6258 sq->sq_evtail->b_next = mp; 6259 sq->sq_evtail = mp; 6260 } 6261 /* 6262 * We have set SQ_EVENTS, so threads will have to 6263 * unwind out of the perimiter, and new entries will 6264 * not grab a putlock. But we still need to know 6265 * how many threads have already made a claim to the 6266 * syncq, so grab the putlocks, and sum the counts. 6267 * If there are no claims on the syncq, we can upgrade 6268 * to exclusive, and run the event list. 6269 * NOTE: We hold the SQLOCK, so we can just grab the 6270 * putlocks. 6271 */ 6272 count = sq->sq_count; 6273 SQ_PUTLOCKS_ENTER(sq); 6274 SUM_SQ_PUTCOUNTS(sq, count); 6275 /* 6276 * We have no claim, so we need to check if there 6277 * are no others, then we can upgrade. 6278 */ 6279 /* 6280 * There are currently no claims on 6281 * the syncq by this thread (at least on this entry). The thread who has 6282 * the claim should drain syncq. 6283 */ 6284 if (count > 0) { 6285 /* 6286 * Can't upgrade - other threads inside. 6287 */ 6288 SQ_PUTLOCKS_EXIT(sq); 6289 mutex_exit(SQLOCK(sq)); 6290 return; 6291 } 6292 /* 6293 * Need to set SQ_EXCL and make a claim on the syncq. 6294 */ 6295 ASSERT((sq->sq_flags & SQ_EXCL) == 0); 6296 sq->sq_flags |= SQ_EXCL; 6297 ASSERT(sq->sq_count == 0); 6298 sq->sq_count++; 6299 SQ_PUTLOCKS_EXIT(sq); 6300 6301 /* Process the events list */ 6302 sq_run_events(sq); 6303 6304 /* 6305 * Release our claim... 6306 */ 6307 sq->sq_count--; 6308 6309 /* 6310 * And release SQ_EXCL. 6311 * We don't need to acquire the putlocks to release 6312 * SQ_EXCL, since we are exclusive, and hold the SQLOCK. 6313 */ 6314 sq->sq_flags &= ~SQ_EXCL; 6315 6316 /* 6317 * sq_run_events should have released SQ_EXCL 6318 */ 6319 ASSERT(!(sq->sq_flags & SQ_EXCL)); 6320 6321 /* 6322 * If anything happened while we were running the 6323 * events (or was there before), we need to process 6324 * them now. We shouldn't be exclusive sine we 6325 * released the perimiter above (plus, we asserted 6326 * for it). 6327 */ 6328 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) 6329 drain_syncq(sq); 6330 else 6331 mutex_exit(SQLOCK(sq)); 6332 } 6333 6334 /* 6335 * Perform delayed processing. The caller has to make sure that it is safe 6336 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are 6337 * set.) 6338 * 6339 * Assume that the caller has NO claims on the syncq. However, a claim 6340 * on the syncq does not indicate that a thread is draining the syncq. 6341 * There may be more claims on the syncq than there are threads draining 6342 * (i.e. #_threads_draining <= sq_count) 6343 * 6344 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set 6345 * in order to preserve qwriter(OUTER) ordering constraints. 6346 * 6347 * sq_putcount only needs to be checked when dispatching the queued 6348 * writer call for CIPUT sync queue, but this is handled in sq_run_events. 6349 */ 6350 void 6351 drain_syncq(syncq_t *sq) 6352 { 6353 queue_t *qp; 6354 uint16_t count; 6355 uint16_t type = sq->sq_type; 6356 uint16_t flags = sq->sq_flags; 6357 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; 6358 6359 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6360 "drain_syncq start:%p", sq); 6361 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6362 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6363 sq->sq_oprev == NULL) || 6364 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6365 sq->sq_oprev != NULL)); 6366 6367 /* 6368 * Drop SQ_SERVICE flag. 6369 */ 6370 if (bg_service) 6371 sq->sq_svcflags &= ~SQ_SERVICE; 6372 6373 /* 6374 * If SQ_EXCL is set, someone else is processing this syncq - let him 6375 * finish the job. 6376 */ 6377 if (flags & SQ_EXCL) { 6378 if (bg_service) { 6379 ASSERT(sq->sq_servcount != 0); 6380 sq->sq_servcount--; 6381 } 6382 mutex_exit(SQLOCK(sq)); 6383 return; 6384 } 6385 6386 /* 6387 * This routine can be called by a background thread if 6388 * it was scheduled by a hi-priority thread. SO, if there are 6389 * NOT messages queued, return (remember, we have the SQLOCK, 6390 * and it cannot change until we release it). Wakeup any waiters also. 6391 */ 6392 if (!(flags & SQ_QUEUED)) { 6393 if (flags & SQ_WANTWAKEUP) { 6394 flags &= ~SQ_WANTWAKEUP; 6395 cv_broadcast(&sq->sq_wait); 6396 } 6397 if (flags & SQ_WANTEXWAKEUP) { 6398 flags &= ~SQ_WANTEXWAKEUP; 6399 cv_broadcast(&sq->sq_exitwait); 6400 } 6401 sq->sq_flags = flags; 6402 if (bg_service) { 6403 ASSERT(sq->sq_servcount != 0); 6404 sq->sq_servcount--; 6405 } 6406 mutex_exit(SQLOCK(sq)); 6407 return; 6408 } 6409 6410 /* 6411 * If this is not a concurrent put perimiter, we need to 6412 * become exclusive to drain. Also, if not CIPUT, we would 6413 * not have acquired a putlock, so we don't need to check 6414 * the putcounts. If not entering with a claim, we test 6415 * for sq_count == 0. 6416 */ 6417 type = sq->sq_type; 6418 if (!(type & SQ_CIPUT)) { 6419 if (sq->sq_count > 1) { 6420 if (bg_service) { 6421 ASSERT(sq->sq_servcount != 0); 6422 sq->sq_servcount--; 6423 } 6424 mutex_exit(SQLOCK(sq)); 6425 return; 6426 } 6427 sq->sq_flags |= SQ_EXCL; 6428 } 6429 6430 /* 6431 * This is where we make a claim to the syncq. 6432 * This can either be done by incrementing a putlock, or 6433 * the sq_count. But since we already have the SQLOCK 6434 * here, we just bump the sq_count. 6435 * 6436 * Note that after we make a claim, we need to let the code 6437 * fall through to the end of this routine to clean itself 6438 * up. A return in the while loop will put the syncq in a 6439 * very bad state. 6440 */ 6441 sq->sq_count++; 6442 ASSERT(sq->sq_count != 0); /* wraparound */ 6443 6444 while ((flags = sq->sq_flags) & SQ_QUEUED) { 6445 /* 6446 * If we are told to stayaway or went exclusive, 6447 * we are done. 6448 */ 6449 if (flags & (SQ_STAYAWAY)) { 6450 break; 6451 } 6452 6453 /* 6454 * If there are events to run, do so. 6455 * We have one claim to the syncq, so if there are 6456 * more than one, other threads are running. 6457 */ 6458 if (sq->sq_evhead != NULL) { 6459 ASSERT(sq->sq_flags & SQ_EVENTS); 6460 6461 count = sq->sq_count; 6462 SQ_PUTLOCKS_ENTER(sq); 6463 SUM_SQ_PUTCOUNTS(sq, count); 6464 if (count > 1) { 6465 SQ_PUTLOCKS_EXIT(sq); 6466 /* Can't upgrade - other threads inside */ 6467 break; 6468 } 6469 ASSERT((flags & SQ_EXCL) == 0); 6470 sq->sq_flags = flags | SQ_EXCL; 6471 SQ_PUTLOCKS_EXIT(sq); 6472 /* 6473 * we have the only claim, run the events, 6474 * sq_run_events will clear the SQ_EXCL flag. 6475 */ 6476 sq_run_events(sq); 6477 6478 /* 6479 * If this is a CIPUT perimiter, we need 6480 * to drop the SQ_EXCL flag so we can properly 6481 * continue draining the syncq. 6482 */ 6483 if (type & SQ_CIPUT) { 6484 ASSERT(sq->sq_flags & SQ_EXCL); 6485 sq->sq_flags &= ~SQ_EXCL; 6486 } 6487 6488 /* 6489 * And go back to the beginning just in case 6490 * anything changed while we were away. 6491 */ 6492 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); 6493 continue; 6494 } 6495 6496 ASSERT(sq->sq_evhead == NULL); 6497 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6498 6499 /* 6500 * Find the queue that is not draining. 6501 * 6502 * q_draining is protected by QLOCK which we do not hold. 6503 * But if it was set, then a thread was draining, and if it gets 6504 * cleared, then it was because the thread has successfully 6505 * drained the syncq, or a GOAWAY state occured. For the GOAWAY 6506 * state to happen, a thread needs the SQLOCK which we hold, and 6507 * if there was such a flag, we whould have already seen it. 6508 */ 6509 6510 for (qp = sq->sq_head; 6511 qp != NULL && (qp->q_draining || 6512 (qp->q_sqflags & Q_SQDRAINING)); 6513 qp = qp->q_sqnext) 6514 ; 6515 6516 if (qp == NULL) 6517 break; 6518 6519 /* 6520 * We have a queue to work on, and we hold the 6521 * SQLOCK and one claim, call qdrain_syncq. 6522 * This means we need to release the SQLOCK and 6523 * aquire the QLOCK (OK since we have a claim). 6524 * Note that qdrain_syncq will actually dequeue 6525 * this queue from the sq_head list when it is 6526 * convinced all the work is done and release 6527 * the QLOCK before returning. 6528 */ 6529 qp->q_sqflags |= Q_SQDRAINING; 6530 mutex_exit(SQLOCK(sq)); 6531 mutex_enter(QLOCK(qp)); 6532 qdrain_syncq(sq, qp); 6533 mutex_enter(SQLOCK(sq)); 6534 6535 /* The queue is drained */ 6536 ASSERT(qp->q_sqflags & Q_SQDRAINING); 6537 qp->q_sqflags &= ~Q_SQDRAINING; 6538 /* 6539 * NOTE: After this point qp should not be used since it may be 6540 * closed. 6541 */ 6542 } 6543 6544 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6545 flags = sq->sq_flags; 6546 6547 /* 6548 * sq->sq_head cannot change because we hold the 6549 * sqlock. However, a thread CAN decide that it is no longer 6550 * going to drain that queue. However, this should be due to 6551 * a GOAWAY state, and we should see that here. 6552 * 6553 * This loop is not very efficient. One solution may be adding a second 6554 * pointer to the "draining" queue, but it is difficult to do when 6555 * queues are inserted in the middle due to priority ordering. Another 6556 * possibility is to yank the queue out of the sq list and put it onto 6557 * the "draining list" and then put it back if it can't be drained. 6558 */ 6559 6560 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || 6561 (type & SQ_CI) || sq->sq_head->q_draining); 6562 6563 /* Drop SQ_EXCL for non-CIPUT perimiters */ 6564 if (!(type & SQ_CIPUT)) 6565 flags &= ~SQ_EXCL; 6566 ASSERT((flags & SQ_EXCL) == 0); 6567 6568 /* Wake up any waiters. */ 6569 if (flags & SQ_WANTWAKEUP) { 6570 flags &= ~SQ_WANTWAKEUP; 6571 cv_broadcast(&sq->sq_wait); 6572 } 6573 if (flags & SQ_WANTEXWAKEUP) { 6574 flags &= ~SQ_WANTEXWAKEUP; 6575 cv_broadcast(&sq->sq_exitwait); 6576 } 6577 sq->sq_flags = flags; 6578 6579 ASSERT(sq->sq_count != 0); 6580 /* Release our claim. */ 6581 sq->sq_count--; 6582 6583 if (bg_service) { 6584 ASSERT(sq->sq_servcount != 0); 6585 sq->sq_servcount--; 6586 } 6587 6588 mutex_exit(SQLOCK(sq)); 6589 6590 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6591 "drain_syncq end:%p", sq); 6592 } 6593 6594 6595 /* 6596 * 6597 * qdrain_syncq can be called (currently) from only one of two places: 6598 * drain_syncq 6599 * putnext (or some variation of it). 6600 * and eventually 6601 * qwait(_sig) 6602 * 6603 * If called from drain_syncq, we found it in the list 6604 * of queue's needing service, so there is work to be done (or it 6605 * wouldn't be on the list). 6606 * 6607 * If called from some putnext variation, it was because the 6608 * perimiter is open, but messages are blocking a putnext and 6609 * there is not a thread working on it. Now a thread could start 6610 * working on it while we are getting ready to do so ourself, but 6611 * the thread would set the q_draining flag, and we can spin out. 6612 * 6613 * As for qwait(_sig), I think I shall let it continue to call 6614 * drain_syncq directly (after all, it will get here eventually). 6615 * 6616 * qdrain_syncq has to terminate when: 6617 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering 6618 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering 6619 * 6620 * ASSUMES: 6621 * One claim 6622 * QLOCK held 6623 * SQLOCK not held 6624 * Will release QLOCK before returning 6625 */ 6626 void 6627 qdrain_syncq(syncq_t *sq, queue_t *q) 6628 { 6629 mblk_t *bp; 6630 boolean_t do_clr; 6631 #ifdef DEBUG 6632 uint16_t count; 6633 #endif 6634 6635 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6636 "drain_syncq start:%p", sq); 6637 ASSERT(q->q_syncq == sq); 6638 ASSERT(MUTEX_HELD(QLOCK(q))); 6639 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6640 /* 6641 * For non-CIPUT perimiters, we should be called with the 6642 * exclusive bit set already. For non-CIPUT perimiters we 6643 * will be doing a concurrent drain, so it better not be set. 6644 */ 6645 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); 6646 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); 6647 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); 6648 /* 6649 * All outer pointers are set, or none of them are 6650 */ 6651 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6652 sq->sq_oprev == NULL) || 6653 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6654 sq->sq_oprev != NULL)); 6655 #ifdef DEBUG 6656 count = sq->sq_count; 6657 /* 6658 * This is OK without the putlocks, because we have one 6659 * claim either from the sq_count, or a putcount. We could 6660 * get an erroneous value from other counts, but ours won't 6661 * change, so one way or another, we will have at least a 6662 * value of one. 6663 */ 6664 SUM_SQ_PUTCOUNTS(sq, count); 6665 ASSERT(count >= 1); 6666 #endif /* DEBUG */ 6667 6668 /* 6669 * The first thing to do here, is find out if a thread is already 6670 * draining this queue or the queue is closing. If so, we are done, 6671 * just return. Also, if there are no messages, we are done as well. 6672 * Note that we check the q_sqhead since there is s window of 6673 * opportunity for us to enter here because Q_SQQUEUED was set, but is 6674 * not anymore. 6675 */ 6676 if (q->q_draining || (q->q_sqhead == NULL)) { 6677 mutex_exit(QLOCK(q)); 6678 return; 6679 } 6680 6681 /* 6682 * If the perimiter is exclusive, there is nothing we can 6683 * do right now, go away. 6684 * Note that there is nothing to prevent this case from changing 6685 * right after this check, but the spin-out will catch it. 6686 */ 6687 6688 /* Tell other threads that we are draining this queue */ 6689 q->q_draining = 1; /* Protected by QLOCK */ 6690 6691 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) { 6692 6693 /* 6694 * Because we can enter this routine just because 6695 * a putnext is blocked, we need to spin out if 6696 * the perimiter wants to go exclusive as well 6697 * as just blocked. We need to spin out also if 6698 * events are queued on the syncq. 6699 * Don't check for SQ_EXCL, because non-CIPUT 6700 * perimiters would set it, and it can't become 6701 * exclusive while we hold a claim. 6702 */ 6703 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { 6704 break; 6705 } 6706 6707 #ifdef DEBUG 6708 /* 6709 * Since we are in qdrain_syncq, we already know the queue, 6710 * but for sanity, we want to check this against the qp that 6711 * was passed in by bp->b_queue. 6712 */ 6713 6714 ASSERT(bp->b_queue == q); 6715 ASSERT(bp->b_queue->q_syncq == sq); 6716 bp->b_queue = NULL; 6717 6718 /* 6719 * We would have the following check in the DEBUG code: 6720 * 6721 * if (bp->b_prev != NULL) { 6722 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp); 6723 * } 6724 * 6725 * This can't be done, however, since IP modifies qinfo 6726 * structure at run-time (switching between IPv4 qinfo and IPv6 6727 * qinfo), invalidating the check. 6728 * So the assignment to func is left here, but the ASSERT itself 6729 * is removed until the whole issue is resolved. 6730 */ 6731 #endif 6732 ASSERT(q->q_sqhead == bp); 6733 q->q_sqhead = bp->b_next; 6734 bp->b_prev = bp->b_next = NULL; 6735 ASSERT(q->q_syncqmsgs > 0); 6736 mutex_exit(QLOCK(q)); 6737 6738 ASSERT(bp->b_datap->db_ref != 0); 6739 6740 (void) (*q->q_qinfo->qi_putp)(q, bp); 6741 6742 mutex_enter(QLOCK(q)); 6743 /* 6744 * We should decrement q_syncqmsgs only after executing the 6745 * put procedure to avoid a possible race with putnext(). 6746 * In putnext() though it sees Q_SQQUEUED is set, there is 6747 * an optimization which allows putnext to call the put 6748 * procedure directly if (q_syncqmsgs == 0) and thus 6749 * a message reodering could otherwise occur. 6750 */ 6751 q->q_syncqmsgs--; 6752 6753 /* 6754 * Clear QFULL in the next service procedure queue if 6755 * this is the last message destined to that queue. 6756 * 6757 * It would make better sense to have some sort of 6758 * tunable for the low water mark, but these symantics 6759 * are not yet defined. So, alas, we use a constant. 6760 */ 6761 do_clr = (q->q_syncqmsgs == 0); 6762 mutex_exit(QLOCK(q)); 6763 6764 if (do_clr) 6765 clr_qfull(q); 6766 6767 mutex_enter(QLOCK(q)); 6768 /* 6769 * Always clear SQ_EXCL when CIPUT in order to handle 6770 * qwriter(INNER). 6771 */ 6772 /* 6773 * The putp() can call qwriter and get exclusive access 6774 * IFF this is the only claim. So, we need to test for 6775 * this possibility so we can aquire the mutex and clear 6776 * the bit. 6777 */ 6778 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { 6779 mutex_enter(SQLOCK(sq)); 6780 sq->sq_flags &= ~SQ_EXCL; 6781 mutex_exit(SQLOCK(sq)); 6782 } 6783 } 6784 6785 /* 6786 * We should either have no queues on the syncq, or we were 6787 * told to goaway by a waiter (which we will wake up at the 6788 * end of this function). 6789 */ 6790 ASSERT((q->q_sqhead == NULL) || 6791 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); 6792 6793 ASSERT(MUTEX_HELD(QLOCK(q))); 6794 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6795 6796 /* 6797 * Remove the q from the syncq list if all the messages are 6798 * drained. 6799 */ 6800 if (q->q_sqhead == NULL) { 6801 mutex_enter(SQLOCK(sq)); 6802 if (q->q_sqflags & Q_SQQUEUED) 6803 SQRM_Q(sq, q); 6804 mutex_exit(SQLOCK(sq)); 6805 /* 6806 * Since the queue is removed from the list, reset its priority. 6807 */ 6808 q->q_spri = 0; 6809 } 6810 6811 /* 6812 * Remember, the q_draining flag is used to let another 6813 * thread know that there is a thread currently draining 6814 * the messages for a queue. Since we are now done with 6815 * this queue (even if there may be messages still there), 6816 * we need to clear this flag so some thread will work 6817 * on it if needed. 6818 */ 6819 ASSERT(q->q_draining); 6820 q->q_draining = 0; 6821 6822 /* called with a claim, so OK to drop all locks. */ 6823 mutex_exit(QLOCK(q)); 6824 6825 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6826 "drain_syncq end:%p", sq); 6827 } 6828 /* END OF QDRAIN_SYNCQ */ 6829 6830 6831 /* 6832 * This is the mate to qdrain_syncq, except that it is putting the 6833 * message onto the the queue instead draining. Since the 6834 * message is destined for the queue that is selected, there is 6835 * no need to identify the function because the message is 6836 * intended for the put routine for the queue. But this 6837 * routine will do it anyway just in case (but only for debug kernels). 6838 * 6839 * After the message is enqueued on the syncq, it calls putnext_tail() 6840 * which will schedule a background thread to actually process the message. 6841 * 6842 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and 6843 * SQLOCK(sq) and QLOCK(q) are not held. 6844 */ 6845 void 6846 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) 6847 { 6848 queue_t *fq = NULL; 6849 6850 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6851 ASSERT(MUTEX_NOT_HELD(QLOCK(q))); 6852 ASSERT(sq->sq_count > 0); 6853 ASSERT(q->q_syncq == sq); 6854 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6855 sq->sq_oprev == NULL) || 6856 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6857 sq->sq_oprev != NULL)); 6858 6859 mutex_enter(QLOCK(q)); 6860 6861 /* 6862 * Set QFULL in next service procedure queue (that cares) if not 6863 * already set and if there are already more messages on the syncq 6864 * than sq_max_size. If sq_max_size is 0, no flow control will be 6865 * asserted on any syncq. 6866 * 6867 * The fq here is the next queue with a service procedure. 6868 * This is where we would fail canputnext, so this is where we 6869 * need to set QFULL. 6870 * 6871 * LOCKING HIERARCHY: In the case when fq != q we need to 6872 * a) Take QLOCK(fq) to set QFULL flag and 6873 * b) Take sd_reflock in the case of the hot stream to update 6874 * sd_refcnt. 6875 * We already have QLOCK at this point. To avoid cross-locks with 6876 * freezestr() which grabs all QLOCKs and with strlock() which grabs 6877 * both SQLOCK and sd_reflock, we need to drop respective locks first. 6878 */ 6879 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) && 6880 (q->q_syncqmsgs > sq_max_size)) { 6881 if ((fq = q->q_nfsrv) == q) { 6882 fq->q_flag |= QFULL; 6883 } else { 6884 mutex_exit(QLOCK(q)); 6885 mutex_enter(QLOCK(fq)); 6886 fq->q_flag |= QFULL; 6887 mutex_exit(QLOCK(fq)); 6888 mutex_enter(QLOCK(q)); 6889 } 6890 } 6891 6892 #ifdef DEBUG 6893 /* 6894 * This is used for debug in the qfill_syncq/qdrain_syncq case 6895 * to trace the queue that the message is intended for. Note 6896 * that the original use was to identify the queue and function 6897 * to call on the drain. In the new syncq, we have the context 6898 * of the queue that we are draining, so call it's putproc and 6899 * don't rely on the saved values. But for debug this is still 6900 * usefull information. 6901 */ 6902 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp; 6903 mp->b_queue = q; 6904 mp->b_next = NULL; 6905 #endif 6906 ASSERT(q->q_syncq == sq); 6907 /* 6908 * Enqueue the message on the list. 6909 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to 6910 * protect it. So its ok to acquire SQLOCK after SQPUT_MP(). 6911 */ 6912 SQPUT_MP(q, mp); 6913 mutex_enter(SQLOCK(sq)); 6914 6915 /* 6916 * And queue on syncq for scheduling, if not already queued. 6917 * Note that we need the SQLOCK for this, and for testing flags 6918 * at the end to see if we will drain. So grab it now, and 6919 * release it before we call qdrain_syncq or return. 6920 */ 6921 if (!(q->q_sqflags & Q_SQQUEUED)) { 6922 q->q_spri = curthread->t_pri; 6923 SQPUT_Q(sq, q); 6924 } 6925 #ifdef DEBUG 6926 else { 6927 /* 6928 * All of these conditions MUST be true! 6929 */ 6930 ASSERT(sq->sq_tail != NULL); 6931 if (sq->sq_tail == sq->sq_head) { 6932 ASSERT((q->q_sqprev == NULL) && 6933 (q->q_sqnext == NULL)); 6934 } else { 6935 ASSERT((q->q_sqprev != NULL) || 6936 (q->q_sqnext != NULL)); 6937 } 6938 ASSERT(sq->sq_flags & SQ_QUEUED); 6939 ASSERT(q->q_syncqmsgs != 0); 6940 ASSERT(q->q_sqflags & Q_SQQUEUED); 6941 } 6942 #endif 6943 mutex_exit(QLOCK(q)); 6944 /* 6945 * SQLOCK is still held, so sq_count can be safely decremented. 6946 */ 6947 sq->sq_count--; 6948 6949 putnext_tail(sq, q, 0); 6950 /* Should not reference sq or q after this point. */ 6951 } 6952 6953 /* End of qfill_syncq */ 6954 6955 /* 6956 * Remove all messages from a syncq (if qp is NULL) or remove all messages 6957 * that would be put into qp by drain_syncq. 6958 * Used when deleting the syncq (qp == NULL) or when detaching 6959 * a queue (qp != NULL). 6960 * Return non-zero if one or more messages were freed. 6961 * 6962 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 6963 * sq_putlocks are used. 6964 * 6965 * NOTE: This function assumes that it is called from the close() context and 6966 * that all the queues in the syncq are going aay. For this reason it doesn't 6967 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is 6968 * currently valid, but it is useful to rethink this function to behave properly 6969 * in other cases. 6970 */ 6971 int 6972 flush_syncq(syncq_t *sq, queue_t *qp) 6973 { 6974 mblk_t *bp, *mp_head, *mp_next, *mp_prev; 6975 queue_t *q; 6976 int ret = 0; 6977 6978 mutex_enter(SQLOCK(sq)); 6979 6980 /* 6981 * Before we leave, we need to make sure there are no 6982 * events listed for this queue. All events for this queue 6983 * will just be freed. 6984 */ 6985 if (qp != NULL && sq->sq_evhead != NULL) { 6986 ASSERT(sq->sq_flags & SQ_EVENTS); 6987 6988 mp_prev = NULL; 6989 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { 6990 mp_next = bp->b_next; 6991 if (bp->b_queue == qp) { 6992 /* Delete this message */ 6993 if (mp_prev != NULL) { 6994 mp_prev->b_next = mp_next; 6995 /* 6996 * Update sq_evtail if the last element 6997 * is removed. 6998 */ 6999 if (bp == sq->sq_evtail) { 7000 ASSERT(mp_next == NULL); 7001 sq->sq_evtail = mp_prev; 7002 } 7003 } else 7004 sq->sq_evhead = mp_next; 7005 if (sq->sq_evhead == NULL) 7006 sq->sq_flags &= ~SQ_EVENTS; 7007 bp->b_prev = bp->b_next = NULL; 7008 freemsg(bp); 7009 ret++; 7010 } else { 7011 mp_prev = bp; 7012 } 7013 } 7014 } 7015 7016 /* 7017 * Walk sq_head and: 7018 * - match qp if qp is set, remove it's messages 7019 * - all if qp is not set 7020 */ 7021 q = sq->sq_head; 7022 while (q != NULL) { 7023 ASSERT(q->q_syncq == sq); 7024 if ((qp == NULL) || (qp == q)) { 7025 /* 7026 * Yank the messages as a list off the queue 7027 */ 7028 mp_head = q->q_sqhead; 7029 /* 7030 * We do not have QLOCK(q) here (which is safe due to 7031 * assumptions mentioned above). To obtain the lock we 7032 * need to release SQLOCK which may allow lots of things 7033 * to change upon us. This place requires more analysis. 7034 */ 7035 q->q_sqhead = q->q_sqtail = NULL; 7036 ASSERT(mp_head->b_queue && 7037 mp_head->b_queue->q_syncq == sq); 7038 7039 /* 7040 * Free each of the messages. 7041 */ 7042 for (bp = mp_head; bp != NULL; bp = mp_next) { 7043 mp_next = bp->b_next; 7044 bp->b_prev = bp->b_next = NULL; 7045 freemsg(bp); 7046 ret++; 7047 } 7048 /* 7049 * Now remove the queue from the syncq. 7050 */ 7051 ASSERT(q->q_sqflags & Q_SQQUEUED); 7052 SQRM_Q(sq, q); 7053 q->q_spri = 0; 7054 q->q_syncqmsgs = 0; 7055 7056 /* 7057 * If qp was specified, we are done with it and are 7058 * going to drop SQLOCK(sq) and return. We wakeup syncq 7059 * waiters while we still have the SQLOCK. 7060 */ 7061 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { 7062 sq->sq_flags &= ~SQ_WANTWAKEUP; 7063 cv_broadcast(&sq->sq_wait); 7064 } 7065 /* Drop SQLOCK across clr_qfull */ 7066 mutex_exit(SQLOCK(sq)); 7067 7068 /* 7069 * We avoid doing the test that drain_syncq does and 7070 * unconditionally clear qfull for every flushed 7071 * message. Since flush_syncq is only called during 7072 * close this should not be a problem. 7073 */ 7074 clr_qfull(q); 7075 if (qp != NULL) { 7076 return (ret); 7077 } else { 7078 mutex_enter(SQLOCK(sq)); 7079 /* 7080 * The head was removed by SQRM_Q above. 7081 * reread the new head and flush it. 7082 */ 7083 q = sq->sq_head; 7084 } 7085 } else { 7086 q = q->q_sqnext; 7087 } 7088 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7089 } 7090 7091 if (sq->sq_flags & SQ_WANTWAKEUP) { 7092 sq->sq_flags &= ~SQ_WANTWAKEUP; 7093 cv_broadcast(&sq->sq_wait); 7094 } 7095 7096 mutex_exit(SQLOCK(sq)); 7097 return (ret); 7098 } 7099 7100 /* 7101 * Propagate all messages from a syncq to the next syncq that are associated 7102 * with the specified queue. If the queue is attached to a driver or if the 7103 * messages have been added due to a qwriter(PERIM_INNER), free the messages. 7104 * 7105 * Assumes that the stream is strlock()'ed. We don't come here if there 7106 * are no messages to propagate. 7107 * 7108 * NOTE : If the queue is attached to a driver, all the messages are freed 7109 * as there is no point in propagating the messages from the driver syncq 7110 * to the closing stream head which will in turn get freed later. 7111 */ 7112 static int 7113 propagate_syncq(queue_t *qp) 7114 { 7115 mblk_t *bp, *head, *tail, *prev, *next; 7116 syncq_t *sq; 7117 queue_t *nqp; 7118 syncq_t *nsq; 7119 boolean_t isdriver; 7120 int moved = 0; 7121 uint16_t flags; 7122 pri_t priority = curthread->t_pri; 7123 #ifdef DEBUG 7124 void (*func)(); 7125 #endif 7126 7127 sq = qp->q_syncq; 7128 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7129 /* debug macro */ 7130 SQ_PUTLOCKS_HELD(sq); 7131 /* 7132 * As entersq() does not increment the sq_count for 7133 * the write side, check sq_count for non-QPERQ 7134 * perimeters alone. 7135 */ 7136 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); 7137 7138 /* 7139 * propagate_syncq() can be called because of either messages on the 7140 * queue syncq or because on events on the queue syncq. Do actual 7141 * message propagations if there are any messages. 7142 */ 7143 if (qp->q_syncqmsgs) { 7144 isdriver = (qp->q_flag & QISDRV); 7145 7146 if (!isdriver) { 7147 nqp = qp->q_next; 7148 nsq = nqp->q_syncq; 7149 ASSERT(MUTEX_HELD(SQLOCK(nsq))); 7150 /* debug macro */ 7151 SQ_PUTLOCKS_HELD(nsq); 7152 #ifdef DEBUG 7153 func = (void (*)())nqp->q_qinfo->qi_putp; 7154 #endif 7155 } 7156 7157 SQRM_Q(sq, qp); 7158 priority = MAX(qp->q_spri, priority); 7159 qp->q_spri = 0; 7160 head = qp->q_sqhead; 7161 tail = qp->q_sqtail; 7162 qp->q_sqhead = qp->q_sqtail = NULL; 7163 qp->q_syncqmsgs = 0; 7164 7165 /* 7166 * Walk the list of messages, and free them if this is a driver, 7167 * otherwise reset the b_prev and b_queue value to the new putp. 7168 * Afterward, we will just add the head to the end of the next 7169 * syncq, and point the tail to the end of this one. 7170 */ 7171 7172 for (bp = head; bp != NULL; bp = next) { 7173 next = bp->b_next; 7174 if (isdriver) { 7175 bp->b_prev = bp->b_next = NULL; 7176 freemsg(bp); 7177 continue; 7178 } 7179 /* Change the q values for this message */ 7180 bp->b_queue = nqp; 7181 #ifdef DEBUG 7182 bp->b_prev = (mblk_t *)func; 7183 #endif 7184 moved++; 7185 } 7186 /* 7187 * Attach list of messages to the end of the new queue (if there 7188 * is a list of messages). 7189 */ 7190 7191 if (!isdriver && head != NULL) { 7192 ASSERT(tail != NULL); 7193 if (nqp->q_sqhead == NULL) { 7194 nqp->q_sqhead = head; 7195 } else { 7196 ASSERT(nqp->q_sqtail != NULL); 7197 nqp->q_sqtail->b_next = head; 7198 } 7199 nqp->q_sqtail = tail; 7200 /* 7201 * When messages are moved from high priority queue to 7202 * another queue, the destination queue priority is 7203 * upgraded. 7204 */ 7205 7206 if (priority > nqp->q_spri) 7207 nqp->q_spri = priority; 7208 7209 SQPUT_Q(nsq, nqp); 7210 7211 nqp->q_syncqmsgs += moved; 7212 ASSERT(nqp->q_syncqmsgs != 0); 7213 } 7214 } 7215 7216 /* 7217 * Before we leave, we need to make sure there are no 7218 * events listed for this queue. All events for this queue 7219 * will just be freed. 7220 */ 7221 if (sq->sq_evhead != NULL) { 7222 ASSERT(sq->sq_flags & SQ_EVENTS); 7223 prev = NULL; 7224 for (bp = sq->sq_evhead; bp != NULL; bp = next) { 7225 next = bp->b_next; 7226 if (bp->b_queue == qp) { 7227 /* Delete this message */ 7228 if (prev != NULL) { 7229 prev->b_next = next; 7230 /* 7231 * Update sq_evtail if the last element 7232 * is removed. 7233 */ 7234 if (bp == sq->sq_evtail) { 7235 ASSERT(next == NULL); 7236 sq->sq_evtail = prev; 7237 } 7238 } else 7239 sq->sq_evhead = next; 7240 if (sq->sq_evhead == NULL) 7241 sq->sq_flags &= ~SQ_EVENTS; 7242 bp->b_prev = bp->b_next = NULL; 7243 freemsg(bp); 7244 } else { 7245 prev = bp; 7246 } 7247 } 7248 } 7249 7250 flags = sq->sq_flags; 7251 7252 /* Wake up any waiter before leaving. */ 7253 if (flags & SQ_WANTWAKEUP) { 7254 flags &= ~SQ_WANTWAKEUP; 7255 cv_broadcast(&sq->sq_wait); 7256 } 7257 sq->sq_flags = flags; 7258 7259 return (moved); 7260 } 7261 7262 /* 7263 * Try and upgrade to exclusive access at the inner perimeter. If this can 7264 * not be done without blocking then request will be queued on the syncq 7265 * and drain_syncq will run it later. 7266 * 7267 * This routine can only be called from put or service procedures plus 7268 * asynchronous callback routines that have properly entered to 7269 * queue (with entersq.) Thus qwriter_inner assumes the caller has one claim 7270 * on the syncq associated with q. 7271 */ 7272 void 7273 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)()) 7274 { 7275 syncq_t *sq = q->q_syncq; 7276 uint16_t count; 7277 7278 mutex_enter(SQLOCK(sq)); 7279 count = sq->sq_count; 7280 SQ_PUTLOCKS_ENTER(sq); 7281 SUM_SQ_PUTCOUNTS(sq, count); 7282 ASSERT(count >= 1); 7283 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); 7284 7285 if (count == 1) { 7286 /* 7287 * Can upgrade. This case also handles nested qwriter calls 7288 * (when the qwriter callback function calls qwriter). In that 7289 * case SQ_EXCL is already set. 7290 */ 7291 sq->sq_flags |= SQ_EXCL; 7292 SQ_PUTLOCKS_EXIT(sq); 7293 mutex_exit(SQLOCK(sq)); 7294 (*func)(q, mp); 7295 /* 7296 * Assumes that leavesq, putnext, and drain_syncq will reset 7297 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on 7298 * until putnext, leavesq, or drain_syncq drops it. 7299 * That way we handle nested qwriter(INNER) without dropping 7300 * SQ_EXCL until the outermost qwriter callback routine is 7301 * done. 7302 */ 7303 return; 7304 } 7305 SQ_PUTLOCKS_EXIT(sq); 7306 sqfill_events(sq, q, mp, func); 7307 } 7308 7309 /* 7310 * Synchronous callback support functions 7311 */ 7312 7313 /* 7314 * Allocate a callback parameter structure. 7315 * Assumes that caller initializes the flags and the id. 7316 * Acquires SQLOCK(sq) if non-NULL is returned. 7317 */ 7318 callbparams_t * 7319 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) 7320 { 7321 callbparams_t *cbp; 7322 size_t size = sizeof (callbparams_t); 7323 7324 cbp = kmem_alloc(size, kmflags & ~KM_PANIC); 7325 7326 /* 7327 * Only try tryhard allocation if the caller is ready to panic. 7328 * Otherwise just fail. 7329 */ 7330 if (cbp == NULL) { 7331 if (kmflags & KM_PANIC) 7332 cbp = kmem_alloc_tryhard(sizeof (callbparams_t), 7333 &size, kmflags); 7334 else 7335 return (NULL); 7336 } 7337 7338 ASSERT(size >= sizeof (callbparams_t)); 7339 cbp->cbp_size = size; 7340 cbp->cbp_sq = sq; 7341 cbp->cbp_func = func; 7342 cbp->cbp_arg = arg; 7343 mutex_enter(SQLOCK(sq)); 7344 cbp->cbp_next = sq->sq_callbpend; 7345 sq->sq_callbpend = cbp; 7346 return (cbp); 7347 } 7348 7349 void 7350 callbparams_free(syncq_t *sq, callbparams_t *cbp) 7351 { 7352 callbparams_t **pp, *p; 7353 7354 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7355 7356 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7357 if (p == cbp) { 7358 *pp = p->cbp_next; 7359 kmem_free(p, p->cbp_size); 7360 return; 7361 } 7362 } 7363 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7364 "callbparams_free: not found\n")); 7365 } 7366 7367 void 7368 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) 7369 { 7370 callbparams_t **pp, *p; 7371 7372 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7373 7374 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7375 if (p->cbp_id == id && p->cbp_flags == flag) { 7376 *pp = p->cbp_next; 7377 kmem_free(p, p->cbp_size); 7378 return; 7379 } 7380 } 7381 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7382 "callbparams_free_id: not found\n")); 7383 } 7384 7385 /* 7386 * Callback wrapper function used by once-only callbacks that can be 7387 * cancelled (qtimeout and qbufcall) 7388 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be 7389 * cancelled by the qun* functions. 7390 */ 7391 void 7392 qcallbwrapper(void *arg) 7393 { 7394 callbparams_t *cbp = arg; 7395 syncq_t *sq; 7396 uint16_t count = 0; 7397 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 7398 uint16_t type; 7399 7400 sq = cbp->cbp_sq; 7401 mutex_enter(SQLOCK(sq)); 7402 type = sq->sq_type; 7403 if (!(type & SQ_CICB)) { 7404 count = sq->sq_count; 7405 SQ_PUTLOCKS_ENTER(sq); 7406 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 7407 SUM_SQ_PUTCOUNTS(sq, count); 7408 sq->sq_needexcl++; 7409 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 7410 waitflags |= SQ_MESSAGES; 7411 } 7412 /* Can not handle exlusive entry at outer perimeter */ 7413 ASSERT(type & SQ_COCB); 7414 7415 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { 7416 if ((sq->sq_callbflags & cbp->cbp_flags) && 7417 (sq->sq_cancelid == cbp->cbp_id)) { 7418 /* timeout has been cancelled */ 7419 sq->sq_callbflags |= SQ_CALLB_BYPASSED; 7420 callbparams_free(sq, cbp); 7421 if (!(type & SQ_CICB)) { 7422 ASSERT(sq->sq_needexcl > 0); 7423 sq->sq_needexcl--; 7424 if (sq->sq_needexcl == 0) { 7425 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7426 } 7427 SQ_PUTLOCKS_EXIT(sq); 7428 } 7429 mutex_exit(SQLOCK(sq)); 7430 return; 7431 } 7432 sq->sq_flags |= SQ_WANTWAKEUP; 7433 if (!(type & SQ_CICB)) { 7434 SQ_PUTLOCKS_EXIT(sq); 7435 } 7436 cv_wait(&sq->sq_wait, SQLOCK(sq)); 7437 if (!(type & SQ_CICB)) { 7438 count = sq->sq_count; 7439 SQ_PUTLOCKS_ENTER(sq); 7440 SUM_SQ_PUTCOUNTS(sq, count); 7441 } 7442 } 7443 7444 sq->sq_count++; 7445 ASSERT(sq->sq_count != 0); /* Wraparound */ 7446 if (!(type & SQ_CICB)) { 7447 ASSERT(count == 0); 7448 sq->sq_flags |= SQ_EXCL; 7449 ASSERT(sq->sq_needexcl > 0); 7450 sq->sq_needexcl--; 7451 if (sq->sq_needexcl == 0) { 7452 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7453 } 7454 SQ_PUTLOCKS_EXIT(sq); 7455 } 7456 7457 mutex_exit(SQLOCK(sq)); 7458 7459 cbp->cbp_func(cbp->cbp_arg); 7460 7461 /* 7462 * We drop the lock only for leavesq to re-acquire it. 7463 * Possible optimization is inline of leavesq. 7464 */ 7465 mutex_enter(SQLOCK(sq)); 7466 callbparams_free(sq, cbp); 7467 mutex_exit(SQLOCK(sq)); 7468 leavesq(sq, SQ_CALLBACK); 7469 } 7470 7471 /* 7472 * no need to grab sq_putlocks here. See comment in strsubr.h that 7473 * explains when sq_putlocks are used. 7474 * 7475 * sq_count (or one of the sq_putcounts) has already been 7476 * decremented by the caller, and if SQ_QUEUED, we need to call 7477 * drain_syncq (the global syncq drain). 7478 * If putnext_tail is called with the SQ_EXCL bit set, we are in 7479 * one of two states, non-CIPUT perimiter, and we need to clear 7480 * it, or we went exclusive in the put procedure. In any case, 7481 * we want to clear the bit now, and it is probably easier to do 7482 * this at the beginning of this function (remember, we hold 7483 * the SQLOCK). Lastly, if there are other messages queued 7484 * on the syncq (and not for our destination), enable the syncq 7485 * for background work. 7486 */ 7487 7488 /* ARGSUSED */ 7489 void 7490 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) 7491 { 7492 uint16_t flags = sq->sq_flags; 7493 7494 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7495 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 7496 7497 /* Clear SQ_EXCL if set in passflags */ 7498 if (passflags & SQ_EXCL) { 7499 flags &= ~SQ_EXCL; 7500 } 7501 if (flags & SQ_WANTWAKEUP) { 7502 flags &= ~SQ_WANTWAKEUP; 7503 cv_broadcast(&sq->sq_wait); 7504 } 7505 if (flags & SQ_WANTEXWAKEUP) { 7506 flags &= ~SQ_WANTEXWAKEUP; 7507 cv_broadcast(&sq->sq_exitwait); 7508 } 7509 sq->sq_flags = flags; 7510 7511 /* 7512 * We have cleared SQ_EXCL if we were asked to, and started 7513 * the wakeup process for waiters. If there are no writers 7514 * then we need to drain the syncq if we were told to, or 7515 * enable the background thread to do it. 7516 */ 7517 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) { 7518 if ((passflags & SQ_QUEUED) || 7519 (sq->sq_svcflags & SQ_DISABLED)) { 7520 /* drain_syncq will take care of events in the list */ 7521 drain_syncq(sq); 7522 return; 7523 } else if (flags & SQ_QUEUED) { 7524 sqenable(sq); 7525 } 7526 } 7527 /* Drop the SQLOCK on exit */ 7528 mutex_exit(SQLOCK(sq)); 7529 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END, 7530 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); 7531 } 7532 7533 void 7534 set_qend(queue_t *q) 7535 { 7536 mutex_enter(QLOCK(q)); 7537 if (!O_SAMESTR(q)) 7538 q->q_flag |= QEND; 7539 else 7540 q->q_flag &= ~QEND; 7541 mutex_exit(QLOCK(q)); 7542 q = _OTHERQ(q); 7543 mutex_enter(QLOCK(q)); 7544 if (!O_SAMESTR(q)) 7545 q->q_flag |= QEND; 7546 else 7547 q->q_flag &= ~QEND; 7548 mutex_exit(QLOCK(q)); 7549 } 7550 7551 7552 void 7553 clr_qfull(queue_t *q) 7554 { 7555 queue_t *oq = q; 7556 7557 q = q->q_nfsrv; 7558 /* Fast check if there is any work to do before getting the lock. */ 7559 if ((q->q_flag & (QFULL|QWANTW)) == 0) { 7560 return; 7561 } 7562 7563 /* 7564 * Do not reset QFULL (and backenable) if the q_count is the reason 7565 * for QFULL being set. 7566 */ 7567 mutex_enter(QLOCK(q)); 7568 /* 7569 * If queue is empty i.e q_mblkcnt is zero, queue can not be full. 7570 * Hence clear the QFULL. 7571 * If both q_count and q_mblkcnt are less than the hiwat mark, 7572 * clear the QFULL. 7573 */ 7574 if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) && 7575 (q->q_mblkcnt < q->q_hiwat))) { 7576 q->q_flag &= ~QFULL; 7577 /* 7578 * A little more confusing, how about this way: 7579 * if someone wants to write, 7580 * AND 7581 * both counts are less than the lowat mark 7582 * OR 7583 * the lowat mark is zero 7584 * THEN 7585 * backenable 7586 */ 7587 if ((q->q_flag & QWANTW) && 7588 (((q->q_count < q->q_lowat) && 7589 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) { 7590 q->q_flag &= ~QWANTW; 7591 mutex_exit(QLOCK(q)); 7592 backenable(oq, 0); 7593 } else 7594 mutex_exit(QLOCK(q)); 7595 } else 7596 mutex_exit(QLOCK(q)); 7597 } 7598 7599 /* 7600 * Set the forward service procedure pointer. 7601 * 7602 * Called at insert-time to cache a queue's next forward service procedure in 7603 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted 7604 * has a service procedure then q_nfsrv points to itself. If the queue to be 7605 * inserted does not have a service procedure, then q_nfsrv points to the next 7606 * queue forward that has a service procedure. If the queue is at the logical 7607 * end of the stream (driver for write side, stream head for the read side) 7608 * and does not have a service procedure, then q_nfsrv also points to itself. 7609 */ 7610 void 7611 set_nfsrv_ptr( 7612 queue_t *rnew, /* read queue pointer to new module */ 7613 queue_t *wnew, /* write queue pointer to new module */ 7614 queue_t *prev_rq, /* read queue pointer to the module above */ 7615 queue_t *prev_wq) /* write queue pointer to the module above */ 7616 { 7617 queue_t *qp; 7618 7619 if (prev_wq->q_next == NULL) { 7620 /* 7621 * Insert the driver, initialize the driver and stream head. 7622 * In this case, prev_rq/prev_wq should be the stream head. 7623 * _I_INSERT does not allow inserting a driver. Make sure 7624 * that it is not an insertion. 7625 */ 7626 ASSERT(!(rnew->q_flag & _QINSERTING)); 7627 wnew->q_nfsrv = wnew; 7628 if (rnew->q_qinfo->qi_srvp) 7629 rnew->q_nfsrv = rnew; 7630 else 7631 rnew->q_nfsrv = prev_rq; 7632 prev_rq->q_nfsrv = prev_rq; 7633 prev_wq->q_nfsrv = prev_wq; 7634 } else { 7635 /* 7636 * set up read side q_nfsrv pointer. This MUST be done 7637 * before setting the write side, because the setting of 7638 * the write side for a fifo may depend on it. 7639 * 7640 * Suppose we have a fifo that only has pipemod pushed. 7641 * pipemod has no read or write service procedures, so 7642 * nfsrv for both pipemod queues points to prev_rq (the 7643 * stream read head). Now push bufmod (which has only a 7644 * read service procedure). Doing the write side first, 7645 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which 7646 * is WRONG; the next queue forward from wnew with a 7647 * service procedure will be rnew, not the stream read head. 7648 * Since the downstream queue (which in the case of a fifo 7649 * is the read queue rnew) can affect upstream queues, it 7650 * needs to be done first. Setting up the read side first 7651 * sets nfsrv for both pipemod queues to rnew and then 7652 * when the write side is set up, wnew-q_nfsrv will also 7653 * point to rnew. 7654 */ 7655 if (rnew->q_qinfo->qi_srvp) { 7656 /* 7657 * use _OTHERQ() because, if this is a pipe, next 7658 * module may have been pushed from other end and 7659 * q_next could be a read queue. 7660 */ 7661 qp = _OTHERQ(prev_wq->q_next); 7662 while (qp && qp->q_nfsrv != qp) { 7663 qp->q_nfsrv = rnew; 7664 qp = backq(qp); 7665 } 7666 rnew->q_nfsrv = rnew; 7667 } else 7668 rnew->q_nfsrv = prev_rq->q_nfsrv; 7669 7670 /* set up write side q_nfsrv pointer */ 7671 if (wnew->q_qinfo->qi_srvp) { 7672 wnew->q_nfsrv = wnew; 7673 7674 /* 7675 * For insertion, need to update nfsrv of the modules 7676 * above which do not have a service routine. 7677 */ 7678 if (rnew->q_flag & _QINSERTING) { 7679 for (qp = prev_wq; 7680 qp != NULL && qp->q_nfsrv != qp; 7681 qp = backq(qp)) { 7682 qp->q_nfsrv = wnew->q_nfsrv; 7683 } 7684 } 7685 } else { 7686 if (prev_wq->q_next == prev_rq) 7687 /* 7688 * Since prev_wq/prev_rq are the middle of a 7689 * fifo, wnew/rnew will also be the middle of 7690 * a fifo and wnew's nfsrv is same as rnew's. 7691 */ 7692 wnew->q_nfsrv = rnew->q_nfsrv; 7693 else 7694 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv; 7695 } 7696 } 7697 } 7698 7699 /* 7700 * Reset the forward service procedure pointer; called at remove-time. 7701 */ 7702 void 7703 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp) 7704 { 7705 queue_t *tmp_qp; 7706 7707 /* Reset the write side q_nfsrv pointer for _I_REMOVE */ 7708 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) { 7709 for (tmp_qp = backq(wqp); 7710 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp; 7711 tmp_qp = backq(tmp_qp)) { 7712 tmp_qp->q_nfsrv = wqp->q_nfsrv; 7713 } 7714 } 7715 7716 /* reset the read side q_nfsrv pointer */ 7717 if (rqp->q_qinfo->qi_srvp) { 7718 if (wqp->q_next) { /* non-driver case */ 7719 tmp_qp = _OTHERQ(wqp->q_next); 7720 while (tmp_qp && tmp_qp->q_nfsrv == rqp) { 7721 /* Note that rqp->q_next cannot be NULL */ 7722 ASSERT(rqp->q_next != NULL); 7723 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv; 7724 tmp_qp = backq(tmp_qp); 7725 } 7726 } 7727 } 7728 } 7729 7730 /* 7731 * This routine should be called after all stream geometry changes to update 7732 * the stream head cached struio() rd/wr queue pointers. Note must be called 7733 * with the streamlock()ed. 7734 * 7735 * Note: only enables Synchronous STREAMS for a side of a Stream which has 7736 * an explicit synchronous barrier module queue. That is, a queue that 7737 * has specified a struio() type. 7738 */ 7739 static void 7740 strsetuio(stdata_t *stp) 7741 { 7742 queue_t *wrq; 7743 7744 if (stp->sd_flag & STPLEX) { 7745 /* 7746 * Not stremahead, but a mux, so no Synchronous STREAMS. 7747 */ 7748 stp->sd_struiowrq = NULL; 7749 stp->sd_struiordq = NULL; 7750 return; 7751 } 7752 /* 7753 * Scan the write queue(s) while synchronous 7754 * until we find a qinfo uio type specified. 7755 */ 7756 wrq = stp->sd_wrq->q_next; 7757 while (wrq) { 7758 if (wrq->q_struiot == STRUIOT_NONE) { 7759 wrq = 0; 7760 break; 7761 } 7762 if (wrq->q_struiot != STRUIOT_DONTCARE) 7763 break; 7764 if (! _SAMESTR(wrq)) { 7765 wrq = 0; 7766 break; 7767 } 7768 wrq = wrq->q_next; 7769 } 7770 stp->sd_struiowrq = wrq; 7771 /* 7772 * Scan the read queue(s) while synchronous 7773 * until we find a qinfo uio type specified. 7774 */ 7775 wrq = stp->sd_wrq->q_next; 7776 while (wrq) { 7777 if (_RD(wrq)->q_struiot == STRUIOT_NONE) { 7778 wrq = 0; 7779 break; 7780 } 7781 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE) 7782 break; 7783 if (! _SAMESTR(wrq)) { 7784 wrq = 0; 7785 break; 7786 } 7787 wrq = wrq->q_next; 7788 } 7789 stp->sd_struiordq = wrq ? _RD(wrq) : 0; 7790 } 7791 7792 /* 7793 * pass_wput, unblocks the passthru queues, so that 7794 * messages can arrive at muxs lower read queue, before 7795 * I_LINK/I_UNLINK is acked/nacked. 7796 */ 7797 static void 7798 pass_wput(queue_t *q, mblk_t *mp) 7799 { 7800 syncq_t *sq; 7801 7802 sq = _RD(q)->q_syncq; 7803 if (sq->sq_flags & SQ_BLOCKED) 7804 unblocksq(sq, SQ_BLOCKED, 0); 7805 putnext(q, mp); 7806 } 7807 7808 /* 7809 * Set up queues for the link/unlink. 7810 * Create a new queue and block it and then insert it 7811 * below the stream head on the lower stream. 7812 * This prevents any messages from arriving during the setq 7813 * as well as while the mux is processing the LINK/I_UNLINK. 7814 * The blocked passq is unblocked once the LINK/I_UNLINK has 7815 * been acked or nacked or if a message is generated and sent 7816 * down muxs write put procedure. 7817 * see pass_wput(). 7818 * 7819 * After the new queue is inserted, all messages coming from below are 7820 * blocked. The call to strlock will ensure that all activity in the stream head 7821 * read queue syncq is stopped (sq_count drops to zero). 7822 */ 7823 static queue_t * 7824 link_addpassthru(stdata_t *stpdown) 7825 { 7826 queue_t *passq; 7827 sqlist_t sqlist; 7828 7829 passq = allocq(); 7830 STREAM(passq) = STREAM(_WR(passq)) = stpdown; 7831 /* setq might sleep in allocator - avoid holding locks. */ 7832 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ, 7833 SQ_CI|SQ_CO, B_FALSE); 7834 claimq(passq); 7835 blocksq(passq->q_syncq, SQ_BLOCKED, 1); 7836 insertq(STREAM(passq), passq); 7837 7838 /* 7839 * Use strlock() to wait for the stream head sq_count to drop to zero 7840 * since we are going to change q_ptr in the stream head. Note that 7841 * insertq() doesn't wait for any syncq counts to drop to zero. 7842 */ 7843 sqlist.sqlist_head = NULL; 7844 sqlist.sqlist_index = 0; 7845 sqlist.sqlist_size = sizeof (sqlist_t); 7846 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq); 7847 strlock(stpdown, &sqlist); 7848 strunlock(stpdown, &sqlist); 7849 7850 releaseq(passq); 7851 return (passq); 7852 } 7853 7854 /* 7855 * Let messages flow up into the mux by removing 7856 * the passq. 7857 */ 7858 static void 7859 link_rempassthru(queue_t *passq) 7860 { 7861 claimq(passq); 7862 removeq(passq); 7863 releaseq(passq); 7864 freeq(passq); 7865 } 7866 7867 /* 7868 * Wait for the condition variable pointed to by `cvp' to be signaled, 7869 * or for `tim' milliseconds to elapse, whichever comes first. If `tim' 7870 * is negative, then there is no time limit. If `nosigs' is non-zero, 7871 * then the wait will be non-interruptible. 7872 * 7873 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout. 7874 */ 7875 clock_t 7876 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs) 7877 { 7878 clock_t ret, now, tick; 7879 7880 if (tim < 0) { 7881 if (nosigs) { 7882 cv_wait(cvp, mp); 7883 ret = 1; 7884 } else { 7885 ret = cv_wait_sig(cvp, mp); 7886 } 7887 } else if (tim > 0) { 7888 /* 7889 * convert milliseconds to clock ticks 7890 */ 7891 tick = MSEC_TO_TICK_ROUNDUP(tim); 7892 time_to_wait(&now, tick); 7893 if (nosigs) { 7894 ret = cv_timedwait(cvp, mp, now); 7895 } else { 7896 ret = cv_timedwait_sig(cvp, mp, now); 7897 } 7898 } else { 7899 ret = -1; 7900 } 7901 return (ret); 7902 } 7903 7904 /* 7905 * Wait until the stream head can determine if it is at the mark but 7906 * don't wait forever to prevent a race condition between the "mark" state 7907 * in the stream head and any mark state in the caller/user of this routine. 7908 * 7909 * This is used by sockets and for a socket it would be incorrect 7910 * to return a failure for SIOCATMARK when there is no data in the receive 7911 * queue and the marked urgent data is traveling up the stream. 7912 * 7913 * This routine waits until the mark is known by waiting for one of these 7914 * three events: 7915 * The stream head read queue becoming non-empty (including an EOF) 7916 * The STRATMARK flag being set. (Due to a MSGMARKNEXT message.) 7917 * The STRNOTATMARK flag being set (which indicates that the transport 7918 * has sent a MSGNOTMARKNEXT message to indicate that it is not at 7919 * the mark). 7920 * 7921 * The routine returns 1 if the stream is at the mark; 0 if it can 7922 * be determined that the stream is not at the mark. 7923 * If the wait times out and it can't determine 7924 * whether or not the stream might be at the mark the routine will return -1. 7925 * 7926 * Note: This routine should only be used when a mark is pending i.e., 7927 * in the socket case the SIGURG has been posted. 7928 * Note2: This can not wakeup just because synchronous streams indicate 7929 * that data is available since it is not possible to use the synchronous 7930 * streams interfaces to determine the b_flag value for the data queued below 7931 * the stream head. 7932 */ 7933 int 7934 strwaitmark(vnode_t *vp) 7935 { 7936 struct stdata *stp = vp->v_stream; 7937 queue_t *rq = _RD(stp->sd_wrq); 7938 int mark; 7939 7940 mutex_enter(&stp->sd_lock); 7941 while (rq->q_first == NULL && 7942 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) { 7943 stp->sd_flag |= RSLEEP; 7944 7945 /* Wait for 100 milliseconds for any state change. */ 7946 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) { 7947 mutex_exit(&stp->sd_lock); 7948 return (-1); 7949 } 7950 } 7951 if (stp->sd_flag & STRATMARK) 7952 mark = 1; 7953 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK)) 7954 mark = 1; 7955 else 7956 mark = 0; 7957 7958 mutex_exit(&stp->sd_lock); 7959 return (mark); 7960 } 7961 7962 /* 7963 * Set a read side error. If persist is set change the socket error 7964 * to persistent. If errfunc is set install the function as the exported 7965 * error handler. 7966 */ 7967 void 7968 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7969 { 7970 struct stdata *stp = vp->v_stream; 7971 7972 mutex_enter(&stp->sd_lock); 7973 stp->sd_rerror = error; 7974 if (error == 0 && errfunc == NULL) 7975 stp->sd_flag &= ~STRDERR; 7976 else 7977 stp->sd_flag |= STRDERR; 7978 if (persist) { 7979 stp->sd_flag &= ~STRDERRNONPERSIST; 7980 } else { 7981 stp->sd_flag |= STRDERRNONPERSIST; 7982 } 7983 stp->sd_rderrfunc = errfunc; 7984 if (error != 0 || errfunc != NULL) { 7985 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7986 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7987 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 7988 7989 mutex_exit(&stp->sd_lock); 7990 pollwakeup(&stp->sd_pollist, POLLERR); 7991 mutex_enter(&stp->sd_lock); 7992 7993 if (stp->sd_sigflags & S_ERROR) 7994 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 7995 } 7996 mutex_exit(&stp->sd_lock); 7997 } 7998 7999 /* 8000 * Set a write side error. If persist is set change the socket error 8001 * to persistent. 8002 */ 8003 void 8004 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 8005 { 8006 struct stdata *stp = vp->v_stream; 8007 8008 mutex_enter(&stp->sd_lock); 8009 stp->sd_werror = error; 8010 if (error == 0 && errfunc == NULL) 8011 stp->sd_flag &= ~STWRERR; 8012 else 8013 stp->sd_flag |= STWRERR; 8014 if (persist) { 8015 stp->sd_flag &= ~STWRERRNONPERSIST; 8016 } else { 8017 stp->sd_flag |= STWRERRNONPERSIST; 8018 } 8019 stp->sd_wrerrfunc = errfunc; 8020 if (error != 0 || errfunc != NULL) { 8021 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 8022 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 8023 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 8024 8025 mutex_exit(&stp->sd_lock); 8026 pollwakeup(&stp->sd_pollist, POLLERR); 8027 mutex_enter(&stp->sd_lock); 8028 8029 if (stp->sd_sigflags & S_ERROR) 8030 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 8031 } 8032 mutex_exit(&stp->sd_lock); 8033 } 8034 8035 /* 8036 * Make the stream return 0 (EOF) when all data has been read. 8037 * No effect on write side. 8038 */ 8039 void 8040 strseteof(vnode_t *vp, int eof) 8041 { 8042 struct stdata *stp = vp->v_stream; 8043 8044 mutex_enter(&stp->sd_lock); 8045 if (!eof) { 8046 stp->sd_flag &= ~STREOF; 8047 mutex_exit(&stp->sd_lock); 8048 return; 8049 } 8050 stp->sd_flag |= STREOF; 8051 if (stp->sd_flag & RSLEEP) { 8052 stp->sd_flag &= ~RSLEEP; 8053 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); 8054 } 8055 8056 mutex_exit(&stp->sd_lock); 8057 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM); 8058 mutex_enter(&stp->sd_lock); 8059 8060 if (stp->sd_sigflags & (S_INPUT|S_RDNORM)) 8061 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0); 8062 mutex_exit(&stp->sd_lock); 8063 } 8064 8065 void 8066 strflushrq(vnode_t *vp, int flag) 8067 { 8068 struct stdata *stp = vp->v_stream; 8069 8070 mutex_enter(&stp->sd_lock); 8071 flushq(_RD(stp->sd_wrq), flag); 8072 mutex_exit(&stp->sd_lock); 8073 } 8074 8075 void 8076 strsetrputhooks(vnode_t *vp, uint_t flags, 8077 msgfunc_t protofunc, msgfunc_t miscfunc) 8078 { 8079 struct stdata *stp = vp->v_stream; 8080 8081 mutex_enter(&stp->sd_lock); 8082 8083 if (protofunc == NULL) 8084 stp->sd_rprotofunc = strrput_proto; 8085 else 8086 stp->sd_rprotofunc = protofunc; 8087 8088 if (miscfunc == NULL) 8089 stp->sd_rmiscfunc = strrput_misc; 8090 else 8091 stp->sd_rmiscfunc = miscfunc; 8092 8093 if (flags & SH_CONSOL_DATA) 8094 stp->sd_rput_opt |= SR_CONSOL_DATA; 8095 else 8096 stp->sd_rput_opt &= ~SR_CONSOL_DATA; 8097 8098 if (flags & SH_SIGALLDATA) 8099 stp->sd_rput_opt |= SR_SIGALLDATA; 8100 else 8101 stp->sd_rput_opt &= ~SR_SIGALLDATA; 8102 8103 if (flags & SH_IGN_ZEROLEN) 8104 stp->sd_rput_opt |= SR_IGN_ZEROLEN; 8105 else 8106 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN; 8107 8108 mutex_exit(&stp->sd_lock); 8109 } 8110 8111 void 8112 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime) 8113 { 8114 struct stdata *stp = vp->v_stream; 8115 8116 mutex_enter(&stp->sd_lock); 8117 stp->sd_closetime = closetime; 8118 8119 if (flags & SH_SIGPIPE) 8120 stp->sd_wput_opt |= SW_SIGPIPE; 8121 else 8122 stp->sd_wput_opt &= ~SW_SIGPIPE; 8123 if (flags & SH_RECHECK_ERR) 8124 stp->sd_wput_opt |= SW_RECHECK_ERR; 8125 else 8126 stp->sd_wput_opt &= ~SW_RECHECK_ERR; 8127 8128 mutex_exit(&stp->sd_lock); 8129 } 8130 8131 void 8132 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc) 8133 { 8134 struct stdata *stp = vp->v_stream; 8135 8136 mutex_enter(&stp->sd_lock); 8137 8138 stp->sd_rputdatafunc = rdatafunc; 8139 stp->sd_wputdatafunc = wdatafunc; 8140 8141 mutex_exit(&stp->sd_lock); 8142 } 8143 8144 /* Used within framework when the queue is already locked */ 8145 void 8146 qenable_locked(queue_t *q) 8147 { 8148 stdata_t *stp = STREAM(q); 8149 8150 ASSERT(MUTEX_HELD(QLOCK(q))); 8151 8152 if (!q->q_qinfo->qi_srvp) 8153 return; 8154 8155 /* 8156 * Do not place on run queue if already enabled or closing. 8157 */ 8158 if (q->q_flag & (QWCLOSE|QENAB)) 8159 return; 8160 8161 /* 8162 * mark queue enabled and place on run list if it is not already being 8163 * serviced. If it is serviced, the runservice() function will detect 8164 * that QENAB is set and call service procedure before clearing 8165 * QINSERVICE flag. 8166 */ 8167 q->q_flag |= QENAB; 8168 if (q->q_flag & QINSERVICE) 8169 return; 8170 8171 /* Record the time of qenable */ 8172 q->q_qtstamp = lbolt; 8173 8174 /* 8175 * Put the queue in the stp list and schedule it for background 8176 * processing if it is not already scheduled or if stream head does not 8177 * intent to process it in the foreground later by setting 8178 * STRS_WILLSERVICE flag. 8179 */ 8180 mutex_enter(&stp->sd_qlock); 8181 /* 8182 * If there are already something on the list, stp flags should show 8183 * intention to drain it. 8184 */ 8185 IMPLY(STREAM_NEEDSERVICE(stp), 8186 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))); 8187 8188 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link); 8189 stp->sd_nqueues++; 8190 8191 /* 8192 * If no one will drain this stream we are the first producer and 8193 * need to schedule it for background thread. 8194 */ 8195 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) { 8196 /* 8197 * No one will service this stream later, so we have to 8198 * schedule it now. 8199 */ 8200 STRSTAT(stenables); 8201 stp->sd_svcflags |= STRS_SCHEDULED; 8202 stp->sd_servid = (void *)taskq_dispatch(streams_taskq, 8203 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE); 8204 8205 if (stp->sd_servid == NULL) { 8206 /* 8207 * Task queue failed so fail over to the backup 8208 * servicing thread. 8209 */ 8210 STRSTAT(taskqfails); 8211 /* 8212 * It is safe to clear STRS_SCHEDULED flag because it 8213 * was set by this thread above. 8214 */ 8215 stp->sd_svcflags &= ~STRS_SCHEDULED; 8216 8217 /* 8218 * Failover scheduling is protected by service_queue 8219 * lock. 8220 */ 8221 mutex_enter(&service_queue); 8222 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q)); 8223 ASSERT(q->q_link == NULL); 8224 /* 8225 * Append the queue to qhead/qtail list. 8226 */ 8227 if (qhead == NULL) 8228 qhead = q; 8229 else 8230 qtail->q_link = q; 8231 qtail = q; 8232 /* 8233 * Clear stp queue list. 8234 */ 8235 stp->sd_qhead = stp->sd_qtail = NULL; 8236 stp->sd_nqueues = 0; 8237 /* 8238 * Wakeup background queue processing thread. 8239 */ 8240 cv_signal(&services_to_run); 8241 mutex_exit(&service_queue); 8242 } 8243 } 8244 mutex_exit(&stp->sd_qlock); 8245 } 8246 8247 static void 8248 queue_service(queue_t *q) 8249 { 8250 /* 8251 * The queue in the list should have 8252 * QENAB flag set and should not have 8253 * QINSERVICE flag set. QINSERVICE is 8254 * set when the queue is dequeued and 8255 * qenable_locked doesn't enqueue a 8256 * queue with QINSERVICE set. 8257 */ 8258 8259 ASSERT(!(q->q_flag & QINSERVICE)); 8260 ASSERT((q->q_flag & QENAB)); 8261 mutex_enter(QLOCK(q)); 8262 q->q_flag &= ~QENAB; 8263 q->q_flag |= QINSERVICE; 8264 mutex_exit(QLOCK(q)); 8265 runservice(q); 8266 } 8267 8268 static void 8269 syncq_service(syncq_t *sq) 8270 { 8271 STRSTAT(syncqservice); 8272 mutex_enter(SQLOCK(sq)); 8273 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); 8274 ASSERT(sq->sq_servcount != 0); 8275 ASSERT(sq->sq_next == NULL); 8276 8277 /* if we came here from the background thread, clear the flag */ 8278 if (sq->sq_svcflags & SQ_BGTHREAD) 8279 sq->sq_svcflags &= ~SQ_BGTHREAD; 8280 8281 /* let drain_syncq know that it's being called in the background */ 8282 sq->sq_svcflags |= SQ_SERVICE; 8283 drain_syncq(sq); 8284 } 8285 8286 static void 8287 qwriter_outer_service(syncq_t *outer) 8288 { 8289 /* 8290 * Note that SQ_WRITER is used on the outer perimeter 8291 * to signal that a qwriter(OUTER) is either investigating 8292 * running or that it is actually running a function. 8293 */ 8294 outer_enter(outer, SQ_BLOCKED|SQ_WRITER); 8295 8296 /* 8297 * All inner syncq are empty and have SQ_WRITER set 8298 * to block entering the outer perimeter. 8299 * 8300 * We do not need to explicitly call write_now since 8301 * outer_exit does it for us. 8302 */ 8303 outer_exit(outer); 8304 } 8305 8306 static void 8307 mblk_free(mblk_t *mp) 8308 { 8309 dblk_t *dbp = mp->b_datap; 8310 frtn_t *frp = dbp->db_frtnp; 8311 8312 mp->b_next = NULL; 8313 if (dbp->db_fthdr != NULL) 8314 str_ftfree(dbp); 8315 8316 ASSERT(dbp->db_fthdr == NULL); 8317 frp->free_func(frp->free_arg); 8318 ASSERT(dbp->db_mblk == mp); 8319 8320 if (dbp->db_credp != NULL) { 8321 crfree(dbp->db_credp); 8322 dbp->db_credp = NULL; 8323 } 8324 dbp->db_cpid = -1; 8325 dbp->db_struioflag = 0; 8326 dbp->db_struioun.cksum.flags = 0; 8327 8328 kmem_cache_free(dbp->db_cache, dbp); 8329 } 8330 8331 /* 8332 * Background processing of the stream queue list. 8333 */ 8334 static void 8335 stream_service(stdata_t *stp) 8336 { 8337 queue_t *q; 8338 8339 mutex_enter(&stp->sd_qlock); 8340 8341 STR_SERVICE(stp, q); 8342 8343 stp->sd_svcflags &= ~STRS_SCHEDULED; 8344 stp->sd_servid = NULL; 8345 cv_signal(&stp->sd_qcv); 8346 mutex_exit(&stp->sd_qlock); 8347 } 8348 8349 /* 8350 * Foreground processing of the stream queue list. 8351 */ 8352 void 8353 stream_runservice(stdata_t *stp) 8354 { 8355 queue_t *q; 8356 8357 mutex_enter(&stp->sd_qlock); 8358 STRSTAT(rservice); 8359 /* 8360 * We are going to drain this stream queue list, so qenable_locked will 8361 * not schedule it until we finish. 8362 */ 8363 stp->sd_svcflags |= STRS_WILLSERVICE; 8364 8365 STR_SERVICE(stp, q); 8366 8367 stp->sd_svcflags &= ~STRS_WILLSERVICE; 8368 mutex_exit(&stp->sd_qlock); 8369 /* 8370 * Help backup background thread to drain the qhead/qtail list. 8371 */ 8372 while (qhead != NULL) { 8373 STRSTAT(qhelps); 8374 mutex_enter(&service_queue); 8375 DQ(q, qhead, qtail, q_link); 8376 mutex_exit(&service_queue); 8377 if (q != NULL) 8378 queue_service(q); 8379 } 8380 } 8381 8382 void 8383 stream_willservice(stdata_t *stp) 8384 { 8385 mutex_enter(&stp->sd_qlock); 8386 stp->sd_svcflags |= STRS_WILLSERVICE; 8387 mutex_exit(&stp->sd_qlock); 8388 } 8389 8390 /* 8391 * Replace the cred currently in the mblk with a different one. 8392 */ 8393 void 8394 mblk_setcred(mblk_t *mp, cred_t *cr) 8395 { 8396 cred_t *ocr = DB_CRED(mp); 8397 8398 ASSERT(cr != NULL); 8399 8400 if (cr != ocr) { 8401 crhold(mp->b_datap->db_credp = cr); 8402 if (ocr != NULL) 8403 crfree(ocr); 8404 } 8405 } 8406 8407 int 8408 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8409 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value, 8410 uint32_t flags, int km_flags) 8411 { 8412 int rc = 0; 8413 8414 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8415 if (mp->b_datap->db_type == M_DATA) { 8416 /* Associate values for M_DATA type */ 8417 DB_CKSUMSTART(mp) = (intptr_t)start; 8418 DB_CKSUMSTUFF(mp) = (intptr_t)stuff; 8419 DB_CKSUMEND(mp) = (intptr_t)end; 8420 DB_CKSUMFLAGS(mp) = flags; 8421 DB_CKSUM16(mp) = (uint16_t)value; 8422 8423 } else { 8424 pattrinfo_t pa_info; 8425 8426 ASSERT(mmd != NULL); 8427 8428 pa_info.type = PATTR_HCKSUM; 8429 pa_info.len = sizeof (pattr_hcksum_t); 8430 8431 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) { 8432 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf; 8433 8434 hck->hcksum_start_offset = start; 8435 hck->hcksum_stuff_offset = stuff; 8436 hck->hcksum_end_offset = end; 8437 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value; 8438 hck->hcksum_flags = flags; 8439 } else { 8440 rc = -1; 8441 } 8442 } 8443 return (rc); 8444 } 8445 8446 void 8447 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8448 uint32_t *start, uint32_t *stuff, uint32_t *end, 8449 uint32_t *value, uint32_t *flags) 8450 { 8451 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8452 if (mp->b_datap->db_type == M_DATA) { 8453 if (flags != NULL) { 8454 *flags = DB_CKSUMFLAGS(mp); 8455 if (*flags & HCK_PARTIALCKSUM) { 8456 if (start != NULL) 8457 *start = (uint32_t)DB_CKSUMSTART(mp); 8458 if (stuff != NULL) 8459 *stuff = (uint32_t)DB_CKSUMSTUFF(mp); 8460 if (end != NULL) 8461 *end = (uint32_t)DB_CKSUMEND(mp); 8462 if (value != NULL) 8463 *value = (uint32_t)DB_CKSUM16(mp); 8464 } else if ((*flags & HW_LSO) && (value != NULL)) 8465 *value = (uint32_t)DB_LSOMSS(mp); 8466 } 8467 } else { 8468 pattrinfo_t hck_attr = {PATTR_HCKSUM}; 8469 8470 ASSERT(mmd != NULL); 8471 8472 /* get hardware checksum attribute */ 8473 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) { 8474 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf; 8475 8476 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t)); 8477 if (flags != NULL) 8478 *flags = hck->hcksum_flags; 8479 if (start != NULL) 8480 *start = hck->hcksum_start_offset; 8481 if (stuff != NULL) 8482 *stuff = hck->hcksum_stuff_offset; 8483 if (end != NULL) 8484 *end = hck->hcksum_end_offset; 8485 if (value != NULL) 8486 *value = (uint32_t) 8487 hck->hcksum_cksum_val.inet_cksum; 8488 } 8489 } 8490 } 8491 8492 /* 8493 * Checksum buffer *bp for len bytes with psum partial checksum, 8494 * or 0 if none, and return the 16 bit partial checksum. 8495 */ 8496 unsigned 8497 bcksum(uchar_t *bp, int len, unsigned int psum) 8498 { 8499 int odd = len & 1; 8500 extern unsigned int ip_ocsum(); 8501 8502 if (((intptr_t)bp & 1) == 0 && !odd) { 8503 /* 8504 * Bp is 16 bit aligned and len is multiple of 16 bit word. 8505 */ 8506 return (ip_ocsum((ushort_t *)bp, len >> 1, psum)); 8507 } 8508 if (((intptr_t)bp & 1) != 0) { 8509 /* 8510 * Bp isn't 16 bit aligned. 8511 */ 8512 unsigned int tsum; 8513 8514 #ifdef _LITTLE_ENDIAN 8515 psum += *bp; 8516 #else 8517 psum += *bp << 8; 8518 #endif 8519 len--; 8520 bp++; 8521 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0); 8522 psum += (tsum << 8) & 0xffff | (tsum >> 8); 8523 if (len & 1) { 8524 bp += len - 1; 8525 #ifdef _LITTLE_ENDIAN 8526 psum += *bp << 8; 8527 #else 8528 psum += *bp; 8529 #endif 8530 } 8531 } else { 8532 /* 8533 * Bp is 16 bit aligned. 8534 */ 8535 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum); 8536 if (odd) { 8537 bp += len - 1; 8538 #ifdef _LITTLE_ENDIAN 8539 psum += *bp; 8540 #else 8541 psum += *bp << 8; 8542 #endif 8543 } 8544 } 8545 /* 8546 * Normalize psum to 16 bits before returning the new partial 8547 * checksum. The max psum value before normalization is 0x3FDFE. 8548 */ 8549 return ((psum >> 16) + (psum & 0xFFFF)); 8550 } 8551 8552 boolean_t 8553 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd) 8554 { 8555 boolean_t rc; 8556 8557 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8558 if (DB_TYPE(mp) == M_DATA) { 8559 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0); 8560 } else { 8561 pattrinfo_t zcopy_attr = {PATTR_ZCOPY}; 8562 8563 ASSERT(mmd != NULL); 8564 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL); 8565 } 8566 return (rc); 8567 } 8568 8569 void 8570 freemsgchain(mblk_t *mp) 8571 { 8572 mblk_t *next; 8573 8574 while (mp != NULL) { 8575 next = mp->b_next; 8576 mp->b_next = NULL; 8577 8578 freemsg(mp); 8579 mp = next; 8580 } 8581 } 8582 8583 mblk_t * 8584 copymsgchain(mblk_t *mp) 8585 { 8586 mblk_t *nmp = NULL; 8587 mblk_t **nmpp = &nmp; 8588 8589 for (; mp != NULL; mp = mp->b_next) { 8590 if ((*nmpp = copymsg(mp)) == NULL) { 8591 freemsgchain(nmp); 8592 return (NULL); 8593 } 8594 8595 nmpp = &((*nmpp)->b_next); 8596 } 8597 8598 return (nmp); 8599 } 8600 8601 /* NOTE: Do not add code after this point. */ 8602 #undef QLOCK 8603 8604 /* 8605 * replacement for QLOCK macro for those that can't use it. 8606 */ 8607 kmutex_t * 8608 QLOCK(queue_t *q) 8609 { 8610 return (&(q)->q_lock); 8611 } 8612 8613 /* 8614 * Dummy runqueues/queuerun functions functions for backwards compatibility. 8615 */ 8616 #undef runqueues 8617 void 8618 runqueues(void) 8619 { 8620 } 8621 8622 #undef queuerun 8623 void 8624 queuerun(void) 8625 { 8626 } 8627 8628 /* 8629 * Initialize the STR stack instance, which tracks autopush and persistent 8630 * links. 8631 */ 8632 /* ARGSUSED */ 8633 static void * 8634 str_stack_init(netstackid_t stackid, netstack_t *ns) 8635 { 8636 str_stack_t *ss; 8637 int i; 8638 8639 ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP); 8640 ss->ss_netstack = ns; 8641 8642 /* 8643 * set up autopush 8644 */ 8645 sad_initspace(ss); 8646 8647 /* 8648 * set up mux_node structures. 8649 */ 8650 ss->ss_devcnt = devcnt; /* In case it should change before free */ 8651 ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) * 8652 ss->ss_devcnt), KM_SLEEP); 8653 for (i = 0; i < ss->ss_devcnt; i++) 8654 ss->ss_mux_nodes[i].mn_imaj = i; 8655 return (ss); 8656 } 8657 8658 /* 8659 * Note: run at zone shutdown and not destroy so that the PLINKs are 8660 * gone by the time other cleanup happens from the destroy callbacks. 8661 */ 8662 static void 8663 str_stack_shutdown(netstackid_t stackid, void *arg) 8664 { 8665 str_stack_t *ss = (str_stack_t *)arg; 8666 int i; 8667 cred_t *cr; 8668 8669 cr = zone_get_kcred(netstackid_to_zoneid(stackid)); 8670 ASSERT(cr != NULL); 8671 8672 /* Undo all the I_PLINKs for this zone */ 8673 for (i = 0; i < ss->ss_devcnt; i++) { 8674 struct mux_edge *ep; 8675 ldi_handle_t lh; 8676 ldi_ident_t li; 8677 int ret; 8678 int rval; 8679 dev_t rdev; 8680 8681 ep = ss->ss_mux_nodes[i].mn_outp; 8682 if (ep == NULL) 8683 continue; 8684 ret = ldi_ident_from_major((major_t)i, &li); 8685 if (ret != 0) { 8686 continue; 8687 } 8688 rdev = ep->me_dev; 8689 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE, 8690 cr, &lh, li); 8691 if (ret != 0) { 8692 ldi_ident_release(li); 8693 continue; 8694 } 8695 8696 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL, 8697 cr, &rval); 8698 if (ret) { 8699 (void) ldi_close(lh, FREAD|FWRITE, cr); 8700 ldi_ident_release(li); 8701 continue; 8702 } 8703 (void) ldi_close(lh, FREAD|FWRITE, cr); 8704 8705 /* Close layered handles */ 8706 ldi_ident_release(li); 8707 } 8708 crfree(cr); 8709 8710 sad_freespace(ss); 8711 8712 kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt); 8713 ss->ss_mux_nodes = NULL; 8714 } 8715 8716 /* 8717 * Free the structure; str_stack_shutdown did the other cleanup work. 8718 */ 8719 /* ARGSUSED */ 8720 static void 8721 str_stack_fini(netstackid_t stackid, void *arg) 8722 { 8723 str_stack_t *ss = (str_stack_t *)arg; 8724 8725 kmem_free(ss, sizeof (*ss)); 8726 } 8727