1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 22 /* All Rights Reserved */ 23 24 25 /* 26 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 27 * Use is subject to license terms. 28 */ 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 #include <sys/types.h> 33 #include <sys/sysmacros.h> 34 #include <sys/param.h> 35 #include <sys/errno.h> 36 #include <sys/signal.h> 37 #include <sys/proc.h> 38 #include <sys/conf.h> 39 #include <sys/cred.h> 40 #include <sys/user.h> 41 #include <sys/vnode.h> 42 #include <sys/file.h> 43 #include <sys/session.h> 44 #include <sys/stream.h> 45 #include <sys/strsubr.h> 46 #include <sys/stropts.h> 47 #include <sys/poll.h> 48 #include <sys/systm.h> 49 #include <sys/cpuvar.h> 50 #include <sys/uio.h> 51 #include <sys/cmn_err.h> 52 #include <sys/priocntl.h> 53 #include <sys/procset.h> 54 #include <sys/vmem.h> 55 #include <sys/bitmap.h> 56 #include <sys/kmem.h> 57 #include <sys/siginfo.h> 58 #include <sys/vtrace.h> 59 #include <sys/callb.h> 60 #include <sys/debug.h> 61 #include <sys/modctl.h> 62 #include <sys/vmsystm.h> 63 #include <vm/page.h> 64 #include <sys/atomic.h> 65 #include <sys/suntpi.h> 66 #include <sys/strlog.h> 67 #include <sys/promif.h> 68 #include <sys/project.h> 69 #include <sys/vm.h> 70 #include <sys/taskq.h> 71 #include <sys/sunddi.h> 72 #include <sys/sunldi_impl.h> 73 #include <sys/strsun.h> 74 #include <sys/isa_defs.h> 75 #include <sys/multidata.h> 76 #include <sys/pattr.h> 77 #include <sys/strft.h> 78 #include <sys/fs/snode.h> 79 #include <sys/zone.h> 80 #include <sys/open.h> 81 #include <sys/sunldi.h> 82 #include <sys/sad.h> 83 #include <sys/netstack.h> 84 85 #define O_SAMESTR(q) (((q)->q_next) && \ 86 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) 87 88 /* 89 * WARNING: 90 * The variables and routines in this file are private, belonging 91 * to the STREAMS subsystem. These should not be used by modules 92 * or drivers. Compatibility will not be guaranteed. 93 */ 94 95 /* 96 * Id value used to distinguish between different multiplexor links. 97 */ 98 static int32_t lnk_id = 0; 99 100 #define STREAMS_LOPRI MINCLSYSPRI 101 static pri_t streams_lopri = STREAMS_LOPRI; 102 103 #define STRSTAT(x) (str_statistics.x.value.ui64++) 104 typedef struct str_stat { 105 kstat_named_t sqenables; 106 kstat_named_t stenables; 107 kstat_named_t syncqservice; 108 kstat_named_t freebs; 109 kstat_named_t qwr_outer; 110 kstat_named_t rservice; 111 kstat_named_t strwaits; 112 kstat_named_t taskqfails; 113 kstat_named_t bufcalls; 114 kstat_named_t qhelps; 115 kstat_named_t qremoved; 116 kstat_named_t sqremoved; 117 kstat_named_t bcwaits; 118 kstat_named_t sqtoomany; 119 } str_stat_t; 120 121 static str_stat_t str_statistics = { 122 { "sqenables", KSTAT_DATA_UINT64 }, 123 { "stenables", KSTAT_DATA_UINT64 }, 124 { "syncqservice", KSTAT_DATA_UINT64 }, 125 { "freebs", KSTAT_DATA_UINT64 }, 126 { "qwr_outer", KSTAT_DATA_UINT64 }, 127 { "rservice", KSTAT_DATA_UINT64 }, 128 { "strwaits", KSTAT_DATA_UINT64 }, 129 { "taskqfails", KSTAT_DATA_UINT64 }, 130 { "bufcalls", KSTAT_DATA_UINT64 }, 131 { "qhelps", KSTAT_DATA_UINT64 }, 132 { "qremoved", KSTAT_DATA_UINT64 }, 133 { "sqremoved", KSTAT_DATA_UINT64 }, 134 { "bcwaits", KSTAT_DATA_UINT64 }, 135 { "sqtoomany", KSTAT_DATA_UINT64 }, 136 }; 137 138 static kstat_t *str_kstat; 139 140 /* 141 * qrunflag was used previously to control background scheduling of queues. It 142 * is not used anymore, but kept here in case some module still wants to access 143 * it via qready() and setqsched macros. 144 */ 145 char qrunflag; /* Unused */ 146 147 /* 148 * Most of the streams scheduling is done via task queues. Task queues may fail 149 * for non-sleep dispatches, so there are two backup threads servicing failed 150 * requests for queues and syncqs. Both of these threads also service failed 151 * dispatches freebs requests. Queues are put in the list specified by `qhead' 152 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs 153 * requests are put into `freebs_list' which has no tail pointer. All three 154 * lists are protected by a single `service_queue' lock and use 155 * `services_to_run' condition variable for signaling background threads. Use of 156 * a single lock should not be a problem because it is only used under heavy 157 * loads when task queues start to fail and at that time it may be a good idea 158 * to throttle scheduling requests. 159 * 160 * NOTE: queues and syncqs should be scheduled by two separate threads because 161 * queue servicing may be blocked waiting for a syncq which may be also 162 * scheduled for background execution. This may create a deadlock when only one 163 * thread is used for both. 164 */ 165 166 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ 167 168 static kmutex_t service_queue; /* protects all of servicing vars */ 169 static kcondvar_t services_to_run; /* wake up background service thread */ 170 static kcondvar_t syncqs_to_run; /* wake up background service thread */ 171 172 /* 173 * List of queues scheduled for background processing dueue to lack of resources 174 * in the task queues. Protected by service_queue lock; 175 */ 176 static struct queue *qhead; 177 static struct queue *qtail; 178 179 /* 180 * Same list for syncqs 181 */ 182 static syncq_t *sqhead; 183 static syncq_t *sqtail; 184 185 static mblk_t *freebs_list; /* list of buffers to free */ 186 187 /* 188 * Backup threads for servicing queues and syncqs 189 */ 190 kthread_t *streams_qbkgrnd_thread; 191 kthread_t *streams_sqbkgrnd_thread; 192 193 /* 194 * Bufcalls related variables. 195 */ 196 struct bclist strbcalls; /* list of waiting bufcalls */ 197 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ 198 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ 199 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ 200 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ 201 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ 202 203 kmutex_t strresources; /* protects global resources */ 204 kmutex_t muxifier; /* single-threads multiplexor creation */ 205 206 static void *str_stack_init(netstackid_t stackid, netstack_t *ns); 207 static void str_stack_shutdown(netstackid_t stackid, void *arg); 208 static void str_stack_fini(netstackid_t stackid, void *arg); 209 210 extern void time_to_wait(clock_t *, clock_t); 211 212 /* 213 * run_queues is no longer used, but is kept in case some 3-d party 214 * module/driver decides to use it. 215 */ 216 int run_queues = 0; 217 218 /* 219 * sq_max_size is the depth of the syncq (in number of messages) before 220 * qfill_syncq() starts QFULL'ing destination queues. As its primary 221 * consumer - IP is no longer D_MTPERMOD, but there may be other 222 * modules/drivers depend on this syncq flow control, we prefer to 223 * choose a large number as the default value. For potential 224 * performance gain, this value is tunable in /etc/system. 225 */ 226 int sq_max_size = 10000; 227 228 /* 229 * the number of ciputctrl structures per syncq and stream we create when 230 * needed. 231 */ 232 int n_ciputctrl; 233 int max_n_ciputctrl = 16; 234 /* 235 * if n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. 236 */ 237 int min_n_ciputctrl = 2; 238 239 /* 240 * Per-driver/module syncqs 241 * ======================== 242 * 243 * For drivers/modules that use PERMOD or outer syncqs we keep a list of 244 * perdm structures, new entries being added (and new syncqs allocated) when 245 * setq() encounters a module/driver with a streamtab that it hasn't seen 246 * before. 247 * The reason for this mechanism is that some modules and drivers share a 248 * common streamtab and it is necessary for those modules and drivers to also 249 * share a common PERMOD syncq. 250 * 251 * perdm_list --> dm_str == streamtab_1 252 * dm_sq == syncq_1 253 * dm_ref 254 * dm_next --> dm_str == streamtab_2 255 * dm_sq == syncq_2 256 * dm_ref 257 * dm_next --> ... NULL 258 * 259 * The dm_ref field is incremented for each new driver/module that takes 260 * a reference to the perdm structure and hence shares the syncq. 261 * References are held in the fmodsw_impl_t structure for each STREAMS module 262 * or the dev_impl array (indexed by device major number) for each driver. 263 * 264 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL 265 * ^ ^ ^ ^ 266 * | ______________/ | | 267 * | / | | 268 * dev_impl: ...|x|y|... module A module B 269 * 270 * When a module/driver is unloaded the reference count is decremented and, 271 * when it falls to zero, the perdm structure is removed from the list and 272 * the syncq is freed (see rele_dm()). 273 */ 274 perdm_t *perdm_list = NULL; 275 static krwlock_t perdm_rwlock; 276 cdevsw_impl_t *devimpl; 277 278 extern struct qinit strdata; 279 extern struct qinit stwdata; 280 281 static void runservice(queue_t *); 282 static void streams_bufcall_service(void); 283 static void streams_qbkgrnd_service(void); 284 static void streams_sqbkgrnd_service(void); 285 static syncq_t *new_syncq(void); 286 static void free_syncq(syncq_t *); 287 static void outer_insert(syncq_t *, syncq_t *); 288 static void outer_remove(syncq_t *, syncq_t *); 289 static void write_now(syncq_t *); 290 static void clr_qfull(queue_t *); 291 static void enable_svc(queue_t *); 292 static void runbufcalls(void); 293 static void sqenable(syncq_t *); 294 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); 295 static void wait_q_syncq(queue_t *); 296 static void backenable_insertedq(queue_t *); 297 298 static void queue_service(queue_t *); 299 static void stream_service(stdata_t *); 300 static void syncq_service(syncq_t *); 301 static void qwriter_outer_service(syncq_t *); 302 static void mblk_free(mblk_t *); 303 #ifdef DEBUG 304 static int qprocsareon(queue_t *); 305 #endif 306 307 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); 308 static void reset_nfsrv_ptr(queue_t *, queue_t *); 309 310 static void sq_run_events(syncq_t *); 311 static int propagate_syncq(queue_t *); 312 313 static void blocksq(syncq_t *, ushort_t, int); 314 static void unblocksq(syncq_t *, ushort_t, int); 315 static int dropsq(syncq_t *, uint16_t); 316 static void emptysq(syncq_t *); 317 static sqlist_t *sqlist_alloc(struct stdata *, int); 318 static void sqlist_free(sqlist_t *); 319 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); 320 static void sqlist_insert(sqlist_t *, syncq_t *); 321 static void sqlist_insertall(sqlist_t *, queue_t *); 322 323 static void strsetuio(stdata_t *); 324 325 struct kmem_cache *stream_head_cache; 326 struct kmem_cache *queue_cache; 327 struct kmem_cache *syncq_cache; 328 struct kmem_cache *qband_cache; 329 struct kmem_cache *linkinfo_cache; 330 struct kmem_cache *ciputctrl_cache = NULL; 331 332 static linkinfo_t *linkinfo_list; 333 334 /* 335 * Qinit structure and Module_info structures 336 * for passthru read and write queues 337 */ 338 339 static void pass_wput(queue_t *, mblk_t *); 340 static queue_t *link_addpassthru(stdata_t *); 341 static void link_rempassthru(queue_t *); 342 343 struct module_info passthru_info = { 344 0, 345 "passthru", 346 0, 347 INFPSZ, 348 STRHIGH, 349 STRLOW 350 }; 351 352 struct qinit passthru_rinit = { 353 (int (*)())putnext, 354 NULL, 355 NULL, 356 NULL, 357 NULL, 358 &passthru_info, 359 NULL 360 }; 361 362 struct qinit passthru_winit = { 363 (int (*)()) pass_wput, 364 NULL, 365 NULL, 366 NULL, 367 NULL, 368 &passthru_info, 369 NULL 370 }; 371 372 /* 373 * Special form of assertion: verify that X implies Y i.e. when X is true Y 374 * should also be true. 375 */ 376 #define IMPLY(X, Y) ASSERT(!(X) || (Y)) 377 378 /* 379 * Logical equivalence. Verify that both X and Y are either TRUE or FALSE. 380 */ 381 #define EQUIV(X, Y) { IMPLY(X, Y); IMPLY(Y, X); } 382 383 /* 384 * Verify correctness of list head/tail pointers. 385 */ 386 #define LISTCHECK(head, tail, link) { \ 387 EQUIV(head, tail); \ 388 IMPLY(tail != NULL, tail->link == NULL); \ 389 } 390 391 /* 392 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail' 393 * using a `link' field. 394 */ 395 #define ENQUEUE(el, head, tail, link) { \ 396 ASSERT(el->link == NULL); \ 397 LISTCHECK(head, tail, link); \ 398 if (head == NULL) \ 399 head = el; \ 400 else \ 401 tail->link = el; \ 402 tail = el; \ 403 } 404 405 /* 406 * Dequeue the first element of the list denoted by `head' and `tail' pointers 407 * using a `link' field and put result into `el'. 408 */ 409 #define DQ(el, head, tail, link) { \ 410 LISTCHECK(head, tail, link); \ 411 el = head; \ 412 if (head != NULL) { \ 413 head = head->link; \ 414 if (head == NULL) \ 415 tail = NULL; \ 416 el->link = NULL; \ 417 } \ 418 } 419 420 /* 421 * Remove `el' from the list using `chase' and `curr' pointers and return result 422 * in `succeed'. 423 */ 424 #define RMQ(el, head, tail, link, chase, curr, succeed) { \ 425 LISTCHECK(head, tail, link); \ 426 chase = NULL; \ 427 succeed = 0; \ 428 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ 429 chase = curr; \ 430 if (curr != NULL) { \ 431 succeed = 1; \ 432 ASSERT(curr == el); \ 433 if (chase != NULL) \ 434 chase->link = curr->link; \ 435 else \ 436 head = curr->link; \ 437 curr->link = NULL; \ 438 if (curr == tail) \ 439 tail = chase; \ 440 } \ 441 LISTCHECK(head, tail, link); \ 442 } 443 444 /* Handling of delayed messages on the inner syncq. */ 445 446 /* 447 * DEBUG versions should use function versions (to simplify tracing) and 448 * non-DEBUG kernels should use macro versions. 449 */ 450 451 /* 452 * Put a queue on the syncq list of queues. 453 * Assumes SQLOCK held. 454 */ 455 #define SQPUT_Q(sq, qp) \ 456 { \ 457 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 458 if (!(qp->q_sqflags & Q_SQQUEUED)) { \ 459 /* The queue should not be linked anywhere */ \ 460 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ 461 /* Head and tail may only be NULL simultaneously */ \ 462 EQUIV(sq->sq_head, sq->sq_tail); \ 463 /* Queue may be only enqueyed on its syncq */ \ 464 ASSERT(sq == qp->q_syncq); \ 465 /* Check the correctness of SQ_MESSAGES flag */ \ 466 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ 467 /* Sanity check first/last elements of the list */ \ 468 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ 469 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ 470 /* \ 471 * Sanity check of priority field: empty queue should \ 472 * have zero priority \ 473 * and nqueues equal to zero. \ 474 */ \ 475 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ 476 /* Sanity check of sq_nqueues field */ \ 477 EQUIV(sq->sq_head, sq->sq_nqueues); \ 478 if (sq->sq_head == NULL) { \ 479 sq->sq_head = sq->sq_tail = qp; \ 480 sq->sq_flags |= SQ_MESSAGES; \ 481 } else if (qp->q_spri == 0) { \ 482 qp->q_sqprev = sq->sq_tail; \ 483 sq->sq_tail->q_sqnext = qp; \ 484 sq->sq_tail = qp; \ 485 } else { \ 486 /* \ 487 * Put this queue in priority order: higher \ 488 * priority gets closer to the head. \ 489 */ \ 490 queue_t **qpp = &sq->sq_tail; \ 491 queue_t *qnext = NULL; \ 492 \ 493 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ 494 qnext = *qpp; \ 495 qpp = &(*qpp)->q_sqprev; \ 496 } \ 497 qp->q_sqnext = qnext; \ 498 qp->q_sqprev = *qpp; \ 499 if (*qpp != NULL) { \ 500 (*qpp)->q_sqnext = qp; \ 501 } else { \ 502 sq->sq_head = qp; \ 503 sq->sq_pri = sq->sq_head->q_spri; \ 504 } \ 505 *qpp = qp; \ 506 } \ 507 qp->q_sqflags |= Q_SQQUEUED; \ 508 qp->q_sqtstamp = lbolt; \ 509 sq->sq_nqueues++; \ 510 } \ 511 } 512 513 /* 514 * Remove a queue from the syncq list 515 * Assumes SQLOCK held. 516 */ 517 #define SQRM_Q(sq, qp) \ 518 { \ 519 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 520 ASSERT(qp->q_sqflags & Q_SQQUEUED); \ 521 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ 522 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ 523 /* Check that the queue is actually in the list */ \ 524 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ 525 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ 526 ASSERT(sq->sq_nqueues != 0); \ 527 if (qp->q_sqprev == NULL) { \ 528 /* First queue on list, make head q_sqnext */ \ 529 sq->sq_head = qp->q_sqnext; \ 530 } else { \ 531 /* Make prev->next == next */ \ 532 qp->q_sqprev->q_sqnext = qp->q_sqnext; \ 533 } \ 534 if (qp->q_sqnext == NULL) { \ 535 /* Last queue on list, make tail sqprev */ \ 536 sq->sq_tail = qp->q_sqprev; \ 537 } else { \ 538 /* Make next->prev == prev */ \ 539 qp->q_sqnext->q_sqprev = qp->q_sqprev; \ 540 } \ 541 /* clear out references on this queue */ \ 542 qp->q_sqprev = qp->q_sqnext = NULL; \ 543 qp->q_sqflags &= ~Q_SQQUEUED; \ 544 /* If there is nothing queued, clear SQ_MESSAGES */ \ 545 if (sq->sq_head != NULL) { \ 546 sq->sq_pri = sq->sq_head->q_spri; \ 547 } else { \ 548 sq->sq_flags &= ~SQ_MESSAGES; \ 549 sq->sq_pri = 0; \ 550 } \ 551 sq->sq_nqueues--; \ 552 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ 553 (sq->sq_flags & SQ_QUEUED) == 0); \ 554 } 555 556 /* Hide the definition from the header file. */ 557 #ifdef SQPUT_MP 558 #undef SQPUT_MP 559 #endif 560 561 /* 562 * Put a message on the queue syncq. 563 * Assumes QLOCK held. 564 */ 565 #define SQPUT_MP(qp, mp) \ 566 { \ 567 ASSERT(MUTEX_HELD(QLOCK(qp))); \ 568 ASSERT(qp->q_sqhead == NULL || \ 569 (qp->q_sqtail != NULL && \ 570 qp->q_sqtail->b_next == NULL)); \ 571 qp->q_syncqmsgs++; \ 572 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ 573 if (qp->q_sqhead == NULL) { \ 574 qp->q_sqhead = qp->q_sqtail = mp; \ 575 } else { \ 576 qp->q_sqtail->b_next = mp; \ 577 qp->q_sqtail = mp; \ 578 } \ 579 ASSERT(qp->q_syncqmsgs > 0); \ 580 } 581 582 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ 583 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 584 if ((sq)->sq_ciputctrl != NULL) { \ 585 int i; \ 586 int nlocks = (sq)->sq_nciputctrl; \ 587 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 588 ASSERT((sq)->sq_type & SQ_CIPUT); \ 589 for (i = 0; i <= nlocks; i++) { \ 590 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 591 cip[i].ciputctrl_count |= SQ_FASTPUT; \ 592 } \ 593 } \ 594 } 595 596 597 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ 598 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 599 if ((sq)->sq_ciputctrl != NULL) { \ 600 int i; \ 601 int nlocks = (sq)->sq_nciputctrl; \ 602 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 603 ASSERT((sq)->sq_type & SQ_CIPUT); \ 604 for (i = 0; i <= nlocks; i++) { \ 605 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 606 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ 607 } \ 608 } \ 609 } 610 611 /* 612 * Run service procedures for all queues in the stream head. 613 */ 614 #define STR_SERVICE(stp, q) { \ 615 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ 616 while (stp->sd_qhead != NULL) { \ 617 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ 618 ASSERT(stp->sd_nqueues > 0); \ 619 stp->sd_nqueues--; \ 620 ASSERT(!(q->q_flag & QINSERVICE)); \ 621 mutex_exit(&stp->sd_qlock); \ 622 queue_service(q); \ 623 mutex_enter(&stp->sd_qlock); \ 624 } \ 625 ASSERT(stp->sd_nqueues == 0); \ 626 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ 627 } 628 629 /* 630 * constructor/destructor routines for the stream head cache 631 */ 632 /* ARGSUSED */ 633 static int 634 stream_head_constructor(void *buf, void *cdrarg, int kmflags) 635 { 636 stdata_t *stp = buf; 637 638 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); 639 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); 640 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); 641 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); 642 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); 643 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); 644 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); 645 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); 646 stp->sd_wrq = NULL; 647 648 return (0); 649 } 650 651 /* ARGSUSED */ 652 static void 653 stream_head_destructor(void *buf, void *cdrarg) 654 { 655 stdata_t *stp = buf; 656 657 mutex_destroy(&stp->sd_lock); 658 mutex_destroy(&stp->sd_reflock); 659 mutex_destroy(&stp->sd_qlock); 660 cv_destroy(&stp->sd_monitor); 661 cv_destroy(&stp->sd_iocmonitor); 662 cv_destroy(&stp->sd_refmonitor); 663 cv_destroy(&stp->sd_qcv); 664 cv_destroy(&stp->sd_zcopy_wait); 665 } 666 667 /* 668 * constructor/destructor routines for the queue cache 669 */ 670 /* ARGSUSED */ 671 static int 672 queue_constructor(void *buf, void *cdrarg, int kmflags) 673 { 674 queinfo_t *qip = buf; 675 queue_t *qp = &qip->qu_rqueue; 676 queue_t *wqp = &qip->qu_wqueue; 677 syncq_t *sq = &qip->qu_syncq; 678 679 qp->q_first = NULL; 680 qp->q_link = NULL; 681 qp->q_count = 0; 682 qp->q_mblkcnt = 0; 683 qp->q_sqhead = NULL; 684 qp->q_sqtail = NULL; 685 qp->q_sqnext = NULL; 686 qp->q_sqprev = NULL; 687 qp->q_sqflags = 0; 688 qp->q_rwcnt = 0; 689 qp->q_spri = 0; 690 691 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); 692 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); 693 694 wqp->q_first = NULL; 695 wqp->q_link = NULL; 696 wqp->q_count = 0; 697 wqp->q_mblkcnt = 0; 698 wqp->q_sqhead = NULL; 699 wqp->q_sqtail = NULL; 700 wqp->q_sqnext = NULL; 701 wqp->q_sqprev = NULL; 702 wqp->q_sqflags = 0; 703 wqp->q_rwcnt = 0; 704 wqp->q_spri = 0; 705 706 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); 707 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); 708 709 sq->sq_head = NULL; 710 sq->sq_tail = NULL; 711 sq->sq_evhead = NULL; 712 sq->sq_evtail = NULL; 713 sq->sq_callbpend = NULL; 714 sq->sq_outer = NULL; 715 sq->sq_onext = NULL; 716 sq->sq_oprev = NULL; 717 sq->sq_next = NULL; 718 sq->sq_svcflags = 0; 719 sq->sq_servcount = 0; 720 sq->sq_needexcl = 0; 721 sq->sq_nqueues = 0; 722 sq->sq_pri = 0; 723 724 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 725 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 726 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 727 728 return (0); 729 } 730 731 /* ARGSUSED */ 732 static void 733 queue_destructor(void *buf, void *cdrarg) 734 { 735 queinfo_t *qip = buf; 736 queue_t *qp = &qip->qu_rqueue; 737 queue_t *wqp = &qip->qu_wqueue; 738 syncq_t *sq = &qip->qu_syncq; 739 740 ASSERT(qp->q_sqhead == NULL); 741 ASSERT(wqp->q_sqhead == NULL); 742 ASSERT(qp->q_sqnext == NULL); 743 ASSERT(wqp->q_sqnext == NULL); 744 ASSERT(qp->q_rwcnt == 0); 745 ASSERT(wqp->q_rwcnt == 0); 746 747 mutex_destroy(&qp->q_lock); 748 cv_destroy(&qp->q_wait); 749 750 mutex_destroy(&wqp->q_lock); 751 cv_destroy(&wqp->q_wait); 752 753 mutex_destroy(&sq->sq_lock); 754 cv_destroy(&sq->sq_wait); 755 cv_destroy(&sq->sq_exitwait); 756 } 757 758 /* 759 * constructor/destructor routines for the syncq cache 760 */ 761 /* ARGSUSED */ 762 static int 763 syncq_constructor(void *buf, void *cdrarg, int kmflags) 764 { 765 syncq_t *sq = buf; 766 767 bzero(buf, sizeof (syncq_t)); 768 769 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 770 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 771 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 772 773 return (0); 774 } 775 776 /* ARGSUSED */ 777 static void 778 syncq_destructor(void *buf, void *cdrarg) 779 { 780 syncq_t *sq = buf; 781 782 ASSERT(sq->sq_head == NULL); 783 ASSERT(sq->sq_tail == NULL); 784 ASSERT(sq->sq_evhead == NULL); 785 ASSERT(sq->sq_evtail == NULL); 786 ASSERT(sq->sq_callbpend == NULL); 787 ASSERT(sq->sq_callbflags == 0); 788 ASSERT(sq->sq_outer == NULL); 789 ASSERT(sq->sq_onext == NULL); 790 ASSERT(sq->sq_oprev == NULL); 791 ASSERT(sq->sq_next == NULL); 792 ASSERT(sq->sq_needexcl == 0); 793 ASSERT(sq->sq_svcflags == 0); 794 ASSERT(sq->sq_servcount == 0); 795 ASSERT(sq->sq_nqueues == 0); 796 ASSERT(sq->sq_pri == 0); 797 ASSERT(sq->sq_count == 0); 798 ASSERT(sq->sq_rmqcount == 0); 799 ASSERT(sq->sq_cancelid == 0); 800 ASSERT(sq->sq_ciputctrl == NULL); 801 ASSERT(sq->sq_nciputctrl == 0); 802 ASSERT(sq->sq_type == 0); 803 ASSERT(sq->sq_flags == 0); 804 805 mutex_destroy(&sq->sq_lock); 806 cv_destroy(&sq->sq_wait); 807 cv_destroy(&sq->sq_exitwait); 808 } 809 810 /* ARGSUSED */ 811 static int 812 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) 813 { 814 ciputctrl_t *cip = buf; 815 int i; 816 817 for (i = 0; i < n_ciputctrl; i++) { 818 cip[i].ciputctrl_count = SQ_FASTPUT; 819 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); 820 } 821 822 return (0); 823 } 824 825 /* ARGSUSED */ 826 static void 827 ciputctrl_destructor(void *buf, void *cdrarg) 828 { 829 ciputctrl_t *cip = buf; 830 int i; 831 832 for (i = 0; i < n_ciputctrl; i++) { 833 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); 834 mutex_destroy(&cip[i].ciputctrl_lock); 835 } 836 } 837 838 /* 839 * Init routine run from main at boot time. 840 */ 841 void 842 strinit(void) 843 { 844 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 845 846 stream_head_cache = kmem_cache_create("stream_head_cache", 847 sizeof (stdata_t), 0, 848 stream_head_constructor, stream_head_destructor, NULL, 849 NULL, NULL, 0); 850 851 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, 852 queue_constructor, queue_destructor, NULL, NULL, NULL, 0); 853 854 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, 855 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); 856 857 qband_cache = kmem_cache_create("qband_cache", 858 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 859 860 linkinfo_cache = kmem_cache_create("linkinfo_cache", 861 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 862 863 n_ciputctrl = ncpus; 864 n_ciputctrl = 1 << highbit(n_ciputctrl - 1); 865 ASSERT(n_ciputctrl >= 1); 866 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); 867 if (n_ciputctrl >= min_n_ciputctrl) { 868 ciputctrl_cache = kmem_cache_create("ciputctrl_cache", 869 sizeof (ciputctrl_t) * n_ciputctrl, 870 sizeof (ciputctrl_t), ciputctrl_constructor, 871 ciputctrl_destructor, NULL, NULL, NULL, 0); 872 } 873 874 streams_taskq = system_taskq; 875 876 if (streams_taskq == NULL) 877 panic("strinit: no memory for streams taskq!"); 878 879 bc_bkgrnd_thread = thread_create(NULL, 0, 880 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); 881 882 streams_qbkgrnd_thread = thread_create(NULL, 0, 883 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 884 885 streams_sqbkgrnd_thread = thread_create(NULL, 0, 886 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 887 888 /* 889 * Create STREAMS kstats. 890 */ 891 str_kstat = kstat_create("streams", 0, "strstat", 892 "net", KSTAT_TYPE_NAMED, 893 sizeof (str_statistics) / sizeof (kstat_named_t), 894 KSTAT_FLAG_VIRTUAL); 895 896 if (str_kstat != NULL) { 897 str_kstat->ks_data = &str_statistics; 898 kstat_install(str_kstat); 899 } 900 901 /* 902 * TPI support routine initialisation. 903 */ 904 tpi_init(); 905 906 /* 907 * Handle to have autopush and persistent link information per 908 * zone. 909 * Note: uses shutdown hook instead of destroy hook so that the 910 * persistent links can be torn down before the destroy hooks 911 * in the TCP/IP stack are called. 912 */ 913 netstack_register(NS_STR, str_stack_init, str_stack_shutdown, 914 str_stack_fini); 915 } 916 917 void 918 str_sendsig(vnode_t *vp, int event, uchar_t band, int error) 919 { 920 struct stdata *stp; 921 922 ASSERT(vp->v_stream); 923 stp = vp->v_stream; 924 /* Have to hold sd_lock to prevent siglist from changing */ 925 mutex_enter(&stp->sd_lock); 926 if (stp->sd_sigflags & event) 927 strsendsig(stp->sd_siglist, event, band, error); 928 mutex_exit(&stp->sd_lock); 929 } 930 931 /* 932 * Send the "sevent" set of signals to a process. 933 * This might send more than one signal if the process is registered 934 * for multiple events. The caller should pass in an sevent that only 935 * includes the events for which the process has registered. 936 */ 937 static void 938 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, 939 uchar_t band, int error) 940 { 941 ASSERT(MUTEX_HELD(&proc->p_lock)); 942 943 info->si_band = 0; 944 info->si_errno = 0; 945 946 if (sevent & S_ERROR) { 947 sevent &= ~S_ERROR; 948 info->si_code = POLL_ERR; 949 info->si_errno = error; 950 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 951 "strsendsig:proc %p info %p", proc, info); 952 sigaddq(proc, NULL, info, KM_NOSLEEP); 953 info->si_errno = 0; 954 } 955 if (sevent & S_HANGUP) { 956 sevent &= ~S_HANGUP; 957 info->si_code = POLL_HUP; 958 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 959 "strsendsig:proc %p info %p", proc, info); 960 sigaddq(proc, NULL, info, KM_NOSLEEP); 961 } 962 if (sevent & S_HIPRI) { 963 sevent &= ~S_HIPRI; 964 info->si_code = POLL_PRI; 965 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 966 "strsendsig:proc %p info %p", proc, info); 967 sigaddq(proc, NULL, info, KM_NOSLEEP); 968 } 969 if (sevent & S_RDBAND) { 970 sevent &= ~S_RDBAND; 971 if (events & S_BANDURG) 972 sigtoproc(proc, NULL, SIGURG); 973 else 974 sigtoproc(proc, NULL, SIGPOLL); 975 } 976 if (sevent & S_WRBAND) { 977 sevent &= ~S_WRBAND; 978 sigtoproc(proc, NULL, SIGPOLL); 979 } 980 if (sevent & S_INPUT) { 981 sevent &= ~S_INPUT; 982 info->si_code = POLL_IN; 983 info->si_band = band; 984 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 985 "strsendsig:proc %p info %p", proc, info); 986 sigaddq(proc, NULL, info, KM_NOSLEEP); 987 info->si_band = 0; 988 } 989 if (sevent & S_OUTPUT) { 990 sevent &= ~S_OUTPUT; 991 info->si_code = POLL_OUT; 992 info->si_band = band; 993 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 994 "strsendsig:proc %p info %p", proc, info); 995 sigaddq(proc, NULL, info, KM_NOSLEEP); 996 info->si_band = 0; 997 } 998 if (sevent & S_MSG) { 999 sevent &= ~S_MSG; 1000 info->si_code = POLL_MSG; 1001 info->si_band = band; 1002 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1003 "strsendsig:proc %p info %p", proc, info); 1004 sigaddq(proc, NULL, info, KM_NOSLEEP); 1005 info->si_band = 0; 1006 } 1007 if (sevent & S_RDNORM) { 1008 sevent &= ~S_RDNORM; 1009 sigtoproc(proc, NULL, SIGPOLL); 1010 } 1011 if (sevent != 0) { 1012 panic("strsendsig: unknown event(s) %x", sevent); 1013 } 1014 } 1015 1016 /* 1017 * Send SIGPOLL/SIGURG signal to all processes and process groups 1018 * registered on the given signal list that want a signal for at 1019 * least one of the specified events. 1020 * 1021 * Must be called with exclusive access to siglist (caller holding sd_lock). 1022 * 1023 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding 1024 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure 1025 * while it is in the siglist. 1026 * 1027 * For performance reasons (MP scalability) the code drops pidlock 1028 * when sending signals to a single process. 1029 * When sending to a process group the code holds 1030 * pidlock to prevent the membership in the process group from changing 1031 * while walking the p_pglink list. 1032 */ 1033 void 1034 strsendsig(strsig_t *siglist, int event, uchar_t band, int error) 1035 { 1036 strsig_t *ssp; 1037 k_siginfo_t info; 1038 struct pid *pidp; 1039 proc_t *proc; 1040 1041 info.si_signo = SIGPOLL; 1042 info.si_errno = 0; 1043 for (ssp = siglist; ssp; ssp = ssp->ss_next) { 1044 int sevent; 1045 1046 sevent = ssp->ss_events & event; 1047 if (sevent == 0) 1048 continue; 1049 1050 if ((pidp = ssp->ss_pidp) == NULL) { 1051 /* pid was released but still on event list */ 1052 continue; 1053 } 1054 1055 1056 if (ssp->ss_pid > 0) { 1057 /* 1058 * XXX This unfortunately still generates 1059 * a signal when a fd is closed but 1060 * the proc is active. 1061 */ 1062 ASSERT(ssp->ss_pid == pidp->pid_id); 1063 1064 mutex_enter(&pidlock); 1065 proc = prfind_zone(pidp->pid_id, ALL_ZONES); 1066 if (proc == NULL) { 1067 mutex_exit(&pidlock); 1068 continue; 1069 } 1070 mutex_enter(&proc->p_lock); 1071 mutex_exit(&pidlock); 1072 dosendsig(proc, ssp->ss_events, sevent, &info, 1073 band, error); 1074 mutex_exit(&proc->p_lock); 1075 } else { 1076 /* 1077 * Send to process group. Hold pidlock across 1078 * calls to dosendsig(). 1079 */ 1080 pid_t pgrp = -ssp->ss_pid; 1081 1082 mutex_enter(&pidlock); 1083 proc = pgfind_zone(pgrp, ALL_ZONES); 1084 while (proc != NULL) { 1085 mutex_enter(&proc->p_lock); 1086 dosendsig(proc, ssp->ss_events, sevent, 1087 &info, band, error); 1088 mutex_exit(&proc->p_lock); 1089 proc = proc->p_pglink; 1090 } 1091 mutex_exit(&pidlock); 1092 } 1093 } 1094 } 1095 1096 /* 1097 * Attach a stream device or module. 1098 * qp is a read queue; the new queue goes in so its next 1099 * read ptr is the argument, and the write queue corresponding 1100 * to the argument points to this queue. Return 0 on success, 1101 * or a non-zero errno on failure. 1102 */ 1103 int 1104 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, 1105 boolean_t is_insert) 1106 { 1107 major_t major; 1108 cdevsw_impl_t *dp; 1109 struct streamtab *str; 1110 queue_t *rq; 1111 queue_t *wrq; 1112 uint32_t qflag; 1113 uint32_t sqtype; 1114 perdm_t *dmp; 1115 int error; 1116 int sflag; 1117 1118 rq = allocq(); 1119 wrq = _WR(rq); 1120 STREAM(rq) = STREAM(wrq) = STREAM(qp); 1121 1122 if (fp != NULL) { 1123 str = fp->f_str; 1124 qflag = fp->f_qflag; 1125 sqtype = fp->f_sqtype; 1126 dmp = fp->f_dmp; 1127 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 1128 sflag = MODOPEN; 1129 1130 /* 1131 * stash away a pointer to the module structure so we can 1132 * unref it in qdetach. 1133 */ 1134 rq->q_fp = fp; 1135 } else { 1136 ASSERT(!is_insert); 1137 1138 major = getmajor(*devp); 1139 dp = &devimpl[major]; 1140 1141 str = dp->d_str; 1142 ASSERT(str == STREAMSTAB(major)); 1143 1144 qflag = dp->d_qflag; 1145 ASSERT(qflag & QISDRV); 1146 sqtype = dp->d_sqtype; 1147 1148 /* create perdm_t if needed */ 1149 if (NEED_DM(dp->d_dmp, qflag)) 1150 dp->d_dmp = hold_dm(str, qflag, sqtype); 1151 1152 dmp = dp->d_dmp; 1153 sflag = 0; 1154 } 1155 1156 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, 1157 "qattach:qflag == %X(%X)", qflag, *devp); 1158 1159 /* setq might sleep in allocator - avoid holding locks. */ 1160 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); 1161 1162 /* 1163 * Before calling the module's open routine, set up the q_next 1164 * pointer for inserting a module in the middle of a stream. 1165 * 1166 * Note that we can always set _QINSERTING and set up q_next 1167 * pointer for both inserting and pushing a module. Then there 1168 * is no need for the is_insert parameter. In insertq(), called 1169 * by qprocson(), assume that q_next of the new module always points 1170 * to the correct queue and use it for insertion. Everything should 1171 * work out fine. But in the first release of _I_INSERT, we 1172 * distinguish between inserting and pushing to make sure that 1173 * pushing a module follows the same code path as before. 1174 */ 1175 if (is_insert) { 1176 rq->q_flag |= _QINSERTING; 1177 rq->q_next = qp; 1178 } 1179 1180 /* 1181 * If there is an outer perimeter get exclusive access during 1182 * the open procedure. Bump up the reference count on the queue. 1183 */ 1184 entersq(rq->q_syncq, SQ_OPENCLOSE); 1185 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); 1186 if (error != 0) 1187 goto failed; 1188 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1189 ASSERT(qprocsareon(rq)); 1190 return (0); 1191 1192 failed: 1193 rq->q_flag &= ~_QINSERTING; 1194 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) 1195 qprocsoff(rq); 1196 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1197 rq->q_next = wrq->q_next = NULL; 1198 qdetach(rq, 0, 0, crp, B_FALSE); 1199 return (error); 1200 } 1201 1202 /* 1203 * Handle second open of stream. For modules, set the 1204 * last argument to MODOPEN and do not pass any open flags. 1205 * Ignore dummydev since this is not the first open. 1206 */ 1207 int 1208 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) 1209 { 1210 int error; 1211 dev_t dummydev; 1212 queue_t *wqp = _WR(qp); 1213 1214 ASSERT(qp->q_flag & QREADR); 1215 entersq(qp->q_syncq, SQ_OPENCLOSE); 1216 1217 dummydev = *devp; 1218 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, 1219 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { 1220 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1221 mutex_enter(&STREAM(qp)->sd_lock); 1222 qp->q_stream->sd_flag |= STREOPENFAIL; 1223 mutex_exit(&STREAM(qp)->sd_lock); 1224 return (error); 1225 } 1226 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1227 1228 /* 1229 * successful open should have done qprocson() 1230 */ 1231 ASSERT(qprocsareon(_RD(qp))); 1232 return (0); 1233 } 1234 1235 /* 1236 * Detach a stream module or device. 1237 * If clmode == 1 then the module or driver was opened and its 1238 * close routine must be called. If clmode == 0, the module 1239 * or driver was never opened or the open failed, and so its close 1240 * should not be called. 1241 */ 1242 void 1243 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) 1244 { 1245 queue_t *wqp = _WR(qp); 1246 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); 1247 1248 if (STREAM_NEEDSERVICE(STREAM(qp))) 1249 stream_runservice(STREAM(qp)); 1250 1251 if (clmode) { 1252 /* 1253 * Make sure that all the messages on the write side syncq are 1254 * processed and nothing is left. Since we are closing, no new 1255 * messages may appear there. 1256 */ 1257 wait_q_syncq(wqp); 1258 1259 entersq(qp->q_syncq, SQ_OPENCLOSE); 1260 if (is_remove) { 1261 mutex_enter(QLOCK(qp)); 1262 qp->q_flag |= _QREMOVING; 1263 mutex_exit(QLOCK(qp)); 1264 } 1265 (*qp->q_qinfo->qi_qclose)(qp, flag, crp); 1266 /* 1267 * Check that qprocsoff() was actually called. 1268 */ 1269 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); 1270 1271 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1272 } else { 1273 disable_svc(qp); 1274 } 1275 1276 /* 1277 * Allow any threads blocked in entersq to proceed and discover 1278 * the QWCLOSE is set. 1279 * Note: This assumes that all users of entersq check QWCLOSE. 1280 * Currently runservice is the only entersq that can happen 1281 * after removeq has finished. 1282 * Removeq will have discarded all messages destined to the closing 1283 * pair of queues from the syncq. 1284 * NOTE: Calling a function inside an assert is unconventional. 1285 * However, it does not cause any problem since flush_syncq() does 1286 * not change any state except when it returns non-zero i.e. 1287 * when the assert will trigger. 1288 */ 1289 ASSERT(flush_syncq(qp->q_syncq, qp) == 0); 1290 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); 1291 ASSERT((qp->q_flag & QPERMOD) || 1292 ((qp->q_syncq->sq_head == NULL) && 1293 (wqp->q_syncq->sq_head == NULL))); 1294 1295 /* release any fmodsw_impl_t structure held on behalf of the queue */ 1296 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); 1297 if (qp->q_fp != NULL) 1298 fmodsw_rele(qp->q_fp); 1299 1300 /* freeq removes us from the outer perimeter if any */ 1301 freeq(qp); 1302 } 1303 1304 /* Prevent service procedures from being called */ 1305 void 1306 disable_svc(queue_t *qp) 1307 { 1308 queue_t *wqp = _WR(qp); 1309 1310 ASSERT(qp->q_flag & QREADR); 1311 mutex_enter(QLOCK(qp)); 1312 qp->q_flag |= QWCLOSE; 1313 mutex_exit(QLOCK(qp)); 1314 mutex_enter(QLOCK(wqp)); 1315 wqp->q_flag |= QWCLOSE; 1316 mutex_exit(QLOCK(wqp)); 1317 } 1318 1319 /* allow service procedures to be called again */ 1320 void 1321 enable_svc(queue_t *qp) 1322 { 1323 queue_t *wqp = _WR(qp); 1324 1325 ASSERT(qp->q_flag & QREADR); 1326 mutex_enter(QLOCK(qp)); 1327 qp->q_flag &= ~QWCLOSE; 1328 mutex_exit(QLOCK(qp)); 1329 mutex_enter(QLOCK(wqp)); 1330 wqp->q_flag &= ~QWCLOSE; 1331 mutex_exit(QLOCK(wqp)); 1332 } 1333 1334 /* 1335 * Remove queue from qhead/qtail if it is enabled. 1336 * Only reset QENAB if the queue was removed from the runlist. 1337 * A queue goes through 3 stages: 1338 * It is on the service list and QENAB is set. 1339 * It is removed from the service list but QENAB is still set. 1340 * QENAB gets changed to QINSERVICE. 1341 * QINSERVICE is reset (when the service procedure is done) 1342 * Thus we can not reset QENAB unless we actually removed it from the service 1343 * queue. 1344 */ 1345 void 1346 remove_runlist(queue_t *qp) 1347 { 1348 if (qp->q_flag & QENAB && qhead != NULL) { 1349 queue_t *q_chase; 1350 queue_t *q_curr; 1351 int removed; 1352 1353 mutex_enter(&service_queue); 1354 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); 1355 mutex_exit(&service_queue); 1356 if (removed) { 1357 STRSTAT(qremoved); 1358 qp->q_flag &= ~QENAB; 1359 } 1360 } 1361 } 1362 1363 1364 /* 1365 * wait for any pending service processing to complete. 1366 * The removal of queues from the runlist is not atomic with the 1367 * clearing of the QENABLED flag and setting the INSERVICE flag. 1368 * consequently it is possible for remove_runlist in strclose 1369 * to not find the queue on the runlist but for it to be QENABLED 1370 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED 1371 * as well as INSERVICE. 1372 */ 1373 void 1374 wait_svc(queue_t *qp) 1375 { 1376 queue_t *wqp = _WR(qp); 1377 1378 ASSERT(qp->q_flag & QREADR); 1379 1380 /* 1381 * Try to remove queues from qhead/qtail list. 1382 */ 1383 if (qhead != NULL) { 1384 remove_runlist(qp); 1385 remove_runlist(wqp); 1386 } 1387 /* 1388 * Wait till the syncqs associated with the queue 1389 * will dissapear from background processing list. 1390 * This only needs to be done for non-PERMOD perimeters since 1391 * for PERMOD perimeters the syncq may be shared and will only be freed 1392 * when the last module/driver is unloaded. 1393 * If for PERMOD perimeters queue was on the syncq list, removeq() 1394 * should call propagate_syncq() or drain_syncq() for it. Both of these 1395 * function remove the queue from its syncq list, so sqthread will not 1396 * try to access the queue. 1397 */ 1398 if (!(qp->q_flag & QPERMOD)) { 1399 syncq_t *rsq = qp->q_syncq; 1400 syncq_t *wsq = wqp->q_syncq; 1401 1402 /* 1403 * Disable rsq and wsq and wait for any background processing of 1404 * syncq to complete. 1405 */ 1406 wait_sq_svc(rsq); 1407 if (wsq != rsq) 1408 wait_sq_svc(wsq); 1409 } 1410 1411 mutex_enter(QLOCK(qp)); 1412 while (qp->q_flag & (QINSERVICE|QENAB)) 1413 cv_wait(&qp->q_wait, QLOCK(qp)); 1414 mutex_exit(QLOCK(qp)); 1415 mutex_enter(QLOCK(wqp)); 1416 while (wqp->q_flag & (QINSERVICE|QENAB)) 1417 cv_wait(&wqp->q_wait, QLOCK(wqp)); 1418 mutex_exit(QLOCK(wqp)); 1419 } 1420 1421 /* 1422 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'. 1423 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may 1424 * also be set, and is passed through to allocb_cred_wait(). 1425 * 1426 * Returns errno on failure, zero on success. 1427 */ 1428 int 1429 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) 1430 { 1431 mblk_t *tmp; 1432 ssize_t count; 1433 size_t n; 1434 int error = 0; 1435 1436 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || 1437 (flag & (U_TO_K | K_TO_K)) == K_TO_K); 1438 1439 if (bp->b_datap->db_type == M_IOCTL) { 1440 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1441 } else { 1442 ASSERT(bp->b_datap->db_type == M_COPYIN); 1443 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1444 } 1445 /* 1446 * strdoioctl validates ioc_count, so if this assert fails it 1447 * cannot be due to user error. 1448 */ 1449 ASSERT(count >= 0); 1450 1451 while (count > 0) { 1452 n = MIN(MAXIOCBSZ, count); 1453 if ((tmp = allocb_cred_wait(n, (flag & STR_NOSIG), &error, 1454 cr)) == NULL) { 1455 return (error); 1456 } 1457 error = strcopyin(arg, tmp->b_wptr, n, flag & (U_TO_K|K_TO_K)); 1458 if (error != 0) { 1459 freeb(tmp); 1460 return (error); 1461 } 1462 arg += n; 1463 DB_CPID(tmp) = curproc->p_pid; 1464 tmp->b_wptr += n; 1465 count -= n; 1466 bp = (bp->b_cont = tmp); 1467 } 1468 1469 return (0); 1470 } 1471 1472 /* 1473 * Copy ioctl data to user-land. Return non-zero errno on failure, 1474 * 0 for success. 1475 */ 1476 int 1477 getiocd(mblk_t *bp, char *arg, int copymode) 1478 { 1479 ssize_t count; 1480 size_t n; 1481 int error; 1482 1483 if (bp->b_datap->db_type == M_IOCACK) 1484 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1485 else { 1486 ASSERT(bp->b_datap->db_type == M_COPYOUT); 1487 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1488 } 1489 ASSERT(count >= 0); 1490 1491 for (bp = bp->b_cont; bp && count; 1492 count -= n, bp = bp->b_cont, arg += n) { 1493 n = MIN(count, bp->b_wptr - bp->b_rptr); 1494 error = strcopyout(bp->b_rptr, arg, n, copymode); 1495 if (error) 1496 return (error); 1497 } 1498 ASSERT(count == 0); 1499 return (0); 1500 } 1501 1502 /* 1503 * Allocate a linkinfo entry given the write queue of the 1504 * bottom module of the top stream and the write queue of the 1505 * stream head of the bottom stream. 1506 */ 1507 linkinfo_t * 1508 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) 1509 { 1510 linkinfo_t *linkp; 1511 1512 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); 1513 1514 linkp->li_lblk.l_qtop = qup; 1515 linkp->li_lblk.l_qbot = qdown; 1516 linkp->li_fpdown = fpdown; 1517 1518 mutex_enter(&strresources); 1519 linkp->li_next = linkinfo_list; 1520 linkp->li_prev = NULL; 1521 if (linkp->li_next) 1522 linkp->li_next->li_prev = linkp; 1523 linkinfo_list = linkp; 1524 linkp->li_lblk.l_index = ++lnk_id; 1525 ASSERT(lnk_id != 0); /* this should never wrap in practice */ 1526 mutex_exit(&strresources); 1527 1528 return (linkp); 1529 } 1530 1531 /* 1532 * Free a linkinfo entry. 1533 */ 1534 void 1535 lbfree(linkinfo_t *linkp) 1536 { 1537 mutex_enter(&strresources); 1538 if (linkp->li_next) 1539 linkp->li_next->li_prev = linkp->li_prev; 1540 if (linkp->li_prev) 1541 linkp->li_prev->li_next = linkp->li_next; 1542 else 1543 linkinfo_list = linkp->li_next; 1544 mutex_exit(&strresources); 1545 1546 kmem_cache_free(linkinfo_cache, linkp); 1547 } 1548 1549 /* 1550 * Check for a potential linking cycle. 1551 * Return 1 if a link will result in a cycle, 1552 * and 0 otherwise. 1553 */ 1554 int 1555 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss) 1556 { 1557 struct mux_node *np; 1558 struct mux_edge *ep; 1559 int i; 1560 major_t lomaj; 1561 major_t upmaj; 1562 /* 1563 * if the lower stream is a pipe/FIFO, return, since link 1564 * cycles can not happen on pipes/FIFOs 1565 */ 1566 if (lostp->sd_vnode->v_type == VFIFO) 1567 return (0); 1568 1569 for (i = 0; i < ss->ss_devcnt; i++) { 1570 np = &ss->ss_mux_nodes[i]; 1571 MUX_CLEAR(np); 1572 } 1573 lomaj = getmajor(lostp->sd_vnode->v_rdev); 1574 upmaj = getmajor(upstp->sd_vnode->v_rdev); 1575 np = &ss->ss_mux_nodes[lomaj]; 1576 for (;;) { 1577 if (!MUX_DIDVISIT(np)) { 1578 if (np->mn_imaj == upmaj) 1579 return (1); 1580 if (np->mn_outp == NULL) { 1581 MUX_VISIT(np); 1582 if (np->mn_originp == NULL) 1583 return (0); 1584 np = np->mn_originp; 1585 continue; 1586 } 1587 MUX_VISIT(np); 1588 np->mn_startp = np->mn_outp; 1589 } else { 1590 if (np->mn_startp == NULL) { 1591 if (np->mn_originp == NULL) 1592 return (0); 1593 else { 1594 np = np->mn_originp; 1595 continue; 1596 } 1597 } 1598 /* 1599 * If ep->me_nodep is a FIFO (me_nodep == NULL), 1600 * ignore the edge and move on. ep->me_nodep gets 1601 * set to NULL in mux_addedge() if it is a FIFO. 1602 * 1603 */ 1604 ep = np->mn_startp; 1605 np->mn_startp = ep->me_nextp; 1606 if (ep->me_nodep == NULL) 1607 continue; 1608 ep->me_nodep->mn_originp = np; 1609 np = ep->me_nodep; 1610 } 1611 } 1612 } 1613 1614 /* 1615 * Find linkinfo entry corresponding to the parameters. 1616 */ 1617 linkinfo_t * 1618 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss) 1619 { 1620 linkinfo_t *linkp; 1621 struct mux_edge *mep; 1622 struct mux_node *mnp; 1623 queue_t *qup; 1624 1625 mutex_enter(&strresources); 1626 if ((type & LINKTYPEMASK) == LINKNORMAL) { 1627 qup = getendq(stp->sd_wrq); 1628 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1629 if ((qup == linkp->li_lblk.l_qtop) && 1630 (!index || (index == linkp->li_lblk.l_index))) { 1631 mutex_exit(&strresources); 1632 return (linkp); 1633 } 1634 } 1635 } else { 1636 ASSERT((type & LINKTYPEMASK) == LINKPERSIST); 1637 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; 1638 mep = mnp->mn_outp; 1639 while (mep) { 1640 if ((index == 0) || (index == mep->me_muxid)) 1641 break; 1642 mep = mep->me_nextp; 1643 } 1644 if (!mep) { 1645 mutex_exit(&strresources); 1646 return (NULL); 1647 } 1648 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1649 if ((!linkp->li_lblk.l_qtop) && 1650 (mep->me_muxid == linkp->li_lblk.l_index)) { 1651 mutex_exit(&strresources); 1652 return (linkp); 1653 } 1654 } 1655 } 1656 mutex_exit(&strresources); 1657 return (NULL); 1658 } 1659 1660 /* 1661 * Given a queue ptr, follow the chain of q_next pointers until you reach the 1662 * last queue on the chain and return it. 1663 */ 1664 queue_t * 1665 getendq(queue_t *q) 1666 { 1667 ASSERT(q != NULL); 1668 while (_SAMESTR(q)) 1669 q = q->q_next; 1670 return (q); 1671 } 1672 1673 /* 1674 * wait for the syncq count to drop to zero. 1675 * sq could be either outer or inner. 1676 */ 1677 1678 static void 1679 wait_syncq(syncq_t *sq) 1680 { 1681 uint16_t count; 1682 1683 mutex_enter(SQLOCK(sq)); 1684 count = sq->sq_count; 1685 SQ_PUTLOCKS_ENTER(sq); 1686 SUM_SQ_PUTCOUNTS(sq, count); 1687 while (count != 0) { 1688 sq->sq_flags |= SQ_WANTWAKEUP; 1689 SQ_PUTLOCKS_EXIT(sq); 1690 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1691 count = sq->sq_count; 1692 SQ_PUTLOCKS_ENTER(sq); 1693 SUM_SQ_PUTCOUNTS(sq, count); 1694 } 1695 SQ_PUTLOCKS_EXIT(sq); 1696 mutex_exit(SQLOCK(sq)); 1697 } 1698 1699 /* 1700 * Wait while there are any messages for the queue in its syncq. 1701 */ 1702 static void 1703 wait_q_syncq(queue_t *q) 1704 { 1705 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1706 syncq_t *sq = q->q_syncq; 1707 1708 mutex_enter(SQLOCK(sq)); 1709 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1710 sq->sq_flags |= SQ_WANTWAKEUP; 1711 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1712 } 1713 mutex_exit(SQLOCK(sq)); 1714 } 1715 } 1716 1717 1718 int 1719 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, 1720 int lhlink) 1721 { 1722 struct stdata *stp; 1723 struct strioctl strioc; 1724 struct linkinfo *linkp; 1725 struct stdata *stpdown; 1726 struct streamtab *str; 1727 queue_t *passq; 1728 syncq_t *passyncq; 1729 queue_t *rq; 1730 cdevsw_impl_t *dp; 1731 uint32_t qflag; 1732 uint32_t sqtype; 1733 perdm_t *dmp; 1734 int error = 0; 1735 netstack_t *ns; 1736 str_stack_t *ss; 1737 1738 stp = vp->v_stream; 1739 TRACE_1(TR_FAC_STREAMS_FR, 1740 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); 1741 /* 1742 * Test for invalid upper stream 1743 */ 1744 if (stp->sd_flag & STRHUP) { 1745 return (ENXIO); 1746 } 1747 if (vp->v_type == VFIFO) { 1748 return (EINVAL); 1749 } 1750 if (stp->sd_strtab == NULL) { 1751 return (EINVAL); 1752 } 1753 if (!stp->sd_strtab->st_muxwinit) { 1754 return (EINVAL); 1755 } 1756 if (fpdown == NULL) { 1757 return (EBADF); 1758 } 1759 ns = netstack_find_by_cred(crp); 1760 ASSERT(ns != NULL); 1761 ss = ns->netstack_str; 1762 ASSERT(ss != NULL); 1763 1764 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) { 1765 netstack_rele(ss->ss_netstack); 1766 return (EINVAL); 1767 } 1768 mutex_enter(&muxifier); 1769 if (stp->sd_flag & STPLEX) { 1770 mutex_exit(&muxifier); 1771 netstack_rele(ss->ss_netstack); 1772 return (ENXIO); 1773 } 1774 1775 /* 1776 * Test for invalid lower stream. 1777 * The check for the v_type != VFIFO and having a major 1778 * number not >= devcnt is done to avoid problems with 1779 * adding mux_node entry past the end of mux_nodes[]. 1780 * For FIFO's we don't add an entry so this isn't a 1781 * problem. 1782 */ 1783 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || 1784 (stpdown == stp) || (stpdown->sd_flag & 1785 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || 1786 ((stpdown->sd_vnode->v_type != VFIFO) && 1787 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) || 1788 linkcycle(stp, stpdown, ss)) { 1789 mutex_exit(&muxifier); 1790 netstack_rele(ss->ss_netstack); 1791 return (EINVAL); 1792 } 1793 TRACE_1(TR_FAC_STREAMS_FR, 1794 TR_STPDOWN, "stpdown:%p", stpdown); 1795 rq = getendq(stp->sd_wrq); 1796 if (cmd == I_PLINK) 1797 rq = NULL; 1798 1799 linkp = alloclink(rq, stpdown->sd_wrq, fpdown); 1800 1801 strioc.ic_cmd = cmd; 1802 strioc.ic_timout = INFTIM; 1803 strioc.ic_len = sizeof (struct linkblk); 1804 strioc.ic_dp = (char *)&linkp->li_lblk; 1805 1806 /* 1807 * STRPLUMB protects plumbing changes and should be set before 1808 * link_addpassthru()/link_rempassthru() are called, so it is set here 1809 * and cleared in the end of mlink when passthru queue is removed. 1810 * Setting of STRPLUMB prevents reopens of the stream while passthru 1811 * queue is in-place (it is not a proper module and doesn't have open 1812 * entry point). 1813 * 1814 * STPLEX prevents any threads from entering the stream from above. It 1815 * can't be set before the call to link_addpassthru() because putnext 1816 * from below may cause stream head I/O routines to be called and these 1817 * routines assert that STPLEX is not set. After link_addpassthru() 1818 * nothing may come from below since the pass queue syncq is blocked. 1819 * Note also that STPLEX should be cleared before the call to 1820 * link_remmpassthru() since when messages start flowing to the stream 1821 * head (e.g. because of message propagation from the pass queue) stream 1822 * head I/O routines may be called with STPLEX flag set. 1823 * 1824 * When STPLEX is set, nothing may come into the stream from above and 1825 * it is safe to do a setq which will change stream head. So, the 1826 * correct sequence of actions is: 1827 * 1828 * 1) Set STRPLUMB 1829 * 2) Call link_addpassthru() 1830 * 3) Set STPLEX 1831 * 4) Call setq and update the stream state 1832 * 5) Clear STPLEX 1833 * 6) Call link_rempassthru() 1834 * 7) Clear STRPLUMB 1835 * 1836 * The same sequence applies to munlink() code. 1837 */ 1838 mutex_enter(&stpdown->sd_lock); 1839 stpdown->sd_flag |= STRPLUMB; 1840 mutex_exit(&stpdown->sd_lock); 1841 /* 1842 * Add passthru queue below lower mux. This will block 1843 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 1844 */ 1845 passq = link_addpassthru(stpdown); 1846 1847 mutex_enter(&stpdown->sd_lock); 1848 stpdown->sd_flag |= STPLEX; 1849 mutex_exit(&stpdown->sd_lock); 1850 1851 rq = _RD(stpdown->sd_wrq); 1852 /* 1853 * There may be messages in the streamhead's syncq due to messages 1854 * that arrived before link_addpassthru() was done. To avoid 1855 * background processing of the syncq happening simultaneous with 1856 * setq processing, we disable the streamhead syncq and wait until 1857 * existing background thread finishes working on it. 1858 */ 1859 wait_sq_svc(rq->q_syncq); 1860 passyncq = passq->q_syncq; 1861 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1862 blocksq(passyncq, SQ_BLOCKED, 0); 1863 1864 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 1865 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 1866 rq->q_ptr = _WR(rq)->q_ptr = NULL; 1867 1868 /* setq might sleep in allocator - avoid holding locks. */ 1869 /* Note: we are holding muxifier here. */ 1870 1871 str = stp->sd_strtab; 1872 dp = &devimpl[getmajor(vp->v_rdev)]; 1873 ASSERT(dp->d_str == str); 1874 1875 qflag = dp->d_qflag; 1876 sqtype = dp->d_sqtype; 1877 1878 /* create perdm_t if needed */ 1879 if (NEED_DM(dp->d_dmp, qflag)) 1880 dp->d_dmp = hold_dm(str, qflag, sqtype); 1881 1882 dmp = dp->d_dmp; 1883 1884 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, 1885 B_TRUE); 1886 1887 /* 1888 * XXX Remove any "odd" messages from the queue. 1889 * Keep only M_DATA, M_PROTO, M_PCPROTO. 1890 */ 1891 error = strdoioctl(stp, &strioc, FNATIVE, 1892 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 1893 if (error != 0) { 1894 lbfree(linkp); 1895 1896 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1897 blocksq(passyncq, SQ_BLOCKED, 0); 1898 /* 1899 * Restore the stream head queue and then remove 1900 * the passq. Turn off STPLEX before we turn on 1901 * the stream by removing the passq. 1902 */ 1903 rq->q_ptr = _WR(rq)->q_ptr = stpdown; 1904 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, 1905 B_TRUE); 1906 1907 mutex_enter(&stpdown->sd_lock); 1908 stpdown->sd_flag &= ~STPLEX; 1909 mutex_exit(&stpdown->sd_lock); 1910 1911 link_rempassthru(passq); 1912 1913 mutex_enter(&stpdown->sd_lock); 1914 stpdown->sd_flag &= ~STRPLUMB; 1915 /* Wakeup anyone waiting for STRPLUMB to clear. */ 1916 cv_broadcast(&stpdown->sd_monitor); 1917 mutex_exit(&stpdown->sd_lock); 1918 1919 mutex_exit(&muxifier); 1920 netstack_rele(ss->ss_netstack); 1921 return (error); 1922 } 1923 mutex_enter(&fpdown->f_tlock); 1924 fpdown->f_count++; 1925 mutex_exit(&fpdown->f_tlock); 1926 1927 /* 1928 * if we've made it here the linkage is all set up so we should also 1929 * set up the layered driver linkages 1930 */ 1931 1932 ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); 1933 if (cmd == I_LINK) { 1934 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); 1935 } else { 1936 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); 1937 } 1938 1939 link_rempassthru(passq); 1940 1941 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss); 1942 1943 /* 1944 * Mark the upper stream as having dependent links 1945 * so that strclose can clean it up. 1946 */ 1947 if (cmd == I_LINK) { 1948 mutex_enter(&stp->sd_lock); 1949 stp->sd_flag |= STRHASLINKS; 1950 mutex_exit(&stp->sd_lock); 1951 } 1952 /* 1953 * Wake up any other processes that may have been 1954 * waiting on the lower stream. These will all 1955 * error out. 1956 */ 1957 mutex_enter(&stpdown->sd_lock); 1958 /* The passthru module is removed so we may release STRPLUMB */ 1959 stpdown->sd_flag &= ~STRPLUMB; 1960 cv_broadcast(&rq->q_wait); 1961 cv_broadcast(&_WR(rq)->q_wait); 1962 cv_broadcast(&stpdown->sd_monitor); 1963 mutex_exit(&stpdown->sd_lock); 1964 mutex_exit(&muxifier); 1965 *rvalp = linkp->li_lblk.l_index; 1966 netstack_rele(ss->ss_netstack); 1967 return (0); 1968 } 1969 1970 int 1971 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) 1972 { 1973 int ret; 1974 struct file *fpdown; 1975 1976 fpdown = getf(arg); 1977 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); 1978 if (fpdown != NULL) 1979 releasef(arg); 1980 return (ret); 1981 } 1982 1983 /* 1984 * Unlink a multiplexor link. Stp is the controlling stream for the 1985 * link, and linkp points to the link's entry in the linkinfo list. 1986 * The muxifier lock must be held on entry and is dropped on exit. 1987 * 1988 * NOTE : Currently it is assumed that mux would process all the messages 1989 * sitting on it's queue before ACKing the UNLINK. It is the responsibility 1990 * of the mux to handle all the messages that arrive before UNLINK. 1991 * If the mux has to send down messages on its lower stream before 1992 * ACKing I_UNLINK, then it *should* know to handle messages even 1993 * after the UNLINK is acked (actually it should be able to handle till we 1994 * re-block the read side of the pass queue here). If the mux does not 1995 * open up the lower stream, any messages that arrive during UNLINK 1996 * will be put in the stream head. In the case of lower stream opening 1997 * up, some messages might land in the stream head depending on when 1998 * the message arrived and when the read side of the pass queue was 1999 * re-blocked. 2000 */ 2001 int 2002 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp, 2003 str_stack_t *ss) 2004 { 2005 struct strioctl strioc; 2006 struct stdata *stpdown; 2007 queue_t *rq, *wrq; 2008 queue_t *passq; 2009 syncq_t *passyncq; 2010 int error = 0; 2011 file_t *fpdown; 2012 2013 ASSERT(MUTEX_HELD(&muxifier)); 2014 2015 stpdown = linkp->li_fpdown->f_vnode->v_stream; 2016 2017 /* 2018 * See the comment in mlink() concerning STRPLUMB/STPLEX flags. 2019 */ 2020 mutex_enter(&stpdown->sd_lock); 2021 stpdown->sd_flag |= STRPLUMB; 2022 mutex_exit(&stpdown->sd_lock); 2023 2024 /* 2025 * Add passthru queue below lower mux. This will block 2026 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 2027 */ 2028 passq = link_addpassthru(stpdown); 2029 2030 if ((flag & LINKTYPEMASK) == LINKNORMAL) 2031 strioc.ic_cmd = I_UNLINK; 2032 else 2033 strioc.ic_cmd = I_PUNLINK; 2034 strioc.ic_timout = INFTIM; 2035 strioc.ic_len = sizeof (struct linkblk); 2036 strioc.ic_dp = (char *)&linkp->li_lblk; 2037 2038 error = strdoioctl(stp, &strioc, FNATIVE, 2039 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 2040 2041 /* 2042 * If there was an error and this is not called via strclose, 2043 * return to the user. Otherwise, pretend there was no error 2044 * and close the link. 2045 */ 2046 if (error) { 2047 if (flag & LINKCLOSE) { 2048 cmn_err(CE_WARN, "KERNEL: munlink: could not perform " 2049 "unlink ioctl, closing anyway (%d)\n", error); 2050 } else { 2051 link_rempassthru(passq); 2052 mutex_enter(&stpdown->sd_lock); 2053 stpdown->sd_flag &= ~STRPLUMB; 2054 cv_broadcast(&stpdown->sd_monitor); 2055 mutex_exit(&stpdown->sd_lock); 2056 mutex_exit(&muxifier); 2057 return (error); 2058 } 2059 } 2060 2061 mux_rmvedge(stp, linkp->li_lblk.l_index, ss); 2062 fpdown = linkp->li_fpdown; 2063 lbfree(linkp); 2064 2065 /* 2066 * We go ahead and drop muxifier here--it's a nasty global lock that 2067 * can slow others down. It's okay to since attempts to mlink() this 2068 * stream will be stopped because STPLEX is still set in the stdata 2069 * structure, and munlink() is stopped because mux_rmvedge() and 2070 * lbfree() have removed it from mux_nodes[] and linkinfo_list, 2071 * respectively. Note that we defer the closef() of fpdown until 2072 * after we drop muxifier since strclose() can call munlinkall(). 2073 */ 2074 mutex_exit(&muxifier); 2075 2076 wrq = stpdown->sd_wrq; 2077 rq = _RD(wrq); 2078 2079 /* 2080 * Get rid of outstanding service procedure runs, before we make 2081 * it a stream head, since a stream head doesn't have any service 2082 * procedure. 2083 */ 2084 disable_svc(rq); 2085 wait_svc(rq); 2086 2087 /* 2088 * Since we don't disable the syncq for QPERMOD, we wait for whatever 2089 * is queued up to be finished. mux should take care that nothing is 2090 * send down to this queue. We should do it now as we're going to block 2091 * passyncq if it was unblocked. 2092 */ 2093 if (wrq->q_flag & QPERMOD) { 2094 syncq_t *sq = wrq->q_syncq; 2095 2096 mutex_enter(SQLOCK(sq)); 2097 while (wrq->q_sqflags & Q_SQQUEUED) { 2098 sq->sq_flags |= SQ_WANTWAKEUP; 2099 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2100 } 2101 mutex_exit(SQLOCK(sq)); 2102 } 2103 passyncq = passq->q_syncq; 2104 if (!(passyncq->sq_flags & SQ_BLOCKED)) { 2105 2106 syncq_t *sq, *outer; 2107 2108 /* 2109 * Messages could be flowing from underneath. We will 2110 * block the read side of the passq. This would be 2111 * sufficient for QPAIR and QPERQ muxes to ensure 2112 * that no data is flowing up into this queue 2113 * and hence no thread active in this instance of 2114 * lower mux. But for QPERMOD and QMTOUTPERIM there 2115 * could be messages on the inner and outer/inner 2116 * syncqs respectively. We will wait for them to drain. 2117 * Because passq is blocked messages end up in the syncq 2118 * And qfill_syncq could possibly end up setting QFULL 2119 * which will access the rq->q_flag. Hence, we have to 2120 * acquire the QLOCK in setq. 2121 * 2122 * XXX Messages can also flow from top into this 2123 * queue though the unlink is over (Ex. some instance 2124 * in putnext() called from top that has still not 2125 * accessed this queue. And also putq(lowerq) ?). 2126 * Solution : How about blocking the l_qtop queue ? 2127 * Do we really care about such pure D_MP muxes ? 2128 */ 2129 2130 blocksq(passyncq, SQ_BLOCKED, 0); 2131 2132 sq = rq->q_syncq; 2133 if ((outer = sq->sq_outer) != NULL) { 2134 2135 /* 2136 * We have to just wait for the outer sq_count 2137 * drop to zero. As this does not prevent new 2138 * messages to enter the outer perimeter, this 2139 * is subject to starvation. 2140 * 2141 * NOTE :Because of blocksq above, messages could 2142 * be in the inner syncq only because of some 2143 * thread holding the outer perimeter exclusively. 2144 * Hence it would be sufficient to wait for the 2145 * exclusive holder of the outer perimeter to drain 2146 * the inner and outer syncqs. But we will not depend 2147 * on this feature and hence check the inner syncqs 2148 * separately. 2149 */ 2150 wait_syncq(outer); 2151 } 2152 2153 2154 /* 2155 * There could be messages destined for 2156 * this queue. Let the exclusive holder 2157 * drain it. 2158 */ 2159 2160 wait_syncq(sq); 2161 ASSERT((rq->q_flag & QPERMOD) || 2162 ((rq->q_syncq->sq_head == NULL) && 2163 (_WR(rq)->q_syncq->sq_head == NULL))); 2164 } 2165 2166 /* 2167 * We haven't taken care of QPERMOD case yet. QPERMOD is a special 2168 * case as we don't disable its syncq or remove it off the syncq 2169 * service list. 2170 */ 2171 if (rq->q_flag & QPERMOD) { 2172 syncq_t *sq = rq->q_syncq; 2173 2174 mutex_enter(SQLOCK(sq)); 2175 while (rq->q_sqflags & Q_SQQUEUED) { 2176 sq->sq_flags |= SQ_WANTWAKEUP; 2177 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2178 } 2179 mutex_exit(SQLOCK(sq)); 2180 } 2181 2182 /* 2183 * flush_syncq changes states only when there is some messages to 2184 * free. ie when it returns non-zero value to return. 2185 */ 2186 ASSERT(flush_syncq(rq->q_syncq, rq) == 0); 2187 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); 2188 2189 /* 2190 * No body else should know about this queue now. 2191 * If the mux did not process the messages before 2192 * acking the I_UNLINK, free them now. 2193 */ 2194 2195 flushq(rq, FLUSHALL); 2196 flushq(_WR(rq), FLUSHALL); 2197 2198 /* 2199 * Convert the mux lower queue into a stream head queue. 2200 * Turn off STPLEX before we turn on the stream by removing the passq. 2201 */ 2202 rq->q_ptr = wrq->q_ptr = stpdown; 2203 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); 2204 2205 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 2206 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 2207 2208 enable_svc(rq); 2209 2210 /* 2211 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still 2212 * needs to be set to prevent reopen() of the stream - such reopen may 2213 * try to call non-existent pass queue open routine and panic. 2214 */ 2215 mutex_enter(&stpdown->sd_lock); 2216 stpdown->sd_flag &= ~STPLEX; 2217 mutex_exit(&stpdown->sd_lock); 2218 2219 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || 2220 ((flag & LINKTYPEMASK) == LINKPERSIST)); 2221 2222 /* clean up the layered driver linkages */ 2223 if ((flag & LINKTYPEMASK) == LINKNORMAL) { 2224 ldi_munlink_fp(stp, fpdown, LINKNORMAL); 2225 } else { 2226 ldi_munlink_fp(stp, fpdown, LINKPERSIST); 2227 } 2228 2229 link_rempassthru(passq); 2230 2231 /* 2232 * Now all plumbing changes are finished and STRPLUMB is no 2233 * longer needed. 2234 */ 2235 mutex_enter(&stpdown->sd_lock); 2236 stpdown->sd_flag &= ~STRPLUMB; 2237 cv_broadcast(&stpdown->sd_monitor); 2238 mutex_exit(&stpdown->sd_lock); 2239 2240 (void) closef(fpdown); 2241 return (0); 2242 } 2243 2244 /* 2245 * Unlink all multiplexor links for which stp is the controlling stream. 2246 * Return 0, or a non-zero errno on failure. 2247 */ 2248 int 2249 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss) 2250 { 2251 linkinfo_t *linkp; 2252 int error = 0; 2253 2254 mutex_enter(&muxifier); 2255 while (linkp = findlinks(stp, 0, flag, ss)) { 2256 /* 2257 * munlink() releases the muxifier lock. 2258 */ 2259 if (error = munlink(stp, linkp, flag, crp, rvalp, ss)) 2260 return (error); 2261 mutex_enter(&muxifier); 2262 } 2263 mutex_exit(&muxifier); 2264 return (0); 2265 } 2266 2267 /* 2268 * A multiplexor link has been made. Add an 2269 * edge to the directed graph. 2270 */ 2271 void 2272 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss) 2273 { 2274 struct mux_node *np; 2275 struct mux_edge *ep; 2276 major_t upmaj; 2277 major_t lomaj; 2278 2279 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2280 lomaj = getmajor(lostp->sd_vnode->v_rdev); 2281 np = &ss->ss_mux_nodes[upmaj]; 2282 if (np->mn_outp) { 2283 ep = np->mn_outp; 2284 while (ep->me_nextp) 2285 ep = ep->me_nextp; 2286 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2287 ep = ep->me_nextp; 2288 } else { 2289 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2290 ep = np->mn_outp; 2291 } 2292 ep->me_nextp = NULL; 2293 ep->me_muxid = muxid; 2294 /* 2295 * Save the dev_t for the purposes of str_stack_shutdown. 2296 * str_stack_shutdown assumes that the device allows reopen, since 2297 * this dev_t is the one after any cloning by xx_open(). 2298 * Would prefer finding the dev_t from before any cloning, 2299 * but specfs doesn't retain that. 2300 */ 2301 ep->me_dev = upstp->sd_vnode->v_rdev; 2302 if (lostp->sd_vnode->v_type == VFIFO) 2303 ep->me_nodep = NULL; 2304 else 2305 ep->me_nodep = &ss->ss_mux_nodes[lomaj]; 2306 } 2307 2308 /* 2309 * A multiplexor link has been removed. Remove the 2310 * edge in the directed graph. 2311 */ 2312 void 2313 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss) 2314 { 2315 struct mux_node *np; 2316 struct mux_edge *ep; 2317 struct mux_edge *pep = NULL; 2318 major_t upmaj; 2319 2320 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2321 np = &ss->ss_mux_nodes[upmaj]; 2322 ASSERT(np->mn_outp != NULL); 2323 ep = np->mn_outp; 2324 while (ep) { 2325 if (ep->me_muxid == muxid) { 2326 if (pep) 2327 pep->me_nextp = ep->me_nextp; 2328 else 2329 np->mn_outp = ep->me_nextp; 2330 kmem_free(ep, sizeof (struct mux_edge)); 2331 return; 2332 } 2333 pep = ep; 2334 ep = ep->me_nextp; 2335 } 2336 ASSERT(0); /* should not reach here */ 2337 } 2338 2339 /* 2340 * Translate the device flags (from conf.h) to the corresponding 2341 * qflag and sq_flag (type) values. 2342 */ 2343 int 2344 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, 2345 uint32_t *sqtypep) 2346 { 2347 uint32_t qflag = 0; 2348 uint32_t sqtype = 0; 2349 2350 if (devflag & _D_OLD) 2351 goto bad; 2352 2353 /* Inner perimeter presence and scope */ 2354 switch (devflag & D_MTINNER_MASK) { 2355 case D_MP: 2356 qflag |= QMTSAFE; 2357 sqtype |= SQ_CI; 2358 break; 2359 case D_MTPERQ|D_MP: 2360 qflag |= QPERQ; 2361 break; 2362 case D_MTQPAIR|D_MP: 2363 qflag |= QPAIR; 2364 break; 2365 case D_MTPERMOD|D_MP: 2366 qflag |= QPERMOD; 2367 break; 2368 default: 2369 goto bad; 2370 } 2371 2372 /* Outer perimeter */ 2373 if (devflag & D_MTOUTPERIM) { 2374 switch (devflag & D_MTINNER_MASK) { 2375 case D_MP: 2376 case D_MTPERQ|D_MP: 2377 case D_MTQPAIR|D_MP: 2378 break; 2379 default: 2380 goto bad; 2381 } 2382 qflag |= QMTOUTPERIM; 2383 } 2384 2385 /* Inner perimeter modifiers */ 2386 if (devflag & D_MTINNER_MOD) { 2387 switch (devflag & D_MTINNER_MASK) { 2388 case D_MP: 2389 goto bad; 2390 default: 2391 break; 2392 } 2393 if (devflag & D_MTPUTSHARED) 2394 sqtype |= SQ_CIPUT; 2395 if (devflag & _D_MTOCSHARED) { 2396 /* 2397 * The code in putnext assumes that it has the 2398 * highest concurrency by not checking sq_count. 2399 * Thus _D_MTOCSHARED can only be supported when 2400 * D_MTPUTSHARED is set. 2401 */ 2402 if (!(devflag & D_MTPUTSHARED)) 2403 goto bad; 2404 sqtype |= SQ_CIOC; 2405 } 2406 if (devflag & _D_MTCBSHARED) { 2407 /* 2408 * The code in putnext assumes that it has the 2409 * highest concurrency by not checking sq_count. 2410 * Thus _D_MTCBSHARED can only be supported when 2411 * D_MTPUTSHARED is set. 2412 */ 2413 if (!(devflag & D_MTPUTSHARED)) 2414 goto bad; 2415 sqtype |= SQ_CICB; 2416 } 2417 if (devflag & _D_MTSVCSHARED) { 2418 /* 2419 * The code in putnext assumes that it has the 2420 * highest concurrency by not checking sq_count. 2421 * Thus _D_MTSVCSHARED can only be supported when 2422 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is 2423 * supported only for QPERMOD. 2424 */ 2425 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) 2426 goto bad; 2427 sqtype |= SQ_CISVC; 2428 } 2429 } 2430 2431 /* Default outer perimeter concurrency */ 2432 sqtype |= SQ_CO; 2433 2434 /* Outer perimeter modifiers */ 2435 if (devflag & D_MTOCEXCL) { 2436 if (!(devflag & D_MTOUTPERIM)) { 2437 /* No outer perimeter */ 2438 goto bad; 2439 } 2440 sqtype &= ~SQ_COOC; 2441 } 2442 2443 /* Synchronous Streams extended qinit structure */ 2444 if (devflag & D_SYNCSTR) 2445 qflag |= QSYNCSTR; 2446 2447 /* 2448 * Private flag used by a transport module to indicate 2449 * to sockfs that it supports direct-access mode without 2450 * having to go through STREAMS. 2451 */ 2452 if (devflag & _D_DIRECT) { 2453 /* Reject unless the module is fully-MT (no perimeter) */ 2454 if ((qflag & QMT_TYPEMASK) != QMTSAFE) 2455 goto bad; 2456 qflag |= _QDIRECT; 2457 } 2458 2459 *qflagp = qflag; 2460 *sqtypep = sqtype; 2461 return (0); 2462 2463 bad: 2464 cmn_err(CE_WARN, 2465 "stropen: bad MT flags (0x%x) in driver '%s'", 2466 (int)(qflag & D_MTSAFETY_MASK), 2467 stp->st_rdinit->qi_minfo->mi_idname); 2468 2469 return (EINVAL); 2470 } 2471 2472 /* 2473 * Set the interface values for a pair of queues (qinit structure, 2474 * packet sizes, water marks). 2475 * setq assumes that the caller does not have a claim (entersq or claimq) 2476 * on the queue. 2477 */ 2478 void 2479 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, 2480 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) 2481 { 2482 queue_t *wq; 2483 syncq_t *sq, *outer; 2484 2485 ASSERT(rq->q_flag & QREADR); 2486 ASSERT((qflag & QMT_TYPEMASK) != 0); 2487 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 2488 2489 wq = _WR(rq); 2490 rq->q_qinfo = rinit; 2491 rq->q_hiwat = rinit->qi_minfo->mi_hiwat; 2492 rq->q_lowat = rinit->qi_minfo->mi_lowat; 2493 rq->q_minpsz = rinit->qi_minfo->mi_minpsz; 2494 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; 2495 wq->q_qinfo = winit; 2496 wq->q_hiwat = winit->qi_minfo->mi_hiwat; 2497 wq->q_lowat = winit->qi_minfo->mi_lowat; 2498 wq->q_minpsz = winit->qi_minfo->mi_minpsz; 2499 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; 2500 2501 /* Remove old syncqs */ 2502 sq = rq->q_syncq; 2503 outer = sq->sq_outer; 2504 if (outer != NULL) { 2505 ASSERT(wq->q_syncq->sq_outer == outer); 2506 outer_remove(outer, rq->q_syncq); 2507 if (wq->q_syncq != rq->q_syncq) 2508 outer_remove(outer, wq->q_syncq); 2509 } 2510 ASSERT(sq->sq_outer == NULL); 2511 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2512 2513 if (sq != SQ(rq)) { 2514 if (!(rq->q_flag & QPERMOD)) 2515 free_syncq(sq); 2516 if (wq->q_syncq == rq->q_syncq) 2517 wq->q_syncq = NULL; 2518 rq->q_syncq = NULL; 2519 } 2520 if (wq->q_syncq != NULL && wq->q_syncq != sq && 2521 wq->q_syncq != SQ(rq)) { 2522 free_syncq(wq->q_syncq); 2523 wq->q_syncq = NULL; 2524 } 2525 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && 2526 rq->q_syncq->sq_tail == NULL)); 2527 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && 2528 wq->q_syncq->sq_tail == NULL)); 2529 2530 if (!(rq->q_flag & QPERMOD) && 2531 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { 2532 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2533 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, 2534 rq->q_syncq->sq_nciputctrl, 0); 2535 ASSERT(ciputctrl_cache != NULL); 2536 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); 2537 rq->q_syncq->sq_ciputctrl = NULL; 2538 rq->q_syncq->sq_nciputctrl = 0; 2539 } 2540 2541 if (!(wq->q_flag & QPERMOD) && 2542 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { 2543 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2544 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, 2545 wq->q_syncq->sq_nciputctrl, 0); 2546 ASSERT(ciputctrl_cache != NULL); 2547 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); 2548 wq->q_syncq->sq_ciputctrl = NULL; 2549 wq->q_syncq->sq_nciputctrl = 0; 2550 } 2551 2552 sq = SQ(rq); 2553 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 2554 ASSERT(sq->sq_outer == NULL); 2555 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2556 2557 /* 2558 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS 2559 * bits in sq_flag based on the sqtype. 2560 */ 2561 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); 2562 2563 rq->q_syncq = wq->q_syncq = sq; 2564 sq->sq_type = sqtype; 2565 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); 2566 2567 /* 2568 * We are making sq_svcflags zero, 2569 * resetting SQ_DISABLED in case it was set by 2570 * wait_svc() in the munlink path. 2571 * 2572 */ 2573 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); 2574 sq->sq_svcflags = 0; 2575 2576 /* 2577 * We need to acquire the lock here for the mlink and munlink case, 2578 * where canputnext, backenable, etc can access the q_flag. 2579 */ 2580 if (lock_needed) { 2581 mutex_enter(QLOCK(rq)); 2582 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2583 mutex_exit(QLOCK(rq)); 2584 mutex_enter(QLOCK(wq)); 2585 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2586 mutex_exit(QLOCK(wq)); 2587 } else { 2588 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2589 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2590 } 2591 2592 if (qflag & QPERQ) { 2593 /* Allocate a separate syncq for the write side */ 2594 sq = new_syncq(); 2595 sq->sq_type = rq->q_syncq->sq_type; 2596 sq->sq_flags = rq->q_syncq->sq_flags; 2597 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2598 sq->sq_oprev == NULL); 2599 wq->q_syncq = sq; 2600 } 2601 if (qflag & QPERMOD) { 2602 sq = dmp->dm_sq; 2603 2604 /* 2605 * Assert that we do have an inner perimeter syncq and that it 2606 * does not have an outer perimeter associated with it. 2607 */ 2608 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2609 sq->sq_oprev == NULL); 2610 rq->q_syncq = wq->q_syncq = sq; 2611 } 2612 if (qflag & QMTOUTPERIM) { 2613 outer = dmp->dm_sq; 2614 2615 ASSERT(outer->sq_outer == NULL); 2616 outer_insert(outer, rq->q_syncq); 2617 if (wq->q_syncq != rq->q_syncq) 2618 outer_insert(outer, wq->q_syncq); 2619 } 2620 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2621 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2622 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2623 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2624 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); 2625 2626 /* 2627 * Initialize struio() types. 2628 */ 2629 rq->q_struiot = 2630 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; 2631 wq->q_struiot = 2632 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; 2633 } 2634 2635 perdm_t * 2636 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) 2637 { 2638 syncq_t *sq; 2639 perdm_t **pp; 2640 perdm_t *p; 2641 perdm_t *dmp; 2642 2643 ASSERT(str != NULL); 2644 ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); 2645 2646 rw_enter(&perdm_rwlock, RW_READER); 2647 for (p = perdm_list; p != NULL; p = p->dm_next) { 2648 if (p->dm_str == str) { /* found one */ 2649 atomic_add_32(&(p->dm_ref), 1); 2650 rw_exit(&perdm_rwlock); 2651 return (p); 2652 } 2653 } 2654 rw_exit(&perdm_rwlock); 2655 2656 sq = new_syncq(); 2657 if (qflag & QPERMOD) { 2658 sq->sq_type = sqtype | SQ_PERMOD; 2659 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; 2660 } else { 2661 ASSERT(qflag & QMTOUTPERIM); 2662 sq->sq_onext = sq->sq_oprev = sq; 2663 } 2664 2665 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); 2666 dmp->dm_sq = sq; 2667 dmp->dm_str = str; 2668 dmp->dm_ref = 1; 2669 dmp->dm_next = NULL; 2670 2671 rw_enter(&perdm_rwlock, RW_WRITER); 2672 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { 2673 if (p->dm_str == str) { /* already present */ 2674 p->dm_ref++; 2675 rw_exit(&perdm_rwlock); 2676 free_syncq(sq); 2677 kmem_free(dmp, sizeof (perdm_t)); 2678 return (p); 2679 } 2680 } 2681 2682 *pp = dmp; 2683 rw_exit(&perdm_rwlock); 2684 return (dmp); 2685 } 2686 2687 void 2688 rele_dm(perdm_t *dmp) 2689 { 2690 perdm_t **pp; 2691 perdm_t *p; 2692 2693 rw_enter(&perdm_rwlock, RW_WRITER); 2694 ASSERT(dmp->dm_ref > 0); 2695 2696 if (--dmp->dm_ref > 0) { 2697 rw_exit(&perdm_rwlock); 2698 return; 2699 } 2700 2701 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) 2702 if (p == dmp) 2703 break; 2704 ASSERT(p == dmp); 2705 *pp = p->dm_next; 2706 rw_exit(&perdm_rwlock); 2707 2708 /* 2709 * Wait for any background processing that relies on the 2710 * syncq to complete before it is freed. 2711 */ 2712 wait_sq_svc(p->dm_sq); 2713 free_syncq(p->dm_sq); 2714 kmem_free(p, sizeof (perdm_t)); 2715 } 2716 2717 /* 2718 * Make a protocol message given control and data buffers. 2719 * n.b., this can block; be careful of what locks you hold when calling it. 2720 * 2721 * If sd_maxblk is less than *iosize this routine can fail part way through 2722 * (due to an allocation failure). In this case on return *iosize will contain 2723 * the amount that was consumed. Otherwise *iosize will not be modified 2724 * i.e. it will contain the amount that was consumed. 2725 */ 2726 int 2727 strmakemsg( 2728 struct strbuf *mctl, 2729 ssize_t *iosize, 2730 struct uio *uiop, 2731 stdata_t *stp, 2732 int32_t flag, 2733 mblk_t **mpp) 2734 { 2735 mblk_t *mpctl = NULL; 2736 mblk_t *mpdata = NULL; 2737 int error; 2738 2739 ASSERT(uiop != NULL); 2740 2741 *mpp = NULL; 2742 /* Create control part, if any */ 2743 if ((mctl != NULL) && (mctl->len >= 0)) { 2744 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); 2745 if (error) 2746 return (error); 2747 } 2748 /* Create data part, if any */ 2749 if (*iosize >= 0) { 2750 error = strmakedata(iosize, uiop, stp, flag, &mpdata); 2751 if (error) { 2752 freemsg(mpctl); 2753 return (error); 2754 } 2755 } 2756 if (mpctl != NULL) { 2757 if (mpdata != NULL) 2758 linkb(mpctl, mpdata); 2759 *mpp = mpctl; 2760 } else { 2761 *mpp = mpdata; 2762 } 2763 return (0); 2764 } 2765 2766 /* 2767 * Make the control part of a protocol message given a control buffer. 2768 * n.b., this can block; be careful of what locks you hold when calling it. 2769 */ 2770 int 2771 strmakectl( 2772 struct strbuf *mctl, 2773 int32_t flag, 2774 int32_t fflag, 2775 mblk_t **mpp) 2776 { 2777 mblk_t *bp = NULL; 2778 unsigned char msgtype; 2779 int error = 0; 2780 2781 *mpp = NULL; 2782 /* 2783 * Create control part of message, if any. 2784 */ 2785 if ((mctl != NULL) && (mctl->len >= 0)) { 2786 caddr_t base; 2787 int ctlcount; 2788 int allocsz; 2789 2790 if (flag & RS_HIPRI) 2791 msgtype = M_PCPROTO; 2792 else 2793 msgtype = M_PROTO; 2794 2795 ctlcount = mctl->len; 2796 base = mctl->buf; 2797 2798 /* 2799 * Give modules a better chance to reuse M_PROTO/M_PCPROTO 2800 * blocks by increasing the size to something more usable. 2801 */ 2802 allocsz = MAX(ctlcount, 64); 2803 2804 /* 2805 * Range checking has already been done; simply try 2806 * to allocate a message block for the ctl part. 2807 */ 2808 while (!(bp = allocb(allocsz, BPRI_MED))) { 2809 if (fflag & (FNDELAY|FNONBLOCK)) 2810 return (EAGAIN); 2811 if (error = strwaitbuf(allocsz, BPRI_MED)) 2812 return (error); 2813 } 2814 2815 bp->b_datap->db_type = msgtype; 2816 if (copyin(base, bp->b_wptr, ctlcount)) { 2817 freeb(bp); 2818 return (EFAULT); 2819 } 2820 bp->b_wptr += ctlcount; 2821 } 2822 *mpp = bp; 2823 return (0); 2824 } 2825 2826 /* 2827 * Make a protocol message given data buffers. 2828 * n.b., this can block; be careful of what locks you hold when calling it. 2829 * 2830 * If sd_maxblk is less than *iosize this routine can fail part way through 2831 * (due to an allocation failure). In this case on return *iosize will contain 2832 * the amount that was consumed. Otherwise *iosize will not be modified 2833 * i.e. it will contain the amount that was consumed. 2834 */ 2835 int 2836 strmakedata( 2837 ssize_t *iosize, 2838 struct uio *uiop, 2839 stdata_t *stp, 2840 int32_t flag, 2841 mblk_t **mpp) 2842 { 2843 mblk_t *mp = NULL; 2844 mblk_t *bp; 2845 int wroff = (int)stp->sd_wroff; 2846 int tail_len = (int)stp->sd_tail; 2847 int extra = wroff + tail_len; 2848 int error = 0; 2849 ssize_t maxblk; 2850 ssize_t count = *iosize; 2851 cred_t *cr = CRED(); 2852 2853 *mpp = NULL; 2854 if (count < 0) 2855 return (0); 2856 2857 maxblk = stp->sd_maxblk; 2858 if (maxblk == INFPSZ) 2859 maxblk = count; 2860 2861 /* 2862 * Create data part of message, if any. 2863 */ 2864 do { 2865 ssize_t size; 2866 dblk_t *dp; 2867 2868 ASSERT(uiop); 2869 2870 size = MIN(count, maxblk); 2871 2872 while ((bp = allocb_cred(size + extra, cr)) == NULL) { 2873 error = EAGAIN; 2874 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 2875 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) { 2876 if (count == *iosize) { 2877 freemsg(mp); 2878 return (error); 2879 } else { 2880 *iosize -= count; 2881 *mpp = mp; 2882 return (0); 2883 } 2884 } 2885 } 2886 dp = bp->b_datap; 2887 dp->db_cpid = curproc->p_pid; 2888 ASSERT(wroff <= dp->db_lim - bp->b_wptr); 2889 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; 2890 2891 if (flag & STRUIO_POSTPONE) { 2892 /* 2893 * Setup the stream uio portion of the 2894 * dblk for subsequent use by struioget(). 2895 */ 2896 dp->db_struioflag = STRUIO_SPEC; 2897 dp->db_cksumstart = 0; 2898 dp->db_cksumstuff = 0; 2899 dp->db_cksumend = size; 2900 *(long long *)dp->db_struioun.data = 0ll; 2901 bp->b_wptr += size; 2902 } else { 2903 if (stp->sd_copyflag & STRCOPYCACHED) 2904 uiop->uio_extflg |= UIO_COPY_CACHED; 2905 2906 if (size != 0) { 2907 error = uiomove(bp->b_wptr, size, UIO_WRITE, 2908 uiop); 2909 if (error != 0) { 2910 freeb(bp); 2911 freemsg(mp); 2912 return (error); 2913 } 2914 } 2915 bp->b_wptr += size; 2916 2917 if (stp->sd_wputdatafunc != NULL) { 2918 mblk_t *newbp; 2919 2920 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode, 2921 bp, NULL, NULL, NULL, NULL); 2922 if (newbp == NULL) { 2923 freeb(bp); 2924 freemsg(mp); 2925 return (ECOMM); 2926 } 2927 bp = newbp; 2928 } 2929 } 2930 2931 count -= size; 2932 2933 if (mp == NULL) 2934 mp = bp; 2935 else 2936 linkb(mp, bp); 2937 } while (count > 0); 2938 2939 *mpp = mp; 2940 return (0); 2941 } 2942 2943 /* 2944 * Wait for a buffer to become available. Return non-zero errno 2945 * if not able to wait, 0 if buffer is probably there. 2946 */ 2947 int 2948 strwaitbuf(size_t size, int pri) 2949 { 2950 bufcall_id_t id; 2951 2952 mutex_enter(&bcall_monitor); 2953 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, 2954 &ttoproc(curthread)->p_flag_cv)) == 0) { 2955 mutex_exit(&bcall_monitor); 2956 return (ENOSR); 2957 } 2958 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { 2959 unbufcall(id); 2960 mutex_exit(&bcall_monitor); 2961 return (EINTR); 2962 } 2963 unbufcall(id); 2964 mutex_exit(&bcall_monitor); 2965 return (0); 2966 } 2967 2968 /* 2969 * This function waits for a read or write event to happen on a stream. 2970 * fmode can specify FNDELAY and/or FNONBLOCK. 2971 * The timeout is in ms with -1 meaning infinite. 2972 * The flag values work as follows: 2973 * READWAIT Check for read side errors, send M_READ 2974 * GETWAIT Check for read side errors, no M_READ 2975 * WRITEWAIT Check for write side errors. 2976 * NOINTR Do not return error if nonblocking or timeout. 2977 * STR_NOERROR Ignore all errors except STPLEX. 2978 * STR_NOSIG Ignore/hold signals during the duration of the call. 2979 * STR_PEEK Pass through the strgeterr(). 2980 */ 2981 int 2982 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, 2983 int *done) 2984 { 2985 int slpflg, errs; 2986 int error; 2987 kcondvar_t *sleepon; 2988 mblk_t *mp; 2989 ssize_t *rd_count; 2990 clock_t rval; 2991 2992 ASSERT(MUTEX_HELD(&stp->sd_lock)); 2993 if ((flag & READWAIT) || (flag & GETWAIT)) { 2994 slpflg = RSLEEP; 2995 sleepon = &_RD(stp->sd_wrq)->q_wait; 2996 errs = STRDERR|STPLEX; 2997 } else { 2998 slpflg = WSLEEP; 2999 sleepon = &stp->sd_wrq->q_wait; 3000 errs = STWRERR|STRHUP|STPLEX; 3001 } 3002 if (flag & STR_NOERROR) 3003 errs = STPLEX; 3004 3005 if (stp->sd_wakeq & slpflg) { 3006 /* 3007 * A strwakeq() is pending, no need to sleep. 3008 */ 3009 stp->sd_wakeq &= ~slpflg; 3010 *done = 0; 3011 return (0); 3012 } 3013 3014 if (fmode & (FNDELAY|FNONBLOCK)) { 3015 if (!(flag & NOINTR)) 3016 error = EAGAIN; 3017 else 3018 error = 0; 3019 *done = 1; 3020 return (error); 3021 } 3022 3023 if (stp->sd_flag & errs) { 3024 /* 3025 * Check for errors before going to sleep since the 3026 * caller might not have checked this while holding 3027 * sd_lock. 3028 */ 3029 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3030 if (error != 0) { 3031 *done = 1; 3032 return (error); 3033 } 3034 } 3035 3036 /* 3037 * If any module downstream has requested read notification 3038 * by setting SNDMREAD flag using M_SETOPTS, send a message 3039 * down stream. 3040 */ 3041 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) { 3042 mutex_exit(&stp->sd_lock); 3043 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED, 3044 (flag & STR_NOSIG), &error))) { 3045 mutex_enter(&stp->sd_lock); 3046 *done = 1; 3047 return (error); 3048 } 3049 mp->b_datap->db_type = M_READ; 3050 rd_count = (ssize_t *)mp->b_wptr; 3051 *rd_count = count; 3052 mp->b_wptr += sizeof (ssize_t); 3053 /* 3054 * Send the number of bytes requested by the 3055 * read as the argument to M_READ. 3056 */ 3057 stream_willservice(stp); 3058 putnext(stp->sd_wrq, mp); 3059 stream_runservice(stp); 3060 mutex_enter(&stp->sd_lock); 3061 3062 /* 3063 * If any data arrived due to inline processing 3064 * of putnext(), don't sleep. 3065 */ 3066 if (_RD(stp->sd_wrq)->q_first != NULL) { 3067 *done = 0; 3068 return (0); 3069 } 3070 } 3071 3072 stp->sd_flag |= slpflg; 3073 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2, 3074 "strwaitq sleeps (2):%p, %X, %lX, %X, %p", 3075 stp, flag, count, fmode, done); 3076 3077 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG); 3078 if (rval > 0) { 3079 /* EMPTY */ 3080 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2, 3081 "strwaitq awakes(2):%X, %X, %X, %X, %X", 3082 stp, flag, count, fmode, done); 3083 } else if (rval == 0) { 3084 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2, 3085 "strwaitq interrupt #2:%p, %X, %lX, %X, %p", 3086 stp, flag, count, fmode, done); 3087 stp->sd_flag &= ~slpflg; 3088 cv_broadcast(sleepon); 3089 if (!(flag & NOINTR)) 3090 error = EINTR; 3091 else 3092 error = 0; 3093 *done = 1; 3094 return (error); 3095 } else { 3096 /* timeout */ 3097 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME, 3098 "strwaitq timeout:%p, %X, %lX, %X, %p", 3099 stp, flag, count, fmode, done); 3100 *done = 1; 3101 if (!(flag & NOINTR)) 3102 return (ETIME); 3103 else 3104 return (0); 3105 } 3106 /* 3107 * If the caller implements delayed errors (i.e. queued after data) 3108 * we can not check for errors here since data as well as an 3109 * error might have arrived at the stream head. We return to 3110 * have the caller check the read queue before checking for errors. 3111 */ 3112 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) { 3113 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3114 if (error != 0) { 3115 *done = 1; 3116 return (error); 3117 } 3118 } 3119 *done = 0; 3120 return (0); 3121 } 3122 3123 /* 3124 * Perform job control discipline access checks. 3125 * Return 0 for success and the errno for failure. 3126 */ 3127 3128 #define cantsend(p, t, sig) \ 3129 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig)) 3130 3131 int 3132 straccess(struct stdata *stp, enum jcaccess mode) 3133 { 3134 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */ 3135 kthread_t *t = curthread; 3136 proc_t *p = ttoproc(t); 3137 sess_t *sp; 3138 3139 ASSERT(mutex_owned(&stp->sd_lock)); 3140 3141 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO) 3142 return (0); 3143 3144 mutex_enter(&p->p_lock); /* protects p_pgidp */ 3145 3146 for (;;) { 3147 mutex_enter(&p->p_splock); /* protects p->p_sessp */ 3148 sp = p->p_sessp; 3149 mutex_enter(&sp->s_lock); /* protects sp->* */ 3150 3151 /* 3152 * If this is not the calling process's controlling terminal 3153 * or if the calling process is already in the foreground 3154 * then allow access. 3155 */ 3156 if (sp->s_dev != stp->sd_vnode->v_rdev || 3157 p->p_pgidp == stp->sd_pgidp) { 3158 mutex_exit(&sp->s_lock); 3159 mutex_exit(&p->p_splock); 3160 mutex_exit(&p->p_lock); 3161 return (0); 3162 } 3163 3164 /* 3165 * Check to see if controlling terminal has been deallocated. 3166 */ 3167 if (sp->s_vp == NULL) { 3168 if (!cantsend(p, t, SIGHUP)) 3169 sigtoproc(p, t, SIGHUP); 3170 mutex_exit(&sp->s_lock); 3171 mutex_exit(&p->p_splock); 3172 mutex_exit(&p->p_lock); 3173 return (EIO); 3174 } 3175 3176 mutex_exit(&sp->s_lock); 3177 mutex_exit(&p->p_splock); 3178 3179 if (mode == JCGETP) { 3180 mutex_exit(&p->p_lock); 3181 return (0); 3182 } 3183 3184 if (mode == JCREAD) { 3185 if (p->p_detached || cantsend(p, t, SIGTTIN)) { 3186 mutex_exit(&p->p_lock); 3187 return (EIO); 3188 } 3189 mutex_exit(&p->p_lock); 3190 mutex_exit(&stp->sd_lock); 3191 pgsignal(p->p_pgidp, SIGTTIN); 3192 mutex_enter(&stp->sd_lock); 3193 mutex_enter(&p->p_lock); 3194 } else { /* mode == JCWRITE or JCSETP */ 3195 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) || 3196 cantsend(p, t, SIGTTOU)) { 3197 mutex_exit(&p->p_lock); 3198 return (0); 3199 } 3200 if (p->p_detached) { 3201 mutex_exit(&p->p_lock); 3202 return (EIO); 3203 } 3204 mutex_exit(&p->p_lock); 3205 mutex_exit(&stp->sd_lock); 3206 pgsignal(p->p_pgidp, SIGTTOU); 3207 mutex_enter(&stp->sd_lock); 3208 mutex_enter(&p->p_lock); 3209 } 3210 3211 /* 3212 * We call cv_wait_sig_swap() to cause the appropriate 3213 * action for the jobcontrol signal to take place. 3214 * If the signal is being caught, we will take the 3215 * EINTR error return. Otherwise, the default action 3216 * of causing the process to stop will take place. 3217 * In this case, we rely on the periodic cv_broadcast() on 3218 * &lbolt_cv to wake us up to loop around and test again. 3219 * We can't get here if the signal is ignored or 3220 * if the current thread is blocking the signal. 3221 */ 3222 mutex_exit(&stp->sd_lock); 3223 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) { 3224 mutex_exit(&p->p_lock); 3225 mutex_enter(&stp->sd_lock); 3226 return (EINTR); 3227 } 3228 mutex_exit(&p->p_lock); 3229 mutex_enter(&stp->sd_lock); 3230 mutex_enter(&p->p_lock); 3231 } 3232 } 3233 3234 /* 3235 * Return size of message of block type (bp->b_datap->db_type) 3236 */ 3237 size_t 3238 xmsgsize(mblk_t *bp) 3239 { 3240 unsigned char type; 3241 size_t count = 0; 3242 3243 type = bp->b_datap->db_type; 3244 3245 for (; bp; bp = bp->b_cont) { 3246 if (type != bp->b_datap->db_type) 3247 break; 3248 ASSERT(bp->b_wptr >= bp->b_rptr); 3249 count += bp->b_wptr - bp->b_rptr; 3250 } 3251 return (count); 3252 } 3253 3254 /* 3255 * Allocate a stream head. 3256 */ 3257 struct stdata * 3258 shalloc(queue_t *qp) 3259 { 3260 stdata_t *stp; 3261 3262 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP); 3263 3264 stp->sd_wrq = _WR(qp); 3265 stp->sd_strtab = NULL; 3266 stp->sd_iocid = 0; 3267 stp->sd_mate = NULL; 3268 stp->sd_freezer = NULL; 3269 stp->sd_refcnt = 0; 3270 stp->sd_wakeq = 0; 3271 stp->sd_anchor = 0; 3272 stp->sd_struiowrq = NULL; 3273 stp->sd_struiordq = NULL; 3274 stp->sd_struiodnak = 0; 3275 stp->sd_struionak = NULL; 3276 #ifdef C2_AUDIT 3277 stp->sd_t_audit_data = NULL; 3278 #endif 3279 stp->sd_rput_opt = 0; 3280 stp->sd_wput_opt = 0; 3281 stp->sd_read_opt = 0; 3282 stp->sd_rprotofunc = strrput_proto; 3283 stp->sd_rmiscfunc = strrput_misc; 3284 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL; 3285 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL; 3286 stp->sd_ciputctrl = NULL; 3287 stp->sd_nciputctrl = 0; 3288 stp->sd_qhead = NULL; 3289 stp->sd_qtail = NULL; 3290 stp->sd_servid = NULL; 3291 stp->sd_nqueues = 0; 3292 stp->sd_svcflags = 0; 3293 stp->sd_copyflag = 0; 3294 3295 return (stp); 3296 } 3297 3298 /* 3299 * Free a stream head. 3300 */ 3301 void 3302 shfree(stdata_t *stp) 3303 { 3304 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock)); 3305 3306 stp->sd_wrq = NULL; 3307 3308 mutex_enter(&stp->sd_qlock); 3309 while (stp->sd_svcflags & STRS_SCHEDULED) { 3310 STRSTAT(strwaits); 3311 cv_wait(&stp->sd_qcv, &stp->sd_qlock); 3312 } 3313 mutex_exit(&stp->sd_qlock); 3314 3315 if (stp->sd_ciputctrl != NULL) { 3316 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1); 3317 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl, 3318 stp->sd_nciputctrl, 0); 3319 ASSERT(ciputctrl_cache != NULL); 3320 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl); 3321 stp->sd_ciputctrl = NULL; 3322 stp->sd_nciputctrl = 0; 3323 } 3324 ASSERT(stp->sd_qhead == NULL); 3325 ASSERT(stp->sd_qtail == NULL); 3326 ASSERT(stp->sd_nqueues == 0); 3327 kmem_cache_free(stream_head_cache, stp); 3328 } 3329 3330 /* 3331 * Allocate a pair of queues and a syncq for the pair 3332 */ 3333 queue_t * 3334 allocq(void) 3335 { 3336 queinfo_t *qip; 3337 queue_t *qp, *wqp; 3338 syncq_t *sq; 3339 3340 qip = kmem_cache_alloc(queue_cache, KM_SLEEP); 3341 3342 qp = &qip->qu_rqueue; 3343 wqp = &qip->qu_wqueue; 3344 sq = &qip->qu_syncq; 3345 3346 qp->q_last = NULL; 3347 qp->q_next = NULL; 3348 qp->q_ptr = NULL; 3349 qp->q_flag = QUSE | QREADR; 3350 qp->q_bandp = NULL; 3351 qp->q_stream = NULL; 3352 qp->q_syncq = sq; 3353 qp->q_nband = 0; 3354 qp->q_nfsrv = NULL; 3355 qp->q_draining = 0; 3356 qp->q_syncqmsgs = 0; 3357 qp->q_spri = 0; 3358 qp->q_qtstamp = 0; 3359 qp->q_sqtstamp = 0; 3360 qp->q_fp = NULL; 3361 3362 wqp->q_last = NULL; 3363 wqp->q_next = NULL; 3364 wqp->q_ptr = NULL; 3365 wqp->q_flag = QUSE; 3366 wqp->q_bandp = NULL; 3367 wqp->q_stream = NULL; 3368 wqp->q_syncq = sq; 3369 wqp->q_nband = 0; 3370 wqp->q_nfsrv = NULL; 3371 wqp->q_draining = 0; 3372 wqp->q_syncqmsgs = 0; 3373 wqp->q_qtstamp = 0; 3374 wqp->q_sqtstamp = 0; 3375 wqp->q_spri = 0; 3376 3377 sq->sq_count = 0; 3378 sq->sq_rmqcount = 0; 3379 sq->sq_flags = 0; 3380 sq->sq_type = 0; 3381 sq->sq_callbflags = 0; 3382 sq->sq_cancelid = 0; 3383 sq->sq_ciputctrl = NULL; 3384 sq->sq_nciputctrl = 0; 3385 sq->sq_needexcl = 0; 3386 sq->sq_svcflags = 0; 3387 3388 return (qp); 3389 } 3390 3391 /* 3392 * Free a pair of queues and the "attached" syncq. 3393 * Discard any messages left on the syncq(s), remove the syncq(s) from the 3394 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq. 3395 */ 3396 void 3397 freeq(queue_t *qp) 3398 { 3399 qband_t *qbp, *nqbp; 3400 syncq_t *sq, *outer; 3401 queue_t *wqp = _WR(qp); 3402 3403 ASSERT(qp->q_flag & QREADR); 3404 3405 /* 3406 * If a previously dispatched taskq job is scheduled to run 3407 * sync_service() or a service routine is scheduled for the 3408 * queues about to be freed, wait here until all service is 3409 * done on the queue and all associated queues and syncqs. 3410 */ 3411 wait_svc(qp); 3412 3413 (void) flush_syncq(qp->q_syncq, qp); 3414 (void) flush_syncq(wqp->q_syncq, wqp); 3415 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0); 3416 3417 /* 3418 * Flush the queues before q_next is set to NULL This is needed 3419 * in order to backenable any downstream queue before we go away. 3420 * Note: we are already removed from the stream so that the 3421 * backenabling will not cause any messages to be delivered to our 3422 * put procedures. 3423 */ 3424 flushq(qp, FLUSHALL); 3425 flushq(wqp, FLUSHALL); 3426 3427 /* Tidy up - removeq only does a half-remove from stream */ 3428 qp->q_next = wqp->q_next = NULL; 3429 ASSERT(!(qp->q_flag & QENAB)); 3430 ASSERT(!(wqp->q_flag & QENAB)); 3431 3432 outer = qp->q_syncq->sq_outer; 3433 if (outer != NULL) { 3434 outer_remove(outer, qp->q_syncq); 3435 if (wqp->q_syncq != qp->q_syncq) 3436 outer_remove(outer, wqp->q_syncq); 3437 } 3438 /* 3439 * Free any syncqs that are outside what allocq returned. 3440 */ 3441 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD)) 3442 free_syncq(qp->q_syncq); 3443 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp)) 3444 free_syncq(wqp->q_syncq); 3445 3446 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3447 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3448 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 3449 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp))); 3450 sq = SQ(qp); 3451 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 3452 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 3453 ASSERT(sq->sq_outer == NULL); 3454 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 3455 ASSERT(sq->sq_callbpend == NULL); 3456 ASSERT(sq->sq_needexcl == 0); 3457 3458 if (sq->sq_ciputctrl != NULL) { 3459 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 3460 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 3461 sq->sq_nciputctrl, 0); 3462 ASSERT(ciputctrl_cache != NULL); 3463 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 3464 sq->sq_ciputctrl = NULL; 3465 sq->sq_nciputctrl = 0; 3466 } 3467 3468 ASSERT(qp->q_first == NULL && wqp->q_first == NULL); 3469 ASSERT(qp->q_count == 0 && wqp->q_count == 0); 3470 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0); 3471 3472 qp->q_flag &= ~QUSE; 3473 wqp->q_flag &= ~QUSE; 3474 3475 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */ 3476 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */ 3477 3478 qbp = qp->q_bandp; 3479 while (qbp) { 3480 nqbp = qbp->qb_next; 3481 freeband(qbp); 3482 qbp = nqbp; 3483 } 3484 qbp = wqp->q_bandp; 3485 while (qbp) { 3486 nqbp = qbp->qb_next; 3487 freeband(qbp); 3488 qbp = nqbp; 3489 } 3490 kmem_cache_free(queue_cache, qp); 3491 } 3492 3493 /* 3494 * Allocate a qband structure. 3495 */ 3496 qband_t * 3497 allocband(void) 3498 { 3499 qband_t *qbp; 3500 3501 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP); 3502 if (qbp == NULL) 3503 return (NULL); 3504 3505 qbp->qb_next = NULL; 3506 qbp->qb_count = 0; 3507 qbp->qb_mblkcnt = 0; 3508 qbp->qb_first = NULL; 3509 qbp->qb_last = NULL; 3510 qbp->qb_flag = 0; 3511 3512 return (qbp); 3513 } 3514 3515 /* 3516 * Free a qband structure. 3517 */ 3518 void 3519 freeband(qband_t *qbp) 3520 { 3521 kmem_cache_free(qband_cache, qbp); 3522 } 3523 3524 /* 3525 * Just like putnextctl(9F), except that allocb_wait() is used. 3526 * 3527 * Consolidation Private, and of course only callable from the stream head or 3528 * routines that may block. 3529 */ 3530 int 3531 putnextctl_wait(queue_t *q, int type) 3532 { 3533 mblk_t *bp; 3534 int error; 3535 3536 if ((datamsg(type) && (type != M_DELAY)) || 3537 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL) 3538 return (0); 3539 3540 bp->b_datap->db_type = (unsigned char)type; 3541 putnext(q, bp); 3542 return (1); 3543 } 3544 3545 /* 3546 * run any possible bufcalls. 3547 */ 3548 void 3549 runbufcalls(void) 3550 { 3551 strbufcall_t *bcp; 3552 3553 mutex_enter(&bcall_monitor); 3554 mutex_enter(&strbcall_lock); 3555 3556 if (strbcalls.bc_head) { 3557 size_t count; 3558 int nevent; 3559 3560 /* 3561 * count how many events are on the list 3562 * now so we can check to avoid looping 3563 * in low memory situations 3564 */ 3565 nevent = 0; 3566 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next) 3567 nevent++; 3568 3569 /* 3570 * get estimate of available memory from kmem_avail(). 3571 * awake all bufcall functions waiting for 3572 * memory whose request could be satisfied 3573 * by 'count' memory and let 'em fight for it. 3574 */ 3575 count = kmem_avail(); 3576 while ((bcp = strbcalls.bc_head) != NULL && nevent) { 3577 STRSTAT(bufcalls); 3578 --nevent; 3579 if (bcp->bc_size <= count) { 3580 bcp->bc_executor = curthread; 3581 mutex_exit(&strbcall_lock); 3582 (*bcp->bc_func)(bcp->bc_arg); 3583 mutex_enter(&strbcall_lock); 3584 bcp->bc_executor = NULL; 3585 cv_broadcast(&bcall_cv); 3586 strbcalls.bc_head = bcp->bc_next; 3587 kmem_free(bcp, sizeof (strbufcall_t)); 3588 } else { 3589 /* 3590 * too big, try again later - note 3591 * that nevent was decremented above 3592 * so we won't retry this one on this 3593 * iteration of the loop 3594 */ 3595 if (bcp->bc_next != NULL) { 3596 strbcalls.bc_head = bcp->bc_next; 3597 bcp->bc_next = NULL; 3598 strbcalls.bc_tail->bc_next = bcp; 3599 strbcalls.bc_tail = bcp; 3600 } 3601 } 3602 } 3603 if (strbcalls.bc_head == NULL) 3604 strbcalls.bc_tail = NULL; 3605 } 3606 3607 mutex_exit(&strbcall_lock); 3608 mutex_exit(&bcall_monitor); 3609 } 3610 3611 3612 /* 3613 * actually run queue's service routine. 3614 */ 3615 static void 3616 runservice(queue_t *q) 3617 { 3618 qband_t *qbp; 3619 3620 ASSERT(q->q_qinfo->qi_srvp); 3621 again: 3622 entersq(q->q_syncq, SQ_SVC); 3623 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START, 3624 "runservice starts:%p", q); 3625 3626 if (!(q->q_flag & QWCLOSE)) 3627 (*q->q_qinfo->qi_srvp)(q); 3628 3629 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END, 3630 "runservice ends:(%p)", q); 3631 3632 leavesq(q->q_syncq, SQ_SVC); 3633 3634 mutex_enter(QLOCK(q)); 3635 if (q->q_flag & QENAB) { 3636 q->q_flag &= ~QENAB; 3637 mutex_exit(QLOCK(q)); 3638 goto again; 3639 } 3640 q->q_flag &= ~QINSERVICE; 3641 q->q_flag &= ~QBACK; 3642 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next) 3643 qbp->qb_flag &= ~QB_BACK; 3644 /* 3645 * Wakeup thread waiting for the service procedure 3646 * to be run (strclose and qdetach). 3647 */ 3648 cv_broadcast(&q->q_wait); 3649 3650 mutex_exit(QLOCK(q)); 3651 } 3652 3653 /* 3654 * Background processing of bufcalls. 3655 */ 3656 void 3657 streams_bufcall_service(void) 3658 { 3659 callb_cpr_t cprinfo; 3660 3661 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr, 3662 "streams_bufcall_service"); 3663 3664 mutex_enter(&strbcall_lock); 3665 3666 for (;;) { 3667 if (strbcalls.bc_head != NULL && kmem_avail() > 0) { 3668 mutex_exit(&strbcall_lock); 3669 runbufcalls(); 3670 mutex_enter(&strbcall_lock); 3671 } 3672 if (strbcalls.bc_head != NULL) { 3673 clock_t wt, tick; 3674 3675 STRSTAT(bcwaits); 3676 /* Wait for memory to become available */ 3677 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3678 tick = SEC_TO_TICK(60); 3679 time_to_wait(&wt, tick); 3680 (void) cv_timedwait(&memavail_cv, &strbcall_lock, wt); 3681 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3682 } 3683 3684 /* Wait for new work to arrive */ 3685 if (strbcalls.bc_head == NULL) { 3686 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3687 cv_wait(&strbcall_cv, &strbcall_lock); 3688 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3689 } 3690 } 3691 } 3692 3693 /* 3694 * Background processing of streams background tasks which failed 3695 * taskq_dispatch. 3696 */ 3697 static void 3698 streams_qbkgrnd_service(void) 3699 { 3700 callb_cpr_t cprinfo; 3701 queue_t *q; 3702 3703 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3704 "streams_bkgrnd_service"); 3705 3706 mutex_enter(&service_queue); 3707 3708 for (;;) { 3709 /* 3710 * Wait for work to arrive. 3711 */ 3712 while ((freebs_list == NULL) && (qhead == NULL)) { 3713 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3714 cv_wait(&services_to_run, &service_queue); 3715 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3716 } 3717 /* 3718 * Handle all pending freebs requests to free memory. 3719 */ 3720 while (freebs_list != NULL) { 3721 mblk_t *mp = freebs_list; 3722 freebs_list = mp->b_next; 3723 mutex_exit(&service_queue); 3724 mblk_free(mp); 3725 mutex_enter(&service_queue); 3726 } 3727 /* 3728 * Run pending queues. 3729 */ 3730 while (qhead != NULL) { 3731 DQ(q, qhead, qtail, q_link); 3732 ASSERT(q != NULL); 3733 mutex_exit(&service_queue); 3734 queue_service(q); 3735 mutex_enter(&service_queue); 3736 } 3737 ASSERT(qhead == NULL && qtail == NULL); 3738 } 3739 } 3740 3741 /* 3742 * Background processing of streams background tasks which failed 3743 * taskq_dispatch. 3744 */ 3745 static void 3746 streams_sqbkgrnd_service(void) 3747 { 3748 callb_cpr_t cprinfo; 3749 syncq_t *sq; 3750 3751 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3752 "streams_sqbkgrnd_service"); 3753 3754 mutex_enter(&service_queue); 3755 3756 for (;;) { 3757 /* 3758 * Wait for work to arrive. 3759 */ 3760 while (sqhead == NULL) { 3761 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3762 cv_wait(&syncqs_to_run, &service_queue); 3763 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3764 } 3765 3766 /* 3767 * Run pending syncqs. 3768 */ 3769 while (sqhead != NULL) { 3770 DQ(sq, sqhead, sqtail, sq_next); 3771 ASSERT(sq != NULL); 3772 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); 3773 mutex_exit(&service_queue); 3774 syncq_service(sq); 3775 mutex_enter(&service_queue); 3776 } 3777 } 3778 } 3779 3780 /* 3781 * Disable the syncq and wait for background syncq processing to complete. 3782 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the 3783 * list. 3784 */ 3785 void 3786 wait_sq_svc(syncq_t *sq) 3787 { 3788 mutex_enter(SQLOCK(sq)); 3789 sq->sq_svcflags |= SQ_DISABLED; 3790 if (sq->sq_svcflags & SQ_BGTHREAD) { 3791 syncq_t *sq_chase; 3792 syncq_t *sq_curr; 3793 int removed; 3794 3795 ASSERT(sq->sq_servcount == 1); 3796 mutex_enter(&service_queue); 3797 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); 3798 mutex_exit(&service_queue); 3799 if (removed) { 3800 sq->sq_svcflags &= ~SQ_BGTHREAD; 3801 sq->sq_servcount = 0; 3802 STRSTAT(sqremoved); 3803 goto done; 3804 } 3805 } 3806 while (sq->sq_servcount != 0) { 3807 sq->sq_flags |= SQ_WANTWAKEUP; 3808 cv_wait(&sq->sq_wait, SQLOCK(sq)); 3809 } 3810 done: 3811 mutex_exit(SQLOCK(sq)); 3812 } 3813 3814 /* 3815 * Put a syncq on the list of syncq's to be serviced by the sqthread. 3816 * Add the argument to the end of the sqhead list and set the flag 3817 * indicating this syncq has been enabled. If it has already been 3818 * enabled, don't do anything. 3819 * This routine assumes that SQLOCK is held. 3820 * NOTE that the lock order is to have the SQLOCK first, 3821 * so if the service_syncq lock is held, we need to release it 3822 * before aquiring the SQLOCK (mostly relevant for the background 3823 * thread, and this seems to be common among the STREAMS global locks). 3824 * Note the the sq_svcflags are protected by the SQLOCK. 3825 */ 3826 void 3827 sqenable(syncq_t *sq) 3828 { 3829 /* 3830 * This is probably not important except for where I believe it 3831 * is being called. At that point, it should be held (and it 3832 * is a pain to release it just for this routine, so don't do 3833 * it). 3834 */ 3835 ASSERT(MUTEX_HELD(SQLOCK(sq))); 3836 3837 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); 3838 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); 3839 3840 /* 3841 * Do not put on list if background thread is scheduled or 3842 * syncq is disabled. 3843 */ 3844 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) 3845 return; 3846 3847 /* 3848 * Check whether we should enable sq at all. 3849 * Non PERMOD syncqs may be drained by at most one thread. 3850 * PERMOD syncqs may be drained by several threads but we limit the 3851 * total amount to the lesser of 3852 * Number of queues on the squeue and 3853 * Number of CPUs. 3854 */ 3855 if (sq->sq_servcount != 0) { 3856 if (((sq->sq_type & SQ_PERMOD) == 0) || 3857 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { 3858 STRSTAT(sqtoomany); 3859 return; 3860 } 3861 } 3862 3863 sq->sq_tstamp = lbolt; 3864 STRSTAT(sqenables); 3865 3866 /* Attempt a taskq dispatch */ 3867 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, 3868 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); 3869 if (sq->sq_servid != NULL) { 3870 sq->sq_servcount++; 3871 return; 3872 } 3873 3874 /* 3875 * This taskq dispatch failed, but a previous one may have succeeded. 3876 * Don't try to schedule on the background thread whilst there is 3877 * outstanding taskq processing. 3878 */ 3879 if (sq->sq_servcount != 0) 3880 return; 3881 3882 /* 3883 * System is low on resources and can't perform a non-sleeping 3884 * dispatch. Schedule the syncq for a background thread and mark the 3885 * syncq to avoid any further taskq dispatch attempts. 3886 */ 3887 mutex_enter(&service_queue); 3888 STRSTAT(taskqfails); 3889 ENQUEUE(sq, sqhead, sqtail, sq_next); 3890 sq->sq_svcflags |= SQ_BGTHREAD; 3891 sq->sq_servcount = 1; 3892 cv_signal(&syncqs_to_run); 3893 mutex_exit(&service_queue); 3894 } 3895 3896 /* 3897 * Note: fifo_close() depends on the mblk_t on the queue being freed 3898 * asynchronously. The asynchronous freeing of messages breaks the 3899 * recursive call chain of fifo_close() while there are I_SENDFD type of 3900 * messages refering other file pointers on the queue. Then when 3901 * closing pipes it can avoid stack overflow in case of daisy-chained 3902 * pipes, and also avoid deadlock in case of fifonode_t pairs (which 3903 * share the same fifolock_t). 3904 */ 3905 3906 /* ARGSUSED */ 3907 void 3908 freebs_enqueue(mblk_t *mp, dblk_t *dbp) 3909 { 3910 ASSERT(dbp->db_mblk == mp); 3911 3912 /* 3913 * Check data sanity. The dblock should have non-empty free function. 3914 * It is better to panic here then later when the dblock is freed 3915 * asynchronously when the context is lost. 3916 */ 3917 if (dbp->db_frtnp->free_func == NULL) { 3918 panic("freebs_enqueue: dblock %p has a NULL free callback", 3919 (void *) dbp); 3920 } 3921 3922 STRSTAT(freebs); 3923 if (taskq_dispatch(streams_taskq, (task_func_t *)mblk_free, mp, 3924 TQ_NOSLEEP) == NULL) { 3925 /* 3926 * System is low on resources and can't perform a non-sleeping 3927 * dispatch. Schedule for a background thread. 3928 */ 3929 mutex_enter(&service_queue); 3930 STRSTAT(taskqfails); 3931 mp->b_next = freebs_list; 3932 freebs_list = mp; 3933 cv_signal(&services_to_run); 3934 mutex_exit(&service_queue); 3935 } 3936 } 3937 3938 /* 3939 * Set the QBACK or QB_BACK flag in the given queue for 3940 * the given priority band. 3941 */ 3942 void 3943 setqback(queue_t *q, unsigned char pri) 3944 { 3945 int i; 3946 qband_t *qbp; 3947 qband_t **qbpp; 3948 3949 ASSERT(MUTEX_HELD(QLOCK(q))); 3950 if (pri != 0) { 3951 if (pri > q->q_nband) { 3952 qbpp = &q->q_bandp; 3953 while (*qbpp) 3954 qbpp = &(*qbpp)->qb_next; 3955 while (pri > q->q_nband) { 3956 if ((*qbpp = allocband()) == NULL) { 3957 cmn_err(CE_WARN, 3958 "setqback: can't allocate qband\n"); 3959 return; 3960 } 3961 (*qbpp)->qb_hiwat = q->q_hiwat; 3962 (*qbpp)->qb_lowat = q->q_lowat; 3963 q->q_nband++; 3964 qbpp = &(*qbpp)->qb_next; 3965 } 3966 } 3967 qbp = q->q_bandp; 3968 i = pri; 3969 while (--i) 3970 qbp = qbp->qb_next; 3971 qbp->qb_flag |= QB_BACK; 3972 } else { 3973 q->q_flag |= QBACK; 3974 } 3975 } 3976 3977 int 3978 strcopyin(void *from, void *to, size_t len, int copyflag) 3979 { 3980 if (copyflag & U_TO_K) { 3981 ASSERT((copyflag & K_TO_K) == 0); 3982 if (copyin(from, to, len)) 3983 return (EFAULT); 3984 } else { 3985 ASSERT(copyflag & K_TO_K); 3986 bcopy(from, to, len); 3987 } 3988 return (0); 3989 } 3990 3991 int 3992 strcopyout(void *from, void *to, size_t len, int copyflag) 3993 { 3994 if (copyflag & U_TO_K) { 3995 if (copyout(from, to, len)) 3996 return (EFAULT); 3997 } else { 3998 ASSERT(copyflag & K_TO_K); 3999 bcopy(from, to, len); 4000 } 4001 return (0); 4002 } 4003 4004 /* 4005 * strsignal_nolock() posts a signal to the process(es) at the stream head. 4006 * It assumes that the stream head lock is already held, whereas strsignal() 4007 * acquires the lock first. This routine was created because a few callers 4008 * release the stream head lock before calling only to re-acquire it after 4009 * it returns. 4010 */ 4011 void 4012 strsignal_nolock(stdata_t *stp, int sig, int32_t band) 4013 { 4014 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4015 switch (sig) { 4016 case SIGPOLL: 4017 if (stp->sd_sigflags & S_MSG) 4018 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 4019 break; 4020 4021 default: 4022 if (stp->sd_pgidp) { 4023 pgsignal(stp->sd_pgidp, sig); 4024 } 4025 break; 4026 } 4027 } 4028 4029 void 4030 strsignal(stdata_t *stp, int sig, int32_t band) 4031 { 4032 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG, 4033 "strsignal:%p, %X, %X", stp, sig, band); 4034 4035 mutex_enter(&stp->sd_lock); 4036 switch (sig) { 4037 case SIGPOLL: 4038 if (stp->sd_sigflags & S_MSG) 4039 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 4040 break; 4041 4042 default: 4043 if (stp->sd_pgidp) { 4044 pgsignal(stp->sd_pgidp, sig); 4045 } 4046 break; 4047 } 4048 mutex_exit(&stp->sd_lock); 4049 } 4050 4051 void 4052 strhup(stdata_t *stp) 4053 { 4054 ASSERT(mutex_owned(&stp->sd_lock)); 4055 pollwakeup(&stp->sd_pollist, POLLHUP); 4056 if (stp->sd_sigflags & S_HANGUP) 4057 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0); 4058 } 4059 4060 /* 4061 * Backenable the first queue upstream from `q' with a service procedure. 4062 */ 4063 void 4064 backenable(queue_t *q, uchar_t pri) 4065 { 4066 queue_t *nq; 4067 4068 /* 4069 * our presence might not prevent other modules in our own 4070 * stream from popping/pushing since the caller of getq might not 4071 * have a claim on the queue (some drivers do a getq on somebody 4072 * else's queue - they know that the queue itself is not going away 4073 * but the framework has to guarantee q_next in that stream.) 4074 */ 4075 claimstr(q); 4076 4077 /* find nearest back queue with service proc */ 4078 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) { 4079 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq)); 4080 } 4081 4082 if (nq) { 4083 kthread_t *freezer; 4084 /* 4085 * backenable can be called either with no locks held 4086 * or with the stream frozen (the latter occurs when a module 4087 * calls rmvq with the stream frozen.) If the stream is frozen 4088 * by the caller the caller will hold all qlocks in the stream. 4089 * Note that a frozen stream doesn't freeze a mated stream, 4090 * so we explicitly check for that. 4091 */ 4092 freezer = STREAM(q)->sd_freezer; 4093 if (freezer != curthread || STREAM(q) != STREAM(nq)) { 4094 mutex_enter(QLOCK(nq)); 4095 } 4096 #ifdef DEBUG 4097 else { 4098 ASSERT(frozenstr(q)); 4099 ASSERT(MUTEX_HELD(QLOCK(q))); 4100 ASSERT(MUTEX_HELD(QLOCK(nq))); 4101 } 4102 #endif 4103 setqback(nq, pri); 4104 qenable_locked(nq); 4105 if (freezer != curthread || STREAM(q) != STREAM(nq)) 4106 mutex_exit(QLOCK(nq)); 4107 } 4108 releasestr(q); 4109 } 4110 4111 /* 4112 * Return the appropriate errno when one of flags_to_check is set 4113 * in sd_flags. Uses the exported error routines if they are set. 4114 * Will return 0 if non error is set (or if the exported error routines 4115 * do not return an error). 4116 * 4117 * If there is both a read and write error to check we prefer the read error. 4118 * Also, give preference to recorded errno's over the error functions. 4119 * The flags that are handled are: 4120 * STPLEX return EINVAL 4121 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST) 4122 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST) 4123 * STRHUP return sd_werror 4124 * 4125 * If the caller indicates that the operation is a peek a nonpersistent error 4126 * is not cleared. 4127 */ 4128 int 4129 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek) 4130 { 4131 int32_t sd_flag = stp->sd_flag & flags_to_check; 4132 int error = 0; 4133 4134 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4135 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0); 4136 if (sd_flag & STPLEX) 4137 error = EINVAL; 4138 else if (sd_flag & STRDERR) { 4139 error = stp->sd_rerror; 4140 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) { 4141 /* 4142 * Read errors are non-persistent i.e. discarded once 4143 * returned to a non-peeking caller, 4144 */ 4145 stp->sd_rerror = 0; 4146 stp->sd_flag &= ~STRDERR; 4147 } 4148 if (error == 0 && stp->sd_rderrfunc != NULL) { 4149 int clearerr = 0; 4150 4151 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek, 4152 &clearerr); 4153 if (clearerr) { 4154 stp->sd_flag &= ~STRDERR; 4155 stp->sd_rderrfunc = NULL; 4156 } 4157 } 4158 } else if (sd_flag & STWRERR) { 4159 error = stp->sd_werror; 4160 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) { 4161 /* 4162 * Write errors are non-persistent i.e. discarded once 4163 * returned to a non-peeking caller, 4164 */ 4165 stp->sd_werror = 0; 4166 stp->sd_flag &= ~STWRERR; 4167 } 4168 if (error == 0 && stp->sd_wrerrfunc != NULL) { 4169 int clearerr = 0; 4170 4171 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek, 4172 &clearerr); 4173 if (clearerr) { 4174 stp->sd_flag &= ~STWRERR; 4175 stp->sd_wrerrfunc = NULL; 4176 } 4177 } 4178 } else if (sd_flag & STRHUP) { 4179 /* sd_werror set when STRHUP */ 4180 error = stp->sd_werror; 4181 } 4182 return (error); 4183 } 4184 4185 4186 /* 4187 * single-thread open/close/push/pop 4188 * for twisted streams also 4189 */ 4190 int 4191 strstartplumb(stdata_t *stp, int flag, int cmd) 4192 { 4193 int waited = 1; 4194 int error = 0; 4195 4196 if (STRMATED(stp)) { 4197 struct stdata *stmatep = stp->sd_mate; 4198 4199 STRLOCKMATES(stp); 4200 while (waited) { 4201 waited = 0; 4202 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4203 if ((cmd == I_POP) && 4204 (flag & (FNDELAY|FNONBLOCK))) { 4205 STRUNLOCKMATES(stp); 4206 return (EAGAIN); 4207 } 4208 waited = 1; 4209 mutex_exit(&stp->sd_lock); 4210 if (!cv_wait_sig(&stmatep->sd_monitor, 4211 &stmatep->sd_lock)) { 4212 mutex_exit(&stmatep->sd_lock); 4213 return (EINTR); 4214 } 4215 mutex_exit(&stmatep->sd_lock); 4216 STRLOCKMATES(stp); 4217 } 4218 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4219 if ((cmd == I_POP) && 4220 (flag & (FNDELAY|FNONBLOCK))) { 4221 STRUNLOCKMATES(stp); 4222 return (EAGAIN); 4223 } 4224 waited = 1; 4225 mutex_exit(&stmatep->sd_lock); 4226 if (!cv_wait_sig(&stp->sd_monitor, 4227 &stp->sd_lock)) { 4228 mutex_exit(&stp->sd_lock); 4229 return (EINTR); 4230 } 4231 mutex_exit(&stp->sd_lock); 4232 STRLOCKMATES(stp); 4233 } 4234 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4235 error = strgeterr(stp, 4236 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4237 if (error != 0) { 4238 STRUNLOCKMATES(stp); 4239 return (error); 4240 } 4241 } 4242 } 4243 stp->sd_flag |= STRPLUMB; 4244 STRUNLOCKMATES(stp); 4245 } else { 4246 mutex_enter(&stp->sd_lock); 4247 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4248 if (((cmd == I_POP) || (cmd == _I_REMOVE)) && 4249 (flag & (FNDELAY|FNONBLOCK))) { 4250 mutex_exit(&stp->sd_lock); 4251 return (EAGAIN); 4252 } 4253 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) { 4254 mutex_exit(&stp->sd_lock); 4255 return (EINTR); 4256 } 4257 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4258 error = strgeterr(stp, 4259 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4260 if (error != 0) { 4261 mutex_exit(&stp->sd_lock); 4262 return (error); 4263 } 4264 } 4265 } 4266 stp->sd_flag |= STRPLUMB; 4267 mutex_exit(&stp->sd_lock); 4268 } 4269 return (0); 4270 } 4271 4272 /* 4273 * Complete the plumbing operation associated with stream `stp'. 4274 */ 4275 void 4276 strendplumb(stdata_t *stp) 4277 { 4278 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4279 ASSERT(stp->sd_flag & STRPLUMB); 4280 stp->sd_flag &= ~STRPLUMB; 4281 cv_broadcast(&stp->sd_monitor); 4282 } 4283 4284 /* 4285 * This describes how the STREAMS framework handles synchronization 4286 * during open/push and close/pop. 4287 * The key interfaces for open and close are qprocson and qprocsoff, 4288 * respectively. While the close case in general is harder both open 4289 * have close have significant similarities. 4290 * 4291 * During close the STREAMS framework has to both ensure that there 4292 * are no stale references to the queue pair (and syncq) that 4293 * are being closed and also provide the guarantees that are documented 4294 * in qprocsoff(9F). 4295 * If there are stale references to the queue that is closing it can 4296 * result in kernel memory corruption or kernel panics. 4297 * 4298 * Note that is it up to the module/driver to ensure that it itself 4299 * does not have any stale references to the closing queues once its close 4300 * routine returns. This includes: 4301 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines 4302 * associated with the queues. For timeout and bufcall callbacks the 4303 * module/driver also has to ensure (or wait for) any callbacks that 4304 * are in progress. 4305 * - If the module/driver is using esballoc it has to ensure that any 4306 * esballoc free functions do not refer to a queue that has closed. 4307 * (Note that in general the close routine can not wait for the esballoc'ed 4308 * messages to be freed since that can cause a deadlock.) 4309 * - Cancelling any interrupts that refer to the closing queues and 4310 * also ensuring that there are no interrupts in progress that will 4311 * refer to the closing queues once the close routine returns. 4312 * - For multiplexors removing any driver global state that refers to 4313 * the closing queue and also ensuring that there are no threads in 4314 * the multiplexor that has picked up a queue pointer but not yet 4315 * finished using it. 4316 * 4317 * In addition, a driver/module can only reference the q_next pointer 4318 * in its open, close, put, or service procedures or in a 4319 * qtimeout/qbufcall callback procedure executing "on" the correct 4320 * stream. Thus it can not reference the q_next pointer in an interrupt 4321 * routine or a timeout, bufcall or esballoc callback routine. Likewise 4322 * it can not reference q_next of a different queue e.g. in a mux that 4323 * passes messages from one queues put/service procedure to another queue. 4324 * In all the cases when the driver/module can not access the q_next 4325 * field it must use the *next* versions e.g. canputnext instead of 4326 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...). 4327 * 4328 * 4329 * Assuming that the driver/module conforms to the above constraints 4330 * the STREAMS framework has to avoid stale references to q_next for all 4331 * the framework internal cases which include (but are not limited to): 4332 * - Threads in canput/canputnext/backenable and elsewhere that are 4333 * walking q_next. 4334 * - Messages on a syncq that have a reference to the queue through b_queue. 4335 * - Messages on an outer perimeter (syncq) that have a reference to the 4336 * queue through b_queue. 4337 * - Threads that use q_nfsrv (e.g. canput) to find a queue. 4338 * Note that only canput and bcanput use q_nfsrv without any locking. 4339 * 4340 * The STREAMS framework providing the qprocsoff(9F) guarantees means that 4341 * after qprocsoff returns, the framework has to ensure that no threads can 4342 * enter the put or service routines for the closing read or write-side queue. 4343 * In addition to preventing "direct" entry into the put procedures 4344 * the framework also has to prevent messages being drained from 4345 * the syncq or the outer perimeter. 4346 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only 4347 * mechanism to prevent qwriter(PERIM_OUTER) from running after 4348 * qprocsoff has returned. 4349 * Note that if a module/driver uses put(9F) on one of its own queues 4350 * it is up to the module/driver to ensure that the put() doesn't 4351 * get called when the queue is closing. 4352 * 4353 * 4354 * The framework aspects of the above "contract" is implemented by 4355 * qprocsoff, removeq, and strlock: 4356 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from 4357 * entering the service procedures. 4358 * - strlock acquires the sd_lock and sd_reflock to prevent putnext, 4359 * canputnext, backenable etc from dereferencing the q_next that will 4360 * soon change. 4361 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext 4362 * or other q_next walker that uses claimstr/releasestr to finish. 4363 * - optionally for every syncq in the stream strlock acquires all the 4364 * sq_lock's and waits for all sq_counts to drop to a value that indicates 4365 * that no thread executes in the put or service procedures and that no 4366 * thread is draining into the module/driver. This ensures that no 4367 * open, close, put, service, or qtimeout/qbufcall callback procedure is 4368 * currently executing hence no such thread can end up with the old stale 4369 * q_next value and no canput/backenable can have the old stale 4370 * q_nfsrv/q_next. 4371 * - qdetach (wait_svc) makes sure that any scheduled or running threads 4372 * have either finished or observed the QWCLOSE flag and gone away. 4373 */ 4374 4375 4376 /* 4377 * Get all the locks necessary to change q_next. 4378 * 4379 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the 4380 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that 4381 * the only threads inside the sqncq are threads currently calling removeq(). 4382 * Since threads calling removeq() are in the process of removing their queues 4383 * from the stream, we do not need to worry about them accessing a stale q_next 4384 * pointer and thus we do not need to wait for them to exit (in fact, waiting 4385 * for them can cause deadlock). 4386 * 4387 * This routine is subject to starvation since it does not set any flag to 4388 * prevent threads from entering a module in the stream(i.e. sq_count can 4389 * increase on some syncq while it is waiting on some other syncq.) 4390 * 4391 * Assumes that only one thread attempts to call strlock for a given 4392 * stream. If this is not the case the two threads would deadlock. 4393 * This assumption is guaranteed since strlock is only called by insertq 4394 * and removeq and streams plumbing changes are single-threaded for 4395 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags. 4396 * 4397 * For pipes, it is not difficult to atomically designate a pair of streams 4398 * to be mated. Once mated atomically by the framework the twisted pair remain 4399 * configured that way until dismantled atomically by the framework. 4400 * When plumbing takes place on a twisted stream it is necessary to ensure that 4401 * this operation is done exclusively on the twisted stream since two such 4402 * operations, each initiated on different ends of the pipe will deadlock 4403 * waiting for each other to complete. 4404 * 4405 * On entry, no locks should be held. 4406 * The locks acquired and held by strlock depends on a few factors. 4407 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired 4408 * and held on exit and all sq_count are at an acceptable level. 4409 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with 4410 * sd_refcnt being zero. 4411 */ 4412 4413 static void 4414 strlock(struct stdata *stp, sqlist_t *sqlist) 4415 { 4416 syncql_t *sql, *sql2; 4417 retry: 4418 /* 4419 * Wait for any claimstr to go away. 4420 */ 4421 if (STRMATED(stp)) { 4422 struct stdata *stp1, *stp2; 4423 4424 STRLOCKMATES(stp); 4425 /* 4426 * Note that the selection of locking order is not 4427 * important, just that they are always aquired in 4428 * the same order. To assure this, we choose this 4429 * order based on the value of the pointer, and since 4430 * the pointer will not change for the life of this 4431 * pair, we will always grab the locks in the same 4432 * order (and hence, prevent deadlocks). 4433 */ 4434 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) { 4435 stp1 = stp; 4436 stp2 = stp->sd_mate; 4437 } else { 4438 stp2 = stp; 4439 stp1 = stp->sd_mate; 4440 } 4441 mutex_enter(&stp1->sd_reflock); 4442 if (stp1->sd_refcnt > 0) { 4443 STRUNLOCKMATES(stp); 4444 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock); 4445 mutex_exit(&stp1->sd_reflock); 4446 goto retry; 4447 } 4448 mutex_enter(&stp2->sd_reflock); 4449 if (stp2->sd_refcnt > 0) { 4450 STRUNLOCKMATES(stp); 4451 mutex_exit(&stp1->sd_reflock); 4452 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock); 4453 mutex_exit(&stp2->sd_reflock); 4454 goto retry; 4455 } 4456 STREAM_PUTLOCKS_ENTER(stp1); 4457 STREAM_PUTLOCKS_ENTER(stp2); 4458 } else { 4459 mutex_enter(&stp->sd_lock); 4460 mutex_enter(&stp->sd_reflock); 4461 while (stp->sd_refcnt > 0) { 4462 mutex_exit(&stp->sd_lock); 4463 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock); 4464 if (mutex_tryenter(&stp->sd_lock) == 0) { 4465 mutex_exit(&stp->sd_reflock); 4466 mutex_enter(&stp->sd_lock); 4467 mutex_enter(&stp->sd_reflock); 4468 } 4469 } 4470 STREAM_PUTLOCKS_ENTER(stp); 4471 } 4472 4473 if (sqlist == NULL) 4474 return; 4475 4476 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4477 syncq_t *sq = sql->sql_sq; 4478 uint16_t count; 4479 4480 mutex_enter(SQLOCK(sq)); 4481 count = sq->sq_count; 4482 ASSERT(sq->sq_rmqcount <= count); 4483 SQ_PUTLOCKS_ENTER(sq); 4484 SUM_SQ_PUTCOUNTS(sq, count); 4485 if (count == sq->sq_rmqcount) 4486 continue; 4487 4488 /* Failed - drop all locks that we have acquired so far */ 4489 if (STRMATED(stp)) { 4490 STREAM_PUTLOCKS_EXIT(stp); 4491 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4492 STRUNLOCKMATES(stp); 4493 mutex_exit(&stp->sd_reflock); 4494 mutex_exit(&stp->sd_mate->sd_reflock); 4495 } else { 4496 STREAM_PUTLOCKS_EXIT(stp); 4497 mutex_exit(&stp->sd_lock); 4498 mutex_exit(&stp->sd_reflock); 4499 } 4500 for (sql2 = sqlist->sqlist_head; sql2 != sql; 4501 sql2 = sql2->sql_next) { 4502 SQ_PUTLOCKS_EXIT(sql2->sql_sq); 4503 mutex_exit(SQLOCK(sql2->sql_sq)); 4504 } 4505 4506 /* 4507 * The wait loop below may starve when there are many threads 4508 * claiming the syncq. This is especially a problem with permod 4509 * syncqs (IP). To lessen the impact of the problem we increment 4510 * sq_needexcl and clear fastbits so that putnexts will slow 4511 * down and call sqenable instead of draining right away. 4512 */ 4513 sq->sq_needexcl++; 4514 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4515 while (count > sq->sq_rmqcount) { 4516 sq->sq_flags |= SQ_WANTWAKEUP; 4517 SQ_PUTLOCKS_EXIT(sq); 4518 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4519 count = sq->sq_count; 4520 SQ_PUTLOCKS_ENTER(sq); 4521 SUM_SQ_PUTCOUNTS(sq, count); 4522 } 4523 sq->sq_needexcl--; 4524 if (sq->sq_needexcl == 0) 4525 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4526 SQ_PUTLOCKS_EXIT(sq); 4527 ASSERT(count == sq->sq_rmqcount); 4528 mutex_exit(SQLOCK(sq)); 4529 goto retry; 4530 } 4531 } 4532 4533 /* 4534 * Drop all the locks that strlock acquired. 4535 */ 4536 static void 4537 strunlock(struct stdata *stp, sqlist_t *sqlist) 4538 { 4539 syncql_t *sql; 4540 4541 if (STRMATED(stp)) { 4542 STREAM_PUTLOCKS_EXIT(stp); 4543 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4544 STRUNLOCKMATES(stp); 4545 mutex_exit(&stp->sd_reflock); 4546 mutex_exit(&stp->sd_mate->sd_reflock); 4547 } else { 4548 STREAM_PUTLOCKS_EXIT(stp); 4549 mutex_exit(&stp->sd_lock); 4550 mutex_exit(&stp->sd_reflock); 4551 } 4552 4553 if (sqlist == NULL) 4554 return; 4555 4556 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4557 SQ_PUTLOCKS_EXIT(sql->sql_sq); 4558 mutex_exit(SQLOCK(sql->sql_sq)); 4559 } 4560 } 4561 4562 /* 4563 * When the module has service procedure, we need check if the next 4564 * module which has service procedure is in flow control to trigger 4565 * the backenable. 4566 */ 4567 static void 4568 backenable_insertedq(queue_t *q) 4569 { 4570 qband_t *qbp; 4571 4572 claimstr(q); 4573 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) { 4574 if (q->q_next->q_nfsrv->q_flag & QWANTW) 4575 backenable(q, 0); 4576 4577 qbp = q->q_next->q_nfsrv->q_bandp; 4578 for (; qbp != NULL; qbp = qbp->qb_next) 4579 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL) 4580 backenable(q, qbp->qb_first->b_band); 4581 } 4582 releasestr(q); 4583 } 4584 4585 /* 4586 * Given two read queues, insert a new single one after another. 4587 * 4588 * This routine acquires all the necessary locks in order to change 4589 * q_next and related pointer using strlock(). 4590 * It depends on the stream head ensuring that there are no concurrent 4591 * insertq or removeq on the same stream. The stream head ensures this 4592 * using the flags STWOPEN, STRCLOSE, and STRPLUMB. 4593 * 4594 * Note that no syncq locks are held during the q_next change. This is 4595 * applied to all streams since, unlike removeq, there is no problem of stale 4596 * pointers when adding a module to the stream. Thus drivers/modules that do a 4597 * canput(rq->q_next) would never get a closed/freed queue pointer even if we 4598 * applied this optimization to all streams. 4599 */ 4600 void 4601 insertq(struct stdata *stp, queue_t *new) 4602 { 4603 queue_t *after; 4604 queue_t *wafter; 4605 queue_t *wnew = _WR(new); 4606 boolean_t have_fifo = B_FALSE; 4607 4608 if (new->q_flag & _QINSERTING) { 4609 ASSERT(stp->sd_vnode->v_type != VFIFO); 4610 after = new->q_next; 4611 wafter = _WR(new->q_next); 4612 } else { 4613 after = _RD(stp->sd_wrq); 4614 wafter = stp->sd_wrq; 4615 } 4616 4617 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ, 4618 "insertq:%p, %p", after, new); 4619 ASSERT(after->q_flag & QREADR); 4620 ASSERT(new->q_flag & QREADR); 4621 4622 strlock(stp, NULL); 4623 4624 /* Do we have a FIFO? */ 4625 if (wafter->q_next == after) { 4626 have_fifo = B_TRUE; 4627 wnew->q_next = new; 4628 } else { 4629 wnew->q_next = wafter->q_next; 4630 } 4631 new->q_next = after; 4632 4633 set_nfsrv_ptr(new, wnew, after, wafter); 4634 /* 4635 * set_nfsrv_ptr() needs to know if this is an insertion or not, 4636 * so only reset this flag after calling it. 4637 */ 4638 new->q_flag &= ~_QINSERTING; 4639 4640 if (have_fifo) { 4641 wafter->q_next = wnew; 4642 } else { 4643 if (wafter->q_next) 4644 _OTHERQ(wafter->q_next)->q_next = new; 4645 wafter->q_next = wnew; 4646 } 4647 4648 set_qend(new); 4649 /* The QEND flag might have to be updated for the upstream guy */ 4650 set_qend(after); 4651 4652 ASSERT(_SAMESTR(new) == O_SAMESTR(new)); 4653 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew)); 4654 ASSERT(_SAMESTR(after) == O_SAMESTR(after)); 4655 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter)); 4656 strsetuio(stp); 4657 4658 /* 4659 * If this was a module insertion, bump the push count. 4660 */ 4661 if (!(new->q_flag & QISDRV)) 4662 stp->sd_pushcnt++; 4663 4664 strunlock(stp, NULL); 4665 4666 /* check if the write Q needs backenable */ 4667 backenable_insertedq(wnew); 4668 4669 /* check if the read Q needs backenable */ 4670 backenable_insertedq(new); 4671 } 4672 4673 /* 4674 * Given a read queue, unlink it from any neighbors. 4675 * 4676 * This routine acquires all the necessary locks in order to 4677 * change q_next and related pointers and also guard against 4678 * stale references (e.g. through q_next) to the queue that 4679 * is being removed. It also plays part of the role in ensuring 4680 * that the module's/driver's put procedure doesn't get called 4681 * after qprocsoff returns. 4682 * 4683 * Removeq depends on the stream head ensuring that there are 4684 * no concurrent insertq or removeq on the same stream. The 4685 * stream head ensures this using the flags STWOPEN, STRCLOSE and 4686 * STRPLUMB. 4687 * 4688 * The set of locks needed to remove the queue is different in 4689 * different cases: 4690 * 4691 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after 4692 * waiting for the syncq reference count to drop to 0 indicating that no 4693 * non-close threads are present anywhere in the stream. This ensures that any 4694 * module/driver can reference q_next in its open, close, put, or service 4695 * procedures. 4696 * 4697 * The sq_rmqcount counter tracks the number of threads inside removeq(). 4698 * strlock() ensures that there is either no threads executing inside perimeter 4699 * or there is only a thread calling qprocsoff(). 4700 * 4701 * strlock() compares the value of sq_count with the number of threads inside 4702 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup 4703 * any threads waiting in strlock() when the sq_rmqcount increases. 4704 */ 4705 4706 void 4707 removeq(queue_t *qp) 4708 { 4709 queue_t *wqp = _WR(qp); 4710 struct stdata *stp = STREAM(qp); 4711 sqlist_t *sqlist = NULL; 4712 boolean_t isdriver; 4713 int moved; 4714 syncq_t *sq = qp->q_syncq; 4715 syncq_t *wsq = wqp->q_syncq; 4716 4717 ASSERT(stp); 4718 4719 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ, 4720 "removeq:%p %p", qp, wqp); 4721 ASSERT(qp->q_flag&QREADR); 4722 4723 /* 4724 * For queues using Synchronous streams, we must wait for all threads in 4725 * rwnext() to drain out before proceeding. 4726 */ 4727 if (qp->q_flag & QSYNCSTR) { 4728 /* First, we need wakeup any threads blocked in rwnext() */ 4729 mutex_enter(SQLOCK(sq)); 4730 if (sq->sq_flags & SQ_WANTWAKEUP) { 4731 sq->sq_flags &= ~SQ_WANTWAKEUP; 4732 cv_broadcast(&sq->sq_wait); 4733 } 4734 mutex_exit(SQLOCK(sq)); 4735 4736 if (wsq != sq) { 4737 mutex_enter(SQLOCK(wsq)); 4738 if (wsq->sq_flags & SQ_WANTWAKEUP) { 4739 wsq->sq_flags &= ~SQ_WANTWAKEUP; 4740 cv_broadcast(&wsq->sq_wait); 4741 } 4742 mutex_exit(SQLOCK(wsq)); 4743 } 4744 4745 mutex_enter(QLOCK(qp)); 4746 while (qp->q_rwcnt > 0) { 4747 qp->q_flag |= QWANTRMQSYNC; 4748 cv_wait(&qp->q_wait, QLOCK(qp)); 4749 } 4750 mutex_exit(QLOCK(qp)); 4751 4752 mutex_enter(QLOCK(wqp)); 4753 while (wqp->q_rwcnt > 0) { 4754 wqp->q_flag |= QWANTRMQSYNC; 4755 cv_wait(&wqp->q_wait, QLOCK(wqp)); 4756 } 4757 mutex_exit(QLOCK(wqp)); 4758 } 4759 4760 mutex_enter(SQLOCK(sq)); 4761 sq->sq_rmqcount++; 4762 if (sq->sq_flags & SQ_WANTWAKEUP) { 4763 sq->sq_flags &= ~SQ_WANTWAKEUP; 4764 cv_broadcast(&sq->sq_wait); 4765 } 4766 mutex_exit(SQLOCK(sq)); 4767 4768 isdriver = (qp->q_flag & QISDRV); 4769 4770 sqlist = sqlist_build(qp, stp, STRMATED(stp)); 4771 strlock(stp, sqlist); 4772 4773 reset_nfsrv_ptr(qp, wqp); 4774 4775 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp); 4776 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp); 4777 /* Do we have a FIFO? */ 4778 if (wqp->q_next == qp) { 4779 stp->sd_wrq->q_next = _RD(stp->sd_wrq); 4780 } else { 4781 if (wqp->q_next) 4782 backq(qp)->q_next = qp->q_next; 4783 if (qp->q_next) 4784 backq(wqp)->q_next = wqp->q_next; 4785 } 4786 4787 /* The QEND flag might have to be updated for the upstream guy */ 4788 if (qp->q_next) 4789 set_qend(qp->q_next); 4790 4791 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq)); 4792 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq))); 4793 4794 /* 4795 * Move any messages destined for the put procedures to the next 4796 * syncq in line. Otherwise free them. 4797 */ 4798 moved = 0; 4799 /* 4800 * Quick check to see whether there are any messages or events. 4801 */ 4802 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS)) 4803 moved += propagate_syncq(qp); 4804 if (wqp->q_syncqmsgs != 0 || 4805 (wqp->q_syncq->sq_flags & SQ_EVENTS)) 4806 moved += propagate_syncq(wqp); 4807 4808 strsetuio(stp); 4809 4810 /* 4811 * If this was a module removal, decrement the push count. 4812 */ 4813 if (!isdriver) 4814 stp->sd_pushcnt--; 4815 4816 strunlock(stp, sqlist); 4817 sqlist_free(sqlist); 4818 4819 /* 4820 * Make sure any messages that were propagated are drained. 4821 * Also clear any QFULL bit caused by messages that were propagated. 4822 */ 4823 4824 if (qp->q_next != NULL) { 4825 clr_qfull(qp); 4826 /* 4827 * For the driver calling qprocsoff, propagate_syncq 4828 * frees all the messages instead of putting it in 4829 * the stream head 4830 */ 4831 if (!isdriver && (moved > 0)) 4832 emptysq(qp->q_next->q_syncq); 4833 } 4834 if (wqp->q_next != NULL) { 4835 clr_qfull(wqp); 4836 /* 4837 * We come here for any pop of a module except for the 4838 * case of driver being removed. We don't call emptysq 4839 * if we did not move any messages. This will avoid holding 4840 * PERMOD syncq locks in emptysq 4841 */ 4842 if (moved > 0) 4843 emptysq(wqp->q_next->q_syncq); 4844 } 4845 4846 mutex_enter(SQLOCK(sq)); 4847 sq->sq_rmqcount--; 4848 mutex_exit(SQLOCK(sq)); 4849 } 4850 4851 /* 4852 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or 4853 * SQ_WRITER) on a syncq. 4854 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the 4855 * sync queue and waits until sq_count reaches maxcnt. 4856 * 4857 * if maxcnt is -1 there's no need to grab sq_putlocks since the caller 4858 * does not care about putnext threads that are in the middle of calling put 4859 * entry points. 4860 * 4861 * This routine is used for both inner and outer syncqs. 4862 */ 4863 static void 4864 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) 4865 { 4866 uint16_t count = 0; 4867 4868 mutex_enter(SQLOCK(sq)); 4869 /* 4870 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset. 4871 * SQ_FROZEN will be set if there is a frozen stream that has a 4872 * queue which also refers to this "shared" syncq. 4873 * SQ_BLOCKED will be set if there is "off" queue which also 4874 * refers to this "shared" syncq. 4875 */ 4876 if (maxcnt != -1) { 4877 count = sq->sq_count; 4878 SQ_PUTLOCKS_ENTER(sq); 4879 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4880 SUM_SQ_PUTCOUNTS(sq, count); 4881 } 4882 sq->sq_needexcl++; 4883 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 4884 4885 while ((sq->sq_flags & flag) || 4886 (maxcnt != -1 && count > (unsigned)maxcnt)) { 4887 sq->sq_flags |= SQ_WANTWAKEUP; 4888 if (maxcnt != -1) { 4889 SQ_PUTLOCKS_EXIT(sq); 4890 } 4891 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4892 if (maxcnt != -1) { 4893 count = sq->sq_count; 4894 SQ_PUTLOCKS_ENTER(sq); 4895 SUM_SQ_PUTCOUNTS(sq, count); 4896 } 4897 } 4898 sq->sq_needexcl--; 4899 sq->sq_flags |= flag; 4900 ASSERT(maxcnt == -1 || count == maxcnt); 4901 if (maxcnt != -1) { 4902 if (sq->sq_needexcl == 0) { 4903 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4904 } 4905 SQ_PUTLOCKS_EXIT(sq); 4906 } else if (sq->sq_needexcl == 0) { 4907 SQ_PUTCOUNT_SETFAST(sq); 4908 } 4909 4910 mutex_exit(SQLOCK(sq)); 4911 } 4912 4913 /* 4914 * Reset a flag that was set with blocksq. 4915 * 4916 * Can not use this routine to reset SQ_WRITER. 4917 * 4918 * If "isouter" is set then the syncq is assumed to be an outer perimeter 4919 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread 4920 * to handle the queued qwriter operations. 4921 * 4922 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4923 * sq_putlocks are used. 4924 */ 4925 static void 4926 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) 4927 { 4928 uint16_t flags; 4929 4930 mutex_enter(SQLOCK(sq)); 4931 ASSERT(resetflag != SQ_WRITER); 4932 ASSERT(sq->sq_flags & resetflag); 4933 flags = sq->sq_flags & ~resetflag; 4934 sq->sq_flags = flags; 4935 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) { 4936 if (flags & SQ_WANTWAKEUP) { 4937 flags &= ~SQ_WANTWAKEUP; 4938 cv_broadcast(&sq->sq_wait); 4939 } 4940 sq->sq_flags = flags; 4941 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 4942 if (!isouter) { 4943 /* drain_syncq drops SQLOCK */ 4944 drain_syncq(sq); 4945 return; 4946 } 4947 } 4948 } 4949 mutex_exit(SQLOCK(sq)); 4950 } 4951 4952 /* 4953 * Reset a flag that was set with blocksq. 4954 * Does not drain the syncq. Use emptysq() for that. 4955 * Returns 1 if SQ_QUEUED is set. Otherwise 0. 4956 * 4957 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4958 * sq_putlocks are used. 4959 */ 4960 static int 4961 dropsq(syncq_t *sq, uint16_t resetflag) 4962 { 4963 uint16_t flags; 4964 4965 mutex_enter(SQLOCK(sq)); 4966 ASSERT(sq->sq_flags & resetflag); 4967 flags = sq->sq_flags & ~resetflag; 4968 if (flags & SQ_WANTWAKEUP) { 4969 flags &= ~SQ_WANTWAKEUP; 4970 cv_broadcast(&sq->sq_wait); 4971 } 4972 sq->sq_flags = flags; 4973 mutex_exit(SQLOCK(sq)); 4974 if (flags & SQ_QUEUED) 4975 return (1); 4976 return (0); 4977 } 4978 4979 /* 4980 * Empty all the messages on a syncq. 4981 * 4982 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4983 * sq_putlocks are used. 4984 */ 4985 static void 4986 emptysq(syncq_t *sq) 4987 { 4988 uint16_t flags; 4989 4990 mutex_enter(SQLOCK(sq)); 4991 flags = sq->sq_flags; 4992 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 4993 /* 4994 * To prevent potential recursive invocation of drain_syncq we 4995 * do not call drain_syncq if count is non-zero. 4996 */ 4997 if (sq->sq_count == 0) { 4998 /* drain_syncq() drops SQLOCK */ 4999 drain_syncq(sq); 5000 return; 5001 } else 5002 sqenable(sq); 5003 } 5004 mutex_exit(SQLOCK(sq)); 5005 } 5006 5007 /* 5008 * Ordered insert while removing duplicates. 5009 */ 5010 static void 5011 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp) 5012 { 5013 syncql_t *sqlp, **prev_sqlpp, *new_sqlp; 5014 5015 prev_sqlpp = &sqlist->sqlist_head; 5016 while ((sqlp = *prev_sqlpp) != NULL) { 5017 if (sqlp->sql_sq >= sqp) { 5018 if (sqlp->sql_sq == sqp) /* duplicate */ 5019 return; 5020 break; 5021 } 5022 prev_sqlpp = &sqlp->sql_next; 5023 } 5024 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++]; 5025 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size); 5026 new_sqlp->sql_next = sqlp; 5027 new_sqlp->sql_sq = sqp; 5028 *prev_sqlpp = new_sqlp; 5029 } 5030 5031 /* 5032 * Walk the write side queues until we hit either the driver 5033 * or a twist in the stream (_SAMESTR will return false in both 5034 * these cases) then turn around and walk the read side queues 5035 * back up to the stream head. 5036 */ 5037 static void 5038 sqlist_insertall(sqlist_t *sqlist, queue_t *q) 5039 { 5040 while (q != NULL) { 5041 sqlist_insert(sqlist, q->q_syncq); 5042 5043 if (_SAMESTR(q)) 5044 q = q->q_next; 5045 else if (!(q->q_flag & QREADR)) 5046 q = _RD(q); 5047 else 5048 q = NULL; 5049 } 5050 } 5051 5052 /* 5053 * Allocate and build a list of all syncqs in a stream and the syncq(s) 5054 * associated with the "q" parameter. The resulting list is sorted in a 5055 * canonical order and is free of duplicates. 5056 * Assumes the passed queue is a _RD(q). 5057 */ 5058 static sqlist_t * 5059 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist) 5060 { 5061 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP); 5062 5063 /* 5064 * start with the current queue/qpair 5065 */ 5066 ASSERT(q->q_flag & QREADR); 5067 5068 sqlist_insert(sqlist, q->q_syncq); 5069 sqlist_insert(sqlist, _WR(q)->q_syncq); 5070 5071 sqlist_insertall(sqlist, stp->sd_wrq); 5072 if (do_twist) 5073 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq); 5074 5075 return (sqlist); 5076 } 5077 5078 static sqlist_t * 5079 sqlist_alloc(struct stdata *stp, int kmflag) 5080 { 5081 size_t sqlist_size; 5082 sqlist_t *sqlist; 5083 5084 /* 5085 * Allocate 2 syncql_t's for each pushed module. Note that 5086 * the sqlist_t structure already has 4 syncql_t's built in: 5087 * 2 for the stream head, and 2 for the driver/other stream head. 5088 */ 5089 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt + 5090 sizeof (sqlist_t); 5091 if (STRMATED(stp)) 5092 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt; 5093 sqlist = kmem_alloc(sqlist_size, kmflag); 5094 5095 sqlist->sqlist_head = NULL; 5096 sqlist->sqlist_size = sqlist_size; 5097 sqlist->sqlist_index = 0; 5098 5099 return (sqlist); 5100 } 5101 5102 /* 5103 * Free the list created by sqlist_alloc() 5104 */ 5105 static void 5106 sqlist_free(sqlist_t *sqlist) 5107 { 5108 kmem_free(sqlist, sqlist->sqlist_size); 5109 } 5110 5111 /* 5112 * Prevent any new entries into any syncq in this stream. 5113 * Used by freezestr. 5114 */ 5115 void 5116 strblock(queue_t *q) 5117 { 5118 struct stdata *stp; 5119 syncql_t *sql; 5120 sqlist_t *sqlist; 5121 5122 q = _RD(q); 5123 5124 stp = STREAM(q); 5125 ASSERT(stp != NULL); 5126 5127 /* 5128 * Get a sorted list with all the duplicates removed containing 5129 * all the syncqs referenced by this stream. 5130 */ 5131 sqlist = sqlist_build(q, stp, B_FALSE); 5132 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5133 blocksq(sql->sql_sq, SQ_FROZEN, -1); 5134 sqlist_free(sqlist); 5135 } 5136 5137 /* 5138 * Release the block on new entries into this stream 5139 */ 5140 void 5141 strunblock(queue_t *q) 5142 { 5143 struct stdata *stp; 5144 syncql_t *sql; 5145 sqlist_t *sqlist; 5146 int drain_needed; 5147 5148 q = _RD(q); 5149 5150 /* 5151 * Get a sorted list with all the duplicates removed containing 5152 * all the syncqs referenced by this stream. 5153 * Have to drop the SQ_FROZEN flag on all the syncqs before 5154 * starting to drain them; otherwise the draining might 5155 * cause a freezestr in some module on the stream (which 5156 * would deadlock.) 5157 */ 5158 stp = STREAM(q); 5159 ASSERT(stp != NULL); 5160 sqlist = sqlist_build(q, stp, B_FALSE); 5161 drain_needed = 0; 5162 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5163 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN); 5164 if (drain_needed) { 5165 for (sql = sqlist->sqlist_head; sql != NULL; 5166 sql = sql->sql_next) 5167 emptysq(sql->sql_sq); 5168 } 5169 sqlist_free(sqlist); 5170 } 5171 5172 #ifdef DEBUG 5173 static int 5174 qprocsareon(queue_t *rq) 5175 { 5176 if (rq->q_next == NULL) 5177 return (0); 5178 return (_WR(rq->q_next)->q_next == _WR(rq)); 5179 } 5180 5181 int 5182 qclaimed(queue_t *q) 5183 { 5184 uint_t count; 5185 5186 count = q->q_syncq->sq_count; 5187 SUM_SQ_PUTCOUNTS(q->q_syncq, count); 5188 return (count != 0); 5189 } 5190 5191 /* 5192 * Check if anyone has frozen this stream with freezestr 5193 */ 5194 int 5195 frozenstr(queue_t *q) 5196 { 5197 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0); 5198 } 5199 #endif /* DEBUG */ 5200 5201 /* 5202 * Enter a queue. 5203 * Obsoleted interface. Should not be used. 5204 */ 5205 void 5206 enterq(queue_t *q) 5207 { 5208 entersq(q->q_syncq, SQ_CALLBACK); 5209 } 5210 5211 void 5212 leaveq(queue_t *q) 5213 { 5214 leavesq(q->q_syncq, SQ_CALLBACK); 5215 } 5216 5217 /* 5218 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits 5219 * to check. 5220 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter 5221 * calls and the running of open, close and service procedures. 5222 * 5223 * if c_inner bit is set no need to grab sq_putlocks since we don't care 5224 * if other threads have entered or are entering put entry point. 5225 * 5226 * if c_inner bit is set it might have been posible to use 5227 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize 5228 * open/close path for IP) but since the count may need to be decremented in 5229 * qwait() we wouldn't know which counter to decrement. Currently counter is 5230 * selected by current cpu_seqid and current CPU can change at any moment. XXX 5231 * in the future we might use curthread id bits to select the counter and this 5232 * would stay constant across routine calls. 5233 */ 5234 void 5235 entersq(syncq_t *sq, int entrypoint) 5236 { 5237 uint16_t count = 0; 5238 uint16_t flags; 5239 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 5240 uint16_t type; 5241 uint_t c_inner = entrypoint & SQ_CI; 5242 uint_t c_outer = entrypoint & SQ_CO; 5243 5244 /* 5245 * Increment ref count to keep closes out of this queue. 5246 */ 5247 ASSERT(sq); 5248 ASSERT(c_inner && c_outer); 5249 mutex_enter(SQLOCK(sq)); 5250 flags = sq->sq_flags; 5251 type = sq->sq_type; 5252 if (!(type & c_inner)) { 5253 /* Make sure all putcounts now use slowlock. */ 5254 count = sq->sq_count; 5255 SQ_PUTLOCKS_ENTER(sq); 5256 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5257 SUM_SQ_PUTCOUNTS(sq, count); 5258 sq->sq_needexcl++; 5259 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5260 waitflags |= SQ_MESSAGES; 5261 } 5262 /* 5263 * Wait until we can enter the inner perimeter. 5264 * If we want exclusive access we wait until sq_count is 0. 5265 * We have to do this before entering the outer perimeter in order 5266 * to preserve put/close message ordering. 5267 */ 5268 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) { 5269 sq->sq_flags = flags | SQ_WANTWAKEUP; 5270 if (!(type & c_inner)) { 5271 SQ_PUTLOCKS_EXIT(sq); 5272 } 5273 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5274 if (!(type & c_inner)) { 5275 count = sq->sq_count; 5276 SQ_PUTLOCKS_ENTER(sq); 5277 SUM_SQ_PUTCOUNTS(sq, count); 5278 } 5279 flags = sq->sq_flags; 5280 } 5281 5282 if (!(type & c_inner)) { 5283 ASSERT(sq->sq_needexcl > 0); 5284 sq->sq_needexcl--; 5285 if (sq->sq_needexcl == 0) { 5286 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5287 } 5288 } 5289 5290 /* Check if we need to enter the outer perimeter */ 5291 if (!(type & c_outer)) { 5292 /* 5293 * We have to enter the outer perimeter exclusively before 5294 * we can increment sq_count to avoid deadlock. This implies 5295 * that we have to re-check sq_flags and sq_count. 5296 * 5297 * is it possible to have c_inner set when c_outer is not set? 5298 */ 5299 if (!(type & c_inner)) { 5300 SQ_PUTLOCKS_EXIT(sq); 5301 } 5302 mutex_exit(SQLOCK(sq)); 5303 outer_enter(sq->sq_outer, SQ_GOAWAY); 5304 mutex_enter(SQLOCK(sq)); 5305 flags = sq->sq_flags; 5306 /* 5307 * there should be no need to recheck sq_putcounts 5308 * because outer_enter() has already waited for them to clear 5309 * after setting SQ_WRITER. 5310 */ 5311 count = sq->sq_count; 5312 #ifdef DEBUG 5313 /* 5314 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead 5315 * of doing an ASSERT internally. Others should do 5316 * something like 5317 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0); 5318 * without the need to #ifdef DEBUG it. 5319 */ 5320 SUMCHECK_SQ_PUTCOUNTS(sq, 0); 5321 #endif 5322 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) || 5323 (!(type & c_inner) && count != 0)) { 5324 sq->sq_flags = flags | SQ_WANTWAKEUP; 5325 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5326 count = sq->sq_count; 5327 flags = sq->sq_flags; 5328 } 5329 } 5330 5331 sq->sq_count++; 5332 ASSERT(sq->sq_count != 0); /* Wraparound */ 5333 if (!(type & c_inner)) { 5334 /* Exclusive entry */ 5335 ASSERT(sq->sq_count == 1); 5336 sq->sq_flags |= SQ_EXCL; 5337 if (type & c_outer) { 5338 SQ_PUTLOCKS_EXIT(sq); 5339 } 5340 } 5341 mutex_exit(SQLOCK(sq)); 5342 } 5343 5344 /* 5345 * leave a syncq. announce to framework that closes may proceed. 5346 * c_inner and c_outer specifies which concurrency bits 5347 * to check. 5348 * 5349 * must never be called from driver or module put entry point. 5350 * 5351 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5352 * sq_putlocks are used. 5353 */ 5354 void 5355 leavesq(syncq_t *sq, int entrypoint) 5356 { 5357 uint16_t flags; 5358 uint16_t type; 5359 uint_t c_outer = entrypoint & SQ_CO; 5360 #ifdef DEBUG 5361 uint_t c_inner = entrypoint & SQ_CI; 5362 #endif 5363 5364 /* 5365 * decrement ref count, drain the syncq if possible, and wake up 5366 * any waiting close. 5367 */ 5368 ASSERT(sq); 5369 ASSERT(c_inner && c_outer); 5370 mutex_enter(SQLOCK(sq)); 5371 flags = sq->sq_flags; 5372 type = sq->sq_type; 5373 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) { 5374 5375 if (flags & SQ_WANTWAKEUP) { 5376 flags &= ~SQ_WANTWAKEUP; 5377 cv_broadcast(&sq->sq_wait); 5378 } 5379 if (flags & SQ_WANTEXWAKEUP) { 5380 flags &= ~SQ_WANTEXWAKEUP; 5381 cv_broadcast(&sq->sq_exitwait); 5382 } 5383 5384 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) { 5385 /* 5386 * The syncq needs to be drained. "Exit" the syncq 5387 * before calling drain_syncq. 5388 */ 5389 ASSERT(sq->sq_count != 0); 5390 sq->sq_count--; 5391 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5392 sq->sq_flags = flags & ~SQ_EXCL; 5393 drain_syncq(sq); 5394 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 5395 /* Check if we need to exit the outer perimeter */ 5396 /* XXX will this ever be true? */ 5397 if (!(type & c_outer)) 5398 outer_exit(sq->sq_outer); 5399 return; 5400 } 5401 } 5402 ASSERT(sq->sq_count != 0); 5403 sq->sq_count--; 5404 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5405 sq->sq_flags = flags & ~SQ_EXCL; 5406 mutex_exit(SQLOCK(sq)); 5407 5408 /* Check if we need to exit the outer perimeter */ 5409 if (!(sq->sq_type & c_outer)) 5410 outer_exit(sq->sq_outer); 5411 } 5412 5413 /* 5414 * Prevent q_next from changing in this stream by incrementing sq_count. 5415 * 5416 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5417 * sq_putlocks are used. 5418 */ 5419 void 5420 claimq(queue_t *qp) 5421 { 5422 syncq_t *sq = qp->q_syncq; 5423 5424 mutex_enter(SQLOCK(sq)); 5425 sq->sq_count++; 5426 ASSERT(sq->sq_count != 0); /* Wraparound */ 5427 mutex_exit(SQLOCK(sq)); 5428 } 5429 5430 /* 5431 * Undo claimq. 5432 * 5433 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5434 * sq_putlocks are used. 5435 */ 5436 void 5437 releaseq(queue_t *qp) 5438 { 5439 syncq_t *sq = qp->q_syncq; 5440 uint16_t flags; 5441 5442 mutex_enter(SQLOCK(sq)); 5443 ASSERT(sq->sq_count > 0); 5444 sq->sq_count--; 5445 5446 flags = sq->sq_flags; 5447 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) { 5448 if (flags & SQ_WANTWAKEUP) { 5449 flags &= ~SQ_WANTWAKEUP; 5450 cv_broadcast(&sq->sq_wait); 5451 } 5452 sq->sq_flags = flags; 5453 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5454 /* 5455 * To prevent potential recursive invocation of 5456 * drain_syncq we do not call drain_syncq if count is 5457 * non-zero. 5458 */ 5459 if (sq->sq_count == 0) { 5460 drain_syncq(sq); 5461 return; 5462 } else 5463 sqenable(sq); 5464 } 5465 } 5466 mutex_exit(SQLOCK(sq)); 5467 } 5468 5469 /* 5470 * Prevent q_next from changing in this stream by incrementing sd_refcnt. 5471 */ 5472 void 5473 claimstr(queue_t *qp) 5474 { 5475 struct stdata *stp = STREAM(qp); 5476 5477 mutex_enter(&stp->sd_reflock); 5478 stp->sd_refcnt++; 5479 ASSERT(stp->sd_refcnt != 0); /* Wraparound */ 5480 mutex_exit(&stp->sd_reflock); 5481 } 5482 5483 /* 5484 * Undo claimstr. 5485 */ 5486 void 5487 releasestr(queue_t *qp) 5488 { 5489 struct stdata *stp = STREAM(qp); 5490 5491 mutex_enter(&stp->sd_reflock); 5492 ASSERT(stp->sd_refcnt != 0); 5493 if (--stp->sd_refcnt == 0) 5494 cv_broadcast(&stp->sd_refmonitor); 5495 mutex_exit(&stp->sd_reflock); 5496 } 5497 5498 static syncq_t * 5499 new_syncq(void) 5500 { 5501 return (kmem_cache_alloc(syncq_cache, KM_SLEEP)); 5502 } 5503 5504 static void 5505 free_syncq(syncq_t *sq) 5506 { 5507 ASSERT(sq->sq_head == NULL); 5508 ASSERT(sq->sq_outer == NULL); 5509 ASSERT(sq->sq_callbpend == NULL); 5510 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || 5511 (sq->sq_onext == sq && sq->sq_oprev == sq)); 5512 5513 if (sq->sq_ciputctrl != NULL) { 5514 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 5515 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 5516 sq->sq_nciputctrl, 0); 5517 ASSERT(ciputctrl_cache != NULL); 5518 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 5519 } 5520 5521 sq->sq_tail = NULL; 5522 sq->sq_evhead = NULL; 5523 sq->sq_evtail = NULL; 5524 sq->sq_ciputctrl = NULL; 5525 sq->sq_nciputctrl = 0; 5526 sq->sq_count = 0; 5527 sq->sq_rmqcount = 0; 5528 sq->sq_callbflags = 0; 5529 sq->sq_cancelid = 0; 5530 sq->sq_next = NULL; 5531 sq->sq_needexcl = 0; 5532 sq->sq_svcflags = 0; 5533 sq->sq_nqueues = 0; 5534 sq->sq_pri = 0; 5535 sq->sq_onext = NULL; 5536 sq->sq_oprev = NULL; 5537 sq->sq_flags = 0; 5538 sq->sq_type = 0; 5539 sq->sq_servcount = 0; 5540 5541 kmem_cache_free(syncq_cache, sq); 5542 } 5543 5544 /* Outer perimeter code */ 5545 5546 /* 5547 * The outer syncq uses the fields and flags in the syncq slightly 5548 * differently from the inner syncqs. 5549 * sq_count Incremented when there are pending or running 5550 * writers at the outer perimeter to prevent the set of 5551 * inner syncqs that belong to the outer perimeter from 5552 * changing. 5553 * sq_head/tail List of deferred qwriter(OUTER) operations. 5554 * 5555 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while 5556 * inner syncqs are added to or removed from the 5557 * outer perimeter. 5558 * SQ_QUEUED sq_head/tail has messages or eventsqueued. 5559 * 5560 * SQ_WRITER A thread is currently traversing all the inner syncqs 5561 * setting the SQ_WRITER flag. 5562 */ 5563 5564 /* 5565 * Get write access at the outer perimeter. 5566 * Note that read access is done by entersq, putnext, and put by simply 5567 * incrementing sq_count in the inner syncq. 5568 * 5569 * Waits until "flags" is no longer set in the outer to prevent multiple 5570 * threads from having write access at the same time. SQ_WRITER has to be part 5571 * of "flags". 5572 * 5573 * Increases sq_count on the outer syncq to keep away outer_insert/remove 5574 * until the outer_exit is finished. 5575 * 5576 * outer_enter is vulnerable to starvation since it does not prevent new 5577 * threads from entering the inner syncqs while it is waiting for sq_count to 5578 * go to zero. 5579 */ 5580 void 5581 outer_enter(syncq_t *outer, uint16_t flags) 5582 { 5583 syncq_t *sq; 5584 int wait_needed; 5585 uint16_t count; 5586 5587 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5588 outer->sq_oprev != NULL); 5589 ASSERT(flags & SQ_WRITER); 5590 5591 retry: 5592 mutex_enter(SQLOCK(outer)); 5593 while (outer->sq_flags & flags) { 5594 outer->sq_flags |= SQ_WANTWAKEUP; 5595 cv_wait(&outer->sq_wait, SQLOCK(outer)); 5596 } 5597 5598 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5599 outer->sq_flags |= SQ_WRITER; 5600 outer->sq_count++; 5601 ASSERT(outer->sq_count != 0); /* wraparound */ 5602 wait_needed = 0; 5603 /* 5604 * Set SQ_WRITER on all the inner syncqs while holding 5605 * the SQLOCK on the outer syncq. This ensures that the changing 5606 * of SQ_WRITER is atomic under the outer SQLOCK. 5607 */ 5608 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5609 mutex_enter(SQLOCK(sq)); 5610 count = sq->sq_count; 5611 SQ_PUTLOCKS_ENTER(sq); 5612 sq->sq_flags |= SQ_WRITER; 5613 SUM_SQ_PUTCOUNTS(sq, count); 5614 if (count != 0) 5615 wait_needed = 1; 5616 SQ_PUTLOCKS_EXIT(sq); 5617 mutex_exit(SQLOCK(sq)); 5618 } 5619 mutex_exit(SQLOCK(outer)); 5620 5621 /* 5622 * Get everybody out of the syncqs sequentially. 5623 * Note that we don't actually need to aqiure the PUTLOCKS, since 5624 * we have already cleared the fastbit, and set QWRITER. By 5625 * definition, the count can not increase since putnext will 5626 * take the slowlock path (and the purpose of aquiring the 5627 * putlocks was to make sure it didn't increase while we were 5628 * waiting). 5629 * 5630 * Note that we still aquire the PUTLOCKS to be safe. 5631 */ 5632 if (wait_needed) { 5633 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5634 mutex_enter(SQLOCK(sq)); 5635 count = sq->sq_count; 5636 SQ_PUTLOCKS_ENTER(sq); 5637 SUM_SQ_PUTCOUNTS(sq, count); 5638 while (count != 0) { 5639 sq->sq_flags |= SQ_WANTWAKEUP; 5640 SQ_PUTLOCKS_EXIT(sq); 5641 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5642 count = sq->sq_count; 5643 SQ_PUTLOCKS_ENTER(sq); 5644 SUM_SQ_PUTCOUNTS(sq, count); 5645 } 5646 SQ_PUTLOCKS_EXIT(sq); 5647 mutex_exit(SQLOCK(sq)); 5648 } 5649 /* 5650 * Verify that none of the flags got set while we 5651 * were waiting for the sq_counts to drop. 5652 * If this happens we exit and retry entering the 5653 * outer perimeter. 5654 */ 5655 mutex_enter(SQLOCK(outer)); 5656 if (outer->sq_flags & (flags & ~SQ_WRITER)) { 5657 mutex_exit(SQLOCK(outer)); 5658 outer_exit(outer); 5659 goto retry; 5660 } 5661 mutex_exit(SQLOCK(outer)); 5662 } 5663 } 5664 5665 /* 5666 * Drop the write access at the outer perimeter. 5667 * Read access is dropped implicitly (by putnext, put, and leavesq) by 5668 * decrementing sq_count. 5669 */ 5670 void 5671 outer_exit(syncq_t *outer) 5672 { 5673 syncq_t *sq; 5674 int drain_needed; 5675 uint16_t flags; 5676 5677 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5678 outer->sq_oprev != NULL); 5679 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer))); 5680 5681 /* 5682 * Atomically (from the perspective of threads calling become_writer) 5683 * drop the write access at the outer perimeter by holding 5684 * SQLOCK(outer) across all the dropsq calls and the resetting of 5685 * SQ_WRITER. 5686 * This defines a locking order between the outer perimeter 5687 * SQLOCK and the inner perimeter SQLOCKs. 5688 */ 5689 mutex_enter(SQLOCK(outer)); 5690 flags = outer->sq_flags; 5691 ASSERT(outer->sq_flags & SQ_WRITER); 5692 if (flags & SQ_QUEUED) { 5693 write_now(outer); 5694 flags = outer->sq_flags; 5695 } 5696 5697 /* 5698 * sq_onext is stable since sq_count has not yet been decreased. 5699 * Reset the SQ_WRITER flags in all syncqs. 5700 * After dropping SQ_WRITER on the outer syncq we empty all the 5701 * inner syncqs. 5702 */ 5703 drain_needed = 0; 5704 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5705 drain_needed += dropsq(sq, SQ_WRITER); 5706 ASSERT(!(outer->sq_flags & SQ_QUEUED)); 5707 flags &= ~SQ_WRITER; 5708 if (drain_needed) { 5709 outer->sq_flags = flags; 5710 mutex_exit(SQLOCK(outer)); 5711 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5712 emptysq(sq); 5713 mutex_enter(SQLOCK(outer)); 5714 flags = outer->sq_flags; 5715 } 5716 if (flags & SQ_WANTWAKEUP) { 5717 flags &= ~SQ_WANTWAKEUP; 5718 cv_broadcast(&outer->sq_wait); 5719 } 5720 outer->sq_flags = flags; 5721 ASSERT(outer->sq_count > 0); 5722 outer->sq_count--; 5723 mutex_exit(SQLOCK(outer)); 5724 } 5725 5726 /* 5727 * Add another syncq to an outer perimeter. 5728 * Block out all other access to the outer perimeter while it is being 5729 * changed using blocksq. 5730 * Assumes that the caller has *not* done an outer_enter. 5731 * 5732 * Vulnerable to starvation in blocksq. 5733 */ 5734 static void 5735 outer_insert(syncq_t *outer, syncq_t *sq) 5736 { 5737 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5738 outer->sq_oprev != NULL); 5739 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 5740 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ 5741 5742 /* Get exclusive access to the outer perimeter list */ 5743 blocksq(outer, SQ_BLOCKED, 0); 5744 ASSERT(outer->sq_flags & SQ_BLOCKED); 5745 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5746 5747 mutex_enter(SQLOCK(sq)); 5748 sq->sq_outer = outer; 5749 outer->sq_onext->sq_oprev = sq; 5750 sq->sq_onext = outer->sq_onext; 5751 outer->sq_onext = sq; 5752 sq->sq_oprev = outer; 5753 mutex_exit(SQLOCK(sq)); 5754 unblocksq(outer, SQ_BLOCKED, 1); 5755 } 5756 5757 /* 5758 * Remove a syncq from an outer perimeter. 5759 * Block out all other access to the outer perimeter while it is being 5760 * changed using blocksq. 5761 * Assumes that the caller has *not* done an outer_enter. 5762 * 5763 * Vulnerable to starvation in blocksq. 5764 */ 5765 static void 5766 outer_remove(syncq_t *outer, syncq_t *sq) 5767 { 5768 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5769 outer->sq_oprev != NULL); 5770 ASSERT(sq->sq_outer == outer); 5771 5772 /* Get exclusive access to the outer perimeter list */ 5773 blocksq(outer, SQ_BLOCKED, 0); 5774 ASSERT(outer->sq_flags & SQ_BLOCKED); 5775 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5776 5777 mutex_enter(SQLOCK(sq)); 5778 sq->sq_outer = NULL; 5779 sq->sq_onext->sq_oprev = sq->sq_oprev; 5780 sq->sq_oprev->sq_onext = sq->sq_onext; 5781 sq->sq_oprev = sq->sq_onext = NULL; 5782 mutex_exit(SQLOCK(sq)); 5783 unblocksq(outer, SQ_BLOCKED, 1); 5784 } 5785 5786 /* 5787 * Queue a deferred qwriter(OUTER) callback for this outer perimeter. 5788 * If this is the first callback for this outer perimeter then add 5789 * this outer perimeter to the list of outer perimeters that 5790 * the qwriter_outer_thread will process. 5791 * 5792 * Increments sq_count in the outer syncq to prevent the membership 5793 * of the outer perimeter (in terms of inner syncqs) to change while 5794 * the callback is pending. 5795 */ 5796 static void 5797 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp) 5798 { 5799 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5800 5801 mp->b_prev = (mblk_t *)func; 5802 mp->b_queue = q; 5803 mp->b_next = NULL; 5804 outer->sq_count++; /* Decremented when dequeued */ 5805 ASSERT(outer->sq_count != 0); /* Wraparound */ 5806 if (outer->sq_evhead == NULL) { 5807 /* First message. */ 5808 outer->sq_evhead = outer->sq_evtail = mp; 5809 outer->sq_flags |= SQ_EVENTS; 5810 mutex_exit(SQLOCK(outer)); 5811 STRSTAT(qwr_outer); 5812 (void) taskq_dispatch(streams_taskq, 5813 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP); 5814 } else { 5815 ASSERT(outer->sq_flags & SQ_EVENTS); 5816 outer->sq_evtail->b_next = mp; 5817 outer->sq_evtail = mp; 5818 mutex_exit(SQLOCK(outer)); 5819 } 5820 } 5821 5822 /* 5823 * Try and upgrade to write access at the outer perimeter. If this can 5824 * not be done without blocking then queue the callback to be done 5825 * by the qwriter_outer_thread. 5826 * 5827 * This routine can only be called from put or service procedures plus 5828 * asynchronous callback routines that have properly entered to 5829 * queue (with entersq.) Thus qwriter(OUTER) assumes the caller has one claim 5830 * on the syncq associated with q. 5831 */ 5832 void 5833 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)()) 5834 { 5835 syncq_t *osq, *sq, *outer; 5836 int failed; 5837 uint16_t flags; 5838 5839 osq = q->q_syncq; 5840 outer = osq->sq_outer; 5841 if (outer == NULL) 5842 panic("qwriter(PERIM_OUTER): no outer perimeter"); 5843 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5844 outer->sq_oprev != NULL); 5845 5846 mutex_enter(SQLOCK(outer)); 5847 flags = outer->sq_flags; 5848 /* 5849 * If some thread is traversing sq_next, or if we are blocked by 5850 * outer_insert or outer_remove, or if the we already have queued 5851 * callbacks, then queue this callback for later processing. 5852 * 5853 * Also queue the qwriter for an interrupt thread in order 5854 * to reduce the time spent running at high IPL. 5855 * to identify there are events. 5856 */ 5857 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) { 5858 /* 5859 * Queue the become_writer request. 5860 * The queueing is atomic under SQLOCK(outer) in order 5861 * to synchronize with outer_exit. 5862 * queue_writer will drop the outer SQLOCK 5863 */ 5864 if (flags & SQ_BLOCKED) { 5865 /* Must set SQ_WRITER on inner perimeter */ 5866 mutex_enter(SQLOCK(osq)); 5867 osq->sq_flags |= SQ_WRITER; 5868 mutex_exit(SQLOCK(osq)); 5869 } else { 5870 if (!(flags & SQ_WRITER)) { 5871 /* 5872 * The outer could have been SQ_BLOCKED thus 5873 * SQ_WRITER might not be set on the inner. 5874 */ 5875 mutex_enter(SQLOCK(osq)); 5876 osq->sq_flags |= SQ_WRITER; 5877 mutex_exit(SQLOCK(osq)); 5878 } 5879 ASSERT(osq->sq_flags & SQ_WRITER); 5880 } 5881 queue_writer(outer, func, q, mp); 5882 return; 5883 } 5884 /* 5885 * We are half-way to exclusive access to the outer perimeter. 5886 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove 5887 * while the inner syncqs are traversed. 5888 */ 5889 outer->sq_count++; 5890 ASSERT(outer->sq_count != 0); /* wraparound */ 5891 flags |= SQ_WRITER; 5892 /* 5893 * Check if we can run the function immediately. Mark all 5894 * syncqs with the writer flag to prevent new entries into 5895 * put and service procedures. 5896 * 5897 * Set SQ_WRITER on all the inner syncqs while holding 5898 * the SQLOCK on the outer syncq. This ensures that the changing 5899 * of SQ_WRITER is atomic under the outer SQLOCK. 5900 */ 5901 failed = 0; 5902 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5903 uint16_t count; 5904 uint_t maxcnt = (sq == osq) ? 1 : 0; 5905 5906 mutex_enter(SQLOCK(sq)); 5907 count = sq->sq_count; 5908 SQ_PUTLOCKS_ENTER(sq); 5909 SUM_SQ_PUTCOUNTS(sq, count); 5910 if (sq->sq_count > maxcnt) 5911 failed = 1; 5912 sq->sq_flags |= SQ_WRITER; 5913 SQ_PUTLOCKS_EXIT(sq); 5914 mutex_exit(SQLOCK(sq)); 5915 } 5916 if (failed) { 5917 /* 5918 * Some other thread has a read claim on the outer perimeter. 5919 * Queue the callback for deferred processing. 5920 * 5921 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER 5922 * so that other qwriter(OUTER) calls will queue their 5923 * callbacks as well. queue_writer increments sq_count so we 5924 * decrement to compensate for the our increment. 5925 * 5926 * Dropping SQ_WRITER enables the writer thread to work 5927 * on this outer perimeter. 5928 */ 5929 outer->sq_flags = flags; 5930 queue_writer(outer, func, q, mp); 5931 /* queue_writer dropper the lock */ 5932 mutex_enter(SQLOCK(outer)); 5933 ASSERT(outer->sq_count > 0); 5934 outer->sq_count--; 5935 ASSERT(outer->sq_flags & SQ_WRITER); 5936 flags = outer->sq_flags; 5937 flags &= ~SQ_WRITER; 5938 if (flags & SQ_WANTWAKEUP) { 5939 flags &= ~SQ_WANTWAKEUP; 5940 cv_broadcast(&outer->sq_wait); 5941 } 5942 outer->sq_flags = flags; 5943 mutex_exit(SQLOCK(outer)); 5944 return; 5945 } else { 5946 outer->sq_flags = flags; 5947 mutex_exit(SQLOCK(outer)); 5948 } 5949 5950 /* Can run it immediately */ 5951 (*func)(q, mp); 5952 5953 outer_exit(outer); 5954 } 5955 5956 /* 5957 * Dequeue all writer callbacks from the outer perimeter and run them. 5958 */ 5959 static void 5960 write_now(syncq_t *outer) 5961 { 5962 mblk_t *mp; 5963 queue_t *q; 5964 void (*func)(); 5965 5966 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5967 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5968 outer->sq_oprev != NULL); 5969 while ((mp = outer->sq_evhead) != NULL) { 5970 /* 5971 * queues cannot be placed on the queuelist on the outer 5972 * perimiter. 5973 */ 5974 ASSERT(!(outer->sq_flags & SQ_MESSAGES)); 5975 ASSERT((outer->sq_flags & SQ_EVENTS)); 5976 5977 outer->sq_evhead = mp->b_next; 5978 if (outer->sq_evhead == NULL) { 5979 outer->sq_evtail = NULL; 5980 outer->sq_flags &= ~SQ_EVENTS; 5981 } 5982 ASSERT(outer->sq_count != 0); 5983 outer->sq_count--; /* Incremented when enqueued. */ 5984 mutex_exit(SQLOCK(outer)); 5985 /* 5986 * Drop the message if the queue is closing. 5987 * Make sure that the queue is "claimed" when the callback 5988 * is run in order to satisfy various ASSERTs. 5989 */ 5990 q = mp->b_queue; 5991 func = (void (*)())mp->b_prev; 5992 ASSERT(func != NULL); 5993 mp->b_next = mp->b_prev = NULL; 5994 if (q->q_flag & QWCLOSE) { 5995 freemsg(mp); 5996 } else { 5997 claimq(q); 5998 (*func)(q, mp); 5999 releaseq(q); 6000 } 6001 mutex_enter(SQLOCK(outer)); 6002 } 6003 ASSERT(MUTEX_HELD(SQLOCK(outer))); 6004 } 6005 6006 /* 6007 * The list of messages on the inner syncq is effectively hashed 6008 * by destination queue. These destination queues are doubly 6009 * linked lists (hopefully) in priority order. Messages are then 6010 * put on the queue referenced by the q_sqhead/q_sqtail elements. 6011 * Additional messages are linked together by the b_next/b_prev 6012 * elements in the mblk, with (similar to putq()) the first message 6013 * having a NULL b_prev and the last message having a NULL b_next. 6014 * 6015 * Events, such as qwriter callbacks, are put onto a list in FIFO 6016 * order referenced by sq_evhead, and sq_evtail. This is a singly 6017 * linked list, and messages here MUST be processed in the order queued. 6018 */ 6019 6020 /* 6021 * Run the events on the syncq event list (sq_evhead). 6022 * Assumes there is only one claim on the syncq, it is 6023 * already exclusive (SQ_EXCL set), and the SQLOCK held. 6024 * Messages here are processed in order, with the SQ_EXCL bit 6025 * held all the way through till the last message is processed. 6026 */ 6027 void 6028 sq_run_events(syncq_t *sq) 6029 { 6030 mblk_t *bp; 6031 queue_t *qp; 6032 uint16_t flags = sq->sq_flags; 6033 void (*func)(); 6034 6035 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6036 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6037 sq->sq_oprev == NULL) || 6038 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6039 sq->sq_oprev != NULL)); 6040 6041 ASSERT(flags & SQ_EXCL); 6042 ASSERT(sq->sq_count == 1); 6043 6044 /* 6045 * We need to process all of the events on this list. It 6046 * is possible that new events will be added while we are 6047 * away processing a callback, so on every loop, we start 6048 * back at the beginning of the list. 6049 */ 6050 /* 6051 * We have to reaccess sq_evhead since there is a 6052 * possibility of a new entry while we were running 6053 * the callback. 6054 */ 6055 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { 6056 ASSERT(bp->b_queue->q_syncq == sq); 6057 ASSERT(sq->sq_flags & SQ_EVENTS); 6058 6059 qp = bp->b_queue; 6060 func = (void (*)())bp->b_prev; 6061 ASSERT(func != NULL); 6062 6063 /* 6064 * Messages from the event queue must be taken off in 6065 * FIFO order. 6066 */ 6067 ASSERT(sq->sq_evhead == bp); 6068 sq->sq_evhead = bp->b_next; 6069 6070 if (bp->b_next == NULL) { 6071 /* Deleting last */ 6072 ASSERT(sq->sq_evtail == bp); 6073 sq->sq_evtail = NULL; 6074 sq->sq_flags &= ~SQ_EVENTS; 6075 } 6076 bp->b_prev = bp->b_next = NULL; 6077 ASSERT(bp->b_datap->db_ref != 0); 6078 6079 mutex_exit(SQLOCK(sq)); 6080 6081 (*func)(qp, bp); 6082 6083 mutex_enter(SQLOCK(sq)); 6084 /* 6085 * re-read the flags, since they could have changed. 6086 */ 6087 flags = sq->sq_flags; 6088 ASSERT(flags & SQ_EXCL); 6089 } 6090 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); 6091 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6092 6093 if (flags & SQ_WANTWAKEUP) { 6094 flags &= ~SQ_WANTWAKEUP; 6095 cv_broadcast(&sq->sq_wait); 6096 } 6097 if (flags & SQ_WANTEXWAKEUP) { 6098 flags &= ~SQ_WANTEXWAKEUP; 6099 cv_broadcast(&sq->sq_exitwait); 6100 } 6101 sq->sq_flags = flags; 6102 } 6103 6104 /* 6105 * Put messages on the event list. 6106 * If we can go exclusive now, do so and process the event list, otherwise 6107 * let the last claim service this list (or wake the sqthread). 6108 * This procedure assumes SQLOCK is held. To run the event list, it 6109 * must be called with no claims. 6110 */ 6111 static void 6112 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) 6113 { 6114 uint16_t count; 6115 6116 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6117 ASSERT(func != NULL); 6118 6119 /* 6120 * This is a callback. Add it to the list of callbacks 6121 * and see about upgrading. 6122 */ 6123 mp->b_prev = (mblk_t *)func; 6124 mp->b_queue = q; 6125 mp->b_next = NULL; 6126 if (sq->sq_evhead == NULL) { 6127 sq->sq_evhead = sq->sq_evtail = mp; 6128 sq->sq_flags |= SQ_EVENTS; 6129 } else { 6130 ASSERT(sq->sq_evtail != NULL); 6131 ASSERT(sq->sq_evtail->b_next == NULL); 6132 ASSERT(sq->sq_flags & SQ_EVENTS); 6133 sq->sq_evtail->b_next = mp; 6134 sq->sq_evtail = mp; 6135 } 6136 /* 6137 * We have set SQ_EVENTS, so threads will have to 6138 * unwind out of the perimiter, and new entries will 6139 * not grab a putlock. But we still need to know 6140 * how many threads have already made a claim to the 6141 * syncq, so grab the putlocks, and sum the counts. 6142 * If there are no claims on the syncq, we can upgrade 6143 * to exclusive, and run the event list. 6144 * NOTE: We hold the SQLOCK, so we can just grab the 6145 * putlocks. 6146 */ 6147 count = sq->sq_count; 6148 SQ_PUTLOCKS_ENTER(sq); 6149 SUM_SQ_PUTCOUNTS(sq, count); 6150 /* 6151 * We have no claim, so we need to check if there 6152 * are no others, then we can upgrade. 6153 */ 6154 /* 6155 * There are currently no claims on 6156 * the syncq by this thread (at least on this entry). The thread who has 6157 * the claim should drain syncq. 6158 */ 6159 if (count > 0) { 6160 /* 6161 * Can't upgrade - other threads inside. 6162 */ 6163 SQ_PUTLOCKS_EXIT(sq); 6164 mutex_exit(SQLOCK(sq)); 6165 return; 6166 } 6167 /* 6168 * Need to set SQ_EXCL and make a claim on the syncq. 6169 */ 6170 ASSERT((sq->sq_flags & SQ_EXCL) == 0); 6171 sq->sq_flags |= SQ_EXCL; 6172 ASSERT(sq->sq_count == 0); 6173 sq->sq_count++; 6174 SQ_PUTLOCKS_EXIT(sq); 6175 6176 /* Process the events list */ 6177 sq_run_events(sq); 6178 6179 /* 6180 * Release our claim... 6181 */ 6182 sq->sq_count--; 6183 6184 /* 6185 * And release SQ_EXCL. 6186 * We don't need to acquire the putlocks to release 6187 * SQ_EXCL, since we are exclusive, and hold the SQLOCK. 6188 */ 6189 sq->sq_flags &= ~SQ_EXCL; 6190 6191 /* 6192 * sq_run_events should have released SQ_EXCL 6193 */ 6194 ASSERT(!(sq->sq_flags & SQ_EXCL)); 6195 6196 /* 6197 * If anything happened while we were running the 6198 * events (or was there before), we need to process 6199 * them now. We shouldn't be exclusive sine we 6200 * released the perimiter above (plus, we asserted 6201 * for it). 6202 */ 6203 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) 6204 drain_syncq(sq); 6205 else 6206 mutex_exit(SQLOCK(sq)); 6207 } 6208 6209 /* 6210 * Perform delayed processing. The caller has to make sure that it is safe 6211 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are 6212 * set.) 6213 * 6214 * Assume that the caller has NO claims on the syncq. However, a claim 6215 * on the syncq does not indicate that a thread is draining the syncq. 6216 * There may be more claims on the syncq than there are threads draining 6217 * (i.e. #_threads_draining <= sq_count) 6218 * 6219 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set 6220 * in order to preserve qwriter(OUTER) ordering constraints. 6221 * 6222 * sq_putcount only needs to be checked when dispatching the queued 6223 * writer call for CIPUT sync queue, but this is handled in sq_run_events. 6224 */ 6225 void 6226 drain_syncq(syncq_t *sq) 6227 { 6228 queue_t *qp; 6229 uint16_t count; 6230 uint16_t type = sq->sq_type; 6231 uint16_t flags = sq->sq_flags; 6232 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; 6233 6234 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6235 "drain_syncq start:%p", sq); 6236 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6237 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6238 sq->sq_oprev == NULL) || 6239 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6240 sq->sq_oprev != NULL)); 6241 6242 /* 6243 * Drop SQ_SERVICE flag. 6244 */ 6245 if (bg_service) 6246 sq->sq_svcflags &= ~SQ_SERVICE; 6247 6248 /* 6249 * If SQ_EXCL is set, someone else is processing this syncq - let him 6250 * finish the job. 6251 */ 6252 if (flags & SQ_EXCL) { 6253 if (bg_service) { 6254 ASSERT(sq->sq_servcount != 0); 6255 sq->sq_servcount--; 6256 } 6257 mutex_exit(SQLOCK(sq)); 6258 return; 6259 } 6260 6261 /* 6262 * This routine can be called by a background thread if 6263 * it was scheduled by a hi-priority thread. SO, if there are 6264 * NOT messages queued, return (remember, we have the SQLOCK, 6265 * and it cannot change until we release it). Wakeup any waiters also. 6266 */ 6267 if (!(flags & SQ_QUEUED)) { 6268 if (flags & SQ_WANTWAKEUP) { 6269 flags &= ~SQ_WANTWAKEUP; 6270 cv_broadcast(&sq->sq_wait); 6271 } 6272 if (flags & SQ_WANTEXWAKEUP) { 6273 flags &= ~SQ_WANTEXWAKEUP; 6274 cv_broadcast(&sq->sq_exitwait); 6275 } 6276 sq->sq_flags = flags; 6277 if (bg_service) { 6278 ASSERT(sq->sq_servcount != 0); 6279 sq->sq_servcount--; 6280 } 6281 mutex_exit(SQLOCK(sq)); 6282 return; 6283 } 6284 6285 /* 6286 * If this is not a concurrent put perimiter, we need to 6287 * become exclusive to drain. Also, if not CIPUT, we would 6288 * not have acquired a putlock, so we don't need to check 6289 * the putcounts. If not entering with a claim, we test 6290 * for sq_count == 0. 6291 */ 6292 type = sq->sq_type; 6293 if (!(type & SQ_CIPUT)) { 6294 if (sq->sq_count > 1) { 6295 if (bg_service) { 6296 ASSERT(sq->sq_servcount != 0); 6297 sq->sq_servcount--; 6298 } 6299 mutex_exit(SQLOCK(sq)); 6300 return; 6301 } 6302 sq->sq_flags |= SQ_EXCL; 6303 } 6304 6305 /* 6306 * This is where we make a claim to the syncq. 6307 * This can either be done by incrementing a putlock, or 6308 * the sq_count. But since we already have the SQLOCK 6309 * here, we just bump the sq_count. 6310 * 6311 * Note that after we make a claim, we need to let the code 6312 * fall through to the end of this routine to clean itself 6313 * up. A return in the while loop will put the syncq in a 6314 * very bad state. 6315 */ 6316 sq->sq_count++; 6317 ASSERT(sq->sq_count != 0); /* wraparound */ 6318 6319 while ((flags = sq->sq_flags) & SQ_QUEUED) { 6320 /* 6321 * If we are told to stayaway or went exclusive, 6322 * we are done. 6323 */ 6324 if (flags & (SQ_STAYAWAY)) { 6325 break; 6326 } 6327 6328 /* 6329 * If there are events to run, do so. 6330 * We have one claim to the syncq, so if there are 6331 * more than one, other threads are running. 6332 */ 6333 if (sq->sq_evhead != NULL) { 6334 ASSERT(sq->sq_flags & SQ_EVENTS); 6335 6336 count = sq->sq_count; 6337 SQ_PUTLOCKS_ENTER(sq); 6338 SUM_SQ_PUTCOUNTS(sq, count); 6339 if (count > 1) { 6340 SQ_PUTLOCKS_EXIT(sq); 6341 /* Can't upgrade - other threads inside */ 6342 break; 6343 } 6344 ASSERT((flags & SQ_EXCL) == 0); 6345 sq->sq_flags = flags | SQ_EXCL; 6346 SQ_PUTLOCKS_EXIT(sq); 6347 /* 6348 * we have the only claim, run the events, 6349 * sq_run_events will clear the SQ_EXCL flag. 6350 */ 6351 sq_run_events(sq); 6352 6353 /* 6354 * If this is a CIPUT perimiter, we need 6355 * to drop the SQ_EXCL flag so we can properly 6356 * continue draining the syncq. 6357 */ 6358 if (type & SQ_CIPUT) { 6359 ASSERT(sq->sq_flags & SQ_EXCL); 6360 sq->sq_flags &= ~SQ_EXCL; 6361 } 6362 6363 /* 6364 * And go back to the beginning just in case 6365 * anything changed while we were away. 6366 */ 6367 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); 6368 continue; 6369 } 6370 6371 ASSERT(sq->sq_evhead == NULL); 6372 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6373 6374 /* 6375 * Find the queue that is not draining. 6376 * 6377 * q_draining is protected by QLOCK which we do not hold. 6378 * But if it was set, then a thread was draining, and if it gets 6379 * cleared, then it was because the thread has successfully 6380 * drained the syncq, or a GOAWAY state occured. For the GOAWAY 6381 * state to happen, a thread needs the SQLOCK which we hold, and 6382 * if there was such a flag, we whould have already seen it. 6383 */ 6384 6385 for (qp = sq->sq_head; 6386 qp != NULL && (qp->q_draining || 6387 (qp->q_sqflags & Q_SQDRAINING)); 6388 qp = qp->q_sqnext) 6389 ; 6390 6391 if (qp == NULL) 6392 break; 6393 6394 /* 6395 * We have a queue to work on, and we hold the 6396 * SQLOCK and one claim, call qdrain_syncq. 6397 * This means we need to release the SQLOCK and 6398 * aquire the QLOCK (OK since we have a claim). 6399 * Note that qdrain_syncq will actually dequeue 6400 * this queue from the sq_head list when it is 6401 * convinced all the work is done and release 6402 * the QLOCK before returning. 6403 */ 6404 qp->q_sqflags |= Q_SQDRAINING; 6405 mutex_exit(SQLOCK(sq)); 6406 mutex_enter(QLOCK(qp)); 6407 qdrain_syncq(sq, qp); 6408 mutex_enter(SQLOCK(sq)); 6409 6410 /* The queue is drained */ 6411 ASSERT(qp->q_sqflags & Q_SQDRAINING); 6412 qp->q_sqflags &= ~Q_SQDRAINING; 6413 /* 6414 * NOTE: After this point qp should not be used since it may be 6415 * closed. 6416 */ 6417 } 6418 6419 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6420 flags = sq->sq_flags; 6421 6422 /* 6423 * sq->sq_head cannot change because we hold the 6424 * sqlock. However, a thread CAN decide that it is no longer 6425 * going to drain that queue. However, this should be due to 6426 * a GOAWAY state, and we should see that here. 6427 * 6428 * This loop is not very efficient. One solution may be adding a second 6429 * pointer to the "draining" queue, but it is difficult to do when 6430 * queues are inserted in the middle due to priority ordering. Another 6431 * possibility is to yank the queue out of the sq list and put it onto 6432 * the "draining list" and then put it back if it can't be drained. 6433 */ 6434 6435 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || 6436 (type & SQ_CI) || sq->sq_head->q_draining); 6437 6438 /* Drop SQ_EXCL for non-CIPUT perimiters */ 6439 if (!(type & SQ_CIPUT)) 6440 flags &= ~SQ_EXCL; 6441 ASSERT((flags & SQ_EXCL) == 0); 6442 6443 /* Wake up any waiters. */ 6444 if (flags & SQ_WANTWAKEUP) { 6445 flags &= ~SQ_WANTWAKEUP; 6446 cv_broadcast(&sq->sq_wait); 6447 } 6448 if (flags & SQ_WANTEXWAKEUP) { 6449 flags &= ~SQ_WANTEXWAKEUP; 6450 cv_broadcast(&sq->sq_exitwait); 6451 } 6452 sq->sq_flags = flags; 6453 6454 ASSERT(sq->sq_count != 0); 6455 /* Release our claim. */ 6456 sq->sq_count--; 6457 6458 if (bg_service) { 6459 ASSERT(sq->sq_servcount != 0); 6460 sq->sq_servcount--; 6461 } 6462 6463 mutex_exit(SQLOCK(sq)); 6464 6465 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6466 "drain_syncq end:%p", sq); 6467 } 6468 6469 6470 /* 6471 * 6472 * qdrain_syncq can be called (currently) from only one of two places: 6473 * drain_syncq 6474 * putnext (or some variation of it). 6475 * and eventually 6476 * qwait(_sig) 6477 * 6478 * If called from drain_syncq, we found it in the list 6479 * of queue's needing service, so there is work to be done (or it 6480 * wouldn't be on the list). 6481 * 6482 * If called from some putnext variation, it was because the 6483 * perimiter is open, but messages are blocking a putnext and 6484 * there is not a thread working on it. Now a thread could start 6485 * working on it while we are getting ready to do so ourself, but 6486 * the thread would set the q_draining flag, and we can spin out. 6487 * 6488 * As for qwait(_sig), I think I shall let it continue to call 6489 * drain_syncq directly (after all, it will get here eventually). 6490 * 6491 * qdrain_syncq has to terminate when: 6492 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering 6493 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering 6494 * 6495 * ASSUMES: 6496 * One claim 6497 * QLOCK held 6498 * SQLOCK not held 6499 * Will release QLOCK before returning 6500 */ 6501 void 6502 qdrain_syncq(syncq_t *sq, queue_t *q) 6503 { 6504 mblk_t *bp; 6505 boolean_t do_clr; 6506 #ifdef DEBUG 6507 uint16_t count; 6508 #endif 6509 6510 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6511 "drain_syncq start:%p", sq); 6512 ASSERT(q->q_syncq == sq); 6513 ASSERT(MUTEX_HELD(QLOCK(q))); 6514 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6515 /* 6516 * For non-CIPUT perimiters, we should be called with the 6517 * exclusive bit set already. For non-CIPUT perimiters we 6518 * will be doing a concurrent drain, so it better not be set. 6519 */ 6520 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); 6521 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); 6522 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); 6523 /* 6524 * All outer pointers are set, or none of them are 6525 */ 6526 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6527 sq->sq_oprev == NULL) || 6528 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6529 sq->sq_oprev != NULL)); 6530 #ifdef DEBUG 6531 count = sq->sq_count; 6532 /* 6533 * This is OK without the putlocks, because we have one 6534 * claim either from the sq_count, or a putcount. We could 6535 * get an erroneous value from other counts, but ours won't 6536 * change, so one way or another, we will have at least a 6537 * value of one. 6538 */ 6539 SUM_SQ_PUTCOUNTS(sq, count); 6540 ASSERT(count >= 1); 6541 #endif /* DEBUG */ 6542 6543 /* 6544 * The first thing to do here, is find out if a thread is already 6545 * draining this queue or the queue is closing. If so, we are done, 6546 * just return. Also, if there are no messages, we are done as well. 6547 * Note that we check the q_sqhead since there is s window of 6548 * opportunity for us to enter here because Q_SQQUEUED was set, but is 6549 * not anymore. 6550 */ 6551 if (q->q_draining || (q->q_sqhead == NULL)) { 6552 mutex_exit(QLOCK(q)); 6553 return; 6554 } 6555 6556 /* 6557 * If the perimiter is exclusive, there is nothing we can 6558 * do right now, go away. 6559 * Note that there is nothing to prevent this case from changing 6560 * right after this check, but the spin-out will catch it. 6561 */ 6562 6563 /* Tell other threads that we are draining this queue */ 6564 q->q_draining = 1; /* Protected by QLOCK */ 6565 6566 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) { 6567 6568 /* 6569 * Because we can enter this routine just because 6570 * a putnext is blocked, we need to spin out if 6571 * the perimiter wants to go exclusive as well 6572 * as just blocked. We need to spin out also if 6573 * events are queued on the syncq. 6574 * Don't check for SQ_EXCL, because non-CIPUT 6575 * perimiters would set it, and it can't become 6576 * exclusive while we hold a claim. 6577 */ 6578 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { 6579 break; 6580 } 6581 6582 #ifdef DEBUG 6583 /* 6584 * Since we are in qdrain_syncq, we already know the queue, 6585 * but for sanity, we want to check this against the qp that 6586 * was passed in by bp->b_queue. 6587 */ 6588 6589 ASSERT(bp->b_queue == q); 6590 ASSERT(bp->b_queue->q_syncq == sq); 6591 bp->b_queue = NULL; 6592 6593 /* 6594 * We would have the following check in the DEBUG code: 6595 * 6596 * if (bp->b_prev != NULL) { 6597 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp); 6598 * } 6599 * 6600 * This can't be done, however, since IP modifies qinfo 6601 * structure at run-time (switching between IPv4 qinfo and IPv6 6602 * qinfo), invalidating the check. 6603 * So the assignment to func is left here, but the ASSERT itself 6604 * is removed until the whole issue is resolved. 6605 */ 6606 #endif 6607 ASSERT(q->q_sqhead == bp); 6608 q->q_sqhead = bp->b_next; 6609 bp->b_prev = bp->b_next = NULL; 6610 ASSERT(q->q_syncqmsgs > 0); 6611 mutex_exit(QLOCK(q)); 6612 6613 ASSERT(bp->b_datap->db_ref != 0); 6614 6615 (void) (*q->q_qinfo->qi_putp)(q, bp); 6616 6617 mutex_enter(QLOCK(q)); 6618 /* 6619 * We should decrement q_syncqmsgs only after executing the 6620 * put procedure to avoid a possible race with putnext(). 6621 * In putnext() though it sees Q_SQQUEUED is set, there is 6622 * an optimization which allows putnext to call the put 6623 * procedure directly if (q_syncqmsgs == 0) and thus 6624 * a message reodering could otherwise occur. 6625 */ 6626 q->q_syncqmsgs--; 6627 6628 /* 6629 * Clear QFULL in the next service procedure queue if 6630 * this is the last message destined to that queue. 6631 * 6632 * It would make better sense to have some sort of 6633 * tunable for the low water mark, but these symantics 6634 * are not yet defined. So, alas, we use a constant. 6635 */ 6636 do_clr = (q->q_syncqmsgs == 0); 6637 mutex_exit(QLOCK(q)); 6638 6639 if (do_clr) 6640 clr_qfull(q); 6641 6642 mutex_enter(QLOCK(q)); 6643 /* 6644 * Always clear SQ_EXCL when CIPUT in order to handle 6645 * qwriter(INNER). 6646 */ 6647 /* 6648 * The putp() can call qwriter and get exclusive access 6649 * IFF this is the only claim. So, we need to test for 6650 * this possibility so we can aquire the mutex and clear 6651 * the bit. 6652 */ 6653 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { 6654 mutex_enter(SQLOCK(sq)); 6655 sq->sq_flags &= ~SQ_EXCL; 6656 mutex_exit(SQLOCK(sq)); 6657 } 6658 } 6659 6660 /* 6661 * We should either have no queues on the syncq, or we were 6662 * told to goaway by a waiter (which we will wake up at the 6663 * end of this function). 6664 */ 6665 ASSERT((q->q_sqhead == NULL) || 6666 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); 6667 6668 ASSERT(MUTEX_HELD(QLOCK(q))); 6669 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6670 6671 /* 6672 * Remove the q from the syncq list if all the messages are 6673 * drained. 6674 */ 6675 if (q->q_sqhead == NULL) { 6676 mutex_enter(SQLOCK(sq)); 6677 if (q->q_sqflags & Q_SQQUEUED) 6678 SQRM_Q(sq, q); 6679 mutex_exit(SQLOCK(sq)); 6680 /* 6681 * Since the queue is removed from the list, reset its priority. 6682 */ 6683 q->q_spri = 0; 6684 } 6685 6686 /* 6687 * Remember, the q_draining flag is used to let another 6688 * thread know that there is a thread currently draining 6689 * the messages for a queue. Since we are now done with 6690 * this queue (even if there may be messages still there), 6691 * we need to clear this flag so some thread will work 6692 * on it if needed. 6693 */ 6694 ASSERT(q->q_draining); 6695 q->q_draining = 0; 6696 6697 /* called with a claim, so OK to drop all locks. */ 6698 mutex_exit(QLOCK(q)); 6699 6700 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6701 "drain_syncq end:%p", sq); 6702 } 6703 /* END OF QDRAIN_SYNCQ */ 6704 6705 6706 /* 6707 * This is the mate to qdrain_syncq, except that it is putting the 6708 * message onto the the queue instead draining. Since the 6709 * message is destined for the queue that is selected, there is 6710 * no need to identify the function because the message is 6711 * intended for the put routine for the queue. But this 6712 * routine will do it anyway just in case (but only for debug kernels). 6713 * 6714 * After the message is enqueued on the syncq, it calls putnext_tail() 6715 * which will schedule a background thread to actually process the message. 6716 * 6717 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and 6718 * SQLOCK(sq) and QLOCK(q) are not held. 6719 */ 6720 void 6721 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) 6722 { 6723 queue_t *fq = NULL; 6724 6725 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6726 ASSERT(MUTEX_NOT_HELD(QLOCK(q))); 6727 ASSERT(sq->sq_count > 0); 6728 ASSERT(q->q_syncq == sq); 6729 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6730 sq->sq_oprev == NULL) || 6731 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6732 sq->sq_oprev != NULL)); 6733 6734 mutex_enter(QLOCK(q)); 6735 6736 /* 6737 * Set QFULL in next service procedure queue (that cares) if not 6738 * already set and if there are already more messages on the syncq 6739 * than sq_max_size. If sq_max_size is 0, no flow control will be 6740 * asserted on any syncq. 6741 * 6742 * The fq here is the next queue with a service procedure. 6743 * This is where we would fail canputnext, so this is where we 6744 * need to set QFULL. 6745 * 6746 * LOCKING HIERARCHY: In the case when fq != q we need to 6747 * a) Take QLOCK(fq) to set QFULL flag and 6748 * b) Take sd_reflock in the case of the hot stream to update 6749 * sd_refcnt. 6750 * We already have QLOCK at this point. To avoid cross-locks with 6751 * freezestr() which grabs all QLOCKs and with strlock() which grabs 6752 * both SQLOCK and sd_reflock, we need to drop respective locks first. 6753 */ 6754 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) && 6755 (q->q_syncqmsgs > sq_max_size)) { 6756 if ((fq = q->q_nfsrv) == q) { 6757 fq->q_flag |= QFULL; 6758 } else { 6759 mutex_exit(QLOCK(q)); 6760 mutex_enter(QLOCK(fq)); 6761 fq->q_flag |= QFULL; 6762 mutex_exit(QLOCK(fq)); 6763 mutex_enter(QLOCK(q)); 6764 } 6765 } 6766 6767 #ifdef DEBUG 6768 /* 6769 * This is used for debug in the qfill_syncq/qdrain_syncq case 6770 * to trace the queue that the message is intended for. Note 6771 * that the original use was to identify the queue and function 6772 * to call on the drain. In the new syncq, we have the context 6773 * of the queue that we are draining, so call it's putproc and 6774 * don't rely on the saved values. But for debug this is still 6775 * usefull information. 6776 */ 6777 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp; 6778 mp->b_queue = q; 6779 mp->b_next = NULL; 6780 #endif 6781 ASSERT(q->q_syncq == sq); 6782 /* 6783 * Enqueue the message on the list. 6784 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to 6785 * protect it. So its ok to acquire SQLOCK after SQPUT_MP(). 6786 */ 6787 SQPUT_MP(q, mp); 6788 mutex_enter(SQLOCK(sq)); 6789 6790 /* 6791 * And queue on syncq for scheduling, if not already queued. 6792 * Note that we need the SQLOCK for this, and for testing flags 6793 * at the end to see if we will drain. So grab it now, and 6794 * release it before we call qdrain_syncq or return. 6795 */ 6796 if (!(q->q_sqflags & Q_SQQUEUED)) { 6797 q->q_spri = curthread->t_pri; 6798 SQPUT_Q(sq, q); 6799 } 6800 #ifdef DEBUG 6801 else { 6802 /* 6803 * All of these conditions MUST be true! 6804 */ 6805 ASSERT(sq->sq_tail != NULL); 6806 if (sq->sq_tail == sq->sq_head) { 6807 ASSERT((q->q_sqprev == NULL) && 6808 (q->q_sqnext == NULL)); 6809 } else { 6810 ASSERT((q->q_sqprev != NULL) || 6811 (q->q_sqnext != NULL)); 6812 } 6813 ASSERT(sq->sq_flags & SQ_QUEUED); 6814 ASSERT(q->q_syncqmsgs != 0); 6815 ASSERT(q->q_sqflags & Q_SQQUEUED); 6816 } 6817 #endif 6818 mutex_exit(QLOCK(q)); 6819 /* 6820 * SQLOCK is still held, so sq_count can be safely decremented. 6821 */ 6822 sq->sq_count--; 6823 6824 putnext_tail(sq, q, 0); 6825 /* Should not reference sq or q after this point. */ 6826 } 6827 6828 /* End of qfill_syncq */ 6829 6830 /* 6831 * Remove all messages from a syncq (if qp is NULL) or remove all messages 6832 * that would be put into qp by drain_syncq. 6833 * Used when deleting the syncq (qp == NULL) or when detaching 6834 * a queue (qp != NULL). 6835 * Return non-zero if one or more messages were freed. 6836 * 6837 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 6838 * sq_putlocks are used. 6839 * 6840 * NOTE: This function assumes that it is called from the close() context and 6841 * that all the queues in the syncq are going aay. For this reason it doesn't 6842 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is 6843 * currently valid, but it is useful to rethink this function to behave properly 6844 * in other cases. 6845 */ 6846 int 6847 flush_syncq(syncq_t *sq, queue_t *qp) 6848 { 6849 mblk_t *bp, *mp_head, *mp_next, *mp_prev; 6850 queue_t *q; 6851 int ret = 0; 6852 6853 mutex_enter(SQLOCK(sq)); 6854 6855 /* 6856 * Before we leave, we need to make sure there are no 6857 * events listed for this queue. All events for this queue 6858 * will just be freed. 6859 */ 6860 if (qp != NULL && sq->sq_evhead != NULL) { 6861 ASSERT(sq->sq_flags & SQ_EVENTS); 6862 6863 mp_prev = NULL; 6864 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { 6865 mp_next = bp->b_next; 6866 if (bp->b_queue == qp) { 6867 /* Delete this message */ 6868 if (mp_prev != NULL) { 6869 mp_prev->b_next = mp_next; 6870 /* 6871 * Update sq_evtail if the last element 6872 * is removed. 6873 */ 6874 if (bp == sq->sq_evtail) { 6875 ASSERT(mp_next == NULL); 6876 sq->sq_evtail = mp_prev; 6877 } 6878 } else 6879 sq->sq_evhead = mp_next; 6880 if (sq->sq_evhead == NULL) 6881 sq->sq_flags &= ~SQ_EVENTS; 6882 bp->b_prev = bp->b_next = NULL; 6883 freemsg(bp); 6884 ret++; 6885 } else { 6886 mp_prev = bp; 6887 } 6888 } 6889 } 6890 6891 /* 6892 * Walk sq_head and: 6893 * - match qp if qp is set, remove it's messages 6894 * - all if qp is not set 6895 */ 6896 q = sq->sq_head; 6897 while (q != NULL) { 6898 ASSERT(q->q_syncq == sq); 6899 if ((qp == NULL) || (qp == q)) { 6900 /* 6901 * Yank the messages as a list off the queue 6902 */ 6903 mp_head = q->q_sqhead; 6904 /* 6905 * We do not have QLOCK(q) here (which is safe due to 6906 * assumptions mentioned above). To obtain the lock we 6907 * need to release SQLOCK which may allow lots of things 6908 * to change upon us. This place requires more analysis. 6909 */ 6910 q->q_sqhead = q->q_sqtail = NULL; 6911 ASSERT(mp_head->b_queue && 6912 mp_head->b_queue->q_syncq == sq); 6913 6914 /* 6915 * Free each of the messages. 6916 */ 6917 for (bp = mp_head; bp != NULL; bp = mp_next) { 6918 mp_next = bp->b_next; 6919 bp->b_prev = bp->b_next = NULL; 6920 freemsg(bp); 6921 ret++; 6922 } 6923 /* 6924 * Now remove the queue from the syncq. 6925 */ 6926 ASSERT(q->q_sqflags & Q_SQQUEUED); 6927 SQRM_Q(sq, q); 6928 q->q_spri = 0; 6929 q->q_syncqmsgs = 0; 6930 6931 /* 6932 * If qp was specified, we are done with it and are 6933 * going to drop SQLOCK(sq) and return. We wakeup syncq 6934 * waiters while we still have the SQLOCK. 6935 */ 6936 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { 6937 sq->sq_flags &= ~SQ_WANTWAKEUP; 6938 cv_broadcast(&sq->sq_wait); 6939 } 6940 /* Drop SQLOCK across clr_qfull */ 6941 mutex_exit(SQLOCK(sq)); 6942 6943 /* 6944 * We avoid doing the test that drain_syncq does and 6945 * unconditionally clear qfull for every flushed 6946 * message. Since flush_syncq is only called during 6947 * close this should not be a problem. 6948 */ 6949 clr_qfull(q); 6950 if (qp != NULL) { 6951 return (ret); 6952 } else { 6953 mutex_enter(SQLOCK(sq)); 6954 /* 6955 * The head was removed by SQRM_Q above. 6956 * reread the new head and flush it. 6957 */ 6958 q = sq->sq_head; 6959 } 6960 } else { 6961 q = q->q_sqnext; 6962 } 6963 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6964 } 6965 6966 if (sq->sq_flags & SQ_WANTWAKEUP) { 6967 sq->sq_flags &= ~SQ_WANTWAKEUP; 6968 cv_broadcast(&sq->sq_wait); 6969 } 6970 6971 mutex_exit(SQLOCK(sq)); 6972 return (ret); 6973 } 6974 6975 /* 6976 * Propagate all messages from a syncq to the next syncq that are associated 6977 * with the specified queue. If the queue is attached to a driver or if the 6978 * messages have been added due to a qwriter(PERIM_INNER), free the messages. 6979 * 6980 * Assumes that the stream is strlock()'ed. We don't come here if there 6981 * are no messages to propagate. 6982 * 6983 * NOTE : If the queue is attached to a driver, all the messages are freed 6984 * as there is no point in propagating the messages from the driver syncq 6985 * to the closing stream head which will in turn get freed later. 6986 */ 6987 static int 6988 propagate_syncq(queue_t *qp) 6989 { 6990 mblk_t *bp, *head, *tail, *prev, *next; 6991 syncq_t *sq; 6992 queue_t *nqp; 6993 syncq_t *nsq; 6994 boolean_t isdriver; 6995 int moved = 0; 6996 uint16_t flags; 6997 pri_t priority = curthread->t_pri; 6998 #ifdef DEBUG 6999 void (*func)(); 7000 #endif 7001 7002 sq = qp->q_syncq; 7003 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7004 /* debug macro */ 7005 SQ_PUTLOCKS_HELD(sq); 7006 /* 7007 * As entersq() does not increment the sq_count for 7008 * the write side, check sq_count for non-QPERQ 7009 * perimeters alone. 7010 */ 7011 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); 7012 7013 /* 7014 * propagate_syncq() can be called because of either messages on the 7015 * queue syncq or because on events on the queue syncq. Do actual 7016 * message propagations if there are any messages. 7017 */ 7018 if (qp->q_syncqmsgs) { 7019 isdriver = (qp->q_flag & QISDRV); 7020 7021 if (!isdriver) { 7022 nqp = qp->q_next; 7023 nsq = nqp->q_syncq; 7024 ASSERT(MUTEX_HELD(SQLOCK(nsq))); 7025 /* debug macro */ 7026 SQ_PUTLOCKS_HELD(nsq); 7027 #ifdef DEBUG 7028 func = (void (*)())nqp->q_qinfo->qi_putp; 7029 #endif 7030 } 7031 7032 SQRM_Q(sq, qp); 7033 priority = MAX(qp->q_spri, priority); 7034 qp->q_spri = 0; 7035 head = qp->q_sqhead; 7036 tail = qp->q_sqtail; 7037 qp->q_sqhead = qp->q_sqtail = NULL; 7038 qp->q_syncqmsgs = 0; 7039 7040 /* 7041 * Walk the list of messages, and free them if this is a driver, 7042 * otherwise reset the b_prev and b_queue value to the new putp. 7043 * Afterward, we will just add the head to the end of the next 7044 * syncq, and point the tail to the end of this one. 7045 */ 7046 7047 for (bp = head; bp != NULL; bp = next) { 7048 next = bp->b_next; 7049 if (isdriver) { 7050 bp->b_prev = bp->b_next = NULL; 7051 freemsg(bp); 7052 continue; 7053 } 7054 /* Change the q values for this message */ 7055 bp->b_queue = nqp; 7056 #ifdef DEBUG 7057 bp->b_prev = (mblk_t *)func; 7058 #endif 7059 moved++; 7060 } 7061 /* 7062 * Attach list of messages to the end of the new queue (if there 7063 * is a list of messages). 7064 */ 7065 7066 if (!isdriver && head != NULL) { 7067 ASSERT(tail != NULL); 7068 if (nqp->q_sqhead == NULL) { 7069 nqp->q_sqhead = head; 7070 } else { 7071 ASSERT(nqp->q_sqtail != NULL); 7072 nqp->q_sqtail->b_next = head; 7073 } 7074 nqp->q_sqtail = tail; 7075 /* 7076 * When messages are moved from high priority queue to 7077 * another queue, the destination queue priority is 7078 * upgraded. 7079 */ 7080 7081 if (priority > nqp->q_spri) 7082 nqp->q_spri = priority; 7083 7084 SQPUT_Q(nsq, nqp); 7085 7086 nqp->q_syncqmsgs += moved; 7087 ASSERT(nqp->q_syncqmsgs != 0); 7088 } 7089 } 7090 7091 /* 7092 * Before we leave, we need to make sure there are no 7093 * events listed for this queue. All events for this queue 7094 * will just be freed. 7095 */ 7096 if (sq->sq_evhead != NULL) { 7097 ASSERT(sq->sq_flags & SQ_EVENTS); 7098 prev = NULL; 7099 for (bp = sq->sq_evhead; bp != NULL; bp = next) { 7100 next = bp->b_next; 7101 if (bp->b_queue == qp) { 7102 /* Delete this message */ 7103 if (prev != NULL) { 7104 prev->b_next = next; 7105 /* 7106 * Update sq_evtail if the last element 7107 * is removed. 7108 */ 7109 if (bp == sq->sq_evtail) { 7110 ASSERT(next == NULL); 7111 sq->sq_evtail = prev; 7112 } 7113 } else 7114 sq->sq_evhead = next; 7115 if (sq->sq_evhead == NULL) 7116 sq->sq_flags &= ~SQ_EVENTS; 7117 bp->b_prev = bp->b_next = NULL; 7118 freemsg(bp); 7119 } else { 7120 prev = bp; 7121 } 7122 } 7123 } 7124 7125 flags = sq->sq_flags; 7126 7127 /* Wake up any waiter before leaving. */ 7128 if (flags & SQ_WANTWAKEUP) { 7129 flags &= ~SQ_WANTWAKEUP; 7130 cv_broadcast(&sq->sq_wait); 7131 } 7132 sq->sq_flags = flags; 7133 7134 return (moved); 7135 } 7136 7137 /* 7138 * Try and upgrade to exclusive access at the inner perimeter. If this can 7139 * not be done without blocking then request will be queued on the syncq 7140 * and drain_syncq will run it later. 7141 * 7142 * This routine can only be called from put or service procedures plus 7143 * asynchronous callback routines that have properly entered to 7144 * queue (with entersq.) Thus qwriter_inner assumes the caller has one claim 7145 * on the syncq associated with q. 7146 */ 7147 void 7148 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)()) 7149 { 7150 syncq_t *sq = q->q_syncq; 7151 uint16_t count; 7152 7153 mutex_enter(SQLOCK(sq)); 7154 count = sq->sq_count; 7155 SQ_PUTLOCKS_ENTER(sq); 7156 SUM_SQ_PUTCOUNTS(sq, count); 7157 ASSERT(count >= 1); 7158 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); 7159 7160 if (count == 1) { 7161 /* 7162 * Can upgrade. This case also handles nested qwriter calls 7163 * (when the qwriter callback function calls qwriter). In that 7164 * case SQ_EXCL is already set. 7165 */ 7166 sq->sq_flags |= SQ_EXCL; 7167 SQ_PUTLOCKS_EXIT(sq); 7168 mutex_exit(SQLOCK(sq)); 7169 (*func)(q, mp); 7170 /* 7171 * Assumes that leavesq, putnext, and drain_syncq will reset 7172 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on 7173 * until putnext, leavesq, or drain_syncq drops it. 7174 * That way we handle nested qwriter(INNER) without dropping 7175 * SQ_EXCL until the outermost qwriter callback routine is 7176 * done. 7177 */ 7178 return; 7179 } 7180 SQ_PUTLOCKS_EXIT(sq); 7181 sqfill_events(sq, q, mp, func); 7182 } 7183 7184 /* 7185 * Synchronous callback support functions 7186 */ 7187 7188 /* 7189 * Allocate a callback parameter structure. 7190 * Assumes that caller initializes the flags and the id. 7191 * Acquires SQLOCK(sq) if non-NULL is returned. 7192 */ 7193 callbparams_t * 7194 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) 7195 { 7196 callbparams_t *cbp; 7197 size_t size = sizeof (callbparams_t); 7198 7199 cbp = kmem_alloc(size, kmflags & ~KM_PANIC); 7200 7201 /* 7202 * Only try tryhard allocation if the caller is ready to panic. 7203 * Otherwise just fail. 7204 */ 7205 if (cbp == NULL) { 7206 if (kmflags & KM_PANIC) 7207 cbp = kmem_alloc_tryhard(sizeof (callbparams_t), 7208 &size, kmflags); 7209 else 7210 return (NULL); 7211 } 7212 7213 ASSERT(size >= sizeof (callbparams_t)); 7214 cbp->cbp_size = size; 7215 cbp->cbp_sq = sq; 7216 cbp->cbp_func = func; 7217 cbp->cbp_arg = arg; 7218 mutex_enter(SQLOCK(sq)); 7219 cbp->cbp_next = sq->sq_callbpend; 7220 sq->sq_callbpend = cbp; 7221 return (cbp); 7222 } 7223 7224 void 7225 callbparams_free(syncq_t *sq, callbparams_t *cbp) 7226 { 7227 callbparams_t **pp, *p; 7228 7229 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7230 7231 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7232 if (p == cbp) { 7233 *pp = p->cbp_next; 7234 kmem_free(p, p->cbp_size); 7235 return; 7236 } 7237 } 7238 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7239 "callbparams_free: not found\n")); 7240 } 7241 7242 void 7243 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) 7244 { 7245 callbparams_t **pp, *p; 7246 7247 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7248 7249 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7250 if (p->cbp_id == id && p->cbp_flags == flag) { 7251 *pp = p->cbp_next; 7252 kmem_free(p, p->cbp_size); 7253 return; 7254 } 7255 } 7256 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7257 "callbparams_free_id: not found\n")); 7258 } 7259 7260 /* 7261 * Callback wrapper function used by once-only callbacks that can be 7262 * cancelled (qtimeout and qbufcall) 7263 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be 7264 * cancelled by the qun* functions. 7265 */ 7266 void 7267 qcallbwrapper(void *arg) 7268 { 7269 callbparams_t *cbp = arg; 7270 syncq_t *sq; 7271 uint16_t count = 0; 7272 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 7273 uint16_t type; 7274 7275 sq = cbp->cbp_sq; 7276 mutex_enter(SQLOCK(sq)); 7277 type = sq->sq_type; 7278 if (!(type & SQ_CICB)) { 7279 count = sq->sq_count; 7280 SQ_PUTLOCKS_ENTER(sq); 7281 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 7282 SUM_SQ_PUTCOUNTS(sq, count); 7283 sq->sq_needexcl++; 7284 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 7285 waitflags |= SQ_MESSAGES; 7286 } 7287 /* Can not handle exlusive entry at outer perimeter */ 7288 ASSERT(type & SQ_COCB); 7289 7290 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { 7291 if ((sq->sq_callbflags & cbp->cbp_flags) && 7292 (sq->sq_cancelid == cbp->cbp_id)) { 7293 /* timeout has been cancelled */ 7294 sq->sq_callbflags |= SQ_CALLB_BYPASSED; 7295 callbparams_free(sq, cbp); 7296 if (!(type & SQ_CICB)) { 7297 ASSERT(sq->sq_needexcl > 0); 7298 sq->sq_needexcl--; 7299 if (sq->sq_needexcl == 0) { 7300 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7301 } 7302 SQ_PUTLOCKS_EXIT(sq); 7303 } 7304 mutex_exit(SQLOCK(sq)); 7305 return; 7306 } 7307 sq->sq_flags |= SQ_WANTWAKEUP; 7308 if (!(type & SQ_CICB)) { 7309 SQ_PUTLOCKS_EXIT(sq); 7310 } 7311 cv_wait(&sq->sq_wait, SQLOCK(sq)); 7312 if (!(type & SQ_CICB)) { 7313 count = sq->sq_count; 7314 SQ_PUTLOCKS_ENTER(sq); 7315 SUM_SQ_PUTCOUNTS(sq, count); 7316 } 7317 } 7318 7319 sq->sq_count++; 7320 ASSERT(sq->sq_count != 0); /* Wraparound */ 7321 if (!(type & SQ_CICB)) { 7322 ASSERT(count == 0); 7323 sq->sq_flags |= SQ_EXCL; 7324 ASSERT(sq->sq_needexcl > 0); 7325 sq->sq_needexcl--; 7326 if (sq->sq_needexcl == 0) { 7327 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7328 } 7329 SQ_PUTLOCKS_EXIT(sq); 7330 } 7331 7332 mutex_exit(SQLOCK(sq)); 7333 7334 cbp->cbp_func(cbp->cbp_arg); 7335 7336 /* 7337 * We drop the lock only for leavesq to re-acquire it. 7338 * Possible optimization is inline of leavesq. 7339 */ 7340 mutex_enter(SQLOCK(sq)); 7341 callbparams_free(sq, cbp); 7342 mutex_exit(SQLOCK(sq)); 7343 leavesq(sq, SQ_CALLBACK); 7344 } 7345 7346 /* 7347 * no need to grab sq_putlocks here. See comment in strsubr.h that 7348 * explains when sq_putlocks are used. 7349 * 7350 * sq_count (or one of the sq_putcounts) has already been 7351 * decremented by the caller, and if SQ_QUEUED, we need to call 7352 * drain_syncq (the global syncq drain). 7353 * If putnext_tail is called with the SQ_EXCL bit set, we are in 7354 * one of two states, non-CIPUT perimiter, and we need to clear 7355 * it, or we went exclusive in the put procedure. In any case, 7356 * we want to clear the bit now, and it is probably easier to do 7357 * this at the beginning of this function (remember, we hold 7358 * the SQLOCK). Lastly, if there are other messages queued 7359 * on the syncq (and not for our destination), enable the syncq 7360 * for background work. 7361 */ 7362 7363 /* ARGSUSED */ 7364 void 7365 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) 7366 { 7367 uint16_t flags = sq->sq_flags; 7368 7369 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7370 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 7371 7372 /* Clear SQ_EXCL if set in passflags */ 7373 if (passflags & SQ_EXCL) { 7374 flags &= ~SQ_EXCL; 7375 } 7376 if (flags & SQ_WANTWAKEUP) { 7377 flags &= ~SQ_WANTWAKEUP; 7378 cv_broadcast(&sq->sq_wait); 7379 } 7380 if (flags & SQ_WANTEXWAKEUP) { 7381 flags &= ~SQ_WANTEXWAKEUP; 7382 cv_broadcast(&sq->sq_exitwait); 7383 } 7384 sq->sq_flags = flags; 7385 7386 /* 7387 * We have cleared SQ_EXCL if we were asked to, and started 7388 * the wakeup process for waiters. If there are no writers 7389 * then we need to drain the syncq if we were told to, or 7390 * enable the background thread to do it. 7391 */ 7392 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) { 7393 if ((passflags & SQ_QUEUED) || 7394 (sq->sq_svcflags & SQ_DISABLED)) { 7395 /* drain_syncq will take care of events in the list */ 7396 drain_syncq(sq); 7397 return; 7398 } else if (flags & SQ_QUEUED) { 7399 sqenable(sq); 7400 } 7401 } 7402 /* Drop the SQLOCK on exit */ 7403 mutex_exit(SQLOCK(sq)); 7404 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END, 7405 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); 7406 } 7407 7408 void 7409 set_qend(queue_t *q) 7410 { 7411 mutex_enter(QLOCK(q)); 7412 if (!O_SAMESTR(q)) 7413 q->q_flag |= QEND; 7414 else 7415 q->q_flag &= ~QEND; 7416 mutex_exit(QLOCK(q)); 7417 q = _OTHERQ(q); 7418 mutex_enter(QLOCK(q)); 7419 if (!O_SAMESTR(q)) 7420 q->q_flag |= QEND; 7421 else 7422 q->q_flag &= ~QEND; 7423 mutex_exit(QLOCK(q)); 7424 } 7425 7426 7427 void 7428 clr_qfull(queue_t *q) 7429 { 7430 queue_t *oq = q; 7431 7432 q = q->q_nfsrv; 7433 /* Fast check if there is any work to do before getting the lock. */ 7434 if ((q->q_flag & (QFULL|QWANTW)) == 0) { 7435 return; 7436 } 7437 7438 /* 7439 * Do not reset QFULL (and backenable) if the q_count is the reason 7440 * for QFULL being set. 7441 */ 7442 mutex_enter(QLOCK(q)); 7443 /* 7444 * If both q_count and q_mblkcnt are less than the hiwat mark 7445 */ 7446 if ((q->q_count < q->q_hiwat) && (q->q_mblkcnt < q->q_hiwat)) { 7447 q->q_flag &= ~QFULL; 7448 /* 7449 * A little more confusing, how about this way: 7450 * if someone wants to write, 7451 * AND 7452 * both counts are less than the lowat mark 7453 * OR 7454 * the lowat mark is zero 7455 * THEN 7456 * backenable 7457 */ 7458 if ((q->q_flag & QWANTW) && 7459 (((q->q_count < q->q_lowat) && 7460 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) { 7461 q->q_flag &= ~QWANTW; 7462 mutex_exit(QLOCK(q)); 7463 backenable(oq, 0); 7464 } else 7465 mutex_exit(QLOCK(q)); 7466 } else 7467 mutex_exit(QLOCK(q)); 7468 } 7469 7470 /* 7471 * Set the forward service procedure pointer. 7472 * 7473 * Called at insert-time to cache a queue's next forward service procedure in 7474 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted 7475 * has a service procedure then q_nfsrv points to itself. If the queue to be 7476 * inserted does not have a service procedure, then q_nfsrv points to the next 7477 * queue forward that has a service procedure. If the queue is at the logical 7478 * end of the stream (driver for write side, stream head for the read side) 7479 * and does not have a service procedure, then q_nfsrv also points to itself. 7480 */ 7481 void 7482 set_nfsrv_ptr( 7483 queue_t *rnew, /* read queue pointer to new module */ 7484 queue_t *wnew, /* write queue pointer to new module */ 7485 queue_t *prev_rq, /* read queue pointer to the module above */ 7486 queue_t *prev_wq) /* write queue pointer to the module above */ 7487 { 7488 queue_t *qp; 7489 7490 if (prev_wq->q_next == NULL) { 7491 /* 7492 * Insert the driver, initialize the driver and stream head. 7493 * In this case, prev_rq/prev_wq should be the stream head. 7494 * _I_INSERT does not allow inserting a driver. Make sure 7495 * that it is not an insertion. 7496 */ 7497 ASSERT(!(rnew->q_flag & _QINSERTING)); 7498 wnew->q_nfsrv = wnew; 7499 if (rnew->q_qinfo->qi_srvp) 7500 rnew->q_nfsrv = rnew; 7501 else 7502 rnew->q_nfsrv = prev_rq; 7503 prev_rq->q_nfsrv = prev_rq; 7504 prev_wq->q_nfsrv = prev_wq; 7505 } else { 7506 /* 7507 * set up read side q_nfsrv pointer. This MUST be done 7508 * before setting the write side, because the setting of 7509 * the write side for a fifo may depend on it. 7510 * 7511 * Suppose we have a fifo that only has pipemod pushed. 7512 * pipemod has no read or write service procedures, so 7513 * nfsrv for both pipemod queues points to prev_rq (the 7514 * stream read head). Now push bufmod (which has only a 7515 * read service procedure). Doing the write side first, 7516 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which 7517 * is WRONG; the next queue forward from wnew with a 7518 * service procedure will be rnew, not the stream read head. 7519 * Since the downstream queue (which in the case of a fifo 7520 * is the read queue rnew) can affect upstream queues, it 7521 * needs to be done first. Setting up the read side first 7522 * sets nfsrv for both pipemod queues to rnew and then 7523 * when the write side is set up, wnew-q_nfsrv will also 7524 * point to rnew. 7525 */ 7526 if (rnew->q_qinfo->qi_srvp) { 7527 /* 7528 * use _OTHERQ() because, if this is a pipe, next 7529 * module may have been pushed from other end and 7530 * q_next could be a read queue. 7531 */ 7532 qp = _OTHERQ(prev_wq->q_next); 7533 while (qp && qp->q_nfsrv != qp) { 7534 qp->q_nfsrv = rnew; 7535 qp = backq(qp); 7536 } 7537 rnew->q_nfsrv = rnew; 7538 } else 7539 rnew->q_nfsrv = prev_rq->q_nfsrv; 7540 7541 /* set up write side q_nfsrv pointer */ 7542 if (wnew->q_qinfo->qi_srvp) { 7543 wnew->q_nfsrv = wnew; 7544 7545 /* 7546 * For insertion, need to update nfsrv of the modules 7547 * above which do not have a service routine. 7548 */ 7549 if (rnew->q_flag & _QINSERTING) { 7550 for (qp = prev_wq; 7551 qp != NULL && qp->q_nfsrv != qp; 7552 qp = backq(qp)) { 7553 qp->q_nfsrv = wnew->q_nfsrv; 7554 } 7555 } 7556 } else { 7557 if (prev_wq->q_next == prev_rq) 7558 /* 7559 * Since prev_wq/prev_rq are the middle of a 7560 * fifo, wnew/rnew will also be the middle of 7561 * a fifo and wnew's nfsrv is same as rnew's. 7562 */ 7563 wnew->q_nfsrv = rnew->q_nfsrv; 7564 else 7565 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv; 7566 } 7567 } 7568 } 7569 7570 /* 7571 * Reset the forward service procedure pointer; called at remove-time. 7572 */ 7573 void 7574 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp) 7575 { 7576 queue_t *tmp_qp; 7577 7578 /* Reset the write side q_nfsrv pointer for _I_REMOVE */ 7579 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) { 7580 for (tmp_qp = backq(wqp); 7581 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp; 7582 tmp_qp = backq(tmp_qp)) { 7583 tmp_qp->q_nfsrv = wqp->q_nfsrv; 7584 } 7585 } 7586 7587 /* reset the read side q_nfsrv pointer */ 7588 if (rqp->q_qinfo->qi_srvp) { 7589 if (wqp->q_next) { /* non-driver case */ 7590 tmp_qp = _OTHERQ(wqp->q_next); 7591 while (tmp_qp && tmp_qp->q_nfsrv == rqp) { 7592 /* Note that rqp->q_next cannot be NULL */ 7593 ASSERT(rqp->q_next != NULL); 7594 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv; 7595 tmp_qp = backq(tmp_qp); 7596 } 7597 } 7598 } 7599 } 7600 7601 /* 7602 * This routine should be called after all stream geometry changes to update 7603 * the stream head cached struio() rd/wr queue pointers. Note must be called 7604 * with the streamlock()ed. 7605 * 7606 * Note: only enables Synchronous STREAMS for a side of a Stream which has 7607 * an explicit synchronous barrier module queue. That is, a queue that 7608 * has specified a struio() type. 7609 */ 7610 static void 7611 strsetuio(stdata_t *stp) 7612 { 7613 queue_t *wrq; 7614 7615 if (stp->sd_flag & STPLEX) { 7616 /* 7617 * Not stremahead, but a mux, so no Synchronous STREAMS. 7618 */ 7619 stp->sd_struiowrq = NULL; 7620 stp->sd_struiordq = NULL; 7621 return; 7622 } 7623 /* 7624 * Scan the write queue(s) while synchronous 7625 * until we find a qinfo uio type specified. 7626 */ 7627 wrq = stp->sd_wrq->q_next; 7628 while (wrq) { 7629 if (wrq->q_struiot == STRUIOT_NONE) { 7630 wrq = 0; 7631 break; 7632 } 7633 if (wrq->q_struiot != STRUIOT_DONTCARE) 7634 break; 7635 if (! _SAMESTR(wrq)) { 7636 wrq = 0; 7637 break; 7638 } 7639 wrq = wrq->q_next; 7640 } 7641 stp->sd_struiowrq = wrq; 7642 /* 7643 * Scan the read queue(s) while synchronous 7644 * until we find a qinfo uio type specified. 7645 */ 7646 wrq = stp->sd_wrq->q_next; 7647 while (wrq) { 7648 if (_RD(wrq)->q_struiot == STRUIOT_NONE) { 7649 wrq = 0; 7650 break; 7651 } 7652 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE) 7653 break; 7654 if (! _SAMESTR(wrq)) { 7655 wrq = 0; 7656 break; 7657 } 7658 wrq = wrq->q_next; 7659 } 7660 stp->sd_struiordq = wrq ? _RD(wrq) : 0; 7661 } 7662 7663 /* 7664 * pass_wput, unblocks the passthru queues, so that 7665 * messages can arrive at muxs lower read queue, before 7666 * I_LINK/I_UNLINK is acked/nacked. 7667 */ 7668 static void 7669 pass_wput(queue_t *q, mblk_t *mp) 7670 { 7671 syncq_t *sq; 7672 7673 sq = _RD(q)->q_syncq; 7674 if (sq->sq_flags & SQ_BLOCKED) 7675 unblocksq(sq, SQ_BLOCKED, 0); 7676 putnext(q, mp); 7677 } 7678 7679 /* 7680 * Set up queues for the link/unlink. 7681 * Create a new queue and block it and then insert it 7682 * below the stream head on the lower stream. 7683 * This prevents any messages from arriving during the setq 7684 * as well as while the mux is processing the LINK/I_UNLINK. 7685 * The blocked passq is unblocked once the LINK/I_UNLINK has 7686 * been acked or nacked or if a message is generated and sent 7687 * down muxs write put procedure. 7688 * see pass_wput(). 7689 * 7690 * After the new queue is inserted, all messages coming from below are 7691 * blocked. The call to strlock will ensure that all activity in the stream head 7692 * read queue syncq is stopped (sq_count drops to zero). 7693 */ 7694 static queue_t * 7695 link_addpassthru(stdata_t *stpdown) 7696 { 7697 queue_t *passq; 7698 sqlist_t sqlist; 7699 7700 passq = allocq(); 7701 STREAM(passq) = STREAM(_WR(passq)) = stpdown; 7702 /* setq might sleep in allocator - avoid holding locks. */ 7703 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ, 7704 SQ_CI|SQ_CO, B_FALSE); 7705 claimq(passq); 7706 blocksq(passq->q_syncq, SQ_BLOCKED, 1); 7707 insertq(STREAM(passq), passq); 7708 7709 /* 7710 * Use strlock() to wait for the stream head sq_count to drop to zero 7711 * since we are going to change q_ptr in the stream head. Note that 7712 * insertq() doesn't wait for any syncq counts to drop to zero. 7713 */ 7714 sqlist.sqlist_head = NULL; 7715 sqlist.sqlist_index = 0; 7716 sqlist.sqlist_size = sizeof (sqlist_t); 7717 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq); 7718 strlock(stpdown, &sqlist); 7719 strunlock(stpdown, &sqlist); 7720 7721 releaseq(passq); 7722 return (passq); 7723 } 7724 7725 /* 7726 * Let messages flow up into the mux by removing 7727 * the passq. 7728 */ 7729 static void 7730 link_rempassthru(queue_t *passq) 7731 { 7732 claimq(passq); 7733 removeq(passq); 7734 releaseq(passq); 7735 freeq(passq); 7736 } 7737 7738 /* 7739 * Wait for the condition variable pointed to by `cvp' to be signaled, 7740 * or for `tim' milliseconds to elapse, whichever comes first. If `tim' 7741 * is negative, then there is no time limit. If `nosigs' is non-zero, 7742 * then the wait will be non-interruptible. 7743 * 7744 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout. 7745 */ 7746 clock_t 7747 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs) 7748 { 7749 clock_t ret, now, tick; 7750 7751 if (tim < 0) { 7752 if (nosigs) { 7753 cv_wait(cvp, mp); 7754 ret = 1; 7755 } else { 7756 ret = cv_wait_sig(cvp, mp); 7757 } 7758 } else if (tim > 0) { 7759 /* 7760 * convert milliseconds to clock ticks 7761 */ 7762 tick = MSEC_TO_TICK_ROUNDUP(tim); 7763 time_to_wait(&now, tick); 7764 if (nosigs) { 7765 ret = cv_timedwait(cvp, mp, now); 7766 } else { 7767 ret = cv_timedwait_sig(cvp, mp, now); 7768 } 7769 } else { 7770 ret = -1; 7771 } 7772 return (ret); 7773 } 7774 7775 /* 7776 * Wait until the stream head can determine if it is at the mark but 7777 * don't wait forever to prevent a race condition between the "mark" state 7778 * in the stream head and any mark state in the caller/user of this routine. 7779 * 7780 * This is used by sockets and for a socket it would be incorrect 7781 * to return a failure for SIOCATMARK when there is no data in the receive 7782 * queue and the marked urgent data is traveling up the stream. 7783 * 7784 * This routine waits until the mark is known by waiting for one of these 7785 * three events: 7786 * The stream head read queue becoming non-empty (including an EOF) 7787 * The STRATMARK flag being set. (Due to a MSGMARKNEXT message.) 7788 * The STRNOTATMARK flag being set (which indicates that the transport 7789 * has sent a MSGNOTMARKNEXT message to indicate that it is not at 7790 * the mark). 7791 * 7792 * The routine returns 1 if the stream is at the mark; 0 if it can 7793 * be determined that the stream is not at the mark. 7794 * If the wait times out and it can't determine 7795 * whether or not the stream might be at the mark the routine will return -1. 7796 * 7797 * Note: This routine should only be used when a mark is pending i.e., 7798 * in the socket case the SIGURG has been posted. 7799 * Note2: This can not wakeup just because synchronous streams indicate 7800 * that data is available since it is not possible to use the synchronous 7801 * streams interfaces to determine the b_flag value for the data queued below 7802 * the stream head. 7803 */ 7804 int 7805 strwaitmark(vnode_t *vp) 7806 { 7807 struct stdata *stp = vp->v_stream; 7808 queue_t *rq = _RD(stp->sd_wrq); 7809 int mark; 7810 7811 mutex_enter(&stp->sd_lock); 7812 while (rq->q_first == NULL && 7813 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) { 7814 stp->sd_flag |= RSLEEP; 7815 7816 /* Wait for 100 milliseconds for any state change. */ 7817 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) { 7818 mutex_exit(&stp->sd_lock); 7819 return (-1); 7820 } 7821 } 7822 if (stp->sd_flag & STRATMARK) 7823 mark = 1; 7824 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK)) 7825 mark = 1; 7826 else 7827 mark = 0; 7828 7829 mutex_exit(&stp->sd_lock); 7830 return (mark); 7831 } 7832 7833 /* 7834 * Set a read side error. If persist is set change the socket error 7835 * to persistent. If errfunc is set install the function as the exported 7836 * error handler. 7837 */ 7838 void 7839 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7840 { 7841 struct stdata *stp = vp->v_stream; 7842 7843 mutex_enter(&stp->sd_lock); 7844 stp->sd_rerror = error; 7845 if (error == 0 && errfunc == NULL) 7846 stp->sd_flag &= ~STRDERR; 7847 else 7848 stp->sd_flag |= STRDERR; 7849 if (persist) { 7850 stp->sd_flag &= ~STRDERRNONPERSIST; 7851 } else { 7852 stp->sd_flag |= STRDERRNONPERSIST; 7853 } 7854 stp->sd_rderrfunc = errfunc; 7855 if (error != 0 || errfunc != NULL) { 7856 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7857 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7858 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 7859 7860 mutex_exit(&stp->sd_lock); 7861 pollwakeup(&stp->sd_pollist, POLLERR); 7862 mutex_enter(&stp->sd_lock); 7863 7864 if (stp->sd_sigflags & S_ERROR) 7865 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 7866 } 7867 mutex_exit(&stp->sd_lock); 7868 } 7869 7870 /* 7871 * Set a write side error. If persist is set change the socket error 7872 * to persistent. 7873 */ 7874 void 7875 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7876 { 7877 struct stdata *stp = vp->v_stream; 7878 7879 mutex_enter(&stp->sd_lock); 7880 stp->sd_werror = error; 7881 if (error == 0 && errfunc == NULL) 7882 stp->sd_flag &= ~STWRERR; 7883 else 7884 stp->sd_flag |= STWRERR; 7885 if (persist) { 7886 stp->sd_flag &= ~STWRERRNONPERSIST; 7887 } else { 7888 stp->sd_flag |= STWRERRNONPERSIST; 7889 } 7890 stp->sd_wrerrfunc = errfunc; 7891 if (error != 0 || errfunc != NULL) { 7892 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7893 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7894 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 7895 7896 mutex_exit(&stp->sd_lock); 7897 pollwakeup(&stp->sd_pollist, POLLERR); 7898 mutex_enter(&stp->sd_lock); 7899 7900 if (stp->sd_sigflags & S_ERROR) 7901 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 7902 } 7903 mutex_exit(&stp->sd_lock); 7904 } 7905 7906 /* 7907 * Make the stream return 0 (EOF) when all data has been read. 7908 * No effect on write side. 7909 */ 7910 void 7911 strseteof(vnode_t *vp, int eof) 7912 { 7913 struct stdata *stp = vp->v_stream; 7914 7915 mutex_enter(&stp->sd_lock); 7916 if (!eof) { 7917 stp->sd_flag &= ~STREOF; 7918 mutex_exit(&stp->sd_lock); 7919 return; 7920 } 7921 stp->sd_flag |= STREOF; 7922 if (stp->sd_flag & RSLEEP) { 7923 stp->sd_flag &= ~RSLEEP; 7924 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); 7925 } 7926 7927 mutex_exit(&stp->sd_lock); 7928 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM); 7929 mutex_enter(&stp->sd_lock); 7930 7931 if (stp->sd_sigflags & (S_INPUT|S_RDNORM)) 7932 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0); 7933 mutex_exit(&stp->sd_lock); 7934 } 7935 7936 void 7937 strflushrq(vnode_t *vp, int flag) 7938 { 7939 struct stdata *stp = vp->v_stream; 7940 7941 mutex_enter(&stp->sd_lock); 7942 flushq(_RD(stp->sd_wrq), flag); 7943 mutex_exit(&stp->sd_lock); 7944 } 7945 7946 void 7947 strsetrputhooks(vnode_t *vp, uint_t flags, 7948 msgfunc_t protofunc, msgfunc_t miscfunc) 7949 { 7950 struct stdata *stp = vp->v_stream; 7951 7952 mutex_enter(&stp->sd_lock); 7953 7954 if (protofunc == NULL) 7955 stp->sd_rprotofunc = strrput_proto; 7956 else 7957 stp->sd_rprotofunc = protofunc; 7958 7959 if (miscfunc == NULL) 7960 stp->sd_rmiscfunc = strrput_misc; 7961 else 7962 stp->sd_rmiscfunc = miscfunc; 7963 7964 if (flags & SH_CONSOL_DATA) 7965 stp->sd_rput_opt |= SR_CONSOL_DATA; 7966 else 7967 stp->sd_rput_opt &= ~SR_CONSOL_DATA; 7968 7969 if (flags & SH_SIGALLDATA) 7970 stp->sd_rput_opt |= SR_SIGALLDATA; 7971 else 7972 stp->sd_rput_opt &= ~SR_SIGALLDATA; 7973 7974 if (flags & SH_IGN_ZEROLEN) 7975 stp->sd_rput_opt |= SR_IGN_ZEROLEN; 7976 else 7977 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN; 7978 7979 mutex_exit(&stp->sd_lock); 7980 } 7981 7982 void 7983 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime) 7984 { 7985 struct stdata *stp = vp->v_stream; 7986 7987 mutex_enter(&stp->sd_lock); 7988 stp->sd_closetime = closetime; 7989 7990 if (flags & SH_SIGPIPE) 7991 stp->sd_wput_opt |= SW_SIGPIPE; 7992 else 7993 stp->sd_wput_opt &= ~SW_SIGPIPE; 7994 if (flags & SH_RECHECK_ERR) 7995 stp->sd_wput_opt |= SW_RECHECK_ERR; 7996 else 7997 stp->sd_wput_opt &= ~SW_RECHECK_ERR; 7998 7999 mutex_exit(&stp->sd_lock); 8000 } 8001 8002 void 8003 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc) 8004 { 8005 struct stdata *stp = vp->v_stream; 8006 8007 mutex_enter(&stp->sd_lock); 8008 8009 stp->sd_rputdatafunc = rdatafunc; 8010 stp->sd_wputdatafunc = wdatafunc; 8011 8012 mutex_exit(&stp->sd_lock); 8013 } 8014 8015 /* Used within framework when the queue is already locked */ 8016 void 8017 qenable_locked(queue_t *q) 8018 { 8019 stdata_t *stp = STREAM(q); 8020 8021 ASSERT(MUTEX_HELD(QLOCK(q))); 8022 8023 if (!q->q_qinfo->qi_srvp) 8024 return; 8025 8026 /* 8027 * Do not place on run queue if already enabled or closing. 8028 */ 8029 if (q->q_flag & (QWCLOSE|QENAB)) 8030 return; 8031 8032 /* 8033 * mark queue enabled and place on run list if it is not already being 8034 * serviced. If it is serviced, the runservice() function will detect 8035 * that QENAB is set and call service procedure before clearing 8036 * QINSERVICE flag. 8037 */ 8038 q->q_flag |= QENAB; 8039 if (q->q_flag & QINSERVICE) 8040 return; 8041 8042 /* Record the time of qenable */ 8043 q->q_qtstamp = lbolt; 8044 8045 /* 8046 * Put the queue in the stp list and schedule it for background 8047 * processing if it is not already scheduled or if stream head does not 8048 * intent to process it in the foreground later by setting 8049 * STRS_WILLSERVICE flag. 8050 */ 8051 mutex_enter(&stp->sd_qlock); 8052 /* 8053 * If there are already something on the list, stp flags should show 8054 * intention to drain it. 8055 */ 8056 IMPLY(STREAM_NEEDSERVICE(stp), 8057 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))); 8058 8059 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link); 8060 stp->sd_nqueues++; 8061 8062 /* 8063 * If no one will drain this stream we are the first producer and 8064 * need to schedule it for background thread. 8065 */ 8066 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) { 8067 /* 8068 * No one will service this stream later, so we have to 8069 * schedule it now. 8070 */ 8071 STRSTAT(stenables); 8072 stp->sd_svcflags |= STRS_SCHEDULED; 8073 stp->sd_servid = (void *)taskq_dispatch(streams_taskq, 8074 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE); 8075 8076 if (stp->sd_servid == NULL) { 8077 /* 8078 * Task queue failed so fail over to the backup 8079 * servicing thread. 8080 */ 8081 STRSTAT(taskqfails); 8082 /* 8083 * It is safe to clear STRS_SCHEDULED flag because it 8084 * was set by this thread above. 8085 */ 8086 stp->sd_svcflags &= ~STRS_SCHEDULED; 8087 8088 /* 8089 * Failover scheduling is protected by service_queue 8090 * lock. 8091 */ 8092 mutex_enter(&service_queue); 8093 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q)); 8094 ASSERT(q->q_link == NULL); 8095 /* 8096 * Append the queue to qhead/qtail list. 8097 */ 8098 if (qhead == NULL) 8099 qhead = q; 8100 else 8101 qtail->q_link = q; 8102 qtail = q; 8103 /* 8104 * Clear stp queue list. 8105 */ 8106 stp->sd_qhead = stp->sd_qtail = NULL; 8107 stp->sd_nqueues = 0; 8108 /* 8109 * Wakeup background queue processing thread. 8110 */ 8111 cv_signal(&services_to_run); 8112 mutex_exit(&service_queue); 8113 } 8114 } 8115 mutex_exit(&stp->sd_qlock); 8116 } 8117 8118 static void 8119 queue_service(queue_t *q) 8120 { 8121 /* 8122 * The queue in the list should have 8123 * QENAB flag set and should not have 8124 * QINSERVICE flag set. QINSERVICE is 8125 * set when the queue is dequeued and 8126 * qenable_locked doesn't enqueue a 8127 * queue with QINSERVICE set. 8128 */ 8129 8130 ASSERT(!(q->q_flag & QINSERVICE)); 8131 ASSERT((q->q_flag & QENAB)); 8132 mutex_enter(QLOCK(q)); 8133 q->q_flag &= ~QENAB; 8134 q->q_flag |= QINSERVICE; 8135 mutex_exit(QLOCK(q)); 8136 runservice(q); 8137 } 8138 8139 static void 8140 syncq_service(syncq_t *sq) 8141 { 8142 STRSTAT(syncqservice); 8143 mutex_enter(SQLOCK(sq)); 8144 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); 8145 ASSERT(sq->sq_servcount != 0); 8146 ASSERT(sq->sq_next == NULL); 8147 8148 /* if we came here from the background thread, clear the flag */ 8149 if (sq->sq_svcflags & SQ_BGTHREAD) 8150 sq->sq_svcflags &= ~SQ_BGTHREAD; 8151 8152 /* let drain_syncq know that it's being called in the background */ 8153 sq->sq_svcflags |= SQ_SERVICE; 8154 drain_syncq(sq); 8155 } 8156 8157 static void 8158 qwriter_outer_service(syncq_t *outer) 8159 { 8160 /* 8161 * Note that SQ_WRITER is used on the outer perimeter 8162 * to signal that a qwriter(OUTER) is either investigating 8163 * running or that it is actually running a function. 8164 */ 8165 outer_enter(outer, SQ_BLOCKED|SQ_WRITER); 8166 8167 /* 8168 * All inner syncq are empty and have SQ_WRITER set 8169 * to block entering the outer perimeter. 8170 * 8171 * We do not need to explicitly call write_now since 8172 * outer_exit does it for us. 8173 */ 8174 outer_exit(outer); 8175 } 8176 8177 static void 8178 mblk_free(mblk_t *mp) 8179 { 8180 dblk_t *dbp = mp->b_datap; 8181 frtn_t *frp = dbp->db_frtnp; 8182 8183 mp->b_next = NULL; 8184 if (dbp->db_fthdr != NULL) 8185 str_ftfree(dbp); 8186 8187 ASSERT(dbp->db_fthdr == NULL); 8188 frp->free_func(frp->free_arg); 8189 ASSERT(dbp->db_mblk == mp); 8190 8191 if (dbp->db_credp != NULL) { 8192 crfree(dbp->db_credp); 8193 dbp->db_credp = NULL; 8194 } 8195 dbp->db_cpid = -1; 8196 dbp->db_struioflag = 0; 8197 dbp->db_struioun.cksum.flags = 0; 8198 8199 kmem_cache_free(dbp->db_cache, dbp); 8200 } 8201 8202 /* 8203 * Background processing of the stream queue list. 8204 */ 8205 static void 8206 stream_service(stdata_t *stp) 8207 { 8208 queue_t *q; 8209 8210 mutex_enter(&stp->sd_qlock); 8211 8212 STR_SERVICE(stp, q); 8213 8214 stp->sd_svcflags &= ~STRS_SCHEDULED; 8215 stp->sd_servid = NULL; 8216 cv_signal(&stp->sd_qcv); 8217 mutex_exit(&stp->sd_qlock); 8218 } 8219 8220 /* 8221 * Foreground processing of the stream queue list. 8222 */ 8223 void 8224 stream_runservice(stdata_t *stp) 8225 { 8226 queue_t *q; 8227 8228 mutex_enter(&stp->sd_qlock); 8229 STRSTAT(rservice); 8230 /* 8231 * We are going to drain this stream queue list, so qenable_locked will 8232 * not schedule it until we finish. 8233 */ 8234 stp->sd_svcflags |= STRS_WILLSERVICE; 8235 8236 STR_SERVICE(stp, q); 8237 8238 stp->sd_svcflags &= ~STRS_WILLSERVICE; 8239 mutex_exit(&stp->sd_qlock); 8240 /* 8241 * Help backup background thread to drain the qhead/qtail list. 8242 */ 8243 while (qhead != NULL) { 8244 STRSTAT(qhelps); 8245 mutex_enter(&service_queue); 8246 DQ(q, qhead, qtail, q_link); 8247 mutex_exit(&service_queue); 8248 if (q != NULL) 8249 queue_service(q); 8250 } 8251 } 8252 8253 void 8254 stream_willservice(stdata_t *stp) 8255 { 8256 mutex_enter(&stp->sd_qlock); 8257 stp->sd_svcflags |= STRS_WILLSERVICE; 8258 mutex_exit(&stp->sd_qlock); 8259 } 8260 8261 /* 8262 * Replace the cred currently in the mblk with a different one. 8263 */ 8264 void 8265 mblk_setcred(mblk_t *mp, cred_t *cr) 8266 { 8267 cred_t *ocr = DB_CRED(mp); 8268 8269 ASSERT(cr != NULL); 8270 8271 if (cr != ocr) { 8272 crhold(mp->b_datap->db_credp = cr); 8273 if (ocr != NULL) 8274 crfree(ocr); 8275 } 8276 } 8277 8278 int 8279 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8280 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value, 8281 uint32_t flags, int km_flags) 8282 { 8283 int rc = 0; 8284 8285 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8286 if (mp->b_datap->db_type == M_DATA) { 8287 /* Associate values for M_DATA type */ 8288 DB_CKSUMSTART(mp) = (intptr_t)start; 8289 DB_CKSUMSTUFF(mp) = (intptr_t)stuff; 8290 DB_CKSUMEND(mp) = (intptr_t)end; 8291 DB_CKSUMFLAGS(mp) = flags; 8292 DB_CKSUM16(mp) = (uint16_t)value; 8293 8294 } else { 8295 pattrinfo_t pa_info; 8296 8297 ASSERT(mmd != NULL); 8298 8299 pa_info.type = PATTR_HCKSUM; 8300 pa_info.len = sizeof (pattr_hcksum_t); 8301 8302 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) { 8303 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf; 8304 8305 hck->hcksum_start_offset = start; 8306 hck->hcksum_stuff_offset = stuff; 8307 hck->hcksum_end_offset = end; 8308 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value; 8309 hck->hcksum_flags = flags; 8310 } else { 8311 rc = -1; 8312 } 8313 } 8314 return (rc); 8315 } 8316 8317 void 8318 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8319 uint32_t *start, uint32_t *stuff, uint32_t *end, 8320 uint32_t *value, uint32_t *flags) 8321 { 8322 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8323 if (mp->b_datap->db_type == M_DATA) { 8324 if (flags != NULL) { 8325 *flags = DB_CKSUMFLAGS(mp); 8326 if (*flags & HCK_PARTIALCKSUM) { 8327 if (start != NULL) 8328 *start = (uint32_t)DB_CKSUMSTART(mp); 8329 if (stuff != NULL) 8330 *stuff = (uint32_t)DB_CKSUMSTUFF(mp); 8331 if (end != NULL) 8332 *end = (uint32_t)DB_CKSUMEND(mp); 8333 if (value != NULL) 8334 *value = (uint32_t)DB_CKSUM16(mp); 8335 } else if ((*flags & HW_LSO) && (value != NULL)) 8336 *value = (uint32_t)DB_LSOMSS(mp); 8337 } 8338 } else { 8339 pattrinfo_t hck_attr = {PATTR_HCKSUM}; 8340 8341 ASSERT(mmd != NULL); 8342 8343 /* get hardware checksum attribute */ 8344 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) { 8345 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf; 8346 8347 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t)); 8348 if (flags != NULL) 8349 *flags = hck->hcksum_flags; 8350 if (start != NULL) 8351 *start = hck->hcksum_start_offset; 8352 if (stuff != NULL) 8353 *stuff = hck->hcksum_stuff_offset; 8354 if (end != NULL) 8355 *end = hck->hcksum_end_offset; 8356 if (value != NULL) 8357 *value = (uint32_t) 8358 hck->hcksum_cksum_val.inet_cksum; 8359 } 8360 } 8361 } 8362 8363 /* 8364 * Checksum buffer *bp for len bytes with psum partial checksum, 8365 * or 0 if none, and return the 16 bit partial checksum. 8366 */ 8367 unsigned 8368 bcksum(uchar_t *bp, int len, unsigned int psum) 8369 { 8370 int odd = len & 1; 8371 extern unsigned int ip_ocsum(); 8372 8373 if (((intptr_t)bp & 1) == 0 && !odd) { 8374 /* 8375 * Bp is 16 bit aligned and len is multiple of 16 bit word. 8376 */ 8377 return (ip_ocsum((ushort_t *)bp, len >> 1, psum)); 8378 } 8379 if (((intptr_t)bp & 1) != 0) { 8380 /* 8381 * Bp isn't 16 bit aligned. 8382 */ 8383 unsigned int tsum; 8384 8385 #ifdef _LITTLE_ENDIAN 8386 psum += *bp; 8387 #else 8388 psum += *bp << 8; 8389 #endif 8390 len--; 8391 bp++; 8392 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0); 8393 psum += (tsum << 8) & 0xffff | (tsum >> 8); 8394 if (len & 1) { 8395 bp += len - 1; 8396 #ifdef _LITTLE_ENDIAN 8397 psum += *bp << 8; 8398 #else 8399 psum += *bp; 8400 #endif 8401 } 8402 } else { 8403 /* 8404 * Bp is 16 bit aligned. 8405 */ 8406 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum); 8407 if (odd) { 8408 bp += len - 1; 8409 #ifdef _LITTLE_ENDIAN 8410 psum += *bp; 8411 #else 8412 psum += *bp << 8; 8413 #endif 8414 } 8415 } 8416 /* 8417 * Normalize psum to 16 bits before returning the new partial 8418 * checksum. The max psum value before normalization is 0x3FDFE. 8419 */ 8420 return ((psum >> 16) + (psum & 0xFFFF)); 8421 } 8422 8423 boolean_t 8424 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd) 8425 { 8426 boolean_t rc; 8427 8428 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8429 if (DB_TYPE(mp) == M_DATA) { 8430 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0); 8431 } else { 8432 pattrinfo_t zcopy_attr = {PATTR_ZCOPY}; 8433 8434 ASSERT(mmd != NULL); 8435 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL); 8436 } 8437 return (rc); 8438 } 8439 8440 void 8441 freemsgchain(mblk_t *mp) 8442 { 8443 mblk_t *next; 8444 8445 while (mp != NULL) { 8446 next = mp->b_next; 8447 mp->b_next = NULL; 8448 8449 freemsg(mp); 8450 mp = next; 8451 } 8452 } 8453 8454 mblk_t * 8455 copymsgchain(mblk_t *mp) 8456 { 8457 mblk_t *nmp = NULL; 8458 mblk_t **nmpp = &nmp; 8459 8460 for (; mp != NULL; mp = mp->b_next) { 8461 if ((*nmpp = copymsg(mp)) == NULL) { 8462 freemsgchain(nmp); 8463 return (NULL); 8464 } 8465 8466 nmpp = &((*nmpp)->b_next); 8467 } 8468 8469 return (nmp); 8470 } 8471 8472 /* NOTE: Do not add code after this point. */ 8473 #undef QLOCK 8474 8475 /* 8476 * replacement for QLOCK macro for those that can't use it. 8477 */ 8478 kmutex_t * 8479 QLOCK(queue_t *q) 8480 { 8481 return (&(q)->q_lock); 8482 } 8483 8484 /* 8485 * Dummy runqueues/queuerun functions functions for backwards compatibility. 8486 */ 8487 #undef runqueues 8488 void 8489 runqueues(void) 8490 { 8491 } 8492 8493 #undef queuerun 8494 void 8495 queuerun(void) 8496 { 8497 } 8498 8499 /* 8500 * Initialize the STR stack instance, which tracks autopush and persistent 8501 * links. 8502 */ 8503 /* ARGSUSED */ 8504 static void * 8505 str_stack_init(netstackid_t stackid, netstack_t *ns) 8506 { 8507 str_stack_t *ss; 8508 int i; 8509 8510 ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP); 8511 ss->ss_netstack = ns; 8512 8513 /* 8514 * set up autopush 8515 */ 8516 sad_initspace(ss); 8517 8518 /* 8519 * set up mux_node structures. 8520 */ 8521 ss->ss_devcnt = devcnt; /* In case it should change before free */ 8522 ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) * 8523 ss->ss_devcnt), KM_SLEEP); 8524 for (i = 0; i < ss->ss_devcnt; i++) 8525 ss->ss_mux_nodes[i].mn_imaj = i; 8526 return (ss); 8527 } 8528 8529 /* 8530 * Note: run at zone shutdown and not destroy so that the PLINKs are 8531 * gone by the time other cleanup happens from the destroy callbacks. 8532 */ 8533 static void 8534 str_stack_shutdown(netstackid_t stackid, void *arg) 8535 { 8536 str_stack_t *ss = (str_stack_t *)arg; 8537 int i; 8538 cred_t *cr; 8539 8540 cr = zone_get_kcred(netstackid_to_zoneid(stackid)); 8541 ASSERT(cr != NULL); 8542 8543 /* Undo all the I_PLINKs for this zone */ 8544 for (i = 0; i < ss->ss_devcnt; i++) { 8545 struct mux_edge *ep; 8546 ldi_handle_t lh; 8547 ldi_ident_t li; 8548 int ret; 8549 int rval; 8550 dev_t rdev; 8551 8552 ep = ss->ss_mux_nodes[i].mn_outp; 8553 if (ep == NULL) 8554 continue; 8555 ret = ldi_ident_from_major((major_t)i, &li); 8556 if (ret != 0) { 8557 continue; 8558 } 8559 rdev = ep->me_dev; 8560 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE, 8561 cr, &lh, li); 8562 if (ret != 0) { 8563 ldi_ident_release(li); 8564 continue; 8565 } 8566 8567 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL, 8568 cr, &rval); 8569 if (ret) { 8570 (void) ldi_close(lh, FREAD|FWRITE, cr); 8571 ldi_ident_release(li); 8572 continue; 8573 } 8574 (void) ldi_close(lh, FREAD|FWRITE, cr); 8575 8576 /* Close layered handles */ 8577 ldi_ident_release(li); 8578 } 8579 crfree(cr); 8580 8581 sad_freespace(ss); 8582 8583 kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt); 8584 ss->ss_mux_nodes = NULL; 8585 } 8586 8587 /* 8588 * Free the structure; str_stack_shutdown did the other cleanup work. 8589 */ 8590 /* ARGSUSED */ 8591 static void 8592 str_stack_fini(netstackid_t stackid, void *arg) 8593 { 8594 str_stack_t *ss = (str_stack_t *)arg; 8595 8596 kmem_free(ss, sizeof (*ss)); 8597 } 8598