1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 23 /* All Rights Reserved */ 24 25 26 /* 27 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31 #pragma ident "%Z%%M% %I% %E% SMI" 32 33 #include <sys/types.h> 34 #include <sys/sysmacros.h> 35 #include <sys/param.h> 36 #include <sys/errno.h> 37 #include <sys/signal.h> 38 #include <sys/proc.h> 39 #include <sys/conf.h> 40 #include <sys/cred.h> 41 #include <sys/user.h> 42 #include <sys/vnode.h> 43 #include <sys/file.h> 44 #include <sys/session.h> 45 #include <sys/stream.h> 46 #include <sys/strsubr.h> 47 #include <sys/stropts.h> 48 #include <sys/poll.h> 49 #include <sys/systm.h> 50 #include <sys/cpuvar.h> 51 #include <sys/uio.h> 52 #include <sys/cmn_err.h> 53 #include <sys/priocntl.h> 54 #include <sys/procset.h> 55 #include <sys/vmem.h> 56 #include <sys/bitmap.h> 57 #include <sys/kmem.h> 58 #include <sys/siginfo.h> 59 #include <sys/vtrace.h> 60 #include <sys/callb.h> 61 #include <sys/debug.h> 62 #include <sys/modctl.h> 63 #include <sys/vmsystm.h> 64 #include <vm/page.h> 65 #include <sys/atomic.h> 66 #include <sys/suntpi.h> 67 #include <sys/strlog.h> 68 #include <sys/promif.h> 69 #include <sys/project.h> 70 #include <sys/vm.h> 71 #include <sys/taskq.h> 72 #include <sys/sunddi.h> 73 #include <sys/sunldi_impl.h> 74 #include <sys/strsun.h> 75 #include <sys/isa_defs.h> 76 #include <sys/multidata.h> 77 #include <sys/pattr.h> 78 #include <sys/strft.h> 79 #include <sys/fs/snode.h> 80 #include <sys/zone.h> 81 82 #define O_SAMESTR(q) (((q)->q_next) && \ 83 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) 84 85 /* 86 * WARNING: 87 * The variables and routines in this file are private, belonging 88 * to the STREAMS subsystem. These should not be used by modules 89 * or drivers. Compatibility will not be guaranteed. 90 */ 91 92 /* 93 * Id value used to distinguish between different multiplexor links. 94 */ 95 static int32_t lnk_id = 0; 96 97 #define STREAMS_LOPRI MINCLSYSPRI 98 static pri_t streams_lopri = STREAMS_LOPRI; 99 100 #define STRSTAT(x) (str_statistics.x.value.ui64++) 101 typedef struct str_stat { 102 kstat_named_t sqenables; 103 kstat_named_t stenables; 104 kstat_named_t syncqservice; 105 kstat_named_t freebs; 106 kstat_named_t qwr_outer; 107 kstat_named_t rservice; 108 kstat_named_t strwaits; 109 kstat_named_t taskqfails; 110 kstat_named_t bufcalls; 111 kstat_named_t qhelps; 112 kstat_named_t qremoved; 113 kstat_named_t sqremoved; 114 kstat_named_t bcwaits; 115 kstat_named_t sqtoomany; 116 } str_stat_t; 117 118 static str_stat_t str_statistics = { 119 { "sqenables", KSTAT_DATA_UINT64 }, 120 { "stenables", KSTAT_DATA_UINT64 }, 121 { "syncqservice", KSTAT_DATA_UINT64 }, 122 { "freebs", KSTAT_DATA_UINT64 }, 123 { "qwr_outer", KSTAT_DATA_UINT64 }, 124 { "rservice", KSTAT_DATA_UINT64 }, 125 { "strwaits", KSTAT_DATA_UINT64 }, 126 { "taskqfails", KSTAT_DATA_UINT64 }, 127 { "bufcalls", KSTAT_DATA_UINT64 }, 128 { "qhelps", KSTAT_DATA_UINT64 }, 129 { "qremoved", KSTAT_DATA_UINT64 }, 130 { "sqremoved", KSTAT_DATA_UINT64 }, 131 { "bcwaits", KSTAT_DATA_UINT64 }, 132 { "sqtoomany", KSTAT_DATA_UINT64 }, 133 }; 134 135 static kstat_t *str_kstat; 136 137 /* 138 * qrunflag was used previously to control background scheduling of queues. It 139 * is not used anymore, but kept here in case some module still wants to access 140 * it via qready() and setqsched macros. 141 */ 142 char qrunflag; /* Unused */ 143 144 /* 145 * Most of the streams scheduling is done via task queues. Task queues may fail 146 * for non-sleep dispatches, so there are two backup threads servicing failed 147 * requests for queues and syncqs. Both of these threads also service failed 148 * dispatches freebs requests. Queues are put in the list specified by `qhead' 149 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs 150 * requests are put into `freebs_list' which has no tail pointer. All three 151 * lists are protected by a single `service_queue' lock and use 152 * `services_to_run' condition variable for signaling background threads. Use of 153 * a single lock should not be a problem because it is only used under heavy 154 * loads when task queues start to fail and at that time it may be a good idea 155 * to throttle scheduling requests. 156 * 157 * NOTE: queues and syncqs should be scheduled by two separate threads because 158 * queue servicing may be blocked waiting for a syncq which may be also 159 * scheduled for background execution. This may create a deadlock when only one 160 * thread is used for both. 161 */ 162 163 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ 164 165 static kmutex_t service_queue; /* protects all of servicing vars */ 166 static kcondvar_t services_to_run; /* wake up background service thread */ 167 static kcondvar_t syncqs_to_run; /* wake up background service thread */ 168 169 /* 170 * List of queues scheduled for background processing dueue to lack of resources 171 * in the task queues. Protected by service_queue lock; 172 */ 173 static struct queue *qhead; 174 static struct queue *qtail; 175 176 /* 177 * Same list for syncqs 178 */ 179 static syncq_t *sqhead; 180 static syncq_t *sqtail; 181 182 static mblk_t *freebs_list; /* list of buffers to free */ 183 184 /* 185 * Backup threads for servicing queues and syncqs 186 */ 187 kthread_t *streams_qbkgrnd_thread; 188 kthread_t *streams_sqbkgrnd_thread; 189 190 /* 191 * Bufcalls related variables. 192 */ 193 struct bclist strbcalls; /* list of waiting bufcalls */ 194 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ 195 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ 196 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ 197 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ 198 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ 199 200 kmutex_t strresources; /* protects global resources */ 201 kmutex_t muxifier; /* single-threads multiplexor creation */ 202 203 extern void time_to_wait(clock_t *, clock_t); 204 205 /* 206 * run_queues is no longer used, but is kept in case some 3-d party 207 * module/driver decides to use it. 208 */ 209 int run_queues = 0; 210 211 /* 212 * sq_max_size is the depth of the syncq (in number of messages) before 213 * qfill_syncq() starts QFULL'ing destination queues. As its primary 214 * consumer - IP is no longer D_MTPERMOD, but there may be other 215 * modules/drivers depend on this syncq flow control, we prefer to 216 * choose a large number as the default value. For potential 217 * performance gain, this value is tunable in /etc/system. 218 */ 219 int sq_max_size = 10000; 220 221 /* 222 * the number of ciputctrl structures per syncq and stream we create when 223 * needed. 224 */ 225 int n_ciputctrl; 226 int max_n_ciputctrl = 16; 227 /* 228 * if n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. 229 */ 230 int min_n_ciputctrl = 2; 231 232 static struct mux_node *mux_nodes; /* mux info for cycle checking */ 233 234 /* 235 * Per-driver/module syncqs 236 * ======================== 237 * 238 * For drivers/modules that use PERMOD or outer syncqs we keep a list of 239 * perdm structures, new entries being added (and new syncqs allocated) when 240 * setq() encounters a module/driver with a streamtab that it hasn't seen 241 * before. 242 * The reason for this mechanism is that some modules and drivers share a 243 * common streamtab and it is necessary for those modules and drivers to also 244 * share a common PERMOD syncq. 245 * 246 * perdm_list --> dm_str == streamtab_1 247 * dm_sq == syncq_1 248 * dm_ref 249 * dm_next --> dm_str == streamtab_2 250 * dm_sq == syncq_2 251 * dm_ref 252 * dm_next --> ... NULL 253 * 254 * The dm_ref field is incremented for each new driver/module that takes 255 * a reference to the perdm structure and hence shares the syncq. 256 * References are held in the fmodsw_impl_t structure for each STREAMS module 257 * or the dev_impl array (indexed by device major number) for each driver. 258 * 259 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL 260 * ^ ^ ^ ^ 261 * | ______________/ | | 262 * | / | | 263 * dev_impl: ...|x|y|... module A module B 264 * 265 * When a module/driver is unloaded the reference count is decremented and, 266 * when it falls to zero, the perdm structure is removed from the list and 267 * the syncq is freed (see rele_dm()). 268 */ 269 perdm_t *perdm_list = NULL; 270 static krwlock_t perdm_rwlock; 271 cdevsw_impl_t *devimpl; 272 273 extern struct qinit strdata; 274 extern struct qinit stwdata; 275 276 static void runservice(queue_t *); 277 static void streams_bufcall_service(void); 278 static void streams_qbkgrnd_service(void); 279 static void streams_sqbkgrnd_service(void); 280 static syncq_t *new_syncq(void); 281 static void free_syncq(syncq_t *); 282 static void outer_insert(syncq_t *, syncq_t *); 283 static void outer_remove(syncq_t *, syncq_t *); 284 static void write_now(syncq_t *); 285 static void clr_qfull(queue_t *); 286 static void enable_svc(queue_t *); 287 static void runbufcalls(void); 288 static void sqenable(syncq_t *); 289 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); 290 static void wait_q_syncq(queue_t *); 291 static void backenable_insertedq(queue_t *); 292 293 static void queue_service(queue_t *); 294 static void stream_service(stdata_t *); 295 static void syncq_service(syncq_t *); 296 static void qwriter_outer_service(syncq_t *); 297 static void mblk_free(mblk_t *); 298 #ifdef DEBUG 299 static int qprocsareon(queue_t *); 300 #endif 301 302 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); 303 static void reset_nfsrv_ptr(queue_t *, queue_t *); 304 305 static void sq_run_events(syncq_t *); 306 static int propagate_syncq(queue_t *); 307 308 static void blocksq(syncq_t *, ushort_t, int); 309 static void unblocksq(syncq_t *, ushort_t, int); 310 static int dropsq(syncq_t *, uint16_t); 311 static void emptysq(syncq_t *); 312 static sqlist_t *sqlist_alloc(struct stdata *, int); 313 static void sqlist_free(sqlist_t *); 314 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); 315 static void sqlist_insert(sqlist_t *, syncq_t *); 316 static void sqlist_insertall(sqlist_t *, queue_t *); 317 318 static void strsetuio(stdata_t *); 319 320 struct kmem_cache *stream_head_cache; 321 struct kmem_cache *queue_cache; 322 struct kmem_cache *syncq_cache; 323 struct kmem_cache *qband_cache; 324 struct kmem_cache *linkinfo_cache; 325 struct kmem_cache *ciputctrl_cache = NULL; 326 327 static linkinfo_t *linkinfo_list; 328 329 /* 330 * Qinit structure and Module_info structures 331 * for passthru read and write queues 332 */ 333 334 static void pass_wput(queue_t *, mblk_t *); 335 static queue_t *link_addpassthru(stdata_t *); 336 static void link_rempassthru(queue_t *); 337 338 struct module_info passthru_info = { 339 0, 340 "passthru", 341 0, 342 INFPSZ, 343 STRHIGH, 344 STRLOW 345 }; 346 347 struct qinit passthru_rinit = { 348 (int (*)())putnext, 349 NULL, 350 NULL, 351 NULL, 352 NULL, 353 &passthru_info, 354 NULL 355 }; 356 357 struct qinit passthru_winit = { 358 (int (*)()) pass_wput, 359 NULL, 360 NULL, 361 NULL, 362 NULL, 363 &passthru_info, 364 NULL 365 }; 366 367 /* 368 * Special form of assertion: verify that X implies Y i.e. when X is true Y 369 * should also be true. 370 */ 371 #define IMPLY(X, Y) ASSERT(!(X) || (Y)) 372 373 /* 374 * Logical equivalence. Verify that both X and Y are either TRUE or FALSE. 375 */ 376 #define EQUIV(X, Y) { IMPLY(X, Y); IMPLY(Y, X); } 377 378 /* 379 * Verify correctness of list head/tail pointers. 380 */ 381 #define LISTCHECK(head, tail, link) { \ 382 EQUIV(head, tail); \ 383 IMPLY(tail != NULL, tail->link == NULL); \ 384 } 385 386 /* 387 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail' 388 * using a `link' field. 389 */ 390 #define ENQUEUE(el, head, tail, link) { \ 391 ASSERT(el->link == NULL); \ 392 LISTCHECK(head, tail, link); \ 393 if (head == NULL) \ 394 head = el; \ 395 else \ 396 tail->link = el; \ 397 tail = el; \ 398 } 399 400 /* 401 * Dequeue the first element of the list denoted by `head' and `tail' pointers 402 * using a `link' field and put result into `el'. 403 */ 404 #define DQ(el, head, tail, link) { \ 405 LISTCHECK(head, tail, link); \ 406 el = head; \ 407 if (head != NULL) { \ 408 head = head->link; \ 409 if (head == NULL) \ 410 tail = NULL; \ 411 el->link = NULL; \ 412 } \ 413 } 414 415 /* 416 * Remove `el' from the list using `chase' and `curr' pointers and return result 417 * in `succeed'. 418 */ 419 #define RMQ(el, head, tail, link, chase, curr, succeed) { \ 420 LISTCHECK(head, tail, link); \ 421 chase = NULL; \ 422 succeed = 0; \ 423 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ 424 chase = curr; \ 425 if (curr != NULL) { \ 426 succeed = 1; \ 427 ASSERT(curr == el); \ 428 if (chase != NULL) \ 429 chase->link = curr->link; \ 430 else \ 431 head = curr->link; \ 432 curr->link = NULL; \ 433 if (curr == tail) \ 434 tail = chase; \ 435 } \ 436 LISTCHECK(head, tail, link); \ 437 } 438 439 /* Handling of delayed messages on the inner syncq. */ 440 441 /* 442 * DEBUG versions should use function versions (to simplify tracing) and 443 * non-DEBUG kernels should use macro versions. 444 */ 445 446 /* 447 * Put a queue on the syncq list of queues. 448 * Assumes SQLOCK held. 449 */ 450 #define SQPUT_Q(sq, qp) \ 451 { \ 452 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 453 if (!(qp->q_sqflags & Q_SQQUEUED)) { \ 454 /* The queue should not be linked anywhere */ \ 455 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ 456 /* Head and tail may only be NULL simultaneously */ \ 457 EQUIV(sq->sq_head, sq->sq_tail); \ 458 /* Queue may be only enqueyed on its syncq */ \ 459 ASSERT(sq == qp->q_syncq); \ 460 /* Check the correctness of SQ_MESSAGES flag */ \ 461 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ 462 /* Sanity check first/last elements of the list */ \ 463 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ 464 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ 465 /* \ 466 * Sanity check of priority field: empty queue should \ 467 * have zero priority \ 468 * and nqueues equal to zero. \ 469 */ \ 470 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ 471 /* Sanity check of sq_nqueues field */ \ 472 EQUIV(sq->sq_head, sq->sq_nqueues); \ 473 if (sq->sq_head == NULL) { \ 474 sq->sq_head = sq->sq_tail = qp; \ 475 sq->sq_flags |= SQ_MESSAGES; \ 476 } else if (qp->q_spri == 0) { \ 477 qp->q_sqprev = sq->sq_tail; \ 478 sq->sq_tail->q_sqnext = qp; \ 479 sq->sq_tail = qp; \ 480 } else { \ 481 /* \ 482 * Put this queue in priority order: higher \ 483 * priority gets closer to the head. \ 484 */ \ 485 queue_t **qpp = &sq->sq_tail; \ 486 queue_t *qnext = NULL; \ 487 \ 488 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ 489 qnext = *qpp; \ 490 qpp = &(*qpp)->q_sqprev; \ 491 } \ 492 qp->q_sqnext = qnext; \ 493 qp->q_sqprev = *qpp; \ 494 if (*qpp != NULL) { \ 495 (*qpp)->q_sqnext = qp; \ 496 } else { \ 497 sq->sq_head = qp; \ 498 sq->sq_pri = sq->sq_head->q_spri; \ 499 } \ 500 *qpp = qp; \ 501 } \ 502 qp->q_sqflags |= Q_SQQUEUED; \ 503 qp->q_sqtstamp = lbolt; \ 504 sq->sq_nqueues++; \ 505 } \ 506 } 507 508 /* 509 * Remove a queue from the syncq list 510 * Assumes SQLOCK held. 511 */ 512 #define SQRM_Q(sq, qp) \ 513 { \ 514 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 515 ASSERT(qp->q_sqflags & Q_SQQUEUED); \ 516 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ 517 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ 518 /* Check that the queue is actually in the list */ \ 519 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ 520 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ 521 ASSERT(sq->sq_nqueues != 0); \ 522 if (qp->q_sqprev == NULL) { \ 523 /* First queue on list, make head q_sqnext */ \ 524 sq->sq_head = qp->q_sqnext; \ 525 } else { \ 526 /* Make prev->next == next */ \ 527 qp->q_sqprev->q_sqnext = qp->q_sqnext; \ 528 } \ 529 if (qp->q_sqnext == NULL) { \ 530 /* Last queue on list, make tail sqprev */ \ 531 sq->sq_tail = qp->q_sqprev; \ 532 } else { \ 533 /* Make next->prev == prev */ \ 534 qp->q_sqnext->q_sqprev = qp->q_sqprev; \ 535 } \ 536 /* clear out references on this queue */ \ 537 qp->q_sqprev = qp->q_sqnext = NULL; \ 538 qp->q_sqflags &= ~Q_SQQUEUED; \ 539 /* If there is nothing queued, clear SQ_MESSAGES */ \ 540 if (sq->sq_head != NULL) { \ 541 sq->sq_pri = sq->sq_head->q_spri; \ 542 } else { \ 543 sq->sq_flags &= ~SQ_MESSAGES; \ 544 sq->sq_pri = 0; \ 545 } \ 546 sq->sq_nqueues--; \ 547 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ 548 (sq->sq_flags & SQ_QUEUED) == 0); \ 549 } 550 551 /* Hide the definition from the header file. */ 552 #ifdef SQPUT_MP 553 #undef SQPUT_MP 554 #endif 555 556 /* 557 * Put a message on the queue syncq. 558 * Assumes QLOCK held. 559 */ 560 #define SQPUT_MP(qp, mp) \ 561 { \ 562 ASSERT(MUTEX_HELD(QLOCK(qp))); \ 563 ASSERT(qp->q_sqhead == NULL || \ 564 (qp->q_sqtail != NULL && \ 565 qp->q_sqtail->b_next == NULL)); \ 566 qp->q_syncqmsgs++; \ 567 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ 568 if (qp->q_sqhead == NULL) { \ 569 qp->q_sqhead = qp->q_sqtail = mp; \ 570 } else { \ 571 qp->q_sqtail->b_next = mp; \ 572 qp->q_sqtail = mp; \ 573 } \ 574 ASSERT(qp->q_syncqmsgs > 0); \ 575 } 576 577 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ 578 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 579 if ((sq)->sq_ciputctrl != NULL) { \ 580 int i; \ 581 int nlocks = (sq)->sq_nciputctrl; \ 582 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 583 ASSERT((sq)->sq_type & SQ_CIPUT); \ 584 for (i = 0; i <= nlocks; i++) { \ 585 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 586 cip[i].ciputctrl_count |= SQ_FASTPUT; \ 587 } \ 588 } \ 589 } 590 591 592 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ 593 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 594 if ((sq)->sq_ciputctrl != NULL) { \ 595 int i; \ 596 int nlocks = (sq)->sq_nciputctrl; \ 597 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 598 ASSERT((sq)->sq_type & SQ_CIPUT); \ 599 for (i = 0; i <= nlocks; i++) { \ 600 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 601 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ 602 } \ 603 } \ 604 } 605 606 /* 607 * Run service procedures for all queues in the stream head. 608 */ 609 #define STR_SERVICE(stp, q) { \ 610 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ 611 while (stp->sd_qhead != NULL) { \ 612 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ 613 ASSERT(stp->sd_nqueues > 0); \ 614 stp->sd_nqueues--; \ 615 ASSERT(!(q->q_flag & QINSERVICE)); \ 616 mutex_exit(&stp->sd_qlock); \ 617 queue_service(q); \ 618 mutex_enter(&stp->sd_qlock); \ 619 } \ 620 ASSERT(stp->sd_nqueues == 0); \ 621 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ 622 } 623 624 /* 625 * constructor/destructor routines for the stream head cache 626 */ 627 /* ARGSUSED */ 628 static int 629 stream_head_constructor(void *buf, void *cdrarg, int kmflags) 630 { 631 stdata_t *stp = buf; 632 633 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); 634 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); 635 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); 636 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); 637 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); 638 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); 639 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); 640 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); 641 stp->sd_wrq = NULL; 642 643 return (0); 644 } 645 646 /* ARGSUSED */ 647 static void 648 stream_head_destructor(void *buf, void *cdrarg) 649 { 650 stdata_t *stp = buf; 651 652 mutex_destroy(&stp->sd_lock); 653 mutex_destroy(&stp->sd_reflock); 654 mutex_destroy(&stp->sd_qlock); 655 cv_destroy(&stp->sd_monitor); 656 cv_destroy(&stp->sd_iocmonitor); 657 cv_destroy(&stp->sd_refmonitor); 658 cv_destroy(&stp->sd_qcv); 659 cv_destroy(&stp->sd_zcopy_wait); 660 } 661 662 /* 663 * constructor/destructor routines for the queue cache 664 */ 665 /* ARGSUSED */ 666 static int 667 queue_constructor(void *buf, void *cdrarg, int kmflags) 668 { 669 queinfo_t *qip = buf; 670 queue_t *qp = &qip->qu_rqueue; 671 queue_t *wqp = &qip->qu_wqueue; 672 syncq_t *sq = &qip->qu_syncq; 673 674 qp->q_first = NULL; 675 qp->q_link = NULL; 676 qp->q_count = 0; 677 qp->q_mblkcnt = 0; 678 qp->q_sqhead = NULL; 679 qp->q_sqtail = NULL; 680 qp->q_sqnext = NULL; 681 qp->q_sqprev = NULL; 682 qp->q_sqflags = 0; 683 qp->q_rwcnt = 0; 684 qp->q_spri = 0; 685 686 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); 687 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); 688 689 wqp->q_first = NULL; 690 wqp->q_link = NULL; 691 wqp->q_count = 0; 692 wqp->q_mblkcnt = 0; 693 wqp->q_sqhead = NULL; 694 wqp->q_sqtail = NULL; 695 wqp->q_sqnext = NULL; 696 wqp->q_sqprev = NULL; 697 wqp->q_sqflags = 0; 698 wqp->q_rwcnt = 0; 699 wqp->q_spri = 0; 700 701 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); 702 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); 703 704 sq->sq_head = NULL; 705 sq->sq_tail = NULL; 706 sq->sq_evhead = NULL; 707 sq->sq_evtail = NULL; 708 sq->sq_callbpend = NULL; 709 sq->sq_outer = NULL; 710 sq->sq_onext = NULL; 711 sq->sq_oprev = NULL; 712 sq->sq_next = NULL; 713 sq->sq_svcflags = 0; 714 sq->sq_servcount = 0; 715 sq->sq_needexcl = 0; 716 sq->sq_nqueues = 0; 717 sq->sq_pri = 0; 718 719 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 720 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 721 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 722 723 return (0); 724 } 725 726 /* ARGSUSED */ 727 static void 728 queue_destructor(void *buf, void *cdrarg) 729 { 730 queinfo_t *qip = buf; 731 queue_t *qp = &qip->qu_rqueue; 732 queue_t *wqp = &qip->qu_wqueue; 733 syncq_t *sq = &qip->qu_syncq; 734 735 ASSERT(qp->q_sqhead == NULL); 736 ASSERT(wqp->q_sqhead == NULL); 737 ASSERT(qp->q_sqnext == NULL); 738 ASSERT(wqp->q_sqnext == NULL); 739 ASSERT(qp->q_rwcnt == 0); 740 ASSERT(wqp->q_rwcnt == 0); 741 742 mutex_destroy(&qp->q_lock); 743 cv_destroy(&qp->q_wait); 744 745 mutex_destroy(&wqp->q_lock); 746 cv_destroy(&wqp->q_wait); 747 748 mutex_destroy(&sq->sq_lock); 749 cv_destroy(&sq->sq_wait); 750 cv_destroy(&sq->sq_exitwait); 751 } 752 753 /* 754 * constructor/destructor routines for the syncq cache 755 */ 756 /* ARGSUSED */ 757 static int 758 syncq_constructor(void *buf, void *cdrarg, int kmflags) 759 { 760 syncq_t *sq = buf; 761 762 bzero(buf, sizeof (syncq_t)); 763 764 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 765 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 766 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 767 768 return (0); 769 } 770 771 /* ARGSUSED */ 772 static void 773 syncq_destructor(void *buf, void *cdrarg) 774 { 775 syncq_t *sq = buf; 776 777 ASSERT(sq->sq_head == NULL); 778 ASSERT(sq->sq_tail == NULL); 779 ASSERT(sq->sq_evhead == NULL); 780 ASSERT(sq->sq_evtail == NULL); 781 ASSERT(sq->sq_callbpend == NULL); 782 ASSERT(sq->sq_callbflags == 0); 783 ASSERT(sq->sq_outer == NULL); 784 ASSERT(sq->sq_onext == NULL); 785 ASSERT(sq->sq_oprev == NULL); 786 ASSERT(sq->sq_next == NULL); 787 ASSERT(sq->sq_needexcl == 0); 788 ASSERT(sq->sq_svcflags == 0); 789 ASSERT(sq->sq_servcount == 0); 790 ASSERT(sq->sq_nqueues == 0); 791 ASSERT(sq->sq_pri == 0); 792 ASSERT(sq->sq_count == 0); 793 ASSERT(sq->sq_rmqcount == 0); 794 ASSERT(sq->sq_cancelid == 0); 795 ASSERT(sq->sq_ciputctrl == NULL); 796 ASSERT(sq->sq_nciputctrl == 0); 797 ASSERT(sq->sq_type == 0); 798 ASSERT(sq->sq_flags == 0); 799 800 mutex_destroy(&sq->sq_lock); 801 cv_destroy(&sq->sq_wait); 802 cv_destroy(&sq->sq_exitwait); 803 } 804 805 /* ARGSUSED */ 806 static int 807 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) 808 { 809 ciputctrl_t *cip = buf; 810 int i; 811 812 for (i = 0; i < n_ciputctrl; i++) { 813 cip[i].ciputctrl_count = SQ_FASTPUT; 814 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); 815 } 816 817 return (0); 818 } 819 820 /* ARGSUSED */ 821 static void 822 ciputctrl_destructor(void *buf, void *cdrarg) 823 { 824 ciputctrl_t *cip = buf; 825 int i; 826 827 for (i = 0; i < n_ciputctrl; i++) { 828 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); 829 mutex_destroy(&cip[i].ciputctrl_lock); 830 } 831 } 832 833 /* 834 * Init routine run from main at boot time. 835 */ 836 void 837 strinit(void) 838 { 839 int i; 840 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 841 842 /* 843 * Set up mux_node structures. 844 */ 845 mux_nodes = kmem_zalloc((sizeof (struct mux_node) * devcnt), KM_SLEEP); 846 for (i = 0; i < devcnt; i++) 847 mux_nodes[i].mn_imaj = i; 848 849 stream_head_cache = kmem_cache_create("stream_head_cache", 850 sizeof (stdata_t), 0, 851 stream_head_constructor, stream_head_destructor, NULL, 852 NULL, NULL, 0); 853 854 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, 855 queue_constructor, queue_destructor, NULL, NULL, NULL, 0); 856 857 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, 858 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); 859 860 qband_cache = kmem_cache_create("qband_cache", 861 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 862 863 linkinfo_cache = kmem_cache_create("linkinfo_cache", 864 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 865 866 n_ciputctrl = ncpus; 867 n_ciputctrl = 1 << highbit(n_ciputctrl - 1); 868 ASSERT(n_ciputctrl >= 1); 869 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); 870 if (n_ciputctrl >= min_n_ciputctrl) { 871 ciputctrl_cache = kmem_cache_create("ciputctrl_cache", 872 sizeof (ciputctrl_t) * n_ciputctrl, 873 sizeof (ciputctrl_t), ciputctrl_constructor, 874 ciputctrl_destructor, NULL, NULL, NULL, 0); 875 } 876 877 streams_taskq = system_taskq; 878 879 if (streams_taskq == NULL) 880 panic("strinit: no memory for streams taskq!"); 881 882 bc_bkgrnd_thread = thread_create(NULL, 0, 883 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); 884 885 streams_qbkgrnd_thread = thread_create(NULL, 0, 886 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 887 888 streams_sqbkgrnd_thread = thread_create(NULL, 0, 889 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 890 891 /* 892 * Create STREAMS kstats. 893 */ 894 str_kstat = kstat_create("streams", 0, "strstat", 895 "net", KSTAT_TYPE_NAMED, 896 sizeof (str_statistics) / sizeof (kstat_named_t), 897 KSTAT_FLAG_VIRTUAL); 898 899 if (str_kstat != NULL) { 900 str_kstat->ks_data = &str_statistics; 901 kstat_install(str_kstat); 902 } 903 904 /* 905 * TPI support routine initialisation. 906 */ 907 tpi_init(); 908 } 909 910 void 911 str_sendsig(vnode_t *vp, int event, uchar_t band, int error) 912 { 913 struct stdata *stp; 914 915 ASSERT(vp->v_stream); 916 stp = vp->v_stream; 917 /* Have to hold sd_lock to prevent siglist from changing */ 918 mutex_enter(&stp->sd_lock); 919 if (stp->sd_sigflags & event) 920 strsendsig(stp->sd_siglist, event, band, error); 921 mutex_exit(&stp->sd_lock); 922 } 923 924 /* 925 * Send the "sevent" set of signals to a process. 926 * This might send more than one signal if the process is registered 927 * for multiple events. The caller should pass in an sevent that only 928 * includes the events for which the process has registered. 929 */ 930 static void 931 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, 932 uchar_t band, int error) 933 { 934 ASSERT(MUTEX_HELD(&proc->p_lock)); 935 936 info->si_band = 0; 937 info->si_errno = 0; 938 939 if (sevent & S_ERROR) { 940 sevent &= ~S_ERROR; 941 info->si_code = POLL_ERR; 942 info->si_errno = error; 943 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 944 "strsendsig:proc %p info %p", proc, info); 945 sigaddq(proc, NULL, info, KM_NOSLEEP); 946 info->si_errno = 0; 947 } 948 if (sevent & S_HANGUP) { 949 sevent &= ~S_HANGUP; 950 info->si_code = POLL_HUP; 951 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 952 "strsendsig:proc %p info %p", proc, info); 953 sigaddq(proc, NULL, info, KM_NOSLEEP); 954 } 955 if (sevent & S_HIPRI) { 956 sevent &= ~S_HIPRI; 957 info->si_code = POLL_PRI; 958 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 959 "strsendsig:proc %p info %p", proc, info); 960 sigaddq(proc, NULL, info, KM_NOSLEEP); 961 } 962 if (sevent & S_RDBAND) { 963 sevent &= ~S_RDBAND; 964 if (events & S_BANDURG) 965 sigtoproc(proc, NULL, SIGURG); 966 else 967 sigtoproc(proc, NULL, SIGPOLL); 968 } 969 if (sevent & S_WRBAND) { 970 sevent &= ~S_WRBAND; 971 sigtoproc(proc, NULL, SIGPOLL); 972 } 973 if (sevent & S_INPUT) { 974 sevent &= ~S_INPUT; 975 info->si_code = POLL_IN; 976 info->si_band = band; 977 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 978 "strsendsig:proc %p info %p", proc, info); 979 sigaddq(proc, NULL, info, KM_NOSLEEP); 980 info->si_band = 0; 981 } 982 if (sevent & S_OUTPUT) { 983 sevent &= ~S_OUTPUT; 984 info->si_code = POLL_OUT; 985 info->si_band = band; 986 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 987 "strsendsig:proc %p info %p", proc, info); 988 sigaddq(proc, NULL, info, KM_NOSLEEP); 989 info->si_band = 0; 990 } 991 if (sevent & S_MSG) { 992 sevent &= ~S_MSG; 993 info->si_code = POLL_MSG; 994 info->si_band = band; 995 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 996 "strsendsig:proc %p info %p", proc, info); 997 sigaddq(proc, NULL, info, KM_NOSLEEP); 998 info->si_band = 0; 999 } 1000 if (sevent & S_RDNORM) { 1001 sevent &= ~S_RDNORM; 1002 sigtoproc(proc, NULL, SIGPOLL); 1003 } 1004 if (sevent != 0) { 1005 panic("strsendsig: unknown event(s) %x", sevent); 1006 } 1007 } 1008 1009 /* 1010 * Send SIGPOLL/SIGURG signal to all processes and process groups 1011 * registered on the given signal list that want a signal for at 1012 * least one of the specified events. 1013 * 1014 * Must be called with exclusive access to siglist (caller holding sd_lock). 1015 * 1016 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding 1017 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure 1018 * while it is in the siglist. 1019 * 1020 * For performance reasons (MP scalability) the code drops pidlock 1021 * when sending signals to a single process. 1022 * When sending to a process group the code holds 1023 * pidlock to prevent the membership in the process group from changing 1024 * while walking the p_pglink list. 1025 */ 1026 void 1027 strsendsig(strsig_t *siglist, int event, uchar_t band, int error) 1028 { 1029 strsig_t *ssp; 1030 k_siginfo_t info; 1031 struct pid *pidp; 1032 proc_t *proc; 1033 1034 info.si_signo = SIGPOLL; 1035 info.si_errno = 0; 1036 for (ssp = siglist; ssp; ssp = ssp->ss_next) { 1037 int sevent; 1038 1039 sevent = ssp->ss_events & event; 1040 if (sevent == 0) 1041 continue; 1042 1043 if ((pidp = ssp->ss_pidp) == NULL) { 1044 /* pid was released but still on event list */ 1045 continue; 1046 } 1047 1048 1049 if (ssp->ss_pid > 0) { 1050 /* 1051 * XXX This unfortunately still generates 1052 * a signal when a fd is closed but 1053 * the proc is active. 1054 */ 1055 ASSERT(ssp->ss_pid == pidp->pid_id); 1056 1057 mutex_enter(&pidlock); 1058 proc = prfind_zone(pidp->pid_id, ALL_ZONES); 1059 if (proc == NULL) { 1060 mutex_exit(&pidlock); 1061 continue; 1062 } 1063 mutex_enter(&proc->p_lock); 1064 mutex_exit(&pidlock); 1065 dosendsig(proc, ssp->ss_events, sevent, &info, 1066 band, error); 1067 mutex_exit(&proc->p_lock); 1068 } else { 1069 /* 1070 * Send to process group. Hold pidlock across 1071 * calls to dosendsig(). 1072 */ 1073 pid_t pgrp = -ssp->ss_pid; 1074 1075 mutex_enter(&pidlock); 1076 proc = pgfind_zone(pgrp, ALL_ZONES); 1077 while (proc != NULL) { 1078 mutex_enter(&proc->p_lock); 1079 dosendsig(proc, ssp->ss_events, sevent, 1080 &info, band, error); 1081 mutex_exit(&proc->p_lock); 1082 proc = proc->p_pglink; 1083 } 1084 mutex_exit(&pidlock); 1085 } 1086 } 1087 } 1088 1089 /* 1090 * Attach a stream device or module. 1091 * qp is a read queue; the new queue goes in so its next 1092 * read ptr is the argument, and the write queue corresponding 1093 * to the argument points to this queue. Return 0 on success, 1094 * or a non-zero errno on failure. 1095 */ 1096 int 1097 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, 1098 boolean_t is_insert) 1099 { 1100 major_t major; 1101 cdevsw_impl_t *dp; 1102 struct streamtab *str; 1103 queue_t *rq; 1104 queue_t *wrq; 1105 uint32_t qflag; 1106 uint32_t sqtype; 1107 perdm_t *dmp; 1108 int error; 1109 int sflag; 1110 1111 rq = allocq(); 1112 wrq = _WR(rq); 1113 STREAM(rq) = STREAM(wrq) = STREAM(qp); 1114 1115 if (fp != NULL) { 1116 str = fp->f_str; 1117 qflag = fp->f_qflag; 1118 sqtype = fp->f_sqtype; 1119 dmp = fp->f_dmp; 1120 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 1121 sflag = MODOPEN; 1122 1123 /* 1124 * stash away a pointer to the module structure so we can 1125 * unref it in qdetach. 1126 */ 1127 rq->q_fp = fp; 1128 } else { 1129 ASSERT(!is_insert); 1130 1131 major = getmajor(*devp); 1132 dp = &devimpl[major]; 1133 1134 str = dp->d_str; 1135 ASSERT(str == STREAMSTAB(major)); 1136 1137 qflag = dp->d_qflag; 1138 ASSERT(qflag & QISDRV); 1139 sqtype = dp->d_sqtype; 1140 1141 /* create perdm_t if needed */ 1142 if (NEED_DM(dp->d_dmp, qflag)) 1143 dp->d_dmp = hold_dm(str, qflag, sqtype); 1144 1145 dmp = dp->d_dmp; 1146 sflag = 0; 1147 } 1148 1149 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, 1150 "qattach:qflag == %X(%X)", qflag, *devp); 1151 1152 /* setq might sleep in allocator - avoid holding locks. */ 1153 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); 1154 1155 /* 1156 * Before calling the module's open routine, set up the q_next 1157 * pointer for inserting a module in the middle of a stream. 1158 * 1159 * Note that we can always set _QINSERTING and set up q_next 1160 * pointer for both inserting and pushing a module. Then there 1161 * is no need for the is_insert parameter. In insertq(), called 1162 * by qprocson(), assume that q_next of the new module always points 1163 * to the correct queue and use it for insertion. Everything should 1164 * work out fine. But in the first release of _I_INSERT, we 1165 * distinguish between inserting and pushing to make sure that 1166 * pushing a module follows the same code path as before. 1167 */ 1168 if (is_insert) { 1169 rq->q_flag |= _QINSERTING; 1170 rq->q_next = qp; 1171 } 1172 1173 /* 1174 * If there is an outer perimeter get exclusive access during 1175 * the open procedure. Bump up the reference count on the queue. 1176 */ 1177 entersq(rq->q_syncq, SQ_OPENCLOSE); 1178 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); 1179 if (error != 0) 1180 goto failed; 1181 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1182 ASSERT(qprocsareon(rq)); 1183 return (0); 1184 1185 failed: 1186 rq->q_flag &= ~_QINSERTING; 1187 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) 1188 qprocsoff(rq); 1189 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1190 rq->q_next = wrq->q_next = NULL; 1191 qdetach(rq, 0, 0, crp, B_FALSE); 1192 return (error); 1193 } 1194 1195 /* 1196 * Handle second open of stream. For modules, set the 1197 * last argument to MODOPEN and do not pass any open flags. 1198 * Ignore dummydev since this is not the first open. 1199 */ 1200 int 1201 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) 1202 { 1203 int error; 1204 dev_t dummydev; 1205 queue_t *wqp = _WR(qp); 1206 1207 ASSERT(qp->q_flag & QREADR); 1208 entersq(qp->q_syncq, SQ_OPENCLOSE); 1209 1210 dummydev = *devp; 1211 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, 1212 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { 1213 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1214 mutex_enter(&STREAM(qp)->sd_lock); 1215 qp->q_stream->sd_flag |= STREOPENFAIL; 1216 mutex_exit(&STREAM(qp)->sd_lock); 1217 return (error); 1218 } 1219 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1220 1221 /* 1222 * successful open should have done qprocson() 1223 */ 1224 ASSERT(qprocsareon(_RD(qp))); 1225 return (0); 1226 } 1227 1228 /* 1229 * Detach a stream module or device. 1230 * If clmode == 1 then the module or driver was opened and its 1231 * close routine must be called. If clmode == 0, the module 1232 * or driver was never opened or the open failed, and so its close 1233 * should not be called. 1234 */ 1235 void 1236 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) 1237 { 1238 queue_t *wqp = _WR(qp); 1239 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); 1240 1241 if (STREAM_NEEDSERVICE(STREAM(qp))) 1242 stream_runservice(STREAM(qp)); 1243 1244 if (clmode) { 1245 /* 1246 * Make sure that all the messages on the write side syncq are 1247 * processed and nothing is left. Since we are closing, no new 1248 * messages may appear there. 1249 */ 1250 wait_q_syncq(wqp); 1251 1252 entersq(qp->q_syncq, SQ_OPENCLOSE); 1253 if (is_remove) { 1254 mutex_enter(QLOCK(qp)); 1255 qp->q_flag |= _QREMOVING; 1256 mutex_exit(QLOCK(qp)); 1257 } 1258 (*qp->q_qinfo->qi_qclose)(qp, flag, crp); 1259 /* 1260 * Check that qprocsoff() was actually called. 1261 */ 1262 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); 1263 1264 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1265 } else { 1266 disable_svc(qp); 1267 } 1268 1269 /* 1270 * Allow any threads blocked in entersq to proceed and discover 1271 * the QWCLOSE is set. 1272 * Note: This assumes that all users of entersq check QWCLOSE. 1273 * Currently runservice is the only entersq that can happen 1274 * after removeq has finished. 1275 * Removeq will have discarded all messages destined to the closing 1276 * pair of queues from the syncq. 1277 * NOTE: Calling a function inside an assert is unconventional. 1278 * However, it does not cause any problem since flush_syncq() does 1279 * not change any state except when it returns non-zero i.e. 1280 * when the assert will trigger. 1281 */ 1282 ASSERT(flush_syncq(qp->q_syncq, qp) == 0); 1283 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); 1284 ASSERT((qp->q_flag & QPERMOD) || 1285 ((qp->q_syncq->sq_head == NULL) && 1286 (wqp->q_syncq->sq_head == NULL))); 1287 1288 /* 1289 * Flush the queues before q_next is set to NULL. This is needed 1290 * in order to backenable any downstream queue before we go away. 1291 * Note: we are already removed from the stream so that the 1292 * backenabling will not cause any messages to be delivered to our 1293 * put procedures. 1294 */ 1295 flushq(qp, FLUSHALL); 1296 flushq(wqp, FLUSHALL); 1297 1298 /* 1299 * wait for any pending service processing to complete 1300 */ 1301 wait_svc(qp); 1302 1303 /* Tidy up - removeq only does a half-remove from stream */ 1304 qp->q_next = wqp->q_next = NULL; 1305 ASSERT(!(qp->q_flag & QENAB)); 1306 ASSERT(!(wqp->q_flag & QENAB)); 1307 1308 /* release any fmodsw_impl_t structure held on behalf of the queue */ 1309 1310 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); 1311 if (qp->q_fp != NULL) 1312 fmodsw_rele(qp->q_fp); 1313 1314 /* freeq removes us from the outer perimeter if any */ 1315 freeq(qp); 1316 } 1317 1318 /* Prevent service procedures from being called */ 1319 void 1320 disable_svc(queue_t *qp) 1321 { 1322 queue_t *wqp = _WR(qp); 1323 1324 ASSERT(qp->q_flag & QREADR); 1325 mutex_enter(QLOCK(qp)); 1326 qp->q_flag |= QWCLOSE; 1327 mutex_exit(QLOCK(qp)); 1328 mutex_enter(QLOCK(wqp)); 1329 wqp->q_flag |= QWCLOSE; 1330 mutex_exit(QLOCK(wqp)); 1331 } 1332 1333 /* allow service procedures to be called again */ 1334 void 1335 enable_svc(queue_t *qp) 1336 { 1337 queue_t *wqp = _WR(qp); 1338 1339 ASSERT(qp->q_flag & QREADR); 1340 mutex_enter(QLOCK(qp)); 1341 qp->q_flag &= ~QWCLOSE; 1342 mutex_exit(QLOCK(qp)); 1343 mutex_enter(QLOCK(wqp)); 1344 wqp->q_flag &= ~QWCLOSE; 1345 mutex_exit(QLOCK(wqp)); 1346 } 1347 1348 /* 1349 * Remove queue from qhead/qtail if it is enabled. 1350 * Only reset QENAB if the queue was removed from the runlist. 1351 * A queue goes through 3 stages: 1352 * It is on the service list and QENAB is set. 1353 * It is removed from the service list but QENAB is still set. 1354 * QENAB gets changed to QINSERVICE. 1355 * QINSERVICE is reset (when the service procedure is done) 1356 * Thus we can not reset QENAB unless we actually removed it from the service 1357 * queue. 1358 */ 1359 void 1360 remove_runlist(queue_t *qp) 1361 { 1362 if (qp->q_flag & QENAB && qhead != NULL) { 1363 queue_t *q_chase; 1364 queue_t *q_curr; 1365 int removed; 1366 1367 mutex_enter(&service_queue); 1368 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); 1369 mutex_exit(&service_queue); 1370 if (removed) { 1371 STRSTAT(qremoved); 1372 qp->q_flag &= ~QENAB; 1373 } 1374 } 1375 } 1376 1377 1378 /* 1379 * wait for any pending service processing to complete. 1380 * The removal of queues from the runlist is not atomic with the 1381 * clearing of the QENABLED flag and setting the INSERVICE flag. 1382 * consequently it is possible for remove_runlist in strclose 1383 * to not find the queue on the runlist but for it to be QENABLED 1384 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED 1385 * as well as INSERVICE. 1386 */ 1387 void 1388 wait_svc(queue_t *qp) 1389 { 1390 queue_t *wqp = _WR(qp); 1391 1392 ASSERT(qp->q_flag & QREADR); 1393 1394 /* 1395 * Try to remove queues from qhead/qtail list. 1396 */ 1397 if (qhead != NULL) { 1398 remove_runlist(qp); 1399 remove_runlist(wqp); 1400 } 1401 /* 1402 * Wait till the syncqs associated with the queue 1403 * will dissapear from background processing list. 1404 * This only needs to be done for non-PERMOD perimeters since 1405 * for PERMOD perimeters the syncq may be shared and will only be freed 1406 * when the last module/driver is unloaded. 1407 * If for PERMOD perimeters queue was on the syncq list, removeq() 1408 * should call propagate_syncq() or drain_syncq() for it. Both of these 1409 * function remove the queue from its syncq list, so sqthread will not 1410 * try to access the queue. 1411 */ 1412 if (!(qp->q_flag & QPERMOD)) { 1413 syncq_t *rsq = qp->q_syncq; 1414 syncq_t *wsq = wqp->q_syncq; 1415 1416 /* 1417 * Disable rsq and wsq and wait for any background processing of 1418 * syncq to complete. 1419 */ 1420 wait_sq_svc(rsq); 1421 if (wsq != rsq) 1422 wait_sq_svc(wsq); 1423 } 1424 1425 mutex_enter(QLOCK(qp)); 1426 while (qp->q_flag & (QINSERVICE|QENAB)) 1427 cv_wait(&qp->q_wait, QLOCK(qp)); 1428 mutex_exit(QLOCK(qp)); 1429 mutex_enter(QLOCK(wqp)); 1430 while (wqp->q_flag & (QINSERVICE|QENAB)) 1431 cv_wait(&wqp->q_wait, QLOCK(wqp)); 1432 mutex_exit(QLOCK(wqp)); 1433 } 1434 1435 /* 1436 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'. 1437 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may 1438 * also be set, and is passed through to allocb_cred_wait(). 1439 * 1440 * Returns errno on failure, zero on success. 1441 */ 1442 int 1443 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) 1444 { 1445 mblk_t *tmp; 1446 ssize_t count; 1447 size_t n; 1448 int error = 0; 1449 1450 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || 1451 (flag & (U_TO_K | K_TO_K)) == K_TO_K); 1452 1453 if (bp->b_datap->db_type == M_IOCTL) { 1454 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1455 } else { 1456 ASSERT(bp->b_datap->db_type == M_COPYIN); 1457 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1458 } 1459 /* 1460 * strdoioctl validates ioc_count, so if this assert fails it 1461 * cannot be due to user error. 1462 */ 1463 ASSERT(count >= 0); 1464 1465 while (count > 0) { 1466 n = MIN(MAXIOCBSZ, count); 1467 if ((tmp = allocb_cred_wait(n, (flag & STR_NOSIG), &error, 1468 cr)) == NULL) { 1469 return (error); 1470 } 1471 error = strcopyin(arg, tmp->b_wptr, n, flag & (U_TO_K|K_TO_K)); 1472 if (error != 0) { 1473 freeb(tmp); 1474 return (error); 1475 } 1476 arg += n; 1477 DB_CPID(tmp) = curproc->p_pid; 1478 tmp->b_wptr += n; 1479 count -= n; 1480 bp = (bp->b_cont = tmp); 1481 } 1482 1483 return (0); 1484 } 1485 1486 /* 1487 * Copy ioctl data to user-land. Return non-zero errno on failure, 1488 * 0 for success. 1489 */ 1490 int 1491 getiocd(mblk_t *bp, char *arg, int copymode) 1492 { 1493 ssize_t count; 1494 size_t n; 1495 int error; 1496 1497 if (bp->b_datap->db_type == M_IOCACK) 1498 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1499 else { 1500 ASSERT(bp->b_datap->db_type == M_COPYOUT); 1501 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1502 } 1503 ASSERT(count >= 0); 1504 1505 for (bp = bp->b_cont; bp && count; 1506 count -= n, bp = bp->b_cont, arg += n) { 1507 n = MIN(count, bp->b_wptr - bp->b_rptr); 1508 error = strcopyout(bp->b_rptr, arg, n, copymode); 1509 if (error) 1510 return (error); 1511 } 1512 ASSERT(count == 0); 1513 return (0); 1514 } 1515 1516 /* 1517 * Allocate a linkinfo entry given the write queue of the 1518 * bottom module of the top stream and the write queue of the 1519 * stream head of the bottom stream. 1520 */ 1521 linkinfo_t * 1522 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) 1523 { 1524 linkinfo_t *linkp; 1525 1526 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); 1527 1528 linkp->li_lblk.l_qtop = qup; 1529 linkp->li_lblk.l_qbot = qdown; 1530 linkp->li_fpdown = fpdown; 1531 1532 mutex_enter(&strresources); 1533 linkp->li_next = linkinfo_list; 1534 linkp->li_prev = NULL; 1535 if (linkp->li_next) 1536 linkp->li_next->li_prev = linkp; 1537 linkinfo_list = linkp; 1538 linkp->li_lblk.l_index = ++lnk_id; 1539 ASSERT(lnk_id != 0); /* this should never wrap in practice */ 1540 mutex_exit(&strresources); 1541 1542 return (linkp); 1543 } 1544 1545 /* 1546 * Free a linkinfo entry. 1547 */ 1548 void 1549 lbfree(linkinfo_t *linkp) 1550 { 1551 mutex_enter(&strresources); 1552 if (linkp->li_next) 1553 linkp->li_next->li_prev = linkp->li_prev; 1554 if (linkp->li_prev) 1555 linkp->li_prev->li_next = linkp->li_next; 1556 else 1557 linkinfo_list = linkp->li_next; 1558 mutex_exit(&strresources); 1559 1560 kmem_cache_free(linkinfo_cache, linkp); 1561 } 1562 1563 /* 1564 * Check for a potential linking cycle. 1565 * Return 1 if a link will result in a cycle, 1566 * and 0 otherwise. 1567 */ 1568 int 1569 linkcycle(stdata_t *upstp, stdata_t *lostp) 1570 { 1571 struct mux_node *np; 1572 struct mux_edge *ep; 1573 int i; 1574 major_t lomaj; 1575 major_t upmaj; 1576 /* 1577 * if the lower stream is a pipe/FIFO, return, since link 1578 * cycles can not happen on pipes/FIFOs 1579 */ 1580 if (lostp->sd_vnode->v_type == VFIFO) 1581 return (0); 1582 1583 for (i = 0; i < devcnt; i++) { 1584 np = &mux_nodes[i]; 1585 MUX_CLEAR(np); 1586 } 1587 lomaj = getmajor(lostp->sd_vnode->v_rdev); 1588 upmaj = getmajor(upstp->sd_vnode->v_rdev); 1589 np = &mux_nodes[lomaj]; 1590 for (;;) { 1591 if (!MUX_DIDVISIT(np)) { 1592 if (np->mn_imaj == upmaj) 1593 return (1); 1594 if (np->mn_outp == NULL) { 1595 MUX_VISIT(np); 1596 if (np->mn_originp == NULL) 1597 return (0); 1598 np = np->mn_originp; 1599 continue; 1600 } 1601 MUX_VISIT(np); 1602 np->mn_startp = np->mn_outp; 1603 } else { 1604 if (np->mn_startp == NULL) { 1605 if (np->mn_originp == NULL) 1606 return (0); 1607 else { 1608 np = np->mn_originp; 1609 continue; 1610 } 1611 } 1612 /* 1613 * If ep->me_nodep is a FIFO (me_nodep == NULL), 1614 * ignore the edge and move on. ep->me_nodep gets 1615 * set to NULL in mux_addedge() if it is a FIFO. 1616 * 1617 */ 1618 ep = np->mn_startp; 1619 np->mn_startp = ep->me_nextp; 1620 if (ep->me_nodep == NULL) 1621 continue; 1622 ep->me_nodep->mn_originp = np; 1623 np = ep->me_nodep; 1624 } 1625 } 1626 } 1627 1628 /* 1629 * Find linkinfo entry corresponding to the parameters. 1630 */ 1631 linkinfo_t * 1632 findlinks(stdata_t *stp, int index, int type) 1633 { 1634 linkinfo_t *linkp; 1635 struct mux_edge *mep; 1636 struct mux_node *mnp; 1637 queue_t *qup; 1638 1639 mutex_enter(&strresources); 1640 if ((type & LINKTYPEMASK) == LINKNORMAL) { 1641 qup = getendq(stp->sd_wrq); 1642 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1643 if ((qup == linkp->li_lblk.l_qtop) && 1644 (!index || (index == linkp->li_lblk.l_index))) { 1645 mutex_exit(&strresources); 1646 return (linkp); 1647 } 1648 } 1649 } else { 1650 ASSERT((type & LINKTYPEMASK) == LINKPERSIST); 1651 mnp = &mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; 1652 mep = mnp->mn_outp; 1653 while (mep) { 1654 if ((index == 0) || (index == mep->me_muxid)) 1655 break; 1656 mep = mep->me_nextp; 1657 } 1658 if (!mep) { 1659 mutex_exit(&strresources); 1660 return (NULL); 1661 } 1662 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1663 if ((!linkp->li_lblk.l_qtop) && 1664 (mep->me_muxid == linkp->li_lblk.l_index)) { 1665 mutex_exit(&strresources); 1666 return (linkp); 1667 } 1668 } 1669 } 1670 mutex_exit(&strresources); 1671 return (NULL); 1672 } 1673 1674 /* 1675 * Given a queue ptr, follow the chain of q_next pointers until you reach the 1676 * last queue on the chain and return it. 1677 */ 1678 queue_t * 1679 getendq(queue_t *q) 1680 { 1681 ASSERT(q != NULL); 1682 while (_SAMESTR(q)) 1683 q = q->q_next; 1684 return (q); 1685 } 1686 1687 /* 1688 * wait for the syncq count to drop to zero. 1689 * sq could be either outer or inner. 1690 */ 1691 1692 static void 1693 wait_syncq(syncq_t *sq) 1694 { 1695 uint16_t count; 1696 1697 mutex_enter(SQLOCK(sq)); 1698 count = sq->sq_count; 1699 SQ_PUTLOCKS_ENTER(sq); 1700 SUM_SQ_PUTCOUNTS(sq, count); 1701 while (count != 0) { 1702 sq->sq_flags |= SQ_WANTWAKEUP; 1703 SQ_PUTLOCKS_EXIT(sq); 1704 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1705 count = sq->sq_count; 1706 SQ_PUTLOCKS_ENTER(sq); 1707 SUM_SQ_PUTCOUNTS(sq, count); 1708 } 1709 SQ_PUTLOCKS_EXIT(sq); 1710 mutex_exit(SQLOCK(sq)); 1711 } 1712 1713 /* 1714 * Wait while there are any messages for the queue in its syncq. 1715 */ 1716 static void 1717 wait_q_syncq(queue_t *q) 1718 { 1719 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1720 syncq_t *sq = q->q_syncq; 1721 1722 mutex_enter(SQLOCK(sq)); 1723 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1724 sq->sq_flags |= SQ_WANTWAKEUP; 1725 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1726 } 1727 mutex_exit(SQLOCK(sq)); 1728 } 1729 } 1730 1731 1732 int 1733 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, 1734 int lhlink) 1735 { 1736 struct stdata *stp; 1737 struct strioctl strioc; 1738 struct linkinfo *linkp; 1739 struct stdata *stpdown; 1740 struct streamtab *str; 1741 queue_t *passq; 1742 syncq_t *passyncq; 1743 queue_t *rq; 1744 cdevsw_impl_t *dp; 1745 uint32_t qflag; 1746 uint32_t sqtype; 1747 perdm_t *dmp; 1748 int error = 0; 1749 1750 stp = vp->v_stream; 1751 TRACE_1(TR_FAC_STREAMS_FR, 1752 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); 1753 /* 1754 * Test for invalid upper stream 1755 */ 1756 if (stp->sd_flag & STRHUP) { 1757 return (ENXIO); 1758 } 1759 if (vp->v_type == VFIFO) { 1760 return (EINVAL); 1761 } 1762 if (stp->sd_strtab == NULL) { 1763 return (EINVAL); 1764 } 1765 if (!stp->sd_strtab->st_muxwinit) { 1766 return (EINVAL); 1767 } 1768 if (fpdown == NULL) { 1769 return (EBADF); 1770 } 1771 if (getmajor(stp->sd_vnode->v_rdev) >= devcnt) { 1772 return (EINVAL); 1773 } 1774 mutex_enter(&muxifier); 1775 if (stp->sd_flag & STPLEX) { 1776 mutex_exit(&muxifier); 1777 return (ENXIO); 1778 } 1779 1780 /* 1781 * Test for invalid lower stream. 1782 * The check for the v_type != VFIFO and having a major 1783 * number not >= devcnt is done to avoid problems with 1784 * adding mux_node entry past the end of mux_nodes[]. 1785 * For FIFO's we don't add an entry so this isn't a 1786 * problem. 1787 */ 1788 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || 1789 (stpdown == stp) || (stpdown->sd_flag & 1790 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || 1791 ((stpdown->sd_vnode->v_type != VFIFO) && 1792 (getmajor(stpdown->sd_vnode->v_rdev) >= devcnt)) || 1793 linkcycle(stp, stpdown)) { 1794 mutex_exit(&muxifier); 1795 return (EINVAL); 1796 } 1797 TRACE_1(TR_FAC_STREAMS_FR, 1798 TR_STPDOWN, "stpdown:%p", stpdown); 1799 rq = getendq(stp->sd_wrq); 1800 if (cmd == I_PLINK) 1801 rq = NULL; 1802 1803 linkp = alloclink(rq, stpdown->sd_wrq, fpdown); 1804 1805 strioc.ic_cmd = cmd; 1806 strioc.ic_timout = INFTIM; 1807 strioc.ic_len = sizeof (struct linkblk); 1808 strioc.ic_dp = (char *)&linkp->li_lblk; 1809 1810 /* 1811 * STRPLUMB protects plumbing changes and should be set before 1812 * link_addpassthru()/link_rempassthru() are called, so it is set here 1813 * and cleared in the end of mlink when passthru queue is removed. 1814 * Setting of STRPLUMB prevents reopens of the stream while passthru 1815 * queue is in-place (it is not a proper module and doesn't have open 1816 * entry point). 1817 * 1818 * STPLEX prevents any threads from entering the stream from above. It 1819 * can't be set before the call to link_addpassthru() because putnext 1820 * from below may cause stream head I/O routines to be called and these 1821 * routines assert that STPLEX is not set. After link_addpassthru() 1822 * nothing may come from below since the pass queue syncq is blocked. 1823 * Note also that STPLEX should be cleared before the call to 1824 * link_remmpassthru() since when messages start flowing to the stream 1825 * head (e.g. because of message propagation from the pass queue) stream 1826 * head I/O routines may be called with STPLEX flag set. 1827 * 1828 * When STPLEX is set, nothing may come into the stream from above and 1829 * it is safe to do a setq which will change stream head. So, the 1830 * correct sequence of actions is: 1831 * 1832 * 1) Set STRPLUMB 1833 * 2) Call link_addpassthru() 1834 * 3) Set STPLEX 1835 * 4) Call setq and update the stream state 1836 * 5) Clear STPLEX 1837 * 6) Call link_rempassthru() 1838 * 7) Clear STRPLUMB 1839 * 1840 * The same sequence applies to munlink() code. 1841 */ 1842 mutex_enter(&stpdown->sd_lock); 1843 stpdown->sd_flag |= STRPLUMB; 1844 mutex_exit(&stpdown->sd_lock); 1845 /* 1846 * Add passthru queue below lower mux. This will block 1847 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 1848 */ 1849 passq = link_addpassthru(stpdown); 1850 1851 mutex_enter(&stpdown->sd_lock); 1852 stpdown->sd_flag |= STPLEX; 1853 mutex_exit(&stpdown->sd_lock); 1854 1855 rq = _RD(stpdown->sd_wrq); 1856 /* 1857 * There may be messages in the streamhead's syncq due to messages 1858 * that arrived before link_addpassthru() was done. To avoid 1859 * background processing of the syncq happening simultaneous with 1860 * setq processing, we disable the streamhead syncq and wait until 1861 * existing background thread finishes working on it. 1862 */ 1863 wait_sq_svc(rq->q_syncq); 1864 passyncq = passq->q_syncq; 1865 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1866 blocksq(passyncq, SQ_BLOCKED, 0); 1867 1868 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 1869 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 1870 rq->q_ptr = _WR(rq)->q_ptr = NULL; 1871 1872 /* setq might sleep in allocator - avoid holding locks. */ 1873 /* Note: we are holding muxifier here. */ 1874 1875 str = stp->sd_strtab; 1876 dp = &devimpl[getmajor(vp->v_rdev)]; 1877 ASSERT(dp->d_str == str); 1878 1879 qflag = dp->d_qflag; 1880 sqtype = dp->d_sqtype; 1881 1882 /* create perdm_t if needed */ 1883 if (NEED_DM(dp->d_dmp, qflag)) 1884 dp->d_dmp = hold_dm(str, qflag, sqtype); 1885 1886 dmp = dp->d_dmp; 1887 1888 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, 1889 B_TRUE); 1890 1891 /* 1892 * XXX Remove any "odd" messages from the queue. 1893 * Keep only M_DATA, M_PROTO, M_PCPROTO. 1894 */ 1895 error = strdoioctl(stp, &strioc, FNATIVE, 1896 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 1897 if (error != 0) { 1898 lbfree(linkp); 1899 1900 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1901 blocksq(passyncq, SQ_BLOCKED, 0); 1902 /* 1903 * Restore the stream head queue and then remove 1904 * the passq. Turn off STPLEX before we turn on 1905 * the stream by removing the passq. 1906 */ 1907 rq->q_ptr = _WR(rq)->q_ptr = stpdown; 1908 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, 1909 B_TRUE); 1910 1911 mutex_enter(&stpdown->sd_lock); 1912 stpdown->sd_flag &= ~STPLEX; 1913 mutex_exit(&stpdown->sd_lock); 1914 1915 link_rempassthru(passq); 1916 1917 mutex_enter(&stpdown->sd_lock); 1918 stpdown->sd_flag &= ~STRPLUMB; 1919 /* Wakeup anyone waiting for STRPLUMB to clear. */ 1920 cv_broadcast(&stpdown->sd_monitor); 1921 mutex_exit(&stpdown->sd_lock); 1922 1923 mutex_exit(&muxifier); 1924 return (error); 1925 } 1926 mutex_enter(&fpdown->f_tlock); 1927 fpdown->f_count++; 1928 mutex_exit(&fpdown->f_tlock); 1929 1930 /* 1931 * if we've made it here the linkage is all set up so we should also 1932 * set up the layered driver linkages 1933 */ 1934 1935 ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); 1936 if (cmd == I_LINK) { 1937 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); 1938 } else { 1939 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); 1940 } 1941 1942 link_rempassthru(passq); 1943 1944 mux_addedge(stp, stpdown, linkp->li_lblk.l_index); 1945 1946 /* 1947 * Mark the upper stream as having dependent links 1948 * so that strclose can clean it up. 1949 */ 1950 if (cmd == I_LINK) { 1951 mutex_enter(&stp->sd_lock); 1952 stp->sd_flag |= STRHASLINKS; 1953 mutex_exit(&stp->sd_lock); 1954 } 1955 /* 1956 * Wake up any other processes that may have been 1957 * waiting on the lower stream. These will all 1958 * error out. 1959 */ 1960 mutex_enter(&stpdown->sd_lock); 1961 /* The passthru module is removed so we may release STRPLUMB */ 1962 stpdown->sd_flag &= ~STRPLUMB; 1963 cv_broadcast(&rq->q_wait); 1964 cv_broadcast(&_WR(rq)->q_wait); 1965 cv_broadcast(&stpdown->sd_monitor); 1966 mutex_exit(&stpdown->sd_lock); 1967 mutex_exit(&muxifier); 1968 *rvalp = linkp->li_lblk.l_index; 1969 return (0); 1970 } 1971 1972 int 1973 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) 1974 { 1975 int ret; 1976 struct file *fpdown; 1977 1978 fpdown = getf(arg); 1979 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); 1980 if (fpdown != NULL) 1981 releasef(arg); 1982 return (ret); 1983 } 1984 1985 /* 1986 * Unlink a multiplexor link. Stp is the controlling stream for the 1987 * link, and linkp points to the link's entry in the linkinfo list. 1988 * The muxifier lock must be held on entry and is dropped on exit. 1989 * 1990 * NOTE : Currently it is assumed that mux would process all the messages 1991 * sitting on it's queue before ACKing the UNLINK. It is the responsibility 1992 * of the mux to handle all the messages that arrive before UNLINK. 1993 * If the mux has to send down messages on its lower stream before 1994 * ACKing I_UNLINK, then it *should* know to handle messages even 1995 * after the UNLINK is acked (actually it should be able to handle till we 1996 * re-block the read side of the pass queue here). If the mux does not 1997 * open up the lower stream, any messages that arrive during UNLINK 1998 * will be put in the stream head. In the case of lower stream opening 1999 * up, some messages might land in the stream head depending on when 2000 * the message arrived and when the read side of the pass queue was 2001 * re-blocked. 2002 */ 2003 int 2004 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp) 2005 { 2006 struct strioctl strioc; 2007 struct stdata *stpdown; 2008 queue_t *rq, *wrq; 2009 queue_t *passq; 2010 syncq_t *passyncq; 2011 int error = 0; 2012 file_t *fpdown; 2013 2014 ASSERT(MUTEX_HELD(&muxifier)); 2015 2016 stpdown = linkp->li_fpdown->f_vnode->v_stream; 2017 2018 /* 2019 * See the comment in mlink() concerning STRPLUMB/STPLEX flags. 2020 */ 2021 mutex_enter(&stpdown->sd_lock); 2022 stpdown->sd_flag |= STRPLUMB; 2023 mutex_exit(&stpdown->sd_lock); 2024 2025 /* 2026 * Add passthru queue below lower mux. This will block 2027 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 2028 */ 2029 passq = link_addpassthru(stpdown); 2030 2031 if ((flag & LINKTYPEMASK) == LINKNORMAL) 2032 strioc.ic_cmd = I_UNLINK; 2033 else 2034 strioc.ic_cmd = I_PUNLINK; 2035 strioc.ic_timout = INFTIM; 2036 strioc.ic_len = sizeof (struct linkblk); 2037 strioc.ic_dp = (char *)&linkp->li_lblk; 2038 2039 error = strdoioctl(stp, &strioc, FNATIVE, 2040 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 2041 2042 /* 2043 * If there was an error and this is not called via strclose, 2044 * return to the user. Otherwise, pretend there was no error 2045 * and close the link. 2046 */ 2047 if (error) { 2048 if (flag & LINKCLOSE) { 2049 cmn_err(CE_WARN, "KERNEL: munlink: could not perform " 2050 "unlink ioctl, closing anyway (%d)\n", error); 2051 } else { 2052 link_rempassthru(passq); 2053 mutex_enter(&stpdown->sd_lock); 2054 stpdown->sd_flag &= ~STRPLUMB; 2055 cv_broadcast(&stpdown->sd_monitor); 2056 mutex_exit(&stpdown->sd_lock); 2057 mutex_exit(&muxifier); 2058 return (error); 2059 } 2060 } 2061 2062 mux_rmvedge(stp, linkp->li_lblk.l_index); 2063 fpdown = linkp->li_fpdown; 2064 lbfree(linkp); 2065 2066 /* 2067 * We go ahead and drop muxifier here--it's a nasty global lock that 2068 * can slow others down. It's okay to since attempts to mlink() this 2069 * stream will be stopped because STPLEX is still set in the stdata 2070 * structure, and munlink() is stopped because mux_rmvedge() and 2071 * lbfree() have removed it from mux_nodes[] and linkinfo_list, 2072 * respectively. Note that we defer the closef() of fpdown until 2073 * after we drop muxifier since strclose() can call munlinkall(). 2074 */ 2075 mutex_exit(&muxifier); 2076 2077 wrq = stpdown->sd_wrq; 2078 rq = _RD(wrq); 2079 2080 /* 2081 * Get rid of outstanding service procedure runs, before we make 2082 * it a stream head, since a stream head doesn't have any service 2083 * procedure. 2084 */ 2085 disable_svc(rq); 2086 wait_svc(rq); 2087 2088 /* 2089 * Since we don't disable the syncq for QPERMOD, we wait for whatever 2090 * is queued up to be finished. mux should take care that nothing is 2091 * send down to this queue. We should do it now as we're going to block 2092 * passyncq if it was unblocked. 2093 */ 2094 if (wrq->q_flag & QPERMOD) { 2095 syncq_t *sq = wrq->q_syncq; 2096 2097 mutex_enter(SQLOCK(sq)); 2098 while (wrq->q_sqflags & Q_SQQUEUED) { 2099 sq->sq_flags |= SQ_WANTWAKEUP; 2100 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2101 } 2102 mutex_exit(SQLOCK(sq)); 2103 } 2104 passyncq = passq->q_syncq; 2105 if (!(passyncq->sq_flags & SQ_BLOCKED)) { 2106 2107 syncq_t *sq, *outer; 2108 2109 /* 2110 * Messages could be flowing from underneath. We will 2111 * block the read side of the passq. This would be 2112 * sufficient for QPAIR and QPERQ muxes to ensure 2113 * that no data is flowing up into this queue 2114 * and hence no thread active in this instance of 2115 * lower mux. But for QPERMOD and QMTOUTPERIM there 2116 * could be messages on the inner and outer/inner 2117 * syncqs respectively. We will wait for them to drain. 2118 * Because passq is blocked messages end up in the syncq 2119 * And qfill_syncq could possibly end up setting QFULL 2120 * which will access the rq->q_flag. Hence, we have to 2121 * acquire the QLOCK in setq. 2122 * 2123 * XXX Messages can also flow from top into this 2124 * queue though the unlink is over (Ex. some instance 2125 * in putnext() called from top that has still not 2126 * accessed this queue. And also putq(lowerq) ?). 2127 * Solution : How about blocking the l_qtop queue ? 2128 * Do we really care about such pure D_MP muxes ? 2129 */ 2130 2131 blocksq(passyncq, SQ_BLOCKED, 0); 2132 2133 sq = rq->q_syncq; 2134 if ((outer = sq->sq_outer) != NULL) { 2135 2136 /* 2137 * We have to just wait for the outer sq_count 2138 * drop to zero. As this does not prevent new 2139 * messages to enter the outer perimeter, this 2140 * is subject to starvation. 2141 * 2142 * NOTE :Because of blocksq above, messages could 2143 * be in the inner syncq only because of some 2144 * thread holding the outer perimeter exclusively. 2145 * Hence it would be sufficient to wait for the 2146 * exclusive holder of the outer perimeter to drain 2147 * the inner and outer syncqs. But we will not depend 2148 * on this feature and hence check the inner syncqs 2149 * separately. 2150 */ 2151 wait_syncq(outer); 2152 } 2153 2154 2155 /* 2156 * There could be messages destined for 2157 * this queue. Let the exclusive holder 2158 * drain it. 2159 */ 2160 2161 wait_syncq(sq); 2162 ASSERT((rq->q_flag & QPERMOD) || 2163 ((rq->q_syncq->sq_head == NULL) && 2164 (_WR(rq)->q_syncq->sq_head == NULL))); 2165 } 2166 2167 /* 2168 * We haven't taken care of QPERMOD case yet. QPERMOD is a special 2169 * case as we don't disable its syncq or remove it off the syncq 2170 * service list. 2171 */ 2172 if (rq->q_flag & QPERMOD) { 2173 syncq_t *sq = rq->q_syncq; 2174 2175 mutex_enter(SQLOCK(sq)); 2176 while (rq->q_sqflags & Q_SQQUEUED) { 2177 sq->sq_flags |= SQ_WANTWAKEUP; 2178 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2179 } 2180 mutex_exit(SQLOCK(sq)); 2181 } 2182 2183 /* 2184 * flush_syncq changes states only when there is some messages to 2185 * free. ie when it returns non-zero value to return. 2186 */ 2187 ASSERT(flush_syncq(rq->q_syncq, rq) == 0); 2188 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); 2189 2190 /* 2191 * No body else should know about this queue now. 2192 * If the mux did not process the messages before 2193 * acking the I_UNLINK, free them now. 2194 */ 2195 2196 flushq(rq, FLUSHALL); 2197 flushq(_WR(rq), FLUSHALL); 2198 2199 /* 2200 * Convert the mux lower queue into a stream head queue. 2201 * Turn off STPLEX before we turn on the stream by removing the passq. 2202 */ 2203 rq->q_ptr = wrq->q_ptr = stpdown; 2204 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); 2205 2206 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 2207 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 2208 2209 enable_svc(rq); 2210 2211 /* 2212 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still 2213 * needs to be set to prevent reopen() of the stream - such reopen may 2214 * try to call non-existent pass queue open routine and panic. 2215 */ 2216 mutex_enter(&stpdown->sd_lock); 2217 stpdown->sd_flag &= ~STPLEX; 2218 mutex_exit(&stpdown->sd_lock); 2219 2220 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || 2221 ((flag & LINKTYPEMASK) == LINKPERSIST)); 2222 2223 /* clean up the layered driver linkages */ 2224 if ((flag & LINKTYPEMASK) == LINKNORMAL) { 2225 ldi_munlink_fp(stp, fpdown, LINKNORMAL); 2226 } else { 2227 ldi_munlink_fp(stp, fpdown, LINKPERSIST); 2228 } 2229 2230 link_rempassthru(passq); 2231 2232 /* 2233 * Now all plumbing changes are finished and STRPLUMB is no 2234 * longer needed. 2235 */ 2236 mutex_enter(&stpdown->sd_lock); 2237 stpdown->sd_flag &= ~STRPLUMB; 2238 cv_broadcast(&stpdown->sd_monitor); 2239 mutex_exit(&stpdown->sd_lock); 2240 2241 (void) closef(fpdown); 2242 return (0); 2243 } 2244 2245 /* 2246 * Unlink all multiplexor links for which stp is the controlling stream. 2247 * Return 0, or a non-zero errno on failure. 2248 */ 2249 int 2250 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp) 2251 { 2252 linkinfo_t *linkp; 2253 int error = 0; 2254 2255 mutex_enter(&muxifier); 2256 while (linkp = findlinks(stp, 0, flag)) { 2257 /* 2258 * munlink() releases the muxifier lock. 2259 */ 2260 if (error = munlink(stp, linkp, flag, crp, rvalp)) 2261 return (error); 2262 mutex_enter(&muxifier); 2263 } 2264 mutex_exit(&muxifier); 2265 return (0); 2266 } 2267 2268 /* 2269 * A multiplexor link has been made. Add an 2270 * edge to the directed graph. 2271 */ 2272 void 2273 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid) 2274 { 2275 struct mux_node *np; 2276 struct mux_edge *ep; 2277 major_t upmaj; 2278 major_t lomaj; 2279 2280 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2281 lomaj = getmajor(lostp->sd_vnode->v_rdev); 2282 np = &mux_nodes[upmaj]; 2283 if (np->mn_outp) { 2284 ep = np->mn_outp; 2285 while (ep->me_nextp) 2286 ep = ep->me_nextp; 2287 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2288 ep = ep->me_nextp; 2289 } else { 2290 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2291 ep = np->mn_outp; 2292 } 2293 ep->me_nextp = NULL; 2294 ep->me_muxid = muxid; 2295 if (lostp->sd_vnode->v_type == VFIFO) 2296 ep->me_nodep = NULL; 2297 else 2298 ep->me_nodep = &mux_nodes[lomaj]; 2299 } 2300 2301 /* 2302 * A multiplexor link has been removed. Remove the 2303 * edge in the directed graph. 2304 */ 2305 void 2306 mux_rmvedge(stdata_t *upstp, int muxid) 2307 { 2308 struct mux_node *np; 2309 struct mux_edge *ep; 2310 struct mux_edge *pep = NULL; 2311 major_t upmaj; 2312 2313 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2314 np = &mux_nodes[upmaj]; 2315 ASSERT(np->mn_outp != NULL); 2316 ep = np->mn_outp; 2317 while (ep) { 2318 if (ep->me_muxid == muxid) { 2319 if (pep) 2320 pep->me_nextp = ep->me_nextp; 2321 else 2322 np->mn_outp = ep->me_nextp; 2323 kmem_free(ep, sizeof (struct mux_edge)); 2324 return; 2325 } 2326 pep = ep; 2327 ep = ep->me_nextp; 2328 } 2329 ASSERT(0); /* should not reach here */ 2330 } 2331 2332 /* 2333 * Translate the device flags (from conf.h) to the corresponding 2334 * qflag and sq_flag (type) values. 2335 */ 2336 int 2337 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, 2338 uint32_t *sqtypep) 2339 { 2340 uint32_t qflag = 0; 2341 uint32_t sqtype = 0; 2342 2343 if (devflag & _D_OLD) 2344 goto bad; 2345 2346 /* Inner perimeter presence and scope */ 2347 switch (devflag & D_MTINNER_MASK) { 2348 case D_MP: 2349 qflag |= QMTSAFE; 2350 sqtype |= SQ_CI; 2351 break; 2352 case D_MTPERQ|D_MP: 2353 qflag |= QPERQ; 2354 break; 2355 case D_MTQPAIR|D_MP: 2356 qflag |= QPAIR; 2357 break; 2358 case D_MTPERMOD|D_MP: 2359 qflag |= QPERMOD; 2360 break; 2361 default: 2362 goto bad; 2363 } 2364 2365 /* Outer perimeter */ 2366 if (devflag & D_MTOUTPERIM) { 2367 switch (devflag & D_MTINNER_MASK) { 2368 case D_MP: 2369 case D_MTPERQ|D_MP: 2370 case D_MTQPAIR|D_MP: 2371 break; 2372 default: 2373 goto bad; 2374 } 2375 qflag |= QMTOUTPERIM; 2376 } 2377 2378 /* Inner perimeter modifiers */ 2379 if (devflag & D_MTINNER_MOD) { 2380 switch (devflag & D_MTINNER_MASK) { 2381 case D_MP: 2382 goto bad; 2383 default: 2384 break; 2385 } 2386 if (devflag & D_MTPUTSHARED) 2387 sqtype |= SQ_CIPUT; 2388 if (devflag & _D_MTOCSHARED) { 2389 /* 2390 * The code in putnext assumes that it has the 2391 * highest concurrency by not checking sq_count. 2392 * Thus _D_MTOCSHARED can only be supported when 2393 * D_MTPUTSHARED is set. 2394 */ 2395 if (!(devflag & D_MTPUTSHARED)) 2396 goto bad; 2397 sqtype |= SQ_CIOC; 2398 } 2399 if (devflag & _D_MTCBSHARED) { 2400 /* 2401 * The code in putnext assumes that it has the 2402 * highest concurrency by not checking sq_count. 2403 * Thus _D_MTCBSHARED can only be supported when 2404 * D_MTPUTSHARED is set. 2405 */ 2406 if (!(devflag & D_MTPUTSHARED)) 2407 goto bad; 2408 sqtype |= SQ_CICB; 2409 } 2410 if (devflag & _D_MTSVCSHARED) { 2411 /* 2412 * The code in putnext assumes that it has the 2413 * highest concurrency by not checking sq_count. 2414 * Thus _D_MTSVCSHARED can only be supported when 2415 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is 2416 * supported only for QPERMOD. 2417 */ 2418 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) 2419 goto bad; 2420 sqtype |= SQ_CISVC; 2421 } 2422 } 2423 2424 /* Default outer perimeter concurrency */ 2425 sqtype |= SQ_CO; 2426 2427 /* Outer perimeter modifiers */ 2428 if (devflag & D_MTOCEXCL) { 2429 if (!(devflag & D_MTOUTPERIM)) { 2430 /* No outer perimeter */ 2431 goto bad; 2432 } 2433 sqtype &= ~SQ_COOC; 2434 } 2435 2436 /* Synchronous Streams extended qinit structure */ 2437 if (devflag & D_SYNCSTR) 2438 qflag |= QSYNCSTR; 2439 2440 *qflagp = qflag; 2441 *sqtypep = sqtype; 2442 return (0); 2443 2444 bad: 2445 cmn_err(CE_WARN, 2446 "stropen: bad MT flags (0x%x) in driver '%s'", 2447 (int)(qflag & D_MTSAFETY_MASK), 2448 stp->st_rdinit->qi_minfo->mi_idname); 2449 2450 return (EINVAL); 2451 } 2452 2453 /* 2454 * Set the interface values for a pair of queues (qinit structure, 2455 * packet sizes, water marks). 2456 * setq assumes that the caller does not have a claim (entersq or claimq) 2457 * on the queue. 2458 */ 2459 void 2460 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, 2461 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) 2462 { 2463 queue_t *wq; 2464 syncq_t *sq, *outer; 2465 2466 ASSERT(rq->q_flag & QREADR); 2467 ASSERT((qflag & QMT_TYPEMASK) != 0); 2468 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 2469 2470 wq = _WR(rq); 2471 rq->q_qinfo = rinit; 2472 rq->q_hiwat = rinit->qi_minfo->mi_hiwat; 2473 rq->q_lowat = rinit->qi_minfo->mi_lowat; 2474 rq->q_minpsz = rinit->qi_minfo->mi_minpsz; 2475 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; 2476 wq->q_qinfo = winit; 2477 wq->q_hiwat = winit->qi_minfo->mi_hiwat; 2478 wq->q_lowat = winit->qi_minfo->mi_lowat; 2479 wq->q_minpsz = winit->qi_minfo->mi_minpsz; 2480 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; 2481 2482 /* Remove old syncqs */ 2483 sq = rq->q_syncq; 2484 outer = sq->sq_outer; 2485 if (outer != NULL) { 2486 ASSERT(wq->q_syncq->sq_outer == outer); 2487 outer_remove(outer, rq->q_syncq); 2488 if (wq->q_syncq != rq->q_syncq) 2489 outer_remove(outer, wq->q_syncq); 2490 } 2491 ASSERT(sq->sq_outer == NULL); 2492 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2493 2494 if (sq != SQ(rq)) { 2495 if (!(rq->q_flag & QPERMOD)) 2496 free_syncq(sq); 2497 if (wq->q_syncq == rq->q_syncq) 2498 wq->q_syncq = NULL; 2499 rq->q_syncq = NULL; 2500 } 2501 if (wq->q_syncq != NULL && wq->q_syncq != sq && 2502 wq->q_syncq != SQ(rq)) { 2503 free_syncq(wq->q_syncq); 2504 wq->q_syncq = NULL; 2505 } 2506 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && 2507 rq->q_syncq->sq_tail == NULL)); 2508 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && 2509 wq->q_syncq->sq_tail == NULL)); 2510 2511 if (!(rq->q_flag & QPERMOD) && 2512 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { 2513 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2514 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, 2515 rq->q_syncq->sq_nciputctrl, 0); 2516 ASSERT(ciputctrl_cache != NULL); 2517 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); 2518 rq->q_syncq->sq_ciputctrl = NULL; 2519 rq->q_syncq->sq_nciputctrl = 0; 2520 } 2521 2522 if (!(wq->q_flag & QPERMOD) && 2523 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { 2524 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2525 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, 2526 wq->q_syncq->sq_nciputctrl, 0); 2527 ASSERT(ciputctrl_cache != NULL); 2528 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); 2529 wq->q_syncq->sq_ciputctrl = NULL; 2530 wq->q_syncq->sq_nciputctrl = 0; 2531 } 2532 2533 sq = SQ(rq); 2534 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 2535 ASSERT(sq->sq_outer == NULL); 2536 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2537 2538 /* 2539 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS 2540 * bits in sq_flag based on the sqtype. 2541 */ 2542 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); 2543 2544 rq->q_syncq = wq->q_syncq = sq; 2545 sq->sq_type = sqtype; 2546 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); 2547 2548 /* 2549 * We are making sq_svcflags zero, 2550 * resetting SQ_DISABLED in case it was set by 2551 * wait_svc() in the munlink path. 2552 * 2553 */ 2554 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); 2555 sq->sq_svcflags = 0; 2556 2557 /* 2558 * We need to acquire the lock here for the mlink and munlink case, 2559 * where canputnext, backenable, etc can access the q_flag. 2560 */ 2561 if (lock_needed) { 2562 mutex_enter(QLOCK(rq)); 2563 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2564 mutex_exit(QLOCK(rq)); 2565 mutex_enter(QLOCK(wq)); 2566 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2567 mutex_exit(QLOCK(wq)); 2568 } else { 2569 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2570 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2571 } 2572 2573 if (qflag & QPERQ) { 2574 /* Allocate a separate syncq for the write side */ 2575 sq = new_syncq(); 2576 sq->sq_type = rq->q_syncq->sq_type; 2577 sq->sq_flags = rq->q_syncq->sq_flags; 2578 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2579 sq->sq_oprev == NULL); 2580 wq->q_syncq = sq; 2581 } 2582 if (qflag & QPERMOD) { 2583 sq = dmp->dm_sq; 2584 2585 /* 2586 * Assert that we do have an inner perimeter syncq and that it 2587 * does not have an outer perimeter associated with it. 2588 */ 2589 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2590 sq->sq_oprev == NULL); 2591 rq->q_syncq = wq->q_syncq = sq; 2592 } 2593 if (qflag & QMTOUTPERIM) { 2594 outer = dmp->dm_sq; 2595 2596 ASSERT(outer->sq_outer == NULL); 2597 outer_insert(outer, rq->q_syncq); 2598 if (wq->q_syncq != rq->q_syncq) 2599 outer_insert(outer, wq->q_syncq); 2600 } 2601 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2602 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2603 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2604 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2605 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); 2606 2607 /* 2608 * Initialize struio() types. 2609 */ 2610 rq->q_struiot = 2611 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; 2612 wq->q_struiot = 2613 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; 2614 } 2615 2616 perdm_t * 2617 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) 2618 { 2619 syncq_t *sq; 2620 perdm_t **pp; 2621 perdm_t *p; 2622 perdm_t *dmp; 2623 2624 ASSERT(str != NULL); 2625 ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); 2626 2627 rw_enter(&perdm_rwlock, RW_READER); 2628 for (p = perdm_list; p != NULL; p = p->dm_next) { 2629 if (p->dm_str == str) { /* found one */ 2630 atomic_add_32(&(p->dm_ref), 1); 2631 rw_exit(&perdm_rwlock); 2632 return (p); 2633 } 2634 } 2635 rw_exit(&perdm_rwlock); 2636 2637 sq = new_syncq(); 2638 if (qflag & QPERMOD) { 2639 sq->sq_type = sqtype | SQ_PERMOD; 2640 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; 2641 } else { 2642 ASSERT(qflag & QMTOUTPERIM); 2643 sq->sq_onext = sq->sq_oprev = sq; 2644 } 2645 2646 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); 2647 dmp->dm_sq = sq; 2648 dmp->dm_str = str; 2649 dmp->dm_ref = 1; 2650 dmp->dm_next = NULL; 2651 2652 rw_enter(&perdm_rwlock, RW_WRITER); 2653 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { 2654 if (p->dm_str == str) { /* already present */ 2655 p->dm_ref++; 2656 rw_exit(&perdm_rwlock); 2657 free_syncq(sq); 2658 kmem_free(dmp, sizeof (perdm_t)); 2659 return (p); 2660 } 2661 } 2662 2663 *pp = dmp; 2664 rw_exit(&perdm_rwlock); 2665 return (dmp); 2666 } 2667 2668 void 2669 rele_dm(perdm_t *dmp) 2670 { 2671 perdm_t **pp; 2672 perdm_t *p; 2673 2674 rw_enter(&perdm_rwlock, RW_WRITER); 2675 ASSERT(dmp->dm_ref > 0); 2676 2677 if (--dmp->dm_ref > 0) { 2678 rw_exit(&perdm_rwlock); 2679 return; 2680 } 2681 2682 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) 2683 if (p == dmp) 2684 break; 2685 ASSERT(p == dmp); 2686 *pp = p->dm_next; 2687 rw_exit(&perdm_rwlock); 2688 2689 /* 2690 * Wait for any background processing that relies on the 2691 * syncq to complete before it is freed. 2692 */ 2693 wait_sq_svc(p->dm_sq); 2694 free_syncq(p->dm_sq); 2695 kmem_free(p, sizeof (perdm_t)); 2696 } 2697 2698 /* 2699 * Make a protocol message given control and data buffers. 2700 * n.b., this can block; be careful of what locks you hold when calling it. 2701 * 2702 * If sd_maxblk is less than *iosize this routine can fail part way through 2703 * (due to an allocation failure). In this case on return *iosize will contain 2704 * the amount that was consumed. Otherwise *iosize will not be modified 2705 * i.e. it will contain the amount that was consumed. 2706 */ 2707 int 2708 strmakemsg( 2709 struct strbuf *mctl, 2710 ssize_t *iosize, 2711 struct uio *uiop, 2712 stdata_t *stp, 2713 int32_t flag, 2714 mblk_t **mpp) 2715 { 2716 mblk_t *mpctl = NULL; 2717 mblk_t *mpdata = NULL; 2718 int error; 2719 2720 ASSERT(uiop != NULL); 2721 2722 *mpp = NULL; 2723 /* Create control part, if any */ 2724 if ((mctl != NULL) && (mctl->len >= 0)) { 2725 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); 2726 if (error) 2727 return (error); 2728 } 2729 /* Create data part, if any */ 2730 if (*iosize >= 0) { 2731 error = strmakedata(iosize, uiop, stp, flag, &mpdata); 2732 if (error) { 2733 freemsg(mpctl); 2734 return (error); 2735 } 2736 } 2737 if (mpctl != NULL) { 2738 if (mpdata != NULL) 2739 linkb(mpctl, mpdata); 2740 *mpp = mpctl; 2741 } else { 2742 *mpp = mpdata; 2743 } 2744 return (0); 2745 } 2746 2747 /* 2748 * Make the control part of a protocol message given a control buffer. 2749 * n.b., this can block; be careful of what locks you hold when calling it. 2750 */ 2751 int 2752 strmakectl( 2753 struct strbuf *mctl, 2754 int32_t flag, 2755 int32_t fflag, 2756 mblk_t **mpp) 2757 { 2758 mblk_t *bp = NULL; 2759 unsigned char msgtype; 2760 int error = 0; 2761 2762 *mpp = NULL; 2763 /* 2764 * Create control part of message, if any. 2765 */ 2766 if ((mctl != NULL) && (mctl->len >= 0)) { 2767 caddr_t base; 2768 int ctlcount; 2769 int allocsz; 2770 2771 if (flag & RS_HIPRI) 2772 msgtype = M_PCPROTO; 2773 else 2774 msgtype = M_PROTO; 2775 2776 ctlcount = mctl->len; 2777 base = mctl->buf; 2778 2779 /* 2780 * Give modules a better chance to reuse M_PROTO/M_PCPROTO 2781 * blocks by increasing the size to something more usable. 2782 */ 2783 allocsz = MAX(ctlcount, 64); 2784 2785 /* 2786 * Range checking has already been done; simply try 2787 * to allocate a message block for the ctl part. 2788 */ 2789 while (!(bp = allocb(allocsz, BPRI_MED))) { 2790 if (fflag & (FNDELAY|FNONBLOCK)) 2791 return (EAGAIN); 2792 if (error = strwaitbuf(allocsz, BPRI_MED)) 2793 return (error); 2794 } 2795 2796 bp->b_datap->db_type = msgtype; 2797 if (copyin(base, bp->b_wptr, ctlcount)) { 2798 freeb(bp); 2799 return (EFAULT); 2800 } 2801 bp->b_wptr += ctlcount; 2802 } 2803 *mpp = bp; 2804 return (0); 2805 } 2806 2807 /* 2808 * Make a protocol message given data buffers. 2809 * n.b., this can block; be careful of what locks you hold when calling it. 2810 * 2811 * If sd_maxblk is less than *iosize this routine can fail part way through 2812 * (due to an allocation failure). In this case on return *iosize will contain 2813 * the amount that was consumed. Otherwise *iosize will not be modified 2814 * i.e. it will contain the amount that was consumed. 2815 */ 2816 int 2817 strmakedata( 2818 ssize_t *iosize, 2819 struct uio *uiop, 2820 stdata_t *stp, 2821 int32_t flag, 2822 mblk_t **mpp) 2823 { 2824 mblk_t *mp = NULL; 2825 mblk_t *bp; 2826 int wroff = (int)stp->sd_wroff; 2827 int error = 0; 2828 ssize_t maxblk; 2829 ssize_t count = *iosize; 2830 cred_t *cr = CRED(); 2831 2832 *mpp = NULL; 2833 if (count < 0) 2834 return (0); 2835 2836 maxblk = stp->sd_maxblk; 2837 if (maxblk == INFPSZ) 2838 maxblk = count; 2839 2840 /* 2841 * Create data part of message, if any. 2842 */ 2843 do { 2844 ssize_t size; 2845 dblk_t *dp; 2846 2847 ASSERT(uiop); 2848 2849 size = MIN(count, maxblk); 2850 2851 while ((bp = allocb_cred(size + wroff, cr)) == NULL) { 2852 error = EAGAIN; 2853 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 2854 (error = strwaitbuf(size + wroff, BPRI_MED)) != 0) { 2855 if (count == *iosize) { 2856 freemsg(mp); 2857 return (error); 2858 } else { 2859 *iosize -= count; 2860 *mpp = mp; 2861 return (0); 2862 } 2863 } 2864 } 2865 dp = bp->b_datap; 2866 dp->db_cpid = curproc->p_pid; 2867 ASSERT(wroff <= dp->db_lim - bp->b_wptr); 2868 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; 2869 2870 if (flag & STRUIO_POSTPONE) { 2871 /* 2872 * Setup the stream uio portion of the 2873 * dblk for subsequent use by struioget(). 2874 */ 2875 dp->db_struioflag = STRUIO_SPEC; 2876 dp->db_cksumstart = 0; 2877 dp->db_cksumstuff = 0; 2878 dp->db_cksumend = size; 2879 *(long long *)dp->db_struioun.data = 0ll; 2880 } else { 2881 if (stp->sd_copyflag & STRCOPYCACHED) 2882 uiop->uio_extflg |= UIO_COPY_CACHED; 2883 2884 if (size != 0) { 2885 error = uiomove(bp->b_wptr, size, UIO_WRITE, 2886 uiop); 2887 if (error != 0) { 2888 freeb(bp); 2889 freemsg(mp); 2890 return (error); 2891 } 2892 } 2893 } 2894 2895 bp->b_wptr += size; 2896 count -= size; 2897 2898 if (mp == NULL) 2899 mp = bp; 2900 else 2901 linkb(mp, bp); 2902 } while (count > 0); 2903 2904 *mpp = mp; 2905 return (0); 2906 } 2907 2908 /* 2909 * Wait for a buffer to become available. Return non-zero errno 2910 * if not able to wait, 0 if buffer is probably there. 2911 */ 2912 int 2913 strwaitbuf(size_t size, int pri) 2914 { 2915 bufcall_id_t id; 2916 2917 mutex_enter(&bcall_monitor); 2918 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, 2919 &ttoproc(curthread)->p_flag_cv)) == 0) { 2920 mutex_exit(&bcall_monitor); 2921 return (ENOSR); 2922 } 2923 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { 2924 unbufcall(id); 2925 mutex_exit(&bcall_monitor); 2926 return (EINTR); 2927 } 2928 unbufcall(id); 2929 mutex_exit(&bcall_monitor); 2930 return (0); 2931 } 2932 2933 /* 2934 * This function waits for a read or write event to happen on a stream. 2935 * fmode can specify FNDELAY and/or FNONBLOCK. 2936 * The timeout is in ms with -1 meaning infinite. 2937 * The flag values work as follows: 2938 * READWAIT Check for read side errors, send M_READ 2939 * GETWAIT Check for read side errors, no M_READ 2940 * WRITEWAIT Check for write side errors. 2941 * NOINTR Do not return error if nonblocking or timeout. 2942 * STR_NOERROR Ignore all errors except STPLEX. 2943 * STR_NOSIG Ignore/hold signals during the duration of the call. 2944 * STR_PEEK Pass through the strgeterr(). 2945 */ 2946 int 2947 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, 2948 int *done) 2949 { 2950 int slpflg, errs; 2951 int error; 2952 kcondvar_t *sleepon; 2953 mblk_t *mp; 2954 ssize_t *rd_count; 2955 clock_t rval; 2956 2957 ASSERT(MUTEX_HELD(&stp->sd_lock)); 2958 if ((flag & READWAIT) || (flag & GETWAIT)) { 2959 slpflg = RSLEEP; 2960 sleepon = &_RD(stp->sd_wrq)->q_wait; 2961 errs = STRDERR|STPLEX; 2962 } else { 2963 slpflg = WSLEEP; 2964 sleepon = &stp->sd_wrq->q_wait; 2965 errs = STWRERR|STRHUP|STPLEX; 2966 } 2967 if (flag & STR_NOERROR) 2968 errs = STPLEX; 2969 2970 if (stp->sd_wakeq & slpflg) { 2971 /* 2972 * A strwakeq() is pending, no need to sleep. 2973 */ 2974 stp->sd_wakeq &= ~slpflg; 2975 *done = 0; 2976 return (0); 2977 } 2978 2979 if (fmode & (FNDELAY|FNONBLOCK)) { 2980 if (!(flag & NOINTR)) 2981 error = EAGAIN; 2982 else 2983 error = 0; 2984 *done = 1; 2985 return (error); 2986 } 2987 2988 if (stp->sd_flag & errs) { 2989 /* 2990 * Check for errors before going to sleep since the 2991 * caller might not have checked this while holding 2992 * sd_lock. 2993 */ 2994 error = strgeterr(stp, errs, (flag & STR_PEEK)); 2995 if (error != 0) { 2996 *done = 1; 2997 return (error); 2998 } 2999 } 3000 3001 /* 3002 * If any module downstream has requested read notification 3003 * by setting SNDMREAD flag using M_SETOPTS, send a message 3004 * down stream. 3005 */ 3006 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) { 3007 mutex_exit(&stp->sd_lock); 3008 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED, 3009 (flag & STR_NOSIG), &error))) { 3010 mutex_enter(&stp->sd_lock); 3011 *done = 1; 3012 return (error); 3013 } 3014 mp->b_datap->db_type = M_READ; 3015 rd_count = (ssize_t *)mp->b_wptr; 3016 *rd_count = count; 3017 mp->b_wptr += sizeof (ssize_t); 3018 /* 3019 * Send the number of bytes requested by the 3020 * read as the argument to M_READ. 3021 */ 3022 stream_willservice(stp); 3023 putnext(stp->sd_wrq, mp); 3024 stream_runservice(stp); 3025 mutex_enter(&stp->sd_lock); 3026 3027 /* 3028 * If any data arrived due to inline processing 3029 * of putnext(), don't sleep. 3030 */ 3031 if (_RD(stp->sd_wrq)->q_first != NULL) { 3032 *done = 0; 3033 return (0); 3034 } 3035 } 3036 3037 stp->sd_flag |= slpflg; 3038 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2, 3039 "strwaitq sleeps (2):%p, %X, %lX, %X, %p", 3040 stp, flag, count, fmode, done); 3041 3042 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG); 3043 if (rval > 0) { 3044 /* EMPTY */ 3045 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2, 3046 "strwaitq awakes(2):%X, %X, %X, %X, %X", 3047 stp, flag, count, fmode, done); 3048 } else if (rval == 0) { 3049 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2, 3050 "strwaitq interrupt #2:%p, %X, %lX, %X, %p", 3051 stp, flag, count, fmode, done); 3052 stp->sd_flag &= ~slpflg; 3053 cv_broadcast(sleepon); 3054 if (!(flag & NOINTR)) 3055 error = EINTR; 3056 else 3057 error = 0; 3058 *done = 1; 3059 return (error); 3060 } else { 3061 /* timeout */ 3062 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME, 3063 "strwaitq timeout:%p, %X, %lX, %X, %p", 3064 stp, flag, count, fmode, done); 3065 *done = 1; 3066 if (!(flag & NOINTR)) 3067 return (ETIME); 3068 else 3069 return (0); 3070 } 3071 /* 3072 * If the caller implements delayed errors (i.e. queued after data) 3073 * we can not check for errors here since data as well as an 3074 * error might have arrived at the stream head. We return to 3075 * have the caller check the read queue before checking for errors. 3076 */ 3077 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) { 3078 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3079 if (error != 0) { 3080 *done = 1; 3081 return (error); 3082 } 3083 } 3084 *done = 0; 3085 return (0); 3086 } 3087 3088 /* 3089 * Perform job control discipline access checks. 3090 * Return 0 for success and the errno for failure. 3091 */ 3092 3093 #define cantsend(p, t, sig) \ 3094 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig)) 3095 3096 int 3097 straccess(struct stdata *stp, enum jcaccess mode) 3098 { 3099 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */ 3100 kthread_t *t = curthread; 3101 proc_t *p = ttoproc(t); 3102 sess_t *sp; 3103 3104 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO) 3105 return (0); 3106 3107 mutex_enter(&p->p_lock); 3108 sp = p->p_sessp; 3109 3110 for (;;) { 3111 /* 3112 * If this is not the calling process's controlling terminal 3113 * or if the calling process is already in the foreground 3114 * then allow access. 3115 */ 3116 if (sp->s_dev != stp->sd_vnode->v_rdev || 3117 p->p_pgidp == stp->sd_pgidp) { 3118 mutex_exit(&p->p_lock); 3119 return (0); 3120 } 3121 3122 /* 3123 * Check to see if controlling terminal has been deallocated. 3124 */ 3125 if (sp->s_vp == NULL) { 3126 if (!cantsend(p, t, SIGHUP)) 3127 sigtoproc(p, t, SIGHUP); 3128 mutex_exit(&p->p_lock); 3129 return (EIO); 3130 } 3131 3132 if (mode == JCGETP) { 3133 mutex_exit(&p->p_lock); 3134 return (0); 3135 } 3136 3137 if (mode == JCREAD) { 3138 if (p->p_detached || cantsend(p, t, SIGTTIN)) { 3139 mutex_exit(&p->p_lock); 3140 return (EIO); 3141 } 3142 mutex_exit(&p->p_lock); 3143 pgsignal(p->p_pgidp, SIGTTIN); 3144 mutex_enter(&p->p_lock); 3145 } else { /* mode == JCWRITE or JCSETP */ 3146 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) || 3147 cantsend(p, t, SIGTTOU)) { 3148 mutex_exit(&p->p_lock); 3149 return (0); 3150 } 3151 if (p->p_detached) { 3152 mutex_exit(&p->p_lock); 3153 return (EIO); 3154 } 3155 mutex_exit(&p->p_lock); 3156 pgsignal(p->p_pgidp, SIGTTOU); 3157 mutex_enter(&p->p_lock); 3158 } 3159 3160 /* 3161 * We call cv_wait_sig_swap() to cause the appropriate 3162 * action for the jobcontrol signal to take place. 3163 * If the signal is being caught, we will take the 3164 * EINTR error return. Otherwise, the default action 3165 * of causing the process to stop will take place. 3166 * In this case, we rely on the periodic cv_broadcast() on 3167 * &lbolt_cv to wake us up to loop around and test again. 3168 * We can't get here if the signal is ignored or 3169 * if the current thread is blocking the signal. 3170 */ 3171 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) { 3172 mutex_exit(&p->p_lock); 3173 return (EINTR); 3174 } 3175 } 3176 } 3177 3178 /* 3179 * Return size of message of block type (bp->b_datap->db_type) 3180 */ 3181 size_t 3182 xmsgsize(mblk_t *bp) 3183 { 3184 unsigned char type; 3185 size_t count = 0; 3186 3187 type = bp->b_datap->db_type; 3188 3189 for (; bp; bp = bp->b_cont) { 3190 if (type != bp->b_datap->db_type) 3191 break; 3192 ASSERT(bp->b_wptr >= bp->b_rptr); 3193 count += bp->b_wptr - bp->b_rptr; 3194 } 3195 return (count); 3196 } 3197 3198 /* 3199 * Allocate a stream head. 3200 */ 3201 struct stdata * 3202 shalloc(queue_t *qp) 3203 { 3204 stdata_t *stp; 3205 3206 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP); 3207 3208 stp->sd_wrq = _WR(qp); 3209 stp->sd_strtab = NULL; 3210 stp->sd_iocid = 0; 3211 stp->sd_mate = NULL; 3212 stp->sd_freezer = NULL; 3213 stp->sd_refcnt = 0; 3214 stp->sd_wakeq = 0; 3215 stp->sd_anchor = 0; 3216 stp->sd_struiowrq = NULL; 3217 stp->sd_struiordq = NULL; 3218 stp->sd_struiodnak = 0; 3219 stp->sd_struionak = NULL; 3220 #ifdef C2_AUDIT 3221 stp->sd_t_audit_data = NULL; 3222 #endif 3223 stp->sd_rput_opt = 0; 3224 stp->sd_wput_opt = 0; 3225 stp->sd_read_opt = 0; 3226 stp->sd_rprotofunc = strrput_proto; 3227 stp->sd_rmiscfunc = strrput_misc; 3228 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL; 3229 stp->sd_ciputctrl = NULL; 3230 stp->sd_nciputctrl = 0; 3231 stp->sd_qhead = NULL; 3232 stp->sd_qtail = NULL; 3233 stp->sd_servid = NULL; 3234 stp->sd_nqueues = 0; 3235 stp->sd_svcflags = 0; 3236 stp->sd_copyflag = 0; 3237 return (stp); 3238 } 3239 3240 /* 3241 * Free a stream head. 3242 */ 3243 void 3244 shfree(stdata_t *stp) 3245 { 3246 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock)); 3247 3248 stp->sd_wrq = NULL; 3249 3250 mutex_enter(&stp->sd_qlock); 3251 while (stp->sd_svcflags & STRS_SCHEDULED) { 3252 STRSTAT(strwaits); 3253 cv_wait(&stp->sd_qcv, &stp->sd_qlock); 3254 } 3255 mutex_exit(&stp->sd_qlock); 3256 3257 if (stp->sd_ciputctrl != NULL) { 3258 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1); 3259 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl, 3260 stp->sd_nciputctrl, 0); 3261 ASSERT(ciputctrl_cache != NULL); 3262 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl); 3263 stp->sd_ciputctrl = NULL; 3264 stp->sd_nciputctrl = 0; 3265 } 3266 ASSERT(stp->sd_qhead == NULL); 3267 ASSERT(stp->sd_qtail == NULL); 3268 ASSERT(stp->sd_nqueues == 0); 3269 kmem_cache_free(stream_head_cache, stp); 3270 } 3271 3272 /* 3273 * Allocate a pair of queues and a syncq for the pair 3274 */ 3275 queue_t * 3276 allocq(void) 3277 { 3278 queinfo_t *qip; 3279 queue_t *qp, *wqp; 3280 syncq_t *sq; 3281 3282 qip = kmem_cache_alloc(queue_cache, KM_SLEEP); 3283 3284 qp = &qip->qu_rqueue; 3285 wqp = &qip->qu_wqueue; 3286 sq = &qip->qu_syncq; 3287 3288 qp->q_last = NULL; 3289 qp->q_next = NULL; 3290 qp->q_ptr = NULL; 3291 qp->q_flag = QUSE | QREADR; 3292 qp->q_bandp = NULL; 3293 qp->q_stream = NULL; 3294 qp->q_syncq = sq; 3295 qp->q_nband = 0; 3296 qp->q_nfsrv = NULL; 3297 qp->q_draining = 0; 3298 qp->q_syncqmsgs = 0; 3299 qp->q_spri = 0; 3300 qp->q_qtstamp = 0; 3301 qp->q_sqtstamp = 0; 3302 qp->q_fp = NULL; 3303 3304 wqp->q_last = NULL; 3305 wqp->q_next = NULL; 3306 wqp->q_ptr = NULL; 3307 wqp->q_flag = QUSE; 3308 wqp->q_bandp = NULL; 3309 wqp->q_stream = NULL; 3310 wqp->q_syncq = sq; 3311 wqp->q_nband = 0; 3312 wqp->q_nfsrv = NULL; 3313 wqp->q_draining = 0; 3314 wqp->q_syncqmsgs = 0; 3315 wqp->q_qtstamp = 0; 3316 wqp->q_sqtstamp = 0; 3317 wqp->q_spri = 0; 3318 3319 sq->sq_count = 0; 3320 sq->sq_rmqcount = 0; 3321 sq->sq_flags = 0; 3322 sq->sq_type = 0; 3323 sq->sq_callbflags = 0; 3324 sq->sq_cancelid = 0; 3325 sq->sq_ciputctrl = NULL; 3326 sq->sq_nciputctrl = 0; 3327 sq->sq_needexcl = 0; 3328 sq->sq_svcflags = 0; 3329 3330 return (qp); 3331 } 3332 3333 /* 3334 * Free a pair of queues and the "attached" syncq. 3335 * Discard any messages left on the syncq(s), remove the syncq(s) from the 3336 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq. 3337 */ 3338 void 3339 freeq(queue_t *qp) 3340 { 3341 qband_t *qbp, *nqbp; 3342 syncq_t *sq, *outer; 3343 queue_t *wqp = _WR(qp); 3344 3345 ASSERT(qp->q_flag & QREADR); 3346 3347 (void) flush_syncq(qp->q_syncq, qp); 3348 (void) flush_syncq(wqp->q_syncq, wqp); 3349 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0); 3350 3351 outer = qp->q_syncq->sq_outer; 3352 if (outer != NULL) { 3353 outer_remove(outer, qp->q_syncq); 3354 if (wqp->q_syncq != qp->q_syncq) 3355 outer_remove(outer, wqp->q_syncq); 3356 } 3357 /* 3358 * Free any syncqs that are outside what allocq returned. 3359 */ 3360 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD)) 3361 free_syncq(qp->q_syncq); 3362 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp)) 3363 free_syncq(wqp->q_syncq); 3364 3365 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3366 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3367 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 3368 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp))); 3369 sq = SQ(qp); 3370 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 3371 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 3372 ASSERT(sq->sq_outer == NULL); 3373 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 3374 ASSERT(sq->sq_callbpend == NULL); 3375 ASSERT(sq->sq_needexcl == 0); 3376 3377 if (sq->sq_ciputctrl != NULL) { 3378 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 3379 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 3380 sq->sq_nciputctrl, 0); 3381 ASSERT(ciputctrl_cache != NULL); 3382 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 3383 sq->sq_ciputctrl = NULL; 3384 sq->sq_nciputctrl = 0; 3385 } 3386 3387 ASSERT(qp->q_first == NULL && wqp->q_first == NULL); 3388 ASSERT(qp->q_count == 0 && wqp->q_count == 0); 3389 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0); 3390 3391 qp->q_flag &= ~QUSE; 3392 wqp->q_flag &= ~QUSE; 3393 3394 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */ 3395 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */ 3396 3397 qbp = qp->q_bandp; 3398 while (qbp) { 3399 nqbp = qbp->qb_next; 3400 freeband(qbp); 3401 qbp = nqbp; 3402 } 3403 qbp = wqp->q_bandp; 3404 while (qbp) { 3405 nqbp = qbp->qb_next; 3406 freeband(qbp); 3407 qbp = nqbp; 3408 } 3409 kmem_cache_free(queue_cache, qp); 3410 } 3411 3412 /* 3413 * Allocate a qband structure. 3414 */ 3415 qband_t * 3416 allocband(void) 3417 { 3418 qband_t *qbp; 3419 3420 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP); 3421 if (qbp == NULL) 3422 return (NULL); 3423 3424 qbp->qb_next = NULL; 3425 qbp->qb_count = 0; 3426 qbp->qb_mblkcnt = 0; 3427 qbp->qb_first = NULL; 3428 qbp->qb_last = NULL; 3429 qbp->qb_flag = 0; 3430 3431 return (qbp); 3432 } 3433 3434 /* 3435 * Free a qband structure. 3436 */ 3437 void 3438 freeband(qband_t *qbp) 3439 { 3440 kmem_cache_free(qband_cache, qbp); 3441 } 3442 3443 /* 3444 * Just like putnextctl(9F), except that allocb_wait() is used. 3445 * 3446 * Consolidation Private, and of course only callable from the stream head or 3447 * routines that may block. 3448 */ 3449 int 3450 putnextctl_wait(queue_t *q, int type) 3451 { 3452 mblk_t *bp; 3453 int error; 3454 3455 if ((datamsg(type) && (type != M_DELAY)) || 3456 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL) 3457 return (0); 3458 3459 bp->b_datap->db_type = (unsigned char)type; 3460 putnext(q, bp); 3461 return (1); 3462 } 3463 3464 /* 3465 * run any possible bufcalls. 3466 */ 3467 void 3468 runbufcalls(void) 3469 { 3470 strbufcall_t *bcp; 3471 3472 mutex_enter(&bcall_monitor); 3473 mutex_enter(&strbcall_lock); 3474 3475 if (strbcalls.bc_head) { 3476 size_t count; 3477 int nevent; 3478 3479 /* 3480 * count how many events are on the list 3481 * now so we can check to avoid looping 3482 * in low memory situations 3483 */ 3484 nevent = 0; 3485 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next) 3486 nevent++; 3487 3488 /* 3489 * get estimate of available memory from kmem_avail(). 3490 * awake all bufcall functions waiting for 3491 * memory whose request could be satisfied 3492 * by 'count' memory and let 'em fight for it. 3493 */ 3494 count = kmem_avail(); 3495 while ((bcp = strbcalls.bc_head) != NULL && nevent) { 3496 STRSTAT(bufcalls); 3497 --nevent; 3498 if (bcp->bc_size <= count) { 3499 bcp->bc_executor = curthread; 3500 mutex_exit(&strbcall_lock); 3501 (*bcp->bc_func)(bcp->bc_arg); 3502 mutex_enter(&strbcall_lock); 3503 bcp->bc_executor = NULL; 3504 cv_broadcast(&bcall_cv); 3505 strbcalls.bc_head = bcp->bc_next; 3506 kmem_free(bcp, sizeof (strbufcall_t)); 3507 } else { 3508 /* 3509 * too big, try again later - note 3510 * that nevent was decremented above 3511 * so we won't retry this one on this 3512 * iteration of the loop 3513 */ 3514 if (bcp->bc_next != NULL) { 3515 strbcalls.bc_head = bcp->bc_next; 3516 bcp->bc_next = NULL; 3517 strbcalls.bc_tail->bc_next = bcp; 3518 strbcalls.bc_tail = bcp; 3519 } 3520 } 3521 } 3522 if (strbcalls.bc_head == NULL) 3523 strbcalls.bc_tail = NULL; 3524 } 3525 3526 mutex_exit(&strbcall_lock); 3527 mutex_exit(&bcall_monitor); 3528 } 3529 3530 3531 /* 3532 * actually run queue's service routine. 3533 */ 3534 static void 3535 runservice(queue_t *q) 3536 { 3537 qband_t *qbp; 3538 3539 ASSERT(q->q_qinfo->qi_srvp); 3540 again: 3541 entersq(q->q_syncq, SQ_SVC); 3542 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START, 3543 "runservice starts:%p", q); 3544 3545 if (!(q->q_flag & QWCLOSE)) 3546 (*q->q_qinfo->qi_srvp)(q); 3547 3548 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END, 3549 "runservice ends:(%p)", q); 3550 3551 leavesq(q->q_syncq, SQ_SVC); 3552 3553 mutex_enter(QLOCK(q)); 3554 if (q->q_flag & QENAB) { 3555 q->q_flag &= ~QENAB; 3556 mutex_exit(QLOCK(q)); 3557 goto again; 3558 } 3559 q->q_flag &= ~QINSERVICE; 3560 q->q_flag &= ~QBACK; 3561 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next) 3562 qbp->qb_flag &= ~QB_BACK; 3563 /* 3564 * Wakeup thread waiting for the service procedure 3565 * to be run (strclose and qdetach). 3566 */ 3567 cv_broadcast(&q->q_wait); 3568 3569 mutex_exit(QLOCK(q)); 3570 } 3571 3572 /* 3573 * Background processing of bufcalls. 3574 */ 3575 void 3576 streams_bufcall_service(void) 3577 { 3578 callb_cpr_t cprinfo; 3579 3580 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr, 3581 "streams_bufcall_service"); 3582 3583 mutex_enter(&strbcall_lock); 3584 3585 for (;;) { 3586 if (strbcalls.bc_head != NULL && kmem_avail() > 0) { 3587 mutex_exit(&strbcall_lock); 3588 runbufcalls(); 3589 mutex_enter(&strbcall_lock); 3590 } 3591 if (strbcalls.bc_head != NULL) { 3592 clock_t wt, tick; 3593 3594 STRSTAT(bcwaits); 3595 /* Wait for memory to become available */ 3596 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3597 tick = SEC_TO_TICK(60); 3598 time_to_wait(&wt, tick); 3599 (void) cv_timedwait(&memavail_cv, &strbcall_lock, wt); 3600 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3601 } 3602 3603 /* Wait for new work to arrive */ 3604 if (strbcalls.bc_head == NULL) { 3605 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3606 cv_wait(&strbcall_cv, &strbcall_lock); 3607 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3608 } 3609 } 3610 } 3611 3612 /* 3613 * Background processing of streams background tasks which failed 3614 * taskq_dispatch. 3615 */ 3616 static void 3617 streams_qbkgrnd_service(void) 3618 { 3619 callb_cpr_t cprinfo; 3620 queue_t *q; 3621 3622 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3623 "streams_bkgrnd_service"); 3624 3625 mutex_enter(&service_queue); 3626 3627 for (;;) { 3628 /* 3629 * Wait for work to arrive. 3630 */ 3631 while ((freebs_list == NULL) && (qhead == NULL)) { 3632 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3633 cv_wait(&services_to_run, &service_queue); 3634 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3635 } 3636 /* 3637 * Handle all pending freebs requests to free memory. 3638 */ 3639 while (freebs_list != NULL) { 3640 mblk_t *mp = freebs_list; 3641 freebs_list = mp->b_next; 3642 mutex_exit(&service_queue); 3643 mblk_free(mp); 3644 mutex_enter(&service_queue); 3645 } 3646 /* 3647 * Run pending queues. 3648 */ 3649 while (qhead != NULL) { 3650 DQ(q, qhead, qtail, q_link); 3651 ASSERT(q != NULL); 3652 mutex_exit(&service_queue); 3653 queue_service(q); 3654 mutex_enter(&service_queue); 3655 } 3656 ASSERT(qhead == NULL && qtail == NULL); 3657 } 3658 } 3659 3660 /* 3661 * Background processing of streams background tasks which failed 3662 * taskq_dispatch. 3663 */ 3664 static void 3665 streams_sqbkgrnd_service(void) 3666 { 3667 callb_cpr_t cprinfo; 3668 syncq_t *sq; 3669 3670 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3671 "streams_sqbkgrnd_service"); 3672 3673 mutex_enter(&service_queue); 3674 3675 for (;;) { 3676 /* 3677 * Wait for work to arrive. 3678 */ 3679 while (sqhead == NULL) { 3680 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3681 cv_wait(&syncqs_to_run, &service_queue); 3682 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3683 } 3684 3685 /* 3686 * Run pending syncqs. 3687 */ 3688 while (sqhead != NULL) { 3689 DQ(sq, sqhead, sqtail, sq_next); 3690 ASSERT(sq != NULL); 3691 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); 3692 mutex_exit(&service_queue); 3693 syncq_service(sq); 3694 mutex_enter(&service_queue); 3695 } 3696 } 3697 } 3698 3699 /* 3700 * Disable the syncq and wait for background syncq processing to complete. 3701 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the 3702 * list. 3703 */ 3704 void 3705 wait_sq_svc(syncq_t *sq) 3706 { 3707 mutex_enter(SQLOCK(sq)); 3708 sq->sq_svcflags |= SQ_DISABLED; 3709 if (sq->sq_svcflags & SQ_BGTHREAD) { 3710 syncq_t *sq_chase; 3711 syncq_t *sq_curr; 3712 int removed; 3713 3714 ASSERT(sq->sq_servcount == 1); 3715 mutex_enter(&service_queue); 3716 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); 3717 mutex_exit(&service_queue); 3718 if (removed) { 3719 sq->sq_svcflags &= ~SQ_BGTHREAD; 3720 sq->sq_servcount = 0; 3721 STRSTAT(sqremoved); 3722 goto done; 3723 } 3724 } 3725 while (sq->sq_servcount != 0) { 3726 sq->sq_flags |= SQ_WANTWAKEUP; 3727 cv_wait(&sq->sq_wait, SQLOCK(sq)); 3728 } 3729 done: 3730 mutex_exit(SQLOCK(sq)); 3731 } 3732 3733 /* 3734 * Put a syncq on the list of syncq's to be serviced by the sqthread. 3735 * Add the argument to the end of the sqhead list and set the flag 3736 * indicating this syncq has been enabled. If it has already been 3737 * enabled, don't do anything. 3738 * This routine assumes that SQLOCK is held. 3739 * NOTE that the lock order is to have the SQLOCK first, 3740 * so if the service_syncq lock is held, we need to release it 3741 * before aquiring the SQLOCK (mostly relevant for the background 3742 * thread, and this seems to be common among the STREAMS global locks). 3743 * Note the the sq_svcflags are protected by the SQLOCK. 3744 */ 3745 void 3746 sqenable(syncq_t *sq) 3747 { 3748 /* 3749 * This is probably not important except for where I believe it 3750 * is being called. At that point, it should be held (and it 3751 * is a pain to release it just for this routine, so don't do 3752 * it). 3753 */ 3754 ASSERT(MUTEX_HELD(SQLOCK(sq))); 3755 3756 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); 3757 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); 3758 3759 /* 3760 * Do not put on list if background thread is scheduled or 3761 * syncq is disabled. 3762 */ 3763 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) 3764 return; 3765 3766 /* 3767 * Check whether we should enable sq at all. 3768 * Non PERMOD syncqs may be drained by at most one thread. 3769 * PERMOD syncqs may be drained by several threads but we limit the 3770 * total amount to the lesser of 3771 * Number of queues on the squeue and 3772 * Number of CPUs. 3773 */ 3774 if (sq->sq_servcount != 0) { 3775 if (((sq->sq_type & SQ_PERMOD) == 0) || 3776 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { 3777 STRSTAT(sqtoomany); 3778 return; 3779 } 3780 } 3781 3782 sq->sq_tstamp = lbolt; 3783 STRSTAT(sqenables); 3784 3785 /* Attempt a taskq dispatch */ 3786 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, 3787 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); 3788 if (sq->sq_servid != NULL) { 3789 sq->sq_servcount++; 3790 return; 3791 } 3792 3793 /* 3794 * This taskq dispatch failed, but a previous one may have succeeded. 3795 * Don't try to schedule on the background thread whilst there is 3796 * outstanding taskq processing. 3797 */ 3798 if (sq->sq_servcount != 0) 3799 return; 3800 3801 /* 3802 * System is low on resources and can't perform a non-sleeping 3803 * dispatch. Schedule the syncq for a background thread and mark the 3804 * syncq to avoid any further taskq dispatch attempts. 3805 */ 3806 mutex_enter(&service_queue); 3807 STRSTAT(taskqfails); 3808 ENQUEUE(sq, sqhead, sqtail, sq_next); 3809 sq->sq_svcflags |= SQ_BGTHREAD; 3810 sq->sq_servcount = 1; 3811 cv_signal(&syncqs_to_run); 3812 mutex_exit(&service_queue); 3813 } 3814 3815 /* 3816 * Note: fifo_close() depends on the mblk_t on the queue being freed 3817 * asynchronously. The asynchronous freeing of messages breaks the 3818 * recursive call chain of fifo_close() while there are I_SENDFD type of 3819 * messages refering other file pointers on the queue. Then when 3820 * closing pipes it can avoid stack overflow in case of daisy-chained 3821 * pipes, and also avoid deadlock in case of fifonode_t pairs (which 3822 * share the same fifolock_t). 3823 */ 3824 3825 /* ARGSUSED */ 3826 void 3827 freebs_enqueue(mblk_t *mp, dblk_t *dbp) 3828 { 3829 ASSERT(dbp->db_mblk == mp); 3830 3831 /* 3832 * Check data sanity. The dblock should have non-empty free function. 3833 * It is better to panic here then later when the dblock is freed 3834 * asynchronously when the context is lost. 3835 */ 3836 if (dbp->db_frtnp->free_func == NULL) { 3837 panic("freebs_enqueue: dblock %p has a NULL free callback", 3838 (void *) dbp); 3839 } 3840 3841 STRSTAT(freebs); 3842 if (taskq_dispatch(streams_taskq, (task_func_t *)mblk_free, mp, 3843 TQ_NOSLEEP) == NULL) { 3844 /* 3845 * System is low on resources and can't perform a non-sleeping 3846 * dispatch. Schedule for a background thread. 3847 */ 3848 mutex_enter(&service_queue); 3849 STRSTAT(taskqfails); 3850 mp->b_next = freebs_list; 3851 freebs_list = mp; 3852 cv_signal(&services_to_run); 3853 mutex_exit(&service_queue); 3854 } 3855 } 3856 3857 /* 3858 * Set the QBACK or QB_BACK flag in the given queue for 3859 * the given priority band. 3860 */ 3861 void 3862 setqback(queue_t *q, unsigned char pri) 3863 { 3864 int i; 3865 qband_t *qbp; 3866 qband_t **qbpp; 3867 3868 ASSERT(MUTEX_HELD(QLOCK(q))); 3869 if (pri != 0) { 3870 if (pri > q->q_nband) { 3871 qbpp = &q->q_bandp; 3872 while (*qbpp) 3873 qbpp = &(*qbpp)->qb_next; 3874 while (pri > q->q_nband) { 3875 if ((*qbpp = allocband()) == NULL) { 3876 cmn_err(CE_WARN, 3877 "setqback: can't allocate qband\n"); 3878 return; 3879 } 3880 (*qbpp)->qb_hiwat = q->q_hiwat; 3881 (*qbpp)->qb_lowat = q->q_lowat; 3882 q->q_nband++; 3883 qbpp = &(*qbpp)->qb_next; 3884 } 3885 } 3886 qbp = q->q_bandp; 3887 i = pri; 3888 while (--i) 3889 qbp = qbp->qb_next; 3890 qbp->qb_flag |= QB_BACK; 3891 } else { 3892 q->q_flag |= QBACK; 3893 } 3894 } 3895 3896 int 3897 strcopyin(void *from, void *to, size_t len, int copyflag) 3898 { 3899 if (copyflag & U_TO_K) { 3900 ASSERT((copyflag & K_TO_K) == 0); 3901 if (copyin(from, to, len)) 3902 return (EFAULT); 3903 } else { 3904 ASSERT(copyflag & K_TO_K); 3905 bcopy(from, to, len); 3906 } 3907 return (0); 3908 } 3909 3910 int 3911 strcopyout(void *from, void *to, size_t len, int copyflag) 3912 { 3913 if (copyflag & U_TO_K) { 3914 if (copyout(from, to, len)) 3915 return (EFAULT); 3916 } else { 3917 ASSERT(copyflag & K_TO_K); 3918 bcopy(from, to, len); 3919 } 3920 return (0); 3921 } 3922 3923 /* 3924 * strsignal_nolock() posts a signal to the process(es) at the stream head. 3925 * It assumes that the stream head lock is already held, whereas strsignal() 3926 * acquires the lock first. This routine was created because a few callers 3927 * release the stream head lock before calling only to re-acquire it after 3928 * it returns. 3929 */ 3930 void 3931 strsignal_nolock(stdata_t *stp, int sig, int32_t band) 3932 { 3933 ASSERT(MUTEX_HELD(&stp->sd_lock)); 3934 switch (sig) { 3935 case SIGPOLL: 3936 if (stp->sd_sigflags & S_MSG) 3937 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 3938 break; 3939 3940 default: 3941 if (stp->sd_pgidp) { 3942 pgsignal(stp->sd_pgidp, sig); 3943 } 3944 break; 3945 } 3946 } 3947 3948 void 3949 strsignal(stdata_t *stp, int sig, int32_t band) 3950 { 3951 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG, 3952 "strsignal:%p, %X, %X", stp, sig, band); 3953 3954 mutex_enter(&stp->sd_lock); 3955 switch (sig) { 3956 case SIGPOLL: 3957 if (stp->sd_sigflags & S_MSG) 3958 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 3959 break; 3960 3961 default: 3962 if (stp->sd_pgidp) { 3963 pgsignal(stp->sd_pgidp, sig); 3964 } 3965 break; 3966 } 3967 mutex_exit(&stp->sd_lock); 3968 } 3969 3970 void 3971 strhup(stdata_t *stp) 3972 { 3973 pollwakeup(&stp->sd_pollist, POLLHUP); 3974 mutex_enter(&stp->sd_lock); 3975 if (stp->sd_sigflags & S_HANGUP) 3976 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0); 3977 mutex_exit(&stp->sd_lock); 3978 } 3979 3980 void 3981 stralloctty(stdata_t *stp) 3982 { 3983 proc_t *p = curproc; 3984 sess_t *sp = p->p_sessp; 3985 3986 mutex_enter(&stp->sd_lock); 3987 /* 3988 * No need to hold the session lock or do a TTY_HOLD() because 3989 * this is the only thread that can be the session leader and not 3990 * have a controlling tty. 3991 */ 3992 if ((stp->sd_flag & 3993 (STRHUP|STRDERR|STWRERR|STPLEX|STRISTTY)) == STRISTTY && 3994 stp->sd_sidp == NULL && /* not allocated as ctty */ 3995 sp->s_sidp == p->p_pidp && /* session leader */ 3996 sp->s_flag != SESS_CLOSE && /* session is not closing */ 3997 sp->s_vp == NULL) { /* without ctty */ 3998 ASSERT(stp->sd_pgidp == NULL); 3999 alloctty(p, makectty(stp->sd_vnode)); 4000 4001 mutex_enter(&pidlock); 4002 stp->sd_sidp = sp->s_sidp; 4003 stp->sd_pgidp = sp->s_sidp; 4004 PID_HOLD(stp->sd_pgidp); 4005 PID_HOLD(stp->sd_sidp); 4006 mutex_exit(&pidlock); 4007 } 4008 mutex_exit(&stp->sd_lock); 4009 } 4010 4011 void 4012 strfreectty(stdata_t *stp) 4013 { 4014 mutex_enter(&stp->sd_lock); 4015 pgsignal(stp->sd_pgidp, SIGHUP); 4016 mutex_enter(&pidlock); 4017 PID_RELE(stp->sd_pgidp); 4018 PID_RELE(stp->sd_sidp); 4019 stp->sd_pgidp = NULL; 4020 stp->sd_sidp = NULL; 4021 mutex_exit(&pidlock); 4022 mutex_exit(&stp->sd_lock); 4023 if (!(stp->sd_flag & STRHUP)) 4024 strhup(stp); 4025 } 4026 /* 4027 * Backenable the first queue upstream from `q' with a service procedure. 4028 */ 4029 void 4030 backenable(queue_t *q, uchar_t pri) 4031 { 4032 queue_t *nq; 4033 4034 /* 4035 * our presence might not prevent other modules in our own 4036 * stream from popping/pushing since the caller of getq might not 4037 * have a claim on the queue (some drivers do a getq on somebody 4038 * else's queue - they know that the queue itself is not going away 4039 * but the framework has to guarantee q_next in that stream.) 4040 */ 4041 claimstr(q); 4042 4043 /* find nearest back queue with service proc */ 4044 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) { 4045 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq)); 4046 } 4047 4048 if (nq) { 4049 kthread_t *freezer; 4050 /* 4051 * backenable can be called either with no locks held 4052 * or with the stream frozen (the latter occurs when a module 4053 * calls rmvq with the stream frozen.) If the stream is frozen 4054 * by the caller the caller will hold all qlocks in the stream. 4055 */ 4056 freezer = STREAM(q)->sd_freezer; 4057 if (freezer != curthread) { 4058 mutex_enter(QLOCK(nq)); 4059 } 4060 #ifdef DEBUG 4061 else { 4062 ASSERT(frozenstr(q)); 4063 ASSERT(MUTEX_HELD(QLOCK(q))); 4064 ASSERT(MUTEX_HELD(QLOCK(nq))); 4065 } 4066 #endif 4067 setqback(nq, pri); 4068 qenable_locked(nq); 4069 if (freezer != curthread) 4070 mutex_exit(QLOCK(nq)); 4071 } 4072 releasestr(q); 4073 } 4074 4075 /* 4076 * Return the appropriate errno when one of flags_to_check is set 4077 * in sd_flags. Uses the exported error routines if they are set. 4078 * Will return 0 if non error is set (or if the exported error routines 4079 * do not return an error). 4080 * 4081 * If there is both a read and write error to check we prefer the read error. 4082 * Also, give preference to recorded errno's over the error functions. 4083 * The flags that are handled are: 4084 * STPLEX return EINVAL 4085 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST) 4086 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST) 4087 * STRHUP return sd_werror 4088 * 4089 * If the caller indicates that the operation is a peek a nonpersistent error 4090 * is not cleared. 4091 */ 4092 int 4093 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek) 4094 { 4095 int32_t sd_flag = stp->sd_flag & flags_to_check; 4096 int error = 0; 4097 4098 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4099 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0); 4100 if (sd_flag & STPLEX) 4101 error = EINVAL; 4102 else if (sd_flag & STRDERR) { 4103 error = stp->sd_rerror; 4104 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) { 4105 /* 4106 * Read errors are non-persistent i.e. discarded once 4107 * returned to a non-peeking caller, 4108 */ 4109 stp->sd_rerror = 0; 4110 stp->sd_flag &= ~STRDERR; 4111 } 4112 if (error == 0 && stp->sd_rderrfunc != NULL) { 4113 int clearerr = 0; 4114 4115 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek, 4116 &clearerr); 4117 if (clearerr) { 4118 stp->sd_flag &= ~STRDERR; 4119 stp->sd_rderrfunc = NULL; 4120 } 4121 } 4122 } else if (sd_flag & STWRERR) { 4123 error = stp->sd_werror; 4124 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) { 4125 /* 4126 * Write errors are non-persistent i.e. discarded once 4127 * returned to a non-peeking caller, 4128 */ 4129 stp->sd_werror = 0; 4130 stp->sd_flag &= ~STWRERR; 4131 } 4132 if (error == 0 && stp->sd_wrerrfunc != NULL) { 4133 int clearerr = 0; 4134 4135 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek, 4136 &clearerr); 4137 if (clearerr) { 4138 stp->sd_flag &= ~STWRERR; 4139 stp->sd_wrerrfunc = NULL; 4140 } 4141 } 4142 } else if (sd_flag & STRHUP) { 4143 /* sd_werror set when STRHUP */ 4144 error = stp->sd_werror; 4145 } 4146 return (error); 4147 } 4148 4149 4150 /* 4151 * single-thread open/close/push/pop 4152 * for twisted streams also 4153 */ 4154 int 4155 strstartplumb(stdata_t *stp, int flag, int cmd) 4156 { 4157 int waited = 1; 4158 int error = 0; 4159 4160 if (STRMATED(stp)) { 4161 struct stdata *stmatep = stp->sd_mate; 4162 4163 STRLOCKMATES(stp); 4164 while (waited) { 4165 waited = 0; 4166 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4167 if ((cmd == I_POP) && 4168 (flag & (FNDELAY|FNONBLOCK))) { 4169 STRUNLOCKMATES(stp); 4170 return (EAGAIN); 4171 } 4172 waited = 1; 4173 mutex_exit(&stp->sd_lock); 4174 if (!cv_wait_sig(&stmatep->sd_monitor, 4175 &stmatep->sd_lock)) { 4176 mutex_exit(&stmatep->sd_lock); 4177 return (EINTR); 4178 } 4179 mutex_exit(&stmatep->sd_lock); 4180 STRLOCKMATES(stp); 4181 } 4182 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4183 if ((cmd == I_POP) && 4184 (flag & (FNDELAY|FNONBLOCK))) { 4185 STRUNLOCKMATES(stp); 4186 return (EAGAIN); 4187 } 4188 waited = 1; 4189 mutex_exit(&stmatep->sd_lock); 4190 if (!cv_wait_sig(&stp->sd_monitor, 4191 &stp->sd_lock)) { 4192 mutex_exit(&stp->sd_lock); 4193 return (EINTR); 4194 } 4195 mutex_exit(&stp->sd_lock); 4196 STRLOCKMATES(stp); 4197 } 4198 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4199 error = strgeterr(stp, 4200 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4201 if (error != 0) { 4202 STRUNLOCKMATES(stp); 4203 return (error); 4204 } 4205 } 4206 } 4207 stp->sd_flag |= STRPLUMB; 4208 STRUNLOCKMATES(stp); 4209 } else { 4210 mutex_enter(&stp->sd_lock); 4211 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4212 if (((cmd == I_POP) || (cmd == _I_REMOVE)) && 4213 (flag & (FNDELAY|FNONBLOCK))) { 4214 mutex_exit(&stp->sd_lock); 4215 return (EAGAIN); 4216 } 4217 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) { 4218 mutex_exit(&stp->sd_lock); 4219 return (EINTR); 4220 } 4221 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4222 error = strgeterr(stp, 4223 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4224 if (error != 0) { 4225 mutex_exit(&stp->sd_lock); 4226 return (error); 4227 } 4228 } 4229 } 4230 stp->sd_flag |= STRPLUMB; 4231 mutex_exit(&stp->sd_lock); 4232 } 4233 return (0); 4234 } 4235 4236 /* 4237 * Complete the plumbing operation associated with stream `stp'. 4238 */ 4239 void 4240 strendplumb(stdata_t *stp) 4241 { 4242 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4243 ASSERT(stp->sd_flag & STRPLUMB); 4244 stp->sd_flag &= ~STRPLUMB; 4245 cv_broadcast(&stp->sd_monitor); 4246 } 4247 4248 /* 4249 * This describes how the STREAMS framework handles synchronization 4250 * during open/push and close/pop. 4251 * The key interfaces for open and close are qprocson and qprocsoff, 4252 * respectively. While the close case in general is harder both open 4253 * have close have significant similarities. 4254 * 4255 * During close the STREAMS framework has to both ensure that there 4256 * are no stale references to the queue pair (and syncq) that 4257 * are being closed and also provide the guarantees that are documented 4258 * in qprocsoff(9F). 4259 * If there are stale references to the queue that is closing it can 4260 * result in kernel memory corruption or kernel panics. 4261 * 4262 * Note that is it up to the module/driver to ensure that it itself 4263 * does not have any stale references to the closing queues once its close 4264 * routine returns. This includes: 4265 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines 4266 * associated with the queues. For timeout and bufcall callbacks the 4267 * module/driver also has to ensure (or wait for) any callbacks that 4268 * are in progress. 4269 * - If the module/driver is using esballoc it has to ensure that any 4270 * esballoc free functions do not refer to a queue that has closed. 4271 * (Note that in general the close routine can not wait for the esballoc'ed 4272 * messages to be freed since that can cause a deadlock.) 4273 * - Cancelling any interrupts that refer to the closing queues and 4274 * also ensuring that there are no interrupts in progress that will 4275 * refer to the closing queues once the close routine returns. 4276 * - For multiplexors removing any driver global state that refers to 4277 * the closing queue and also ensuring that there are no threads in 4278 * the multiplexor that has picked up a queue pointer but not yet 4279 * finished using it. 4280 * 4281 * In addition, a driver/module can only reference the q_next pointer 4282 * in its open, close, put, or service procedures or in a 4283 * qtimeout/qbufcall callback procedure executing "on" the correct 4284 * stream. Thus it can not reference the q_next pointer in an interrupt 4285 * routine or a timeout, bufcall or esballoc callback routine. Likewise 4286 * it can not reference q_next of a different queue e.g. in a mux that 4287 * passes messages from one queues put/service procedure to another queue. 4288 * In all the cases when the driver/module can not access the q_next 4289 * field it must use the *next* versions e.g. canputnext instead of 4290 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...). 4291 * 4292 * 4293 * Assuming that the driver/module conforms to the above constraints 4294 * the STREAMS framework has to avoid stale references to q_next for all 4295 * the framework internal cases which include (but are not limited to): 4296 * - Threads in canput/canputnext/backenable and elsewhere that are 4297 * walking q_next. 4298 * - Messages on a syncq that have a reference to the queue through b_queue. 4299 * - Messages on an outer perimeter (syncq) that have a reference to the 4300 * queue through b_queue. 4301 * - Threads that use q_nfsrv (e.g. canput) to find a queue. 4302 * Note that only canput and bcanput use q_nfsrv without any locking. 4303 * 4304 * The STREAMS framework providing the qprocsoff(9F) guarantees means that 4305 * after qprocsoff returns, the framework has to ensure that no threads can 4306 * enter the put or service routines for the closing read or write-side queue. 4307 * In addition to preventing "direct" entry into the put procedures 4308 * the framework also has to prevent messages being drained from 4309 * the syncq or the outer perimeter. 4310 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only 4311 * mechanism to prevent qwriter(PERIM_OUTER) from running after 4312 * qprocsoff has returned. 4313 * Note that if a module/driver uses put(9F) on one of its own queues 4314 * it is up to the module/driver to ensure that the put() doesn't 4315 * get called when the queue is closing. 4316 * 4317 * 4318 * The framework aspects of the above "contract" is implemented by 4319 * qprocsoff, removeq, and strlock: 4320 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from 4321 * entering the service procedures. 4322 * - strlock acquires the sd_lock and sd_reflock to prevent putnext, 4323 * canputnext, backenable etc from dereferencing the q_next that will 4324 * soon change. 4325 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext 4326 * or other q_next walker that uses claimstr/releasestr to finish. 4327 * - optionally for every syncq in the stream strlock acquires all the 4328 * sq_lock's and waits for all sq_counts to drop to a value that indicates 4329 * that no thread executes in the put or service procedures and that no 4330 * thread is draining into the module/driver. This ensures that no 4331 * open, close, put, service, or qtimeout/qbufcall callback procedure is 4332 * currently executing hence no such thread can end up with the old stale 4333 * q_next value and no canput/backenable can have the old stale 4334 * q_nfsrv/q_next. 4335 * - qdetach (wait_svc) makes sure that any scheduled or running threads 4336 * have either finished or observed the QWCLOSE flag and gone away. 4337 */ 4338 4339 4340 /* 4341 * Get all the locks necessary to change q_next. 4342 * 4343 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the 4344 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that 4345 * the only threads inside the sqncq are threads currently calling removeq(). 4346 * Since threads calling removeq() are in the process of removing their queues 4347 * from the stream, we do not need to worry about them accessing a stale q_next 4348 * pointer and thus we do not need to wait for them to exit (in fact, waiting 4349 * for them can cause deadlock). 4350 * 4351 * This routine is subject to starvation since it does not set any flag to 4352 * prevent threads from entering a module in the stream(i.e. sq_count can 4353 * increase on some syncq while it is waiting on some other syncq.) 4354 * 4355 * Assumes that only one thread attempts to call strlock for a given 4356 * stream. If this is not the case the two threads would deadlock. 4357 * This assumption is guaranteed since strlock is only called by insertq 4358 * and removeq and streams plumbing changes are single-threaded for 4359 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags. 4360 * 4361 * For pipes, it is not difficult to atomically designate a pair of streams 4362 * to be mated. Once mated atomically by the framework the twisted pair remain 4363 * configured that way until dismantled atomically by the framework. 4364 * When plumbing takes place on a twisted stream it is necessary to ensure that 4365 * this operation is done exclusively on the twisted stream since two such 4366 * operations, each initiated on different ends of the pipe will deadlock 4367 * waiting for each other to complete. 4368 * 4369 * On entry, no locks should be held. 4370 * The locks acquired and held by strlock depends on a few factors. 4371 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired 4372 * and held on exit and all sq_count are at an acceptable level. 4373 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with 4374 * sd_refcnt being zero. 4375 */ 4376 4377 static void 4378 strlock(struct stdata *stp, sqlist_t *sqlist) 4379 { 4380 syncql_t *sql, *sql2; 4381 retry: 4382 /* 4383 * Wait for any claimstr to go away. 4384 */ 4385 if (STRMATED(stp)) { 4386 struct stdata *stp1, *stp2; 4387 4388 STRLOCKMATES(stp); 4389 /* 4390 * Note that the selection of locking order is not 4391 * important, just that they are always aquired in 4392 * the same order. To assure this, we choose this 4393 * order based on the value of the pointer, and since 4394 * the pointer will not change for the life of this 4395 * pair, we will always grab the locks in the same 4396 * order (and hence, prevent deadlocks). 4397 */ 4398 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) { 4399 stp1 = stp; 4400 stp2 = stp->sd_mate; 4401 } else { 4402 stp2 = stp; 4403 stp1 = stp->sd_mate; 4404 } 4405 mutex_enter(&stp1->sd_reflock); 4406 if (stp1->sd_refcnt > 0) { 4407 STRUNLOCKMATES(stp); 4408 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock); 4409 mutex_exit(&stp1->sd_reflock); 4410 goto retry; 4411 } 4412 mutex_enter(&stp2->sd_reflock); 4413 if (stp2->sd_refcnt > 0) { 4414 STRUNLOCKMATES(stp); 4415 mutex_exit(&stp1->sd_reflock); 4416 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock); 4417 mutex_exit(&stp2->sd_reflock); 4418 goto retry; 4419 } 4420 STREAM_PUTLOCKS_ENTER(stp1); 4421 STREAM_PUTLOCKS_ENTER(stp2); 4422 } else { 4423 mutex_enter(&stp->sd_lock); 4424 mutex_enter(&stp->sd_reflock); 4425 while (stp->sd_refcnt > 0) { 4426 mutex_exit(&stp->sd_lock); 4427 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock); 4428 if (mutex_tryenter(&stp->sd_lock) == 0) { 4429 mutex_exit(&stp->sd_reflock); 4430 mutex_enter(&stp->sd_lock); 4431 mutex_enter(&stp->sd_reflock); 4432 } 4433 } 4434 STREAM_PUTLOCKS_ENTER(stp); 4435 } 4436 4437 if (sqlist == NULL) 4438 return; 4439 4440 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4441 syncq_t *sq = sql->sql_sq; 4442 uint16_t count; 4443 4444 mutex_enter(SQLOCK(sq)); 4445 count = sq->sq_count; 4446 ASSERT(sq->sq_rmqcount <= count); 4447 SQ_PUTLOCKS_ENTER(sq); 4448 SUM_SQ_PUTCOUNTS(sq, count); 4449 if (count == sq->sq_rmqcount) 4450 continue; 4451 4452 /* Failed - drop all locks that we have acquired so far */ 4453 if (STRMATED(stp)) { 4454 STREAM_PUTLOCKS_EXIT(stp); 4455 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4456 STRUNLOCKMATES(stp); 4457 mutex_exit(&stp->sd_reflock); 4458 mutex_exit(&stp->sd_mate->sd_reflock); 4459 } else { 4460 STREAM_PUTLOCKS_EXIT(stp); 4461 mutex_exit(&stp->sd_lock); 4462 mutex_exit(&stp->sd_reflock); 4463 } 4464 for (sql2 = sqlist->sqlist_head; sql2 != sql; 4465 sql2 = sql2->sql_next) { 4466 SQ_PUTLOCKS_EXIT(sql2->sql_sq); 4467 mutex_exit(SQLOCK(sql2->sql_sq)); 4468 } 4469 4470 /* 4471 * The wait loop below may starve when there are many threads 4472 * claiming the syncq. This is especially a problem with permod 4473 * syncqs (IP). To lessen the impact of the problem we increment 4474 * sq_needexcl and clear fastbits so that putnexts will slow 4475 * down and call sqenable instead of draining right away. 4476 */ 4477 sq->sq_needexcl++; 4478 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4479 while (count > sq->sq_rmqcount) { 4480 sq->sq_flags |= SQ_WANTWAKEUP; 4481 SQ_PUTLOCKS_EXIT(sq); 4482 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4483 count = sq->sq_count; 4484 SQ_PUTLOCKS_ENTER(sq); 4485 SUM_SQ_PUTCOUNTS(sq, count); 4486 } 4487 sq->sq_needexcl--; 4488 if (sq->sq_needexcl == 0) 4489 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4490 SQ_PUTLOCKS_EXIT(sq); 4491 ASSERT(count == sq->sq_rmqcount); 4492 mutex_exit(SQLOCK(sq)); 4493 goto retry; 4494 } 4495 } 4496 4497 /* 4498 * Drop all the locks that strlock acquired. 4499 */ 4500 static void 4501 strunlock(struct stdata *stp, sqlist_t *sqlist) 4502 { 4503 syncql_t *sql; 4504 4505 if (STRMATED(stp)) { 4506 STREAM_PUTLOCKS_EXIT(stp); 4507 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4508 STRUNLOCKMATES(stp); 4509 mutex_exit(&stp->sd_reflock); 4510 mutex_exit(&stp->sd_mate->sd_reflock); 4511 } else { 4512 STREAM_PUTLOCKS_EXIT(stp); 4513 mutex_exit(&stp->sd_lock); 4514 mutex_exit(&stp->sd_reflock); 4515 } 4516 4517 if (sqlist == NULL) 4518 return; 4519 4520 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4521 SQ_PUTLOCKS_EXIT(sql->sql_sq); 4522 mutex_exit(SQLOCK(sql->sql_sq)); 4523 } 4524 } 4525 4526 /* 4527 * When the module has service procedure, we need check if the next 4528 * module which has service procedure is in flow control to trigger 4529 * the backenable. 4530 */ 4531 static void 4532 backenable_insertedq(queue_t *q) 4533 { 4534 qband_t *qbp; 4535 4536 claimstr(q); 4537 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) { 4538 if (q->q_next->q_nfsrv->q_flag & QWANTW) 4539 backenable(q, 0); 4540 4541 qbp = q->q_next->q_nfsrv->q_bandp; 4542 for (; qbp != NULL; qbp = qbp->qb_next) 4543 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL) 4544 backenable(q, qbp->qb_first->b_band); 4545 } 4546 releasestr(q); 4547 } 4548 4549 /* 4550 * Given two read queues, insert a new single one after another. 4551 * 4552 * This routine acquires all the necessary locks in order to change 4553 * q_next and related pointer using strlock(). 4554 * It depends on the stream head ensuring that there are no concurrent 4555 * insertq or removeq on the same stream. The stream head ensures this 4556 * using the flags STWOPEN, STRCLOSE, and STRPLUMB. 4557 * 4558 * Note that no syncq locks are held during the q_next change. This is 4559 * applied to all streams since, unlike removeq, there is no problem of stale 4560 * pointers when adding a module to the stream. Thus drivers/modules that do a 4561 * canput(rq->q_next) would never get a closed/freed queue pointer even if we 4562 * applied this optimization to all streams. 4563 */ 4564 void 4565 insertq(struct stdata *stp, queue_t *new) 4566 { 4567 queue_t *after; 4568 queue_t *wafter; 4569 queue_t *wnew = _WR(new); 4570 boolean_t have_fifo = B_FALSE; 4571 4572 if (new->q_flag & _QINSERTING) { 4573 ASSERT(stp->sd_vnode->v_type != VFIFO); 4574 after = new->q_next; 4575 wafter = _WR(new->q_next); 4576 } else { 4577 after = _RD(stp->sd_wrq); 4578 wafter = stp->sd_wrq; 4579 } 4580 4581 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ, 4582 "insertq:%p, %p", after, new); 4583 ASSERT(after->q_flag & QREADR); 4584 ASSERT(new->q_flag & QREADR); 4585 4586 strlock(stp, NULL); 4587 4588 /* Do we have a FIFO? */ 4589 if (wafter->q_next == after) { 4590 have_fifo = B_TRUE; 4591 wnew->q_next = new; 4592 } else { 4593 wnew->q_next = wafter->q_next; 4594 } 4595 new->q_next = after; 4596 4597 set_nfsrv_ptr(new, wnew, after, wafter); 4598 /* 4599 * set_nfsrv_ptr() needs to know if this is an insertion or not, 4600 * so only reset this flag after calling it. 4601 */ 4602 new->q_flag &= ~_QINSERTING; 4603 4604 if (have_fifo) { 4605 wafter->q_next = wnew; 4606 } else { 4607 if (wafter->q_next) 4608 _OTHERQ(wafter->q_next)->q_next = new; 4609 wafter->q_next = wnew; 4610 } 4611 4612 set_qend(new); 4613 /* The QEND flag might have to be updated for the upstream guy */ 4614 set_qend(after); 4615 4616 ASSERT(_SAMESTR(new) == O_SAMESTR(new)); 4617 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew)); 4618 ASSERT(_SAMESTR(after) == O_SAMESTR(after)); 4619 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter)); 4620 strsetuio(stp); 4621 4622 /* 4623 * If this was a module insertion, bump the push count. 4624 */ 4625 if (!(new->q_flag & QISDRV)) 4626 stp->sd_pushcnt++; 4627 4628 strunlock(stp, NULL); 4629 4630 /* check if the write Q needs backenable */ 4631 backenable_insertedq(wnew); 4632 4633 /* check if the read Q needs backenable */ 4634 backenable_insertedq(new); 4635 } 4636 4637 /* 4638 * Given a read queue, unlink it from any neighbors. 4639 * 4640 * This routine acquires all the necessary locks in order to 4641 * change q_next and related pointers and also guard against 4642 * stale references (e.g. through q_next) to the queue that 4643 * is being removed. It also plays part of the role in ensuring 4644 * that the module's/driver's put procedure doesn't get called 4645 * after qprocsoff returns. 4646 * 4647 * Removeq depends on the stream head ensuring that there are 4648 * no concurrent insertq or removeq on the same stream. The 4649 * stream head ensures this using the flags STWOPEN, STRCLOSE and 4650 * STRPLUMB. 4651 * 4652 * The set of locks needed to remove the queue is different in 4653 * different cases: 4654 * 4655 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after 4656 * waiting for the syncq reference count to drop to 0 indicating that no 4657 * non-close threads are present anywhere in the stream. This ensures that any 4658 * module/driver can reference q_next in its open, close, put, or service 4659 * procedures. 4660 * 4661 * The sq_rmqcount counter tracks the number of threads inside removeq(). 4662 * strlock() ensures that there is either no threads executing inside perimeter 4663 * or there is only a thread calling qprocsoff(). 4664 * 4665 * strlock() compares the value of sq_count with the number of threads inside 4666 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup 4667 * any threads waiting in strlock() when the sq_rmqcount increases. 4668 */ 4669 4670 void 4671 removeq(queue_t *qp) 4672 { 4673 queue_t *wqp = _WR(qp); 4674 struct stdata *stp = STREAM(qp); 4675 sqlist_t *sqlist = NULL; 4676 boolean_t isdriver; 4677 int moved; 4678 syncq_t *sq = qp->q_syncq; 4679 syncq_t *wsq = wqp->q_syncq; 4680 4681 ASSERT(stp); 4682 4683 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ, 4684 "removeq:%p %p", qp, wqp); 4685 ASSERT(qp->q_flag&QREADR); 4686 4687 /* 4688 * For queues using Synchronous streams, we must wait for all threads in 4689 * rwnext() to drain out before proceeding. 4690 */ 4691 if (qp->q_flag & QSYNCSTR) { 4692 /* First, we need wakeup any threads blocked in rwnext() */ 4693 mutex_enter(SQLOCK(sq)); 4694 if (sq->sq_flags & SQ_WANTWAKEUP) { 4695 sq->sq_flags &= ~SQ_WANTWAKEUP; 4696 cv_broadcast(&sq->sq_wait); 4697 } 4698 mutex_exit(SQLOCK(sq)); 4699 4700 if (wsq != sq) { 4701 mutex_enter(SQLOCK(wsq)); 4702 if (wsq->sq_flags & SQ_WANTWAKEUP) { 4703 wsq->sq_flags &= ~SQ_WANTWAKEUP; 4704 cv_broadcast(&wsq->sq_wait); 4705 } 4706 mutex_exit(SQLOCK(wsq)); 4707 } 4708 4709 mutex_enter(QLOCK(qp)); 4710 while (qp->q_rwcnt > 0) { 4711 qp->q_flag |= QWANTRMQSYNC; 4712 cv_wait(&qp->q_wait, QLOCK(qp)); 4713 } 4714 mutex_exit(QLOCK(qp)); 4715 4716 mutex_enter(QLOCK(wqp)); 4717 while (wqp->q_rwcnt > 0) { 4718 wqp->q_flag |= QWANTRMQSYNC; 4719 cv_wait(&wqp->q_wait, QLOCK(wqp)); 4720 } 4721 mutex_exit(QLOCK(wqp)); 4722 } 4723 4724 mutex_enter(SQLOCK(sq)); 4725 sq->sq_rmqcount++; 4726 if (sq->sq_flags & SQ_WANTWAKEUP) { 4727 sq->sq_flags &= ~SQ_WANTWAKEUP; 4728 cv_broadcast(&sq->sq_wait); 4729 } 4730 mutex_exit(SQLOCK(sq)); 4731 4732 isdriver = (qp->q_flag & QISDRV); 4733 4734 sqlist = sqlist_build(qp, stp, STRMATED(stp)); 4735 strlock(stp, sqlist); 4736 4737 reset_nfsrv_ptr(qp, wqp); 4738 4739 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp); 4740 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp); 4741 /* Do we have a FIFO? */ 4742 if (wqp->q_next == qp) { 4743 stp->sd_wrq->q_next = _RD(stp->sd_wrq); 4744 } else { 4745 if (wqp->q_next) 4746 backq(qp)->q_next = qp->q_next; 4747 if (qp->q_next) 4748 backq(wqp)->q_next = wqp->q_next; 4749 } 4750 4751 /* The QEND flag might have to be updated for the upstream guy */ 4752 if (qp->q_next) 4753 set_qend(qp->q_next); 4754 4755 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq)); 4756 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq))); 4757 4758 /* 4759 * Move any messages destined for the put procedures to the next 4760 * syncq in line. Otherwise free them. 4761 */ 4762 moved = 0; 4763 /* 4764 * Quick check to see whether there are any messages or events. 4765 */ 4766 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS)) 4767 moved += propagate_syncq(qp); 4768 if (wqp->q_syncqmsgs != 0 || 4769 (wqp->q_syncq->sq_flags & SQ_EVENTS)) 4770 moved += propagate_syncq(wqp); 4771 4772 strsetuio(stp); 4773 4774 /* 4775 * If this was a module removal, decrement the push count. 4776 */ 4777 if (!isdriver) 4778 stp->sd_pushcnt--; 4779 4780 strunlock(stp, sqlist); 4781 sqlist_free(sqlist); 4782 4783 /* 4784 * Make sure any messages that were propagated are drained. 4785 * Also clear any QFULL bit caused by messages that were propagated. 4786 */ 4787 4788 if (qp->q_next != NULL) { 4789 clr_qfull(qp); 4790 /* 4791 * For the driver calling qprocsoff, propagate_syncq 4792 * frees all the messages instead of putting it in 4793 * the stream head 4794 */ 4795 if (!isdriver && (moved > 0)) 4796 emptysq(qp->q_next->q_syncq); 4797 } 4798 if (wqp->q_next != NULL) { 4799 clr_qfull(wqp); 4800 /* 4801 * We come here for any pop of a module except for the 4802 * case of driver being removed. We don't call emptysq 4803 * if we did not move any messages. This will avoid holding 4804 * PERMOD syncq locks in emptysq 4805 */ 4806 if (moved > 0) 4807 emptysq(wqp->q_next->q_syncq); 4808 } 4809 4810 mutex_enter(SQLOCK(sq)); 4811 sq->sq_rmqcount--; 4812 mutex_exit(SQLOCK(sq)); 4813 } 4814 4815 /* 4816 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or 4817 * SQ_WRITER) on a syncq. 4818 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the 4819 * sync queue and waits until sq_count reaches maxcnt. 4820 * 4821 * if maxcnt is -1 there's no need to grab sq_putlocks since the caller 4822 * does not care about putnext threads that are in the middle of calling put 4823 * entry points. 4824 * 4825 * This routine is used for both inner and outer syncqs. 4826 */ 4827 static void 4828 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) 4829 { 4830 uint16_t count = 0; 4831 4832 mutex_enter(SQLOCK(sq)); 4833 /* 4834 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset. 4835 * SQ_FROZEN will be set if there is a frozen stream that has a 4836 * queue which also refers to this "shared" syncq. 4837 * SQ_BLOCKED will be set if there is "off" queue which also 4838 * refers to this "shared" syncq. 4839 */ 4840 if (maxcnt != -1) { 4841 count = sq->sq_count; 4842 SQ_PUTLOCKS_ENTER(sq); 4843 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4844 SUM_SQ_PUTCOUNTS(sq, count); 4845 } 4846 sq->sq_needexcl++; 4847 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 4848 4849 while ((sq->sq_flags & flag) || 4850 (maxcnt != -1 && count > (unsigned)maxcnt)) { 4851 sq->sq_flags |= SQ_WANTWAKEUP; 4852 if (maxcnt != -1) { 4853 SQ_PUTLOCKS_EXIT(sq); 4854 } 4855 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4856 if (maxcnt != -1) { 4857 count = sq->sq_count; 4858 SQ_PUTLOCKS_ENTER(sq); 4859 SUM_SQ_PUTCOUNTS(sq, count); 4860 } 4861 } 4862 sq->sq_needexcl--; 4863 sq->sq_flags |= flag; 4864 ASSERT(maxcnt == -1 || count == maxcnt); 4865 if (maxcnt != -1) { 4866 if (sq->sq_needexcl == 0) { 4867 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4868 } 4869 SQ_PUTLOCKS_EXIT(sq); 4870 } else if (sq->sq_needexcl == 0) { 4871 SQ_PUTCOUNT_SETFAST(sq); 4872 } 4873 4874 mutex_exit(SQLOCK(sq)); 4875 } 4876 4877 /* 4878 * Reset a flag that was set with blocksq. 4879 * 4880 * Can not use this routine to reset SQ_WRITER. 4881 * 4882 * If "isouter" is set then the syncq is assumed to be an outer perimeter 4883 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread 4884 * to handle the queued qwriter operations. 4885 * 4886 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4887 * sq_putlocks are used. 4888 */ 4889 static void 4890 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) 4891 { 4892 uint16_t flags; 4893 4894 mutex_enter(SQLOCK(sq)); 4895 ASSERT(resetflag != SQ_WRITER); 4896 ASSERT(sq->sq_flags & resetflag); 4897 flags = sq->sq_flags & ~resetflag; 4898 sq->sq_flags = flags; 4899 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) { 4900 if (flags & SQ_WANTWAKEUP) { 4901 flags &= ~SQ_WANTWAKEUP; 4902 cv_broadcast(&sq->sq_wait); 4903 } 4904 sq->sq_flags = flags; 4905 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 4906 if (!isouter) { 4907 /* drain_syncq drops SQLOCK */ 4908 drain_syncq(sq); 4909 return; 4910 } 4911 } 4912 } 4913 mutex_exit(SQLOCK(sq)); 4914 } 4915 4916 /* 4917 * Reset a flag that was set with blocksq. 4918 * Does not drain the syncq. Use emptysq() for that. 4919 * Returns 1 if SQ_QUEUED is set. Otherwise 0. 4920 * 4921 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4922 * sq_putlocks are used. 4923 */ 4924 static int 4925 dropsq(syncq_t *sq, uint16_t resetflag) 4926 { 4927 uint16_t flags; 4928 4929 mutex_enter(SQLOCK(sq)); 4930 ASSERT(sq->sq_flags & resetflag); 4931 flags = sq->sq_flags & ~resetflag; 4932 if (flags & SQ_WANTWAKEUP) { 4933 flags &= ~SQ_WANTWAKEUP; 4934 cv_broadcast(&sq->sq_wait); 4935 } 4936 sq->sq_flags = flags; 4937 mutex_exit(SQLOCK(sq)); 4938 if (flags & SQ_QUEUED) 4939 return (1); 4940 return (0); 4941 } 4942 4943 /* 4944 * Empty all the messages on a syncq. 4945 * 4946 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4947 * sq_putlocks are used. 4948 */ 4949 static void 4950 emptysq(syncq_t *sq) 4951 { 4952 uint16_t flags; 4953 4954 mutex_enter(SQLOCK(sq)); 4955 flags = sq->sq_flags; 4956 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 4957 /* 4958 * To prevent potential recursive invocation of drain_syncq we 4959 * do not call drain_syncq if count is non-zero. 4960 */ 4961 if (sq->sq_count == 0) { 4962 /* drain_syncq() drops SQLOCK */ 4963 drain_syncq(sq); 4964 return; 4965 } else 4966 sqenable(sq); 4967 } 4968 mutex_exit(SQLOCK(sq)); 4969 } 4970 4971 /* 4972 * Ordered insert while removing duplicates. 4973 */ 4974 static void 4975 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp) 4976 { 4977 syncql_t *sqlp, **prev_sqlpp, *new_sqlp; 4978 4979 prev_sqlpp = &sqlist->sqlist_head; 4980 while ((sqlp = *prev_sqlpp) != NULL) { 4981 if (sqlp->sql_sq >= sqp) { 4982 if (sqlp->sql_sq == sqp) /* duplicate */ 4983 return; 4984 break; 4985 } 4986 prev_sqlpp = &sqlp->sql_next; 4987 } 4988 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++]; 4989 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size); 4990 new_sqlp->sql_next = sqlp; 4991 new_sqlp->sql_sq = sqp; 4992 *prev_sqlpp = new_sqlp; 4993 } 4994 4995 /* 4996 * Walk the write side queues until we hit either the driver 4997 * or a twist in the stream (_SAMESTR will return false in both 4998 * these cases) then turn around and walk the read side queues 4999 * back up to the stream head. 5000 */ 5001 static void 5002 sqlist_insertall(sqlist_t *sqlist, queue_t *q) 5003 { 5004 while (q != NULL) { 5005 sqlist_insert(sqlist, q->q_syncq); 5006 5007 if (_SAMESTR(q)) 5008 q = q->q_next; 5009 else if (!(q->q_flag & QREADR)) 5010 q = _RD(q); 5011 else 5012 q = NULL; 5013 } 5014 } 5015 5016 /* 5017 * Allocate and build a list of all syncqs in a stream and the syncq(s) 5018 * associated with the "q" parameter. The resulting list is sorted in a 5019 * canonical order and is free of duplicates. 5020 * Assumes the passed queue is a _RD(q). 5021 */ 5022 static sqlist_t * 5023 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist) 5024 { 5025 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP); 5026 5027 /* 5028 * start with the current queue/qpair 5029 */ 5030 ASSERT(q->q_flag & QREADR); 5031 5032 sqlist_insert(sqlist, q->q_syncq); 5033 sqlist_insert(sqlist, _WR(q)->q_syncq); 5034 5035 sqlist_insertall(sqlist, stp->sd_wrq); 5036 if (do_twist) 5037 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq); 5038 5039 return (sqlist); 5040 } 5041 5042 static sqlist_t * 5043 sqlist_alloc(struct stdata *stp, int kmflag) 5044 { 5045 size_t sqlist_size; 5046 sqlist_t *sqlist; 5047 5048 /* 5049 * Allocate 2 syncql_t's for each pushed module. Note that 5050 * the sqlist_t structure already has 4 syncql_t's built in: 5051 * 2 for the stream head, and 2 for the driver/other stream head. 5052 */ 5053 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt + 5054 sizeof (sqlist_t); 5055 if (STRMATED(stp)) 5056 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt; 5057 sqlist = kmem_alloc(sqlist_size, kmflag); 5058 5059 sqlist->sqlist_head = NULL; 5060 sqlist->sqlist_size = sqlist_size; 5061 sqlist->sqlist_index = 0; 5062 5063 return (sqlist); 5064 } 5065 5066 /* 5067 * Free the list created by sqlist_alloc() 5068 */ 5069 static void 5070 sqlist_free(sqlist_t *sqlist) 5071 { 5072 kmem_free(sqlist, sqlist->sqlist_size); 5073 } 5074 5075 /* 5076 * Prevent any new entries into any syncq in this stream. 5077 * Used by freezestr. 5078 */ 5079 void 5080 strblock(queue_t *q) 5081 { 5082 struct stdata *stp; 5083 syncql_t *sql; 5084 sqlist_t *sqlist; 5085 5086 q = _RD(q); 5087 5088 stp = STREAM(q); 5089 ASSERT(stp != NULL); 5090 5091 /* 5092 * Get a sorted list with all the duplicates removed containing 5093 * all the syncqs referenced by this stream. 5094 */ 5095 sqlist = sqlist_build(q, stp, B_FALSE); 5096 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5097 blocksq(sql->sql_sq, SQ_FROZEN, -1); 5098 sqlist_free(sqlist); 5099 } 5100 5101 /* 5102 * Release the block on new entries into this stream 5103 */ 5104 void 5105 strunblock(queue_t *q) 5106 { 5107 struct stdata *stp; 5108 syncql_t *sql; 5109 sqlist_t *sqlist; 5110 int drain_needed; 5111 5112 q = _RD(q); 5113 5114 /* 5115 * Get a sorted list with all the duplicates removed containing 5116 * all the syncqs referenced by this stream. 5117 * Have to drop the SQ_FROZEN flag on all the syncqs before 5118 * starting to drain them; otherwise the draining might 5119 * cause a freezestr in some module on the stream (which 5120 * would deadlock.) 5121 */ 5122 stp = STREAM(q); 5123 ASSERT(stp != NULL); 5124 sqlist = sqlist_build(q, stp, B_FALSE); 5125 drain_needed = 0; 5126 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5127 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN); 5128 if (drain_needed) { 5129 for (sql = sqlist->sqlist_head; sql != NULL; 5130 sql = sql->sql_next) 5131 emptysq(sql->sql_sq); 5132 } 5133 sqlist_free(sqlist); 5134 } 5135 5136 #ifdef DEBUG 5137 static int 5138 qprocsareon(queue_t *rq) 5139 { 5140 if (rq->q_next == NULL) 5141 return (0); 5142 return (_WR(rq->q_next)->q_next == _WR(rq)); 5143 } 5144 5145 int 5146 qclaimed(queue_t *q) 5147 { 5148 uint_t count; 5149 5150 count = q->q_syncq->sq_count; 5151 SUM_SQ_PUTCOUNTS(q->q_syncq, count); 5152 return (count != 0); 5153 } 5154 5155 /* 5156 * Check if anyone has frozen this stream with freezestr 5157 */ 5158 int 5159 frozenstr(queue_t *q) 5160 { 5161 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0); 5162 } 5163 #endif /* DEBUG */ 5164 5165 /* 5166 * Enter a queue. 5167 * Obsoleted interface. Should not be used. 5168 */ 5169 void 5170 enterq(queue_t *q) 5171 { 5172 entersq(q->q_syncq, SQ_CALLBACK); 5173 } 5174 5175 void 5176 leaveq(queue_t *q) 5177 { 5178 leavesq(q->q_syncq, SQ_CALLBACK); 5179 } 5180 5181 /* 5182 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits 5183 * to check. 5184 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter 5185 * calls and the running of open, close and service procedures. 5186 * 5187 * if c_inner bit is set no need to grab sq_putlocks since we don't care 5188 * if other threads have entered or are entering put entry point. 5189 * 5190 * if c_inner bit is set it might have been posible to use 5191 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize 5192 * open/close path for IP) but since the count may need to be decremented in 5193 * qwait() we wouldn't know which counter to decrement. Currently counter is 5194 * selected by current cpu_seqid and current CPU can change at any moment. XXX 5195 * in the future we might use curthread id bits to select the counter and this 5196 * would stay constant across routine calls. 5197 */ 5198 void 5199 entersq(syncq_t *sq, int entrypoint) 5200 { 5201 uint16_t count = 0; 5202 uint16_t flags; 5203 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 5204 uint16_t type; 5205 uint_t c_inner = entrypoint & SQ_CI; 5206 uint_t c_outer = entrypoint & SQ_CO; 5207 5208 /* 5209 * Increment ref count to keep closes out of this queue. 5210 */ 5211 ASSERT(sq); 5212 ASSERT(c_inner && c_outer); 5213 mutex_enter(SQLOCK(sq)); 5214 flags = sq->sq_flags; 5215 type = sq->sq_type; 5216 if (!(type & c_inner)) { 5217 /* Make sure all putcounts now use slowlock. */ 5218 count = sq->sq_count; 5219 SQ_PUTLOCKS_ENTER(sq); 5220 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5221 SUM_SQ_PUTCOUNTS(sq, count); 5222 sq->sq_needexcl++; 5223 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5224 waitflags |= SQ_MESSAGES; 5225 } 5226 /* 5227 * Wait until we can enter the inner perimeter. 5228 * If we want exclusive access we wait until sq_count is 0. 5229 * We have to do this before entering the outer perimeter in order 5230 * to preserve put/close message ordering. 5231 */ 5232 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) { 5233 sq->sq_flags = flags | SQ_WANTWAKEUP; 5234 if (!(type & c_inner)) { 5235 SQ_PUTLOCKS_EXIT(sq); 5236 } 5237 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5238 if (!(type & c_inner)) { 5239 count = sq->sq_count; 5240 SQ_PUTLOCKS_ENTER(sq); 5241 SUM_SQ_PUTCOUNTS(sq, count); 5242 } 5243 flags = sq->sq_flags; 5244 } 5245 5246 if (!(type & c_inner)) { 5247 ASSERT(sq->sq_needexcl > 0); 5248 sq->sq_needexcl--; 5249 if (sq->sq_needexcl == 0) { 5250 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5251 } 5252 } 5253 5254 /* Check if we need to enter the outer perimeter */ 5255 if (!(type & c_outer)) { 5256 /* 5257 * We have to enter the outer perimeter exclusively before 5258 * we can increment sq_count to avoid deadlock. This implies 5259 * that we have to re-check sq_flags and sq_count. 5260 * 5261 * is it possible to have c_inner set when c_outer is not set? 5262 */ 5263 if (!(type & c_inner)) { 5264 SQ_PUTLOCKS_EXIT(sq); 5265 } 5266 mutex_exit(SQLOCK(sq)); 5267 outer_enter(sq->sq_outer, SQ_GOAWAY); 5268 mutex_enter(SQLOCK(sq)); 5269 flags = sq->sq_flags; 5270 /* 5271 * there should be no need to recheck sq_putcounts 5272 * because outer_enter() has already waited for them to clear 5273 * after setting SQ_WRITER. 5274 */ 5275 count = sq->sq_count; 5276 #ifdef DEBUG 5277 /* 5278 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead 5279 * of doing an ASSERT internally. Others should do 5280 * something like 5281 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0); 5282 * without the need to #ifdef DEBUG it. 5283 */ 5284 SUMCHECK_SQ_PUTCOUNTS(sq, 0); 5285 #endif 5286 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) || 5287 (!(type & c_inner) && count != 0)) { 5288 sq->sq_flags = flags | SQ_WANTWAKEUP; 5289 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5290 count = sq->sq_count; 5291 flags = sq->sq_flags; 5292 } 5293 } 5294 5295 sq->sq_count++; 5296 ASSERT(sq->sq_count != 0); /* Wraparound */ 5297 if (!(type & c_inner)) { 5298 /* Exclusive entry */ 5299 ASSERT(sq->sq_count == 1); 5300 sq->sq_flags |= SQ_EXCL; 5301 if (type & c_outer) { 5302 SQ_PUTLOCKS_EXIT(sq); 5303 } 5304 } 5305 mutex_exit(SQLOCK(sq)); 5306 } 5307 5308 /* 5309 * leave a syncq. announce to framework that closes may proceed. 5310 * c_inner and c_outer specifies which concurrency bits 5311 * to check. 5312 * 5313 * must never be called from driver or module put entry point. 5314 * 5315 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5316 * sq_putlocks are used. 5317 */ 5318 void 5319 leavesq(syncq_t *sq, int entrypoint) 5320 { 5321 uint16_t flags; 5322 uint16_t type; 5323 uint_t c_outer = entrypoint & SQ_CO; 5324 #ifdef DEBUG 5325 uint_t c_inner = entrypoint & SQ_CI; 5326 #endif 5327 5328 /* 5329 * decrement ref count, drain the syncq if possible, and wake up 5330 * any waiting close. 5331 */ 5332 ASSERT(sq); 5333 ASSERT(c_inner && c_outer); 5334 mutex_enter(SQLOCK(sq)); 5335 flags = sq->sq_flags; 5336 type = sq->sq_type; 5337 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) { 5338 5339 if (flags & SQ_WANTWAKEUP) { 5340 flags &= ~SQ_WANTWAKEUP; 5341 cv_broadcast(&sq->sq_wait); 5342 } 5343 if (flags & SQ_WANTEXWAKEUP) { 5344 flags &= ~SQ_WANTEXWAKEUP; 5345 cv_broadcast(&sq->sq_exitwait); 5346 } 5347 5348 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) { 5349 /* 5350 * The syncq needs to be drained. "Exit" the syncq 5351 * before calling drain_syncq. 5352 */ 5353 ASSERT(sq->sq_count != 0); 5354 sq->sq_count--; 5355 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5356 sq->sq_flags = flags & ~SQ_EXCL; 5357 drain_syncq(sq); 5358 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 5359 /* Check if we need to exit the outer perimeter */ 5360 /* XXX will this ever be true? */ 5361 if (!(type & c_outer)) 5362 outer_exit(sq->sq_outer); 5363 return; 5364 } 5365 } 5366 ASSERT(sq->sq_count != 0); 5367 sq->sq_count--; 5368 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5369 sq->sq_flags = flags & ~SQ_EXCL; 5370 mutex_exit(SQLOCK(sq)); 5371 5372 /* Check if we need to exit the outer perimeter */ 5373 if (!(sq->sq_type & c_outer)) 5374 outer_exit(sq->sq_outer); 5375 } 5376 5377 /* 5378 * Prevent q_next from changing in this stream by incrementing sq_count. 5379 * 5380 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5381 * sq_putlocks are used. 5382 */ 5383 void 5384 claimq(queue_t *qp) 5385 { 5386 syncq_t *sq = qp->q_syncq; 5387 5388 mutex_enter(SQLOCK(sq)); 5389 sq->sq_count++; 5390 ASSERT(sq->sq_count != 0); /* Wraparound */ 5391 mutex_exit(SQLOCK(sq)); 5392 } 5393 5394 /* 5395 * Undo claimq. 5396 * 5397 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5398 * sq_putlocks are used. 5399 */ 5400 void 5401 releaseq(queue_t *qp) 5402 { 5403 syncq_t *sq = qp->q_syncq; 5404 uint16_t flags; 5405 5406 mutex_enter(SQLOCK(sq)); 5407 ASSERT(sq->sq_count > 0); 5408 sq->sq_count--; 5409 5410 flags = sq->sq_flags; 5411 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) { 5412 if (flags & SQ_WANTWAKEUP) { 5413 flags &= ~SQ_WANTWAKEUP; 5414 cv_broadcast(&sq->sq_wait); 5415 } 5416 sq->sq_flags = flags; 5417 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5418 /* 5419 * To prevent potential recursive invocation of 5420 * drain_syncq we do not call drain_syncq if count is 5421 * non-zero. 5422 */ 5423 if (sq->sq_count == 0) { 5424 drain_syncq(sq); 5425 return; 5426 } else 5427 sqenable(sq); 5428 } 5429 } 5430 mutex_exit(SQLOCK(sq)); 5431 } 5432 5433 /* 5434 * Prevent q_next from changing in this stream by incrementing sd_refcnt. 5435 */ 5436 void 5437 claimstr(queue_t *qp) 5438 { 5439 struct stdata *stp = STREAM(qp); 5440 5441 mutex_enter(&stp->sd_reflock); 5442 stp->sd_refcnt++; 5443 ASSERT(stp->sd_refcnt != 0); /* Wraparound */ 5444 mutex_exit(&stp->sd_reflock); 5445 } 5446 5447 /* 5448 * Undo claimstr. 5449 */ 5450 void 5451 releasestr(queue_t *qp) 5452 { 5453 struct stdata *stp = STREAM(qp); 5454 5455 mutex_enter(&stp->sd_reflock); 5456 ASSERT(stp->sd_refcnt != 0); 5457 if (--stp->sd_refcnt == 0) 5458 cv_broadcast(&stp->sd_refmonitor); 5459 mutex_exit(&stp->sd_reflock); 5460 } 5461 5462 static syncq_t * 5463 new_syncq(void) 5464 { 5465 return (kmem_cache_alloc(syncq_cache, KM_SLEEP)); 5466 } 5467 5468 static void 5469 free_syncq(syncq_t *sq) 5470 { 5471 ASSERT(sq->sq_head == NULL); 5472 ASSERT(sq->sq_outer == NULL); 5473 ASSERT(sq->sq_callbpend == NULL); 5474 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || 5475 (sq->sq_onext == sq && sq->sq_oprev == sq)); 5476 5477 if (sq->sq_ciputctrl != NULL) { 5478 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 5479 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 5480 sq->sq_nciputctrl, 0); 5481 ASSERT(ciputctrl_cache != NULL); 5482 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 5483 } 5484 5485 sq->sq_tail = NULL; 5486 sq->sq_evhead = NULL; 5487 sq->sq_evtail = NULL; 5488 sq->sq_ciputctrl = NULL; 5489 sq->sq_nciputctrl = 0; 5490 sq->sq_count = 0; 5491 sq->sq_rmqcount = 0; 5492 sq->sq_callbflags = 0; 5493 sq->sq_cancelid = 0; 5494 sq->sq_next = NULL; 5495 sq->sq_needexcl = 0; 5496 sq->sq_svcflags = 0; 5497 sq->sq_nqueues = 0; 5498 sq->sq_pri = 0; 5499 sq->sq_onext = NULL; 5500 sq->sq_oprev = NULL; 5501 sq->sq_flags = 0; 5502 sq->sq_type = 0; 5503 sq->sq_servcount = 0; 5504 5505 kmem_cache_free(syncq_cache, sq); 5506 } 5507 5508 /* Outer perimeter code */ 5509 5510 /* 5511 * The outer syncq uses the fields and flags in the syncq slightly 5512 * differently from the inner syncqs. 5513 * sq_count Incremented when there are pending or running 5514 * writers at the outer perimeter to prevent the set of 5515 * inner syncqs that belong to the outer perimeter from 5516 * changing. 5517 * sq_head/tail List of deferred qwriter(OUTER) operations. 5518 * 5519 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while 5520 * inner syncqs are added to or removed from the 5521 * outer perimeter. 5522 * SQ_QUEUED sq_head/tail has messages or eventsqueued. 5523 * 5524 * SQ_WRITER A thread is currently traversing all the inner syncqs 5525 * setting the SQ_WRITER flag. 5526 */ 5527 5528 /* 5529 * Get write access at the outer perimeter. 5530 * Note that read access is done by entersq, putnext, and put by simply 5531 * incrementing sq_count in the inner syncq. 5532 * 5533 * Waits until "flags" is no longer set in the outer to prevent multiple 5534 * threads from having write access at the same time. SQ_WRITER has to be part 5535 * of "flags". 5536 * 5537 * Increases sq_count on the outer syncq to keep away outer_insert/remove 5538 * until the outer_exit is finished. 5539 * 5540 * outer_enter is vulnerable to starvation since it does not prevent new 5541 * threads from entering the inner syncqs while it is waiting for sq_count to 5542 * go to zero. 5543 */ 5544 void 5545 outer_enter(syncq_t *outer, uint16_t flags) 5546 { 5547 syncq_t *sq; 5548 int wait_needed; 5549 uint16_t count; 5550 5551 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5552 outer->sq_oprev != NULL); 5553 ASSERT(flags & SQ_WRITER); 5554 5555 retry: 5556 mutex_enter(SQLOCK(outer)); 5557 while (outer->sq_flags & flags) { 5558 outer->sq_flags |= SQ_WANTWAKEUP; 5559 cv_wait(&outer->sq_wait, SQLOCK(outer)); 5560 } 5561 5562 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5563 outer->sq_flags |= SQ_WRITER; 5564 outer->sq_count++; 5565 ASSERT(outer->sq_count != 0); /* wraparound */ 5566 wait_needed = 0; 5567 /* 5568 * Set SQ_WRITER on all the inner syncqs while holding 5569 * the SQLOCK on the outer syncq. This ensures that the changing 5570 * of SQ_WRITER is atomic under the outer SQLOCK. 5571 */ 5572 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5573 mutex_enter(SQLOCK(sq)); 5574 count = sq->sq_count; 5575 SQ_PUTLOCKS_ENTER(sq); 5576 sq->sq_flags |= SQ_WRITER; 5577 SUM_SQ_PUTCOUNTS(sq, count); 5578 if (count != 0) 5579 wait_needed = 1; 5580 SQ_PUTLOCKS_EXIT(sq); 5581 mutex_exit(SQLOCK(sq)); 5582 } 5583 mutex_exit(SQLOCK(outer)); 5584 5585 /* 5586 * Get everybody out of the syncqs sequentially. 5587 * Note that we don't actually need to aqiure the PUTLOCKS, since 5588 * we have already cleared the fastbit, and set QWRITER. By 5589 * definition, the count can not increase since putnext will 5590 * take the slowlock path (and the purpose of aquiring the 5591 * putlocks was to make sure it didn't increase while we were 5592 * waiting). 5593 * 5594 * Note that we still aquire the PUTLOCKS to be safe. 5595 */ 5596 if (wait_needed) { 5597 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5598 mutex_enter(SQLOCK(sq)); 5599 count = sq->sq_count; 5600 SQ_PUTLOCKS_ENTER(sq); 5601 SUM_SQ_PUTCOUNTS(sq, count); 5602 while (count != 0) { 5603 sq->sq_flags |= SQ_WANTWAKEUP; 5604 SQ_PUTLOCKS_EXIT(sq); 5605 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5606 count = sq->sq_count; 5607 SQ_PUTLOCKS_ENTER(sq); 5608 SUM_SQ_PUTCOUNTS(sq, count); 5609 } 5610 SQ_PUTLOCKS_EXIT(sq); 5611 mutex_exit(SQLOCK(sq)); 5612 } 5613 /* 5614 * Verify that none of the flags got set while we 5615 * were waiting for the sq_counts to drop. 5616 * If this happens we exit and retry entering the 5617 * outer perimeter. 5618 */ 5619 mutex_enter(SQLOCK(outer)); 5620 if (outer->sq_flags & (flags & ~SQ_WRITER)) { 5621 mutex_exit(SQLOCK(outer)); 5622 outer_exit(outer); 5623 goto retry; 5624 } 5625 mutex_exit(SQLOCK(outer)); 5626 } 5627 } 5628 5629 /* 5630 * Drop the write access at the outer perimeter. 5631 * Read access is dropped implicitly (by putnext, put, and leavesq) by 5632 * decrementing sq_count. 5633 */ 5634 void 5635 outer_exit(syncq_t *outer) 5636 { 5637 syncq_t *sq; 5638 int drain_needed; 5639 uint16_t flags; 5640 5641 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5642 outer->sq_oprev != NULL); 5643 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer))); 5644 5645 /* 5646 * Atomically (from the perspective of threads calling become_writer) 5647 * drop the write access at the outer perimeter by holding 5648 * SQLOCK(outer) across all the dropsq calls and the resetting of 5649 * SQ_WRITER. 5650 * This defines a locking order between the outer perimeter 5651 * SQLOCK and the inner perimeter SQLOCKs. 5652 */ 5653 mutex_enter(SQLOCK(outer)); 5654 flags = outer->sq_flags; 5655 ASSERT(outer->sq_flags & SQ_WRITER); 5656 if (flags & SQ_QUEUED) { 5657 write_now(outer); 5658 flags = outer->sq_flags; 5659 } 5660 5661 /* 5662 * sq_onext is stable since sq_count has not yet been decreased. 5663 * Reset the SQ_WRITER flags in all syncqs. 5664 * After dropping SQ_WRITER on the outer syncq we empty all the 5665 * inner syncqs. 5666 */ 5667 drain_needed = 0; 5668 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5669 drain_needed += dropsq(sq, SQ_WRITER); 5670 ASSERT(!(outer->sq_flags & SQ_QUEUED)); 5671 flags &= ~SQ_WRITER; 5672 if (drain_needed) { 5673 outer->sq_flags = flags; 5674 mutex_exit(SQLOCK(outer)); 5675 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5676 emptysq(sq); 5677 mutex_enter(SQLOCK(outer)); 5678 flags = outer->sq_flags; 5679 } 5680 if (flags & SQ_WANTWAKEUP) { 5681 flags &= ~SQ_WANTWAKEUP; 5682 cv_broadcast(&outer->sq_wait); 5683 } 5684 outer->sq_flags = flags; 5685 ASSERT(outer->sq_count > 0); 5686 outer->sq_count--; 5687 mutex_exit(SQLOCK(outer)); 5688 } 5689 5690 /* 5691 * Add another syncq to an outer perimeter. 5692 * Block out all other access to the outer perimeter while it is being 5693 * changed using blocksq. 5694 * Assumes that the caller has *not* done an outer_enter. 5695 * 5696 * Vulnerable to starvation in blocksq. 5697 */ 5698 static void 5699 outer_insert(syncq_t *outer, syncq_t *sq) 5700 { 5701 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5702 outer->sq_oprev != NULL); 5703 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 5704 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ 5705 5706 /* Get exclusive access to the outer perimeter list */ 5707 blocksq(outer, SQ_BLOCKED, 0); 5708 ASSERT(outer->sq_flags & SQ_BLOCKED); 5709 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5710 5711 mutex_enter(SQLOCK(sq)); 5712 sq->sq_outer = outer; 5713 outer->sq_onext->sq_oprev = sq; 5714 sq->sq_onext = outer->sq_onext; 5715 outer->sq_onext = sq; 5716 sq->sq_oprev = outer; 5717 mutex_exit(SQLOCK(sq)); 5718 unblocksq(outer, SQ_BLOCKED, 1); 5719 } 5720 5721 /* 5722 * Remove a syncq from an outer perimeter. 5723 * Block out all other access to the outer perimeter while it is being 5724 * changed using blocksq. 5725 * Assumes that the caller has *not* done an outer_enter. 5726 * 5727 * Vulnerable to starvation in blocksq. 5728 */ 5729 static void 5730 outer_remove(syncq_t *outer, syncq_t *sq) 5731 { 5732 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5733 outer->sq_oprev != NULL); 5734 ASSERT(sq->sq_outer == outer); 5735 5736 /* Get exclusive access to the outer perimeter list */ 5737 blocksq(outer, SQ_BLOCKED, 0); 5738 ASSERT(outer->sq_flags & SQ_BLOCKED); 5739 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5740 5741 mutex_enter(SQLOCK(sq)); 5742 sq->sq_outer = NULL; 5743 sq->sq_onext->sq_oprev = sq->sq_oprev; 5744 sq->sq_oprev->sq_onext = sq->sq_onext; 5745 sq->sq_oprev = sq->sq_onext = NULL; 5746 mutex_exit(SQLOCK(sq)); 5747 unblocksq(outer, SQ_BLOCKED, 1); 5748 } 5749 5750 /* 5751 * Queue a deferred qwriter(OUTER) callback for this outer perimeter. 5752 * If this is the first callback for this outer perimeter then add 5753 * this outer perimeter to the list of outer perimeters that 5754 * the qwriter_outer_thread will process. 5755 * 5756 * Increments sq_count in the outer syncq to prevent the membership 5757 * of the outer perimeter (in terms of inner syncqs) to change while 5758 * the callback is pending. 5759 */ 5760 static void 5761 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp) 5762 { 5763 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5764 5765 mp->b_prev = (mblk_t *)func; 5766 mp->b_queue = q; 5767 mp->b_next = NULL; 5768 outer->sq_count++; /* Decremented when dequeued */ 5769 ASSERT(outer->sq_count != 0); /* Wraparound */ 5770 if (outer->sq_evhead == NULL) { 5771 /* First message. */ 5772 outer->sq_evhead = outer->sq_evtail = mp; 5773 outer->sq_flags |= SQ_EVENTS; 5774 mutex_exit(SQLOCK(outer)); 5775 STRSTAT(qwr_outer); 5776 (void) taskq_dispatch(streams_taskq, 5777 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP); 5778 } else { 5779 ASSERT(outer->sq_flags & SQ_EVENTS); 5780 outer->sq_evtail->b_next = mp; 5781 outer->sq_evtail = mp; 5782 mutex_exit(SQLOCK(outer)); 5783 } 5784 } 5785 5786 /* 5787 * Try and upgrade to write access at the outer perimeter. If this can 5788 * not be done without blocking then queue the callback to be done 5789 * by the qwriter_outer_thread. 5790 * 5791 * This routine can only be called from put or service procedures plus 5792 * asynchronous callback routines that have properly entered to 5793 * queue (with entersq.) Thus qwriter(OUTER) assumes the caller has one claim 5794 * on the syncq associated with q. 5795 */ 5796 void 5797 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)()) 5798 { 5799 syncq_t *osq, *sq, *outer; 5800 int failed; 5801 uint16_t flags; 5802 5803 osq = q->q_syncq; 5804 outer = osq->sq_outer; 5805 if (outer == NULL) 5806 panic("qwriter(PERIM_OUTER): no outer perimeter"); 5807 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5808 outer->sq_oprev != NULL); 5809 5810 mutex_enter(SQLOCK(outer)); 5811 flags = outer->sq_flags; 5812 /* 5813 * If some thread is traversing sq_next, or if we are blocked by 5814 * outer_insert or outer_remove, or if the we already have queued 5815 * callbacks, then queue this callback for later processing. 5816 * 5817 * Also queue the qwriter for an interrupt thread in order 5818 * to reduce the time spent running at high IPL. 5819 * to identify there are events. 5820 */ 5821 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) { 5822 /* 5823 * Queue the become_writer request. 5824 * The queueing is atomic under SQLOCK(outer) in order 5825 * to synchronize with outer_exit. 5826 * queue_writer will drop the outer SQLOCK 5827 */ 5828 if (flags & SQ_BLOCKED) { 5829 /* Must set SQ_WRITER on inner perimeter */ 5830 mutex_enter(SQLOCK(osq)); 5831 osq->sq_flags |= SQ_WRITER; 5832 mutex_exit(SQLOCK(osq)); 5833 } else { 5834 if (!(flags & SQ_WRITER)) { 5835 /* 5836 * The outer could have been SQ_BLOCKED thus 5837 * SQ_WRITER might not be set on the inner. 5838 */ 5839 mutex_enter(SQLOCK(osq)); 5840 osq->sq_flags |= SQ_WRITER; 5841 mutex_exit(SQLOCK(osq)); 5842 } 5843 ASSERT(osq->sq_flags & SQ_WRITER); 5844 } 5845 queue_writer(outer, func, q, mp); 5846 return; 5847 } 5848 /* 5849 * We are half-way to exclusive access to the outer perimeter. 5850 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove 5851 * while the inner syncqs are traversed. 5852 */ 5853 outer->sq_count++; 5854 ASSERT(outer->sq_count != 0); /* wraparound */ 5855 flags |= SQ_WRITER; 5856 /* 5857 * Check if we can run the function immediately. Mark all 5858 * syncqs with the writer flag to prevent new entries into 5859 * put and service procedures. 5860 * 5861 * Set SQ_WRITER on all the inner syncqs while holding 5862 * the SQLOCK on the outer syncq. This ensures that the changing 5863 * of SQ_WRITER is atomic under the outer SQLOCK. 5864 */ 5865 failed = 0; 5866 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5867 uint16_t count; 5868 uint_t maxcnt = (sq == osq) ? 1 : 0; 5869 5870 mutex_enter(SQLOCK(sq)); 5871 count = sq->sq_count; 5872 SQ_PUTLOCKS_ENTER(sq); 5873 SUM_SQ_PUTCOUNTS(sq, count); 5874 if (sq->sq_count > maxcnt) 5875 failed = 1; 5876 sq->sq_flags |= SQ_WRITER; 5877 SQ_PUTLOCKS_EXIT(sq); 5878 mutex_exit(SQLOCK(sq)); 5879 } 5880 if (failed) { 5881 /* 5882 * Some other thread has a read claim on the outer perimeter. 5883 * Queue the callback for deferred processing. 5884 * 5885 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER 5886 * so that other qwriter(OUTER) calls will queue their 5887 * callbacks as well. queue_writer increments sq_count so we 5888 * decrement to compensate for the our increment. 5889 * 5890 * Dropping SQ_WRITER enables the writer thread to work 5891 * on this outer perimeter. 5892 */ 5893 outer->sq_flags = flags; 5894 queue_writer(outer, func, q, mp); 5895 /* queue_writer dropper the lock */ 5896 mutex_enter(SQLOCK(outer)); 5897 ASSERT(outer->sq_count > 0); 5898 outer->sq_count--; 5899 ASSERT(outer->sq_flags & SQ_WRITER); 5900 flags = outer->sq_flags; 5901 flags &= ~SQ_WRITER; 5902 if (flags & SQ_WANTWAKEUP) { 5903 flags &= ~SQ_WANTWAKEUP; 5904 cv_broadcast(&outer->sq_wait); 5905 } 5906 outer->sq_flags = flags; 5907 mutex_exit(SQLOCK(outer)); 5908 return; 5909 } else { 5910 outer->sq_flags = flags; 5911 mutex_exit(SQLOCK(outer)); 5912 } 5913 5914 /* Can run it immediately */ 5915 (*func)(q, mp); 5916 5917 outer_exit(outer); 5918 } 5919 5920 /* 5921 * Dequeue all writer callbacks from the outer perimeter and run them. 5922 */ 5923 static void 5924 write_now(syncq_t *outer) 5925 { 5926 mblk_t *mp; 5927 queue_t *q; 5928 void (*func)(); 5929 5930 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5931 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5932 outer->sq_oprev != NULL); 5933 while ((mp = outer->sq_evhead) != NULL) { 5934 /* 5935 * queues cannot be placed on the queuelist on the outer 5936 * perimiter. 5937 */ 5938 ASSERT(!(outer->sq_flags & SQ_MESSAGES)); 5939 ASSERT((outer->sq_flags & SQ_EVENTS)); 5940 5941 outer->sq_evhead = mp->b_next; 5942 if (outer->sq_evhead == NULL) { 5943 outer->sq_evtail = NULL; 5944 outer->sq_flags &= ~SQ_EVENTS; 5945 } 5946 ASSERT(outer->sq_count != 0); 5947 outer->sq_count--; /* Incremented when enqueued. */ 5948 mutex_exit(SQLOCK(outer)); 5949 /* 5950 * Drop the message if the queue is closing. 5951 * Make sure that the queue is "claimed" when the callback 5952 * is run in order to satisfy various ASSERTs. 5953 */ 5954 q = mp->b_queue; 5955 func = (void (*)())mp->b_prev; 5956 ASSERT(func != NULL); 5957 mp->b_next = mp->b_prev = NULL; 5958 if (q->q_flag & QWCLOSE) { 5959 freemsg(mp); 5960 } else { 5961 claimq(q); 5962 (*func)(q, mp); 5963 releaseq(q); 5964 } 5965 mutex_enter(SQLOCK(outer)); 5966 } 5967 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5968 } 5969 5970 /* 5971 * The list of messages on the inner syncq is effectively hashed 5972 * by destination queue. These destination queues are doubly 5973 * linked lists (hopefully) in priority order. Messages are then 5974 * put on the queue referenced by the q_sqhead/q_sqtail elements. 5975 * Additional messages are linked together by the b_next/b_prev 5976 * elements in the mblk, with (similar to putq()) the first message 5977 * having a NULL b_prev and the last message having a NULL b_next. 5978 * 5979 * Events, such as qwriter callbacks, are put onto a list in FIFO 5980 * order referenced by sq_evhead, and sq_evtail. This is a singly 5981 * linked list, and messages here MUST be processed in the order queued. 5982 */ 5983 5984 /* 5985 * Run the events on the syncq event list (sq_evhead). 5986 * Assumes there is only one claim on the syncq, it is 5987 * already exclusive (SQ_EXCL set), and the SQLOCK held. 5988 * Messages here are processed in order, with the SQ_EXCL bit 5989 * held all the way through till the last message is processed. 5990 */ 5991 void 5992 sq_run_events(syncq_t *sq) 5993 { 5994 mblk_t *bp; 5995 queue_t *qp; 5996 uint16_t flags = sq->sq_flags; 5997 void (*func)(); 5998 5999 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6000 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6001 sq->sq_oprev == NULL) || 6002 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6003 sq->sq_oprev != NULL)); 6004 6005 ASSERT(flags & SQ_EXCL); 6006 ASSERT(sq->sq_count == 1); 6007 6008 /* 6009 * We need to process all of the events on this list. It 6010 * is possible that new events will be added while we are 6011 * away processing a callback, so on every loop, we start 6012 * back at the beginning of the list. 6013 */ 6014 /* 6015 * We have to reaccess sq_evhead since there is a 6016 * possibility of a new entry while we were running 6017 * the callback. 6018 */ 6019 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { 6020 ASSERT(bp->b_queue->q_syncq == sq); 6021 ASSERT(sq->sq_flags & SQ_EVENTS); 6022 6023 qp = bp->b_queue; 6024 func = (void (*)())bp->b_prev; 6025 ASSERT(func != NULL); 6026 6027 /* 6028 * Messages from the event queue must be taken off in 6029 * FIFO order. 6030 */ 6031 ASSERT(sq->sq_evhead == bp); 6032 sq->sq_evhead = bp->b_next; 6033 6034 if (bp->b_next == NULL) { 6035 /* Deleting last */ 6036 ASSERT(sq->sq_evtail == bp); 6037 sq->sq_evtail = NULL; 6038 sq->sq_flags &= ~SQ_EVENTS; 6039 } 6040 bp->b_prev = bp->b_next = NULL; 6041 ASSERT(bp->b_datap->db_ref != 0); 6042 6043 mutex_exit(SQLOCK(sq)); 6044 6045 (*func)(qp, bp); 6046 6047 mutex_enter(SQLOCK(sq)); 6048 /* 6049 * re-read the flags, since they could have changed. 6050 */ 6051 flags = sq->sq_flags; 6052 ASSERT(flags & SQ_EXCL); 6053 } 6054 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); 6055 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6056 6057 if (flags & SQ_WANTWAKEUP) { 6058 flags &= ~SQ_WANTWAKEUP; 6059 cv_broadcast(&sq->sq_wait); 6060 } 6061 if (flags & SQ_WANTEXWAKEUP) { 6062 flags &= ~SQ_WANTEXWAKEUP; 6063 cv_broadcast(&sq->sq_exitwait); 6064 } 6065 sq->sq_flags = flags; 6066 } 6067 6068 /* 6069 * Put messages on the event list. 6070 * If we can go exclusive now, do so and process the event list, otherwise 6071 * let the last claim service this list (or wake the sqthread). 6072 * This procedure assumes SQLOCK is held. To run the event list, it 6073 * must be called with no claims. 6074 */ 6075 static void 6076 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) 6077 { 6078 uint16_t count; 6079 6080 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6081 ASSERT(func != NULL); 6082 6083 /* 6084 * This is a callback. Add it to the list of callbacks 6085 * and see about upgrading. 6086 */ 6087 mp->b_prev = (mblk_t *)func; 6088 mp->b_queue = q; 6089 mp->b_next = NULL; 6090 if (sq->sq_evhead == NULL) { 6091 sq->sq_evhead = sq->sq_evtail = mp; 6092 sq->sq_flags |= SQ_EVENTS; 6093 } else { 6094 ASSERT(sq->sq_evtail != NULL); 6095 ASSERT(sq->sq_evtail->b_next == NULL); 6096 ASSERT(sq->sq_flags & SQ_EVENTS); 6097 sq->sq_evtail->b_next = mp; 6098 sq->sq_evtail = mp; 6099 } 6100 /* 6101 * We have set SQ_EVENTS, so threads will have to 6102 * unwind out of the perimiter, and new entries will 6103 * not grab a putlock. But we still need to know 6104 * how many threads have already made a claim to the 6105 * syncq, so grab the putlocks, and sum the counts. 6106 * If there are no claims on the syncq, we can upgrade 6107 * to exclusive, and run the event list. 6108 * NOTE: We hold the SQLOCK, so we can just grab the 6109 * putlocks. 6110 */ 6111 count = sq->sq_count; 6112 SQ_PUTLOCKS_ENTER(sq); 6113 SUM_SQ_PUTCOUNTS(sq, count); 6114 /* 6115 * We have no claim, so we need to check if there 6116 * are no others, then we can upgrade. 6117 */ 6118 /* 6119 * There are currently no claims on 6120 * the syncq by this thread (at least on this entry). The thread who has 6121 * the claim should drain syncq. 6122 */ 6123 if (count > 0) { 6124 /* 6125 * Can't upgrade - other threads inside. 6126 */ 6127 SQ_PUTLOCKS_EXIT(sq); 6128 mutex_exit(SQLOCK(sq)); 6129 return; 6130 } 6131 /* 6132 * Need to set SQ_EXCL and make a claim on the syncq. 6133 */ 6134 ASSERT((sq->sq_flags & SQ_EXCL) == 0); 6135 sq->sq_flags |= SQ_EXCL; 6136 ASSERT(sq->sq_count == 0); 6137 sq->sq_count++; 6138 SQ_PUTLOCKS_EXIT(sq); 6139 6140 /* Process the events list */ 6141 sq_run_events(sq); 6142 6143 /* 6144 * Release our claim... 6145 */ 6146 sq->sq_count--; 6147 6148 /* 6149 * And release SQ_EXCL. 6150 * We don't need to acquire the putlocks to release 6151 * SQ_EXCL, since we are exclusive, and hold the SQLOCK. 6152 */ 6153 sq->sq_flags &= ~SQ_EXCL; 6154 6155 /* 6156 * sq_run_events should have released SQ_EXCL 6157 */ 6158 ASSERT(!(sq->sq_flags & SQ_EXCL)); 6159 6160 /* 6161 * If anything happened while we were running the 6162 * events (or was there before), we need to process 6163 * them now. We shouldn't be exclusive sine we 6164 * released the perimiter above (plus, we asserted 6165 * for it). 6166 */ 6167 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) 6168 drain_syncq(sq); 6169 else 6170 mutex_exit(SQLOCK(sq)); 6171 } 6172 6173 /* 6174 * Perform delayed processing. The caller has to make sure that it is safe 6175 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are 6176 * set.) 6177 * 6178 * Assume that the caller has NO claims on the syncq. However, a claim 6179 * on the syncq does not indicate that a thread is draining the syncq. 6180 * There may be more claims on the syncq than there are threads draining 6181 * (i.e. #_threads_draining <= sq_count) 6182 * 6183 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set 6184 * in order to preserve qwriter(OUTER) ordering constraints. 6185 * 6186 * sq_putcount only needs to be checked when dispatching the queued 6187 * writer call for CIPUT sync queue, but this is handled in sq_run_events. 6188 */ 6189 void 6190 drain_syncq(syncq_t *sq) 6191 { 6192 queue_t *qp; 6193 uint16_t count; 6194 uint16_t type = sq->sq_type; 6195 uint16_t flags = sq->sq_flags; 6196 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; 6197 6198 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6199 "drain_syncq start:%p", sq); 6200 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6201 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6202 sq->sq_oprev == NULL) || 6203 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6204 sq->sq_oprev != NULL)); 6205 6206 /* 6207 * Drop SQ_SERVICE flag. 6208 */ 6209 if (bg_service) 6210 sq->sq_svcflags &= ~SQ_SERVICE; 6211 6212 /* 6213 * If SQ_EXCL is set, someone else is processing this syncq - let him 6214 * finish the job. 6215 */ 6216 if (flags & SQ_EXCL) { 6217 if (bg_service) { 6218 ASSERT(sq->sq_servcount != 0); 6219 sq->sq_servcount--; 6220 } 6221 mutex_exit(SQLOCK(sq)); 6222 return; 6223 } 6224 6225 /* 6226 * This routine can be called by a background thread if 6227 * it was scheduled by a hi-priority thread. SO, if there are 6228 * NOT messages queued, return (remember, we have the SQLOCK, 6229 * and it cannot change until we release it). Wakeup any waiters also. 6230 */ 6231 if (!(flags & SQ_QUEUED)) { 6232 if (flags & SQ_WANTWAKEUP) { 6233 flags &= ~SQ_WANTWAKEUP; 6234 cv_broadcast(&sq->sq_wait); 6235 } 6236 if (flags & SQ_WANTEXWAKEUP) { 6237 flags &= ~SQ_WANTEXWAKEUP; 6238 cv_broadcast(&sq->sq_exitwait); 6239 } 6240 sq->sq_flags = flags; 6241 if (bg_service) { 6242 ASSERT(sq->sq_servcount != 0); 6243 sq->sq_servcount--; 6244 } 6245 mutex_exit(SQLOCK(sq)); 6246 return; 6247 } 6248 6249 /* 6250 * If this is not a concurrent put perimiter, we need to 6251 * become exclusive to drain. Also, if not CIPUT, we would 6252 * not have acquired a putlock, so we don't need to check 6253 * the putcounts. If not entering with a claim, we test 6254 * for sq_count == 0. 6255 */ 6256 type = sq->sq_type; 6257 if (!(type & SQ_CIPUT)) { 6258 if (sq->sq_count > 1) { 6259 if (bg_service) { 6260 ASSERT(sq->sq_servcount != 0); 6261 sq->sq_servcount--; 6262 } 6263 mutex_exit(SQLOCK(sq)); 6264 return; 6265 } 6266 sq->sq_flags |= SQ_EXCL; 6267 } 6268 6269 /* 6270 * This is where we make a claim to the syncq. 6271 * This can either be done by incrementing a putlock, or 6272 * the sq_count. But since we already have the SQLOCK 6273 * here, we just bump the sq_count. 6274 * 6275 * Note that after we make a claim, we need to let the code 6276 * fall through to the end of this routine to clean itself 6277 * up. A return in the while loop will put the syncq in a 6278 * very bad state. 6279 */ 6280 sq->sq_count++; 6281 ASSERT(sq->sq_count != 0); /* wraparound */ 6282 6283 while ((flags = sq->sq_flags) & SQ_QUEUED) { 6284 /* 6285 * If we are told to stayaway or went exclusive, 6286 * we are done. 6287 */ 6288 if (flags & (SQ_STAYAWAY)) { 6289 break; 6290 } 6291 6292 /* 6293 * If there are events to run, do so. 6294 * We have one claim to the syncq, so if there are 6295 * more than one, other threads are running. 6296 */ 6297 if (sq->sq_evhead != NULL) { 6298 ASSERT(sq->sq_flags & SQ_EVENTS); 6299 6300 count = sq->sq_count; 6301 SQ_PUTLOCKS_ENTER(sq); 6302 SUM_SQ_PUTCOUNTS(sq, count); 6303 if (count > 1) { 6304 SQ_PUTLOCKS_EXIT(sq); 6305 /* Can't upgrade - other threads inside */ 6306 break; 6307 } 6308 ASSERT((flags & SQ_EXCL) == 0); 6309 sq->sq_flags = flags | SQ_EXCL; 6310 SQ_PUTLOCKS_EXIT(sq); 6311 /* 6312 * we have the only claim, run the events, 6313 * sq_run_events will clear the SQ_EXCL flag. 6314 */ 6315 sq_run_events(sq); 6316 6317 /* 6318 * If this is a CIPUT perimiter, we need 6319 * to drop the SQ_EXCL flag so we can properly 6320 * continue draining the syncq. 6321 */ 6322 if (type & SQ_CIPUT) { 6323 ASSERT(sq->sq_flags & SQ_EXCL); 6324 sq->sq_flags &= ~SQ_EXCL; 6325 } 6326 6327 /* 6328 * And go back to the beginning just in case 6329 * anything changed while we were away. 6330 */ 6331 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); 6332 continue; 6333 } 6334 6335 ASSERT(sq->sq_evhead == NULL); 6336 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6337 6338 /* 6339 * Find the queue that is not draining. 6340 * 6341 * q_draining is protected by QLOCK which we do not hold. 6342 * But if it was set, then a thread was draining, and if it gets 6343 * cleared, then it was because the thread has successfully 6344 * drained the syncq, or a GOAWAY state occured. For the GOAWAY 6345 * state to happen, a thread needs the SQLOCK which we hold, and 6346 * if there was such a flag, we whould have already seen it. 6347 */ 6348 6349 for (qp = sq->sq_head; 6350 qp != NULL && (qp->q_draining || 6351 (qp->q_sqflags & Q_SQDRAINING)); 6352 qp = qp->q_sqnext) 6353 ; 6354 6355 if (qp == NULL) 6356 break; 6357 6358 /* 6359 * We have a queue to work on, and we hold the 6360 * SQLOCK and one claim, call qdrain_syncq. 6361 * This means we need to release the SQLOCK and 6362 * aquire the QLOCK (OK since we have a claim). 6363 * Note that qdrain_syncq will actually dequeue 6364 * this queue from the sq_head list when it is 6365 * convinced all the work is done and release 6366 * the QLOCK before returning. 6367 */ 6368 qp->q_sqflags |= Q_SQDRAINING; 6369 mutex_exit(SQLOCK(sq)); 6370 mutex_enter(QLOCK(qp)); 6371 qdrain_syncq(sq, qp); 6372 mutex_enter(SQLOCK(sq)); 6373 6374 /* The queue is drained */ 6375 ASSERT(qp->q_sqflags & Q_SQDRAINING); 6376 qp->q_sqflags &= ~Q_SQDRAINING; 6377 /* 6378 * NOTE: After this point qp should not be used since it may be 6379 * closed. 6380 */ 6381 } 6382 6383 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6384 flags = sq->sq_flags; 6385 6386 /* 6387 * sq->sq_head cannot change because we hold the 6388 * sqlock. However, a thread CAN decide that it is no longer 6389 * going to drain that queue. However, this should be due to 6390 * a GOAWAY state, and we should see that here. 6391 * 6392 * This loop is not very efficient. One solution may be adding a second 6393 * pointer to the "draining" queue, but it is difficult to do when 6394 * queues are inserted in the middle due to priority ordering. Another 6395 * possibility is to yank the queue out of the sq list and put it onto 6396 * the "draining list" and then put it back if it can't be drained. 6397 */ 6398 6399 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || 6400 (type & SQ_CI) || sq->sq_head->q_draining); 6401 6402 /* Drop SQ_EXCL for non-CIPUT perimiters */ 6403 if (!(type & SQ_CIPUT)) 6404 flags &= ~SQ_EXCL; 6405 ASSERT((flags & SQ_EXCL) == 0); 6406 6407 /* Wake up any waiters. */ 6408 if (flags & SQ_WANTWAKEUP) { 6409 flags &= ~SQ_WANTWAKEUP; 6410 cv_broadcast(&sq->sq_wait); 6411 } 6412 if (flags & SQ_WANTEXWAKEUP) { 6413 flags &= ~SQ_WANTEXWAKEUP; 6414 cv_broadcast(&sq->sq_exitwait); 6415 } 6416 sq->sq_flags = flags; 6417 6418 ASSERT(sq->sq_count != 0); 6419 /* Release our claim. */ 6420 sq->sq_count--; 6421 6422 if (bg_service) { 6423 ASSERT(sq->sq_servcount != 0); 6424 sq->sq_servcount--; 6425 } 6426 6427 mutex_exit(SQLOCK(sq)); 6428 6429 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6430 "drain_syncq end:%p", sq); 6431 } 6432 6433 6434 /* 6435 * 6436 * qdrain_syncq can be called (currently) from only one of two places: 6437 * drain_syncq 6438 * putnext (or some variation of it). 6439 * and eventually 6440 * qwait(_sig) 6441 * 6442 * If called from drain_syncq, we found it in the list 6443 * of queue's needing service, so there is work to be done (or it 6444 * wouldn't be on the list). 6445 * 6446 * If called from some putnext variation, it was because the 6447 * perimiter is open, but messages are blocking a putnext and 6448 * there is not a thread working on it. Now a thread could start 6449 * working on it while we are getting ready to do so ourself, but 6450 * the thread would set the q_draining flag, and we can spin out. 6451 * 6452 * As for qwait(_sig), I think I shall let it continue to call 6453 * drain_syncq directly (after all, it will get here eventually). 6454 * 6455 * qdrain_syncq has to terminate when: 6456 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering 6457 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering 6458 * 6459 * ASSUMES: 6460 * One claim 6461 * QLOCK held 6462 * SQLOCK not held 6463 * Will release QLOCK before returning 6464 */ 6465 void 6466 qdrain_syncq(syncq_t *sq, queue_t *q) 6467 { 6468 mblk_t *bp; 6469 boolean_t do_clr; 6470 #ifdef DEBUG 6471 uint16_t count; 6472 #endif 6473 6474 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6475 "drain_syncq start:%p", sq); 6476 ASSERT(q->q_syncq == sq); 6477 ASSERT(MUTEX_HELD(QLOCK(q))); 6478 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6479 /* 6480 * For non-CIPUT perimiters, we should be called with the 6481 * exclusive bit set already. For non-CIPUT perimiters we 6482 * will be doing a concurrent drain, so it better not be set. 6483 */ 6484 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); 6485 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); 6486 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); 6487 /* 6488 * All outer pointers are set, or none of them are 6489 */ 6490 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6491 sq->sq_oprev == NULL) || 6492 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6493 sq->sq_oprev != NULL)); 6494 #ifdef DEBUG 6495 count = sq->sq_count; 6496 /* 6497 * This is OK without the putlocks, because we have one 6498 * claim either from the sq_count, or a putcount. We could 6499 * get an erroneous value from other counts, but ours won't 6500 * change, so one way or another, we will have at least a 6501 * value of one. 6502 */ 6503 SUM_SQ_PUTCOUNTS(sq, count); 6504 ASSERT(count >= 1); 6505 #endif /* DEBUG */ 6506 6507 /* 6508 * The first thing to do here, is find out if a thread is already 6509 * draining this queue or the queue is closing. If so, we are done, 6510 * just return. Also, if there are no messages, we are done as well. 6511 * Note that we check the q_sqhead since there is s window of 6512 * opportunity for us to enter here because Q_SQQUEUED was set, but is 6513 * not anymore. 6514 */ 6515 if (q->q_draining || (q->q_sqhead == NULL)) { 6516 mutex_exit(QLOCK(q)); 6517 return; 6518 } 6519 6520 /* 6521 * If the perimiter is exclusive, there is nothing we can 6522 * do right now, go away. 6523 * Note that there is nothing to prevent this case from changing 6524 * right after this check, but the spin-out will catch it. 6525 */ 6526 6527 /* Tell other threads that we are draining this queue */ 6528 q->q_draining = 1; /* Protected by QLOCK */ 6529 6530 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) { 6531 6532 /* 6533 * Because we can enter this routine just because 6534 * a putnext is blocked, we need to spin out if 6535 * the perimiter wants to go exclusive as well 6536 * as just blocked. We need to spin out also if 6537 * events are queued on the syncq. 6538 * Don't check for SQ_EXCL, because non-CIPUT 6539 * perimiters would set it, and it can't become 6540 * exclusive while we hold a claim. 6541 */ 6542 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { 6543 break; 6544 } 6545 6546 #ifdef DEBUG 6547 /* 6548 * Since we are in qdrain_syncq, we already know the queue, 6549 * but for sanity, we want to check this against the qp that 6550 * was passed in by bp->b_queue. 6551 */ 6552 6553 ASSERT(bp->b_queue == q); 6554 ASSERT(bp->b_queue->q_syncq == sq); 6555 bp->b_queue = NULL; 6556 6557 /* 6558 * We would have the following check in the DEBUG code: 6559 * 6560 * if (bp->b_prev != NULL) { 6561 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp); 6562 * } 6563 * 6564 * This can't be done, however, since IP modifies qinfo 6565 * structure at run-time (switching between IPv4 qinfo and IPv6 6566 * qinfo), invalidating the check. 6567 * So the assignment to func is left here, but the ASSERT itself 6568 * is removed until the whole issue is resolved. 6569 */ 6570 #endif 6571 ASSERT(q->q_sqhead == bp); 6572 q->q_sqhead = bp->b_next; 6573 bp->b_prev = bp->b_next = NULL; 6574 ASSERT(q->q_syncqmsgs > 0); 6575 mutex_exit(QLOCK(q)); 6576 6577 ASSERT(bp->b_datap->db_ref != 0); 6578 6579 (void) (*q->q_qinfo->qi_putp)(q, bp); 6580 6581 mutex_enter(QLOCK(q)); 6582 /* 6583 * We should decrement q_syncqmsgs only after executing the 6584 * put procedure to avoid a possible race with putnext(). 6585 * In putnext() though it sees Q_SQQUEUED is set, there is 6586 * an optimization which allows putnext to call the put 6587 * procedure directly if (q_syncqmsgs == 0) and thus 6588 * a message reodering could otherwise occur. 6589 */ 6590 q->q_syncqmsgs--; 6591 6592 /* 6593 * Clear QFULL in the next service procedure queue if 6594 * this is the last message destined to that queue. 6595 * 6596 * It would make better sense to have some sort of 6597 * tunable for the low water mark, but these symantics 6598 * are not yet defined. So, alas, we use a constant. 6599 */ 6600 do_clr = (q->q_syncqmsgs == 0); 6601 mutex_exit(QLOCK(q)); 6602 6603 if (do_clr) 6604 clr_qfull(q); 6605 6606 mutex_enter(QLOCK(q)); 6607 /* 6608 * Always clear SQ_EXCL when CIPUT in order to handle 6609 * qwriter(INNER). 6610 */ 6611 /* 6612 * The putp() can call qwriter and get exclusive access 6613 * IFF this is the only claim. So, we need to test for 6614 * this possibility so we can aquire the mutex and clear 6615 * the bit. 6616 */ 6617 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { 6618 mutex_enter(SQLOCK(sq)); 6619 sq->sq_flags &= ~SQ_EXCL; 6620 mutex_exit(SQLOCK(sq)); 6621 } 6622 } 6623 6624 /* 6625 * We should either have no queues on the syncq, or we were 6626 * told to goaway by a waiter (which we will wake up at the 6627 * end of this function). 6628 */ 6629 ASSERT((q->q_sqhead == NULL) || 6630 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); 6631 6632 ASSERT(MUTEX_HELD(QLOCK(q))); 6633 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6634 6635 /* 6636 * Remove the q from the syncq list if all the messages are 6637 * drained. 6638 */ 6639 if (q->q_sqhead == NULL) { 6640 mutex_enter(SQLOCK(sq)); 6641 if (q->q_sqflags & Q_SQQUEUED) 6642 SQRM_Q(sq, q); 6643 mutex_exit(SQLOCK(sq)); 6644 /* 6645 * Since the queue is removed from the list, reset its priority. 6646 */ 6647 q->q_spri = 0; 6648 } 6649 6650 /* 6651 * Remember, the q_draining flag is used to let another 6652 * thread know that there is a thread currently draining 6653 * the messages for a queue. Since we are now done with 6654 * this queue (even if there may be messages still there), 6655 * we need to clear this flag so some thread will work 6656 * on it if needed. 6657 */ 6658 ASSERT(q->q_draining); 6659 q->q_draining = 0; 6660 6661 /* called with a claim, so OK to drop all locks. */ 6662 mutex_exit(QLOCK(q)); 6663 6664 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6665 "drain_syncq end:%p", sq); 6666 } 6667 /* END OF QDRAIN_SYNCQ */ 6668 6669 6670 /* 6671 * This is the mate to qdrain_syncq, except that it is putting the 6672 * message onto the the queue instead draining. Since the 6673 * message is destined for the queue that is selected, there is 6674 * no need to identify the function because the message is 6675 * intended for the put routine for the queue. But this 6676 * routine will do it anyway just in case (but only for debug kernels). 6677 * 6678 * After the message is enqueued on the syncq, it calls putnext_tail() 6679 * which will schedule a background thread to actually process the message. 6680 * 6681 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and 6682 * SQLOCK(sq) and QLOCK(q) are not held. 6683 */ 6684 void 6685 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) 6686 { 6687 queue_t *fq = NULL; 6688 6689 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6690 ASSERT(MUTEX_NOT_HELD(QLOCK(q))); 6691 ASSERT(sq->sq_count > 0); 6692 ASSERT(q->q_syncq == sq); 6693 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6694 sq->sq_oprev == NULL) || 6695 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6696 sq->sq_oprev != NULL)); 6697 6698 mutex_enter(QLOCK(q)); 6699 6700 /* 6701 * Set QFULL in next service procedure queue (that cares) if not 6702 * already set and if there are already more messages on the syncq 6703 * than sq_max_size. If sq_max_size is 0, no flow control will be 6704 * asserted on any syncq. 6705 * 6706 * The fq here is the next queue with a service procedure. 6707 * This is where we would fail canputnext, so this is where we 6708 * need to set QFULL. 6709 * 6710 * LOCKING HIERARCHY: In the case when fq != q we need to 6711 * a) Take QLOCK(fq) to set QFULL flag and 6712 * b) Take sd_reflock in the case of the hot stream to update 6713 * sd_refcnt. 6714 * We already have QLOCK at this point. To avoid cross-locks with 6715 * freezestr() which grabs all QLOCKs and with strlock() which grabs 6716 * both SQLOCK and sd_reflock, we need to drop respective locks first. 6717 */ 6718 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) && 6719 (q->q_syncqmsgs > sq_max_size)) { 6720 if ((fq = q->q_nfsrv) == q) { 6721 fq->q_flag |= QFULL; 6722 } else { 6723 mutex_exit(QLOCK(q)); 6724 mutex_enter(QLOCK(fq)); 6725 fq->q_flag |= QFULL; 6726 mutex_exit(QLOCK(fq)); 6727 mutex_enter(QLOCK(q)); 6728 } 6729 } 6730 6731 #ifdef DEBUG 6732 /* 6733 * This is used for debug in the qfill_syncq/qdrain_syncq case 6734 * to trace the queue that the message is intended for. Note 6735 * that the original use was to identify the queue and function 6736 * to call on the drain. In the new syncq, we have the context 6737 * of the queue that we are draining, so call it's putproc and 6738 * don't rely on the saved values. But for debug this is still 6739 * usefull information. 6740 */ 6741 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp; 6742 mp->b_queue = q; 6743 mp->b_next = NULL; 6744 #endif 6745 ASSERT(q->q_syncq == sq); 6746 /* 6747 * Enqueue the message on the list. 6748 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to 6749 * protect it. So its ok to acquire SQLOCK after SQPUT_MP(). 6750 */ 6751 SQPUT_MP(q, mp); 6752 mutex_enter(SQLOCK(sq)); 6753 6754 /* 6755 * And queue on syncq for scheduling, if not already queued. 6756 * Note that we need the SQLOCK for this, and for testing flags 6757 * at the end to see if we will drain. So grab it now, and 6758 * release it before we call qdrain_syncq or return. 6759 */ 6760 if (!(q->q_sqflags & Q_SQQUEUED)) { 6761 q->q_spri = curthread->t_pri; 6762 SQPUT_Q(sq, q); 6763 } 6764 #ifdef DEBUG 6765 else { 6766 /* 6767 * All of these conditions MUST be true! 6768 */ 6769 ASSERT(sq->sq_tail != NULL); 6770 if (sq->sq_tail == sq->sq_head) { 6771 ASSERT((q->q_sqprev == NULL) && 6772 (q->q_sqnext == NULL)); 6773 } else { 6774 ASSERT((q->q_sqprev != NULL) || 6775 (q->q_sqnext != NULL)); 6776 } 6777 ASSERT(sq->sq_flags & SQ_QUEUED); 6778 ASSERT(q->q_syncqmsgs != 0); 6779 ASSERT(q->q_sqflags & Q_SQQUEUED); 6780 } 6781 #endif 6782 mutex_exit(QLOCK(q)); 6783 /* 6784 * SQLOCK is still held, so sq_count can be safely decremented. 6785 */ 6786 sq->sq_count--; 6787 6788 putnext_tail(sq, q, 0); 6789 /* Should not reference sq or q after this point. */ 6790 } 6791 6792 /* End of qfill_syncq */ 6793 6794 /* 6795 * Remove all messages from a syncq (if qp is NULL) or remove all messages 6796 * that would be put into qp by drain_syncq. 6797 * Used when deleting the syncq (qp == NULL) or when detaching 6798 * a queue (qp != NULL). 6799 * Return non-zero if one or more messages were freed. 6800 * 6801 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 6802 * sq_putlocks are used. 6803 * 6804 * NOTE: This function assumes that it is called from the close() context and 6805 * that all the queues in the syncq are going aay. For this reason it doesn't 6806 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is 6807 * currently valid, but it is useful to rethink this function to behave properly 6808 * in other cases. 6809 */ 6810 int 6811 flush_syncq(syncq_t *sq, queue_t *qp) 6812 { 6813 mblk_t *bp, *mp_head, *mp_next, *mp_prev; 6814 queue_t *q; 6815 int ret = 0; 6816 6817 mutex_enter(SQLOCK(sq)); 6818 6819 /* 6820 * Before we leave, we need to make sure there are no 6821 * events listed for this queue. All events for this queue 6822 * will just be freed. 6823 */ 6824 if (qp != NULL && sq->sq_evhead != NULL) { 6825 ASSERT(sq->sq_flags & SQ_EVENTS); 6826 6827 mp_prev = NULL; 6828 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { 6829 mp_next = bp->b_next; 6830 if (bp->b_queue == qp) { 6831 /* Delete this message */ 6832 if (mp_prev != NULL) { 6833 mp_prev->b_next = mp_next; 6834 /* 6835 * Update sq_evtail if the last element 6836 * is removed. 6837 */ 6838 if (bp == sq->sq_evtail) { 6839 ASSERT(mp_next == NULL); 6840 sq->sq_evtail = mp_prev; 6841 } 6842 } else 6843 sq->sq_evhead = mp_next; 6844 if (sq->sq_evhead == NULL) 6845 sq->sq_flags &= ~SQ_EVENTS; 6846 bp->b_prev = bp->b_next = NULL; 6847 freemsg(bp); 6848 ret++; 6849 } else { 6850 mp_prev = bp; 6851 } 6852 } 6853 } 6854 6855 /* 6856 * Walk sq_head and: 6857 * - match qp if qp is set, remove it's messages 6858 * - all if qp is not set 6859 */ 6860 q = sq->sq_head; 6861 while (q != NULL) { 6862 ASSERT(q->q_syncq == sq); 6863 if ((qp == NULL) || (qp == q)) { 6864 /* 6865 * Yank the messages as a list off the queue 6866 */ 6867 mp_head = q->q_sqhead; 6868 /* 6869 * We do not have QLOCK(q) here (which is safe due to 6870 * assumptions mentioned above). To obtain the lock we 6871 * need to release SQLOCK which may allow lots of things 6872 * to change upon us. This place requires more analysis. 6873 */ 6874 q->q_sqhead = q->q_sqtail = NULL; 6875 ASSERT(mp_head->b_queue && 6876 mp_head->b_queue->q_syncq == sq); 6877 6878 /* 6879 * Free each of the messages. 6880 */ 6881 for (bp = mp_head; bp != NULL; bp = mp_next) { 6882 mp_next = bp->b_next; 6883 bp->b_prev = bp->b_next = NULL; 6884 freemsg(bp); 6885 ret++; 6886 } 6887 /* 6888 * Now remove the queue from the syncq. 6889 */ 6890 ASSERT(q->q_sqflags & Q_SQQUEUED); 6891 SQRM_Q(sq, q); 6892 q->q_spri = 0; 6893 q->q_syncqmsgs = 0; 6894 6895 /* 6896 * If qp was specified, we are done with it and are 6897 * going to drop SQLOCK(sq) and return. We wakeup syncq 6898 * waiters while we still have the SQLOCK. 6899 */ 6900 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { 6901 sq->sq_flags &= ~SQ_WANTWAKEUP; 6902 cv_broadcast(&sq->sq_wait); 6903 } 6904 /* Drop SQLOCK across clr_qfull */ 6905 mutex_exit(SQLOCK(sq)); 6906 6907 /* 6908 * We avoid doing the test that drain_syncq does and 6909 * unconditionally clear qfull for every flushed 6910 * message. Since flush_syncq is only called during 6911 * close this should not be a problem. 6912 */ 6913 clr_qfull(q); 6914 if (qp != NULL) { 6915 return (ret); 6916 } else { 6917 mutex_enter(SQLOCK(sq)); 6918 /* 6919 * The head was removed by SQRM_Q above. 6920 * reread the new head and flush it. 6921 */ 6922 q = sq->sq_head; 6923 } 6924 } else { 6925 q = q->q_sqnext; 6926 } 6927 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6928 } 6929 6930 if (sq->sq_flags & SQ_WANTWAKEUP) { 6931 sq->sq_flags &= ~SQ_WANTWAKEUP; 6932 cv_broadcast(&sq->sq_wait); 6933 } 6934 6935 mutex_exit(SQLOCK(sq)); 6936 return (ret); 6937 } 6938 6939 /* 6940 * Propagate all messages from a syncq to the next syncq that are associated 6941 * with the specified queue. If the queue is attached to a driver or if the 6942 * messages have been added due to a qwriter(PERIM_INNER), free the messages. 6943 * 6944 * Assumes that the stream is strlock()'ed. We don't come here if there 6945 * are no messages to propagate. 6946 * 6947 * NOTE : If the queue is attached to a driver, all the messages are freed 6948 * as there is no point in propagating the messages from the driver syncq 6949 * to the closing stream head which will in turn get freed later. 6950 */ 6951 static int 6952 propagate_syncq(queue_t *qp) 6953 { 6954 mblk_t *bp, *head, *tail, *prev, *next; 6955 syncq_t *sq; 6956 queue_t *nqp; 6957 syncq_t *nsq; 6958 boolean_t isdriver; 6959 int moved = 0; 6960 uint16_t flags; 6961 pri_t priority = curthread->t_pri; 6962 #ifdef DEBUG 6963 void (*func)(); 6964 #endif 6965 6966 sq = qp->q_syncq; 6967 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6968 /* debug macro */ 6969 SQ_PUTLOCKS_HELD(sq); 6970 /* 6971 * As entersq() does not increment the sq_count for 6972 * the write side, check sq_count for non-QPERQ 6973 * perimeters alone. 6974 */ 6975 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); 6976 6977 /* 6978 * propagate_syncq() can be called because of either messages on the 6979 * queue syncq or because on events on the queue syncq. Do actual 6980 * message propagations if there are any messages. 6981 */ 6982 if (qp->q_syncqmsgs) { 6983 isdriver = (qp->q_flag & QISDRV); 6984 6985 if (!isdriver) { 6986 nqp = qp->q_next; 6987 nsq = nqp->q_syncq; 6988 ASSERT(MUTEX_HELD(SQLOCK(nsq))); 6989 /* debug macro */ 6990 SQ_PUTLOCKS_HELD(nsq); 6991 #ifdef DEBUG 6992 func = (void (*)())nqp->q_qinfo->qi_putp; 6993 #endif 6994 } 6995 6996 SQRM_Q(sq, qp); 6997 priority = MAX(qp->q_spri, priority); 6998 qp->q_spri = 0; 6999 head = qp->q_sqhead; 7000 tail = qp->q_sqtail; 7001 qp->q_sqhead = qp->q_sqtail = NULL; 7002 qp->q_syncqmsgs = 0; 7003 7004 /* 7005 * Walk the list of messages, and free them if this is a driver, 7006 * otherwise reset the b_prev and b_queue value to the new putp. 7007 * Afterward, we will just add the head to the end of the next 7008 * syncq, and point the tail to the end of this one. 7009 */ 7010 7011 for (bp = head; bp != NULL; bp = next) { 7012 next = bp->b_next; 7013 if (isdriver) { 7014 bp->b_prev = bp->b_next = NULL; 7015 freemsg(bp); 7016 continue; 7017 } 7018 /* Change the q values for this message */ 7019 bp->b_queue = nqp; 7020 #ifdef DEBUG 7021 bp->b_prev = (mblk_t *)func; 7022 #endif 7023 moved++; 7024 } 7025 /* 7026 * Attach list of messages to the end of the new queue (if there 7027 * is a list of messages). 7028 */ 7029 7030 if (!isdriver && head != NULL) { 7031 ASSERT(tail != NULL); 7032 if (nqp->q_sqhead == NULL) { 7033 nqp->q_sqhead = head; 7034 } else { 7035 ASSERT(nqp->q_sqtail != NULL); 7036 nqp->q_sqtail->b_next = head; 7037 } 7038 nqp->q_sqtail = tail; 7039 /* 7040 * When messages are moved from high priority queue to 7041 * another queue, the destination queue priority is 7042 * upgraded. 7043 */ 7044 7045 if (priority > nqp->q_spri) 7046 nqp->q_spri = priority; 7047 7048 SQPUT_Q(nsq, nqp); 7049 7050 nqp->q_syncqmsgs += moved; 7051 ASSERT(nqp->q_syncqmsgs != 0); 7052 } 7053 } 7054 7055 /* 7056 * Before we leave, we need to make sure there are no 7057 * events listed for this queue. All events for this queue 7058 * will just be freed. 7059 */ 7060 if (sq->sq_evhead != NULL) { 7061 ASSERT(sq->sq_flags & SQ_EVENTS); 7062 prev = NULL; 7063 for (bp = sq->sq_evhead; bp != NULL; bp = next) { 7064 next = bp->b_next; 7065 if (bp->b_queue == qp) { 7066 /* Delete this message */ 7067 if (prev != NULL) { 7068 prev->b_next = next; 7069 /* 7070 * Update sq_evtail if the last element 7071 * is removed. 7072 */ 7073 if (bp == sq->sq_evtail) { 7074 ASSERT(next == NULL); 7075 sq->sq_evtail = prev; 7076 } 7077 } else 7078 sq->sq_evhead = next; 7079 if (sq->sq_evhead == NULL) 7080 sq->sq_flags &= ~SQ_EVENTS; 7081 bp->b_prev = bp->b_next = NULL; 7082 freemsg(bp); 7083 } else { 7084 prev = bp; 7085 } 7086 } 7087 } 7088 7089 flags = sq->sq_flags; 7090 7091 /* Wake up any waiter before leaving. */ 7092 if (flags & SQ_WANTWAKEUP) { 7093 flags &= ~SQ_WANTWAKEUP; 7094 cv_broadcast(&sq->sq_wait); 7095 } 7096 sq->sq_flags = flags; 7097 7098 return (moved); 7099 } 7100 7101 /* 7102 * Try and upgrade to exclusive access at the inner perimeter. If this can 7103 * not be done without blocking then request will be queued on the syncq 7104 * and drain_syncq will run it later. 7105 * 7106 * This routine can only be called from put or service procedures plus 7107 * asynchronous callback routines that have properly entered to 7108 * queue (with entersq.) Thus qwriter_inner assumes the caller has one claim 7109 * on the syncq associated with q. 7110 */ 7111 void 7112 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)()) 7113 { 7114 syncq_t *sq = q->q_syncq; 7115 uint16_t count; 7116 7117 mutex_enter(SQLOCK(sq)); 7118 count = sq->sq_count; 7119 SQ_PUTLOCKS_ENTER(sq); 7120 SUM_SQ_PUTCOUNTS(sq, count); 7121 ASSERT(count >= 1); 7122 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); 7123 7124 if (count == 1) { 7125 /* 7126 * Can upgrade. This case also handles nested qwriter calls 7127 * (when the qwriter callback function calls qwriter). In that 7128 * case SQ_EXCL is already set. 7129 */ 7130 sq->sq_flags |= SQ_EXCL; 7131 SQ_PUTLOCKS_EXIT(sq); 7132 mutex_exit(SQLOCK(sq)); 7133 (*func)(q, mp); 7134 /* 7135 * Assumes that leavesq, putnext, and drain_syncq will reset 7136 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on 7137 * until putnext, leavesq, or drain_syncq drops it. 7138 * That way we handle nested qwriter(INNER) without dropping 7139 * SQ_EXCL until the outermost qwriter callback routine is 7140 * done. 7141 */ 7142 return; 7143 } 7144 SQ_PUTLOCKS_EXIT(sq); 7145 sqfill_events(sq, q, mp, func); 7146 } 7147 7148 /* 7149 * Synchronous callback support functions 7150 */ 7151 7152 /* 7153 * Allocate a callback parameter structure. 7154 * Assumes that caller initializes the flags and the id. 7155 * Acquires SQLOCK(sq) if non-NULL is returned. 7156 */ 7157 callbparams_t * 7158 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) 7159 { 7160 callbparams_t *cbp; 7161 size_t size = sizeof (callbparams_t); 7162 7163 cbp = kmem_alloc(size, kmflags & ~KM_PANIC); 7164 7165 /* 7166 * Only try tryhard allocation if the caller is ready to panic. 7167 * Otherwise just fail. 7168 */ 7169 if (cbp == NULL) { 7170 if (kmflags & KM_PANIC) 7171 cbp = kmem_alloc_tryhard(sizeof (callbparams_t), 7172 &size, kmflags); 7173 else 7174 return (NULL); 7175 } 7176 7177 ASSERT(size >= sizeof (callbparams_t)); 7178 cbp->cbp_size = size; 7179 cbp->cbp_sq = sq; 7180 cbp->cbp_func = func; 7181 cbp->cbp_arg = arg; 7182 mutex_enter(SQLOCK(sq)); 7183 cbp->cbp_next = sq->sq_callbpend; 7184 sq->sq_callbpend = cbp; 7185 return (cbp); 7186 } 7187 7188 void 7189 callbparams_free(syncq_t *sq, callbparams_t *cbp) 7190 { 7191 callbparams_t **pp, *p; 7192 7193 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7194 7195 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7196 if (p == cbp) { 7197 *pp = p->cbp_next; 7198 kmem_free(p, p->cbp_size); 7199 return; 7200 } 7201 } 7202 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7203 "callbparams_free: not found\n")); 7204 } 7205 7206 void 7207 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) 7208 { 7209 callbparams_t **pp, *p; 7210 7211 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7212 7213 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7214 if (p->cbp_id == id && p->cbp_flags == flag) { 7215 *pp = p->cbp_next; 7216 kmem_free(p, p->cbp_size); 7217 return; 7218 } 7219 } 7220 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7221 "callbparams_free_id: not found\n")); 7222 } 7223 7224 /* 7225 * Callback wrapper function used by once-only callbacks that can be 7226 * cancelled (qtimeout and qbufcall) 7227 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be 7228 * cancelled by the qun* functions. 7229 */ 7230 void 7231 qcallbwrapper(void *arg) 7232 { 7233 callbparams_t *cbp = arg; 7234 syncq_t *sq; 7235 uint16_t count = 0; 7236 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 7237 uint16_t type; 7238 7239 sq = cbp->cbp_sq; 7240 mutex_enter(SQLOCK(sq)); 7241 type = sq->sq_type; 7242 if (!(type & SQ_CICB)) { 7243 count = sq->sq_count; 7244 SQ_PUTLOCKS_ENTER(sq); 7245 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 7246 SUM_SQ_PUTCOUNTS(sq, count); 7247 sq->sq_needexcl++; 7248 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 7249 waitflags |= SQ_MESSAGES; 7250 } 7251 /* Can not handle exlusive entry at outer perimeter */ 7252 ASSERT(type & SQ_COCB); 7253 7254 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { 7255 if ((sq->sq_callbflags & cbp->cbp_flags) && 7256 (sq->sq_cancelid == cbp->cbp_id)) { 7257 /* timeout has been cancelled */ 7258 sq->sq_callbflags |= SQ_CALLB_BYPASSED; 7259 callbparams_free(sq, cbp); 7260 if (!(type & SQ_CICB)) { 7261 ASSERT(sq->sq_needexcl > 0); 7262 sq->sq_needexcl--; 7263 if (sq->sq_needexcl == 0) { 7264 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7265 } 7266 SQ_PUTLOCKS_EXIT(sq); 7267 } 7268 mutex_exit(SQLOCK(sq)); 7269 return; 7270 } 7271 sq->sq_flags |= SQ_WANTWAKEUP; 7272 if (!(type & SQ_CICB)) { 7273 SQ_PUTLOCKS_EXIT(sq); 7274 } 7275 cv_wait(&sq->sq_wait, SQLOCK(sq)); 7276 if (!(type & SQ_CICB)) { 7277 count = sq->sq_count; 7278 SQ_PUTLOCKS_ENTER(sq); 7279 SUM_SQ_PUTCOUNTS(sq, count); 7280 } 7281 } 7282 7283 sq->sq_count++; 7284 ASSERT(sq->sq_count != 0); /* Wraparound */ 7285 if (!(type & SQ_CICB)) { 7286 ASSERT(count == 0); 7287 sq->sq_flags |= SQ_EXCL; 7288 ASSERT(sq->sq_needexcl > 0); 7289 sq->sq_needexcl--; 7290 if (sq->sq_needexcl == 0) { 7291 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7292 } 7293 SQ_PUTLOCKS_EXIT(sq); 7294 } 7295 7296 mutex_exit(SQLOCK(sq)); 7297 7298 cbp->cbp_func(cbp->cbp_arg); 7299 7300 /* 7301 * We drop the lock only for leavesq to re-acquire it. 7302 * Possible optimization is inline of leavesq. 7303 */ 7304 mutex_enter(SQLOCK(sq)); 7305 callbparams_free(sq, cbp); 7306 mutex_exit(SQLOCK(sq)); 7307 leavesq(sq, SQ_CALLBACK); 7308 } 7309 7310 /* 7311 * no need to grab sq_putlocks here. See comment in strsubr.h that 7312 * explains when sq_putlocks are used. 7313 * 7314 * sq_count (or one of the sq_putcounts) has already been 7315 * decremented by the caller, and if SQ_QUEUED, we need to call 7316 * drain_syncq (the global syncq drain). 7317 * If putnext_tail is called with the SQ_EXCL bit set, we are in 7318 * one of two states, non-CIPUT perimiter, and we need to clear 7319 * it, or we went exclusive in the put procedure. In any case, 7320 * we want to clear the bit now, and it is probably easier to do 7321 * this at the beginning of this function (remember, we hold 7322 * the SQLOCK). Lastly, if there are other messages queued 7323 * on the syncq (and not for our destination), enable the syncq 7324 * for background work. 7325 */ 7326 7327 /* ARGSUSED */ 7328 void 7329 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) 7330 { 7331 uint16_t flags = sq->sq_flags; 7332 7333 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7334 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 7335 7336 /* Clear SQ_EXCL if set in passflags */ 7337 if (passflags & SQ_EXCL) { 7338 flags &= ~SQ_EXCL; 7339 } 7340 if (flags & SQ_WANTWAKEUP) { 7341 flags &= ~SQ_WANTWAKEUP; 7342 cv_broadcast(&sq->sq_wait); 7343 } 7344 if (flags & SQ_WANTEXWAKEUP) { 7345 flags &= ~SQ_WANTEXWAKEUP; 7346 cv_broadcast(&sq->sq_exitwait); 7347 } 7348 sq->sq_flags = flags; 7349 7350 /* 7351 * We have cleared SQ_EXCL if we were asked to, and started 7352 * the wakeup process for waiters. If there are no writers 7353 * then we need to drain the syncq if we were told to, or 7354 * enable the background thread to do it. 7355 */ 7356 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) { 7357 if ((passflags & SQ_QUEUED) || 7358 (sq->sq_svcflags & SQ_DISABLED)) { 7359 /* drain_syncq will take care of events in the list */ 7360 drain_syncq(sq); 7361 return; 7362 } else if (flags & SQ_QUEUED) { 7363 sqenable(sq); 7364 } 7365 } 7366 /* Drop the SQLOCK on exit */ 7367 mutex_exit(SQLOCK(sq)); 7368 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END, 7369 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); 7370 } 7371 7372 void 7373 set_qend(queue_t *q) 7374 { 7375 mutex_enter(QLOCK(q)); 7376 if (!O_SAMESTR(q)) 7377 q->q_flag |= QEND; 7378 else 7379 q->q_flag &= ~QEND; 7380 mutex_exit(QLOCK(q)); 7381 q = _OTHERQ(q); 7382 mutex_enter(QLOCK(q)); 7383 if (!O_SAMESTR(q)) 7384 q->q_flag |= QEND; 7385 else 7386 q->q_flag &= ~QEND; 7387 mutex_exit(QLOCK(q)); 7388 } 7389 7390 7391 void 7392 clr_qfull(queue_t *q) 7393 { 7394 queue_t *oq = q; 7395 7396 q = q->q_nfsrv; 7397 /* Fast check if there is any work to do before getting the lock. */ 7398 if ((q->q_flag & (QFULL|QWANTW)) == 0) { 7399 return; 7400 } 7401 7402 /* 7403 * Do not reset QFULL (and backenable) if the q_count is the reason 7404 * for QFULL being set. 7405 */ 7406 mutex_enter(QLOCK(q)); 7407 /* 7408 * If both q_count and q_mblkcnt are less than the hiwat mark 7409 */ 7410 if ((q->q_count < q->q_hiwat) && (q->q_mblkcnt < q->q_hiwat)) { 7411 q->q_flag &= ~QFULL; 7412 /* 7413 * A little more confusing, how about this way: 7414 * if someone wants to write, 7415 * AND 7416 * both counts are less than the lowat mark 7417 * OR 7418 * the lowat mark is zero 7419 * THEN 7420 * backenable 7421 */ 7422 if ((q->q_flag & QWANTW) && 7423 (((q->q_count < q->q_lowat) && 7424 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) { 7425 q->q_flag &= ~QWANTW; 7426 mutex_exit(QLOCK(q)); 7427 backenable(oq, 0); 7428 } else 7429 mutex_exit(QLOCK(q)); 7430 } else 7431 mutex_exit(QLOCK(q)); 7432 } 7433 7434 /* 7435 * Set the forward service procedure pointer. 7436 * 7437 * Called at insert-time to cache a queue's next forward service procedure in 7438 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted 7439 * has a service procedure then q_nfsrv points to itself. If the queue to be 7440 * inserted does not have a service procedure, then q_nfsrv points to the next 7441 * queue forward that has a service procedure. If the queue is at the logical 7442 * end of the stream (driver for write side, stream head for the read side) 7443 * and does not have a service procedure, then q_nfsrv also points to itself. 7444 */ 7445 void 7446 set_nfsrv_ptr( 7447 queue_t *rnew, /* read queue pointer to new module */ 7448 queue_t *wnew, /* write queue pointer to new module */ 7449 queue_t *prev_rq, /* read queue pointer to the module above */ 7450 queue_t *prev_wq) /* write queue pointer to the module above */ 7451 { 7452 queue_t *qp; 7453 7454 if (prev_wq->q_next == NULL) { 7455 /* 7456 * Insert the driver, initialize the driver and stream head. 7457 * In this case, prev_rq/prev_wq should be the stream head. 7458 * _I_INSERT does not allow inserting a driver. Make sure 7459 * that it is not an insertion. 7460 */ 7461 ASSERT(!(rnew->q_flag & _QINSERTING)); 7462 wnew->q_nfsrv = wnew; 7463 if (rnew->q_qinfo->qi_srvp) 7464 rnew->q_nfsrv = rnew; 7465 else 7466 rnew->q_nfsrv = prev_rq; 7467 prev_rq->q_nfsrv = prev_rq; 7468 prev_wq->q_nfsrv = prev_wq; 7469 } else { 7470 /* 7471 * set up read side q_nfsrv pointer. This MUST be done 7472 * before setting the write side, because the setting of 7473 * the write side for a fifo may depend on it. 7474 * 7475 * Suppose we have a fifo that only has pipemod pushed. 7476 * pipemod has no read or write service procedures, so 7477 * nfsrv for both pipemod queues points to prev_rq (the 7478 * stream read head). Now push bufmod (which has only a 7479 * read service procedure). Doing the write side first, 7480 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which 7481 * is WRONG; the next queue forward from wnew with a 7482 * service procedure will be rnew, not the stream read head. 7483 * Since the downstream queue (which in the case of a fifo 7484 * is the read queue rnew) can affect upstream queues, it 7485 * needs to be done first. Setting up the read side first 7486 * sets nfsrv for both pipemod queues to rnew and then 7487 * when the write side is set up, wnew-q_nfsrv will also 7488 * point to rnew. 7489 */ 7490 if (rnew->q_qinfo->qi_srvp) { 7491 /* 7492 * use _OTHERQ() because, if this is a pipe, next 7493 * module may have been pushed from other end and 7494 * q_next could be a read queue. 7495 */ 7496 qp = _OTHERQ(prev_wq->q_next); 7497 while (qp && qp->q_nfsrv != qp) { 7498 qp->q_nfsrv = rnew; 7499 qp = backq(qp); 7500 } 7501 rnew->q_nfsrv = rnew; 7502 } else 7503 rnew->q_nfsrv = prev_rq->q_nfsrv; 7504 7505 /* set up write side q_nfsrv pointer */ 7506 if (wnew->q_qinfo->qi_srvp) { 7507 wnew->q_nfsrv = wnew; 7508 7509 /* 7510 * For insertion, need to update nfsrv of the modules 7511 * above which do not have a service routine. 7512 */ 7513 if (rnew->q_flag & _QINSERTING) { 7514 for (qp = prev_wq; 7515 qp != NULL && qp->q_nfsrv != qp; 7516 qp = backq(qp)) { 7517 qp->q_nfsrv = wnew->q_nfsrv; 7518 } 7519 } 7520 } else { 7521 if (prev_wq->q_next == prev_rq) 7522 /* 7523 * Since prev_wq/prev_rq are the middle of a 7524 * fifo, wnew/rnew will also be the middle of 7525 * a fifo and wnew's nfsrv is same as rnew's. 7526 */ 7527 wnew->q_nfsrv = rnew->q_nfsrv; 7528 else 7529 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv; 7530 } 7531 } 7532 } 7533 7534 /* 7535 * Reset the forward service procedure pointer; called at remove-time. 7536 */ 7537 void 7538 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp) 7539 { 7540 queue_t *tmp_qp; 7541 7542 /* Reset the write side q_nfsrv pointer for _I_REMOVE */ 7543 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) { 7544 for (tmp_qp = backq(wqp); 7545 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp; 7546 tmp_qp = backq(tmp_qp)) { 7547 tmp_qp->q_nfsrv = wqp->q_nfsrv; 7548 } 7549 } 7550 7551 /* reset the read side q_nfsrv pointer */ 7552 if (rqp->q_qinfo->qi_srvp) { 7553 if (wqp->q_next) { /* non-driver case */ 7554 tmp_qp = _OTHERQ(wqp->q_next); 7555 while (tmp_qp && tmp_qp->q_nfsrv == rqp) { 7556 /* Note that rqp->q_next cannot be NULL */ 7557 ASSERT(rqp->q_next != NULL); 7558 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv; 7559 tmp_qp = backq(tmp_qp); 7560 } 7561 } 7562 } 7563 } 7564 7565 /* 7566 * This routine should be called after all stream geometry changes to update 7567 * the stream head cached struio() rd/wr queue pointers. Note must be called 7568 * with the streamlock()ed. 7569 * 7570 * Note: only enables Synchronous STREAMS for a side of a Stream which has 7571 * an explicit synchronous barrier module queue. That is, a queue that 7572 * has specified a struio() type. 7573 */ 7574 static void 7575 strsetuio(stdata_t *stp) 7576 { 7577 queue_t *wrq; 7578 7579 if (stp->sd_flag & STPLEX) { 7580 /* 7581 * Not stremahead, but a mux, so no Synchronous STREAMS. 7582 */ 7583 stp->sd_struiowrq = NULL; 7584 stp->sd_struiordq = NULL; 7585 return; 7586 } 7587 /* 7588 * Scan the write queue(s) while synchronous 7589 * until we find a qinfo uio type specified. 7590 */ 7591 wrq = stp->sd_wrq->q_next; 7592 while (wrq) { 7593 if (wrq->q_struiot == STRUIOT_NONE) { 7594 wrq = 0; 7595 break; 7596 } 7597 if (wrq->q_struiot != STRUIOT_DONTCARE) 7598 break; 7599 if (! _SAMESTR(wrq)) { 7600 wrq = 0; 7601 break; 7602 } 7603 wrq = wrq->q_next; 7604 } 7605 stp->sd_struiowrq = wrq; 7606 /* 7607 * Scan the read queue(s) while synchronous 7608 * until we find a qinfo uio type specified. 7609 */ 7610 wrq = stp->sd_wrq->q_next; 7611 while (wrq) { 7612 if (_RD(wrq)->q_struiot == STRUIOT_NONE) { 7613 wrq = 0; 7614 break; 7615 } 7616 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE) 7617 break; 7618 if (! _SAMESTR(wrq)) { 7619 wrq = 0; 7620 break; 7621 } 7622 wrq = wrq->q_next; 7623 } 7624 stp->sd_struiordq = wrq ? _RD(wrq) : 0; 7625 } 7626 7627 /* 7628 * pass_wput, unblocks the passthru queues, so that 7629 * messages can arrive at muxs lower read queue, before 7630 * I_LINK/I_UNLINK is acked/nacked. 7631 */ 7632 static void 7633 pass_wput(queue_t *q, mblk_t *mp) 7634 { 7635 syncq_t *sq; 7636 7637 sq = _RD(q)->q_syncq; 7638 if (sq->sq_flags & SQ_BLOCKED) 7639 unblocksq(sq, SQ_BLOCKED, 0); 7640 putnext(q, mp); 7641 } 7642 7643 /* 7644 * Set up queues for the link/unlink. 7645 * Create a new queue and block it and then insert it 7646 * below the stream head on the lower stream. 7647 * This prevents any messages from arriving during the setq 7648 * as well as while the mux is processing the LINK/I_UNLINK. 7649 * The blocked passq is unblocked once the LINK/I_UNLINK has 7650 * been acked or nacked or if a message is generated and sent 7651 * down muxs write put procedure. 7652 * see pass_wput(). 7653 * 7654 * After the new queue is inserted, all messages coming from below are 7655 * blocked. The call to strlock will ensure that all activity in the stream head 7656 * read queue syncq is stopped (sq_count drops to zero). 7657 */ 7658 static queue_t * 7659 link_addpassthru(stdata_t *stpdown) 7660 { 7661 queue_t *passq; 7662 sqlist_t sqlist; 7663 7664 passq = allocq(); 7665 STREAM(passq) = STREAM(_WR(passq)) = stpdown; 7666 /* setq might sleep in allocator - avoid holding locks. */ 7667 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ, 7668 SQ_CI|SQ_CO, B_FALSE); 7669 claimq(passq); 7670 blocksq(passq->q_syncq, SQ_BLOCKED, 1); 7671 insertq(STREAM(passq), passq); 7672 7673 /* 7674 * Use strlock() to wait for the stream head sq_count to drop to zero 7675 * since we are going to change q_ptr in the stream head. Note that 7676 * insertq() doesn't wait for any syncq counts to drop to zero. 7677 */ 7678 sqlist.sqlist_head = NULL; 7679 sqlist.sqlist_index = 0; 7680 sqlist.sqlist_size = sizeof (sqlist_t); 7681 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq); 7682 strlock(stpdown, &sqlist); 7683 strunlock(stpdown, &sqlist); 7684 7685 releaseq(passq); 7686 return (passq); 7687 } 7688 7689 /* 7690 * Let messages flow up into the mux by removing 7691 * the passq. 7692 */ 7693 static void 7694 link_rempassthru(queue_t *passq) 7695 { 7696 claimq(passq); 7697 removeq(passq); 7698 releaseq(passq); 7699 freeq(passq); 7700 } 7701 7702 /* 7703 * Wait for the condition variable pointed to by `cvp' to be signaled, 7704 * or for `tim' milliseconds to elapse, whichever comes first. If `tim' 7705 * is negative, then there is no time limit. If `nosigs' is non-zero, 7706 * then the wait will be non-interruptible. 7707 * 7708 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout. 7709 */ 7710 clock_t 7711 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs) 7712 { 7713 clock_t ret, now, tick; 7714 7715 if (tim < 0) { 7716 if (nosigs) { 7717 cv_wait(cvp, mp); 7718 ret = 1; 7719 } else { 7720 ret = cv_wait_sig(cvp, mp); 7721 } 7722 } else if (tim > 0) { 7723 /* 7724 * convert milliseconds to clock ticks 7725 */ 7726 tick = MSEC_TO_TICK_ROUNDUP(tim); 7727 time_to_wait(&now, tick); 7728 if (nosigs) { 7729 ret = cv_timedwait(cvp, mp, now); 7730 } else { 7731 ret = cv_timedwait_sig(cvp, mp, now); 7732 } 7733 } else { 7734 ret = -1; 7735 } 7736 return (ret); 7737 } 7738 7739 /* 7740 * Wait until the stream head can determine if it is at the mark but 7741 * don't wait forever to prevent a race condition between the "mark" state 7742 * in the stream head and any mark state in the caller/user of this routine. 7743 * 7744 * This is used by sockets and for a socket it would be incorrect 7745 * to return a failure for SIOCATMARK when there is no data in the receive 7746 * queue and the marked urgent data is traveling up the stream. 7747 * 7748 * This routine waits until the mark is known by waiting for one of these 7749 * three events: 7750 * The stream head read queue becoming non-empty (including an EOF) 7751 * The STRATMARK flag being set. (Due to a MSGMARKNEXT message.) 7752 * The STRNOTATMARK flag being set (which indicates that the transport 7753 * has sent a MSGNOTMARKNEXT message to indicate that it is not at 7754 * the mark). 7755 * 7756 * The routine returns 1 if the stream is at the mark; 0 if it can 7757 * be determined that the stream is not at the mark. 7758 * If the wait times out and it can't determine 7759 * whether or not the stream might be at the mark the routine will return -1. 7760 * 7761 * Note: This routine should only be used when a mark is pending i.e., 7762 * in the socket case the SIGURG has been posted. 7763 * Note2: This can not wakeup just because synchronous streams indicate 7764 * that data is available since it is not possible to use the synchronous 7765 * streams interfaces to determine the b_flag value for the data queued below 7766 * the stream head. 7767 */ 7768 int 7769 strwaitmark(vnode_t *vp) 7770 { 7771 struct stdata *stp = vp->v_stream; 7772 queue_t *rq = _RD(stp->sd_wrq); 7773 int mark; 7774 7775 mutex_enter(&stp->sd_lock); 7776 while (rq->q_first == NULL && 7777 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) { 7778 stp->sd_flag |= RSLEEP; 7779 7780 /* Wait for 100 milliseconds for any state change. */ 7781 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) { 7782 mutex_exit(&stp->sd_lock); 7783 return (-1); 7784 } 7785 } 7786 if (stp->sd_flag & STRATMARK) 7787 mark = 1; 7788 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK)) 7789 mark = 1; 7790 else 7791 mark = 0; 7792 7793 mutex_exit(&stp->sd_lock); 7794 return (mark); 7795 } 7796 7797 /* 7798 * Set a read side error. If persist is set change the socket error 7799 * to persistent. If errfunc is set install the function as the exported 7800 * error handler. 7801 */ 7802 void 7803 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7804 { 7805 struct stdata *stp = vp->v_stream; 7806 7807 mutex_enter(&stp->sd_lock); 7808 stp->sd_rerror = error; 7809 if (error == 0 && errfunc == NULL) 7810 stp->sd_flag &= ~STRDERR; 7811 else 7812 stp->sd_flag |= STRDERR; 7813 if (persist) { 7814 stp->sd_flag &= ~STRDERRNONPERSIST; 7815 } else { 7816 stp->sd_flag |= STRDERRNONPERSIST; 7817 } 7818 stp->sd_rderrfunc = errfunc; 7819 if (error != 0 || errfunc != NULL) { 7820 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7821 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7822 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 7823 7824 mutex_exit(&stp->sd_lock); 7825 pollwakeup(&stp->sd_pollist, POLLERR); 7826 mutex_enter(&stp->sd_lock); 7827 7828 if (stp->sd_sigflags & S_ERROR) 7829 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 7830 } 7831 mutex_exit(&stp->sd_lock); 7832 } 7833 7834 /* 7835 * Set a write side error. If persist is set change the socket error 7836 * to persistent. 7837 */ 7838 void 7839 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7840 { 7841 struct stdata *stp = vp->v_stream; 7842 7843 mutex_enter(&stp->sd_lock); 7844 stp->sd_werror = error; 7845 if (error == 0 && errfunc == NULL) 7846 stp->sd_flag &= ~STWRERR; 7847 else 7848 stp->sd_flag |= STWRERR; 7849 if (persist) { 7850 stp->sd_flag &= ~STWRERRNONPERSIST; 7851 } else { 7852 stp->sd_flag |= STWRERRNONPERSIST; 7853 } 7854 stp->sd_wrerrfunc = errfunc; 7855 if (error != 0 || errfunc != NULL) { 7856 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7857 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7858 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 7859 7860 mutex_exit(&stp->sd_lock); 7861 pollwakeup(&stp->sd_pollist, POLLERR); 7862 mutex_enter(&stp->sd_lock); 7863 7864 if (stp->sd_sigflags & S_ERROR) 7865 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 7866 } 7867 mutex_exit(&stp->sd_lock); 7868 } 7869 7870 /* 7871 * Make the stream return 0 (EOF) when all data has been read. 7872 * No effect on write side. 7873 */ 7874 void 7875 strseteof(vnode_t *vp, int eof) 7876 { 7877 struct stdata *stp = vp->v_stream; 7878 7879 mutex_enter(&stp->sd_lock); 7880 if (!eof) { 7881 stp->sd_flag &= ~STREOF; 7882 mutex_exit(&stp->sd_lock); 7883 return; 7884 } 7885 stp->sd_flag |= STREOF; 7886 if (stp->sd_flag & RSLEEP) { 7887 stp->sd_flag &= ~RSLEEP; 7888 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); 7889 } 7890 7891 mutex_exit(&stp->sd_lock); 7892 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM); 7893 mutex_enter(&stp->sd_lock); 7894 7895 if (stp->sd_sigflags & (S_INPUT|S_RDNORM)) 7896 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0); 7897 mutex_exit(&stp->sd_lock); 7898 } 7899 7900 void 7901 strflushrq(vnode_t *vp, int flag) 7902 { 7903 struct stdata *stp = vp->v_stream; 7904 7905 mutex_enter(&stp->sd_lock); 7906 flushq(_RD(stp->sd_wrq), flag); 7907 mutex_exit(&stp->sd_lock); 7908 } 7909 7910 void 7911 strsetrputhooks(vnode_t *vp, uint_t flags, 7912 msgfunc_t protofunc, msgfunc_t miscfunc) 7913 { 7914 struct stdata *stp = vp->v_stream; 7915 7916 mutex_enter(&stp->sd_lock); 7917 7918 if (protofunc == NULL) 7919 stp->sd_rprotofunc = strrput_proto; 7920 else 7921 stp->sd_rprotofunc = protofunc; 7922 7923 if (miscfunc == NULL) 7924 stp->sd_rmiscfunc = strrput_misc; 7925 else 7926 stp->sd_rmiscfunc = miscfunc; 7927 7928 if (flags & SH_CONSOL_DATA) 7929 stp->sd_rput_opt |= SR_CONSOL_DATA; 7930 else 7931 stp->sd_rput_opt &= ~SR_CONSOL_DATA; 7932 7933 if (flags & SH_SIGALLDATA) 7934 stp->sd_rput_opt |= SR_SIGALLDATA; 7935 else 7936 stp->sd_rput_opt &= ~SR_SIGALLDATA; 7937 7938 if (flags & SH_IGN_ZEROLEN) 7939 stp->sd_rput_opt |= SR_IGN_ZEROLEN; 7940 else 7941 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN; 7942 7943 mutex_exit(&stp->sd_lock); 7944 } 7945 7946 void 7947 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime) 7948 { 7949 struct stdata *stp = vp->v_stream; 7950 7951 mutex_enter(&stp->sd_lock); 7952 stp->sd_closetime = closetime; 7953 7954 if (flags & SH_SIGPIPE) 7955 stp->sd_wput_opt |= SW_SIGPIPE; 7956 else 7957 stp->sd_wput_opt &= ~SW_SIGPIPE; 7958 if (flags & SH_RECHECK_ERR) 7959 stp->sd_wput_opt |= SW_RECHECK_ERR; 7960 else 7961 stp->sd_wput_opt &= ~SW_RECHECK_ERR; 7962 7963 mutex_exit(&stp->sd_lock); 7964 } 7965 7966 /* Used within framework when the queue is already locked */ 7967 void 7968 qenable_locked(queue_t *q) 7969 { 7970 stdata_t *stp = STREAM(q); 7971 7972 ASSERT(MUTEX_HELD(QLOCK(q))); 7973 7974 if (!q->q_qinfo->qi_srvp) 7975 return; 7976 7977 /* 7978 * Do not place on run queue if already enabled or closing. 7979 */ 7980 if (q->q_flag & (QWCLOSE|QENAB)) 7981 return; 7982 7983 /* 7984 * mark queue enabled and place on run list if it is not already being 7985 * serviced. If it is serviced, the runservice() function will detect 7986 * that QENAB is set and call service procedure before clearing 7987 * QINSERVICE flag. 7988 */ 7989 q->q_flag |= QENAB; 7990 if (q->q_flag & QINSERVICE) 7991 return; 7992 7993 /* Record the time of qenable */ 7994 q->q_qtstamp = lbolt; 7995 7996 /* 7997 * Put the queue in the stp list and schedule it for background 7998 * processing if it is not already scheduled or if stream head does not 7999 * intent to process it in the foreground later by setting 8000 * STRS_WILLSERVICE flag. 8001 */ 8002 mutex_enter(&stp->sd_qlock); 8003 /* 8004 * If there are already something on the list, stp flags should show 8005 * intention to drain it. 8006 */ 8007 IMPLY(STREAM_NEEDSERVICE(stp), 8008 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))); 8009 8010 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link); 8011 stp->sd_nqueues++; 8012 8013 /* 8014 * If no one will drain this stream we are the first producer and 8015 * need to schedule it for background thread. 8016 */ 8017 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) { 8018 /* 8019 * No one will service this stream later, so we have to 8020 * schedule it now. 8021 */ 8022 STRSTAT(stenables); 8023 stp->sd_svcflags |= STRS_SCHEDULED; 8024 stp->sd_servid = (void *)taskq_dispatch(streams_taskq, 8025 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE); 8026 8027 if (stp->sd_servid == NULL) { 8028 /* 8029 * Task queue failed so fail over to the backup 8030 * servicing thread. 8031 */ 8032 STRSTAT(taskqfails); 8033 /* 8034 * It is safe to clear STRS_SCHEDULED flag because it 8035 * was set by this thread above. 8036 */ 8037 stp->sd_svcflags &= ~STRS_SCHEDULED; 8038 8039 /* 8040 * Failover scheduling is protected by service_queue 8041 * lock. 8042 */ 8043 mutex_enter(&service_queue); 8044 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q)); 8045 ASSERT(q->q_link == NULL); 8046 /* 8047 * Append the queue to qhead/qtail list. 8048 */ 8049 if (qhead == NULL) 8050 qhead = q; 8051 else 8052 qtail->q_link = q; 8053 qtail = q; 8054 /* 8055 * Clear stp queue list. 8056 */ 8057 stp->sd_qhead = stp->sd_qtail = NULL; 8058 stp->sd_nqueues = 0; 8059 /* 8060 * Wakeup background queue processing thread. 8061 */ 8062 cv_signal(&services_to_run); 8063 mutex_exit(&service_queue); 8064 } 8065 } 8066 mutex_exit(&stp->sd_qlock); 8067 } 8068 8069 static void 8070 queue_service(queue_t *q) 8071 { 8072 /* 8073 * The queue in the list should have 8074 * QENAB flag set and should not have 8075 * QINSERVICE flag set. QINSERVICE is 8076 * set when the queue is dequeued and 8077 * qenable_locked doesn't enqueue a 8078 * queue with QINSERVICE set. 8079 */ 8080 8081 ASSERT(!(q->q_flag & QINSERVICE)); 8082 ASSERT((q->q_flag & QENAB)); 8083 mutex_enter(QLOCK(q)); 8084 q->q_flag &= ~QENAB; 8085 q->q_flag |= QINSERVICE; 8086 mutex_exit(QLOCK(q)); 8087 runservice(q); 8088 } 8089 8090 static void 8091 syncq_service(syncq_t *sq) 8092 { 8093 STRSTAT(syncqservice); 8094 mutex_enter(SQLOCK(sq)); 8095 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); 8096 ASSERT(sq->sq_servcount != 0); 8097 ASSERT(sq->sq_next == NULL); 8098 8099 /* if we came here from the background thread, clear the flag */ 8100 if (sq->sq_svcflags & SQ_BGTHREAD) 8101 sq->sq_svcflags &= ~SQ_BGTHREAD; 8102 8103 /* let drain_syncq know that it's being called in the background */ 8104 sq->sq_svcflags |= SQ_SERVICE; 8105 drain_syncq(sq); 8106 } 8107 8108 static void 8109 qwriter_outer_service(syncq_t *outer) 8110 { 8111 /* 8112 * Note that SQ_WRITER is used on the outer perimeter 8113 * to signal that a qwriter(OUTER) is either investigating 8114 * running or that it is actually running a function. 8115 */ 8116 outer_enter(outer, SQ_BLOCKED|SQ_WRITER); 8117 8118 /* 8119 * All inner syncq are empty and have SQ_WRITER set 8120 * to block entering the outer perimeter. 8121 * 8122 * We do not need to explicitly call write_now since 8123 * outer_exit does it for us. 8124 */ 8125 outer_exit(outer); 8126 } 8127 8128 static void 8129 mblk_free(mblk_t *mp) 8130 { 8131 dblk_t *dbp = mp->b_datap; 8132 frtn_t *frp = dbp->db_frtnp; 8133 8134 mp->b_next = NULL; 8135 if (dbp->db_fthdr != NULL) 8136 str_ftfree(dbp); 8137 8138 ASSERT(dbp->db_fthdr == NULL); 8139 frp->free_func(frp->free_arg); 8140 ASSERT(dbp->db_mblk == mp); 8141 8142 if (dbp->db_credp != NULL) { 8143 crfree(dbp->db_credp); 8144 dbp->db_credp = NULL; 8145 } 8146 dbp->db_cpid = -1; 8147 dbp->db_struioflag = 0; 8148 dbp->db_struioun.cksum.flags = 0; 8149 8150 kmem_cache_free(dbp->db_cache, dbp); 8151 } 8152 8153 /* 8154 * Background processing of the stream queue list. 8155 */ 8156 static void 8157 stream_service(stdata_t *stp) 8158 { 8159 queue_t *q; 8160 8161 mutex_enter(&stp->sd_qlock); 8162 8163 STR_SERVICE(stp, q); 8164 8165 stp->sd_svcflags &= ~STRS_SCHEDULED; 8166 stp->sd_servid = NULL; 8167 cv_signal(&stp->sd_qcv); 8168 mutex_exit(&stp->sd_qlock); 8169 } 8170 8171 /* 8172 * Foreground processing of the stream queue list. 8173 */ 8174 void 8175 stream_runservice(stdata_t *stp) 8176 { 8177 queue_t *q; 8178 8179 mutex_enter(&stp->sd_qlock); 8180 STRSTAT(rservice); 8181 /* 8182 * We are going to drain this stream queue list, so qenable_locked will 8183 * not schedule it until we finish. 8184 */ 8185 stp->sd_svcflags |= STRS_WILLSERVICE; 8186 8187 STR_SERVICE(stp, q); 8188 8189 stp->sd_svcflags &= ~STRS_WILLSERVICE; 8190 mutex_exit(&stp->sd_qlock); 8191 /* 8192 * Help backup background thread to drain the qhead/qtail list. 8193 */ 8194 while (qhead != NULL) { 8195 STRSTAT(qhelps); 8196 mutex_enter(&service_queue); 8197 DQ(q, qhead, qtail, q_link); 8198 mutex_exit(&service_queue); 8199 if (q != NULL) 8200 queue_service(q); 8201 } 8202 } 8203 8204 void 8205 stream_willservice(stdata_t *stp) 8206 { 8207 mutex_enter(&stp->sd_qlock); 8208 stp->sd_svcflags |= STRS_WILLSERVICE; 8209 mutex_exit(&stp->sd_qlock); 8210 } 8211 8212 /* 8213 * Replace the cred currently in the mblk with a different one. 8214 */ 8215 void 8216 mblk_setcred(mblk_t *mp, cred_t *cr) 8217 { 8218 cred_t *ocr = DB_CRED(mp); 8219 8220 ASSERT(cr != NULL); 8221 8222 if (cr != ocr) { 8223 crhold(mp->b_datap->db_credp = cr); 8224 if (ocr != NULL) 8225 crfree(ocr); 8226 } 8227 } 8228 8229 int 8230 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8231 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value, 8232 uint32_t flags, int km_flags) 8233 { 8234 int rc = 0; 8235 8236 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8237 if (mp->b_datap->db_type == M_DATA) { 8238 /* Associate values for M_DATA type */ 8239 mp->b_datap->db_cksumstart = (intptr_t)start; 8240 mp->b_datap->db_cksumstuff = (intptr_t)stuff; 8241 mp->b_datap->db_cksumend = (intptr_t)end; 8242 mp->b_datap->db_struioun.cksum.flags = flags; 8243 mp->b_datap->db_cksum16 = (uint16_t)value; 8244 8245 } else { 8246 pattrinfo_t pa_info; 8247 8248 ASSERT(mmd != NULL); 8249 8250 pa_info.type = PATTR_HCKSUM; 8251 pa_info.len = sizeof (pattr_hcksum_t); 8252 8253 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) { 8254 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf; 8255 8256 hck->hcksum_start_offset = start; 8257 hck->hcksum_stuff_offset = stuff; 8258 hck->hcksum_end_offset = end; 8259 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value; 8260 hck->hcksum_flags = flags; 8261 } 8262 } 8263 return (rc); 8264 } 8265 8266 void 8267 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8268 uint32_t *start, uint32_t *stuff, uint32_t *end, 8269 uint32_t *value, uint32_t *flags) 8270 { 8271 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8272 if (mp->b_datap->db_type == M_DATA) { 8273 if (flags != NULL) { 8274 *flags = mp->b_datap->db_struioun.cksum.flags; 8275 if (*flags & HCK_PARTIALCKSUM) { 8276 if (start != NULL) 8277 *start = (uint32_t) 8278 mp->b_datap->db_cksumstart; 8279 if (stuff != NULL) 8280 *stuff = (uint32_t) 8281 mp->b_datap->db_cksumstuff; 8282 if (end != NULL) 8283 *end = 8284 (uint32_t)mp->b_datap->db_cksumend; 8285 if (value != NULL) 8286 *value = 8287 (uint32_t)mp->b_datap->db_cksum16; 8288 } 8289 } 8290 } else { 8291 pattrinfo_t hck_attr = {PATTR_HCKSUM}; 8292 8293 ASSERT(mmd != NULL); 8294 8295 /* get hardware checksum attribute */ 8296 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) { 8297 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf; 8298 8299 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t)); 8300 if (flags != NULL) 8301 *flags = hck->hcksum_flags; 8302 if (start != NULL) 8303 *start = hck->hcksum_start_offset; 8304 if (stuff != NULL) 8305 *stuff = hck->hcksum_stuff_offset; 8306 if (end != NULL) 8307 *end = hck->hcksum_end_offset; 8308 if (value != NULL) 8309 *value = (uint32_t) 8310 hck->hcksum_cksum_val.inet_cksum; 8311 } 8312 } 8313 } 8314 8315 /* 8316 * Checksum buffer *bp for len bytes with psum partial checksum, 8317 * or 0 if none, and return the 16 bit partial checksum. 8318 */ 8319 unsigned 8320 bcksum(uchar_t *bp, int len, unsigned int psum) 8321 { 8322 int odd = len & 1; 8323 extern unsigned int ip_ocsum(); 8324 8325 if (((intptr_t)bp & 1) == 0 && !odd) { 8326 /* 8327 * Bp is 16 bit aligned and len is multiple of 16 bit word. 8328 */ 8329 return (ip_ocsum((ushort_t *)bp, len >> 1, psum)); 8330 } 8331 if (((intptr_t)bp & 1) != 0) { 8332 /* 8333 * Bp isn't 16 bit aligned. 8334 */ 8335 unsigned int tsum; 8336 8337 #ifdef _LITTLE_ENDIAN 8338 psum += *bp; 8339 #else 8340 psum += *bp << 8; 8341 #endif 8342 len--; 8343 bp++; 8344 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0); 8345 psum += (tsum << 8) & 0xffff | (tsum >> 8); 8346 if (len & 1) { 8347 bp += len - 1; 8348 #ifdef _LITTLE_ENDIAN 8349 psum += *bp << 8; 8350 #else 8351 psum += *bp; 8352 #endif 8353 } 8354 } else { 8355 /* 8356 * Bp is 16 bit aligned. 8357 */ 8358 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum); 8359 if (odd) { 8360 bp += len - 1; 8361 #ifdef _LITTLE_ENDIAN 8362 psum += *bp; 8363 #else 8364 psum += *bp << 8; 8365 #endif 8366 } 8367 } 8368 /* 8369 * Normalize psum to 16 bits before returning the new partial 8370 * checksum. The max psum value before normalization is 0x3FDFE. 8371 */ 8372 return ((psum >> 16) + (psum & 0xFFFF)); 8373 } 8374 8375 boolean_t 8376 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd) 8377 { 8378 boolean_t rc; 8379 8380 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8381 if (DB_TYPE(mp) == M_DATA) { 8382 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0); 8383 } else { 8384 pattrinfo_t zcopy_attr = {PATTR_ZCOPY}; 8385 8386 ASSERT(mmd != NULL); 8387 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL); 8388 } 8389 return (rc); 8390 } 8391 8392 void 8393 freemsgchain(mblk_t *mp) 8394 { 8395 mblk_t *next; 8396 8397 while (mp != NULL) { 8398 next = mp->b_next; 8399 mp->b_next = NULL; 8400 8401 freemsg(mp); 8402 mp = next; 8403 } 8404 } 8405 8406 mblk_t * 8407 copymsgchain(mblk_t *mp) 8408 { 8409 mblk_t *nmp = NULL; 8410 mblk_t **nmpp = &nmp; 8411 8412 for (; mp != NULL; mp = mp->b_next) { 8413 if ((*nmpp = copymsg(mp)) == NULL) { 8414 freemsgchain(nmp); 8415 return (NULL); 8416 } 8417 8418 nmpp = &((*nmpp)->b_next); 8419 } 8420 8421 return (nmp); 8422 } 8423 8424 /* NOTE: Do not add code after this point. */ 8425 #undef QLOCK 8426 8427 /* 8428 * replacement for QLOCK macro for those that can't use it. 8429 */ 8430 kmutex_t * 8431 QLOCK(queue_t *q) 8432 { 8433 return (&(q)->q_lock); 8434 } 8435 8436 /* 8437 * Dummy runqueues/queuerun functions functions for backwards compatibility. 8438 */ 8439 #undef runqueues 8440 void 8441 runqueues(void) 8442 { 8443 } 8444 8445 #undef queuerun 8446 void 8447 queuerun(void) 8448 { 8449 } 8450