1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 22 /* All Rights Reserved */ 23 24 25 /* 26 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 27 * Use is subject to license terms. 28 */ 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 #include <sys/types.h> 33 #include <sys/sysmacros.h> 34 #include <sys/param.h> 35 #include <sys/errno.h> 36 #include <sys/signal.h> 37 #include <sys/proc.h> 38 #include <sys/conf.h> 39 #include <sys/cred.h> 40 #include <sys/user.h> 41 #include <sys/vnode.h> 42 #include <sys/file.h> 43 #include <sys/session.h> 44 #include <sys/stream.h> 45 #include <sys/strsubr.h> 46 #include <sys/stropts.h> 47 #include <sys/poll.h> 48 #include <sys/systm.h> 49 #include <sys/cpuvar.h> 50 #include <sys/uio.h> 51 #include <sys/cmn_err.h> 52 #include <sys/priocntl.h> 53 #include <sys/procset.h> 54 #include <sys/vmem.h> 55 #include <sys/bitmap.h> 56 #include <sys/kmem.h> 57 #include <sys/siginfo.h> 58 #include <sys/vtrace.h> 59 #include <sys/callb.h> 60 #include <sys/debug.h> 61 #include <sys/modctl.h> 62 #include <sys/vmsystm.h> 63 #include <vm/page.h> 64 #include <sys/atomic.h> 65 #include <sys/suntpi.h> 66 #include <sys/strlog.h> 67 #include <sys/promif.h> 68 #include <sys/project.h> 69 #include <sys/vm.h> 70 #include <sys/taskq.h> 71 #include <sys/sunddi.h> 72 #include <sys/sunldi_impl.h> 73 #include <sys/strsun.h> 74 #include <sys/isa_defs.h> 75 #include <sys/multidata.h> 76 #include <sys/pattr.h> 77 #include <sys/strft.h> 78 #include <sys/fs/snode.h> 79 #include <sys/zone.h> 80 81 #define O_SAMESTR(q) (((q)->q_next) && \ 82 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) 83 84 /* 85 * WARNING: 86 * The variables and routines in this file are private, belonging 87 * to the STREAMS subsystem. These should not be used by modules 88 * or drivers. Compatibility will not be guaranteed. 89 */ 90 91 /* 92 * Id value used to distinguish between different multiplexor links. 93 */ 94 static int32_t lnk_id = 0; 95 96 #define STREAMS_LOPRI MINCLSYSPRI 97 static pri_t streams_lopri = STREAMS_LOPRI; 98 99 #define STRSTAT(x) (str_statistics.x.value.ui64++) 100 typedef struct str_stat { 101 kstat_named_t sqenables; 102 kstat_named_t stenables; 103 kstat_named_t syncqservice; 104 kstat_named_t freebs; 105 kstat_named_t qwr_outer; 106 kstat_named_t rservice; 107 kstat_named_t strwaits; 108 kstat_named_t taskqfails; 109 kstat_named_t bufcalls; 110 kstat_named_t qhelps; 111 kstat_named_t qremoved; 112 kstat_named_t sqremoved; 113 kstat_named_t bcwaits; 114 kstat_named_t sqtoomany; 115 } str_stat_t; 116 117 static str_stat_t str_statistics = { 118 { "sqenables", KSTAT_DATA_UINT64 }, 119 { "stenables", KSTAT_DATA_UINT64 }, 120 { "syncqservice", KSTAT_DATA_UINT64 }, 121 { "freebs", KSTAT_DATA_UINT64 }, 122 { "qwr_outer", KSTAT_DATA_UINT64 }, 123 { "rservice", KSTAT_DATA_UINT64 }, 124 { "strwaits", KSTAT_DATA_UINT64 }, 125 { "taskqfails", KSTAT_DATA_UINT64 }, 126 { "bufcalls", KSTAT_DATA_UINT64 }, 127 { "qhelps", KSTAT_DATA_UINT64 }, 128 { "qremoved", KSTAT_DATA_UINT64 }, 129 { "sqremoved", KSTAT_DATA_UINT64 }, 130 { "bcwaits", KSTAT_DATA_UINT64 }, 131 { "sqtoomany", KSTAT_DATA_UINT64 }, 132 }; 133 134 static kstat_t *str_kstat; 135 136 /* 137 * qrunflag was used previously to control background scheduling of queues. It 138 * is not used anymore, but kept here in case some module still wants to access 139 * it via qready() and setqsched macros. 140 */ 141 char qrunflag; /* Unused */ 142 143 /* 144 * Most of the streams scheduling is done via task queues. Task queues may fail 145 * for non-sleep dispatches, so there are two backup threads servicing failed 146 * requests for queues and syncqs. Both of these threads also service failed 147 * dispatches freebs requests. Queues are put in the list specified by `qhead' 148 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs 149 * requests are put into `freebs_list' which has no tail pointer. All three 150 * lists are protected by a single `service_queue' lock and use 151 * `services_to_run' condition variable for signaling background threads. Use of 152 * a single lock should not be a problem because it is only used under heavy 153 * loads when task queues start to fail and at that time it may be a good idea 154 * to throttle scheduling requests. 155 * 156 * NOTE: queues and syncqs should be scheduled by two separate threads because 157 * queue servicing may be blocked waiting for a syncq which may be also 158 * scheduled for background execution. This may create a deadlock when only one 159 * thread is used for both. 160 */ 161 162 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ 163 164 static kmutex_t service_queue; /* protects all of servicing vars */ 165 static kcondvar_t services_to_run; /* wake up background service thread */ 166 static kcondvar_t syncqs_to_run; /* wake up background service thread */ 167 168 /* 169 * List of queues scheduled for background processing dueue to lack of resources 170 * in the task queues. Protected by service_queue lock; 171 */ 172 static struct queue *qhead; 173 static struct queue *qtail; 174 175 /* 176 * Same list for syncqs 177 */ 178 static syncq_t *sqhead; 179 static syncq_t *sqtail; 180 181 static mblk_t *freebs_list; /* list of buffers to free */ 182 183 /* 184 * Backup threads for servicing queues and syncqs 185 */ 186 kthread_t *streams_qbkgrnd_thread; 187 kthread_t *streams_sqbkgrnd_thread; 188 189 /* 190 * Bufcalls related variables. 191 */ 192 struct bclist strbcalls; /* list of waiting bufcalls */ 193 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ 194 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ 195 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ 196 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ 197 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ 198 199 kmutex_t strresources; /* protects global resources */ 200 kmutex_t muxifier; /* single-threads multiplexor creation */ 201 202 extern void time_to_wait(clock_t *, clock_t); 203 204 /* 205 * run_queues is no longer used, but is kept in case some 3-d party 206 * module/driver decides to use it. 207 */ 208 int run_queues = 0; 209 210 /* 211 * sq_max_size is the depth of the syncq (in number of messages) before 212 * qfill_syncq() starts QFULL'ing destination queues. As its primary 213 * consumer - IP is no longer D_MTPERMOD, but there may be other 214 * modules/drivers depend on this syncq flow control, we prefer to 215 * choose a large number as the default value. For potential 216 * performance gain, this value is tunable in /etc/system. 217 */ 218 int sq_max_size = 10000; 219 220 /* 221 * the number of ciputctrl structures per syncq and stream we create when 222 * needed. 223 */ 224 int n_ciputctrl; 225 int max_n_ciputctrl = 16; 226 /* 227 * if n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. 228 */ 229 int min_n_ciputctrl = 2; 230 231 static struct mux_node *mux_nodes; /* mux info for cycle checking */ 232 233 /* 234 * Per-driver/module syncqs 235 * ======================== 236 * 237 * For drivers/modules that use PERMOD or outer syncqs we keep a list of 238 * perdm structures, new entries being added (and new syncqs allocated) when 239 * setq() encounters a module/driver with a streamtab that it hasn't seen 240 * before. 241 * The reason for this mechanism is that some modules and drivers share a 242 * common streamtab and it is necessary for those modules and drivers to also 243 * share a common PERMOD syncq. 244 * 245 * perdm_list --> dm_str == streamtab_1 246 * dm_sq == syncq_1 247 * dm_ref 248 * dm_next --> dm_str == streamtab_2 249 * dm_sq == syncq_2 250 * dm_ref 251 * dm_next --> ... NULL 252 * 253 * The dm_ref field is incremented for each new driver/module that takes 254 * a reference to the perdm structure and hence shares the syncq. 255 * References are held in the fmodsw_impl_t structure for each STREAMS module 256 * or the dev_impl array (indexed by device major number) for each driver. 257 * 258 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL 259 * ^ ^ ^ ^ 260 * | ______________/ | | 261 * | / | | 262 * dev_impl: ...|x|y|... module A module B 263 * 264 * When a module/driver is unloaded the reference count is decremented and, 265 * when it falls to zero, the perdm structure is removed from the list and 266 * the syncq is freed (see rele_dm()). 267 */ 268 perdm_t *perdm_list = NULL; 269 static krwlock_t perdm_rwlock; 270 cdevsw_impl_t *devimpl; 271 272 extern struct qinit strdata; 273 extern struct qinit stwdata; 274 275 static void runservice(queue_t *); 276 static void streams_bufcall_service(void); 277 static void streams_qbkgrnd_service(void); 278 static void streams_sqbkgrnd_service(void); 279 static syncq_t *new_syncq(void); 280 static void free_syncq(syncq_t *); 281 static void outer_insert(syncq_t *, syncq_t *); 282 static void outer_remove(syncq_t *, syncq_t *); 283 static void write_now(syncq_t *); 284 static void clr_qfull(queue_t *); 285 static void enable_svc(queue_t *); 286 static void runbufcalls(void); 287 static void sqenable(syncq_t *); 288 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); 289 static void wait_q_syncq(queue_t *); 290 static void backenable_insertedq(queue_t *); 291 292 static void queue_service(queue_t *); 293 static void stream_service(stdata_t *); 294 static void syncq_service(syncq_t *); 295 static void qwriter_outer_service(syncq_t *); 296 static void mblk_free(mblk_t *); 297 #ifdef DEBUG 298 static int qprocsareon(queue_t *); 299 #endif 300 301 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); 302 static void reset_nfsrv_ptr(queue_t *, queue_t *); 303 304 static void sq_run_events(syncq_t *); 305 static int propagate_syncq(queue_t *); 306 307 static void blocksq(syncq_t *, ushort_t, int); 308 static void unblocksq(syncq_t *, ushort_t, int); 309 static int dropsq(syncq_t *, uint16_t); 310 static void emptysq(syncq_t *); 311 static sqlist_t *sqlist_alloc(struct stdata *, int); 312 static void sqlist_free(sqlist_t *); 313 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); 314 static void sqlist_insert(sqlist_t *, syncq_t *); 315 static void sqlist_insertall(sqlist_t *, queue_t *); 316 317 static void strsetuio(stdata_t *); 318 319 struct kmem_cache *stream_head_cache; 320 struct kmem_cache *queue_cache; 321 struct kmem_cache *syncq_cache; 322 struct kmem_cache *qband_cache; 323 struct kmem_cache *linkinfo_cache; 324 struct kmem_cache *ciputctrl_cache = NULL; 325 326 static linkinfo_t *linkinfo_list; 327 328 /* 329 * Qinit structure and Module_info structures 330 * for passthru read and write queues 331 */ 332 333 static void pass_wput(queue_t *, mblk_t *); 334 static queue_t *link_addpassthru(stdata_t *); 335 static void link_rempassthru(queue_t *); 336 337 struct module_info passthru_info = { 338 0, 339 "passthru", 340 0, 341 INFPSZ, 342 STRHIGH, 343 STRLOW 344 }; 345 346 struct qinit passthru_rinit = { 347 (int (*)())putnext, 348 NULL, 349 NULL, 350 NULL, 351 NULL, 352 &passthru_info, 353 NULL 354 }; 355 356 struct qinit passthru_winit = { 357 (int (*)()) pass_wput, 358 NULL, 359 NULL, 360 NULL, 361 NULL, 362 &passthru_info, 363 NULL 364 }; 365 366 /* 367 * Special form of assertion: verify that X implies Y i.e. when X is true Y 368 * should also be true. 369 */ 370 #define IMPLY(X, Y) ASSERT(!(X) || (Y)) 371 372 /* 373 * Logical equivalence. Verify that both X and Y are either TRUE or FALSE. 374 */ 375 #define EQUIV(X, Y) { IMPLY(X, Y); IMPLY(Y, X); } 376 377 /* 378 * Verify correctness of list head/tail pointers. 379 */ 380 #define LISTCHECK(head, tail, link) { \ 381 EQUIV(head, tail); \ 382 IMPLY(tail != NULL, tail->link == NULL); \ 383 } 384 385 /* 386 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail' 387 * using a `link' field. 388 */ 389 #define ENQUEUE(el, head, tail, link) { \ 390 ASSERT(el->link == NULL); \ 391 LISTCHECK(head, tail, link); \ 392 if (head == NULL) \ 393 head = el; \ 394 else \ 395 tail->link = el; \ 396 tail = el; \ 397 } 398 399 /* 400 * Dequeue the first element of the list denoted by `head' and `tail' pointers 401 * using a `link' field and put result into `el'. 402 */ 403 #define DQ(el, head, tail, link) { \ 404 LISTCHECK(head, tail, link); \ 405 el = head; \ 406 if (head != NULL) { \ 407 head = head->link; \ 408 if (head == NULL) \ 409 tail = NULL; \ 410 el->link = NULL; \ 411 } \ 412 } 413 414 /* 415 * Remove `el' from the list using `chase' and `curr' pointers and return result 416 * in `succeed'. 417 */ 418 #define RMQ(el, head, tail, link, chase, curr, succeed) { \ 419 LISTCHECK(head, tail, link); \ 420 chase = NULL; \ 421 succeed = 0; \ 422 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ 423 chase = curr; \ 424 if (curr != NULL) { \ 425 succeed = 1; \ 426 ASSERT(curr == el); \ 427 if (chase != NULL) \ 428 chase->link = curr->link; \ 429 else \ 430 head = curr->link; \ 431 curr->link = NULL; \ 432 if (curr == tail) \ 433 tail = chase; \ 434 } \ 435 LISTCHECK(head, tail, link); \ 436 } 437 438 /* Handling of delayed messages on the inner syncq. */ 439 440 /* 441 * DEBUG versions should use function versions (to simplify tracing) and 442 * non-DEBUG kernels should use macro versions. 443 */ 444 445 /* 446 * Put a queue on the syncq list of queues. 447 * Assumes SQLOCK held. 448 */ 449 #define SQPUT_Q(sq, qp) \ 450 { \ 451 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 452 if (!(qp->q_sqflags & Q_SQQUEUED)) { \ 453 /* The queue should not be linked anywhere */ \ 454 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ 455 /* Head and tail may only be NULL simultaneously */ \ 456 EQUIV(sq->sq_head, sq->sq_tail); \ 457 /* Queue may be only enqueyed on its syncq */ \ 458 ASSERT(sq == qp->q_syncq); \ 459 /* Check the correctness of SQ_MESSAGES flag */ \ 460 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ 461 /* Sanity check first/last elements of the list */ \ 462 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ 463 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ 464 /* \ 465 * Sanity check of priority field: empty queue should \ 466 * have zero priority \ 467 * and nqueues equal to zero. \ 468 */ \ 469 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ 470 /* Sanity check of sq_nqueues field */ \ 471 EQUIV(sq->sq_head, sq->sq_nqueues); \ 472 if (sq->sq_head == NULL) { \ 473 sq->sq_head = sq->sq_tail = qp; \ 474 sq->sq_flags |= SQ_MESSAGES; \ 475 } else if (qp->q_spri == 0) { \ 476 qp->q_sqprev = sq->sq_tail; \ 477 sq->sq_tail->q_sqnext = qp; \ 478 sq->sq_tail = qp; \ 479 } else { \ 480 /* \ 481 * Put this queue in priority order: higher \ 482 * priority gets closer to the head. \ 483 */ \ 484 queue_t **qpp = &sq->sq_tail; \ 485 queue_t *qnext = NULL; \ 486 \ 487 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ 488 qnext = *qpp; \ 489 qpp = &(*qpp)->q_sqprev; \ 490 } \ 491 qp->q_sqnext = qnext; \ 492 qp->q_sqprev = *qpp; \ 493 if (*qpp != NULL) { \ 494 (*qpp)->q_sqnext = qp; \ 495 } else { \ 496 sq->sq_head = qp; \ 497 sq->sq_pri = sq->sq_head->q_spri; \ 498 } \ 499 *qpp = qp; \ 500 } \ 501 qp->q_sqflags |= Q_SQQUEUED; \ 502 qp->q_sqtstamp = lbolt; \ 503 sq->sq_nqueues++; \ 504 } \ 505 } 506 507 /* 508 * Remove a queue from the syncq list 509 * Assumes SQLOCK held. 510 */ 511 #define SQRM_Q(sq, qp) \ 512 { \ 513 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 514 ASSERT(qp->q_sqflags & Q_SQQUEUED); \ 515 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ 516 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ 517 /* Check that the queue is actually in the list */ \ 518 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ 519 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ 520 ASSERT(sq->sq_nqueues != 0); \ 521 if (qp->q_sqprev == NULL) { \ 522 /* First queue on list, make head q_sqnext */ \ 523 sq->sq_head = qp->q_sqnext; \ 524 } else { \ 525 /* Make prev->next == next */ \ 526 qp->q_sqprev->q_sqnext = qp->q_sqnext; \ 527 } \ 528 if (qp->q_sqnext == NULL) { \ 529 /* Last queue on list, make tail sqprev */ \ 530 sq->sq_tail = qp->q_sqprev; \ 531 } else { \ 532 /* Make next->prev == prev */ \ 533 qp->q_sqnext->q_sqprev = qp->q_sqprev; \ 534 } \ 535 /* clear out references on this queue */ \ 536 qp->q_sqprev = qp->q_sqnext = NULL; \ 537 qp->q_sqflags &= ~Q_SQQUEUED; \ 538 /* If there is nothing queued, clear SQ_MESSAGES */ \ 539 if (sq->sq_head != NULL) { \ 540 sq->sq_pri = sq->sq_head->q_spri; \ 541 } else { \ 542 sq->sq_flags &= ~SQ_MESSAGES; \ 543 sq->sq_pri = 0; \ 544 } \ 545 sq->sq_nqueues--; \ 546 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ 547 (sq->sq_flags & SQ_QUEUED) == 0); \ 548 } 549 550 /* Hide the definition from the header file. */ 551 #ifdef SQPUT_MP 552 #undef SQPUT_MP 553 #endif 554 555 /* 556 * Put a message on the queue syncq. 557 * Assumes QLOCK held. 558 */ 559 #define SQPUT_MP(qp, mp) \ 560 { \ 561 ASSERT(MUTEX_HELD(QLOCK(qp))); \ 562 ASSERT(qp->q_sqhead == NULL || \ 563 (qp->q_sqtail != NULL && \ 564 qp->q_sqtail->b_next == NULL)); \ 565 qp->q_syncqmsgs++; \ 566 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ 567 if (qp->q_sqhead == NULL) { \ 568 qp->q_sqhead = qp->q_sqtail = mp; \ 569 } else { \ 570 qp->q_sqtail->b_next = mp; \ 571 qp->q_sqtail = mp; \ 572 } \ 573 ASSERT(qp->q_syncqmsgs > 0); \ 574 } 575 576 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ 577 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 578 if ((sq)->sq_ciputctrl != NULL) { \ 579 int i; \ 580 int nlocks = (sq)->sq_nciputctrl; \ 581 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 582 ASSERT((sq)->sq_type & SQ_CIPUT); \ 583 for (i = 0; i <= nlocks; i++) { \ 584 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 585 cip[i].ciputctrl_count |= SQ_FASTPUT; \ 586 } \ 587 } \ 588 } 589 590 591 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ 592 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 593 if ((sq)->sq_ciputctrl != NULL) { \ 594 int i; \ 595 int nlocks = (sq)->sq_nciputctrl; \ 596 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 597 ASSERT((sq)->sq_type & SQ_CIPUT); \ 598 for (i = 0; i <= nlocks; i++) { \ 599 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 600 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ 601 } \ 602 } \ 603 } 604 605 /* 606 * Run service procedures for all queues in the stream head. 607 */ 608 #define STR_SERVICE(stp, q) { \ 609 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ 610 while (stp->sd_qhead != NULL) { \ 611 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ 612 ASSERT(stp->sd_nqueues > 0); \ 613 stp->sd_nqueues--; \ 614 ASSERT(!(q->q_flag & QINSERVICE)); \ 615 mutex_exit(&stp->sd_qlock); \ 616 queue_service(q); \ 617 mutex_enter(&stp->sd_qlock); \ 618 } \ 619 ASSERT(stp->sd_nqueues == 0); \ 620 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ 621 } 622 623 /* 624 * constructor/destructor routines for the stream head cache 625 */ 626 /* ARGSUSED */ 627 static int 628 stream_head_constructor(void *buf, void *cdrarg, int kmflags) 629 { 630 stdata_t *stp = buf; 631 632 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); 633 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); 634 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); 635 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); 636 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); 637 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); 638 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); 639 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); 640 stp->sd_wrq = NULL; 641 642 return (0); 643 } 644 645 /* ARGSUSED */ 646 static void 647 stream_head_destructor(void *buf, void *cdrarg) 648 { 649 stdata_t *stp = buf; 650 651 mutex_destroy(&stp->sd_lock); 652 mutex_destroy(&stp->sd_reflock); 653 mutex_destroy(&stp->sd_qlock); 654 cv_destroy(&stp->sd_monitor); 655 cv_destroy(&stp->sd_iocmonitor); 656 cv_destroy(&stp->sd_refmonitor); 657 cv_destroy(&stp->sd_qcv); 658 cv_destroy(&stp->sd_zcopy_wait); 659 } 660 661 /* 662 * constructor/destructor routines for the queue cache 663 */ 664 /* ARGSUSED */ 665 static int 666 queue_constructor(void *buf, void *cdrarg, int kmflags) 667 { 668 queinfo_t *qip = buf; 669 queue_t *qp = &qip->qu_rqueue; 670 queue_t *wqp = &qip->qu_wqueue; 671 syncq_t *sq = &qip->qu_syncq; 672 673 qp->q_first = NULL; 674 qp->q_link = NULL; 675 qp->q_count = 0; 676 qp->q_mblkcnt = 0; 677 qp->q_sqhead = NULL; 678 qp->q_sqtail = NULL; 679 qp->q_sqnext = NULL; 680 qp->q_sqprev = NULL; 681 qp->q_sqflags = 0; 682 qp->q_rwcnt = 0; 683 qp->q_spri = 0; 684 685 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); 686 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); 687 688 wqp->q_first = NULL; 689 wqp->q_link = NULL; 690 wqp->q_count = 0; 691 wqp->q_mblkcnt = 0; 692 wqp->q_sqhead = NULL; 693 wqp->q_sqtail = NULL; 694 wqp->q_sqnext = NULL; 695 wqp->q_sqprev = NULL; 696 wqp->q_sqflags = 0; 697 wqp->q_rwcnt = 0; 698 wqp->q_spri = 0; 699 700 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); 701 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); 702 703 sq->sq_head = NULL; 704 sq->sq_tail = NULL; 705 sq->sq_evhead = NULL; 706 sq->sq_evtail = NULL; 707 sq->sq_callbpend = NULL; 708 sq->sq_outer = NULL; 709 sq->sq_onext = NULL; 710 sq->sq_oprev = NULL; 711 sq->sq_next = NULL; 712 sq->sq_svcflags = 0; 713 sq->sq_servcount = 0; 714 sq->sq_needexcl = 0; 715 sq->sq_nqueues = 0; 716 sq->sq_pri = 0; 717 718 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 719 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 720 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 721 722 return (0); 723 } 724 725 /* ARGSUSED */ 726 static void 727 queue_destructor(void *buf, void *cdrarg) 728 { 729 queinfo_t *qip = buf; 730 queue_t *qp = &qip->qu_rqueue; 731 queue_t *wqp = &qip->qu_wqueue; 732 syncq_t *sq = &qip->qu_syncq; 733 734 ASSERT(qp->q_sqhead == NULL); 735 ASSERT(wqp->q_sqhead == NULL); 736 ASSERT(qp->q_sqnext == NULL); 737 ASSERT(wqp->q_sqnext == NULL); 738 ASSERT(qp->q_rwcnt == 0); 739 ASSERT(wqp->q_rwcnt == 0); 740 741 mutex_destroy(&qp->q_lock); 742 cv_destroy(&qp->q_wait); 743 744 mutex_destroy(&wqp->q_lock); 745 cv_destroy(&wqp->q_wait); 746 747 mutex_destroy(&sq->sq_lock); 748 cv_destroy(&sq->sq_wait); 749 cv_destroy(&sq->sq_exitwait); 750 } 751 752 /* 753 * constructor/destructor routines for the syncq cache 754 */ 755 /* ARGSUSED */ 756 static int 757 syncq_constructor(void *buf, void *cdrarg, int kmflags) 758 { 759 syncq_t *sq = buf; 760 761 bzero(buf, sizeof (syncq_t)); 762 763 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 764 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 765 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 766 767 return (0); 768 } 769 770 /* ARGSUSED */ 771 static void 772 syncq_destructor(void *buf, void *cdrarg) 773 { 774 syncq_t *sq = buf; 775 776 ASSERT(sq->sq_head == NULL); 777 ASSERT(sq->sq_tail == NULL); 778 ASSERT(sq->sq_evhead == NULL); 779 ASSERT(sq->sq_evtail == NULL); 780 ASSERT(sq->sq_callbpend == NULL); 781 ASSERT(sq->sq_callbflags == 0); 782 ASSERT(sq->sq_outer == NULL); 783 ASSERT(sq->sq_onext == NULL); 784 ASSERT(sq->sq_oprev == NULL); 785 ASSERT(sq->sq_next == NULL); 786 ASSERT(sq->sq_needexcl == 0); 787 ASSERT(sq->sq_svcflags == 0); 788 ASSERT(sq->sq_servcount == 0); 789 ASSERT(sq->sq_nqueues == 0); 790 ASSERT(sq->sq_pri == 0); 791 ASSERT(sq->sq_count == 0); 792 ASSERT(sq->sq_rmqcount == 0); 793 ASSERT(sq->sq_cancelid == 0); 794 ASSERT(sq->sq_ciputctrl == NULL); 795 ASSERT(sq->sq_nciputctrl == 0); 796 ASSERT(sq->sq_type == 0); 797 ASSERT(sq->sq_flags == 0); 798 799 mutex_destroy(&sq->sq_lock); 800 cv_destroy(&sq->sq_wait); 801 cv_destroy(&sq->sq_exitwait); 802 } 803 804 /* ARGSUSED */ 805 static int 806 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) 807 { 808 ciputctrl_t *cip = buf; 809 int i; 810 811 for (i = 0; i < n_ciputctrl; i++) { 812 cip[i].ciputctrl_count = SQ_FASTPUT; 813 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); 814 } 815 816 return (0); 817 } 818 819 /* ARGSUSED */ 820 static void 821 ciputctrl_destructor(void *buf, void *cdrarg) 822 { 823 ciputctrl_t *cip = buf; 824 int i; 825 826 for (i = 0; i < n_ciputctrl; i++) { 827 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); 828 mutex_destroy(&cip[i].ciputctrl_lock); 829 } 830 } 831 832 /* 833 * Init routine run from main at boot time. 834 */ 835 void 836 strinit(void) 837 { 838 int i; 839 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 840 841 /* 842 * Set up mux_node structures. 843 */ 844 mux_nodes = kmem_zalloc((sizeof (struct mux_node) * devcnt), KM_SLEEP); 845 for (i = 0; i < devcnt; i++) 846 mux_nodes[i].mn_imaj = i; 847 848 stream_head_cache = kmem_cache_create("stream_head_cache", 849 sizeof (stdata_t), 0, 850 stream_head_constructor, stream_head_destructor, NULL, 851 NULL, NULL, 0); 852 853 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, 854 queue_constructor, queue_destructor, NULL, NULL, NULL, 0); 855 856 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, 857 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); 858 859 qband_cache = kmem_cache_create("qband_cache", 860 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 861 862 linkinfo_cache = kmem_cache_create("linkinfo_cache", 863 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 864 865 n_ciputctrl = ncpus; 866 n_ciputctrl = 1 << highbit(n_ciputctrl - 1); 867 ASSERT(n_ciputctrl >= 1); 868 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); 869 if (n_ciputctrl >= min_n_ciputctrl) { 870 ciputctrl_cache = kmem_cache_create("ciputctrl_cache", 871 sizeof (ciputctrl_t) * n_ciputctrl, 872 sizeof (ciputctrl_t), ciputctrl_constructor, 873 ciputctrl_destructor, NULL, NULL, NULL, 0); 874 } 875 876 streams_taskq = system_taskq; 877 878 if (streams_taskq == NULL) 879 panic("strinit: no memory for streams taskq!"); 880 881 bc_bkgrnd_thread = thread_create(NULL, 0, 882 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); 883 884 streams_qbkgrnd_thread = thread_create(NULL, 0, 885 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 886 887 streams_sqbkgrnd_thread = thread_create(NULL, 0, 888 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 889 890 /* 891 * Create STREAMS kstats. 892 */ 893 str_kstat = kstat_create("streams", 0, "strstat", 894 "net", KSTAT_TYPE_NAMED, 895 sizeof (str_statistics) / sizeof (kstat_named_t), 896 KSTAT_FLAG_VIRTUAL); 897 898 if (str_kstat != NULL) { 899 str_kstat->ks_data = &str_statistics; 900 kstat_install(str_kstat); 901 } 902 903 /* 904 * TPI support routine initialisation. 905 */ 906 tpi_init(); 907 } 908 909 void 910 str_sendsig(vnode_t *vp, int event, uchar_t band, int error) 911 { 912 struct stdata *stp; 913 914 ASSERT(vp->v_stream); 915 stp = vp->v_stream; 916 /* Have to hold sd_lock to prevent siglist from changing */ 917 mutex_enter(&stp->sd_lock); 918 if (stp->sd_sigflags & event) 919 strsendsig(stp->sd_siglist, event, band, error); 920 mutex_exit(&stp->sd_lock); 921 } 922 923 /* 924 * Send the "sevent" set of signals to a process. 925 * This might send more than one signal if the process is registered 926 * for multiple events. The caller should pass in an sevent that only 927 * includes the events for which the process has registered. 928 */ 929 static void 930 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, 931 uchar_t band, int error) 932 { 933 ASSERT(MUTEX_HELD(&proc->p_lock)); 934 935 info->si_band = 0; 936 info->si_errno = 0; 937 938 if (sevent & S_ERROR) { 939 sevent &= ~S_ERROR; 940 info->si_code = POLL_ERR; 941 info->si_errno = error; 942 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 943 "strsendsig:proc %p info %p", proc, info); 944 sigaddq(proc, NULL, info, KM_NOSLEEP); 945 info->si_errno = 0; 946 } 947 if (sevent & S_HANGUP) { 948 sevent &= ~S_HANGUP; 949 info->si_code = POLL_HUP; 950 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 951 "strsendsig:proc %p info %p", proc, info); 952 sigaddq(proc, NULL, info, KM_NOSLEEP); 953 } 954 if (sevent & S_HIPRI) { 955 sevent &= ~S_HIPRI; 956 info->si_code = POLL_PRI; 957 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 958 "strsendsig:proc %p info %p", proc, info); 959 sigaddq(proc, NULL, info, KM_NOSLEEP); 960 } 961 if (sevent & S_RDBAND) { 962 sevent &= ~S_RDBAND; 963 if (events & S_BANDURG) 964 sigtoproc(proc, NULL, SIGURG); 965 else 966 sigtoproc(proc, NULL, SIGPOLL); 967 } 968 if (sevent & S_WRBAND) { 969 sevent &= ~S_WRBAND; 970 sigtoproc(proc, NULL, SIGPOLL); 971 } 972 if (sevent & S_INPUT) { 973 sevent &= ~S_INPUT; 974 info->si_code = POLL_IN; 975 info->si_band = band; 976 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 977 "strsendsig:proc %p info %p", proc, info); 978 sigaddq(proc, NULL, info, KM_NOSLEEP); 979 info->si_band = 0; 980 } 981 if (sevent & S_OUTPUT) { 982 sevent &= ~S_OUTPUT; 983 info->si_code = POLL_OUT; 984 info->si_band = band; 985 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 986 "strsendsig:proc %p info %p", proc, info); 987 sigaddq(proc, NULL, info, KM_NOSLEEP); 988 info->si_band = 0; 989 } 990 if (sevent & S_MSG) { 991 sevent &= ~S_MSG; 992 info->si_code = POLL_MSG; 993 info->si_band = band; 994 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 995 "strsendsig:proc %p info %p", proc, info); 996 sigaddq(proc, NULL, info, KM_NOSLEEP); 997 info->si_band = 0; 998 } 999 if (sevent & S_RDNORM) { 1000 sevent &= ~S_RDNORM; 1001 sigtoproc(proc, NULL, SIGPOLL); 1002 } 1003 if (sevent != 0) { 1004 panic("strsendsig: unknown event(s) %x", sevent); 1005 } 1006 } 1007 1008 /* 1009 * Send SIGPOLL/SIGURG signal to all processes and process groups 1010 * registered on the given signal list that want a signal for at 1011 * least one of the specified events. 1012 * 1013 * Must be called with exclusive access to siglist (caller holding sd_lock). 1014 * 1015 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding 1016 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure 1017 * while it is in the siglist. 1018 * 1019 * For performance reasons (MP scalability) the code drops pidlock 1020 * when sending signals to a single process. 1021 * When sending to a process group the code holds 1022 * pidlock to prevent the membership in the process group from changing 1023 * while walking the p_pglink list. 1024 */ 1025 void 1026 strsendsig(strsig_t *siglist, int event, uchar_t band, int error) 1027 { 1028 strsig_t *ssp; 1029 k_siginfo_t info; 1030 struct pid *pidp; 1031 proc_t *proc; 1032 1033 info.si_signo = SIGPOLL; 1034 info.si_errno = 0; 1035 for (ssp = siglist; ssp; ssp = ssp->ss_next) { 1036 int sevent; 1037 1038 sevent = ssp->ss_events & event; 1039 if (sevent == 0) 1040 continue; 1041 1042 if ((pidp = ssp->ss_pidp) == NULL) { 1043 /* pid was released but still on event list */ 1044 continue; 1045 } 1046 1047 1048 if (ssp->ss_pid > 0) { 1049 /* 1050 * XXX This unfortunately still generates 1051 * a signal when a fd is closed but 1052 * the proc is active. 1053 */ 1054 ASSERT(ssp->ss_pid == pidp->pid_id); 1055 1056 mutex_enter(&pidlock); 1057 proc = prfind_zone(pidp->pid_id, ALL_ZONES); 1058 if (proc == NULL) { 1059 mutex_exit(&pidlock); 1060 continue; 1061 } 1062 mutex_enter(&proc->p_lock); 1063 mutex_exit(&pidlock); 1064 dosendsig(proc, ssp->ss_events, sevent, &info, 1065 band, error); 1066 mutex_exit(&proc->p_lock); 1067 } else { 1068 /* 1069 * Send to process group. Hold pidlock across 1070 * calls to dosendsig(). 1071 */ 1072 pid_t pgrp = -ssp->ss_pid; 1073 1074 mutex_enter(&pidlock); 1075 proc = pgfind_zone(pgrp, ALL_ZONES); 1076 while (proc != NULL) { 1077 mutex_enter(&proc->p_lock); 1078 dosendsig(proc, ssp->ss_events, sevent, 1079 &info, band, error); 1080 mutex_exit(&proc->p_lock); 1081 proc = proc->p_pglink; 1082 } 1083 mutex_exit(&pidlock); 1084 } 1085 } 1086 } 1087 1088 /* 1089 * Attach a stream device or module. 1090 * qp is a read queue; the new queue goes in so its next 1091 * read ptr is the argument, and the write queue corresponding 1092 * to the argument points to this queue. Return 0 on success, 1093 * or a non-zero errno on failure. 1094 */ 1095 int 1096 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, 1097 boolean_t is_insert) 1098 { 1099 major_t major; 1100 cdevsw_impl_t *dp; 1101 struct streamtab *str; 1102 queue_t *rq; 1103 queue_t *wrq; 1104 uint32_t qflag; 1105 uint32_t sqtype; 1106 perdm_t *dmp; 1107 int error; 1108 int sflag; 1109 1110 rq = allocq(); 1111 wrq = _WR(rq); 1112 STREAM(rq) = STREAM(wrq) = STREAM(qp); 1113 1114 if (fp != NULL) { 1115 str = fp->f_str; 1116 qflag = fp->f_qflag; 1117 sqtype = fp->f_sqtype; 1118 dmp = fp->f_dmp; 1119 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 1120 sflag = MODOPEN; 1121 1122 /* 1123 * stash away a pointer to the module structure so we can 1124 * unref it in qdetach. 1125 */ 1126 rq->q_fp = fp; 1127 } else { 1128 ASSERT(!is_insert); 1129 1130 major = getmajor(*devp); 1131 dp = &devimpl[major]; 1132 1133 str = dp->d_str; 1134 ASSERT(str == STREAMSTAB(major)); 1135 1136 qflag = dp->d_qflag; 1137 ASSERT(qflag & QISDRV); 1138 sqtype = dp->d_sqtype; 1139 1140 /* create perdm_t if needed */ 1141 if (NEED_DM(dp->d_dmp, qflag)) 1142 dp->d_dmp = hold_dm(str, qflag, sqtype); 1143 1144 dmp = dp->d_dmp; 1145 sflag = 0; 1146 } 1147 1148 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, 1149 "qattach:qflag == %X(%X)", qflag, *devp); 1150 1151 /* setq might sleep in allocator - avoid holding locks. */ 1152 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); 1153 1154 /* 1155 * Before calling the module's open routine, set up the q_next 1156 * pointer for inserting a module in the middle of a stream. 1157 * 1158 * Note that we can always set _QINSERTING and set up q_next 1159 * pointer for both inserting and pushing a module. Then there 1160 * is no need for the is_insert parameter. In insertq(), called 1161 * by qprocson(), assume that q_next of the new module always points 1162 * to the correct queue and use it for insertion. Everything should 1163 * work out fine. But in the first release of _I_INSERT, we 1164 * distinguish between inserting and pushing to make sure that 1165 * pushing a module follows the same code path as before. 1166 */ 1167 if (is_insert) { 1168 rq->q_flag |= _QINSERTING; 1169 rq->q_next = qp; 1170 } 1171 1172 /* 1173 * If there is an outer perimeter get exclusive access during 1174 * the open procedure. Bump up the reference count on the queue. 1175 */ 1176 entersq(rq->q_syncq, SQ_OPENCLOSE); 1177 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); 1178 if (error != 0) 1179 goto failed; 1180 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1181 ASSERT(qprocsareon(rq)); 1182 return (0); 1183 1184 failed: 1185 rq->q_flag &= ~_QINSERTING; 1186 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) 1187 qprocsoff(rq); 1188 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1189 rq->q_next = wrq->q_next = NULL; 1190 qdetach(rq, 0, 0, crp, B_FALSE); 1191 return (error); 1192 } 1193 1194 /* 1195 * Handle second open of stream. For modules, set the 1196 * last argument to MODOPEN and do not pass any open flags. 1197 * Ignore dummydev since this is not the first open. 1198 */ 1199 int 1200 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) 1201 { 1202 int error; 1203 dev_t dummydev; 1204 queue_t *wqp = _WR(qp); 1205 1206 ASSERT(qp->q_flag & QREADR); 1207 entersq(qp->q_syncq, SQ_OPENCLOSE); 1208 1209 dummydev = *devp; 1210 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, 1211 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { 1212 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1213 mutex_enter(&STREAM(qp)->sd_lock); 1214 qp->q_stream->sd_flag |= STREOPENFAIL; 1215 mutex_exit(&STREAM(qp)->sd_lock); 1216 return (error); 1217 } 1218 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1219 1220 /* 1221 * successful open should have done qprocson() 1222 */ 1223 ASSERT(qprocsareon(_RD(qp))); 1224 return (0); 1225 } 1226 1227 /* 1228 * Detach a stream module or device. 1229 * If clmode == 1 then the module or driver was opened and its 1230 * close routine must be called. If clmode == 0, the module 1231 * or driver was never opened or the open failed, and so its close 1232 * should not be called. 1233 */ 1234 void 1235 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) 1236 { 1237 queue_t *wqp = _WR(qp); 1238 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); 1239 1240 if (STREAM_NEEDSERVICE(STREAM(qp))) 1241 stream_runservice(STREAM(qp)); 1242 1243 if (clmode) { 1244 /* 1245 * Make sure that all the messages on the write side syncq are 1246 * processed and nothing is left. Since we are closing, no new 1247 * messages may appear there. 1248 */ 1249 wait_q_syncq(wqp); 1250 1251 entersq(qp->q_syncq, SQ_OPENCLOSE); 1252 if (is_remove) { 1253 mutex_enter(QLOCK(qp)); 1254 qp->q_flag |= _QREMOVING; 1255 mutex_exit(QLOCK(qp)); 1256 } 1257 (*qp->q_qinfo->qi_qclose)(qp, flag, crp); 1258 /* 1259 * Check that qprocsoff() was actually called. 1260 */ 1261 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); 1262 1263 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1264 } else { 1265 disable_svc(qp); 1266 } 1267 1268 /* 1269 * Allow any threads blocked in entersq to proceed and discover 1270 * the QWCLOSE is set. 1271 * Note: This assumes that all users of entersq check QWCLOSE. 1272 * Currently runservice is the only entersq that can happen 1273 * after removeq has finished. 1274 * Removeq will have discarded all messages destined to the closing 1275 * pair of queues from the syncq. 1276 * NOTE: Calling a function inside an assert is unconventional. 1277 * However, it does not cause any problem since flush_syncq() does 1278 * not change any state except when it returns non-zero i.e. 1279 * when the assert will trigger. 1280 */ 1281 ASSERT(flush_syncq(qp->q_syncq, qp) == 0); 1282 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); 1283 ASSERT((qp->q_flag & QPERMOD) || 1284 ((qp->q_syncq->sq_head == NULL) && 1285 (wqp->q_syncq->sq_head == NULL))); 1286 1287 /* release any fmodsw_impl_t structure held on behalf of the queue */ 1288 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); 1289 if (qp->q_fp != NULL) 1290 fmodsw_rele(qp->q_fp); 1291 1292 /* freeq removes us from the outer perimeter if any */ 1293 freeq(qp); 1294 } 1295 1296 /* Prevent service procedures from being called */ 1297 void 1298 disable_svc(queue_t *qp) 1299 { 1300 queue_t *wqp = _WR(qp); 1301 1302 ASSERT(qp->q_flag & QREADR); 1303 mutex_enter(QLOCK(qp)); 1304 qp->q_flag |= QWCLOSE; 1305 mutex_exit(QLOCK(qp)); 1306 mutex_enter(QLOCK(wqp)); 1307 wqp->q_flag |= QWCLOSE; 1308 mutex_exit(QLOCK(wqp)); 1309 } 1310 1311 /* allow service procedures to be called again */ 1312 void 1313 enable_svc(queue_t *qp) 1314 { 1315 queue_t *wqp = _WR(qp); 1316 1317 ASSERT(qp->q_flag & QREADR); 1318 mutex_enter(QLOCK(qp)); 1319 qp->q_flag &= ~QWCLOSE; 1320 mutex_exit(QLOCK(qp)); 1321 mutex_enter(QLOCK(wqp)); 1322 wqp->q_flag &= ~QWCLOSE; 1323 mutex_exit(QLOCK(wqp)); 1324 } 1325 1326 /* 1327 * Remove queue from qhead/qtail if it is enabled. 1328 * Only reset QENAB if the queue was removed from the runlist. 1329 * A queue goes through 3 stages: 1330 * It is on the service list and QENAB is set. 1331 * It is removed from the service list but QENAB is still set. 1332 * QENAB gets changed to QINSERVICE. 1333 * QINSERVICE is reset (when the service procedure is done) 1334 * Thus we can not reset QENAB unless we actually removed it from the service 1335 * queue. 1336 */ 1337 void 1338 remove_runlist(queue_t *qp) 1339 { 1340 if (qp->q_flag & QENAB && qhead != NULL) { 1341 queue_t *q_chase; 1342 queue_t *q_curr; 1343 int removed; 1344 1345 mutex_enter(&service_queue); 1346 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); 1347 mutex_exit(&service_queue); 1348 if (removed) { 1349 STRSTAT(qremoved); 1350 qp->q_flag &= ~QENAB; 1351 } 1352 } 1353 } 1354 1355 1356 /* 1357 * wait for any pending service processing to complete. 1358 * The removal of queues from the runlist is not atomic with the 1359 * clearing of the QENABLED flag and setting the INSERVICE flag. 1360 * consequently it is possible for remove_runlist in strclose 1361 * to not find the queue on the runlist but for it to be QENABLED 1362 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED 1363 * as well as INSERVICE. 1364 */ 1365 void 1366 wait_svc(queue_t *qp) 1367 { 1368 queue_t *wqp = _WR(qp); 1369 1370 ASSERT(qp->q_flag & QREADR); 1371 1372 /* 1373 * Try to remove queues from qhead/qtail list. 1374 */ 1375 if (qhead != NULL) { 1376 remove_runlist(qp); 1377 remove_runlist(wqp); 1378 } 1379 /* 1380 * Wait till the syncqs associated with the queue 1381 * will dissapear from background processing list. 1382 * This only needs to be done for non-PERMOD perimeters since 1383 * for PERMOD perimeters the syncq may be shared and will only be freed 1384 * when the last module/driver is unloaded. 1385 * If for PERMOD perimeters queue was on the syncq list, removeq() 1386 * should call propagate_syncq() or drain_syncq() for it. Both of these 1387 * function remove the queue from its syncq list, so sqthread will not 1388 * try to access the queue. 1389 */ 1390 if (!(qp->q_flag & QPERMOD)) { 1391 syncq_t *rsq = qp->q_syncq; 1392 syncq_t *wsq = wqp->q_syncq; 1393 1394 /* 1395 * Disable rsq and wsq and wait for any background processing of 1396 * syncq to complete. 1397 */ 1398 wait_sq_svc(rsq); 1399 if (wsq != rsq) 1400 wait_sq_svc(wsq); 1401 } 1402 1403 mutex_enter(QLOCK(qp)); 1404 while (qp->q_flag & (QINSERVICE|QENAB)) 1405 cv_wait(&qp->q_wait, QLOCK(qp)); 1406 mutex_exit(QLOCK(qp)); 1407 mutex_enter(QLOCK(wqp)); 1408 while (wqp->q_flag & (QINSERVICE|QENAB)) 1409 cv_wait(&wqp->q_wait, QLOCK(wqp)); 1410 mutex_exit(QLOCK(wqp)); 1411 } 1412 1413 /* 1414 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'. 1415 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may 1416 * also be set, and is passed through to allocb_cred_wait(). 1417 * 1418 * Returns errno on failure, zero on success. 1419 */ 1420 int 1421 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) 1422 { 1423 mblk_t *tmp; 1424 ssize_t count; 1425 size_t n; 1426 int error = 0; 1427 1428 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || 1429 (flag & (U_TO_K | K_TO_K)) == K_TO_K); 1430 1431 if (bp->b_datap->db_type == M_IOCTL) { 1432 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1433 } else { 1434 ASSERT(bp->b_datap->db_type == M_COPYIN); 1435 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1436 } 1437 /* 1438 * strdoioctl validates ioc_count, so if this assert fails it 1439 * cannot be due to user error. 1440 */ 1441 ASSERT(count >= 0); 1442 1443 while (count > 0) { 1444 n = MIN(MAXIOCBSZ, count); 1445 if ((tmp = allocb_cred_wait(n, (flag & STR_NOSIG), &error, 1446 cr)) == NULL) { 1447 return (error); 1448 } 1449 error = strcopyin(arg, tmp->b_wptr, n, flag & (U_TO_K|K_TO_K)); 1450 if (error != 0) { 1451 freeb(tmp); 1452 return (error); 1453 } 1454 arg += n; 1455 DB_CPID(tmp) = curproc->p_pid; 1456 tmp->b_wptr += n; 1457 count -= n; 1458 bp = (bp->b_cont = tmp); 1459 } 1460 1461 return (0); 1462 } 1463 1464 /* 1465 * Copy ioctl data to user-land. Return non-zero errno on failure, 1466 * 0 for success. 1467 */ 1468 int 1469 getiocd(mblk_t *bp, char *arg, int copymode) 1470 { 1471 ssize_t count; 1472 size_t n; 1473 int error; 1474 1475 if (bp->b_datap->db_type == M_IOCACK) 1476 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1477 else { 1478 ASSERT(bp->b_datap->db_type == M_COPYOUT); 1479 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1480 } 1481 ASSERT(count >= 0); 1482 1483 for (bp = bp->b_cont; bp && count; 1484 count -= n, bp = bp->b_cont, arg += n) { 1485 n = MIN(count, bp->b_wptr - bp->b_rptr); 1486 error = strcopyout(bp->b_rptr, arg, n, copymode); 1487 if (error) 1488 return (error); 1489 } 1490 ASSERT(count == 0); 1491 return (0); 1492 } 1493 1494 /* 1495 * Allocate a linkinfo entry given the write queue of the 1496 * bottom module of the top stream and the write queue of the 1497 * stream head of the bottom stream. 1498 */ 1499 linkinfo_t * 1500 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) 1501 { 1502 linkinfo_t *linkp; 1503 1504 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); 1505 1506 linkp->li_lblk.l_qtop = qup; 1507 linkp->li_lblk.l_qbot = qdown; 1508 linkp->li_fpdown = fpdown; 1509 1510 mutex_enter(&strresources); 1511 linkp->li_next = linkinfo_list; 1512 linkp->li_prev = NULL; 1513 if (linkp->li_next) 1514 linkp->li_next->li_prev = linkp; 1515 linkinfo_list = linkp; 1516 linkp->li_lblk.l_index = ++lnk_id; 1517 ASSERT(lnk_id != 0); /* this should never wrap in practice */ 1518 mutex_exit(&strresources); 1519 1520 return (linkp); 1521 } 1522 1523 /* 1524 * Free a linkinfo entry. 1525 */ 1526 void 1527 lbfree(linkinfo_t *linkp) 1528 { 1529 mutex_enter(&strresources); 1530 if (linkp->li_next) 1531 linkp->li_next->li_prev = linkp->li_prev; 1532 if (linkp->li_prev) 1533 linkp->li_prev->li_next = linkp->li_next; 1534 else 1535 linkinfo_list = linkp->li_next; 1536 mutex_exit(&strresources); 1537 1538 kmem_cache_free(linkinfo_cache, linkp); 1539 } 1540 1541 /* 1542 * Check for a potential linking cycle. 1543 * Return 1 if a link will result in a cycle, 1544 * and 0 otherwise. 1545 */ 1546 int 1547 linkcycle(stdata_t *upstp, stdata_t *lostp) 1548 { 1549 struct mux_node *np; 1550 struct mux_edge *ep; 1551 int i; 1552 major_t lomaj; 1553 major_t upmaj; 1554 /* 1555 * if the lower stream is a pipe/FIFO, return, since link 1556 * cycles can not happen on pipes/FIFOs 1557 */ 1558 if (lostp->sd_vnode->v_type == VFIFO) 1559 return (0); 1560 1561 for (i = 0; i < devcnt; i++) { 1562 np = &mux_nodes[i]; 1563 MUX_CLEAR(np); 1564 } 1565 lomaj = getmajor(lostp->sd_vnode->v_rdev); 1566 upmaj = getmajor(upstp->sd_vnode->v_rdev); 1567 np = &mux_nodes[lomaj]; 1568 for (;;) { 1569 if (!MUX_DIDVISIT(np)) { 1570 if (np->mn_imaj == upmaj) 1571 return (1); 1572 if (np->mn_outp == NULL) { 1573 MUX_VISIT(np); 1574 if (np->mn_originp == NULL) 1575 return (0); 1576 np = np->mn_originp; 1577 continue; 1578 } 1579 MUX_VISIT(np); 1580 np->mn_startp = np->mn_outp; 1581 } else { 1582 if (np->mn_startp == NULL) { 1583 if (np->mn_originp == NULL) 1584 return (0); 1585 else { 1586 np = np->mn_originp; 1587 continue; 1588 } 1589 } 1590 /* 1591 * If ep->me_nodep is a FIFO (me_nodep == NULL), 1592 * ignore the edge and move on. ep->me_nodep gets 1593 * set to NULL in mux_addedge() if it is a FIFO. 1594 * 1595 */ 1596 ep = np->mn_startp; 1597 np->mn_startp = ep->me_nextp; 1598 if (ep->me_nodep == NULL) 1599 continue; 1600 ep->me_nodep->mn_originp = np; 1601 np = ep->me_nodep; 1602 } 1603 } 1604 } 1605 1606 /* 1607 * Find linkinfo entry corresponding to the parameters. 1608 */ 1609 linkinfo_t * 1610 findlinks(stdata_t *stp, int index, int type) 1611 { 1612 linkinfo_t *linkp; 1613 struct mux_edge *mep; 1614 struct mux_node *mnp; 1615 queue_t *qup; 1616 1617 mutex_enter(&strresources); 1618 if ((type & LINKTYPEMASK) == LINKNORMAL) { 1619 qup = getendq(stp->sd_wrq); 1620 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1621 if ((qup == linkp->li_lblk.l_qtop) && 1622 (!index || (index == linkp->li_lblk.l_index))) { 1623 mutex_exit(&strresources); 1624 return (linkp); 1625 } 1626 } 1627 } else { 1628 ASSERT((type & LINKTYPEMASK) == LINKPERSIST); 1629 mnp = &mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; 1630 mep = mnp->mn_outp; 1631 while (mep) { 1632 if ((index == 0) || (index == mep->me_muxid)) 1633 break; 1634 mep = mep->me_nextp; 1635 } 1636 if (!mep) { 1637 mutex_exit(&strresources); 1638 return (NULL); 1639 } 1640 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1641 if ((!linkp->li_lblk.l_qtop) && 1642 (mep->me_muxid == linkp->li_lblk.l_index)) { 1643 mutex_exit(&strresources); 1644 return (linkp); 1645 } 1646 } 1647 } 1648 mutex_exit(&strresources); 1649 return (NULL); 1650 } 1651 1652 /* 1653 * Given a queue ptr, follow the chain of q_next pointers until you reach the 1654 * last queue on the chain and return it. 1655 */ 1656 queue_t * 1657 getendq(queue_t *q) 1658 { 1659 ASSERT(q != NULL); 1660 while (_SAMESTR(q)) 1661 q = q->q_next; 1662 return (q); 1663 } 1664 1665 /* 1666 * wait for the syncq count to drop to zero. 1667 * sq could be either outer or inner. 1668 */ 1669 1670 static void 1671 wait_syncq(syncq_t *sq) 1672 { 1673 uint16_t count; 1674 1675 mutex_enter(SQLOCK(sq)); 1676 count = sq->sq_count; 1677 SQ_PUTLOCKS_ENTER(sq); 1678 SUM_SQ_PUTCOUNTS(sq, count); 1679 while (count != 0) { 1680 sq->sq_flags |= SQ_WANTWAKEUP; 1681 SQ_PUTLOCKS_EXIT(sq); 1682 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1683 count = sq->sq_count; 1684 SQ_PUTLOCKS_ENTER(sq); 1685 SUM_SQ_PUTCOUNTS(sq, count); 1686 } 1687 SQ_PUTLOCKS_EXIT(sq); 1688 mutex_exit(SQLOCK(sq)); 1689 } 1690 1691 /* 1692 * Wait while there are any messages for the queue in its syncq. 1693 */ 1694 static void 1695 wait_q_syncq(queue_t *q) 1696 { 1697 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1698 syncq_t *sq = q->q_syncq; 1699 1700 mutex_enter(SQLOCK(sq)); 1701 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1702 sq->sq_flags |= SQ_WANTWAKEUP; 1703 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1704 } 1705 mutex_exit(SQLOCK(sq)); 1706 } 1707 } 1708 1709 1710 int 1711 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, 1712 int lhlink) 1713 { 1714 struct stdata *stp; 1715 struct strioctl strioc; 1716 struct linkinfo *linkp; 1717 struct stdata *stpdown; 1718 struct streamtab *str; 1719 queue_t *passq; 1720 syncq_t *passyncq; 1721 queue_t *rq; 1722 cdevsw_impl_t *dp; 1723 uint32_t qflag; 1724 uint32_t sqtype; 1725 perdm_t *dmp; 1726 int error = 0; 1727 1728 stp = vp->v_stream; 1729 TRACE_1(TR_FAC_STREAMS_FR, 1730 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); 1731 /* 1732 * Test for invalid upper stream 1733 */ 1734 if (stp->sd_flag & STRHUP) { 1735 return (ENXIO); 1736 } 1737 if (vp->v_type == VFIFO) { 1738 return (EINVAL); 1739 } 1740 if (stp->sd_strtab == NULL) { 1741 return (EINVAL); 1742 } 1743 if (!stp->sd_strtab->st_muxwinit) { 1744 return (EINVAL); 1745 } 1746 if (fpdown == NULL) { 1747 return (EBADF); 1748 } 1749 if (getmajor(stp->sd_vnode->v_rdev) >= devcnt) { 1750 return (EINVAL); 1751 } 1752 mutex_enter(&muxifier); 1753 if (stp->sd_flag & STPLEX) { 1754 mutex_exit(&muxifier); 1755 return (ENXIO); 1756 } 1757 1758 /* 1759 * Test for invalid lower stream. 1760 * The check for the v_type != VFIFO and having a major 1761 * number not >= devcnt is done to avoid problems with 1762 * adding mux_node entry past the end of mux_nodes[]. 1763 * For FIFO's we don't add an entry so this isn't a 1764 * problem. 1765 */ 1766 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || 1767 (stpdown == stp) || (stpdown->sd_flag & 1768 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || 1769 ((stpdown->sd_vnode->v_type != VFIFO) && 1770 (getmajor(stpdown->sd_vnode->v_rdev) >= devcnt)) || 1771 linkcycle(stp, stpdown)) { 1772 mutex_exit(&muxifier); 1773 return (EINVAL); 1774 } 1775 TRACE_1(TR_FAC_STREAMS_FR, 1776 TR_STPDOWN, "stpdown:%p", stpdown); 1777 rq = getendq(stp->sd_wrq); 1778 if (cmd == I_PLINK) 1779 rq = NULL; 1780 1781 linkp = alloclink(rq, stpdown->sd_wrq, fpdown); 1782 1783 strioc.ic_cmd = cmd; 1784 strioc.ic_timout = INFTIM; 1785 strioc.ic_len = sizeof (struct linkblk); 1786 strioc.ic_dp = (char *)&linkp->li_lblk; 1787 1788 /* 1789 * STRPLUMB protects plumbing changes and should be set before 1790 * link_addpassthru()/link_rempassthru() are called, so it is set here 1791 * and cleared in the end of mlink when passthru queue is removed. 1792 * Setting of STRPLUMB prevents reopens of the stream while passthru 1793 * queue is in-place (it is not a proper module and doesn't have open 1794 * entry point). 1795 * 1796 * STPLEX prevents any threads from entering the stream from above. It 1797 * can't be set before the call to link_addpassthru() because putnext 1798 * from below may cause stream head I/O routines to be called and these 1799 * routines assert that STPLEX is not set. After link_addpassthru() 1800 * nothing may come from below since the pass queue syncq is blocked. 1801 * Note also that STPLEX should be cleared before the call to 1802 * link_remmpassthru() since when messages start flowing to the stream 1803 * head (e.g. because of message propagation from the pass queue) stream 1804 * head I/O routines may be called with STPLEX flag set. 1805 * 1806 * When STPLEX is set, nothing may come into the stream from above and 1807 * it is safe to do a setq which will change stream head. So, the 1808 * correct sequence of actions is: 1809 * 1810 * 1) Set STRPLUMB 1811 * 2) Call link_addpassthru() 1812 * 3) Set STPLEX 1813 * 4) Call setq and update the stream state 1814 * 5) Clear STPLEX 1815 * 6) Call link_rempassthru() 1816 * 7) Clear STRPLUMB 1817 * 1818 * The same sequence applies to munlink() code. 1819 */ 1820 mutex_enter(&stpdown->sd_lock); 1821 stpdown->sd_flag |= STRPLUMB; 1822 mutex_exit(&stpdown->sd_lock); 1823 /* 1824 * Add passthru queue below lower mux. This will block 1825 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 1826 */ 1827 passq = link_addpassthru(stpdown); 1828 1829 mutex_enter(&stpdown->sd_lock); 1830 stpdown->sd_flag |= STPLEX; 1831 mutex_exit(&stpdown->sd_lock); 1832 1833 rq = _RD(stpdown->sd_wrq); 1834 /* 1835 * There may be messages in the streamhead's syncq due to messages 1836 * that arrived before link_addpassthru() was done. To avoid 1837 * background processing of the syncq happening simultaneous with 1838 * setq processing, we disable the streamhead syncq and wait until 1839 * existing background thread finishes working on it. 1840 */ 1841 wait_sq_svc(rq->q_syncq); 1842 passyncq = passq->q_syncq; 1843 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1844 blocksq(passyncq, SQ_BLOCKED, 0); 1845 1846 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 1847 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 1848 rq->q_ptr = _WR(rq)->q_ptr = NULL; 1849 1850 /* setq might sleep in allocator - avoid holding locks. */ 1851 /* Note: we are holding muxifier here. */ 1852 1853 str = stp->sd_strtab; 1854 dp = &devimpl[getmajor(vp->v_rdev)]; 1855 ASSERT(dp->d_str == str); 1856 1857 qflag = dp->d_qflag; 1858 sqtype = dp->d_sqtype; 1859 1860 /* create perdm_t if needed */ 1861 if (NEED_DM(dp->d_dmp, qflag)) 1862 dp->d_dmp = hold_dm(str, qflag, sqtype); 1863 1864 dmp = dp->d_dmp; 1865 1866 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, 1867 B_TRUE); 1868 1869 /* 1870 * XXX Remove any "odd" messages from the queue. 1871 * Keep only M_DATA, M_PROTO, M_PCPROTO. 1872 */ 1873 error = strdoioctl(stp, &strioc, FNATIVE, 1874 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 1875 if (error != 0) { 1876 lbfree(linkp); 1877 1878 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1879 blocksq(passyncq, SQ_BLOCKED, 0); 1880 /* 1881 * Restore the stream head queue and then remove 1882 * the passq. Turn off STPLEX before we turn on 1883 * the stream by removing the passq. 1884 */ 1885 rq->q_ptr = _WR(rq)->q_ptr = stpdown; 1886 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, 1887 B_TRUE); 1888 1889 mutex_enter(&stpdown->sd_lock); 1890 stpdown->sd_flag &= ~STPLEX; 1891 mutex_exit(&stpdown->sd_lock); 1892 1893 link_rempassthru(passq); 1894 1895 mutex_enter(&stpdown->sd_lock); 1896 stpdown->sd_flag &= ~STRPLUMB; 1897 /* Wakeup anyone waiting for STRPLUMB to clear. */ 1898 cv_broadcast(&stpdown->sd_monitor); 1899 mutex_exit(&stpdown->sd_lock); 1900 1901 mutex_exit(&muxifier); 1902 return (error); 1903 } 1904 mutex_enter(&fpdown->f_tlock); 1905 fpdown->f_count++; 1906 mutex_exit(&fpdown->f_tlock); 1907 1908 /* 1909 * if we've made it here the linkage is all set up so we should also 1910 * set up the layered driver linkages 1911 */ 1912 1913 ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); 1914 if (cmd == I_LINK) { 1915 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); 1916 } else { 1917 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); 1918 } 1919 1920 link_rempassthru(passq); 1921 1922 mux_addedge(stp, stpdown, linkp->li_lblk.l_index); 1923 1924 /* 1925 * Mark the upper stream as having dependent links 1926 * so that strclose can clean it up. 1927 */ 1928 if (cmd == I_LINK) { 1929 mutex_enter(&stp->sd_lock); 1930 stp->sd_flag |= STRHASLINKS; 1931 mutex_exit(&stp->sd_lock); 1932 } 1933 /* 1934 * Wake up any other processes that may have been 1935 * waiting on the lower stream. These will all 1936 * error out. 1937 */ 1938 mutex_enter(&stpdown->sd_lock); 1939 /* The passthru module is removed so we may release STRPLUMB */ 1940 stpdown->sd_flag &= ~STRPLUMB; 1941 cv_broadcast(&rq->q_wait); 1942 cv_broadcast(&_WR(rq)->q_wait); 1943 cv_broadcast(&stpdown->sd_monitor); 1944 mutex_exit(&stpdown->sd_lock); 1945 mutex_exit(&muxifier); 1946 *rvalp = linkp->li_lblk.l_index; 1947 return (0); 1948 } 1949 1950 int 1951 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) 1952 { 1953 int ret; 1954 struct file *fpdown; 1955 1956 fpdown = getf(arg); 1957 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); 1958 if (fpdown != NULL) 1959 releasef(arg); 1960 return (ret); 1961 } 1962 1963 /* 1964 * Unlink a multiplexor link. Stp is the controlling stream for the 1965 * link, and linkp points to the link's entry in the linkinfo list. 1966 * The muxifier lock must be held on entry and is dropped on exit. 1967 * 1968 * NOTE : Currently it is assumed that mux would process all the messages 1969 * sitting on it's queue before ACKing the UNLINK. It is the responsibility 1970 * of the mux to handle all the messages that arrive before UNLINK. 1971 * If the mux has to send down messages on its lower stream before 1972 * ACKing I_UNLINK, then it *should* know to handle messages even 1973 * after the UNLINK is acked (actually it should be able to handle till we 1974 * re-block the read side of the pass queue here). If the mux does not 1975 * open up the lower stream, any messages that arrive during UNLINK 1976 * will be put in the stream head. In the case of lower stream opening 1977 * up, some messages might land in the stream head depending on when 1978 * the message arrived and when the read side of the pass queue was 1979 * re-blocked. 1980 */ 1981 int 1982 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp) 1983 { 1984 struct strioctl strioc; 1985 struct stdata *stpdown; 1986 queue_t *rq, *wrq; 1987 queue_t *passq; 1988 syncq_t *passyncq; 1989 int error = 0; 1990 file_t *fpdown; 1991 1992 ASSERT(MUTEX_HELD(&muxifier)); 1993 1994 stpdown = linkp->li_fpdown->f_vnode->v_stream; 1995 1996 /* 1997 * See the comment in mlink() concerning STRPLUMB/STPLEX flags. 1998 */ 1999 mutex_enter(&stpdown->sd_lock); 2000 stpdown->sd_flag |= STRPLUMB; 2001 mutex_exit(&stpdown->sd_lock); 2002 2003 /* 2004 * Add passthru queue below lower mux. This will block 2005 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 2006 */ 2007 passq = link_addpassthru(stpdown); 2008 2009 if ((flag & LINKTYPEMASK) == LINKNORMAL) 2010 strioc.ic_cmd = I_UNLINK; 2011 else 2012 strioc.ic_cmd = I_PUNLINK; 2013 strioc.ic_timout = INFTIM; 2014 strioc.ic_len = sizeof (struct linkblk); 2015 strioc.ic_dp = (char *)&linkp->li_lblk; 2016 2017 error = strdoioctl(stp, &strioc, FNATIVE, 2018 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 2019 2020 /* 2021 * If there was an error and this is not called via strclose, 2022 * return to the user. Otherwise, pretend there was no error 2023 * and close the link. 2024 */ 2025 if (error) { 2026 if (flag & LINKCLOSE) { 2027 cmn_err(CE_WARN, "KERNEL: munlink: could not perform " 2028 "unlink ioctl, closing anyway (%d)\n", error); 2029 } else { 2030 link_rempassthru(passq); 2031 mutex_enter(&stpdown->sd_lock); 2032 stpdown->sd_flag &= ~STRPLUMB; 2033 cv_broadcast(&stpdown->sd_monitor); 2034 mutex_exit(&stpdown->sd_lock); 2035 mutex_exit(&muxifier); 2036 return (error); 2037 } 2038 } 2039 2040 mux_rmvedge(stp, linkp->li_lblk.l_index); 2041 fpdown = linkp->li_fpdown; 2042 lbfree(linkp); 2043 2044 /* 2045 * We go ahead and drop muxifier here--it's a nasty global lock that 2046 * can slow others down. It's okay to since attempts to mlink() this 2047 * stream will be stopped because STPLEX is still set in the stdata 2048 * structure, and munlink() is stopped because mux_rmvedge() and 2049 * lbfree() have removed it from mux_nodes[] and linkinfo_list, 2050 * respectively. Note that we defer the closef() of fpdown until 2051 * after we drop muxifier since strclose() can call munlinkall(). 2052 */ 2053 mutex_exit(&muxifier); 2054 2055 wrq = stpdown->sd_wrq; 2056 rq = _RD(wrq); 2057 2058 /* 2059 * Get rid of outstanding service procedure runs, before we make 2060 * it a stream head, since a stream head doesn't have any service 2061 * procedure. 2062 */ 2063 disable_svc(rq); 2064 wait_svc(rq); 2065 2066 /* 2067 * Since we don't disable the syncq for QPERMOD, we wait for whatever 2068 * is queued up to be finished. mux should take care that nothing is 2069 * send down to this queue. We should do it now as we're going to block 2070 * passyncq if it was unblocked. 2071 */ 2072 if (wrq->q_flag & QPERMOD) { 2073 syncq_t *sq = wrq->q_syncq; 2074 2075 mutex_enter(SQLOCK(sq)); 2076 while (wrq->q_sqflags & Q_SQQUEUED) { 2077 sq->sq_flags |= SQ_WANTWAKEUP; 2078 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2079 } 2080 mutex_exit(SQLOCK(sq)); 2081 } 2082 passyncq = passq->q_syncq; 2083 if (!(passyncq->sq_flags & SQ_BLOCKED)) { 2084 2085 syncq_t *sq, *outer; 2086 2087 /* 2088 * Messages could be flowing from underneath. We will 2089 * block the read side of the passq. This would be 2090 * sufficient for QPAIR and QPERQ muxes to ensure 2091 * that no data is flowing up into this queue 2092 * and hence no thread active in this instance of 2093 * lower mux. But for QPERMOD and QMTOUTPERIM there 2094 * could be messages on the inner and outer/inner 2095 * syncqs respectively. We will wait for them to drain. 2096 * Because passq is blocked messages end up in the syncq 2097 * And qfill_syncq could possibly end up setting QFULL 2098 * which will access the rq->q_flag. Hence, we have to 2099 * acquire the QLOCK in setq. 2100 * 2101 * XXX Messages can also flow from top into this 2102 * queue though the unlink is over (Ex. some instance 2103 * in putnext() called from top that has still not 2104 * accessed this queue. And also putq(lowerq) ?). 2105 * Solution : How about blocking the l_qtop queue ? 2106 * Do we really care about such pure D_MP muxes ? 2107 */ 2108 2109 blocksq(passyncq, SQ_BLOCKED, 0); 2110 2111 sq = rq->q_syncq; 2112 if ((outer = sq->sq_outer) != NULL) { 2113 2114 /* 2115 * We have to just wait for the outer sq_count 2116 * drop to zero. As this does not prevent new 2117 * messages to enter the outer perimeter, this 2118 * is subject to starvation. 2119 * 2120 * NOTE :Because of blocksq above, messages could 2121 * be in the inner syncq only because of some 2122 * thread holding the outer perimeter exclusively. 2123 * Hence it would be sufficient to wait for the 2124 * exclusive holder of the outer perimeter to drain 2125 * the inner and outer syncqs. But we will not depend 2126 * on this feature and hence check the inner syncqs 2127 * separately. 2128 */ 2129 wait_syncq(outer); 2130 } 2131 2132 2133 /* 2134 * There could be messages destined for 2135 * this queue. Let the exclusive holder 2136 * drain it. 2137 */ 2138 2139 wait_syncq(sq); 2140 ASSERT((rq->q_flag & QPERMOD) || 2141 ((rq->q_syncq->sq_head == NULL) && 2142 (_WR(rq)->q_syncq->sq_head == NULL))); 2143 } 2144 2145 /* 2146 * We haven't taken care of QPERMOD case yet. QPERMOD is a special 2147 * case as we don't disable its syncq or remove it off the syncq 2148 * service list. 2149 */ 2150 if (rq->q_flag & QPERMOD) { 2151 syncq_t *sq = rq->q_syncq; 2152 2153 mutex_enter(SQLOCK(sq)); 2154 while (rq->q_sqflags & Q_SQQUEUED) { 2155 sq->sq_flags |= SQ_WANTWAKEUP; 2156 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2157 } 2158 mutex_exit(SQLOCK(sq)); 2159 } 2160 2161 /* 2162 * flush_syncq changes states only when there is some messages to 2163 * free. ie when it returns non-zero value to return. 2164 */ 2165 ASSERT(flush_syncq(rq->q_syncq, rq) == 0); 2166 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); 2167 2168 /* 2169 * No body else should know about this queue now. 2170 * If the mux did not process the messages before 2171 * acking the I_UNLINK, free them now. 2172 */ 2173 2174 flushq(rq, FLUSHALL); 2175 flushq(_WR(rq), FLUSHALL); 2176 2177 /* 2178 * Convert the mux lower queue into a stream head queue. 2179 * Turn off STPLEX before we turn on the stream by removing the passq. 2180 */ 2181 rq->q_ptr = wrq->q_ptr = stpdown; 2182 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); 2183 2184 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 2185 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 2186 2187 enable_svc(rq); 2188 2189 /* 2190 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still 2191 * needs to be set to prevent reopen() of the stream - such reopen may 2192 * try to call non-existent pass queue open routine and panic. 2193 */ 2194 mutex_enter(&stpdown->sd_lock); 2195 stpdown->sd_flag &= ~STPLEX; 2196 mutex_exit(&stpdown->sd_lock); 2197 2198 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || 2199 ((flag & LINKTYPEMASK) == LINKPERSIST)); 2200 2201 /* clean up the layered driver linkages */ 2202 if ((flag & LINKTYPEMASK) == LINKNORMAL) { 2203 ldi_munlink_fp(stp, fpdown, LINKNORMAL); 2204 } else { 2205 ldi_munlink_fp(stp, fpdown, LINKPERSIST); 2206 } 2207 2208 link_rempassthru(passq); 2209 2210 /* 2211 * Now all plumbing changes are finished and STRPLUMB is no 2212 * longer needed. 2213 */ 2214 mutex_enter(&stpdown->sd_lock); 2215 stpdown->sd_flag &= ~STRPLUMB; 2216 cv_broadcast(&stpdown->sd_monitor); 2217 mutex_exit(&stpdown->sd_lock); 2218 2219 (void) closef(fpdown); 2220 return (0); 2221 } 2222 2223 /* 2224 * Unlink all multiplexor links for which stp is the controlling stream. 2225 * Return 0, or a non-zero errno on failure. 2226 */ 2227 int 2228 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp) 2229 { 2230 linkinfo_t *linkp; 2231 int error = 0; 2232 2233 mutex_enter(&muxifier); 2234 while (linkp = findlinks(stp, 0, flag)) { 2235 /* 2236 * munlink() releases the muxifier lock. 2237 */ 2238 if (error = munlink(stp, linkp, flag, crp, rvalp)) 2239 return (error); 2240 mutex_enter(&muxifier); 2241 } 2242 mutex_exit(&muxifier); 2243 return (0); 2244 } 2245 2246 /* 2247 * A multiplexor link has been made. Add an 2248 * edge to the directed graph. 2249 */ 2250 void 2251 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid) 2252 { 2253 struct mux_node *np; 2254 struct mux_edge *ep; 2255 major_t upmaj; 2256 major_t lomaj; 2257 2258 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2259 lomaj = getmajor(lostp->sd_vnode->v_rdev); 2260 np = &mux_nodes[upmaj]; 2261 if (np->mn_outp) { 2262 ep = np->mn_outp; 2263 while (ep->me_nextp) 2264 ep = ep->me_nextp; 2265 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2266 ep = ep->me_nextp; 2267 } else { 2268 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2269 ep = np->mn_outp; 2270 } 2271 ep->me_nextp = NULL; 2272 ep->me_muxid = muxid; 2273 if (lostp->sd_vnode->v_type == VFIFO) 2274 ep->me_nodep = NULL; 2275 else 2276 ep->me_nodep = &mux_nodes[lomaj]; 2277 } 2278 2279 /* 2280 * A multiplexor link has been removed. Remove the 2281 * edge in the directed graph. 2282 */ 2283 void 2284 mux_rmvedge(stdata_t *upstp, int muxid) 2285 { 2286 struct mux_node *np; 2287 struct mux_edge *ep; 2288 struct mux_edge *pep = NULL; 2289 major_t upmaj; 2290 2291 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2292 np = &mux_nodes[upmaj]; 2293 ASSERT(np->mn_outp != NULL); 2294 ep = np->mn_outp; 2295 while (ep) { 2296 if (ep->me_muxid == muxid) { 2297 if (pep) 2298 pep->me_nextp = ep->me_nextp; 2299 else 2300 np->mn_outp = ep->me_nextp; 2301 kmem_free(ep, sizeof (struct mux_edge)); 2302 return; 2303 } 2304 pep = ep; 2305 ep = ep->me_nextp; 2306 } 2307 ASSERT(0); /* should not reach here */ 2308 } 2309 2310 /* 2311 * Translate the device flags (from conf.h) to the corresponding 2312 * qflag and sq_flag (type) values. 2313 */ 2314 int 2315 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, 2316 uint32_t *sqtypep) 2317 { 2318 uint32_t qflag = 0; 2319 uint32_t sqtype = 0; 2320 2321 if (devflag & _D_OLD) 2322 goto bad; 2323 2324 /* Inner perimeter presence and scope */ 2325 switch (devflag & D_MTINNER_MASK) { 2326 case D_MP: 2327 qflag |= QMTSAFE; 2328 sqtype |= SQ_CI; 2329 break; 2330 case D_MTPERQ|D_MP: 2331 qflag |= QPERQ; 2332 break; 2333 case D_MTQPAIR|D_MP: 2334 qflag |= QPAIR; 2335 break; 2336 case D_MTPERMOD|D_MP: 2337 qflag |= QPERMOD; 2338 break; 2339 default: 2340 goto bad; 2341 } 2342 2343 /* Outer perimeter */ 2344 if (devflag & D_MTOUTPERIM) { 2345 switch (devflag & D_MTINNER_MASK) { 2346 case D_MP: 2347 case D_MTPERQ|D_MP: 2348 case D_MTQPAIR|D_MP: 2349 break; 2350 default: 2351 goto bad; 2352 } 2353 qflag |= QMTOUTPERIM; 2354 } 2355 2356 /* Inner perimeter modifiers */ 2357 if (devflag & D_MTINNER_MOD) { 2358 switch (devflag & D_MTINNER_MASK) { 2359 case D_MP: 2360 goto bad; 2361 default: 2362 break; 2363 } 2364 if (devflag & D_MTPUTSHARED) 2365 sqtype |= SQ_CIPUT; 2366 if (devflag & _D_MTOCSHARED) { 2367 /* 2368 * The code in putnext assumes that it has the 2369 * highest concurrency by not checking sq_count. 2370 * Thus _D_MTOCSHARED can only be supported when 2371 * D_MTPUTSHARED is set. 2372 */ 2373 if (!(devflag & D_MTPUTSHARED)) 2374 goto bad; 2375 sqtype |= SQ_CIOC; 2376 } 2377 if (devflag & _D_MTCBSHARED) { 2378 /* 2379 * The code in putnext assumes that it has the 2380 * highest concurrency by not checking sq_count. 2381 * Thus _D_MTCBSHARED can only be supported when 2382 * D_MTPUTSHARED is set. 2383 */ 2384 if (!(devflag & D_MTPUTSHARED)) 2385 goto bad; 2386 sqtype |= SQ_CICB; 2387 } 2388 if (devflag & _D_MTSVCSHARED) { 2389 /* 2390 * The code in putnext assumes that it has the 2391 * highest concurrency by not checking sq_count. 2392 * Thus _D_MTSVCSHARED can only be supported when 2393 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is 2394 * supported only for QPERMOD. 2395 */ 2396 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) 2397 goto bad; 2398 sqtype |= SQ_CISVC; 2399 } 2400 } 2401 2402 /* Default outer perimeter concurrency */ 2403 sqtype |= SQ_CO; 2404 2405 /* Outer perimeter modifiers */ 2406 if (devflag & D_MTOCEXCL) { 2407 if (!(devflag & D_MTOUTPERIM)) { 2408 /* No outer perimeter */ 2409 goto bad; 2410 } 2411 sqtype &= ~SQ_COOC; 2412 } 2413 2414 /* Synchronous Streams extended qinit structure */ 2415 if (devflag & D_SYNCSTR) 2416 qflag |= QSYNCSTR; 2417 2418 /* 2419 * Private flag used by a transport module to indicate 2420 * to sockfs that it supports direct-access mode without 2421 * having to go through STREAMS. 2422 */ 2423 if (devflag & _D_DIRECT) { 2424 /* Reject unless the module is fully-MT (no perimeter) */ 2425 if ((qflag & QMT_TYPEMASK) != QMTSAFE) 2426 goto bad; 2427 qflag |= _QDIRECT; 2428 } 2429 2430 *qflagp = qflag; 2431 *sqtypep = sqtype; 2432 return (0); 2433 2434 bad: 2435 cmn_err(CE_WARN, 2436 "stropen: bad MT flags (0x%x) in driver '%s'", 2437 (int)(qflag & D_MTSAFETY_MASK), 2438 stp->st_rdinit->qi_minfo->mi_idname); 2439 2440 return (EINVAL); 2441 } 2442 2443 /* 2444 * Set the interface values for a pair of queues (qinit structure, 2445 * packet sizes, water marks). 2446 * setq assumes that the caller does not have a claim (entersq or claimq) 2447 * on the queue. 2448 */ 2449 void 2450 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, 2451 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) 2452 { 2453 queue_t *wq; 2454 syncq_t *sq, *outer; 2455 2456 ASSERT(rq->q_flag & QREADR); 2457 ASSERT((qflag & QMT_TYPEMASK) != 0); 2458 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 2459 2460 wq = _WR(rq); 2461 rq->q_qinfo = rinit; 2462 rq->q_hiwat = rinit->qi_minfo->mi_hiwat; 2463 rq->q_lowat = rinit->qi_minfo->mi_lowat; 2464 rq->q_minpsz = rinit->qi_minfo->mi_minpsz; 2465 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; 2466 wq->q_qinfo = winit; 2467 wq->q_hiwat = winit->qi_minfo->mi_hiwat; 2468 wq->q_lowat = winit->qi_minfo->mi_lowat; 2469 wq->q_minpsz = winit->qi_minfo->mi_minpsz; 2470 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; 2471 2472 /* Remove old syncqs */ 2473 sq = rq->q_syncq; 2474 outer = sq->sq_outer; 2475 if (outer != NULL) { 2476 ASSERT(wq->q_syncq->sq_outer == outer); 2477 outer_remove(outer, rq->q_syncq); 2478 if (wq->q_syncq != rq->q_syncq) 2479 outer_remove(outer, wq->q_syncq); 2480 } 2481 ASSERT(sq->sq_outer == NULL); 2482 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2483 2484 if (sq != SQ(rq)) { 2485 if (!(rq->q_flag & QPERMOD)) 2486 free_syncq(sq); 2487 if (wq->q_syncq == rq->q_syncq) 2488 wq->q_syncq = NULL; 2489 rq->q_syncq = NULL; 2490 } 2491 if (wq->q_syncq != NULL && wq->q_syncq != sq && 2492 wq->q_syncq != SQ(rq)) { 2493 free_syncq(wq->q_syncq); 2494 wq->q_syncq = NULL; 2495 } 2496 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && 2497 rq->q_syncq->sq_tail == NULL)); 2498 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && 2499 wq->q_syncq->sq_tail == NULL)); 2500 2501 if (!(rq->q_flag & QPERMOD) && 2502 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { 2503 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2504 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, 2505 rq->q_syncq->sq_nciputctrl, 0); 2506 ASSERT(ciputctrl_cache != NULL); 2507 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); 2508 rq->q_syncq->sq_ciputctrl = NULL; 2509 rq->q_syncq->sq_nciputctrl = 0; 2510 } 2511 2512 if (!(wq->q_flag & QPERMOD) && 2513 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { 2514 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2515 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, 2516 wq->q_syncq->sq_nciputctrl, 0); 2517 ASSERT(ciputctrl_cache != NULL); 2518 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); 2519 wq->q_syncq->sq_ciputctrl = NULL; 2520 wq->q_syncq->sq_nciputctrl = 0; 2521 } 2522 2523 sq = SQ(rq); 2524 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 2525 ASSERT(sq->sq_outer == NULL); 2526 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2527 2528 /* 2529 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS 2530 * bits in sq_flag based on the sqtype. 2531 */ 2532 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); 2533 2534 rq->q_syncq = wq->q_syncq = sq; 2535 sq->sq_type = sqtype; 2536 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); 2537 2538 /* 2539 * We are making sq_svcflags zero, 2540 * resetting SQ_DISABLED in case it was set by 2541 * wait_svc() in the munlink path. 2542 * 2543 */ 2544 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); 2545 sq->sq_svcflags = 0; 2546 2547 /* 2548 * We need to acquire the lock here for the mlink and munlink case, 2549 * where canputnext, backenable, etc can access the q_flag. 2550 */ 2551 if (lock_needed) { 2552 mutex_enter(QLOCK(rq)); 2553 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2554 mutex_exit(QLOCK(rq)); 2555 mutex_enter(QLOCK(wq)); 2556 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2557 mutex_exit(QLOCK(wq)); 2558 } else { 2559 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2560 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2561 } 2562 2563 if (qflag & QPERQ) { 2564 /* Allocate a separate syncq for the write side */ 2565 sq = new_syncq(); 2566 sq->sq_type = rq->q_syncq->sq_type; 2567 sq->sq_flags = rq->q_syncq->sq_flags; 2568 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2569 sq->sq_oprev == NULL); 2570 wq->q_syncq = sq; 2571 } 2572 if (qflag & QPERMOD) { 2573 sq = dmp->dm_sq; 2574 2575 /* 2576 * Assert that we do have an inner perimeter syncq and that it 2577 * does not have an outer perimeter associated with it. 2578 */ 2579 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2580 sq->sq_oprev == NULL); 2581 rq->q_syncq = wq->q_syncq = sq; 2582 } 2583 if (qflag & QMTOUTPERIM) { 2584 outer = dmp->dm_sq; 2585 2586 ASSERT(outer->sq_outer == NULL); 2587 outer_insert(outer, rq->q_syncq); 2588 if (wq->q_syncq != rq->q_syncq) 2589 outer_insert(outer, wq->q_syncq); 2590 } 2591 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2592 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2593 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2594 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2595 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); 2596 2597 /* 2598 * Initialize struio() types. 2599 */ 2600 rq->q_struiot = 2601 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; 2602 wq->q_struiot = 2603 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; 2604 } 2605 2606 perdm_t * 2607 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) 2608 { 2609 syncq_t *sq; 2610 perdm_t **pp; 2611 perdm_t *p; 2612 perdm_t *dmp; 2613 2614 ASSERT(str != NULL); 2615 ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); 2616 2617 rw_enter(&perdm_rwlock, RW_READER); 2618 for (p = perdm_list; p != NULL; p = p->dm_next) { 2619 if (p->dm_str == str) { /* found one */ 2620 atomic_add_32(&(p->dm_ref), 1); 2621 rw_exit(&perdm_rwlock); 2622 return (p); 2623 } 2624 } 2625 rw_exit(&perdm_rwlock); 2626 2627 sq = new_syncq(); 2628 if (qflag & QPERMOD) { 2629 sq->sq_type = sqtype | SQ_PERMOD; 2630 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; 2631 } else { 2632 ASSERT(qflag & QMTOUTPERIM); 2633 sq->sq_onext = sq->sq_oprev = sq; 2634 } 2635 2636 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); 2637 dmp->dm_sq = sq; 2638 dmp->dm_str = str; 2639 dmp->dm_ref = 1; 2640 dmp->dm_next = NULL; 2641 2642 rw_enter(&perdm_rwlock, RW_WRITER); 2643 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { 2644 if (p->dm_str == str) { /* already present */ 2645 p->dm_ref++; 2646 rw_exit(&perdm_rwlock); 2647 free_syncq(sq); 2648 kmem_free(dmp, sizeof (perdm_t)); 2649 return (p); 2650 } 2651 } 2652 2653 *pp = dmp; 2654 rw_exit(&perdm_rwlock); 2655 return (dmp); 2656 } 2657 2658 void 2659 rele_dm(perdm_t *dmp) 2660 { 2661 perdm_t **pp; 2662 perdm_t *p; 2663 2664 rw_enter(&perdm_rwlock, RW_WRITER); 2665 ASSERT(dmp->dm_ref > 0); 2666 2667 if (--dmp->dm_ref > 0) { 2668 rw_exit(&perdm_rwlock); 2669 return; 2670 } 2671 2672 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) 2673 if (p == dmp) 2674 break; 2675 ASSERT(p == dmp); 2676 *pp = p->dm_next; 2677 rw_exit(&perdm_rwlock); 2678 2679 /* 2680 * Wait for any background processing that relies on the 2681 * syncq to complete before it is freed. 2682 */ 2683 wait_sq_svc(p->dm_sq); 2684 free_syncq(p->dm_sq); 2685 kmem_free(p, sizeof (perdm_t)); 2686 } 2687 2688 /* 2689 * Make a protocol message given control and data buffers. 2690 * n.b., this can block; be careful of what locks you hold when calling it. 2691 * 2692 * If sd_maxblk is less than *iosize this routine can fail part way through 2693 * (due to an allocation failure). In this case on return *iosize will contain 2694 * the amount that was consumed. Otherwise *iosize will not be modified 2695 * i.e. it will contain the amount that was consumed. 2696 */ 2697 int 2698 strmakemsg( 2699 struct strbuf *mctl, 2700 ssize_t *iosize, 2701 struct uio *uiop, 2702 stdata_t *stp, 2703 int32_t flag, 2704 mblk_t **mpp) 2705 { 2706 mblk_t *mpctl = NULL; 2707 mblk_t *mpdata = NULL; 2708 int error; 2709 2710 ASSERT(uiop != NULL); 2711 2712 *mpp = NULL; 2713 /* Create control part, if any */ 2714 if ((mctl != NULL) && (mctl->len >= 0)) { 2715 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); 2716 if (error) 2717 return (error); 2718 } 2719 /* Create data part, if any */ 2720 if (*iosize >= 0) { 2721 error = strmakedata(iosize, uiop, stp, flag, &mpdata); 2722 if (error) { 2723 freemsg(mpctl); 2724 return (error); 2725 } 2726 } 2727 if (mpctl != NULL) { 2728 if (mpdata != NULL) 2729 linkb(mpctl, mpdata); 2730 *mpp = mpctl; 2731 } else { 2732 *mpp = mpdata; 2733 } 2734 return (0); 2735 } 2736 2737 /* 2738 * Make the control part of a protocol message given a control buffer. 2739 * n.b., this can block; be careful of what locks you hold when calling it. 2740 */ 2741 int 2742 strmakectl( 2743 struct strbuf *mctl, 2744 int32_t flag, 2745 int32_t fflag, 2746 mblk_t **mpp) 2747 { 2748 mblk_t *bp = NULL; 2749 unsigned char msgtype; 2750 int error = 0; 2751 2752 *mpp = NULL; 2753 /* 2754 * Create control part of message, if any. 2755 */ 2756 if ((mctl != NULL) && (mctl->len >= 0)) { 2757 caddr_t base; 2758 int ctlcount; 2759 int allocsz; 2760 2761 if (flag & RS_HIPRI) 2762 msgtype = M_PCPROTO; 2763 else 2764 msgtype = M_PROTO; 2765 2766 ctlcount = mctl->len; 2767 base = mctl->buf; 2768 2769 /* 2770 * Give modules a better chance to reuse M_PROTO/M_PCPROTO 2771 * blocks by increasing the size to something more usable. 2772 */ 2773 allocsz = MAX(ctlcount, 64); 2774 2775 /* 2776 * Range checking has already been done; simply try 2777 * to allocate a message block for the ctl part. 2778 */ 2779 while (!(bp = allocb(allocsz, BPRI_MED))) { 2780 if (fflag & (FNDELAY|FNONBLOCK)) 2781 return (EAGAIN); 2782 if (error = strwaitbuf(allocsz, BPRI_MED)) 2783 return (error); 2784 } 2785 2786 bp->b_datap->db_type = msgtype; 2787 if (copyin(base, bp->b_wptr, ctlcount)) { 2788 freeb(bp); 2789 return (EFAULT); 2790 } 2791 bp->b_wptr += ctlcount; 2792 } 2793 *mpp = bp; 2794 return (0); 2795 } 2796 2797 /* 2798 * Make a protocol message given data buffers. 2799 * n.b., this can block; be careful of what locks you hold when calling it. 2800 * 2801 * If sd_maxblk is less than *iosize this routine can fail part way through 2802 * (due to an allocation failure). In this case on return *iosize will contain 2803 * the amount that was consumed. Otherwise *iosize will not be modified 2804 * i.e. it will contain the amount that was consumed. 2805 */ 2806 int 2807 strmakedata( 2808 ssize_t *iosize, 2809 struct uio *uiop, 2810 stdata_t *stp, 2811 int32_t flag, 2812 mblk_t **mpp) 2813 { 2814 mblk_t *mp = NULL; 2815 mblk_t *bp; 2816 int wroff = (int)stp->sd_wroff; 2817 int tail_len = (int)stp->sd_tail; 2818 int extra = wroff + tail_len; 2819 int error = 0; 2820 ssize_t maxblk; 2821 ssize_t count = *iosize; 2822 cred_t *cr = CRED(); 2823 2824 *mpp = NULL; 2825 if (count < 0) 2826 return (0); 2827 2828 maxblk = stp->sd_maxblk; 2829 if (maxblk == INFPSZ) 2830 maxblk = count; 2831 2832 /* 2833 * Create data part of message, if any. 2834 */ 2835 do { 2836 ssize_t size; 2837 dblk_t *dp; 2838 2839 ASSERT(uiop); 2840 2841 size = MIN(count, maxblk); 2842 2843 while ((bp = allocb_cred(size + extra, cr)) == NULL) { 2844 error = EAGAIN; 2845 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 2846 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) { 2847 if (count == *iosize) { 2848 freemsg(mp); 2849 return (error); 2850 } else { 2851 *iosize -= count; 2852 *mpp = mp; 2853 return (0); 2854 } 2855 } 2856 } 2857 dp = bp->b_datap; 2858 dp->db_cpid = curproc->p_pid; 2859 ASSERT(wroff <= dp->db_lim - bp->b_wptr); 2860 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; 2861 2862 if (flag & STRUIO_POSTPONE) { 2863 /* 2864 * Setup the stream uio portion of the 2865 * dblk for subsequent use by struioget(). 2866 */ 2867 dp->db_struioflag = STRUIO_SPEC; 2868 dp->db_cksumstart = 0; 2869 dp->db_cksumstuff = 0; 2870 dp->db_cksumend = size; 2871 *(long long *)dp->db_struioun.data = 0ll; 2872 bp->b_wptr += size; 2873 } else { 2874 if (stp->sd_copyflag & STRCOPYCACHED) 2875 uiop->uio_extflg |= UIO_COPY_CACHED; 2876 2877 if (size != 0) { 2878 error = uiomove(bp->b_wptr, size, UIO_WRITE, 2879 uiop); 2880 if (error != 0) { 2881 freeb(bp); 2882 freemsg(mp); 2883 return (error); 2884 } 2885 } 2886 bp->b_wptr += size; 2887 2888 if (stp->sd_wputdatafunc != NULL) { 2889 mblk_t *newbp; 2890 2891 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode, 2892 bp, NULL, NULL, NULL, NULL); 2893 if (newbp == NULL) { 2894 freeb(bp); 2895 freemsg(mp); 2896 return (ECOMM); 2897 } 2898 bp = newbp; 2899 } 2900 } 2901 2902 count -= size; 2903 2904 if (mp == NULL) 2905 mp = bp; 2906 else 2907 linkb(mp, bp); 2908 } while (count > 0); 2909 2910 *mpp = mp; 2911 return (0); 2912 } 2913 2914 /* 2915 * Wait for a buffer to become available. Return non-zero errno 2916 * if not able to wait, 0 if buffer is probably there. 2917 */ 2918 int 2919 strwaitbuf(size_t size, int pri) 2920 { 2921 bufcall_id_t id; 2922 2923 mutex_enter(&bcall_monitor); 2924 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, 2925 &ttoproc(curthread)->p_flag_cv)) == 0) { 2926 mutex_exit(&bcall_monitor); 2927 return (ENOSR); 2928 } 2929 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { 2930 unbufcall(id); 2931 mutex_exit(&bcall_monitor); 2932 return (EINTR); 2933 } 2934 unbufcall(id); 2935 mutex_exit(&bcall_monitor); 2936 return (0); 2937 } 2938 2939 /* 2940 * This function waits for a read or write event to happen on a stream. 2941 * fmode can specify FNDELAY and/or FNONBLOCK. 2942 * The timeout is in ms with -1 meaning infinite. 2943 * The flag values work as follows: 2944 * READWAIT Check for read side errors, send M_READ 2945 * GETWAIT Check for read side errors, no M_READ 2946 * WRITEWAIT Check for write side errors. 2947 * NOINTR Do not return error if nonblocking or timeout. 2948 * STR_NOERROR Ignore all errors except STPLEX. 2949 * STR_NOSIG Ignore/hold signals during the duration of the call. 2950 * STR_PEEK Pass through the strgeterr(). 2951 */ 2952 int 2953 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, 2954 int *done) 2955 { 2956 int slpflg, errs; 2957 int error; 2958 kcondvar_t *sleepon; 2959 mblk_t *mp; 2960 ssize_t *rd_count; 2961 clock_t rval; 2962 2963 ASSERT(MUTEX_HELD(&stp->sd_lock)); 2964 if ((flag & READWAIT) || (flag & GETWAIT)) { 2965 slpflg = RSLEEP; 2966 sleepon = &_RD(stp->sd_wrq)->q_wait; 2967 errs = STRDERR|STPLEX; 2968 } else { 2969 slpflg = WSLEEP; 2970 sleepon = &stp->sd_wrq->q_wait; 2971 errs = STWRERR|STRHUP|STPLEX; 2972 } 2973 if (flag & STR_NOERROR) 2974 errs = STPLEX; 2975 2976 if (stp->sd_wakeq & slpflg) { 2977 /* 2978 * A strwakeq() is pending, no need to sleep. 2979 */ 2980 stp->sd_wakeq &= ~slpflg; 2981 *done = 0; 2982 return (0); 2983 } 2984 2985 if (fmode & (FNDELAY|FNONBLOCK)) { 2986 if (!(flag & NOINTR)) 2987 error = EAGAIN; 2988 else 2989 error = 0; 2990 *done = 1; 2991 return (error); 2992 } 2993 2994 if (stp->sd_flag & errs) { 2995 /* 2996 * Check for errors before going to sleep since the 2997 * caller might not have checked this while holding 2998 * sd_lock. 2999 */ 3000 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3001 if (error != 0) { 3002 *done = 1; 3003 return (error); 3004 } 3005 } 3006 3007 /* 3008 * If any module downstream has requested read notification 3009 * by setting SNDMREAD flag using M_SETOPTS, send a message 3010 * down stream. 3011 */ 3012 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) { 3013 mutex_exit(&stp->sd_lock); 3014 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED, 3015 (flag & STR_NOSIG), &error))) { 3016 mutex_enter(&stp->sd_lock); 3017 *done = 1; 3018 return (error); 3019 } 3020 mp->b_datap->db_type = M_READ; 3021 rd_count = (ssize_t *)mp->b_wptr; 3022 *rd_count = count; 3023 mp->b_wptr += sizeof (ssize_t); 3024 /* 3025 * Send the number of bytes requested by the 3026 * read as the argument to M_READ. 3027 */ 3028 stream_willservice(stp); 3029 putnext(stp->sd_wrq, mp); 3030 stream_runservice(stp); 3031 mutex_enter(&stp->sd_lock); 3032 3033 /* 3034 * If any data arrived due to inline processing 3035 * of putnext(), don't sleep. 3036 */ 3037 if (_RD(stp->sd_wrq)->q_first != NULL) { 3038 *done = 0; 3039 return (0); 3040 } 3041 } 3042 3043 stp->sd_flag |= slpflg; 3044 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2, 3045 "strwaitq sleeps (2):%p, %X, %lX, %X, %p", 3046 stp, flag, count, fmode, done); 3047 3048 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG); 3049 if (rval > 0) { 3050 /* EMPTY */ 3051 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2, 3052 "strwaitq awakes(2):%X, %X, %X, %X, %X", 3053 stp, flag, count, fmode, done); 3054 } else if (rval == 0) { 3055 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2, 3056 "strwaitq interrupt #2:%p, %X, %lX, %X, %p", 3057 stp, flag, count, fmode, done); 3058 stp->sd_flag &= ~slpflg; 3059 cv_broadcast(sleepon); 3060 if (!(flag & NOINTR)) 3061 error = EINTR; 3062 else 3063 error = 0; 3064 *done = 1; 3065 return (error); 3066 } else { 3067 /* timeout */ 3068 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME, 3069 "strwaitq timeout:%p, %X, %lX, %X, %p", 3070 stp, flag, count, fmode, done); 3071 *done = 1; 3072 if (!(flag & NOINTR)) 3073 return (ETIME); 3074 else 3075 return (0); 3076 } 3077 /* 3078 * If the caller implements delayed errors (i.e. queued after data) 3079 * we can not check for errors here since data as well as an 3080 * error might have arrived at the stream head. We return to 3081 * have the caller check the read queue before checking for errors. 3082 */ 3083 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) { 3084 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3085 if (error != 0) { 3086 *done = 1; 3087 return (error); 3088 } 3089 } 3090 *done = 0; 3091 return (0); 3092 } 3093 3094 /* 3095 * Perform job control discipline access checks. 3096 * Return 0 for success and the errno for failure. 3097 */ 3098 3099 #define cantsend(p, t, sig) \ 3100 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig)) 3101 3102 int 3103 straccess(struct stdata *stp, enum jcaccess mode) 3104 { 3105 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */ 3106 kthread_t *t = curthread; 3107 proc_t *p = ttoproc(t); 3108 sess_t *sp; 3109 3110 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO) 3111 return (0); 3112 3113 mutex_enter(&p->p_lock); 3114 sp = p->p_sessp; 3115 3116 for (;;) { 3117 /* 3118 * If this is not the calling process's controlling terminal 3119 * or if the calling process is already in the foreground 3120 * then allow access. 3121 */ 3122 if (sp->s_dev != stp->sd_vnode->v_rdev || 3123 p->p_pgidp == stp->sd_pgidp) { 3124 mutex_exit(&p->p_lock); 3125 return (0); 3126 } 3127 3128 /* 3129 * Check to see if controlling terminal has been deallocated. 3130 */ 3131 if (sp->s_vp == NULL) { 3132 if (!cantsend(p, t, SIGHUP)) 3133 sigtoproc(p, t, SIGHUP); 3134 mutex_exit(&p->p_lock); 3135 return (EIO); 3136 } 3137 3138 if (mode == JCGETP) { 3139 mutex_exit(&p->p_lock); 3140 return (0); 3141 } 3142 3143 if (mode == JCREAD) { 3144 if (p->p_detached || cantsend(p, t, SIGTTIN)) { 3145 mutex_exit(&p->p_lock); 3146 return (EIO); 3147 } 3148 mutex_exit(&p->p_lock); 3149 pgsignal(p->p_pgidp, SIGTTIN); 3150 mutex_enter(&p->p_lock); 3151 } else { /* mode == JCWRITE or JCSETP */ 3152 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) || 3153 cantsend(p, t, SIGTTOU)) { 3154 mutex_exit(&p->p_lock); 3155 return (0); 3156 } 3157 if (p->p_detached) { 3158 mutex_exit(&p->p_lock); 3159 return (EIO); 3160 } 3161 mutex_exit(&p->p_lock); 3162 pgsignal(p->p_pgidp, SIGTTOU); 3163 mutex_enter(&p->p_lock); 3164 } 3165 3166 /* 3167 * We call cv_wait_sig_swap() to cause the appropriate 3168 * action for the jobcontrol signal to take place. 3169 * If the signal is being caught, we will take the 3170 * EINTR error return. Otherwise, the default action 3171 * of causing the process to stop will take place. 3172 * In this case, we rely on the periodic cv_broadcast() on 3173 * &lbolt_cv to wake us up to loop around and test again. 3174 * We can't get here if the signal is ignored or 3175 * if the current thread is blocking the signal. 3176 */ 3177 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) { 3178 mutex_exit(&p->p_lock); 3179 return (EINTR); 3180 } 3181 } 3182 } 3183 3184 /* 3185 * Return size of message of block type (bp->b_datap->db_type) 3186 */ 3187 size_t 3188 xmsgsize(mblk_t *bp) 3189 { 3190 unsigned char type; 3191 size_t count = 0; 3192 3193 type = bp->b_datap->db_type; 3194 3195 for (; bp; bp = bp->b_cont) { 3196 if (type != bp->b_datap->db_type) 3197 break; 3198 ASSERT(bp->b_wptr >= bp->b_rptr); 3199 count += bp->b_wptr - bp->b_rptr; 3200 } 3201 return (count); 3202 } 3203 3204 /* 3205 * Allocate a stream head. 3206 */ 3207 struct stdata * 3208 shalloc(queue_t *qp) 3209 { 3210 stdata_t *stp; 3211 3212 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP); 3213 3214 stp->sd_wrq = _WR(qp); 3215 stp->sd_strtab = NULL; 3216 stp->sd_iocid = 0; 3217 stp->sd_mate = NULL; 3218 stp->sd_freezer = NULL; 3219 stp->sd_refcnt = 0; 3220 stp->sd_wakeq = 0; 3221 stp->sd_anchor = 0; 3222 stp->sd_struiowrq = NULL; 3223 stp->sd_struiordq = NULL; 3224 stp->sd_struiodnak = 0; 3225 stp->sd_struionak = NULL; 3226 #ifdef C2_AUDIT 3227 stp->sd_t_audit_data = NULL; 3228 #endif 3229 stp->sd_rput_opt = 0; 3230 stp->sd_wput_opt = 0; 3231 stp->sd_read_opt = 0; 3232 stp->sd_rprotofunc = strrput_proto; 3233 stp->sd_rmiscfunc = strrput_misc; 3234 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL; 3235 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL; 3236 stp->sd_ciputctrl = NULL; 3237 stp->sd_nciputctrl = 0; 3238 stp->sd_qhead = NULL; 3239 stp->sd_qtail = NULL; 3240 stp->sd_servid = NULL; 3241 stp->sd_nqueues = 0; 3242 stp->sd_svcflags = 0; 3243 stp->sd_copyflag = 0; 3244 3245 return (stp); 3246 } 3247 3248 /* 3249 * Free a stream head. 3250 */ 3251 void 3252 shfree(stdata_t *stp) 3253 { 3254 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock)); 3255 3256 stp->sd_wrq = NULL; 3257 3258 mutex_enter(&stp->sd_qlock); 3259 while (stp->sd_svcflags & STRS_SCHEDULED) { 3260 STRSTAT(strwaits); 3261 cv_wait(&stp->sd_qcv, &stp->sd_qlock); 3262 } 3263 mutex_exit(&stp->sd_qlock); 3264 3265 if (stp->sd_ciputctrl != NULL) { 3266 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1); 3267 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl, 3268 stp->sd_nciputctrl, 0); 3269 ASSERT(ciputctrl_cache != NULL); 3270 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl); 3271 stp->sd_ciputctrl = NULL; 3272 stp->sd_nciputctrl = 0; 3273 } 3274 ASSERT(stp->sd_qhead == NULL); 3275 ASSERT(stp->sd_qtail == NULL); 3276 ASSERT(stp->sd_nqueues == 0); 3277 kmem_cache_free(stream_head_cache, stp); 3278 } 3279 3280 /* 3281 * Allocate a pair of queues and a syncq for the pair 3282 */ 3283 queue_t * 3284 allocq(void) 3285 { 3286 queinfo_t *qip; 3287 queue_t *qp, *wqp; 3288 syncq_t *sq; 3289 3290 qip = kmem_cache_alloc(queue_cache, KM_SLEEP); 3291 3292 qp = &qip->qu_rqueue; 3293 wqp = &qip->qu_wqueue; 3294 sq = &qip->qu_syncq; 3295 3296 qp->q_last = NULL; 3297 qp->q_next = NULL; 3298 qp->q_ptr = NULL; 3299 qp->q_flag = QUSE | QREADR; 3300 qp->q_bandp = NULL; 3301 qp->q_stream = NULL; 3302 qp->q_syncq = sq; 3303 qp->q_nband = 0; 3304 qp->q_nfsrv = NULL; 3305 qp->q_draining = 0; 3306 qp->q_syncqmsgs = 0; 3307 qp->q_spri = 0; 3308 qp->q_qtstamp = 0; 3309 qp->q_sqtstamp = 0; 3310 qp->q_fp = NULL; 3311 3312 wqp->q_last = NULL; 3313 wqp->q_next = NULL; 3314 wqp->q_ptr = NULL; 3315 wqp->q_flag = QUSE; 3316 wqp->q_bandp = NULL; 3317 wqp->q_stream = NULL; 3318 wqp->q_syncq = sq; 3319 wqp->q_nband = 0; 3320 wqp->q_nfsrv = NULL; 3321 wqp->q_draining = 0; 3322 wqp->q_syncqmsgs = 0; 3323 wqp->q_qtstamp = 0; 3324 wqp->q_sqtstamp = 0; 3325 wqp->q_spri = 0; 3326 3327 sq->sq_count = 0; 3328 sq->sq_rmqcount = 0; 3329 sq->sq_flags = 0; 3330 sq->sq_type = 0; 3331 sq->sq_callbflags = 0; 3332 sq->sq_cancelid = 0; 3333 sq->sq_ciputctrl = NULL; 3334 sq->sq_nciputctrl = 0; 3335 sq->sq_needexcl = 0; 3336 sq->sq_svcflags = 0; 3337 3338 return (qp); 3339 } 3340 3341 /* 3342 * Free a pair of queues and the "attached" syncq. 3343 * Discard any messages left on the syncq(s), remove the syncq(s) from the 3344 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq. 3345 */ 3346 void 3347 freeq(queue_t *qp) 3348 { 3349 qband_t *qbp, *nqbp; 3350 syncq_t *sq, *outer; 3351 queue_t *wqp = _WR(qp); 3352 3353 ASSERT(qp->q_flag & QREADR); 3354 3355 /* 3356 * If a previously dispatched taskq job is scheduled to run 3357 * sync_service() or a service routine is scheduled for the 3358 * queues about to be freed, wait here until all service is 3359 * done on the queue and all associated queues and syncqs. 3360 */ 3361 wait_svc(qp); 3362 3363 (void) flush_syncq(qp->q_syncq, qp); 3364 (void) flush_syncq(wqp->q_syncq, wqp); 3365 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0); 3366 3367 /* 3368 * Flush the queues before q_next is set to NULL This is needed 3369 * in order to backenable any downstream queue before we go away. 3370 * Note: we are already removed from the stream so that the 3371 * backenabling will not cause any messages to be delivered to our 3372 * put procedures. 3373 */ 3374 flushq(qp, FLUSHALL); 3375 flushq(wqp, FLUSHALL); 3376 3377 /* Tidy up - removeq only does a half-remove from stream */ 3378 qp->q_next = wqp->q_next = NULL; 3379 ASSERT(!(qp->q_flag & QENAB)); 3380 ASSERT(!(wqp->q_flag & QENAB)); 3381 3382 outer = qp->q_syncq->sq_outer; 3383 if (outer != NULL) { 3384 outer_remove(outer, qp->q_syncq); 3385 if (wqp->q_syncq != qp->q_syncq) 3386 outer_remove(outer, wqp->q_syncq); 3387 } 3388 /* 3389 * Free any syncqs that are outside what allocq returned. 3390 */ 3391 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD)) 3392 free_syncq(qp->q_syncq); 3393 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp)) 3394 free_syncq(wqp->q_syncq); 3395 3396 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3397 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3398 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 3399 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp))); 3400 sq = SQ(qp); 3401 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 3402 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 3403 ASSERT(sq->sq_outer == NULL); 3404 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 3405 ASSERT(sq->sq_callbpend == NULL); 3406 ASSERT(sq->sq_needexcl == 0); 3407 3408 if (sq->sq_ciputctrl != NULL) { 3409 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 3410 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 3411 sq->sq_nciputctrl, 0); 3412 ASSERT(ciputctrl_cache != NULL); 3413 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 3414 sq->sq_ciputctrl = NULL; 3415 sq->sq_nciputctrl = 0; 3416 } 3417 3418 ASSERT(qp->q_first == NULL && wqp->q_first == NULL); 3419 ASSERT(qp->q_count == 0 && wqp->q_count == 0); 3420 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0); 3421 3422 qp->q_flag &= ~QUSE; 3423 wqp->q_flag &= ~QUSE; 3424 3425 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */ 3426 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */ 3427 3428 qbp = qp->q_bandp; 3429 while (qbp) { 3430 nqbp = qbp->qb_next; 3431 freeband(qbp); 3432 qbp = nqbp; 3433 } 3434 qbp = wqp->q_bandp; 3435 while (qbp) { 3436 nqbp = qbp->qb_next; 3437 freeband(qbp); 3438 qbp = nqbp; 3439 } 3440 kmem_cache_free(queue_cache, qp); 3441 } 3442 3443 /* 3444 * Allocate a qband structure. 3445 */ 3446 qband_t * 3447 allocband(void) 3448 { 3449 qband_t *qbp; 3450 3451 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP); 3452 if (qbp == NULL) 3453 return (NULL); 3454 3455 qbp->qb_next = NULL; 3456 qbp->qb_count = 0; 3457 qbp->qb_mblkcnt = 0; 3458 qbp->qb_first = NULL; 3459 qbp->qb_last = NULL; 3460 qbp->qb_flag = 0; 3461 3462 return (qbp); 3463 } 3464 3465 /* 3466 * Free a qband structure. 3467 */ 3468 void 3469 freeband(qband_t *qbp) 3470 { 3471 kmem_cache_free(qband_cache, qbp); 3472 } 3473 3474 /* 3475 * Just like putnextctl(9F), except that allocb_wait() is used. 3476 * 3477 * Consolidation Private, and of course only callable from the stream head or 3478 * routines that may block. 3479 */ 3480 int 3481 putnextctl_wait(queue_t *q, int type) 3482 { 3483 mblk_t *bp; 3484 int error; 3485 3486 if ((datamsg(type) && (type != M_DELAY)) || 3487 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL) 3488 return (0); 3489 3490 bp->b_datap->db_type = (unsigned char)type; 3491 putnext(q, bp); 3492 return (1); 3493 } 3494 3495 /* 3496 * run any possible bufcalls. 3497 */ 3498 void 3499 runbufcalls(void) 3500 { 3501 strbufcall_t *bcp; 3502 3503 mutex_enter(&bcall_monitor); 3504 mutex_enter(&strbcall_lock); 3505 3506 if (strbcalls.bc_head) { 3507 size_t count; 3508 int nevent; 3509 3510 /* 3511 * count how many events are on the list 3512 * now so we can check to avoid looping 3513 * in low memory situations 3514 */ 3515 nevent = 0; 3516 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next) 3517 nevent++; 3518 3519 /* 3520 * get estimate of available memory from kmem_avail(). 3521 * awake all bufcall functions waiting for 3522 * memory whose request could be satisfied 3523 * by 'count' memory and let 'em fight for it. 3524 */ 3525 count = kmem_avail(); 3526 while ((bcp = strbcalls.bc_head) != NULL && nevent) { 3527 STRSTAT(bufcalls); 3528 --nevent; 3529 if (bcp->bc_size <= count) { 3530 bcp->bc_executor = curthread; 3531 mutex_exit(&strbcall_lock); 3532 (*bcp->bc_func)(bcp->bc_arg); 3533 mutex_enter(&strbcall_lock); 3534 bcp->bc_executor = NULL; 3535 cv_broadcast(&bcall_cv); 3536 strbcalls.bc_head = bcp->bc_next; 3537 kmem_free(bcp, sizeof (strbufcall_t)); 3538 } else { 3539 /* 3540 * too big, try again later - note 3541 * that nevent was decremented above 3542 * so we won't retry this one on this 3543 * iteration of the loop 3544 */ 3545 if (bcp->bc_next != NULL) { 3546 strbcalls.bc_head = bcp->bc_next; 3547 bcp->bc_next = NULL; 3548 strbcalls.bc_tail->bc_next = bcp; 3549 strbcalls.bc_tail = bcp; 3550 } 3551 } 3552 } 3553 if (strbcalls.bc_head == NULL) 3554 strbcalls.bc_tail = NULL; 3555 } 3556 3557 mutex_exit(&strbcall_lock); 3558 mutex_exit(&bcall_monitor); 3559 } 3560 3561 3562 /* 3563 * actually run queue's service routine. 3564 */ 3565 static void 3566 runservice(queue_t *q) 3567 { 3568 qband_t *qbp; 3569 3570 ASSERT(q->q_qinfo->qi_srvp); 3571 again: 3572 entersq(q->q_syncq, SQ_SVC); 3573 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START, 3574 "runservice starts:%p", q); 3575 3576 if (!(q->q_flag & QWCLOSE)) 3577 (*q->q_qinfo->qi_srvp)(q); 3578 3579 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END, 3580 "runservice ends:(%p)", q); 3581 3582 leavesq(q->q_syncq, SQ_SVC); 3583 3584 mutex_enter(QLOCK(q)); 3585 if (q->q_flag & QENAB) { 3586 q->q_flag &= ~QENAB; 3587 mutex_exit(QLOCK(q)); 3588 goto again; 3589 } 3590 q->q_flag &= ~QINSERVICE; 3591 q->q_flag &= ~QBACK; 3592 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next) 3593 qbp->qb_flag &= ~QB_BACK; 3594 /* 3595 * Wakeup thread waiting for the service procedure 3596 * to be run (strclose and qdetach). 3597 */ 3598 cv_broadcast(&q->q_wait); 3599 3600 mutex_exit(QLOCK(q)); 3601 } 3602 3603 /* 3604 * Background processing of bufcalls. 3605 */ 3606 void 3607 streams_bufcall_service(void) 3608 { 3609 callb_cpr_t cprinfo; 3610 3611 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr, 3612 "streams_bufcall_service"); 3613 3614 mutex_enter(&strbcall_lock); 3615 3616 for (;;) { 3617 if (strbcalls.bc_head != NULL && kmem_avail() > 0) { 3618 mutex_exit(&strbcall_lock); 3619 runbufcalls(); 3620 mutex_enter(&strbcall_lock); 3621 } 3622 if (strbcalls.bc_head != NULL) { 3623 clock_t wt, tick; 3624 3625 STRSTAT(bcwaits); 3626 /* Wait for memory to become available */ 3627 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3628 tick = SEC_TO_TICK(60); 3629 time_to_wait(&wt, tick); 3630 (void) cv_timedwait(&memavail_cv, &strbcall_lock, wt); 3631 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3632 } 3633 3634 /* Wait for new work to arrive */ 3635 if (strbcalls.bc_head == NULL) { 3636 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3637 cv_wait(&strbcall_cv, &strbcall_lock); 3638 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3639 } 3640 } 3641 } 3642 3643 /* 3644 * Background processing of streams background tasks which failed 3645 * taskq_dispatch. 3646 */ 3647 static void 3648 streams_qbkgrnd_service(void) 3649 { 3650 callb_cpr_t cprinfo; 3651 queue_t *q; 3652 3653 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3654 "streams_bkgrnd_service"); 3655 3656 mutex_enter(&service_queue); 3657 3658 for (;;) { 3659 /* 3660 * Wait for work to arrive. 3661 */ 3662 while ((freebs_list == NULL) && (qhead == NULL)) { 3663 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3664 cv_wait(&services_to_run, &service_queue); 3665 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3666 } 3667 /* 3668 * Handle all pending freebs requests to free memory. 3669 */ 3670 while (freebs_list != NULL) { 3671 mblk_t *mp = freebs_list; 3672 freebs_list = mp->b_next; 3673 mutex_exit(&service_queue); 3674 mblk_free(mp); 3675 mutex_enter(&service_queue); 3676 } 3677 /* 3678 * Run pending queues. 3679 */ 3680 while (qhead != NULL) { 3681 DQ(q, qhead, qtail, q_link); 3682 ASSERT(q != NULL); 3683 mutex_exit(&service_queue); 3684 queue_service(q); 3685 mutex_enter(&service_queue); 3686 } 3687 ASSERT(qhead == NULL && qtail == NULL); 3688 } 3689 } 3690 3691 /* 3692 * Background processing of streams background tasks which failed 3693 * taskq_dispatch. 3694 */ 3695 static void 3696 streams_sqbkgrnd_service(void) 3697 { 3698 callb_cpr_t cprinfo; 3699 syncq_t *sq; 3700 3701 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3702 "streams_sqbkgrnd_service"); 3703 3704 mutex_enter(&service_queue); 3705 3706 for (;;) { 3707 /* 3708 * Wait for work to arrive. 3709 */ 3710 while (sqhead == NULL) { 3711 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3712 cv_wait(&syncqs_to_run, &service_queue); 3713 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3714 } 3715 3716 /* 3717 * Run pending syncqs. 3718 */ 3719 while (sqhead != NULL) { 3720 DQ(sq, sqhead, sqtail, sq_next); 3721 ASSERT(sq != NULL); 3722 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); 3723 mutex_exit(&service_queue); 3724 syncq_service(sq); 3725 mutex_enter(&service_queue); 3726 } 3727 } 3728 } 3729 3730 /* 3731 * Disable the syncq and wait for background syncq processing to complete. 3732 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the 3733 * list. 3734 */ 3735 void 3736 wait_sq_svc(syncq_t *sq) 3737 { 3738 mutex_enter(SQLOCK(sq)); 3739 sq->sq_svcflags |= SQ_DISABLED; 3740 if (sq->sq_svcflags & SQ_BGTHREAD) { 3741 syncq_t *sq_chase; 3742 syncq_t *sq_curr; 3743 int removed; 3744 3745 ASSERT(sq->sq_servcount == 1); 3746 mutex_enter(&service_queue); 3747 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); 3748 mutex_exit(&service_queue); 3749 if (removed) { 3750 sq->sq_svcflags &= ~SQ_BGTHREAD; 3751 sq->sq_servcount = 0; 3752 STRSTAT(sqremoved); 3753 goto done; 3754 } 3755 } 3756 while (sq->sq_servcount != 0) { 3757 sq->sq_flags |= SQ_WANTWAKEUP; 3758 cv_wait(&sq->sq_wait, SQLOCK(sq)); 3759 } 3760 done: 3761 mutex_exit(SQLOCK(sq)); 3762 } 3763 3764 /* 3765 * Put a syncq on the list of syncq's to be serviced by the sqthread. 3766 * Add the argument to the end of the sqhead list and set the flag 3767 * indicating this syncq has been enabled. If it has already been 3768 * enabled, don't do anything. 3769 * This routine assumes that SQLOCK is held. 3770 * NOTE that the lock order is to have the SQLOCK first, 3771 * so if the service_syncq lock is held, we need to release it 3772 * before aquiring the SQLOCK (mostly relevant for the background 3773 * thread, and this seems to be common among the STREAMS global locks). 3774 * Note the the sq_svcflags are protected by the SQLOCK. 3775 */ 3776 void 3777 sqenable(syncq_t *sq) 3778 { 3779 /* 3780 * This is probably not important except for where I believe it 3781 * is being called. At that point, it should be held (and it 3782 * is a pain to release it just for this routine, so don't do 3783 * it). 3784 */ 3785 ASSERT(MUTEX_HELD(SQLOCK(sq))); 3786 3787 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); 3788 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); 3789 3790 /* 3791 * Do not put on list if background thread is scheduled or 3792 * syncq is disabled. 3793 */ 3794 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) 3795 return; 3796 3797 /* 3798 * Check whether we should enable sq at all. 3799 * Non PERMOD syncqs may be drained by at most one thread. 3800 * PERMOD syncqs may be drained by several threads but we limit the 3801 * total amount to the lesser of 3802 * Number of queues on the squeue and 3803 * Number of CPUs. 3804 */ 3805 if (sq->sq_servcount != 0) { 3806 if (((sq->sq_type & SQ_PERMOD) == 0) || 3807 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { 3808 STRSTAT(sqtoomany); 3809 return; 3810 } 3811 } 3812 3813 sq->sq_tstamp = lbolt; 3814 STRSTAT(sqenables); 3815 3816 /* Attempt a taskq dispatch */ 3817 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, 3818 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); 3819 if (sq->sq_servid != NULL) { 3820 sq->sq_servcount++; 3821 return; 3822 } 3823 3824 /* 3825 * This taskq dispatch failed, but a previous one may have succeeded. 3826 * Don't try to schedule on the background thread whilst there is 3827 * outstanding taskq processing. 3828 */ 3829 if (sq->sq_servcount != 0) 3830 return; 3831 3832 /* 3833 * System is low on resources and can't perform a non-sleeping 3834 * dispatch. Schedule the syncq for a background thread and mark the 3835 * syncq to avoid any further taskq dispatch attempts. 3836 */ 3837 mutex_enter(&service_queue); 3838 STRSTAT(taskqfails); 3839 ENQUEUE(sq, sqhead, sqtail, sq_next); 3840 sq->sq_svcflags |= SQ_BGTHREAD; 3841 sq->sq_servcount = 1; 3842 cv_signal(&syncqs_to_run); 3843 mutex_exit(&service_queue); 3844 } 3845 3846 /* 3847 * Note: fifo_close() depends on the mblk_t on the queue being freed 3848 * asynchronously. The asynchronous freeing of messages breaks the 3849 * recursive call chain of fifo_close() while there are I_SENDFD type of 3850 * messages refering other file pointers on the queue. Then when 3851 * closing pipes it can avoid stack overflow in case of daisy-chained 3852 * pipes, and also avoid deadlock in case of fifonode_t pairs (which 3853 * share the same fifolock_t). 3854 */ 3855 3856 /* ARGSUSED */ 3857 void 3858 freebs_enqueue(mblk_t *mp, dblk_t *dbp) 3859 { 3860 ASSERT(dbp->db_mblk == mp); 3861 3862 /* 3863 * Check data sanity. The dblock should have non-empty free function. 3864 * It is better to panic here then later when the dblock is freed 3865 * asynchronously when the context is lost. 3866 */ 3867 if (dbp->db_frtnp->free_func == NULL) { 3868 panic("freebs_enqueue: dblock %p has a NULL free callback", 3869 (void *) dbp); 3870 } 3871 3872 STRSTAT(freebs); 3873 if (taskq_dispatch(streams_taskq, (task_func_t *)mblk_free, mp, 3874 TQ_NOSLEEP) == NULL) { 3875 /* 3876 * System is low on resources and can't perform a non-sleeping 3877 * dispatch. Schedule for a background thread. 3878 */ 3879 mutex_enter(&service_queue); 3880 STRSTAT(taskqfails); 3881 mp->b_next = freebs_list; 3882 freebs_list = mp; 3883 cv_signal(&services_to_run); 3884 mutex_exit(&service_queue); 3885 } 3886 } 3887 3888 /* 3889 * Set the QBACK or QB_BACK flag in the given queue for 3890 * the given priority band. 3891 */ 3892 void 3893 setqback(queue_t *q, unsigned char pri) 3894 { 3895 int i; 3896 qband_t *qbp; 3897 qband_t **qbpp; 3898 3899 ASSERT(MUTEX_HELD(QLOCK(q))); 3900 if (pri != 0) { 3901 if (pri > q->q_nband) { 3902 qbpp = &q->q_bandp; 3903 while (*qbpp) 3904 qbpp = &(*qbpp)->qb_next; 3905 while (pri > q->q_nband) { 3906 if ((*qbpp = allocband()) == NULL) { 3907 cmn_err(CE_WARN, 3908 "setqback: can't allocate qband\n"); 3909 return; 3910 } 3911 (*qbpp)->qb_hiwat = q->q_hiwat; 3912 (*qbpp)->qb_lowat = q->q_lowat; 3913 q->q_nband++; 3914 qbpp = &(*qbpp)->qb_next; 3915 } 3916 } 3917 qbp = q->q_bandp; 3918 i = pri; 3919 while (--i) 3920 qbp = qbp->qb_next; 3921 qbp->qb_flag |= QB_BACK; 3922 } else { 3923 q->q_flag |= QBACK; 3924 } 3925 } 3926 3927 int 3928 strcopyin(void *from, void *to, size_t len, int copyflag) 3929 { 3930 if (copyflag & U_TO_K) { 3931 ASSERT((copyflag & K_TO_K) == 0); 3932 if (copyin(from, to, len)) 3933 return (EFAULT); 3934 } else { 3935 ASSERT(copyflag & K_TO_K); 3936 bcopy(from, to, len); 3937 } 3938 return (0); 3939 } 3940 3941 int 3942 strcopyout(void *from, void *to, size_t len, int copyflag) 3943 { 3944 if (copyflag & U_TO_K) { 3945 if (copyout(from, to, len)) 3946 return (EFAULT); 3947 } else { 3948 ASSERT(copyflag & K_TO_K); 3949 bcopy(from, to, len); 3950 } 3951 return (0); 3952 } 3953 3954 /* 3955 * strsignal_nolock() posts a signal to the process(es) at the stream head. 3956 * It assumes that the stream head lock is already held, whereas strsignal() 3957 * acquires the lock first. This routine was created because a few callers 3958 * release the stream head lock before calling only to re-acquire it after 3959 * it returns. 3960 */ 3961 void 3962 strsignal_nolock(stdata_t *stp, int sig, int32_t band) 3963 { 3964 ASSERT(MUTEX_HELD(&stp->sd_lock)); 3965 switch (sig) { 3966 case SIGPOLL: 3967 if (stp->sd_sigflags & S_MSG) 3968 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 3969 break; 3970 3971 default: 3972 if (stp->sd_pgidp) { 3973 pgsignal(stp->sd_pgidp, sig); 3974 } 3975 break; 3976 } 3977 } 3978 3979 void 3980 strsignal(stdata_t *stp, int sig, int32_t band) 3981 { 3982 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG, 3983 "strsignal:%p, %X, %X", stp, sig, band); 3984 3985 mutex_enter(&stp->sd_lock); 3986 switch (sig) { 3987 case SIGPOLL: 3988 if (stp->sd_sigflags & S_MSG) 3989 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 3990 break; 3991 3992 default: 3993 if (stp->sd_pgidp) { 3994 pgsignal(stp->sd_pgidp, sig); 3995 } 3996 break; 3997 } 3998 mutex_exit(&stp->sd_lock); 3999 } 4000 4001 void 4002 strhup(stdata_t *stp) 4003 { 4004 pollwakeup(&stp->sd_pollist, POLLHUP); 4005 mutex_enter(&stp->sd_lock); 4006 if (stp->sd_sigflags & S_HANGUP) 4007 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0); 4008 mutex_exit(&stp->sd_lock); 4009 } 4010 4011 void 4012 stralloctty(stdata_t *stp) 4013 { 4014 proc_t *p = curproc; 4015 sess_t *sp = p->p_sessp; 4016 4017 mutex_enter(&stp->sd_lock); 4018 /* 4019 * No need to hold the session lock or do a TTY_HOLD() because 4020 * this is the only thread that can be the session leader and not 4021 * have a controlling tty. 4022 */ 4023 if ((stp->sd_flag & 4024 (STRHUP|STRDERR|STWRERR|STPLEX|STRISTTY)) == STRISTTY && 4025 stp->sd_sidp == NULL && /* not allocated as ctty */ 4026 sp->s_sidp == p->p_pidp && /* session leader */ 4027 sp->s_flag != SESS_CLOSE && /* session is not closing */ 4028 sp->s_vp == NULL) { /* without ctty */ 4029 ASSERT(stp->sd_pgidp == NULL); 4030 alloctty(p, makectty(stp->sd_vnode)); 4031 4032 mutex_enter(&pidlock); 4033 stp->sd_sidp = sp->s_sidp; 4034 stp->sd_pgidp = sp->s_sidp; 4035 PID_HOLD(stp->sd_pgidp); 4036 PID_HOLD(stp->sd_sidp); 4037 mutex_exit(&pidlock); 4038 } 4039 mutex_exit(&stp->sd_lock); 4040 } 4041 4042 void 4043 strfreectty(stdata_t *stp) 4044 { 4045 mutex_enter(&stp->sd_lock); 4046 pgsignal(stp->sd_pgidp, SIGHUP); 4047 mutex_enter(&pidlock); 4048 PID_RELE(stp->sd_pgidp); 4049 PID_RELE(stp->sd_sidp); 4050 stp->sd_pgidp = NULL; 4051 stp->sd_sidp = NULL; 4052 mutex_exit(&pidlock); 4053 mutex_exit(&stp->sd_lock); 4054 if (!(stp->sd_flag & STRHUP)) 4055 strhup(stp); 4056 } 4057 /* 4058 * Backenable the first queue upstream from `q' with a service procedure. 4059 */ 4060 void 4061 backenable(queue_t *q, uchar_t pri) 4062 { 4063 queue_t *nq; 4064 4065 /* 4066 * our presence might not prevent other modules in our own 4067 * stream from popping/pushing since the caller of getq might not 4068 * have a claim on the queue (some drivers do a getq on somebody 4069 * else's queue - they know that the queue itself is not going away 4070 * but the framework has to guarantee q_next in that stream.) 4071 */ 4072 claimstr(q); 4073 4074 /* find nearest back queue with service proc */ 4075 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) { 4076 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq)); 4077 } 4078 4079 if (nq) { 4080 kthread_t *freezer; 4081 /* 4082 * backenable can be called either with no locks held 4083 * or with the stream frozen (the latter occurs when a module 4084 * calls rmvq with the stream frozen.) If the stream is frozen 4085 * by the caller the caller will hold all qlocks in the stream. 4086 */ 4087 freezer = STREAM(q)->sd_freezer; 4088 if (freezer != curthread) { 4089 mutex_enter(QLOCK(nq)); 4090 } 4091 #ifdef DEBUG 4092 else { 4093 ASSERT(frozenstr(q)); 4094 ASSERT(MUTEX_HELD(QLOCK(q))); 4095 ASSERT(MUTEX_HELD(QLOCK(nq))); 4096 } 4097 #endif 4098 setqback(nq, pri); 4099 qenable_locked(nq); 4100 if (freezer != curthread) 4101 mutex_exit(QLOCK(nq)); 4102 } 4103 releasestr(q); 4104 } 4105 4106 /* 4107 * Return the appropriate errno when one of flags_to_check is set 4108 * in sd_flags. Uses the exported error routines if they are set. 4109 * Will return 0 if non error is set (or if the exported error routines 4110 * do not return an error). 4111 * 4112 * If there is both a read and write error to check we prefer the read error. 4113 * Also, give preference to recorded errno's over the error functions. 4114 * The flags that are handled are: 4115 * STPLEX return EINVAL 4116 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST) 4117 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST) 4118 * STRHUP return sd_werror 4119 * 4120 * If the caller indicates that the operation is a peek a nonpersistent error 4121 * is not cleared. 4122 */ 4123 int 4124 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek) 4125 { 4126 int32_t sd_flag = stp->sd_flag & flags_to_check; 4127 int error = 0; 4128 4129 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4130 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0); 4131 if (sd_flag & STPLEX) 4132 error = EINVAL; 4133 else if (sd_flag & STRDERR) { 4134 error = stp->sd_rerror; 4135 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) { 4136 /* 4137 * Read errors are non-persistent i.e. discarded once 4138 * returned to a non-peeking caller, 4139 */ 4140 stp->sd_rerror = 0; 4141 stp->sd_flag &= ~STRDERR; 4142 } 4143 if (error == 0 && stp->sd_rderrfunc != NULL) { 4144 int clearerr = 0; 4145 4146 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek, 4147 &clearerr); 4148 if (clearerr) { 4149 stp->sd_flag &= ~STRDERR; 4150 stp->sd_rderrfunc = NULL; 4151 } 4152 } 4153 } else if (sd_flag & STWRERR) { 4154 error = stp->sd_werror; 4155 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) { 4156 /* 4157 * Write errors are non-persistent i.e. discarded once 4158 * returned to a non-peeking caller, 4159 */ 4160 stp->sd_werror = 0; 4161 stp->sd_flag &= ~STWRERR; 4162 } 4163 if (error == 0 && stp->sd_wrerrfunc != NULL) { 4164 int clearerr = 0; 4165 4166 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek, 4167 &clearerr); 4168 if (clearerr) { 4169 stp->sd_flag &= ~STWRERR; 4170 stp->sd_wrerrfunc = NULL; 4171 } 4172 } 4173 } else if (sd_flag & STRHUP) { 4174 /* sd_werror set when STRHUP */ 4175 error = stp->sd_werror; 4176 } 4177 return (error); 4178 } 4179 4180 4181 /* 4182 * single-thread open/close/push/pop 4183 * for twisted streams also 4184 */ 4185 int 4186 strstartplumb(stdata_t *stp, int flag, int cmd) 4187 { 4188 int waited = 1; 4189 int error = 0; 4190 4191 if (STRMATED(stp)) { 4192 struct stdata *stmatep = stp->sd_mate; 4193 4194 STRLOCKMATES(stp); 4195 while (waited) { 4196 waited = 0; 4197 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4198 if ((cmd == I_POP) && 4199 (flag & (FNDELAY|FNONBLOCK))) { 4200 STRUNLOCKMATES(stp); 4201 return (EAGAIN); 4202 } 4203 waited = 1; 4204 mutex_exit(&stp->sd_lock); 4205 if (!cv_wait_sig(&stmatep->sd_monitor, 4206 &stmatep->sd_lock)) { 4207 mutex_exit(&stmatep->sd_lock); 4208 return (EINTR); 4209 } 4210 mutex_exit(&stmatep->sd_lock); 4211 STRLOCKMATES(stp); 4212 } 4213 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4214 if ((cmd == I_POP) && 4215 (flag & (FNDELAY|FNONBLOCK))) { 4216 STRUNLOCKMATES(stp); 4217 return (EAGAIN); 4218 } 4219 waited = 1; 4220 mutex_exit(&stmatep->sd_lock); 4221 if (!cv_wait_sig(&stp->sd_monitor, 4222 &stp->sd_lock)) { 4223 mutex_exit(&stp->sd_lock); 4224 return (EINTR); 4225 } 4226 mutex_exit(&stp->sd_lock); 4227 STRLOCKMATES(stp); 4228 } 4229 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4230 error = strgeterr(stp, 4231 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4232 if (error != 0) { 4233 STRUNLOCKMATES(stp); 4234 return (error); 4235 } 4236 } 4237 } 4238 stp->sd_flag |= STRPLUMB; 4239 STRUNLOCKMATES(stp); 4240 } else { 4241 mutex_enter(&stp->sd_lock); 4242 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4243 if (((cmd == I_POP) || (cmd == _I_REMOVE)) && 4244 (flag & (FNDELAY|FNONBLOCK))) { 4245 mutex_exit(&stp->sd_lock); 4246 return (EAGAIN); 4247 } 4248 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) { 4249 mutex_exit(&stp->sd_lock); 4250 return (EINTR); 4251 } 4252 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4253 error = strgeterr(stp, 4254 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4255 if (error != 0) { 4256 mutex_exit(&stp->sd_lock); 4257 return (error); 4258 } 4259 } 4260 } 4261 stp->sd_flag |= STRPLUMB; 4262 mutex_exit(&stp->sd_lock); 4263 } 4264 return (0); 4265 } 4266 4267 /* 4268 * Complete the plumbing operation associated with stream `stp'. 4269 */ 4270 void 4271 strendplumb(stdata_t *stp) 4272 { 4273 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4274 ASSERT(stp->sd_flag & STRPLUMB); 4275 stp->sd_flag &= ~STRPLUMB; 4276 cv_broadcast(&stp->sd_monitor); 4277 } 4278 4279 /* 4280 * This describes how the STREAMS framework handles synchronization 4281 * during open/push and close/pop. 4282 * The key interfaces for open and close are qprocson and qprocsoff, 4283 * respectively. While the close case in general is harder both open 4284 * have close have significant similarities. 4285 * 4286 * During close the STREAMS framework has to both ensure that there 4287 * are no stale references to the queue pair (and syncq) that 4288 * are being closed and also provide the guarantees that are documented 4289 * in qprocsoff(9F). 4290 * If there are stale references to the queue that is closing it can 4291 * result in kernel memory corruption or kernel panics. 4292 * 4293 * Note that is it up to the module/driver to ensure that it itself 4294 * does not have any stale references to the closing queues once its close 4295 * routine returns. This includes: 4296 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines 4297 * associated with the queues. For timeout and bufcall callbacks the 4298 * module/driver also has to ensure (or wait for) any callbacks that 4299 * are in progress. 4300 * - If the module/driver is using esballoc it has to ensure that any 4301 * esballoc free functions do not refer to a queue that has closed. 4302 * (Note that in general the close routine can not wait for the esballoc'ed 4303 * messages to be freed since that can cause a deadlock.) 4304 * - Cancelling any interrupts that refer to the closing queues and 4305 * also ensuring that there are no interrupts in progress that will 4306 * refer to the closing queues once the close routine returns. 4307 * - For multiplexors removing any driver global state that refers to 4308 * the closing queue and also ensuring that there are no threads in 4309 * the multiplexor that has picked up a queue pointer but not yet 4310 * finished using it. 4311 * 4312 * In addition, a driver/module can only reference the q_next pointer 4313 * in its open, close, put, or service procedures or in a 4314 * qtimeout/qbufcall callback procedure executing "on" the correct 4315 * stream. Thus it can not reference the q_next pointer in an interrupt 4316 * routine or a timeout, bufcall or esballoc callback routine. Likewise 4317 * it can not reference q_next of a different queue e.g. in a mux that 4318 * passes messages from one queues put/service procedure to another queue. 4319 * In all the cases when the driver/module can not access the q_next 4320 * field it must use the *next* versions e.g. canputnext instead of 4321 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...). 4322 * 4323 * 4324 * Assuming that the driver/module conforms to the above constraints 4325 * the STREAMS framework has to avoid stale references to q_next for all 4326 * the framework internal cases which include (but are not limited to): 4327 * - Threads in canput/canputnext/backenable and elsewhere that are 4328 * walking q_next. 4329 * - Messages on a syncq that have a reference to the queue through b_queue. 4330 * - Messages on an outer perimeter (syncq) that have a reference to the 4331 * queue through b_queue. 4332 * - Threads that use q_nfsrv (e.g. canput) to find a queue. 4333 * Note that only canput and bcanput use q_nfsrv without any locking. 4334 * 4335 * The STREAMS framework providing the qprocsoff(9F) guarantees means that 4336 * after qprocsoff returns, the framework has to ensure that no threads can 4337 * enter the put or service routines for the closing read or write-side queue. 4338 * In addition to preventing "direct" entry into the put procedures 4339 * the framework also has to prevent messages being drained from 4340 * the syncq or the outer perimeter. 4341 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only 4342 * mechanism to prevent qwriter(PERIM_OUTER) from running after 4343 * qprocsoff has returned. 4344 * Note that if a module/driver uses put(9F) on one of its own queues 4345 * it is up to the module/driver to ensure that the put() doesn't 4346 * get called when the queue is closing. 4347 * 4348 * 4349 * The framework aspects of the above "contract" is implemented by 4350 * qprocsoff, removeq, and strlock: 4351 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from 4352 * entering the service procedures. 4353 * - strlock acquires the sd_lock and sd_reflock to prevent putnext, 4354 * canputnext, backenable etc from dereferencing the q_next that will 4355 * soon change. 4356 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext 4357 * or other q_next walker that uses claimstr/releasestr to finish. 4358 * - optionally for every syncq in the stream strlock acquires all the 4359 * sq_lock's and waits for all sq_counts to drop to a value that indicates 4360 * that no thread executes in the put or service procedures and that no 4361 * thread is draining into the module/driver. This ensures that no 4362 * open, close, put, service, or qtimeout/qbufcall callback procedure is 4363 * currently executing hence no such thread can end up with the old stale 4364 * q_next value and no canput/backenable can have the old stale 4365 * q_nfsrv/q_next. 4366 * - qdetach (wait_svc) makes sure that any scheduled or running threads 4367 * have either finished or observed the QWCLOSE flag and gone away. 4368 */ 4369 4370 4371 /* 4372 * Get all the locks necessary to change q_next. 4373 * 4374 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the 4375 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that 4376 * the only threads inside the sqncq are threads currently calling removeq(). 4377 * Since threads calling removeq() are in the process of removing their queues 4378 * from the stream, we do not need to worry about them accessing a stale q_next 4379 * pointer and thus we do not need to wait for them to exit (in fact, waiting 4380 * for them can cause deadlock). 4381 * 4382 * This routine is subject to starvation since it does not set any flag to 4383 * prevent threads from entering a module in the stream(i.e. sq_count can 4384 * increase on some syncq while it is waiting on some other syncq.) 4385 * 4386 * Assumes that only one thread attempts to call strlock for a given 4387 * stream. If this is not the case the two threads would deadlock. 4388 * This assumption is guaranteed since strlock is only called by insertq 4389 * and removeq and streams plumbing changes are single-threaded for 4390 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags. 4391 * 4392 * For pipes, it is not difficult to atomically designate a pair of streams 4393 * to be mated. Once mated atomically by the framework the twisted pair remain 4394 * configured that way until dismantled atomically by the framework. 4395 * When plumbing takes place on a twisted stream it is necessary to ensure that 4396 * this operation is done exclusively on the twisted stream since two such 4397 * operations, each initiated on different ends of the pipe will deadlock 4398 * waiting for each other to complete. 4399 * 4400 * On entry, no locks should be held. 4401 * The locks acquired and held by strlock depends on a few factors. 4402 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired 4403 * and held on exit and all sq_count are at an acceptable level. 4404 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with 4405 * sd_refcnt being zero. 4406 */ 4407 4408 static void 4409 strlock(struct stdata *stp, sqlist_t *sqlist) 4410 { 4411 syncql_t *sql, *sql2; 4412 retry: 4413 /* 4414 * Wait for any claimstr to go away. 4415 */ 4416 if (STRMATED(stp)) { 4417 struct stdata *stp1, *stp2; 4418 4419 STRLOCKMATES(stp); 4420 /* 4421 * Note that the selection of locking order is not 4422 * important, just that they are always aquired in 4423 * the same order. To assure this, we choose this 4424 * order based on the value of the pointer, and since 4425 * the pointer will not change for the life of this 4426 * pair, we will always grab the locks in the same 4427 * order (and hence, prevent deadlocks). 4428 */ 4429 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) { 4430 stp1 = stp; 4431 stp2 = stp->sd_mate; 4432 } else { 4433 stp2 = stp; 4434 stp1 = stp->sd_mate; 4435 } 4436 mutex_enter(&stp1->sd_reflock); 4437 if (stp1->sd_refcnt > 0) { 4438 STRUNLOCKMATES(stp); 4439 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock); 4440 mutex_exit(&stp1->sd_reflock); 4441 goto retry; 4442 } 4443 mutex_enter(&stp2->sd_reflock); 4444 if (stp2->sd_refcnt > 0) { 4445 STRUNLOCKMATES(stp); 4446 mutex_exit(&stp1->sd_reflock); 4447 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock); 4448 mutex_exit(&stp2->sd_reflock); 4449 goto retry; 4450 } 4451 STREAM_PUTLOCKS_ENTER(stp1); 4452 STREAM_PUTLOCKS_ENTER(stp2); 4453 } else { 4454 mutex_enter(&stp->sd_lock); 4455 mutex_enter(&stp->sd_reflock); 4456 while (stp->sd_refcnt > 0) { 4457 mutex_exit(&stp->sd_lock); 4458 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock); 4459 if (mutex_tryenter(&stp->sd_lock) == 0) { 4460 mutex_exit(&stp->sd_reflock); 4461 mutex_enter(&stp->sd_lock); 4462 mutex_enter(&stp->sd_reflock); 4463 } 4464 } 4465 STREAM_PUTLOCKS_ENTER(stp); 4466 } 4467 4468 if (sqlist == NULL) 4469 return; 4470 4471 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4472 syncq_t *sq = sql->sql_sq; 4473 uint16_t count; 4474 4475 mutex_enter(SQLOCK(sq)); 4476 count = sq->sq_count; 4477 ASSERT(sq->sq_rmqcount <= count); 4478 SQ_PUTLOCKS_ENTER(sq); 4479 SUM_SQ_PUTCOUNTS(sq, count); 4480 if (count == sq->sq_rmqcount) 4481 continue; 4482 4483 /* Failed - drop all locks that we have acquired so far */ 4484 if (STRMATED(stp)) { 4485 STREAM_PUTLOCKS_EXIT(stp); 4486 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4487 STRUNLOCKMATES(stp); 4488 mutex_exit(&stp->sd_reflock); 4489 mutex_exit(&stp->sd_mate->sd_reflock); 4490 } else { 4491 STREAM_PUTLOCKS_EXIT(stp); 4492 mutex_exit(&stp->sd_lock); 4493 mutex_exit(&stp->sd_reflock); 4494 } 4495 for (sql2 = sqlist->sqlist_head; sql2 != sql; 4496 sql2 = sql2->sql_next) { 4497 SQ_PUTLOCKS_EXIT(sql2->sql_sq); 4498 mutex_exit(SQLOCK(sql2->sql_sq)); 4499 } 4500 4501 /* 4502 * The wait loop below may starve when there are many threads 4503 * claiming the syncq. This is especially a problem with permod 4504 * syncqs (IP). To lessen the impact of the problem we increment 4505 * sq_needexcl and clear fastbits so that putnexts will slow 4506 * down and call sqenable instead of draining right away. 4507 */ 4508 sq->sq_needexcl++; 4509 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4510 while (count > sq->sq_rmqcount) { 4511 sq->sq_flags |= SQ_WANTWAKEUP; 4512 SQ_PUTLOCKS_EXIT(sq); 4513 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4514 count = sq->sq_count; 4515 SQ_PUTLOCKS_ENTER(sq); 4516 SUM_SQ_PUTCOUNTS(sq, count); 4517 } 4518 sq->sq_needexcl--; 4519 if (sq->sq_needexcl == 0) 4520 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4521 SQ_PUTLOCKS_EXIT(sq); 4522 ASSERT(count == sq->sq_rmqcount); 4523 mutex_exit(SQLOCK(sq)); 4524 goto retry; 4525 } 4526 } 4527 4528 /* 4529 * Drop all the locks that strlock acquired. 4530 */ 4531 static void 4532 strunlock(struct stdata *stp, sqlist_t *sqlist) 4533 { 4534 syncql_t *sql; 4535 4536 if (STRMATED(stp)) { 4537 STREAM_PUTLOCKS_EXIT(stp); 4538 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4539 STRUNLOCKMATES(stp); 4540 mutex_exit(&stp->sd_reflock); 4541 mutex_exit(&stp->sd_mate->sd_reflock); 4542 } else { 4543 STREAM_PUTLOCKS_EXIT(stp); 4544 mutex_exit(&stp->sd_lock); 4545 mutex_exit(&stp->sd_reflock); 4546 } 4547 4548 if (sqlist == NULL) 4549 return; 4550 4551 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4552 SQ_PUTLOCKS_EXIT(sql->sql_sq); 4553 mutex_exit(SQLOCK(sql->sql_sq)); 4554 } 4555 } 4556 4557 /* 4558 * When the module has service procedure, we need check if the next 4559 * module which has service procedure is in flow control to trigger 4560 * the backenable. 4561 */ 4562 static void 4563 backenable_insertedq(queue_t *q) 4564 { 4565 qband_t *qbp; 4566 4567 claimstr(q); 4568 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) { 4569 if (q->q_next->q_nfsrv->q_flag & QWANTW) 4570 backenable(q, 0); 4571 4572 qbp = q->q_next->q_nfsrv->q_bandp; 4573 for (; qbp != NULL; qbp = qbp->qb_next) 4574 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL) 4575 backenable(q, qbp->qb_first->b_band); 4576 } 4577 releasestr(q); 4578 } 4579 4580 /* 4581 * Given two read queues, insert a new single one after another. 4582 * 4583 * This routine acquires all the necessary locks in order to change 4584 * q_next and related pointer using strlock(). 4585 * It depends on the stream head ensuring that there are no concurrent 4586 * insertq or removeq on the same stream. The stream head ensures this 4587 * using the flags STWOPEN, STRCLOSE, and STRPLUMB. 4588 * 4589 * Note that no syncq locks are held during the q_next change. This is 4590 * applied to all streams since, unlike removeq, there is no problem of stale 4591 * pointers when adding a module to the stream. Thus drivers/modules that do a 4592 * canput(rq->q_next) would never get a closed/freed queue pointer even if we 4593 * applied this optimization to all streams. 4594 */ 4595 void 4596 insertq(struct stdata *stp, queue_t *new) 4597 { 4598 queue_t *after; 4599 queue_t *wafter; 4600 queue_t *wnew = _WR(new); 4601 boolean_t have_fifo = B_FALSE; 4602 4603 if (new->q_flag & _QINSERTING) { 4604 ASSERT(stp->sd_vnode->v_type != VFIFO); 4605 after = new->q_next; 4606 wafter = _WR(new->q_next); 4607 } else { 4608 after = _RD(stp->sd_wrq); 4609 wafter = stp->sd_wrq; 4610 } 4611 4612 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ, 4613 "insertq:%p, %p", after, new); 4614 ASSERT(after->q_flag & QREADR); 4615 ASSERT(new->q_flag & QREADR); 4616 4617 strlock(stp, NULL); 4618 4619 /* Do we have a FIFO? */ 4620 if (wafter->q_next == after) { 4621 have_fifo = B_TRUE; 4622 wnew->q_next = new; 4623 } else { 4624 wnew->q_next = wafter->q_next; 4625 } 4626 new->q_next = after; 4627 4628 set_nfsrv_ptr(new, wnew, after, wafter); 4629 /* 4630 * set_nfsrv_ptr() needs to know if this is an insertion or not, 4631 * so only reset this flag after calling it. 4632 */ 4633 new->q_flag &= ~_QINSERTING; 4634 4635 if (have_fifo) { 4636 wafter->q_next = wnew; 4637 } else { 4638 if (wafter->q_next) 4639 _OTHERQ(wafter->q_next)->q_next = new; 4640 wafter->q_next = wnew; 4641 } 4642 4643 set_qend(new); 4644 /* The QEND flag might have to be updated for the upstream guy */ 4645 set_qend(after); 4646 4647 ASSERT(_SAMESTR(new) == O_SAMESTR(new)); 4648 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew)); 4649 ASSERT(_SAMESTR(after) == O_SAMESTR(after)); 4650 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter)); 4651 strsetuio(stp); 4652 4653 /* 4654 * If this was a module insertion, bump the push count. 4655 */ 4656 if (!(new->q_flag & QISDRV)) 4657 stp->sd_pushcnt++; 4658 4659 strunlock(stp, NULL); 4660 4661 /* check if the write Q needs backenable */ 4662 backenable_insertedq(wnew); 4663 4664 /* check if the read Q needs backenable */ 4665 backenable_insertedq(new); 4666 } 4667 4668 /* 4669 * Given a read queue, unlink it from any neighbors. 4670 * 4671 * This routine acquires all the necessary locks in order to 4672 * change q_next and related pointers and also guard against 4673 * stale references (e.g. through q_next) to the queue that 4674 * is being removed. It also plays part of the role in ensuring 4675 * that the module's/driver's put procedure doesn't get called 4676 * after qprocsoff returns. 4677 * 4678 * Removeq depends on the stream head ensuring that there are 4679 * no concurrent insertq or removeq on the same stream. The 4680 * stream head ensures this using the flags STWOPEN, STRCLOSE and 4681 * STRPLUMB. 4682 * 4683 * The set of locks needed to remove the queue is different in 4684 * different cases: 4685 * 4686 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after 4687 * waiting for the syncq reference count to drop to 0 indicating that no 4688 * non-close threads are present anywhere in the stream. This ensures that any 4689 * module/driver can reference q_next in its open, close, put, or service 4690 * procedures. 4691 * 4692 * The sq_rmqcount counter tracks the number of threads inside removeq(). 4693 * strlock() ensures that there is either no threads executing inside perimeter 4694 * or there is only a thread calling qprocsoff(). 4695 * 4696 * strlock() compares the value of sq_count with the number of threads inside 4697 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup 4698 * any threads waiting in strlock() when the sq_rmqcount increases. 4699 */ 4700 4701 void 4702 removeq(queue_t *qp) 4703 { 4704 queue_t *wqp = _WR(qp); 4705 struct stdata *stp = STREAM(qp); 4706 sqlist_t *sqlist = NULL; 4707 boolean_t isdriver; 4708 int moved; 4709 syncq_t *sq = qp->q_syncq; 4710 syncq_t *wsq = wqp->q_syncq; 4711 4712 ASSERT(stp); 4713 4714 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ, 4715 "removeq:%p %p", qp, wqp); 4716 ASSERT(qp->q_flag&QREADR); 4717 4718 /* 4719 * For queues using Synchronous streams, we must wait for all threads in 4720 * rwnext() to drain out before proceeding. 4721 */ 4722 if (qp->q_flag & QSYNCSTR) { 4723 /* First, we need wakeup any threads blocked in rwnext() */ 4724 mutex_enter(SQLOCK(sq)); 4725 if (sq->sq_flags & SQ_WANTWAKEUP) { 4726 sq->sq_flags &= ~SQ_WANTWAKEUP; 4727 cv_broadcast(&sq->sq_wait); 4728 } 4729 mutex_exit(SQLOCK(sq)); 4730 4731 if (wsq != sq) { 4732 mutex_enter(SQLOCK(wsq)); 4733 if (wsq->sq_flags & SQ_WANTWAKEUP) { 4734 wsq->sq_flags &= ~SQ_WANTWAKEUP; 4735 cv_broadcast(&wsq->sq_wait); 4736 } 4737 mutex_exit(SQLOCK(wsq)); 4738 } 4739 4740 mutex_enter(QLOCK(qp)); 4741 while (qp->q_rwcnt > 0) { 4742 qp->q_flag |= QWANTRMQSYNC; 4743 cv_wait(&qp->q_wait, QLOCK(qp)); 4744 } 4745 mutex_exit(QLOCK(qp)); 4746 4747 mutex_enter(QLOCK(wqp)); 4748 while (wqp->q_rwcnt > 0) { 4749 wqp->q_flag |= QWANTRMQSYNC; 4750 cv_wait(&wqp->q_wait, QLOCK(wqp)); 4751 } 4752 mutex_exit(QLOCK(wqp)); 4753 } 4754 4755 mutex_enter(SQLOCK(sq)); 4756 sq->sq_rmqcount++; 4757 if (sq->sq_flags & SQ_WANTWAKEUP) { 4758 sq->sq_flags &= ~SQ_WANTWAKEUP; 4759 cv_broadcast(&sq->sq_wait); 4760 } 4761 mutex_exit(SQLOCK(sq)); 4762 4763 isdriver = (qp->q_flag & QISDRV); 4764 4765 sqlist = sqlist_build(qp, stp, STRMATED(stp)); 4766 strlock(stp, sqlist); 4767 4768 reset_nfsrv_ptr(qp, wqp); 4769 4770 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp); 4771 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp); 4772 /* Do we have a FIFO? */ 4773 if (wqp->q_next == qp) { 4774 stp->sd_wrq->q_next = _RD(stp->sd_wrq); 4775 } else { 4776 if (wqp->q_next) 4777 backq(qp)->q_next = qp->q_next; 4778 if (qp->q_next) 4779 backq(wqp)->q_next = wqp->q_next; 4780 } 4781 4782 /* The QEND flag might have to be updated for the upstream guy */ 4783 if (qp->q_next) 4784 set_qend(qp->q_next); 4785 4786 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq)); 4787 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq))); 4788 4789 /* 4790 * Move any messages destined for the put procedures to the next 4791 * syncq in line. Otherwise free them. 4792 */ 4793 moved = 0; 4794 /* 4795 * Quick check to see whether there are any messages or events. 4796 */ 4797 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS)) 4798 moved += propagate_syncq(qp); 4799 if (wqp->q_syncqmsgs != 0 || 4800 (wqp->q_syncq->sq_flags & SQ_EVENTS)) 4801 moved += propagate_syncq(wqp); 4802 4803 strsetuio(stp); 4804 4805 /* 4806 * If this was a module removal, decrement the push count. 4807 */ 4808 if (!isdriver) 4809 stp->sd_pushcnt--; 4810 4811 strunlock(stp, sqlist); 4812 sqlist_free(sqlist); 4813 4814 /* 4815 * Make sure any messages that were propagated are drained. 4816 * Also clear any QFULL bit caused by messages that were propagated. 4817 */ 4818 4819 if (qp->q_next != NULL) { 4820 clr_qfull(qp); 4821 /* 4822 * For the driver calling qprocsoff, propagate_syncq 4823 * frees all the messages instead of putting it in 4824 * the stream head 4825 */ 4826 if (!isdriver && (moved > 0)) 4827 emptysq(qp->q_next->q_syncq); 4828 } 4829 if (wqp->q_next != NULL) { 4830 clr_qfull(wqp); 4831 /* 4832 * We come here for any pop of a module except for the 4833 * case of driver being removed. We don't call emptysq 4834 * if we did not move any messages. This will avoid holding 4835 * PERMOD syncq locks in emptysq 4836 */ 4837 if (moved > 0) 4838 emptysq(wqp->q_next->q_syncq); 4839 } 4840 4841 mutex_enter(SQLOCK(sq)); 4842 sq->sq_rmqcount--; 4843 mutex_exit(SQLOCK(sq)); 4844 } 4845 4846 /* 4847 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or 4848 * SQ_WRITER) on a syncq. 4849 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the 4850 * sync queue and waits until sq_count reaches maxcnt. 4851 * 4852 * if maxcnt is -1 there's no need to grab sq_putlocks since the caller 4853 * does not care about putnext threads that are in the middle of calling put 4854 * entry points. 4855 * 4856 * This routine is used for both inner and outer syncqs. 4857 */ 4858 static void 4859 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) 4860 { 4861 uint16_t count = 0; 4862 4863 mutex_enter(SQLOCK(sq)); 4864 /* 4865 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset. 4866 * SQ_FROZEN will be set if there is a frozen stream that has a 4867 * queue which also refers to this "shared" syncq. 4868 * SQ_BLOCKED will be set if there is "off" queue which also 4869 * refers to this "shared" syncq. 4870 */ 4871 if (maxcnt != -1) { 4872 count = sq->sq_count; 4873 SQ_PUTLOCKS_ENTER(sq); 4874 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4875 SUM_SQ_PUTCOUNTS(sq, count); 4876 } 4877 sq->sq_needexcl++; 4878 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 4879 4880 while ((sq->sq_flags & flag) || 4881 (maxcnt != -1 && count > (unsigned)maxcnt)) { 4882 sq->sq_flags |= SQ_WANTWAKEUP; 4883 if (maxcnt != -1) { 4884 SQ_PUTLOCKS_EXIT(sq); 4885 } 4886 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4887 if (maxcnt != -1) { 4888 count = sq->sq_count; 4889 SQ_PUTLOCKS_ENTER(sq); 4890 SUM_SQ_PUTCOUNTS(sq, count); 4891 } 4892 } 4893 sq->sq_needexcl--; 4894 sq->sq_flags |= flag; 4895 ASSERT(maxcnt == -1 || count == maxcnt); 4896 if (maxcnt != -1) { 4897 if (sq->sq_needexcl == 0) { 4898 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4899 } 4900 SQ_PUTLOCKS_EXIT(sq); 4901 } else if (sq->sq_needexcl == 0) { 4902 SQ_PUTCOUNT_SETFAST(sq); 4903 } 4904 4905 mutex_exit(SQLOCK(sq)); 4906 } 4907 4908 /* 4909 * Reset a flag that was set with blocksq. 4910 * 4911 * Can not use this routine to reset SQ_WRITER. 4912 * 4913 * If "isouter" is set then the syncq is assumed to be an outer perimeter 4914 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread 4915 * to handle the queued qwriter operations. 4916 * 4917 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4918 * sq_putlocks are used. 4919 */ 4920 static void 4921 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) 4922 { 4923 uint16_t flags; 4924 4925 mutex_enter(SQLOCK(sq)); 4926 ASSERT(resetflag != SQ_WRITER); 4927 ASSERT(sq->sq_flags & resetflag); 4928 flags = sq->sq_flags & ~resetflag; 4929 sq->sq_flags = flags; 4930 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) { 4931 if (flags & SQ_WANTWAKEUP) { 4932 flags &= ~SQ_WANTWAKEUP; 4933 cv_broadcast(&sq->sq_wait); 4934 } 4935 sq->sq_flags = flags; 4936 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 4937 if (!isouter) { 4938 /* drain_syncq drops SQLOCK */ 4939 drain_syncq(sq); 4940 return; 4941 } 4942 } 4943 } 4944 mutex_exit(SQLOCK(sq)); 4945 } 4946 4947 /* 4948 * Reset a flag that was set with blocksq. 4949 * Does not drain the syncq. Use emptysq() for that. 4950 * Returns 1 if SQ_QUEUED is set. Otherwise 0. 4951 * 4952 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4953 * sq_putlocks are used. 4954 */ 4955 static int 4956 dropsq(syncq_t *sq, uint16_t resetflag) 4957 { 4958 uint16_t flags; 4959 4960 mutex_enter(SQLOCK(sq)); 4961 ASSERT(sq->sq_flags & resetflag); 4962 flags = sq->sq_flags & ~resetflag; 4963 if (flags & SQ_WANTWAKEUP) { 4964 flags &= ~SQ_WANTWAKEUP; 4965 cv_broadcast(&sq->sq_wait); 4966 } 4967 sq->sq_flags = flags; 4968 mutex_exit(SQLOCK(sq)); 4969 if (flags & SQ_QUEUED) 4970 return (1); 4971 return (0); 4972 } 4973 4974 /* 4975 * Empty all the messages on a syncq. 4976 * 4977 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4978 * sq_putlocks are used. 4979 */ 4980 static void 4981 emptysq(syncq_t *sq) 4982 { 4983 uint16_t flags; 4984 4985 mutex_enter(SQLOCK(sq)); 4986 flags = sq->sq_flags; 4987 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 4988 /* 4989 * To prevent potential recursive invocation of drain_syncq we 4990 * do not call drain_syncq if count is non-zero. 4991 */ 4992 if (sq->sq_count == 0) { 4993 /* drain_syncq() drops SQLOCK */ 4994 drain_syncq(sq); 4995 return; 4996 } else 4997 sqenable(sq); 4998 } 4999 mutex_exit(SQLOCK(sq)); 5000 } 5001 5002 /* 5003 * Ordered insert while removing duplicates. 5004 */ 5005 static void 5006 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp) 5007 { 5008 syncql_t *sqlp, **prev_sqlpp, *new_sqlp; 5009 5010 prev_sqlpp = &sqlist->sqlist_head; 5011 while ((sqlp = *prev_sqlpp) != NULL) { 5012 if (sqlp->sql_sq >= sqp) { 5013 if (sqlp->sql_sq == sqp) /* duplicate */ 5014 return; 5015 break; 5016 } 5017 prev_sqlpp = &sqlp->sql_next; 5018 } 5019 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++]; 5020 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size); 5021 new_sqlp->sql_next = sqlp; 5022 new_sqlp->sql_sq = sqp; 5023 *prev_sqlpp = new_sqlp; 5024 } 5025 5026 /* 5027 * Walk the write side queues until we hit either the driver 5028 * or a twist in the stream (_SAMESTR will return false in both 5029 * these cases) then turn around and walk the read side queues 5030 * back up to the stream head. 5031 */ 5032 static void 5033 sqlist_insertall(sqlist_t *sqlist, queue_t *q) 5034 { 5035 while (q != NULL) { 5036 sqlist_insert(sqlist, q->q_syncq); 5037 5038 if (_SAMESTR(q)) 5039 q = q->q_next; 5040 else if (!(q->q_flag & QREADR)) 5041 q = _RD(q); 5042 else 5043 q = NULL; 5044 } 5045 } 5046 5047 /* 5048 * Allocate and build a list of all syncqs in a stream and the syncq(s) 5049 * associated with the "q" parameter. The resulting list is sorted in a 5050 * canonical order and is free of duplicates. 5051 * Assumes the passed queue is a _RD(q). 5052 */ 5053 static sqlist_t * 5054 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist) 5055 { 5056 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP); 5057 5058 /* 5059 * start with the current queue/qpair 5060 */ 5061 ASSERT(q->q_flag & QREADR); 5062 5063 sqlist_insert(sqlist, q->q_syncq); 5064 sqlist_insert(sqlist, _WR(q)->q_syncq); 5065 5066 sqlist_insertall(sqlist, stp->sd_wrq); 5067 if (do_twist) 5068 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq); 5069 5070 return (sqlist); 5071 } 5072 5073 static sqlist_t * 5074 sqlist_alloc(struct stdata *stp, int kmflag) 5075 { 5076 size_t sqlist_size; 5077 sqlist_t *sqlist; 5078 5079 /* 5080 * Allocate 2 syncql_t's for each pushed module. Note that 5081 * the sqlist_t structure already has 4 syncql_t's built in: 5082 * 2 for the stream head, and 2 for the driver/other stream head. 5083 */ 5084 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt + 5085 sizeof (sqlist_t); 5086 if (STRMATED(stp)) 5087 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt; 5088 sqlist = kmem_alloc(sqlist_size, kmflag); 5089 5090 sqlist->sqlist_head = NULL; 5091 sqlist->sqlist_size = sqlist_size; 5092 sqlist->sqlist_index = 0; 5093 5094 return (sqlist); 5095 } 5096 5097 /* 5098 * Free the list created by sqlist_alloc() 5099 */ 5100 static void 5101 sqlist_free(sqlist_t *sqlist) 5102 { 5103 kmem_free(sqlist, sqlist->sqlist_size); 5104 } 5105 5106 /* 5107 * Prevent any new entries into any syncq in this stream. 5108 * Used by freezestr. 5109 */ 5110 void 5111 strblock(queue_t *q) 5112 { 5113 struct stdata *stp; 5114 syncql_t *sql; 5115 sqlist_t *sqlist; 5116 5117 q = _RD(q); 5118 5119 stp = STREAM(q); 5120 ASSERT(stp != NULL); 5121 5122 /* 5123 * Get a sorted list with all the duplicates removed containing 5124 * all the syncqs referenced by this stream. 5125 */ 5126 sqlist = sqlist_build(q, stp, B_FALSE); 5127 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5128 blocksq(sql->sql_sq, SQ_FROZEN, -1); 5129 sqlist_free(sqlist); 5130 } 5131 5132 /* 5133 * Release the block on new entries into this stream 5134 */ 5135 void 5136 strunblock(queue_t *q) 5137 { 5138 struct stdata *stp; 5139 syncql_t *sql; 5140 sqlist_t *sqlist; 5141 int drain_needed; 5142 5143 q = _RD(q); 5144 5145 /* 5146 * Get a sorted list with all the duplicates removed containing 5147 * all the syncqs referenced by this stream. 5148 * Have to drop the SQ_FROZEN flag on all the syncqs before 5149 * starting to drain them; otherwise the draining might 5150 * cause a freezestr in some module on the stream (which 5151 * would deadlock.) 5152 */ 5153 stp = STREAM(q); 5154 ASSERT(stp != NULL); 5155 sqlist = sqlist_build(q, stp, B_FALSE); 5156 drain_needed = 0; 5157 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5158 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN); 5159 if (drain_needed) { 5160 for (sql = sqlist->sqlist_head; sql != NULL; 5161 sql = sql->sql_next) 5162 emptysq(sql->sql_sq); 5163 } 5164 sqlist_free(sqlist); 5165 } 5166 5167 #ifdef DEBUG 5168 static int 5169 qprocsareon(queue_t *rq) 5170 { 5171 if (rq->q_next == NULL) 5172 return (0); 5173 return (_WR(rq->q_next)->q_next == _WR(rq)); 5174 } 5175 5176 int 5177 qclaimed(queue_t *q) 5178 { 5179 uint_t count; 5180 5181 count = q->q_syncq->sq_count; 5182 SUM_SQ_PUTCOUNTS(q->q_syncq, count); 5183 return (count != 0); 5184 } 5185 5186 /* 5187 * Check if anyone has frozen this stream with freezestr 5188 */ 5189 int 5190 frozenstr(queue_t *q) 5191 { 5192 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0); 5193 } 5194 #endif /* DEBUG */ 5195 5196 /* 5197 * Enter a queue. 5198 * Obsoleted interface. Should not be used. 5199 */ 5200 void 5201 enterq(queue_t *q) 5202 { 5203 entersq(q->q_syncq, SQ_CALLBACK); 5204 } 5205 5206 void 5207 leaveq(queue_t *q) 5208 { 5209 leavesq(q->q_syncq, SQ_CALLBACK); 5210 } 5211 5212 /* 5213 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits 5214 * to check. 5215 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter 5216 * calls and the running of open, close and service procedures. 5217 * 5218 * if c_inner bit is set no need to grab sq_putlocks since we don't care 5219 * if other threads have entered or are entering put entry point. 5220 * 5221 * if c_inner bit is set it might have been posible to use 5222 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize 5223 * open/close path for IP) but since the count may need to be decremented in 5224 * qwait() we wouldn't know which counter to decrement. Currently counter is 5225 * selected by current cpu_seqid and current CPU can change at any moment. XXX 5226 * in the future we might use curthread id bits to select the counter and this 5227 * would stay constant across routine calls. 5228 */ 5229 void 5230 entersq(syncq_t *sq, int entrypoint) 5231 { 5232 uint16_t count = 0; 5233 uint16_t flags; 5234 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 5235 uint16_t type; 5236 uint_t c_inner = entrypoint & SQ_CI; 5237 uint_t c_outer = entrypoint & SQ_CO; 5238 5239 /* 5240 * Increment ref count to keep closes out of this queue. 5241 */ 5242 ASSERT(sq); 5243 ASSERT(c_inner && c_outer); 5244 mutex_enter(SQLOCK(sq)); 5245 flags = sq->sq_flags; 5246 type = sq->sq_type; 5247 if (!(type & c_inner)) { 5248 /* Make sure all putcounts now use slowlock. */ 5249 count = sq->sq_count; 5250 SQ_PUTLOCKS_ENTER(sq); 5251 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5252 SUM_SQ_PUTCOUNTS(sq, count); 5253 sq->sq_needexcl++; 5254 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5255 waitflags |= SQ_MESSAGES; 5256 } 5257 /* 5258 * Wait until we can enter the inner perimeter. 5259 * If we want exclusive access we wait until sq_count is 0. 5260 * We have to do this before entering the outer perimeter in order 5261 * to preserve put/close message ordering. 5262 */ 5263 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) { 5264 sq->sq_flags = flags | SQ_WANTWAKEUP; 5265 if (!(type & c_inner)) { 5266 SQ_PUTLOCKS_EXIT(sq); 5267 } 5268 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5269 if (!(type & c_inner)) { 5270 count = sq->sq_count; 5271 SQ_PUTLOCKS_ENTER(sq); 5272 SUM_SQ_PUTCOUNTS(sq, count); 5273 } 5274 flags = sq->sq_flags; 5275 } 5276 5277 if (!(type & c_inner)) { 5278 ASSERT(sq->sq_needexcl > 0); 5279 sq->sq_needexcl--; 5280 if (sq->sq_needexcl == 0) { 5281 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5282 } 5283 } 5284 5285 /* Check if we need to enter the outer perimeter */ 5286 if (!(type & c_outer)) { 5287 /* 5288 * We have to enter the outer perimeter exclusively before 5289 * we can increment sq_count to avoid deadlock. This implies 5290 * that we have to re-check sq_flags and sq_count. 5291 * 5292 * is it possible to have c_inner set when c_outer is not set? 5293 */ 5294 if (!(type & c_inner)) { 5295 SQ_PUTLOCKS_EXIT(sq); 5296 } 5297 mutex_exit(SQLOCK(sq)); 5298 outer_enter(sq->sq_outer, SQ_GOAWAY); 5299 mutex_enter(SQLOCK(sq)); 5300 flags = sq->sq_flags; 5301 /* 5302 * there should be no need to recheck sq_putcounts 5303 * because outer_enter() has already waited for them to clear 5304 * after setting SQ_WRITER. 5305 */ 5306 count = sq->sq_count; 5307 #ifdef DEBUG 5308 /* 5309 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead 5310 * of doing an ASSERT internally. Others should do 5311 * something like 5312 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0); 5313 * without the need to #ifdef DEBUG it. 5314 */ 5315 SUMCHECK_SQ_PUTCOUNTS(sq, 0); 5316 #endif 5317 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) || 5318 (!(type & c_inner) && count != 0)) { 5319 sq->sq_flags = flags | SQ_WANTWAKEUP; 5320 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5321 count = sq->sq_count; 5322 flags = sq->sq_flags; 5323 } 5324 } 5325 5326 sq->sq_count++; 5327 ASSERT(sq->sq_count != 0); /* Wraparound */ 5328 if (!(type & c_inner)) { 5329 /* Exclusive entry */ 5330 ASSERT(sq->sq_count == 1); 5331 sq->sq_flags |= SQ_EXCL; 5332 if (type & c_outer) { 5333 SQ_PUTLOCKS_EXIT(sq); 5334 } 5335 } 5336 mutex_exit(SQLOCK(sq)); 5337 } 5338 5339 /* 5340 * leave a syncq. announce to framework that closes may proceed. 5341 * c_inner and c_outer specifies which concurrency bits 5342 * to check. 5343 * 5344 * must never be called from driver or module put entry point. 5345 * 5346 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5347 * sq_putlocks are used. 5348 */ 5349 void 5350 leavesq(syncq_t *sq, int entrypoint) 5351 { 5352 uint16_t flags; 5353 uint16_t type; 5354 uint_t c_outer = entrypoint & SQ_CO; 5355 #ifdef DEBUG 5356 uint_t c_inner = entrypoint & SQ_CI; 5357 #endif 5358 5359 /* 5360 * decrement ref count, drain the syncq if possible, and wake up 5361 * any waiting close. 5362 */ 5363 ASSERT(sq); 5364 ASSERT(c_inner && c_outer); 5365 mutex_enter(SQLOCK(sq)); 5366 flags = sq->sq_flags; 5367 type = sq->sq_type; 5368 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) { 5369 5370 if (flags & SQ_WANTWAKEUP) { 5371 flags &= ~SQ_WANTWAKEUP; 5372 cv_broadcast(&sq->sq_wait); 5373 } 5374 if (flags & SQ_WANTEXWAKEUP) { 5375 flags &= ~SQ_WANTEXWAKEUP; 5376 cv_broadcast(&sq->sq_exitwait); 5377 } 5378 5379 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) { 5380 /* 5381 * The syncq needs to be drained. "Exit" the syncq 5382 * before calling drain_syncq. 5383 */ 5384 ASSERT(sq->sq_count != 0); 5385 sq->sq_count--; 5386 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5387 sq->sq_flags = flags & ~SQ_EXCL; 5388 drain_syncq(sq); 5389 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 5390 /* Check if we need to exit the outer perimeter */ 5391 /* XXX will this ever be true? */ 5392 if (!(type & c_outer)) 5393 outer_exit(sq->sq_outer); 5394 return; 5395 } 5396 } 5397 ASSERT(sq->sq_count != 0); 5398 sq->sq_count--; 5399 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5400 sq->sq_flags = flags & ~SQ_EXCL; 5401 mutex_exit(SQLOCK(sq)); 5402 5403 /* Check if we need to exit the outer perimeter */ 5404 if (!(sq->sq_type & c_outer)) 5405 outer_exit(sq->sq_outer); 5406 } 5407 5408 /* 5409 * Prevent q_next from changing in this stream by incrementing sq_count. 5410 * 5411 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5412 * sq_putlocks are used. 5413 */ 5414 void 5415 claimq(queue_t *qp) 5416 { 5417 syncq_t *sq = qp->q_syncq; 5418 5419 mutex_enter(SQLOCK(sq)); 5420 sq->sq_count++; 5421 ASSERT(sq->sq_count != 0); /* Wraparound */ 5422 mutex_exit(SQLOCK(sq)); 5423 } 5424 5425 /* 5426 * Undo claimq. 5427 * 5428 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5429 * sq_putlocks are used. 5430 */ 5431 void 5432 releaseq(queue_t *qp) 5433 { 5434 syncq_t *sq = qp->q_syncq; 5435 uint16_t flags; 5436 5437 mutex_enter(SQLOCK(sq)); 5438 ASSERT(sq->sq_count > 0); 5439 sq->sq_count--; 5440 5441 flags = sq->sq_flags; 5442 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) { 5443 if (flags & SQ_WANTWAKEUP) { 5444 flags &= ~SQ_WANTWAKEUP; 5445 cv_broadcast(&sq->sq_wait); 5446 } 5447 sq->sq_flags = flags; 5448 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5449 /* 5450 * To prevent potential recursive invocation of 5451 * drain_syncq we do not call drain_syncq if count is 5452 * non-zero. 5453 */ 5454 if (sq->sq_count == 0) { 5455 drain_syncq(sq); 5456 return; 5457 } else 5458 sqenable(sq); 5459 } 5460 } 5461 mutex_exit(SQLOCK(sq)); 5462 } 5463 5464 /* 5465 * Prevent q_next from changing in this stream by incrementing sd_refcnt. 5466 */ 5467 void 5468 claimstr(queue_t *qp) 5469 { 5470 struct stdata *stp = STREAM(qp); 5471 5472 mutex_enter(&stp->sd_reflock); 5473 stp->sd_refcnt++; 5474 ASSERT(stp->sd_refcnt != 0); /* Wraparound */ 5475 mutex_exit(&stp->sd_reflock); 5476 } 5477 5478 /* 5479 * Undo claimstr. 5480 */ 5481 void 5482 releasestr(queue_t *qp) 5483 { 5484 struct stdata *stp = STREAM(qp); 5485 5486 mutex_enter(&stp->sd_reflock); 5487 ASSERT(stp->sd_refcnt != 0); 5488 if (--stp->sd_refcnt == 0) 5489 cv_broadcast(&stp->sd_refmonitor); 5490 mutex_exit(&stp->sd_reflock); 5491 } 5492 5493 static syncq_t * 5494 new_syncq(void) 5495 { 5496 return (kmem_cache_alloc(syncq_cache, KM_SLEEP)); 5497 } 5498 5499 static void 5500 free_syncq(syncq_t *sq) 5501 { 5502 ASSERT(sq->sq_head == NULL); 5503 ASSERT(sq->sq_outer == NULL); 5504 ASSERT(sq->sq_callbpend == NULL); 5505 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || 5506 (sq->sq_onext == sq && sq->sq_oprev == sq)); 5507 5508 if (sq->sq_ciputctrl != NULL) { 5509 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 5510 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 5511 sq->sq_nciputctrl, 0); 5512 ASSERT(ciputctrl_cache != NULL); 5513 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 5514 } 5515 5516 sq->sq_tail = NULL; 5517 sq->sq_evhead = NULL; 5518 sq->sq_evtail = NULL; 5519 sq->sq_ciputctrl = NULL; 5520 sq->sq_nciputctrl = 0; 5521 sq->sq_count = 0; 5522 sq->sq_rmqcount = 0; 5523 sq->sq_callbflags = 0; 5524 sq->sq_cancelid = 0; 5525 sq->sq_next = NULL; 5526 sq->sq_needexcl = 0; 5527 sq->sq_svcflags = 0; 5528 sq->sq_nqueues = 0; 5529 sq->sq_pri = 0; 5530 sq->sq_onext = NULL; 5531 sq->sq_oprev = NULL; 5532 sq->sq_flags = 0; 5533 sq->sq_type = 0; 5534 sq->sq_servcount = 0; 5535 5536 kmem_cache_free(syncq_cache, sq); 5537 } 5538 5539 /* Outer perimeter code */ 5540 5541 /* 5542 * The outer syncq uses the fields and flags in the syncq slightly 5543 * differently from the inner syncqs. 5544 * sq_count Incremented when there are pending or running 5545 * writers at the outer perimeter to prevent the set of 5546 * inner syncqs that belong to the outer perimeter from 5547 * changing. 5548 * sq_head/tail List of deferred qwriter(OUTER) operations. 5549 * 5550 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while 5551 * inner syncqs are added to or removed from the 5552 * outer perimeter. 5553 * SQ_QUEUED sq_head/tail has messages or eventsqueued. 5554 * 5555 * SQ_WRITER A thread is currently traversing all the inner syncqs 5556 * setting the SQ_WRITER flag. 5557 */ 5558 5559 /* 5560 * Get write access at the outer perimeter. 5561 * Note that read access is done by entersq, putnext, and put by simply 5562 * incrementing sq_count in the inner syncq. 5563 * 5564 * Waits until "flags" is no longer set in the outer to prevent multiple 5565 * threads from having write access at the same time. SQ_WRITER has to be part 5566 * of "flags". 5567 * 5568 * Increases sq_count on the outer syncq to keep away outer_insert/remove 5569 * until the outer_exit is finished. 5570 * 5571 * outer_enter is vulnerable to starvation since it does not prevent new 5572 * threads from entering the inner syncqs while it is waiting for sq_count to 5573 * go to zero. 5574 */ 5575 void 5576 outer_enter(syncq_t *outer, uint16_t flags) 5577 { 5578 syncq_t *sq; 5579 int wait_needed; 5580 uint16_t count; 5581 5582 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5583 outer->sq_oprev != NULL); 5584 ASSERT(flags & SQ_WRITER); 5585 5586 retry: 5587 mutex_enter(SQLOCK(outer)); 5588 while (outer->sq_flags & flags) { 5589 outer->sq_flags |= SQ_WANTWAKEUP; 5590 cv_wait(&outer->sq_wait, SQLOCK(outer)); 5591 } 5592 5593 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5594 outer->sq_flags |= SQ_WRITER; 5595 outer->sq_count++; 5596 ASSERT(outer->sq_count != 0); /* wraparound */ 5597 wait_needed = 0; 5598 /* 5599 * Set SQ_WRITER on all the inner syncqs while holding 5600 * the SQLOCK on the outer syncq. This ensures that the changing 5601 * of SQ_WRITER is atomic under the outer SQLOCK. 5602 */ 5603 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5604 mutex_enter(SQLOCK(sq)); 5605 count = sq->sq_count; 5606 SQ_PUTLOCKS_ENTER(sq); 5607 sq->sq_flags |= SQ_WRITER; 5608 SUM_SQ_PUTCOUNTS(sq, count); 5609 if (count != 0) 5610 wait_needed = 1; 5611 SQ_PUTLOCKS_EXIT(sq); 5612 mutex_exit(SQLOCK(sq)); 5613 } 5614 mutex_exit(SQLOCK(outer)); 5615 5616 /* 5617 * Get everybody out of the syncqs sequentially. 5618 * Note that we don't actually need to aqiure the PUTLOCKS, since 5619 * we have already cleared the fastbit, and set QWRITER. By 5620 * definition, the count can not increase since putnext will 5621 * take the slowlock path (and the purpose of aquiring the 5622 * putlocks was to make sure it didn't increase while we were 5623 * waiting). 5624 * 5625 * Note that we still aquire the PUTLOCKS to be safe. 5626 */ 5627 if (wait_needed) { 5628 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5629 mutex_enter(SQLOCK(sq)); 5630 count = sq->sq_count; 5631 SQ_PUTLOCKS_ENTER(sq); 5632 SUM_SQ_PUTCOUNTS(sq, count); 5633 while (count != 0) { 5634 sq->sq_flags |= SQ_WANTWAKEUP; 5635 SQ_PUTLOCKS_EXIT(sq); 5636 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5637 count = sq->sq_count; 5638 SQ_PUTLOCKS_ENTER(sq); 5639 SUM_SQ_PUTCOUNTS(sq, count); 5640 } 5641 SQ_PUTLOCKS_EXIT(sq); 5642 mutex_exit(SQLOCK(sq)); 5643 } 5644 /* 5645 * Verify that none of the flags got set while we 5646 * were waiting for the sq_counts to drop. 5647 * If this happens we exit and retry entering the 5648 * outer perimeter. 5649 */ 5650 mutex_enter(SQLOCK(outer)); 5651 if (outer->sq_flags & (flags & ~SQ_WRITER)) { 5652 mutex_exit(SQLOCK(outer)); 5653 outer_exit(outer); 5654 goto retry; 5655 } 5656 mutex_exit(SQLOCK(outer)); 5657 } 5658 } 5659 5660 /* 5661 * Drop the write access at the outer perimeter. 5662 * Read access is dropped implicitly (by putnext, put, and leavesq) by 5663 * decrementing sq_count. 5664 */ 5665 void 5666 outer_exit(syncq_t *outer) 5667 { 5668 syncq_t *sq; 5669 int drain_needed; 5670 uint16_t flags; 5671 5672 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5673 outer->sq_oprev != NULL); 5674 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer))); 5675 5676 /* 5677 * Atomically (from the perspective of threads calling become_writer) 5678 * drop the write access at the outer perimeter by holding 5679 * SQLOCK(outer) across all the dropsq calls and the resetting of 5680 * SQ_WRITER. 5681 * This defines a locking order between the outer perimeter 5682 * SQLOCK and the inner perimeter SQLOCKs. 5683 */ 5684 mutex_enter(SQLOCK(outer)); 5685 flags = outer->sq_flags; 5686 ASSERT(outer->sq_flags & SQ_WRITER); 5687 if (flags & SQ_QUEUED) { 5688 write_now(outer); 5689 flags = outer->sq_flags; 5690 } 5691 5692 /* 5693 * sq_onext is stable since sq_count has not yet been decreased. 5694 * Reset the SQ_WRITER flags in all syncqs. 5695 * After dropping SQ_WRITER on the outer syncq we empty all the 5696 * inner syncqs. 5697 */ 5698 drain_needed = 0; 5699 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5700 drain_needed += dropsq(sq, SQ_WRITER); 5701 ASSERT(!(outer->sq_flags & SQ_QUEUED)); 5702 flags &= ~SQ_WRITER; 5703 if (drain_needed) { 5704 outer->sq_flags = flags; 5705 mutex_exit(SQLOCK(outer)); 5706 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5707 emptysq(sq); 5708 mutex_enter(SQLOCK(outer)); 5709 flags = outer->sq_flags; 5710 } 5711 if (flags & SQ_WANTWAKEUP) { 5712 flags &= ~SQ_WANTWAKEUP; 5713 cv_broadcast(&outer->sq_wait); 5714 } 5715 outer->sq_flags = flags; 5716 ASSERT(outer->sq_count > 0); 5717 outer->sq_count--; 5718 mutex_exit(SQLOCK(outer)); 5719 } 5720 5721 /* 5722 * Add another syncq to an outer perimeter. 5723 * Block out all other access to the outer perimeter while it is being 5724 * changed using blocksq. 5725 * Assumes that the caller has *not* done an outer_enter. 5726 * 5727 * Vulnerable to starvation in blocksq. 5728 */ 5729 static void 5730 outer_insert(syncq_t *outer, syncq_t *sq) 5731 { 5732 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5733 outer->sq_oprev != NULL); 5734 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 5735 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ 5736 5737 /* Get exclusive access to the outer perimeter list */ 5738 blocksq(outer, SQ_BLOCKED, 0); 5739 ASSERT(outer->sq_flags & SQ_BLOCKED); 5740 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5741 5742 mutex_enter(SQLOCK(sq)); 5743 sq->sq_outer = outer; 5744 outer->sq_onext->sq_oprev = sq; 5745 sq->sq_onext = outer->sq_onext; 5746 outer->sq_onext = sq; 5747 sq->sq_oprev = outer; 5748 mutex_exit(SQLOCK(sq)); 5749 unblocksq(outer, SQ_BLOCKED, 1); 5750 } 5751 5752 /* 5753 * Remove a syncq from an outer perimeter. 5754 * Block out all other access to the outer perimeter while it is being 5755 * changed using blocksq. 5756 * Assumes that the caller has *not* done an outer_enter. 5757 * 5758 * Vulnerable to starvation in blocksq. 5759 */ 5760 static void 5761 outer_remove(syncq_t *outer, syncq_t *sq) 5762 { 5763 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5764 outer->sq_oprev != NULL); 5765 ASSERT(sq->sq_outer == outer); 5766 5767 /* Get exclusive access to the outer perimeter list */ 5768 blocksq(outer, SQ_BLOCKED, 0); 5769 ASSERT(outer->sq_flags & SQ_BLOCKED); 5770 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5771 5772 mutex_enter(SQLOCK(sq)); 5773 sq->sq_outer = NULL; 5774 sq->sq_onext->sq_oprev = sq->sq_oprev; 5775 sq->sq_oprev->sq_onext = sq->sq_onext; 5776 sq->sq_oprev = sq->sq_onext = NULL; 5777 mutex_exit(SQLOCK(sq)); 5778 unblocksq(outer, SQ_BLOCKED, 1); 5779 } 5780 5781 /* 5782 * Queue a deferred qwriter(OUTER) callback for this outer perimeter. 5783 * If this is the first callback for this outer perimeter then add 5784 * this outer perimeter to the list of outer perimeters that 5785 * the qwriter_outer_thread will process. 5786 * 5787 * Increments sq_count in the outer syncq to prevent the membership 5788 * of the outer perimeter (in terms of inner syncqs) to change while 5789 * the callback is pending. 5790 */ 5791 static void 5792 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp) 5793 { 5794 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5795 5796 mp->b_prev = (mblk_t *)func; 5797 mp->b_queue = q; 5798 mp->b_next = NULL; 5799 outer->sq_count++; /* Decremented when dequeued */ 5800 ASSERT(outer->sq_count != 0); /* Wraparound */ 5801 if (outer->sq_evhead == NULL) { 5802 /* First message. */ 5803 outer->sq_evhead = outer->sq_evtail = mp; 5804 outer->sq_flags |= SQ_EVENTS; 5805 mutex_exit(SQLOCK(outer)); 5806 STRSTAT(qwr_outer); 5807 (void) taskq_dispatch(streams_taskq, 5808 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP); 5809 } else { 5810 ASSERT(outer->sq_flags & SQ_EVENTS); 5811 outer->sq_evtail->b_next = mp; 5812 outer->sq_evtail = mp; 5813 mutex_exit(SQLOCK(outer)); 5814 } 5815 } 5816 5817 /* 5818 * Try and upgrade to write access at the outer perimeter. If this can 5819 * not be done without blocking then queue the callback to be done 5820 * by the qwriter_outer_thread. 5821 * 5822 * This routine can only be called from put or service procedures plus 5823 * asynchronous callback routines that have properly entered to 5824 * queue (with entersq.) Thus qwriter(OUTER) assumes the caller has one claim 5825 * on the syncq associated with q. 5826 */ 5827 void 5828 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)()) 5829 { 5830 syncq_t *osq, *sq, *outer; 5831 int failed; 5832 uint16_t flags; 5833 5834 osq = q->q_syncq; 5835 outer = osq->sq_outer; 5836 if (outer == NULL) 5837 panic("qwriter(PERIM_OUTER): no outer perimeter"); 5838 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5839 outer->sq_oprev != NULL); 5840 5841 mutex_enter(SQLOCK(outer)); 5842 flags = outer->sq_flags; 5843 /* 5844 * If some thread is traversing sq_next, or if we are blocked by 5845 * outer_insert or outer_remove, or if the we already have queued 5846 * callbacks, then queue this callback for later processing. 5847 * 5848 * Also queue the qwriter for an interrupt thread in order 5849 * to reduce the time spent running at high IPL. 5850 * to identify there are events. 5851 */ 5852 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) { 5853 /* 5854 * Queue the become_writer request. 5855 * The queueing is atomic under SQLOCK(outer) in order 5856 * to synchronize with outer_exit. 5857 * queue_writer will drop the outer SQLOCK 5858 */ 5859 if (flags & SQ_BLOCKED) { 5860 /* Must set SQ_WRITER on inner perimeter */ 5861 mutex_enter(SQLOCK(osq)); 5862 osq->sq_flags |= SQ_WRITER; 5863 mutex_exit(SQLOCK(osq)); 5864 } else { 5865 if (!(flags & SQ_WRITER)) { 5866 /* 5867 * The outer could have been SQ_BLOCKED thus 5868 * SQ_WRITER might not be set on the inner. 5869 */ 5870 mutex_enter(SQLOCK(osq)); 5871 osq->sq_flags |= SQ_WRITER; 5872 mutex_exit(SQLOCK(osq)); 5873 } 5874 ASSERT(osq->sq_flags & SQ_WRITER); 5875 } 5876 queue_writer(outer, func, q, mp); 5877 return; 5878 } 5879 /* 5880 * We are half-way to exclusive access to the outer perimeter. 5881 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove 5882 * while the inner syncqs are traversed. 5883 */ 5884 outer->sq_count++; 5885 ASSERT(outer->sq_count != 0); /* wraparound */ 5886 flags |= SQ_WRITER; 5887 /* 5888 * Check if we can run the function immediately. Mark all 5889 * syncqs with the writer flag to prevent new entries into 5890 * put and service procedures. 5891 * 5892 * Set SQ_WRITER on all the inner syncqs while holding 5893 * the SQLOCK on the outer syncq. This ensures that the changing 5894 * of SQ_WRITER is atomic under the outer SQLOCK. 5895 */ 5896 failed = 0; 5897 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5898 uint16_t count; 5899 uint_t maxcnt = (sq == osq) ? 1 : 0; 5900 5901 mutex_enter(SQLOCK(sq)); 5902 count = sq->sq_count; 5903 SQ_PUTLOCKS_ENTER(sq); 5904 SUM_SQ_PUTCOUNTS(sq, count); 5905 if (sq->sq_count > maxcnt) 5906 failed = 1; 5907 sq->sq_flags |= SQ_WRITER; 5908 SQ_PUTLOCKS_EXIT(sq); 5909 mutex_exit(SQLOCK(sq)); 5910 } 5911 if (failed) { 5912 /* 5913 * Some other thread has a read claim on the outer perimeter. 5914 * Queue the callback for deferred processing. 5915 * 5916 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER 5917 * so that other qwriter(OUTER) calls will queue their 5918 * callbacks as well. queue_writer increments sq_count so we 5919 * decrement to compensate for the our increment. 5920 * 5921 * Dropping SQ_WRITER enables the writer thread to work 5922 * on this outer perimeter. 5923 */ 5924 outer->sq_flags = flags; 5925 queue_writer(outer, func, q, mp); 5926 /* queue_writer dropper the lock */ 5927 mutex_enter(SQLOCK(outer)); 5928 ASSERT(outer->sq_count > 0); 5929 outer->sq_count--; 5930 ASSERT(outer->sq_flags & SQ_WRITER); 5931 flags = outer->sq_flags; 5932 flags &= ~SQ_WRITER; 5933 if (flags & SQ_WANTWAKEUP) { 5934 flags &= ~SQ_WANTWAKEUP; 5935 cv_broadcast(&outer->sq_wait); 5936 } 5937 outer->sq_flags = flags; 5938 mutex_exit(SQLOCK(outer)); 5939 return; 5940 } else { 5941 outer->sq_flags = flags; 5942 mutex_exit(SQLOCK(outer)); 5943 } 5944 5945 /* Can run it immediately */ 5946 (*func)(q, mp); 5947 5948 outer_exit(outer); 5949 } 5950 5951 /* 5952 * Dequeue all writer callbacks from the outer perimeter and run them. 5953 */ 5954 static void 5955 write_now(syncq_t *outer) 5956 { 5957 mblk_t *mp; 5958 queue_t *q; 5959 void (*func)(); 5960 5961 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5962 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5963 outer->sq_oprev != NULL); 5964 while ((mp = outer->sq_evhead) != NULL) { 5965 /* 5966 * queues cannot be placed on the queuelist on the outer 5967 * perimiter. 5968 */ 5969 ASSERT(!(outer->sq_flags & SQ_MESSAGES)); 5970 ASSERT((outer->sq_flags & SQ_EVENTS)); 5971 5972 outer->sq_evhead = mp->b_next; 5973 if (outer->sq_evhead == NULL) { 5974 outer->sq_evtail = NULL; 5975 outer->sq_flags &= ~SQ_EVENTS; 5976 } 5977 ASSERT(outer->sq_count != 0); 5978 outer->sq_count--; /* Incremented when enqueued. */ 5979 mutex_exit(SQLOCK(outer)); 5980 /* 5981 * Drop the message if the queue is closing. 5982 * Make sure that the queue is "claimed" when the callback 5983 * is run in order to satisfy various ASSERTs. 5984 */ 5985 q = mp->b_queue; 5986 func = (void (*)())mp->b_prev; 5987 ASSERT(func != NULL); 5988 mp->b_next = mp->b_prev = NULL; 5989 if (q->q_flag & QWCLOSE) { 5990 freemsg(mp); 5991 } else { 5992 claimq(q); 5993 (*func)(q, mp); 5994 releaseq(q); 5995 } 5996 mutex_enter(SQLOCK(outer)); 5997 } 5998 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5999 } 6000 6001 /* 6002 * The list of messages on the inner syncq is effectively hashed 6003 * by destination queue. These destination queues are doubly 6004 * linked lists (hopefully) in priority order. Messages are then 6005 * put on the queue referenced by the q_sqhead/q_sqtail elements. 6006 * Additional messages are linked together by the b_next/b_prev 6007 * elements in the mblk, with (similar to putq()) the first message 6008 * having a NULL b_prev and the last message having a NULL b_next. 6009 * 6010 * Events, such as qwriter callbacks, are put onto a list in FIFO 6011 * order referenced by sq_evhead, and sq_evtail. This is a singly 6012 * linked list, and messages here MUST be processed in the order queued. 6013 */ 6014 6015 /* 6016 * Run the events on the syncq event list (sq_evhead). 6017 * Assumes there is only one claim on the syncq, it is 6018 * already exclusive (SQ_EXCL set), and the SQLOCK held. 6019 * Messages here are processed in order, with the SQ_EXCL bit 6020 * held all the way through till the last message is processed. 6021 */ 6022 void 6023 sq_run_events(syncq_t *sq) 6024 { 6025 mblk_t *bp; 6026 queue_t *qp; 6027 uint16_t flags = sq->sq_flags; 6028 void (*func)(); 6029 6030 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6031 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6032 sq->sq_oprev == NULL) || 6033 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6034 sq->sq_oprev != NULL)); 6035 6036 ASSERT(flags & SQ_EXCL); 6037 ASSERT(sq->sq_count == 1); 6038 6039 /* 6040 * We need to process all of the events on this list. It 6041 * is possible that new events will be added while we are 6042 * away processing a callback, so on every loop, we start 6043 * back at the beginning of the list. 6044 */ 6045 /* 6046 * We have to reaccess sq_evhead since there is a 6047 * possibility of a new entry while we were running 6048 * the callback. 6049 */ 6050 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { 6051 ASSERT(bp->b_queue->q_syncq == sq); 6052 ASSERT(sq->sq_flags & SQ_EVENTS); 6053 6054 qp = bp->b_queue; 6055 func = (void (*)())bp->b_prev; 6056 ASSERT(func != NULL); 6057 6058 /* 6059 * Messages from the event queue must be taken off in 6060 * FIFO order. 6061 */ 6062 ASSERT(sq->sq_evhead == bp); 6063 sq->sq_evhead = bp->b_next; 6064 6065 if (bp->b_next == NULL) { 6066 /* Deleting last */ 6067 ASSERT(sq->sq_evtail == bp); 6068 sq->sq_evtail = NULL; 6069 sq->sq_flags &= ~SQ_EVENTS; 6070 } 6071 bp->b_prev = bp->b_next = NULL; 6072 ASSERT(bp->b_datap->db_ref != 0); 6073 6074 mutex_exit(SQLOCK(sq)); 6075 6076 (*func)(qp, bp); 6077 6078 mutex_enter(SQLOCK(sq)); 6079 /* 6080 * re-read the flags, since they could have changed. 6081 */ 6082 flags = sq->sq_flags; 6083 ASSERT(flags & SQ_EXCL); 6084 } 6085 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); 6086 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6087 6088 if (flags & SQ_WANTWAKEUP) { 6089 flags &= ~SQ_WANTWAKEUP; 6090 cv_broadcast(&sq->sq_wait); 6091 } 6092 if (flags & SQ_WANTEXWAKEUP) { 6093 flags &= ~SQ_WANTEXWAKEUP; 6094 cv_broadcast(&sq->sq_exitwait); 6095 } 6096 sq->sq_flags = flags; 6097 } 6098 6099 /* 6100 * Put messages on the event list. 6101 * If we can go exclusive now, do so and process the event list, otherwise 6102 * let the last claim service this list (or wake the sqthread). 6103 * This procedure assumes SQLOCK is held. To run the event list, it 6104 * must be called with no claims. 6105 */ 6106 static void 6107 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) 6108 { 6109 uint16_t count; 6110 6111 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6112 ASSERT(func != NULL); 6113 6114 /* 6115 * This is a callback. Add it to the list of callbacks 6116 * and see about upgrading. 6117 */ 6118 mp->b_prev = (mblk_t *)func; 6119 mp->b_queue = q; 6120 mp->b_next = NULL; 6121 if (sq->sq_evhead == NULL) { 6122 sq->sq_evhead = sq->sq_evtail = mp; 6123 sq->sq_flags |= SQ_EVENTS; 6124 } else { 6125 ASSERT(sq->sq_evtail != NULL); 6126 ASSERT(sq->sq_evtail->b_next == NULL); 6127 ASSERT(sq->sq_flags & SQ_EVENTS); 6128 sq->sq_evtail->b_next = mp; 6129 sq->sq_evtail = mp; 6130 } 6131 /* 6132 * We have set SQ_EVENTS, so threads will have to 6133 * unwind out of the perimiter, and new entries will 6134 * not grab a putlock. But we still need to know 6135 * how many threads have already made a claim to the 6136 * syncq, so grab the putlocks, and sum the counts. 6137 * If there are no claims on the syncq, we can upgrade 6138 * to exclusive, and run the event list. 6139 * NOTE: We hold the SQLOCK, so we can just grab the 6140 * putlocks. 6141 */ 6142 count = sq->sq_count; 6143 SQ_PUTLOCKS_ENTER(sq); 6144 SUM_SQ_PUTCOUNTS(sq, count); 6145 /* 6146 * We have no claim, so we need to check if there 6147 * are no others, then we can upgrade. 6148 */ 6149 /* 6150 * There are currently no claims on 6151 * the syncq by this thread (at least on this entry). The thread who has 6152 * the claim should drain syncq. 6153 */ 6154 if (count > 0) { 6155 /* 6156 * Can't upgrade - other threads inside. 6157 */ 6158 SQ_PUTLOCKS_EXIT(sq); 6159 mutex_exit(SQLOCK(sq)); 6160 return; 6161 } 6162 /* 6163 * Need to set SQ_EXCL and make a claim on the syncq. 6164 */ 6165 ASSERT((sq->sq_flags & SQ_EXCL) == 0); 6166 sq->sq_flags |= SQ_EXCL; 6167 ASSERT(sq->sq_count == 0); 6168 sq->sq_count++; 6169 SQ_PUTLOCKS_EXIT(sq); 6170 6171 /* Process the events list */ 6172 sq_run_events(sq); 6173 6174 /* 6175 * Release our claim... 6176 */ 6177 sq->sq_count--; 6178 6179 /* 6180 * And release SQ_EXCL. 6181 * We don't need to acquire the putlocks to release 6182 * SQ_EXCL, since we are exclusive, and hold the SQLOCK. 6183 */ 6184 sq->sq_flags &= ~SQ_EXCL; 6185 6186 /* 6187 * sq_run_events should have released SQ_EXCL 6188 */ 6189 ASSERT(!(sq->sq_flags & SQ_EXCL)); 6190 6191 /* 6192 * If anything happened while we were running the 6193 * events (or was there before), we need to process 6194 * them now. We shouldn't be exclusive sine we 6195 * released the perimiter above (plus, we asserted 6196 * for it). 6197 */ 6198 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) 6199 drain_syncq(sq); 6200 else 6201 mutex_exit(SQLOCK(sq)); 6202 } 6203 6204 /* 6205 * Perform delayed processing. The caller has to make sure that it is safe 6206 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are 6207 * set.) 6208 * 6209 * Assume that the caller has NO claims on the syncq. However, a claim 6210 * on the syncq does not indicate that a thread is draining the syncq. 6211 * There may be more claims on the syncq than there are threads draining 6212 * (i.e. #_threads_draining <= sq_count) 6213 * 6214 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set 6215 * in order to preserve qwriter(OUTER) ordering constraints. 6216 * 6217 * sq_putcount only needs to be checked when dispatching the queued 6218 * writer call for CIPUT sync queue, but this is handled in sq_run_events. 6219 */ 6220 void 6221 drain_syncq(syncq_t *sq) 6222 { 6223 queue_t *qp; 6224 uint16_t count; 6225 uint16_t type = sq->sq_type; 6226 uint16_t flags = sq->sq_flags; 6227 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; 6228 6229 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6230 "drain_syncq start:%p", sq); 6231 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6232 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6233 sq->sq_oprev == NULL) || 6234 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6235 sq->sq_oprev != NULL)); 6236 6237 /* 6238 * Drop SQ_SERVICE flag. 6239 */ 6240 if (bg_service) 6241 sq->sq_svcflags &= ~SQ_SERVICE; 6242 6243 /* 6244 * If SQ_EXCL is set, someone else is processing this syncq - let him 6245 * finish the job. 6246 */ 6247 if (flags & SQ_EXCL) { 6248 if (bg_service) { 6249 ASSERT(sq->sq_servcount != 0); 6250 sq->sq_servcount--; 6251 } 6252 mutex_exit(SQLOCK(sq)); 6253 return; 6254 } 6255 6256 /* 6257 * This routine can be called by a background thread if 6258 * it was scheduled by a hi-priority thread. SO, if there are 6259 * NOT messages queued, return (remember, we have the SQLOCK, 6260 * and it cannot change until we release it). Wakeup any waiters also. 6261 */ 6262 if (!(flags & SQ_QUEUED)) { 6263 if (flags & SQ_WANTWAKEUP) { 6264 flags &= ~SQ_WANTWAKEUP; 6265 cv_broadcast(&sq->sq_wait); 6266 } 6267 if (flags & SQ_WANTEXWAKEUP) { 6268 flags &= ~SQ_WANTEXWAKEUP; 6269 cv_broadcast(&sq->sq_exitwait); 6270 } 6271 sq->sq_flags = flags; 6272 if (bg_service) { 6273 ASSERT(sq->sq_servcount != 0); 6274 sq->sq_servcount--; 6275 } 6276 mutex_exit(SQLOCK(sq)); 6277 return; 6278 } 6279 6280 /* 6281 * If this is not a concurrent put perimiter, we need to 6282 * become exclusive to drain. Also, if not CIPUT, we would 6283 * not have acquired a putlock, so we don't need to check 6284 * the putcounts. If not entering with a claim, we test 6285 * for sq_count == 0. 6286 */ 6287 type = sq->sq_type; 6288 if (!(type & SQ_CIPUT)) { 6289 if (sq->sq_count > 1) { 6290 if (bg_service) { 6291 ASSERT(sq->sq_servcount != 0); 6292 sq->sq_servcount--; 6293 } 6294 mutex_exit(SQLOCK(sq)); 6295 return; 6296 } 6297 sq->sq_flags |= SQ_EXCL; 6298 } 6299 6300 /* 6301 * This is where we make a claim to the syncq. 6302 * This can either be done by incrementing a putlock, or 6303 * the sq_count. But since we already have the SQLOCK 6304 * here, we just bump the sq_count. 6305 * 6306 * Note that after we make a claim, we need to let the code 6307 * fall through to the end of this routine to clean itself 6308 * up. A return in the while loop will put the syncq in a 6309 * very bad state. 6310 */ 6311 sq->sq_count++; 6312 ASSERT(sq->sq_count != 0); /* wraparound */ 6313 6314 while ((flags = sq->sq_flags) & SQ_QUEUED) { 6315 /* 6316 * If we are told to stayaway or went exclusive, 6317 * we are done. 6318 */ 6319 if (flags & (SQ_STAYAWAY)) { 6320 break; 6321 } 6322 6323 /* 6324 * If there are events to run, do so. 6325 * We have one claim to the syncq, so if there are 6326 * more than one, other threads are running. 6327 */ 6328 if (sq->sq_evhead != NULL) { 6329 ASSERT(sq->sq_flags & SQ_EVENTS); 6330 6331 count = sq->sq_count; 6332 SQ_PUTLOCKS_ENTER(sq); 6333 SUM_SQ_PUTCOUNTS(sq, count); 6334 if (count > 1) { 6335 SQ_PUTLOCKS_EXIT(sq); 6336 /* Can't upgrade - other threads inside */ 6337 break; 6338 } 6339 ASSERT((flags & SQ_EXCL) == 0); 6340 sq->sq_flags = flags | SQ_EXCL; 6341 SQ_PUTLOCKS_EXIT(sq); 6342 /* 6343 * we have the only claim, run the events, 6344 * sq_run_events will clear the SQ_EXCL flag. 6345 */ 6346 sq_run_events(sq); 6347 6348 /* 6349 * If this is a CIPUT perimiter, we need 6350 * to drop the SQ_EXCL flag so we can properly 6351 * continue draining the syncq. 6352 */ 6353 if (type & SQ_CIPUT) { 6354 ASSERT(sq->sq_flags & SQ_EXCL); 6355 sq->sq_flags &= ~SQ_EXCL; 6356 } 6357 6358 /* 6359 * And go back to the beginning just in case 6360 * anything changed while we were away. 6361 */ 6362 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); 6363 continue; 6364 } 6365 6366 ASSERT(sq->sq_evhead == NULL); 6367 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6368 6369 /* 6370 * Find the queue that is not draining. 6371 * 6372 * q_draining is protected by QLOCK which we do not hold. 6373 * But if it was set, then a thread was draining, and if it gets 6374 * cleared, then it was because the thread has successfully 6375 * drained the syncq, or a GOAWAY state occured. For the GOAWAY 6376 * state to happen, a thread needs the SQLOCK which we hold, and 6377 * if there was such a flag, we whould have already seen it. 6378 */ 6379 6380 for (qp = sq->sq_head; 6381 qp != NULL && (qp->q_draining || 6382 (qp->q_sqflags & Q_SQDRAINING)); 6383 qp = qp->q_sqnext) 6384 ; 6385 6386 if (qp == NULL) 6387 break; 6388 6389 /* 6390 * We have a queue to work on, and we hold the 6391 * SQLOCK and one claim, call qdrain_syncq. 6392 * This means we need to release the SQLOCK and 6393 * aquire the QLOCK (OK since we have a claim). 6394 * Note that qdrain_syncq will actually dequeue 6395 * this queue from the sq_head list when it is 6396 * convinced all the work is done and release 6397 * the QLOCK before returning. 6398 */ 6399 qp->q_sqflags |= Q_SQDRAINING; 6400 mutex_exit(SQLOCK(sq)); 6401 mutex_enter(QLOCK(qp)); 6402 qdrain_syncq(sq, qp); 6403 mutex_enter(SQLOCK(sq)); 6404 6405 /* The queue is drained */ 6406 ASSERT(qp->q_sqflags & Q_SQDRAINING); 6407 qp->q_sqflags &= ~Q_SQDRAINING; 6408 /* 6409 * NOTE: After this point qp should not be used since it may be 6410 * closed. 6411 */ 6412 } 6413 6414 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6415 flags = sq->sq_flags; 6416 6417 /* 6418 * sq->sq_head cannot change because we hold the 6419 * sqlock. However, a thread CAN decide that it is no longer 6420 * going to drain that queue. However, this should be due to 6421 * a GOAWAY state, and we should see that here. 6422 * 6423 * This loop is not very efficient. One solution may be adding a second 6424 * pointer to the "draining" queue, but it is difficult to do when 6425 * queues are inserted in the middle due to priority ordering. Another 6426 * possibility is to yank the queue out of the sq list and put it onto 6427 * the "draining list" and then put it back if it can't be drained. 6428 */ 6429 6430 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || 6431 (type & SQ_CI) || sq->sq_head->q_draining); 6432 6433 /* Drop SQ_EXCL for non-CIPUT perimiters */ 6434 if (!(type & SQ_CIPUT)) 6435 flags &= ~SQ_EXCL; 6436 ASSERT((flags & SQ_EXCL) == 0); 6437 6438 /* Wake up any waiters. */ 6439 if (flags & SQ_WANTWAKEUP) { 6440 flags &= ~SQ_WANTWAKEUP; 6441 cv_broadcast(&sq->sq_wait); 6442 } 6443 if (flags & SQ_WANTEXWAKEUP) { 6444 flags &= ~SQ_WANTEXWAKEUP; 6445 cv_broadcast(&sq->sq_exitwait); 6446 } 6447 sq->sq_flags = flags; 6448 6449 ASSERT(sq->sq_count != 0); 6450 /* Release our claim. */ 6451 sq->sq_count--; 6452 6453 if (bg_service) { 6454 ASSERT(sq->sq_servcount != 0); 6455 sq->sq_servcount--; 6456 } 6457 6458 mutex_exit(SQLOCK(sq)); 6459 6460 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6461 "drain_syncq end:%p", sq); 6462 } 6463 6464 6465 /* 6466 * 6467 * qdrain_syncq can be called (currently) from only one of two places: 6468 * drain_syncq 6469 * putnext (or some variation of it). 6470 * and eventually 6471 * qwait(_sig) 6472 * 6473 * If called from drain_syncq, we found it in the list 6474 * of queue's needing service, so there is work to be done (or it 6475 * wouldn't be on the list). 6476 * 6477 * If called from some putnext variation, it was because the 6478 * perimiter is open, but messages are blocking a putnext and 6479 * there is not a thread working on it. Now a thread could start 6480 * working on it while we are getting ready to do so ourself, but 6481 * the thread would set the q_draining flag, and we can spin out. 6482 * 6483 * As for qwait(_sig), I think I shall let it continue to call 6484 * drain_syncq directly (after all, it will get here eventually). 6485 * 6486 * qdrain_syncq has to terminate when: 6487 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering 6488 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering 6489 * 6490 * ASSUMES: 6491 * One claim 6492 * QLOCK held 6493 * SQLOCK not held 6494 * Will release QLOCK before returning 6495 */ 6496 void 6497 qdrain_syncq(syncq_t *sq, queue_t *q) 6498 { 6499 mblk_t *bp; 6500 boolean_t do_clr; 6501 #ifdef DEBUG 6502 uint16_t count; 6503 #endif 6504 6505 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6506 "drain_syncq start:%p", sq); 6507 ASSERT(q->q_syncq == sq); 6508 ASSERT(MUTEX_HELD(QLOCK(q))); 6509 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6510 /* 6511 * For non-CIPUT perimiters, we should be called with the 6512 * exclusive bit set already. For non-CIPUT perimiters we 6513 * will be doing a concurrent drain, so it better not be set. 6514 */ 6515 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); 6516 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); 6517 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); 6518 /* 6519 * All outer pointers are set, or none of them are 6520 */ 6521 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6522 sq->sq_oprev == NULL) || 6523 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6524 sq->sq_oprev != NULL)); 6525 #ifdef DEBUG 6526 count = sq->sq_count; 6527 /* 6528 * This is OK without the putlocks, because we have one 6529 * claim either from the sq_count, or a putcount. We could 6530 * get an erroneous value from other counts, but ours won't 6531 * change, so one way or another, we will have at least a 6532 * value of one. 6533 */ 6534 SUM_SQ_PUTCOUNTS(sq, count); 6535 ASSERT(count >= 1); 6536 #endif /* DEBUG */ 6537 6538 /* 6539 * The first thing to do here, is find out if a thread is already 6540 * draining this queue or the queue is closing. If so, we are done, 6541 * just return. Also, if there are no messages, we are done as well. 6542 * Note that we check the q_sqhead since there is s window of 6543 * opportunity for us to enter here because Q_SQQUEUED was set, but is 6544 * not anymore. 6545 */ 6546 if (q->q_draining || (q->q_sqhead == NULL)) { 6547 mutex_exit(QLOCK(q)); 6548 return; 6549 } 6550 6551 /* 6552 * If the perimiter is exclusive, there is nothing we can 6553 * do right now, go away. 6554 * Note that there is nothing to prevent this case from changing 6555 * right after this check, but the spin-out will catch it. 6556 */ 6557 6558 /* Tell other threads that we are draining this queue */ 6559 q->q_draining = 1; /* Protected by QLOCK */ 6560 6561 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) { 6562 6563 /* 6564 * Because we can enter this routine just because 6565 * a putnext is blocked, we need to spin out if 6566 * the perimiter wants to go exclusive as well 6567 * as just blocked. We need to spin out also if 6568 * events are queued on the syncq. 6569 * Don't check for SQ_EXCL, because non-CIPUT 6570 * perimiters would set it, and it can't become 6571 * exclusive while we hold a claim. 6572 */ 6573 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { 6574 break; 6575 } 6576 6577 #ifdef DEBUG 6578 /* 6579 * Since we are in qdrain_syncq, we already know the queue, 6580 * but for sanity, we want to check this against the qp that 6581 * was passed in by bp->b_queue. 6582 */ 6583 6584 ASSERT(bp->b_queue == q); 6585 ASSERT(bp->b_queue->q_syncq == sq); 6586 bp->b_queue = NULL; 6587 6588 /* 6589 * We would have the following check in the DEBUG code: 6590 * 6591 * if (bp->b_prev != NULL) { 6592 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp); 6593 * } 6594 * 6595 * This can't be done, however, since IP modifies qinfo 6596 * structure at run-time (switching between IPv4 qinfo and IPv6 6597 * qinfo), invalidating the check. 6598 * So the assignment to func is left here, but the ASSERT itself 6599 * is removed until the whole issue is resolved. 6600 */ 6601 #endif 6602 ASSERT(q->q_sqhead == bp); 6603 q->q_sqhead = bp->b_next; 6604 bp->b_prev = bp->b_next = NULL; 6605 ASSERT(q->q_syncqmsgs > 0); 6606 mutex_exit(QLOCK(q)); 6607 6608 ASSERT(bp->b_datap->db_ref != 0); 6609 6610 (void) (*q->q_qinfo->qi_putp)(q, bp); 6611 6612 mutex_enter(QLOCK(q)); 6613 /* 6614 * We should decrement q_syncqmsgs only after executing the 6615 * put procedure to avoid a possible race with putnext(). 6616 * In putnext() though it sees Q_SQQUEUED is set, there is 6617 * an optimization which allows putnext to call the put 6618 * procedure directly if (q_syncqmsgs == 0) and thus 6619 * a message reodering could otherwise occur. 6620 */ 6621 q->q_syncqmsgs--; 6622 6623 /* 6624 * Clear QFULL in the next service procedure queue if 6625 * this is the last message destined to that queue. 6626 * 6627 * It would make better sense to have some sort of 6628 * tunable for the low water mark, but these symantics 6629 * are not yet defined. So, alas, we use a constant. 6630 */ 6631 do_clr = (q->q_syncqmsgs == 0); 6632 mutex_exit(QLOCK(q)); 6633 6634 if (do_clr) 6635 clr_qfull(q); 6636 6637 mutex_enter(QLOCK(q)); 6638 /* 6639 * Always clear SQ_EXCL when CIPUT in order to handle 6640 * qwriter(INNER). 6641 */ 6642 /* 6643 * The putp() can call qwriter and get exclusive access 6644 * IFF this is the only claim. So, we need to test for 6645 * this possibility so we can aquire the mutex and clear 6646 * the bit. 6647 */ 6648 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { 6649 mutex_enter(SQLOCK(sq)); 6650 sq->sq_flags &= ~SQ_EXCL; 6651 mutex_exit(SQLOCK(sq)); 6652 } 6653 } 6654 6655 /* 6656 * We should either have no queues on the syncq, or we were 6657 * told to goaway by a waiter (which we will wake up at the 6658 * end of this function). 6659 */ 6660 ASSERT((q->q_sqhead == NULL) || 6661 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); 6662 6663 ASSERT(MUTEX_HELD(QLOCK(q))); 6664 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6665 6666 /* 6667 * Remove the q from the syncq list if all the messages are 6668 * drained. 6669 */ 6670 if (q->q_sqhead == NULL) { 6671 mutex_enter(SQLOCK(sq)); 6672 if (q->q_sqflags & Q_SQQUEUED) 6673 SQRM_Q(sq, q); 6674 mutex_exit(SQLOCK(sq)); 6675 /* 6676 * Since the queue is removed from the list, reset its priority. 6677 */ 6678 q->q_spri = 0; 6679 } 6680 6681 /* 6682 * Remember, the q_draining flag is used to let another 6683 * thread know that there is a thread currently draining 6684 * the messages for a queue. Since we are now done with 6685 * this queue (even if there may be messages still there), 6686 * we need to clear this flag so some thread will work 6687 * on it if needed. 6688 */ 6689 ASSERT(q->q_draining); 6690 q->q_draining = 0; 6691 6692 /* called with a claim, so OK to drop all locks. */ 6693 mutex_exit(QLOCK(q)); 6694 6695 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6696 "drain_syncq end:%p", sq); 6697 } 6698 /* END OF QDRAIN_SYNCQ */ 6699 6700 6701 /* 6702 * This is the mate to qdrain_syncq, except that it is putting the 6703 * message onto the the queue instead draining. Since the 6704 * message is destined for the queue that is selected, there is 6705 * no need to identify the function because the message is 6706 * intended for the put routine for the queue. But this 6707 * routine will do it anyway just in case (but only for debug kernels). 6708 * 6709 * After the message is enqueued on the syncq, it calls putnext_tail() 6710 * which will schedule a background thread to actually process the message. 6711 * 6712 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and 6713 * SQLOCK(sq) and QLOCK(q) are not held. 6714 */ 6715 void 6716 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) 6717 { 6718 queue_t *fq = NULL; 6719 6720 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6721 ASSERT(MUTEX_NOT_HELD(QLOCK(q))); 6722 ASSERT(sq->sq_count > 0); 6723 ASSERT(q->q_syncq == sq); 6724 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6725 sq->sq_oprev == NULL) || 6726 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6727 sq->sq_oprev != NULL)); 6728 6729 mutex_enter(QLOCK(q)); 6730 6731 /* 6732 * Set QFULL in next service procedure queue (that cares) if not 6733 * already set and if there are already more messages on the syncq 6734 * than sq_max_size. If sq_max_size is 0, no flow control will be 6735 * asserted on any syncq. 6736 * 6737 * The fq here is the next queue with a service procedure. 6738 * This is where we would fail canputnext, so this is where we 6739 * need to set QFULL. 6740 * 6741 * LOCKING HIERARCHY: In the case when fq != q we need to 6742 * a) Take QLOCK(fq) to set QFULL flag and 6743 * b) Take sd_reflock in the case of the hot stream to update 6744 * sd_refcnt. 6745 * We already have QLOCK at this point. To avoid cross-locks with 6746 * freezestr() which grabs all QLOCKs and with strlock() which grabs 6747 * both SQLOCK and sd_reflock, we need to drop respective locks first. 6748 */ 6749 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) && 6750 (q->q_syncqmsgs > sq_max_size)) { 6751 if ((fq = q->q_nfsrv) == q) { 6752 fq->q_flag |= QFULL; 6753 } else { 6754 mutex_exit(QLOCK(q)); 6755 mutex_enter(QLOCK(fq)); 6756 fq->q_flag |= QFULL; 6757 mutex_exit(QLOCK(fq)); 6758 mutex_enter(QLOCK(q)); 6759 } 6760 } 6761 6762 #ifdef DEBUG 6763 /* 6764 * This is used for debug in the qfill_syncq/qdrain_syncq case 6765 * to trace the queue that the message is intended for. Note 6766 * that the original use was to identify the queue and function 6767 * to call on the drain. In the new syncq, we have the context 6768 * of the queue that we are draining, so call it's putproc and 6769 * don't rely on the saved values. But for debug this is still 6770 * usefull information. 6771 */ 6772 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp; 6773 mp->b_queue = q; 6774 mp->b_next = NULL; 6775 #endif 6776 ASSERT(q->q_syncq == sq); 6777 /* 6778 * Enqueue the message on the list. 6779 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to 6780 * protect it. So its ok to acquire SQLOCK after SQPUT_MP(). 6781 */ 6782 SQPUT_MP(q, mp); 6783 mutex_enter(SQLOCK(sq)); 6784 6785 /* 6786 * And queue on syncq for scheduling, if not already queued. 6787 * Note that we need the SQLOCK for this, and for testing flags 6788 * at the end to see if we will drain. So grab it now, and 6789 * release it before we call qdrain_syncq or return. 6790 */ 6791 if (!(q->q_sqflags & Q_SQQUEUED)) { 6792 q->q_spri = curthread->t_pri; 6793 SQPUT_Q(sq, q); 6794 } 6795 #ifdef DEBUG 6796 else { 6797 /* 6798 * All of these conditions MUST be true! 6799 */ 6800 ASSERT(sq->sq_tail != NULL); 6801 if (sq->sq_tail == sq->sq_head) { 6802 ASSERT((q->q_sqprev == NULL) && 6803 (q->q_sqnext == NULL)); 6804 } else { 6805 ASSERT((q->q_sqprev != NULL) || 6806 (q->q_sqnext != NULL)); 6807 } 6808 ASSERT(sq->sq_flags & SQ_QUEUED); 6809 ASSERT(q->q_syncqmsgs != 0); 6810 ASSERT(q->q_sqflags & Q_SQQUEUED); 6811 } 6812 #endif 6813 mutex_exit(QLOCK(q)); 6814 /* 6815 * SQLOCK is still held, so sq_count can be safely decremented. 6816 */ 6817 sq->sq_count--; 6818 6819 putnext_tail(sq, q, 0); 6820 /* Should not reference sq or q after this point. */ 6821 } 6822 6823 /* End of qfill_syncq */ 6824 6825 /* 6826 * Remove all messages from a syncq (if qp is NULL) or remove all messages 6827 * that would be put into qp by drain_syncq. 6828 * Used when deleting the syncq (qp == NULL) or when detaching 6829 * a queue (qp != NULL). 6830 * Return non-zero if one or more messages were freed. 6831 * 6832 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 6833 * sq_putlocks are used. 6834 * 6835 * NOTE: This function assumes that it is called from the close() context and 6836 * that all the queues in the syncq are going aay. For this reason it doesn't 6837 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is 6838 * currently valid, but it is useful to rethink this function to behave properly 6839 * in other cases. 6840 */ 6841 int 6842 flush_syncq(syncq_t *sq, queue_t *qp) 6843 { 6844 mblk_t *bp, *mp_head, *mp_next, *mp_prev; 6845 queue_t *q; 6846 int ret = 0; 6847 6848 mutex_enter(SQLOCK(sq)); 6849 6850 /* 6851 * Before we leave, we need to make sure there are no 6852 * events listed for this queue. All events for this queue 6853 * will just be freed. 6854 */ 6855 if (qp != NULL && sq->sq_evhead != NULL) { 6856 ASSERT(sq->sq_flags & SQ_EVENTS); 6857 6858 mp_prev = NULL; 6859 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { 6860 mp_next = bp->b_next; 6861 if (bp->b_queue == qp) { 6862 /* Delete this message */ 6863 if (mp_prev != NULL) { 6864 mp_prev->b_next = mp_next; 6865 /* 6866 * Update sq_evtail if the last element 6867 * is removed. 6868 */ 6869 if (bp == sq->sq_evtail) { 6870 ASSERT(mp_next == NULL); 6871 sq->sq_evtail = mp_prev; 6872 } 6873 } else 6874 sq->sq_evhead = mp_next; 6875 if (sq->sq_evhead == NULL) 6876 sq->sq_flags &= ~SQ_EVENTS; 6877 bp->b_prev = bp->b_next = NULL; 6878 freemsg(bp); 6879 ret++; 6880 } else { 6881 mp_prev = bp; 6882 } 6883 } 6884 } 6885 6886 /* 6887 * Walk sq_head and: 6888 * - match qp if qp is set, remove it's messages 6889 * - all if qp is not set 6890 */ 6891 q = sq->sq_head; 6892 while (q != NULL) { 6893 ASSERT(q->q_syncq == sq); 6894 if ((qp == NULL) || (qp == q)) { 6895 /* 6896 * Yank the messages as a list off the queue 6897 */ 6898 mp_head = q->q_sqhead; 6899 /* 6900 * We do not have QLOCK(q) here (which is safe due to 6901 * assumptions mentioned above). To obtain the lock we 6902 * need to release SQLOCK which may allow lots of things 6903 * to change upon us. This place requires more analysis. 6904 */ 6905 q->q_sqhead = q->q_sqtail = NULL; 6906 ASSERT(mp_head->b_queue && 6907 mp_head->b_queue->q_syncq == sq); 6908 6909 /* 6910 * Free each of the messages. 6911 */ 6912 for (bp = mp_head; bp != NULL; bp = mp_next) { 6913 mp_next = bp->b_next; 6914 bp->b_prev = bp->b_next = NULL; 6915 freemsg(bp); 6916 ret++; 6917 } 6918 /* 6919 * Now remove the queue from the syncq. 6920 */ 6921 ASSERT(q->q_sqflags & Q_SQQUEUED); 6922 SQRM_Q(sq, q); 6923 q->q_spri = 0; 6924 q->q_syncqmsgs = 0; 6925 6926 /* 6927 * If qp was specified, we are done with it and are 6928 * going to drop SQLOCK(sq) and return. We wakeup syncq 6929 * waiters while we still have the SQLOCK. 6930 */ 6931 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { 6932 sq->sq_flags &= ~SQ_WANTWAKEUP; 6933 cv_broadcast(&sq->sq_wait); 6934 } 6935 /* Drop SQLOCK across clr_qfull */ 6936 mutex_exit(SQLOCK(sq)); 6937 6938 /* 6939 * We avoid doing the test that drain_syncq does and 6940 * unconditionally clear qfull for every flushed 6941 * message. Since flush_syncq is only called during 6942 * close this should not be a problem. 6943 */ 6944 clr_qfull(q); 6945 if (qp != NULL) { 6946 return (ret); 6947 } else { 6948 mutex_enter(SQLOCK(sq)); 6949 /* 6950 * The head was removed by SQRM_Q above. 6951 * reread the new head and flush it. 6952 */ 6953 q = sq->sq_head; 6954 } 6955 } else { 6956 q = q->q_sqnext; 6957 } 6958 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6959 } 6960 6961 if (sq->sq_flags & SQ_WANTWAKEUP) { 6962 sq->sq_flags &= ~SQ_WANTWAKEUP; 6963 cv_broadcast(&sq->sq_wait); 6964 } 6965 6966 mutex_exit(SQLOCK(sq)); 6967 return (ret); 6968 } 6969 6970 /* 6971 * Propagate all messages from a syncq to the next syncq that are associated 6972 * with the specified queue. If the queue is attached to a driver or if the 6973 * messages have been added due to a qwriter(PERIM_INNER), free the messages. 6974 * 6975 * Assumes that the stream is strlock()'ed. We don't come here if there 6976 * are no messages to propagate. 6977 * 6978 * NOTE : If the queue is attached to a driver, all the messages are freed 6979 * as there is no point in propagating the messages from the driver syncq 6980 * to the closing stream head which will in turn get freed later. 6981 */ 6982 static int 6983 propagate_syncq(queue_t *qp) 6984 { 6985 mblk_t *bp, *head, *tail, *prev, *next; 6986 syncq_t *sq; 6987 queue_t *nqp; 6988 syncq_t *nsq; 6989 boolean_t isdriver; 6990 int moved = 0; 6991 uint16_t flags; 6992 pri_t priority = curthread->t_pri; 6993 #ifdef DEBUG 6994 void (*func)(); 6995 #endif 6996 6997 sq = qp->q_syncq; 6998 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6999 /* debug macro */ 7000 SQ_PUTLOCKS_HELD(sq); 7001 /* 7002 * As entersq() does not increment the sq_count for 7003 * the write side, check sq_count for non-QPERQ 7004 * perimeters alone. 7005 */ 7006 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); 7007 7008 /* 7009 * propagate_syncq() can be called because of either messages on the 7010 * queue syncq or because on events on the queue syncq. Do actual 7011 * message propagations if there are any messages. 7012 */ 7013 if (qp->q_syncqmsgs) { 7014 isdriver = (qp->q_flag & QISDRV); 7015 7016 if (!isdriver) { 7017 nqp = qp->q_next; 7018 nsq = nqp->q_syncq; 7019 ASSERT(MUTEX_HELD(SQLOCK(nsq))); 7020 /* debug macro */ 7021 SQ_PUTLOCKS_HELD(nsq); 7022 #ifdef DEBUG 7023 func = (void (*)())nqp->q_qinfo->qi_putp; 7024 #endif 7025 } 7026 7027 SQRM_Q(sq, qp); 7028 priority = MAX(qp->q_spri, priority); 7029 qp->q_spri = 0; 7030 head = qp->q_sqhead; 7031 tail = qp->q_sqtail; 7032 qp->q_sqhead = qp->q_sqtail = NULL; 7033 qp->q_syncqmsgs = 0; 7034 7035 /* 7036 * Walk the list of messages, and free them if this is a driver, 7037 * otherwise reset the b_prev and b_queue value to the new putp. 7038 * Afterward, we will just add the head to the end of the next 7039 * syncq, and point the tail to the end of this one. 7040 */ 7041 7042 for (bp = head; bp != NULL; bp = next) { 7043 next = bp->b_next; 7044 if (isdriver) { 7045 bp->b_prev = bp->b_next = NULL; 7046 freemsg(bp); 7047 continue; 7048 } 7049 /* Change the q values for this message */ 7050 bp->b_queue = nqp; 7051 #ifdef DEBUG 7052 bp->b_prev = (mblk_t *)func; 7053 #endif 7054 moved++; 7055 } 7056 /* 7057 * Attach list of messages to the end of the new queue (if there 7058 * is a list of messages). 7059 */ 7060 7061 if (!isdriver && head != NULL) { 7062 ASSERT(tail != NULL); 7063 if (nqp->q_sqhead == NULL) { 7064 nqp->q_sqhead = head; 7065 } else { 7066 ASSERT(nqp->q_sqtail != NULL); 7067 nqp->q_sqtail->b_next = head; 7068 } 7069 nqp->q_sqtail = tail; 7070 /* 7071 * When messages are moved from high priority queue to 7072 * another queue, the destination queue priority is 7073 * upgraded. 7074 */ 7075 7076 if (priority > nqp->q_spri) 7077 nqp->q_spri = priority; 7078 7079 SQPUT_Q(nsq, nqp); 7080 7081 nqp->q_syncqmsgs += moved; 7082 ASSERT(nqp->q_syncqmsgs != 0); 7083 } 7084 } 7085 7086 /* 7087 * Before we leave, we need to make sure there are no 7088 * events listed for this queue. All events for this queue 7089 * will just be freed. 7090 */ 7091 if (sq->sq_evhead != NULL) { 7092 ASSERT(sq->sq_flags & SQ_EVENTS); 7093 prev = NULL; 7094 for (bp = sq->sq_evhead; bp != NULL; bp = next) { 7095 next = bp->b_next; 7096 if (bp->b_queue == qp) { 7097 /* Delete this message */ 7098 if (prev != NULL) { 7099 prev->b_next = next; 7100 /* 7101 * Update sq_evtail if the last element 7102 * is removed. 7103 */ 7104 if (bp == sq->sq_evtail) { 7105 ASSERT(next == NULL); 7106 sq->sq_evtail = prev; 7107 } 7108 } else 7109 sq->sq_evhead = next; 7110 if (sq->sq_evhead == NULL) 7111 sq->sq_flags &= ~SQ_EVENTS; 7112 bp->b_prev = bp->b_next = NULL; 7113 freemsg(bp); 7114 } else { 7115 prev = bp; 7116 } 7117 } 7118 } 7119 7120 flags = sq->sq_flags; 7121 7122 /* Wake up any waiter before leaving. */ 7123 if (flags & SQ_WANTWAKEUP) { 7124 flags &= ~SQ_WANTWAKEUP; 7125 cv_broadcast(&sq->sq_wait); 7126 } 7127 sq->sq_flags = flags; 7128 7129 return (moved); 7130 } 7131 7132 /* 7133 * Try and upgrade to exclusive access at the inner perimeter. If this can 7134 * not be done without blocking then request will be queued on the syncq 7135 * and drain_syncq will run it later. 7136 * 7137 * This routine can only be called from put or service procedures plus 7138 * asynchronous callback routines that have properly entered to 7139 * queue (with entersq.) Thus qwriter_inner assumes the caller has one claim 7140 * on the syncq associated with q. 7141 */ 7142 void 7143 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)()) 7144 { 7145 syncq_t *sq = q->q_syncq; 7146 uint16_t count; 7147 7148 mutex_enter(SQLOCK(sq)); 7149 count = sq->sq_count; 7150 SQ_PUTLOCKS_ENTER(sq); 7151 SUM_SQ_PUTCOUNTS(sq, count); 7152 ASSERT(count >= 1); 7153 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); 7154 7155 if (count == 1) { 7156 /* 7157 * Can upgrade. This case also handles nested qwriter calls 7158 * (when the qwriter callback function calls qwriter). In that 7159 * case SQ_EXCL is already set. 7160 */ 7161 sq->sq_flags |= SQ_EXCL; 7162 SQ_PUTLOCKS_EXIT(sq); 7163 mutex_exit(SQLOCK(sq)); 7164 (*func)(q, mp); 7165 /* 7166 * Assumes that leavesq, putnext, and drain_syncq will reset 7167 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on 7168 * until putnext, leavesq, or drain_syncq drops it. 7169 * That way we handle nested qwriter(INNER) without dropping 7170 * SQ_EXCL until the outermost qwriter callback routine is 7171 * done. 7172 */ 7173 return; 7174 } 7175 SQ_PUTLOCKS_EXIT(sq); 7176 sqfill_events(sq, q, mp, func); 7177 } 7178 7179 /* 7180 * Synchronous callback support functions 7181 */ 7182 7183 /* 7184 * Allocate a callback parameter structure. 7185 * Assumes that caller initializes the flags and the id. 7186 * Acquires SQLOCK(sq) if non-NULL is returned. 7187 */ 7188 callbparams_t * 7189 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) 7190 { 7191 callbparams_t *cbp; 7192 size_t size = sizeof (callbparams_t); 7193 7194 cbp = kmem_alloc(size, kmflags & ~KM_PANIC); 7195 7196 /* 7197 * Only try tryhard allocation if the caller is ready to panic. 7198 * Otherwise just fail. 7199 */ 7200 if (cbp == NULL) { 7201 if (kmflags & KM_PANIC) 7202 cbp = kmem_alloc_tryhard(sizeof (callbparams_t), 7203 &size, kmflags); 7204 else 7205 return (NULL); 7206 } 7207 7208 ASSERT(size >= sizeof (callbparams_t)); 7209 cbp->cbp_size = size; 7210 cbp->cbp_sq = sq; 7211 cbp->cbp_func = func; 7212 cbp->cbp_arg = arg; 7213 mutex_enter(SQLOCK(sq)); 7214 cbp->cbp_next = sq->sq_callbpend; 7215 sq->sq_callbpend = cbp; 7216 return (cbp); 7217 } 7218 7219 void 7220 callbparams_free(syncq_t *sq, callbparams_t *cbp) 7221 { 7222 callbparams_t **pp, *p; 7223 7224 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7225 7226 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7227 if (p == cbp) { 7228 *pp = p->cbp_next; 7229 kmem_free(p, p->cbp_size); 7230 return; 7231 } 7232 } 7233 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7234 "callbparams_free: not found\n")); 7235 } 7236 7237 void 7238 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) 7239 { 7240 callbparams_t **pp, *p; 7241 7242 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7243 7244 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7245 if (p->cbp_id == id && p->cbp_flags == flag) { 7246 *pp = p->cbp_next; 7247 kmem_free(p, p->cbp_size); 7248 return; 7249 } 7250 } 7251 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7252 "callbparams_free_id: not found\n")); 7253 } 7254 7255 /* 7256 * Callback wrapper function used by once-only callbacks that can be 7257 * cancelled (qtimeout and qbufcall) 7258 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be 7259 * cancelled by the qun* functions. 7260 */ 7261 void 7262 qcallbwrapper(void *arg) 7263 { 7264 callbparams_t *cbp = arg; 7265 syncq_t *sq; 7266 uint16_t count = 0; 7267 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 7268 uint16_t type; 7269 7270 sq = cbp->cbp_sq; 7271 mutex_enter(SQLOCK(sq)); 7272 type = sq->sq_type; 7273 if (!(type & SQ_CICB)) { 7274 count = sq->sq_count; 7275 SQ_PUTLOCKS_ENTER(sq); 7276 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 7277 SUM_SQ_PUTCOUNTS(sq, count); 7278 sq->sq_needexcl++; 7279 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 7280 waitflags |= SQ_MESSAGES; 7281 } 7282 /* Can not handle exlusive entry at outer perimeter */ 7283 ASSERT(type & SQ_COCB); 7284 7285 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { 7286 if ((sq->sq_callbflags & cbp->cbp_flags) && 7287 (sq->sq_cancelid == cbp->cbp_id)) { 7288 /* timeout has been cancelled */ 7289 sq->sq_callbflags |= SQ_CALLB_BYPASSED; 7290 callbparams_free(sq, cbp); 7291 if (!(type & SQ_CICB)) { 7292 ASSERT(sq->sq_needexcl > 0); 7293 sq->sq_needexcl--; 7294 if (sq->sq_needexcl == 0) { 7295 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7296 } 7297 SQ_PUTLOCKS_EXIT(sq); 7298 } 7299 mutex_exit(SQLOCK(sq)); 7300 return; 7301 } 7302 sq->sq_flags |= SQ_WANTWAKEUP; 7303 if (!(type & SQ_CICB)) { 7304 SQ_PUTLOCKS_EXIT(sq); 7305 } 7306 cv_wait(&sq->sq_wait, SQLOCK(sq)); 7307 if (!(type & SQ_CICB)) { 7308 count = sq->sq_count; 7309 SQ_PUTLOCKS_ENTER(sq); 7310 SUM_SQ_PUTCOUNTS(sq, count); 7311 } 7312 } 7313 7314 sq->sq_count++; 7315 ASSERT(sq->sq_count != 0); /* Wraparound */ 7316 if (!(type & SQ_CICB)) { 7317 ASSERT(count == 0); 7318 sq->sq_flags |= SQ_EXCL; 7319 ASSERT(sq->sq_needexcl > 0); 7320 sq->sq_needexcl--; 7321 if (sq->sq_needexcl == 0) { 7322 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7323 } 7324 SQ_PUTLOCKS_EXIT(sq); 7325 } 7326 7327 mutex_exit(SQLOCK(sq)); 7328 7329 cbp->cbp_func(cbp->cbp_arg); 7330 7331 /* 7332 * We drop the lock only for leavesq to re-acquire it. 7333 * Possible optimization is inline of leavesq. 7334 */ 7335 mutex_enter(SQLOCK(sq)); 7336 callbparams_free(sq, cbp); 7337 mutex_exit(SQLOCK(sq)); 7338 leavesq(sq, SQ_CALLBACK); 7339 } 7340 7341 /* 7342 * no need to grab sq_putlocks here. See comment in strsubr.h that 7343 * explains when sq_putlocks are used. 7344 * 7345 * sq_count (or one of the sq_putcounts) has already been 7346 * decremented by the caller, and if SQ_QUEUED, we need to call 7347 * drain_syncq (the global syncq drain). 7348 * If putnext_tail is called with the SQ_EXCL bit set, we are in 7349 * one of two states, non-CIPUT perimiter, and we need to clear 7350 * it, or we went exclusive in the put procedure. In any case, 7351 * we want to clear the bit now, and it is probably easier to do 7352 * this at the beginning of this function (remember, we hold 7353 * the SQLOCK). Lastly, if there are other messages queued 7354 * on the syncq (and not for our destination), enable the syncq 7355 * for background work. 7356 */ 7357 7358 /* ARGSUSED */ 7359 void 7360 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) 7361 { 7362 uint16_t flags = sq->sq_flags; 7363 7364 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7365 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 7366 7367 /* Clear SQ_EXCL if set in passflags */ 7368 if (passflags & SQ_EXCL) { 7369 flags &= ~SQ_EXCL; 7370 } 7371 if (flags & SQ_WANTWAKEUP) { 7372 flags &= ~SQ_WANTWAKEUP; 7373 cv_broadcast(&sq->sq_wait); 7374 } 7375 if (flags & SQ_WANTEXWAKEUP) { 7376 flags &= ~SQ_WANTEXWAKEUP; 7377 cv_broadcast(&sq->sq_exitwait); 7378 } 7379 sq->sq_flags = flags; 7380 7381 /* 7382 * We have cleared SQ_EXCL if we were asked to, and started 7383 * the wakeup process for waiters. If there are no writers 7384 * then we need to drain the syncq if we were told to, or 7385 * enable the background thread to do it. 7386 */ 7387 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) { 7388 if ((passflags & SQ_QUEUED) || 7389 (sq->sq_svcflags & SQ_DISABLED)) { 7390 /* drain_syncq will take care of events in the list */ 7391 drain_syncq(sq); 7392 return; 7393 } else if (flags & SQ_QUEUED) { 7394 sqenable(sq); 7395 } 7396 } 7397 /* Drop the SQLOCK on exit */ 7398 mutex_exit(SQLOCK(sq)); 7399 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END, 7400 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); 7401 } 7402 7403 void 7404 set_qend(queue_t *q) 7405 { 7406 mutex_enter(QLOCK(q)); 7407 if (!O_SAMESTR(q)) 7408 q->q_flag |= QEND; 7409 else 7410 q->q_flag &= ~QEND; 7411 mutex_exit(QLOCK(q)); 7412 q = _OTHERQ(q); 7413 mutex_enter(QLOCK(q)); 7414 if (!O_SAMESTR(q)) 7415 q->q_flag |= QEND; 7416 else 7417 q->q_flag &= ~QEND; 7418 mutex_exit(QLOCK(q)); 7419 } 7420 7421 7422 void 7423 clr_qfull(queue_t *q) 7424 { 7425 queue_t *oq = q; 7426 7427 q = q->q_nfsrv; 7428 /* Fast check if there is any work to do before getting the lock. */ 7429 if ((q->q_flag & (QFULL|QWANTW)) == 0) { 7430 return; 7431 } 7432 7433 /* 7434 * Do not reset QFULL (and backenable) if the q_count is the reason 7435 * for QFULL being set. 7436 */ 7437 mutex_enter(QLOCK(q)); 7438 /* 7439 * If both q_count and q_mblkcnt are less than the hiwat mark 7440 */ 7441 if ((q->q_count < q->q_hiwat) && (q->q_mblkcnt < q->q_hiwat)) { 7442 q->q_flag &= ~QFULL; 7443 /* 7444 * A little more confusing, how about this way: 7445 * if someone wants to write, 7446 * AND 7447 * both counts are less than the lowat mark 7448 * OR 7449 * the lowat mark is zero 7450 * THEN 7451 * backenable 7452 */ 7453 if ((q->q_flag & QWANTW) && 7454 (((q->q_count < q->q_lowat) && 7455 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) { 7456 q->q_flag &= ~QWANTW; 7457 mutex_exit(QLOCK(q)); 7458 backenable(oq, 0); 7459 } else 7460 mutex_exit(QLOCK(q)); 7461 } else 7462 mutex_exit(QLOCK(q)); 7463 } 7464 7465 /* 7466 * Set the forward service procedure pointer. 7467 * 7468 * Called at insert-time to cache a queue's next forward service procedure in 7469 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted 7470 * has a service procedure then q_nfsrv points to itself. If the queue to be 7471 * inserted does not have a service procedure, then q_nfsrv points to the next 7472 * queue forward that has a service procedure. If the queue is at the logical 7473 * end of the stream (driver for write side, stream head for the read side) 7474 * and does not have a service procedure, then q_nfsrv also points to itself. 7475 */ 7476 void 7477 set_nfsrv_ptr( 7478 queue_t *rnew, /* read queue pointer to new module */ 7479 queue_t *wnew, /* write queue pointer to new module */ 7480 queue_t *prev_rq, /* read queue pointer to the module above */ 7481 queue_t *prev_wq) /* write queue pointer to the module above */ 7482 { 7483 queue_t *qp; 7484 7485 if (prev_wq->q_next == NULL) { 7486 /* 7487 * Insert the driver, initialize the driver and stream head. 7488 * In this case, prev_rq/prev_wq should be the stream head. 7489 * _I_INSERT does not allow inserting a driver. Make sure 7490 * that it is not an insertion. 7491 */ 7492 ASSERT(!(rnew->q_flag & _QINSERTING)); 7493 wnew->q_nfsrv = wnew; 7494 if (rnew->q_qinfo->qi_srvp) 7495 rnew->q_nfsrv = rnew; 7496 else 7497 rnew->q_nfsrv = prev_rq; 7498 prev_rq->q_nfsrv = prev_rq; 7499 prev_wq->q_nfsrv = prev_wq; 7500 } else { 7501 /* 7502 * set up read side q_nfsrv pointer. This MUST be done 7503 * before setting the write side, because the setting of 7504 * the write side for a fifo may depend on it. 7505 * 7506 * Suppose we have a fifo that only has pipemod pushed. 7507 * pipemod has no read or write service procedures, so 7508 * nfsrv for both pipemod queues points to prev_rq (the 7509 * stream read head). Now push bufmod (which has only a 7510 * read service procedure). Doing the write side first, 7511 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which 7512 * is WRONG; the next queue forward from wnew with a 7513 * service procedure will be rnew, not the stream read head. 7514 * Since the downstream queue (which in the case of a fifo 7515 * is the read queue rnew) can affect upstream queues, it 7516 * needs to be done first. Setting up the read side first 7517 * sets nfsrv for both pipemod queues to rnew and then 7518 * when the write side is set up, wnew-q_nfsrv will also 7519 * point to rnew. 7520 */ 7521 if (rnew->q_qinfo->qi_srvp) { 7522 /* 7523 * use _OTHERQ() because, if this is a pipe, next 7524 * module may have been pushed from other end and 7525 * q_next could be a read queue. 7526 */ 7527 qp = _OTHERQ(prev_wq->q_next); 7528 while (qp && qp->q_nfsrv != qp) { 7529 qp->q_nfsrv = rnew; 7530 qp = backq(qp); 7531 } 7532 rnew->q_nfsrv = rnew; 7533 } else 7534 rnew->q_nfsrv = prev_rq->q_nfsrv; 7535 7536 /* set up write side q_nfsrv pointer */ 7537 if (wnew->q_qinfo->qi_srvp) { 7538 wnew->q_nfsrv = wnew; 7539 7540 /* 7541 * For insertion, need to update nfsrv of the modules 7542 * above which do not have a service routine. 7543 */ 7544 if (rnew->q_flag & _QINSERTING) { 7545 for (qp = prev_wq; 7546 qp != NULL && qp->q_nfsrv != qp; 7547 qp = backq(qp)) { 7548 qp->q_nfsrv = wnew->q_nfsrv; 7549 } 7550 } 7551 } else { 7552 if (prev_wq->q_next == prev_rq) 7553 /* 7554 * Since prev_wq/prev_rq are the middle of a 7555 * fifo, wnew/rnew will also be the middle of 7556 * a fifo and wnew's nfsrv is same as rnew's. 7557 */ 7558 wnew->q_nfsrv = rnew->q_nfsrv; 7559 else 7560 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv; 7561 } 7562 } 7563 } 7564 7565 /* 7566 * Reset the forward service procedure pointer; called at remove-time. 7567 */ 7568 void 7569 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp) 7570 { 7571 queue_t *tmp_qp; 7572 7573 /* Reset the write side q_nfsrv pointer for _I_REMOVE */ 7574 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) { 7575 for (tmp_qp = backq(wqp); 7576 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp; 7577 tmp_qp = backq(tmp_qp)) { 7578 tmp_qp->q_nfsrv = wqp->q_nfsrv; 7579 } 7580 } 7581 7582 /* reset the read side q_nfsrv pointer */ 7583 if (rqp->q_qinfo->qi_srvp) { 7584 if (wqp->q_next) { /* non-driver case */ 7585 tmp_qp = _OTHERQ(wqp->q_next); 7586 while (tmp_qp && tmp_qp->q_nfsrv == rqp) { 7587 /* Note that rqp->q_next cannot be NULL */ 7588 ASSERT(rqp->q_next != NULL); 7589 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv; 7590 tmp_qp = backq(tmp_qp); 7591 } 7592 } 7593 } 7594 } 7595 7596 /* 7597 * This routine should be called after all stream geometry changes to update 7598 * the stream head cached struio() rd/wr queue pointers. Note must be called 7599 * with the streamlock()ed. 7600 * 7601 * Note: only enables Synchronous STREAMS for a side of a Stream which has 7602 * an explicit synchronous barrier module queue. That is, a queue that 7603 * has specified a struio() type. 7604 */ 7605 static void 7606 strsetuio(stdata_t *stp) 7607 { 7608 queue_t *wrq; 7609 7610 if (stp->sd_flag & STPLEX) { 7611 /* 7612 * Not stremahead, but a mux, so no Synchronous STREAMS. 7613 */ 7614 stp->sd_struiowrq = NULL; 7615 stp->sd_struiordq = NULL; 7616 return; 7617 } 7618 /* 7619 * Scan the write queue(s) while synchronous 7620 * until we find a qinfo uio type specified. 7621 */ 7622 wrq = stp->sd_wrq->q_next; 7623 while (wrq) { 7624 if (wrq->q_struiot == STRUIOT_NONE) { 7625 wrq = 0; 7626 break; 7627 } 7628 if (wrq->q_struiot != STRUIOT_DONTCARE) 7629 break; 7630 if (! _SAMESTR(wrq)) { 7631 wrq = 0; 7632 break; 7633 } 7634 wrq = wrq->q_next; 7635 } 7636 stp->sd_struiowrq = wrq; 7637 /* 7638 * Scan the read queue(s) while synchronous 7639 * until we find a qinfo uio type specified. 7640 */ 7641 wrq = stp->sd_wrq->q_next; 7642 while (wrq) { 7643 if (_RD(wrq)->q_struiot == STRUIOT_NONE) { 7644 wrq = 0; 7645 break; 7646 } 7647 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE) 7648 break; 7649 if (! _SAMESTR(wrq)) { 7650 wrq = 0; 7651 break; 7652 } 7653 wrq = wrq->q_next; 7654 } 7655 stp->sd_struiordq = wrq ? _RD(wrq) : 0; 7656 } 7657 7658 /* 7659 * pass_wput, unblocks the passthru queues, so that 7660 * messages can arrive at muxs lower read queue, before 7661 * I_LINK/I_UNLINK is acked/nacked. 7662 */ 7663 static void 7664 pass_wput(queue_t *q, mblk_t *mp) 7665 { 7666 syncq_t *sq; 7667 7668 sq = _RD(q)->q_syncq; 7669 if (sq->sq_flags & SQ_BLOCKED) 7670 unblocksq(sq, SQ_BLOCKED, 0); 7671 putnext(q, mp); 7672 } 7673 7674 /* 7675 * Set up queues for the link/unlink. 7676 * Create a new queue and block it and then insert it 7677 * below the stream head on the lower stream. 7678 * This prevents any messages from arriving during the setq 7679 * as well as while the mux is processing the LINK/I_UNLINK. 7680 * The blocked passq is unblocked once the LINK/I_UNLINK has 7681 * been acked or nacked or if a message is generated and sent 7682 * down muxs write put procedure. 7683 * see pass_wput(). 7684 * 7685 * After the new queue is inserted, all messages coming from below are 7686 * blocked. The call to strlock will ensure that all activity in the stream head 7687 * read queue syncq is stopped (sq_count drops to zero). 7688 */ 7689 static queue_t * 7690 link_addpassthru(stdata_t *stpdown) 7691 { 7692 queue_t *passq; 7693 sqlist_t sqlist; 7694 7695 passq = allocq(); 7696 STREAM(passq) = STREAM(_WR(passq)) = stpdown; 7697 /* setq might sleep in allocator - avoid holding locks. */ 7698 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ, 7699 SQ_CI|SQ_CO, B_FALSE); 7700 claimq(passq); 7701 blocksq(passq->q_syncq, SQ_BLOCKED, 1); 7702 insertq(STREAM(passq), passq); 7703 7704 /* 7705 * Use strlock() to wait for the stream head sq_count to drop to zero 7706 * since we are going to change q_ptr in the stream head. Note that 7707 * insertq() doesn't wait for any syncq counts to drop to zero. 7708 */ 7709 sqlist.sqlist_head = NULL; 7710 sqlist.sqlist_index = 0; 7711 sqlist.sqlist_size = sizeof (sqlist_t); 7712 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq); 7713 strlock(stpdown, &sqlist); 7714 strunlock(stpdown, &sqlist); 7715 7716 releaseq(passq); 7717 return (passq); 7718 } 7719 7720 /* 7721 * Let messages flow up into the mux by removing 7722 * the passq. 7723 */ 7724 static void 7725 link_rempassthru(queue_t *passq) 7726 { 7727 claimq(passq); 7728 removeq(passq); 7729 releaseq(passq); 7730 freeq(passq); 7731 } 7732 7733 /* 7734 * Wait for the condition variable pointed to by `cvp' to be signaled, 7735 * or for `tim' milliseconds to elapse, whichever comes first. If `tim' 7736 * is negative, then there is no time limit. If `nosigs' is non-zero, 7737 * then the wait will be non-interruptible. 7738 * 7739 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout. 7740 */ 7741 clock_t 7742 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs) 7743 { 7744 clock_t ret, now, tick; 7745 7746 if (tim < 0) { 7747 if (nosigs) { 7748 cv_wait(cvp, mp); 7749 ret = 1; 7750 } else { 7751 ret = cv_wait_sig(cvp, mp); 7752 } 7753 } else if (tim > 0) { 7754 /* 7755 * convert milliseconds to clock ticks 7756 */ 7757 tick = MSEC_TO_TICK_ROUNDUP(tim); 7758 time_to_wait(&now, tick); 7759 if (nosigs) { 7760 ret = cv_timedwait(cvp, mp, now); 7761 } else { 7762 ret = cv_timedwait_sig(cvp, mp, now); 7763 } 7764 } else { 7765 ret = -1; 7766 } 7767 return (ret); 7768 } 7769 7770 /* 7771 * Wait until the stream head can determine if it is at the mark but 7772 * don't wait forever to prevent a race condition between the "mark" state 7773 * in the stream head and any mark state in the caller/user of this routine. 7774 * 7775 * This is used by sockets and for a socket it would be incorrect 7776 * to return a failure for SIOCATMARK when there is no data in the receive 7777 * queue and the marked urgent data is traveling up the stream. 7778 * 7779 * This routine waits until the mark is known by waiting for one of these 7780 * three events: 7781 * The stream head read queue becoming non-empty (including an EOF) 7782 * The STRATMARK flag being set. (Due to a MSGMARKNEXT message.) 7783 * The STRNOTATMARK flag being set (which indicates that the transport 7784 * has sent a MSGNOTMARKNEXT message to indicate that it is not at 7785 * the mark). 7786 * 7787 * The routine returns 1 if the stream is at the mark; 0 if it can 7788 * be determined that the stream is not at the mark. 7789 * If the wait times out and it can't determine 7790 * whether or not the stream might be at the mark the routine will return -1. 7791 * 7792 * Note: This routine should only be used when a mark is pending i.e., 7793 * in the socket case the SIGURG has been posted. 7794 * Note2: This can not wakeup just because synchronous streams indicate 7795 * that data is available since it is not possible to use the synchronous 7796 * streams interfaces to determine the b_flag value for the data queued below 7797 * the stream head. 7798 */ 7799 int 7800 strwaitmark(vnode_t *vp) 7801 { 7802 struct stdata *stp = vp->v_stream; 7803 queue_t *rq = _RD(stp->sd_wrq); 7804 int mark; 7805 7806 mutex_enter(&stp->sd_lock); 7807 while (rq->q_first == NULL && 7808 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) { 7809 stp->sd_flag |= RSLEEP; 7810 7811 /* Wait for 100 milliseconds for any state change. */ 7812 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) { 7813 mutex_exit(&stp->sd_lock); 7814 return (-1); 7815 } 7816 } 7817 if (stp->sd_flag & STRATMARK) 7818 mark = 1; 7819 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK)) 7820 mark = 1; 7821 else 7822 mark = 0; 7823 7824 mutex_exit(&stp->sd_lock); 7825 return (mark); 7826 } 7827 7828 /* 7829 * Set a read side error. If persist is set change the socket error 7830 * to persistent. If errfunc is set install the function as the exported 7831 * error handler. 7832 */ 7833 void 7834 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7835 { 7836 struct stdata *stp = vp->v_stream; 7837 7838 mutex_enter(&stp->sd_lock); 7839 stp->sd_rerror = error; 7840 if (error == 0 && errfunc == NULL) 7841 stp->sd_flag &= ~STRDERR; 7842 else 7843 stp->sd_flag |= STRDERR; 7844 if (persist) { 7845 stp->sd_flag &= ~STRDERRNONPERSIST; 7846 } else { 7847 stp->sd_flag |= STRDERRNONPERSIST; 7848 } 7849 stp->sd_rderrfunc = errfunc; 7850 if (error != 0 || errfunc != NULL) { 7851 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7852 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7853 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 7854 7855 mutex_exit(&stp->sd_lock); 7856 pollwakeup(&stp->sd_pollist, POLLERR); 7857 mutex_enter(&stp->sd_lock); 7858 7859 if (stp->sd_sigflags & S_ERROR) 7860 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 7861 } 7862 mutex_exit(&stp->sd_lock); 7863 } 7864 7865 /* 7866 * Set a write side error. If persist is set change the socket error 7867 * to persistent. 7868 */ 7869 void 7870 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7871 { 7872 struct stdata *stp = vp->v_stream; 7873 7874 mutex_enter(&stp->sd_lock); 7875 stp->sd_werror = error; 7876 if (error == 0 && errfunc == NULL) 7877 stp->sd_flag &= ~STWRERR; 7878 else 7879 stp->sd_flag |= STWRERR; 7880 if (persist) { 7881 stp->sd_flag &= ~STWRERRNONPERSIST; 7882 } else { 7883 stp->sd_flag |= STWRERRNONPERSIST; 7884 } 7885 stp->sd_wrerrfunc = errfunc; 7886 if (error != 0 || errfunc != NULL) { 7887 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7888 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7889 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 7890 7891 mutex_exit(&stp->sd_lock); 7892 pollwakeup(&stp->sd_pollist, POLLERR); 7893 mutex_enter(&stp->sd_lock); 7894 7895 if (stp->sd_sigflags & S_ERROR) 7896 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 7897 } 7898 mutex_exit(&stp->sd_lock); 7899 } 7900 7901 /* 7902 * Make the stream return 0 (EOF) when all data has been read. 7903 * No effect on write side. 7904 */ 7905 void 7906 strseteof(vnode_t *vp, int eof) 7907 { 7908 struct stdata *stp = vp->v_stream; 7909 7910 mutex_enter(&stp->sd_lock); 7911 if (!eof) { 7912 stp->sd_flag &= ~STREOF; 7913 mutex_exit(&stp->sd_lock); 7914 return; 7915 } 7916 stp->sd_flag |= STREOF; 7917 if (stp->sd_flag & RSLEEP) { 7918 stp->sd_flag &= ~RSLEEP; 7919 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); 7920 } 7921 7922 mutex_exit(&stp->sd_lock); 7923 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM); 7924 mutex_enter(&stp->sd_lock); 7925 7926 if (stp->sd_sigflags & (S_INPUT|S_RDNORM)) 7927 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0); 7928 mutex_exit(&stp->sd_lock); 7929 } 7930 7931 void 7932 strflushrq(vnode_t *vp, int flag) 7933 { 7934 struct stdata *stp = vp->v_stream; 7935 7936 mutex_enter(&stp->sd_lock); 7937 flushq(_RD(stp->sd_wrq), flag); 7938 mutex_exit(&stp->sd_lock); 7939 } 7940 7941 void 7942 strsetrputhooks(vnode_t *vp, uint_t flags, 7943 msgfunc_t protofunc, msgfunc_t miscfunc) 7944 { 7945 struct stdata *stp = vp->v_stream; 7946 7947 mutex_enter(&stp->sd_lock); 7948 7949 if (protofunc == NULL) 7950 stp->sd_rprotofunc = strrput_proto; 7951 else 7952 stp->sd_rprotofunc = protofunc; 7953 7954 if (miscfunc == NULL) 7955 stp->sd_rmiscfunc = strrput_misc; 7956 else 7957 stp->sd_rmiscfunc = miscfunc; 7958 7959 if (flags & SH_CONSOL_DATA) 7960 stp->sd_rput_opt |= SR_CONSOL_DATA; 7961 else 7962 stp->sd_rput_opt &= ~SR_CONSOL_DATA; 7963 7964 if (flags & SH_SIGALLDATA) 7965 stp->sd_rput_opt |= SR_SIGALLDATA; 7966 else 7967 stp->sd_rput_opt &= ~SR_SIGALLDATA; 7968 7969 if (flags & SH_IGN_ZEROLEN) 7970 stp->sd_rput_opt |= SR_IGN_ZEROLEN; 7971 else 7972 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN; 7973 7974 mutex_exit(&stp->sd_lock); 7975 } 7976 7977 void 7978 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime) 7979 { 7980 struct stdata *stp = vp->v_stream; 7981 7982 mutex_enter(&stp->sd_lock); 7983 stp->sd_closetime = closetime; 7984 7985 if (flags & SH_SIGPIPE) 7986 stp->sd_wput_opt |= SW_SIGPIPE; 7987 else 7988 stp->sd_wput_opt &= ~SW_SIGPIPE; 7989 if (flags & SH_RECHECK_ERR) 7990 stp->sd_wput_opt |= SW_RECHECK_ERR; 7991 else 7992 stp->sd_wput_opt &= ~SW_RECHECK_ERR; 7993 7994 mutex_exit(&stp->sd_lock); 7995 } 7996 7997 void 7998 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc) 7999 { 8000 struct stdata *stp = vp->v_stream; 8001 8002 mutex_enter(&stp->sd_lock); 8003 8004 stp->sd_rputdatafunc = rdatafunc; 8005 stp->sd_wputdatafunc = wdatafunc; 8006 8007 mutex_exit(&stp->sd_lock); 8008 } 8009 8010 /* Used within framework when the queue is already locked */ 8011 void 8012 qenable_locked(queue_t *q) 8013 { 8014 stdata_t *stp = STREAM(q); 8015 8016 ASSERT(MUTEX_HELD(QLOCK(q))); 8017 8018 if (!q->q_qinfo->qi_srvp) 8019 return; 8020 8021 /* 8022 * Do not place on run queue if already enabled or closing. 8023 */ 8024 if (q->q_flag & (QWCLOSE|QENAB)) 8025 return; 8026 8027 /* 8028 * mark queue enabled and place on run list if it is not already being 8029 * serviced. If it is serviced, the runservice() function will detect 8030 * that QENAB is set and call service procedure before clearing 8031 * QINSERVICE flag. 8032 */ 8033 q->q_flag |= QENAB; 8034 if (q->q_flag & QINSERVICE) 8035 return; 8036 8037 /* Record the time of qenable */ 8038 q->q_qtstamp = lbolt; 8039 8040 /* 8041 * Put the queue in the stp list and schedule it for background 8042 * processing if it is not already scheduled or if stream head does not 8043 * intent to process it in the foreground later by setting 8044 * STRS_WILLSERVICE flag. 8045 */ 8046 mutex_enter(&stp->sd_qlock); 8047 /* 8048 * If there are already something on the list, stp flags should show 8049 * intention to drain it. 8050 */ 8051 IMPLY(STREAM_NEEDSERVICE(stp), 8052 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))); 8053 8054 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link); 8055 stp->sd_nqueues++; 8056 8057 /* 8058 * If no one will drain this stream we are the first producer and 8059 * need to schedule it for background thread. 8060 */ 8061 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) { 8062 /* 8063 * No one will service this stream later, so we have to 8064 * schedule it now. 8065 */ 8066 STRSTAT(stenables); 8067 stp->sd_svcflags |= STRS_SCHEDULED; 8068 stp->sd_servid = (void *)taskq_dispatch(streams_taskq, 8069 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE); 8070 8071 if (stp->sd_servid == NULL) { 8072 /* 8073 * Task queue failed so fail over to the backup 8074 * servicing thread. 8075 */ 8076 STRSTAT(taskqfails); 8077 /* 8078 * It is safe to clear STRS_SCHEDULED flag because it 8079 * was set by this thread above. 8080 */ 8081 stp->sd_svcflags &= ~STRS_SCHEDULED; 8082 8083 /* 8084 * Failover scheduling is protected by service_queue 8085 * lock. 8086 */ 8087 mutex_enter(&service_queue); 8088 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q)); 8089 ASSERT(q->q_link == NULL); 8090 /* 8091 * Append the queue to qhead/qtail list. 8092 */ 8093 if (qhead == NULL) 8094 qhead = q; 8095 else 8096 qtail->q_link = q; 8097 qtail = q; 8098 /* 8099 * Clear stp queue list. 8100 */ 8101 stp->sd_qhead = stp->sd_qtail = NULL; 8102 stp->sd_nqueues = 0; 8103 /* 8104 * Wakeup background queue processing thread. 8105 */ 8106 cv_signal(&services_to_run); 8107 mutex_exit(&service_queue); 8108 } 8109 } 8110 mutex_exit(&stp->sd_qlock); 8111 } 8112 8113 static void 8114 queue_service(queue_t *q) 8115 { 8116 /* 8117 * The queue in the list should have 8118 * QENAB flag set and should not have 8119 * QINSERVICE flag set. QINSERVICE is 8120 * set when the queue is dequeued and 8121 * qenable_locked doesn't enqueue a 8122 * queue with QINSERVICE set. 8123 */ 8124 8125 ASSERT(!(q->q_flag & QINSERVICE)); 8126 ASSERT((q->q_flag & QENAB)); 8127 mutex_enter(QLOCK(q)); 8128 q->q_flag &= ~QENAB; 8129 q->q_flag |= QINSERVICE; 8130 mutex_exit(QLOCK(q)); 8131 runservice(q); 8132 } 8133 8134 static void 8135 syncq_service(syncq_t *sq) 8136 { 8137 STRSTAT(syncqservice); 8138 mutex_enter(SQLOCK(sq)); 8139 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); 8140 ASSERT(sq->sq_servcount != 0); 8141 ASSERT(sq->sq_next == NULL); 8142 8143 /* if we came here from the background thread, clear the flag */ 8144 if (sq->sq_svcflags & SQ_BGTHREAD) 8145 sq->sq_svcflags &= ~SQ_BGTHREAD; 8146 8147 /* let drain_syncq know that it's being called in the background */ 8148 sq->sq_svcflags |= SQ_SERVICE; 8149 drain_syncq(sq); 8150 } 8151 8152 static void 8153 qwriter_outer_service(syncq_t *outer) 8154 { 8155 /* 8156 * Note that SQ_WRITER is used on the outer perimeter 8157 * to signal that a qwriter(OUTER) is either investigating 8158 * running or that it is actually running a function. 8159 */ 8160 outer_enter(outer, SQ_BLOCKED|SQ_WRITER); 8161 8162 /* 8163 * All inner syncq are empty and have SQ_WRITER set 8164 * to block entering the outer perimeter. 8165 * 8166 * We do not need to explicitly call write_now since 8167 * outer_exit does it for us. 8168 */ 8169 outer_exit(outer); 8170 } 8171 8172 static void 8173 mblk_free(mblk_t *mp) 8174 { 8175 dblk_t *dbp = mp->b_datap; 8176 frtn_t *frp = dbp->db_frtnp; 8177 8178 mp->b_next = NULL; 8179 if (dbp->db_fthdr != NULL) 8180 str_ftfree(dbp); 8181 8182 ASSERT(dbp->db_fthdr == NULL); 8183 frp->free_func(frp->free_arg); 8184 ASSERT(dbp->db_mblk == mp); 8185 8186 if (dbp->db_credp != NULL) { 8187 crfree(dbp->db_credp); 8188 dbp->db_credp = NULL; 8189 } 8190 dbp->db_cpid = -1; 8191 dbp->db_struioflag = 0; 8192 dbp->db_struioun.cksum.flags = 0; 8193 8194 kmem_cache_free(dbp->db_cache, dbp); 8195 } 8196 8197 /* 8198 * Background processing of the stream queue list. 8199 */ 8200 static void 8201 stream_service(stdata_t *stp) 8202 { 8203 queue_t *q; 8204 8205 mutex_enter(&stp->sd_qlock); 8206 8207 STR_SERVICE(stp, q); 8208 8209 stp->sd_svcflags &= ~STRS_SCHEDULED; 8210 stp->sd_servid = NULL; 8211 cv_signal(&stp->sd_qcv); 8212 mutex_exit(&stp->sd_qlock); 8213 } 8214 8215 /* 8216 * Foreground processing of the stream queue list. 8217 */ 8218 void 8219 stream_runservice(stdata_t *stp) 8220 { 8221 queue_t *q; 8222 8223 mutex_enter(&stp->sd_qlock); 8224 STRSTAT(rservice); 8225 /* 8226 * We are going to drain this stream queue list, so qenable_locked will 8227 * not schedule it until we finish. 8228 */ 8229 stp->sd_svcflags |= STRS_WILLSERVICE; 8230 8231 STR_SERVICE(stp, q); 8232 8233 stp->sd_svcflags &= ~STRS_WILLSERVICE; 8234 mutex_exit(&stp->sd_qlock); 8235 /* 8236 * Help backup background thread to drain the qhead/qtail list. 8237 */ 8238 while (qhead != NULL) { 8239 STRSTAT(qhelps); 8240 mutex_enter(&service_queue); 8241 DQ(q, qhead, qtail, q_link); 8242 mutex_exit(&service_queue); 8243 if (q != NULL) 8244 queue_service(q); 8245 } 8246 } 8247 8248 void 8249 stream_willservice(stdata_t *stp) 8250 { 8251 mutex_enter(&stp->sd_qlock); 8252 stp->sd_svcflags |= STRS_WILLSERVICE; 8253 mutex_exit(&stp->sd_qlock); 8254 } 8255 8256 /* 8257 * Replace the cred currently in the mblk with a different one. 8258 */ 8259 void 8260 mblk_setcred(mblk_t *mp, cred_t *cr) 8261 { 8262 cred_t *ocr = DB_CRED(mp); 8263 8264 ASSERT(cr != NULL); 8265 8266 if (cr != ocr) { 8267 crhold(mp->b_datap->db_credp = cr); 8268 if (ocr != NULL) 8269 crfree(ocr); 8270 } 8271 } 8272 8273 int 8274 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8275 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value, 8276 uint32_t flags, int km_flags) 8277 { 8278 int rc = 0; 8279 8280 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8281 if (mp->b_datap->db_type == M_DATA) { 8282 /* Associate values for M_DATA type */ 8283 DB_CKSUMSTART(mp) = (intptr_t)start; 8284 DB_CKSUMSTUFF(mp) = (intptr_t)stuff; 8285 DB_CKSUMEND(mp) = (intptr_t)end; 8286 DB_CKSUMFLAGS(mp) = flags; 8287 DB_CKSUM16(mp) = (uint16_t)value; 8288 8289 } else { 8290 pattrinfo_t pa_info; 8291 8292 ASSERT(mmd != NULL); 8293 8294 pa_info.type = PATTR_HCKSUM; 8295 pa_info.len = sizeof (pattr_hcksum_t); 8296 8297 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) { 8298 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf; 8299 8300 hck->hcksum_start_offset = start; 8301 hck->hcksum_stuff_offset = stuff; 8302 hck->hcksum_end_offset = end; 8303 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value; 8304 hck->hcksum_flags = flags; 8305 } else { 8306 rc = -1; 8307 } 8308 } 8309 return (rc); 8310 } 8311 8312 void 8313 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8314 uint32_t *start, uint32_t *stuff, uint32_t *end, 8315 uint32_t *value, uint32_t *flags) 8316 { 8317 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8318 if (mp->b_datap->db_type == M_DATA) { 8319 if (flags != NULL) { 8320 *flags = DB_CKSUMFLAGS(mp); 8321 if (*flags & HCK_PARTIALCKSUM) { 8322 if (start != NULL) 8323 *start = (uint32_t)DB_CKSUMSTART(mp); 8324 if (stuff != NULL) 8325 *stuff = (uint32_t)DB_CKSUMSTUFF(mp); 8326 if (end != NULL) 8327 *end = (uint32_t)DB_CKSUMEND(mp); 8328 if (value != NULL) 8329 *value = (uint32_t)DB_CKSUM16(mp); 8330 } 8331 } 8332 } else { 8333 pattrinfo_t hck_attr = {PATTR_HCKSUM}; 8334 8335 ASSERT(mmd != NULL); 8336 8337 /* get hardware checksum attribute */ 8338 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) { 8339 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf; 8340 8341 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t)); 8342 if (flags != NULL) 8343 *flags = hck->hcksum_flags; 8344 if (start != NULL) 8345 *start = hck->hcksum_start_offset; 8346 if (stuff != NULL) 8347 *stuff = hck->hcksum_stuff_offset; 8348 if (end != NULL) 8349 *end = hck->hcksum_end_offset; 8350 if (value != NULL) 8351 *value = (uint32_t) 8352 hck->hcksum_cksum_val.inet_cksum; 8353 } 8354 } 8355 } 8356 8357 /* 8358 * Checksum buffer *bp for len bytes with psum partial checksum, 8359 * or 0 if none, and return the 16 bit partial checksum. 8360 */ 8361 unsigned 8362 bcksum(uchar_t *bp, int len, unsigned int psum) 8363 { 8364 int odd = len & 1; 8365 extern unsigned int ip_ocsum(); 8366 8367 if (((intptr_t)bp & 1) == 0 && !odd) { 8368 /* 8369 * Bp is 16 bit aligned and len is multiple of 16 bit word. 8370 */ 8371 return (ip_ocsum((ushort_t *)bp, len >> 1, psum)); 8372 } 8373 if (((intptr_t)bp & 1) != 0) { 8374 /* 8375 * Bp isn't 16 bit aligned. 8376 */ 8377 unsigned int tsum; 8378 8379 #ifdef _LITTLE_ENDIAN 8380 psum += *bp; 8381 #else 8382 psum += *bp << 8; 8383 #endif 8384 len--; 8385 bp++; 8386 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0); 8387 psum += (tsum << 8) & 0xffff | (tsum >> 8); 8388 if (len & 1) { 8389 bp += len - 1; 8390 #ifdef _LITTLE_ENDIAN 8391 psum += *bp << 8; 8392 #else 8393 psum += *bp; 8394 #endif 8395 } 8396 } else { 8397 /* 8398 * Bp is 16 bit aligned. 8399 */ 8400 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum); 8401 if (odd) { 8402 bp += len - 1; 8403 #ifdef _LITTLE_ENDIAN 8404 psum += *bp; 8405 #else 8406 psum += *bp << 8; 8407 #endif 8408 } 8409 } 8410 /* 8411 * Normalize psum to 16 bits before returning the new partial 8412 * checksum. The max psum value before normalization is 0x3FDFE. 8413 */ 8414 return ((psum >> 16) + (psum & 0xFFFF)); 8415 } 8416 8417 boolean_t 8418 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd) 8419 { 8420 boolean_t rc; 8421 8422 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8423 if (DB_TYPE(mp) == M_DATA) { 8424 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0); 8425 } else { 8426 pattrinfo_t zcopy_attr = {PATTR_ZCOPY}; 8427 8428 ASSERT(mmd != NULL); 8429 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL); 8430 } 8431 return (rc); 8432 } 8433 8434 void 8435 freemsgchain(mblk_t *mp) 8436 { 8437 mblk_t *next; 8438 8439 while (mp != NULL) { 8440 next = mp->b_next; 8441 mp->b_next = NULL; 8442 8443 freemsg(mp); 8444 mp = next; 8445 } 8446 } 8447 8448 mblk_t * 8449 copymsgchain(mblk_t *mp) 8450 { 8451 mblk_t *nmp = NULL; 8452 mblk_t **nmpp = &nmp; 8453 8454 for (; mp != NULL; mp = mp->b_next) { 8455 if ((*nmpp = copymsg(mp)) == NULL) { 8456 freemsgchain(nmp); 8457 return (NULL); 8458 } 8459 8460 nmpp = &((*nmpp)->b_next); 8461 } 8462 8463 return (nmp); 8464 } 8465 8466 /* NOTE: Do not add code after this point. */ 8467 #undef QLOCK 8468 8469 /* 8470 * replacement for QLOCK macro for those that can't use it. 8471 */ 8472 kmutex_t * 8473 QLOCK(queue_t *q) 8474 { 8475 return (&(q)->q_lock); 8476 } 8477 8478 /* 8479 * Dummy runqueues/queuerun functions functions for backwards compatibility. 8480 */ 8481 #undef runqueues 8482 void 8483 runqueues(void) 8484 { 8485 } 8486 8487 #undef queuerun 8488 void 8489 queuerun(void) 8490 { 8491 } 8492